geo/server/internal/tenant/app/knowledge_url_parser.go

package app

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/geo-platform/tenant-api/internal/shared/config"
	"github.com/geo-platform/tenant-api/internal/shared/llm"
	"github.com/geo-platform/tenant-api/internal/tenant/prompts"
	"go.uber.org/zap"
)

const (
	defaultKnowledgeArkBaseURL             = "https://ark.cn-beijing.volces.com/api/v3"
	defaultKnowledgeURLParseTimeout        = 60 * time.Second
	defaultKnowledgeURLMarkdownTimeout     = 90 * time.Second
	defaultKnowledgeURLMarkdownChunkChars  = 12000
	defaultKnowledgeURLMarkdownOutputToken = 12000
)

var errBrowserFetchNotAvailable = fmt.Errorf("browser fetch fallback is not available")

type sourceFetchRestrictedError struct {
	URL    string
	Status int
	Cause  error
}

func (e sourceFetchRestrictedError) Error() string {
	cause := ""
	if e.Cause != nil {
		cause = strings.TrimSpace(e.Cause.Error())
	}
	if cause == "" {
		cause = "source website requires browser rendering or blocks server-side fetch"
	}
	if e.Status > 0 {
		return fmt.Sprintf("source website fetch restricted status=%d: %s", e.Status, cause)
	}
	return "source website fetch restricted: " + cause
}

func (e sourceFetchRestrictedError) Unwrap() error {
	return e.Cause
}

func isSourceFetchRestrictedError(err error) bool {
	var restricted sourceFetchRestrictedError
	return errors.As(err, &restricted)
}

type arkToolExecuteRequest struct {
	ActionName string         `json:"action_name"`
	ToolName   string         `json:"tool_name"`
	Timeout    int            `json:"timeout,omitempty"`
	Parameters map[string]any `json:"parameters,omitempty"`
}

type arkToolExecuteResponse struct {
	StatusCode int                `json:"status_code"`
	Data       arkToolExecuteData `json:"data"`
}

type arkToolExecuteData struct {
	ArkWebDataList []arkWebDataItem `json:"ark_web_data_list"`
}

type arkWebDataItem struct {
	URL          string `json:"url"`
	Title        string `json:"title"`
	Content      string `json:"content"`
	ErrorMessage string `json:"error_message"`
	ErrorCode    string `json:"error_code"`
}

func (s *KnowledgeService) parseWebsiteKnowledge(ctx context.Context, rawURL string) (*knowledgeParsedContent, error) {
	item, err := s.executeLinkReader(ctx, rawURL)
	if err != nil {
		if !s.shouldTryBrowserFetchFallback(rawURL, err) {
			return nil, classifySourceFetchError(rawURL, err)
		}
		fallback, fallbackErr := s.parseWebsiteKnowledgeWithBrowserFetch(ctx, rawURL)
		if fallbackErr == nil {
			return fallback, nil
		}
		if s.logger != nil {
			s.logger.Warn("knowledge browser fetch fallback failed",
				zap.String("url", rawURL),
				zap.Error(fallbackErr),
			)
		}
		return nil, classifySourceFetchError(rawURL, err)
	}

	text := normalizeKnowledgeText(item.Content)
	if text == "" {
		return nil, fmt.Errorf("webpage parser returned empty content")
	}

	title := strings.TrimSpace(item.Title)
	markdown, err := s.formatWebsiteMarkdown(ctx, title, text, rawURL)
	if err != nil {
		if s.logger != nil {
			s.logger.Warn("knowledge website markdown formatting failed, fallback to local markdown",
				zap.String("url", rawURL),
				zap.Error(err),
			)
		}
		markdown = buildKnowledgeMarkdownFromText(title, text)
	}

	return &knowledgeParsedContent{
		Text:     text,
		Markdown: markdown,
	}, nil
}

func (s *KnowledgeService) parseWebsiteKnowledgeWithBrowserFetch(ctx context.Context, rawURL string) (*knowledgeParsedContent, error) {
	runtimeCfg := s.runtimeConfig()
	browserCfg := runtimeCfg.BrowserFetch
	config.NormalizeBrowserFetchConfig(&browserCfg)
	if !browserCfg.Enabled {
		return nil, errBrowserFetchNotAvailable
	}

	provider := strings.ToLower(strings.TrimSpace(browserCfg.Provider))
	if provider != "lightpanda" && provider != "lightpanda_service" {
		return nil, fmt.Errorf("unsupported browser fetch provider %q", browserCfg.Provider)
	}

	if strings.TrimSpace(browserCfg.ServiceURL) == "" {
		return nil, fmt.Errorf("%w: browser_fetch.service_url is empty", errBrowserFetchNotAvailable)
	}

	callCtx := ctx
	if browserCfg.RequestTimeout > 0 {
		var cancel context.CancelFunc
		callCtx, cancel = context.WithTimeout(ctx, browserCfg.RequestTimeout)
		defer cancel()
	}

	result, err := s.browserFetchQueue(browserCfg).Fetch(callCtx, rawURL, browserCfg)
	if err != nil {
		return nil, err
	}

	text := normalizeKnowledgeText(result.Text)
	if text == "" {
		text = normalizeKnowledgeText(result.Markdown)
	}
	if text == "" {
		return nil, fmt.Errorf("browser fetch returned empty content")
	}

	title := strings.TrimSpace(result.Title)
	markdown := strings.TrimSpace(result.Markdown)
	if markdown == "" {
		var err error
		markdown, err = s.formatWebsiteMarkdown(ctx, title, text, rawURL)
		if err != nil {
			if s.logger != nil {
				s.logger.Warn("knowledge browser fetch markdown formatting failed, fallback to local markdown",
					zap.String("url", rawURL),
					zap.Error(err),
				)
			}
			markdown = buildKnowledgeMarkdownFromText(title, text)
		}
	}

	return &knowledgeParsedContent{
		Text:     text,
		Markdown: markdown,
	}, nil
}

func (s *KnowledgeService) executeLinkReader(ctx context.Context, rawURL string) (*arkWebDataItem, error) {
	runtimeCfg := s.runtimeConfig()
	baseURL := strings.TrimRight(strings.TrimSpace(runtimeCfg.ArkBaseURL), "/")
	if baseURL == "" {
		baseURL = defaultKnowledgeArkBaseURL
	}

	timeout := defaultKnowledgeURLParseTimeout
	callCtx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()

	body, err := json.Marshal(arkToolExecuteRequest{
		ActionName: "LinkReader",
		ToolName:   "LinkReader",
		Timeout:    int(timeout / time.Second),
		Parameters: map[string]any{
			"url_list": []string{rawURL},
		},
	})
	if err != nil {
		return nil, fmt.Errorf("marshal webpage parser request: %w", err)
	}

	req, err := http.NewRequestWithContext(callCtx, http.MethodPost, baseURL+"/tools/execute", bytes.NewReader(body))
	if err != nil {
		return nil, fmt.Errorf("create webpage parser request: %w", err)
	}
	req.Header.Set("Authorization", "Bearer "+runtimeCfg.ArkAPIKey)
	req.Header.Set("Content-Type", "application/json")

	resp, err := s.httpClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("execute webpage parser: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode >= http.StatusBadRequest {
		data, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
		return nil, fmt.Errorf("webpage parser failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(data)))
	}

	var parsed arkToolExecuteResponse
	if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
		return nil, fmt.Errorf("decode webpage parser response: %w", err)
	}
	if parsed.StatusCode != 0 && parsed.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("webpage parser returned status_code=%d", parsed.StatusCode)
	}
	if len(parsed.Data.ArkWebDataList) == 0 {
		return nil, fmt.Errorf("webpage parser returned no content")
	}

	item := parsed.Data.ArkWebDataList[0]
	if strings.TrimSpace(item.ErrorCode) != "" || strings.TrimSpace(item.ErrorMessage) != "" {
		return nil, fmt.Errorf("webpage parser returned error: %s %s", strings.TrimSpace(item.ErrorCode), strings.TrimSpace(item.ErrorMessage))
	}
	return &item, nil
}

func classifySourceFetchError(rawURL string, err error) error {
	if err == nil {
		return nil
	}
	status := extractHTTPStatusFromError(err)
	if status == http.StatusForbidden || status == http.StatusTooManyRequests || status >= http.StatusInternalServerError {
		return sourceFetchRestrictedError{URL: rawURL, Status: status, Cause: err}
	}
	return err
}

func extractHTTPStatusFromError(err error) int {
	if err == nil {
		return 0
	}
	message := err.Error()
	idx := strings.Index(message, "status=")
	if idx < 0 {
		return 0
	}
	value := message[idx+len("status="):]
	end := 0
	for end < len(value) && value[end] >= '0' && value[end] <= '9' {
		end++
	}
	if end == 0 {
		return 0
	}
	status, parseErr := strconv.Atoi(value[:end])
	if parseErr != nil {
		return 0
	}
	return status
}

func (s *KnowledgeService) shouldTryBrowserFetchFallback(rawURL string, err error) bool {
	if s == nil {
		return false
	}
	runtimeCfg := s.runtimeConfig()
	browserCfg := runtimeCfg.BrowserFetch
	config.NormalizeBrowserFetchConfig(&browserCfg)
	if !browserCfg.Enabled {
		return false
	}
	if !browserFetchDomainAllowed(rawURL, browserCfg.AllowedDomains) {
		return false
	}
	status := extractHTTPStatusFromError(err)
	if status <= 0 {
		return false
	}
	for _, trigger := range browserCfg.TriggerStatus {
		if status == trigger {
			return true
		}
	}
	return false
}

func browserFetchDomainAllowed(rawURL string, allowedDomains []string) bool {
	if len(allowedDomains) == 0 {
		return false
	}
	parsed, err := url.Parse(rawURL)
	if err != nil {
		return false
	}
	host := strings.ToLower(strings.TrimSpace(parsed.Hostname()))
	if host == "" {
		return false
	}
	for _, domain := range allowedDomains {
		domain = strings.ToLower(strings.TrimPrefix(strings.TrimSpace(domain), "."))
		if domain == "" {
			continue
		}
		if host == domain || strings.HasSuffix(host, "."+domain) {
			return true
		}
	}
	return false
}

func (s *KnowledgeService) formatWebsiteMarkdown(ctx context.Context, title, content, rawURL string) (string, error) {
	content = strings.TrimSpace(content)
	if content == "" {
		return "", fmt.Errorf("website content is empty")
	}

	chunks := splitKnowledgeMarkdownChunks(content, defaultKnowledgeURLMarkdownChunkChars)
	if len(chunks) == 0 {
		return "", fmt.Errorf("website content is empty after chunking")
	}

	sections := make([]string, 0, len(chunks)+1)
	runtimeCfg := s.runtimeConfig()
	for index, chunk := range chunks {
		result, err := s.llmClient.Generate(ctx, llm.GenerateRequest{
			Model:           runtimeCfg.URLMarkdownModel,
			Prompt:          prompts.KnowledgeWebsiteMarkdownPrompt(title, rawURL, chunk, index, len(chunks)),
			Timeout:         defaultKnowledgeURLMarkdownTimeout,
			MaxOutputTokens: defaultKnowledgeURLMarkdownOutputToken,
		}, nil)
		if err != nil {
			return "", err
		}

		section := sanitizeLLMMarkdown(strings.TrimSpace(result.Content))
		if section == "" {
			continue
		}
		if index > 0 && !strings.HasPrefix(section, "#") {
			section = fmt.Sprintf("## 第 %d 部分\n\n%s", index+1, section)
		}
		sections = append(sections, section)
	}

	if len(sections) == 0 {
		return "", fmt.Errorf("website formatter returned empty markdown")
	}

	markdown := strings.TrimSpace(strings.Join(sections, "\n\n"))
	if title != "" && !strings.HasPrefix(markdown, "# ") {
		markdown = "# " + title + "\n\n" + markdown
	}
	return markdown, nil
}

func splitKnowledgeMarkdownChunks(content string, maxChars int) []string {
	if maxChars <= 0 {
		maxChars = defaultKnowledgeURLMarkdownChunkChars
	}

	lines := strings.Split(content, "\n")
	chunks := make([]string, 0, len(lines))
	current := make([]string, 0, 16)
	currentLen := 0

	flush := func() {
		if len(current) == 0 {
			return
		}
		chunks = append(chunks, strings.TrimSpace(strings.Join(current, "\n")))
		current = current[:0]
		currentLen = 0
	}

	for _, line := range lines {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}

		if len([]rune(line)) > maxChars {
			flush()
			for _, segment := range splitKnowledgeLongLine(line, maxChars) {
				if strings.TrimSpace(segment) != "" {
					chunks = append(chunks, segment)
				}
			}
			continue
		}

		nextLen := currentLen + len([]rune(line))
		if len(current) > 0 {
			nextLen++
		}
		if nextLen > maxChars {
			flush()
		}

		current = append(current, line)
		currentLen += len([]rune(line))
		if len(current) > 1 {
			currentLen++
		}
	}

	flush()
	return chunks
}

func splitKnowledgeLongLine(line string, maxChars int) []string {
	runes := []rune(line)
	if len(runes) <= maxChars {
		return []string{line}
	}

	segments := make([]string, 0, (len(runes)/maxChars)+1)
	for start := 0; start < len(runes); start += maxChars {
		end := start + maxChars
		if end > len(runes) {
			end = len(runes)
		}
		segments = append(segments, strings.TrimSpace(string(runes[start:end])))
	}
	return segments
}

func sanitizeLLMMarkdown(content string) string {
	content = strings.TrimSpace(content)
	if !strings.HasPrefix(content, "```") {
		return content
	}

	lines := strings.Split(content, "\n")
	if len(lines) < 3 {
		return strings.Trim(content, "`")
	}
	if !strings.HasPrefix(lines[0], "```") || strings.TrimSpace(lines[len(lines)-1]) != "```" {
		return content
	}
	return strings.TrimSpace(strings.Join(lines[1:len(lines)-1], "\n"))
}