package app import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "strconv" "strings" "time" "github.com/geo-platform/tenant-api/internal/shared/config" "github.com/geo-platform/tenant-api/internal/shared/llm" "github.com/geo-platform/tenant-api/internal/tenant/prompts" "go.uber.org/zap" ) const ( defaultKnowledgeArkBaseURL = "https://ark.cn-beijing.volces.com/api/v3" defaultKnowledgeURLParseTimeout = 60 * time.Second defaultKnowledgeURLMarkdownTimeout = 90 * time.Second defaultKnowledgeURLMarkdownChunkChars = 12000 defaultKnowledgeURLMarkdownOutputToken = 12000 ) var errBrowserFetchNotAvailable = fmt.Errorf("browser fetch fallback is not available") type sourceFetchRestrictedError struct { URL string Status int Cause error } func (e sourceFetchRestrictedError) Error() string { cause := "" if e.Cause != nil { cause = strings.TrimSpace(e.Cause.Error()) } if cause == "" { cause = "source website requires browser rendering or blocks server-side fetch" } if e.Status > 0 { return fmt.Sprintf("source website fetch restricted status=%d: %s", e.Status, cause) } return "source website fetch restricted: " + cause } func (e sourceFetchRestrictedError) Unwrap() error { return e.Cause } func isSourceFetchRestrictedError(err error) bool { var restricted sourceFetchRestrictedError return errors.As(err, &restricted) } type arkToolExecuteRequest struct { ActionName string `json:"action_name"` ToolName string `json:"tool_name"` Timeout int `json:"timeout,omitempty"` Parameters map[string]any `json:"parameters,omitempty"` } type arkToolExecuteResponse struct { StatusCode int `json:"status_code"` Data arkToolExecuteData `json:"data"` } type arkToolExecuteData struct { ArkWebDataList []arkWebDataItem `json:"ark_web_data_list"` } type arkWebDataItem struct { URL string `json:"url"` Title string `json:"title"` Content string `json:"content"` ErrorMessage string `json:"error_message"` ErrorCode string `json:"error_code"` } func (s *KnowledgeService) parseWebsiteKnowledge(ctx context.Context, rawURL string) (*knowledgeParsedContent, error) { item, err := s.executeLinkReader(ctx, rawURL) if err != nil { if !s.shouldTryBrowserFetchFallback(rawURL, err) { return nil, classifySourceFetchError(rawURL, err) } fallback, fallbackErr := s.parseWebsiteKnowledgeWithBrowserFetch(ctx, rawURL) if fallbackErr == nil { return fallback, nil } if s.logger != nil { s.logger.Warn("knowledge browser fetch fallback failed", zap.String("url", rawURL), zap.Error(fallbackErr), ) } return nil, classifySourceFetchError(rawURL, err) } text := normalizeKnowledgeText(item.Content) if text == "" { return nil, fmt.Errorf("webpage parser returned empty content") } title := strings.TrimSpace(item.Title) markdown, err := s.formatWebsiteMarkdown(ctx, title, text, rawURL) if err != nil { if s.logger != nil { s.logger.Warn("knowledge website markdown formatting failed, fallback to local markdown", zap.String("url", rawURL), zap.Error(err), ) } markdown = buildKnowledgeMarkdownFromText(title, text) } return &knowledgeParsedContent{ Text: text, Markdown: markdown, }, nil } func (s *KnowledgeService) parseWebsiteKnowledgeWithBrowserFetch(ctx context.Context, rawURL string) (*knowledgeParsedContent, error) { runtimeCfg := s.runtimeConfig() browserCfg := runtimeCfg.BrowserFetch config.NormalizeBrowserFetchConfig(&browserCfg) if !browserCfg.Enabled { return nil, errBrowserFetchNotAvailable } provider := strings.ToLower(strings.TrimSpace(browserCfg.Provider)) if provider != "lightpanda" && provider != "lightpanda_service" { return nil, fmt.Errorf("unsupported browser fetch provider %q", browserCfg.Provider) } if strings.TrimSpace(browserCfg.ServiceURL) == "" { return nil, fmt.Errorf("%w: browser_fetch.service_url is empty", errBrowserFetchNotAvailable) } callCtx := ctx if browserCfg.RequestTimeout > 0 { var cancel context.CancelFunc callCtx, cancel = context.WithTimeout(ctx, browserCfg.RequestTimeout) defer cancel() } result, err := s.browserFetchQueue(browserCfg).Fetch(callCtx, rawURL, browserCfg) if err != nil { return nil, err } text := normalizeKnowledgeText(result.Text) if text == "" { text = normalizeKnowledgeText(result.Markdown) } if text == "" { return nil, fmt.Errorf("browser fetch returned empty content") } title := strings.TrimSpace(result.Title) markdown := strings.TrimSpace(result.Markdown) if markdown == "" { var err error markdown, err = s.formatWebsiteMarkdown(ctx, title, text, rawURL) if err != nil { if s.logger != nil { s.logger.Warn("knowledge browser fetch markdown formatting failed, fallback to local markdown", zap.String("url", rawURL), zap.Error(err), ) } markdown = buildKnowledgeMarkdownFromText(title, text) } } return &knowledgeParsedContent{ Text: text, Markdown: markdown, }, nil } func (s *KnowledgeService) executeLinkReader(ctx context.Context, rawURL string) (*arkWebDataItem, error) { runtimeCfg := s.runtimeConfig() baseURL := strings.TrimRight(strings.TrimSpace(runtimeCfg.ArkBaseURL), "/") if baseURL == "" { baseURL = defaultKnowledgeArkBaseURL } timeout := defaultKnowledgeURLParseTimeout callCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() body, err := json.Marshal(arkToolExecuteRequest{ ActionName: "LinkReader", ToolName: "LinkReader", Timeout: int(timeout / time.Second), Parameters: map[string]any{ "url_list": []string{rawURL}, }, }) if err != nil { return nil, fmt.Errorf("marshal webpage parser request: %w", err) } req, err := http.NewRequestWithContext(callCtx, http.MethodPost, baseURL+"/tools/execute", bytes.NewReader(body)) if err != nil { return nil, fmt.Errorf("create webpage parser request: %w", err) } req.Header.Set("Authorization", "Bearer "+runtimeCfg.ArkAPIKey) req.Header.Set("Content-Type", "application/json") resp, err := s.httpClient.Do(req) if err != nil { return nil, fmt.Errorf("execute webpage parser: %w", err) } defer resp.Body.Close() if resp.StatusCode >= http.StatusBadRequest { data, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) return nil, fmt.Errorf("webpage parser failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(data))) } var parsed arkToolExecuteResponse if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { return nil, fmt.Errorf("decode webpage parser response: %w", err) } if parsed.StatusCode != 0 && parsed.StatusCode != http.StatusOK { return nil, fmt.Errorf("webpage parser returned status_code=%d", parsed.StatusCode) } if len(parsed.Data.ArkWebDataList) == 0 { return nil, fmt.Errorf("webpage parser returned no content") } item := parsed.Data.ArkWebDataList[0] if strings.TrimSpace(item.ErrorCode) != "" || strings.TrimSpace(item.ErrorMessage) != "" { return nil, fmt.Errorf("webpage parser returned error: %s %s", strings.TrimSpace(item.ErrorCode), strings.TrimSpace(item.ErrorMessage)) } return &item, nil } func classifySourceFetchError(rawURL string, err error) error { if err == nil { return nil } status := extractHTTPStatusFromError(err) if status == http.StatusForbidden || status == http.StatusTooManyRequests || status >= http.StatusInternalServerError { return sourceFetchRestrictedError{URL: rawURL, Status: status, Cause: err} } return err } func extractHTTPStatusFromError(err error) int { if err == nil { return 0 } message := err.Error() idx := strings.Index(message, "status=") if idx < 0 { return 0 } value := message[idx+len("status="):] end := 0 for end < len(value) && value[end] >= '0' && value[end] <= '9' { end++ } if end == 0 { return 0 } status, parseErr := strconv.Atoi(value[:end]) if parseErr != nil { return 0 } return status } func (s *KnowledgeService) shouldTryBrowserFetchFallback(rawURL string, err error) bool { if s == nil { return false } runtimeCfg := s.runtimeConfig() browserCfg := runtimeCfg.BrowserFetch config.NormalizeBrowserFetchConfig(&browserCfg) if !browserCfg.Enabled { return false } if !browserFetchDomainAllowed(rawURL, browserCfg.AllowedDomains) { return false } status := extractHTTPStatusFromError(err) if status <= 0 { return false } for _, trigger := range browserCfg.TriggerStatus { if status == trigger { return true } } return false } func browserFetchDomainAllowed(rawURL string, allowedDomains []string) bool { if len(allowedDomains) == 0 { return false } parsed, err := url.Parse(rawURL) if err != nil { return false } host := strings.ToLower(strings.TrimSpace(parsed.Hostname())) if host == "" { return false } for _, domain := range allowedDomains { domain = strings.ToLower(strings.TrimPrefix(strings.TrimSpace(domain), ".")) if domain == "" { continue } if host == domain || strings.HasSuffix(host, "."+domain) { return true } } return false } func (s *KnowledgeService) formatWebsiteMarkdown(ctx context.Context, title, content, rawURL string) (string, error) { content = strings.TrimSpace(content) if content == "" { return "", fmt.Errorf("website content is empty") } chunks := splitKnowledgeMarkdownChunks(content, defaultKnowledgeURLMarkdownChunkChars) if len(chunks) == 0 { return "", fmt.Errorf("website content is empty after chunking") } sections := make([]string, 0, len(chunks)+1) runtimeCfg := s.runtimeConfig() for index, chunk := range chunks { result, err := s.llmClient.Generate(ctx, llm.GenerateRequest{ Model: runtimeCfg.URLMarkdownModel, Prompt: prompts.KnowledgeWebsiteMarkdownPrompt(title, rawURL, chunk, index, len(chunks)), Timeout: defaultKnowledgeURLMarkdownTimeout, MaxOutputTokens: defaultKnowledgeURLMarkdownOutputToken, }, nil) if err != nil { return "", err } section := sanitizeLLMMarkdown(strings.TrimSpace(result.Content)) if section == "" { continue } if index > 0 && !strings.HasPrefix(section, "#") { section = fmt.Sprintf("## 第 %d 部分\n\n%s", index+1, section) } sections = append(sections, section) } if len(sections) == 0 { return "", fmt.Errorf("website formatter returned empty markdown") } markdown := strings.TrimSpace(strings.Join(sections, "\n\n")) if title != "" && !strings.HasPrefix(markdown, "# ") { markdown = "# " + title + "\n\n" + markdown } return markdown, nil } func splitKnowledgeMarkdownChunks(content string, maxChars int) []string { if maxChars <= 0 { maxChars = defaultKnowledgeURLMarkdownChunkChars } lines := strings.Split(content, "\n") chunks := make([]string, 0, len(lines)) current := make([]string, 0, 16) currentLen := 0 flush := func() { if len(current) == 0 { return } chunks = append(chunks, strings.TrimSpace(strings.Join(current, "\n"))) current = current[:0] currentLen = 0 } for _, line := range lines { line = strings.TrimSpace(line) if line == "" { continue } if len([]rune(line)) > maxChars { flush() for _, segment := range splitKnowledgeLongLine(line, maxChars) { if strings.TrimSpace(segment) != "" { chunks = append(chunks, segment) } } continue } nextLen := currentLen + len([]rune(line)) if len(current) > 0 { nextLen++ } if nextLen > maxChars { flush() } current = append(current, line) currentLen += len([]rune(line)) if len(current) > 1 { currentLen++ } } flush() return chunks } func splitKnowledgeLongLine(line string, maxChars int) []string { runes := []rune(line) if len(runes) <= maxChars { return []string{line} } segments := make([]string, 0, (len(runes)/maxChars)+1) for start := 0; start < len(runes); start += maxChars { end := start + maxChars if end > len(runes) { end = len(runes) } segments = append(segments, strings.TrimSpace(string(runes[start:end]))) } return segments } func sanitizeLLMMarkdown(content string) string { content = strings.TrimSpace(content) if !strings.HasPrefix(content, "```") { return content } lines := strings.Split(content, "\n") if len(lines) < 3 { return strings.Trim(content, "`") } if !strings.HasPrefix(lines[0], "```") || strings.TrimSpace(lines[len(lines)-1]) != "```" { return content } return strings.TrimSpace(strings.Join(lines[1:len(lines)-1], "\n")) }