feat(browser-fetch): add lightpanda-backed fetch service and knowledge URL fallback
Adds a new browser-fetch microservice that wraps Lightpanda for rendering JS-heavy pages, and wires it into the knowledge URL parser as a fallback when the upstream link reader returns 403/429/5xx for an allow-listed domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload diff support, and full deploy assets (Dockerfile target, docker-compose, k3s manifests, image-build/package scripts).
This commit is contained in:
@@ -4,12 +4,16 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/config"
|
||||
"github.com/geo-platform/tenant-api/internal/shared/llm"
|
||||
"github.com/geo-platform/tenant-api/internal/tenant/prompts"
|
||||
"go.uber.org/zap"
|
||||
@@ -23,6 +27,37 @@ const (
|
||||
defaultKnowledgeURLMarkdownOutputToken = 12000
|
||||
)
|
||||
|
||||
var errBrowserFetchNotAvailable = fmt.Errorf("browser fetch fallback is not available")
|
||||
|
||||
type sourceFetchRestrictedError struct {
|
||||
URL string
|
||||
Status int
|
||||
Cause error
|
||||
}
|
||||
|
||||
func (e sourceFetchRestrictedError) Error() string {
|
||||
cause := ""
|
||||
if e.Cause != nil {
|
||||
cause = strings.TrimSpace(e.Cause.Error())
|
||||
}
|
||||
if cause == "" {
|
||||
cause = "source website requires browser rendering or blocks server-side fetch"
|
||||
}
|
||||
if e.Status > 0 {
|
||||
return fmt.Sprintf("source website fetch restricted status=%d: %s", e.Status, cause)
|
||||
}
|
||||
return "source website fetch restricted: " + cause
|
||||
}
|
||||
|
||||
func (e sourceFetchRestrictedError) Unwrap() error {
|
||||
return e.Cause
|
||||
}
|
||||
|
||||
func isSourceFetchRestrictedError(err error) bool {
|
||||
var restricted sourceFetchRestrictedError
|
||||
return errors.As(err, &restricted)
|
||||
}
|
||||
|
||||
type arkToolExecuteRequest struct {
|
||||
ActionName string `json:"action_name"`
|
||||
ToolName string `json:"tool_name"`
|
||||
@@ -50,7 +85,20 @@ type arkWebDataItem struct {
|
||||
func (s *KnowledgeService) parseWebsiteKnowledge(ctx context.Context, rawURL string) (*knowledgeParsedContent, error) {
|
||||
item, err := s.executeLinkReader(ctx, rawURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
if !s.shouldTryBrowserFetchFallback(rawURL, err) {
|
||||
return nil, classifySourceFetchError(rawURL, err)
|
||||
}
|
||||
fallback, fallbackErr := s.parseWebsiteKnowledgeWithBrowserFetch(ctx, rawURL)
|
||||
if fallbackErr == nil {
|
||||
return fallback, nil
|
||||
}
|
||||
if s.logger != nil {
|
||||
s.logger.Warn("knowledge browser fetch fallback failed",
|
||||
zap.String("url", rawURL),
|
||||
zap.Error(fallbackErr),
|
||||
)
|
||||
}
|
||||
return nil, classifySourceFetchError(rawURL, err)
|
||||
}
|
||||
|
||||
text := normalizeKnowledgeText(item.Content)
|
||||
@@ -76,6 +124,65 @@ func (s *KnowledgeService) parseWebsiteKnowledge(ctx context.Context, rawURL str
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *KnowledgeService) parseWebsiteKnowledgeWithBrowserFetch(ctx context.Context, rawURL string) (*knowledgeParsedContent, error) {
|
||||
runtimeCfg := s.runtimeConfig()
|
||||
browserCfg := runtimeCfg.BrowserFetch
|
||||
config.NormalizeBrowserFetchConfig(&browserCfg)
|
||||
if !browserCfg.Enabled {
|
||||
return nil, errBrowserFetchNotAvailable
|
||||
}
|
||||
|
||||
provider := strings.ToLower(strings.TrimSpace(browserCfg.Provider))
|
||||
if provider != "lightpanda" && provider != "lightpanda_service" {
|
||||
return nil, fmt.Errorf("unsupported browser fetch provider %q", browserCfg.Provider)
|
||||
}
|
||||
|
||||
if strings.TrimSpace(browserCfg.ServiceURL) == "" {
|
||||
return nil, fmt.Errorf("%w: browser_fetch.service_url is empty", errBrowserFetchNotAvailable)
|
||||
}
|
||||
|
||||
callCtx := ctx
|
||||
if browserCfg.RequestTimeout > 0 {
|
||||
var cancel context.CancelFunc
|
||||
callCtx, cancel = context.WithTimeout(ctx, browserCfg.RequestTimeout)
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
result, err := s.browserFetchQueue(browserCfg).Fetch(callCtx, rawURL, browserCfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
text := normalizeKnowledgeText(result.Text)
|
||||
if text == "" {
|
||||
text = normalizeKnowledgeText(result.Markdown)
|
||||
}
|
||||
if text == "" {
|
||||
return nil, fmt.Errorf("browser fetch returned empty content")
|
||||
}
|
||||
|
||||
title := strings.TrimSpace(result.Title)
|
||||
markdown := strings.TrimSpace(result.Markdown)
|
||||
if markdown == "" {
|
||||
var err error
|
||||
markdown, err = s.formatWebsiteMarkdown(ctx, title, text, rawURL)
|
||||
if err != nil {
|
||||
if s.logger != nil {
|
||||
s.logger.Warn("knowledge browser fetch markdown formatting failed, fallback to local markdown",
|
||||
zap.String("url", rawURL),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
markdown = buildKnowledgeMarkdownFromText(title, text)
|
||||
}
|
||||
}
|
||||
|
||||
return &knowledgeParsedContent{
|
||||
Text: text,
|
||||
Markdown: markdown,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *KnowledgeService) executeLinkReader(ctx context.Context, rawURL string) (*arkWebDataItem, error) {
|
||||
runtimeCfg := s.runtimeConfig()
|
||||
baseURL := strings.TrimRight(strings.TrimSpace(runtimeCfg.ArkBaseURL), "/")
|
||||
@@ -135,6 +242,90 @@ func (s *KnowledgeService) executeLinkReader(ctx context.Context, rawURL string)
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
func classifySourceFetchError(rawURL string, err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
status := extractHTTPStatusFromError(err)
|
||||
if status == http.StatusForbidden || status == http.StatusTooManyRequests || status >= http.StatusInternalServerError {
|
||||
return sourceFetchRestrictedError{URL: rawURL, Status: status, Cause: err}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func extractHTTPStatusFromError(err error) int {
|
||||
if err == nil {
|
||||
return 0
|
||||
}
|
||||
message := err.Error()
|
||||
idx := strings.Index(message, "status=")
|
||||
if idx < 0 {
|
||||
return 0
|
||||
}
|
||||
value := message[idx+len("status="):]
|
||||
end := 0
|
||||
for end < len(value) && value[end] >= '0' && value[end] <= '9' {
|
||||
end++
|
||||
}
|
||||
if end == 0 {
|
||||
return 0
|
||||
}
|
||||
status, parseErr := strconv.Atoi(value[:end])
|
||||
if parseErr != nil {
|
||||
return 0
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
func (s *KnowledgeService) shouldTryBrowserFetchFallback(rawURL string, err error) bool {
|
||||
if s == nil {
|
||||
return false
|
||||
}
|
||||
runtimeCfg := s.runtimeConfig()
|
||||
browserCfg := runtimeCfg.BrowserFetch
|
||||
config.NormalizeBrowserFetchConfig(&browserCfg)
|
||||
if !browserCfg.Enabled {
|
||||
return false
|
||||
}
|
||||
if !browserFetchDomainAllowed(rawURL, browserCfg.AllowedDomains) {
|
||||
return false
|
||||
}
|
||||
status := extractHTTPStatusFromError(err)
|
||||
if status <= 0 {
|
||||
return false
|
||||
}
|
||||
for _, trigger := range browserCfg.TriggerStatus {
|
||||
if status == trigger {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func browserFetchDomainAllowed(rawURL string, allowedDomains []string) bool {
|
||||
if len(allowedDomains) == 0 {
|
||||
return false
|
||||
}
|
||||
parsed, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(strings.TrimSpace(parsed.Hostname()))
|
||||
if host == "" {
|
||||
return false
|
||||
}
|
||||
for _, domain := range allowedDomains {
|
||||
domain = strings.ToLower(strings.TrimPrefix(strings.TrimSpace(domain), "."))
|
||||
if domain == "" {
|
||||
continue
|
||||
}
|
||||
if host == domain || strings.HasSuffix(host, "."+domain) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *KnowledgeService) formatWebsiteMarkdown(ctx context.Context, title, content, rawURL string) (string, error) {
|
||||
content = strings.TrimSpace(content)
|
||||
if content == "" {
|
||||
|
||||
Reference in New Issue
Block a user