feat(browser-fetch): add lightpanda-backed fetch service and knowledge URL fallback

Adds a new browser-fetch microservice that wraps Lightpanda for rendering JS-heavy pages, and wires it into the knowledge URL parser as a fallback when the upstream link reader returns 403/429/5xx for an allow-listed domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload diff support, and full deploy assets (Dockerfile target, docker-compose, k3s manifests, image-build/package scripts).
2026-05-11 11:11:21 +08:00
parent 708a45ba16
commit 7f08d92958
22 changed files with 1770 additions and 8 deletions
@@ -0,0 +1,236 @@
+package app
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/geo-platform/tenant-api/internal/shared/config"
+	"go.uber.org/zap"
+)
+
+var errBrowserFetchQueueFull = errors.New("browser fetch queue is full")
+
+type browserFetchResult struct {
+	URL      string `json:"url,omitempty"`
+	Title    string `json:"title,omitempty"`
+	Text     string `json:"text,omitempty"`
+	Markdown string `json:"markdown,omitempty"`
+}
+
+type browserFetchRequest struct {
+	URL       string                         `json:"url"`
+	TimeoutMS int64                          `json:"timeout_ms,omitempty"`
+	Options   map[string]interface{}         `json:"options,omitempty"`
+	ctx       context.Context                `json:"-"`
+	cfg       config.BrowserFetchConfig      `json:"-"`
+	resp      chan browserFetchRequestResult `json:"-"`
+}
+
+type browserFetchRequestResult struct {
+	result browserFetchResult
+	err    error
+}
+
+type browserFetchQueue struct {
+	logger *zap.Logger
+	client *http.Client
+	jobs   chan browserFetchRequest
+	cache  *browserFetchMemoryCache
+}
+
+type browserFetchCacheEntry struct {
+	result    browserFetchResult
+	expiresAt time.Time
+}
+
+type browserFetchMemoryCache struct {
+	mu      sync.Mutex
+	entries map[string]browserFetchCacheEntry
+}
+
+func (s *KnowledgeService) browserFetchQueue(cfg config.BrowserFetchConfig) *browserFetchQueue {
+	s.browserFetchMu.Lock()
+	defer s.browserFetchMu.Unlock()
+	if s.browserFetchQ != nil {
+		return s.browserFetchQ
+	}
+
+	config.NormalizeBrowserFetchConfig(&cfg)
+	queue := &browserFetchQueue{
+		logger: s.logger,
+		client: &http.Client{},
+		jobs:   make(chan browserFetchRequest, cfg.QueueSize),
+		cache: &browserFetchMemoryCache{
+			entries: make(map[string]browserFetchCacheEntry),
+		},
+	}
+
+	for i := 0; i < cfg.WorkerConcurrency; i++ {
+		go queue.runWorker(i)
+	}
+	s.browserFetchQ = queue
+	return queue
+}
+
+func (q *browserFetchQueue) Fetch(ctx context.Context, rawURL string, cfg config.BrowserFetchConfig) (browserFetchResult, error) {
+	if q == nil {
+		return browserFetchResult{}, errBrowserFetchNotAvailable
+	}
+	config.NormalizeBrowserFetchConfig(&cfg)
+	cacheKey := browserFetchCacheKey(rawURL)
+	if cfg.CacheTTL > 0 {
+		if result, ok := q.cache.Get(cacheKey); ok {
+			return result, nil
+		}
+	}
+
+	request := browserFetchRequest{
+		URL:       rawURL,
+		TimeoutMS: int64(cfg.RequestTimeout / time.Millisecond),
+		Options: map[string]interface{}{
+			"provider": cfg.Provider,
+		},
+		ctx:  ctx,
+		cfg:  cfg,
+		resp: make(chan browserFetchRequestResult, 1),
+	}
+
+	select {
+	case q.jobs <- request:
+	case <-ctx.Done():
+		return browserFetchResult{}, ctx.Err()
+	default:
+		return browserFetchResult{}, errBrowserFetchQueueFull
+	}
+
+	select {
+	case result := <-request.resp:
+		if result.err != nil {
+			return browserFetchResult{}, result.err
+		}
+		if cfg.CacheTTL > 0 {
+			q.cache.Set(cacheKey, result.result, cfg.CacheTTL)
+		}
+		return result.result, nil
+	case <-ctx.Done():
+		return browserFetchResult{}, ctx.Err()
+	}
+}
+
+func (q *browserFetchQueue) runWorker(workerID int) {
+	for request := range q.jobs {
+		result, err := q.execute(context.Background(), request)
+		if err != nil && q.logger != nil {
+			q.logger.Warn("browser fetch request failed",
+				zap.Int("worker_id", workerID),
+				zap.String("url", request.URL),
+				zap.Error(err),
+			)
+		}
+		request.resp <- browserFetchRequestResult{result: result, err: err}
+	}
+}
+
+func (q *browserFetchQueue) execute(parent context.Context, request browserFetchRequest) (browserFetchResult, error) {
+	if request.ctx != nil {
+		parent = request.ctx
+	}
+	timeout := time.Duration(request.TimeoutMS) * time.Millisecond
+	if timeout <= 0 {
+		timeout = 45 * time.Second
+	}
+	ctx, cancel := context.WithTimeout(parent, timeout)
+	defer cancel()
+
+	return q.callService(ctx, request.URL, request.cfg)
+}
+
+func (q *browserFetchQueue) callService(ctx context.Context, rawURL string, cfg config.BrowserFetchConfig) (browserFetchResult, error) {
+	config.NormalizeBrowserFetchConfig(&cfg)
+	endpoint := strings.TrimRight(cfg.ServiceURL, "/") + "/fetch"
+	payload := browserFetchRequest{
+		URL:       rawURL,
+		TimeoutMS: int64(cfg.RequestTimeout / time.Millisecond),
+	}
+	body, err := json.Marshal(payload)
+	if err != nil {
+		return browserFetchResult{}, fmt.Errorf("marshal browser fetch request: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))
+	if err != nil {
+		return browserFetchResult{}, fmt.Errorf("create browser fetch request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if cfg.Token != "" {
+		req.Header.Set("Authorization", "Bearer "+cfg.Token)
+	}
+
+	resp, err := q.client.Do(req)
+	if err != nil {
+		return browserFetchResult{}, fmt.Errorf("execute browser fetch request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= http.StatusBadRequest {
+		data, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+		return browserFetchResult{}, fmt.Errorf("browser fetch service failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(data)))
+	}
+
+	var result browserFetchResult
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return browserFetchResult{}, fmt.Errorf("decode browser fetch response: %w", err)
+	}
+	return result, nil
+}
+
+func (c *browserFetchMemoryCache) Get(key string) (browserFetchResult, bool) {
+	if c == nil || key == "" {
+		return browserFetchResult{}, false
+	}
+	now := time.Now()
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	entry, ok := c.entries[key]
+	if !ok {
+		return browserFetchResult{}, false
+	}
+	if now.After(entry.expiresAt) {
+		delete(c.entries, key)
+		return browserFetchResult{}, false
+	}
+	return entry.result, true
+}
+
+func (c *browserFetchMemoryCache) Set(key string, result browserFetchResult, ttl time.Duration) {
+	if c == nil || key == "" || ttl <= 0 {
+		return
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.entries[key] = browserFetchCacheEntry{
+		result:    result,
+		expiresAt: time.Now().Add(ttl),
+	}
+}
+
+func browserFetchCacheKey(rawURL string) string {
+	normalized := strings.TrimSpace(rawURL)
+	if parsed, err := url.Parse(normalized); err == nil && parsed.Host != "" {
+		parsed.Fragment = ""
+		normalized = parsed.String()
+	}
+	sum := sha256.Sum256([]byte(normalized))
+	return hex.EncodeToString(sum[:])
+}
@@ -49,6 +49,8 @@ type KnowledgeService struct {
 	jobs            chan knowledgeParseJob
 	cleanupMu       sync.Mutex
 	cleanupInFlight map[string]struct{}
+	browserFetchMu  sync.Mutex
+	browserFetchQ   *browserFetchQueue
 }

 type knowledgeParseJob struct {
@@ -199,7 +201,7 @@ func (s *KnowledgeService) WithConfigProvider(provider runtimeConfigProvider) *K
 func (s *KnowledgeService) runtimeConfig() knowledgeRuntimeConfig {
 	if s != nil && s.configProvider != nil {
 		if cfg := s.configProvider.Current(); cfg != nil {
-			return knowledgeRuntimeFromConfig(cfg.Retrieval, cfg.LLM)
+			return knowledgeRuntimeFromFullConfig(cfg)
 		}
 	}
 	if s == nil {
@@ -4,12 +4,16 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
+	"strconv"
 	"strings"
 	"time"

+	"github.com/geo-platform/tenant-api/internal/shared/config"
 	"github.com/geo-platform/tenant-api/internal/shared/llm"
 	"github.com/geo-platform/tenant-api/internal/tenant/prompts"
 	"go.uber.org/zap"
@@ -23,6 +27,37 @@ const (
 	defaultKnowledgeURLMarkdownOutputToken = 12000
 )

+var errBrowserFetchNotAvailable = fmt.Errorf("browser fetch fallback is not available")
+
+type sourceFetchRestrictedError struct {
+	URL    string
+	Status int
+	Cause  error
+}
+
+func (e sourceFetchRestrictedError) Error() string {
+	cause := ""
+	if e.Cause != nil {
+		cause = strings.TrimSpace(e.Cause.Error())
+	}
+	if cause == "" {
+		cause = "source website requires browser rendering or blocks server-side fetch"
+	}
+	if e.Status > 0 {
+		return fmt.Sprintf("source website fetch restricted status=%d: %s", e.Status, cause)
+	}
+	return "source website fetch restricted: " + cause
+}
+
+func (e sourceFetchRestrictedError) Unwrap() error {
+	return e.Cause
+}
+
+func isSourceFetchRestrictedError(err error) bool {
+	var restricted sourceFetchRestrictedError
+	return errors.As(err, &restricted)
+}
+
 type arkToolExecuteRequest struct {
 	ActionName string         `json:"action_name"`
 	ToolName   string         `json:"tool_name"`
@@ -50,7 +85,20 @@ type arkWebDataItem struct {
 func (s *KnowledgeService) parseWebsiteKnowledge(ctx context.Context, rawURL string) (*knowledgeParsedContent, error) {
 	item, err := s.executeLinkReader(ctx, rawURL)
 	if err != nil {
-		return nil, err
+		if !s.shouldTryBrowserFetchFallback(rawURL, err) {
+			return nil, classifySourceFetchError(rawURL, err)
+		}
+		fallback, fallbackErr := s.parseWebsiteKnowledgeWithBrowserFetch(ctx, rawURL)
+		if fallbackErr == nil {
+			return fallback, nil
+		}
+		if s.logger != nil {
+			s.logger.Warn("knowledge browser fetch fallback failed",
+				zap.String("url", rawURL),
+				zap.Error(fallbackErr),
+			)
+		}
+		return nil, classifySourceFetchError(rawURL, err)
 	}

 	text := normalizeKnowledgeText(item.Content)
@@ -76,6 +124,65 @@ func (s *KnowledgeService) parseWebsiteKnowledge(ctx context.Context, rawURL str
 	}, nil
 }

+func (s *KnowledgeService) parseWebsiteKnowledgeWithBrowserFetch(ctx context.Context, rawURL string) (*knowledgeParsedContent, error) {
+	runtimeCfg := s.runtimeConfig()
+	browserCfg := runtimeCfg.BrowserFetch
+	config.NormalizeBrowserFetchConfig(&browserCfg)
+	if !browserCfg.Enabled {
+		return nil, errBrowserFetchNotAvailable
+	}
+
+	provider := strings.ToLower(strings.TrimSpace(browserCfg.Provider))
+	if provider != "lightpanda" && provider != "lightpanda_service" {
+		return nil, fmt.Errorf("unsupported browser fetch provider %q", browserCfg.Provider)
+	}
+
+	if strings.TrimSpace(browserCfg.ServiceURL) == "" {
+		return nil, fmt.Errorf("%w: browser_fetch.service_url is empty", errBrowserFetchNotAvailable)
+	}
+
+	callCtx := ctx
+	if browserCfg.RequestTimeout > 0 {
+		var cancel context.CancelFunc
+		callCtx, cancel = context.WithTimeout(ctx, browserCfg.RequestTimeout)
+		defer cancel()
+	}
+
+	result, err := s.browserFetchQueue(browserCfg).Fetch(callCtx, rawURL, browserCfg)
+	if err != nil {
+		return nil, err
+	}
+
+	text := normalizeKnowledgeText(result.Text)
+	if text == "" {
+		text = normalizeKnowledgeText(result.Markdown)
+	}
+	if text == "" {
+		return nil, fmt.Errorf("browser fetch returned empty content")
+	}
+
+	title := strings.TrimSpace(result.Title)
+	markdown := strings.TrimSpace(result.Markdown)
+	if markdown == "" {
+		var err error
+		markdown, err = s.formatWebsiteMarkdown(ctx, title, text, rawURL)
+		if err != nil {
+			if s.logger != nil {
+				s.logger.Warn("knowledge browser fetch markdown formatting failed, fallback to local markdown",
+					zap.String("url", rawURL),
+					zap.Error(err),
+				)
+			}
+			markdown = buildKnowledgeMarkdownFromText(title, text)
+		}
+	}
+
+	return &knowledgeParsedContent{
+		Text:     text,
+		Markdown: markdown,
+	}, nil
+}
+
 func (s *KnowledgeService) executeLinkReader(ctx context.Context, rawURL string) (*arkWebDataItem, error) {
 	runtimeCfg := s.runtimeConfig()
 	baseURL := strings.TrimRight(strings.TrimSpace(runtimeCfg.ArkBaseURL), "/")
@@ -135,6 +242,90 @@ func (s *KnowledgeService) executeLinkReader(ctx context.Context, rawURL string)
 	return &item, nil
 }

+func classifySourceFetchError(rawURL string, err error) error {
+	if err == nil {
+		return nil
+	}
+	status := extractHTTPStatusFromError(err)
+	if status == http.StatusForbidden || status == http.StatusTooManyRequests || status >= http.StatusInternalServerError {
+		return sourceFetchRestrictedError{URL: rawURL, Status: status, Cause: err}
+	}
+	return err
+}
+
+func extractHTTPStatusFromError(err error) int {
+	if err == nil {
+		return 0
+	}
+	message := err.Error()
+	idx := strings.Index(message, "status=")
+	if idx < 0 {
+		return 0
+	}
+	value := message[idx+len("status="):]
+	end := 0
+	for end < len(value) && value[end] >= '0' && value[end] <= '9' {
+		end++
+	}
+	if end == 0 {
+		return 0
+	}
+	status, parseErr := strconv.Atoi(value[:end])
+	if parseErr != nil {
+		return 0
+	}
+	return status
+}
+
+func (s *KnowledgeService) shouldTryBrowserFetchFallback(rawURL string, err error) bool {
+	if s == nil {
+		return false
+	}
+	runtimeCfg := s.runtimeConfig()
+	browserCfg := runtimeCfg.BrowserFetch
+	config.NormalizeBrowserFetchConfig(&browserCfg)
+	if !browserCfg.Enabled {
+		return false
+	}
+	if !browserFetchDomainAllowed(rawURL, browserCfg.AllowedDomains) {
+		return false
+	}
+	status := extractHTTPStatusFromError(err)
+	if status <= 0 {
+		return false
+	}
+	for _, trigger := range browserCfg.TriggerStatus {
+		if status == trigger {
+			return true
+		}
+	}
+	return false
+}
+
+func browserFetchDomainAllowed(rawURL string, allowedDomains []string) bool {
+	if len(allowedDomains) == 0 {
+		return false
+	}
+	parsed, err := url.Parse(rawURL)
+	if err != nil {
+		return false
+	}
+	host := strings.ToLower(strings.TrimSpace(parsed.Hostname()))
+	if host == "" {
+		return false
+	}
+	for _, domain := range allowedDomains {
+		domain = strings.ToLower(strings.TrimPrefix(strings.TrimSpace(domain), "."))
+		if domain == "" {
+			continue
+		}
+		if host == domain || strings.HasSuffix(host, "."+domain) {
+			return true
+		}
+	}
+	return false
+}
+
 func (s *KnowledgeService) formatWebsiteMarkdown(ctx context.Context, title, content, rawURL string) (string, error) {
 	content = strings.TrimSpace(content)
 	if content == "" {
@@ -26,6 +26,7 @@ type knowledgeRuntimeConfig struct {
 	ArkBaseURL         string
 	ArkAPIKey          string
 	URLMarkdownModel   string
+	BrowserFetch       config.BrowserFetchConfig
 }

 type generationStreamDisabledConfigProvider struct {
@@ -103,3 +104,14 @@ func knowledgeRuntimeFromConfig(retrievalCfg config.RetrievalConfig, llmCfg conf
 		URLMarkdownModel:   model,
 	}
 }
+
+func knowledgeRuntimeFromFullConfig(cfg *config.Config) knowledgeRuntimeConfig {
+	if cfg == nil {
+		return knowledgeRuntimeFromConfig(config.RetrievalConfig{}, config.LLMConfig{})
+	}
+	runtime := knowledgeRuntimeFromConfig(cfg.Retrieval, cfg.LLM)
+	browserFetch := cfg.BrowserFetch
+	config.NormalizeBrowserFetchConfig(&browserFetch)
+	runtime.BrowserFetch = browserFetch
+	return runtime
+}