feat(browser-fetch): add lightpanda-backed fetch service and knowledge URL fallback
Adds a new browser-fetch microservice that wraps Lightpanda for rendering JS-heavy pages, and wires it into the knowledge URL parser as a fallback when the upstream link reader returns 403/429/5xx for an allow-listed domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload diff support, and full deploy assets (Dockerfile target, docker-compose, k3s manifests, image-build/package scripts).
This commit is contained in:
@@ -0,0 +1,236 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/config"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
var errBrowserFetchQueueFull = errors.New("browser fetch queue is full")
|
||||
|
||||
type browserFetchResult struct {
|
||||
URL string `json:"url,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Text string `json:"text,omitempty"`
|
||||
Markdown string `json:"markdown,omitempty"`
|
||||
}
|
||||
|
||||
type browserFetchRequest struct {
|
||||
URL string `json:"url"`
|
||||
TimeoutMS int64 `json:"timeout_ms,omitempty"`
|
||||
Options map[string]interface{} `json:"options,omitempty"`
|
||||
ctx context.Context `json:"-"`
|
||||
cfg config.BrowserFetchConfig `json:"-"`
|
||||
resp chan browserFetchRequestResult `json:"-"`
|
||||
}
|
||||
|
||||
type browserFetchRequestResult struct {
|
||||
result browserFetchResult
|
||||
err error
|
||||
}
|
||||
|
||||
type browserFetchQueue struct {
|
||||
logger *zap.Logger
|
||||
client *http.Client
|
||||
jobs chan browserFetchRequest
|
||||
cache *browserFetchMemoryCache
|
||||
}
|
||||
|
||||
type browserFetchCacheEntry struct {
|
||||
result browserFetchResult
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
type browserFetchMemoryCache struct {
|
||||
mu sync.Mutex
|
||||
entries map[string]browserFetchCacheEntry
|
||||
}
|
||||
|
||||
func (s *KnowledgeService) browserFetchQueue(cfg config.BrowserFetchConfig) *browserFetchQueue {
|
||||
s.browserFetchMu.Lock()
|
||||
defer s.browserFetchMu.Unlock()
|
||||
if s.browserFetchQ != nil {
|
||||
return s.browserFetchQ
|
||||
}
|
||||
|
||||
config.NormalizeBrowserFetchConfig(&cfg)
|
||||
queue := &browserFetchQueue{
|
||||
logger: s.logger,
|
||||
client: &http.Client{},
|
||||
jobs: make(chan browserFetchRequest, cfg.QueueSize),
|
||||
cache: &browserFetchMemoryCache{
|
||||
entries: make(map[string]browserFetchCacheEntry),
|
||||
},
|
||||
}
|
||||
|
||||
for i := 0; i < cfg.WorkerConcurrency; i++ {
|
||||
go queue.runWorker(i)
|
||||
}
|
||||
s.browserFetchQ = queue
|
||||
return queue
|
||||
}
|
||||
|
||||
func (q *browserFetchQueue) Fetch(ctx context.Context, rawURL string, cfg config.BrowserFetchConfig) (browserFetchResult, error) {
|
||||
if q == nil {
|
||||
return browserFetchResult{}, errBrowserFetchNotAvailable
|
||||
}
|
||||
config.NormalizeBrowserFetchConfig(&cfg)
|
||||
cacheKey := browserFetchCacheKey(rawURL)
|
||||
if cfg.CacheTTL > 0 {
|
||||
if result, ok := q.cache.Get(cacheKey); ok {
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
|
||||
request := browserFetchRequest{
|
||||
URL: rawURL,
|
||||
TimeoutMS: int64(cfg.RequestTimeout / time.Millisecond),
|
||||
Options: map[string]interface{}{
|
||||
"provider": cfg.Provider,
|
||||
},
|
||||
ctx: ctx,
|
||||
cfg: cfg,
|
||||
resp: make(chan browserFetchRequestResult, 1),
|
||||
}
|
||||
|
||||
select {
|
||||
case q.jobs <- request:
|
||||
case <-ctx.Done():
|
||||
return browserFetchResult{}, ctx.Err()
|
||||
default:
|
||||
return browserFetchResult{}, errBrowserFetchQueueFull
|
||||
}
|
||||
|
||||
select {
|
||||
case result := <-request.resp:
|
||||
if result.err != nil {
|
||||
return browserFetchResult{}, result.err
|
||||
}
|
||||
if cfg.CacheTTL > 0 {
|
||||
q.cache.Set(cacheKey, result.result, cfg.CacheTTL)
|
||||
}
|
||||
return result.result, nil
|
||||
case <-ctx.Done():
|
||||
return browserFetchResult{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (q *browserFetchQueue) runWorker(workerID int) {
|
||||
for request := range q.jobs {
|
||||
result, err := q.execute(context.Background(), request)
|
||||
if err != nil && q.logger != nil {
|
||||
q.logger.Warn("browser fetch request failed",
|
||||
zap.Int("worker_id", workerID),
|
||||
zap.String("url", request.URL),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
request.resp <- browserFetchRequestResult{result: result, err: err}
|
||||
}
|
||||
}
|
||||
|
||||
func (q *browserFetchQueue) execute(parent context.Context, request browserFetchRequest) (browserFetchResult, error) {
|
||||
if request.ctx != nil {
|
||||
parent = request.ctx
|
||||
}
|
||||
timeout := time.Duration(request.TimeoutMS) * time.Millisecond
|
||||
if timeout <= 0 {
|
||||
timeout = 45 * time.Second
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(parent, timeout)
|
||||
defer cancel()
|
||||
|
||||
return q.callService(ctx, request.URL, request.cfg)
|
||||
}
|
||||
|
||||
func (q *browserFetchQueue) callService(ctx context.Context, rawURL string, cfg config.BrowserFetchConfig) (browserFetchResult, error) {
|
||||
config.NormalizeBrowserFetchConfig(&cfg)
|
||||
endpoint := strings.TrimRight(cfg.ServiceURL, "/") + "/fetch"
|
||||
payload := browserFetchRequest{
|
||||
URL: rawURL,
|
||||
TimeoutMS: int64(cfg.RequestTimeout / time.Millisecond),
|
||||
}
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return browserFetchResult{}, fmt.Errorf("marshal browser fetch request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return browserFetchResult{}, fmt.Errorf("create browser fetch request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if cfg.Token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+cfg.Token)
|
||||
}
|
||||
|
||||
resp, err := q.client.Do(req)
|
||||
if err != nil {
|
||||
return browserFetchResult{}, fmt.Errorf("execute browser fetch request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= http.StatusBadRequest {
|
||||
data, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||
return browserFetchResult{}, fmt.Errorf("browser fetch service failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(data)))
|
||||
}
|
||||
|
||||
var result browserFetchResult
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return browserFetchResult{}, fmt.Errorf("decode browser fetch response: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *browserFetchMemoryCache) Get(key string) (browserFetchResult, bool) {
|
||||
if c == nil || key == "" {
|
||||
return browserFetchResult{}, false
|
||||
}
|
||||
now := time.Now()
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
entry, ok := c.entries[key]
|
||||
if !ok {
|
||||
return browserFetchResult{}, false
|
||||
}
|
||||
if now.After(entry.expiresAt) {
|
||||
delete(c.entries, key)
|
||||
return browserFetchResult{}, false
|
||||
}
|
||||
return entry.result, true
|
||||
}
|
||||
|
||||
func (c *browserFetchMemoryCache) Set(key string, result browserFetchResult, ttl time.Duration) {
|
||||
if c == nil || key == "" || ttl <= 0 {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.entries[key] = browserFetchCacheEntry{
|
||||
result: result,
|
||||
expiresAt: time.Now().Add(ttl),
|
||||
}
|
||||
}
|
||||
|
||||
func browserFetchCacheKey(rawURL string) string {
|
||||
normalized := strings.TrimSpace(rawURL)
|
||||
if parsed, err := url.Parse(normalized); err == nil && parsed.Host != "" {
|
||||
parsed.Fragment = ""
|
||||
normalized = parsed.String()
|
||||
}
|
||||
sum := sha256.Sum256([]byte(normalized))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
Reference in New Issue
Block a user