feat(browser-fetch): add lightpanda-backed fetch service and knowledge URL fallback
Adds a new browser-fetch microservice that wraps Lightpanda for rendering JS-heavy pages, and wires it into the knowledge URL parser as a fallback when the upstream link reader returns 403/429/5xx for an allow-listed domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload diff support, and full deploy assets (Dockerfile target, docker-compose, k3s manifests, image-build/package scripts).
This commit is contained in:
@@ -38,6 +38,7 @@ type Config struct {
|
||||
Compliance ComplianceConfig `mapstructure:"compliance"`
|
||||
Retrieval RetrievalConfig `mapstructure:"retrieval"`
|
||||
Generation GenerationConfig `mapstructure:"generation"`
|
||||
BrowserFetch BrowserFetchConfig `mapstructure:"browser_fetch"`
|
||||
}
|
||||
|
||||
type ServerConfig struct {
|
||||
@@ -348,6 +349,19 @@ type GenerationConfig struct {
|
||||
TaskStateCheckLookback time.Duration `mapstructure:"task_state_check_lookback"`
|
||||
}
|
||||
|
||||
type BrowserFetchConfig struct {
|
||||
Enabled bool `mapstructure:"enabled"`
|
||||
Provider string `mapstructure:"provider"`
|
||||
ServiceURL string `mapstructure:"service_url"`
|
||||
Token string `mapstructure:"token"`
|
||||
QueueSize int `mapstructure:"queue_size"`
|
||||
WorkerConcurrency int `mapstructure:"worker_concurrency"`
|
||||
RequestTimeout time.Duration `mapstructure:"request_timeout"`
|
||||
CacheTTL time.Duration `mapstructure:"cache_ttl"`
|
||||
AllowedDomains []string `mapstructure:"allowed_domains"`
|
||||
TriggerStatus []int `mapstructure:"trigger_status"`
|
||||
}
|
||||
|
||||
func Load(configPath string) (*Config, error) {
|
||||
cfg, _, err := loadWithFiles(configPath)
|
||||
return cfg, err
|
||||
@@ -382,6 +396,7 @@ func loadWithFiles(configPath string) (*Config, []string, error) {
|
||||
normalizeBrandLibraryConfig(&cfg.BrandLibrary)
|
||||
NormalizeComplianceConfig(&cfg.Compliance)
|
||||
NormalizeGenerationConfig(&cfg.Generation)
|
||||
NormalizeBrowserFetchConfig(&cfg.BrowserFetch)
|
||||
|
||||
files := []string{configFile}
|
||||
if localConfigFile != "" {
|
||||
@@ -511,6 +526,42 @@ func compactStringSlice(values []string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
func compactIntSlice(values []int) []int {
|
||||
if values == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]int, 0, len(values))
|
||||
seen := make(map[int]struct{}, len(values))
|
||||
for _, value := range values {
|
||||
if value <= 0 {
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[value]; exists {
|
||||
continue
|
||||
}
|
||||
seen[value] = struct{}{}
|
||||
out = append(out, value)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseIntList(value string) []int {
|
||||
parts := strings.Split(value, ",")
|
||||
out := make([]int, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
n, err := strconv.Atoi(part)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func normalizeMonitoringDispatchConfig(cfg *MonitoringDispatchConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
@@ -643,6 +694,35 @@ func NormalizeGenerationConfig(cfg *GenerationConfig) {
|
||||
}
|
||||
}
|
||||
|
||||
func NormalizeBrowserFetchConfig(cfg *BrowserFetchConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
cfg.Provider = strings.ToLower(strings.TrimSpace(cfg.Provider))
|
||||
if cfg.Provider == "" {
|
||||
cfg.Provider = "lightpanda"
|
||||
}
|
||||
cfg.ServiceURL = strings.TrimRight(strings.TrimSpace(cfg.ServiceURL), "/")
|
||||
cfg.Token = strings.TrimSpace(cfg.Token)
|
||||
if cfg.QueueSize <= 0 {
|
||||
cfg.QueueSize = 64
|
||||
}
|
||||
if cfg.WorkerConcurrency <= 0 {
|
||||
cfg.WorkerConcurrency = 1
|
||||
}
|
||||
if cfg.RequestTimeout <= 0 {
|
||||
cfg.RequestTimeout = 45 * time.Second
|
||||
}
|
||||
if cfg.CacheTTL <= 0 {
|
||||
cfg.CacheTTL = 6 * time.Hour
|
||||
}
|
||||
cfg.AllowedDomains = compactStringSlice(cfg.AllowedDomains)
|
||||
cfg.TriggerStatus = compactIntSlice(cfg.TriggerStatus)
|
||||
if len(cfg.TriggerStatus) == 0 {
|
||||
cfg.TriggerStatus = []int{403, 429, 500, 502, 503, 504}
|
||||
}
|
||||
}
|
||||
|
||||
func NormalizeComplianceConfig(cfg *ComplianceConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
@@ -759,6 +839,36 @@ func applyEnvOverrides(cfg *Config) {
|
||||
if d, ok := lookupDurationEnv("GENERATION_TASK_STATE_CHECK_LOOKBACK"); ok {
|
||||
cfg.Generation.TaskStateCheckLookback = d
|
||||
}
|
||||
if enabled, ok := lookupBoolEnv("BROWSER_FETCH_ENABLED"); ok {
|
||||
cfg.BrowserFetch.Enabled = enabled
|
||||
}
|
||||
if provider, ok := lookupNonEmptyEnv("BROWSER_FETCH_PROVIDER"); ok {
|
||||
cfg.BrowserFetch.Provider = provider
|
||||
}
|
||||
if serviceURL, ok := lookupFirstNonEmptyEnv("BROWSER_FETCH_SERVICE_URL", "LIGHTPANDA_BROWSER_FETCH_URL"); ok {
|
||||
cfg.BrowserFetch.ServiceURL = serviceURL
|
||||
}
|
||||
if token, ok := lookupNonEmptyEnv("BROWSER_FETCH_TOKEN"); ok {
|
||||
cfg.BrowserFetch.Token = token
|
||||
}
|
||||
if n, ok := lookupIntEnv("BROWSER_FETCH_QUEUE_SIZE"); ok {
|
||||
cfg.BrowserFetch.QueueSize = n
|
||||
}
|
||||
if n, ok := lookupIntEnv("BROWSER_FETCH_WORKER_CONCURRENCY"); ok {
|
||||
cfg.BrowserFetch.WorkerConcurrency = n
|
||||
}
|
||||
if d, ok := lookupDurationEnv("BROWSER_FETCH_REQUEST_TIMEOUT"); ok {
|
||||
cfg.BrowserFetch.RequestTimeout = d
|
||||
}
|
||||
if d, ok := lookupDurationEnv("BROWSER_FETCH_CACHE_TTL"); ok {
|
||||
cfg.BrowserFetch.CacheTTL = d
|
||||
}
|
||||
if domains, ok := lookupNonEmptyEnv("BROWSER_FETCH_ALLOWED_DOMAINS"); ok {
|
||||
cfg.BrowserFetch.AllowedDomains = strings.Split(domains, ",")
|
||||
}
|
||||
if statuses, ok := lookupNonEmptyEnv("BROWSER_FETCH_TRIGGER_STATUS"); ok {
|
||||
cfg.BrowserFetch.TriggerStatus = parseIntList(statuses)
|
||||
}
|
||||
if apiKey, ok := lookupNonEmptyEnv("QDRANT_API_KEY"); ok {
|
||||
cfg.Qdrant.APIKey = apiKey
|
||||
}
|
||||
|
||||
@@ -557,6 +557,93 @@ generation: {}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAppliesBrowserFetchDefaultsAndOverrides(t *testing.T) {
|
||||
configPath := writeTestConfig(t, `
|
||||
browser_fetch:
|
||||
enabled: true
|
||||
service_url: " http://browser-fetch:8082/ "
|
||||
token: " token "
|
||||
allowed_domains:
|
||||
- " zhuanlan.zhihu.com "
|
||||
- ""
|
||||
trigger_status:
|
||||
- 403
|
||||
- 403
|
||||
- 500
|
||||
`)
|
||||
|
||||
cfg, err := Load(configPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
|
||||
if !cfg.BrowserFetch.Enabled {
|
||||
t.Fatalf("expected browser fetch enabled")
|
||||
}
|
||||
if cfg.BrowserFetch.Provider != "lightpanda" {
|
||||
t.Fatalf("expected default provider lightpanda, got %q", cfg.BrowserFetch.Provider)
|
||||
}
|
||||
if cfg.BrowserFetch.ServiceURL != "http://browser-fetch:8082" {
|
||||
t.Fatalf("expected normalized service url, got %q", cfg.BrowserFetch.ServiceURL)
|
||||
}
|
||||
if cfg.BrowserFetch.Token != "token" {
|
||||
t.Fatalf("expected normalized token, got %q", cfg.BrowserFetch.Token)
|
||||
}
|
||||
if cfg.BrowserFetch.QueueSize != 64 ||
|
||||
cfg.BrowserFetch.WorkerConcurrency != 1 ||
|
||||
cfg.BrowserFetch.RequestTimeout != 45*time.Second ||
|
||||
cfg.BrowserFetch.CacheTTL != 6*time.Hour {
|
||||
t.Fatalf("unexpected browser fetch defaults: %#v", cfg.BrowserFetch)
|
||||
}
|
||||
if len(cfg.BrowserFetch.AllowedDomains) != 1 || cfg.BrowserFetch.AllowedDomains[0] != "zhuanlan.zhihu.com" {
|
||||
t.Fatalf("unexpected allowed domains: %#v", cfg.BrowserFetch.AllowedDomains)
|
||||
}
|
||||
if len(cfg.BrowserFetch.TriggerStatus) != 2 ||
|
||||
cfg.BrowserFetch.TriggerStatus[0] != 403 ||
|
||||
cfg.BrowserFetch.TriggerStatus[1] != 500 {
|
||||
t.Fatalf("unexpected trigger status: %#v", cfg.BrowserFetch.TriggerStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAppliesBrowserFetchEnvOverrides(t *testing.T) {
|
||||
t.Setenv("BROWSER_FETCH_ENABLED", "true")
|
||||
t.Setenv("BROWSER_FETCH_PROVIDER", "lightpanda_service")
|
||||
t.Setenv("BROWSER_FETCH_SERVICE_URL", "http://env-browser-fetch:8082/")
|
||||
t.Setenv("BROWSER_FETCH_TOKEN", "env-token")
|
||||
t.Setenv("BROWSER_FETCH_QUEUE_SIZE", "11")
|
||||
t.Setenv("BROWSER_FETCH_WORKER_CONCURRENCY", "3")
|
||||
t.Setenv("BROWSER_FETCH_REQUEST_TIMEOUT", "12s")
|
||||
t.Setenv("BROWSER_FETCH_CACHE_TTL", "30m")
|
||||
t.Setenv("BROWSER_FETCH_ALLOWED_DOMAINS", "zhihu.com,example.com")
|
||||
t.Setenv("BROWSER_FETCH_TRIGGER_STATUS", "403,429,broken,503")
|
||||
|
||||
configPath := writeTestConfig(t, `
|
||||
browser_fetch: {}
|
||||
`)
|
||||
|
||||
cfg, err := Load(configPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
|
||||
if !cfg.BrowserFetch.Enabled ||
|
||||
cfg.BrowserFetch.Provider != "lightpanda_service" ||
|
||||
cfg.BrowserFetch.ServiceURL != "http://env-browser-fetch:8082" ||
|
||||
cfg.BrowserFetch.Token != "env-token" ||
|
||||
cfg.BrowserFetch.QueueSize != 11 ||
|
||||
cfg.BrowserFetch.WorkerConcurrency != 3 ||
|
||||
cfg.BrowserFetch.RequestTimeout != 12*time.Second ||
|
||||
cfg.BrowserFetch.CacheTTL != 30*time.Minute {
|
||||
t.Fatalf("unexpected browser fetch env config: %#v", cfg.BrowserFetch)
|
||||
}
|
||||
if got := cfg.BrowserFetch.AllowedDomains; len(got) != 2 || got[0] != "zhihu.com" || got[1] != "example.com" {
|
||||
t.Fatalf("unexpected env allowed domains: %#v", got)
|
||||
}
|
||||
if got := cfg.BrowserFetch.TriggerStatus; len(got) != 3 || got[0] != 403 || got[1] != 429 || got[2] != 503 {
|
||||
t.Fatalf("unexpected env trigger status: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreReloadsConfigAndKeepsPreviousOnInvalidYAML(t *testing.T) {
|
||||
configPath := writeTestConfig(t, `
|
||||
jwt:
|
||||
|
||||
@@ -80,6 +80,12 @@ func Diff(previous, current *Config) []FieldChange {
|
||||
if previous.Retrieval != current.Retrieval {
|
||||
addChange("retrieval", true)
|
||||
}
|
||||
if previous.BrowserFetch.QueueSize != current.BrowserFetch.QueueSize ||
|
||||
previous.BrowserFetch.WorkerConcurrency != current.BrowserFetch.WorkerConcurrency {
|
||||
addChange("browser_fetch.worker", false)
|
||||
} else if !reflect.DeepEqual(previous.BrowserFetch, current.BrowserFetch) {
|
||||
addChange("browser_fetch", true)
|
||||
}
|
||||
if previous.Generation.QueueSize != current.Generation.QueueSize ||
|
||||
previous.Generation.WorkerConcurrency != current.Generation.WorkerConcurrency {
|
||||
addChange("generation.worker", false)
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/config"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
var errBrowserFetchQueueFull = errors.New("browser fetch queue is full")
|
||||
|
||||
type browserFetchResult struct {
|
||||
URL string `json:"url,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Text string `json:"text,omitempty"`
|
||||
Markdown string `json:"markdown,omitempty"`
|
||||
}
|
||||
|
||||
type browserFetchRequest struct {
|
||||
URL string `json:"url"`
|
||||
TimeoutMS int64 `json:"timeout_ms,omitempty"`
|
||||
Options map[string]interface{} `json:"options,omitempty"`
|
||||
ctx context.Context `json:"-"`
|
||||
cfg config.BrowserFetchConfig `json:"-"`
|
||||
resp chan browserFetchRequestResult `json:"-"`
|
||||
}
|
||||
|
||||
type browserFetchRequestResult struct {
|
||||
result browserFetchResult
|
||||
err error
|
||||
}
|
||||
|
||||
type browserFetchQueue struct {
|
||||
logger *zap.Logger
|
||||
client *http.Client
|
||||
jobs chan browserFetchRequest
|
||||
cache *browserFetchMemoryCache
|
||||
}
|
||||
|
||||
type browserFetchCacheEntry struct {
|
||||
result browserFetchResult
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
type browserFetchMemoryCache struct {
|
||||
mu sync.Mutex
|
||||
entries map[string]browserFetchCacheEntry
|
||||
}
|
||||
|
||||
func (s *KnowledgeService) browserFetchQueue(cfg config.BrowserFetchConfig) *browserFetchQueue {
|
||||
s.browserFetchMu.Lock()
|
||||
defer s.browserFetchMu.Unlock()
|
||||
if s.browserFetchQ != nil {
|
||||
return s.browserFetchQ
|
||||
}
|
||||
|
||||
config.NormalizeBrowserFetchConfig(&cfg)
|
||||
queue := &browserFetchQueue{
|
||||
logger: s.logger,
|
||||
client: &http.Client{},
|
||||
jobs: make(chan browserFetchRequest, cfg.QueueSize),
|
||||
cache: &browserFetchMemoryCache{
|
||||
entries: make(map[string]browserFetchCacheEntry),
|
||||
},
|
||||
}
|
||||
|
||||
for i := 0; i < cfg.WorkerConcurrency; i++ {
|
||||
go queue.runWorker(i)
|
||||
}
|
||||
s.browserFetchQ = queue
|
||||
return queue
|
||||
}
|
||||
|
||||
func (q *browserFetchQueue) Fetch(ctx context.Context, rawURL string, cfg config.BrowserFetchConfig) (browserFetchResult, error) {
|
||||
if q == nil {
|
||||
return browserFetchResult{}, errBrowserFetchNotAvailable
|
||||
}
|
||||
config.NormalizeBrowserFetchConfig(&cfg)
|
||||
cacheKey := browserFetchCacheKey(rawURL)
|
||||
if cfg.CacheTTL > 0 {
|
||||
if result, ok := q.cache.Get(cacheKey); ok {
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
|
||||
request := browserFetchRequest{
|
||||
URL: rawURL,
|
||||
TimeoutMS: int64(cfg.RequestTimeout / time.Millisecond),
|
||||
Options: map[string]interface{}{
|
||||
"provider": cfg.Provider,
|
||||
},
|
||||
ctx: ctx,
|
||||
cfg: cfg,
|
||||
resp: make(chan browserFetchRequestResult, 1),
|
||||
}
|
||||
|
||||
select {
|
||||
case q.jobs <- request:
|
||||
case <-ctx.Done():
|
||||
return browserFetchResult{}, ctx.Err()
|
||||
default:
|
||||
return browserFetchResult{}, errBrowserFetchQueueFull
|
||||
}
|
||||
|
||||
select {
|
||||
case result := <-request.resp:
|
||||
if result.err != nil {
|
||||
return browserFetchResult{}, result.err
|
||||
}
|
||||
if cfg.CacheTTL > 0 {
|
||||
q.cache.Set(cacheKey, result.result, cfg.CacheTTL)
|
||||
}
|
||||
return result.result, nil
|
||||
case <-ctx.Done():
|
||||
return browserFetchResult{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (q *browserFetchQueue) runWorker(workerID int) {
|
||||
for request := range q.jobs {
|
||||
result, err := q.execute(context.Background(), request)
|
||||
if err != nil && q.logger != nil {
|
||||
q.logger.Warn("browser fetch request failed",
|
||||
zap.Int("worker_id", workerID),
|
||||
zap.String("url", request.URL),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
request.resp <- browserFetchRequestResult{result: result, err: err}
|
||||
}
|
||||
}
|
||||
|
||||
func (q *browserFetchQueue) execute(parent context.Context, request browserFetchRequest) (browserFetchResult, error) {
|
||||
if request.ctx != nil {
|
||||
parent = request.ctx
|
||||
}
|
||||
timeout := time.Duration(request.TimeoutMS) * time.Millisecond
|
||||
if timeout <= 0 {
|
||||
timeout = 45 * time.Second
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(parent, timeout)
|
||||
defer cancel()
|
||||
|
||||
return q.callService(ctx, request.URL, request.cfg)
|
||||
}
|
||||
|
||||
func (q *browserFetchQueue) callService(ctx context.Context, rawURL string, cfg config.BrowserFetchConfig) (browserFetchResult, error) {
|
||||
config.NormalizeBrowserFetchConfig(&cfg)
|
||||
endpoint := strings.TrimRight(cfg.ServiceURL, "/") + "/fetch"
|
||||
payload := browserFetchRequest{
|
||||
URL: rawURL,
|
||||
TimeoutMS: int64(cfg.RequestTimeout / time.Millisecond),
|
||||
}
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return browserFetchResult{}, fmt.Errorf("marshal browser fetch request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return browserFetchResult{}, fmt.Errorf("create browser fetch request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if cfg.Token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+cfg.Token)
|
||||
}
|
||||
|
||||
resp, err := q.client.Do(req)
|
||||
if err != nil {
|
||||
return browserFetchResult{}, fmt.Errorf("execute browser fetch request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= http.StatusBadRequest {
|
||||
data, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||
return browserFetchResult{}, fmt.Errorf("browser fetch service failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(data)))
|
||||
}
|
||||
|
||||
var result browserFetchResult
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return browserFetchResult{}, fmt.Errorf("decode browser fetch response: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *browserFetchMemoryCache) Get(key string) (browserFetchResult, bool) {
|
||||
if c == nil || key == "" {
|
||||
return browserFetchResult{}, false
|
||||
}
|
||||
now := time.Now()
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
entry, ok := c.entries[key]
|
||||
if !ok {
|
||||
return browserFetchResult{}, false
|
||||
}
|
||||
if now.After(entry.expiresAt) {
|
||||
delete(c.entries, key)
|
||||
return browserFetchResult{}, false
|
||||
}
|
||||
return entry.result, true
|
||||
}
|
||||
|
||||
func (c *browserFetchMemoryCache) Set(key string, result browserFetchResult, ttl time.Duration) {
|
||||
if c == nil || key == "" || ttl <= 0 {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.entries[key] = browserFetchCacheEntry{
|
||||
result: result,
|
||||
expiresAt: time.Now().Add(ttl),
|
||||
}
|
||||
}
|
||||
|
||||
func browserFetchCacheKey(rawURL string) string {
|
||||
normalized := strings.TrimSpace(rawURL)
|
||||
if parsed, err := url.Parse(normalized); err == nil && parsed.Host != "" {
|
||||
parsed.Fragment = ""
|
||||
normalized = parsed.String()
|
||||
}
|
||||
sum := sha256.Sum256([]byte(normalized))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -49,6 +49,8 @@ type KnowledgeService struct {
|
||||
jobs chan knowledgeParseJob
|
||||
cleanupMu sync.Mutex
|
||||
cleanupInFlight map[string]struct{}
|
||||
browserFetchMu sync.Mutex
|
||||
browserFetchQ *browserFetchQueue
|
||||
}
|
||||
|
||||
type knowledgeParseJob struct {
|
||||
@@ -199,7 +201,7 @@ func (s *KnowledgeService) WithConfigProvider(provider runtimeConfigProvider) *K
|
||||
func (s *KnowledgeService) runtimeConfig() knowledgeRuntimeConfig {
|
||||
if s != nil && s.configProvider != nil {
|
||||
if cfg := s.configProvider.Current(); cfg != nil {
|
||||
return knowledgeRuntimeFromConfig(cfg.Retrieval, cfg.LLM)
|
||||
return knowledgeRuntimeFromFullConfig(cfg)
|
||||
}
|
||||
}
|
||||
if s == nil {
|
||||
|
||||
@@ -4,12 +4,16 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/config"
|
||||
"github.com/geo-platform/tenant-api/internal/shared/llm"
|
||||
"github.com/geo-platform/tenant-api/internal/tenant/prompts"
|
||||
"go.uber.org/zap"
|
||||
@@ -23,6 +27,37 @@ const (
|
||||
defaultKnowledgeURLMarkdownOutputToken = 12000
|
||||
)
|
||||
|
||||
var errBrowserFetchNotAvailable = fmt.Errorf("browser fetch fallback is not available")
|
||||
|
||||
type sourceFetchRestrictedError struct {
|
||||
URL string
|
||||
Status int
|
||||
Cause error
|
||||
}
|
||||
|
||||
func (e sourceFetchRestrictedError) Error() string {
|
||||
cause := ""
|
||||
if e.Cause != nil {
|
||||
cause = strings.TrimSpace(e.Cause.Error())
|
||||
}
|
||||
if cause == "" {
|
||||
cause = "source website requires browser rendering or blocks server-side fetch"
|
||||
}
|
||||
if e.Status > 0 {
|
||||
return fmt.Sprintf("source website fetch restricted status=%d: %s", e.Status, cause)
|
||||
}
|
||||
return "source website fetch restricted: " + cause
|
||||
}
|
||||
|
||||
func (e sourceFetchRestrictedError) Unwrap() error {
|
||||
return e.Cause
|
||||
}
|
||||
|
||||
func isSourceFetchRestrictedError(err error) bool {
|
||||
var restricted sourceFetchRestrictedError
|
||||
return errors.As(err, &restricted)
|
||||
}
|
||||
|
||||
type arkToolExecuteRequest struct {
|
||||
ActionName string `json:"action_name"`
|
||||
ToolName string `json:"tool_name"`
|
||||
@@ -50,7 +85,20 @@ type arkWebDataItem struct {
|
||||
func (s *KnowledgeService) parseWebsiteKnowledge(ctx context.Context, rawURL string) (*knowledgeParsedContent, error) {
|
||||
item, err := s.executeLinkReader(ctx, rawURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
if !s.shouldTryBrowserFetchFallback(rawURL, err) {
|
||||
return nil, classifySourceFetchError(rawURL, err)
|
||||
}
|
||||
fallback, fallbackErr := s.parseWebsiteKnowledgeWithBrowserFetch(ctx, rawURL)
|
||||
if fallbackErr == nil {
|
||||
return fallback, nil
|
||||
}
|
||||
if s.logger != nil {
|
||||
s.logger.Warn("knowledge browser fetch fallback failed",
|
||||
zap.String("url", rawURL),
|
||||
zap.Error(fallbackErr),
|
||||
)
|
||||
}
|
||||
return nil, classifySourceFetchError(rawURL, err)
|
||||
}
|
||||
|
||||
text := normalizeKnowledgeText(item.Content)
|
||||
@@ -76,6 +124,65 @@ func (s *KnowledgeService) parseWebsiteKnowledge(ctx context.Context, rawURL str
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *KnowledgeService) parseWebsiteKnowledgeWithBrowserFetch(ctx context.Context, rawURL string) (*knowledgeParsedContent, error) {
|
||||
runtimeCfg := s.runtimeConfig()
|
||||
browserCfg := runtimeCfg.BrowserFetch
|
||||
config.NormalizeBrowserFetchConfig(&browserCfg)
|
||||
if !browserCfg.Enabled {
|
||||
return nil, errBrowserFetchNotAvailable
|
||||
}
|
||||
|
||||
provider := strings.ToLower(strings.TrimSpace(browserCfg.Provider))
|
||||
if provider != "lightpanda" && provider != "lightpanda_service" {
|
||||
return nil, fmt.Errorf("unsupported browser fetch provider %q", browserCfg.Provider)
|
||||
}
|
||||
|
||||
if strings.TrimSpace(browserCfg.ServiceURL) == "" {
|
||||
return nil, fmt.Errorf("%w: browser_fetch.service_url is empty", errBrowserFetchNotAvailable)
|
||||
}
|
||||
|
||||
callCtx := ctx
|
||||
if browserCfg.RequestTimeout > 0 {
|
||||
var cancel context.CancelFunc
|
||||
callCtx, cancel = context.WithTimeout(ctx, browserCfg.RequestTimeout)
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
result, err := s.browserFetchQueue(browserCfg).Fetch(callCtx, rawURL, browserCfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
text := normalizeKnowledgeText(result.Text)
|
||||
if text == "" {
|
||||
text = normalizeKnowledgeText(result.Markdown)
|
||||
}
|
||||
if text == "" {
|
||||
return nil, fmt.Errorf("browser fetch returned empty content")
|
||||
}
|
||||
|
||||
title := strings.TrimSpace(result.Title)
|
||||
markdown := strings.TrimSpace(result.Markdown)
|
||||
if markdown == "" {
|
||||
var err error
|
||||
markdown, err = s.formatWebsiteMarkdown(ctx, title, text, rawURL)
|
||||
if err != nil {
|
||||
if s.logger != nil {
|
||||
s.logger.Warn("knowledge browser fetch markdown formatting failed, fallback to local markdown",
|
||||
zap.String("url", rawURL),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
markdown = buildKnowledgeMarkdownFromText(title, text)
|
||||
}
|
||||
}
|
||||
|
||||
return &knowledgeParsedContent{
|
||||
Text: text,
|
||||
Markdown: markdown,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *KnowledgeService) executeLinkReader(ctx context.Context, rawURL string) (*arkWebDataItem, error) {
|
||||
runtimeCfg := s.runtimeConfig()
|
||||
baseURL := strings.TrimRight(strings.TrimSpace(runtimeCfg.ArkBaseURL), "/")
|
||||
@@ -135,6 +242,90 @@ func (s *KnowledgeService) executeLinkReader(ctx context.Context, rawURL string)
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
func classifySourceFetchError(rawURL string, err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
status := extractHTTPStatusFromError(err)
|
||||
if status == http.StatusForbidden || status == http.StatusTooManyRequests || status >= http.StatusInternalServerError {
|
||||
return sourceFetchRestrictedError{URL: rawURL, Status: status, Cause: err}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func extractHTTPStatusFromError(err error) int {
|
||||
if err == nil {
|
||||
return 0
|
||||
}
|
||||
message := err.Error()
|
||||
idx := strings.Index(message, "status=")
|
||||
if idx < 0 {
|
||||
return 0
|
||||
}
|
||||
value := message[idx+len("status="):]
|
||||
end := 0
|
||||
for end < len(value) && value[end] >= '0' && value[end] <= '9' {
|
||||
end++
|
||||
}
|
||||
if end == 0 {
|
||||
return 0
|
||||
}
|
||||
status, parseErr := strconv.Atoi(value[:end])
|
||||
if parseErr != nil {
|
||||
return 0
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
func (s *KnowledgeService) shouldTryBrowserFetchFallback(rawURL string, err error) bool {
|
||||
if s == nil {
|
||||
return false
|
||||
}
|
||||
runtimeCfg := s.runtimeConfig()
|
||||
browserCfg := runtimeCfg.BrowserFetch
|
||||
config.NormalizeBrowserFetchConfig(&browserCfg)
|
||||
if !browserCfg.Enabled {
|
||||
return false
|
||||
}
|
||||
if !browserFetchDomainAllowed(rawURL, browserCfg.AllowedDomains) {
|
||||
return false
|
||||
}
|
||||
status := extractHTTPStatusFromError(err)
|
||||
if status <= 0 {
|
||||
return false
|
||||
}
|
||||
for _, trigger := range browserCfg.TriggerStatus {
|
||||
if status == trigger {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func browserFetchDomainAllowed(rawURL string, allowedDomains []string) bool {
|
||||
if len(allowedDomains) == 0 {
|
||||
return false
|
||||
}
|
||||
parsed, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(strings.TrimSpace(parsed.Hostname()))
|
||||
if host == "" {
|
||||
return false
|
||||
}
|
||||
for _, domain := range allowedDomains {
|
||||
domain = strings.ToLower(strings.TrimPrefix(strings.TrimSpace(domain), "."))
|
||||
if domain == "" {
|
||||
continue
|
||||
}
|
||||
if host == domain || strings.HasSuffix(host, "."+domain) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *KnowledgeService) formatWebsiteMarkdown(ctx context.Context, title, content, rawURL string) (string, error) {
|
||||
content = strings.TrimSpace(content)
|
||||
if content == "" {
|
||||
|
||||
@@ -26,6 +26,7 @@ type knowledgeRuntimeConfig struct {
|
||||
ArkBaseURL string
|
||||
ArkAPIKey string
|
||||
URLMarkdownModel string
|
||||
BrowserFetch config.BrowserFetchConfig
|
||||
}
|
||||
|
||||
type generationStreamDisabledConfigProvider struct {
|
||||
@@ -103,3 +104,14 @@ func knowledgeRuntimeFromConfig(retrievalCfg config.RetrievalConfig, llmCfg conf
|
||||
URLMarkdownModel: model,
|
||||
}
|
||||
}
|
||||
|
||||
func knowledgeRuntimeFromFullConfig(cfg *config.Config) knowledgeRuntimeConfig {
|
||||
if cfg == nil {
|
||||
return knowledgeRuntimeFromConfig(config.RetrievalConfig{}, config.LLMConfig{})
|
||||
}
|
||||
runtime := knowledgeRuntimeFromConfig(cfg.Retrieval, cfg.LLM)
|
||||
browserFetch := cfg.BrowserFetch
|
||||
config.NormalizeBrowserFetchConfig(&browserFetch)
|
||||
runtime.BrowserFetch = browserFetch
|
||||
return runtime
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user