root/geo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(browser-fetch): add lightpanda-backed fetch service and knowledge URL fallback

Browse Source 
Adds a new browser-fetch microservice that wraps Lightpanda for rendering
JS-heavy pages, and wires it into the knowledge URL parser as a fallback
when the upstream link reader returns 403/429/5xx for an allow-listed
domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload
diff support, and full deploy assets (Dockerfile target, docker-compose,
k3s manifests, image-build/package scripts).
This commit is contained in:
liangxu
2026-05-11 11:11:21 +08:00
parent 708a45ba16
commit 7f08d92958
22 changed files with 1770 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/shared/config/config.go
+110
View File
@@ -38,6 +38,7 @@ type Config struct {
Compliance ComplianceConfig `mapstructure:"compliance"`
Retrieval RetrievalConfig `mapstructure:"retrieval"`
Generation GenerationConfig `mapstructure:"generation"`
BrowserFetch BrowserFetchConfig `mapstructure:"browser_fetch"`
}
type ServerConfig struct {
@@ -348,6 +349,19 @@ type GenerationConfig struct {
TaskStateCheckLookback time.Duration `mapstructure:"task_state_check_lookback"`
}
type BrowserFetchConfig struct {
Enabled bool `mapstructure:"enabled"`
Provider string `mapstructure:"provider"`
ServiceURL string `mapstructure:"service_url"`
Token string `mapstructure:"token"`
QueueSize int `mapstructure:"queue_size"`
WorkerConcurrency int `mapstructure:"worker_concurrency"`
RequestTimeout time.Duration `mapstructure:"request_timeout"`
CacheTTL time.Duration `mapstructure:"cache_ttl"`
AllowedDomains []string `mapstructure:"allowed_domains"`
TriggerStatus []int `mapstructure:"trigger_status"`
}
func Load(configPath string) (*Config, error) {
cfg, _, err := loadWithFiles(configPath)
return cfg, err
@@ -382,6 +396,7 @@ func loadWithFiles(configPath string) (*Config, []string, error) {
normalizeBrandLibraryConfig(&cfg.BrandLibrary)
NormalizeComplianceConfig(&cfg.Compliance)
NormalizeGenerationConfig(&cfg.Generation)
NormalizeBrowserFetchConfig(&cfg.BrowserFetch)
files := []string{configFile}
if localConfigFile != "" {
@@ -511,6 +526,42 @@ func compactStringSlice(values []string) []string {
return out
}
func compactIntSlice(values []int) []int {
if values == nil {
return nil
}
out := make([]int, 0, len(values))
seen := make(map[int]struct{}, len(values))
for _, value := range values {
if value <= 0 {
continue
}
if _, exists := seen[value]; exists {
continue
}
seen[value] = struct{}{}
out = append(out, value)
}
return out
}
func parseIntList(value string) []int {
parts := strings.Split(value, ",")
out := make([]int, 0, len(parts))
for _, part := range parts {
part = strings.TrimSpace(part)
if part == "" {
continue
}
n, err := strconv.Atoi(part)
if err != nil {
continue
}
out = append(out, n)
}
return out
}
func normalizeMonitoringDispatchConfig(cfg *MonitoringDispatchConfig) {
if cfg == nil {
return
@@ -643,6 +694,35 @@ func NormalizeGenerationConfig(cfg *GenerationConfig) {
}
}
func NormalizeBrowserFetchConfig(cfg *BrowserFetchConfig) {
if cfg == nil {
return
}
cfg.Provider = strings.ToLower(strings.TrimSpace(cfg.Provider))
if cfg.Provider == "" {
cfg.Provider = "lightpanda"
}
cfg.ServiceURL = strings.TrimRight(strings.TrimSpace(cfg.ServiceURL), "/")
cfg.Token = strings.TrimSpace(cfg.Token)
if cfg.QueueSize <= 0 {
cfg.QueueSize = 64
}
if cfg.WorkerConcurrency <= 0 {
cfg.WorkerConcurrency = 1
}
if cfg.RequestTimeout <= 0 {
cfg.RequestTimeout = 45 * time.Second
}
if cfg.CacheTTL <= 0 {
cfg.CacheTTL = 6 * time.Hour
}
cfg.AllowedDomains = compactStringSlice(cfg.AllowedDomains)
cfg.TriggerStatus = compactIntSlice(cfg.TriggerStatus)
if len(cfg.TriggerStatus) == 0 {
cfg.TriggerStatus = []int{403, 429, 500, 502, 503, 504}
}
}
func NormalizeComplianceConfig(cfg *ComplianceConfig) {
if cfg == nil {
return
@@ -759,6 +839,36 @@ func applyEnvOverrides(cfg *Config) {
if d, ok := lookupDurationEnv("GENERATION_TASK_STATE_CHECK_LOOKBACK"); ok {
cfg.Generation.TaskStateCheckLookback = d
}
if enabled, ok := lookupBoolEnv("BROWSER_FETCH_ENABLED"); ok {
cfg.BrowserFetch.Enabled = enabled
}
if provider, ok := lookupNonEmptyEnv("BROWSER_FETCH_PROVIDER"); ok {
cfg.BrowserFetch.Provider = provider
}
if serviceURL, ok := lookupFirstNonEmptyEnv("BROWSER_FETCH_SERVICE_URL", "LIGHTPANDA_BROWSER_FETCH_URL"); ok {
cfg.BrowserFetch.ServiceURL = serviceURL
}
if token, ok := lookupNonEmptyEnv("BROWSER_FETCH_TOKEN"); ok {
cfg.BrowserFetch.Token = token
}
if n, ok := lookupIntEnv("BROWSER_FETCH_QUEUE_SIZE"); ok {
cfg.BrowserFetch.QueueSize = n
}
if n, ok := lookupIntEnv("BROWSER_FETCH_WORKER_CONCURRENCY"); ok {
cfg.BrowserFetch.WorkerConcurrency = n
}
if d, ok := lookupDurationEnv("BROWSER_FETCH_REQUEST_TIMEOUT"); ok {
cfg.BrowserFetch.RequestTimeout = d
}
if d, ok := lookupDurationEnv("BROWSER_FETCH_CACHE_TTL"); ok {
cfg.BrowserFetch.CacheTTL = d
}
if domains, ok := lookupNonEmptyEnv("BROWSER_FETCH_ALLOWED_DOMAINS"); ok {
cfg.BrowserFetch.AllowedDomains = strings.Split(domains, ",")
}
if statuses, ok := lookupNonEmptyEnv("BROWSER_FETCH_TRIGGER_STATUS"); ok {
cfg.BrowserFetch.TriggerStatus = parseIntList(statuses)
}
if apiKey, ok := lookupNonEmptyEnv("QDRANT_API_KEY"); ok {
cfg.Qdrant.APIKey = apiKey
}
server/internal/shared/config/config_test.go
+87
View File
@@ -557,6 +557,93 @@ generation: {}
}
}
func TestLoadAppliesBrowserFetchDefaultsAndOverrides(t *testing.T) {
configPath := writeTestConfig(t, `
browser_fetch:
enabled: true
service_url: " http://browser-fetch:8082/ "
token: " token "
allowed_domains:
- " zhuanlan.zhihu.com "
- ""
trigger_status:
- 403
- 403
- 500
`)
cfg, err := Load(configPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if !cfg.BrowserFetch.Enabled {
t.Fatalf("expected browser fetch enabled")
}
if cfg.BrowserFetch.Provider != "lightpanda" {
t.Fatalf("expected default provider lightpanda, got %q", cfg.BrowserFetch.Provider)
}
if cfg.BrowserFetch.ServiceURL != "http://browser-fetch:8082" {
t.Fatalf("expected normalized service url, got %q", cfg.BrowserFetch.ServiceURL)
}
if cfg.BrowserFetch.Token != "token" {
t.Fatalf("expected normalized token, got %q", cfg.BrowserFetch.Token)
}
if cfg.BrowserFetch.QueueSize != 64 ||
cfg.BrowserFetch.WorkerConcurrency != 1 ||
cfg.BrowserFetch.RequestTimeout != 45*time.Second ||
cfg.BrowserFetch.CacheTTL != 6*time.Hour {
t.Fatalf("unexpected browser fetch defaults: %#v", cfg.BrowserFetch)
}
if len(cfg.BrowserFetch.AllowedDomains) != 1 || cfg.BrowserFetch.AllowedDomains[0] != "zhuanlan.zhihu.com" {
t.Fatalf("unexpected allowed domains: %#v", cfg.BrowserFetch.AllowedDomains)
}
if len(cfg.BrowserFetch.TriggerStatus) != 2 ||
cfg.BrowserFetch.TriggerStatus[0] != 403 ||
cfg.BrowserFetch.TriggerStatus[1] != 500 {
t.Fatalf("unexpected trigger status: %#v", cfg.BrowserFetch.TriggerStatus)
}
}
func TestLoadAppliesBrowserFetchEnvOverrides(t *testing.T) {
t.Setenv("BROWSER_FETCH_ENABLED", "true")
t.Setenv("BROWSER_FETCH_PROVIDER", "lightpanda_service")
t.Setenv("BROWSER_FETCH_SERVICE_URL", "http://env-browser-fetch:8082/")
t.Setenv("BROWSER_FETCH_TOKEN", "env-token")
t.Setenv("BROWSER_FETCH_QUEUE_SIZE", "11")
t.Setenv("BROWSER_FETCH_WORKER_CONCURRENCY", "3")
t.Setenv("BROWSER_FETCH_REQUEST_TIMEOUT", "12s")
t.Setenv("BROWSER_FETCH_CACHE_TTL", "30m")
t.Setenv("BROWSER_FETCH_ALLOWED_DOMAINS", "zhihu.com,example.com")
t.Setenv("BROWSER_FETCH_TRIGGER_STATUS", "403,429,broken,503")
configPath := writeTestConfig(t, `
browser_fetch: {}
`)
cfg, err := Load(configPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if !cfg.BrowserFetch.Enabled ||
cfg.BrowserFetch.Provider != "lightpanda_service" ||
cfg.BrowserFetch.ServiceURL != "http://env-browser-fetch:8082" ||
cfg.BrowserFetch.Token != "env-token" ||
cfg.BrowserFetch.QueueSize != 11 ||
cfg.BrowserFetch.WorkerConcurrency != 3 ||
cfg.BrowserFetch.RequestTimeout != 12*time.Second ||
cfg.BrowserFetch.CacheTTL != 30*time.Minute {
t.Fatalf("unexpected browser fetch env config: %#v", cfg.BrowserFetch)
}
if got := cfg.BrowserFetch.AllowedDomains; len(got) != 2 || got[0] != "zhihu.com" || got[1] != "example.com" {
t.Fatalf("unexpected env allowed domains: %#v", got)
}
if got := cfg.BrowserFetch.TriggerStatus; len(got) != 3 || got[0] != 403 || got[1] != 429 || got[2] != 503 {
t.Fatalf("unexpected env trigger status: %#v", got)
}
}
func TestStoreReloadsConfigAndKeepsPreviousOnInvalidYAML(t *testing.T) {
configPath := writeTestConfig(t, `
jwt:
server/internal/shared/config/reload.go
+6
View File
@@ -80,6 +80,12 @@ func Diff(previous, current *Config) []FieldChange {
if previous.Retrieval != current.Retrieval {
addChange("retrieval", true)
}
if previous.BrowserFetch.QueueSize != current.BrowserFetch.QueueSize ||
previous.BrowserFetch.WorkerConcurrency != current.BrowserFetch.WorkerConcurrency {
addChange("browser_fetch.worker", false)
} else if !reflect.DeepEqual(previous.BrowserFetch, current.BrowserFetch) {
addChange("browser_fetch", true)
}
if previous.Generation.QueueSize != current.Generation.QueueSize ||
previous.Generation.WorkerConcurrency != current.Generation.WorkerConcurrency {
addChange("generation.worker", false)