feat(browser-fetch): add lightpanda-backed fetch service and knowledge URL fallback
Adds a new browser-fetch microservice that wraps Lightpanda for rendering JS-heavy pages, and wires it into the knowledge URL parser as a fallback when the upstream link reader returns 403/429/5xx for an allow-listed domain (e.g. zhuanlan.zhihu.com). Includes config/env plumbing, hot-reload diff support, and full deploy assets (Dockerfile target, docker-compose, k3s manifests, image-build/package scripts).
This commit is contained in:
@@ -0,0 +1,877 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultPort = 8082
|
||||
defaultQueueSize = 128
|
||||
defaultWorkerConcurrency = 2
|
||||
defaultPerDomainConcurrency = 1
|
||||
defaultRequestTimeout = 45 * time.Second
|
||||
defaultCacheTTL = 6 * time.Hour
|
||||
defaultBreakerThreshold = 5
|
||||
defaultBreakerCooldown = 2 * time.Minute
|
||||
)
|
||||
|
||||
type fetchConfig struct {
|
||||
Port int
|
||||
Token string
|
||||
QueueSize int
|
||||
WorkerConcurrency int
|
||||
PerDomainConcurrency int
|
||||
RequestTimeout time.Duration
|
||||
CacheTTL time.Duration
|
||||
AllowedDomains []string
|
||||
LightpandaDriver string
|
||||
LightpandaBin string
|
||||
LightpandaArgs []string
|
||||
LightpandaFetchArgs []string
|
||||
WorkerMaxRequests int
|
||||
BreakerThreshold int
|
||||
BreakerCooldown time.Duration
|
||||
}
|
||||
|
||||
type fetchService struct {
|
||||
cfg fetchConfig
|
||||
jobs chan fetchJob
|
||||
cache *memoryCache
|
||||
limiters *domainLimiters
|
||||
breakers *domainBreakers
|
||||
metrics *fetchMetrics
|
||||
shuttingDown atomic.Bool
|
||||
}
|
||||
|
||||
type fetchJob struct {
|
||||
rawURL string
|
||||
timeout time.Duration
|
||||
host string
|
||||
release func()
|
||||
resp chan fetchJobResult
|
||||
}
|
||||
|
||||
type fetchJobResult struct {
|
||||
result fetchResponse
|
||||
err error
|
||||
}
|
||||
|
||||
type fetchRequest struct {
|
||||
URL string `json:"url"`
|
||||
TimeoutMS int64 `json:"timeout_ms,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
}
|
||||
|
||||
type fetchResponse struct {
|
||||
URL string `json:"url,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Text string `json:"text,omitempty"`
|
||||
Markdown string `json:"markdown,omitempty"`
|
||||
}
|
||||
|
||||
type memoryCacheEntry struct {
|
||||
result fetchResponse
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
type memoryCache struct {
|
||||
mu sync.Mutex
|
||||
entries map[string]memoryCacheEntry
|
||||
}
|
||||
|
||||
type domainLimiters struct {
|
||||
mu sync.Mutex
|
||||
capacity int
|
||||
items map[string]chan struct{}
|
||||
}
|
||||
|
||||
type domainBreakerState struct {
|
||||
failures int
|
||||
openedAt time.Time
|
||||
}
|
||||
|
||||
type domainBreakers struct {
|
||||
mu sync.Mutex
|
||||
threshold int
|
||||
cooldown time.Duration
|
||||
items map[string]domainBreakerState
|
||||
}
|
||||
|
||||
type fetchMetrics struct {
|
||||
accepted atomic.Int64
|
||||
cacheHits atomic.Int64
|
||||
queueFull atomic.Int64
|
||||
domainLimited atomic.Int64
|
||||
authFailed atomic.Int64
|
||||
success atomic.Int64
|
||||
failed atomic.Int64
|
||||
breakerOpen atomic.Int64
|
||||
}
|
||||
|
||||
type lightpandaWorker struct {
|
||||
id int
|
||||
cfg fetchConfig
|
||||
client *mcpClient
|
||||
handled int
|
||||
}
|
||||
|
||||
type mcpClient struct {
|
||||
cfg fetchConfig
|
||||
cmd *exec.Cmd
|
||||
stdin io.WriteCloser
|
||||
stdout *bufio.Scanner
|
||||
stderr bytes.Buffer
|
||||
nextID int64
|
||||
started bool
|
||||
}
|
||||
|
||||
type mcpResponse struct {
|
||||
JSONRPC string `json:"jsonrpc"`
|
||||
ID int64 `json:"id,omitempty"`
|
||||
Result json.RawMessage `json:"result,omitempty"`
|
||||
Error *mcpError `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type mcpError struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
type mcpToolResult struct {
|
||||
Content []mcpContentItem `json:"content"`
|
||||
IsError bool `json:"isError"`
|
||||
}
|
||||
|
||||
type mcpContentItem struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg := loadFetchConfig()
|
||||
if len(os.Args) > 1 && os.Args[1] == "-healthcheck" {
|
||||
runHealthcheck(cfg)
|
||||
return
|
||||
}
|
||||
service := newFetchService(cfg)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
service.Start(ctx)
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/health/live", service.handleLive)
|
||||
mux.HandleFunc("/health/ready", service.handleReady)
|
||||
mux.HandleFunc("/metrics", service.handleMetrics)
|
||||
mux.HandleFunc("/fetch", service.handleFetch)
|
||||
|
||||
server := &http.Server{
|
||||
Addr: fmt.Sprintf(":%d", cfg.Port),
|
||||
Handler: mux,
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
}
|
||||
|
||||
log.Printf("browser-fetch listening on :%d workers=%d queue=%d domains=%v", cfg.Port, cfg.WorkerConcurrency, cfg.QueueSize, cfg.AllowedDomains)
|
||||
if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
log.Fatalf("browser-fetch server stopped: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func runHealthcheck(cfg fetchConfig) {
|
||||
client := &http.Client{Timeout: 2 * time.Second}
|
||||
resp, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/health/ready", cfg.Port))
|
||||
if err != nil {
|
||||
log.Fatalf("healthcheck failed: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= http.StatusBadRequest {
|
||||
log.Fatalf("healthcheck failed: status=%d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func loadFetchConfig() fetchConfig {
|
||||
cfg := fetchConfig{
|
||||
Port: envInt("BROWSER_FETCH_PORT", defaultPort),
|
||||
Token: strings.TrimSpace(os.Getenv("BROWSER_FETCH_TOKEN")),
|
||||
QueueSize: envInt("BROWSER_FETCH_QUEUE_SIZE", defaultQueueSize),
|
||||
WorkerConcurrency: envInt("BROWSER_FETCH_WORKER_CONCURRENCY", defaultWorkerConcurrency),
|
||||
PerDomainConcurrency: envInt("BROWSER_FETCH_PER_DOMAIN_CONCURRENCY", defaultPerDomainConcurrency),
|
||||
RequestTimeout: envDuration("BROWSER_FETCH_REQUEST_TIMEOUT", defaultRequestTimeout),
|
||||
CacheTTL: envDuration("BROWSER_FETCH_CACHE_TTL", defaultCacheTTL),
|
||||
AllowedDomains: splitCSV(os.Getenv("BROWSER_FETCH_ALLOWED_DOMAINS")),
|
||||
LightpandaDriver: strings.ToLower(strings.TrimSpace(envString("LIGHTPANDA_DRIVER", "fetch"))),
|
||||
LightpandaBin: strings.TrimSpace(envString("LIGHTPANDA_BIN", "lightpanda")),
|
||||
LightpandaArgs: splitCommandArgs(envString("LIGHTPANDA_ARGS", "mcp")),
|
||||
LightpandaFetchArgs: splitCommandArgs(envString("LIGHTPANDA_FETCH_ARGS", "--dump markdown")),
|
||||
WorkerMaxRequests: envInt("LIGHTPANDA_WORKER_MAX_REQUESTS", 100),
|
||||
BreakerThreshold: envInt("BROWSER_FETCH_BREAKER_THRESHOLD", defaultBreakerThreshold),
|
||||
BreakerCooldown: envDuration("BROWSER_FETCH_BREAKER_COOLDOWN", defaultBreakerCooldown),
|
||||
}
|
||||
if cfg.QueueSize <= 0 {
|
||||
cfg.QueueSize = defaultQueueSize
|
||||
}
|
||||
if cfg.WorkerConcurrency <= 0 {
|
||||
cfg.WorkerConcurrency = defaultWorkerConcurrency
|
||||
}
|
||||
if cfg.PerDomainConcurrency <= 0 {
|
||||
cfg.PerDomainConcurrency = defaultPerDomainConcurrency
|
||||
}
|
||||
if cfg.RequestTimeout <= 0 {
|
||||
cfg.RequestTimeout = defaultRequestTimeout
|
||||
}
|
||||
if cfg.CacheTTL < 0 {
|
||||
cfg.CacheTTL = 0
|
||||
}
|
||||
if cfg.BreakerThreshold <= 0 {
|
||||
cfg.BreakerThreshold = defaultBreakerThreshold
|
||||
}
|
||||
if cfg.BreakerCooldown <= 0 {
|
||||
cfg.BreakerCooldown = defaultBreakerCooldown
|
||||
}
|
||||
if cfg.LightpandaDriver == "" {
|
||||
cfg.LightpandaDriver = "fetch"
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func newFetchService(cfg fetchConfig) *fetchService {
|
||||
return &fetchService{
|
||||
cfg: cfg,
|
||||
jobs: make(chan fetchJob, cfg.QueueSize),
|
||||
cache: &memoryCache{entries: make(map[string]memoryCacheEntry)},
|
||||
limiters: &domainLimiters{capacity: cfg.PerDomainConcurrency, items: make(map[string]chan struct{})},
|
||||
breakers: &domainBreakers{threshold: cfg.BreakerThreshold, cooldown: cfg.BreakerCooldown, items: make(map[string]domainBreakerState)},
|
||||
metrics: &fetchMetrics{},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *fetchService) Start(ctx context.Context) {
|
||||
for i := 0; i < s.cfg.WorkerConcurrency; i++ {
|
||||
worker := &lightpandaWorker{id: i, cfg: s.cfg}
|
||||
go worker.Run(ctx, s.jobs, s.metrics, s.breakers)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *fetchService) handleLive(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]string{"status": "alive"})
|
||||
}
|
||||
|
||||
func (s *fetchService) handleReady(w http.ResponseWriter, _ *http.Request) {
|
||||
if s.cfg.Token == "" {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "BROWSER_FETCH_TOKEN is required"})
|
||||
return
|
||||
}
|
||||
if len(s.cfg.AllowedDomains) == 0 {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "BROWSER_FETCH_ALLOWED_DOMAINS is required"})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"status": "ready"})
|
||||
}
|
||||
|
||||
func (s *fetchService) handleMetrics(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.authorized(r) {
|
||||
s.metrics.authFailed.Add(1)
|
||||
writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"})
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
_, _ = fmt.Fprintf(w, "browser_fetch_requests_accepted_total %d\n", s.metrics.accepted.Load())
|
||||
_, _ = fmt.Fprintf(w, "browser_fetch_cache_hits_total %d\n", s.metrics.cacheHits.Load())
|
||||
_, _ = fmt.Fprintf(w, "browser_fetch_queue_full_total %d\n", s.metrics.queueFull.Load())
|
||||
_, _ = fmt.Fprintf(w, "browser_fetch_domain_limited_total %d\n", s.metrics.domainLimited.Load())
|
||||
_, _ = fmt.Fprintf(w, "browser_fetch_auth_failed_total %d\n", s.metrics.authFailed.Load())
|
||||
_, _ = fmt.Fprintf(w, "browser_fetch_success_total %d\n", s.metrics.success.Load())
|
||||
_, _ = fmt.Fprintf(w, "browser_fetch_failed_total %d\n", s.metrics.failed.Load())
|
||||
_, _ = fmt.Fprintf(w, "browser_fetch_breaker_open_total %d\n", s.metrics.breakerOpen.Load())
|
||||
_, _ = fmt.Fprintf(w, "browser_fetch_queue_depth %d\n", len(s.jobs))
|
||||
}
|
||||
|
||||
func (s *fetchService) handleFetch(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
|
||||
return
|
||||
}
|
||||
if !s.authorized(r) {
|
||||
s.metrics.authFailed.Add(1)
|
||||
writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"})
|
||||
return
|
||||
}
|
||||
|
||||
var req fetchRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&req); err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid json"})
|
||||
return
|
||||
}
|
||||
rawURL := strings.TrimSpace(req.URL)
|
||||
host, err := validateFetchURL(rawURL, s.cfg.AllowedDomains)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
if s.breakers.IsOpen(host) {
|
||||
s.metrics.breakerOpen.Add(1)
|
||||
writeJSON(w, http.StatusTooManyRequests, map[string]string{"error": "domain circuit breaker is open"})
|
||||
return
|
||||
}
|
||||
|
||||
cacheKey := fetchCacheKey(rawURL)
|
||||
if s.cfg.CacheTTL > 0 {
|
||||
if cached, ok := s.cache.Get(cacheKey); ok {
|
||||
s.metrics.cacheHits.Add(1)
|
||||
writeJSON(w, http.StatusOK, cached)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
release, ok := s.limiters.TryAcquire(host)
|
||||
if !ok {
|
||||
s.metrics.domainLimited.Add(1)
|
||||
writeJSON(w, http.StatusTooManyRequests, map[string]string{"error": "domain concurrency limit reached"})
|
||||
return
|
||||
}
|
||||
|
||||
timeout := s.cfg.RequestTimeout
|
||||
if req.TimeoutMS > 0 {
|
||||
timeout = time.Duration(req.TimeoutMS) * time.Millisecond
|
||||
if timeout > s.cfg.RequestTimeout {
|
||||
timeout = s.cfg.RequestTimeout
|
||||
}
|
||||
}
|
||||
job := fetchJob{
|
||||
rawURL: rawURL,
|
||||
timeout: timeout,
|
||||
host: host,
|
||||
release: release,
|
||||
resp: make(chan fetchJobResult, 1),
|
||||
}
|
||||
|
||||
select {
|
||||
case s.jobs <- job:
|
||||
s.metrics.accepted.Add(1)
|
||||
default:
|
||||
release()
|
||||
s.metrics.queueFull.Add(1)
|
||||
writeJSON(w, http.StatusTooManyRequests, map[string]string{"error": "browser fetch queue is full"})
|
||||
return
|
||||
}
|
||||
|
||||
select {
|
||||
case result := <-job.resp:
|
||||
if result.err != nil {
|
||||
writeJSON(w, http.StatusBadGateway, map[string]string{"error": result.err.Error()})
|
||||
return
|
||||
}
|
||||
if s.cfg.CacheTTL > 0 {
|
||||
s.cache.Set(cacheKey, result.result, s.cfg.CacheTTL)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, result.result)
|
||||
case <-r.Context().Done():
|
||||
writeJSON(w, http.StatusGatewayTimeout, map[string]string{"error": "request cancelled"})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *fetchService) authorized(r *http.Request) bool {
|
||||
expected := s.cfg.Token
|
||||
if expected == "" {
|
||||
return false
|
||||
}
|
||||
value := strings.TrimSpace(r.Header.Get("Authorization"))
|
||||
value = strings.TrimPrefix(value, "Bearer ")
|
||||
if value == "" {
|
||||
value = strings.TrimSpace(r.URL.Query().Get("token"))
|
||||
}
|
||||
return subtle.ConstantTimeCompare([]byte(value), []byte(expected)) == 1
|
||||
}
|
||||
|
||||
func (w *lightpandaWorker) Run(ctx context.Context, jobs <-chan fetchJob, metrics *fetchMetrics, breakers *domainBreakers) {
|
||||
defer w.closeClient()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case job := <-jobs:
|
||||
result, err := w.Fetch(ctx, job.rawURL, job.timeout)
|
||||
job.release()
|
||||
if err != nil {
|
||||
metrics.failed.Add(1)
|
||||
breakers.RecordFailure(job.host)
|
||||
} else {
|
||||
metrics.success.Add(1)
|
||||
breakers.RecordSuccess(job.host)
|
||||
}
|
||||
job.resp <- fetchJobResult{result: result, err: err}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *lightpandaWorker) Fetch(parent context.Context, rawURL string, timeout time.Duration) (fetchResponse, error) {
|
||||
if timeout <= 0 {
|
||||
timeout = w.cfg.RequestTimeout
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(parent, timeout)
|
||||
defer cancel()
|
||||
|
||||
var markdown string
|
||||
var err error
|
||||
if w.cfg.LightpandaDriver == "mcp" {
|
||||
markdown, err = w.fetchViaMCP(ctx, rawURL)
|
||||
if err != nil {
|
||||
w.closeClient()
|
||||
return fetchResponse{}, err
|
||||
}
|
||||
} else {
|
||||
markdown, err = w.fetchViaCommand(ctx, rawURL)
|
||||
if err != nil {
|
||||
return fetchResponse{}, err
|
||||
}
|
||||
}
|
||||
w.handled++
|
||||
|
||||
markdown = strings.TrimSpace(markdown)
|
||||
text := normalizeText(markdown)
|
||||
if text == "" {
|
||||
return fetchResponse{}, errors.New("lightpanda returned empty markdown")
|
||||
}
|
||||
return fetchResponse{
|
||||
URL: rawURL,
|
||||
Title: extractMarkdownTitle(markdown),
|
||||
Text: text,
|
||||
Markdown: markdown,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (w *lightpandaWorker) fetchViaCommand(ctx context.Context, rawURL string) (string, error) {
|
||||
args := append([]string{"fetch"}, w.cfg.LightpandaFetchArgs...)
|
||||
args = append(args, rawURL)
|
||||
cmd := exec.CommandContext(ctx, w.cfg.LightpandaBin, args...)
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
msg := strings.TrimSpace(stderr.String())
|
||||
if msg != "" {
|
||||
return "", fmt.Errorf("lightpanda fetch failed: %w: %s", err, truncateString(msg, 2048))
|
||||
}
|
||||
return "", fmt.Errorf("lightpanda fetch failed: %w", err)
|
||||
}
|
||||
return string(out), nil
|
||||
}
|
||||
|
||||
func (w *lightpandaWorker) fetchViaMCP(ctx context.Context, rawURL string) (string, error) {
|
||||
if w.client == nil || (w.cfg.WorkerMaxRequests > 0 && w.handled >= w.cfg.WorkerMaxRequests) {
|
||||
w.closeClient()
|
||||
w.client = newMCPClient(w.cfg)
|
||||
w.handled = 0
|
||||
}
|
||||
return w.client.Markdown(ctx, rawURL)
|
||||
}
|
||||
|
||||
func (w *lightpandaWorker) closeClient() {
|
||||
if w.client != nil {
|
||||
w.client.Close()
|
||||
w.client = nil
|
||||
}
|
||||
}
|
||||
|
||||
func newMCPClient(cfg fetchConfig) *mcpClient {
|
||||
return &mcpClient{cfg: cfg}
|
||||
}
|
||||
|
||||
func (c *mcpClient) Markdown(ctx context.Context, rawURL string) (string, error) {
|
||||
if err := c.ensureStarted(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
id := atomic.AddInt64(&c.nextID, 1)
|
||||
req := map[string]any{
|
||||
"jsonrpc": "2.0",
|
||||
"id": id,
|
||||
"method": "tools/call",
|
||||
"params": map[string]any{
|
||||
"name": "markdown",
|
||||
"arguments": map[string]any{
|
||||
"url": rawURL,
|
||||
},
|
||||
},
|
||||
}
|
||||
resp, err := c.call(ctx, id, req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var toolResult mcpToolResult
|
||||
if err := json.Unmarshal(resp.Result, &toolResult); err != nil {
|
||||
return "", fmt.Errorf("decode lightpanda markdown result: %w", err)
|
||||
}
|
||||
texts := make([]string, 0, len(toolResult.Content))
|
||||
for _, item := range toolResult.Content {
|
||||
if item.Type == "text" && strings.TrimSpace(item.Text) != "" {
|
||||
texts = append(texts, item.Text)
|
||||
}
|
||||
}
|
||||
text := strings.TrimSpace(strings.Join(texts, "\n\n"))
|
||||
if toolResult.IsError {
|
||||
if text == "" {
|
||||
text = "lightpanda tool returned error"
|
||||
}
|
||||
return "", errors.New(text)
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (c *mcpClient) ensureStarted(ctx context.Context) error {
|
||||
if c.started {
|
||||
return nil
|
||||
}
|
||||
args := append([]string{}, c.cfg.LightpandaArgs...)
|
||||
cmd := exec.Command(c.cfg.LightpandaBin, args...)
|
||||
stdin, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return fmt.Errorf("open lightpanda stdin: %w", err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return fmt.Errorf("open lightpanda stdout: %w", err)
|
||||
}
|
||||
cmd.Stderr = &c.stderr
|
||||
if err := cmd.Start(); err != nil {
|
||||
return fmt.Errorf("start lightpanda mcp: %w", err)
|
||||
}
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
scanner.Buffer(make([]byte, 64*1024), 32*1024*1024)
|
||||
c.cmd = cmd
|
||||
c.stdin = stdin
|
||||
c.stdout = scanner
|
||||
c.started = true
|
||||
|
||||
id := atomic.AddInt64(&c.nextID, 1)
|
||||
req := map[string]any{
|
||||
"jsonrpc": "2.0",
|
||||
"id": id,
|
||||
"method": "initialize",
|
||||
"params": map[string]any{
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": map[string]any{},
|
||||
"clientInfo": map[string]string{
|
||||
"name": "geo-browser-fetch",
|
||||
"version": "1.0.0",
|
||||
},
|
||||
},
|
||||
}
|
||||
if _, err := c.call(ctx, id, req); err != nil {
|
||||
c.Close()
|
||||
return fmt.Errorf("initialize lightpanda mcp: %w", err)
|
||||
}
|
||||
initialized := map[string]any{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "notifications/initialized",
|
||||
"params": map[string]any{},
|
||||
}
|
||||
return c.writeJSON(initialized)
|
||||
}
|
||||
|
||||
func (c *mcpClient) call(ctx context.Context, id int64, payload map[string]any) (mcpResponse, error) {
|
||||
if err := c.writeJSON(payload); err != nil {
|
||||
return mcpResponse{}, err
|
||||
}
|
||||
type responseResult struct {
|
||||
resp mcpResponse
|
||||
err error
|
||||
}
|
||||
done := make(chan responseResult, 1)
|
||||
go func() {
|
||||
if c.stdout == nil {
|
||||
done <- responseResult{err: errors.New("lightpanda mcp stdout is not open")}
|
||||
return
|
||||
}
|
||||
for c.stdout.Scan() {
|
||||
line := strings.TrimSpace(c.stdout.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var resp mcpResponse
|
||||
if err := json.Unmarshal([]byte(line), &resp); err != nil {
|
||||
continue
|
||||
}
|
||||
if resp.ID != id {
|
||||
continue
|
||||
}
|
||||
if resp.Error != nil {
|
||||
done <- responseResult{err: fmt.Errorf("lightpanda mcp error %d: %s", resp.Error.Code, resp.Error.Message)}
|
||||
return
|
||||
}
|
||||
done <- responseResult{resp: resp}
|
||||
return
|
||||
}
|
||||
err := c.stdout.Err()
|
||||
if err == nil {
|
||||
err = errors.New("lightpanda mcp stdout closed")
|
||||
}
|
||||
stderr := strings.TrimSpace(c.stderr.String())
|
||||
if stderr != "" {
|
||||
err = fmt.Errorf("%w: %s", err, truncateString(stderr, 2048))
|
||||
}
|
||||
done <- responseResult{err: err}
|
||||
}()
|
||||
|
||||
select {
|
||||
case result := <-done:
|
||||
return result.resp, result.err
|
||||
case <-ctx.Done():
|
||||
return mcpResponse{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (c *mcpClient) writeJSON(payload map[string]any) error {
|
||||
data, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
_, err = c.stdin.Write(data)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *mcpClient) Close() {
|
||||
if c.stdin != nil {
|
||||
_ = c.stdin.Close()
|
||||
}
|
||||
if c.cmd != nil && c.cmd.Process != nil {
|
||||
_ = c.cmd.Process.Kill()
|
||||
_, _ = c.cmd.Process.Wait()
|
||||
}
|
||||
c.started = false
|
||||
c.cmd = nil
|
||||
c.stdin = nil
|
||||
c.stdout = nil
|
||||
c.stderr.Reset()
|
||||
}
|
||||
|
||||
func (c *memoryCache) Get(key string) (fetchResponse, bool) {
|
||||
now := time.Now()
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
entry, ok := c.entries[key]
|
||||
if !ok {
|
||||
return fetchResponse{}, false
|
||||
}
|
||||
if now.After(entry.expiresAt) {
|
||||
delete(c.entries, key)
|
||||
return fetchResponse{}, false
|
||||
}
|
||||
return entry.result, true
|
||||
}
|
||||
|
||||
func (c *memoryCache) Set(key string, result fetchResponse, ttl time.Duration) {
|
||||
if ttl <= 0 {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.entries[key] = memoryCacheEntry{result: result, expiresAt: time.Now().Add(ttl)}
|
||||
}
|
||||
|
||||
func (l *domainLimiters) TryAcquire(host string) (func(), bool) {
|
||||
l.mu.Lock()
|
||||
ch := l.items[host]
|
||||
if ch == nil {
|
||||
ch = make(chan struct{}, l.capacity)
|
||||
l.items[host] = ch
|
||||
}
|
||||
l.mu.Unlock()
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
return func() { <-ch }, true
|
||||
default:
|
||||
return func() {}, false
|
||||
}
|
||||
}
|
||||
|
||||
func (b *domainBreakers) IsOpen(host string) bool {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
state := b.items[host]
|
||||
if state.openedAt.IsZero() {
|
||||
return false
|
||||
}
|
||||
if time.Since(state.openedAt) >= b.cooldown {
|
||||
delete(b.items, host)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (b *domainBreakers) RecordSuccess(host string) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
delete(b.items, host)
|
||||
}
|
||||
|
||||
func (b *domainBreakers) RecordFailure(host string) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
state := b.items[host]
|
||||
if !state.openedAt.IsZero() && time.Since(state.openedAt) < b.cooldown {
|
||||
b.items[host] = state
|
||||
return
|
||||
}
|
||||
state.failures++
|
||||
if state.failures >= b.threshold {
|
||||
state.openedAt = time.Now()
|
||||
}
|
||||
b.items[host] = state
|
||||
}
|
||||
|
||||
func validateFetchURL(rawURL string, allowedDomains []string) (string, error) {
|
||||
parsed, err := url.Parse(rawURL)
|
||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||
return "", errors.New("invalid url")
|
||||
}
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return "", errors.New("only http and https urls are allowed")
|
||||
}
|
||||
host := strings.ToLower(strings.TrimSpace(parsed.Hostname()))
|
||||
if host == "" {
|
||||
return "", errors.New("url host is required")
|
||||
}
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
return "", errors.New("ip literal urls are not allowed")
|
||||
}
|
||||
if !domainAllowed(host, allowedDomains) {
|
||||
return "", errors.New("domain is not allowed")
|
||||
}
|
||||
return host, nil
|
||||
}
|
||||
|
||||
func domainAllowed(host string, allowedDomains []string) bool {
|
||||
if len(allowedDomains) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, domain := range allowedDomains {
|
||||
domain = strings.ToLower(strings.TrimPrefix(strings.TrimSpace(domain), "."))
|
||||
if domain == "" {
|
||||
continue
|
||||
}
|
||||
if host == domain || strings.HasSuffix(host, "."+domain) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func fetchCacheKey(rawURL string) string {
|
||||
normalized := strings.TrimSpace(rawURL)
|
||||
if parsed, err := url.Parse(normalized); err == nil && parsed.Host != "" {
|
||||
parsed.Fragment = ""
|
||||
normalized = parsed.String()
|
||||
}
|
||||
sum := sha256.Sum256([]byte(normalized))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func extractMarkdownTitle(markdown string) string {
|
||||
for _, line := range strings.Split(markdown, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if strings.HasPrefix(line, "# ") {
|
||||
return strings.TrimSpace(strings.TrimPrefix(line, "# "))
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func normalizeText(value string) string {
|
||||
lines := strings.Split(value, "\n")
|
||||
out := make([]string, 0, len(lines))
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line != "" {
|
||||
out = append(out, line)
|
||||
}
|
||||
}
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, payload any) {
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(payload)
|
||||
}
|
||||
|
||||
func splitCSV(value string) []string {
|
||||
parts := strings.Split(value, ",")
|
||||
out := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
out = append(out, part)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func splitCommandArgs(value string) []string {
|
||||
parts := strings.Fields(value)
|
||||
if len(parts) == 0 {
|
||||
return nil
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
func envString(key, fallback string) string {
|
||||
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func envInt(key string, fallback int) int {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func envDuration(key string, fallback time.Duration) time.Duration {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := time.ParseDuration(value)
|
||||
if err == nil {
|
||||
return parsed
|
||||
}
|
||||
ms, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
return time.Duration(ms) * time.Millisecond
|
||||
}
|
||||
|
||||
func truncateString(value string, max int) string {
|
||||
if len(value) <= max {
|
||||
return value
|
||||
}
|
||||
return value[:max]
|
||||
}
|
||||
Reference in New Issue
Block a user