geo/server/cmd/browser-fetch/main.go

package main

import (
	"bufio"
	"bytes"
	"context"
	"crypto/sha256"
	"crypto/subtle"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

const (
	defaultPort                 = 8082
	defaultQueueSize            = 128
	defaultWorkerConcurrency    = 2
	defaultPerDomainConcurrency = 1
	defaultRequestTimeout       = 45 * time.Second
	defaultCacheTTL             = 6 * time.Hour
	defaultBreakerThreshold     = 5
	defaultBreakerCooldown      = 2 * time.Minute
)

type fetchConfig struct {
	Port                 int
	Token                string
	QueueSize            int
	WorkerConcurrency    int
	PerDomainConcurrency int
	RequestTimeout       time.Duration
	CacheTTL             time.Duration
	AllowedDomains       []string
	LightpandaDriver     string
	LightpandaBin        string
	LightpandaArgs       []string
	LightpandaFetchArgs  []string
	WorkerMaxRequests    int
	BreakerThreshold     int
	BreakerCooldown      time.Duration
}

type fetchService struct {
	cfg          fetchConfig
	jobs         chan fetchJob
	cache        *memoryCache
	limiters     *domainLimiters
	breakers     *domainBreakers
	metrics      *fetchMetrics
	shuttingDown atomic.Bool
}

type fetchJob struct {
	rawURL  string
	timeout time.Duration
	host    string
	release func()
	resp    chan fetchJobResult
}

type fetchJobResult struct {
	result fetchResponse
	err    error
}

type fetchRequest struct {
	URL       string         `json:"url"`
	TimeoutMS int64          `json:"timeout_ms,omitempty"`
	Options   map[string]any `json:"options,omitempty"`
}

type fetchResponse struct {
	URL      string `json:"url,omitempty"`
	Title    string `json:"title,omitempty"`
	Text     string `json:"text,omitempty"`
	Markdown string `json:"markdown,omitempty"`
}

type memoryCacheEntry struct {
	result    fetchResponse
	expiresAt time.Time
}

type memoryCache struct {
	mu      sync.Mutex
	entries map[string]memoryCacheEntry
}

type domainLimiters struct {
	mu       sync.Mutex
	capacity int
	items    map[string]chan struct{}
}

type domainBreakerState struct {
	failures int
	openedAt time.Time
}

type domainBreakers struct {
	mu        sync.Mutex
	threshold int
	cooldown  time.Duration
	items     map[string]domainBreakerState
}

type fetchMetrics struct {
	accepted      atomic.Int64
	cacheHits     atomic.Int64
	queueFull     atomic.Int64
	domainLimited atomic.Int64
	authFailed    atomic.Int64
	success       atomic.Int64
	failed        atomic.Int64
	breakerOpen   atomic.Int64
}

type lightpandaWorker struct {
	id      int
	cfg     fetchConfig
	client  *mcpClient
	handled int
}

type mcpClient struct {
	cfg     fetchConfig
	cmd     *exec.Cmd
	stdin   io.WriteCloser
	stdout  *bufio.Scanner
	stderr  bytes.Buffer
	nextID  int64
	started bool
}

type mcpResponse struct {
	JSONRPC string          `json:"jsonrpc"`
	ID      int64           `json:"id,omitempty"`
	Result  json.RawMessage `json:"result,omitempty"`
	Error   *mcpError       `json:"error,omitempty"`
}

type mcpError struct {
	Code    int    `json:"code"`
	Message string `json:"message"`
}

type mcpToolResult struct {
	Content []mcpContentItem `json:"content"`
	IsError bool             `json:"isError"`
}

type mcpContentItem struct {
	Type string `json:"type"`
	Text string `json:"text"`
}

func main() {
	cfg := loadFetchConfig()
	if len(os.Args) > 1 && os.Args[1] == "-healthcheck" {
		runHealthcheck(cfg)
		return
	}
	service := newFetchService(cfg)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	service.Start(ctx)

	mux := http.NewServeMux()
	mux.HandleFunc("/health/live", service.handleLive)
	mux.HandleFunc("/health/ready", service.handleReady)
	mux.HandleFunc("/metrics", service.handleMetrics)
	mux.HandleFunc("/fetch", service.handleFetch)

	server := &http.Server{
		Addr:              fmt.Sprintf(":%d", cfg.Port),
		Handler:           mux,
		ReadHeaderTimeout: 5 * time.Second,
	}

	log.Printf("browser-fetch listening on :%d workers=%d queue=%d domains=%v", cfg.Port, cfg.WorkerConcurrency, cfg.QueueSize, cfg.AllowedDomains)
	if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
		log.Fatalf("browser-fetch server stopped: %v", err)
	}
}

func runHealthcheck(cfg fetchConfig) {
	client := &http.Client{Timeout: 2 * time.Second}
	resp, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/health/ready", cfg.Port))
	if err != nil {
		log.Fatalf("healthcheck failed: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode >= http.StatusBadRequest {
		log.Fatalf("healthcheck failed: status=%d", resp.StatusCode)
	}
}

func loadFetchConfig() fetchConfig {
	cfg := fetchConfig{
		Port:                 envInt("BROWSER_FETCH_PORT", defaultPort),
		Token:                strings.TrimSpace(os.Getenv("BROWSER_FETCH_TOKEN")),
		QueueSize:            envInt("BROWSER_FETCH_QUEUE_SIZE", defaultQueueSize),
		WorkerConcurrency:    envInt("BROWSER_FETCH_WORKER_CONCURRENCY", defaultWorkerConcurrency),
		PerDomainConcurrency: envInt("BROWSER_FETCH_PER_DOMAIN_CONCURRENCY", defaultPerDomainConcurrency),
		RequestTimeout:       envDuration("BROWSER_FETCH_REQUEST_TIMEOUT", defaultRequestTimeout),
		CacheTTL:             envDuration("BROWSER_FETCH_CACHE_TTL", defaultCacheTTL),
		AllowedDomains:       splitCSV(os.Getenv("BROWSER_FETCH_ALLOWED_DOMAINS")),
		LightpandaDriver:     strings.ToLower(strings.TrimSpace(envString("LIGHTPANDA_DRIVER", "fetch"))),
		LightpandaBin:        strings.TrimSpace(envString("LIGHTPANDA_BIN", "lightpanda")),
		LightpandaArgs:       splitCommandArgs(envString("LIGHTPANDA_ARGS", "mcp")),
		LightpandaFetchArgs:  splitCommandArgs(envString("LIGHTPANDA_FETCH_ARGS", "--dump markdown")),
		WorkerMaxRequests:    envInt("LIGHTPANDA_WORKER_MAX_REQUESTS", 100),
		BreakerThreshold:     envInt("BROWSER_FETCH_BREAKER_THRESHOLD", defaultBreakerThreshold),
		BreakerCooldown:      envDuration("BROWSER_FETCH_BREAKER_COOLDOWN", defaultBreakerCooldown),
	}
	if cfg.QueueSize <= 0 {
		cfg.QueueSize = defaultQueueSize
	}
	if cfg.WorkerConcurrency <= 0 {
		cfg.WorkerConcurrency = defaultWorkerConcurrency
	}
	if cfg.PerDomainConcurrency <= 0 {
		cfg.PerDomainConcurrency = defaultPerDomainConcurrency
	}
	if cfg.RequestTimeout <= 0 {
		cfg.RequestTimeout = defaultRequestTimeout
	}
	if cfg.CacheTTL < 0 {
		cfg.CacheTTL = 0
	}
	if cfg.BreakerThreshold <= 0 {
		cfg.BreakerThreshold = defaultBreakerThreshold
	}
	if cfg.BreakerCooldown <= 0 {
		cfg.BreakerCooldown = defaultBreakerCooldown
	}
	if cfg.LightpandaDriver == "" {
		cfg.LightpandaDriver = "fetch"
	}
	return cfg
}

func newFetchService(cfg fetchConfig) *fetchService {
	return &fetchService{
		cfg:      cfg,
		jobs:     make(chan fetchJob, cfg.QueueSize),
		cache:    &memoryCache{entries: make(map[string]memoryCacheEntry)},
		limiters: &domainLimiters{capacity: cfg.PerDomainConcurrency, items: make(map[string]chan struct{})},
		breakers: &domainBreakers{threshold: cfg.BreakerThreshold, cooldown: cfg.BreakerCooldown, items: make(map[string]domainBreakerState)},
		metrics:  &fetchMetrics{},
	}
}

func (s *fetchService) Start(ctx context.Context) {
	for i := 0; i < s.cfg.WorkerConcurrency; i++ {
		worker := &lightpandaWorker{id: i, cfg: s.cfg}
		go worker.Run(ctx, s.jobs, s.metrics, s.breakers)
	}
}

func (s *fetchService) handleLive(w http.ResponseWriter, _ *http.Request) {
	writeJSON(w, http.StatusOK, map[string]string{"status": "alive"})
}

func (s *fetchService) handleReady(w http.ResponseWriter, _ *http.Request) {
	if s.cfg.Token == "" {
		writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "BROWSER_FETCH_TOKEN is required"})
		return
	}
	if len(s.cfg.AllowedDomains) == 0 {
		writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "BROWSER_FETCH_ALLOWED_DOMAINS is required"})
		return
	}
	writeJSON(w, http.StatusOK, map[string]string{"status": "ready"})
}

func (s *fetchService) handleMetrics(w http.ResponseWriter, r *http.Request) {
	if !s.authorized(r) {
		s.metrics.authFailed.Add(1)
		writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"})
		return
	}
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	_, _ = fmt.Fprintf(w, "browser_fetch_requests_accepted_total %d\n", s.metrics.accepted.Load())
	_, _ = fmt.Fprintf(w, "browser_fetch_cache_hits_total %d\n", s.metrics.cacheHits.Load())
	_, _ = fmt.Fprintf(w, "browser_fetch_queue_full_total %d\n", s.metrics.queueFull.Load())
	_, _ = fmt.Fprintf(w, "browser_fetch_domain_limited_total %d\n", s.metrics.domainLimited.Load())
	_, _ = fmt.Fprintf(w, "browser_fetch_auth_failed_total %d\n", s.metrics.authFailed.Load())
	_, _ = fmt.Fprintf(w, "browser_fetch_success_total %d\n", s.metrics.success.Load())
	_, _ = fmt.Fprintf(w, "browser_fetch_failed_total %d\n", s.metrics.failed.Load())
	_, _ = fmt.Fprintf(w, "browser_fetch_breaker_open_total %d\n", s.metrics.breakerOpen.Load())
	_, _ = fmt.Fprintf(w, "browser_fetch_queue_depth %d\n", len(s.jobs))
}

func (s *fetchService) handleFetch(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost {
		writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
		return
	}
	if !s.authorized(r) {
		s.metrics.authFailed.Add(1)
		writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"})
		return
	}

	var req fetchRequest
	if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&req); err != nil {
		writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid json"})
		return
	}
	rawURL := strings.TrimSpace(req.URL)
	host, err := validateFetchURL(rawURL, s.cfg.AllowedDomains)
	if err != nil {
		writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
		return
	}
	if s.breakers.IsOpen(host) {
		s.metrics.breakerOpen.Add(1)
		writeJSON(w, http.StatusTooManyRequests, map[string]string{"error": "domain circuit breaker is open"})
		return
	}

	cacheKey := fetchCacheKey(rawURL)
	if s.cfg.CacheTTL > 0 {
		if cached, ok := s.cache.Get(cacheKey); ok {
			s.metrics.cacheHits.Add(1)
			writeJSON(w, http.StatusOK, cached)
			return
		}
	}

	release, ok := s.limiters.TryAcquire(host)
	if !ok {
		s.metrics.domainLimited.Add(1)
		writeJSON(w, http.StatusTooManyRequests, map[string]string{"error": "domain concurrency limit reached"})
		return
	}

	timeout := s.cfg.RequestTimeout
	if req.TimeoutMS > 0 {
		timeout = time.Duration(req.TimeoutMS) * time.Millisecond
		if timeout > s.cfg.RequestTimeout {
			timeout = s.cfg.RequestTimeout
		}
	}
	job := fetchJob{
		rawURL:  rawURL,
		timeout: timeout,
		host:    host,
		release: release,
		resp:    make(chan fetchJobResult, 1),
	}

	select {
	case s.jobs <- job:
		s.metrics.accepted.Add(1)
	default:
		release()
		s.metrics.queueFull.Add(1)
		writeJSON(w, http.StatusTooManyRequests, map[string]string{"error": "browser fetch queue is full"})
		return
	}

	select {
	case result := <-job.resp:
		if result.err != nil {
			writeJSON(w, http.StatusBadGateway, map[string]string{"error": result.err.Error()})
			return
		}
		if s.cfg.CacheTTL > 0 {
			s.cache.Set(cacheKey, result.result, s.cfg.CacheTTL)
		}
		writeJSON(w, http.StatusOK, result.result)
	case <-r.Context().Done():
		writeJSON(w, http.StatusGatewayTimeout, map[string]string{"error": "request cancelled"})
	}
}

func (s *fetchService) authorized(r *http.Request) bool {
	expected := s.cfg.Token
	if expected == "" {
		return false
	}
	value := strings.TrimSpace(r.Header.Get("Authorization"))
	value = strings.TrimPrefix(value, "Bearer ")
	if value == "" {
		value = strings.TrimSpace(r.URL.Query().Get("token"))
	}
	return subtle.ConstantTimeCompare([]byte(value), []byte(expected)) == 1
}

func (w *lightpandaWorker) Run(ctx context.Context, jobs <-chan fetchJob, metrics *fetchMetrics, breakers *domainBreakers) {
	defer w.closeClient()
	for {
		select {
		case <-ctx.Done():
			return
		case job := <-jobs:
			result, err := w.Fetch(ctx, job.rawURL, job.timeout)
			job.release()
			if err != nil {
				metrics.failed.Add(1)
				breakers.RecordFailure(job.host)
			} else {
				metrics.success.Add(1)
				breakers.RecordSuccess(job.host)
			}
			job.resp <- fetchJobResult{result: result, err: err}
		}
	}
}

func (w *lightpandaWorker) Fetch(parent context.Context, rawURL string, timeout time.Duration) (fetchResponse, error) {
	if timeout <= 0 {
		timeout = w.cfg.RequestTimeout
	}
	ctx, cancel := context.WithTimeout(parent, timeout)
	defer cancel()

	var markdown string
	var err error
	if w.cfg.LightpandaDriver == "mcp" {
		markdown, err = w.fetchViaMCP(ctx, rawURL)
		if err != nil {
			w.closeClient()
			return fetchResponse{}, err
		}
	} else {
		markdown, err = w.fetchViaCommand(ctx, rawURL)
		if err != nil {
			return fetchResponse{}, err
		}
	}
	w.handled++

	markdown = strings.TrimSpace(markdown)
	text := normalizeText(markdown)
	if text == "" {
		return fetchResponse{}, errors.New("lightpanda returned empty markdown")
	}
	return fetchResponse{
		URL:      rawURL,
		Title:    extractMarkdownTitle(markdown),
		Text:     text,
		Markdown: markdown,
	}, nil
}

func (w *lightpandaWorker) fetchViaCommand(ctx context.Context, rawURL string) (string, error) {
	args := append([]string{"fetch"}, w.cfg.LightpandaFetchArgs...)
	args = append(args, rawURL)
	cmd := exec.CommandContext(ctx, w.cfg.LightpandaBin, args...)
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	out, err := cmd.Output()
	if err != nil {
		msg := strings.TrimSpace(stderr.String())
		if msg != "" {
			return "", fmt.Errorf("lightpanda fetch failed: %w: %s", err, truncateString(msg, 2048))
		}
		return "", fmt.Errorf("lightpanda fetch failed: %w", err)
	}
	return string(out), nil
}

func (w *lightpandaWorker) fetchViaMCP(ctx context.Context, rawURL string) (string, error) {
	if w.client == nil || (w.cfg.WorkerMaxRequests > 0 && w.handled >= w.cfg.WorkerMaxRequests) {
		w.closeClient()
		w.client = newMCPClient(w.cfg)
		w.handled = 0
	}
	return w.client.Markdown(ctx, rawURL)
}

func (w *lightpandaWorker) closeClient() {
	if w.client != nil {
		w.client.Close()
		w.client = nil
	}
}

func newMCPClient(cfg fetchConfig) *mcpClient {
	return &mcpClient{cfg: cfg}
}

func (c *mcpClient) Markdown(ctx context.Context, rawURL string) (string, error) {
	if err := c.ensureStarted(ctx); err != nil {
		return "", err
	}
	id := atomic.AddInt64(&c.nextID, 1)
	req := map[string]any{
		"jsonrpc": "2.0",
		"id":      id,
		"method":  "tools/call",
		"params": map[string]any{
			"name": "markdown",
			"arguments": map[string]any{
				"url": rawURL,
			},
		},
	}
	resp, err := c.call(ctx, id, req)
	if err != nil {
		return "", err
	}
	var toolResult mcpToolResult
	if err := json.Unmarshal(resp.Result, &toolResult); err != nil {
		return "", fmt.Errorf("decode lightpanda markdown result: %w", err)
	}
	texts := make([]string, 0, len(toolResult.Content))
	for _, item := range toolResult.Content {
		if item.Type == "text" && strings.TrimSpace(item.Text) != "" {
			texts = append(texts, item.Text)
		}
	}
	text := strings.TrimSpace(strings.Join(texts, "\n\n"))
	if toolResult.IsError {
		if text == "" {
			text = "lightpanda tool returned error"
		}
		return "", errors.New(text)
	}
	return text, nil
}

func (c *mcpClient) ensureStarted(ctx context.Context) error {
	if c.started {
		return nil
	}
	args := append([]string{}, c.cfg.LightpandaArgs...)
	cmd := exec.Command(c.cfg.LightpandaBin, args...)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return fmt.Errorf("open lightpanda stdin: %w", err)
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return fmt.Errorf("open lightpanda stdout: %w", err)
	}
	cmd.Stderr = &c.stderr
	if err := cmd.Start(); err != nil {
		return fmt.Errorf("start lightpanda mcp: %w", err)
	}
	scanner := bufio.NewScanner(stdout)
	scanner.Buffer(make([]byte, 64*1024), 32*1024*1024)
	c.cmd = cmd
	c.stdin = stdin
	c.stdout = scanner
	c.started = true

	id := atomic.AddInt64(&c.nextID, 1)
	req := map[string]any{
		"jsonrpc": "2.0",
		"id":      id,
		"method":  "initialize",
		"params": map[string]any{
			"protocolVersion": "2024-11-05",
			"capabilities":    map[string]any{},
			"clientInfo": map[string]string{
				"name":    "geo-browser-fetch",
				"version": "1.0.0",
			},
		},
	}
	if _, err := c.call(ctx, id, req); err != nil {
		c.Close()
		return fmt.Errorf("initialize lightpanda mcp: %w", err)
	}
	initialized := map[string]any{
		"jsonrpc": "2.0",
		"method":  "notifications/initialized",
		"params":  map[string]any{},
	}
	return c.writeJSON(initialized)
}

func (c *mcpClient) call(ctx context.Context, id int64, payload map[string]any) (mcpResponse, error) {
	if err := c.writeJSON(payload); err != nil {
		return mcpResponse{}, err
	}
	type responseResult struct {
		resp mcpResponse
		err  error
	}
	done := make(chan responseResult, 1)
	go func() {
		if c.stdout == nil {
			done <- responseResult{err: errors.New("lightpanda mcp stdout is not open")}
			return
		}
		for c.stdout.Scan() {
			line := strings.TrimSpace(c.stdout.Text())
			if line == "" {
				continue
			}
			var resp mcpResponse
			if err := json.Unmarshal([]byte(line), &resp); err != nil {
				continue
			}
			if resp.ID != id {
				continue
			}
			if resp.Error != nil {
				done <- responseResult{err: fmt.Errorf("lightpanda mcp error %d: %s", resp.Error.Code, resp.Error.Message)}
				return
			}
			done <- responseResult{resp: resp}
			return
		}
		err := c.stdout.Err()
		if err == nil {
			err = errors.New("lightpanda mcp stdout closed")
		}
		stderr := strings.TrimSpace(c.stderr.String())
		if stderr != "" {
			err = fmt.Errorf("%w: %s", err, truncateString(stderr, 2048))
		}
		done <- responseResult{err: err}
	}()

	select {
	case result := <-done:
		return result.resp, result.err
	case <-ctx.Done():
		return mcpResponse{}, ctx.Err()
	}
}

func (c *mcpClient) writeJSON(payload map[string]any) error {
	data, err := json.Marshal(payload)
	if err != nil {
		return err
	}
	data = append(data, '\n')
	_, err = c.stdin.Write(data)
	return err
}

func (c *mcpClient) Close() {
	if c.stdin != nil {
		_ = c.stdin.Close()
	}
	if c.cmd != nil && c.cmd.Process != nil {
		_ = c.cmd.Process.Kill()
		_, _ = c.cmd.Process.Wait()
	}
	c.started = false
	c.cmd = nil
	c.stdin = nil
	c.stdout = nil
	c.stderr.Reset()
}

func (c *memoryCache) Get(key string) (fetchResponse, bool) {
	now := time.Now()
	c.mu.Lock()
	defer c.mu.Unlock()
	entry, ok := c.entries[key]
	if !ok {
		return fetchResponse{}, false
	}
	if now.After(entry.expiresAt) {
		delete(c.entries, key)
		return fetchResponse{}, false
	}
	return entry.result, true
}

func (c *memoryCache) Set(key string, result fetchResponse, ttl time.Duration) {
	if ttl <= 0 {
		return
	}
	c.mu.Lock()
	defer c.mu.Unlock()
	c.entries[key] = memoryCacheEntry{result: result, expiresAt: time.Now().Add(ttl)}
}

func (l *domainLimiters) TryAcquire(host string) (func(), bool) {
	l.mu.Lock()
	ch := l.items[host]
	if ch == nil {
		ch = make(chan struct{}, l.capacity)
		l.items[host] = ch
	}
	l.mu.Unlock()
	select {
	case ch <- struct{}{}:
		return func() { <-ch }, true
	default:
		return func() {}, false
	}
}

func (b *domainBreakers) IsOpen(host string) bool {
	b.mu.Lock()
	defer b.mu.Unlock()
	state := b.items[host]
	if state.openedAt.IsZero() {
		return false
	}
	if time.Since(state.openedAt) >= b.cooldown {
		delete(b.items, host)
		return false
	}
	return true
}

func (b *domainBreakers) RecordSuccess(host string) {
	b.mu.Lock()
	defer b.mu.Unlock()
	delete(b.items, host)
}

func (b *domainBreakers) RecordFailure(host string) {
	b.mu.Lock()
	defer b.mu.Unlock()
	state := b.items[host]
	if !state.openedAt.IsZero() && time.Since(state.openedAt) < b.cooldown {
		b.items[host] = state
		return
	}
	state.failures++
	if state.failures >= b.threshold {
		state.openedAt = time.Now()
	}
	b.items[host] = state
}

func validateFetchURL(rawURL string, allowedDomains []string) (string, error) {
	parsed, err := url.Parse(rawURL)
	if err != nil || parsed.Scheme == "" || parsed.Host == "" {
		return "", errors.New("invalid url")
	}
	if parsed.Scheme != "http" && parsed.Scheme != "https" {
		return "", errors.New("only http and https urls are allowed")
	}
	host := strings.ToLower(strings.TrimSpace(parsed.Hostname()))
	if host == "" {
		return "", errors.New("url host is required")
	}
	if ip := net.ParseIP(host); ip != nil {
		return "", errors.New("ip literal urls are not allowed")
	}
	if !domainAllowed(host, allowedDomains) {
		return "", errors.New("domain is not allowed")
	}
	return host, nil
}

func domainAllowed(host string, allowedDomains []string) bool {
	if len(allowedDomains) == 0 {
		return false
	}
	for _, domain := range allowedDomains {
		domain = strings.ToLower(strings.TrimPrefix(strings.TrimSpace(domain), "."))
		if domain == "" {
			continue
		}
		if host == domain || strings.HasSuffix(host, "."+domain) {
			return true
		}
	}
	return false
}

func fetchCacheKey(rawURL string) string {
	normalized := strings.TrimSpace(rawURL)
	if parsed, err := url.Parse(normalized); err == nil && parsed.Host != "" {
		parsed.Fragment = ""
		normalized = parsed.String()
	}
	sum := sha256.Sum256([]byte(normalized))
	return hex.EncodeToString(sum[:])
}

func extractMarkdownTitle(markdown string) string {
	for _, line := range strings.Split(markdown, "\n") {
		line = strings.TrimSpace(line)
		if strings.HasPrefix(line, "# ") {
			return strings.TrimSpace(strings.TrimPrefix(line, "# "))
		}
	}
	return ""
}

func normalizeText(value string) string {
	lines := strings.Split(value, "\n")
	out := make([]string, 0, len(lines))
	for _, line := range lines {
		line = strings.TrimSpace(line)
		if line != "" {
			out = append(out, line)
		}
	}
	return strings.Join(out, "\n")
}

func writeJSON(w http.ResponseWriter, status int, payload any) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.WriteHeader(status)
	_ = json.NewEncoder(w).Encode(payload)
}

func splitCSV(value string) []string {
	parts := strings.Split(value, ",")
	out := make([]string, 0, len(parts))
	for _, part := range parts {
		part = strings.TrimSpace(part)
		if part != "" {
			out = append(out, part)
		}
	}
	return out
}

func splitCommandArgs(value string) []string {
	parts := strings.Fields(value)
	if len(parts) == 0 {
		return nil
	}
	return parts
}

func envString(key, fallback string) string {
	if value := strings.TrimSpace(os.Getenv(key)); value != "" {
		return value
	}
	return fallback
}

func envInt(key string, fallback int) int {
	value := strings.TrimSpace(os.Getenv(key))
	if value == "" {
		return fallback
	}
	parsed, err := strconv.Atoi(value)
	if err != nil {
		return fallback
	}
	return parsed
}

func envDuration(key string, fallback time.Duration) time.Duration {
	value := strings.TrimSpace(os.Getenv(key))
	if value == "" {
		return fallback
	}
	parsed, err := time.ParseDuration(value)
	if err == nil {
		return parsed
	}
	ms, err := strconv.Atoi(value)
	if err != nil {
		return fallback
	}
	return time.Duration(ms) * time.Millisecond
}

func truncateString(value string, max int) string {
	if len(value) <= max {
		return value
	}
	return value[:max]
}