feat(cache,worker): overhaul cache layer and add generation task lease recovery

- shared/cache: add Options/L1/async/metrics/prefix decorators, multi-key ops, Redis pool tuning, and JSON readthrough metrics - worker-generate: claim tasks via DB lease + heartbeat, requeue stale queued tasks, expire dead leases with refund/cache invalidation - tenant: version article cache keys so worker recovery invalidations propagate cleanly - shared/config: expand Redis (pool/timeouts/TLS) and Generation (lease/recovery) configs with defaults Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-03 02:02:39 +08:00
parent bbfeabdaa5
commit c1ef717d17
37 changed files with 2502 additions and 93 deletions
@@ -145,6 +145,117 @@ scheduler:
 	}
 }

+func TestLoadAppliesRedisAndCacheDefaults(t *testing.T) {
+	configPath := writeTestConfig(t, `
+redis: {}
+cache: {}
+`)
+
+	cfg, err := Load(configPath)
+	if err != nil {
+		t.Fatalf("Load() error = %v", err)
+	}
+
+	if cfg.Redis.Addr != "localhost:6379" {
+		t.Fatalf("expected default redis addr, got %q", cfg.Redis.Addr)
+	}
+	if cfg.Redis.DialTimeout != 5*time.Second {
+		t.Fatalf("expected default redis dial timeout 5s, got %s", cfg.Redis.DialTimeout)
+	}
+	if cfg.Redis.ReadTimeout != 3*time.Second {
+		t.Fatalf("expected default redis read timeout 3s, got %s", cfg.Redis.ReadTimeout)
+	}
+	if cfg.Redis.WriteTimeout != 3*time.Second {
+		t.Fatalf("expected default redis write timeout 3s, got %s", cfg.Redis.WriteTimeout)
+	}
+	if cfg.Redis.PoolSize != 16 {
+		t.Fatalf("expected default redis pool size 16, got %d", cfg.Redis.PoolSize)
+	}
+	if cfg.Redis.PoolTimeout != 4*time.Second {
+		t.Fatalf("expected default redis pool timeout 4s, got %s", cfg.Redis.PoolTimeout)
+	}
+	if cfg.Cache.Driver != "redis" {
+		t.Fatalf("expected default cache driver redis, got %q", cfg.Cache.Driver)
+	}
+	if cfg.Cache.Namespace != "geo" {
+		t.Fatalf("expected default cache namespace geo, got %q", cfg.Cache.Namespace)
+	}
+	if cfg.Cache.JitterRatio != 0.1 {
+		t.Fatalf("expected default jitter ratio 0.1, got %f", cfg.Cache.JitterRatio)
+	}
+	if !cfg.Cache.MetricsEnabled {
+		t.Fatalf("expected cache metrics enabled by default")
+	}
+	if cfg.Cache.AsyncFillWorkers != 2 || cfg.Cache.AsyncFillBuffer != 1024 || cfg.Cache.AsyncFillTimeout != 2*time.Second {
+		t.Fatalf("unexpected async fill defaults: workers=%d buffer=%d timeout=%s", cfg.Cache.AsyncFillWorkers, cfg.Cache.AsyncFillBuffer, cfg.Cache.AsyncFillTimeout)
+	}
+	if cfg.Cache.L1TTL != 30*time.Second {
+		t.Fatalf("expected default l1 ttl 30s, got %s", cfg.Cache.L1TTL)
+	}
+	if cfg.Cache.DeleteScanCount != 500 {
+		t.Fatalf("expected default delete scan count 500, got %d", cfg.Cache.DeleteScanCount)
+	}
+}
+
+func TestLoadAppliesRedisAndCacheOverrides(t *testing.T) {
+	configPath := writeTestConfig(t, `
+redis:
+  addr: redis.internal:6380
+  password: secret
+  db: 2
+  dial_timeout: 7s
+  read_timeout: 8s
+  write_timeout: 9s
+  pool_size: 32
+  min_idle_conns: 4
+  max_retries: 5
+  pool_timeout: 6s
+  tls_enabled: true
+cache:
+  driver: memory
+  namespace: tenant-cache
+  jitter_ratio: 0.25
+  metrics_enabled: false
+  async_fill_enabled: true
+  async_fill_workers: 3
+  async_fill_buffer: 64
+  async_fill_timeout: 1500ms
+  l1_enabled: true
+  l1_ttl: 5s
+  delete_scan_count: 1200
+`)
+
+	cfg, err := Load(configPath)
+	if err != nil {
+		t.Fatalf("Load() error = %v", err)
+	}
+
+	if cfg.Redis.Addr != "redis.internal:6380" || cfg.Redis.Password != "secret" || cfg.Redis.DB != 2 {
+		t.Fatalf("unexpected redis endpoint config: %#v", cfg.Redis)
+	}
+	if cfg.Redis.DialTimeout != 7*time.Second || cfg.Redis.ReadTimeout != 8*time.Second || cfg.Redis.WriteTimeout != 9*time.Second {
+		t.Fatalf("unexpected redis timeout config: %#v", cfg.Redis)
+	}
+	if cfg.Redis.PoolSize != 32 || cfg.Redis.MinIdleConns != 4 || cfg.Redis.MaxRetries != 5 || cfg.Redis.PoolTimeout != 6*time.Second {
+		t.Fatalf("unexpected redis pool config: %#v", cfg.Redis)
+	}
+	if cfg.Redis.TLSConfig() == nil {
+		t.Fatalf("expected redis TLS config")
+	}
+	if cfg.Cache.Driver != "memory" || cfg.Cache.Namespace != "tenant-cache" || cfg.Cache.JitterRatio != 0.25 {
+		t.Fatalf("unexpected cache base config: %#v", cfg.Cache)
+	}
+	if cfg.Cache.MetricsEnabled {
+		t.Fatalf("expected cache metrics override false")
+	}
+	if !cfg.Cache.AsyncFillEnabled || cfg.Cache.AsyncFillWorkers != 3 || cfg.Cache.AsyncFillBuffer != 64 || cfg.Cache.AsyncFillTimeout != 1500*time.Millisecond {
+		t.Fatalf("unexpected cache async config: %#v", cfg.Cache)
+	}
+	if !cfg.Cache.L1Enabled || cfg.Cache.L1TTL != 5*time.Second || cfg.Cache.DeleteScanCount != 1200 {
+		t.Fatalf("unexpected cache l1/delete config: %#v", cfg.Cache)
+	}
+}
+
 func TestLoadPrefersEnvSchedulerMetricsSettings(t *testing.T) {
 	t.Setenv("SCHEDULER_HTTP_HOST", "0.0.0.0")
 	t.Setenv("SCHEDULER_INTERNAL_METRICS_TOKEN", "env-metrics-token")
@@ -281,6 +392,75 @@ generation:
 	}
 }

+func TestLoadAppliesGenerationRecoveryDefaults(t *testing.T) {
+	configPath := writeTestConfig(t, `
+generation:
+  article_timeout: 90s
+`)
+
+	cfg, err := Load(configPath)
+	if err != nil {
+		t.Fatalf("Load() error = %v", err)
+	}
+
+	if cfg.Generation.QueueSize != 128 {
+		t.Fatalf("expected default queue size 128, got %d", cfg.Generation.QueueSize)
+	}
+	if cfg.Generation.WorkerConcurrency != 1 {
+		t.Fatalf("expected default worker concurrency 1, got %d", cfg.Generation.WorkerConcurrency)
+	}
+	if cfg.Generation.TaskLeaseTTL != 3*time.Minute+30*time.Second {
+		t.Fatalf("expected default task lease ttl article_timeout+2m, got %s", cfg.Generation.TaskLeaseTTL)
+	}
+	if cfg.Generation.TaskRecoveryInterval != time.Minute {
+		t.Fatalf("expected default task recovery interval 1m, got %s", cfg.Generation.TaskRecoveryInterval)
+	}
+	if cfg.Generation.TaskRecoveryTimeout != 30*time.Second {
+		t.Fatalf("expected default task recovery timeout 30s, got %s", cfg.Generation.TaskRecoveryTimeout)
+	}
+	if cfg.Generation.TaskRecoveryBatchSize != 100 {
+		t.Fatalf("expected default task recovery batch size 100, got %d", cfg.Generation.TaskRecoveryBatchSize)
+	}
+	if cfg.Generation.TaskQueuedStaleAfter != 2*time.Minute {
+		t.Fatalf("expected default queued stale after 2m, got %s", cfg.Generation.TaskQueuedStaleAfter)
+	}
+	if cfg.Generation.TaskMaxAttempts != 3 {
+		t.Fatalf("expected default task max attempts 3, got %d", cfg.Generation.TaskMaxAttempts)
+	}
+}
+
+func TestLoadAppliesGenerationRecoveryOverrides(t *testing.T) {
+	configPath := writeTestConfig(t, `
+generation:
+  queue_size: 64
+  worker_concurrency: 4
+  article_timeout: 5m
+  task_lease_ttl: 7m
+  task_recovery_interval: 15s
+  task_recovery_timeout: 10s
+  task_recovery_batch_size: 12
+  task_queued_stale_after: 45s
+  task_max_attempts: 5
+`)
+
+	cfg, err := Load(configPath)
+	if err != nil {
+		t.Fatalf("Load() error = %v", err)
+	}
+
+	if cfg.Generation.QueueSize != 64 || cfg.Generation.WorkerConcurrency != 4 || cfg.Generation.ArticleTimeout != 5*time.Minute {
+		t.Fatalf("unexpected generation base config: %#v", cfg.Generation)
+	}
+	if cfg.Generation.TaskLeaseTTL != 7*time.Minute ||
+		cfg.Generation.TaskRecoveryInterval != 15*time.Second ||
+		cfg.Generation.TaskRecoveryTimeout != 10*time.Second ||
+		cfg.Generation.TaskRecoveryBatchSize != 12 ||
+		cfg.Generation.TaskQueuedStaleAfter != 45*time.Second ||
+		cfg.Generation.TaskMaxAttempts != 5 {
+		t.Fatalf("unexpected generation recovery config: %#v", cfg.Generation)
+	}
+}
+
 func TestStoreReloadsConfigAndKeepsPreviousOnInvalidYAML(t *testing.T) {
 	configPath := writeTestConfig(t, `
 jwt: