feat(cache,worker): overhaul cache layer and add generation task lease recovery

- shared/cache: add Options/L1/async/metrics/prefix decorators, multi-key ops, Redis pool tuning, and JSON readthrough metrics
- worker-generate: claim tasks via DB lease + heartbeat, requeue stale queued tasks, expire dead leases with refund/cache invalidation
- tenant: version article cache keys so worker recovery invalidations propagate cleanly
- shared/config: expand Redis (pool/timeouts/TLS) and Generation (lease/recovery) configs with defaults

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

server/internal/shared/config/config.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -57,8 +58,24 @@ func (d DatabaseConfig) DSN() string {
 }
 
 type RedisConfig struct {
-	Addr string `mapstructure:"addr"`
-	DB   int    `mapstructure:"db"`
+	Addr         string        `mapstructure:"addr"`
+	Password     string        `mapstructure:"password"`
+	DB           int           `mapstructure:"db"`
+	DialTimeout  time.Duration `mapstructure:"dial_timeout"`
+	ReadTimeout  time.Duration `mapstructure:"read_timeout"`
+	WriteTimeout time.Duration `mapstructure:"write_timeout"`
+	PoolSize     int           `mapstructure:"pool_size"`
+	MinIdleConns int           `mapstructure:"min_idle_conns"`
+	MaxRetries   int           `mapstructure:"max_retries"`
+	PoolTimeout  time.Duration `mapstructure:"pool_timeout"`
+	TLSEnabled   bool          `mapstructure:"tls_enabled"`
+}
+
+func (r RedisConfig) TLSConfig() *tls.Config {
+	if !r.TLSEnabled {
+		return nil
+	}
+	return &tls.Config{MinVersion: tls.VersionTLS12}
 }
 
 type RabbitMQConfig struct {
@@ -272,14 +289,30 @@ type RetrievalConfig struct {
 }
 
 type CacheConfig struct {
-	Driver string `mapstructure:"driver"` // "redis" or "memory"
+	Driver           string        `mapstructure:"driver"` // "redis" or "memory"
+	Namespace        string        `mapstructure:"namespace"`
+	JitterRatio      float64       `mapstructure:"jitter_ratio"`
+	MetricsEnabled   bool          `mapstructure:"metrics_enabled"`
+	AsyncFillEnabled bool          `mapstructure:"async_fill_enabled"`
+	AsyncFillWorkers int           `mapstructure:"async_fill_workers"`
+	AsyncFillBuffer  int           `mapstructure:"async_fill_buffer"`
+	AsyncFillTimeout time.Duration `mapstructure:"async_fill_timeout"`
+	L1Enabled        bool          `mapstructure:"l1_enabled"`
+	L1TTL            time.Duration `mapstructure:"l1_ttl"`
+	DeleteScanCount  int64         `mapstructure:"delete_scan_count"`
 }
 
 type GenerationConfig struct {
-	QueueSize         int           `mapstructure:"queue_size"`
-	WorkerConcurrency int           `mapstructure:"worker_concurrency"`
-	StreamEnabled     bool          `mapstructure:"stream_enabled"`
-	ArticleTimeout    time.Duration `mapstructure:"article_timeout"`
+	QueueSize             int           `mapstructure:"queue_size"`
+	WorkerConcurrency     int           `mapstructure:"worker_concurrency"`
+	StreamEnabled         bool          `mapstructure:"stream_enabled"`
+	ArticleTimeout        time.Duration `mapstructure:"article_timeout"`
+	TaskLeaseTTL          time.Duration `mapstructure:"task_lease_ttl"`
+	TaskRecoveryInterval  time.Duration `mapstructure:"task_recovery_interval"`
+	TaskRecoveryTimeout   time.Duration `mapstructure:"task_recovery_timeout"`
+	TaskRecoveryBatchSize int           `mapstructure:"task_recovery_batch_size"`
+	TaskQueuedStaleAfter  time.Duration `mapstructure:"task_queued_stale_after"`
+	TaskMaxAttempts       int           `mapstructure:"task_max_attempts"`
 }
 
 func Load(configPath string) (*Config, error) {
@@ -305,12 +338,15 @@ func loadWithFiles(configPath string) (*Config, []string, error) {
 	}
 
 	applyEnvOverrides(cfg)
+	NormalizeRedisConfig(&cfg.Redis)
+	NormalizeCacheConfig(&cfg.Cache)
 	normalizeRabbitMQConfig(&cfg.RabbitMQ)
 	normalizeSchedulerConfig(&cfg.Scheduler)
 	normalizeMonitoringConfig(&cfg.MonitoringWorkers)
 	normalizeMonitoringDispatchConfig(&cfg.MonitoringDispatch)
 	normalizeMembershipConfig(&cfg.Membership)
 	normalizeBrandLibraryConfig(&cfg.BrandLibrary)
+	NormalizeGenerationConfig(&cfg.Generation)
 
 	files := []string{configFile}
 	if localConfigFile != "" {
@@ -360,6 +396,7 @@ func decodeResolvedConfig(configFile, localConfigFile string) (*Config, error) {
 	if err := resolved.Scan(&settings); err != nil {
 		return nil, err
 	}
+	applyConfigDefaults(settings)
 
 	var cfg Config
 	decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
@@ -380,6 +417,25 @@ func decodeResolvedConfig(configFile, localConfigFile string) (*Config, error) {
 	return &cfg, nil
 }
 
+func applyConfigDefaults(settings map[string]any) {
+	cacheSettings := ensureMapSetting(settings, "cache")
+	if _, ok := cacheSettings["metrics_enabled"]; !ok {
+		cacheSettings["metrics_enabled"] = true
+	}
+}
+
+func ensureMapSetting(settings map[string]any, key string) map[string]any {
+	if settings == nil {
+		return map[string]any{}
+	}
+	if existing, ok := settings[key].(map[string]any); ok {
+		return existing
+	}
+	next := make(map[string]any)
+	settings[key] = next
+	return next
+}
+
 func normalizeMonitoringDispatchConfig(cfg *MonitoringDispatchConfig) {
 	if cfg == nil {
 		return
@@ -392,6 +448,114 @@ func normalizeMonitoringDispatchConfig(cfg *MonitoringDispatchConfig) {
 	}
 }
 
+func NormalizeRedisConfig(cfg *RedisConfig) {
+	if cfg == nil {
+		return
+	}
+	cfg.Addr = strings.TrimSpace(cfg.Addr)
+	if cfg.Addr == "" {
+		cfg.Addr = "localhost:6379"
+	}
+	if cfg.DialTimeout <= 0 {
+		cfg.DialTimeout = 5 * time.Second
+	}
+	if cfg.ReadTimeout <= 0 {
+		cfg.ReadTimeout = 3 * time.Second
+	}
+	if cfg.WriteTimeout <= 0 {
+		cfg.WriteTimeout = 3 * time.Second
+	}
+	if cfg.PoolSize <= 0 {
+		cfg.PoolSize = 16
+	}
+	if cfg.MinIdleConns < 0 {
+		cfg.MinIdleConns = 0
+	}
+	if cfg.MinIdleConns > cfg.PoolSize {
+		cfg.MinIdleConns = cfg.PoolSize
+	}
+	if cfg.MaxRetries < 0 {
+		cfg.MaxRetries = 0
+	}
+	if cfg.PoolTimeout <= 0 {
+		cfg.PoolTimeout = 4 * time.Second
+	}
+}
+
+func NormalizeCacheConfig(cfg *CacheConfig) {
+	if cfg == nil {
+		return
+	}
+	cfg.Driver = strings.ToLower(strings.TrimSpace(cfg.Driver))
+	if cfg.Driver == "" {
+		cfg.Driver = "redis"
+	}
+	if cfg.Driver != "redis" && cfg.Driver != "memory" {
+		cfg.Driver = "memory"
+	}
+	cfg.Namespace = strings.Trim(strings.TrimSpace(cfg.Namespace), ":")
+	if cfg.Namespace == "" {
+		cfg.Namespace = "geo"
+	}
+	if cfg.JitterRatio <= 0 {
+		cfg.JitterRatio = 0.1
+	}
+	if cfg.JitterRatio > 1 {
+		cfg.JitterRatio = 1
+	}
+	if cfg.AsyncFillWorkers <= 0 {
+		cfg.AsyncFillWorkers = 2
+	}
+	if cfg.AsyncFillBuffer <= 0 {
+		cfg.AsyncFillBuffer = 1024
+	}
+	if cfg.AsyncFillTimeout <= 0 {
+		cfg.AsyncFillTimeout = 2 * time.Second
+	}
+	if cfg.L1TTL <= 0 {
+		cfg.L1TTL = 30 * time.Second
+	}
+	if cfg.DeleteScanCount <= 0 {
+		cfg.DeleteScanCount = 500
+	}
+}
+
+func NormalizeGenerationConfig(cfg *GenerationConfig) {
+	if cfg == nil {
+		return
+	}
+	if cfg.QueueSize <= 0 {
+		cfg.QueueSize = 128
+	}
+	if cfg.WorkerConcurrency <= 0 {
+		cfg.WorkerConcurrency = 1
+	}
+	if cfg.ArticleTimeout <= 0 {
+		cfg.ArticleTimeout = 8 * time.Minute
+	}
+	if cfg.TaskLeaseTTL <= 0 {
+		cfg.TaskLeaseTTL = cfg.ArticleTimeout + 2*time.Minute
+	}
+	if cfg.TaskLeaseTTL < time.Minute {
+		cfg.TaskLeaseTTL = time.Minute
+	}
+	if cfg.TaskRecoveryInterval <= 0 {
+		cfg.TaskRecoveryInterval = time.Minute
+	}
+	if cfg.TaskRecoveryTimeout <= 0 {
+		cfg.TaskRecoveryTimeout = 30 * time.Second
+	}
+	if cfg.TaskRecoveryBatchSize <= 0 {
+		cfg.TaskRecoveryBatchSize = 100
+	}
+	if cfg.TaskQueuedStaleAfter <= 0 {
+		cfg.TaskQueuedStaleAfter = 2 * time.Minute
+	}
+	if cfg.TaskMaxAttempts <= 0 {
+		cfg.TaskMaxAttempts = 3
+	}
+}
+
 func candidateConfigPaths(configPath string, local bool) []string {
 	trimmed := strings.TrimSpace(configPath)
 	if trimmed == "" {