root/geo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(cache,worker): overhaul cache layer and add generation task lease recovery
Deployment Config CI / Deployment Config (push) Successful in 24s
Details
Backend CI / Backend (push) Successful in 14m33s
Details

Browse Source 
- shared/cache: add Options/L1/async/metrics/prefix decorators, multi-key ops, Redis pool tuning, and JSON readthrough metrics
- worker-generate: claim tasks via DB lease + heartbeat, requeue stale queued tasks, expire dead leases with refund/cache invalidation
- tenant: version article cache keys so worker recovery invalidations propagate cleanly
- shared/config: expand Redis (pool/timeouts/TLS) and Generation (lease/recovery) configs with defaults

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
liangxu
2026-05-03 02:02:39 +08:00
parent bbfeabdaa5
commit c1ef717d17
37 changed files with 2502 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/shared/config/config.go
+171 -7
View File
@@ -1,6 +1,7 @@
package config
import (
"crypto/tls"
"encoding/json"
"fmt"
"os"
@@ -57,8 +58,24 @@ func (d DatabaseConfig) DSN() string {
}
type RedisConfig struct {
Addr string `mapstructure:"addr"`
DB int `mapstructure:"db"`
Addr string `mapstructure:"addr"`
Password string `mapstructure:"password"`
DB int `mapstructure:"db"`
DialTimeout time.Duration `mapstructure:"dial_timeout"`
ReadTimeout time.Duration `mapstructure:"read_timeout"`
WriteTimeout time.Duration `mapstructure:"write_timeout"`
PoolSize int `mapstructure:"pool_size"`
MinIdleConns int `mapstructure:"min_idle_conns"`
MaxRetries int `mapstructure:"max_retries"`
PoolTimeout time.Duration `mapstructure:"pool_timeout"`
TLSEnabled bool `mapstructure:"tls_enabled"`
}
func (r RedisConfig) TLSConfig() *tls.Config {
if !r.TLSEnabled {
return nil
}
return &tls.Config{MinVersion: tls.VersionTLS12}
}
type RabbitMQConfig struct {
@@ -272,14 +289,30 @@ type RetrievalConfig struct {
}
type CacheConfig struct {
Driver string `mapstructure:"driver"` // "redis" or "memory"
Driver string `mapstructure:"driver"` // "redis" or "memory"
Namespace string `mapstructure:"namespace"`
JitterRatio float64 `mapstructure:"jitter_ratio"`
MetricsEnabled bool `mapstructure:"metrics_enabled"`
AsyncFillEnabled bool `mapstructure:"async_fill_enabled"`
AsyncFillWorkers int `mapstructure:"async_fill_workers"`
AsyncFillBuffer int `mapstructure:"async_fill_buffer"`
AsyncFillTimeout time.Duration `mapstructure:"async_fill_timeout"`
L1Enabled bool `mapstructure:"l1_enabled"`
L1TTL time.Duration `mapstructure:"l1_ttl"`
DeleteScanCount int64 `mapstructure:"delete_scan_count"`
}
type GenerationConfig struct {
QueueSize int `mapstructure:"queue_size"`
WorkerConcurrency int `mapstructure:"worker_concurrency"`
StreamEnabled bool `mapstructure:"stream_enabled"`
ArticleTimeout time.Duration `mapstructure:"article_timeout"`
QueueSize int `mapstructure:"queue_size"`
WorkerConcurrency int `mapstructure:"worker_concurrency"`
StreamEnabled bool `mapstructure:"stream_enabled"`
ArticleTimeout time.Duration `mapstructure:"article_timeout"`
TaskLeaseTTL time.Duration `mapstructure:"task_lease_ttl"`
TaskRecoveryInterval time.Duration `mapstructure:"task_recovery_interval"`
TaskRecoveryTimeout time.Duration `mapstructure:"task_recovery_timeout"`
TaskRecoveryBatchSize int `mapstructure:"task_recovery_batch_size"`
TaskQueuedStaleAfter time.Duration `mapstructure:"task_queued_stale_after"`
TaskMaxAttempts int `mapstructure:"task_max_attempts"`
}
func Load(configPath string) (*Config, error) {
@@ -305,12 +338,15 @@ func loadWithFiles(configPath string) (*Config, []string, error) {
}
applyEnvOverrides(cfg)
NormalizeRedisConfig(&cfg.Redis)
NormalizeCacheConfig(&cfg.Cache)
normalizeRabbitMQConfig(&cfg.RabbitMQ)
normalizeSchedulerConfig(&cfg.Scheduler)
normalizeMonitoringConfig(&cfg.MonitoringWorkers)
normalizeMonitoringDispatchConfig(&cfg.MonitoringDispatch)
normalizeMembershipConfig(&cfg.Membership)
normalizeBrandLibraryConfig(&cfg.BrandLibrary)
NormalizeGenerationConfig(&cfg.Generation)
files := []string{configFile}
if localConfigFile != "" {
@@ -360,6 +396,7 @@ func decodeResolvedConfig(configFile, localConfigFile string) (*Config, error) {
if err := resolved.Scan(&settings); err != nil {
return nil, err
}
applyConfigDefaults(settings)
var cfg Config
decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
@@ -380,6 +417,25 @@ func decodeResolvedConfig(configFile, localConfigFile string) (*Config, error) {
return &cfg, nil
}
func applyConfigDefaults(settings map[string]any) {
cacheSettings := ensureMapSetting(settings, "cache")
if _, ok := cacheSettings["metrics_enabled"]; !ok {
cacheSettings["metrics_enabled"] = true
}
}
func ensureMapSetting(settings map[string]any, key string) map[string]any {
if settings == nil {
return map[string]any{}
}
if existing, ok := settings[key].(map[string]any); ok {
return existing
}
next := make(map[string]any)
settings[key] = next
return next
}
func normalizeMonitoringDispatchConfig(cfg *MonitoringDispatchConfig) {
if cfg == nil {
return
@@ -392,6 +448,114 @@ func normalizeMonitoringDispatchConfig(cfg *MonitoringDispatchConfig) {
}
}
func NormalizeRedisConfig(cfg *RedisConfig) {
if cfg == nil {
return
}
cfg.Addr = strings.TrimSpace(cfg.Addr)
if cfg.Addr == "" {
cfg.Addr = "localhost:6379"
}
if cfg.DialTimeout <= 0 {
cfg.DialTimeout = 5 * time.Second
}
if cfg.ReadTimeout <= 0 {
cfg.ReadTimeout = 3 * time.Second
}
if cfg.WriteTimeout <= 0 {
cfg.WriteTimeout = 3 * time.Second
}
if cfg.PoolSize <= 0 {
cfg.PoolSize = 16
}
if cfg.MinIdleConns < 0 {
cfg.MinIdleConns = 0
}
if cfg.MinIdleConns > cfg.PoolSize {
cfg.MinIdleConns = cfg.PoolSize
}
if cfg.MaxRetries < 0 {
cfg.MaxRetries = 0
}
if cfg.PoolTimeout <= 0 {
cfg.PoolTimeout = 4 * time.Second
}
}
func NormalizeCacheConfig(cfg *CacheConfig) {
if cfg == nil {
return
}
cfg.Driver = strings.ToLower(strings.TrimSpace(cfg.Driver))
if cfg.Driver == "" {
cfg.Driver = "redis"
}
if cfg.Driver != "redis" && cfg.Driver != "memory" {
cfg.Driver = "memory"
}
cfg.Namespace = strings.Trim(strings.TrimSpace(cfg.Namespace), ":")
if cfg.Namespace == "" {
cfg.Namespace = "geo"
}
if cfg.JitterRatio <= 0 {
cfg.JitterRatio = 0.1
}
if cfg.JitterRatio > 1 {
cfg.JitterRatio = 1
}
if cfg.AsyncFillWorkers <= 0 {
cfg.AsyncFillWorkers = 2
}
if cfg.AsyncFillBuffer <= 0 {
cfg.AsyncFillBuffer = 1024
}
if cfg.AsyncFillTimeout <= 0 {
cfg.AsyncFillTimeout = 2 * time.Second
}
if cfg.L1TTL <= 0 {
cfg.L1TTL = 30 * time.Second
}
if cfg.DeleteScanCount <= 0 {
cfg.DeleteScanCount = 500
}
}
func NormalizeGenerationConfig(cfg *GenerationConfig) {
if cfg == nil {
return
}
if cfg.QueueSize <= 0 {
cfg.QueueSize = 128
}
if cfg.WorkerConcurrency <= 0 {
cfg.WorkerConcurrency = 1
}
if cfg.ArticleTimeout <= 0 {
cfg.ArticleTimeout = 8 * time.Minute
}
if cfg.TaskLeaseTTL <= 0 {
cfg.TaskLeaseTTL = cfg.ArticleTimeout + 2*time.Minute
}
if cfg.TaskLeaseTTL < time.Minute {
cfg.TaskLeaseTTL = time.Minute
}
if cfg.TaskRecoveryInterval <= 0 {
cfg.TaskRecoveryInterval = time.Minute
}
if cfg.TaskRecoveryTimeout <= 0 {
cfg.TaskRecoveryTimeout = 30 * time.Second
}
if cfg.TaskRecoveryBatchSize <= 0 {
cfg.TaskRecoveryBatchSize = 100
}
if cfg.TaskQueuedStaleAfter <= 0 {
cfg.TaskQueuedStaleAfter = 2 * time.Minute
}
if cfg.TaskMaxAttempts <= 0 {
cfg.TaskMaxAttempts = 3
}
}
func candidateConfigPaths(configPath string, local bool) []string {
trimmed := strings.TrimSpace(configPath)
if trimmed == "" {
server/internal/shared/config/config_test.go
+180
View File
@@ -145,6 +145,117 @@ scheduler:
}
}
func TestLoadAppliesRedisAndCacheDefaults(t *testing.T) {
configPath := writeTestConfig(t, `
redis: {}
cache: {}
`)
cfg, err := Load(configPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Redis.Addr != "localhost:6379" {
t.Fatalf("expected default redis addr, got %q", cfg.Redis.Addr)
}
if cfg.Redis.DialTimeout != 5*time.Second {
t.Fatalf("expected default redis dial timeout 5s, got %s", cfg.Redis.DialTimeout)
}
if cfg.Redis.ReadTimeout != 3*time.Second {
t.Fatalf("expected default redis read timeout 3s, got %s", cfg.Redis.ReadTimeout)
}
if cfg.Redis.WriteTimeout != 3*time.Second {
t.Fatalf("expected default redis write timeout 3s, got %s", cfg.Redis.WriteTimeout)
}
if cfg.Redis.PoolSize != 16 {
t.Fatalf("expected default redis pool size 16, got %d", cfg.Redis.PoolSize)
}
if cfg.Redis.PoolTimeout != 4*time.Second {
t.Fatalf("expected default redis pool timeout 4s, got %s", cfg.Redis.PoolTimeout)
}
if cfg.Cache.Driver != "redis" {
t.Fatalf("expected default cache driver redis, got %q", cfg.Cache.Driver)
}
if cfg.Cache.Namespace != "geo" {
t.Fatalf("expected default cache namespace geo, got %q", cfg.Cache.Namespace)
}
if cfg.Cache.JitterRatio != 0.1 {
t.Fatalf("expected default jitter ratio 0.1, got %f", cfg.Cache.JitterRatio)
}
if !cfg.Cache.MetricsEnabled {
t.Fatalf("expected cache metrics enabled by default")
}
if cfg.Cache.AsyncFillWorkers != 2 || cfg.Cache.AsyncFillBuffer != 1024 || cfg.Cache.AsyncFillTimeout != 2*time.Second {
t.Fatalf("unexpected async fill defaults: workers=%d buffer=%d timeout=%s", cfg.Cache.AsyncFillWorkers, cfg.Cache.AsyncFillBuffer, cfg.Cache.AsyncFillTimeout)
}
if cfg.Cache.L1TTL != 30*time.Second {
t.Fatalf("expected default l1 ttl 30s, got %s", cfg.Cache.L1TTL)
}
if cfg.Cache.DeleteScanCount != 500 {
t.Fatalf("expected default delete scan count 500, got %d", cfg.Cache.DeleteScanCount)
}
}
func TestLoadAppliesRedisAndCacheOverrides(t *testing.T) {
configPath := writeTestConfig(t, `
redis:
addr: redis.internal:6380
password: secret
db: 2
dial_timeout: 7s
read_timeout: 8s
write_timeout: 9s
pool_size: 32
min_idle_conns: 4
max_retries: 5
pool_timeout: 6s
tls_enabled: true
cache:
driver: memory
namespace: tenant-cache
jitter_ratio: 0.25
metrics_enabled: false
async_fill_enabled: true
async_fill_workers: 3
async_fill_buffer: 64
async_fill_timeout: 1500ms
l1_enabled: true
l1_ttl: 5s
delete_scan_count: 1200
`)
cfg, err := Load(configPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Redis.Addr != "redis.internal:6380" || cfg.Redis.Password != "secret" || cfg.Redis.DB != 2 {
t.Fatalf("unexpected redis endpoint config: %#v", cfg.Redis)
}
if cfg.Redis.DialTimeout != 7*time.Second || cfg.Redis.ReadTimeout != 8*time.Second || cfg.Redis.WriteTimeout != 9*time.Second {
t.Fatalf("unexpected redis timeout config: %#v", cfg.Redis)
}
if cfg.Redis.PoolSize != 32 || cfg.Redis.MinIdleConns != 4 || cfg.Redis.MaxRetries != 5 || cfg.Redis.PoolTimeout != 6*time.Second {
t.Fatalf("unexpected redis pool config: %#v", cfg.Redis)
}
if cfg.Redis.TLSConfig() == nil {
t.Fatalf("expected redis TLS config")
}
if cfg.Cache.Driver != "memory" || cfg.Cache.Namespace != "tenant-cache" || cfg.Cache.JitterRatio != 0.25 {
t.Fatalf("unexpected cache base config: %#v", cfg.Cache)
}
if cfg.Cache.MetricsEnabled {
t.Fatalf("expected cache metrics override false")
}
if !cfg.Cache.AsyncFillEnabled || cfg.Cache.AsyncFillWorkers != 3 || cfg.Cache.AsyncFillBuffer != 64 || cfg.Cache.AsyncFillTimeout != 1500*time.Millisecond {
t.Fatalf("unexpected cache async config: %#v", cfg.Cache)
}
if !cfg.Cache.L1Enabled || cfg.Cache.L1TTL != 5*time.Second || cfg.Cache.DeleteScanCount != 1200 {
t.Fatalf("unexpected cache l1/delete config: %#v", cfg.Cache)
}
}
func TestLoadPrefersEnvSchedulerMetricsSettings(t *testing.T) {
t.Setenv("SCHEDULER_HTTP_HOST", "0.0.0.0")
t.Setenv("SCHEDULER_INTERNAL_METRICS_TOKEN", "env-metrics-token")
@@ -281,6 +392,75 @@ generation:
}
}
func TestLoadAppliesGenerationRecoveryDefaults(t *testing.T) {
configPath := writeTestConfig(t, `
generation:
article_timeout: 90s
`)
cfg, err := Load(configPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Generation.QueueSize != 128 {
t.Fatalf("expected default queue size 128, got %d", cfg.Generation.QueueSize)
}
if cfg.Generation.WorkerConcurrency != 1 {
t.Fatalf("expected default worker concurrency 1, got %d", cfg.Generation.WorkerConcurrency)
}
if cfg.Generation.TaskLeaseTTL != 3*time.Minute+30*time.Second {
t.Fatalf("expected default task lease ttl article_timeout+2m, got %s", cfg.Generation.TaskLeaseTTL)
}
if cfg.Generation.TaskRecoveryInterval != time.Minute {
t.Fatalf("expected default task recovery interval 1m, got %s", cfg.Generation.TaskRecoveryInterval)
}
if cfg.Generation.TaskRecoveryTimeout != 30*time.Second {
t.Fatalf("expected default task recovery timeout 30s, got %s", cfg.Generation.TaskRecoveryTimeout)
}
if cfg.Generation.TaskRecoveryBatchSize != 100 {
t.Fatalf("expected default task recovery batch size 100, got %d", cfg.Generation.TaskRecoveryBatchSize)
}
if cfg.Generation.TaskQueuedStaleAfter != 2*time.Minute {
t.Fatalf("expected default queued stale after 2m, got %s", cfg.Generation.TaskQueuedStaleAfter)
}
if cfg.Generation.TaskMaxAttempts != 3 {
t.Fatalf("expected default task max attempts 3, got %d", cfg.Generation.TaskMaxAttempts)
}
}
func TestLoadAppliesGenerationRecoveryOverrides(t *testing.T) {
configPath := writeTestConfig(t, `
generation:
queue_size: 64
worker_concurrency: 4
article_timeout: 5m
task_lease_ttl: 7m
task_recovery_interval: 15s
task_recovery_timeout: 10s
task_recovery_batch_size: 12
task_queued_stale_after: 45s
task_max_attempts: 5
`)
cfg, err := Load(configPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Generation.QueueSize != 64 || cfg.Generation.WorkerConcurrency != 4 || cfg.Generation.ArticleTimeout != 5*time.Minute {
t.Fatalf("unexpected generation base config: %#v", cfg.Generation)
}
if cfg.Generation.TaskLeaseTTL != 7*time.Minute ||
cfg.Generation.TaskRecoveryInterval != 15*time.Second ||
cfg.Generation.TaskRecoveryTimeout != 10*time.Second ||
cfg.Generation.TaskRecoveryBatchSize != 12 ||
cfg.Generation.TaskQueuedStaleAfter != 45*time.Second ||
cfg.Generation.TaskMaxAttempts != 5 {
t.Fatalf("unexpected generation recovery config: %#v", cfg.Generation)
}
}
func TestStoreReloadsConfigAndKeepsPreviousOnInvalidYAML(t *testing.T) {
configPath := writeTestConfig(t, `
jwt:
server/internal/shared/config/reload.go
+7 -1
View File
@@ -85,7 +85,13 @@ func Diff(previous, current *Config) []FieldChange {
addChange("generation.worker", false)
}
if previous.Generation.StreamEnabled != current.Generation.StreamEnabled ||
previous.Generation.ArticleTimeout != current.Generation.ArticleTimeout {
previous.Generation.ArticleTimeout != current.Generation.ArticleTimeout ||
previous.Generation.TaskLeaseTTL != current.Generation.TaskLeaseTTL ||
previous.Generation.TaskRecoveryInterval != current.Generation.TaskRecoveryInterval ||
previous.Generation.TaskRecoveryTimeout != current.Generation.TaskRecoveryTimeout ||
previous.Generation.TaskRecoveryBatchSize != current.Generation.TaskRecoveryBatchSize ||
previous.Generation.TaskQueuedStaleAfter != current.Generation.TaskQueuedStaleAfter ||
previous.Generation.TaskMaxAttempts != current.Generation.TaskMaxAttempts {
addChange("generation", true)
}