feat(cache,worker): overhaul cache layer and add generation task lease recovery

- shared/cache: add Options/L1/async/metrics/prefix decorators, multi-key ops, Redis pool tuning, and JSON readthrough metrics
- worker-generate: claim tasks via DB lease + heartbeat, requeue stale queued tasks, expire dead leases with refund/cache invalidation
- tenant: version article cache keys so worker recovery invalidations propagate cleanly
- shared/config: expand Redis (pool/timeouts/TLS) and Generation (lease/recovery) configs with defaults

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

server/internal/shared/cache/async.go

@@ -0,0 +1,178 @@
+package cache
+
+import (
+	"context"
+	"errors"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+var ErrAsyncQueueFull = errors.New("cache: async fill queue full")
+
+type asyncCache struct {
+	inner   Cache
+	fanout  *fanout
+	timeout time.Duration
+}
+
+func newAsyncCache(inner Cache, workers, buffer int, timeout time.Duration) Cache {
+	return &asyncCache{
+		inner:   inner,
+		fanout:  newFanout("cache_fill", workers, buffer),
+		timeout: timeout,
+	}
+}
+
+func (c *asyncCache) Get(ctx context.Context, key string) ([]byte, error) {
+	return c.inner.Get(ctx, key)
+}
+
+func (c *asyncCache) GetMulti(ctx context.Context, keys []string) (map[string][]byte, error) {
+	return c.inner.GetMulti(ctx, keys)
+}
+
+func (c *asyncCache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
+	if isCacheControlKey(key) {
+		return c.inner.Set(ctx, key, value, ttl)
+	}
+	err := c.fanout.Do(ctx, func(taskCtx context.Context) {
+		cctx, cancel := context.WithTimeout(taskCtx, c.timeout)
+		defer cancel()
+		if setErr := c.inner.Set(cctx, key, value, ttl); setErr != nil {
+			recordCacheOperation("set", "error", time.Duration(0))
+		}
+	})
+	recordCacheAsyncQueue(c.fanout.Len())
+	if err != nil {
+		recordCacheAsyncDrop("set")
+		return err
+	}
+	return nil
+}
+
+func (c *asyncCache) SetMulti(ctx context.Context, items []Item) error {
+	err := c.fanout.Do(ctx, func(taskCtx context.Context) {
+		cctx, cancel := context.WithTimeout(taskCtx, c.timeout)
+		defer cancel()
+		if setErr := c.inner.SetMulti(cctx, items); setErr != nil {
+			recordCacheOperation("set_multi", "error", time.Duration(0))
+		}
+	})
+	recordCacheAsyncQueue(c.fanout.Len())
+	if err != nil {
+		recordCacheAsyncDrop("set_multi")
+		return err
+	}
+	return nil
+}
+
+func (c *asyncCache) Delete(ctx context.Context, key string) error {
+	return c.inner.Delete(ctx, key)
+}
+
+func (c *asyncCache) DeleteMany(ctx context.Context, keys []string) error {
+	return c.inner.DeleteMany(ctx, keys)
+}
+
+func (c *asyncCache) DeletePrefix(ctx context.Context, prefix string) error {
+	return c.inner.DeletePrefix(ctx, prefix)
+}
+
+type fanoutTask struct {
+	ctx context.Context
+	fn  func(context.Context)
+}
+
+type fanout struct {
+	name   string
+	ch     chan fanoutTask
+	ctx    context.Context
+	cancel context.CancelFunc
+	wg     sync.WaitGroup
+}
+
+func newFanout(name string, workers, buffer int) *fanout {
+	if workers <= 0 {
+		workers = defaultAsyncFillWorkers
+	}
+	if buffer <= 0 {
+		buffer = defaultAsyncFillBuffer
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	f := &fanout{
+		name:   name,
+		ch:     make(chan fanoutTask, buffer),
+		ctx:    ctx,
+		cancel: cancel,
+	}
+	f.wg.Add(workers)
+	for idx := 0; idx < workers; idx++ {
+		go f.run()
+	}
+	return f
+}
+
+func (f *fanout) Do(ctx context.Context, fn func(context.Context)) error {
+	if fn == nil {
+		return nil
+	}
+	if err := f.ctx.Err(); err != nil {
+		return err
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	select {
+	case f.ch <- fanoutTask{ctx: contextWithoutCancel(ctx), fn: fn}:
+		return nil
+	default:
+		return ErrAsyncQueueFull
+	}
+}
+
+func (f *fanout) Len() int {
+	if f == nil {
+		return 0
+	}
+	return len(f.ch)
+}
+
+func (f *fanout) Close() {
+	if f == nil {
+		return
+	}
+	f.cancel()
+	f.wg.Wait()
+}
+
+func (f *fanout) run() {
+	defer f.wg.Done()
+	for {
+		select {
+		case task := <-f.ch:
+			f.safeRun(task)
+			recordCacheAsyncQueue(len(f.ch))
+		case <-f.ctx.Done():
+			return
+		}
+	}
+}
+
+func (f *fanout) safeRun(task fanoutTask) {
+	defer func() {
+		if recover() != nil {
+			recordCacheAsyncPanic()
+			_ = debug.Stack()
+		}
+	}()
+	task.fn(task.ctx)
+}
+
+func contextWithoutCancel(ctx context.Context) context.Context {
+	if ctx == nil {
+		return context.Background()
+	}
+	return context.WithoutCancel(ctx)
+}

server/internal/shared/cache/cache.go

@@ -8,9 +8,18 @@ import (
 
 var ErrNotFound = errors.New("cache: key not found")
 
+type Item struct {
+	Key   string
+	Value []byte
+	TTL   time.Duration
+}
+
 type Cache interface {
 	Get(ctx context.Context, key string) ([]byte, error)
+	GetMulti(ctx context.Context, keys []string) (map[string][]byte, error)
 	Set(ctx context.Context, key string, value []byte, ttl time.Duration) error
+	SetMulti(ctx context.Context, items []Item) error
 	Delete(ctx context.Context, key string) error
+	DeleteMany(ctx context.Context, keys []string) error
 	DeletePrefix(ctx context.Context, prefix string) error
 }

server/internal/shared/cache/factory.go

@@ -2,11 +2,39 @@ package cache
 
 import goredis "github.com/redis/go-redis/v9"
 
-func New(driver string, rdb *goredis.Client) Cache {
+func New(driver string, rdb *goredis.Client, opts ...Option) Cache {
+	options := DefaultOptions()
+	for _, opt := range opts {
+		if opt != nil {
+			opt(&options)
+		}
+	}
+	return NewWithOptions(driver, rdb, options)
+}
+
+func NewWithOptions(driver string, rdb *goredis.Client, options Options) Cache {
+	options = normalizeOptions(options)
+
+	var backend Cache
 	switch driver {
 	case "redis":
-		return newRedisCache(rdb)
+		backend = newRedisCache(rdb, options.DeleteScanCount)
 	default:
-		return newMemoryCache()
+		backend = newMemoryCache()
 	}
+
+	if options.Namespace != "" {
+		backend = newPrefixCache(backend, options.Namespace)
+	}
+	if options.L1Enabled && driver == "redis" {
+		backend = newL1Cache(backend, options.L1TTL)
+	}
+	if options.MetricsEnabled {
+		backend = newObservedCache(backend, driver)
+	}
+	if options.AsyncFillEnabled {
+		backend = newAsyncCache(backend, options.AsyncFillWorkers, options.AsyncFillBuffer, options.AsyncFillTimeout)
+	}
+
+	return newOptionsCache(backend, options)
 }

server/internal/shared/cache/l1.go

@@ -0,0 +1,117 @@
+package cache
+
+import (
+	"context"
+	"strings"
+	"time"
+)
+
+type l1Cache struct {
+	inner Cache
+	local Cache
+	ttl   time.Duration
+}
+
+func newL1Cache(inner Cache, ttl time.Duration) Cache {
+	if ttl <= 0 {
+		return inner
+	}
+	return &l1Cache{inner: inner, local: newMemoryCache(), ttl: ttl}
+}
+
+func (c *l1Cache) Get(ctx context.Context, key string) ([]byte, error) {
+	if isCacheControlKey(key) {
+		return c.inner.Get(ctx, key)
+	}
+	if value, err := c.local.Get(ctx, key); err == nil {
+		recordCacheL1("hit")
+		return value, nil
+	}
+	recordCacheL1("miss")
+
+	value, err := c.inner.Get(ctx, key)
+	if err != nil {
+		return nil, err
+	}
+	_ = c.local.Set(ctx, key, value, c.ttl)
+	return value, nil
+}
+
+func (c *l1Cache) GetMulti(ctx context.Context, keys []string) (map[string][]byte, error) {
+	if len(keys) == 0 {
+		return map[string][]byte{}, nil
+	}
+
+	result := make(map[string][]byte, len(keys))
+	miss := make([]string, 0, len(keys))
+	for _, key := range keys {
+		value, err := c.local.Get(ctx, key)
+		if err == nil {
+			recordCacheL1("hit")
+			result[key] = value
+			continue
+		}
+		recordCacheL1("miss")
+		miss = append(miss, key)
+	}
+	if len(miss) == 0 {
+		return result, nil
+	}
+
+	values, err := c.inner.GetMulti(ctx, miss)
+	if err != nil {
+		return nil, err
+	}
+	for key, value := range values {
+		result[key] = value
+		_ = c.local.Set(ctx, key, value, c.ttl)
+	}
+	return result, nil
+}
+
+func (c *l1Cache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
+	if isCacheControlKey(key) {
+		return c.inner.Set(ctx, key, value, ttl)
+	}
+	if err := c.inner.Set(ctx, key, value, ttl); err != nil {
+		return err
+	}
+	return c.local.Set(ctx, key, value, c.l1TTL(ttl))
+}
+
+func (c *l1Cache) SetMulti(ctx context.Context, items []Item) error {
+	if err := c.inner.SetMulti(ctx, items); err != nil {
+		return err
+	}
+	for _, item := range items {
+		_ = c.local.Set(ctx, item.Key, item.Value, c.l1TTL(item.TTL))
+	}
+	return nil
+}
+
+func (c *l1Cache) Delete(ctx context.Context, key string) error {
+	_ = c.local.Delete(ctx, key)
+	return c.inner.Delete(ctx, key)
+}
+
+func (c *l1Cache) DeleteMany(ctx context.Context, keys []string) error {
+	_ = c.local.DeleteMany(ctx, keys)
+	return c.inner.DeleteMany(ctx, keys)
+}
+
+func (c *l1Cache) DeletePrefix(ctx context.Context, prefix string) error {
+	_ = c.local.DeletePrefix(ctx, prefix)
+	return c.inner.DeletePrefix(ctx, prefix)
+}
+
+func (c *l1Cache) l1TTL(ttl time.Duration) time.Duration {
+	if ttl > 0 && ttl < c.ttl {
+		return ttl
+	}
+	return c.ttl
+}
+
+func isCacheControlKey(key string) bool {
+	key = strings.TrimSpace(key)
+	return strings.HasPrefix(key, "cache:version:") || strings.Contains(key, ":cache:version:")
+}

server/internal/shared/cache/memory.go

@@ -41,6 +41,25 @@ func (c *memoryCache) Get(_ context.Context, key string) ([]byte, error) {
 	return dst, nil
 }
 
+func (c *memoryCache) GetMulti(ctx context.Context, keys []string) (map[string][]byte, error) {
+	if len(keys) == 0 {
+		return map[string][]byte{}, nil
+	}
+
+	result := make(map[string][]byte, len(keys))
+	for _, key := range keys {
+		value, err := c.Get(ctx, key)
+		if err != nil {
+			if err == ErrNotFound {
+				continue
+			}
+			return nil, err
+		}
+		result[key] = value
+	}
+	return result, nil
+}
+
 func (c *memoryCache) Set(_ context.Context, key string, value []byte, ttl time.Duration) error {
 	dst := make([]byte, len(value))
 	copy(dst, value)
@@ -51,6 +70,15 @@ func (c *memoryCache) Set(_ context.Context, key string, value []byte, ttl time.
 	return nil
 }
 
+func (c *memoryCache) SetMulti(ctx context.Context, items []Item) error {
+	for _, item := range items {
+		if err := c.Set(ctx, item.Key, item.Value, item.TTL); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (c *memoryCache) Delete(_ context.Context, key string) error {
 	c.mu.Lock()
 	delete(c.entries, key)
@@ -58,6 +86,15 @@ func (c *memoryCache) Delete(_ context.Context, key string) error {
 	return nil
 }
 
+func (c *memoryCache) DeleteMany(ctx context.Context, keys []string) error {
+	for _, key := range keys {
+		if err := c.Delete(ctx, key); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (c *memoryCache) DeletePrefix(_ context.Context, prefix string) error {
 	if prefix == "" {
 		return nil

server/internal/shared/cache/memory_test.go

@@ -0,0 +1,44 @@
+package cache
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestMemoryCacheMultiAndDeleteMany(t *testing.T) {
+	ctx := context.Background()
+	c := newMemoryCache()
+
+	if err := c.SetMulti(ctx, []Item{
+		{Key: "a", Value: []byte("one"), TTL: time.Minute},
+		{Key: "b", Value: []byte("two"), TTL: time.Minute},
+	}); err != nil {
+		t.Fatalf("SetMulti() error = %v", err)
+	}
+
+	values, err := c.GetMulti(ctx, []string{"a", "missing", "b"})
+	if err != nil {
+		t.Fatalf("GetMulti() error = %v", err)
+	}
+	if got := string(values["a"]); got != "one" {
+		t.Fatalf("values[a] = %q, want one", got)
+	}
+	if got := string(values["b"]); got != "two" {
+		t.Fatalf("values[b] = %q, want two", got)
+	}
+	if _, ok := values["missing"]; ok {
+		t.Fatalf("missing key should not be returned")
+	}
+
+	if err := c.DeleteMany(ctx, []string{"a", "b"}); err != nil {
+		t.Fatalf("DeleteMany() error = %v", err)
+	}
+	values, err = c.GetMulti(ctx, []string{"a", "b"})
+	if err != nil {
+		t.Fatalf("GetMulti() after DeleteMany error = %v", err)
+	}
+	if len(values) != 0 {
+		t.Fatalf("values after DeleteMany = %v, want empty", values)
+	}
+}

server/internal/shared/cache/metrics.go

@@ -0,0 +1,263 @@
+package cache
+
+import (
+	"context"
+	"errors"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type metricsKey struct {
+	Operation string
+	Result    string
+}
+
+type cacheMetricsRecorder struct {
+	operations       sync.Map
+	l1HitTotal       atomic.Int64
+	l1MissTotal      atomic.Int64
+	asyncDropTotal   atomic.Int64
+	asyncPanicTotal  atomic.Int64
+	asyncQueueLength atomic.Int64
+}
+
+type cacheOperationMetrics struct {
+	Count          atomic.Int64
+	DurationNanos atomic.Int64
+}
+
+var cacheMetrics = &cacheMetricsRecorder{}
+
+type observedCache struct {
+	inner  Cache
+	driver string
+}
+
+func newObservedCache(inner Cache, driver string) Cache {
+	driver = strings.TrimSpace(driver)
+	if driver == "" {
+		driver = "unknown"
+	}
+	return &observedCache{inner: inner, driver: driver}
+}
+
+func (c *observedCache) Get(ctx context.Context, key string) ([]byte, error) {
+	start := time.Now()
+	value, err := c.inner.Get(ctx, key)
+	recordCacheOperation("get", cacheResult(err), time.Since(start))
+	return value, err
+}
+
+func (c *observedCache) GetMulti(ctx context.Context, keys []string) (map[string][]byte, error) {
+	start := time.Now()
+	values, err := c.inner.GetMulti(ctx, keys)
+	result := cacheResult(err)
+	if err == nil {
+		recordCacheOperation("get_multi_hit", "ok", time.Duration(0))
+		missCount := len(keys) - len(values)
+		if missCount > 0 {
+			for i := 0; i < missCount; i++ {
+				recordCacheOperation("get_multi_miss", "not_found", time.Duration(0))
+			}
+		}
+	}
+	recordCacheOperation("get_multi", result, time.Since(start))
+	return values, err
+}
+
+func (c *observedCache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
+	start := time.Now()
+	err := c.inner.Set(ctx, key, value, ttl)
+	recordCacheOperation("set", cacheResult(err), time.Since(start))
+	return err
+}
+
+func (c *observedCache) SetMulti(ctx context.Context, items []Item) error {
+	start := time.Now()
+	err := c.inner.SetMulti(ctx, items)
+	recordCacheOperation("set_multi", cacheResult(err), time.Since(start))
+	return err
+}
+
+func (c *observedCache) Delete(ctx context.Context, key string) error {
+	start := time.Now()
+	err := c.inner.Delete(ctx, key)
+	recordCacheOperation("delete", cacheResult(err), time.Since(start))
+	return err
+}
+
+func (c *observedCache) DeleteMany(ctx context.Context, keys []string) error {
+	start := time.Now()
+	err := c.inner.DeleteMany(ctx, keys)
+	recordCacheOperation("delete_many", cacheResult(err), time.Since(start))
+	return err
+}
+
+func (c *observedCache) DeletePrefix(ctx context.Context, prefix string) error {
+	start := time.Now()
+	err := c.inner.DeletePrefix(ctx, prefix)
+	recordCacheOperation("delete_prefix", cacheResult(err), time.Since(start))
+	return err
+}
+
+func cacheResult(err error) string {
+	switch {
+	case err == nil:
+		return "ok"
+	case errors.Is(err, ErrNotFound):
+		return "not_found"
+	case errors.Is(err, ErrAsyncQueueFull):
+		return "async_queue_full"
+	default:
+		return "error"
+	}
+}
+
+func recordCacheOperation(operation, result string, duration time.Duration) {
+	if operation == "" {
+		operation = "unknown"
+	}
+	if result == "" {
+		result = "unknown"
+	}
+	valueAny, _ := cacheMetrics.operations.LoadOrStore(metricsKey{
+		Operation: operation,
+		Result:    result,
+	}, &cacheOperationMetrics{})
+	value := valueAny.(*cacheOperationMetrics)
+	value.Count.Add(1)
+	if duration > 0 {
+		value.DurationNanos.Add(duration.Nanoseconds())
+	}
+}
+
+func recordCacheL1(result string) {
+	if result == "hit" {
+		cacheMetrics.l1HitTotal.Add(1)
+		return
+	}
+	cacheMetrics.l1MissTotal.Add(1)
+}
+
+func recordCacheAsyncDrop(operation string) {
+	cacheMetrics.asyncDropTotal.Add(1)
+	recordCacheOperation("async_drop_"+operation, "dropped", time.Duration(0))
+}
+
+func recordCacheAsyncPanic() {
+	cacheMetrics.asyncPanicTotal.Add(1)
+}
+
+func recordCacheAsyncQueue(length int) {
+	cacheMetrics.asyncQueueLength.Store(int64(length))
+}
+
+type MetricsSnapshot struct {
+	Operations       []OperationMetric `json:"operations"`
+	L1HitTotal       int64             `json:"l1_hit_total"`
+	L1MissTotal      int64             `json:"l1_miss_total"`
+	AsyncDropTotal   int64             `json:"async_drop_total"`
+	AsyncPanicTotal  int64             `json:"async_panic_total"`
+	AsyncQueueLength int64             `json:"async_queue_length"`
+}
+
+type OperationMetric struct {
+	Operation       string  `json:"operation"`
+	Result          string  `json:"result"`
+	Count           int64   `json:"count"`
+	DurationSeconds float64 `json:"duration_seconds"`
+}
+
+func MetricsSnapshotValue() MetricsSnapshot {
+	snapshot := MetricsSnapshot{
+		L1HitTotal:       cacheMetrics.l1HitTotal.Load(),
+		L1MissTotal:      cacheMetrics.l1MissTotal.Load(),
+		AsyncDropTotal:   cacheMetrics.asyncDropTotal.Load(),
+		AsyncPanicTotal:  cacheMetrics.asyncPanicTotal.Load(),
+		AsyncQueueLength: cacheMetrics.asyncQueueLength.Load(),
+	}
+	cacheMetrics.operations.Range(func(keyAny, valueAny any) bool {
+		key, ok := keyAny.(metricsKey)
+		if !ok {
+			return true
+		}
+		value, ok := valueAny.(*cacheOperationMetrics)
+		if !ok {
+			return true
+		}
+		snapshot.Operations = append(snapshot.Operations, OperationMetric{
+			Operation:       key.Operation,
+			Result:          key.Result,
+			Count:           value.Count.Load(),
+			DurationSeconds: float64(value.DurationNanos.Load()) / float64(time.Second),
+		})
+		return true
+	})
+	sort.Slice(snapshot.Operations, func(i, j int) bool {
+		if snapshot.Operations[i].Operation == snapshot.Operations[j].Operation {
+			return snapshot.Operations[i].Result < snapshot.Operations[j].Result
+		}
+		return snapshot.Operations[i].Operation < snapshot.Operations[j].Operation
+	})
+	return snapshot
+}
+
+func ResetMetricsForTest() {
+	cacheMetrics = &cacheMetricsRecorder{}
+}
+
+type MetricsCollector struct{}
+
+func (MetricsCollector) Describe(chan<- *prometheus.Desc) {}
+
+func (MetricsCollector) Collect(ch chan<- prometheus.Metric) {
+	snapshot := MetricsSnapshotValue()
+	for _, item := range snapshot.Operations {
+		ch <- prometheus.MustNewConstMetric(
+			prometheus.NewDesc("cache_operations_total", "Total cache operations by operation and result.", []string{"operation", "result"}, nil),
+			prometheus.CounterValue,
+			float64(item.Count),
+			item.Operation,
+			item.Result,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			prometheus.NewDesc("cache_operation_duration_seconds_total", "Total cache operation duration by operation and result.", []string{"operation", "result"}, nil),
+			prometheus.CounterValue,
+			item.DurationSeconds,
+			item.Operation,
+			item.Result,
+		)
+	}
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc("cache_l1_access_total", "Total L1 cache access by result.", []string{"result"}, nil),
+		prometheus.CounterValue,
+		float64(snapshot.L1HitTotal),
+		"hit",
+	)
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc("cache_l1_access_total", "Total L1 cache access by result.", []string{"result"}, nil),
+		prometheus.CounterValue,
+		float64(snapshot.L1MissTotal),
+		"miss",
+	)
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc("cache_async_drops_total", "Total async cache fill tasks dropped because the queue was full.", nil, nil),
+		prometheus.CounterValue,
+		float64(snapshot.AsyncDropTotal),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc("cache_async_panics_total", "Total panics recovered in async cache fill workers.", nil, nil),
+		prometheus.CounterValue,
+		float64(snapshot.AsyncPanicTotal),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc("cache_async_queue_length", "Current async cache fill queue length.", nil, nil),
+		prometheus.GaugeValue,
+		float64(snapshot.AsyncQueueLength),
+	)
+}

server/internal/shared/cache/options.go

@@ -0,0 +1,159 @@
+package cache
+
+import (
+	"context"
+	"strings"
+	"time"
+)
+
+const (
+	defaultNamespace        = "geo"
+	defaultCacheJitterRatio = 0.1
+	defaultDeleteScanCount  = int64(500)
+	defaultAsyncFillWorkers = 2
+	defaultAsyncFillBuffer  = 1024
+	defaultAsyncFillTimeout = 2 * time.Second
+	defaultL1TTL            = 30 * time.Second
+)
+
+type Options struct {
+	Namespace        string
+	JitterRatio      float64
+	MetricsEnabled   bool
+	AsyncFillEnabled bool
+	AsyncFillWorkers int
+	AsyncFillBuffer  int
+	AsyncFillTimeout time.Duration
+	L1Enabled        bool
+	L1TTL            time.Duration
+	DeleteScanCount  int64
+}
+
+func DefaultOptions() Options {
+	return Options{
+		Namespace:        defaultNamespace,
+		JitterRatio:      defaultCacheJitterRatio,
+		MetricsEnabled:   true,
+		AsyncFillWorkers: defaultAsyncFillWorkers,
+		AsyncFillBuffer:  defaultAsyncFillBuffer,
+		AsyncFillTimeout: defaultAsyncFillTimeout,
+		L1TTL:            defaultL1TTL,
+		DeleteScanCount:  defaultDeleteScanCount,
+	}
+}
+
+type Option func(*Options)
+
+func WithNamespace(namespace string) Option {
+	return func(o *Options) {
+		o.Namespace = namespace
+	}
+}
+
+func WithJitterRatio(ratio float64) Option {
+	return func(o *Options) {
+		o.JitterRatio = ratio
+	}
+}
+
+func WithMetrics(enabled bool) Option {
+	return func(o *Options) {
+		o.MetricsEnabled = enabled
+	}
+}
+
+func WithAsyncFill(enabled bool, workers, buffer int, timeout time.Duration) Option {
+	return func(o *Options) {
+		o.AsyncFillEnabled = enabled
+		o.AsyncFillWorkers = workers
+		o.AsyncFillBuffer = buffer
+		o.AsyncFillTimeout = timeout
+	}
+}
+
+func WithL1(enabled bool, ttl time.Duration) Option {
+	return func(o *Options) {
+		o.L1Enabled = enabled
+		o.L1TTL = ttl
+	}
+}
+
+func WithDeleteScanCount(count int64) Option {
+	return func(o *Options) {
+		o.DeleteScanCount = count
+	}
+}
+
+func normalizeOptions(options Options) Options {
+	options.Namespace = strings.TrimSpace(options.Namespace)
+	if options.Namespace == "" {
+		options.Namespace = defaultNamespace
+	}
+	if options.JitterRatio < 0 {
+		options.JitterRatio = 0
+	}
+	if options.JitterRatio > 1 {
+		options.JitterRatio = 1
+	}
+	if options.AsyncFillWorkers <= 0 {
+		options.AsyncFillWorkers = defaultAsyncFillWorkers
+	}
+	if options.AsyncFillBuffer <= 0 {
+		options.AsyncFillBuffer = defaultAsyncFillBuffer
+	}
+	if options.AsyncFillTimeout <= 0 {
+		options.AsyncFillTimeout = defaultAsyncFillTimeout
+	}
+	if options.L1TTL <= 0 {
+		options.L1TTL = defaultL1TTL
+	}
+	if options.DeleteScanCount <= 0 {
+		options.DeleteScanCount = defaultDeleteScanCount
+	}
+	return options
+}
+
+type optionsProvider interface {
+	CacheOptions() Options
+}
+
+type optionsCache struct {
+	inner   Cache
+	options Options
+}
+
+func newOptionsCache(inner Cache, options Options) Cache {
+	return &optionsCache{inner: inner, options: normalizeOptions(options)}
+}
+
+func (c *optionsCache) CacheOptions() Options {
+	return c.options
+}
+
+func (c *optionsCache) Get(ctx context.Context, key string) ([]byte, error) {
+	return c.inner.Get(ctx, key)
+}
+
+func (c *optionsCache) GetMulti(ctx context.Context, keys []string) (map[string][]byte, error) {
+	return c.inner.GetMulti(ctx, keys)
+}
+
+func (c *optionsCache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
+	return c.inner.Set(ctx, key, value, ttl)
+}
+
+func (c *optionsCache) SetMulti(ctx context.Context, items []Item) error {
+	return c.inner.SetMulti(ctx, items)
+}
+
+func (c *optionsCache) Delete(ctx context.Context, key string) error {
+	return c.inner.Delete(ctx, key)
+}
+
+func (c *optionsCache) DeleteMany(ctx context.Context, keys []string) error {
+	return c.inner.DeleteMany(ctx, keys)
+}
+
+func (c *optionsCache) DeletePrefix(ctx context.Context, prefix string) error {
+	return c.inner.DeletePrefix(ctx, prefix)
+}

server/internal/shared/cache/prefix.go

@@ -0,0 +1,91 @@
+package cache
+
+import (
+	"context"
+	"strings"
+	"time"
+)
+
+type prefixCache struct {
+	inner  Cache
+	prefix string
+}
+
+func newPrefixCache(inner Cache, namespace string) Cache {
+	namespace = strings.Trim(strings.TrimSpace(namespace), ":")
+	if namespace == "" {
+		return inner
+	}
+	return &prefixCache{inner: inner, prefix: namespace + ":"}
+}
+
+func (c *prefixCache) Get(ctx context.Context, key string) ([]byte, error) {
+	return c.inner.Get(ctx, c.key(key))
+}
+
+func (c *prefixCache) GetMulti(ctx context.Context, keys []string) (map[string][]byte, error) {
+	if len(keys) == 0 {
+		return map[string][]byte{}, nil
+	}
+
+	prefixed := make([]string, len(keys))
+	originalByPrefixed := make(map[string]string, len(keys))
+	for idx, key := range keys {
+		prefixedKey := c.key(key)
+		prefixed[idx] = prefixedKey
+		originalByPrefixed[prefixedKey] = key
+	}
+
+	values, err := c.inner.GetMulti(ctx, prefixed)
+	if err != nil {
+		return nil, err
+	}
+	result := make(map[string][]byte, len(values))
+	for prefixedKey, value := range values {
+		key, ok := originalByPrefixed[prefixedKey]
+		if !ok {
+			continue
+		}
+		result[key] = value
+	}
+	return result, nil
+}
+
+func (c *prefixCache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
+	return c.inner.Set(ctx, c.key(key), value, ttl)
+}
+
+func (c *prefixCache) SetMulti(ctx context.Context, items []Item) error {
+	if len(items) == 0 {
+		return nil
+	}
+	prefixed := make([]Item, len(items))
+	for idx, item := range items {
+		item.Key = c.key(item.Key)
+		prefixed[idx] = item
+	}
+	return c.inner.SetMulti(ctx, prefixed)
+}
+
+func (c *prefixCache) Delete(ctx context.Context, key string) error {
+	return c.inner.Delete(ctx, c.key(key))
+}
+
+func (c *prefixCache) DeleteMany(ctx context.Context, keys []string) error {
+	if len(keys) == 0 {
+		return nil
+	}
+	prefixed := make([]string, len(keys))
+	for idx, key := range keys {
+		prefixed[idx] = c.key(key)
+	}
+	return c.inner.DeleteMany(ctx, prefixed)
+}
+
+func (c *prefixCache) DeletePrefix(ctx context.Context, prefix string) error {
+	return c.inner.DeletePrefix(ctx, c.key(prefix))
+}
+
+func (c *prefixCache) key(key string) string {
+	return c.prefix + strings.TrimPrefix(key, c.prefix)
+}

server/internal/shared/cache/readthrough.go

@@ -3,6 +3,7 @@ package cache
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"math/rand"
 	"sync"
 	"time"
@@ -11,7 +12,7 @@ import (
 )
 
 const (
-	defaultJitterRatio = 0.1
+	readthroughDefaultJitterRatio = 0.1
 )
 
 var (
@@ -62,6 +63,7 @@ func loadJSONInternal[T any](
 ) (T, bool, error) {
 	var zero T
 	if c == nil {
+		recordCacheOperation("readthrough_load", "cache_disabled", time.Duration(0))
 		return loader(ctx)
 	}
 
@@ -69,14 +71,23 @@ func loadJSONInternal[T any](
 		var envelope payloadEnvelope[T]
 		if unmarshalErr := json.Unmarshal(raw, &envelope); unmarshalErr == nil {
 			if envelope.Empty {
+				recordCacheOperation("readthrough_empty", "hit", time.Duration(0))
 				return zero, false, nil
 			}
+			recordCacheOperation("readthrough_hit", "ok", time.Duration(0))
 			return envelope.Value, true, nil
 		}
+		recordCacheOperation("readthrough_unmarshal", "error", time.Duration(0))
+	} else if errors.Is(err, ErrNotFound) {
+		recordCacheOperation("readthrough_miss", "not_found", time.Duration(0))
+	} else {
+		recordCacheOperation("readthrough_get", "error", time.Duration(0))
 	}
 
 	result, err, _ := group.Do(key, func() (interface{}, error) {
+		start := time.Now()
 		value, found, loadErr := loader(ctx)
+		recordCacheOperation("readthrough_load", cacheResult(loadErr), time.Since(start))
 		if loadErr != nil {
 			return nil, loadErr
 		}
@@ -86,13 +97,18 @@ func loadJSONInternal[T any](
 			Value: value,
 		}
 		if raw, marshalErr := json.Marshal(envelope); marshalErr == nil {
+			recordCacheOperation("readthrough_marshal", "ok", time.Duration(0))
 			cacheTTL := ttl
 			if !found {
 				cacheTTL = emptyTTL
 			}
 			if cacheTTL > 0 {
-				_ = c.Set(ctx, key, raw, ApplyJitter(cacheTTL))
+				setStart := time.Now()
+				setErr := c.Set(ctx, key, raw, ApplyJitterWithRatio(cacheTTL, cacheJitterRatio(c)))
+				recordCacheOperation("readthrough_set", cacheResult(setErr), time.Since(setStart))
 			}
+		} else {
+			recordCacheOperation("readthrough_marshal", "error", time.Duration(0))
 		}
 
 		return envelope, nil
@@ -112,11 +128,21 @@ func loadJSONInternal[T any](
 }
 
 func ApplyJitter(ttl time.Duration) time.Duration {
+	return ApplyJitterWithRatio(ttl, readthroughDefaultJitterRatio)
+}
+
+func ApplyJitterWithRatio(ttl time.Duration, ratio float64) time.Duration {
 	if ttl <= 0 {
 		return ttl
 	}
+	if ratio <= 0 {
+		return ttl
+	}
+	if ratio > 1 {
+		ratio = 1
+	}
 
-	maxJitter := int64(float64(ttl) * defaultJitterRatio)
+	maxJitter := int64(float64(ttl) * ratio)
 	if maxJitter <= 0 {
 		return ttl
 	}
@@ -126,3 +152,13 @@ func ApplyJitter(ttl time.Duration) time.Duration {
 
 	return ttl + time.Duration(jitterRand.Int63n(maxJitter+1))
 }
+
+func cacheJitterRatio(c Cache) float64 {
+	if c == nil {
+		return readthroughDefaultJitterRatio
+	}
+	if provider, ok := c.(optionsProvider); ok {
+		return provider.CacheOptions().JitterRatio
+	}
+	return readthroughDefaultJitterRatio
+}

server/internal/shared/cache/readthrough_test.go

@@ -0,0 +1,63 @@
+package cache
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"golang.org/x/sync/singleflight"
+)
+
+func TestLoadJSONWithEmptyCachesEmptyAndRecordsMetrics(t *testing.T) {
+	ResetMetricsForTest()
+	ctx := context.Background()
+	c := NewWithOptions("memory", nil, Options{
+		Namespace:      "test",
+		JitterRatio:    0,
+		MetricsEnabled: true,
+	})
+	var group singleflight.Group
+	loads := 0
+
+	value, found, err := LoadJSONWithEmpty(ctx, c, &group, "empty", time.Minute, time.Minute, func(context.Context) (string, bool, error) {
+		loads++
+		return "", false, nil
+	})
+	if err != nil {
+		t.Fatalf("LoadJSONWithEmpty() error = %v", err)
+	}
+	if found || value != "" {
+		t.Fatalf("first load value=%q found=%v, want empty not found", value, found)
+	}
+
+	value, found, err = LoadJSONWithEmpty(ctx, c, &group, "empty", time.Minute, time.Minute, func(context.Context) (string, bool, error) {
+		loads++
+		return "unexpected", true, nil
+	})
+	if err != nil {
+		t.Fatalf("LoadJSONWithEmpty() cached error = %v", err)
+	}
+	if found || value != "" {
+		t.Fatalf("cached value=%q found=%v, want empty not found", value, found)
+	}
+	if loads != 1 {
+		t.Fatalf("loader calls = %d, want 1", loads)
+	}
+
+	snapshot := MetricsSnapshotValue()
+	if metricCount(snapshot, "readthrough_empty", "hit") != 1 {
+		t.Fatalf("readthrough_empty hit metric missing in %#v", snapshot.Operations)
+	}
+	if metricCount(snapshot, "readthrough_load", "ok") != 1 {
+		t.Fatalf("readthrough_load ok metric missing in %#v", snapshot.Operations)
+	}
+}
+
+func metricCount(snapshot MetricsSnapshot, operation, result string) int64 {
+	for _, item := range snapshot.Operations {
+		if item.Operation == operation && item.Result == result {
+			return item.Count
+		}
+	}
+	return 0
+}

server/internal/shared/cache/redis.go

@@ -3,17 +3,19 @@ package cache
 import (
 	"context"
 	"errors"
+	"fmt"
 	"time"
 
 	goredis "github.com/redis/go-redis/v9"
 )
 
 type redisCache struct {
-	client *goredis.Client
+	client          *goredis.Client
+	deleteScanCount int64
 }
 
-func newRedisCache(client *goredis.Client) Cache {
-	return &redisCache{client: client}
+func newRedisCache(client *goredis.Client, deleteScanCount int64) Cache {
+	return &redisCache{client: client, deleteScanCount: deleteScanCount}
 }
 
 func (c *redisCache) Get(ctx context.Context, key string) ([]byte, error) {
@@ -24,28 +26,81 @@ func (c *redisCache) Get(ctx context.Context, key string) ([]byte, error) {
 	return val, err
 }
 
+func (c *redisCache) GetMulti(ctx context.Context, keys []string) (map[string][]byte, error) {
+	if len(keys) == 0 {
+		return map[string][]byte{}, nil
+	}
+
+	values, err := c.client.MGet(ctx, keys...).Result()
+	if err != nil {
+		return nil, err
+	}
+
+	result := make(map[string][]byte, len(keys))
+	for idx, value := range values {
+		if value == nil {
+			continue
+		}
+		switch typed := value.(type) {
+		case string:
+			result[keys[idx]] = []byte(typed)
+		case []byte:
+			dst := make([]byte, len(typed))
+			copy(dst, typed)
+			result[keys[idx]] = dst
+		default:
+			result[keys[idx]] = []byte(fmt.Sprint(value))
+		}
+	}
+	return result, nil
+}
+
 func (c *redisCache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
 	return c.client.Set(ctx, key, value, ttl).Err()
 }
 
+func (c *redisCache) SetMulti(ctx context.Context, items []Item) error {
+	if len(items) == 0 {
+		return nil
+	}
+
+	pipe := c.client.Pipeline()
+	for _, item := range items {
+		pipe.Set(ctx, item.Key, item.Value, item.TTL)
+	}
+	_, err := pipe.Exec(ctx)
+	return err
+}
+
 func (c *redisCache) Delete(ctx context.Context, key string) error {
 	return c.client.Del(ctx, key).Err()
 }
 
+func (c *redisCache) DeleteMany(ctx context.Context, keys []string) error {
+	if len(keys) == 0 {
+		return nil
+	}
+	return c.client.Del(ctx, keys...).Err()
+}
+
 func (c *redisCache) DeletePrefix(ctx context.Context, prefix string) error {
 	if prefix == "" {
 		return nil
 	}
 
 	pattern := prefix + "*"
+	scanCount := c.deleteScanCount
+	if scanCount <= 0 {
+		scanCount = defaultDeleteScanCount
+	}
 	var cursor uint64
 	for {
-		keys, nextCursor, err := c.client.Scan(ctx, cursor, pattern, 100).Result()
+		keys, nextCursor, err := c.client.Scan(ctx, cursor, pattern, scanCount).Result()
 		if err != nil {
 			return err
 		}
 		if len(keys) > 0 {
-			if err := c.client.Del(ctx, keys...).Err(); err != nil {
+			if err := c.DeleteMany(ctx, keys); err != nil {
 				return err
 			}
 		}