feat(cache,worker): overhaul cache layer and add generation task lease recovery

- shared/cache: add Options/L1/async/metrics/prefix decorators, multi-key ops, Redis pool tuning, and JSON readthrough metrics
- worker-generate: claim tasks via DB lease + heartbeat, requeue stale queued tasks, expire dead leases with refund/cache invalidation
- tenant: version article cache keys so worker recovery invalidations propagate cleanly
- shared/config: expand Redis (pool/timeouts/TLS) and Generation (lease/recovery) configs with defaults

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

server/internal/shared/cache/async.go

@@ -0,0 +1,178 @@
+package cache
+
+import (
+	"context"
+	"errors"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+var ErrAsyncQueueFull = errors.New("cache: async fill queue full")
+
+type asyncCache struct {
+	inner   Cache
+	fanout  *fanout
+	timeout time.Duration
+}
+
+func newAsyncCache(inner Cache, workers, buffer int, timeout time.Duration) Cache {
+	return &asyncCache{
+		inner:   inner,
+		fanout:  newFanout("cache_fill", workers, buffer),
+		timeout: timeout,
+	}
+}
+
+func (c *asyncCache) Get(ctx context.Context, key string) ([]byte, error) {
+	return c.inner.Get(ctx, key)
+}
+
+func (c *asyncCache) GetMulti(ctx context.Context, keys []string) (map[string][]byte, error) {
+	return c.inner.GetMulti(ctx, keys)
+}
+
+func (c *asyncCache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
+	if isCacheControlKey(key) {
+		return c.inner.Set(ctx, key, value, ttl)
+	}
+	err := c.fanout.Do(ctx, func(taskCtx context.Context) {
+		cctx, cancel := context.WithTimeout(taskCtx, c.timeout)
+		defer cancel()
+		if setErr := c.inner.Set(cctx, key, value, ttl); setErr != nil {
+			recordCacheOperation("set", "error", time.Duration(0))
+		}
+	})
+	recordCacheAsyncQueue(c.fanout.Len())
+	if err != nil {
+		recordCacheAsyncDrop("set")
+		return err
+	}
+	return nil
+}
+
+func (c *asyncCache) SetMulti(ctx context.Context, items []Item) error {
+	err := c.fanout.Do(ctx, func(taskCtx context.Context) {
+		cctx, cancel := context.WithTimeout(taskCtx, c.timeout)
+		defer cancel()
+		if setErr := c.inner.SetMulti(cctx, items); setErr != nil {
+			recordCacheOperation("set_multi", "error", time.Duration(0))
+		}
+	})
+	recordCacheAsyncQueue(c.fanout.Len())
+	if err != nil {
+		recordCacheAsyncDrop("set_multi")
+		return err
+	}
+	return nil
+}
+
+func (c *asyncCache) Delete(ctx context.Context, key string) error {
+	return c.inner.Delete(ctx, key)
+}
+
+func (c *asyncCache) DeleteMany(ctx context.Context, keys []string) error {
+	return c.inner.DeleteMany(ctx, keys)
+}
+
+func (c *asyncCache) DeletePrefix(ctx context.Context, prefix string) error {
+	return c.inner.DeletePrefix(ctx, prefix)
+}
+
+type fanoutTask struct {
+	ctx context.Context
+	fn  func(context.Context)
+}
+
+type fanout struct {
+	name   string
+	ch     chan fanoutTask
+	ctx    context.Context
+	cancel context.CancelFunc
+	wg     sync.WaitGroup
+}
+
+func newFanout(name string, workers, buffer int) *fanout {
+	if workers <= 0 {
+		workers = defaultAsyncFillWorkers
+	}
+	if buffer <= 0 {
+		buffer = defaultAsyncFillBuffer
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	f := &fanout{
+		name:   name,
+		ch:     make(chan fanoutTask, buffer),
+		ctx:    ctx,
+		cancel: cancel,
+	}
+	f.wg.Add(workers)
+	for idx := 0; idx < workers; idx++ {
+		go f.run()
+	}
+	return f
+}
+
+func (f *fanout) Do(ctx context.Context, fn func(context.Context)) error {
+	if fn == nil {
+		return nil
+	}
+	if err := f.ctx.Err(); err != nil {
+		return err
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	select {
+	case f.ch <- fanoutTask{ctx: contextWithoutCancel(ctx), fn: fn}:
+		return nil
+	default:
+		return ErrAsyncQueueFull
+	}
+}
+
+func (f *fanout) Len() int {
+	if f == nil {
+		return 0
+	}
+	return len(f.ch)
+}
+
+func (f *fanout) Close() {
+	if f == nil {
+		return
+	}
+	f.cancel()
+	f.wg.Wait()
+}
+
+func (f *fanout) run() {
+	defer f.wg.Done()
+	for {
+		select {
+		case task := <-f.ch:
+			f.safeRun(task)
+			recordCacheAsyncQueue(len(f.ch))
+		case <-f.ctx.Done():
+			return
+		}
+	}
+}
+
+func (f *fanout) safeRun(task fanoutTask) {
+	defer func() {
+		if recover() != nil {
+			recordCacheAsyncPanic()
+			_ = debug.Stack()
+		}
+	}()
+	task.fn(task.ctx)
+}
+
+func contextWithoutCancel(ctx context.Context) context.Context {
+	if ctx == nil {
+		return context.Background()
+	}
+	return context.WithoutCancel(ctx)
+}