feat(cache,worker): overhaul cache layer and add generation task lease recovery
- shared/cache: add Options/L1/async/metrics/prefix decorators, multi-key ops, Redis pool tuning, and JSON readthrough metrics - worker-generate: claim tasks via DB lease + heartbeat, requeue stale queued tasks, expire dead leases with refund/cache invalidation - tenant: version article cache keys so worker recovery invalidations propagate cleanly - shared/config: expand Redis (pool/timeouts/TLS) and Generation (lease/recovery) configs with defaults Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
+263
@@ -0,0 +1,263 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
type metricsKey struct {
|
||||
Operation string
|
||||
Result string
|
||||
}
|
||||
|
||||
type cacheMetricsRecorder struct {
|
||||
operations sync.Map
|
||||
l1HitTotal atomic.Int64
|
||||
l1MissTotal atomic.Int64
|
||||
asyncDropTotal atomic.Int64
|
||||
asyncPanicTotal atomic.Int64
|
||||
asyncQueueLength atomic.Int64
|
||||
}
|
||||
|
||||
type cacheOperationMetrics struct {
|
||||
Count atomic.Int64
|
||||
DurationNanos atomic.Int64
|
||||
}
|
||||
|
||||
var cacheMetrics = &cacheMetricsRecorder{}
|
||||
|
||||
type observedCache struct {
|
||||
inner Cache
|
||||
driver string
|
||||
}
|
||||
|
||||
func newObservedCache(inner Cache, driver string) Cache {
|
||||
driver = strings.TrimSpace(driver)
|
||||
if driver == "" {
|
||||
driver = "unknown"
|
||||
}
|
||||
return &observedCache{inner: inner, driver: driver}
|
||||
}
|
||||
|
||||
func (c *observedCache) Get(ctx context.Context, key string) ([]byte, error) {
|
||||
start := time.Now()
|
||||
value, err := c.inner.Get(ctx, key)
|
||||
recordCacheOperation("get", cacheResult(err), time.Since(start))
|
||||
return value, err
|
||||
}
|
||||
|
||||
func (c *observedCache) GetMulti(ctx context.Context, keys []string) (map[string][]byte, error) {
|
||||
start := time.Now()
|
||||
values, err := c.inner.GetMulti(ctx, keys)
|
||||
result := cacheResult(err)
|
||||
if err == nil {
|
||||
recordCacheOperation("get_multi_hit", "ok", time.Duration(0))
|
||||
missCount := len(keys) - len(values)
|
||||
if missCount > 0 {
|
||||
for i := 0; i < missCount; i++ {
|
||||
recordCacheOperation("get_multi_miss", "not_found", time.Duration(0))
|
||||
}
|
||||
}
|
||||
}
|
||||
recordCacheOperation("get_multi", result, time.Since(start))
|
||||
return values, err
|
||||
}
|
||||
|
||||
func (c *observedCache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error {
|
||||
start := time.Now()
|
||||
err := c.inner.Set(ctx, key, value, ttl)
|
||||
recordCacheOperation("set", cacheResult(err), time.Since(start))
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *observedCache) SetMulti(ctx context.Context, items []Item) error {
|
||||
start := time.Now()
|
||||
err := c.inner.SetMulti(ctx, items)
|
||||
recordCacheOperation("set_multi", cacheResult(err), time.Since(start))
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *observedCache) Delete(ctx context.Context, key string) error {
|
||||
start := time.Now()
|
||||
err := c.inner.Delete(ctx, key)
|
||||
recordCacheOperation("delete", cacheResult(err), time.Since(start))
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *observedCache) DeleteMany(ctx context.Context, keys []string) error {
|
||||
start := time.Now()
|
||||
err := c.inner.DeleteMany(ctx, keys)
|
||||
recordCacheOperation("delete_many", cacheResult(err), time.Since(start))
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *observedCache) DeletePrefix(ctx context.Context, prefix string) error {
|
||||
start := time.Now()
|
||||
err := c.inner.DeletePrefix(ctx, prefix)
|
||||
recordCacheOperation("delete_prefix", cacheResult(err), time.Since(start))
|
||||
return err
|
||||
}
|
||||
|
||||
func cacheResult(err error) string {
|
||||
switch {
|
||||
case err == nil:
|
||||
return "ok"
|
||||
case errors.Is(err, ErrNotFound):
|
||||
return "not_found"
|
||||
case errors.Is(err, ErrAsyncQueueFull):
|
||||
return "async_queue_full"
|
||||
default:
|
||||
return "error"
|
||||
}
|
||||
}
|
||||
|
||||
func recordCacheOperation(operation, result string, duration time.Duration) {
|
||||
if operation == "" {
|
||||
operation = "unknown"
|
||||
}
|
||||
if result == "" {
|
||||
result = "unknown"
|
||||
}
|
||||
valueAny, _ := cacheMetrics.operations.LoadOrStore(metricsKey{
|
||||
Operation: operation,
|
||||
Result: result,
|
||||
}, &cacheOperationMetrics{})
|
||||
value := valueAny.(*cacheOperationMetrics)
|
||||
value.Count.Add(1)
|
||||
if duration > 0 {
|
||||
value.DurationNanos.Add(duration.Nanoseconds())
|
||||
}
|
||||
}
|
||||
|
||||
func recordCacheL1(result string) {
|
||||
if result == "hit" {
|
||||
cacheMetrics.l1HitTotal.Add(1)
|
||||
return
|
||||
}
|
||||
cacheMetrics.l1MissTotal.Add(1)
|
||||
}
|
||||
|
||||
func recordCacheAsyncDrop(operation string) {
|
||||
cacheMetrics.asyncDropTotal.Add(1)
|
||||
recordCacheOperation("async_drop_"+operation, "dropped", time.Duration(0))
|
||||
}
|
||||
|
||||
func recordCacheAsyncPanic() {
|
||||
cacheMetrics.asyncPanicTotal.Add(1)
|
||||
}
|
||||
|
||||
func recordCacheAsyncQueue(length int) {
|
||||
cacheMetrics.asyncQueueLength.Store(int64(length))
|
||||
}
|
||||
|
||||
type MetricsSnapshot struct {
|
||||
Operations []OperationMetric `json:"operations"`
|
||||
L1HitTotal int64 `json:"l1_hit_total"`
|
||||
L1MissTotal int64 `json:"l1_miss_total"`
|
||||
AsyncDropTotal int64 `json:"async_drop_total"`
|
||||
AsyncPanicTotal int64 `json:"async_panic_total"`
|
||||
AsyncQueueLength int64 `json:"async_queue_length"`
|
||||
}
|
||||
|
||||
type OperationMetric struct {
|
||||
Operation string `json:"operation"`
|
||||
Result string `json:"result"`
|
||||
Count int64 `json:"count"`
|
||||
DurationSeconds float64 `json:"duration_seconds"`
|
||||
}
|
||||
|
||||
func MetricsSnapshotValue() MetricsSnapshot {
|
||||
snapshot := MetricsSnapshot{
|
||||
L1HitTotal: cacheMetrics.l1HitTotal.Load(),
|
||||
L1MissTotal: cacheMetrics.l1MissTotal.Load(),
|
||||
AsyncDropTotal: cacheMetrics.asyncDropTotal.Load(),
|
||||
AsyncPanicTotal: cacheMetrics.asyncPanicTotal.Load(),
|
||||
AsyncQueueLength: cacheMetrics.asyncQueueLength.Load(),
|
||||
}
|
||||
cacheMetrics.operations.Range(func(keyAny, valueAny any) bool {
|
||||
key, ok := keyAny.(metricsKey)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
value, ok := valueAny.(*cacheOperationMetrics)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
snapshot.Operations = append(snapshot.Operations, OperationMetric{
|
||||
Operation: key.Operation,
|
||||
Result: key.Result,
|
||||
Count: value.Count.Load(),
|
||||
DurationSeconds: float64(value.DurationNanos.Load()) / float64(time.Second),
|
||||
})
|
||||
return true
|
||||
})
|
||||
sort.Slice(snapshot.Operations, func(i, j int) bool {
|
||||
if snapshot.Operations[i].Operation == snapshot.Operations[j].Operation {
|
||||
return snapshot.Operations[i].Result < snapshot.Operations[j].Result
|
||||
}
|
||||
return snapshot.Operations[i].Operation < snapshot.Operations[j].Operation
|
||||
})
|
||||
return snapshot
|
||||
}
|
||||
|
||||
func ResetMetricsForTest() {
|
||||
cacheMetrics = &cacheMetricsRecorder{}
|
||||
}
|
||||
|
||||
type MetricsCollector struct{}
|
||||
|
||||
func (MetricsCollector) Describe(chan<- *prometheus.Desc) {}
|
||||
|
||||
func (MetricsCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
snapshot := MetricsSnapshotValue()
|
||||
for _, item := range snapshot.Operations {
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc("cache_operations_total", "Total cache operations by operation and result.", []string{"operation", "result"}, nil),
|
||||
prometheus.CounterValue,
|
||||
float64(item.Count),
|
||||
item.Operation,
|
||||
item.Result,
|
||||
)
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc("cache_operation_duration_seconds_total", "Total cache operation duration by operation and result.", []string{"operation", "result"}, nil),
|
||||
prometheus.CounterValue,
|
||||
item.DurationSeconds,
|
||||
item.Operation,
|
||||
item.Result,
|
||||
)
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc("cache_l1_access_total", "Total L1 cache access by result.", []string{"result"}, nil),
|
||||
prometheus.CounterValue,
|
||||
float64(snapshot.L1HitTotal),
|
||||
"hit",
|
||||
)
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc("cache_l1_access_total", "Total L1 cache access by result.", []string{"result"}, nil),
|
||||
prometheus.CounterValue,
|
||||
float64(snapshot.L1MissTotal),
|
||||
"miss",
|
||||
)
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc("cache_async_drops_total", "Total async cache fill tasks dropped because the queue was full.", nil, nil),
|
||||
prometheus.CounterValue,
|
||||
float64(snapshot.AsyncDropTotal),
|
||||
)
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc("cache_async_panics_total", "Total panics recovered in async cache fill workers.", nil, nil),
|
||||
prometheus.CounterValue,
|
||||
float64(snapshot.AsyncPanicTotal),
|
||||
)
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc("cache_async_queue_length", "Current async cache fill queue length.", nil, nil),
|
||||
prometheus.GaugeValue,
|
||||
float64(snapshot.AsyncQueueLength),
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user