feat(cache,worker): overhaul cache layer and add generation task lease recovery

- shared/cache: add Options/L1/async/metrics/prefix decorators, multi-key ops, Redis pool tuning, and JSON readthrough metrics
- worker-generate: claim tasks via DB lease + heartbeat, requeue stale queued tasks, expire dead leases with refund/cache invalidation
- tenant: version article cache keys so worker recovery invalidations propagate cleanly
- shared/config: expand Redis (pool/timeouts/TLS) and Generation (lease/recovery) configs with defaults

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

server/internal/tenant/app/article_service.go

@@ -136,7 +136,8 @@ type ArticleListResponse struct {
 
 func (s *ArticleService) List(ctx context.Context, params ArticleListParams) (*ArticleListResponse, error) {
 	actor := auth.MustActor(ctx)
-	return sharedcache.LoadJSON(ctx, s.cache, &s.cacheGroup, articleListCacheKey(actor.TenantID, params), defaultCacheTTL(), func(loadCtx context.Context) (*ArticleListResponse, error) {
+	version := articleCacheVersion(ctx, s.cache, actor.TenantID)
+	return sharedcache.LoadJSON(ctx, s.cache, &s.cacheGroup, articleListCacheKey(actor.TenantID, version, params), defaultCacheTTL(), func(loadCtx context.Context) (*ArticleListResponse, error) {
 		return s.loadArticles(loadCtx, actor.TenantID, params)
 	})
 }
@@ -167,7 +168,8 @@ type ArticleDetailResponse struct {
 
 func (s *ArticleService) Detail(ctx context.Context, id int64) (*ArticleDetailResponse, error) {
 	actor := auth.MustActor(ctx)
-	record, found, err := sharedcache.LoadJSONWithEmpty(ctx, s.cache, &s.cacheGroup, articleDetailCacheKey(actor.TenantID, id), defaultCacheTTL(), defaultCacheEmptyTTL(), func(loadCtx context.Context) (*ArticleDetailResponse, bool, error) {
+	version := articleCacheVersion(ctx, s.cache, actor.TenantID)
+	record, found, err := sharedcache.LoadJSONWithEmpty(ctx, s.cache, &s.cacheGroup, articleDetailCacheKey(actor.TenantID, id, version), defaultCacheTTL(), defaultCacheEmptyTTL(), func(loadCtx context.Context) (*ArticleDetailResponse, bool, error) {
 		return s.loadArticleDetail(loadCtx, actor.TenantID, id)
 	})
 	if err != nil {

server/internal/tenant/app/cache_support.go

@@ -6,14 +6,18 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"strings"
 	"time"
 
+	"github.com/google/uuid"
+
 	sharedcache "github.com/geo-platform/tenant-api/internal/shared/cache"
 )
 
 const (
 	defaultReadCacheTTL      = 5 * time.Minute
 	defaultReadCacheEmptyTTL = 1 * time.Minute
+	articleCacheVersionTTL   = 24 * time.Hour
 )
 
 func defaultCacheTTL() time.Duration {
@@ -203,18 +207,54 @@ func articleDetailCachePrefix(tenantID int64) string {
 	return fmt.Sprintf("article:detail:%d:", tenantID)
 }
 
-func articleDetailCacheKey(tenantID, articleID int64) string {
-	return fmt.Sprintf("%s%d", articleDetailCachePrefix(tenantID), articleID)
+func articleDetailCacheArticlePrefix(tenantID, articleID int64) string {
+	return fmt.Sprintf("%s%d:", articleDetailCachePrefix(tenantID), articleID)
 }
 
-func articleListCacheKey(tenantID int64, params interface{}) string {
-	return articleListCachePrefix(tenantID) + digestCacheKey(params)
+func articleDetailCacheKey(tenantID, articleID int64, version string) string {
+	return fmt.Sprintf("%s%s", articleDetailCacheArticlePrefix(tenantID, articleID), normalizeArticleCacheVersion(version))
+}
+
+func articleListCacheKey(tenantID int64, version string, params interface{}) string {
+	return articleListCachePrefix(tenantID) + normalizeArticleCacheVersion(version) + ":" + digestCacheKey(params)
+}
+
+func articleCacheVersionKey(tenantID int64) string {
+	return fmt.Sprintf("cache:version:article:%d", tenantID)
+}
+
+func articleCacheVersion(ctx context.Context, c sharedcache.Cache, tenantID int64) string {
+	if c == nil || tenantID <= 0 {
+		return "0"
+	}
+	raw, err := c.Get(ctx, articleCacheVersionKey(tenantID))
+	if err != nil || len(raw) == 0 {
+		return "0"
+	}
+	return normalizeArticleCacheVersion(string(raw))
+}
+
+func bumpArticleCacheVersion(ctx context.Context, c sharedcache.Cache, tenantID int64) {
+	if c == nil || tenantID <= 0 {
+		return
+	}
+	version := uuid.NewString()
+	_ = c.Set(ctx, articleCacheVersionKey(tenantID), []byte(version), articleCacheVersionTTL)
+}
+
+func normalizeArticleCacheVersion(version string) string {
+	version = strings.TrimSpace(version)
+	if version == "" {
+		return "0"
+	}
+	return version
 }
 
 func invalidateArticleCaches(ctx context.Context, c sharedcache.Cache, tenantID int64, articleID *int64) {
+	bumpArticleCacheVersion(ctx, c, tenantID)
 	deleteCachePrefix(ctx, c, articleListCachePrefix(tenantID))
 	if articleID != nil && *articleID > 0 {
-		deleteCacheKey(ctx, c, articleDetailCacheKey(tenantID, *articleID))
+		deleteCachePrefix(ctx, c, articleDetailCacheArticlePrefix(tenantID, *articleID))
 	} else {
 		deleteCachePrefix(ctx, c, articleDetailCachePrefix(tenantID))
 	}

server/internal/tenant/app/cache_support_test.go

@@ -0,0 +1,50 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	sharedcache "github.com/geo-platform/tenant-api/internal/shared/cache"
+)
+
+func TestArticleCacheVersionSeparatesLateWritesAfterInvalidation(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	c := sharedcache.New("memory", nil, sharedcache.WithNamespace("test"))
+	tenantID := int64(42)
+	articleID := int64(7)
+	params := ArticleListParams{Page: 1, PageSize: 10}
+
+	oldVersion := articleCacheVersion(ctx, c, tenantID)
+	oldListKey := articleListCacheKey(tenantID, oldVersion, params)
+	oldDetailKey := articleDetailCacheKey(tenantID, articleID, oldVersion)
+	if err := c.Set(ctx, oldListKey, []byte("generating-list"), time.Minute); err != nil {
+		t.Fatalf("set old list cache: %v", err)
+	}
+	if err := c.Set(ctx, oldDetailKey, []byte("generating-detail"), time.Minute); err != nil {
+		t.Fatalf("set old detail cache: %v", err)
+	}
+
+	invalidateArticleCaches(ctx, c, tenantID, &articleID)
+	newVersion := articleCacheVersion(ctx, c, tenantID)
+	if newVersion == oldVersion {
+		t.Fatalf("expected article cache version to change, stayed %q", newVersion)
+	}
+
+	if err := c.Set(ctx, oldListKey, []byte("late-generating-list"), time.Minute); err != nil {
+		t.Fatalf("late set old list cache: %v", err)
+	}
+	if err := c.Set(ctx, oldDetailKey, []byte("late-generating-detail"), time.Minute); err != nil {
+		t.Fatalf("late set old detail cache: %v", err)
+	}
+
+	if _, err := c.Get(ctx, articleListCacheKey(tenantID, newVersion, params)); !errors.Is(err, sharedcache.ErrNotFound) {
+		t.Fatalf("expected new list key to miss stale late write, got err %v", err)
+	}
+	if _, err := c.Get(ctx, articleDetailCacheKey(tenantID, articleID, newVersion)); !errors.Is(err, sharedcache.ErrNotFound) {
+		t.Fatalf("expected new detail key to miss stale late write, got err %v", err)
+	}
+}

server/internal/tenant/app/generation_task_runtime.go

@@ -52,6 +52,9 @@ func HandleClaimedGenerationTaskFailure(
 	defer cancel()
 
 	errMsg := formatGenerationFailure("task_load", taskErr)
+	if taskErr != nil && strings.Contains(taskErr.Error(), "[worker_recovery]") {
+		errMsg = strings.TrimSpace(taskErr.Error())
+	}
 	now := time.Now()
 
 	auditRepo := repository.NewAuditRepository(pool)

server/internal/tenant/app/knowledge_service.go

@@ -933,7 +933,7 @@ func (s *KnowledgeService) logKnowledgeResolve(
 		zap.Int("candidate_count", len(candidates)),
 		zap.Int("selected_count", len(selected)),
 		zap.Int("precise_fact_count", len(knowledgeCtx.PreciseFacts)),
-		zap.Strings("precise_facts", summarizeKnowledgeFactsForLog(knowledgeCtx.PreciseFacts)),
+		// zap.Strings("precise_facts", summarizeKnowledgeFactsForLog(knowledgeCtx.PreciseFacts)),
 	)
 }
 

server/internal/tenant/app/monitoring_metrics_prometheus.go

@@ -9,10 +9,12 @@ import (
 	promcollectors "github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/prometheus/common/expfmt"
+
+	"github.com/geo-platform/tenant-api/internal/shared/cache"
 )
 
 func MonitoringSchedulerMetricsPrometheusHandler() http.Handler {
-	return monitoringPrometheusHandler(true, monitoringDailyTaskMetricsCollector{})
+	return monitoringPrometheusHandler(true, monitoringDailyTaskMetricsCollector{}, cache.MetricsCollector{})
 }
 
 func MonitoringDailyTaskMetricsPrometheus() string {

server/internal/tenant/repository/generated/audit.sql.go

@@ -125,7 +125,12 @@ func (q *Queries) CreateTaskRecord(ctx context.Context, arg CreateTaskRecordPara
 
 const updateGenerationTaskStatus = `-- name: UpdateGenerationTaskStatus :exec
 UPDATE generation_tasks SET status = $1, error_message = $2,
-       started_at = $3, completed_at = $4, updated_at = NOW()
+       started_at = $3,
+       completed_at = $4,
+       lease_token = CASE WHEN $1::text IN ('completed', 'failed') THEN NULL ELSE lease_token END,
+       lease_owner = CASE WHEN $1::text IN ('completed', 'failed') THEN NULL ELSE lease_owner END,
+       lease_expires_at = CASE WHEN $1::text IN ('completed', 'failed') THEN NULL ELSE lease_expires_at END,
+       updated_at = NOW()
 WHERE id = $5 AND tenant_id = $6
 `
 

server/internal/tenant/repository/generated/models.go

@@ -265,6 +265,11 @@ type GenerationTask struct {
 	CreatedAt          pgtype.Timestamptz `json:"created_at"`
 	UpdatedAt          pgtype.Timestamptz `json:"updated_at"`
 	OperatorID         pgtype.Int8        `json:"operator_id"`
+	LeaseToken         pgtype.Text        `json:"lease_token"`
+	LeaseOwner         pgtype.Text        `json:"lease_owner"`
+	LeaseExpiresAt     pgtype.Timestamptz `json:"lease_expires_at"`
+	AttemptCount       int32              `json:"attempt_count"`
+	LastHeartbeatAt    pgtype.Timestamptz `json:"last_heartbeat_at"`
 }
 
 type ImageAsset struct {

server/internal/tenant/repository/queries/audit.sql

@@ -40,5 +40,10 @@ WHERE tenant_id = sqlc.arg(tenant_id)
 
 -- name: UpdateGenerationTaskStatus :exec
 UPDATE generation_tasks SET status = sqlc.arg(status), error_message = sqlc.arg(error_message),
-       started_at = sqlc.arg(started_at), completed_at = sqlc.arg(completed_at), updated_at = NOW()
+       started_at = sqlc.arg(started_at),
+       completed_at = sqlc.arg(completed_at),
+       lease_token = CASE WHEN sqlc.arg(status)::text IN ('completed', 'failed') THEN NULL ELSE lease_token END,
+       lease_owner = CASE WHEN sqlc.arg(status)::text IN ('completed', 'failed') THEN NULL ELSE lease_owner END,
+       lease_expires_at = CASE WHEN sqlc.arg(status)::text IN ('completed', 'failed') THEN NULL ELSE lease_expires_at END,
+       updated_at = NOW()
 WHERE id = sqlc.arg(id) AND tenant_id = sqlc.arg(tenant_id);