harden article generation reliability

server/internal/worker/generate/article_generation_recovery.go

@@ -32,6 +32,13 @@ type generationTaskRecoveryConfig struct {
 	MaxAttempts      int
 }
 
+type generationTaskStateCheckConfig struct {
+	Interval  time.Duration
+	Timeout   time.Duration
+	BatchSize int
+	Lookback  time.Duration
+}
+
 type claimedGenerationTaskLease struct {
 	task  generationTaskRecoveryRecord
 	token string
@@ -139,6 +146,17 @@ func normalizeGenerationTaskRecoveryConfig(cfg config.GenerationConfig) generati
 	}
 }
 
+func normalizeGenerationTaskStateCheckConfig(cfg config.GenerationConfig) generationTaskStateCheckConfig {
+	normalized := cfg
+	config.NormalizeGenerationConfig(&normalized)
+	return generationTaskStateCheckConfig{
+		Interval:  normalized.TaskStateCheckInterval,
+		Timeout:   normalized.TaskStateCheckTimeout,
+		BatchSize: normalized.TaskStateCheckBatchSize,
+		Lookback:  normalized.TaskStateCheckLookback,
+	}
+}
+
 func (w *ArticleGenerationWorker) claimGenerationTask(ctx context.Context, taskID int64, owner string, leaseTTL time.Duration) (*claimedGenerationTaskLease, bool, error) {
 	if leaseTTL <= 0 {
 		leaseTTL = normalizeGenerationTaskRecoveryConfig(config.GenerationConfig{}).LeaseTTL
@@ -362,6 +380,20 @@ func (w *ArticleGenerationWorker) requeueStaleGenerationTasks(ctx context.Contex
 			}
 			continue
 		}
+		logCtx := tenantapp.GenerationTaskLogContext{
+			TaskID:       item.ID,
+			TenantID:     item.TenantID,
+			ArticleID:    item.ArticleID,
+			TaskType:     item.TaskType,
+			Stage:        "recovery_stale_queued",
+			StatusBefore: "queued",
+			StatusAfter:  "queued",
+			Result:       tenantapp.GenerationTaskResultSuccess,
+			Severity:     "warning",
+			AttemptCount: item.AttemptCount,
+		}
+		tenantapp.RecordGenerationTaskEvent(tenantapp.GenerationTaskEventRecoveryRequeue, logCtx)
+		tenantapp.LogGenerationTaskWarn(w.logger, "article generation stale queued task republished", logCtx)
 		requeued++
 	}
 
@@ -538,17 +570,39 @@ func (w *ArticleGenerationWorker) requeueExpiredRunningGenerationTask(ctx contex
 		return false, err
 	}
 	if !shouldRequeue {
+		logCtx := tenantapp.GenerationTaskLogContext{
+			TaskID:       item.ID,
+			TenantID:     tenantID,
+			ArticleID:    articleID,
+			TaskType:     taskType,
+			Stage:        "recovery_expired_running",
+			StatusBefore: "running",
+			StatusAfter:  "failed",
+			Result:       tenantapp.GenerationTaskResultSkipped,
+			Severity:     "warning",
+		}
+		tenantapp.RecordGenerationTaskEvent(tenantapp.GenerationTaskEventRecoveryFinalize, logCtx)
 		if w.logger != nil {
-			w.logger.Info("article generation expired task finalized because article is already failed",
-				zap.Int64("task_id", item.ID),
-				zap.Int64("article_id", articleID),
-			)
+			tenantapp.LogGenerationTaskInfo(w.logger, "article generation expired task finalized because article is already failed", logCtx)
 		}
 		return true, nil
 	}
 	if err := w.publishGenerationTask(ctx, item.ID, taskType, tenantID, articleID); err != nil {
 		return false, err
 	}
+	logCtx := tenantapp.GenerationTaskLogContext{
+		TaskID:       item.ID,
+		TenantID:     tenantID,
+		ArticleID:    articleID,
+		TaskType:     taskType,
+		Stage:        "recovery_expired_running",
+		StatusBefore: "running",
+		StatusAfter:  "queued",
+		Result:       tenantapp.GenerationTaskResultSuccess,
+		Severity:     "warning",
+	}
+	tenantapp.RecordGenerationTaskEvent(tenantapp.GenerationTaskEventRecoveryRequeue, logCtx)
+	tenantapp.LogGenerationTaskWarn(w.logger, "article generation expired task requeued", logCtx)
 	return true, nil
 }
 
@@ -672,6 +726,21 @@ func (w *ArticleGenerationWorker) finalizeExpiredGenerationTask(ctx context.Cont
 	if item.ArticleID > 0 {
 		tenantapp.InvalidateArticleCaches(cacheCtx, w.cache, item.TenantID, &item.ArticleID)
 	}
+	logCtx := tenantapp.GenerationTaskLogContext{
+		TaskID:        item.ID,
+		TenantID:      item.TenantID,
+		OperatorID:    item.OperatorID,
+		ArticleID:     item.ArticleID,
+		ReservationID: item.QuotaReservationID,
+		TaskType:      item.TaskType,
+		Stage:         "recovery_finalize",
+		StatusAfter:   "failed",
+		Result:        tenantapp.GenerationTaskResultSuccess,
+		Severity:      "critical",
+		AttemptCount:  item.AttemptCount,
+	}
+	tenantapp.RecordGenerationTaskEvent(tenantapp.GenerationTaskEventRecoveryFinalize, logCtx)
+	tenantapp.LogGenerationTaskWarn(w.logger, "article generation stale task finalized", logCtx)
 	return true, nil
 }