harden article generation reliability

server/internal/tenant/app/article_imitation_service.go

@@ -3,6 +3,7 @@ package app
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/url"
 	"strings"
@@ -246,6 +247,7 @@ func (s *ArticleImitationService) Generate(ctx context.Context, req GenerateImit
 		TenantID:  actor.TenantID,
 		ArticleID: articleID,
 	}); err != nil {
+		RecordGenerationTaskEvent(GenerationTaskEventQueueFailure, articleImitationJobLogContext(job, "queue_enqueue", "queued", "failed", GenerationTaskResultFailure))
 		s.failGeneration(context.Background(), job, "queue_enqueue", err)
 		return nil, response.ErrServiceUnavailable(50310, "generation_queue_unavailable", "generation queue is busy, please retry")
 	}
@@ -302,12 +304,17 @@ func (s *ArticleImitationService) executeGeneration(ctx context.Context, job art
 	}
 
 	auditRepo := repository.NewAuditRepository(s.pool)
-	_ = auditRepo.UpdateGenerationTaskStatus(ctx, repository.GenerationTaskStatusInput{
+	if err := auditRepo.UpdateGenerationTaskStatus(ctx, repository.GenerationTaskStatusInput{
 		ID:        job.TaskID,
 		TenantID:  job.TenantID,
 		Status:    "running",
 		StartedAt: &now,
-	})
+	}); err != nil {
+		s.failGeneration(ctx, job, "task_status_running", err)
+		return
+	}
+	RecordGenerationTaskEvent(GenerationTaskEventTransition, articleImitationJobLogContext(job, "task_status_running", "", GenerationTaskStatusRunning, GenerationTaskResultSuccess))
+	LogGenerationTaskInfo(s.logger, "article generation task marked running", articleImitationJobLogContext(job, "task_status_running", "", GenerationTaskStatusRunning, GenerationTaskResultSuccess))
 
 	sourceURL := strings.TrimSpace(extractString(job.InputParams, "source_url"))
 	if sourceURL == "" {
@@ -458,6 +465,9 @@ func (s *ArticleImitationService) failGeneration(ctx context.Context, job articl
 
 	errMsg := formatGenerationFailure(stage, genErr)
 	now := time.Now()
+	logCtx := articleImitationJobLogContext(job, stage, "", "failed", GenerationTaskResultFailure)
+	RecordGenerationTaskEvent(GenerationTaskEventFailure, logCtx)
+	LogGenerationTaskWarn(s.logger, "imitation generation failed", logCtx, zap.Error(genErr))
 	title := strings.TrimSpace(job.InitialTitle)
 	if title == "" {
 		title = articleImitationFallbackTitle
@@ -467,6 +477,7 @@ func (s *ArticleImitationService) failGeneration(ctx context.Context, job articl
 	articleRepo := repository.NewArticleRepository(s.pool)
 	quotaRepo := repository.NewQuotaRepository(s.pool)
 
+	var cleanupErr error
 	if err := auditRepo.UpdateGenerationTaskStatus(cleanupCtx, repository.GenerationTaskStatusInput{
 		ID:           job.TaskID,
 		TenantID:     job.TenantID,
@@ -480,6 +491,7 @@ func (s *ArticleImitationService) failGeneration(ctx context.Context, job articl
 			zap.Int64("article_id", job.ArticleID),
 			zap.String("stage", stage),
 		)
+		cleanupErr = errors.Join(cleanupErr, fmt.Errorf("update generation task failed status: %w", err))
 	}
 	if job.ArticleID > 0 {
 		if err := articleRepo.UpdateArticleGenerateStatus(cleanupCtx, job.ArticleID, job.TenantID, "failed"); err != nil && s.logger != nil {
@@ -489,6 +501,7 @@ func (s *ArticleImitationService) failGeneration(ctx context.Context, job articl
 				zap.Int64("article_id", job.ArticleID),
 				zap.String("stage", stage),
 			)
+			cleanupErr = errors.Join(cleanupErr, fmt.Errorf("update article failed status: %w", err))
 		}
 		_, _ = s.pool.Exec(cleanupCtx, `
 			UPDATE articles
@@ -500,11 +513,13 @@ func (s *ArticleImitationService) failGeneration(ctx context.Context, job articl
 
 	if job.ReservationID > 0 {
 		balance, balanceErr := quotaRepo.GetCurrentBalance(cleanupCtx, job.TenantID, "article_generation")
-		if balanceErr == nil {
+		if balanceErr != nil {
+			cleanupErr = errors.Join(cleanupErr, fmt.Errorf("load quota balance for refund: %w", balanceErr))
+		} else {
 			reason := "generation_refund"
 			referenceType := "generation_task"
 			taskReferenceID := job.TaskID
-			_, _ = quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
+			if _, err := quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
 				TenantID:      job.TenantID,
 				OperatorID:    job.OperatorID,
 				QuotaType:     "article_generation",
@@ -513,14 +528,22 @@ func (s *ArticleImitationService) failGeneration(ctx context.Context, job articl
 				Reason:        &reason,
 				ReferenceType: &referenceType,
 				ReferenceID:   &taskReferenceID,
-			})
+			}); err != nil {
+				cleanupErr = errors.Join(cleanupErr, fmt.Errorf("insert generation refund ledger: %w", err))
+			}
+		}
+		if err := quotaRepo.RefundReservation(cleanupCtx, job.ReservationID, job.TenantID); err != nil {
+			cleanupErr = errors.Join(cleanupErr, fmt.Errorf("refund generation reservation: %w", err))
 		}
-		_ = quotaRepo.RefundReservation(cleanupCtx, job.ReservationID, job.TenantID)
 	}
 
 	if job.ArticleID > 0 {
 		invalidateArticleCaches(cleanupCtx, s.cache, job.TenantID, &job.ArticleID)
 	}
+	if cleanupErr != nil {
+		RecordGenerationTaskEvent(GenerationTaskEventCleanupFailure, logCtx)
+		LogGenerationTaskError(s.logger, "imitation generation failure cleanup failed", logCtx, zap.Error(cleanupErr))
+	}
 
 	runtimeCfg := s.runtimeConfig()
 	if runtimeCfg.StreamEnabled && s.streams != nil && job.ArticleID > 0 {
@@ -528,6 +551,21 @@ func (s *ArticleImitationService) failGeneration(ctx context.Context, job articl
 	}
 }
 
+func articleImitationJobLogContext(job articleImitationJob, stage, statusBefore, statusAfter, result string) GenerationTaskLogContext {
+	return GenerationTaskLogContext{
+		TaskID:        job.TaskID,
+		TenantID:      job.TenantID,
+		OperatorID:    job.OperatorID,
+		ArticleID:     job.ArticleID,
+		ReservationID: job.ReservationID,
+		TaskType:      articleImitationTaskType,
+		Stage:         stage,
+		StatusBefore:  statusBefore,
+		StatusAfter:   statusAfter,
+		Result:        result,
+	}
+}
+
 func normalizeImitationSourceURL(raw string) (string, error) {
 	trimmed := strings.TrimSpace(raw)
 	if trimmed == "" {