harden article generation reliability

server/internal/tenant/app/prompt_generate_service.go

@@ -3,6 +3,7 @@ package app
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strconv"
 	"strings"
@@ -445,6 +446,7 @@ func (s *PromptRuleGenerationService) enqueuePromptRuleGeneration(ctx context.Co
 		TenantID:  input.TenantID,
 		ArticleID: articleID,
 	}); err != nil {
+		RecordGenerationTaskEvent(GenerationTaskEventQueueFailure, promptRuleGenerationJobLogContext(job, "queue_enqueue", "queued", "failed", GenerationTaskResultFailure))
 		s.failGeneration(context.Background(), job, promptRuleGeneratingTitlePlaceholder, "queue_enqueue", err)
 		return nil, response.ErrServiceUnavailable(50310, "generation_queue_unavailable", "generation queue is busy, please retry")
 	}
@@ -508,12 +510,17 @@ func (s *PromptRuleGenerationService) executeGeneration(ctx context.Context, job
 	}
 
 	auditRepo := repository.NewAuditRepository(s.pool)
-	_ = auditRepo.UpdateGenerationTaskStatus(ctx, repository.GenerationTaskStatusInput{
+	if err := auditRepo.UpdateGenerationTaskStatus(ctx, repository.GenerationTaskStatusInput{
 		ID:        job.TaskID,
 		TenantID:  job.TenantID,
 		Status:    "running",
 		StartedAt: &now,
-	})
+	}); err != nil {
+		s.failGeneration(ctx, job, title, "task_status_running", err)
+		return
+	}
+	RecordGenerationTaskEvent(GenerationTaskEventTransition, promptRuleGenerationJobLogContext(job, "task_status_running", "", GenerationTaskStatusRunning, GenerationTaskResultSuccess))
+	LogGenerationTaskInfo(s.logger, "article generation task marked running", promptRuleGenerationJobLogContext(job, "task_status_running", "", GenerationTaskStatusRunning, GenerationTaskResultSuccess))
 
 	runtimeCfg := s.runtimeConfig()
 	req := llm.GenerateRequest{
@@ -686,6 +693,9 @@ func (s *PromptRuleGenerationService) failGeneration(ctx context.Context, job pr
 
 	errMsg := formatGenerationFailure(stage, genErr)
 	now := time.Now()
+	logCtx := promptRuleGenerationJobLogContext(job, stage, "", "failed", GenerationTaskResultFailure)
+	RecordGenerationTaskEvent(GenerationTaskEventFailure, logCtx)
+	LogGenerationTaskWarn(s.logger, "prompt rule generation failed", logCtx, zap.Error(genErr))
 	failureTitle := strings.TrimSpace(extractString(job.InputParams, "task_name"))
 	if failureTitle == "" {
 		failureTitle = title
@@ -695,6 +705,7 @@ func (s *PromptRuleGenerationService) failGeneration(ctx context.Context, job pr
 	articleRepo := repository.NewArticleRepository(s.pool)
 	quotaRepo := repository.NewQuotaRepository(s.pool)
 
+	var cleanupErr error
 	if err := auditRepo.UpdateGenerationTaskStatus(cleanupCtx, repository.GenerationTaskStatusInput{
 		ID:           job.TaskID,
 		TenantID:     job.TenantID,
@@ -708,6 +719,7 @@ func (s *PromptRuleGenerationService) failGeneration(ctx context.Context, job pr
 			zap.Int64("article_id", job.ArticleID),
 			zap.String("stage", stage),
 		)
+		cleanupErr = errors.Join(cleanupErr, fmt.Errorf("update generation task failed status: %w", err))
 	}
 	if err := articleRepo.UpdateArticleGenerateStatus(cleanupCtx, job.ArticleID, job.TenantID, "failed"); err != nil && s.logger != nil {
 		s.logger.Warn("prompt rule generation article failed status update failed",
@@ -716,6 +728,7 @@ func (s *PromptRuleGenerationService) failGeneration(ctx context.Context, job pr
 			zap.Int64("article_id", job.ArticleID),
 			zap.String("stage", stage),
 		)
+		cleanupErr = errors.Join(cleanupErr, fmt.Errorf("update article failed status: %w", err))
 	}
 	if failureTitle != "" {
 		_, _ = s.pool.Exec(cleanupCtx, `
@@ -727,11 +740,13 @@ func (s *PromptRuleGenerationService) failGeneration(ctx context.Context, job pr
 	}
 
 	balance, balanceErr := quotaRepo.GetCurrentBalance(cleanupCtx, job.TenantID, "article_generation")
-	if balanceErr == nil {
+	if balanceErr != nil {
+		cleanupErr = errors.Join(cleanupErr, fmt.Errorf("load quota balance for refund: %w", balanceErr))
+	} else {
 		reason := "generation_refund"
 		referenceType := "generation_task"
 		taskReferenceID := job.TaskID
-		_, _ = quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
+		if _, err := quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
 			TenantID:      job.TenantID,
 			OperatorID:    job.OperatorID,
 			QuotaType:     "article_generation",
@@ -740,11 +755,19 @@ func (s *PromptRuleGenerationService) failGeneration(ctx context.Context, job pr
 			Reason:        &reason,
 			ReferenceType: &referenceType,
 			ReferenceID:   &taskReferenceID,
-		})
+		}); err != nil {
+			cleanupErr = errors.Join(cleanupErr, fmt.Errorf("insert generation refund ledger: %w", err))
+		}
 	}
 
-	_ = quotaRepo.RefundReservation(cleanupCtx, job.ReservationID, job.TenantID)
+	if err := quotaRepo.RefundReservation(cleanupCtx, job.ReservationID, job.TenantID); err != nil {
+		cleanupErr = errors.Join(cleanupErr, fmt.Errorf("refund generation reservation: %w", err))
+	}
 	invalidateArticleCaches(cleanupCtx, s.cache, job.TenantID, &job.ArticleID)
+	if cleanupErr != nil {
+		RecordGenerationTaskEvent(GenerationTaskEventCleanupFailure, logCtx)
+		LogGenerationTaskError(s.logger, "prompt rule generation failure cleanup failed", logCtx, zap.Error(cleanupErr))
+	}
 
 	runtimeCfg := s.runtimeConfig()
 	if runtimeCfg.StreamEnabled && s.streams != nil {
@@ -752,6 +775,21 @@ func (s *PromptRuleGenerationService) failGeneration(ctx context.Context, job pr
 	}
 }
 
+func promptRuleGenerationJobLogContext(job promptRuleGenerationJob, stage, statusBefore, statusAfter, result string) GenerationTaskLogContext {
+	return GenerationTaskLogContext{
+		TaskID:        job.TaskID,
+		TenantID:      job.TenantID,
+		OperatorID:    job.OperatorID,
+		ArticleID:     job.ArticleID,
+		ReservationID: job.ReservationID,
+		TaskType:      "custom_generation",
+		Stage:         stage,
+		StatusBefore:  statusBefore,
+		StatusAfter:   statusAfter,
+		Result:        result,
+	}
+}
+
 func buildPromptRuleGenerationPrompt(rule *promptRuleGenerationRecord, params map[string]interface{}, knowledgePrompt string) string {
 	sections := []string{strings.TrimSpace(rule.PromptContent)}