harden article generation reliability

server/internal/tenant/app/template_service.go

@@ -409,6 +409,7 @@ func (s *TemplateService) Generate(ctx context.Context, templateID int64, req Ge
 		TenantID:  actor.TenantID,
 		ArticleID: articleID,
 	}); err != nil {
+		RecordGenerationTaskEvent(GenerationTaskEventQueueFailure, generationJobLogContext(job, "queue_enqueue", "queued", "failed", GenerationTaskResultFailure))
 		s.failGeneration(context.Background(), job, initialTitle, "queue_enqueue", err)
 		return nil, response.ErrServiceUnavailable(50310, "generation_queue_unavailable", "generation queue is busy, please retry")
 	}
@@ -457,12 +458,17 @@ func (s *TemplateService) executeGeneration(ctx context.Context, job generationJ
 	}
 
 	auditRepo := repository.NewAuditRepository(s.pool)
-	_ = auditRepo.UpdateGenerationTaskStatus(ctx, repository.GenerationTaskStatusInput{
+	if err := auditRepo.UpdateGenerationTaskStatus(ctx, repository.GenerationTaskStatusInput{
 		ID:        job.TaskID,
 		TenantID:  job.TenantID,
 		Status:    "running",
 		StartedAt: &now,
-	})
+	}); err != nil {
+		s.failGeneration(ctx, job, title, "task_status_running", err)
+		return
+	}
+	RecordGenerationTaskEvent(GenerationTaskEventTransition, generationJobLogContext(job, "task_status_running", "", GenerationTaskStatusRunning, GenerationTaskResultSuccess))
+	LogGenerationTaskInfo(s.logger, "article generation task marked running", generationJobLogContext(job, "task_status_running", "", GenerationTaskStatusRunning, GenerationTaskResultSuccess))
 
 	knowledgePrompt := ""
 	if knowledgeGroupIDs := extractKnowledgeGroupIDs(job.Params["knowledge_group_ids"]); len(knowledgeGroupIDs) > 0 {
@@ -586,11 +592,15 @@ func (s *TemplateService) failGeneration(ctx context.Context, job generationJob,
 
 	errMsg := formatGenerationFailure(stage, genErr)
 	now := time.Now()
+	logCtx := generationJobLogContext(job, stage, "", "failed", GenerationTaskResultFailure)
+	RecordGenerationTaskEvent(GenerationTaskEventFailure, logCtx)
+	LogGenerationTaskWarn(s.logger, "template generation failed", logCtx, zap.Error(genErr))
 
 	auditRepo := repository.NewAuditRepository(s.pool)
 	articleRepo := repository.NewArticleRepository(s.pool)
 	quotaRepo := repository.NewQuotaRepository(s.pool)
 
+	var cleanupErr error
 	if err := auditRepo.UpdateGenerationTaskStatus(cleanupCtx, repository.GenerationTaskStatusInput{
 		ID:           job.TaskID,
 		TenantID:     job.TenantID,
@@ -604,6 +614,7 @@ func (s *TemplateService) failGeneration(ctx context.Context, job generationJob,
 			zap.Int64("article_id", job.ArticleID),
 			zap.String("stage", stage),
 		)
+		cleanupErr = errors.Join(cleanupErr, fmt.Errorf("update generation task failed status: %w", err))
 	}
 	if err := articleRepo.UpdateArticleGenerateStatus(cleanupCtx, job.ArticleID, job.TenantID, "failed"); err != nil && s.logger != nil {
 		s.logger.Warn("template generation article failed status update failed",
@@ -612,14 +623,17 @@ func (s *TemplateService) failGeneration(ctx context.Context, job generationJob,
 			zap.Int64("article_id", job.ArticleID),
 			zap.String("stage", stage),
 		)
+		cleanupErr = errors.Join(cleanupErr, fmt.Errorf("update article failed status: %w", err))
 	}
 
 	balance, balanceErr := quotaRepo.GetCurrentBalance(cleanupCtx, job.TenantID, "article_generation")
-	if balanceErr == nil {
+	if balanceErr != nil {
+		cleanupErr = errors.Join(cleanupErr, fmt.Errorf("load quota balance for refund: %w", balanceErr))
+	} else {
 		reason := "generation_refund"
 		referenceType := "generation_task"
 		taskReferenceID := job.TaskID
-		_, _ = quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
+		if _, err := quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
 			TenantID:      job.TenantID,
 			OperatorID:    job.OperatorID,
 			QuotaType:     "article_generation",
@@ -628,11 +642,19 @@ func (s *TemplateService) failGeneration(ctx context.Context, job generationJob,
 			Reason:        &reason,
 			ReferenceType: &referenceType,
 			ReferenceID:   &taskReferenceID,
-		})
+		}); err != nil {
+			cleanupErr = errors.Join(cleanupErr, fmt.Errorf("insert generation refund ledger: %w", err))
+		}
 	}
 
-	_ = quotaRepo.RefundReservation(cleanupCtx, job.ReservationID, job.TenantID)
+	if err := quotaRepo.RefundReservation(cleanupCtx, job.ReservationID, job.TenantID); err != nil {
+		cleanupErr = errors.Join(cleanupErr, fmt.Errorf("refund generation reservation: %w", err))
+	}
 	invalidateArticleCaches(cleanupCtx, s.cache, job.TenantID, &job.ArticleID)
+	if cleanupErr != nil {
+		RecordGenerationTaskEvent(GenerationTaskEventCleanupFailure, logCtx)
+		LogGenerationTaskError(s.logger, "template generation failure cleanup failed", logCtx, zap.Error(cleanupErr))
+	}
 
 	runtimeCfg := s.runtimeConfig()
 	if runtimeCfg.StreamEnabled && s.streams != nil {
@@ -640,6 +662,21 @@ func (s *TemplateService) failGeneration(ctx context.Context, job generationJob,
 	}
 }
 
+func generationJobLogContext(job generationJob, stage, statusBefore, statusAfter, result string) GenerationTaskLogContext {
+	return GenerationTaskLogContext{
+		TaskID:        job.TaskID,
+		TenantID:      job.TenantID,
+		OperatorID:    job.OperatorID,
+		ArticleID:     job.ArticleID,
+		ReservationID: job.ReservationID,
+		TaskType:      "template",
+		Stage:         stage,
+		StatusBefore:  statusBefore,
+		StatusAfter:   statusAfter,
+		Result:        result,
+	}
+}
+
 func formatGenerationFailure(stage string, genErr error) string {
 	cause := ""
 	if genErr != nil {
@@ -657,6 +694,8 @@ func generationFailureStageLabel(stage string) string {
 	switch stage {
 	case "task_load":
 		return "加载生成任务失败"
+	case "task_status_running":
+		return "更新生成任务运行状态失败"
 	case "queue_enqueue":
 		return "任务入队失败"
 	case "llm_generate":