harden article generation reliability

server/internal/tenant/app/kol_generation_service.go

@@ -327,6 +327,19 @@ func (s *KolGenerationService) Submit(ctx context.Context, actor auth.Actor, sub
 		TenantID:  actor.TenantID,
 		ArticleID: articleID,
 	}); err != nil {
+		logCtx := GenerationTaskLogContext{
+			TaskID:        task.ID,
+			TenantID:      task.TenantID,
+			OperatorID:    task.OperatorID,
+			ArticleID:     task.ArticleID,
+			ReservationID: task.QuotaReservationID,
+			TaskType:      kolGenerationTaskType,
+			Stage:         "queue_enqueue",
+			StatusBefore:  "queued",
+			StatusAfter:   "failed",
+			Result:        GenerationTaskResultFailure,
+		}
+		RecordGenerationTaskEvent(GenerationTaskEventQueueFailure, logCtx)
 		_ = HandleKolGenerationTaskFailure(context.Background(), s.pool, s.cache, task, "queue_enqueue", err)
 		return nil, response.ErrServiceUnavailable(50343, "generation_queue_unavailable", "generation queue is busy, please retry")
 	}
@@ -385,6 +398,18 @@ func HandleKolGenerationTaskFailure(
 
 	errMsg := formatGenerationFailure(stage, taskErr)
 	now := time.Now()
+	logCtx := GenerationTaskLogContext{
+		TaskID:        task.ID,
+		TenantID:      task.TenantID,
+		OperatorID:    task.OperatorID,
+		ArticleID:     task.ArticleID,
+		ReservationID: task.QuotaReservationID,
+		TaskType:      kolGenerationTaskType,
+		Stage:         stage,
+		StatusAfter:   "failed",
+		Result:        GenerationTaskResultFailure,
+	}
+	RecordGenerationTaskEvent(GenerationTaskEventFailure, logCtx)
 
 	auditRepo := repository.NewAuditRepository(pool)
 	articleRepo := repository.NewArticleRepository(pool)
@@ -421,11 +446,13 @@ func HandleKolGenerationTaskFailure(
 
 	if task.QuotaReservationID > 0 {
 		balance, balanceErr := quotaRepo.GetCurrentBalance(cleanupCtx, task.TenantID, "article_generation")
-		if balanceErr == nil && task.OperatorID > 0 {
+		if balanceErr != nil {
+			failureErr = errors.Join(failureErr, fmt.Errorf("load quota balance for refund: %w", balanceErr))
+		} else if task.OperatorID > 0 {
 			reason := "generation_refund"
 			referenceType := "generation_task"
 			taskReferenceID := task.ID
-			_, _ = quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
+			if _, err := quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
 				TenantID:      task.TenantID,
 				OperatorID:    task.OperatorID,
 				QuotaType:     "article_generation",
@@ -434,11 +461,18 @@ func HandleKolGenerationTaskFailure(
 				Reason:        &reason,
 				ReferenceType: &referenceType,
 				ReferenceID:   &taskReferenceID,
-			})
+			}); err != nil {
+				failureErr = errors.Join(failureErr, fmt.Errorf("insert generation refund ledger: %w", err))
+			}
+		}
+		if err := quotaRepo.RefundReservation(cleanupCtx, task.QuotaReservationID, task.TenantID); err != nil {
+			failureErr = errors.Join(failureErr, fmt.Errorf("refund generation reservation: %w", err))
 		}
-		_ = quotaRepo.RefundReservation(cleanupCtx, task.QuotaReservationID, task.TenantID)
 	}
 
 	InvalidateArticleCaches(cleanupCtx, cache, task.TenantID, &task.ArticleID)
+	if failureErr != nil {
+		RecordGenerationTaskEvent(GenerationTaskEventCleanupFailure, logCtx)
+	}
 	return failureErr
 }