harden article generation reliability

server/internal/tenant/app/article_imitation_service.go

@@ -3,6 +3,7 @@ package app
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/url"
 	"strings"
@@ -246,6 +247,7 @@ func (s *ArticleImitationService) Generate(ctx context.Context, req GenerateImit
 		TenantID:  actor.TenantID,
 		ArticleID: articleID,
 	}); err != nil {
+		RecordGenerationTaskEvent(GenerationTaskEventQueueFailure, articleImitationJobLogContext(job, "queue_enqueue", "queued", "failed", GenerationTaskResultFailure))
 		s.failGeneration(context.Background(), job, "queue_enqueue", err)
 		return nil, response.ErrServiceUnavailable(50310, "generation_queue_unavailable", "generation queue is busy, please retry")
 	}
@@ -302,12 +304,17 @@ func (s *ArticleImitationService) executeGeneration(ctx context.Context, job art
 	}
 
 	auditRepo := repository.NewAuditRepository(s.pool)
-	_ = auditRepo.UpdateGenerationTaskStatus(ctx, repository.GenerationTaskStatusInput{
+	if err := auditRepo.UpdateGenerationTaskStatus(ctx, repository.GenerationTaskStatusInput{
 		ID:        job.TaskID,
 		TenantID:  job.TenantID,
 		Status:    "running",
 		StartedAt: &now,
-	})
+	}); err != nil {
+		s.failGeneration(ctx, job, "task_status_running", err)
+		return
+	}
+	RecordGenerationTaskEvent(GenerationTaskEventTransition, articleImitationJobLogContext(job, "task_status_running", "", GenerationTaskStatusRunning, GenerationTaskResultSuccess))
+	LogGenerationTaskInfo(s.logger, "article generation task marked running", articleImitationJobLogContext(job, "task_status_running", "", GenerationTaskStatusRunning, GenerationTaskResultSuccess))
 
 	sourceURL := strings.TrimSpace(extractString(job.InputParams, "source_url"))
 	if sourceURL == "" {
@@ -458,6 +465,9 @@ func (s *ArticleImitationService) failGeneration(ctx context.Context, job articl
 
 	errMsg := formatGenerationFailure(stage, genErr)
 	now := time.Now()
+	logCtx := articleImitationJobLogContext(job, stage, "", "failed", GenerationTaskResultFailure)
+	RecordGenerationTaskEvent(GenerationTaskEventFailure, logCtx)
+	LogGenerationTaskWarn(s.logger, "imitation generation failed", logCtx, zap.Error(genErr))
 	title := strings.TrimSpace(job.InitialTitle)
 	if title == "" {
 		title = articleImitationFallbackTitle
@@ -467,6 +477,7 @@ func (s *ArticleImitationService) failGeneration(ctx context.Context, job articl
 	articleRepo := repository.NewArticleRepository(s.pool)
 	quotaRepo := repository.NewQuotaRepository(s.pool)
 
+	var cleanupErr error
 	if err := auditRepo.UpdateGenerationTaskStatus(cleanupCtx, repository.GenerationTaskStatusInput{
 		ID:           job.TaskID,
 		TenantID:     job.TenantID,
@@ -480,6 +491,7 @@ func (s *ArticleImitationService) failGeneration(ctx context.Context, job articl
 			zap.Int64("article_id", job.ArticleID),
 			zap.String("stage", stage),
 		)
+		cleanupErr = errors.Join(cleanupErr, fmt.Errorf("update generation task failed status: %w", err))
 	}
 	if job.ArticleID > 0 {
 		if err := articleRepo.UpdateArticleGenerateStatus(cleanupCtx, job.ArticleID, job.TenantID, "failed"); err != nil && s.logger != nil {
@@ -489,6 +501,7 @@ func (s *ArticleImitationService) failGeneration(ctx context.Context, job articl
 				zap.Int64("article_id", job.ArticleID),
 				zap.String("stage", stage),
 			)
+			cleanupErr = errors.Join(cleanupErr, fmt.Errorf("update article failed status: %w", err))
 		}
 		_, _ = s.pool.Exec(cleanupCtx, `
 			UPDATE articles
@@ -500,11 +513,13 @@ func (s *ArticleImitationService) failGeneration(ctx context.Context, job articl
 
 	if job.ReservationID > 0 {
 		balance, balanceErr := quotaRepo.GetCurrentBalance(cleanupCtx, job.TenantID, "article_generation")
-		if balanceErr == nil {
+		if balanceErr != nil {
+			cleanupErr = errors.Join(cleanupErr, fmt.Errorf("load quota balance for refund: %w", balanceErr))
+		} else {
 			reason := "generation_refund"
 			referenceType := "generation_task"
 			taskReferenceID := job.TaskID
-			_, _ = quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
+			if _, err := quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
 				TenantID:      job.TenantID,
 				OperatorID:    job.OperatorID,
 				QuotaType:     "article_generation",
@@ -513,14 +528,22 @@ func (s *ArticleImitationService) failGeneration(ctx context.Context, job articl
 				Reason:        &reason,
 				ReferenceType: &referenceType,
 				ReferenceID:   &taskReferenceID,
-			})
+			}); err != nil {
+				cleanupErr = errors.Join(cleanupErr, fmt.Errorf("insert generation refund ledger: %w", err))
+			}
+		}
+		if err := quotaRepo.RefundReservation(cleanupCtx, job.ReservationID, job.TenantID); err != nil {
+			cleanupErr = errors.Join(cleanupErr, fmt.Errorf("refund generation reservation: %w", err))
 		}
-		_ = quotaRepo.RefundReservation(cleanupCtx, job.ReservationID, job.TenantID)
 	}
 
 	if job.ArticleID > 0 {
 		invalidateArticleCaches(cleanupCtx, s.cache, job.TenantID, &job.ArticleID)
 	}
+	if cleanupErr != nil {
+		RecordGenerationTaskEvent(GenerationTaskEventCleanupFailure, logCtx)
+		LogGenerationTaskError(s.logger, "imitation generation failure cleanup failed", logCtx, zap.Error(cleanupErr))
+	}
 
 	runtimeCfg := s.runtimeConfig()
 	if runtimeCfg.StreamEnabled && s.streams != nil && job.ArticleID > 0 {
@@ -528,6 +551,21 @@ func (s *ArticleImitationService) failGeneration(ctx context.Context, job articl
 	}
 }
 
+func articleImitationJobLogContext(job articleImitationJob, stage, statusBefore, statusAfter, result string) GenerationTaskLogContext {
+	return GenerationTaskLogContext{
+		TaskID:        job.TaskID,
+		TenantID:      job.TenantID,
+		OperatorID:    job.OperatorID,
+		ArticleID:     job.ArticleID,
+		ReservationID: job.ReservationID,
+		TaskType:      articleImitationTaskType,
+		Stage:         stage,
+		StatusBefore:  statusBefore,
+		StatusAfter:   statusAfter,
+		Result:        result,
+	}
+}
+
 func normalizeImitationSourceURL(raw string) (string, error) {
 	trimmed := strings.TrimSpace(raw)
 	if trimmed == "" {

server/internal/tenant/app/generation_observability.go

@@ -0,0 +1,240 @@
+package app
+
+import (
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"go.uber.org/zap"
+)
+
+const GenerationTaskStatusRunning = "running"
+
+const (
+	GenerationTaskEventTransition        = "transition"
+	GenerationTaskEventFailure           = "failure"
+	GenerationTaskEventCleanupFailure    = "cleanup_failure"
+	GenerationTaskEventQueueFailure      = "queue_failure"
+	GenerationTaskEventRecoveryRequeue   = "recovery_requeue"
+	GenerationTaskEventRecoveryFinalize  = "recovery_finalize"
+	GenerationTaskEventRecoverySkipped   = "recovery_skipped"
+	GenerationTaskEventStateAnomaly      = "state_anomaly"
+	GenerationTaskEventStateCheckFailure = "state_check_failure"
+)
+
+const (
+	GenerationTaskResultSuccess = "success"
+	GenerationTaskResultFailure = "failure"
+	GenerationTaskResultSkipped = "skipped"
+)
+
+type GenerationTaskLogContext struct {
+	TaskID        int64
+	TenantID      int64
+	OperatorID    int64
+	ArticleID     int64
+	ReservationID int64
+	TaskType      string
+	Stage         string
+	AttemptCount  int
+	LeaseOwner    string
+	StatusBefore  string
+	StatusAfter   string
+	Result        string
+	AnomalyType   string
+	Severity      string
+	Duration      time.Duration
+}
+
+func GenerationTaskLogFields(ctx GenerationTaskLogContext) []zap.Field {
+	fields := []zap.Field{zap.String("event_domain", "article_generation")}
+	appendStringField := func(key, value string) {
+		if value = strings.TrimSpace(value); value != "" {
+			fields = append(fields, zap.String(key, value))
+		}
+	}
+	appendInt64Field := func(key string, value int64) {
+		if value > 0 {
+			fields = append(fields, zap.Int64(key, value))
+		}
+	}
+
+	appendInt64Field("task_id", ctx.TaskID)
+	appendInt64Field("tenant_id", ctx.TenantID)
+	appendInt64Field("operator_id", ctx.OperatorID)
+	appendInt64Field("article_id", ctx.ArticleID)
+	appendInt64Field("quota_reservation_id", ctx.ReservationID)
+	appendStringField("task_type", ctx.TaskType)
+	appendStringField("stage", ctx.Stage)
+	if ctx.AttemptCount > 0 {
+		fields = append(fields, zap.Int("attempt_count", ctx.AttemptCount))
+	}
+	appendStringField("lease_owner", ctx.LeaseOwner)
+	appendStringField("status_before", ctx.StatusBefore)
+	appendStringField("status_after", ctx.StatusAfter)
+	appendStringField("result", ctx.Result)
+	appendStringField("anomaly_type", ctx.AnomalyType)
+	appendStringField("severity", ctx.Severity)
+	if ctx.Duration > 0 {
+		fields = append(fields, zap.Duration("duration", ctx.Duration))
+		fields = append(fields, zap.Int64("duration_ms", ctx.Duration.Milliseconds()))
+	}
+	return fields
+}
+
+func LogGenerationTaskInfo(logger *zap.Logger, message string, ctx GenerationTaskLogContext, fields ...zap.Field) {
+	if logger == nil {
+		return
+	}
+	logger.Info(message, append(GenerationTaskLogFields(ctx), fields...)...)
+}
+
+func LogGenerationTaskWarn(logger *zap.Logger, message string, ctx GenerationTaskLogContext, fields ...zap.Field) {
+	if logger == nil {
+		return
+	}
+	logger.Warn(message, append(GenerationTaskLogFields(ctx), fields...)...)
+}
+
+func LogGenerationTaskError(logger *zap.Logger, message string, ctx GenerationTaskLogContext, fields ...zap.Field) {
+	if logger == nil {
+		return
+	}
+	logger.Error(message, append(GenerationTaskLogFields(ctx), fields...)...)
+}
+
+type GenerationTaskMetricsSnapshot struct {
+	TransitionTotal        int64      `json:"transition_total"`
+	FailureTotal           int64      `json:"failure_total"`
+	CleanupFailureTotal    int64      `json:"cleanup_failure_total"`
+	QueueFailureTotal      int64      `json:"queue_failure_total"`
+	RecoveryRequeueTotal   int64      `json:"recovery_requeue_total"`
+	RecoveryFinalizeTotal  int64      `json:"recovery_finalize_total"`
+	StateAnomalyTotal      int64      `json:"state_anomaly_total"`
+	StateCheckFailureTotal int64      `json:"state_check_failure_total"`
+	LastEventAt            *time.Time `json:"last_event_at,omitempty"`
+	LastEvent              string     `json:"last_event,omitempty"`
+	LastTaskType           string     `json:"last_task_type,omitempty"`
+	LastStage              string     `json:"last_stage,omitempty"`
+	LastResult             string     `json:"last_result,omitempty"`
+	LastAnomalyType        string     `json:"last_anomaly_type,omitempty"`
+	LastSeverity           string     `json:"last_severity,omitempty"`
+}
+
+type generationTaskMetricsRecorder struct {
+	transitionTotal        atomic.Int64
+	failureTotal           atomic.Int64
+	cleanupFailureTotal    atomic.Int64
+	queueFailureTotal      atomic.Int64
+	recoveryRequeueTotal   atomic.Int64
+	recoveryFinalizeTotal  atomic.Int64
+	stateAnomalyTotal      atomic.Int64
+	stateCheckFailureTotal atomic.Int64
+	lastEventAtUnixNano    atomic.Int64
+	lastEvent              atomic.Value
+	lastTaskType           atomic.Value
+	lastStage              atomic.Value
+	lastResult             atomic.Value
+	lastAnomalyType        atomic.Value
+	lastSeverity           atomic.Value
+}
+
+var generationTaskMetrics = newGenerationTaskMetricsRecorder()
+
+func newGenerationTaskMetricsRecorder() *generationTaskMetricsRecorder {
+	return &generationTaskMetricsRecorder{}
+}
+
+func RecordGenerationTaskEvent(event string, ctx GenerationTaskLogContext) {
+	generationTaskMetrics.record(event, ctx)
+}
+
+func GenerationTaskMetricsSnapshotValue() GenerationTaskMetricsSnapshot {
+	return generationTaskMetrics.snapshot()
+}
+
+func resetGenerationTaskMetricsForTest() {
+	generationTaskMetrics = newGenerationTaskMetricsRecorder()
+}
+
+func (r *generationTaskMetricsRecorder) record(event string, ctx GenerationTaskLogContext) {
+	if r == nil {
+		return
+	}
+	event = strings.TrimSpace(event)
+	if event == "" {
+		event = "unknown"
+	}
+	switch event {
+	case GenerationTaskEventTransition:
+		r.transitionTotal.Add(1)
+	case GenerationTaskEventFailure:
+		r.failureTotal.Add(1)
+	case GenerationTaskEventCleanupFailure:
+		r.cleanupFailureTotal.Add(1)
+	case GenerationTaskEventQueueFailure:
+		r.queueFailureTotal.Add(1)
+	case GenerationTaskEventRecoveryRequeue:
+		r.recoveryRequeueTotal.Add(1)
+	case GenerationTaskEventRecoveryFinalize:
+		r.recoveryFinalizeTotal.Add(1)
+	case GenerationTaskEventStateAnomaly:
+		r.stateAnomalyTotal.Add(1)
+	case GenerationTaskEventStateCheckFailure:
+		r.stateCheckFailureTotal.Add(1)
+	}
+	r.lastEventAtUnixNano.Store(time.Now().UTC().UnixNano())
+	r.lastEvent.Store(event)
+	storeGenerationMetricString(&r.lastTaskType, ctx.TaskType)
+	storeGenerationMetricString(&r.lastStage, ctx.Stage)
+	storeGenerationMetricString(&r.lastResult, ctx.Result)
+	storeGenerationMetricString(&r.lastAnomalyType, ctx.AnomalyType)
+	storeGenerationMetricString(&r.lastSeverity, ctx.Severity)
+}
+
+func (r *generationTaskMetricsRecorder) snapshot() GenerationTaskMetricsSnapshot {
+	if r == nil {
+		return GenerationTaskMetricsSnapshot{}
+	}
+	return GenerationTaskMetricsSnapshot{
+		TransitionTotal:        r.transitionTotal.Load(),
+		FailureTotal:           r.failureTotal.Load(),
+		CleanupFailureTotal:    r.cleanupFailureTotal.Load(),
+		QueueFailureTotal:      r.queueFailureTotal.Load(),
+		RecoveryRequeueTotal:   r.recoveryRequeueTotal.Load(),
+		RecoveryFinalizeTotal:  r.recoveryFinalizeTotal.Load(),
+		StateAnomalyTotal:      r.stateAnomalyTotal.Load(),
+		StateCheckFailureTotal: r.stateCheckFailureTotal.Load(),
+		LastEventAt:            generationMetricUnixNanoTime(r.lastEventAtUnixNano.Load()),
+		LastEvent:              loadGenerationMetricString(&r.lastEvent),
+		LastTaskType:           loadGenerationMetricString(&r.lastTaskType),
+		LastStage:              loadGenerationMetricString(&r.lastStage),
+		LastResult:             loadGenerationMetricString(&r.lastResult),
+		LastAnomalyType:        loadGenerationMetricString(&r.lastAnomalyType),
+		LastSeverity:           loadGenerationMetricString(&r.lastSeverity),
+	}
+}
+
+func storeGenerationMetricString(target *atomic.Value, value string) {
+	if target == nil {
+		return
+	}
+	target.Store(strings.TrimSpace(value))
+}
+
+func loadGenerationMetricString(value *atomic.Value) string {
+	if value == nil {
+		return ""
+	}
+	raw := value.Load()
+	text, _ := raw.(string)
+	return text
+}
+
+func generationMetricUnixNanoTime(value int64) *time.Time {
+	if value <= 0 {
+		return nil
+	}
+	t := time.Unix(0, value).UTC()
+	return &t
+}

server/internal/tenant/app/generation_observability_test.go

@@ -0,0 +1,101 @@
+package app
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGenerationTaskLogFieldsUsesStableStructuredKeys(t *testing.T) {
+	fields := GenerationTaskLogFields(GenerationTaskLogContext{
+		TaskID:        11,
+		TenantID:      22,
+		OperatorID:    33,
+		ArticleID:     44,
+		ReservationID: 55,
+		TaskType:      "template",
+		Stage:         "llm_generate",
+		AttemptCount:  2,
+		LeaseOwner:    "worker-1",
+		StatusBefore:  "running",
+		StatusAfter:   "failed",
+		Result:        GenerationTaskResultFailure,
+		AnomalyType:   "article_failed_task_running",
+		Severity:      "critical",
+	})
+
+	values := map[string]any{}
+	for _, field := range fields {
+		values[field.Key] = field.Interface
+		if field.String != "" {
+			values[field.Key] = field.String
+		}
+		if field.Integer != 0 {
+			values[field.Key] = field.Integer
+		}
+	}
+
+	assert.Equal(t, "article_generation", values["event_domain"])
+	assert.Equal(t, int64(11), values["task_id"])
+	assert.Equal(t, int64(22), values["tenant_id"])
+	assert.Equal(t, int64(44), values["article_id"])
+	assert.Equal(t, "template", values["task_type"])
+	assert.Equal(t, "llm_generate", values["stage"])
+	assert.Equal(t, "failed", values["status_after"])
+	assert.Equal(t, "critical", values["severity"])
+}
+
+func TestGenerationTaskMetricsRecordsEventsAndPrometheus(t *testing.T) {
+	resetGenerationTaskMetricsForTest()
+	t.Cleanup(resetGenerationTaskMetricsForTest)
+
+	RecordGenerationTaskEvent(GenerationTaskEventTransition, GenerationTaskLogContext{
+		TaskType:    "template",
+		Stage:       "worker_claim",
+		StatusAfter: GenerationTaskStatusRunning,
+		Result:      GenerationTaskResultSuccess,
+	})
+	RecordGenerationTaskEvent(GenerationTaskEventCleanupFailure, GenerationTaskLogContext{
+		TaskType: "template",
+		Stage:    "refund_reservation",
+		Result:   GenerationTaskResultFailure,
+		Severity: "critical",
+	})
+	RecordGenerationTaskEvent(GenerationTaskEventStateAnomaly, GenerationTaskLogContext{
+		TaskType:    "template",
+		Stage:       "state_check",
+		AnomalyType: "completed_task_missing_version",
+		Severity:    "critical",
+	})
+
+	snapshot := GenerationTaskMetricsSnapshotValue()
+	assert.Equal(t, int64(1), snapshot.TransitionTotal)
+	assert.Equal(t, int64(1), snapshot.CleanupFailureTotal)
+	assert.Equal(t, int64(1), snapshot.StateAnomalyTotal)
+	assert.Equal(t, "state_anomaly", snapshot.LastEvent)
+	assert.Equal(t, "completed_task_missing_version", snapshot.LastAnomalyType)
+	assert.NotNil(t, snapshot.LastEventAt)
+
+	prometheus := GenerationTaskMetricsPrometheus()
+	assert.Contains(t, prometheus, "article_generation_task_transitions_total 1")
+	assert.Contains(t, prometheus, "article_generation_task_cleanup_failures_total 1")
+	assert.Contains(t, prometheus, "article_generation_task_state_anomalies_total 1")
+	lastEventLine := findPrometheusMetricLine(prometheus, "article_generation_task_last_event_info")
+	assert.Contains(t, lastEventLine, `event="state_anomaly"`)
+	assert.Contains(t, lastEventLine, `task_type="template"`)
+	assert.Contains(t, lastEventLine, `stage="state_check"`)
+	assert.Contains(t, lastEventLine, `result=""`)
+	assert.Contains(t, lastEventLine, `anomaly_type="completed_task_missing_version"`)
+	assert.Contains(t, lastEventLine, `severity="critical"`)
+	assert.True(t, strings.HasSuffix(lastEventLine, " 1"), lastEventLine)
+}
+
+func findPrometheusMetricLine(output, metricName string) string {
+	for _, line := range strings.Split(output, "\n") {
+		if strings.HasPrefix(line, metricName+"{") || strings.HasPrefix(line, metricName+" ") {
+			return line
+		}
+	}
+	return ""
+}

server/internal/tenant/app/generation_task_runtime.go

@@ -80,11 +80,13 @@ func HandleClaimedGenerationTaskFailure(
 
 	if task.QuotaReservationID > 0 {
 		balance, balanceErr := quotaRepo.GetCurrentBalance(cleanupCtx, task.TenantID, "article_generation")
-		if balanceErr == nil && task.OperatorID > 0 {
+		if balanceErr != nil {
+			failureErr = errors.Join(failureErr, fmt.Errorf("load quota balance for refund: %w", balanceErr))
+		} else if task.OperatorID > 0 {
 			reason := "generation_refund"
 			referenceType := "generation_task"
 			taskReferenceID := task.ID
-			_, _ = quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
+			if _, err := quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
 				TenantID:      task.TenantID,
 				OperatorID:    task.OperatorID,
 				QuotaType:     "article_generation",
@@ -93,10 +95,14 @@ func HandleClaimedGenerationTaskFailure(
 				Reason:        &reason,
 				ReferenceType: &referenceType,
 				ReferenceID:   &taskReferenceID,
-			})
+			}); err != nil {
+				failureErr = errors.Join(failureErr, fmt.Errorf("insert generation refund ledger: %w", err))
+			}
 		}
 
-		_ = quotaRepo.RefundReservation(cleanupCtx, task.QuotaReservationID, task.TenantID)
+		if err := quotaRepo.RefundReservation(cleanupCtx, task.QuotaReservationID, task.TenantID); err != nil {
+			failureErr = errors.Join(failureErr, fmt.Errorf("refund generation reservation: %w", err))
+		}
 	}
 
 	if task.ArticleID > 0 {

server/internal/tenant/app/kol_generation_service.go

@@ -327,6 +327,19 @@ func (s *KolGenerationService) Submit(ctx context.Context, actor auth.Actor, sub
 		TenantID:  actor.TenantID,
 		ArticleID: articleID,
 	}); err != nil {
+		logCtx := GenerationTaskLogContext{
+			TaskID:        task.ID,
+			TenantID:      task.TenantID,
+			OperatorID:    task.OperatorID,
+			ArticleID:     task.ArticleID,
+			ReservationID: task.QuotaReservationID,
+			TaskType:      kolGenerationTaskType,
+			Stage:         "queue_enqueue",
+			StatusBefore:  "queued",
+			StatusAfter:   "failed",
+			Result:        GenerationTaskResultFailure,
+		}
+		RecordGenerationTaskEvent(GenerationTaskEventQueueFailure, logCtx)
 		_ = HandleKolGenerationTaskFailure(context.Background(), s.pool, s.cache, task, "queue_enqueue", err)
 		return nil, response.ErrServiceUnavailable(50343, "generation_queue_unavailable", "generation queue is busy, please retry")
 	}
@@ -385,6 +398,18 @@ func HandleKolGenerationTaskFailure(
 
 	errMsg := formatGenerationFailure(stage, taskErr)
 	now := time.Now()
+	logCtx := GenerationTaskLogContext{
+		TaskID:        task.ID,
+		TenantID:      task.TenantID,
+		OperatorID:    task.OperatorID,
+		ArticleID:     task.ArticleID,
+		ReservationID: task.QuotaReservationID,
+		TaskType:      kolGenerationTaskType,
+		Stage:         stage,
+		StatusAfter:   "failed",
+		Result:        GenerationTaskResultFailure,
+	}
+	RecordGenerationTaskEvent(GenerationTaskEventFailure, logCtx)
 
 	auditRepo := repository.NewAuditRepository(pool)
 	articleRepo := repository.NewArticleRepository(pool)
@@ -421,11 +446,13 @@ func HandleKolGenerationTaskFailure(
 
 	if task.QuotaReservationID > 0 {
 		balance, balanceErr := quotaRepo.GetCurrentBalance(cleanupCtx, task.TenantID, "article_generation")
-		if balanceErr == nil && task.OperatorID > 0 {
+		if balanceErr != nil {
+			failureErr = errors.Join(failureErr, fmt.Errorf("load quota balance for refund: %w", balanceErr))
+		} else if task.OperatorID > 0 {
 			reason := "generation_refund"
 			referenceType := "generation_task"
 			taskReferenceID := task.ID
-			_, _ = quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
+			if _, err := quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
 				TenantID:      task.TenantID,
 				OperatorID:    task.OperatorID,
 				QuotaType:     "article_generation",
@@ -434,11 +461,18 @@ func HandleKolGenerationTaskFailure(
 				Reason:        &reason,
 				ReferenceType: &referenceType,
 				ReferenceID:   &taskReferenceID,
-			})
+			}); err != nil {
+				failureErr = errors.Join(failureErr, fmt.Errorf("insert generation refund ledger: %w", err))
+			}
+		}
+		if err := quotaRepo.RefundReservation(cleanupCtx, task.QuotaReservationID, task.TenantID); err != nil {
+			failureErr = errors.Join(failureErr, fmt.Errorf("refund generation reservation: %w", err))
 		}
-		_ = quotaRepo.RefundReservation(cleanupCtx, task.QuotaReservationID, task.TenantID)
 	}
 
 	InvalidateArticleCaches(cleanupCtx, cache, task.TenantID, &task.ArticleID)
+	if failureErr != nil {
+		RecordGenerationTaskEvent(GenerationTaskEventCleanupFailure, logCtx)
+	}
 	return failureErr
 }

server/internal/tenant/app/monitoring_metrics_prometheus.go

@@ -14,7 +14,7 @@ import (
 )
 
 func MonitoringSchedulerMetricsPrometheusHandler() http.Handler {
-	return monitoringPrometheusHandler(true, monitoringDailyTaskMetricsCollector{}, cache.MetricsCollector{})
+	return monitoringPrometheusHandler(true, monitoringDailyTaskMetricsCollector{}, generationTaskMetricsCollector{}, cache.MetricsCollector{})
 }
 
 func MonitoringDailyTaskMetricsPrometheus() string {
@@ -41,6 +41,14 @@ func MonitoringHeartbeatSnapshotMetricsPrometheusHandler() http.Handler {
 	return monitoringPrometheusHandler(false, monitoringHeartbeatSnapshotMetricsCollector{})
 }
 
+func GenerationTaskMetricsPrometheus() string {
+	return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, generationTaskMetricsCollector{}))
+}
+
+func GenerationTaskMetricsPrometheusHandler() http.Handler {
+	return monitoringPrometheusHandler(false, generationTaskMetricsCollector{})
+}
+
 func monitoringPrometheusHandler(includeRuntime bool, metricCollectors ...prometheus.Collector) http.Handler {
 	return promhttp.HandlerFor(monitoringPrometheusRegistry(includeRuntime, metricCollectors...), promhttp.HandlerOpts{})
 }
@@ -155,6 +163,33 @@ func (monitoringHeartbeatSnapshotMetricsCollector) Collect(ch chan<- prometheus.
 	}
 }
 
+type generationTaskMetricsCollector struct{}
+
+func (generationTaskMetricsCollector) Describe(chan<- *prometheus.Desc) {}
+
+func (generationTaskMetricsCollector) Collect(ch chan<- prometheus.Metric) {
+	snapshot := GenerationTaskMetricsSnapshotValue()
+	monitoringCounter(ch, "article_generation_task_transitions_total", "Total article generation task state transition attempts.", float64(snapshot.TransitionTotal), nil)
+	monitoringCounter(ch, "article_generation_task_failures_total", "Total article generation task failures recorded by business stage.", float64(snapshot.FailureTotal), nil)
+	monitoringCounter(ch, "article_generation_task_cleanup_failures_total", "Total article generation failure-cleanup errors that need operator attention.", float64(snapshot.CleanupFailureTotal), nil)
+	monitoringCounter(ch, "article_generation_task_queue_failures_total", "Total article generation task queue publish or delivery errors.", float64(snapshot.QueueFailureTotal), nil)
+	monitoringCounter(ch, "article_generation_task_recovery_total", "Total article generation recovery actions by action.", float64(snapshot.RecoveryRequeueTotal), []string{"action"}, "requeue")
+	monitoringCounter(ch, "article_generation_task_recovery_total", "Total article generation recovery actions by action.", float64(snapshot.RecoveryFinalizeTotal), []string{"action"}, "finalize")
+	monitoringCounter(ch, "article_generation_task_state_anomalies_total", "Total article generation task/article state anomalies detected by bounded consistency checks.", float64(snapshot.StateAnomalyTotal), nil)
+	monitoringCounter(ch, "article_generation_task_state_check_failures_total", "Total article generation state consistency check failures.", float64(snapshot.StateCheckFailureTotal), nil)
+	if snapshot.LastEvent != "" {
+		monitoringGauge(ch, "article_generation_task_last_event_info", "Last article generation task observability event.", 1, []string{"event", "task_type", "stage", "result", "anomaly_type", "severity"},
+			snapshot.LastEvent,
+			snapshot.LastTaskType,
+			snapshot.LastStage,
+			snapshot.LastResult,
+			snapshot.LastAnomalyType,
+			snapshot.LastSeverity,
+		)
+	}
+	monitoringGauge(ch, "article_generation_task_last_event_timestamp_seconds", "Unix timestamp of the last article generation task observability event.", monitoringDailyTimeSeconds(snapshot.LastEventAt), nil)
+}
+
 func monitoringCounter(ch chan<- prometheus.Metric, name, help string, value float64, labelNames []string, labelValues ...string) {
 	monitoringMetric(ch, name, help, prometheus.CounterValue, value, labelNames, labelValues...)
 }

server/internal/tenant/app/prompt_generate_service.go

@@ -3,6 +3,7 @@ package app
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strconv"
 	"strings"
@@ -445,6 +446,7 @@ func (s *PromptRuleGenerationService) enqueuePromptRuleGeneration(ctx context.Co
 		TenantID:  input.TenantID,
 		ArticleID: articleID,
 	}); err != nil {
+		RecordGenerationTaskEvent(GenerationTaskEventQueueFailure, promptRuleGenerationJobLogContext(job, "queue_enqueue", "queued", "failed", GenerationTaskResultFailure))
 		s.failGeneration(context.Background(), job, promptRuleGeneratingTitlePlaceholder, "queue_enqueue", err)
 		return nil, response.ErrServiceUnavailable(50310, "generation_queue_unavailable", "generation queue is busy, please retry")
 	}
@@ -508,12 +510,17 @@ func (s *PromptRuleGenerationService) executeGeneration(ctx context.Context, job
 	}
 
 	auditRepo := repository.NewAuditRepository(s.pool)
-	_ = auditRepo.UpdateGenerationTaskStatus(ctx, repository.GenerationTaskStatusInput{
+	if err := auditRepo.UpdateGenerationTaskStatus(ctx, repository.GenerationTaskStatusInput{
 		ID:        job.TaskID,
 		TenantID:  job.TenantID,
 		Status:    "running",
 		StartedAt: &now,
-	})
+	}); err != nil {
+		s.failGeneration(ctx, job, title, "task_status_running", err)
+		return
+	}
+	RecordGenerationTaskEvent(GenerationTaskEventTransition, promptRuleGenerationJobLogContext(job, "task_status_running", "", GenerationTaskStatusRunning, GenerationTaskResultSuccess))
+	LogGenerationTaskInfo(s.logger, "article generation task marked running", promptRuleGenerationJobLogContext(job, "task_status_running", "", GenerationTaskStatusRunning, GenerationTaskResultSuccess))
 
 	runtimeCfg := s.runtimeConfig()
 	req := llm.GenerateRequest{
@@ -686,6 +693,9 @@ func (s *PromptRuleGenerationService) failGeneration(ctx context.Context, job pr
 
 	errMsg := formatGenerationFailure(stage, genErr)
 	now := time.Now()
+	logCtx := promptRuleGenerationJobLogContext(job, stage, "", "failed", GenerationTaskResultFailure)
+	RecordGenerationTaskEvent(GenerationTaskEventFailure, logCtx)
+	LogGenerationTaskWarn(s.logger, "prompt rule generation failed", logCtx, zap.Error(genErr))
 	failureTitle := strings.TrimSpace(extractString(job.InputParams, "task_name"))
 	if failureTitle == "" {
 		failureTitle = title
@@ -695,6 +705,7 @@ func (s *PromptRuleGenerationService) failGeneration(ctx context.Context, job pr
 	articleRepo := repository.NewArticleRepository(s.pool)
 	quotaRepo := repository.NewQuotaRepository(s.pool)
 
+	var cleanupErr error
 	if err := auditRepo.UpdateGenerationTaskStatus(cleanupCtx, repository.GenerationTaskStatusInput{
 		ID:           job.TaskID,
 		TenantID:     job.TenantID,
@@ -708,6 +719,7 @@ func (s *PromptRuleGenerationService) failGeneration(ctx context.Context, job pr
 			zap.Int64("article_id", job.ArticleID),
 			zap.String("stage", stage),
 		)
+		cleanupErr = errors.Join(cleanupErr, fmt.Errorf("update generation task failed status: %w", err))
 	}
 	if err := articleRepo.UpdateArticleGenerateStatus(cleanupCtx, job.ArticleID, job.TenantID, "failed"); err != nil && s.logger != nil {
 		s.logger.Warn("prompt rule generation article failed status update failed",
@@ -716,6 +728,7 @@ func (s *PromptRuleGenerationService) failGeneration(ctx context.Context, job pr
 			zap.Int64("article_id", job.ArticleID),
 			zap.String("stage", stage),
 		)
+		cleanupErr = errors.Join(cleanupErr, fmt.Errorf("update article failed status: %w", err))
 	}
 	if failureTitle != "" {
 		_, _ = s.pool.Exec(cleanupCtx, `
@@ -727,11 +740,13 @@ func (s *PromptRuleGenerationService) failGeneration(ctx context.Context, job pr
 	}
 
 	balance, balanceErr := quotaRepo.GetCurrentBalance(cleanupCtx, job.TenantID, "article_generation")
-	if balanceErr == nil {
+	if balanceErr != nil {
+		cleanupErr = errors.Join(cleanupErr, fmt.Errorf("load quota balance for refund: %w", balanceErr))
+	} else {
 		reason := "generation_refund"
 		referenceType := "generation_task"
 		taskReferenceID := job.TaskID
-		_, _ = quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
+		if _, err := quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
 			TenantID:      job.TenantID,
 			OperatorID:    job.OperatorID,
 			QuotaType:     "article_generation",
@@ -740,11 +755,19 @@ func (s *PromptRuleGenerationService) failGeneration(ctx context.Context, job pr
 			Reason:        &reason,
 			ReferenceType: &referenceType,
 			ReferenceID:   &taskReferenceID,
-		})
+		}); err != nil {
+			cleanupErr = errors.Join(cleanupErr, fmt.Errorf("insert generation refund ledger: %w", err))
+		}
 	}
 
-	_ = quotaRepo.RefundReservation(cleanupCtx, job.ReservationID, job.TenantID)
+	if err := quotaRepo.RefundReservation(cleanupCtx, job.ReservationID, job.TenantID); err != nil {
+		cleanupErr = errors.Join(cleanupErr, fmt.Errorf("refund generation reservation: %w", err))
+	}
 	invalidateArticleCaches(cleanupCtx, s.cache, job.TenantID, &job.ArticleID)
+	if cleanupErr != nil {
+		RecordGenerationTaskEvent(GenerationTaskEventCleanupFailure, logCtx)
+		LogGenerationTaskError(s.logger, "prompt rule generation failure cleanup failed", logCtx, zap.Error(cleanupErr))
+	}
 
 	runtimeCfg := s.runtimeConfig()
 	if runtimeCfg.StreamEnabled && s.streams != nil {
@@ -752,6 +775,21 @@ func (s *PromptRuleGenerationService) failGeneration(ctx context.Context, job pr
 	}
 }
 
+func promptRuleGenerationJobLogContext(job promptRuleGenerationJob, stage, statusBefore, statusAfter, result string) GenerationTaskLogContext {
+	return GenerationTaskLogContext{
+		TaskID:        job.TaskID,
+		TenantID:      job.TenantID,
+		OperatorID:    job.OperatorID,
+		ArticleID:     job.ArticleID,
+		ReservationID: job.ReservationID,
+		TaskType:      "custom_generation",
+		Stage:         stage,
+		StatusBefore:  statusBefore,
+		StatusAfter:   statusAfter,
+		Result:        result,
+	}
+}
+
 func buildPromptRuleGenerationPrompt(rule *promptRuleGenerationRecord, params map[string]interface{}, knowledgePrompt string) string {
 	sections := []string{strings.TrimSpace(rule.PromptContent)}
 

server/internal/tenant/app/template_service.go

@@ -409,6 +409,7 @@ func (s *TemplateService) Generate(ctx context.Context, templateID int64, req Ge
 		TenantID:  actor.TenantID,
 		ArticleID: articleID,
 	}); err != nil {
+		RecordGenerationTaskEvent(GenerationTaskEventQueueFailure, generationJobLogContext(job, "queue_enqueue", "queued", "failed", GenerationTaskResultFailure))
 		s.failGeneration(context.Background(), job, initialTitle, "queue_enqueue", err)
 		return nil, response.ErrServiceUnavailable(50310, "generation_queue_unavailable", "generation queue is busy, please retry")
 	}
@@ -457,12 +458,17 @@ func (s *TemplateService) executeGeneration(ctx context.Context, job generationJ
 	}
 
 	auditRepo := repository.NewAuditRepository(s.pool)
-	_ = auditRepo.UpdateGenerationTaskStatus(ctx, repository.GenerationTaskStatusInput{
+	if err := auditRepo.UpdateGenerationTaskStatus(ctx, repository.GenerationTaskStatusInput{
 		ID:        job.TaskID,
 		TenantID:  job.TenantID,
 		Status:    "running",
 		StartedAt: &now,
-	})
+	}); err != nil {
+		s.failGeneration(ctx, job, title, "task_status_running", err)
+		return
+	}
+	RecordGenerationTaskEvent(GenerationTaskEventTransition, generationJobLogContext(job, "task_status_running", "", GenerationTaskStatusRunning, GenerationTaskResultSuccess))
+	LogGenerationTaskInfo(s.logger, "article generation task marked running", generationJobLogContext(job, "task_status_running", "", GenerationTaskStatusRunning, GenerationTaskResultSuccess))
 
 	knowledgePrompt := ""
 	if knowledgeGroupIDs := extractKnowledgeGroupIDs(job.Params["knowledge_group_ids"]); len(knowledgeGroupIDs) > 0 {
@@ -586,11 +592,15 @@ func (s *TemplateService) failGeneration(ctx context.Context, job generationJob,
 
 	errMsg := formatGenerationFailure(stage, genErr)
 	now := time.Now()
+	logCtx := generationJobLogContext(job, stage, "", "failed", GenerationTaskResultFailure)
+	RecordGenerationTaskEvent(GenerationTaskEventFailure, logCtx)
+	LogGenerationTaskWarn(s.logger, "template generation failed", logCtx, zap.Error(genErr))
 
 	auditRepo := repository.NewAuditRepository(s.pool)
 	articleRepo := repository.NewArticleRepository(s.pool)
 	quotaRepo := repository.NewQuotaRepository(s.pool)
 
+	var cleanupErr error
 	if err := auditRepo.UpdateGenerationTaskStatus(cleanupCtx, repository.GenerationTaskStatusInput{
 		ID:           job.TaskID,
 		TenantID:     job.TenantID,
@@ -604,6 +614,7 @@ func (s *TemplateService) failGeneration(ctx context.Context, job generationJob,
 			zap.Int64("article_id", job.ArticleID),
 			zap.String("stage", stage),
 		)
+		cleanupErr = errors.Join(cleanupErr, fmt.Errorf("update generation task failed status: %w", err))
 	}
 	if err := articleRepo.UpdateArticleGenerateStatus(cleanupCtx, job.ArticleID, job.TenantID, "failed"); err != nil && s.logger != nil {
 		s.logger.Warn("template generation article failed status update failed",
@@ -612,14 +623,17 @@ func (s *TemplateService) failGeneration(ctx context.Context, job generationJob,
 			zap.Int64("article_id", job.ArticleID),
 			zap.String("stage", stage),
 		)
+		cleanupErr = errors.Join(cleanupErr, fmt.Errorf("update article failed status: %w", err))
 	}
 
 	balance, balanceErr := quotaRepo.GetCurrentBalance(cleanupCtx, job.TenantID, "article_generation")
-	if balanceErr == nil {
+	if balanceErr != nil {
+		cleanupErr = errors.Join(cleanupErr, fmt.Errorf("load quota balance for refund: %w", balanceErr))
+	} else {
 		reason := "generation_refund"
 		referenceType := "generation_task"
 		taskReferenceID := job.TaskID
-		_, _ = quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
+		if _, err := quotaRepo.InsertQuotaLedger(cleanupCtx, repository.QuotaLedgerInput{
 			TenantID:      job.TenantID,
 			OperatorID:    job.OperatorID,
 			QuotaType:     "article_generation",
@@ -628,11 +642,19 @@ func (s *TemplateService) failGeneration(ctx context.Context, job generationJob,
 			Reason:        &reason,
 			ReferenceType: &referenceType,
 			ReferenceID:   &taskReferenceID,
-		})
+		}); err != nil {
+			cleanupErr = errors.Join(cleanupErr, fmt.Errorf("insert generation refund ledger: %w", err))
+		}
 	}
 
-	_ = quotaRepo.RefundReservation(cleanupCtx, job.ReservationID, job.TenantID)
+	if err := quotaRepo.RefundReservation(cleanupCtx, job.ReservationID, job.TenantID); err != nil {
+		cleanupErr = errors.Join(cleanupErr, fmt.Errorf("refund generation reservation: %w", err))
+	}
 	invalidateArticleCaches(cleanupCtx, s.cache, job.TenantID, &job.ArticleID)
+	if cleanupErr != nil {
+		RecordGenerationTaskEvent(GenerationTaskEventCleanupFailure, logCtx)
+		LogGenerationTaskError(s.logger, "template generation failure cleanup failed", logCtx, zap.Error(cleanupErr))
+	}
 
 	runtimeCfg := s.runtimeConfig()
 	if runtimeCfg.StreamEnabled && s.streams != nil {
@@ -640,6 +662,21 @@ func (s *TemplateService) failGeneration(ctx context.Context, job generationJob,
 	}
 }
 
+func generationJobLogContext(job generationJob, stage, statusBefore, statusAfter, result string) GenerationTaskLogContext {
+	return GenerationTaskLogContext{
+		TaskID:        job.TaskID,
+		TenantID:      job.TenantID,
+		OperatorID:    job.OperatorID,
+		ArticleID:     job.ArticleID,
+		ReservationID: job.ReservationID,
+		TaskType:      "template",
+		Stage:         stage,
+		StatusBefore:  statusBefore,
+		StatusAfter:   statusAfter,
+		Result:        result,
+	}
+}
+
 func formatGenerationFailure(stage string, genErr error) string {
 	cause := ""
 	if genErr != nil {
@@ -657,6 +694,8 @@ func generationFailureStageLabel(stage string) string {
 	switch stage {
 	case "task_load":
 		return "加载生成任务失败"
+	case "task_status_running":
+		return "更新生成任务运行状态失败"
 	case "queue_enqueue":
 		return "任务入队失败"
 	case "llm_generate":