harden article generation reliability

server/internal/tenant/app/generation_observability_test.go

@@ -0,0 +1,101 @@
+package app
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGenerationTaskLogFieldsUsesStableStructuredKeys(t *testing.T) {
+	fields := GenerationTaskLogFields(GenerationTaskLogContext{
+		TaskID:        11,
+		TenantID:      22,
+		OperatorID:    33,
+		ArticleID:     44,
+		ReservationID: 55,
+		TaskType:      "template",
+		Stage:         "llm_generate",
+		AttemptCount:  2,
+		LeaseOwner:    "worker-1",
+		StatusBefore:  "running",
+		StatusAfter:   "failed",
+		Result:        GenerationTaskResultFailure,
+		AnomalyType:   "article_failed_task_running",
+		Severity:      "critical",
+	})
+
+	values := map[string]any{}
+	for _, field := range fields {
+		values[field.Key] = field.Interface
+		if field.String != "" {
+			values[field.Key] = field.String
+		}
+		if field.Integer != 0 {
+			values[field.Key] = field.Integer
+		}
+	}
+
+	assert.Equal(t, "article_generation", values["event_domain"])
+	assert.Equal(t, int64(11), values["task_id"])
+	assert.Equal(t, int64(22), values["tenant_id"])
+	assert.Equal(t, int64(44), values["article_id"])
+	assert.Equal(t, "template", values["task_type"])
+	assert.Equal(t, "llm_generate", values["stage"])
+	assert.Equal(t, "failed", values["status_after"])
+	assert.Equal(t, "critical", values["severity"])
+}
+
+func TestGenerationTaskMetricsRecordsEventsAndPrometheus(t *testing.T) {
+	resetGenerationTaskMetricsForTest()
+	t.Cleanup(resetGenerationTaskMetricsForTest)
+
+	RecordGenerationTaskEvent(GenerationTaskEventTransition, GenerationTaskLogContext{
+		TaskType:    "template",
+		Stage:       "worker_claim",
+		StatusAfter: GenerationTaskStatusRunning,
+		Result:      GenerationTaskResultSuccess,
+	})
+	RecordGenerationTaskEvent(GenerationTaskEventCleanupFailure, GenerationTaskLogContext{
+		TaskType: "template",
+		Stage:    "refund_reservation",
+		Result:   GenerationTaskResultFailure,
+		Severity: "critical",
+	})
+	RecordGenerationTaskEvent(GenerationTaskEventStateAnomaly, GenerationTaskLogContext{
+		TaskType:    "template",
+		Stage:       "state_check",
+		AnomalyType: "completed_task_missing_version",
+		Severity:    "critical",
+	})
+
+	snapshot := GenerationTaskMetricsSnapshotValue()
+	assert.Equal(t, int64(1), snapshot.TransitionTotal)
+	assert.Equal(t, int64(1), snapshot.CleanupFailureTotal)
+	assert.Equal(t, int64(1), snapshot.StateAnomalyTotal)
+	assert.Equal(t, "state_anomaly", snapshot.LastEvent)
+	assert.Equal(t, "completed_task_missing_version", snapshot.LastAnomalyType)
+	assert.NotNil(t, snapshot.LastEventAt)
+
+	prometheus := GenerationTaskMetricsPrometheus()
+	assert.Contains(t, prometheus, "article_generation_task_transitions_total 1")
+	assert.Contains(t, prometheus, "article_generation_task_cleanup_failures_total 1")
+	assert.Contains(t, prometheus, "article_generation_task_state_anomalies_total 1")
+	lastEventLine := findPrometheusMetricLine(prometheus, "article_generation_task_last_event_info")
+	assert.Contains(t, lastEventLine, `event="state_anomaly"`)
+	assert.Contains(t, lastEventLine, `task_type="template"`)
+	assert.Contains(t, lastEventLine, `stage="state_check"`)
+	assert.Contains(t, lastEventLine, `result=""`)
+	assert.Contains(t, lastEventLine, `anomaly_type="completed_task_missing_version"`)
+	assert.Contains(t, lastEventLine, `severity="critical"`)
+	assert.True(t, strings.HasSuffix(lastEventLine, " 1"), lastEventLine)
+}
+
+func findPrometheusMetricLine(output, metricName string) string {
+	for _, line := range strings.Split(output, "\n") {
+		if strings.HasPrefix(line, metricName+"{") || strings.HasPrefix(line, metricName+" ") {
+			return line
+		}
+	}
+	return ""
+}