harden article generation reliability

2026-05-05 23:43:10 +08:00
parent 3c912949e4
commit 65e9b7e293
24 changed files with 1259 additions and 52 deletions
@@ -0,0 +1,56 @@
+groups:
+  - name: article-generation-reliability
+    rules:
+      - alert: ArticleGenerationCleanupFailure
+        expr: increase(article_generation_task_cleanup_failures_total[5m]) > 0
+        labels:
+          severity: critical
+          component: article_generation
+        annotations:
+          summary: Article generation failure cleanup is failing
+          description: Failure cleanup could not persist task/article/quota state. Check structured logs with event_domain=article_generation and event=cleanup_failure.
+
+      - alert: ArticleGenerationStateAnomaly
+        expr: increase(article_generation_task_state_anomalies_total[5m]) > 0
+        labels:
+          severity: critical
+          component: article_generation
+        annotations:
+          summary: Article generation task state anomaly detected
+          description: Bounded state consistency checks found task/article/version mismatch. Inspect anomaly_type, task_id, tenant_id, and article_id in structured logs.
+
+      - alert: ArticleGenerationStateCheckFailure
+        expr: increase(article_generation_task_state_check_failures_total[15m]) > 0
+        labels:
+          severity: warning
+          component: article_generation
+        annotations:
+          summary: Article generation state check is failing
+          description: The scheduler could not complete the bounded generation task state check. Inspect scheduler logs and database health.
+
+      - alert: ArticleGenerationQueueFailure
+        expr: increase(article_generation_task_queue_failures_total[10m]) > 0
+        labels:
+          severity: warning
+          component: article_generation
+        annotations:
+          summary: Article generation queue delivery failed
+          description: Queue publish or delivery reject errors were observed. Inspect RabbitMQ health and generation task structured logs.
+
+      - alert: ArticleGenerationFailureSpike
+        expr: increase(article_generation_task_failures_total[15m]) >= 5
+        labels:
+          severity: warning
+          component: article_generation
+        annotations:
+          summary: Article generation failures are spiking
+          description: Multiple article generation failures occurred in a short window. Group by stage/task_type from structured logs to identify the failing dependency.
+
+      - alert: ArticleGenerationRecoveryFinalize
+        expr: increase(article_generation_task_recovery_total{action="finalize"}[15m]) > 0
+        labels:
+          severity: warning
+          component: article_generation
+        annotations:
+          summary: Article generation recovery finalized stale tasks
+          description: Recovery had to mark stale generation tasks failed instead of normal worker completion. Inspect task_id/article_id logs and worker health.