harden article generation reliability
Deployment Config CI / Deployment Config (push) Successful in 24s
Backend CI / Backend (push) Successful in 14m12s

This commit is contained in:
2026-05-05 23:43:10 +08:00
parent 3c912949e4
commit 65e9b7e293
24 changed files with 1259 additions and 52 deletions
@@ -14,7 +14,7 @@ import (
)
func MonitoringSchedulerMetricsPrometheusHandler() http.Handler {
return monitoringPrometheusHandler(true, monitoringDailyTaskMetricsCollector{}, cache.MetricsCollector{})
return monitoringPrometheusHandler(true, monitoringDailyTaskMetricsCollector{}, generationTaskMetricsCollector{}, cache.MetricsCollector{})
}
func MonitoringDailyTaskMetricsPrometheus() string {
@@ -41,6 +41,14 @@ func MonitoringHeartbeatSnapshotMetricsPrometheusHandler() http.Handler {
return monitoringPrometheusHandler(false, monitoringHeartbeatSnapshotMetricsCollector{})
}
func GenerationTaskMetricsPrometheus() string {
return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, generationTaskMetricsCollector{}))
}
func GenerationTaskMetricsPrometheusHandler() http.Handler {
return monitoringPrometheusHandler(false, generationTaskMetricsCollector{})
}
func monitoringPrometheusHandler(includeRuntime bool, metricCollectors ...prometheus.Collector) http.Handler {
return promhttp.HandlerFor(monitoringPrometheusRegistry(includeRuntime, metricCollectors...), promhttp.HandlerOpts{})
}
@@ -155,6 +163,33 @@ func (monitoringHeartbeatSnapshotMetricsCollector) Collect(ch chan<- prometheus.
}
}
type generationTaskMetricsCollector struct{}
func (generationTaskMetricsCollector) Describe(chan<- *prometheus.Desc) {}
func (generationTaskMetricsCollector) Collect(ch chan<- prometheus.Metric) {
snapshot := GenerationTaskMetricsSnapshotValue()
monitoringCounter(ch, "article_generation_task_transitions_total", "Total article generation task state transition attempts.", float64(snapshot.TransitionTotal), nil)
monitoringCounter(ch, "article_generation_task_failures_total", "Total article generation task failures recorded by business stage.", float64(snapshot.FailureTotal), nil)
monitoringCounter(ch, "article_generation_task_cleanup_failures_total", "Total article generation failure-cleanup errors that need operator attention.", float64(snapshot.CleanupFailureTotal), nil)
monitoringCounter(ch, "article_generation_task_queue_failures_total", "Total article generation task queue publish or delivery errors.", float64(snapshot.QueueFailureTotal), nil)
monitoringCounter(ch, "article_generation_task_recovery_total", "Total article generation recovery actions by action.", float64(snapshot.RecoveryRequeueTotal), []string{"action"}, "requeue")
monitoringCounter(ch, "article_generation_task_recovery_total", "Total article generation recovery actions by action.", float64(snapshot.RecoveryFinalizeTotal), []string{"action"}, "finalize")
monitoringCounter(ch, "article_generation_task_state_anomalies_total", "Total article generation task/article state anomalies detected by bounded consistency checks.", float64(snapshot.StateAnomalyTotal), nil)
monitoringCounter(ch, "article_generation_task_state_check_failures_total", "Total article generation state consistency check failures.", float64(snapshot.StateCheckFailureTotal), nil)
if snapshot.LastEvent != "" {
monitoringGauge(ch, "article_generation_task_last_event_info", "Last article generation task observability event.", 1, []string{"event", "task_type", "stage", "result", "anomaly_type", "severity"},
snapshot.LastEvent,
snapshot.LastTaskType,
snapshot.LastStage,
snapshot.LastResult,
snapshot.LastAnomalyType,
snapshot.LastSeverity,
)
}
monitoringGauge(ch, "article_generation_task_last_event_timestamp_seconds", "Unix timestamp of the last article generation task observability event.", monitoringDailyTimeSeconds(snapshot.LastEventAt), nil)
}
func monitoringCounter(ch chan<- prometheus.Metric, name, help string, value float64, labelNames []string, labelValues ...string) {
monitoringMetric(ch, name, help, prometheus.CounterValue, value, labelNames, labelValues...)
}