geo/server/internal/tenant/app/monitoring_metrics_prometheus.go

package app

import (
	"bytes"
	"net/http"
	"strings"

	"github.com/prometheus/client_golang/prometheus"
	promcollectors "github.com/prometheus/client_golang/prometheus/collectors"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/common/expfmt"

	"github.com/geo-platform/tenant-api/internal/shared/cache"
)

func MonitoringSchedulerMetricsPrometheusHandler() http.Handler {
	return monitoringPrometheusHandler(true, monitoringDailyTaskMetricsCollector{}, generationTaskMetricsCollector{}, cache.MetricsCollector{})
}

func MonitoringDailyTaskMetricsPrometheus() string {
	return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringDailyTaskMetricsCollector{}))
}

func MonitoringDailyTaskMetricsPrometheusHandler() http.Handler {
	return monitoringPrometheusHandler(false, monitoringDailyTaskMetricsCollector{})
}

func MonitoringHeartbeatPrimaryLeaseMetricsPrometheus() string {
	return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringHeartbeatPrimaryLeaseMetricsCollector{}))
}

func MonitoringHeartbeatPrimaryLeaseMetricsPrometheusHandler() http.Handler {
	return monitoringPrometheusHandler(false, monitoringHeartbeatPrimaryLeaseMetricsCollector{})
}

func MonitoringHeartbeatSnapshotMetricsPrometheus() string {
	return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringHeartbeatSnapshotMetricsCollector{}))
}

func MonitoringHeartbeatSnapshotMetricsPrometheusHandler() http.Handler {
	return monitoringPrometheusHandler(false, monitoringHeartbeatSnapshotMetricsCollector{})
}

func GenerationTaskMetricsPrometheus() string {
	return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, generationTaskMetricsCollector{}))
}

func GenerationTaskMetricsPrometheusHandler() http.Handler {
	return monitoringPrometheusHandler(false, generationTaskMetricsCollector{})
}

func monitoringPrometheusHandler(includeRuntime bool, metricCollectors ...prometheus.Collector) http.Handler {
	return promhttp.HandlerFor(monitoringPrometheusRegistry(includeRuntime, metricCollectors...), promhttp.HandlerOpts{})
}

func monitoringPrometheusRegistry(includeRuntime bool, metricCollectors ...prometheus.Collector) *prometheus.Registry {
	registry := prometheus.NewRegistry()
	if includeRuntime {
		registry.MustRegister(
			promcollectors.NewGoCollector(),
			promcollectors.NewProcessCollector(promcollectors.ProcessCollectorOpts{}),
		)
	}
	for _, collector := range metricCollectors {
		registry.MustRegister(collector)
	}
	return registry
}

func renderMonitoringPrometheus(gatherer prometheus.Gatherer) string {
	metricFamilies, err := gatherer.Gather()
	if err != nil {
		return ""
	}
	var b bytes.Buffer
	encoder := expfmt.NewEncoder(&b, expfmt.NewFormat(expfmt.TypeTextPlain))
	for _, metricFamily := range metricFamilies {
		if err := encoder.Encode(metricFamily); err != nil {
			return ""
		}
	}
	if closer, ok := encoder.(expfmt.Closer); ok {
		_ = closer.Close()
	}
	return b.String()
}

type monitoringDailyTaskMetricsCollector struct{}

func (monitoringDailyTaskMetricsCollector) Describe(chan<- *prometheus.Desc) {}

func (monitoringDailyTaskMetricsCollector) Collect(ch chan<- prometheus.Metric) {
	snapshot := MonitoringDailyTaskMetricsSnapshotValue()
	monitoringCounter(ch, "monitoring_daily_task_worker_runs_total", "Total daily monitoring worker runs by terminal status.", float64(snapshot.RunCompletedTotal), []string{"status"}, "completed")
	monitoringCounter(ch, "monitoring_daily_task_worker_runs_total", "Total daily monitoring worker runs by terminal status.", float64(snapshot.RunFailedTotal), []string{"status"}, "failed")
	monitoringCounter(ch, "monitoring_daily_task_worker_partial_failure_runs_total", "Total completed daily monitoring worker runs with at least one plan or brand failure.", float64(snapshot.RunPartialFailureTotal), nil)
	monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.PlanFailureTotal), []string{"scope"}, "plan")
	monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.BrandFailureTotal), []string{"scope"}, "brand")
	monitoringCounter(ch, "monitoring_daily_task_generation_skipped_plans_total", "Total daily monitoring plans skipped before generation.", float64(snapshot.SkippedPlanTotal), nil)
	monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.PlannedTaskTotal), []string{"stage"}, "planned")
	monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DueTaskTotal), []string{"stage"}, "due")
	monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.CreatedTaskTotal), []string{"stage"}, "created")
	monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DesktopTaskTotal), []string{"stage"}, "desktop")
	monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DeferredTaskTotal), []string{"stage"}, "deferred")
	monitoringGauge(ch, "monitoring_daily_task_worker_last_run_timestamp_seconds", "Unix timestamp of the last daily monitoring worker run completion.", monitoringDailyTimeSeconds(snapshot.LastRunAt), nil)
	monitoringGauge(ch, "monitoring_daily_task_worker_last_success_timestamp_seconds", "Unix timestamp of the last completed daily monitoring worker run.", monitoringDailyTimeSeconds(snapshot.LastSuccessAt), nil)
	monitoringGauge(ch, "monitoring_daily_task_worker_last_error_timestamp_seconds", "Unix timestamp of the last failed daily monitoring worker run.", monitoringDailyTimeSeconds(snapshot.LastErrorAt), nil)
	monitoringGauge(ch, "monitoring_daily_task_worker_last_duration_seconds", "Duration of the last daily monitoring worker run.", snapshot.LastDurationSeconds, nil)
	monitoringGauge(ch, "monitoring_daily_task_worker_last_scope_count", "Tenant and brand counts observed in the last daily monitoring worker run.", float64(snapshot.LastTenantCount), []string{"scope"}, "tenant")
	monitoringGauge(ch, "monitoring_daily_task_worker_last_scope_count", "Tenant and brand counts observed in the last daily monitoring worker run.", float64(snapshot.LastBrandCount), []string{"scope"}, "brand")
	if snapshot.LastBusinessDate != "" {
		monitoringGauge(ch, "monitoring_daily_task_worker_last_business_date_info", "Business date processed by the last daily monitoring worker run.", 1, []string{"business_date"}, snapshot.LastBusinessDate)
	}
	if snapshot.LastErrorCode != "" {
		monitoringGauge(ch, "monitoring_daily_task_worker_last_error_info", "Last fatal daily monitoring worker error code.", 1, []string{"code"}, snapshot.LastErrorCode)
	}
}

type monitoringHeartbeatPrimaryLeaseMetricsCollector struct{}

func (monitoringHeartbeatPrimaryLeaseMetricsCollector) Describe(chan<- *prometheus.Desc) {}

func (monitoringHeartbeatPrimaryLeaseMetricsCollector) Collect(ch chan<- prometheus.Metric) {
	snapshot := MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue()
	help := "Total monitoring heartbeat primary lease resolutions by outcome."
	monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.PrimaryHitTotal), []string{"outcome"}, "primary_hit")
	monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.NonPrimaryLiveTotal), []string{"outcome"}, "non_primary_live")
	monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.InsertedTotal), []string{"outcome"}, "inserted")
	monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.RenewedTotal), []string{"outcome"}, "renewed")
	monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ClaimedExpiredTotal), []string{"outcome"}, "claimed_expired")
	monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ClaimedUnavailableTotal), []string{"outcome"}, "claimed_unavailable")
	monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ContentionTotal), []string{"outcome"}, "contention")
	monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ErrorTotal), []string{"outcome"}, "error")
	monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBInsertTotal), []string{"operation"}, "insert")
	monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBRenewTotal), []string{"operation"}, "renew")
	monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBClaimTotal), []string{"operation"}, "claim")
	monitoringSummary(ch, "monitoring_heartbeat_primary_lease_duration_seconds", "Monitoring heartbeat primary lease resolution duration summary.", snapshot.DurationCount, snapshot.DurationSumSeconds, nil)
	monitoringGauge(ch, "monitoring_heartbeat_primary_lease_duration_seconds_last", "Last monitoring heartbeat primary lease resolution duration in seconds.", snapshot.LastDurationSeconds, nil)
	if snapshot.LastOutcome != "" {
		monitoringGauge(ch, "monitoring_heartbeat_primary_lease_last_outcome_info", "Last monitoring heartbeat primary lease resolution outcome.", 1, []string{"outcome"}, snapshot.LastOutcome)
	}
}

type monitoringHeartbeatSnapshotMetricsCollector struct{}

func (monitoringHeartbeatSnapshotMetricsCollector) Describe(chan<- *prometheus.Desc) {}

func (monitoringHeartbeatSnapshotMetricsCollector) Collect(ch chan<- prometheus.Metric) {
	snapshot := MonitoringHeartbeatSnapshotMetricsSnapshotValue()
	monitoringCounter(ch, "monitoring_heartbeat_snapshot_platform_reports_total", "Total monitoring heartbeat platform reports received.", float64(snapshot.PlatformReportedTotal), nil)
	monitoringCounter(ch, "monitoring_heartbeat_snapshot_transactions_total", "Total monitoring heartbeat snapshot transactions opened.", float64(snapshot.TransactionOpenedTotal), nil)
	monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotInsertedTotal), []string{"operation"}, "insert")
	monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotUpdatedTotal), []string{"operation"}, "update")
	monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotRefreshedTotal), []string{"operation"}, "refresh")
	monitoringCounter(ch, "monitoring_heartbeat_snapshot_skips_total", "Total monitoring heartbeat access snapshot reports skipped because state was fresh and unchanged.", float64(snapshot.SnapshotSkippedTotal), nil)
	monitoringCounter(ch, "monitoring_heartbeat_snapshot_reconciled_tasks_total", "Total pending monitoring tasks skipped by heartbeat access reconciliation.", float64(snapshot.ReconciledSkippedTaskTotal), nil)
	monitoringCounter(ch, "monitoring_heartbeat_snapshot_projection_rebuilds_total", "Total monitoring daily projection rebuilds triggered by heartbeat snapshots.", float64(snapshot.ProjectionRebuildTotal), nil)
	monitoringCounter(ch, "monitoring_heartbeat_snapshot_errors_total", "Total monitoring heartbeat snapshot errors.", float64(snapshot.ErrorTotal), nil)
	monitoringSummary(ch, "monitoring_heartbeat_snapshot_duration_seconds", "Monitoring heartbeat snapshot processing duration summary.", snapshot.DurationCount, snapshot.DurationSumSeconds, nil)
	monitoringGauge(ch, "monitoring_heartbeat_snapshot_duration_seconds_last", "Last monitoring heartbeat snapshot processing duration in seconds.", snapshot.LastDurationSeconds, nil)
	if snapshot.LastErrorCode != "" {
		monitoringGauge(ch, "monitoring_heartbeat_snapshot_last_error_info", "Last monitoring heartbeat snapshot error code.", 1, []string{"code"}, snapshot.LastErrorCode)
	}
}

type generationTaskMetricsCollector struct{}

func (generationTaskMetricsCollector) Describe(chan<- *prometheus.Desc) {}

func (generationTaskMetricsCollector) Collect(ch chan<- prometheus.Metric) {
	snapshot := GenerationTaskMetricsSnapshotValue()
	monitoringCounter(ch, "article_generation_task_transitions_total", "Total article generation task state transition attempts.", float64(snapshot.TransitionTotal), nil)
	monitoringCounter(ch, "article_generation_task_failures_total", "Total article generation task failures recorded by business stage.", float64(snapshot.FailureTotal), nil)
	monitoringCounter(ch, "article_generation_task_cleanup_failures_total", "Total article generation failure-cleanup errors that need operator attention.", float64(snapshot.CleanupFailureTotal), nil)
	monitoringCounter(ch, "article_generation_task_queue_failures_total", "Total article generation task queue publish or delivery errors.", float64(snapshot.QueueFailureTotal), nil)
	monitoringCounter(ch, "article_generation_task_recovery_total", "Total article generation recovery actions by action.", float64(snapshot.RecoveryRequeueTotal), []string{"action"}, "requeue")
	monitoringCounter(ch, "article_generation_task_recovery_total", "Total article generation recovery actions by action.", float64(snapshot.RecoveryFinalizeTotal), []string{"action"}, "finalize")
	monitoringCounter(ch, "article_generation_task_state_anomalies_total", "Total article generation task/article state anomalies detected by bounded consistency checks.", float64(snapshot.StateAnomalyTotal), nil)
	monitoringCounter(ch, "article_generation_task_state_check_failures_total", "Total article generation state consistency check failures.", float64(snapshot.StateCheckFailureTotal), nil)
	if snapshot.LastEvent != "" {
		monitoringGauge(ch, "article_generation_task_last_event_info", "Last article generation task observability event.", 1, []string{"event", "task_type", "stage", "result", "anomaly_type", "severity"},
			snapshot.LastEvent,
			snapshot.LastTaskType,
			snapshot.LastStage,
			snapshot.LastResult,
			snapshot.LastAnomalyType,
			snapshot.LastSeverity,
		)
	}
	monitoringGauge(ch, "article_generation_task_last_event_timestamp_seconds", "Unix timestamp of the last article generation task observability event.", monitoringDailyTimeSeconds(snapshot.LastEventAt), nil)
}

func monitoringCounter(ch chan<- prometheus.Metric, name, help string, value float64, labelNames []string, labelValues ...string) {
	monitoringMetric(ch, name, help, prometheus.CounterValue, value, labelNames, labelValues...)
}

func monitoringGauge(ch chan<- prometheus.Metric, name, help string, value float64, labelNames []string, labelValues ...string) {
	monitoringMetric(ch, name, help, prometheus.GaugeValue, value, labelNames, labelValues...)
}

func monitoringMetric(ch chan<- prometheus.Metric, name, help string, valueType prometheus.ValueType, value float64, labelNames []string, labelValues ...string) {
	ch <- prometheus.MustNewConstMetric(
		prometheus.NewDesc(name, help, labelNames, nil),
		valueType,
		value,
		monitoringPrometheusLabelValues(labelValues)...,
	)
}

func monitoringSummary(ch chan<- prometheus.Metric, name, help string, count int64, sum float64, labelNames []string, labelValues ...string) {
	if count < 0 {
		count = 0
	}
	ch <- prometheus.MustNewConstSummary(
		prometheus.NewDesc(name, help, labelNames, nil),
		uint64(count),
		sum,
		nil,
		monitoringPrometheusLabelValues(labelValues)...,
	)
}

func monitoringPrometheusLabelValues(values []string) []string {
	if len(values) == 0 {
		return values
	}
	result := make([]string, len(values))
	for idx, value := range values {
		result[idx] = strings.ReplaceAll(value, "\r", `\r`)
	}
	return result
}