root
ff86933c24
Adds a MonitoringDailyTaskWorker that materializes and dispatches daily collect tasks from active subscription/desktop-client state, with per-run atomic metrics and a Prometheus collector. Dashboard and question-detail APIs now surface a platform_authorization_status so the UI can distinguish no-desktop-client, no-authorized-platforms, and authorized states; quota/access-state lookups resolve by request workspace instead of tenant. Hardens heartbeat: a desktop_client_primary_leases table gives per-(tenant, workspace) sticky primary selection, read-mostly lease resolution avoids a per-heartbeat transaction, and unchanged platform access reports skip snapshot upserts outside a 10-minute refresh window. Adds heartbeat primary/snapshot metrics exposed via token-protected tenant-api /api/internal/metrics endpoints plus Prometheus. Monitoring quota seed is removed from dev-seed since daily plans now derive from desktop bindings, and the projection uses the canonical six-platform catalog so platforms with zero samples still render. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
196 lines
13 KiB
Go
196 lines
13 KiB
Go
package app
|
|
|
|
import (
|
|
"bytes"
|
|
"net/http"
|
|
"strings"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
promcollectors "github.com/prometheus/client_golang/prometheus/collectors"
|
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
|
"github.com/prometheus/common/expfmt"
|
|
)
|
|
|
|
func MonitoringSchedulerMetricsPrometheusHandler() http.Handler {
|
|
return monitoringPrometheusHandler(true, monitoringDailyTaskMetricsCollector{})
|
|
}
|
|
|
|
func MonitoringDailyTaskMetricsPrometheus() string {
|
|
return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringDailyTaskMetricsCollector{}))
|
|
}
|
|
|
|
func MonitoringDailyTaskMetricsPrometheusHandler() http.Handler {
|
|
return monitoringPrometheusHandler(false, monitoringDailyTaskMetricsCollector{})
|
|
}
|
|
|
|
func MonitoringHeartbeatPrimaryLeaseMetricsPrometheus() string {
|
|
return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringHeartbeatPrimaryLeaseMetricsCollector{}))
|
|
}
|
|
|
|
func MonitoringHeartbeatPrimaryLeaseMetricsPrometheusHandler() http.Handler {
|
|
return monitoringPrometheusHandler(false, monitoringHeartbeatPrimaryLeaseMetricsCollector{})
|
|
}
|
|
|
|
func MonitoringHeartbeatSnapshotMetricsPrometheus() string {
|
|
return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringHeartbeatSnapshotMetricsCollector{}))
|
|
}
|
|
|
|
func MonitoringHeartbeatSnapshotMetricsPrometheusHandler() http.Handler {
|
|
return monitoringPrometheusHandler(false, monitoringHeartbeatSnapshotMetricsCollector{})
|
|
}
|
|
|
|
func monitoringPrometheusHandler(includeRuntime bool, metricCollectors ...prometheus.Collector) http.Handler {
|
|
return promhttp.HandlerFor(monitoringPrometheusRegistry(includeRuntime, metricCollectors...), promhttp.HandlerOpts{})
|
|
}
|
|
|
|
func monitoringPrometheusRegistry(includeRuntime bool, metricCollectors ...prometheus.Collector) *prometheus.Registry {
|
|
registry := prometheus.NewRegistry()
|
|
if includeRuntime {
|
|
registry.MustRegister(
|
|
promcollectors.NewGoCollector(),
|
|
promcollectors.NewProcessCollector(promcollectors.ProcessCollectorOpts{}),
|
|
)
|
|
}
|
|
for _, collector := range metricCollectors {
|
|
registry.MustRegister(collector)
|
|
}
|
|
return registry
|
|
}
|
|
|
|
func renderMonitoringPrometheus(gatherer prometheus.Gatherer) string {
|
|
metricFamilies, err := gatherer.Gather()
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
var b bytes.Buffer
|
|
encoder := expfmt.NewEncoder(&b, expfmt.NewFormat(expfmt.TypeTextPlain))
|
|
for _, metricFamily := range metricFamilies {
|
|
if err := encoder.Encode(metricFamily); err != nil {
|
|
return ""
|
|
}
|
|
}
|
|
if closer, ok := encoder.(expfmt.Closer); ok {
|
|
_ = closer.Close()
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
type monitoringDailyTaskMetricsCollector struct{}
|
|
|
|
func (monitoringDailyTaskMetricsCollector) Describe(chan<- *prometheus.Desc) {}
|
|
|
|
func (monitoringDailyTaskMetricsCollector) Collect(ch chan<- prometheus.Metric) {
|
|
snapshot := MonitoringDailyTaskMetricsSnapshotValue()
|
|
monitoringCounter(ch, "monitoring_daily_task_worker_runs_total", "Total daily monitoring worker runs by terminal status.", float64(snapshot.RunCompletedTotal), []string{"status"}, "completed")
|
|
monitoringCounter(ch, "monitoring_daily_task_worker_runs_total", "Total daily monitoring worker runs by terminal status.", float64(snapshot.RunFailedTotal), []string{"status"}, "failed")
|
|
monitoringCounter(ch, "monitoring_daily_task_worker_partial_failure_runs_total", "Total completed daily monitoring worker runs with at least one plan or brand failure.", float64(snapshot.RunPartialFailureTotal), nil)
|
|
monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.PlanFailureTotal), []string{"scope"}, "plan")
|
|
monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.BrandFailureTotal), []string{"scope"}, "brand")
|
|
monitoringCounter(ch, "monitoring_daily_task_generation_skipped_plans_total", "Total daily monitoring plans skipped before generation.", float64(snapshot.SkippedPlanTotal), nil)
|
|
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.PlannedTaskTotal), []string{"stage"}, "planned")
|
|
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DueTaskTotal), []string{"stage"}, "due")
|
|
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.CreatedTaskTotal), []string{"stage"}, "created")
|
|
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DesktopTaskTotal), []string{"stage"}, "desktop")
|
|
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DeferredTaskTotal), []string{"stage"}, "deferred")
|
|
monitoringGauge(ch, "monitoring_daily_task_worker_last_run_timestamp_seconds", "Unix timestamp of the last daily monitoring worker run completion.", monitoringDailyTimeSeconds(snapshot.LastRunAt), nil)
|
|
monitoringGauge(ch, "monitoring_daily_task_worker_last_success_timestamp_seconds", "Unix timestamp of the last completed daily monitoring worker run.", monitoringDailyTimeSeconds(snapshot.LastSuccessAt), nil)
|
|
monitoringGauge(ch, "monitoring_daily_task_worker_last_error_timestamp_seconds", "Unix timestamp of the last failed daily monitoring worker run.", monitoringDailyTimeSeconds(snapshot.LastErrorAt), nil)
|
|
monitoringGauge(ch, "monitoring_daily_task_worker_last_duration_seconds", "Duration of the last daily monitoring worker run.", snapshot.LastDurationSeconds, nil)
|
|
monitoringGauge(ch, "monitoring_daily_task_worker_last_scope_count", "Tenant and brand counts observed in the last daily monitoring worker run.", float64(snapshot.LastTenantCount), []string{"scope"}, "tenant")
|
|
monitoringGauge(ch, "monitoring_daily_task_worker_last_scope_count", "Tenant and brand counts observed in the last daily monitoring worker run.", float64(snapshot.LastBrandCount), []string{"scope"}, "brand")
|
|
if snapshot.LastBusinessDate != "" {
|
|
monitoringGauge(ch, "monitoring_daily_task_worker_last_business_date_info", "Business date processed by the last daily monitoring worker run.", 1, []string{"business_date"}, snapshot.LastBusinessDate)
|
|
}
|
|
if snapshot.LastErrorCode != "" {
|
|
monitoringGauge(ch, "monitoring_daily_task_worker_last_error_info", "Last fatal daily monitoring worker error code.", 1, []string{"code"}, snapshot.LastErrorCode)
|
|
}
|
|
}
|
|
|
|
type monitoringHeartbeatPrimaryLeaseMetricsCollector struct{}
|
|
|
|
func (monitoringHeartbeatPrimaryLeaseMetricsCollector) Describe(chan<- *prometheus.Desc) {}
|
|
|
|
func (monitoringHeartbeatPrimaryLeaseMetricsCollector) Collect(ch chan<- prometheus.Metric) {
|
|
snapshot := MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue()
|
|
help := "Total monitoring heartbeat primary lease resolutions by outcome."
|
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.PrimaryHitTotal), []string{"outcome"}, "primary_hit")
|
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.NonPrimaryLiveTotal), []string{"outcome"}, "non_primary_live")
|
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.InsertedTotal), []string{"outcome"}, "inserted")
|
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.RenewedTotal), []string{"outcome"}, "renewed")
|
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ClaimedExpiredTotal), []string{"outcome"}, "claimed_expired")
|
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ClaimedUnavailableTotal), []string{"outcome"}, "claimed_unavailable")
|
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ContentionTotal), []string{"outcome"}, "contention")
|
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ErrorTotal), []string{"outcome"}, "error")
|
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBInsertTotal), []string{"operation"}, "insert")
|
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBRenewTotal), []string{"operation"}, "renew")
|
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBClaimTotal), []string{"operation"}, "claim")
|
|
monitoringSummary(ch, "monitoring_heartbeat_primary_lease_duration_seconds", "Monitoring heartbeat primary lease resolution duration summary.", snapshot.DurationCount, snapshot.DurationSumSeconds, nil)
|
|
monitoringGauge(ch, "monitoring_heartbeat_primary_lease_duration_seconds_last", "Last monitoring heartbeat primary lease resolution duration in seconds.", snapshot.LastDurationSeconds, nil)
|
|
if snapshot.LastOutcome != "" {
|
|
monitoringGauge(ch, "monitoring_heartbeat_primary_lease_last_outcome_info", "Last monitoring heartbeat primary lease resolution outcome.", 1, []string{"outcome"}, snapshot.LastOutcome)
|
|
}
|
|
}
|
|
|
|
type monitoringHeartbeatSnapshotMetricsCollector struct{}
|
|
|
|
func (monitoringHeartbeatSnapshotMetricsCollector) Describe(chan<- *prometheus.Desc) {}
|
|
|
|
func (monitoringHeartbeatSnapshotMetricsCollector) Collect(ch chan<- prometheus.Metric) {
|
|
snapshot := MonitoringHeartbeatSnapshotMetricsSnapshotValue()
|
|
monitoringCounter(ch, "monitoring_heartbeat_snapshot_platform_reports_total", "Total monitoring heartbeat platform reports received.", float64(snapshot.PlatformReportedTotal), nil)
|
|
monitoringCounter(ch, "monitoring_heartbeat_snapshot_transactions_total", "Total monitoring heartbeat snapshot transactions opened.", float64(snapshot.TransactionOpenedTotal), nil)
|
|
monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotInsertedTotal), []string{"operation"}, "insert")
|
|
monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotUpdatedTotal), []string{"operation"}, "update")
|
|
monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotRefreshedTotal), []string{"operation"}, "refresh")
|
|
monitoringCounter(ch, "monitoring_heartbeat_snapshot_skips_total", "Total monitoring heartbeat access snapshot reports skipped because state was fresh and unchanged.", float64(snapshot.SnapshotSkippedTotal), nil)
|
|
monitoringCounter(ch, "monitoring_heartbeat_snapshot_reconciled_tasks_total", "Total pending monitoring tasks skipped by heartbeat access reconciliation.", float64(snapshot.ReconciledSkippedTaskTotal), nil)
|
|
monitoringCounter(ch, "monitoring_heartbeat_snapshot_projection_rebuilds_total", "Total monitoring daily projection rebuilds triggered by heartbeat snapshots.", float64(snapshot.ProjectionRebuildTotal), nil)
|
|
monitoringCounter(ch, "monitoring_heartbeat_snapshot_errors_total", "Total monitoring heartbeat snapshot errors.", float64(snapshot.ErrorTotal), nil)
|
|
monitoringSummary(ch, "monitoring_heartbeat_snapshot_duration_seconds", "Monitoring heartbeat snapshot processing duration summary.", snapshot.DurationCount, snapshot.DurationSumSeconds, nil)
|
|
monitoringGauge(ch, "monitoring_heartbeat_snapshot_duration_seconds_last", "Last monitoring heartbeat snapshot processing duration in seconds.", snapshot.LastDurationSeconds, nil)
|
|
if snapshot.LastErrorCode != "" {
|
|
monitoringGauge(ch, "monitoring_heartbeat_snapshot_last_error_info", "Last monitoring heartbeat snapshot error code.", 1, []string{"code"}, snapshot.LastErrorCode)
|
|
}
|
|
}
|
|
|
|
func monitoringCounter(ch chan<- prometheus.Metric, name, help string, value float64, labelNames []string, labelValues ...string) {
|
|
monitoringMetric(ch, name, help, prometheus.CounterValue, value, labelNames, labelValues...)
|
|
}
|
|
|
|
func monitoringGauge(ch chan<- prometheus.Metric, name, help string, value float64, labelNames []string, labelValues ...string) {
|
|
monitoringMetric(ch, name, help, prometheus.GaugeValue, value, labelNames, labelValues...)
|
|
}
|
|
|
|
func monitoringMetric(ch chan<- prometheus.Metric, name, help string, valueType prometheus.ValueType, value float64, labelNames []string, labelValues ...string) {
|
|
ch <- prometheus.MustNewConstMetric(
|
|
prometheus.NewDesc(name, help, labelNames, nil),
|
|
valueType,
|
|
value,
|
|
monitoringPrometheusLabelValues(labelValues)...,
|
|
)
|
|
}
|
|
|
|
func monitoringSummary(ch chan<- prometheus.Metric, name, help string, count int64, sum float64, labelNames []string, labelValues ...string) {
|
|
if count < 0 {
|
|
count = 0
|
|
}
|
|
ch <- prometheus.MustNewConstSummary(
|
|
prometheus.NewDesc(name, help, labelNames, nil),
|
|
uint64(count),
|
|
sum,
|
|
nil,
|
|
monitoringPrometheusLabelValues(labelValues)...,
|
|
)
|
|
}
|
|
|
|
func monitoringPrometheusLabelValues(values []string) []string {
|
|
if len(values) == 0 {
|
|
return values
|
|
}
|
|
result := make([]string, len(values))
|
|
for idx, value := range values {
|
|
result[idx] = strings.ReplaceAll(value, "\r", `\r`)
|
|
}
|
|
return result
|
|
}
|