feat(monitoring): generate daily tasks and optimize heartbeat writes
Adds a MonitoringDailyTaskWorker that materializes and dispatches daily collect tasks from active subscription/desktop-client state, with per-run atomic metrics and a Prometheus collector. Dashboard and question-detail APIs now surface a platform_authorization_status so the UI can distinguish no-desktop-client, no-authorized-platforms, and authorized states; quota/access-state lookups resolve by request workspace instead of tenant. Hardens heartbeat: a desktop_client_primary_leases table gives per-(tenant, workspace) sticky primary selection, read-mostly lease resolution avoids a per-heartbeat transaction, and unchanged platform access reports skip snapshot upserts outside a 10-minute refresh window. Adds heartbeat primary/snapshot metrics exposed via token-protected tenant-api /api/internal/metrics endpoints plus Prometheus. Monitoring quota seed is removed from dev-seed since daily plans now derive from desktop bindings, and the projection uses the canonical six-platform catalog so platforms with zero samples still render. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,195 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
promcollectors "github.com/prometheus/client_golang/prometheus/collectors"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"github.com/prometheus/common/expfmt"
|
||||
)
|
||||
|
||||
func MonitoringSchedulerMetricsPrometheusHandler() http.Handler {
|
||||
return monitoringPrometheusHandler(true, monitoringDailyTaskMetricsCollector{})
|
||||
}
|
||||
|
||||
func MonitoringDailyTaskMetricsPrometheus() string {
|
||||
return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringDailyTaskMetricsCollector{}))
|
||||
}
|
||||
|
||||
func MonitoringDailyTaskMetricsPrometheusHandler() http.Handler {
|
||||
return monitoringPrometheusHandler(false, monitoringDailyTaskMetricsCollector{})
|
||||
}
|
||||
|
||||
func MonitoringHeartbeatPrimaryLeaseMetricsPrometheus() string {
|
||||
return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringHeartbeatPrimaryLeaseMetricsCollector{}))
|
||||
}
|
||||
|
||||
func MonitoringHeartbeatPrimaryLeaseMetricsPrometheusHandler() http.Handler {
|
||||
return monitoringPrometheusHandler(false, monitoringHeartbeatPrimaryLeaseMetricsCollector{})
|
||||
}
|
||||
|
||||
func MonitoringHeartbeatSnapshotMetricsPrometheus() string {
|
||||
return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringHeartbeatSnapshotMetricsCollector{}))
|
||||
}
|
||||
|
||||
func MonitoringHeartbeatSnapshotMetricsPrometheusHandler() http.Handler {
|
||||
return monitoringPrometheusHandler(false, monitoringHeartbeatSnapshotMetricsCollector{})
|
||||
}
|
||||
|
||||
func monitoringPrometheusHandler(includeRuntime bool, metricCollectors ...prometheus.Collector) http.Handler {
|
||||
return promhttp.HandlerFor(monitoringPrometheusRegistry(includeRuntime, metricCollectors...), promhttp.HandlerOpts{})
|
||||
}
|
||||
|
||||
func monitoringPrometheusRegistry(includeRuntime bool, metricCollectors ...prometheus.Collector) *prometheus.Registry {
|
||||
registry := prometheus.NewRegistry()
|
||||
if includeRuntime {
|
||||
registry.MustRegister(
|
||||
promcollectors.NewGoCollector(),
|
||||
promcollectors.NewProcessCollector(promcollectors.ProcessCollectorOpts{}),
|
||||
)
|
||||
}
|
||||
for _, collector := range metricCollectors {
|
||||
registry.MustRegister(collector)
|
||||
}
|
||||
return registry
|
||||
}
|
||||
|
||||
func renderMonitoringPrometheus(gatherer prometheus.Gatherer) string {
|
||||
metricFamilies, err := gatherer.Gather()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
var b bytes.Buffer
|
||||
encoder := expfmt.NewEncoder(&b, expfmt.NewFormat(expfmt.TypeTextPlain))
|
||||
for _, metricFamily := range metricFamilies {
|
||||
if err := encoder.Encode(metricFamily); err != nil {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
if closer, ok := encoder.(expfmt.Closer); ok {
|
||||
_ = closer.Close()
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
type monitoringDailyTaskMetricsCollector struct{}
|
||||
|
||||
func (monitoringDailyTaskMetricsCollector) Describe(chan<- *prometheus.Desc) {}
|
||||
|
||||
func (monitoringDailyTaskMetricsCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
snapshot := MonitoringDailyTaskMetricsSnapshotValue()
|
||||
monitoringCounter(ch, "monitoring_daily_task_worker_runs_total", "Total daily monitoring worker runs by terminal status.", float64(snapshot.RunCompletedTotal), []string{"status"}, "completed")
|
||||
monitoringCounter(ch, "monitoring_daily_task_worker_runs_total", "Total daily monitoring worker runs by terminal status.", float64(snapshot.RunFailedTotal), []string{"status"}, "failed")
|
||||
monitoringCounter(ch, "monitoring_daily_task_worker_partial_failure_runs_total", "Total completed daily monitoring worker runs with at least one plan or brand failure.", float64(snapshot.RunPartialFailureTotal), nil)
|
||||
monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.PlanFailureTotal), []string{"scope"}, "plan")
|
||||
monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.BrandFailureTotal), []string{"scope"}, "brand")
|
||||
monitoringCounter(ch, "monitoring_daily_task_generation_skipped_plans_total", "Total daily monitoring plans skipped before generation.", float64(snapshot.SkippedPlanTotal), nil)
|
||||
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.PlannedTaskTotal), []string{"stage"}, "planned")
|
||||
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DueTaskTotal), []string{"stage"}, "due")
|
||||
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.CreatedTaskTotal), []string{"stage"}, "created")
|
||||
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DesktopTaskTotal), []string{"stage"}, "desktop")
|
||||
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DeferredTaskTotal), []string{"stage"}, "deferred")
|
||||
monitoringGauge(ch, "monitoring_daily_task_worker_last_run_timestamp_seconds", "Unix timestamp of the last daily monitoring worker run completion.", monitoringDailyTimeSeconds(snapshot.LastRunAt), nil)
|
||||
monitoringGauge(ch, "monitoring_daily_task_worker_last_success_timestamp_seconds", "Unix timestamp of the last completed daily monitoring worker run.", monitoringDailyTimeSeconds(snapshot.LastSuccessAt), nil)
|
||||
monitoringGauge(ch, "monitoring_daily_task_worker_last_error_timestamp_seconds", "Unix timestamp of the last failed daily monitoring worker run.", monitoringDailyTimeSeconds(snapshot.LastErrorAt), nil)
|
||||
monitoringGauge(ch, "monitoring_daily_task_worker_last_duration_seconds", "Duration of the last daily monitoring worker run.", snapshot.LastDurationSeconds, nil)
|
||||
monitoringGauge(ch, "monitoring_daily_task_worker_last_scope_count", "Tenant and brand counts observed in the last daily monitoring worker run.", float64(snapshot.LastTenantCount), []string{"scope"}, "tenant")
|
||||
monitoringGauge(ch, "monitoring_daily_task_worker_last_scope_count", "Tenant and brand counts observed in the last daily monitoring worker run.", float64(snapshot.LastBrandCount), []string{"scope"}, "brand")
|
||||
if snapshot.LastBusinessDate != "" {
|
||||
monitoringGauge(ch, "monitoring_daily_task_worker_last_business_date_info", "Business date processed by the last daily monitoring worker run.", 1, []string{"business_date"}, snapshot.LastBusinessDate)
|
||||
}
|
||||
if snapshot.LastErrorCode != "" {
|
||||
monitoringGauge(ch, "monitoring_daily_task_worker_last_error_info", "Last fatal daily monitoring worker error code.", 1, []string{"code"}, snapshot.LastErrorCode)
|
||||
}
|
||||
}
|
||||
|
||||
type monitoringHeartbeatPrimaryLeaseMetricsCollector struct{}
|
||||
|
||||
func (monitoringHeartbeatPrimaryLeaseMetricsCollector) Describe(chan<- *prometheus.Desc) {}
|
||||
|
||||
func (monitoringHeartbeatPrimaryLeaseMetricsCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
snapshot := MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue()
|
||||
help := "Total monitoring heartbeat primary lease resolutions by outcome."
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.PrimaryHitTotal), []string{"outcome"}, "primary_hit")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.NonPrimaryLiveTotal), []string{"outcome"}, "non_primary_live")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.InsertedTotal), []string{"outcome"}, "inserted")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.RenewedTotal), []string{"outcome"}, "renewed")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ClaimedExpiredTotal), []string{"outcome"}, "claimed_expired")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ClaimedUnavailableTotal), []string{"outcome"}, "claimed_unavailable")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ContentionTotal), []string{"outcome"}, "contention")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ErrorTotal), []string{"outcome"}, "error")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBInsertTotal), []string{"operation"}, "insert")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBRenewTotal), []string{"operation"}, "renew")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBClaimTotal), []string{"operation"}, "claim")
|
||||
monitoringSummary(ch, "monitoring_heartbeat_primary_lease_duration_seconds", "Monitoring heartbeat primary lease resolution duration summary.", snapshot.DurationCount, snapshot.DurationSumSeconds, nil)
|
||||
monitoringGauge(ch, "monitoring_heartbeat_primary_lease_duration_seconds_last", "Last monitoring heartbeat primary lease resolution duration in seconds.", snapshot.LastDurationSeconds, nil)
|
||||
if snapshot.LastOutcome != "" {
|
||||
monitoringGauge(ch, "monitoring_heartbeat_primary_lease_last_outcome_info", "Last monitoring heartbeat primary lease resolution outcome.", 1, []string{"outcome"}, snapshot.LastOutcome)
|
||||
}
|
||||
}
|
||||
|
||||
type monitoringHeartbeatSnapshotMetricsCollector struct{}
|
||||
|
||||
func (monitoringHeartbeatSnapshotMetricsCollector) Describe(chan<- *prometheus.Desc) {}
|
||||
|
||||
func (monitoringHeartbeatSnapshotMetricsCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
snapshot := MonitoringHeartbeatSnapshotMetricsSnapshotValue()
|
||||
monitoringCounter(ch, "monitoring_heartbeat_snapshot_platform_reports_total", "Total monitoring heartbeat platform reports received.", float64(snapshot.PlatformReportedTotal), nil)
|
||||
monitoringCounter(ch, "monitoring_heartbeat_snapshot_transactions_total", "Total monitoring heartbeat snapshot transactions opened.", float64(snapshot.TransactionOpenedTotal), nil)
|
||||
monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotInsertedTotal), []string{"operation"}, "insert")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotUpdatedTotal), []string{"operation"}, "update")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotRefreshedTotal), []string{"operation"}, "refresh")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_snapshot_skips_total", "Total monitoring heartbeat access snapshot reports skipped because state was fresh and unchanged.", float64(snapshot.SnapshotSkippedTotal), nil)
|
||||
monitoringCounter(ch, "monitoring_heartbeat_snapshot_reconciled_tasks_total", "Total pending monitoring tasks skipped by heartbeat access reconciliation.", float64(snapshot.ReconciledSkippedTaskTotal), nil)
|
||||
monitoringCounter(ch, "monitoring_heartbeat_snapshot_projection_rebuilds_total", "Total monitoring daily projection rebuilds triggered by heartbeat snapshots.", float64(snapshot.ProjectionRebuildTotal), nil)
|
||||
monitoringCounter(ch, "monitoring_heartbeat_snapshot_errors_total", "Total monitoring heartbeat snapshot errors.", float64(snapshot.ErrorTotal), nil)
|
||||
monitoringSummary(ch, "monitoring_heartbeat_snapshot_duration_seconds", "Monitoring heartbeat snapshot processing duration summary.", snapshot.DurationCount, snapshot.DurationSumSeconds, nil)
|
||||
monitoringGauge(ch, "monitoring_heartbeat_snapshot_duration_seconds_last", "Last monitoring heartbeat snapshot processing duration in seconds.", snapshot.LastDurationSeconds, nil)
|
||||
if snapshot.LastErrorCode != "" {
|
||||
monitoringGauge(ch, "monitoring_heartbeat_snapshot_last_error_info", "Last monitoring heartbeat snapshot error code.", 1, []string{"code"}, snapshot.LastErrorCode)
|
||||
}
|
||||
}
|
||||
|
||||
func monitoringCounter(ch chan<- prometheus.Metric, name, help string, value float64, labelNames []string, labelValues ...string) {
|
||||
monitoringMetric(ch, name, help, prometheus.CounterValue, value, labelNames, labelValues...)
|
||||
}
|
||||
|
||||
func monitoringGauge(ch chan<- prometheus.Metric, name, help string, value float64, labelNames []string, labelValues ...string) {
|
||||
monitoringMetric(ch, name, help, prometheus.GaugeValue, value, labelNames, labelValues...)
|
||||
}
|
||||
|
||||
func monitoringMetric(ch chan<- prometheus.Metric, name, help string, valueType prometheus.ValueType, value float64, labelNames []string, labelValues ...string) {
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(name, help, labelNames, nil),
|
||||
valueType,
|
||||
value,
|
||||
monitoringPrometheusLabelValues(labelValues)...,
|
||||
)
|
||||
}
|
||||
|
||||
func monitoringSummary(ch chan<- prometheus.Metric, name, help string, count int64, sum float64, labelNames []string, labelValues ...string) {
|
||||
if count < 0 {
|
||||
count = 0
|
||||
}
|
||||
ch <- prometheus.MustNewConstSummary(
|
||||
prometheus.NewDesc(name, help, labelNames, nil),
|
||||
uint64(count),
|
||||
sum,
|
||||
nil,
|
||||
monitoringPrometheusLabelValues(labelValues)...,
|
||||
)
|
||||
}
|
||||
|
||||
func monitoringPrometheusLabelValues(values []string) []string {
|
||||
if len(values) == 0 {
|
||||
return values
|
||||
}
|
||||
result := make([]string, len(values))
|
||||
for idx, value := range values {
|
||||
result[idx] = strings.ReplaceAll(value, "\r", `\r`)
|
||||
}
|
||||
return result
|
||||
}
|
||||
Reference in New Issue
Block a user