Files
geo/server/internal/tenant/app/monitoring_heartbeat_snapshot_metrics.go
T
root ff86933c24 feat(monitoring): generate daily tasks and optimize heartbeat writes
Adds a MonitoringDailyTaskWorker that materializes and dispatches
daily collect tasks from active subscription/desktop-client state,
with per-run atomic metrics and a Prometheus collector. Dashboard
and question-detail APIs now surface a platform_authorization_status
so the UI can distinguish no-desktop-client, no-authorized-platforms,
and authorized states; quota/access-state lookups resolve by request
workspace instead of tenant.

Hardens heartbeat: a desktop_client_primary_leases table gives
per-(tenant, workspace) sticky primary selection, read-mostly lease
resolution avoids a per-heartbeat transaction, and unchanged platform
access reports skip snapshot upserts outside a 10-minute refresh
window. Adds heartbeat primary/snapshot metrics exposed via
token-protected tenant-api /api/internal/metrics endpoints plus
Prometheus. Monitoring quota seed is removed from dev-seed since
daily plans now derive from desktop bindings, and the projection
uses the canonical six-platform catalog so platforms with zero
samples still render.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-24 22:19:57 +08:00

124 lines
4.9 KiB
Go

package app
import (
"sync/atomic"
"time"
)
type MonitoringHeartbeatSnapshotMetricsSnapshot struct {
PlatformReportedTotal int64 `json:"platform_reported_total"`
TransactionOpenedTotal int64 `json:"transaction_opened_total"`
SnapshotInsertedTotal int64 `json:"snapshot_inserted_total"`
SnapshotUpdatedTotal int64 `json:"snapshot_updated_total"`
SnapshotRefreshedTotal int64 `json:"snapshot_refreshed_total"`
SnapshotSkippedTotal int64 `json:"snapshot_skipped_total"`
ReconciledSkippedTaskTotal int64 `json:"reconciled_skipped_task_total"`
ProjectionRebuildTotal int64 `json:"projection_rebuild_total"`
ErrorTotal int64 `json:"error_total"`
DurationCount int64 `json:"duration_count"`
DurationSumSeconds float64 `json:"duration_sum_seconds"`
LastDurationSeconds float64 `json:"last_duration_seconds"`
LastErrorCode string `json:"last_error_code,omitempty"`
LastErrorMessage string `json:"last_error_message,omitempty"`
}
type monitoringHeartbeatSnapshotMetricsEvent struct {
Active bool
PlatformCount int
TransactionOpened bool
SnapshotInsertedCount int64
SnapshotUpdatedCount int64
SnapshotRefreshedCount int64
SnapshotSkippedCount int64
ReconciledSkippedTaskCount int64
ProjectionRebuildCount int64
Duration time.Duration
Err error
}
type monitoringHeartbeatSnapshotMetricsRecorder struct {
platformReportedTotal atomic.Int64
transactionOpenedTotal atomic.Int64
snapshotInsertedTotal atomic.Int64
snapshotUpdatedTotal atomic.Int64
snapshotRefreshedTotal atomic.Int64
snapshotSkippedTotal atomic.Int64
reconciledSkippedTaskTotal atomic.Int64
projectionRebuildTotal atomic.Int64
errorTotal atomic.Int64
durationCount atomic.Int64
durationSumNanos atomic.Int64
lastDurationNanos atomic.Int64
lastErrorCode atomic.Value
lastErrorMessage atomic.Value
}
var monitoringHeartbeatSnapshotMetrics = newMonitoringHeartbeatSnapshotMetricsRecorder()
func newMonitoringHeartbeatSnapshotMetricsRecorder() *monitoringHeartbeatSnapshotMetricsRecorder {
return &monitoringHeartbeatSnapshotMetricsRecorder{}
}
func recordMonitoringHeartbeatSnapshotMetrics(event monitoringHeartbeatSnapshotMetricsEvent) {
monitoringHeartbeatSnapshotMetrics.record(event)
}
func MonitoringHeartbeatSnapshotMetricsSnapshotValue() MonitoringHeartbeatSnapshotMetricsSnapshot {
return monitoringHeartbeatSnapshotMetrics.snapshot()
}
func resetMonitoringHeartbeatSnapshotMetricsForTest() {
monitoringHeartbeatSnapshotMetrics = newMonitoringHeartbeatSnapshotMetricsRecorder()
}
func (r *monitoringHeartbeatSnapshotMetricsRecorder) record(event monitoringHeartbeatSnapshotMetricsEvent) {
if r == nil {
return
}
if event.Duration < 0 {
event.Duration = 0
}
r.platformReportedTotal.Add(int64(event.PlatformCount))
if event.TransactionOpened {
r.transactionOpenedTotal.Add(1)
}
r.snapshotInsertedTotal.Add(event.SnapshotInsertedCount)
r.snapshotUpdatedTotal.Add(event.SnapshotUpdatedCount)
r.snapshotRefreshedTotal.Add(event.SnapshotRefreshedCount)
r.snapshotSkippedTotal.Add(event.SnapshotSkippedCount)
r.reconciledSkippedTaskTotal.Add(event.ReconciledSkippedTaskCount)
r.projectionRebuildTotal.Add(event.ProjectionRebuildCount)
r.durationCount.Add(1)
r.durationSumNanos.Add(event.Duration.Nanoseconds())
r.lastDurationNanos.Store(event.Duration.Nanoseconds())
if event.Err != nil {
code, message := monitoringDailyMetricError(event.Err)
r.errorTotal.Add(1)
r.lastErrorCode.Store(code)
r.lastErrorMessage.Store(message)
}
}
func (r *monitoringHeartbeatSnapshotMetricsRecorder) snapshot() MonitoringHeartbeatSnapshotMetricsSnapshot {
if r == nil {
return MonitoringHeartbeatSnapshotMetricsSnapshot{}
}
return MonitoringHeartbeatSnapshotMetricsSnapshot{
PlatformReportedTotal: r.platformReportedTotal.Load(),
TransactionOpenedTotal: r.transactionOpenedTotal.Load(),
SnapshotInsertedTotal: r.snapshotInsertedTotal.Load(),
SnapshotUpdatedTotal: r.snapshotUpdatedTotal.Load(),
SnapshotRefreshedTotal: r.snapshotRefreshedTotal.Load(),
SnapshotSkippedTotal: r.snapshotSkippedTotal.Load(),
ReconciledSkippedTaskTotal: r.reconciledSkippedTaskTotal.Load(),
ProjectionRebuildTotal: r.projectionRebuildTotal.Load(),
ErrorTotal: r.errorTotal.Load(),
DurationCount: r.durationCount.Load(),
DurationSumSeconds: float64(r.durationSumNanos.Load()) / float64(time.Second),
LastDurationSeconds: float64(r.lastDurationNanos.Load()) / float64(time.Second),
LastErrorCode: monitoringDailyLoadString(&r.lastErrorCode),
LastErrorMessage: monitoringDailyLoadString(&r.lastErrorMessage),
}
}