ff86933c24
Adds a MonitoringDailyTaskWorker that materializes and dispatches daily collect tasks from active subscription/desktop-client state, with per-run atomic metrics and a Prometheus collector. Dashboard and question-detail APIs now surface a platform_authorization_status so the UI can distinguish no-desktop-client, no-authorized-platforms, and authorized states; quota/access-state lookups resolve by request workspace instead of tenant. Hardens heartbeat: a desktop_client_primary_leases table gives per-(tenant, workspace) sticky primary selection, read-mostly lease resolution avoids a per-heartbeat transaction, and unchanged platform access reports skip snapshot upserts outside a 10-minute refresh window. Adds heartbeat primary/snapshot metrics exposed via token-protected tenant-api /api/internal/metrics endpoints plus Prometheus. Monitoring quota seed is removed from dev-seed since daily plans now derive from desktop bindings, and the projection uses the canonical six-platform catalog so platforms with zero samples still render. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
124 lines
4.9 KiB
Go
124 lines
4.9 KiB
Go
package app
|
|
|
|
import (
|
|
"sync/atomic"
|
|
"time"
|
|
)
|
|
|
|
type MonitoringHeartbeatSnapshotMetricsSnapshot struct {
|
|
PlatformReportedTotal int64 `json:"platform_reported_total"`
|
|
TransactionOpenedTotal int64 `json:"transaction_opened_total"`
|
|
SnapshotInsertedTotal int64 `json:"snapshot_inserted_total"`
|
|
SnapshotUpdatedTotal int64 `json:"snapshot_updated_total"`
|
|
SnapshotRefreshedTotal int64 `json:"snapshot_refreshed_total"`
|
|
SnapshotSkippedTotal int64 `json:"snapshot_skipped_total"`
|
|
ReconciledSkippedTaskTotal int64 `json:"reconciled_skipped_task_total"`
|
|
ProjectionRebuildTotal int64 `json:"projection_rebuild_total"`
|
|
ErrorTotal int64 `json:"error_total"`
|
|
DurationCount int64 `json:"duration_count"`
|
|
DurationSumSeconds float64 `json:"duration_sum_seconds"`
|
|
LastDurationSeconds float64 `json:"last_duration_seconds"`
|
|
LastErrorCode string `json:"last_error_code,omitempty"`
|
|
LastErrorMessage string `json:"last_error_message,omitempty"`
|
|
}
|
|
|
|
type monitoringHeartbeatSnapshotMetricsEvent struct {
|
|
Active bool
|
|
PlatformCount int
|
|
TransactionOpened bool
|
|
SnapshotInsertedCount int64
|
|
SnapshotUpdatedCount int64
|
|
SnapshotRefreshedCount int64
|
|
SnapshotSkippedCount int64
|
|
ReconciledSkippedTaskCount int64
|
|
ProjectionRebuildCount int64
|
|
Duration time.Duration
|
|
Err error
|
|
}
|
|
|
|
type monitoringHeartbeatSnapshotMetricsRecorder struct {
|
|
platformReportedTotal atomic.Int64
|
|
transactionOpenedTotal atomic.Int64
|
|
snapshotInsertedTotal atomic.Int64
|
|
snapshotUpdatedTotal atomic.Int64
|
|
snapshotRefreshedTotal atomic.Int64
|
|
snapshotSkippedTotal atomic.Int64
|
|
reconciledSkippedTaskTotal atomic.Int64
|
|
projectionRebuildTotal atomic.Int64
|
|
errorTotal atomic.Int64
|
|
durationCount atomic.Int64
|
|
durationSumNanos atomic.Int64
|
|
lastDurationNanos atomic.Int64
|
|
lastErrorCode atomic.Value
|
|
lastErrorMessage atomic.Value
|
|
}
|
|
|
|
var monitoringHeartbeatSnapshotMetrics = newMonitoringHeartbeatSnapshotMetricsRecorder()
|
|
|
|
func newMonitoringHeartbeatSnapshotMetricsRecorder() *monitoringHeartbeatSnapshotMetricsRecorder {
|
|
return &monitoringHeartbeatSnapshotMetricsRecorder{}
|
|
}
|
|
|
|
func recordMonitoringHeartbeatSnapshotMetrics(event monitoringHeartbeatSnapshotMetricsEvent) {
|
|
monitoringHeartbeatSnapshotMetrics.record(event)
|
|
}
|
|
|
|
func MonitoringHeartbeatSnapshotMetricsSnapshotValue() MonitoringHeartbeatSnapshotMetricsSnapshot {
|
|
return monitoringHeartbeatSnapshotMetrics.snapshot()
|
|
}
|
|
|
|
func resetMonitoringHeartbeatSnapshotMetricsForTest() {
|
|
monitoringHeartbeatSnapshotMetrics = newMonitoringHeartbeatSnapshotMetricsRecorder()
|
|
}
|
|
|
|
func (r *monitoringHeartbeatSnapshotMetricsRecorder) record(event monitoringHeartbeatSnapshotMetricsEvent) {
|
|
if r == nil {
|
|
return
|
|
}
|
|
if event.Duration < 0 {
|
|
event.Duration = 0
|
|
}
|
|
r.platformReportedTotal.Add(int64(event.PlatformCount))
|
|
if event.TransactionOpened {
|
|
r.transactionOpenedTotal.Add(1)
|
|
}
|
|
r.snapshotInsertedTotal.Add(event.SnapshotInsertedCount)
|
|
r.snapshotUpdatedTotal.Add(event.SnapshotUpdatedCount)
|
|
r.snapshotRefreshedTotal.Add(event.SnapshotRefreshedCount)
|
|
r.snapshotSkippedTotal.Add(event.SnapshotSkippedCount)
|
|
r.reconciledSkippedTaskTotal.Add(event.ReconciledSkippedTaskCount)
|
|
r.projectionRebuildTotal.Add(event.ProjectionRebuildCount)
|
|
r.durationCount.Add(1)
|
|
r.durationSumNanos.Add(event.Duration.Nanoseconds())
|
|
r.lastDurationNanos.Store(event.Duration.Nanoseconds())
|
|
|
|
if event.Err != nil {
|
|
code, message := monitoringDailyMetricError(event.Err)
|
|
r.errorTotal.Add(1)
|
|
r.lastErrorCode.Store(code)
|
|
r.lastErrorMessage.Store(message)
|
|
}
|
|
}
|
|
|
|
func (r *monitoringHeartbeatSnapshotMetricsRecorder) snapshot() MonitoringHeartbeatSnapshotMetricsSnapshot {
|
|
if r == nil {
|
|
return MonitoringHeartbeatSnapshotMetricsSnapshot{}
|
|
}
|
|
return MonitoringHeartbeatSnapshotMetricsSnapshot{
|
|
PlatformReportedTotal: r.platformReportedTotal.Load(),
|
|
TransactionOpenedTotal: r.transactionOpenedTotal.Load(),
|
|
SnapshotInsertedTotal: r.snapshotInsertedTotal.Load(),
|
|
SnapshotUpdatedTotal: r.snapshotUpdatedTotal.Load(),
|
|
SnapshotRefreshedTotal: r.snapshotRefreshedTotal.Load(),
|
|
SnapshotSkippedTotal: r.snapshotSkippedTotal.Load(),
|
|
ReconciledSkippedTaskTotal: r.reconciledSkippedTaskTotal.Load(),
|
|
ProjectionRebuildTotal: r.projectionRebuildTotal.Load(),
|
|
ErrorTotal: r.errorTotal.Load(),
|
|
DurationCount: r.durationCount.Load(),
|
|
DurationSumSeconds: float64(r.durationSumNanos.Load()) / float64(time.Second),
|
|
LastDurationSeconds: float64(r.lastDurationNanos.Load()) / float64(time.Second),
|
|
LastErrorCode: monitoringDailyLoadString(&r.lastErrorCode),
|
|
LastErrorMessage: monitoringDailyLoadString(&r.lastErrorMessage),
|
|
}
|
|
}
|