ff86933c24
Adds a MonitoringDailyTaskWorker that materializes and dispatches daily collect tasks from active subscription/desktop-client state, with per-run atomic metrics and a Prometheus collector. Dashboard and question-detail APIs now surface a platform_authorization_status so the UI can distinguish no-desktop-client, no-authorized-platforms, and authorized states; quota/access-state lookups resolve by request workspace instead of tenant. Hardens heartbeat: a desktop_client_primary_leases table gives per-(tenant, workspace) sticky primary selection, read-mostly lease resolution avoids a per-heartbeat transaction, and unchanged platform access reports skip snapshot upserts outside a 10-minute refresh window. Adds heartbeat primary/snapshot metrics exposed via token-protected tenant-api /api/internal/metrics endpoints plus Prometheus. Monitoring quota seed is removed from dev-seed since daily plans now derive from desktop bindings, and the projection uses the canonical six-platform catalog so platforms with zero samples still render. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
142 lines
4.7 KiB
Go
142 lines
4.7 KiB
Go
package app
|
|
|
|
import (
|
|
"sync/atomic"
|
|
"time"
|
|
)
|
|
|
|
type MonitoringHeartbeatPrimaryLeaseMetricsSnapshot struct {
|
|
PrimaryHitTotal int64 `json:"primary_hit_total"`
|
|
NonPrimaryLiveTotal int64 `json:"non_primary_live_total"`
|
|
InsertedTotal int64 `json:"inserted_total"`
|
|
RenewedTotal int64 `json:"renewed_total"`
|
|
ClaimedExpiredTotal int64 `json:"claimed_expired_total"`
|
|
ClaimedUnavailableTotal int64 `json:"claimed_unavailable_total"`
|
|
ContentionTotal int64 `json:"contention_total"`
|
|
ErrorTotal int64 `json:"error_total"`
|
|
DBInsertTotal int64 `json:"db_insert_total"`
|
|
DBRenewTotal int64 `json:"db_renew_total"`
|
|
DBClaimTotal int64 `json:"db_claim_total"`
|
|
DurationCount int64 `json:"duration_count"`
|
|
DurationSumSeconds float64 `json:"duration_sum_seconds"`
|
|
LastDurationSeconds float64 `json:"last_duration_seconds"`
|
|
LastOutcome string `json:"last_outcome,omitempty"`
|
|
}
|
|
|
|
type monitoringHeartbeatPrimaryLeaseMetricsRecorder struct {
|
|
primaryHitTotal atomic.Int64
|
|
nonPrimaryLiveTotal atomic.Int64
|
|
insertedTotal atomic.Int64
|
|
renewedTotal atomic.Int64
|
|
claimedExpiredTotal atomic.Int64
|
|
claimedUnavailableTotal atomic.Int64
|
|
contentionTotal atomic.Int64
|
|
errorTotal atomic.Int64
|
|
dbInsertTotal atomic.Int64
|
|
dbRenewTotal atomic.Int64
|
|
dbClaimTotal atomic.Int64
|
|
durationCount atomic.Int64
|
|
durationSumNanos atomic.Int64
|
|
lastDurationNanos atomic.Int64
|
|
lastOutcome atomic.Value
|
|
}
|
|
|
|
var monitoringHeartbeatPrimaryLeaseMetrics = newMonitoringHeartbeatPrimaryLeaseMetricsRecorder()
|
|
|
|
func newMonitoringHeartbeatPrimaryLeaseMetricsRecorder() *monitoringHeartbeatPrimaryLeaseMetricsRecorder {
|
|
return &monitoringHeartbeatPrimaryLeaseMetricsRecorder{}
|
|
}
|
|
|
|
func recordHeartbeatPrimaryLeaseMetric(outcome string, duration time.Duration) {
|
|
monitoringHeartbeatPrimaryLeaseMetrics.record(outcome, duration)
|
|
}
|
|
|
|
func recordHeartbeatPrimaryLeaseWriteMetric(operation string) {
|
|
monitoringHeartbeatPrimaryLeaseMetrics.recordWrite(operation)
|
|
}
|
|
|
|
func MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue() MonitoringHeartbeatPrimaryLeaseMetricsSnapshot {
|
|
return monitoringHeartbeatPrimaryLeaseMetrics.snapshot()
|
|
}
|
|
|
|
func resetMonitoringHeartbeatPrimaryLeaseMetricsForTest() {
|
|
monitoringHeartbeatPrimaryLeaseMetrics = newMonitoringHeartbeatPrimaryLeaseMetricsRecorder()
|
|
}
|
|
|
|
func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) record(outcome string, duration time.Duration) {
|
|
if r == nil {
|
|
return
|
|
}
|
|
if duration < 0 {
|
|
duration = 0
|
|
}
|
|
switch outcome {
|
|
case "primary_hit":
|
|
r.primaryHitTotal.Add(1)
|
|
case "non_primary_live":
|
|
r.nonPrimaryLiveTotal.Add(1)
|
|
case "inserted":
|
|
r.insertedTotal.Add(1)
|
|
case "renewed":
|
|
r.renewedTotal.Add(1)
|
|
case "claimed_expired":
|
|
r.claimedExpiredTotal.Add(1)
|
|
case "claimed_unavailable":
|
|
r.claimedUnavailableTotal.Add(1)
|
|
case "contention":
|
|
r.contentionTotal.Add(1)
|
|
case "error":
|
|
r.errorTotal.Add(1)
|
|
}
|
|
r.durationCount.Add(1)
|
|
r.durationSumNanos.Add(duration.Nanoseconds())
|
|
r.lastDurationNanos.Store(duration.Nanoseconds())
|
|
r.lastOutcome.Store(outcome)
|
|
}
|
|
|
|
func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) recordWrite(operation string) {
|
|
if r == nil {
|
|
return
|
|
}
|
|
switch operation {
|
|
case "insert":
|
|
r.dbInsertTotal.Add(1)
|
|
case "renew":
|
|
r.dbRenewTotal.Add(1)
|
|
case "claim":
|
|
r.dbClaimTotal.Add(1)
|
|
}
|
|
}
|
|
|
|
func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) snapshot() MonitoringHeartbeatPrimaryLeaseMetricsSnapshot {
|
|
if r == nil {
|
|
return MonitoringHeartbeatPrimaryLeaseMetricsSnapshot{}
|
|
}
|
|
return MonitoringHeartbeatPrimaryLeaseMetricsSnapshot{
|
|
PrimaryHitTotal: r.primaryHitTotal.Load(),
|
|
NonPrimaryLiveTotal: r.nonPrimaryLiveTotal.Load(),
|
|
InsertedTotal: r.insertedTotal.Load(),
|
|
RenewedTotal: r.renewedTotal.Load(),
|
|
ClaimedExpiredTotal: r.claimedExpiredTotal.Load(),
|
|
ClaimedUnavailableTotal: r.claimedUnavailableTotal.Load(),
|
|
ContentionTotal: r.contentionTotal.Load(),
|
|
ErrorTotal: r.errorTotal.Load(),
|
|
DBInsertTotal: r.dbInsertTotal.Load(),
|
|
DBRenewTotal: r.dbRenewTotal.Load(),
|
|
DBClaimTotal: r.dbClaimTotal.Load(),
|
|
DurationCount: r.durationCount.Load(),
|
|
DurationSumSeconds: float64(r.durationSumNanos.Load()) / float64(time.Second),
|
|
LastDurationSeconds: float64(r.lastDurationNanos.Load()) / float64(time.Second),
|
|
LastOutcome: heartbeatPrimaryMetricLoadString(&r.lastOutcome),
|
|
}
|
|
}
|
|
|
|
func heartbeatPrimaryMetricLoadString(value *atomic.Value) string {
|
|
if value == nil {
|
|
return ""
|
|
}
|
|
raw := value.Load()
|
|
result, _ := raw.(string)
|
|
return result
|
|
}
|