Files
geo/server/internal/tenant/app/monitoring_heartbeat_primary_metrics.go
T
root ff86933c24 feat(monitoring): generate daily tasks and optimize heartbeat writes
Adds a MonitoringDailyTaskWorker that materializes and dispatches
daily collect tasks from active subscription/desktop-client state,
with per-run atomic metrics and a Prometheus collector. Dashboard
and question-detail APIs now surface a platform_authorization_status
so the UI can distinguish no-desktop-client, no-authorized-platforms,
and authorized states; quota/access-state lookups resolve by request
workspace instead of tenant.

Hardens heartbeat: a desktop_client_primary_leases table gives
per-(tenant, workspace) sticky primary selection, read-mostly lease
resolution avoids a per-heartbeat transaction, and unchanged platform
access reports skip snapshot upserts outside a 10-minute refresh
window. Adds heartbeat primary/snapshot metrics exposed via
token-protected tenant-api /api/internal/metrics endpoints plus
Prometheus. Monitoring quota seed is removed from dev-seed since
daily plans now derive from desktop bindings, and the projection
uses the canonical six-platform catalog so platforms with zero
samples still render.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-24 22:19:57 +08:00

142 lines
4.7 KiB
Go

package app
import (
"sync/atomic"
"time"
)
type MonitoringHeartbeatPrimaryLeaseMetricsSnapshot struct {
PrimaryHitTotal int64 `json:"primary_hit_total"`
NonPrimaryLiveTotal int64 `json:"non_primary_live_total"`
InsertedTotal int64 `json:"inserted_total"`
RenewedTotal int64 `json:"renewed_total"`
ClaimedExpiredTotal int64 `json:"claimed_expired_total"`
ClaimedUnavailableTotal int64 `json:"claimed_unavailable_total"`
ContentionTotal int64 `json:"contention_total"`
ErrorTotal int64 `json:"error_total"`
DBInsertTotal int64 `json:"db_insert_total"`
DBRenewTotal int64 `json:"db_renew_total"`
DBClaimTotal int64 `json:"db_claim_total"`
DurationCount int64 `json:"duration_count"`
DurationSumSeconds float64 `json:"duration_sum_seconds"`
LastDurationSeconds float64 `json:"last_duration_seconds"`
LastOutcome string `json:"last_outcome,omitempty"`
}
type monitoringHeartbeatPrimaryLeaseMetricsRecorder struct {
primaryHitTotal atomic.Int64
nonPrimaryLiveTotal atomic.Int64
insertedTotal atomic.Int64
renewedTotal atomic.Int64
claimedExpiredTotal atomic.Int64
claimedUnavailableTotal atomic.Int64
contentionTotal atomic.Int64
errorTotal atomic.Int64
dbInsertTotal atomic.Int64
dbRenewTotal atomic.Int64
dbClaimTotal atomic.Int64
durationCount atomic.Int64
durationSumNanos atomic.Int64
lastDurationNanos atomic.Int64
lastOutcome atomic.Value
}
var monitoringHeartbeatPrimaryLeaseMetrics = newMonitoringHeartbeatPrimaryLeaseMetricsRecorder()
func newMonitoringHeartbeatPrimaryLeaseMetricsRecorder() *monitoringHeartbeatPrimaryLeaseMetricsRecorder {
return &monitoringHeartbeatPrimaryLeaseMetricsRecorder{}
}
func recordHeartbeatPrimaryLeaseMetric(outcome string, duration time.Duration) {
monitoringHeartbeatPrimaryLeaseMetrics.record(outcome, duration)
}
func recordHeartbeatPrimaryLeaseWriteMetric(operation string) {
monitoringHeartbeatPrimaryLeaseMetrics.recordWrite(operation)
}
func MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue() MonitoringHeartbeatPrimaryLeaseMetricsSnapshot {
return monitoringHeartbeatPrimaryLeaseMetrics.snapshot()
}
func resetMonitoringHeartbeatPrimaryLeaseMetricsForTest() {
monitoringHeartbeatPrimaryLeaseMetrics = newMonitoringHeartbeatPrimaryLeaseMetricsRecorder()
}
func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) record(outcome string, duration time.Duration) {
if r == nil {
return
}
if duration < 0 {
duration = 0
}
switch outcome {
case "primary_hit":
r.primaryHitTotal.Add(1)
case "non_primary_live":
r.nonPrimaryLiveTotal.Add(1)
case "inserted":
r.insertedTotal.Add(1)
case "renewed":
r.renewedTotal.Add(1)
case "claimed_expired":
r.claimedExpiredTotal.Add(1)
case "claimed_unavailable":
r.claimedUnavailableTotal.Add(1)
case "contention":
r.contentionTotal.Add(1)
case "error":
r.errorTotal.Add(1)
}
r.durationCount.Add(1)
r.durationSumNanos.Add(duration.Nanoseconds())
r.lastDurationNanos.Store(duration.Nanoseconds())
r.lastOutcome.Store(outcome)
}
func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) recordWrite(operation string) {
if r == nil {
return
}
switch operation {
case "insert":
r.dbInsertTotal.Add(1)
case "renew":
r.dbRenewTotal.Add(1)
case "claim":
r.dbClaimTotal.Add(1)
}
}
func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) snapshot() MonitoringHeartbeatPrimaryLeaseMetricsSnapshot {
if r == nil {
return MonitoringHeartbeatPrimaryLeaseMetricsSnapshot{}
}
return MonitoringHeartbeatPrimaryLeaseMetricsSnapshot{
PrimaryHitTotal: r.primaryHitTotal.Load(),
NonPrimaryLiveTotal: r.nonPrimaryLiveTotal.Load(),
InsertedTotal: r.insertedTotal.Load(),
RenewedTotal: r.renewedTotal.Load(),
ClaimedExpiredTotal: r.claimedExpiredTotal.Load(),
ClaimedUnavailableTotal: r.claimedUnavailableTotal.Load(),
ContentionTotal: r.contentionTotal.Load(),
ErrorTotal: r.errorTotal.Load(),
DBInsertTotal: r.dbInsertTotal.Load(),
DBRenewTotal: r.dbRenewTotal.Load(),
DBClaimTotal: r.dbClaimTotal.Load(),
DurationCount: r.durationCount.Load(),
DurationSumSeconds: float64(r.durationSumNanos.Load()) / float64(time.Second),
LastDurationSeconds: float64(r.lastDurationNanos.Load()) / float64(time.Second),
LastOutcome: heartbeatPrimaryMetricLoadString(&r.lastOutcome),
}
}
func heartbeatPrimaryMetricLoadString(value *atomic.Value) string {
if value == nil {
return ""
}
raw := value.Load()
result, _ := raw.(string)
return result
}