feat(monitoring): generate daily tasks and optimize heartbeat writes

Adds a MonitoringDailyTaskWorker that materializes and dispatches daily collect tasks from active subscription/desktop-client state, with per-run atomic metrics and a Prometheus collector. Dashboard and question-detail APIs now surface a platform_authorization_status so the UI can distinguish no-desktop-client, no-authorized-platforms, and authorized states; quota/access-state lookups resolve by request workspace instead of tenant. Hardens heartbeat: a desktop_client_primary_leases table gives per-(tenant, workspace) sticky primary selection, read-mostly lease resolution avoids a per-heartbeat transaction, and unchanged platform access reports skip snapshot upserts outside a 10-minute refresh window. Adds heartbeat primary/snapshot metrics exposed via token-protected tenant-api /api/internal/metrics endpoints plus Prometheus. Monitoring quota seed is removed from dev-seed since daily plans now derive from desktop bindings, and the projection uses the canonical six-platform catalog so platforms with zero samples still render. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-24 22:19:57 +08:00
parent ffe4d335aa
commit ff86933c24
26 changed files with 3546 additions and 277 deletions
@@ -0,0 +1,141 @@
+package app
+
+import (
+	"sync/atomic"
+	"time"
+)
+
+type MonitoringHeartbeatPrimaryLeaseMetricsSnapshot struct {
+	PrimaryHitTotal         int64   `json:"primary_hit_total"`
+	NonPrimaryLiveTotal     int64   `json:"non_primary_live_total"`
+	InsertedTotal           int64   `json:"inserted_total"`
+	RenewedTotal            int64   `json:"renewed_total"`
+	ClaimedExpiredTotal     int64   `json:"claimed_expired_total"`
+	ClaimedUnavailableTotal int64   `json:"claimed_unavailable_total"`
+	ContentionTotal         int64   `json:"contention_total"`
+	ErrorTotal              int64   `json:"error_total"`
+	DBInsertTotal           int64   `json:"db_insert_total"`
+	DBRenewTotal            int64   `json:"db_renew_total"`
+	DBClaimTotal            int64   `json:"db_claim_total"`
+	DurationCount           int64   `json:"duration_count"`
+	DurationSumSeconds      float64 `json:"duration_sum_seconds"`
+	LastDurationSeconds     float64 `json:"last_duration_seconds"`
+	LastOutcome             string  `json:"last_outcome,omitempty"`
+}
+
+type monitoringHeartbeatPrimaryLeaseMetricsRecorder struct {
+	primaryHitTotal         atomic.Int64
+	nonPrimaryLiveTotal     atomic.Int64
+	insertedTotal           atomic.Int64
+	renewedTotal            atomic.Int64
+	claimedExpiredTotal     atomic.Int64
+	claimedUnavailableTotal atomic.Int64
+	contentionTotal         atomic.Int64
+	errorTotal              atomic.Int64
+	dbInsertTotal           atomic.Int64
+	dbRenewTotal            atomic.Int64
+	dbClaimTotal            atomic.Int64
+	durationCount           atomic.Int64
+	durationSumNanos        atomic.Int64
+	lastDurationNanos       atomic.Int64
+	lastOutcome             atomic.Value
+}
+
+var monitoringHeartbeatPrimaryLeaseMetrics = newMonitoringHeartbeatPrimaryLeaseMetricsRecorder()
+
+func newMonitoringHeartbeatPrimaryLeaseMetricsRecorder() *monitoringHeartbeatPrimaryLeaseMetricsRecorder {
+	return &monitoringHeartbeatPrimaryLeaseMetricsRecorder{}
+}
+
+func recordHeartbeatPrimaryLeaseMetric(outcome string, duration time.Duration) {
+	monitoringHeartbeatPrimaryLeaseMetrics.record(outcome, duration)
+}
+
+func recordHeartbeatPrimaryLeaseWriteMetric(operation string) {
+	monitoringHeartbeatPrimaryLeaseMetrics.recordWrite(operation)
+}
+
+func MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue() MonitoringHeartbeatPrimaryLeaseMetricsSnapshot {
+	return monitoringHeartbeatPrimaryLeaseMetrics.snapshot()
+}
+
+func resetMonitoringHeartbeatPrimaryLeaseMetricsForTest() {
+	monitoringHeartbeatPrimaryLeaseMetrics = newMonitoringHeartbeatPrimaryLeaseMetricsRecorder()
+}
+
+func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) record(outcome string, duration time.Duration) {
+	if r == nil {
+		return
+	}
+	if duration < 0 {
+		duration = 0
+	}
+	switch outcome {
+	case "primary_hit":
+		r.primaryHitTotal.Add(1)
+	case "non_primary_live":
+		r.nonPrimaryLiveTotal.Add(1)
+	case "inserted":
+		r.insertedTotal.Add(1)
+	case "renewed":
+		r.renewedTotal.Add(1)
+	case "claimed_expired":
+		r.claimedExpiredTotal.Add(1)
+	case "claimed_unavailable":
+		r.claimedUnavailableTotal.Add(1)
+	case "contention":
+		r.contentionTotal.Add(1)
+	case "error":
+		r.errorTotal.Add(1)
+	}
+	r.durationCount.Add(1)
+	r.durationSumNanos.Add(duration.Nanoseconds())
+	r.lastDurationNanos.Store(duration.Nanoseconds())
+	r.lastOutcome.Store(outcome)
+}
+
+func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) recordWrite(operation string) {
+	if r == nil {
+		return
+	}
+	switch operation {
+	case "insert":
+		r.dbInsertTotal.Add(1)
+	case "renew":
+		r.dbRenewTotal.Add(1)
+	case "claim":
+		r.dbClaimTotal.Add(1)
+	}
+}
+
+func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) snapshot() MonitoringHeartbeatPrimaryLeaseMetricsSnapshot {
+	if r == nil {
+		return MonitoringHeartbeatPrimaryLeaseMetricsSnapshot{}
+	}
+	return MonitoringHeartbeatPrimaryLeaseMetricsSnapshot{
+		PrimaryHitTotal:         r.primaryHitTotal.Load(),
+		NonPrimaryLiveTotal:     r.nonPrimaryLiveTotal.Load(),
+		InsertedTotal:           r.insertedTotal.Load(),
+		RenewedTotal:            r.renewedTotal.Load(),
+		ClaimedExpiredTotal:     r.claimedExpiredTotal.Load(),
+		ClaimedUnavailableTotal: r.claimedUnavailableTotal.Load(),
+		ContentionTotal:         r.contentionTotal.Load(),
+		ErrorTotal:              r.errorTotal.Load(),
+		DBInsertTotal:           r.dbInsertTotal.Load(),
+		DBRenewTotal:            r.dbRenewTotal.Load(),
+		DBClaimTotal:            r.dbClaimTotal.Load(),
+		DurationCount:           r.durationCount.Load(),
+		DurationSumSeconds:      float64(r.durationSumNanos.Load()) / float64(time.Second),
+		LastDurationSeconds:     float64(r.lastDurationNanos.Load()) / float64(time.Second),
+		LastOutcome:             heartbeatPrimaryMetricLoadString(&r.lastOutcome),
+	}
+}
+
+func heartbeatPrimaryMetricLoadString(value *atomic.Value) string {
+	if value == nil {
+		return ""
+	}
+	raw := value.Load()
+	result, _ := raw.(string)
+	return result
+}