feat(monitoring): generate daily tasks and optimize heartbeat writes

Adds a MonitoringDailyTaskWorker that materializes and dispatches daily collect tasks from active subscription/desktop-client state, with per-run atomic metrics and a Prometheus collector. Dashboard and question-detail APIs now surface a platform_authorization_status so the UI can distinguish no-desktop-client, no-authorized-platforms, and authorized states; quota/access-state lookups resolve by request workspace instead of tenant. Hardens heartbeat: a desktop_client_primary_leases table gives per-(tenant, workspace) sticky primary selection, read-mostly lease resolution avoids a per-heartbeat transaction, and unchanged platform access reports skip snapshot upserts outside a 10-minute refresh window. Adds heartbeat primary/snapshot metrics exposed via token-protected tenant-api /api/internal/metrics endpoints plus Prometheus. Monitoring quota seed is removed from dev-seed since daily plans now derive from desktop bindings, and the projection uses the canonical six-platform catalog so platforms with zero samples still render. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-24 22:19:57 +08:00
parent ffe4d335aa
commit ff86933c24
26 changed files with 3546 additions and 277 deletions
@@ -0,0 +1,123 @@
+package app
+
+import (
+	"sync/atomic"
+	"time"
+)
+
+type MonitoringHeartbeatSnapshotMetricsSnapshot struct {
+	PlatformReportedTotal      int64   `json:"platform_reported_total"`
+	TransactionOpenedTotal     int64   `json:"transaction_opened_total"`
+	SnapshotInsertedTotal      int64   `json:"snapshot_inserted_total"`
+	SnapshotUpdatedTotal       int64   `json:"snapshot_updated_total"`
+	SnapshotRefreshedTotal     int64   `json:"snapshot_refreshed_total"`
+	SnapshotSkippedTotal       int64   `json:"snapshot_skipped_total"`
+	ReconciledSkippedTaskTotal int64   `json:"reconciled_skipped_task_total"`
+	ProjectionRebuildTotal     int64   `json:"projection_rebuild_total"`
+	ErrorTotal                 int64   `json:"error_total"`
+	DurationCount              int64   `json:"duration_count"`
+	DurationSumSeconds         float64 `json:"duration_sum_seconds"`
+	LastDurationSeconds        float64 `json:"last_duration_seconds"`
+	LastErrorCode              string  `json:"last_error_code,omitempty"`
+	LastErrorMessage           string  `json:"last_error_message,omitempty"`
+}
+
+type monitoringHeartbeatSnapshotMetricsEvent struct {
+	Active                     bool
+	PlatformCount              int
+	TransactionOpened          bool
+	SnapshotInsertedCount      int64
+	SnapshotUpdatedCount       int64
+	SnapshotRefreshedCount     int64
+	SnapshotSkippedCount       int64
+	ReconciledSkippedTaskCount int64
+	ProjectionRebuildCount     int64
+	Duration                   time.Duration
+	Err                        error
+}
+
+type monitoringHeartbeatSnapshotMetricsRecorder struct {
+	platformReportedTotal      atomic.Int64
+	transactionOpenedTotal     atomic.Int64
+	snapshotInsertedTotal      atomic.Int64
+	snapshotUpdatedTotal       atomic.Int64
+	snapshotRefreshedTotal     atomic.Int64
+	snapshotSkippedTotal       atomic.Int64
+	reconciledSkippedTaskTotal atomic.Int64
+	projectionRebuildTotal     atomic.Int64
+	errorTotal                 atomic.Int64
+	durationCount              atomic.Int64
+	durationSumNanos           atomic.Int64
+	lastDurationNanos          atomic.Int64
+	lastErrorCode              atomic.Value
+	lastErrorMessage           atomic.Value
+}
+
+var monitoringHeartbeatSnapshotMetrics = newMonitoringHeartbeatSnapshotMetricsRecorder()
+
+func newMonitoringHeartbeatSnapshotMetricsRecorder() *monitoringHeartbeatSnapshotMetricsRecorder {
+	return &monitoringHeartbeatSnapshotMetricsRecorder{}
+}
+
+func recordMonitoringHeartbeatSnapshotMetrics(event monitoringHeartbeatSnapshotMetricsEvent) {
+	monitoringHeartbeatSnapshotMetrics.record(event)
+}
+
+func MonitoringHeartbeatSnapshotMetricsSnapshotValue() MonitoringHeartbeatSnapshotMetricsSnapshot {
+	return monitoringHeartbeatSnapshotMetrics.snapshot()
+}
+
+func resetMonitoringHeartbeatSnapshotMetricsForTest() {
+	monitoringHeartbeatSnapshotMetrics = newMonitoringHeartbeatSnapshotMetricsRecorder()
+}
+
+func (r *monitoringHeartbeatSnapshotMetricsRecorder) record(event monitoringHeartbeatSnapshotMetricsEvent) {
+	if r == nil {
+		return
+	}
+	if event.Duration < 0 {
+		event.Duration = 0
+	}
+	r.platformReportedTotal.Add(int64(event.PlatformCount))
+	if event.TransactionOpened {
+		r.transactionOpenedTotal.Add(1)
+	}
+	r.snapshotInsertedTotal.Add(event.SnapshotInsertedCount)
+	r.snapshotUpdatedTotal.Add(event.SnapshotUpdatedCount)
+	r.snapshotRefreshedTotal.Add(event.SnapshotRefreshedCount)
+	r.snapshotSkippedTotal.Add(event.SnapshotSkippedCount)
+	r.reconciledSkippedTaskTotal.Add(event.ReconciledSkippedTaskCount)
+	r.projectionRebuildTotal.Add(event.ProjectionRebuildCount)
+	r.durationCount.Add(1)
+	r.durationSumNanos.Add(event.Duration.Nanoseconds())
+	r.lastDurationNanos.Store(event.Duration.Nanoseconds())
+
+	if event.Err != nil {
+		code, message := monitoringDailyMetricError(event.Err)
+		r.errorTotal.Add(1)
+		r.lastErrorCode.Store(code)
+		r.lastErrorMessage.Store(message)
+	}
+}
+
+func (r *monitoringHeartbeatSnapshotMetricsRecorder) snapshot() MonitoringHeartbeatSnapshotMetricsSnapshot {
+	if r == nil {
+		return MonitoringHeartbeatSnapshotMetricsSnapshot{}
+	}
+	return MonitoringHeartbeatSnapshotMetricsSnapshot{
+		PlatformReportedTotal:      r.platformReportedTotal.Load(),
+		TransactionOpenedTotal:     r.transactionOpenedTotal.Load(),
+		SnapshotInsertedTotal:      r.snapshotInsertedTotal.Load(),
+		SnapshotUpdatedTotal:       r.snapshotUpdatedTotal.Load(),
+		SnapshotRefreshedTotal:     r.snapshotRefreshedTotal.Load(),
+		SnapshotSkippedTotal:       r.snapshotSkippedTotal.Load(),
+		ReconciledSkippedTaskTotal: r.reconciledSkippedTaskTotal.Load(),
+		ProjectionRebuildTotal:     r.projectionRebuildTotal.Load(),
+		ErrorTotal:                 r.errorTotal.Load(),
+		DurationCount:              r.durationCount.Load(),
+		DurationSumSeconds:         float64(r.durationSumNanos.Load()) / float64(time.Second),
+		LastDurationSeconds:        float64(r.lastDurationNanos.Load()) / float64(time.Second),
+		LastErrorCode:              monitoringDailyLoadString(&r.lastErrorCode),
+		LastErrorMessage:           monitoringDailyLoadString(&r.lastErrorMessage),
+	}
+}