feat(monitoring): generate daily tasks and optimize heartbeat writes
Adds a MonitoringDailyTaskWorker that materializes and dispatches daily collect tasks from active subscription/desktop-client state, with per-run atomic metrics and a Prometheus collector. Dashboard and question-detail APIs now surface a platform_authorization_status so the UI can distinguish no-desktop-client, no-authorized-platforms, and authorized states; quota/access-state lookups resolve by request workspace instead of tenant. Hardens heartbeat: a desktop_client_primary_leases table gives per-(tenant, workspace) sticky primary selection, read-mostly lease resolution avoids a per-heartbeat transaction, and unchanged platform access reports skip snapshot upserts outside a 10-minute refresh window. Adds heartbeat primary/snapshot metrics exposed via token-protected tenant-api /api/internal/metrics endpoints plus Prometheus. Monitoring quota seed is removed from dev-seed since daily plans now derive from desktop bindings, and the projection uses the canonical six-platform catalog so platforms with zero samples still render. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,201 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/response"
|
||||
)
|
||||
|
||||
type MonitoringDailyTaskMetricsSnapshot struct {
|
||||
RunTotal int64 `json:"run_total"`
|
||||
RunCompletedTotal int64 `json:"run_completed_total"`
|
||||
RunFailedTotal int64 `json:"run_failed_total"`
|
||||
RunPartialFailureTotal int64 `json:"run_partial_failure_total"`
|
||||
PlanFailureTotal int64 `json:"plan_failure_total"`
|
||||
BrandFailureTotal int64 `json:"brand_failure_total"`
|
||||
SkippedPlanTotal int64 `json:"skipped_plan_total"`
|
||||
PlannedTaskTotal int64 `json:"planned_task_total"`
|
||||
DueTaskTotal int64 `json:"due_task_total"`
|
||||
CreatedTaskTotal int64 `json:"created_task_total"`
|
||||
DesktopTaskTotal int64 `json:"desktop_task_total"`
|
||||
DeferredTaskTotal int64 `json:"deferred_task_total"`
|
||||
LastRunAt *time.Time `json:"last_run_at,omitempty"`
|
||||
LastSuccessAt *time.Time `json:"last_success_at,omitempty"`
|
||||
LastErrorAt *time.Time `json:"last_error_at,omitempty"`
|
||||
LastDurationSeconds float64 `json:"last_duration_seconds"`
|
||||
LastBusinessDate string `json:"last_business_date,omitempty"`
|
||||
LastTenantCount int64 `json:"last_tenant_count"`
|
||||
LastBrandCount int64 `json:"last_brand_count"`
|
||||
LastErrorCode string `json:"last_error_code,omitempty"`
|
||||
LastErrorMessage string `json:"last_error_message,omitempty"`
|
||||
}
|
||||
|
||||
type monitoringDailyTaskMetricsRecorder struct {
|
||||
runTotal atomic.Int64
|
||||
runCompletedTotal atomic.Int64
|
||||
runFailedTotal atomic.Int64
|
||||
runPartialFailureTotal atomic.Int64
|
||||
planFailureTotal atomic.Int64
|
||||
brandFailureTotal atomic.Int64
|
||||
skippedPlanTotal atomic.Int64
|
||||
plannedTaskTotal atomic.Int64
|
||||
dueTaskTotal atomic.Int64
|
||||
createdTaskTotal atomic.Int64
|
||||
desktopTaskTotal atomic.Int64
|
||||
deferredTaskTotal atomic.Int64
|
||||
lastRunAtUnixNano atomic.Int64
|
||||
lastSuccessAtUnixNano atomic.Int64
|
||||
lastErrorAtUnixNano atomic.Int64
|
||||
lastDurationNanos atomic.Int64
|
||||
lastTenantCount atomic.Int64
|
||||
lastBrandCount atomic.Int64
|
||||
lastBusinessDate atomic.Value
|
||||
lastErrorCode atomic.Value
|
||||
lastErrorMessage atomic.Value
|
||||
}
|
||||
|
||||
var monitoringDailyTaskMetrics = newMonitoringDailyTaskMetricsRecorder()
|
||||
|
||||
func newMonitoringDailyTaskMetricsRecorder() *monitoringDailyTaskMetricsRecorder {
|
||||
return &monitoringDailyTaskMetricsRecorder{}
|
||||
}
|
||||
|
||||
func recordMonitoringDailyTaskMetrics(summary *MonitoringDailyTaskGenerationSummary, duration time.Duration, err error) {
|
||||
monitoringDailyTaskMetrics.record(summary, duration, err)
|
||||
}
|
||||
|
||||
func MonitoringDailyTaskMetricsSnapshotValue() MonitoringDailyTaskMetricsSnapshot {
|
||||
return monitoringDailyTaskMetrics.snapshot()
|
||||
}
|
||||
|
||||
func resetMonitoringDailyTaskMetricsForTest() {
|
||||
monitoringDailyTaskMetrics = newMonitoringDailyTaskMetricsRecorder()
|
||||
}
|
||||
|
||||
func (r *monitoringDailyTaskMetricsRecorder) record(summary *MonitoringDailyTaskGenerationSummary, duration time.Duration, err error) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if duration < 0 {
|
||||
duration = 0
|
||||
}
|
||||
|
||||
r.runTotal.Add(1)
|
||||
r.lastRunAtUnixNano.Store(now.UnixNano())
|
||||
r.lastDurationNanos.Store(duration.Nanoseconds())
|
||||
|
||||
if summary != nil {
|
||||
r.lastBusinessDate.Store(summary.BusinessDate)
|
||||
r.lastTenantCount.Store(int64(summary.TenantCount))
|
||||
r.lastBrandCount.Store(int64(summary.BrandCount))
|
||||
r.planFailureTotal.Add(int64(summary.FailedPlanCount))
|
||||
r.brandFailureTotal.Add(int64(summary.FailedBrandCount))
|
||||
r.skippedPlanTotal.Add(int64(summary.SkippedPlanCount))
|
||||
r.plannedTaskTotal.Add(summary.PlannedTaskCount)
|
||||
r.dueTaskTotal.Add(summary.DueTaskCount)
|
||||
r.createdTaskTotal.Add(summary.CreatedTaskCount)
|
||||
r.desktopTaskTotal.Add(summary.DesktopTaskCount)
|
||||
r.deferredTaskTotal.Add(summary.DeferredTaskCount)
|
||||
if summary.FailedPlanCount > 0 || summary.FailedBrandCount > 0 {
|
||||
r.runPartialFailureTotal.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
code, message := monitoringDailyMetricError(err)
|
||||
r.runFailedTotal.Add(1)
|
||||
r.lastErrorAtUnixNano.Store(now.UnixNano())
|
||||
r.lastErrorCode.Store(code)
|
||||
r.lastErrorMessage.Store(message)
|
||||
return
|
||||
}
|
||||
|
||||
r.runCompletedTotal.Add(1)
|
||||
r.lastSuccessAtUnixNano.Store(now.UnixNano())
|
||||
}
|
||||
|
||||
func (r *monitoringDailyTaskMetricsRecorder) snapshot() MonitoringDailyTaskMetricsSnapshot {
|
||||
if r == nil {
|
||||
return MonitoringDailyTaskMetricsSnapshot{}
|
||||
}
|
||||
return MonitoringDailyTaskMetricsSnapshot{
|
||||
RunTotal: r.runTotal.Load(),
|
||||
RunCompletedTotal: r.runCompletedTotal.Load(),
|
||||
RunFailedTotal: r.runFailedTotal.Load(),
|
||||
RunPartialFailureTotal: r.runPartialFailureTotal.Load(),
|
||||
PlanFailureTotal: r.planFailureTotal.Load(),
|
||||
BrandFailureTotal: r.brandFailureTotal.Load(),
|
||||
SkippedPlanTotal: r.skippedPlanTotal.Load(),
|
||||
PlannedTaskTotal: r.plannedTaskTotal.Load(),
|
||||
DueTaskTotal: r.dueTaskTotal.Load(),
|
||||
CreatedTaskTotal: r.createdTaskTotal.Load(),
|
||||
DesktopTaskTotal: r.desktopTaskTotal.Load(),
|
||||
DeferredTaskTotal: r.deferredTaskTotal.Load(),
|
||||
LastRunAt: monitoringDailyUnixNanoTime(r.lastRunAtUnixNano.Load()),
|
||||
LastSuccessAt: monitoringDailyUnixNanoTime(r.lastSuccessAtUnixNano.Load()),
|
||||
LastErrorAt: monitoringDailyUnixNanoTime(r.lastErrorAtUnixNano.Load()),
|
||||
LastDurationSeconds: float64(r.lastDurationNanos.Load()) / float64(time.Second),
|
||||
LastBusinessDate: monitoringDailyLoadString(&r.lastBusinessDate),
|
||||
LastTenantCount: r.lastTenantCount.Load(),
|
||||
LastBrandCount: r.lastBrandCount.Load(),
|
||||
LastErrorCode: monitoringDailyLoadString(&r.lastErrorCode),
|
||||
LastErrorMessage: monitoringDailyLoadString(&r.lastErrorMessage),
|
||||
}
|
||||
}
|
||||
|
||||
func monitoringDailyMetricError(err error) (string, string) {
|
||||
normalized := normalizeMonitoringWorkerError(err)
|
||||
if normalized == nil {
|
||||
return "", ""
|
||||
}
|
||||
switch {
|
||||
case errors.Is(normalized, context.Canceled):
|
||||
return "context_canceled", normalized.Error()
|
||||
case errors.Is(normalized, context.DeadlineExceeded):
|
||||
return "context_deadline_exceeded", normalized.Error()
|
||||
}
|
||||
var appErr *response.AppError
|
||||
if errors.As(normalized, &appErr) {
|
||||
if strings.TrimSpace(appErr.Message) != "" {
|
||||
return appErr.Message, appErr.Error()
|
||||
}
|
||||
return "app_error", appErr.Error()
|
||||
}
|
||||
code := strings.TrimLeft(fmt.Sprintf("%T", normalized), "*")
|
||||
code = strings.ReplaceAll(code, "/", "_")
|
||||
code = strings.ReplaceAll(code, ".", "_")
|
||||
if code == "" {
|
||||
code = "error"
|
||||
}
|
||||
return code, normalized.Error()
|
||||
}
|
||||
|
||||
func monitoringDailyUnixNanoTime(value int64) *time.Time {
|
||||
if value <= 0 {
|
||||
return nil
|
||||
}
|
||||
t := time.Unix(0, value).UTC()
|
||||
return &t
|
||||
}
|
||||
|
||||
func monitoringDailyLoadString(value *atomic.Value) string {
|
||||
if value == nil {
|
||||
return ""
|
||||
}
|
||||
raw := value.Load()
|
||||
result, _ := raw.(string)
|
||||
return result
|
||||
}
|
||||
|
||||
func monitoringDailyTimeSeconds(value *time.Time) float64 {
|
||||
if value == nil || value.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return float64(value.UnixNano()) / float64(time.Second)
|
||||
}
|
||||
Reference in New Issue
Block a user