Files
geo/server/internal/tenant/app/monitoring_daily_task_metrics.go
T
root ff86933c24 feat(monitoring): generate daily tasks and optimize heartbeat writes
Adds a MonitoringDailyTaskWorker that materializes and dispatches
daily collect tasks from active subscription/desktop-client state,
with per-run atomic metrics and a Prometheus collector. Dashboard
and question-detail APIs now surface a platform_authorization_status
so the UI can distinguish no-desktop-client, no-authorized-platforms,
and authorized states; quota/access-state lookups resolve by request
workspace instead of tenant.

Hardens heartbeat: a desktop_client_primary_leases table gives
per-(tenant, workspace) sticky primary selection, read-mostly lease
resolution avoids a per-heartbeat transaction, and unchanged platform
access reports skip snapshot upserts outside a 10-minute refresh
window. Adds heartbeat primary/snapshot metrics exposed via
token-protected tenant-api /api/internal/metrics endpoints plus
Prometheus. Monitoring quota seed is removed from dev-seed since
daily plans now derive from desktop bindings, and the projection
uses the canonical six-platform catalog so platforms with zero
samples still render.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-24 22:19:57 +08:00

202 lines
7.0 KiB
Go

package app
import (
"context"
"errors"
"fmt"
"strings"
"sync/atomic"
"time"
"github.com/geo-platform/tenant-api/internal/shared/response"
)
type MonitoringDailyTaskMetricsSnapshot struct {
RunTotal int64 `json:"run_total"`
RunCompletedTotal int64 `json:"run_completed_total"`
RunFailedTotal int64 `json:"run_failed_total"`
RunPartialFailureTotal int64 `json:"run_partial_failure_total"`
PlanFailureTotal int64 `json:"plan_failure_total"`
BrandFailureTotal int64 `json:"brand_failure_total"`
SkippedPlanTotal int64 `json:"skipped_plan_total"`
PlannedTaskTotal int64 `json:"planned_task_total"`
DueTaskTotal int64 `json:"due_task_total"`
CreatedTaskTotal int64 `json:"created_task_total"`
DesktopTaskTotal int64 `json:"desktop_task_total"`
DeferredTaskTotal int64 `json:"deferred_task_total"`
LastRunAt *time.Time `json:"last_run_at,omitempty"`
LastSuccessAt *time.Time `json:"last_success_at,omitempty"`
LastErrorAt *time.Time `json:"last_error_at,omitempty"`
LastDurationSeconds float64 `json:"last_duration_seconds"`
LastBusinessDate string `json:"last_business_date,omitempty"`
LastTenantCount int64 `json:"last_tenant_count"`
LastBrandCount int64 `json:"last_brand_count"`
LastErrorCode string `json:"last_error_code,omitempty"`
LastErrorMessage string `json:"last_error_message,omitempty"`
}
type monitoringDailyTaskMetricsRecorder struct {
runTotal atomic.Int64
runCompletedTotal atomic.Int64
runFailedTotal atomic.Int64
runPartialFailureTotal atomic.Int64
planFailureTotal atomic.Int64
brandFailureTotal atomic.Int64
skippedPlanTotal atomic.Int64
plannedTaskTotal atomic.Int64
dueTaskTotal atomic.Int64
createdTaskTotal atomic.Int64
desktopTaskTotal atomic.Int64
deferredTaskTotal atomic.Int64
lastRunAtUnixNano atomic.Int64
lastSuccessAtUnixNano atomic.Int64
lastErrorAtUnixNano atomic.Int64
lastDurationNanos atomic.Int64
lastTenantCount atomic.Int64
lastBrandCount atomic.Int64
lastBusinessDate atomic.Value
lastErrorCode atomic.Value
lastErrorMessage atomic.Value
}
var monitoringDailyTaskMetrics = newMonitoringDailyTaskMetricsRecorder()
func newMonitoringDailyTaskMetricsRecorder() *monitoringDailyTaskMetricsRecorder {
return &monitoringDailyTaskMetricsRecorder{}
}
func recordMonitoringDailyTaskMetrics(summary *MonitoringDailyTaskGenerationSummary, duration time.Duration, err error) {
monitoringDailyTaskMetrics.record(summary, duration, err)
}
func MonitoringDailyTaskMetricsSnapshotValue() MonitoringDailyTaskMetricsSnapshot {
return monitoringDailyTaskMetrics.snapshot()
}
func resetMonitoringDailyTaskMetricsForTest() {
monitoringDailyTaskMetrics = newMonitoringDailyTaskMetricsRecorder()
}
func (r *monitoringDailyTaskMetricsRecorder) record(summary *MonitoringDailyTaskGenerationSummary, duration time.Duration, err error) {
if r == nil {
return
}
now := time.Now().UTC()
if duration < 0 {
duration = 0
}
r.runTotal.Add(1)
r.lastRunAtUnixNano.Store(now.UnixNano())
r.lastDurationNanos.Store(duration.Nanoseconds())
if summary != nil {
r.lastBusinessDate.Store(summary.BusinessDate)
r.lastTenantCount.Store(int64(summary.TenantCount))
r.lastBrandCount.Store(int64(summary.BrandCount))
r.planFailureTotal.Add(int64(summary.FailedPlanCount))
r.brandFailureTotal.Add(int64(summary.FailedBrandCount))
r.skippedPlanTotal.Add(int64(summary.SkippedPlanCount))
r.plannedTaskTotal.Add(summary.PlannedTaskCount)
r.dueTaskTotal.Add(summary.DueTaskCount)
r.createdTaskTotal.Add(summary.CreatedTaskCount)
r.desktopTaskTotal.Add(summary.DesktopTaskCount)
r.deferredTaskTotal.Add(summary.DeferredTaskCount)
if summary.FailedPlanCount > 0 || summary.FailedBrandCount > 0 {
r.runPartialFailureTotal.Add(1)
}
}
if err != nil {
code, message := monitoringDailyMetricError(err)
r.runFailedTotal.Add(1)
r.lastErrorAtUnixNano.Store(now.UnixNano())
r.lastErrorCode.Store(code)
r.lastErrorMessage.Store(message)
return
}
r.runCompletedTotal.Add(1)
r.lastSuccessAtUnixNano.Store(now.UnixNano())
}
func (r *monitoringDailyTaskMetricsRecorder) snapshot() MonitoringDailyTaskMetricsSnapshot {
if r == nil {
return MonitoringDailyTaskMetricsSnapshot{}
}
return MonitoringDailyTaskMetricsSnapshot{
RunTotal: r.runTotal.Load(),
RunCompletedTotal: r.runCompletedTotal.Load(),
RunFailedTotal: r.runFailedTotal.Load(),
RunPartialFailureTotal: r.runPartialFailureTotal.Load(),
PlanFailureTotal: r.planFailureTotal.Load(),
BrandFailureTotal: r.brandFailureTotal.Load(),
SkippedPlanTotal: r.skippedPlanTotal.Load(),
PlannedTaskTotal: r.plannedTaskTotal.Load(),
DueTaskTotal: r.dueTaskTotal.Load(),
CreatedTaskTotal: r.createdTaskTotal.Load(),
DesktopTaskTotal: r.desktopTaskTotal.Load(),
DeferredTaskTotal: r.deferredTaskTotal.Load(),
LastRunAt: monitoringDailyUnixNanoTime(r.lastRunAtUnixNano.Load()),
LastSuccessAt: monitoringDailyUnixNanoTime(r.lastSuccessAtUnixNano.Load()),
LastErrorAt: monitoringDailyUnixNanoTime(r.lastErrorAtUnixNano.Load()),
LastDurationSeconds: float64(r.lastDurationNanos.Load()) / float64(time.Second),
LastBusinessDate: monitoringDailyLoadString(&r.lastBusinessDate),
LastTenantCount: r.lastTenantCount.Load(),
LastBrandCount: r.lastBrandCount.Load(),
LastErrorCode: monitoringDailyLoadString(&r.lastErrorCode),
LastErrorMessage: monitoringDailyLoadString(&r.lastErrorMessage),
}
}
func monitoringDailyMetricError(err error) (string, string) {
normalized := normalizeMonitoringWorkerError(err)
if normalized == nil {
return "", ""
}
switch {
case errors.Is(normalized, context.Canceled):
return "context_canceled", normalized.Error()
case errors.Is(normalized, context.DeadlineExceeded):
return "context_deadline_exceeded", normalized.Error()
}
var appErr *response.AppError
if errors.As(normalized, &appErr) {
if strings.TrimSpace(appErr.Message) != "" {
return appErr.Message, appErr.Error()
}
return "app_error", appErr.Error()
}
code := strings.TrimLeft(fmt.Sprintf("%T", normalized), "*")
code = strings.ReplaceAll(code, "/", "_")
code = strings.ReplaceAll(code, ".", "_")
if code == "" {
code = "error"
}
return code, normalized.Error()
}
func monitoringDailyUnixNanoTime(value int64) *time.Time {
if value <= 0 {
return nil
}
t := time.Unix(0, value).UTC()
return &t
}
func monitoringDailyLoadString(value *atomic.Value) string {
if value == nil {
return ""
}
raw := value.Load()
result, _ := raw.(string)
return result
}
func monitoringDailyTimeSeconds(value *time.Time) float64 {
if value == nil || value.IsZero() {
return 0
}
return float64(value.UnixNano()) / float64(time.Second)
}