From ff86933c24cac822defdb9f32715e811a3b63096 Mon Sep 17 00:00:00 2001 From: liangxu Date: Fri, 24 Apr 2026 22:19:57 +0800 Subject: [PATCH] feat(monitoring): generate daily tasks and optimize heartbeat writes Adds a MonitoringDailyTaskWorker that materializes and dispatches daily collect tasks from active subscription/desktop-client state, with per-run atomic metrics and a Prometheus collector. Dashboard and question-detail APIs now surface a platform_authorization_status so the UI can distinguish no-desktop-client, no-authorized-platforms, and authorized states; quota/access-state lookups resolve by request workspace instead of tenant. Hardens heartbeat: a desktop_client_primary_leases table gives per-(tenant, workspace) sticky primary selection, read-mostly lease resolution avoids a per-heartbeat transaction, and unchanged platform access reports skip snapshot upserts outside a 10-minute refresh window. Adds heartbeat primary/snapshot metrics exposed via token-protected tenant-api /api/internal/metrics endpoints plus Prometheus. Monitoring quota seed is removed from dev-seed since daily plans now derive from desktop bindings, and the projection uses the canonical six-platform catalog so platforms with zero samples still render. Co-Authored-By: Claude Opus 4.7 (1M context) --- server/cmd/dev-seed/main.go | 26 - server/cmd/tenant-api/main.go | 2 +- server/go.mod | 6 + server/go.sum | 14 + server/internal/shared/config/config.go | 17 + server/internal/shared/config/config_test.go | 52 + .../internal/tenant/app/desktop_presence.go | 3 + .../tenant/app/monitoring_callback_service.go | 578 ++++++-- .../app/monitoring_callback_service_test.go | 278 ++++ .../app/monitoring_daily_task_metrics.go | 201 +++ .../app/monitoring_daily_task_worker.go | 1295 +++++++++++++++++ .../app/monitoring_daily_task_worker_test.go | 337 +++++ .../monitoring_heartbeat_primary_metrics.go | 141 ++ .../monitoring_heartbeat_snapshot_metrics.go | 123 ++ .../app/monitoring_metrics_prometheus.go | 195 +++ .../app/monitoring_phase2_desktop_tasks.go | 9 +- .../tenant/app/monitoring_projection.go | 29 +- .../internal/tenant/app/monitoring_service.go | 347 +++-- .../app/monitoring_service_date_test.go | 43 +- .../transport/internal_metrics_handler.go | 61 + .../tenant/transport/monitoring_handler.go | 2 +- server/internal/tenant/transport/router.go | 3 + ...130000_add_desktop_primary_leases.down.sql | 4 + ...24130000_add_desktop_primary_leases.up.sql | 52 + ..._pending_platform_reconcile_index.down.sql | 1 + ...ng_pending_platform_reconcile_index.up.sql | 4 + 26 files changed, 3546 insertions(+), 277 deletions(-) create mode 100644 server/internal/tenant/app/monitoring_daily_task_metrics.go create mode 100644 server/internal/tenant/app/monitoring_daily_task_worker.go create mode 100644 server/internal/tenant/app/monitoring_daily_task_worker_test.go create mode 100644 server/internal/tenant/app/monitoring_heartbeat_primary_metrics.go create mode 100644 server/internal/tenant/app/monitoring_heartbeat_snapshot_metrics.go create mode 100644 server/internal/tenant/app/monitoring_metrics_prometheus.go create mode 100644 server/internal/tenant/transport/internal_metrics_handler.go create mode 100644 server/migrations/20260424130000_add_desktop_primary_leases.down.sql create mode 100644 server/migrations/20260424130000_add_desktop_primary_leases.up.sql create mode 100644 server/migrations_monitoring/20260424133000_add_monitoring_pending_platform_reconcile_index.down.sql create mode 100644 server/migrations_monitoring/20260424133000_add_monitoring_pending_platform_reconcile_index.up.sql diff --git a/server/cmd/dev-seed/main.go b/server/cmd/dev-seed/main.go index 507b44d..fa3b505 100644 --- a/server/cmd/dev-seed/main.go +++ b/server/cmd/dev-seed/main.go @@ -246,10 +246,6 @@ func seedBusinessData(ctx context.Context, pool *pgxpool.Pool, membershipCfg con return nil, err } - if err := ensureMonitoringQuota(ctx, tx, state); err != nil { - return nil, err - } - if err := tx.Commit(ctx); err != nil { return nil, fmt.Errorf("commit business seed: %w", err) } @@ -692,28 +688,6 @@ func ensureDesktopClient(ctx context.Context, tx pgx.Tx, tenantID, workspaceID, return ensuredID, nil } -func ensureMonitoringQuota(ctx context.Context, tx pgx.Tx, state *seedState) error { - enabledPlatforms, _ := json.Marshal([]string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"}) - _, err := tx.Exec(ctx, ` - INSERT INTO tenant_monitoring_quotas ( - tenant_id, workspace_id, max_brands, collection_mode, question_selection_mode, collect_frequency, - enabled_platforms, primary_client_id, task_daily_hard_cap, plan_tier - ) - VALUES ($1, $2, 1, 'desktop', 'best_effort_all', 'daily', $3::jsonb, $4, 36, 'free') - ON CONFLICT (tenant_id) DO UPDATE - SET workspace_id = EXCLUDED.workspace_id, - collection_mode = EXCLUDED.collection_mode, - question_selection_mode = EXCLUDED.question_selection_mode, - collect_frequency = EXCLUDED.collect_frequency, - enabled_platforms = EXCLUDED.enabled_platforms, - primary_client_id = EXCLUDED.primary_client_id, - task_daily_hard_cap = EXCLUDED.task_daily_hard_cap, - plan_tier = EXCLUDED.plan_tier, - updated_at = NOW() - `, state.TenantID, state.WorkspaceID, string(enabledPlatforms), state.DesktopClientID) - return wrapSeedErr("upsert monitoring quota", err) -} - func clearMonitoringData(ctx context.Context, tx pgx.Tx, state *seedState) error { pairs := []string{ `DELETE FROM monitoring_citation_facts WHERE tenant_id = $1 AND brand_id = $2`, diff --git a/server/cmd/tenant-api/main.go b/server/cmd/tenant-api/main.go index 9ad1ac7..dbdc0a3 100644 --- a/server/cmd/tenant-api/main.go +++ b/server/cmd/tenant-api/main.go @@ -32,7 +32,7 @@ func main() { app.DesktopTaskStreams.Run(workerCtx) app.DesktopDispatch.Run(workerCtx) - monitoringService := tenantapp.NewMonitoringService(app.DB, app.MonitoringDB, app.RabbitMQ, app.Config.MonitoringDispatch, app.Logger) + monitoringService := tenantapp.NewMonitoringService(app.DB, app.MonitoringDB, app.RabbitMQ, app.Config.MonitoringDispatch, app.Config.BrandLibrary, app.Logger) monitoringCallbackService := tenantapp.NewMonitoringCallbackService(app.DB, app.MonitoringDB, app.RabbitMQ, app.LLM, app.Logger) tenantapp.NewMonitoringCollectOutboxWorker(monitoringService, app.Logger).Start(workerCtx) tenantapp.NewMonitoringResultIngestWorker(monitoringCallbackService, app.RabbitMQ, app.Logger, app.Config.MonitoringWorkers).Start(workerCtx) diff --git a/server/go.mod b/server/go.mod index 9002d3d..b093667 100644 --- a/server/go.mod +++ b/server/go.mod @@ -14,6 +14,8 @@ require ( github.com/jackc/pgx/v5 v5.5.5 github.com/ledongthuc/pdf v0.0.0-20240314090751-a2a84ec735c3 github.com/minio/minio-go/v7 v7.0.83 + github.com/prometheus/client_golang v1.20.5 + github.com/prometheus/common v0.60.1 github.com/qdrant/go-client v1.13.0 github.com/rabbitmq/amqp091-go v1.10.0 github.com/redis/go-redis/v9 v9.5.1 @@ -32,6 +34,7 @@ require ( ) require ( + github.com/beorn7/perks v1.0.1 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect @@ -63,8 +66,11 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/richardlehane/mscfb v1.0.4 // indirect github.com/richardlehane/msoleps v1.0.3 // indirect github.com/rs/xid v1.6.0 // indirect diff --git a/server/go.sum b/server/go.sum index c283b60..eb13bb4 100644 --- a/server/go.sum +++ b/server/go.sum @@ -5,6 +5,8 @@ github.com/alicebob/miniredis/v2 v2.37.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -115,6 +117,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/ledongthuc/pdf v0.0.0-20240314090751-a2a84ec735c3 h1:DDPccSesc/SJu8CiCEhHhWjK9C9U1fhrRKB3jRNw4f4= github.com/ledongthuc/pdf v0.0.0-20240314090751-a2a84ec735c3/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= @@ -136,13 +140,23 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc= +github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/qdrant/go-client v1.13.0 h1:qeWKCs1vxvfF2MLLFnP2qDG0R8wI18HyAoSfc7wJim8= github.com/qdrant/go-client v1.13.0/go.mod h1:iO8ts78jL4x6LDHFOViyYWELVtIBDTjOykBmiOTHLnQ= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= diff --git a/server/internal/shared/config/config.go b/server/internal/shared/config/config.go index 294bc44..82fddac 100644 --- a/server/internal/shared/config/config.go +++ b/server/internal/shared/config/config.go @@ -105,6 +105,9 @@ type RabbitMQConfig struct { } type SchedulerConfig struct { + HTTPHost string `mapstructure:"http_host"` + HTTPPort int `mapstructure:"http_port"` + InternalMetricsToken string `mapstructure:"internal_metrics_token"` DispatchInterval time.Duration `mapstructure:"dispatch_interval"` DispatchTimeout time.Duration `mapstructure:"dispatch_timeout"` DispatchBatchSize int `mapstructure:"dispatch_batch_size"` @@ -427,6 +430,12 @@ func applyEnvOverrides(cfg *Config) { if model, ok := lookupFirstNonEmptyEnv("SILICONFLOW_RERANKER_MODEL", "RETRIEVAL_RERANKER_MODEL"); ok { cfg.Retrieval.RerankerModel = model } + if host, ok := lookupNonEmptyEnv("SCHEDULER_HTTP_HOST"); ok { + cfg.Scheduler.HTTPHost = host + } + if token, ok := lookupFirstNonEmptyEnv("SCHEDULER_INTERNAL_METRICS_TOKEN", "SCHEDULER_METRICS_TOKEN"); ok { + cfg.Scheduler.InternalMetricsToken = token + } } func normalizeRabbitMQConfig(cfg *RabbitMQConfig) { @@ -569,6 +578,14 @@ func normalizeSchedulerConfig(cfg *SchedulerConfig) { return } + if cfg.HTTPPort == 0 { + cfg.HTTPPort = 8081 + } + cfg.HTTPHost = strings.TrimSpace(cfg.HTTPHost) + if cfg.HTTPHost == "" { + cfg.HTTPHost = "127.0.0.1" + } + cfg.InternalMetricsToken = strings.TrimSpace(cfg.InternalMetricsToken) if cfg.DispatchInterval <= 0 { cfg.DispatchInterval = 15 * time.Second } diff --git a/server/internal/shared/config/config_test.go b/server/internal/shared/config/config_test.go index fe584f9..59a0924 100644 --- a/server/internal/shared/config/config_test.go +++ b/server/internal/shared/config/config_test.go @@ -114,6 +114,58 @@ brand_library: {} } } +func TestLoadAppliesSchedulerHTTPDefaultsAndDisableSentinel(t *testing.T) { + defaultConfigPath := writeTestConfig(t, ` +scheduler: {} +`) + + cfg, err := Load(defaultConfigPath) + if err != nil { + t.Fatalf("Load() default error = %v", err) + } + if cfg.Scheduler.HTTPHost != "127.0.0.1" { + t.Fatalf("expected default scheduler http host 127.0.0.1, got %q", cfg.Scheduler.HTTPHost) + } + if cfg.Scheduler.HTTPPort != 8081 { + t.Fatalf("expected default scheduler http port 8081, got %d", cfg.Scheduler.HTTPPort) + } + + disabledConfigPath := writeTestConfig(t, ` +scheduler: + http_port: -1 +`) + + cfg, err = Load(disabledConfigPath) + if err != nil { + t.Fatalf("Load() disabled error = %v", err) + } + if cfg.Scheduler.HTTPPort != -1 { + t.Fatalf("expected scheduler http port -1 to disable server, got %d", cfg.Scheduler.HTTPPort) + } +} + +func TestLoadPrefersEnvSchedulerMetricsSettings(t *testing.T) { + t.Setenv("SCHEDULER_HTTP_HOST", "0.0.0.0") + t.Setenv("SCHEDULER_INTERNAL_METRICS_TOKEN", "env-metrics-token") + + configPath := writeTestConfig(t, ` +scheduler: + http_host: 127.0.0.1 + internal_metrics_token: config-token +`) + + cfg, err := Load(configPath) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if cfg.Scheduler.HTTPHost != "0.0.0.0" { + t.Fatalf("expected env scheduler http host, got %q", cfg.Scheduler.HTTPHost) + } + if cfg.Scheduler.InternalMetricsToken != "env-metrics-token" { + t.Fatalf("expected env scheduler metrics token, got %q", cfg.Scheduler.InternalMetricsToken) + } +} + func TestLoadAppliesMembershipDefaults(t *testing.T) { configPath := writeTestConfig(t, ` membership: {} diff --git a/server/internal/tenant/app/desktop_presence.go b/server/internal/tenant/app/desktop_presence.go index b6e9423..8f7e46f 100644 --- a/server/internal/tenant/app/desktop_presence.go +++ b/server/internal/tenant/app/desktop_presence.go @@ -9,6 +9,9 @@ import ( goredis "github.com/redis/go-redis/v9" ) +// desktopClientPresenceTTL is also used as the monitoring primary-client lease TTL. +// Desktop heartbeat interval must stay below TTL/2, so one missed heartbeat does +// not make another client eligible to steal a still-healthy primary lease. const desktopClientPresenceTTL = 90 * time.Second const desktopAccountPresenceTTL = 90 * time.Second diff --git a/server/internal/tenant/app/monitoring_callback_service.go b/server/internal/tenant/app/monitoring_callback_service.go index 190ec77..bd419c3 100644 --- a/server/internal/tenant/app/monitoring_callback_service.go +++ b/server/internal/tenant/app/monitoring_callback_service.go @@ -30,6 +30,14 @@ const ( maxMonitoringLeaseLimit = 50 monitoringLeaseTTL = 15 * time.Minute monitoringTaskResultQueueIOTimeout = 5 * time.Second + // The current primary renews the DB lease only after this interval. With the + // desktop client's 25s heartbeat and a 90s lease TTL, this keeps normal + // heartbeats read-only while still tolerating one missed heartbeat. + monitoringHeartbeatPrimaryRenewAfter = desktopClientPresenceTTL / 3 + // Access snapshots are semantic state, not a liveness clock. Stable heartbeat + // reports skip writes and only refresh the row at this interval so dashboards + // do not depend on per-heartbeat database churn. + monitoringHeartbeatAccessSnapshotRefreshAfter = 10 * time.Minute ) type MonitoringCallbackService struct { @@ -205,6 +213,16 @@ type monitoringPlatformAccessSnapshotState struct { Accessible bool ReasonCode *string ReasonMessage *string + UpdatedAt time.Time +} + +type monitoringHeartbeatSnapshotDecision struct { + WriteSnapshot bool + StateChanged bool + RefreshDue bool + PendingReconcileCandidate bool + ReconcilePending bool + SnapshotOperation string } type monitoringCollectTask struct { @@ -434,7 +452,18 @@ func (s *MonitoringCallbackService) handleTaskResultForInstallation( }, nil } -func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationID string, installationToken string, req MonitoringHeartbeatRequest) (*MonitoringHeartbeatResponse, error) { +func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationID string, installationToken string, req MonitoringHeartbeatRequest) (_ *MonitoringHeartbeatResponse, err error) { + startedAt := time.Now() + snapshotMetrics := monitoringHeartbeatSnapshotMetricsEvent{} + defer func() { + if !snapshotMetrics.Active { + return + } + snapshotMetrics.Duration = time.Since(startedAt) + snapshotMetrics.Err = err + recordMonitoringHeartbeatSnapshotMetrics(snapshotMetrics) + }() + installation, err := s.authenticateInstallation(ctx, installationID, installationToken) if err != nil { return nil, err @@ -451,28 +480,111 @@ func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationI if err != nil { return nil, err } + snapshotMetrics.Active = true + snapshotMetrics.PlatformCount = len(platforms) + + if len(platforms) == 0 { + _ = s.touchInstallation(ctx, installation.ID) + return &MonitoringHeartbeatResponse{ + InstallationID: installation.ID, + BusinessDate: businessDate, + UpdatedPlatformCount: 0, + ReconciledSkippedTaskCount: 0, + }, nil + } + + platformIDs := monitoringHeartbeatPlatformIDs(platforms) + previousStates, err := s.loadPlatformAccessSnapshotStates(ctx, installation.TenantID, installation.ID, businessDate, platformIDs) + if err != nil { + return nil, err + } + + type heartbeatPlatformAction struct { + platform MonitoringHeartbeatPlatform + decision monitoringHeartbeatSnapshotDecision + } + + actions := make([]heartbeatPlatformAction, 0, len(platforms)) + inaccessiblePlatformIDs := make([]string, 0, len(platforms)) + needsTransaction := false + for _, platform := range platforms { + nextState := monitoringPlatformAccessSnapshotStateFromHeartbeat(platform) + decision := decideMonitoringHeartbeatSnapshot(previousStates[platform.AIPlatformID], nextState, now, isPrimaryInstallation) + if decision.WriteSnapshot { + needsTransaction = true + } else { + snapshotMetrics.SnapshotSkippedCount++ + } + if decision.PendingReconcileCandidate { + inaccessiblePlatformIDs = append(inaccessiblePlatformIDs, platform.AIPlatformID) + } + actions = append(actions, heartbeatPlatformAction{ + platform: platform, + decision: decision, + }) + } + + if isPrimaryInstallation && len(inaccessiblePlatformIDs) > 0 { + pendingPlatforms, pendingErr := s.loadPendingMonitoringTaskPlatformIDs(ctx, installation.TenantID, businessDate, inaccessiblePlatformIDs) + if pendingErr != nil { + return nil, pendingErr + } + for idx := range actions { + if !actions[idx].decision.PendingReconcileCandidate { + continue + } + if _, ok := pendingPlatforms[actions[idx].platform.AIPlatformID]; ok { + actions[idx].decision.ReconcilePending = true + needsTransaction = true + } + } + } + + if !needsTransaction { + _ = s.touchInstallation(ctx, installation.ID) + return &MonitoringHeartbeatResponse{ + InstallationID: installation.ID, + BusinessDate: businessDate, + UpdatedPlatformCount: len(platforms), + ReconciledSkippedTaskCount: 0, + }, nil + } tx, err := s.monitoringPool.Begin(ctx) if err != nil { return nil, response.ErrInternal(50041, "begin_failed", "failed to start monitoring heartbeat") } + snapshotMetrics.TransactionOpened = true defer tx.Rollback(ctx) var reconciledSkippedTaskCount int64 + var snapshotInsertedCount int64 + var snapshotUpdatedCount int64 + var snapshotRefreshedCount int64 + var projectionRebuildCount int64 affectedBrandIDs := make(map[int64]struct{}) - for _, platform := range platforms { - accessChanged, err := s.upsertPlatformAccessSnapshot(ctx, tx, installation, businessDate, platform) - if err != nil { - return nil, err + for _, action := range actions { + if action.decision.WriteSnapshot { + if err := s.upsertPlatformAccessSnapshot(ctx, tx, installation, businessDate, action.platform); err != nil { + return nil, err + } + switch action.decision.SnapshotOperation { + case "insert": + snapshotInsertedCount++ + case "update": + snapshotUpdatedCount++ + case "refresh": + snapshotRefreshedCount++ + } } if !isPrimaryInstallation { continue } - needsProjectionRefresh := accessChanged - if monitoringStringValue(platform.AccessStatus) != "accessible" { - count, reconcileErr := s.reconcilePendingTasksForPlatform(ctx, tx, installation.TenantID, businessDate, platform.AIPlatformID) + needsProjectionRefresh := action.decision.StateChanged + if action.decision.ReconcilePending { + count, reconcileErr := s.reconcilePendingTasksForPlatform(ctx, tx, installation.TenantID, businessDate, action.platform.AIPlatformID) if reconcileErr != nil { return nil, reconcileErr } @@ -486,7 +598,7 @@ func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationI continue } - brandIDs, loadErr := s.loadMonitoringAffectedBrandIDs(ctx, tx, installation.TenantID, businessDay, platform.AIPlatformID) + brandIDs, loadErr := s.loadMonitoringAffectedBrandIDs(ctx, tx, installation.TenantID, businessDay, action.platform.AIPlatformID) if loadErr != nil { return nil, loadErr } @@ -500,6 +612,7 @@ func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationI if err := s.rebuildBrandDailyProjection(ctx, tx, installation.TenantID, brandID, businessDay); err != nil { return nil, err } + projectionRebuildCount++ } } @@ -507,6 +620,12 @@ func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationI return nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring heartbeat") } + snapshotMetrics.SnapshotInsertedCount = snapshotInsertedCount + snapshotMetrics.SnapshotUpdatedCount = snapshotUpdatedCount + snapshotMetrics.SnapshotRefreshedCount = snapshotRefreshedCount + snapshotMetrics.ReconciledSkippedTaskCount = reconciledSkippedTaskCount + snapshotMetrics.ProjectionRebuildCount = projectionRebuildCount + _ = s.touchInstallation(ctx, installation.ID) if isPrimaryInstallation && len(affectedBrandIDs) > 0 { s.dispatchMonitoringProjectionRebuilds(installation.TenantID, businessDay, affectedBrandIDs, "heartbeat") @@ -963,80 +1082,224 @@ func (s *MonitoringCallbackService) authenticateInstallation(ctx context.Context } func (s *MonitoringCallbackService) resolveHeartbeatPrimaryInstallation(ctx context.Context, tenantID int64, installation *monitoringInstallationIdentity) (bool, error) { - var preferred sql.NullString - err := s.businessPool.QueryRow(ctx, ` - SELECT primary_client_id::text - FROM tenant_monitoring_quotas - WHERE tenant_id = $1 - `, tenantID).Scan(&preferred) - if err != nil && err != pgx.ErrNoRows { - return false, response.ErrInternal(50041, "query_failed", "failed to inspect primary monitoring desktop client") + if installation == nil || strings.TrimSpace(installation.ID) == "" { + return false, nil + } + isPrimary, err := s.claimHeartbeatPrimaryLease(ctx, tenantID, installation.WorkspaceID, installation.ID) + if err != nil { + return false, err + } + return isPrimary, nil +} + +func (s *MonitoringCallbackService) claimHeartbeatPrimaryLease(ctx context.Context, tenantID, workspaceID int64, installationID string) (bool, error) { + startedAt := time.Now() + installationID = strings.TrimSpace(installationID) + leaseTTL := formatPgInterval(desktopClientPresenceTTL) + + state, err := s.loadHeartbeatPrimaryLeaseState(ctx, tenantID, workspaceID, installationID) + if err != nil { + recordHeartbeatPrimaryLeaseMetric("error", time.Since(startedAt)) + return false, err + } + if !state.Found { + inserted, insertErr := s.tryInsertHeartbeatPrimaryLease(ctx, tenantID, workspaceID, installationID, leaseTTL) + if insertErr != nil { + recordHeartbeatPrimaryLeaseMetric("error", time.Since(startedAt)) + return false, insertErr + } + if inserted { + recordHeartbeatPrimaryLeaseMetric("inserted", time.Since(startedAt)) + recordHeartbeatPrimaryLeaseWriteMetric("insert") + return true, nil + } + recordHeartbeatPrimaryLeaseMetric("contention", time.Since(startedAt)) + return s.resolveHeartbeatPrimaryAfterContention(ctx, tenantID, workspaceID, installationID) } - if !preferred.Valid || preferred.String == installation.ID { - if !preferred.Valid { - if err := s.persistHeartbeatPrimaryInstallationID(ctx, tenantID, installation); err != nil { - return false, err - } - } + decision := decideHeartbeatPrimaryLease(state) + switch decision.Action { + case heartbeatPrimaryLeaseActionPrimaryHit: + recordHeartbeatPrimaryLeaseMetric("primary_hit", time.Since(startedAt)) return true, nil - } - - online, err := s.isMonitoringInstallationRecentlyOnline(ctx, tenantID, installation.WorkspaceID, preferred.String) - if err != nil { - return false, err - } - if online { + case heartbeatPrimaryLeaseActionNonPrimaryLive: + recordHeartbeatPrimaryLeaseMetric("non_primary_live", time.Since(startedAt)) return false, nil - } - - if err := s.persistHeartbeatPrimaryInstallationID(ctx, tenantID, installation); err != nil { - return false, err - } - return true, nil -} - -func (s *MonitoringCallbackService) isMonitoringInstallationRecentlyOnline(ctx context.Context, tenantID, workspaceID int64, installationID string) (bool, error) { - var lastSeenAt sql.NullTime - err := s.businessPool.QueryRow(ctx, ` - SELECT last_seen_at - FROM desktop_clients - WHERE id = $1::uuid - AND tenant_id = $2 - AND workspace_id = $3 - AND revoked_at IS NULL - `, strings.TrimSpace(installationID), tenantID, workspaceID).Scan(&lastSeenAt) - if err != nil { - if err == pgx.ErrNoRows { - return false, nil + case heartbeatPrimaryLeaseActionRenew: + updated, updateErr := s.renewHeartbeatPrimaryLease(ctx, tenantID, workspaceID, installationID, leaseTTL) + if updateErr != nil { + recordHeartbeatPrimaryLeaseMetric("error", time.Since(startedAt)) + return false, updateErr + } + if updated { + recordHeartbeatPrimaryLeaseMetric("renewed", time.Since(startedAt)) + recordHeartbeatPrimaryLeaseWriteMetric("renew") + return true, nil + } + case heartbeatPrimaryLeaseActionClaimExpired, heartbeatPrimaryLeaseActionClaimUnavailable: + updated, updateErr := s.claimExpiredHeartbeatPrimaryLease(ctx, tenantID, workspaceID, installationID, state.PrimaryInstallationID, leaseTTL, decision.Action) + if updateErr != nil { + recordHeartbeatPrimaryLeaseMetric("error", time.Since(startedAt)) + return false, updateErr + } + if updated { + if decision.Action == heartbeatPrimaryLeaseActionClaimExpired { + recordHeartbeatPrimaryLeaseMetric("claimed_expired", time.Since(startedAt)) + } else { + recordHeartbeatPrimaryLeaseMetric("claimed_unavailable", time.Since(startedAt)) + } + recordHeartbeatPrimaryLeaseWriteMetric("claim") + return true, nil } - return false, response.ErrInternal(50041, "query_failed", "failed to inspect monitoring desktop client heartbeat") } - if !lastSeenAt.Valid { - return false, nil - } - return time.Since(lastSeenAt.Time) <= monitoringOnlineDuration, nil + recordHeartbeatPrimaryLeaseMetric("contention", time.Since(startedAt)) + return s.resolveHeartbeatPrimaryAfterContention(ctx, tenantID, workspaceID, installationID) } -func (s *MonitoringCallbackService) persistHeartbeatPrimaryInstallationID(ctx context.Context, tenantID int64, installation *monitoringInstallationIdentity) error { - if _, err := s.businessPool.Exec(ctx, ` - INSERT INTO tenant_monitoring_quotas ( - tenant_id, workspace_id, primary_client_id +func (s *MonitoringCallbackService) tryInsertHeartbeatPrimaryLease(ctx context.Context, tenantID, workspaceID int64, installationID, leaseTTL string) (bool, error) { + tag, err := s.businessPool.Exec(ctx, ` + INSERT INTO desktop_client_primary_leases ( + tenant_id, + workspace_id, + primary_client_id, + lease_expires_at ) - VALUES ( - $1, - $2, - $3::uuid - ) - ON CONFLICT (tenant_id) DO UPDATE - SET workspace_id = EXCLUDED.workspace_id, - primary_client_id = EXCLUDED.primary_client_id, - updated_at = NOW() - `, tenantID, installation.WorkspaceID, installation.ID); err != nil { - return response.ErrInternal(50041, "update_failed", "failed to persist primary monitoring desktop client") + VALUES ($1, $2, $3::uuid, NOW() + $4::interval) + ON CONFLICT (tenant_id, workspace_id) DO NOTHING + `, tenantID, workspaceID, installationID, leaseTTL) + if err != nil { + return false, response.ErrInternal(50041, "primary_lease_insert_failed", "failed to initialize monitoring desktop primary lease") } - return nil + return tag.RowsAffected() > 0, nil +} + +type heartbeatPrimaryLeaseState struct { + Found bool + PrimaryInstallationID string + CurrentIsPrimary bool + LeaseLive bool + RenewDue bool + PrimaryAvailable bool +} + +func (s *MonitoringCallbackService) loadHeartbeatPrimaryLeaseState(ctx context.Context, tenantID, workspaceID int64, installationID string) (heartbeatPrimaryLeaseState, error) { + var state heartbeatPrimaryLeaseState + renewAfter := formatPgInterval(monitoringHeartbeatPrimaryRenewAfter) + if err := s.businessPool.QueryRow(ctx, ` + SELECT + l.primary_client_id::text, + l.primary_client_id = $3::uuid AS current_is_primary, + l.lease_expires_at > NOW() AS lease_live, + l.updated_at <= NOW() - $4::interval AS renew_due, + dc.id IS NOT NULL AND dc.revoked_at IS NULL AS primary_available + FROM desktop_client_primary_leases l + LEFT JOIN desktop_clients dc ON dc.id = l.primary_client_id + WHERE l.tenant_id = $1 + AND l.workspace_id = $2 + `, tenantID, workspaceID, installationID, renewAfter).Scan( + &state.PrimaryInstallationID, + &state.CurrentIsPrimary, + &state.LeaseLive, + &state.RenewDue, + &state.PrimaryAvailable, + ); err != nil { + if err == pgx.ErrNoRows { + return state, nil + } + return state, response.ErrInternal(50041, "query_failed", "failed to inspect monitoring desktop primary lease") + } + state.Found = true + return state, nil +} + +type heartbeatPrimaryLeaseAction string + +const ( + heartbeatPrimaryLeaseActionPrimaryHit heartbeatPrimaryLeaseAction = "primary_hit" + heartbeatPrimaryLeaseActionNonPrimaryLive heartbeatPrimaryLeaseAction = "non_primary_live" + heartbeatPrimaryLeaseActionRenew heartbeatPrimaryLeaseAction = "renew" + heartbeatPrimaryLeaseActionClaimExpired heartbeatPrimaryLeaseAction = "claim_expired" + heartbeatPrimaryLeaseActionClaimUnavailable heartbeatPrimaryLeaseAction = "claim_unavailable" +) + +type heartbeatPrimaryLeaseDecision struct { + Action heartbeatPrimaryLeaseAction +} + +func decideHeartbeatPrimaryLease(state heartbeatPrimaryLeaseState) heartbeatPrimaryLeaseDecision { + if !state.Found { + return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionNonPrimaryLive} + } + if state.CurrentIsPrimary { + if !state.LeaseLive || !state.PrimaryAvailable || state.RenewDue { + return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionRenew} + } + return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionPrimaryHit} + } + if !state.LeaseLive { + return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionClaimExpired} + } + if !state.PrimaryAvailable { + return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionClaimUnavailable} + } + return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionNonPrimaryLive} +} + +func (s *MonitoringCallbackService) renewHeartbeatPrimaryLease(ctx context.Context, tenantID, workspaceID int64, installationID, leaseTTL string) (bool, error) { + tag, err := s.businessPool.Exec(ctx, ` + UPDATE desktop_client_primary_leases + SET lease_expires_at = NOW() + $4::interval, + updated_at = NOW() + WHERE tenant_id = $1 + AND workspace_id = $2 + AND primary_client_id = $3::uuid + `, tenantID, workspaceID, installationID, leaseTTL) + if err != nil { + return false, response.ErrInternal(50041, "primary_lease_update_failed", "failed to renew monitoring desktop primary lease") + } + return tag.RowsAffected() > 0, nil +} + +func (s *MonitoringCallbackService) claimExpiredHeartbeatPrimaryLease( + ctx context.Context, + tenantID, workspaceID int64, + installationID string, + previousPrimaryInstallationID string, + leaseTTL string, + action heartbeatPrimaryLeaseAction, +) (bool, error) { + claimPredicate := "lease_expires_at <= NOW()" + if action == heartbeatPrimaryLeaseActionClaimUnavailable { + claimPredicate = `NOT EXISTS ( + SELECT 1 + FROM desktop_clients dc + WHERE dc.id = desktop_client_primary_leases.primary_client_id + AND dc.revoked_at IS NULL + )` + } + query := ` + UPDATE desktop_client_primary_leases + SET primary_client_id = $3::uuid, + lease_expires_at = NOW() + $5::interval, + updated_at = NOW() + WHERE tenant_id = $1 + AND workspace_id = $2 + AND primary_client_id = $4::uuid + AND ` + claimPredicate + tag, err := s.businessPool.Exec(ctx, query, tenantID, workspaceID, installationID, previousPrimaryInstallationID, leaseTTL) + if err != nil { + return false, response.ErrInternal(50041, "primary_lease_update_failed", "failed to claim monitoring desktop primary lease") + } + return tag.RowsAffected() > 0, nil +} + +func (s *MonitoringCallbackService) resolveHeartbeatPrimaryAfterContention(ctx context.Context, tenantID, workspaceID int64, installationID string) (bool, error) { + state, err := s.loadHeartbeatPrimaryLeaseState(ctx, tenantID, workspaceID, installationID) + if err != nil { + return false, err + } + return state.CurrentIsPrimary && state.LeaseLive && state.PrimaryAvailable, nil } func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantID, taskID int64) (*monitoringCollectTask, error) { @@ -1464,20 +1727,7 @@ func (s *MonitoringCallbackService) markTaskResultReceived(ctx context.Context, return nil } -func (s *MonitoringCallbackService) upsertPlatformAccessSnapshot(ctx context.Context, tx pgx.Tx, installation *monitoringInstallationIdentity, businessDate string, platform MonitoringHeartbeatPlatform) (bool, error) { - previous, err := s.loadPlatformAccessSnapshotState(ctx, tx, installation.TenantID, installation.ID, businessDate, platform.AIPlatformID) - if err != nil { - return false, err - } - - nextState := monitoringPlatformAccessSnapshotState{ - AccessStatus: monitoringStringValue(platform.AccessStatus), - Connected: derefBool(platform.Connected), - Accessible: derefBool(platform.Accessible), - ReasonCode: optionalString(monitoringStringValue(platform.ReasonCode)), - ReasonMessage: optionalString(monitoringStringValue(platform.ReasonMessage)), - } - +func (s *MonitoringCallbackService) upsertPlatformAccessSnapshot(ctx context.Context, tx pgx.Tx, installation *monitoringInstallationIdentity, businessDate string, platform MonitoringHeartbeatPlatform) error { if _, err := tx.Exec(ctx, ` INSERT INTO monitoring_platform_access_snapshots ( tenant_id, client_id, ai_platform_id, business_date, access_status, @@ -1498,44 +1748,101 @@ func (s *MonitoringCallbackService) upsertPlatformAccessSnapshot(ctx context.Con installation.ID, platform.AIPlatformID, businessDate, - platform.AccessStatus, + monitoringStringValue(platform.AccessStatus), derefBool(platform.Connected), derefBool(platform.Accessible), resolveHeartbeatDetectedAt(platform.DetectedAt), nullableString(platform.ReasonCode), nullableString(platform.ReasonMessage), ); err != nil { - return false, response.ErrInternal(50041, "access_snapshot_upsert_failed", "failed to persist monitoring platform access snapshot") + return response.ErrInternal(50041, "access_snapshot_upsert_failed", "failed to persist monitoring platform access snapshot") } - return hasPlatformAccessSnapshotChanged(previous, nextState), nil + return nil } -func (s *MonitoringCallbackService) loadPlatformAccessSnapshotState(ctx context.Context, tx pgx.Tx, tenantID int64, installationID string, businessDate, aiPlatformID string) (*monitoringPlatformAccessSnapshotState, error) { - item := &monitoringPlatformAccessSnapshotState{} - var reasonCode sql.NullString - var reasonMessage sql.NullString - if err := tx.QueryRow(ctx, ` - SELECT access_status, connected, accessible, reason_code, reason_message +func (s *MonitoringCallbackService) loadPlatformAccessSnapshotStates(ctx context.Context, tenantID int64, installationID string, businessDate string, aiPlatformIDs []string) (map[string]*monitoringPlatformAccessSnapshotState, error) { + states := make(map[string]*monitoringPlatformAccessSnapshotState, len(aiPlatformIDs)) + if len(aiPlatformIDs) == 0 { + return states, nil + } + + rows, err := s.monitoringPool.Query(ctx, ` + SELECT ai_platform_id, access_status, connected, accessible, reason_code, reason_message, updated_at FROM monitoring_platform_access_snapshots WHERE tenant_id = $1 AND client_id = $2::uuid - AND ai_platform_id = $3 - AND business_date = $4::date - `, tenantID, installationID, aiPlatformID, businessDate).Scan( - &item.AccessStatus, - &item.Connected, - &item.Accessible, - &reasonCode, - &reasonMessage, - ); err != nil { - if err == pgx.ErrNoRows { - return nil, nil - } - return nil, response.ErrInternal(50041, "access_snapshot_lookup_failed", "failed to inspect monitoring platform access snapshot") + AND business_date = $3::date + AND ai_platform_id = ANY($4::text[]) + `, tenantID, installationID, businessDate, aiPlatformIDs) + if err != nil { + return nil, response.ErrInternal(50041, "access_snapshot_lookup_failed", "failed to inspect monitoring platform access snapshots") + } + defer rows.Close() + + for rows.Next() { + item := &monitoringPlatformAccessSnapshotState{} + var platformID string + var reasonCode sql.NullString + var reasonMessage sql.NullString + if scanErr := rows.Scan( + &platformID, + &item.AccessStatus, + &item.Connected, + &item.Accessible, + &reasonCode, + &reasonMessage, + &item.UpdatedAt, + ); scanErr != nil { + return nil, response.ErrInternal(50041, "access_snapshot_scan_failed", "failed to parse monitoring platform access snapshots") + } + if reasonCode.Valid { + item.ReasonCode = optionalString(reasonCode.String) + } + if reasonMessage.Valid { + item.ReasonMessage = optionalString(reasonMessage.String) + } + states[platformID] = item + } + if err := rows.Err(); err != nil { + return nil, response.ErrInternal(50041, "access_snapshot_scan_failed", "failed to iterate monitoring platform access snapshots") + } + + return states, nil +} + +func monitoringPlatformAccessSnapshotStateFromHeartbeat(platform MonitoringHeartbeatPlatform) monitoringPlatformAccessSnapshotState { + return monitoringPlatformAccessSnapshotState{ + AccessStatus: monitoringStringValue(platform.AccessStatus), + Connected: derefBool(platform.Connected), + Accessible: derefBool(platform.Accessible), + ReasonCode: optionalString(monitoringStringValue(platform.ReasonCode)), + ReasonMessage: optionalString(monitoringStringValue(platform.ReasonMessage)), + } +} + +func decideMonitoringHeartbeatSnapshot(previous *monitoringPlatformAccessSnapshotState, next monitoringPlatformAccessSnapshotState, now time.Time, isPrimary bool) monitoringHeartbeatSnapshotDecision { + stateChanged := hasPlatformAccessSnapshotChanged(previous, next) + refreshDue := false + if previous != nil && !stateChanged { + refreshDue = previous.UpdatedAt.IsZero() || !previous.UpdatedAt.After(now.Add(-monitoringHeartbeatAccessSnapshotRefreshAfter)) + } + + operation := "skip" + if previous == nil { + operation = "insert" + } else if stateChanged { + operation = "update" + } else if refreshDue { + operation = "refresh" + } + + return monitoringHeartbeatSnapshotDecision{ + WriteSnapshot: stateChanged || refreshDue, + StateChanged: stateChanged, + RefreshDue: refreshDue, + PendingReconcileCandidate: isPrimary && next.AccessStatus != "accessible", + SnapshotOperation: operation, } - item.ReasonCode = optionalString(reasonCode.String) - item.ReasonMessage = optionalString(reasonMessage.String) - return item, nil } func hasPlatformAccessSnapshotChanged(previous *monitoringPlatformAccessSnapshotState, next monitoringPlatformAccessSnapshotState) bool { @@ -1558,6 +1865,41 @@ func hasPlatformAccessSnapshotChanged(previous *monitoringPlatformAccessSnapshot return false } +func (s *MonitoringCallbackService) loadPendingMonitoringTaskPlatformIDs(ctx context.Context, tenantID int64, businessDate string, aiPlatformIDs []string) (map[string]struct{}, error) { + result := make(map[string]struct{}, len(aiPlatformIDs)) + if len(aiPlatformIDs) == 0 { + return result, nil + } + + rows, err := s.monitoringPool.Query(ctx, ` + SELECT ai_platform_id + FROM monitoring_collect_tasks + WHERE tenant_id = $1 + AND ai_platform_id = ANY($2::text[]) + AND collector_type = $3 + AND business_date = $4::date + AND status = 'pending' + GROUP BY ai_platform_id + `, tenantID, aiPlatformIDs, monitoringCollectorType, businessDate) + if err != nil { + return nil, response.ErrInternal(50041, "pending_task_platform_lookup_failed", "failed to inspect pending monitoring task platforms") + } + defer rows.Close() + + for rows.Next() { + var platformID string + if scanErr := rows.Scan(&platformID); scanErr != nil { + return nil, response.ErrInternal(50041, "pending_task_platform_scan_failed", "failed to parse pending monitoring task platforms") + } + result[platformID] = struct{}{} + } + if err := rows.Err(); err != nil { + return nil, response.ErrInternal(50041, "pending_task_platform_scan_failed", "failed to iterate pending monitoring task platforms") + } + + return result, nil +} + func (s *MonitoringCallbackService) reconcilePendingTasksForPlatform(ctx context.Context, tx pgx.Tx, tenantID int64, businessDate, aiPlatformID string) (int64, error) { tag, err := tx.Exec(ctx, ` UPDATE monitoring_collect_tasks @@ -2530,6 +2872,20 @@ func normalizeMonitoringLeasePlatformIDs(values []string) []string { return result } +func monitoringHeartbeatPlatformIDs(platforms []MonitoringHeartbeatPlatform) []string { + if len(platforms) == 0 { + return nil + } + result := make([]string, 0, len(platforms)) + for _, platform := range platforms { + platformID := strings.TrimSpace(platform.AIPlatformID) + if platformID != "" { + result = append(result, platformID) + } + } + return result +} + func normalizeHeartbeatPlatforms(items []MonitoringHeartbeatPlatform, fallbackDetectedAt time.Time) ([]MonitoringHeartbeatPlatform, error) { if len(items) == 0 { return nil, nil diff --git a/server/internal/tenant/app/monitoring_callback_service_test.go b/server/internal/tenant/app/monitoring_callback_service_test.go index 3a9409b..dc133e8 100644 --- a/server/internal/tenant/app/monitoring_callback_service_test.go +++ b/server/internal/tenant/app/monitoring_callback_service_test.go @@ -2,9 +2,287 @@ package app import ( "encoding/json" + "errors" + "strings" "testing" + "time" ) +func TestDecideHeartbeatPrimaryLease(t *testing.T) { + tests := []struct { + name string + state heartbeatPrimaryLeaseState + want heartbeatPrimaryLeaseAction + }{ + { + name: "current primary fast path", + state: heartbeatPrimaryLeaseState{ + Found: true, + CurrentIsPrimary: true, + LeaseLive: true, + PrimaryAvailable: true, + }, + want: heartbeatPrimaryLeaseActionPrimaryHit, + }, + { + name: "current primary renews when due", + state: heartbeatPrimaryLeaseState{ + Found: true, + CurrentIsPrimary: true, + LeaseLive: true, + RenewDue: true, + PrimaryAvailable: true, + }, + want: heartbeatPrimaryLeaseActionRenew, + }, + { + name: "non primary does not steal live lease", + state: heartbeatPrimaryLeaseState{ + Found: true, + CurrentIsPrimary: false, + LeaseLive: true, + PrimaryAvailable: true, + }, + want: heartbeatPrimaryLeaseActionNonPrimaryLive, + }, + { + name: "expired lease can be claimed", + state: heartbeatPrimaryLeaseState{ + Found: true, + CurrentIsPrimary: false, + LeaseLive: false, + PrimaryAvailable: true, + }, + want: heartbeatPrimaryLeaseActionClaimExpired, + }, + { + name: "revoked primary can be replaced", + state: heartbeatPrimaryLeaseState{ + Found: true, + CurrentIsPrimary: false, + LeaseLive: true, + PrimaryAvailable: false, + }, + want: heartbeatPrimaryLeaseActionClaimUnavailable, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := decideHeartbeatPrimaryLease(tt.state) + if got.Action != tt.want { + t.Fatalf("decideHeartbeatPrimaryLease() = %v, want %v", got.Action, tt.want) + } + }) + } +} + +func TestMonitoringHeartbeatPrimaryLeaseMetrics(t *testing.T) { + resetMonitoringHeartbeatPrimaryLeaseMetricsForTest() + t.Cleanup(resetMonitoringHeartbeatPrimaryLeaseMetricsForTest) + + recordHeartbeatPrimaryLeaseMetric("primary_hit", 20*time.Millisecond) + recordHeartbeatPrimaryLeaseMetric("contention", 10*time.Millisecond) + recordHeartbeatPrimaryLeaseWriteMetric("renew") + + snapshot := MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue() + if snapshot.PrimaryHitTotal != 1 { + t.Fatalf("PrimaryHitTotal = %d, want 1", snapshot.PrimaryHitTotal) + } + if snapshot.ContentionTotal != 1 { + t.Fatalf("ContentionTotal = %d, want 1", snapshot.ContentionTotal) + } + if snapshot.DBRenewTotal != 1 { + t.Fatalf("DBRenewTotal = %d, want 1", snapshot.DBRenewTotal) + } + if snapshot.DurationCount != 2 { + t.Fatalf("DurationCount = %d, want 2", snapshot.DurationCount) + } + + prometheus := MonitoringHeartbeatPrimaryLeaseMetricsPrometheus() + if !strings.Contains(prometheus, `monitoring_heartbeat_primary_lease_resolutions_total{outcome="primary_hit"} 1`) { + t.Fatalf("prometheus output missing primary_hit metric: %s", prometheus) + } + if !strings.Contains(prometheus, `monitoring_heartbeat_primary_lease_db_writes_total{operation="renew"} 1`) { + t.Fatalf("prometheus output missing renew write metric: %s", prometheus) + } +} + +func TestDecideMonitoringHeartbeatSnapshot(t *testing.T) { + now := time.Date(2026, 4, 24, 3, 0, 0, 0, time.UTC) + base := monitoringPlatformAccessSnapshotState{ + AccessStatus: "accessible", + Connected: true, + Accessible: true, + UpdatedAt: now.Add(-time.Minute), + } + + tests := []struct { + name string + previous *monitoringPlatformAccessSnapshotState + next monitoringPlatformAccessSnapshotState + primary bool + want monitoringHeartbeatSnapshotDecision + }{ + { + name: "missing snapshot inserts", + next: base, + primary: true, + want: monitoringHeartbeatSnapshotDecision{ + WriteSnapshot: true, + StateChanged: true, + SnapshotOperation: "insert", + }, + }, + { + name: "fresh unchanged accessible snapshot skips write", + previous: &base, + next: base, + primary: true, + want: monitoringHeartbeatSnapshotDecision{ + SnapshotOperation: "skip", + }, + }, + { + name: "stale unchanged snapshot refreshes", + previous: &monitoringPlatformAccessSnapshotState{ + AccessStatus: "accessible", + Connected: true, + Accessible: true, + UpdatedAt: now.Add(-monitoringHeartbeatAccessSnapshotRefreshAfter - time.Second), + }, + next: base, + primary: true, + want: monitoringHeartbeatSnapshotDecision{ + WriteSnapshot: true, + RefreshDue: true, + SnapshotOperation: "refresh", + }, + }, + { + name: "changed snapshot updates", + previous: &base, + next: monitoringPlatformAccessSnapshotState{ + AccessStatus: "unavailable", + Connected: true, + Accessible: false, + }, + primary: true, + want: monitoringHeartbeatSnapshotDecision{ + WriteSnapshot: true, + StateChanged: true, + PendingReconcileCandidate: true, + SnapshotOperation: "update", + }, + }, + { + name: "fresh unchanged primary inaccessible snapshot checks pending tasks without writing snapshot", + previous: &monitoringPlatformAccessSnapshotState{ + AccessStatus: "not_logged_in", + Connected: false, + Accessible: false, + UpdatedAt: now.Add(-time.Minute), + }, + next: monitoringPlatformAccessSnapshotState{ + AccessStatus: "not_logged_in", + Connected: false, + Accessible: false, + }, + primary: true, + want: monitoringHeartbeatSnapshotDecision{ + PendingReconcileCandidate: true, + SnapshotOperation: "skip", + }, + }, + { + name: "non-primary inaccessible snapshot does not reconcile", + previous: &monitoringPlatformAccessSnapshotState{ + AccessStatus: "not_logged_in", + Connected: false, + Accessible: false, + UpdatedAt: now.Add(-time.Minute), + }, + next: monitoringPlatformAccessSnapshotState{ + AccessStatus: "not_logged_in", + Connected: false, + Accessible: false, + }, + primary: false, + want: monitoringHeartbeatSnapshotDecision{ + SnapshotOperation: "skip", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := decideMonitoringHeartbeatSnapshot(tt.previous, tt.next, now, tt.primary) + if got.WriteSnapshot != tt.want.WriteSnapshot || + got.StateChanged != tt.want.StateChanged || + got.RefreshDue != tt.want.RefreshDue || + got.PendingReconcileCandidate != tt.want.PendingReconcileCandidate || + got.ReconcilePending != tt.want.ReconcilePending || + got.SnapshotOperation != tt.want.SnapshotOperation { + t.Fatalf("decideMonitoringHeartbeatSnapshot() = %+v, want %+v", got, tt.want) + } + }) + } +} + +func TestMonitoringHeartbeatSnapshotMetrics(t *testing.T) { + resetMonitoringHeartbeatSnapshotMetricsForTest() + t.Cleanup(resetMonitoringHeartbeatSnapshotMetricsForTest) + + recordMonitoringHeartbeatSnapshotMetrics(monitoringHeartbeatSnapshotMetricsEvent{ + PlatformCount: 3, + TransactionOpened: true, + SnapshotInsertedCount: 1, + SnapshotSkippedCount: 2, + ReconciledSkippedTaskCount: 5, + ProjectionRebuildCount: 2, + Duration: 20 * time.Millisecond, + }) + recordMonitoringHeartbeatSnapshotMetrics(monitoringHeartbeatSnapshotMetricsEvent{ + PlatformCount: 1, + Duration: 10 * time.Millisecond, + Err: errors.New("snapshot failed"), + }) + + snapshot := MonitoringHeartbeatSnapshotMetricsSnapshotValue() + if snapshot.PlatformReportedTotal != 4 { + t.Fatalf("PlatformReportedTotal = %d, want 4", snapshot.PlatformReportedTotal) + } + if snapshot.TransactionOpenedTotal != 1 { + t.Fatalf("TransactionOpenedTotal = %d, want 1", snapshot.TransactionOpenedTotal) + } + if snapshot.SnapshotInsertedTotal != 1 { + t.Fatalf("SnapshotInsertedTotal = %d, want 1", snapshot.SnapshotInsertedTotal) + } + if snapshot.SnapshotSkippedTotal != 2 { + t.Fatalf("SnapshotSkippedTotal = %d, want 2", snapshot.SnapshotSkippedTotal) + } + if snapshot.ReconciledSkippedTaskTotal != 5 { + t.Fatalf("ReconciledSkippedTaskTotal = %d, want 5", snapshot.ReconciledSkippedTaskTotal) + } + if snapshot.ProjectionRebuildTotal != 2 { + t.Fatalf("ProjectionRebuildTotal = %d, want 2", snapshot.ProjectionRebuildTotal) + } + if snapshot.ErrorTotal != 1 { + t.Fatalf("ErrorTotal = %d, want 1", snapshot.ErrorTotal) + } + if snapshot.DurationCount != 2 { + t.Fatalf("DurationCount = %d, want 2", snapshot.DurationCount) + } + + prometheus := MonitoringHeartbeatSnapshotMetricsPrometheus() + if !strings.Contains(prometheus, `monitoring_heartbeat_snapshot_writes_total{operation="insert"} 1`) { + t.Fatalf("prometheus output missing insert write metric: %s", prometheus) + } + if !strings.Contains(prometheus, "monitoring_heartbeat_snapshot_transactions_total 1") { + t.Fatalf("prometheus output missing transaction metric: %s", prometheus) + } +} + func TestBuildMonitoringRawPayloadKimiExcludesSearchResultsFromCitationInputs(t *testing.T) { citationTitle := "真实引用文章" searchTitle := "搜索网页结果" diff --git a/server/internal/tenant/app/monitoring_daily_task_metrics.go b/server/internal/tenant/app/monitoring_daily_task_metrics.go new file mode 100644 index 0000000..fdd20a2 --- /dev/null +++ b/server/internal/tenant/app/monitoring_daily_task_metrics.go @@ -0,0 +1,201 @@ +package app + +import ( + "context" + "errors" + "fmt" + "strings" + "sync/atomic" + "time" + + "github.com/geo-platform/tenant-api/internal/shared/response" +) + +type MonitoringDailyTaskMetricsSnapshot struct { + RunTotal int64 `json:"run_total"` + RunCompletedTotal int64 `json:"run_completed_total"` + RunFailedTotal int64 `json:"run_failed_total"` + RunPartialFailureTotal int64 `json:"run_partial_failure_total"` + PlanFailureTotal int64 `json:"plan_failure_total"` + BrandFailureTotal int64 `json:"brand_failure_total"` + SkippedPlanTotal int64 `json:"skipped_plan_total"` + PlannedTaskTotal int64 `json:"planned_task_total"` + DueTaskTotal int64 `json:"due_task_total"` + CreatedTaskTotal int64 `json:"created_task_total"` + DesktopTaskTotal int64 `json:"desktop_task_total"` + DeferredTaskTotal int64 `json:"deferred_task_total"` + LastRunAt *time.Time `json:"last_run_at,omitempty"` + LastSuccessAt *time.Time `json:"last_success_at,omitempty"` + LastErrorAt *time.Time `json:"last_error_at,omitempty"` + LastDurationSeconds float64 `json:"last_duration_seconds"` + LastBusinessDate string `json:"last_business_date,omitempty"` + LastTenantCount int64 `json:"last_tenant_count"` + LastBrandCount int64 `json:"last_brand_count"` + LastErrorCode string `json:"last_error_code,omitempty"` + LastErrorMessage string `json:"last_error_message,omitempty"` +} + +type monitoringDailyTaskMetricsRecorder struct { + runTotal atomic.Int64 + runCompletedTotal atomic.Int64 + runFailedTotal atomic.Int64 + runPartialFailureTotal atomic.Int64 + planFailureTotal atomic.Int64 + brandFailureTotal atomic.Int64 + skippedPlanTotal atomic.Int64 + plannedTaskTotal atomic.Int64 + dueTaskTotal atomic.Int64 + createdTaskTotal atomic.Int64 + desktopTaskTotal atomic.Int64 + deferredTaskTotal atomic.Int64 + lastRunAtUnixNano atomic.Int64 + lastSuccessAtUnixNano atomic.Int64 + lastErrorAtUnixNano atomic.Int64 + lastDurationNanos atomic.Int64 + lastTenantCount atomic.Int64 + lastBrandCount atomic.Int64 + lastBusinessDate atomic.Value + lastErrorCode atomic.Value + lastErrorMessage atomic.Value +} + +var monitoringDailyTaskMetrics = newMonitoringDailyTaskMetricsRecorder() + +func newMonitoringDailyTaskMetricsRecorder() *monitoringDailyTaskMetricsRecorder { + return &monitoringDailyTaskMetricsRecorder{} +} + +func recordMonitoringDailyTaskMetrics(summary *MonitoringDailyTaskGenerationSummary, duration time.Duration, err error) { + monitoringDailyTaskMetrics.record(summary, duration, err) +} + +func MonitoringDailyTaskMetricsSnapshotValue() MonitoringDailyTaskMetricsSnapshot { + return monitoringDailyTaskMetrics.snapshot() +} + +func resetMonitoringDailyTaskMetricsForTest() { + monitoringDailyTaskMetrics = newMonitoringDailyTaskMetricsRecorder() +} + +func (r *monitoringDailyTaskMetricsRecorder) record(summary *MonitoringDailyTaskGenerationSummary, duration time.Duration, err error) { + if r == nil { + return + } + now := time.Now().UTC() + if duration < 0 { + duration = 0 + } + + r.runTotal.Add(1) + r.lastRunAtUnixNano.Store(now.UnixNano()) + r.lastDurationNanos.Store(duration.Nanoseconds()) + + if summary != nil { + r.lastBusinessDate.Store(summary.BusinessDate) + r.lastTenantCount.Store(int64(summary.TenantCount)) + r.lastBrandCount.Store(int64(summary.BrandCount)) + r.planFailureTotal.Add(int64(summary.FailedPlanCount)) + r.brandFailureTotal.Add(int64(summary.FailedBrandCount)) + r.skippedPlanTotal.Add(int64(summary.SkippedPlanCount)) + r.plannedTaskTotal.Add(summary.PlannedTaskCount) + r.dueTaskTotal.Add(summary.DueTaskCount) + r.createdTaskTotal.Add(summary.CreatedTaskCount) + r.desktopTaskTotal.Add(summary.DesktopTaskCount) + r.deferredTaskTotal.Add(summary.DeferredTaskCount) + if summary.FailedPlanCount > 0 || summary.FailedBrandCount > 0 { + r.runPartialFailureTotal.Add(1) + } + } + + if err != nil { + code, message := monitoringDailyMetricError(err) + r.runFailedTotal.Add(1) + r.lastErrorAtUnixNano.Store(now.UnixNano()) + r.lastErrorCode.Store(code) + r.lastErrorMessage.Store(message) + return + } + + r.runCompletedTotal.Add(1) + r.lastSuccessAtUnixNano.Store(now.UnixNano()) +} + +func (r *monitoringDailyTaskMetricsRecorder) snapshot() MonitoringDailyTaskMetricsSnapshot { + if r == nil { + return MonitoringDailyTaskMetricsSnapshot{} + } + return MonitoringDailyTaskMetricsSnapshot{ + RunTotal: r.runTotal.Load(), + RunCompletedTotal: r.runCompletedTotal.Load(), + RunFailedTotal: r.runFailedTotal.Load(), + RunPartialFailureTotal: r.runPartialFailureTotal.Load(), + PlanFailureTotal: r.planFailureTotal.Load(), + BrandFailureTotal: r.brandFailureTotal.Load(), + SkippedPlanTotal: r.skippedPlanTotal.Load(), + PlannedTaskTotal: r.plannedTaskTotal.Load(), + DueTaskTotal: r.dueTaskTotal.Load(), + CreatedTaskTotal: r.createdTaskTotal.Load(), + DesktopTaskTotal: r.desktopTaskTotal.Load(), + DeferredTaskTotal: r.deferredTaskTotal.Load(), + LastRunAt: monitoringDailyUnixNanoTime(r.lastRunAtUnixNano.Load()), + LastSuccessAt: monitoringDailyUnixNanoTime(r.lastSuccessAtUnixNano.Load()), + LastErrorAt: monitoringDailyUnixNanoTime(r.lastErrorAtUnixNano.Load()), + LastDurationSeconds: float64(r.lastDurationNanos.Load()) / float64(time.Second), + LastBusinessDate: monitoringDailyLoadString(&r.lastBusinessDate), + LastTenantCount: r.lastTenantCount.Load(), + LastBrandCount: r.lastBrandCount.Load(), + LastErrorCode: monitoringDailyLoadString(&r.lastErrorCode), + LastErrorMessage: monitoringDailyLoadString(&r.lastErrorMessage), + } +} + +func monitoringDailyMetricError(err error) (string, string) { + normalized := normalizeMonitoringWorkerError(err) + if normalized == nil { + return "", "" + } + switch { + case errors.Is(normalized, context.Canceled): + return "context_canceled", normalized.Error() + case errors.Is(normalized, context.DeadlineExceeded): + return "context_deadline_exceeded", normalized.Error() + } + var appErr *response.AppError + if errors.As(normalized, &appErr) { + if strings.TrimSpace(appErr.Message) != "" { + return appErr.Message, appErr.Error() + } + return "app_error", appErr.Error() + } + code := strings.TrimLeft(fmt.Sprintf("%T", normalized), "*") + code = strings.ReplaceAll(code, "/", "_") + code = strings.ReplaceAll(code, ".", "_") + if code == "" { + code = "error" + } + return code, normalized.Error() +} + +func monitoringDailyUnixNanoTime(value int64) *time.Time { + if value <= 0 { + return nil + } + t := time.Unix(0, value).UTC() + return &t +} + +func monitoringDailyLoadString(value *atomic.Value) string { + if value == nil { + return "" + } + raw := value.Load() + result, _ := raw.(string) + return result +} + +func monitoringDailyTimeSeconds(value *time.Time) float64 { + if value == nil || value.IsZero() { + return 0 + } + return float64(value.UnixNano()) / float64(time.Second) +} diff --git a/server/internal/tenant/app/monitoring_daily_task_worker.go b/server/internal/tenant/app/monitoring_daily_task_worker.go new file mode 100644 index 0000000..9515b9a --- /dev/null +++ b/server/internal/tenant/app/monitoring_daily_task_worker.go @@ -0,0 +1,1295 @@ +package app + +import ( + "context" + "encoding/json" + "fmt" + "hash/fnv" + "sort" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "go.uber.org/zap" + + "github.com/geo-platform/tenant-api/internal/shared/config" + "github.com/geo-platform/tenant-api/internal/shared/response" + "github.com/geo-platform/tenant-api/internal/shared/stream" +) + +const ( + defaultMonitoringDailyTaskInterval = 5 * time.Minute + defaultMonitoringDailyTaskTimeout = 45 * time.Second + defaultMonitoringDailyTaskHardCap = 36 + defaultMonitoringDailyBrandLimit = 1 + defaultMonitoringDailyLookahead = 10 * time.Minute + defaultMonitoringDailyMaterializeWindow = 2 * time.Hour + defaultMonitoringDailyDesktopRetryCooldown = 15 * time.Minute + defaultMonitoringDailyPublishTimeout = 5 * time.Second + defaultMonitoringDailyMaxMaterializePerRun = 64 + defaultMonitoringDailyMaxMaterializePerPlan = 12 + defaultMonitoringDailyMaxMaterializePerBrand = 4 + monitoringDailyMaterializeStartOffset = 5 * time.Minute +) + +type MonitoringDailyTaskWorker struct { + service *MonitoringService + projectionService *MonitoringCallbackService + logger *zap.Logger + interval time.Duration + timeout time.Duration +} + +type MonitoringDailyTaskGenerationSummary struct { + BusinessDate string + TenantCount int + BrandCount int + FailedPlanCount int + FailedBrandCount int + PlannedTaskCount int64 + DueTaskCount int64 + CreatedTaskCount int64 + DesktopTaskCount int64 + DeferredTaskCount int64 + SkippedPlanCount int +} + +type monitoringDailyPlan struct { + TenantID int64 + WorkspaceID int64 + MaxBrands int + EnabledJSON []byte + PrimaryClientID *uuid.UUID + MaxQuestionsPerBrand int +} + +type monitoringDailyPlanClientCandidate struct { + TenantID int64 + WorkspaceID int64 + PrimaryClientID uuid.UUID + MaxBrands int + MaxQuestionsPerBrand int +} + +type monitoringDailyPlanClientKey struct { + TenantID int64 + WorkspaceID int64 + ClientID uuid.UUID +} + +type monitoringDailyBrand struct { + TenantID int64 + WorkspaceID int64 + BrandID int64 +} + +type monitoringDailyTaskCandidate struct { + Question monitoringConfiguredQuestion + Platform monitoringPlatformMetadata + MaterializeAt time.Time + DispatchAfter time.Time + sortKey uint64 +} + +type monitoringDailyTaskKey struct { + QuestionID int64 + PlatformID string +} + +func NewMonitoringDailyTaskWorker(service *MonitoringService, projectionService *MonitoringCallbackService, logger *zap.Logger) *MonitoringDailyTaskWorker { + return &MonitoringDailyTaskWorker{ + service: service, + projectionService: projectionService, + logger: logger, + interval: defaultMonitoringDailyTaskInterval, + timeout: defaultMonitoringDailyTaskTimeout, + } +} + +func (w *MonitoringDailyTaskWorker) Start(ctx context.Context) { + go w.Run(ctx) +} + +func (w *MonitoringDailyTaskWorker) Run(ctx context.Context) { + if w == nil || w.service == nil || w.service.businessPool == nil || w.service.monitoringPool == nil { + return + } + w.run(ctx) +} + +func (w *MonitoringDailyTaskWorker) run(ctx context.Context) { + w.runOnce(context.Background()) + + ticker := time.NewTicker(w.interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + w.runOnce(context.Background()) + } + } +} + +func (w *MonitoringDailyTaskWorker) runOnce(parent context.Context) { + ctx, cancel := context.WithTimeout(parent, w.timeout) + defer cancel() + + startedAt := time.Now() + summary, err := w.service.GenerateDailyMonitoringTasks(ctx, w.projectionService, time.Now().UTC()) + recordMonitoringDailyTaskMetrics(summary, time.Since(startedAt), err) + if err != nil { + if w.logger != nil { + w.logger.Warn("monitoring daily task generation failed", MonitoringWorkerErrorFields(err)...) + } + return + } + if summary == nil || w.logger == nil { + return + } + if summary.CreatedTaskCount == 0 && + summary.DesktopTaskCount == 0 && + summary.DeferredTaskCount == 0 && + summary.SkippedPlanCount == 0 && + summary.FailedPlanCount == 0 && + summary.FailedBrandCount == 0 { + return + } + w.logger.Info("monitoring daily task generation completed", + zap.String("business_date", summary.BusinessDate), + zap.Int("tenant_count", summary.TenantCount), + zap.Int("brand_count", summary.BrandCount), + zap.Int("skipped_plan_count", summary.SkippedPlanCount), + zap.Int("failed_plan_count", summary.FailedPlanCount), + zap.Int("failed_brand_count", summary.FailedBrandCount), + zap.Int64("planned_task_count", summary.PlannedTaskCount), + zap.Int64("due_task_count", summary.DueTaskCount), + zap.Int64("created_task_count", summary.CreatedTaskCount), + zap.Int64("desktop_task_count", summary.DesktopTaskCount), + zap.Int64("deferred_task_count", summary.DeferredTaskCount), + ) +} + +func (s *MonitoringService) GenerateDailyMonitoringTasks( + ctx context.Context, + projectionService *MonitoringCallbackService, + now time.Time, +) (*MonitoringDailyTaskGenerationSummary, error) { + if s == nil || s.businessPool == nil || s.monitoringPool == nil { + return nil, nil + } + + businessDay, businessDate := monitoringBusinessDayAndDateAt(now) + plans, err := s.loadDailyMonitoringPlans(ctx) + if err != nil { + return nil, err + } + plans = orderDailyMonitoringPlans(plans, businessDate, now) + + summary := &MonitoringDailyTaskGenerationSummary{ + BusinessDate: businessDate, + TenantCount: len(plans), + } + globalRemaining := defaultMonitoringDailyMaxMaterializePerRun + + for _, plan := range plans { + if globalRemaining <= 0 { + break + } + executionOwner, ok, skipReason := s.resolveDailyMonitoringExecutionOwner(plan) + if !ok { + summary.SkippedPlanCount++ + if s.logger != nil { + s.logger.Debug("monitoring daily task generation skipped", + zap.Int64("tenant_id", plan.TenantID), + zap.String("business_date", businessDate), + zap.String("reason", skipReason), + ) + } + continue + } + + brands, err := s.loadDailyMonitoringBrands(ctx, plan) + if err != nil { + if abortErr := monitoringDailyAbortErr(ctx); abortErr != nil { + return nil, abortErr + } + summary.FailedPlanCount++ + s.logDailyMonitoringPlanError("monitoring daily brand plan load failed", plan, businessDate, err) + continue + } + brands = orderDailyMonitoringBrands(brands, businessDate, now) + + enabledPlatformIDs, err := decodeDailyMonitoringEnabledPlatforms(plan.EnabledJSON) + if err != nil { + summary.FailedPlanCount++ + s.logDailyMonitoringPlanError("monitoring daily enabled platform config decode failed", plan, businessDate, err) + continue + } + platforms := resolveMonitoringPlatforms(enabledPlatformIDs) + if len(platforms) == 0 { + continue + } + + planRunRemaining := minMonitoringDailyInt(defaultMonitoringDailyMaxMaterializePerPlan, globalRemaining) + + for _, brand := range brands { + if planRunRemaining <= 0 || globalRemaining <= 0 { + break + } + + questions, err := s.loadConfiguredQuestions(ctx, plan.TenantID, brand.BrandID, nil) + if err != nil { + if abortErr := monitoringDailyAbortErr(ctx); abortErr != nil { + return nil, abortErr + } + summary.FailedBrandCount++ + s.logDailyMonitoringBrandError("monitoring daily question load failed", plan, brand, businessDate, err) + continue + } + questions = limitMonitoringDailyQuestions(questions, plan.MaxQuestionsPerBrand) + if len(questions) == 0 { + continue + } + dailyHardCap := len(questions) * len(platforms) + materializedCount, err := s.countDailyMonitoringMaterializedTasks(ctx, plan.TenantID, brand.BrandID, businessDay) + if err != nil { + if abortErr := monitoringDailyAbortErr(ctx); abortErr != nil { + return nil, abortErr + } + summary.FailedBrandCount++ + s.logDailyMonitoringBrandError("monitoring daily materialized task count failed", plan, brand, businessDate, err) + continue + } + dailyRemaining := monitoringDailyRemainingCapacity(dailyHardCap, materializedCount) + + candidates := buildMonitoringDailyTaskCandidates(plan.TenantID, brand.BrandID, businessDay, questions, platforms, dailyHardCap) + if len(candidates) == 0 { + continue + } + + dispatchLimit := minMonitoringDailyInt(defaultMonitoringDailyMaxMaterializePerBrand, planRunRemaining, globalRemaining) + materializeLimit := minMonitoringDailyInt(dispatchLimit, dailyRemaining) + createdCount, dueCount, desktopCount, deferredCount, err := s.generateDailyMonitoringTasksForBrand( + ctx, + projectionService, + plan, + brand, + businessDay, + now, + questions, + candidates, + materializeLimit, + dispatchLimit, + executionOwner, + ) + if err != nil { + if abortErr := monitoringDailyAbortErr(ctx); abortErr != nil { + return nil, abortErr + } + summary.FailedBrandCount++ + s.logDailyMonitoringBrandError("monitoring daily brand task generation failed", plan, brand, businessDate, err) + continue + } + + dailyRemaining = decrementMonitoringDailyRemaining(dailyRemaining, createdCount) + processedCount := int(maxMonitoringDailyInt(dueCount, desktopCount+deferredCount)) + if processedCount > 0 { + planRunRemaining -= processedCount + globalRemaining -= processedCount + } + summary.BrandCount++ + summary.PlannedTaskCount += int64(len(candidates)) + summary.DueTaskCount += dueCount + summary.CreatedTaskCount += createdCount + summary.DesktopTaskCount += desktopCount + summary.DeferredTaskCount += deferredCount + } + } + + return summary, nil +} + +func (s *MonitoringService) resolveDailyMonitoringExecutionOwner(plan monitoringDailyPlan) (string, bool, string) { + if plan.PrimaryClientID == nil { + return "", false, "missing_primary_client" + } + if s == nil || !s.shouldUseDesktopTasksExecution(plan.TenantID) { + return "", false, "desktop_tasks_rollout_disabled" + } + return "desktop_tasks", true, "" +} + +func monitoringDailyAbortErr(ctx context.Context) error { + if ctx == nil { + return nil + } + return ctx.Err() +} + +func (s *MonitoringService) logDailyMonitoringPlanError(message string, plan monitoringDailyPlan, businessDate string, err error) { + if s == nil || s.logger == nil { + return + } + fields := []zap.Field{ + zap.Int64("tenant_id", plan.TenantID), + zap.Int64("workspace_id", plan.WorkspaceID), + zap.String("business_date", businessDate), + } + fields = append(fields, MonitoringWorkerErrorFields(err)...) + s.logger.Warn(message, fields...) +} + +func (s *MonitoringService) logDailyMonitoringBrandError(message string, plan monitoringDailyPlan, brand monitoringDailyBrand, businessDate string, err error) { + if s == nil || s.logger == nil { + return + } + fields := []zap.Field{ + zap.Int64("tenant_id", plan.TenantID), + zap.Int64("workspace_id", plan.WorkspaceID), + zap.Int64("brand_id", brand.BrandID), + zap.String("business_date", businessDate), + } + fields = append(fields, MonitoringWorkerErrorFields(err)...) + s.logger.Warn(message, fields...) +} + +func (s *MonitoringService) generateDailyMonitoringTasksForBrand( + ctx context.Context, + projectionService *MonitoringCallbackService, + plan monitoringDailyPlan, + brand monitoringDailyBrand, + businessDay time.Time, + now time.Time, + questions []monitoringConfiguredQuestion, + candidates []monitoringDailyTaskCandidate, + materializeLimit int, + dispatchLimit int, + executionOwner string, +) (int64, int64, int64, int64, error) { + if materializeLimit <= 0 && dispatchLimit <= 0 { + return 0, 0, 0, 0, nil + } + if executionOwner != "desktop_tasks" || plan.PrimaryClientID == nil { + return 0, 0, 0, 0, nil + } + + var createdCount int64 + var dueCount int64 + if materializeLimit > 0 { + tx, err := s.monitoringPool.Begin(ctx) + if err != nil { + return 0, 0, 0, 0, response.ErrInternal(50041, "begin_failed", "failed to start monitoring daily task generation") + } + defer tx.Rollback(ctx) + + locked, err := acquireMonitoringDailyBrandLock(ctx, tx, plan.TenantID, brand.BrandID, monitoringBusinessDateText(businessDay)) + if err != nil { + return 0, 0, 0, 0, err + } + + if locked { + existingKeys, err := s.loadDailyMonitoringTaskKeys(ctx, tx, plan.TenantID, brand.BrandID, businessDay) + if err != nil { + return 0, 0, 0, 0, err + } + + dueCandidates := selectDueMonitoringDailyCandidates(candidates, existingKeys, now, defaultMonitoringDailyLookahead, materializeLimit) + dueCount = int64(len(dueCandidates)) + if len(dueCandidates) > 0 { + if err := s.syncMonitoringQuestionSnapshots(ctx, tx, plan.TenantID, brand.BrandID, questions, true); err != nil { + return 0, 0, 0, 0, err + } + + createdCount, err = s.ensureDailyMonitoringCollectTasks(ctx, tx, plan, brand, businessDay, dueCandidates, executionOwner) + if err != nil { + return 0, 0, 0, 0, err + } + + if projectionService != nil && createdCount > 0 { + if err := projectionService.rebuildBrandDailyProjection(ctx, tx, plan.TenantID, brand.BrandID, businessDay); err != nil { + return 0, 0, 0, 0, err + } + } + } + } + + if err := tx.Commit(ctx); err != nil { + return 0, 0, 0, 0, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring daily task generation") + } + } + + if dispatchLimit <= 0 { + return createdCount, dueCount, 0, 0, nil + } + specs, err := s.loadDueDailyMonitorDesktopTaskSpecs( + ctx, + s.monitoringPool, + plan.TenantID, + plan.WorkspaceID, + brand.BrandID, + businessDay, + now.Add(defaultMonitoringDailyLookahead), + now.Add(-defaultMonitoringDailyDesktopRetryCooldown), + *plan.PrimaryClientID, + dispatchLimit, + ) + if err != nil { + return createdCount, dueCount, 0, 0, err + } + if len(specs) == 0 { + return createdCount, dueCount, 0, 0, nil + } + + publishedTasks, deferredTaskIDs, err := s.dispatchMonitorDesktopTasks(ctx, specs) + if err != nil { + if s.logger != nil { + s.logger.Warn("daily monitor desktop dispatch failed; pending tasks will be retried", + zap.Int("task_count", len(specs)), + zap.Error(err), + ) + } + return createdCount, dueCount, 0, 0, err + } + + for _, task := range publishedTasks { + publishCtx, cancel := context.WithTimeout(ctx, defaultMonitoringDailyPublishTimeout) + if publishErr := publishMonitorDesktopTaskAvailable( + publishCtx, + task.TargetClientID.String(), + task, + func(publishCtx context.Context, event DesktopTaskEvent) error { + return publishDesktopTaskEvent(publishCtx, s.rabbitMQ, event) + }, + func(publishCtx context.Context, event stream.DesktopDispatchEvent) error { + return publishDesktopDispatchEvent(publishCtx, s.rabbitMQ, s.logger, event) + }, + ); publishErr != nil && s.logger != nil { + cancel() + s.logger.Warn("daily monitor desktop task available publish failed", + zap.String("task_id", task.DesktopID.String()), + zap.Error(publishErr), + ) + continue + } + cancel() + } + + return createdCount, dueCount, int64(len(publishedTasks)), int64(len(deferredTaskIDs)), nil +} + +func (s *MonitoringService) loadDailyMonitoringPlans(ctx context.Context) ([]monitoringDailyPlan, error) { + freeBrandLimit := normalizeMonitoringDailyBrandLimit(s.brandLibraryConfig.FreeBrandLimit) + paidBrandLimit := normalizeMonitoringDailyBrandLimit(s.brandLibraryConfig.PaidBrandLimit) + questionLimit := monitoringDailyQuestionLimit(s.brandLibraryConfig) + supportedPlatforms := monitoringPlatformIDs(defaultMonitoringPlatforms) + + candidates, err := s.loadDailyMonitoringPrimaryClientPlans(ctx, freeBrandLimit, paidBrandLimit, questionLimit, supportedPlatforms) + if err != nil { + return nil, err + } + if len(candidates) == 0 { + return nil, nil + } + + enabledByClient, err := s.loadDailyMonitoringPlanPlatformJSON(ctx, candidates, supportedPlatforms) + if err != nil { + return nil, err + } + return buildDailyMonitoringPlansFromSelectedClients(candidates, enabledByClient), nil +} + +func (s *MonitoringService) loadDailyMonitoringPrimaryClientPlans( + ctx context.Context, + freeBrandLimit int, + paidBrandLimit int, + questionLimit int, + supportedPlatforms []string, +) ([]monitoringDailyPlanClientCandidate, error) { + rows, err := s.businessPool.Query(ctx, ` + WITH eligible_clients AS ( + SELECT + dc.tenant_id, + dc.workspace_id, + dc.id AS primary_client_id, + dc.created_at, + p.plan_code, + p.quota_policy_json + FROM desktop_clients dc + JOIN tenant_plan_subscriptions s + ON s.tenant_id = dc.tenant_id + AND s.status = 'active' + AND s.deleted_at IS NULL + AND s.start_at <= NOW() + AND s.end_at > NOW() + JOIN plans p + ON p.id = s.plan_id + AND p.status = 'active' + AND p.deleted_at IS NULL + WHERE dc.revoked_at IS NULL + AND EXISTS ( + SELECT 1 + FROM platform_accounts pa + WHERE pa.tenant_id = dc.tenant_id + AND pa.workspace_id = dc.workspace_id + AND pa.client_id = dc.id + AND pa.deleted_at IS NULL + AND pa.platform_id = ANY($4::text[]) + ) + ), + selected_clients AS ( + SELECT DISTINCT ON (ec.tenant_id, ec.workspace_id) + ec.tenant_id, + ec.workspace_id, + ec.primary_client_id, + ec.plan_code, + ec.quota_policy_json + FROM eligible_clients ec + LEFT JOIN desktop_client_primary_leases l + ON l.tenant_id = ec.tenant_id + AND l.workspace_id = ec.workspace_id + AND l.primary_client_id = ec.primary_client_id + ORDER BY + ec.tenant_id ASC, + ec.workspace_id ASC, + CASE WHEN l.primary_client_id IS NOT NULL THEN 0 ELSE 1 END ASC, + l.updated_at DESC NULLS LAST, + ec.created_at ASC, + ec.primary_client_id ASC + ) + SELECT + c.tenant_id, + c.workspace_id, + c.primary_client_id, + GREATEST( + COALESCE( + NULLIF((c.quota_policy_json ->> 'brand_limit')::int, 0), + CASE WHEN c.plan_code = 'free' THEN $1::int ELSE $2::int END + ), + 0 + )::int AS max_brands, + $3::int AS max_questions_per_brand + FROM selected_clients c + ORDER BY c.tenant_id ASC, c.workspace_id ASC + `, freeBrandLimit, paidBrandLimit, questionLimit, supportedPlatforms) + if err != nil { + return nil, response.ErrInternal(50041, "query_failed", "failed to load daily monitoring plans") + } + defer rows.Close() + + candidates := make([]monitoringDailyPlanClientCandidate, 0) + for rows.Next() { + var item monitoringDailyPlanClientCandidate + if scanErr := rows.Scan( + &item.TenantID, + &item.WorkspaceID, + &item.PrimaryClientID, + &item.MaxBrands, + &item.MaxQuestionsPerBrand, + ); scanErr != nil { + return nil, response.ErrInternal(50041, "scan_failed", "failed to parse daily monitoring plans") + } + candidates = append(candidates, item) + } + if err := rows.Err(); err != nil { + return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate daily monitoring plans") + } + return candidates, nil +} + +func (s *MonitoringService) loadDailyMonitoringPlanPlatformJSON( + ctx context.Context, + candidates []monitoringDailyPlanClientCandidate, + supportedPlatforms []string, +) (map[monitoringDailyPlanClientKey][]byte, error) { + result := make(map[monitoringDailyPlanClientKey][]byte, len(candidates)) + if len(candidates) == 0 { + return result, nil + } + + tenantIDs := make([]int64, 0, len(candidates)) + workspaceIDs := make([]int64, 0, len(candidates)) + clientIDs := make([]uuid.UUID, 0, len(candidates)) + seen := make(map[monitoringDailyPlanClientKey]struct{}, len(candidates)) + for _, candidate := range candidates { + key := candidate.monitoringDailyPlanClientKey() + if _, exists := seen[key]; exists { + continue + } + seen[key] = struct{}{} + tenantIDs = append(tenantIDs, candidate.TenantID) + workspaceIDs = append(workspaceIDs, candidate.WorkspaceID) + clientIDs = append(clientIDs, candidate.PrimaryClientID) + } + if len(clientIDs) == 0 { + return result, nil + } + + rows, err := s.businessPool.Query(ctx, ` + WITH selected_clients AS ( + SELECT * + FROM unnest( + $1::bigint[], + $2::bigint[], + $3::uuid[] + ) AS input(tenant_id, workspace_id, client_id) + ) + SELECT + p.tenant_id, + p.workspace_id, + p.client_id, + jsonb_agg(p.platform_id ORDER BY p.platform_id) AS enabled_platforms + FROM ( + SELECT DISTINCT + sc.tenant_id, + sc.workspace_id, + sc.client_id, + pa.platform_id + FROM selected_clients sc + JOIN platform_accounts pa + ON pa.tenant_id = sc.tenant_id + AND pa.workspace_id = sc.workspace_id + AND pa.client_id = sc.client_id + WHERE pa.deleted_at IS NULL + AND pa.platform_id = ANY($4::text[]) + ) p + GROUP BY p.tenant_id, p.workspace_id, p.client_id + ORDER BY p.tenant_id ASC, p.workspace_id ASC, p.client_id ASC + `, tenantIDs, workspaceIDs, clientIDs, supportedPlatforms) + if err != nil { + return nil, response.ErrInternal(50041, "query_failed", "failed to load daily monitoring plan platforms") + } + defer rows.Close() + + for rows.Next() { + var ( + key monitoringDailyPlanClientKey + enabled []byte + ) + if scanErr := rows.Scan( + &key.TenantID, + &key.WorkspaceID, + &key.ClientID, + &enabled, + ); scanErr != nil { + return nil, response.ErrInternal(50041, "scan_failed", "failed to parse daily monitoring plan platforms") + } + result[key] = enabled + } + if err := rows.Err(); err != nil { + return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate daily monitoring plan platforms") + } + return result, nil +} + +func buildDailyMonitoringPlansFromSelectedClients( + candidates []monitoringDailyPlanClientCandidate, + enabledByClient map[monitoringDailyPlanClientKey][]byte, +) []monitoringDailyPlan { + if len(candidates) == 0 { + return nil + } + plans := make([]monitoringDailyPlan, 0, len(candidates)) + for _, candidate := range candidates { + enabledJSON, ok := enabledByClient[candidate.monitoringDailyPlanClientKey()] + if !ok || len(enabledJSON) == 0 { + continue + } + clientID := candidate.PrimaryClientID + plans = append(plans, monitoringDailyPlan{ + TenantID: candidate.TenantID, + WorkspaceID: candidate.WorkspaceID, + MaxBrands: candidate.MaxBrands, + EnabledJSON: append([]byte(nil), enabledJSON...), + PrimaryClientID: &clientID, + MaxQuestionsPerBrand: candidate.MaxQuestionsPerBrand, + }) + } + return plans +} + +func (candidate monitoringDailyPlanClientCandidate) monitoringDailyPlanClientKey() monitoringDailyPlanClientKey { + return monitoringDailyPlanClientKey{ + TenantID: candidate.TenantID, + WorkspaceID: candidate.WorkspaceID, + ClientID: candidate.PrimaryClientID, + } +} + +func (s *MonitoringService) loadDailyMonitoringBrands(ctx context.Context, plan monitoringDailyPlan) ([]monitoringDailyBrand, error) { + limit := plan.MaxBrands + if limit <= 0 { + return nil, nil + } + + rows, err := s.businessPool.Query(ctx, ` + SELECT id + FROM brands + WHERE tenant_id = $1 + AND deleted_at IS NULL + ORDER BY created_at ASC, id ASC + LIMIT $2 + `, plan.TenantID, limit) + if err != nil { + return nil, response.ErrInternal(50041, "query_failed", "failed to load daily monitoring brands") + } + defer rows.Close() + + brands := make([]monitoringDailyBrand, 0, limit) + for rows.Next() { + item := monitoringDailyBrand{ + TenantID: plan.TenantID, + WorkspaceID: plan.WorkspaceID, + } + if scanErr := rows.Scan(&item.BrandID); scanErr != nil { + return nil, response.ErrInternal(50041, "scan_failed", "failed to parse daily monitoring brands") + } + brands = append(brands, item) + } + if err := rows.Err(); err != nil { + return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate daily monitoring brands") + } + return brands, nil +} + +func (s *MonitoringService) loadDailyMonitoringTaskKeys(ctx context.Context, q monitoringCollectTaskQueryer, tenantID, brandID int64, businessDay time.Time) (map[monitoringDailyTaskKey]struct{}, error) { + rows, err := q.Query(ctx, ` + SELECT question_id, ai_platform_id + FROM monitoring_collect_tasks + WHERE tenant_id = $1 + AND brand_id = $2 + AND collector_type = $3 + AND run_mode = 'desktop_standard' + AND business_date = $4::date + `, tenantID, brandID, monitoringCollectorType, monitoringBusinessDateText(businessDay)) + if err != nil { + return nil, response.ErrInternal(50041, "query_failed", "failed to load existing daily monitoring tasks") + } + defer rows.Close() + + keys := make(map[monitoringDailyTaskKey]struct{}) + for rows.Next() { + var key monitoringDailyTaskKey + if scanErr := rows.Scan(&key.QuestionID, &key.PlatformID); scanErr != nil { + return nil, response.ErrInternal(50041, "scan_failed", "failed to parse existing daily monitoring tasks") + } + key.PlatformID = normalizeMonitoringPlatformID(key.PlatformID) + if key.QuestionID > 0 && key.PlatformID != "" { + keys[key] = struct{}{} + } + } + if err := rows.Err(); err != nil { + return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate existing daily monitoring tasks") + } + return keys, nil +} + +func (s *MonitoringService) countDailyMonitoringMaterializedTasks(ctx context.Context, tenantID, brandID int64, businessDay time.Time) (int64, error) { + if s == nil || s.monitoringPool == nil { + return 0, nil + } + var count int64 + if err := s.monitoringPool.QueryRow(ctx, ` + SELECT COUNT(*) + FROM monitoring_collect_tasks + WHERE tenant_id = $1 + AND brand_id = $2 + AND collector_type = $3 + AND trigger_source = 'automatic' + AND run_mode = 'desktop_standard' + AND business_date = $4::date + AND COALESCE(execution_owner, 'legacy') = 'desktop_tasks' + `, tenantID, brandID, monitoringCollectorType, monitoringBusinessDateText(businessDay)).Scan(&count); err != nil { + return 0, response.ErrInternal(50041, "query_failed", "failed to count daily monitoring materialized tasks") + } + return count, nil +} + +func (s *MonitoringService) ensureDailyMonitoringCollectTasks( + ctx context.Context, + tx pgx.Tx, + plan monitoringDailyPlan, + brand monitoringDailyBrand, + businessDay time.Time, + candidates []monitoringDailyTaskCandidate, + executionOwner string, +) (int64, error) { + if len(candidates) == 0 { + return 0, nil + } + if strings.TrimSpace(executionOwner) != "desktop_tasks" { + return 0, response.ErrInternal(50041, "invalid_execution_owner", "daily monitoring tasks require desktop task execution") + } + + businessDate := monitoringBusinessDateText(businessDay) + shardKey := strings.Join([]string{int64Text(plan.TenantID), int64Text(brand.BrandID), businessDate}, ":") + + questionIDs := make([]int64, 0, len(candidates)) + questionHashes := make([][]byte, 0, len(candidates)) + platformIDs := make([]string, 0, len(candidates)) + dispatchAfterValues := make([]time.Time, 0, len(candidates)) + now := time.Now().UTC() + for _, candidate := range candidates { + dispatchAfter := candidate.DispatchAfter + if dispatchAfter.IsZero() { + dispatchAfter = now + } + questionIDs = append(questionIDs, candidate.Question.ID) + questionHashes = append(questionHashes, candidate.Question.QuestionHash) + platformIDs = append(platformIDs, candidate.Platform.ID) + dispatchAfterValues = append(dispatchAfterValues, dispatchAfter) + } + + tag, err := tx.Exec(ctx, ` + INSERT INTO monitoring_collect_tasks ( + tenant_id, brand_id, question_id, question_hash, ai_platform_id, + collector_type, trigger_source, run_mode, business_date, planned_at, status, + dispatch_priority, dispatch_lane, target_client_id, dispatch_after, + ingest_shard_key, execution_owner + ) + SELECT + $1, + $2, + input.question_id, + input.question_hash, + input.ai_platform_id, + $3, + 'automatic', + 'desktop_standard', + $4::date, + input.dispatch_after, + 'pending', + 100, + 'normal', + $5, + input.dispatch_after, + $6, + $7 + FROM unnest( + $8::bigint[], + $9::bytea[], + $10::text[], + $11::timestamptz[] + ) AS input(question_id, question_hash, ai_platform_id, dispatch_after) + ON CONFLICT ( + tenant_id, brand_id, question_id, ai_platform_id, collector_type, run_mode, business_date + ) + DO NOTHING + `, + plan.TenantID, + brand.BrandID, + monitoringCollectorType, + businessDate, + nullableUUID(plan.PrimaryClientID), + shardKey, + executionOwner, + questionIDs, + questionHashes, + platformIDs, + dispatchAfterValues, + ) + if err != nil { + return 0, response.ErrInternal(50041, "insert_failed", "failed to create daily monitoring tasks") + } + + return tag.RowsAffected(), nil +} + +func (s *MonitoringService) loadDueDailyMonitorDesktopTaskSpecs( + ctx context.Context, + q monitoringCollectTaskQueryer, + tenantID, workspaceID, brandID int64, + businessDate time.Time, + cutoff time.Time, + retryBefore time.Time, + targetClientID uuid.UUID, + limit int, +) ([]monitorDesktopTaskSpec, error) { + if q == nil || limit <= 0 { + return nil, nil + } + + rows, err := q.Query(ctx, ` + WITH candidate_tasks AS ( + SELECT t.id + FROM monitoring_collect_tasks t + WHERE t.tenant_id = $1 + AND t.brand_id = $2 + AND t.collector_type = $3 + AND t.business_date = $4::date + AND t.status = 'pending' + AND t.trigger_source = 'automatic' + AND t.run_mode = 'desktop_standard' + AND COALESCE(t.execution_owner, 'legacy') = 'desktop_tasks' + AND t.target_client_id = $5 + AND (t.dispatch_after IS NULL OR t.dispatch_after <= $6) + AND (t.last_dispatched_at IS NULL OR t.last_dispatched_at <= $7) + ORDER BY t.dispatch_after ASC NULLS FIRST, t.dispatch_priority DESC, t.id ASC + LIMIT $8 + FOR UPDATE SKIP LOCKED + ), + claimed_tasks AS ( + UPDATE monitoring_collect_tasks t + SET last_dispatched_at = NOW(), + dispatch_attempts = COALESCE(t.dispatch_attempts, 0) + 1, + updated_at = NOW() + FROM candidate_tasks c + WHERE t.id = c.id + RETURNING t.* + ) + SELECT + t.id, + t.tenant_id, + t.ai_platform_id, + t.business_date, + t.trigger_source, + t.run_mode, + t.question_id, + t.question_hash, + t.dispatch_priority, + t.dispatch_lane, + t.interrupt_generation, + COALESCE(( + SELECT question_text_snapshot + FROM monitoring_question_config_snapshots s + WHERE s.tenant_id = t.tenant_id + AND s.brand_id = t.brand_id + AND s.question_id = t.question_id + AND s.question_hash = t.question_hash + AND s.deleted_at IS NULL + ORDER BY s.projected_at DESC, s.id DESC + LIMIT 1 + ), '') AS question_text_snapshot + FROM claimed_tasks t + ORDER BY t.dispatch_after ASC NULLS FIRST, t.dispatch_priority DESC, t.id ASC + `, tenantID, brandID, monitoringCollectorType, monitoringBusinessDateText(businessDate), targetClientID, cutoff.UTC(), retryBefore.UTC(), limit) + if err != nil { + return nil, response.ErrInternal(50041, "query_failed", "failed to load due daily monitor desktop task specs") + } + defer rows.Close() + + result := make([]monitorDesktopTaskSpec, 0, limit) + for rows.Next() { + var ( + spec monitorDesktopTaskSpec + questionHash []byte + businessDay time.Time + ) + if scanErr := rows.Scan( + &spec.MonitorTaskID, + &spec.TenantID, + &spec.PlatformID, + &businessDay, + &spec.TriggerSource, + &spec.RunMode, + &spec.QuestionID, + &questionHash, + &spec.Priority, + &spec.Lane, + &spec.InterruptGeneration, + &spec.QuestionText, + ); scanErr != nil { + return nil, response.ErrInternal(50041, "scan_failed", "failed to parse due daily monitor desktop task specs") + } + spec.WorkspaceID = workspaceID + spec.TargetClientID = targetClientID + spec.BusinessDate = businessDay.Format("2006-01-02") + spec.QuestionHash = encodeQuestionHash(questionHash) + spec.SchedulerGroupKey = monitoringSchedulerGroupKey(spec.QuestionHash, spec.QuestionID, spec.QuestionText) + result = append(result, spec) + } + if err := rows.Err(); err != nil { + return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate due daily monitor desktop task specs") + } + return result, nil +} + +func buildMonitoringDailyTaskCandidates( + tenantID, brandID int64, + businessDay time.Time, + questions []monitoringConfiguredQuestion, + platforms []monitoringPlatformMetadata, + limit int, +) []monitoringDailyTaskCandidate { + if len(questions) == 0 || len(platforms) == 0 || limit <= 0 { + return nil + } + + businessDate := monitoringBusinessDateText(businessDay) + candidates := make([]monitoringDailyTaskCandidate, 0, minMonitoringDailyInt(limit, len(questions)*len(platforms))) + for _, question := range questions { + if question.ID <= 0 { + continue + } + for _, platform := range platforms { + platformID := normalizeMonitoringPlatformID(platform.ID) + if platformID == "" { + continue + } + candidates = append(candidates, monitoringDailyTaskCandidate{ + Question: question, + Platform: monitoringPlatformMetadata{ + ID: platformID, + Name: platform.Name, + }, + sortKey: monitoringDailyStableHash("candidate", int64Text(tenantID), int64Text(brandID), businessDate, int64Text(question.ID), platformID), + }) + } + } + sort.SliceStable(candidates, func(i, j int) bool { + if candidates[i].sortKey == candidates[j].sortKey { + if candidates[i].Question.ID == candidates[j].Question.ID { + return candidates[i].Platform.ID < candidates[j].Platform.ID + } + return candidates[i].Question.ID < candidates[j].Question.ID + } + return candidates[i].sortKey < candidates[j].sortKey + }) + if len(candidates) > limit { + candidates = candidates[:limit] + } + assignMonitoringDailyScheduleTimes(tenantID, brandID, businessDay, candidates) + return candidates +} + +func assignMonitoringDailyScheduleTimes(tenantID, brandID int64, businessDay time.Time, candidates []monitoringDailyTaskCandidate) { + if len(candidates) == 0 { + return + } + + start := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset) + window := defaultMonitoringDailyMaterializeWindow + if window <= 0 { + window = time.Hour + } + slot := window / time.Duration(len(candidates)) + if slot <= 0 { + slot = time.Second + } + latest := start.Add(window) + businessDate := monitoringBusinessDateText(businessDay) + + for i := range candidates { + jitter := time.Duration(monitoringDailyStableHash( + "dispatch_jitter", + int64Text(tenantID), + int64Text(brandID), + businessDate, + int64Text(candidates[i].Question.ID), + candidates[i].Platform.ID, + ) % uint64(slot)) + materializeAt := start.Add(time.Duration(i)*slot + jitter) + if materializeAt.After(latest) { + materializeAt = latest + } + candidates[i].MaterializeAt = materializeAt.UTC() + candidates[i].DispatchAfter = materializeAt.UTC() + } + + sort.SliceStable(candidates, func(i, j int) bool { + if candidates[i].MaterializeAt.Equal(candidates[j].MaterializeAt) { + return candidates[i].sortKey < candidates[j].sortKey + } + return candidates[i].MaterializeAt.Before(candidates[j].MaterializeAt) + }) +} + +func selectDueMonitoringDailyCandidates( + candidates []monitoringDailyTaskCandidate, + existing map[monitoringDailyTaskKey]struct{}, + now time.Time, + lookahead time.Duration, + limit int, +) []monitoringDailyTaskCandidate { + if len(candidates) == 0 || limit <= 0 { + return nil + } + + cutoff := now.Add(lookahead) + result := make([]monitoringDailyTaskCandidate, 0, limit) + for _, candidate := range candidates { + if candidate.MaterializeAt.IsZero() || candidate.MaterializeAt.After(cutoff) { + continue + } + key := monitoringDailyTaskKey{ + QuestionID: candidate.Question.ID, + PlatformID: normalizeMonitoringPlatformID(candidate.Platform.ID), + } + if _, ok := existing[key]; ok { + continue + } + result = append(result, candidate) + if len(result) >= limit { + return result + } + } + return result +} + +func acquireMonitoringDailyBrandLock(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDate string) (bool, error) { + var locked bool + if err := tx.QueryRow(ctx, `SELECT pg_try_advisory_xact_lock($1)`, monitoringDailyBrandLockKey(tenantID, brandID, businessDate)).Scan(&locked); err != nil { + return false, response.ErrInternal(50041, "lock_failed", "failed to acquire monitoring daily brand lock") + } + return locked, nil +} + +func monitoringDailyBrandLockKey(tenantID, brandID int64, businessDate string) int64 { + return int64(monitoringDailyStableHash("daily_brand_lock", int64Text(tenantID), int64Text(brandID), businessDate)) +} + +func orderDailyMonitoringPlans(plans []monitoringDailyPlan, businessDate string, now time.Time) []monitoringDailyPlan { + if len(plans) <= 1 { + return plans + } + ordered := append([]monitoringDailyPlan(nil), plans...) + bucket := int64(0) + if defaultMonitoringDailyTaskInterval > 0 { + bucket = now.Unix() / int64(defaultMonitoringDailyTaskInterval/time.Second) + } + sort.SliceStable(ordered, func(i, j int) bool { + left := monitoringDailyStableHash("plan_order", businessDate, int64Text(bucket), int64Text(ordered[i].TenantID)) + right := monitoringDailyStableHash("plan_order", businessDate, int64Text(bucket), int64Text(ordered[j].TenantID)) + if left == right { + return ordered[i].TenantID < ordered[j].TenantID + } + return left < right + }) + return ordered +} + +func orderDailyMonitoringBrands(brands []monitoringDailyBrand, businessDate string, now time.Time) []monitoringDailyBrand { + if len(brands) <= 1 { + return brands + } + ordered := append([]monitoringDailyBrand(nil), brands...) + bucket := int64(0) + if defaultMonitoringDailyTaskInterval > 0 { + bucket = now.Unix() / int64(defaultMonitoringDailyTaskInterval/time.Second) + } + sort.SliceStable(ordered, func(i, j int) bool { + left := monitoringDailyStableHash("brand_order", businessDate, int64Text(bucket), int64Text(ordered[i].TenantID), int64Text(ordered[i].BrandID)) + right := monitoringDailyStableHash("brand_order", businessDate, int64Text(bucket), int64Text(ordered[j].TenantID), int64Text(ordered[j].BrandID)) + if left == right { + return ordered[i].BrandID < ordered[j].BrandID + } + return left < right + }) + return ordered +} + +func decodeDailyMonitoringEnabledPlatforms(raw []byte) ([]string, error) { + var enabled []string + if len(raw) > 0 { + if err := json.Unmarshal(raw, &enabled); err != nil { + return nil, fmt.Errorf("decode daily monitoring enabled platforms: %w", err) + } + } + return reconcileEnabledMonitoringPlatforms(enabled), nil +} + +func normalizeMonitoringDailyTaskHardCap(value int) int { + if value <= 0 { + return defaultMonitoringDailyTaskHardCap + } + return value +} + +func normalizeMonitoringDailyBrandLimit(value int) int { + if value <= 0 { + return defaultMonitoringDailyBrandLimit + } + return value +} + +func monitoringDailyQuestionLimit(cfg config.BrandLibraryConfig) int { + maxKeywords := cfg.MaxKeywords + if maxKeywords <= 0 { + maxKeywords = 5 + } + maxQuestionsPerKeyword := cfg.MaxQuestionsPerKeyword + if maxQuestionsPerKeyword <= 0 { + maxQuestionsPerKeyword = 5 + } + return maxKeywords * maxQuestionsPerKeyword +} + +func normalizeMonitoringDailyQuestionLimit(value int) int { + if value <= 0 { + return monitoringDailyQuestionLimit(config.BrandLibraryConfig{}) + } + return value +} + +func limitMonitoringDailyQuestions(questions []monitoringConfiguredQuestion, limit int) []monitoringConfiguredQuestion { + limit = normalizeMonitoringDailyQuestionLimit(limit) + if len(questions) <= limit { + return questions + } + return questions[:limit] +} + +func monitoringDailyRemainingCapacity(hardCap int, materializedCount int64) int { + capacity := normalizeMonitoringDailyTaskHardCap(hardCap) + if materializedCount <= 0 { + return capacity + } + if materializedCount >= int64(capacity) { + return 0 + } + return capacity - int(materializedCount) +} + +func decrementMonitoringDailyRemaining(remaining int, createdCount int64) int { + if remaining <= 0 || createdCount <= 0 { + return remaining + } + if createdCount >= int64(remaining) { + return 0 + } + return remaining - int(createdCount) +} + +func nullableUUID(value *uuid.UUID) interface{} { + if value == nil { + return nil + } + return *value +} + +func monitoringDailyStableHash(parts ...string) uint64 { + h := fnv.New64a() + for _, part := range parts { + _, _ = h.Write([]byte(part)) + _, _ = h.Write([]byte{0}) + } + return h.Sum64() +} + +func int64Text(value int64) string { + return strconv.FormatInt(value, 10) +} + +func minMonitoringDailyInt(values ...int) int { + if len(values) == 0 { + return 0 + } + result := values[0] + for _, value := range values[1:] { + if value < result { + result = value + } + } + return result +} + +func maxMonitoringDailyInt(left, right int64) int64 { + if left > right { + return left + } + return right +} diff --git a/server/internal/tenant/app/monitoring_daily_task_worker_test.go b/server/internal/tenant/app/monitoring_daily_task_worker_test.go new file mode 100644 index 0000000..0b37229 --- /dev/null +++ b/server/internal/tenant/app/monitoring_daily_task_worker_test.go @@ -0,0 +1,337 @@ +package app + +import ( + "context" + "net/http" + "net/http/httptest" + "runtime" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/geo-platform/tenant-api/internal/shared/config" +) + +func TestBuildMonitoringDailyTaskCandidatesMaterializesAfterMidnight(t *testing.T) { + businessDay := time.Date(2026, 4, 24, 0, 0, 0, 0, monitoringBusinessLocation()) + questions := []monitoringConfiguredQuestion{ + {ID: 101, QuestionHash: []byte("q101")}, + {ID: 102, QuestionHash: []byte("q102")}, + } + platforms := []monitoringPlatformMetadata{ + {ID: "qwen", Name: "Qwen"}, + {ID: "doubao", Name: "Doubao"}, + {ID: "deepseek", Name: "DeepSeek"}, + } + + candidates := buildMonitoringDailyTaskCandidates(7, 11, businessDay, questions, platforms, 4) + + require.Len(t, candidates, 4) + start := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset).UTC() + end := start.Add(defaultMonitoringDailyMaterializeWindow) + for i, candidate := range candidates { + assert.False(t, candidate.MaterializeAt.Before(start)) + assert.False(t, candidate.MaterializeAt.After(end)) + assert.Equal(t, candidate.MaterializeAt, candidate.DispatchAfter) + if i > 0 { + assert.False(t, candidate.MaterializeAt.Before(candidates[i-1].MaterializeAt)) + } + } +} + +func TestSelectDueMonitoringDailyCandidatesSkipsExistingAndHonorsLimit(t *testing.T) { + businessDay := time.Date(2026, 4, 24, 0, 0, 0, 0, monitoringBusinessLocation()) + candidates := buildMonitoringDailyTaskCandidates( + 7, + 11, + businessDay, + []monitoringConfiguredQuestion{ + {ID: 101, QuestionHash: []byte("q101")}, + {ID: 102, QuestionHash: []byte("q102")}, + }, + []monitoringPlatformMetadata{ + {ID: "qwen"}, + {ID: "doubao"}, + {ID: "deepseek"}, + }, + 6, + ) + require.Len(t, candidates, 6) + + existing := map[monitoringDailyTaskKey]struct{}{ + { + QuestionID: candidates[0].Question.ID, + PlatformID: candidates[0].Platform.ID, + }: {}, + } + now := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset + defaultMonitoringDailyMaterializeWindow) + + due := selectDueMonitoringDailyCandidates(candidates, existing, now, defaultMonitoringDailyLookahead, 2) + + require.Len(t, due, 2) + for _, candidate := range due { + _, exists := existing[monitoringDailyTaskKey{ + QuestionID: candidate.Question.ID, + PlatformID: candidate.Platform.ID, + }] + assert.False(t, exists) + } +} + +func TestMonitoringDailyRemainingKeepsFullCandidateHorizon(t *testing.T) { + businessDay := time.Date(2026, 4, 24, 0, 0, 0, 0, monitoringBusinessLocation()) + hardCap := 6 + remaining := monitoringDailyRemainingCapacity(hardCap, 4) + assert.Equal(t, 2, remaining) + + candidates := buildMonitoringDailyTaskCandidates( + 7, + 11, + businessDay, + []monitoringConfiguredQuestion{ + {ID: 101, QuestionHash: []byte("q101")}, + {ID: 102, QuestionHash: []byte("q102")}, + {ID: 103, QuestionHash: []byte("q103")}, + }, + []monitoringPlatformMetadata{ + {ID: "qwen"}, + {ID: "doubao"}, + }, + hardCap, + ) + require.Len(t, candidates, hardCap) + + existing := make(map[monitoringDailyTaskKey]struct{}, 4) + for _, candidate := range candidates[:4] { + existing[monitoringDailyTaskKey{ + QuestionID: candidate.Question.ID, + PlatformID: candidate.Platform.ID, + }] = struct{}{} + } + + now := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset + defaultMonitoringDailyMaterializeWindow) + due := selectDueMonitoringDailyCandidates(candidates, existing, now, defaultMonitoringDailyLookahead, remaining) + + require.Len(t, due, remaining) + for _, candidate := range due { + _, exists := existing[monitoringDailyTaskKey{ + QuestionID: candidate.Question.ID, + PlatformID: candidate.Platform.ID, + }] + assert.False(t, exists) + } + assert.Equal(t, 1, decrementMonitoringDailyRemaining(remaining, 1)) + assert.Equal(t, 0, decrementMonitoringDailyRemaining(remaining, 2)) +} + +func TestMonitoringDailyQuestionLimitUsesBrandLibraryConfig(t *testing.T) { + assert.Equal(t, 25, monitoringDailyQuestionLimit(config.BrandLibraryConfig{ + MaxKeywords: 5, + MaxQuestionsPerKeyword: 5, + })) + assert.Equal(t, 6, monitoringDailyQuestionLimit(config.BrandLibraryConfig{ + MaxKeywords: 2, + MaxQuestionsPerKeyword: 3, + })) + + questions := []monitoringConfiguredQuestion{ + {ID: 1}, + {ID: 2}, + {ID: 3}, + } + assert.Len(t, limitMonitoringDailyQuestions(questions, 2), 2) + assert.Len(t, limitMonitoringDailyQuestions(questions, 25), 3) +} + +func TestDecodeDailyMonitoringEnabledPlatformsReportsMalformedJSON(t *testing.T) { + platforms, err := decodeDailyMonitoringEnabledPlatforms([]byte(`["qwen"`)) + require.Error(t, err) + assert.Nil(t, platforms) +} + +func TestDecodeDailyMonitoringEnabledPlatformsReconcilesSupportedPlatforms(t *testing.T) { + platforms, err := decodeDailyMonitoringEnabledPlatforms([]byte(`["qwen","unknown","doubao"]`)) + require.NoError(t, err) + assert.Equal(t, []string{"doubao", "qwen"}, platforms) +} + +func TestBuildDailyMonitoringPlansFromSelectedClientsRequiresPlatformSnapshot(t *testing.T) { + clientOne := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000001") + clientTwo := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000002") + candidates := []monitoringDailyPlanClientCandidate{ + { + TenantID: 1, + WorkspaceID: 10, + PrimaryClientID: clientOne, + MaxBrands: 3, + MaxQuestionsPerBrand: 25, + }, + { + TenantID: 1, + WorkspaceID: 11, + PrimaryClientID: clientTwo, + MaxBrands: 2, + MaxQuestionsPerBrand: 10, + }, + } + enabledByClient := map[monitoringDailyPlanClientKey][]byte{ + candidates[0].monitoringDailyPlanClientKey(): []byte(`["qwen"]`), + } + + plans := buildDailyMonitoringPlansFromSelectedClients(candidates, enabledByClient) + + require.Len(t, plans, 1) + assert.Equal(t, int64(1), plans[0].TenantID) + assert.Equal(t, int64(10), plans[0].WorkspaceID) + assert.Equal(t, 3, plans[0].MaxBrands) + assert.Equal(t, 25, plans[0].MaxQuestionsPerBrand) + require.NotNil(t, plans[0].PrimaryClientID) + assert.Equal(t, clientOne, *plans[0].PrimaryClientID) + assert.Equal(t, []byte(`["qwen"]`), plans[0].EnabledJSON) +} + +func TestOrderDailyMonitoringPlansRotatesDeterministicallyByBucket(t *testing.T) { + plans := []monitoringDailyPlan{ + {TenantID: 1}, + {TenantID: 2}, + {TenantID: 3}, + } + now := time.Date(2026, 4, 24, 1, 0, 0, 0, time.UTC) + + first := orderDailyMonitoringPlans(plans, "2026-04-24", now) + second := orderDailyMonitoringPlans(plans, "2026-04-24", now) + + assert.Equal(t, first, second) + assert.Equal(t, []monitoringDailyPlan{{TenantID: 1}, {TenantID: 2}, {TenantID: 3}}, plans) +} + +func TestResolveDailyMonitoringExecutionOwnerRequiresDesktopRolloutAndClient(t *testing.T) { + clientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000001") + + disabled := &MonitoringService{desktopTasksRolloutPercentage: 0} + owner, ok, reason := disabled.resolveDailyMonitoringExecutionOwner(monitoringDailyPlan{ + TenantID: 1, + PrimaryClientID: &clientID, + }) + assert.False(t, ok) + assert.Empty(t, owner) + assert.Equal(t, "desktop_tasks_rollout_disabled", reason) + + enabled := &MonitoringService{desktopTasksRolloutPercentage: 100} + owner, ok, reason = enabled.resolveDailyMonitoringExecutionOwner(monitoringDailyPlan{TenantID: 1}) + assert.False(t, ok) + assert.Empty(t, owner) + assert.Equal(t, "missing_primary_client", reason) + + owner, ok, reason = enabled.resolveDailyMonitoringExecutionOwner(monitoringDailyPlan{ + TenantID: 1, + PrimaryClientID: &clientID, + }) + assert.True(t, ok) + assert.Equal(t, "desktop_tasks", owner) + assert.Empty(t, reason) +} + +func TestMonitoringDailyTaskMetricsRecordsSummaryCounters(t *testing.T) { + resetMonitoringDailyTaskMetricsForTest() + t.Cleanup(resetMonitoringDailyTaskMetricsForTest) + + recordMonitoringDailyTaskMetrics(&MonitoringDailyTaskGenerationSummary{ + BusinessDate: "2026-04-24", + TenantCount: 3, + BrandCount: 2, + FailedPlanCount: 1, + FailedBrandCount: 2, + PlannedTaskCount: 20, + DueTaskCount: 8, + CreatedTaskCount: 6, + DesktopTaskCount: 4, + DeferredTaskCount: 2, + SkippedPlanCount: 1, + }, 1500*time.Millisecond, nil) + + snapshot := MonitoringDailyTaskMetricsSnapshotValue() + assert.Equal(t, int64(1), snapshot.RunTotal) + assert.Equal(t, int64(1), snapshot.RunCompletedTotal) + assert.Equal(t, int64(0), snapshot.RunFailedTotal) + assert.Equal(t, int64(1), snapshot.RunPartialFailureTotal) + assert.Equal(t, int64(1), snapshot.PlanFailureTotal) + assert.Equal(t, int64(2), snapshot.BrandFailureTotal) + assert.Equal(t, int64(1), snapshot.SkippedPlanTotal) + assert.Equal(t, int64(20), snapshot.PlannedTaskTotal) + assert.Equal(t, int64(8), snapshot.DueTaskTotal) + assert.Equal(t, int64(6), snapshot.CreatedTaskTotal) + assert.Equal(t, int64(4), snapshot.DesktopTaskTotal) + assert.Equal(t, int64(2), snapshot.DeferredTaskTotal) + assert.Equal(t, "2026-04-24", snapshot.LastBusinessDate) + assert.Equal(t, int64(3), snapshot.LastTenantCount) + assert.Equal(t, int64(2), snapshot.LastBrandCount) + assert.NotNil(t, snapshot.LastRunAt) + assert.NotNil(t, snapshot.LastSuccessAt) + + prometheus := MonitoringDailyTaskMetricsPrometheus() + assert.Contains(t, prometheus, `monitoring_daily_task_worker_runs_total{status="completed"} 1`) + assert.Contains(t, prometheus, `monitoring_daily_task_generation_failures_total{scope="plan"} 1`) + assert.Contains(t, prometheus, `monitoring_daily_task_generation_tasks_total{stage="created"} 6`) + assert.Contains(t, prometheus, `monitoring_daily_task_worker_last_business_date_info{business_date="2026-04-24"} 1`) +} + +func TestMonitoringDailyTaskMetricsRecordsFatalError(t *testing.T) { + resetMonitoringDailyTaskMetricsForTest() + t.Cleanup(resetMonitoringDailyTaskMetricsForTest) + + recordMonitoringDailyTaskMetrics(nil, time.Second, context.DeadlineExceeded) + + snapshot := MonitoringDailyTaskMetricsSnapshotValue() + assert.Equal(t, int64(1), snapshot.RunTotal) + assert.Equal(t, int64(0), snapshot.RunCompletedTotal) + assert.Equal(t, int64(1), snapshot.RunFailedTotal) + assert.Equal(t, "context_deadline_exceeded", snapshot.LastErrorCode) + assert.NotNil(t, snapshot.LastRunAt) + assert.NotNil(t, snapshot.LastErrorAt) + + prometheus := MonitoringDailyTaskMetricsPrometheus() + assert.Contains(t, prometheus, `monitoring_daily_task_worker_runs_total{status="failed"} 1`) + assert.Contains(t, prometheus, `monitoring_daily_task_worker_last_error_info{code="context_deadline_exceeded"} 1`) +} + +func TestMonitoringDailyTaskPrometheusEscapesCarriageReturnLabels(t *testing.T) { + resetMonitoringDailyTaskMetricsForTest() + t.Cleanup(resetMonitoringDailyTaskMetricsForTest) + + monitoringDailyTaskMetrics.lastBusinessDate.Store("2026-04-24\rbad") + + prometheus := MonitoringDailyTaskMetricsPrometheus() + assert.NotContains(t, prometheus, "\r") + assert.Contains(t, prometheus, `monitoring_daily_task_worker_last_business_date_info{business_date="2026-04-24\\rbad"} 1`) +} + +func TestMonitoringSchedulerMetricsPrometheusHandlerIncludesRuntimeAndProcessCollectors(t *testing.T) { + resetMonitoringDailyTaskMetricsForTest() + t.Cleanup(resetMonitoringDailyTaskMetricsForTest) + recordMonitoringDailyTaskMetrics(&MonitoringDailyTaskGenerationSummary{CreatedTaskCount: 1}, time.Second, nil) + + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/metrics", nil) + MonitoringSchedulerMetricsPrometheusHandler().ServeHTTP(recorder, request) + + body := recorder.Body.String() + assert.Equal(t, http.StatusOK, recorder.Code) + assert.Contains(t, body, `monitoring_daily_task_worker_runs_total{status="completed"} 1`) + assert.Contains(t, body, "go_goroutines") + if runtime.GOOS == "linux" || runtime.GOOS == "windows" { + assert.Contains(t, body, "process_cpu_seconds_total") + } + assert.True(t, strings.HasPrefix(recorder.Header().Get("Content-Type"), "text/plain")) +} + +func mustParseDailyMonitoringUUID(t *testing.T, value string) uuid.UUID { + t.Helper() + id, err := uuid.Parse(value) + require.NoError(t, err) + return id +} diff --git a/server/internal/tenant/app/monitoring_heartbeat_primary_metrics.go b/server/internal/tenant/app/monitoring_heartbeat_primary_metrics.go new file mode 100644 index 0000000..7c9b3d9 --- /dev/null +++ b/server/internal/tenant/app/monitoring_heartbeat_primary_metrics.go @@ -0,0 +1,141 @@ +package app + +import ( + "sync/atomic" + "time" +) + +type MonitoringHeartbeatPrimaryLeaseMetricsSnapshot struct { + PrimaryHitTotal int64 `json:"primary_hit_total"` + NonPrimaryLiveTotal int64 `json:"non_primary_live_total"` + InsertedTotal int64 `json:"inserted_total"` + RenewedTotal int64 `json:"renewed_total"` + ClaimedExpiredTotal int64 `json:"claimed_expired_total"` + ClaimedUnavailableTotal int64 `json:"claimed_unavailable_total"` + ContentionTotal int64 `json:"contention_total"` + ErrorTotal int64 `json:"error_total"` + DBInsertTotal int64 `json:"db_insert_total"` + DBRenewTotal int64 `json:"db_renew_total"` + DBClaimTotal int64 `json:"db_claim_total"` + DurationCount int64 `json:"duration_count"` + DurationSumSeconds float64 `json:"duration_sum_seconds"` + LastDurationSeconds float64 `json:"last_duration_seconds"` + LastOutcome string `json:"last_outcome,omitempty"` +} + +type monitoringHeartbeatPrimaryLeaseMetricsRecorder struct { + primaryHitTotal atomic.Int64 + nonPrimaryLiveTotal atomic.Int64 + insertedTotal atomic.Int64 + renewedTotal atomic.Int64 + claimedExpiredTotal atomic.Int64 + claimedUnavailableTotal atomic.Int64 + contentionTotal atomic.Int64 + errorTotal atomic.Int64 + dbInsertTotal atomic.Int64 + dbRenewTotal atomic.Int64 + dbClaimTotal atomic.Int64 + durationCount atomic.Int64 + durationSumNanos atomic.Int64 + lastDurationNanos atomic.Int64 + lastOutcome atomic.Value +} + +var monitoringHeartbeatPrimaryLeaseMetrics = newMonitoringHeartbeatPrimaryLeaseMetricsRecorder() + +func newMonitoringHeartbeatPrimaryLeaseMetricsRecorder() *monitoringHeartbeatPrimaryLeaseMetricsRecorder { + return &monitoringHeartbeatPrimaryLeaseMetricsRecorder{} +} + +func recordHeartbeatPrimaryLeaseMetric(outcome string, duration time.Duration) { + monitoringHeartbeatPrimaryLeaseMetrics.record(outcome, duration) +} + +func recordHeartbeatPrimaryLeaseWriteMetric(operation string) { + monitoringHeartbeatPrimaryLeaseMetrics.recordWrite(operation) +} + +func MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue() MonitoringHeartbeatPrimaryLeaseMetricsSnapshot { + return monitoringHeartbeatPrimaryLeaseMetrics.snapshot() +} + +func resetMonitoringHeartbeatPrimaryLeaseMetricsForTest() { + monitoringHeartbeatPrimaryLeaseMetrics = newMonitoringHeartbeatPrimaryLeaseMetricsRecorder() +} + +func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) record(outcome string, duration time.Duration) { + if r == nil { + return + } + if duration < 0 { + duration = 0 + } + switch outcome { + case "primary_hit": + r.primaryHitTotal.Add(1) + case "non_primary_live": + r.nonPrimaryLiveTotal.Add(1) + case "inserted": + r.insertedTotal.Add(1) + case "renewed": + r.renewedTotal.Add(1) + case "claimed_expired": + r.claimedExpiredTotal.Add(1) + case "claimed_unavailable": + r.claimedUnavailableTotal.Add(1) + case "contention": + r.contentionTotal.Add(1) + case "error": + r.errorTotal.Add(1) + } + r.durationCount.Add(1) + r.durationSumNanos.Add(duration.Nanoseconds()) + r.lastDurationNanos.Store(duration.Nanoseconds()) + r.lastOutcome.Store(outcome) +} + +func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) recordWrite(operation string) { + if r == nil { + return + } + switch operation { + case "insert": + r.dbInsertTotal.Add(1) + case "renew": + r.dbRenewTotal.Add(1) + case "claim": + r.dbClaimTotal.Add(1) + } +} + +func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) snapshot() MonitoringHeartbeatPrimaryLeaseMetricsSnapshot { + if r == nil { + return MonitoringHeartbeatPrimaryLeaseMetricsSnapshot{} + } + return MonitoringHeartbeatPrimaryLeaseMetricsSnapshot{ + PrimaryHitTotal: r.primaryHitTotal.Load(), + NonPrimaryLiveTotal: r.nonPrimaryLiveTotal.Load(), + InsertedTotal: r.insertedTotal.Load(), + RenewedTotal: r.renewedTotal.Load(), + ClaimedExpiredTotal: r.claimedExpiredTotal.Load(), + ClaimedUnavailableTotal: r.claimedUnavailableTotal.Load(), + ContentionTotal: r.contentionTotal.Load(), + ErrorTotal: r.errorTotal.Load(), + DBInsertTotal: r.dbInsertTotal.Load(), + DBRenewTotal: r.dbRenewTotal.Load(), + DBClaimTotal: r.dbClaimTotal.Load(), + DurationCount: r.durationCount.Load(), + DurationSumSeconds: float64(r.durationSumNanos.Load()) / float64(time.Second), + LastDurationSeconds: float64(r.lastDurationNanos.Load()) / float64(time.Second), + LastOutcome: heartbeatPrimaryMetricLoadString(&r.lastOutcome), + } +} + +func heartbeatPrimaryMetricLoadString(value *atomic.Value) string { + if value == nil { + return "" + } + raw := value.Load() + result, _ := raw.(string) + return result +} diff --git a/server/internal/tenant/app/monitoring_heartbeat_snapshot_metrics.go b/server/internal/tenant/app/monitoring_heartbeat_snapshot_metrics.go new file mode 100644 index 0000000..f05f773 --- /dev/null +++ b/server/internal/tenant/app/monitoring_heartbeat_snapshot_metrics.go @@ -0,0 +1,123 @@ +package app + +import ( + "sync/atomic" + "time" +) + +type MonitoringHeartbeatSnapshotMetricsSnapshot struct { + PlatformReportedTotal int64 `json:"platform_reported_total"` + TransactionOpenedTotal int64 `json:"transaction_opened_total"` + SnapshotInsertedTotal int64 `json:"snapshot_inserted_total"` + SnapshotUpdatedTotal int64 `json:"snapshot_updated_total"` + SnapshotRefreshedTotal int64 `json:"snapshot_refreshed_total"` + SnapshotSkippedTotal int64 `json:"snapshot_skipped_total"` + ReconciledSkippedTaskTotal int64 `json:"reconciled_skipped_task_total"` + ProjectionRebuildTotal int64 `json:"projection_rebuild_total"` + ErrorTotal int64 `json:"error_total"` + DurationCount int64 `json:"duration_count"` + DurationSumSeconds float64 `json:"duration_sum_seconds"` + LastDurationSeconds float64 `json:"last_duration_seconds"` + LastErrorCode string `json:"last_error_code,omitempty"` + LastErrorMessage string `json:"last_error_message,omitempty"` +} + +type monitoringHeartbeatSnapshotMetricsEvent struct { + Active bool + PlatformCount int + TransactionOpened bool + SnapshotInsertedCount int64 + SnapshotUpdatedCount int64 + SnapshotRefreshedCount int64 + SnapshotSkippedCount int64 + ReconciledSkippedTaskCount int64 + ProjectionRebuildCount int64 + Duration time.Duration + Err error +} + +type monitoringHeartbeatSnapshotMetricsRecorder struct { + platformReportedTotal atomic.Int64 + transactionOpenedTotal atomic.Int64 + snapshotInsertedTotal atomic.Int64 + snapshotUpdatedTotal atomic.Int64 + snapshotRefreshedTotal atomic.Int64 + snapshotSkippedTotal atomic.Int64 + reconciledSkippedTaskTotal atomic.Int64 + projectionRebuildTotal atomic.Int64 + errorTotal atomic.Int64 + durationCount atomic.Int64 + durationSumNanos atomic.Int64 + lastDurationNanos atomic.Int64 + lastErrorCode atomic.Value + lastErrorMessage atomic.Value +} + +var monitoringHeartbeatSnapshotMetrics = newMonitoringHeartbeatSnapshotMetricsRecorder() + +func newMonitoringHeartbeatSnapshotMetricsRecorder() *monitoringHeartbeatSnapshotMetricsRecorder { + return &monitoringHeartbeatSnapshotMetricsRecorder{} +} + +func recordMonitoringHeartbeatSnapshotMetrics(event monitoringHeartbeatSnapshotMetricsEvent) { + monitoringHeartbeatSnapshotMetrics.record(event) +} + +func MonitoringHeartbeatSnapshotMetricsSnapshotValue() MonitoringHeartbeatSnapshotMetricsSnapshot { + return monitoringHeartbeatSnapshotMetrics.snapshot() +} + +func resetMonitoringHeartbeatSnapshotMetricsForTest() { + monitoringHeartbeatSnapshotMetrics = newMonitoringHeartbeatSnapshotMetricsRecorder() +} + +func (r *monitoringHeartbeatSnapshotMetricsRecorder) record(event monitoringHeartbeatSnapshotMetricsEvent) { + if r == nil { + return + } + if event.Duration < 0 { + event.Duration = 0 + } + r.platformReportedTotal.Add(int64(event.PlatformCount)) + if event.TransactionOpened { + r.transactionOpenedTotal.Add(1) + } + r.snapshotInsertedTotal.Add(event.SnapshotInsertedCount) + r.snapshotUpdatedTotal.Add(event.SnapshotUpdatedCount) + r.snapshotRefreshedTotal.Add(event.SnapshotRefreshedCount) + r.snapshotSkippedTotal.Add(event.SnapshotSkippedCount) + r.reconciledSkippedTaskTotal.Add(event.ReconciledSkippedTaskCount) + r.projectionRebuildTotal.Add(event.ProjectionRebuildCount) + r.durationCount.Add(1) + r.durationSumNanos.Add(event.Duration.Nanoseconds()) + r.lastDurationNanos.Store(event.Duration.Nanoseconds()) + + if event.Err != nil { + code, message := monitoringDailyMetricError(event.Err) + r.errorTotal.Add(1) + r.lastErrorCode.Store(code) + r.lastErrorMessage.Store(message) + } +} + +func (r *monitoringHeartbeatSnapshotMetricsRecorder) snapshot() MonitoringHeartbeatSnapshotMetricsSnapshot { + if r == nil { + return MonitoringHeartbeatSnapshotMetricsSnapshot{} + } + return MonitoringHeartbeatSnapshotMetricsSnapshot{ + PlatformReportedTotal: r.platformReportedTotal.Load(), + TransactionOpenedTotal: r.transactionOpenedTotal.Load(), + SnapshotInsertedTotal: r.snapshotInsertedTotal.Load(), + SnapshotUpdatedTotal: r.snapshotUpdatedTotal.Load(), + SnapshotRefreshedTotal: r.snapshotRefreshedTotal.Load(), + SnapshotSkippedTotal: r.snapshotSkippedTotal.Load(), + ReconciledSkippedTaskTotal: r.reconciledSkippedTaskTotal.Load(), + ProjectionRebuildTotal: r.projectionRebuildTotal.Load(), + ErrorTotal: r.errorTotal.Load(), + DurationCount: r.durationCount.Load(), + DurationSumSeconds: float64(r.durationSumNanos.Load()) / float64(time.Second), + LastDurationSeconds: float64(r.lastDurationNanos.Load()) / float64(time.Second), + LastErrorCode: monitoringDailyLoadString(&r.lastErrorCode), + LastErrorMessage: monitoringDailyLoadString(&r.lastErrorMessage), + } +} diff --git a/server/internal/tenant/app/monitoring_metrics_prometheus.go b/server/internal/tenant/app/monitoring_metrics_prometheus.go new file mode 100644 index 0000000..5c30bb9 --- /dev/null +++ b/server/internal/tenant/app/monitoring_metrics_prometheus.go @@ -0,0 +1,195 @@ +package app + +import ( + "bytes" + "net/http" + "strings" + + "github.com/prometheus/client_golang/prometheus" + promcollectors "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/common/expfmt" +) + +func MonitoringSchedulerMetricsPrometheusHandler() http.Handler { + return monitoringPrometheusHandler(true, monitoringDailyTaskMetricsCollector{}) +} + +func MonitoringDailyTaskMetricsPrometheus() string { + return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringDailyTaskMetricsCollector{})) +} + +func MonitoringDailyTaskMetricsPrometheusHandler() http.Handler { + return monitoringPrometheusHandler(false, monitoringDailyTaskMetricsCollector{}) +} + +func MonitoringHeartbeatPrimaryLeaseMetricsPrometheus() string { + return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringHeartbeatPrimaryLeaseMetricsCollector{})) +} + +func MonitoringHeartbeatPrimaryLeaseMetricsPrometheusHandler() http.Handler { + return monitoringPrometheusHandler(false, monitoringHeartbeatPrimaryLeaseMetricsCollector{}) +} + +func MonitoringHeartbeatSnapshotMetricsPrometheus() string { + return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringHeartbeatSnapshotMetricsCollector{})) +} + +func MonitoringHeartbeatSnapshotMetricsPrometheusHandler() http.Handler { + return monitoringPrometheusHandler(false, monitoringHeartbeatSnapshotMetricsCollector{}) +} + +func monitoringPrometheusHandler(includeRuntime bool, metricCollectors ...prometheus.Collector) http.Handler { + return promhttp.HandlerFor(monitoringPrometheusRegistry(includeRuntime, metricCollectors...), promhttp.HandlerOpts{}) +} + +func monitoringPrometheusRegistry(includeRuntime bool, metricCollectors ...prometheus.Collector) *prometheus.Registry { + registry := prometheus.NewRegistry() + if includeRuntime { + registry.MustRegister( + promcollectors.NewGoCollector(), + promcollectors.NewProcessCollector(promcollectors.ProcessCollectorOpts{}), + ) + } + for _, collector := range metricCollectors { + registry.MustRegister(collector) + } + return registry +} + +func renderMonitoringPrometheus(gatherer prometheus.Gatherer) string { + metricFamilies, err := gatherer.Gather() + if err != nil { + return "" + } + var b bytes.Buffer + encoder := expfmt.NewEncoder(&b, expfmt.NewFormat(expfmt.TypeTextPlain)) + for _, metricFamily := range metricFamilies { + if err := encoder.Encode(metricFamily); err != nil { + return "" + } + } + if closer, ok := encoder.(expfmt.Closer); ok { + _ = closer.Close() + } + return b.String() +} + +type monitoringDailyTaskMetricsCollector struct{} + +func (monitoringDailyTaskMetricsCollector) Describe(chan<- *prometheus.Desc) {} + +func (monitoringDailyTaskMetricsCollector) Collect(ch chan<- prometheus.Metric) { + snapshot := MonitoringDailyTaskMetricsSnapshotValue() + monitoringCounter(ch, "monitoring_daily_task_worker_runs_total", "Total daily monitoring worker runs by terminal status.", float64(snapshot.RunCompletedTotal), []string{"status"}, "completed") + monitoringCounter(ch, "monitoring_daily_task_worker_runs_total", "Total daily monitoring worker runs by terminal status.", float64(snapshot.RunFailedTotal), []string{"status"}, "failed") + monitoringCounter(ch, "monitoring_daily_task_worker_partial_failure_runs_total", "Total completed daily monitoring worker runs with at least one plan or brand failure.", float64(snapshot.RunPartialFailureTotal), nil) + monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.PlanFailureTotal), []string{"scope"}, "plan") + monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.BrandFailureTotal), []string{"scope"}, "brand") + monitoringCounter(ch, "monitoring_daily_task_generation_skipped_plans_total", "Total daily monitoring plans skipped before generation.", float64(snapshot.SkippedPlanTotal), nil) + monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.PlannedTaskTotal), []string{"stage"}, "planned") + monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DueTaskTotal), []string{"stage"}, "due") + monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.CreatedTaskTotal), []string{"stage"}, "created") + monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DesktopTaskTotal), []string{"stage"}, "desktop") + monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DeferredTaskTotal), []string{"stage"}, "deferred") + monitoringGauge(ch, "monitoring_daily_task_worker_last_run_timestamp_seconds", "Unix timestamp of the last daily monitoring worker run completion.", monitoringDailyTimeSeconds(snapshot.LastRunAt), nil) + monitoringGauge(ch, "monitoring_daily_task_worker_last_success_timestamp_seconds", "Unix timestamp of the last completed daily monitoring worker run.", monitoringDailyTimeSeconds(snapshot.LastSuccessAt), nil) + monitoringGauge(ch, "monitoring_daily_task_worker_last_error_timestamp_seconds", "Unix timestamp of the last failed daily monitoring worker run.", monitoringDailyTimeSeconds(snapshot.LastErrorAt), nil) + monitoringGauge(ch, "monitoring_daily_task_worker_last_duration_seconds", "Duration of the last daily monitoring worker run.", snapshot.LastDurationSeconds, nil) + monitoringGauge(ch, "monitoring_daily_task_worker_last_scope_count", "Tenant and brand counts observed in the last daily monitoring worker run.", float64(snapshot.LastTenantCount), []string{"scope"}, "tenant") + monitoringGauge(ch, "monitoring_daily_task_worker_last_scope_count", "Tenant and brand counts observed in the last daily monitoring worker run.", float64(snapshot.LastBrandCount), []string{"scope"}, "brand") + if snapshot.LastBusinessDate != "" { + monitoringGauge(ch, "monitoring_daily_task_worker_last_business_date_info", "Business date processed by the last daily monitoring worker run.", 1, []string{"business_date"}, snapshot.LastBusinessDate) + } + if snapshot.LastErrorCode != "" { + monitoringGauge(ch, "monitoring_daily_task_worker_last_error_info", "Last fatal daily monitoring worker error code.", 1, []string{"code"}, snapshot.LastErrorCode) + } +} + +type monitoringHeartbeatPrimaryLeaseMetricsCollector struct{} + +func (monitoringHeartbeatPrimaryLeaseMetricsCollector) Describe(chan<- *prometheus.Desc) {} + +func (monitoringHeartbeatPrimaryLeaseMetricsCollector) Collect(ch chan<- prometheus.Metric) { + snapshot := MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue() + help := "Total monitoring heartbeat primary lease resolutions by outcome." + monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.PrimaryHitTotal), []string{"outcome"}, "primary_hit") + monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.NonPrimaryLiveTotal), []string{"outcome"}, "non_primary_live") + monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.InsertedTotal), []string{"outcome"}, "inserted") + monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.RenewedTotal), []string{"outcome"}, "renewed") + monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ClaimedExpiredTotal), []string{"outcome"}, "claimed_expired") + monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ClaimedUnavailableTotal), []string{"outcome"}, "claimed_unavailable") + monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ContentionTotal), []string{"outcome"}, "contention") + monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ErrorTotal), []string{"outcome"}, "error") + monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBInsertTotal), []string{"operation"}, "insert") + monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBRenewTotal), []string{"operation"}, "renew") + monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBClaimTotal), []string{"operation"}, "claim") + monitoringSummary(ch, "monitoring_heartbeat_primary_lease_duration_seconds", "Monitoring heartbeat primary lease resolution duration summary.", snapshot.DurationCount, snapshot.DurationSumSeconds, nil) + monitoringGauge(ch, "monitoring_heartbeat_primary_lease_duration_seconds_last", "Last monitoring heartbeat primary lease resolution duration in seconds.", snapshot.LastDurationSeconds, nil) + if snapshot.LastOutcome != "" { + monitoringGauge(ch, "monitoring_heartbeat_primary_lease_last_outcome_info", "Last monitoring heartbeat primary lease resolution outcome.", 1, []string{"outcome"}, snapshot.LastOutcome) + } +} + +type monitoringHeartbeatSnapshotMetricsCollector struct{} + +func (monitoringHeartbeatSnapshotMetricsCollector) Describe(chan<- *prometheus.Desc) {} + +func (monitoringHeartbeatSnapshotMetricsCollector) Collect(ch chan<- prometheus.Metric) { + snapshot := MonitoringHeartbeatSnapshotMetricsSnapshotValue() + monitoringCounter(ch, "monitoring_heartbeat_snapshot_platform_reports_total", "Total monitoring heartbeat platform reports received.", float64(snapshot.PlatformReportedTotal), nil) + monitoringCounter(ch, "monitoring_heartbeat_snapshot_transactions_total", "Total monitoring heartbeat snapshot transactions opened.", float64(snapshot.TransactionOpenedTotal), nil) + monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotInsertedTotal), []string{"operation"}, "insert") + monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotUpdatedTotal), []string{"operation"}, "update") + monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotRefreshedTotal), []string{"operation"}, "refresh") + monitoringCounter(ch, "monitoring_heartbeat_snapshot_skips_total", "Total monitoring heartbeat access snapshot reports skipped because state was fresh and unchanged.", float64(snapshot.SnapshotSkippedTotal), nil) + monitoringCounter(ch, "monitoring_heartbeat_snapshot_reconciled_tasks_total", "Total pending monitoring tasks skipped by heartbeat access reconciliation.", float64(snapshot.ReconciledSkippedTaskTotal), nil) + monitoringCounter(ch, "monitoring_heartbeat_snapshot_projection_rebuilds_total", "Total monitoring daily projection rebuilds triggered by heartbeat snapshots.", float64(snapshot.ProjectionRebuildTotal), nil) + monitoringCounter(ch, "monitoring_heartbeat_snapshot_errors_total", "Total monitoring heartbeat snapshot errors.", float64(snapshot.ErrorTotal), nil) + monitoringSummary(ch, "monitoring_heartbeat_snapshot_duration_seconds", "Monitoring heartbeat snapshot processing duration summary.", snapshot.DurationCount, snapshot.DurationSumSeconds, nil) + monitoringGauge(ch, "monitoring_heartbeat_snapshot_duration_seconds_last", "Last monitoring heartbeat snapshot processing duration in seconds.", snapshot.LastDurationSeconds, nil) + if snapshot.LastErrorCode != "" { + monitoringGauge(ch, "monitoring_heartbeat_snapshot_last_error_info", "Last monitoring heartbeat snapshot error code.", 1, []string{"code"}, snapshot.LastErrorCode) + } +} + +func monitoringCounter(ch chan<- prometheus.Metric, name, help string, value float64, labelNames []string, labelValues ...string) { + monitoringMetric(ch, name, help, prometheus.CounterValue, value, labelNames, labelValues...) +} + +func monitoringGauge(ch chan<- prometheus.Metric, name, help string, value float64, labelNames []string, labelValues ...string) { + monitoringMetric(ch, name, help, prometheus.GaugeValue, value, labelNames, labelValues...) +} + +func monitoringMetric(ch chan<- prometheus.Metric, name, help string, valueType prometheus.ValueType, value float64, labelNames []string, labelValues ...string) { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc(name, help, labelNames, nil), + valueType, + value, + monitoringPrometheusLabelValues(labelValues)..., + ) +} + +func monitoringSummary(ch chan<- prometheus.Metric, name, help string, count int64, sum float64, labelNames []string, labelValues ...string) { + if count < 0 { + count = 0 + } + ch <- prometheus.MustNewConstSummary( + prometheus.NewDesc(name, help, labelNames, nil), + uint64(count), + sum, + nil, + monitoringPrometheusLabelValues(labelValues)..., + ) +} + +func monitoringPrometheusLabelValues(values []string) []string { + if len(values) == 0 { + return values + } + result := make([]string, len(values)) + for idx, value := range values { + result[idx] = strings.ReplaceAll(value, "\r", `\r`) + } + return result +} diff --git a/server/internal/tenant/app/monitoring_phase2_desktop_tasks.go b/server/internal/tenant/app/monitoring_phase2_desktop_tasks.go index 5c87565..9070470 100644 --- a/server/internal/tenant/app/monitoring_phase2_desktop_tasks.go +++ b/server/internal/tenant/app/monitoring_phase2_desktop_tasks.go @@ -223,8 +223,7 @@ func (s *MonitoringService) loadMonitorDesktopAccountMap( rows, err := s.businessPool.Query(ctx, ` SELECT DISTINCT ON (platform_id) platform_id, - desktop_id, - health + desktop_id FROM platform_accounts WHERE tenant_id = $1 AND workspace_id = $2 @@ -251,14 +250,10 @@ func (s *MonitoringService) loadMonitorDesktopAccountMap( var ( platformID string desktopID uuid.UUID - health string ) - if scanErr := rows.Scan(&platformID, &desktopID, &health); scanErr != nil { + if scanErr := rows.Scan(&platformID, &desktopID); scanErr != nil { return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to parse monitor desktop accounts") } - if strings.TrimSpace(health) != "live" { - continue - } result[platformID] = desktopID } if err := rows.Err(); err != nil { diff --git a/server/internal/tenant/app/monitoring_projection.go b/server/internal/tenant/app/monitoring_projection.go index 44e7219..342d2e3 100644 --- a/server/internal/tenant/app/monitoring_projection.go +++ b/server/internal/tenant/app/monitoring_projection.go @@ -3,11 +3,9 @@ package app import ( "context" "database/sql" - "encoding/json" "time" "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgxpool" ) type monitoringBrandDailyAggregate struct { @@ -33,7 +31,7 @@ type monitoringPlatformDailyAggregate struct { } func (s *MonitoringCallbackService) rebuildBrandDailyProjection(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDate time.Time) error { - platforms, err := loadMonitoringProjectionPlatforms(ctx, s.businessPool, tenantID) + platforms, err := loadMonitoringProjectionPlatforms(ctx, tx, tenantID, brandID, businessDate) if err != nil { return err } @@ -133,29 +131,8 @@ func (s *MonitoringCallbackService) rebuildBrandDailyProjection(ctx context.Cont return nil } -func loadMonitoringProjectionPlatforms(ctx context.Context, businessPool *pgxpool.Pool, tenantID int64) ([]monitoringPlatformMetadata, error) { - enabled := []string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"} - - var enabledJSON []byte - if err := businessPool.QueryRow(ctx, ` - SELECT enabled_platforms - FROM tenant_monitoring_quotas - WHERE tenant_id = $1 - `, tenantID).Scan(&enabledJSON); err != nil { - if err == pgx.ErrNoRows { - return resolveMonitoringPlatforms(enabled), nil - } - return nil, monitoringInternalError(50041, "quota_lookup_failed", "failed to load monitoring quota", err) - } - - if len(enabledJSON) > 0 { - var configured []string - if jsonErr := json.Unmarshal(enabledJSON, &configured); jsonErr == nil && len(configured) > 0 { - enabled = monitoringPlatformIDs(defaultMonitoringPlatforms) - } - } - - return resolveMonitoringPlatforms(enabled), nil +func loadMonitoringProjectionPlatforms(_ context.Context, _ pgx.Tx, _ int64, _ int64, _ time.Time) ([]monitoringPlatformMetadata, error) { + return defaultMonitoringPlatformMetadata(), nil } func (s *MonitoringCallbackService) loadMonitoringEnabledQuestionCount(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDate time.Time) (int64, error) { diff --git a/server/internal/tenant/app/monitoring_service.go b/server/internal/tenant/app/monitoring_service.go index 5963d70..fd59624 100644 --- a/server/internal/tenant/app/monitoring_service.go +++ b/server/internal/tenant/app/monitoring_service.go @@ -30,6 +30,10 @@ const ( monitoringCollectorType = "desktop" monitoringOnlineDuration = 20 * time.Minute monitoringCollectNowQuestionLimit = 5 + + monitoringPlatformAuthorizationAuthorized = "authorized" + monitoringPlatformAuthorizationNoDesktopClient = "no_desktop_client" + monitoringPlatformAuthorizationNoAuthorizedPlatforms = "no_authorized_platforms" ) type MonitoringService struct { @@ -38,12 +42,14 @@ type MonitoringService struct { rabbitMQ *rabbitmq.Client logger *zap.Logger desktopTasksRolloutPercentage int + brandLibraryConfig config.BrandLibraryConfig } func NewMonitoringService( businessPool, monitoringPool *pgxpool.Pool, rabbitMQClient *rabbitmq.Client, dispatchCfg config.MonitoringDispatchConfig, + brandLibraryCfg config.BrandLibraryConfig, logger *zap.Logger, ) *MonitoringService { return &MonitoringService{ @@ -52,6 +58,7 @@ func NewMonitoringService( rabbitMQ: rabbitMQClient, logger: logger, desktopTasksRolloutPercentage: dispatchCfg.DesktopTasksRolloutPercentage, + brandLibraryConfig: brandLibraryCfg, } } @@ -65,7 +72,10 @@ type MonitoringDashboardCompositeResponse struct { } type MonitoringDashboardRuntime struct { - CurrentUserClientOnline bool `json:"current_user_client_online"` + CurrentUserClientOnline bool `json:"current_user_client_online"` + PlatformAuthorizationStatus string `json:"platform_authorization_status"` + AuthorizedMonitoringPlatformIDs []string `json:"authorized_monitoring_platform_ids"` + AuthorizedMonitoringPlatformCount int `json:"authorized_monitoring_platform_count"` } type MonitoringOverview struct { @@ -124,13 +134,14 @@ type MonitoringCitedArticle struct { } type MonitoringQuestionDetailResponse struct { - QuestionID int64 `json:"question_id"` - QuestionHash string `json:"question_hash"` - QuestionText string `json:"question_text"` - TimeWindow MonitoringQuestionDetailTimeWindow `json:"time_window"` - Platforms []MonitoringQuestionDetailPlatform `json:"platforms"` - CitationAnalysis []MonitoringQuestionCitationStats `json:"citation_analysis"` - ContentCitations []MonitoringQuestionContentCitation `json:"content_citations"` + QuestionID int64 `json:"question_id"` + QuestionHash string `json:"question_hash"` + QuestionText string `json:"question_text"` + PlatformAuthorizationStatus string `json:"platform_authorization_status"` + TimeWindow MonitoringQuestionDetailTimeWindow `json:"time_window"` + Platforms []MonitoringQuestionDetailPlatform `json:"platforms"` + CitationAnalysis []MonitoringQuestionCitationStats `json:"citation_analysis"` + ContentCitations []MonitoringQuestionContentCitation `json:"content_citations"` } type MonitoringQuestionDetailTimeWindow struct { @@ -216,6 +227,16 @@ type monitoringQuotaConfig struct { EnabledPlatforms []string } +func monitoringPlatformAuthorizationStatus(quota monitoringQuotaConfig) string { + if quota.PrimaryClientID == nil { + return monitoringPlatformAuthorizationNoDesktopClient + } + if len(quota.EnabledPlatforms) == 0 { + return monitoringPlatformAuthorizationNoAuthorizedPlatforms + } + return monitoringPlatformAuthorizationAuthorized +} + type monitoringBrandIdentity struct { ID int64 Name string @@ -290,6 +311,7 @@ func (s *MonitoringService) DashboardComposite( aiPlatformID *string, ) (*MonitoringDashboardCompositeResponse, error) { actor := auth.MustActor(ctx) + workspaceID := auth.CurrentWorkspaceID(ctx) brand, err := s.resolveBrand(ctx, actor.TenantID, brandID) if err != nil { @@ -302,12 +324,12 @@ func (s *MonitoringService) DashboardComposite( } } - quota, err := s.loadQuota(ctx, actor.TenantID) + quota, err := s.loadQuota(ctx, actor, workspaceID) if err != nil { return nil, err } - runtime, err := s.loadDashboardRuntime(ctx, actor) + runtime, err := s.loadDashboardRuntime(ctx, actor, workspaceID, quota) if err != nil { return nil, err } @@ -318,7 +340,7 @@ func (s *MonitoringService) DashboardComposite( } questionIDs := configuredQuestionIDs(configuredQuestions) - platforms := resolveMonitoringPlatforms(quota.EnabledPlatforms) + platforms := defaultMonitoringPlatformMetadata() selectedPlatformID := normalizedOptionalMonitoringPlatformPointer(aiPlatformID) filteredPlatforms := filterMonitoringPlatforms(platforms, selectedPlatformID) startDate, endDate, err := resolveDashboardDateWindow(days, businessDate) @@ -334,7 +356,7 @@ func (s *MonitoringService) DashboardComposite( return nil, err } - accessStates, err := s.loadAccessStates(ctx, actor.TenantID, quota.PrimaryClientID, endDate) + accessStates, err := s.loadAccessStates(ctx, actor.TenantID, workspaceID, quota.PrimaryClientID, endDate) if err != nil { return nil, err } @@ -369,19 +391,27 @@ func (s *MonitoringService) DashboardComposite( }, nil } -func (s *MonitoringService) loadDashboardRuntime(ctx context.Context, actor auth.Actor) (MonitoringDashboardRuntime, error) { - online, err := s.hasOnlineClientForUser(ctx, actor.TenantID, actor.PrimaryWorkspaceID, actor.UserID) - if err != nil { - return MonitoringDashboardRuntime{}, err +func (s *MonitoringService) loadDashboardRuntime(ctx context.Context, actor auth.Actor, workspaceID int64, quota monitoringQuotaConfig) (MonitoringDashboardRuntime, error) { + online := false + if quota.PrimaryClientID != nil { + var err error + online, err = s.isClientOnline(ctx, actor.TenantID, workspaceID, *quota.PrimaryClientID) + if err != nil { + return MonitoringDashboardRuntime{}, err + } } return MonitoringDashboardRuntime{ - CurrentUserClientOnline: online, + CurrentUserClientOnline: online, + PlatformAuthorizationStatus: monitoringPlatformAuthorizationStatus(quota), + AuthorizedMonitoringPlatformIDs: append([]string(nil), quota.EnabledPlatforms...), + AuthorizedMonitoringPlatformCount: len(quota.EnabledPlatforms), }, nil } func (s *MonitoringService) QuestionDetail(ctx context.Context, brandID, questionID int64, dateFrom, dateTo string, questionHash string, aiPlatformID *string) (*MonitoringQuestionDetailResponse, error) { actor := auth.MustActor(ctx) + workspaceID := auth.CurrentWorkspaceID(ctx) brand, err := s.resolveBrand(ctx, actor.TenantID, brandID) if err != nil { @@ -393,12 +423,12 @@ func (s *MonitoringService) QuestionDetail(ctx context.Context, brandID, questio return nil, err } - quota, err := s.loadQuota(ctx, actor.TenantID) + quota, err := s.loadQuota(ctx, actor, workspaceID) if err != nil { return nil, err } - platforms := resolveMonitoringPlatforms(quota.EnabledPlatforms) selectedPlatformID := normalizedOptionalMonitoringPlatformPointer(aiPlatformID) + platforms := filterMonitoringPlatforms(defaultMonitoringPlatformMetadata(), selectedPlatformID) questionText, err := s.loadQuestionText(ctx, actor.TenantID, brand.ID, questionID) if err != nil { @@ -418,7 +448,7 @@ func (s *MonitoringService) QuestionDetail(ctx context.Context, brandID, questio } } - accessStates, err := s.loadAccessStates(ctx, actor.TenantID, quota.PrimaryClientID, toDate) + accessStates, err := s.loadAccessStates(ctx, actor.TenantID, workspaceID, quota.PrimaryClientID, toDate) if err != nil { return nil, err } @@ -481,9 +511,10 @@ func (s *MonitoringService) QuestionDetail(ctx context.Context, brandID, questio } return &MonitoringQuestionDetailResponse{ - QuestionID: questionID, - QuestionHash: encodeQuestionHash(hashBytes), - QuestionText: questionText, + QuestionID: questionID, + QuestionHash: encodeQuestionHash(hashBytes), + QuestionText: questionText, + PlatformAuthorizationStatus: monitoringPlatformAuthorizationStatus(quota), TimeWindow: MonitoringQuestionDetailTimeWindow{ DateFrom: fromDate.Format("2006-01-02"), DateTo: toDate.Format("2006-01-02"), @@ -501,6 +532,7 @@ func (s *MonitoringService) CollectNow( options MonitoringCollectNowOptions, ) (*MonitoringCollectNowResponse, error) { actor := auth.MustActor(ctx) + workspaceID := auth.CurrentWorkspaceID(ctx) brand, err := s.resolveBrand(ctx, actor.TenantID, brandID) if err != nil { @@ -513,7 +545,7 @@ func (s *MonitoringService) CollectNow( } } - quota, err := s.loadQuota(ctx, actor.TenantID) + quota, err := s.loadQuota(ctx, actor, workspaceID) if err != nil { return nil, err } @@ -540,19 +572,7 @@ func (s *MonitoringService) CollectNow( }, nil } - platforms := resolveMonitoringPlatforms(quota.EnabledPlatforms) - platforms, err = intersectMonitoringPlatforms(platforms, options.PlatformIDs) - if err != nil { - return nil, err - } - questionIDs := configuredQuestionIDs(configuredQuestions) - platformIDs := monitoringPlatformIDs(platforms) - executionOwner := "legacy" - if s.shouldUseDesktopTasksExecution(actor.TenantID) { - executionOwner = "desktop_tasks" - } - - clientID, err := s.resolveCollectNowClientID(ctx, actor.TenantID, actor.PrimaryWorkspaceID, actor.UserID, options.TargetClientID) + clientID, err := s.resolveCollectNowClientID(ctx, actor.TenantID, workspaceID, actor.UserID, quota.PrimaryClientID, options.TargetClientID) if err != nil { return nil, err } @@ -560,10 +580,37 @@ func (s *MonitoringService) CollectNow( return nil, response.ErrConflict(40941, "desktop_client_offline", "current user's monitoring desktop client is offline") } - if err := s.ensureClientOnline(ctx, actor.TenantID, actor.PrimaryWorkspaceID, *clientID); err != nil { + if err := s.ensureClientOnline(ctx, actor.TenantID, workspaceID, *clientID); err != nil { return nil, err } + targetPlatformIDs, err := s.loadMonitoringClientPlatformIDs(ctx, actor.TenantID, workspaceID, *clientID) + if err != nil { + return nil, err + } + platforms := resolveMonitoringPlatforms(targetPlatformIDs) + platforms, err = intersectMonitoringPlatforms(platforms, options.PlatformIDs) + if err != nil { + return nil, err + } + if len(platforms) == 0 { + return &MonitoringCollectNowResponse{ + CollectionMode: quota.CollectionMode, + RefreshedTaskCount: 0, + CreatedTaskCount: 0, + LeasedTaskCount: 0, + CompletedTaskCount: 0, + HasEffectiveSnapshot: true, + Message: "当前桌面端暂无已授权的 AI 平台，未创建采集任务", + }, nil + } + questionIDs := configuredQuestionIDs(configuredQuestions) + platformIDs := monitoringPlatformIDs(platforms) + executionOwner := "legacy" + if s.shouldUseDesktopTasksExecution(actor.TenantID) { + executionOwner = "desktop_tasks" + } + now := time.Now().UTC() todayTime, _ := monitoringBusinessDayAndDateAt(now) requestID := uuid.New() @@ -584,7 +631,7 @@ func (s *MonitoringService) CollectNow( ctx, tx, actor.TenantID, - actor.PrimaryWorkspaceID, + workspaceID, brand.ID, actor.UserID, scopeHash, @@ -663,7 +710,7 @@ func (s *MonitoringService) CollectNow( ctx, tx, actor.TenantID, - actor.PrimaryWorkspaceID, + workspaceID, todayTime, questionIDs, platformIDs, @@ -722,7 +769,7 @@ func (s *MonitoringService) CollectNow( $12, $13, $14, $15, $16 ) - `, requestID, actor.TenantID, actor.PrimaryWorkspaceID, brand.ID, nullableInt64(keywordID), actor.UserID, *clientID, scopeHash, requestScopeJSON, affectedTaskCount, affectedTaskCount, abortedQueuedCount, int64(len(interruptTaskIDs)), interruptGeneration, message, ttlExpiresAt); err != nil { + `, requestID, actor.TenantID, workspaceID, brand.ID, nullableInt64(keywordID), actor.UserID, *clientID, scopeHash, requestScopeJSON, affectedTaskCount, affectedTaskCount, abortedQueuedCount, int64(len(interruptTaskIDs)), interruptGeneration, message, ttlExpiresAt); err != nil { return nil, response.ErrInternal(50041, "request_create_failed", "failed to persist monitoring collect-now request") } @@ -730,7 +777,7 @@ func (s *MonitoringService) CollectNow( ctx, tx, requestID, - actor.PrimaryWorkspaceID, + workspaceID, *clientID, affectedTaskCount, interruptGeneration, @@ -748,8 +795,6 @@ func (s *MonitoringService) CollectNow( return nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring collect-now") } - _ = s.persistPrimaryClientID(ctx, actor.TenantID, actor.PrimaryWorkspaceID, *clientID) - phase2PublishedTasks := make([]*repository.DesktopTask, 0) phase2DispatchReady := executionOwner == "desktop_tasks" if len(phase2TaskSpecs) > 0 { @@ -877,6 +922,7 @@ func (s *MonitoringService) CollectNow( func (s *MonitoringService) resolveCollectNowClientID( ctx context.Context, tenantID, workspaceID, userID int64, + defaultClientID *uuid.UUID, preferredClientID *uuid.UUID, ) (*uuid.UUID, error) { if preferredClientID != nil { @@ -885,7 +931,7 @@ func (s *MonitoringService) resolveCollectNowClientID( } return preferredClientID, nil } - return s.findLatestOnlineClientForUser(ctx, tenantID, workspaceID, userID) + return defaultClientID, nil } func (s *MonitoringService) hasOnlineClientForUser(ctx context.Context, tenantID, workspaceID, userID int64) (bool, error) { @@ -968,6 +1014,10 @@ func monitoringPlatformIDs(items []monitoringPlatformMetadata) []string { return result } +func defaultMonitoringPlatformMetadata() []monitoringPlatformMetadata { + return append([]monitoringPlatformMetadata(nil), defaultMonitoringPlatforms...) +} + func normalizeMonitoringPlatformID(platformID string) string { return strings.ToLower(strings.TrimSpace(platformID)) } @@ -996,10 +1046,30 @@ func monitoringPlatformQueryIDs(value *string) []string { } func reconcileEnabledMonitoringPlatforms(enabled []string) []string { - // Old quota rows may still contain partial or aliased platform ids. Admin monitoring - // should always expose the full canonical desktop-supported set for the tenant. - _ = enabled - return monitoringPlatformIDs(defaultMonitoringPlatforms) + if len(enabled) == 0 { + return nil + } + requested := make(map[string]struct{}, len(enabled)) + for _, item := range enabled { + id := normalizeMonitoringPlatformID(item) + if id == "" { + continue + } + if _, supported := monitoringPlatformNames[id]; !supported { + continue + } + requested[id] = struct{}{} + } + if len(requested) == 0 { + return nil + } + result := make([]string, 0, len(requested)) + for _, platform := range defaultMonitoringPlatforms { + if _, ok := requested[platform.ID]; ok { + result = append(result, platform.ID) + } + } + return result } func monitoringPlatformSortIndex(platformID string) int { @@ -1054,26 +1124,6 @@ func (s *MonitoringService) findLatestOnlineClientForUser(ctx context.Context, t return &clientID, nil } -func (s *MonitoringService) persistPrimaryClientID(ctx context.Context, tenantID, workspaceID int64, clientID uuid.UUID) error { - if _, err := s.businessPool.Exec(ctx, ` - INSERT INTO tenant_monitoring_quotas ( - tenant_id, workspace_id, primary_client_id - ) - VALUES ( - $1, - $2, - $3 - ) - ON CONFLICT (tenant_id) DO UPDATE - SET workspace_id = EXCLUDED.workspace_id, - primary_client_id = EXCLUDED.primary_client_id, - updated_at = NOW() - `, tenantID, workspaceID, clientID); err != nil { - return response.ErrInternal(50041, "update_failed", "failed to persist primary monitoring desktop client") - } - return nil -} - func formatPgInterval(duration time.Duration) string { return fmt.Sprintf("%d seconds", int64(duration/time.Second)) } @@ -1431,40 +1481,117 @@ func (s *MonitoringService) deferQueuedNormalTasks( return tag.RowsAffected(), nil } -func (s *MonitoringService) loadQuota(ctx context.Context, tenantID int64) (monitoringQuotaConfig, error) { +func (s *MonitoringService) loadQuota(ctx context.Context, actor auth.Actor, workspaceID int64) (monitoringQuotaConfig, error) { cfg := monitoringQuotaConfig{ - CollectionMode: monitoringCollectorType, - EnabledPlatforms: reconcileEnabledMonitoringPlatforms(nil), + CollectionMode: monitoringCollectorType, } - var enabledJSON []byte - var clientID uuid.NullUUID + clientID, err := s.findLatestRegisteredClientWithMonitoringAccountForUser(ctx, actor.TenantID, workspaceID, actor.UserID) + if err != nil { + return cfg, err + } + if clientID == nil { + clientID, err = s.findLatestRegisteredClientForUser(ctx, actor.TenantID, workspaceID, actor.UserID) + if err != nil { + return cfg, err + } + } + cfg.PrimaryClientID = clientID + if clientID == nil { + return cfg, nil + } + platforms, err := s.loadMonitoringClientPlatformIDs(ctx, actor.TenantID, workspaceID, *clientID) + if err != nil { + return cfg, err + } + cfg.EnabledPlatforms = platforms + return cfg, nil +} + +// findLatestRegisteredClientWithMonitoringAccountForUser returns the newest +// non-revoked desktop client that has at least one bound monitoring platform +// account. Platform account health is not authorization. +func (s *MonitoringService) findLatestRegisteredClientWithMonitoringAccountForUser(ctx context.Context, tenantID, workspaceID, userID int64) (*uuid.UUID, error) { + var clientID uuid.UUID err := s.businessPool.QueryRow(ctx, ` - SELECT collection_mode, enabled_platforms, primary_client_id - FROM tenant_monitoring_quotas - WHERE tenant_id = $1 - `, tenantID).Scan(&cfg.CollectionMode, &enabledJSON, &clientID) + SELECT dc.id + FROM desktop_clients dc + WHERE dc.tenant_id = $1 + AND dc.workspace_id = $2 + AND dc.user_id = $3 + AND dc.revoked_at IS NULL + AND EXISTS ( + SELECT 1 + FROM platform_accounts pa + WHERE pa.tenant_id = dc.tenant_id + AND pa.workspace_id = dc.workspace_id + AND pa.client_id = dc.id + AND pa.deleted_at IS NULL + AND pa.platform_id = ANY($4::text[]) + ) + ORDER BY dc.created_at DESC, dc.id DESC + LIMIT 1 + `, tenantID, workspaceID, userID, monitoringPlatformIDs(defaultMonitoringPlatforms)).Scan(&clientID) if err != nil { if err == pgx.ErrNoRows { - return cfg, nil + return nil, nil } - return cfg, response.ErrInternal(50041, "query_failed", "failed to load monitoring quota") + return nil, response.ErrInternal(50041, "query_failed", "failed to inspect monitoring desktop client authorization") } + return &clientID, nil +} - if len(enabledJSON) > 0 { - var enabled []string - if jsonErr := json.Unmarshal(enabledJSON, &enabled); jsonErr == nil { - cfg.EnabledPlatforms = reconcileEnabledMonitoringPlatforms(enabled) +// findLatestRegisteredClientForUser returns the newest non-revoked desktop client +// registered by creation order. It intentionally ignores last_seen_at and health; +// use findLatestOnlineClientForUser when online presence is required. +func (s *MonitoringService) findLatestRegisteredClientForUser(ctx context.Context, tenantID, workspaceID, userID int64) (*uuid.UUID, error) { + var clientID uuid.UUID + err := s.businessPool.QueryRow(ctx, ` + SELECT id + FROM desktop_clients + WHERE tenant_id = $1 + AND workspace_id = $2 + AND user_id = $3 + AND revoked_at IS NULL + ORDER BY created_at DESC, id DESC + LIMIT 1 + `, tenantID, workspaceID, userID).Scan(&clientID) + if err != nil { + if err == pgx.ErrNoRows { + return nil, nil } + return nil, response.ErrInternal(50041, "query_failed", "failed to inspect monitoring desktop client") } - if clientID.Valid { - id := clientID.UUID - cfg.PrimaryClientID = &id + return &clientID, nil +} + +func (s *MonitoringService) loadMonitoringClientPlatformIDs(ctx context.Context, tenantID, workspaceID int64, clientID uuid.UUID) ([]string, error) { + rows, err := s.businessPool.Query(ctx, ` + SELECT DISTINCT platform_id + FROM platform_accounts + WHERE tenant_id = $1 + AND workspace_id = $2 + AND client_id = $3 + AND deleted_at IS NULL + AND platform_id = ANY($4::text[]) + `, tenantID, workspaceID, clientID, monitoringPlatformIDs(defaultMonitoringPlatforms)) + if err != nil { + return nil, response.ErrInternal(50041, "query_failed", "failed to load monitoring desktop platform accounts") } - if strings.TrimSpace(cfg.CollectionMode) == "" { - cfg.CollectionMode = monitoringCollectorType + defer rows.Close() + + platformIDs := make([]string, 0) + for rows.Next() { + var platformID string + if scanErr := rows.Scan(&platformID); scanErr != nil { + return nil, response.ErrInternal(50041, "scan_failed", "failed to parse monitoring desktop platform accounts") + } + platformIDs = append(platformIDs, platformID) } - return cfg, nil + if err := rows.Err(); err != nil { + return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate monitoring desktop platform accounts") + } + return reconcileEnabledMonitoringPlatforms(platformIDs), nil } func newMonitoringDerivedMetrics() monitoringDerivedMetrics { @@ -2021,11 +2148,18 @@ func applyDerivedRatesToOverviewSnapshot(snapshot *monitoringOverviewSnapshot, d } } -func (s *MonitoringService) loadAccessStates(ctx context.Context, tenantID int64, clientID *uuid.UUID, businessDate time.Time) (map[string]monitoringAccessState, error) { +func (s *MonitoringService) loadAccessStates(ctx context.Context, tenantID, workspaceID int64, clientID *uuid.UUID, businessDate time.Time) (map[string]monitoringAccessState, error) { states := map[string]monitoringAccessState{} if clientID == nil { return states, nil } + belongs, err := s.monitoringClientBelongsToWorkspace(ctx, tenantID, workspaceID, *clientID) + if err != nil { + return nil, err + } + if !belongs { + return states, nil + } rows, err := s.monitoringPool.Query(ctx, ` SELECT ai_platform_id, access_status @@ -2054,10 +2188,30 @@ func (s *MonitoringService) loadAccessStates(ctx context.Context, tenantID int64 } states[platformID] = monitoringAccessState{AccessStatus: accessStatus} } + if err := rows.Err(); err != nil { + return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate platform access states") + } return states, nil } +func (s *MonitoringService) monitoringClientBelongsToWorkspace(ctx context.Context, tenantID, workspaceID int64, clientID uuid.UUID) (bool, error) { + var exists bool + if err := s.businessPool.QueryRow(ctx, ` + SELECT EXISTS ( + SELECT 1 + FROM desktop_clients + WHERE id = $1 + AND tenant_id = $2 + AND workspace_id = $3 + AND revoked_at IS NULL + ) + `, clientID, tenantID, workspaceID).Scan(&exists); err != nil { + return false, response.ErrInternal(50041, "query_failed", "failed to validate monitoring desktop client workspace") + } + return exists, nil +} + func (s *MonitoringService) loadPlatformBreakdown(ctx context.Context, tenantID, brandID int64, businessDate time.Time, collectionMode string, platforms []monitoringPlatformMetadata, accessStates map[string]monitoringAccessState, derived monitoringDerivedMetrics) ([]MonitoringPlatformDaily, error) { rows, err := s.monitoringPool.Query(ctx, ` SELECT @@ -3117,7 +3271,7 @@ func normalizeTrackingDays(days int) int { func resolveMonitoringPlatforms(enabled []string) []monitoringPlatformMetadata { if len(enabled) == 0 { - return defaultMonitoringPlatforms + return nil } platforms := make([]monitoringPlatformMetadata, 0, len(enabled)) seen := make(map[string]struct{}, len(enabled)) @@ -3139,7 +3293,7 @@ func resolveMonitoringPlatforms(enabled []string) []monitoringPlatformMetadata { }) } if len(platforms) == 0 { - return defaultMonitoringPlatforms + return nil } return platforms } @@ -3192,12 +3346,7 @@ func filterMonitoringPlatforms(platforms []monitoringPlatformMetadata, aiPlatfor } } - return []monitoringPlatformMetadata{ - { - ID: platformID, - Name: platformDisplayName(platformID), - }, - } + return nil } func platformDisplayName(platformID string) string { diff --git a/server/internal/tenant/app/monitoring_service_date_test.go b/server/internal/tenant/app/monitoring_service_date_test.go index b46ff62..475a9f3 100644 --- a/server/internal/tenant/app/monitoring_service_date_test.go +++ b/server/internal/tenant/app/monitoring_service_date_test.go @@ -4,6 +4,7 @@ import ( "testing" "time" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -81,7 +82,7 @@ func TestFilterMonitoringPlatformsReturnsSelectedPlatform(t *testing.T) { }, filtered) } -func TestFilterMonitoringPlatformsFallsBackToDisplayName(t *testing.T) { +func TestFilterMonitoringPlatformsDropsUnauthorizedSelection(t *testing.T) { platforms := []monitoringPlatformMetadata{ {ID: "deepseek", Name: "DeepSeek"}, } @@ -89,9 +90,7 @@ func TestFilterMonitoringPlatformsFallsBackToDisplayName(t *testing.T) { filtered := filterMonitoringPlatforms(platforms, &selected) - assert.Equal(t, []monitoringPlatformMetadata{ - {ID: "doubao", Name: "豆包"}, - }, filtered) + assert.Empty(t, filtered) } func TestNormalizeMonitoringPlatformIDCanonicalizesLatestIDs(t *testing.T) { @@ -127,8 +126,40 @@ func TestIntersectMonitoringPlatformsAcceptsCanonicalIDs(t *testing.T) { }, filtered) } -func TestReconcileEnabledMonitoringPlatformsBackfillsSupportedCatalog(t *testing.T) { - assert.Equal(t, []string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"}, reconcileEnabledMonitoringPlatforms([]string{"deepseek", "qwen", "doubao"})) +func TestReconcileEnabledMonitoringPlatformsKeepsAuthorizedSubset(t *testing.T) { + assert.Equal(t, []string{"deepseek", "doubao", "qwen"}, reconcileEnabledMonitoringPlatforms([]string{"deepseek", "qwen", "doubao"})) + assert.Empty(t, reconcileEnabledMonitoringPlatforms(nil)) + assert.Empty(t, resolveMonitoringPlatforms(nil)) +} + +func TestMonitoringPlatformAuthorizationStatus(t *testing.T) { + clientID := uuid.New() + + assert.Equal(t, monitoringPlatformAuthorizationNoDesktopClient, monitoringPlatformAuthorizationStatus(monitoringQuotaConfig{})) + assert.Equal(t, monitoringPlatformAuthorizationNoAuthorizedPlatforms, monitoringPlatformAuthorizationStatus(monitoringQuotaConfig{ + PrimaryClientID: &clientID, + })) + assert.Equal(t, monitoringPlatformAuthorizationAuthorized, monitoringPlatformAuthorizationStatus(monitoringQuotaConfig{ + PrimaryClientID: &clientID, + EnabledPlatforms: []string{"qwen"}, + })) +} + +func TestDefaultMonitoringPlatformMetadataReturnsFullCatalogCopy(t *testing.T) { + platforms := defaultMonitoringPlatformMetadata() + + require.Len(t, platforms, 6) + assert.Equal(t, []string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"}, monitoringPlatformIDs(platforms)) + + platforms[0].ID = "mutated" + assert.Equal(t, "yuanbao", defaultMonitoringPlatformMetadata()[0].ID) +} + +func TestLoadMonitoringProjectionPlatformsReturnsFullCatalog(t *testing.T) { + platforms, err := loadMonitoringProjectionPlatforms(nil, nil, 0, 0, time.Time{}) + require.NoError(t, err) + + assert.Equal(t, []string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"}, monitoringPlatformIDs(platforms)) } func TestApplyDerivedRatesToOverviewSnapshotUsesAllSamplesForMentionRate(t *testing.T) { diff --git a/server/internal/tenant/transport/internal_metrics_handler.go b/server/internal/tenant/transport/internal_metrics_handler.go new file mode 100644 index 0000000..8d801cc --- /dev/null +++ b/server/internal/tenant/transport/internal_metrics_handler.go @@ -0,0 +1,61 @@ +package transport + +import ( + "strings" + + "github.com/gin-gonic/gin" + + "github.com/geo-platform/tenant-api/internal/bootstrap" + "github.com/geo-platform/tenant-api/internal/shared/response" + tenantapp "github.com/geo-platform/tenant-api/internal/tenant/app" +) + +func registerInternalMetricsRoutes(app *bootstrap.App) { + token := strings.TrimSpace(app.Config.Scheduler.InternalMetricsToken) + if token == "" { + app.Logger.Warn("tenant-api internal metrics routes not registered: scheduler.internal_metrics_token is required") + return + } + + metrics := app.Engine.Group("/api/internal/metrics") + metrics.Use(internalMetricsAuthMiddleware(token)) + metrics.GET("/monitoring/heartbeat-primary-lease", func(c *gin.Context) { + response.Success(c, tenantapp.MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue()) + }) + primaryLeasePrometheusHandler := tenantapp.MonitoringHeartbeatPrimaryLeaseMetricsPrometheusHandler() + metrics.GET("/monitoring/heartbeat-primary-lease/prometheus", func(c *gin.Context) { + primaryLeasePrometheusHandler.ServeHTTP(c.Writer, c.Request) + }) + metrics.GET("/monitoring/heartbeat-snapshots", func(c *gin.Context) { + response.Success(c, tenantapp.MonitoringHeartbeatSnapshotMetricsSnapshotValue()) + }) + snapshotPrometheusHandler := tenantapp.MonitoringHeartbeatSnapshotMetricsPrometheusHandler() + metrics.GET("/monitoring/heartbeat-snapshots/prometheus", func(c *gin.Context) { + snapshotPrometheusHandler.ServeHTTP(c.Writer, c.Request) + }) +} + +func internalMetricsAuthMiddleware(token string) gin.HandlerFunc { + expected := strings.TrimSpace(token) + return func(c *gin.Context) { + if expected == "" || !internalMetricsTokenMatches(c, expected) { + response.Error(c, response.ErrUnauthorized(40181, "internal_metrics_unauthorized", "valid internal metrics token is required")) + c.Abort() + return + } + c.Next() + } +} + +func internalMetricsTokenMatches(c *gin.Context, expected string) bool { + if c == nil { + return false + } + header := strings.TrimSpace(c.GetHeader("Authorization")) + if strings.HasPrefix(strings.ToLower(header), "bearer ") { + if strings.TrimSpace(header[len("bearer "):]) == expected { + return true + } + } + return strings.TrimSpace(c.GetHeader("X-Internal-Metrics-Token")) == expected +} diff --git a/server/internal/tenant/transport/monitoring_handler.go b/server/internal/tenant/transport/monitoring_handler.go index 18e7d24..3319569 100644 --- a/server/internal/tenant/transport/monitoring_handler.go +++ b/server/internal/tenant/transport/monitoring_handler.go @@ -26,7 +26,7 @@ type monitoringCollectNowRequest struct { func NewMonitoringHandler(a *bootstrap.App) *MonitoringHandler { return &MonitoringHandler{ - svc: app.NewMonitoringService(a.DB, a.MonitoringDB, a.RabbitMQ, a.Config.MonitoringDispatch, a.Logger), + svc: app.NewMonitoringService(a.DB, a.MonitoringDB, a.RabbitMQ, a.Config.MonitoringDispatch, a.Config.BrandLibrary, a.Logger), } } diff --git a/server/internal/tenant/transport/router.go b/server/internal/tenant/transport/router.go index 43d74c4..c20bdbc 100644 --- a/server/internal/tenant/transport/router.go +++ b/server/internal/tenant/transport/router.go @@ -8,6 +8,8 @@ import ( ) func RegisterRoutes(app *bootstrap.App) { + registerInternalMetricsRoutes(app) + authAPI := app.Engine.Group("/api/auth") authHandler := NewAuthHandler(app) authAPI.POST("/login", authHandler.Login) @@ -21,6 +23,7 @@ func RegisterRoutes(app *bootstrap.App) { protected := app.Engine.Group("/api") protected.Use(auth.Middleware(app.JWT, app.Sessions)) protected.Use(middleware.TenantScope()) + protected.Use(middleware.WorkspaceScope(app.DB)) protected.GET("/auth/me", authHandler.Me) protected.POST("/auth/logout", authHandler.Logout) diff --git a/server/migrations/20260424130000_add_desktop_primary_leases.down.sql b/server/migrations/20260424130000_add_desktop_primary_leases.down.sql new file mode 100644 index 0000000..7915749 --- /dev/null +++ b/server/migrations/20260424130000_add_desktop_primary_leases.down.sql @@ -0,0 +1,4 @@ +DROP INDEX IF EXISTS idx_desktop_clients_tenant_workspace_user_seen_active; +DROP INDEX IF EXISTS idx_desktop_clients_tenant_workspace_seen_active; +DROP INDEX IF EXISTS idx_desktop_client_primary_leases_primary_client; +DROP TABLE IF EXISTS desktop_client_primary_leases; diff --git a/server/migrations/20260424130000_add_desktop_primary_leases.up.sql b/server/migrations/20260424130000_add_desktop_primary_leases.up.sql new file mode 100644 index 0000000..02d3c7d --- /dev/null +++ b/server/migrations/20260424130000_add_desktop_primary_leases.up.sql @@ -0,0 +1,52 @@ +-- Backfilled primary leases use the same 90s TTL as +-- server/internal/tenant/app/desktop_presence.go:desktopClientPresenceTTL. +-- Keep these values aligned; desktop heartbeat interval must be < TTL/2 so +-- a single missed heartbeat does not let another client steal primary. +CREATE TABLE IF NOT EXISTS desktop_client_primary_leases ( + tenant_id BIGINT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE, + workspace_id BIGINT NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + primary_client_id UUID NOT NULL REFERENCES desktop_clients(id) ON DELETE CASCADE, + lease_expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (tenant_id, workspace_id) +); + +CREATE INDEX IF NOT EXISTS idx_desktop_client_primary_leases_primary_client + ON desktop_client_primary_leases (primary_client_id); + +CREATE INDEX IF NOT EXISTS idx_desktop_clients_tenant_workspace_seen_active + ON desktop_clients (tenant_id, workspace_id, last_seen_at DESC, created_at DESC, id DESC) + WHERE revoked_at IS NULL; + +CREATE INDEX IF NOT EXISTS idx_desktop_clients_tenant_workspace_user_seen_active + ON desktop_clients (tenant_id, workspace_id, user_id, last_seen_at DESC, created_at DESC, id DESC) + WHERE revoked_at IS NULL; + +INSERT INTO desktop_client_primary_leases ( + tenant_id, + workspace_id, + primary_client_id, + lease_expires_at +) +SELECT + tenant_id, + workspace_id, + id, + COALESCE(last_seen_at, created_at) + INTERVAL '90 seconds' +FROM ( + SELECT + dc.id, + dc.tenant_id, + dc.workspace_id, + dc.created_at, + dc.last_seen_at, + ROW_NUMBER() OVER ( + PARTITION BY dc.tenant_id, dc.workspace_id + ORDER BY dc.last_seen_at DESC NULLS LAST, dc.created_at DESC, dc.id DESC + ) AS rn + FROM desktop_clients dc + WHERE dc.revoked_at IS NULL +) ranked_clients +WHERE rn = 1 +ON CONFLICT (tenant_id, workspace_id) DO NOTHING; diff --git a/server/migrations_monitoring/20260424133000_add_monitoring_pending_platform_reconcile_index.down.sql b/server/migrations_monitoring/20260424133000_add_monitoring_pending_platform_reconcile_index.down.sql new file mode 100644 index 0000000..5e040dd --- /dev/null +++ b/server/migrations_monitoring/20260424133000_add_monitoring_pending_platform_reconcile_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_collect_tasks_pending_platform_reconcile; diff --git a/server/migrations_monitoring/20260424133000_add_monitoring_pending_platform_reconcile_index.up.sql b/server/migrations_monitoring/20260424133000_add_monitoring_pending_platform_reconcile_index.up.sql new file mode 100644 index 0000000..75adff7 --- /dev/null +++ b/server/migrations_monitoring/20260424133000_add_monitoring_pending_platform_reconcile_index.up.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS idx_collect_tasks_pending_platform_reconcile + ON monitoring_collect_tasks (tenant_id, business_date, ai_platform_id) + WHERE collector_type = 'desktop' + AND status = 'pending';