feat(monitoring): generate daily tasks and optimize heartbeat writes

Adds a MonitoringDailyTaskWorker that materializes and dispatches
daily collect tasks from active subscription/desktop-client state,
with per-run atomic metrics and a Prometheus collector. Dashboard
and question-detail APIs now surface a platform_authorization_status
so the UI can distinguish no-desktop-client, no-authorized-platforms,
and authorized states; quota/access-state lookups resolve by request
workspace instead of tenant.

Hardens heartbeat: a desktop_client_primary_leases table gives
per-(tenant, workspace) sticky primary selection, read-mostly lease
resolution avoids a per-heartbeat transaction, and unchanged platform
access reports skip snapshot upserts outside a 10-minute refresh
window. Adds heartbeat primary/snapshot metrics exposed via
token-protected tenant-api /api/internal/metrics endpoints plus
Prometheus. Monitoring quota seed is removed from dev-seed since
daily plans now derive from desktop bindings, and the projection
uses the canonical six-platform catalog so platforms with zero
samples still render.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-04-24 22:19:57 +08:00
parent ffe4d335aa
commit ff86933c24
26 changed files with 3546 additions and 277 deletions
-26
View File
@@ -246,10 +246,6 @@ func seedBusinessData(ctx context.Context, pool *pgxpool.Pool, membershipCfg con
return nil, err return nil, err
} }
if err := ensureMonitoringQuota(ctx, tx, state); err != nil {
return nil, err
}
if err := tx.Commit(ctx); err != nil { if err := tx.Commit(ctx); err != nil {
return nil, fmt.Errorf("commit business seed: %w", err) return nil, fmt.Errorf("commit business seed: %w", err)
} }
@@ -692,28 +688,6 @@ func ensureDesktopClient(ctx context.Context, tx pgx.Tx, tenantID, workspaceID,
return ensuredID, nil return ensuredID, nil
} }
func ensureMonitoringQuota(ctx context.Context, tx pgx.Tx, state *seedState) error {
enabledPlatforms, _ := json.Marshal([]string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"})
_, err := tx.Exec(ctx, `
INSERT INTO tenant_monitoring_quotas (
tenant_id, workspace_id, max_brands, collection_mode, question_selection_mode, collect_frequency,
enabled_platforms, primary_client_id, task_daily_hard_cap, plan_tier
)
VALUES ($1, $2, 1, 'desktop', 'best_effort_all', 'daily', $3::jsonb, $4, 36, 'free')
ON CONFLICT (tenant_id) DO UPDATE
SET workspace_id = EXCLUDED.workspace_id,
collection_mode = EXCLUDED.collection_mode,
question_selection_mode = EXCLUDED.question_selection_mode,
collect_frequency = EXCLUDED.collect_frequency,
enabled_platforms = EXCLUDED.enabled_platforms,
primary_client_id = EXCLUDED.primary_client_id,
task_daily_hard_cap = EXCLUDED.task_daily_hard_cap,
plan_tier = EXCLUDED.plan_tier,
updated_at = NOW()
`, state.TenantID, state.WorkspaceID, string(enabledPlatforms), state.DesktopClientID)
return wrapSeedErr("upsert monitoring quota", err)
}
func clearMonitoringData(ctx context.Context, tx pgx.Tx, state *seedState) error { func clearMonitoringData(ctx context.Context, tx pgx.Tx, state *seedState) error {
pairs := []string{ pairs := []string{
`DELETE FROM monitoring_citation_facts WHERE tenant_id = $1 AND brand_id = $2`, `DELETE FROM monitoring_citation_facts WHERE tenant_id = $1 AND brand_id = $2`,
+1 -1
View File
@@ -32,7 +32,7 @@ func main() {
app.DesktopTaskStreams.Run(workerCtx) app.DesktopTaskStreams.Run(workerCtx)
app.DesktopDispatch.Run(workerCtx) app.DesktopDispatch.Run(workerCtx)
monitoringService := tenantapp.NewMonitoringService(app.DB, app.MonitoringDB, app.RabbitMQ, app.Config.MonitoringDispatch, app.Logger) monitoringService := tenantapp.NewMonitoringService(app.DB, app.MonitoringDB, app.RabbitMQ, app.Config.MonitoringDispatch, app.Config.BrandLibrary, app.Logger)
monitoringCallbackService := tenantapp.NewMonitoringCallbackService(app.DB, app.MonitoringDB, app.RabbitMQ, app.LLM, app.Logger) monitoringCallbackService := tenantapp.NewMonitoringCallbackService(app.DB, app.MonitoringDB, app.RabbitMQ, app.LLM, app.Logger)
tenantapp.NewMonitoringCollectOutboxWorker(monitoringService, app.Logger).Start(workerCtx) tenantapp.NewMonitoringCollectOutboxWorker(monitoringService, app.Logger).Start(workerCtx)
tenantapp.NewMonitoringResultIngestWorker(monitoringCallbackService, app.RabbitMQ, app.Logger, app.Config.MonitoringWorkers).Start(workerCtx) tenantapp.NewMonitoringResultIngestWorker(monitoringCallbackService, app.RabbitMQ, app.Logger, app.Config.MonitoringWorkers).Start(workerCtx)
+6
View File
@@ -14,6 +14,8 @@ require (
github.com/jackc/pgx/v5 v5.5.5 github.com/jackc/pgx/v5 v5.5.5
github.com/ledongthuc/pdf v0.0.0-20240314090751-a2a84ec735c3 github.com/ledongthuc/pdf v0.0.0-20240314090751-a2a84ec735c3
github.com/minio/minio-go/v7 v7.0.83 github.com/minio/minio-go/v7 v7.0.83
github.com/prometheus/client_golang v1.20.5
github.com/prometheus/common v0.60.1
github.com/qdrant/go-client v1.13.0 github.com/qdrant/go-client v1.13.0
github.com/rabbitmq/amqp091-go v1.10.0 github.com/rabbitmq/amqp091-go v1.10.0
github.com/redis/go-redis/v9 v9.5.1 github.com/redis/go-redis/v9 v9.5.1
@@ -32,6 +34,7 @@ require (
) )
require ( require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/bytedance/sonic v1.9.1 // indirect github.com/bytedance/sonic v1.9.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
@@ -63,8 +66,11 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/richardlehane/mscfb v1.0.4 // indirect github.com/richardlehane/mscfb v1.0.4 // indirect
github.com/richardlehane/msoleps v1.0.3 // indirect github.com/richardlehane/msoleps v1.0.3 // indirect
github.com/rs/xid v1.6.0 // indirect github.com/rs/xid v1.6.0 // indirect
+14
View File
@@ -5,6 +5,8 @@ github.com/alicebob/miniredis/v2 v2.37.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g=
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
@@ -115,6 +117,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/ledongthuc/pdf v0.0.0-20240314090751-a2a84ec735c3 h1:DDPccSesc/SJu8CiCEhHhWjK9C9U1fhrRKB3jRNw4f4= github.com/ledongthuc/pdf v0.0.0-20240314090751-a2a84ec735c3 h1:DDPccSesc/SJu8CiCEhHhWjK9C9U1fhrRKB3jRNw4f4=
github.com/ledongthuc/pdf v0.0.0-20240314090751-a2a84ec735c3/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/ledongthuc/pdf v0.0.0-20240314090751-a2a84ec735c3/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
@@ -136,13 +140,23 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc=
github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/qdrant/go-client v1.13.0 h1:qeWKCs1vxvfF2MLLFnP2qDG0R8wI18HyAoSfc7wJim8= github.com/qdrant/go-client v1.13.0 h1:qeWKCs1vxvfF2MLLFnP2qDG0R8wI18HyAoSfc7wJim8=
github.com/qdrant/go-client v1.13.0/go.mod h1:iO8ts78jL4x6LDHFOViyYWELVtIBDTjOykBmiOTHLnQ= github.com/qdrant/go-client v1.13.0/go.mod h1:iO8ts78jL4x6LDHFOViyYWELVtIBDTjOykBmiOTHLnQ=
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
+17
View File
@@ -105,6 +105,9 @@ type RabbitMQConfig struct {
} }
type SchedulerConfig struct { type SchedulerConfig struct {
HTTPHost string `mapstructure:"http_host"`
HTTPPort int `mapstructure:"http_port"`
InternalMetricsToken string `mapstructure:"internal_metrics_token"`
DispatchInterval time.Duration `mapstructure:"dispatch_interval"` DispatchInterval time.Duration `mapstructure:"dispatch_interval"`
DispatchTimeout time.Duration `mapstructure:"dispatch_timeout"` DispatchTimeout time.Duration `mapstructure:"dispatch_timeout"`
DispatchBatchSize int `mapstructure:"dispatch_batch_size"` DispatchBatchSize int `mapstructure:"dispatch_batch_size"`
@@ -427,6 +430,12 @@ func applyEnvOverrides(cfg *Config) {
if model, ok := lookupFirstNonEmptyEnv("SILICONFLOW_RERANKER_MODEL", "RETRIEVAL_RERANKER_MODEL"); ok { if model, ok := lookupFirstNonEmptyEnv("SILICONFLOW_RERANKER_MODEL", "RETRIEVAL_RERANKER_MODEL"); ok {
cfg.Retrieval.RerankerModel = model cfg.Retrieval.RerankerModel = model
} }
if host, ok := lookupNonEmptyEnv("SCHEDULER_HTTP_HOST"); ok {
cfg.Scheduler.HTTPHost = host
}
if token, ok := lookupFirstNonEmptyEnv("SCHEDULER_INTERNAL_METRICS_TOKEN", "SCHEDULER_METRICS_TOKEN"); ok {
cfg.Scheduler.InternalMetricsToken = token
}
} }
func normalizeRabbitMQConfig(cfg *RabbitMQConfig) { func normalizeRabbitMQConfig(cfg *RabbitMQConfig) {
@@ -569,6 +578,14 @@ func normalizeSchedulerConfig(cfg *SchedulerConfig) {
return return
} }
if cfg.HTTPPort == 0 {
cfg.HTTPPort = 8081
}
cfg.HTTPHost = strings.TrimSpace(cfg.HTTPHost)
if cfg.HTTPHost == "" {
cfg.HTTPHost = "127.0.0.1"
}
cfg.InternalMetricsToken = strings.TrimSpace(cfg.InternalMetricsToken)
if cfg.DispatchInterval <= 0 { if cfg.DispatchInterval <= 0 {
cfg.DispatchInterval = 15 * time.Second cfg.DispatchInterval = 15 * time.Second
} }
@@ -114,6 +114,58 @@ brand_library: {}
} }
} }
func TestLoadAppliesSchedulerHTTPDefaultsAndDisableSentinel(t *testing.T) {
defaultConfigPath := writeTestConfig(t, `
scheduler: {}
`)
cfg, err := Load(defaultConfigPath)
if err != nil {
t.Fatalf("Load() default error = %v", err)
}
if cfg.Scheduler.HTTPHost != "127.0.0.1" {
t.Fatalf("expected default scheduler http host 127.0.0.1, got %q", cfg.Scheduler.HTTPHost)
}
if cfg.Scheduler.HTTPPort != 8081 {
t.Fatalf("expected default scheduler http port 8081, got %d", cfg.Scheduler.HTTPPort)
}
disabledConfigPath := writeTestConfig(t, `
scheduler:
http_port: -1
`)
cfg, err = Load(disabledConfigPath)
if err != nil {
t.Fatalf("Load() disabled error = %v", err)
}
if cfg.Scheduler.HTTPPort != -1 {
t.Fatalf("expected scheduler http port -1 to disable server, got %d", cfg.Scheduler.HTTPPort)
}
}
func TestLoadPrefersEnvSchedulerMetricsSettings(t *testing.T) {
t.Setenv("SCHEDULER_HTTP_HOST", "0.0.0.0")
t.Setenv("SCHEDULER_INTERNAL_METRICS_TOKEN", "env-metrics-token")
configPath := writeTestConfig(t, `
scheduler:
http_host: 127.0.0.1
internal_metrics_token: config-token
`)
cfg, err := Load(configPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Scheduler.HTTPHost != "0.0.0.0" {
t.Fatalf("expected env scheduler http host, got %q", cfg.Scheduler.HTTPHost)
}
if cfg.Scheduler.InternalMetricsToken != "env-metrics-token" {
t.Fatalf("expected env scheduler metrics token, got %q", cfg.Scheduler.InternalMetricsToken)
}
}
func TestLoadAppliesMembershipDefaults(t *testing.T) { func TestLoadAppliesMembershipDefaults(t *testing.T) {
configPath := writeTestConfig(t, ` configPath := writeTestConfig(t, `
membership: {} membership: {}
@@ -9,6 +9,9 @@ import (
goredis "github.com/redis/go-redis/v9" goredis "github.com/redis/go-redis/v9"
) )
// desktopClientPresenceTTL is also used as the monitoring primary-client lease TTL.
// Desktop heartbeat interval must stay below TTL/2, so one missed heartbeat does
// not make another client eligible to steal a still-healthy primary lease.
const desktopClientPresenceTTL = 90 * time.Second const desktopClientPresenceTTL = 90 * time.Second
const desktopAccountPresenceTTL = 90 * time.Second const desktopAccountPresenceTTL = 90 * time.Second
@@ -30,6 +30,14 @@ const (
maxMonitoringLeaseLimit = 50 maxMonitoringLeaseLimit = 50
monitoringLeaseTTL = 15 * time.Minute monitoringLeaseTTL = 15 * time.Minute
monitoringTaskResultQueueIOTimeout = 5 * time.Second monitoringTaskResultQueueIOTimeout = 5 * time.Second
// The current primary renews the DB lease only after this interval. With the
// desktop client's 25s heartbeat and a 90s lease TTL, this keeps normal
// heartbeats read-only while still tolerating one missed heartbeat.
monitoringHeartbeatPrimaryRenewAfter = desktopClientPresenceTTL / 3
// Access snapshots are semantic state, not a liveness clock. Stable heartbeat
// reports skip writes and only refresh the row at this interval so dashboards
// do not depend on per-heartbeat database churn.
monitoringHeartbeatAccessSnapshotRefreshAfter = 10 * time.Minute
) )
type MonitoringCallbackService struct { type MonitoringCallbackService struct {
@@ -205,6 +213,16 @@ type monitoringPlatformAccessSnapshotState struct {
Accessible bool Accessible bool
ReasonCode *string ReasonCode *string
ReasonMessage *string ReasonMessage *string
UpdatedAt time.Time
}
type monitoringHeartbeatSnapshotDecision struct {
WriteSnapshot bool
StateChanged bool
RefreshDue bool
PendingReconcileCandidate bool
ReconcilePending bool
SnapshotOperation string
} }
type monitoringCollectTask struct { type monitoringCollectTask struct {
@@ -434,7 +452,18 @@ func (s *MonitoringCallbackService) handleTaskResultForInstallation(
}, nil }, nil
} }
func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationID string, installationToken string, req MonitoringHeartbeatRequest) (*MonitoringHeartbeatResponse, error) { func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationID string, installationToken string, req MonitoringHeartbeatRequest) (_ *MonitoringHeartbeatResponse, err error) {
startedAt := time.Now()
snapshotMetrics := monitoringHeartbeatSnapshotMetricsEvent{}
defer func() {
if !snapshotMetrics.Active {
return
}
snapshotMetrics.Duration = time.Since(startedAt)
snapshotMetrics.Err = err
recordMonitoringHeartbeatSnapshotMetrics(snapshotMetrics)
}()
installation, err := s.authenticateInstallation(ctx, installationID, installationToken) installation, err := s.authenticateInstallation(ctx, installationID, installationToken)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -451,28 +480,111 @@ func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationI
if err != nil { if err != nil {
return nil, err return nil, err
} }
snapshotMetrics.Active = true
snapshotMetrics.PlatformCount = len(platforms)
if len(platforms) == 0 {
_ = s.touchInstallation(ctx, installation.ID)
return &MonitoringHeartbeatResponse{
InstallationID: installation.ID,
BusinessDate: businessDate,
UpdatedPlatformCount: 0,
ReconciledSkippedTaskCount: 0,
}, nil
}
platformIDs := monitoringHeartbeatPlatformIDs(platforms)
previousStates, err := s.loadPlatformAccessSnapshotStates(ctx, installation.TenantID, installation.ID, businessDate, platformIDs)
if err != nil {
return nil, err
}
type heartbeatPlatformAction struct {
platform MonitoringHeartbeatPlatform
decision monitoringHeartbeatSnapshotDecision
}
actions := make([]heartbeatPlatformAction, 0, len(platforms))
inaccessiblePlatformIDs := make([]string, 0, len(platforms))
needsTransaction := false
for _, platform := range platforms {
nextState := monitoringPlatformAccessSnapshotStateFromHeartbeat(platform)
decision := decideMonitoringHeartbeatSnapshot(previousStates[platform.AIPlatformID], nextState, now, isPrimaryInstallation)
if decision.WriteSnapshot {
needsTransaction = true
} else {
snapshotMetrics.SnapshotSkippedCount++
}
if decision.PendingReconcileCandidate {
inaccessiblePlatformIDs = append(inaccessiblePlatformIDs, platform.AIPlatformID)
}
actions = append(actions, heartbeatPlatformAction{
platform: platform,
decision: decision,
})
}
if isPrimaryInstallation && len(inaccessiblePlatformIDs) > 0 {
pendingPlatforms, pendingErr := s.loadPendingMonitoringTaskPlatformIDs(ctx, installation.TenantID, businessDate, inaccessiblePlatformIDs)
if pendingErr != nil {
return nil, pendingErr
}
for idx := range actions {
if !actions[idx].decision.PendingReconcileCandidate {
continue
}
if _, ok := pendingPlatforms[actions[idx].platform.AIPlatformID]; ok {
actions[idx].decision.ReconcilePending = true
needsTransaction = true
}
}
}
if !needsTransaction {
_ = s.touchInstallation(ctx, installation.ID)
return &MonitoringHeartbeatResponse{
InstallationID: installation.ID,
BusinessDate: businessDate,
UpdatedPlatformCount: len(platforms),
ReconciledSkippedTaskCount: 0,
}, nil
}
tx, err := s.monitoringPool.Begin(ctx) tx, err := s.monitoringPool.Begin(ctx)
if err != nil { if err != nil {
return nil, response.ErrInternal(50041, "begin_failed", "failed to start monitoring heartbeat") return nil, response.ErrInternal(50041, "begin_failed", "failed to start monitoring heartbeat")
} }
snapshotMetrics.TransactionOpened = true
defer tx.Rollback(ctx) defer tx.Rollback(ctx)
var reconciledSkippedTaskCount int64 var reconciledSkippedTaskCount int64
var snapshotInsertedCount int64
var snapshotUpdatedCount int64
var snapshotRefreshedCount int64
var projectionRebuildCount int64
affectedBrandIDs := make(map[int64]struct{}) affectedBrandIDs := make(map[int64]struct{})
for _, platform := range platforms { for _, action := range actions {
accessChanged, err := s.upsertPlatformAccessSnapshot(ctx, tx, installation, businessDate, platform) if action.decision.WriteSnapshot {
if err != nil { if err := s.upsertPlatformAccessSnapshot(ctx, tx, installation, businessDate, action.platform); err != nil {
return nil, err return nil, err
} }
switch action.decision.SnapshotOperation {
case "insert":
snapshotInsertedCount++
case "update":
snapshotUpdatedCount++
case "refresh":
snapshotRefreshedCount++
}
}
if !isPrimaryInstallation { if !isPrimaryInstallation {
continue continue
} }
needsProjectionRefresh := accessChanged needsProjectionRefresh := action.decision.StateChanged
if monitoringStringValue(platform.AccessStatus) != "accessible" { if action.decision.ReconcilePending {
count, reconcileErr := s.reconcilePendingTasksForPlatform(ctx, tx, installation.TenantID, businessDate, platform.AIPlatformID) count, reconcileErr := s.reconcilePendingTasksForPlatform(ctx, tx, installation.TenantID, businessDate, action.platform.AIPlatformID)
if reconcileErr != nil { if reconcileErr != nil {
return nil, reconcileErr return nil, reconcileErr
} }
@@ -486,7 +598,7 @@ func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationI
continue continue
} }
brandIDs, loadErr := s.loadMonitoringAffectedBrandIDs(ctx, tx, installation.TenantID, businessDay, platform.AIPlatformID) brandIDs, loadErr := s.loadMonitoringAffectedBrandIDs(ctx, tx, installation.TenantID, businessDay, action.platform.AIPlatformID)
if loadErr != nil { if loadErr != nil {
return nil, loadErr return nil, loadErr
} }
@@ -500,6 +612,7 @@ func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationI
if err := s.rebuildBrandDailyProjection(ctx, tx, installation.TenantID, brandID, businessDay); err != nil { if err := s.rebuildBrandDailyProjection(ctx, tx, installation.TenantID, brandID, businessDay); err != nil {
return nil, err return nil, err
} }
projectionRebuildCount++
} }
} }
@@ -507,6 +620,12 @@ func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationI
return nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring heartbeat") return nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring heartbeat")
} }
snapshotMetrics.SnapshotInsertedCount = snapshotInsertedCount
snapshotMetrics.SnapshotUpdatedCount = snapshotUpdatedCount
snapshotMetrics.SnapshotRefreshedCount = snapshotRefreshedCount
snapshotMetrics.ReconciledSkippedTaskCount = reconciledSkippedTaskCount
snapshotMetrics.ProjectionRebuildCount = projectionRebuildCount
_ = s.touchInstallation(ctx, installation.ID) _ = s.touchInstallation(ctx, installation.ID)
if isPrimaryInstallation && len(affectedBrandIDs) > 0 { if isPrimaryInstallation && len(affectedBrandIDs) > 0 {
s.dispatchMonitoringProjectionRebuilds(installation.TenantID, businessDay, affectedBrandIDs, "heartbeat") s.dispatchMonitoringProjectionRebuilds(installation.TenantID, businessDay, affectedBrandIDs, "heartbeat")
@@ -963,80 +1082,224 @@ func (s *MonitoringCallbackService) authenticateInstallation(ctx context.Context
} }
func (s *MonitoringCallbackService) resolveHeartbeatPrimaryInstallation(ctx context.Context, tenantID int64, installation *monitoringInstallationIdentity) (bool, error) { func (s *MonitoringCallbackService) resolveHeartbeatPrimaryInstallation(ctx context.Context, tenantID int64, installation *monitoringInstallationIdentity) (bool, error) {
var preferred sql.NullString if installation == nil || strings.TrimSpace(installation.ID) == "" {
err := s.businessPool.QueryRow(ctx, `
SELECT primary_client_id::text
FROM tenant_monitoring_quotas
WHERE tenant_id = $1
`, tenantID).Scan(&preferred)
if err != nil && err != pgx.ErrNoRows {
return false, response.ErrInternal(50041, "query_failed", "failed to inspect primary monitoring desktop client")
}
if !preferred.Valid || preferred.String == installation.ID {
if !preferred.Valid {
if err := s.persistHeartbeatPrimaryInstallationID(ctx, tenantID, installation); err != nil {
return false, err
}
}
return true, nil
}
online, err := s.isMonitoringInstallationRecentlyOnline(ctx, tenantID, installation.WorkspaceID, preferred.String)
if err != nil {
return false, err
}
if online {
return false, nil return false, nil
} }
isPrimary, err := s.claimHeartbeatPrimaryLease(ctx, tenantID, installation.WorkspaceID, installation.ID)
if err := s.persistHeartbeatPrimaryInstallationID(ctx, tenantID, installation); err != nil { if err != nil {
return false, err return false, err
} }
return true, nil return isPrimary, nil
} }
func (s *MonitoringCallbackService) isMonitoringInstallationRecentlyOnline(ctx context.Context, tenantID, workspaceID int64, installationID string) (bool, error) { func (s *MonitoringCallbackService) claimHeartbeatPrimaryLease(ctx context.Context, tenantID, workspaceID int64, installationID string) (bool, error) {
var lastSeenAt sql.NullTime startedAt := time.Now()
err := s.businessPool.QueryRow(ctx, ` installationID = strings.TrimSpace(installationID)
SELECT last_seen_at leaseTTL := formatPgInterval(desktopClientPresenceTTL)
FROM desktop_clients
WHERE id = $1::uuid state, err := s.loadHeartbeatPrimaryLeaseState(ctx, tenantID, workspaceID, installationID)
AND tenant_id = $2
AND workspace_id = $3
AND revoked_at IS NULL
`, strings.TrimSpace(installationID), tenantID, workspaceID).Scan(&lastSeenAt)
if err != nil { if err != nil {
recordHeartbeatPrimaryLeaseMetric("error", time.Since(startedAt))
return false, err
}
if !state.Found {
inserted, insertErr := s.tryInsertHeartbeatPrimaryLease(ctx, tenantID, workspaceID, installationID, leaseTTL)
if insertErr != nil {
recordHeartbeatPrimaryLeaseMetric("error", time.Since(startedAt))
return false, insertErr
}
if inserted {
recordHeartbeatPrimaryLeaseMetric("inserted", time.Since(startedAt))
recordHeartbeatPrimaryLeaseWriteMetric("insert")
return true, nil
}
recordHeartbeatPrimaryLeaseMetric("contention", time.Since(startedAt))
return s.resolveHeartbeatPrimaryAfterContention(ctx, tenantID, workspaceID, installationID)
}
decision := decideHeartbeatPrimaryLease(state)
switch decision.Action {
case heartbeatPrimaryLeaseActionPrimaryHit:
recordHeartbeatPrimaryLeaseMetric("primary_hit", time.Since(startedAt))
return true, nil
case heartbeatPrimaryLeaseActionNonPrimaryLive:
recordHeartbeatPrimaryLeaseMetric("non_primary_live", time.Since(startedAt))
return false, nil
case heartbeatPrimaryLeaseActionRenew:
updated, updateErr := s.renewHeartbeatPrimaryLease(ctx, tenantID, workspaceID, installationID, leaseTTL)
if updateErr != nil {
recordHeartbeatPrimaryLeaseMetric("error", time.Since(startedAt))
return false, updateErr
}
if updated {
recordHeartbeatPrimaryLeaseMetric("renewed", time.Since(startedAt))
recordHeartbeatPrimaryLeaseWriteMetric("renew")
return true, nil
}
case heartbeatPrimaryLeaseActionClaimExpired, heartbeatPrimaryLeaseActionClaimUnavailable:
updated, updateErr := s.claimExpiredHeartbeatPrimaryLease(ctx, tenantID, workspaceID, installationID, state.PrimaryInstallationID, leaseTTL, decision.Action)
if updateErr != nil {
recordHeartbeatPrimaryLeaseMetric("error", time.Since(startedAt))
return false, updateErr
}
if updated {
if decision.Action == heartbeatPrimaryLeaseActionClaimExpired {
recordHeartbeatPrimaryLeaseMetric("claimed_expired", time.Since(startedAt))
} else {
recordHeartbeatPrimaryLeaseMetric("claimed_unavailable", time.Since(startedAt))
}
recordHeartbeatPrimaryLeaseWriteMetric("claim")
return true, nil
}
}
recordHeartbeatPrimaryLeaseMetric("contention", time.Since(startedAt))
return s.resolveHeartbeatPrimaryAfterContention(ctx, tenantID, workspaceID, installationID)
}
func (s *MonitoringCallbackService) tryInsertHeartbeatPrimaryLease(ctx context.Context, tenantID, workspaceID int64, installationID, leaseTTL string) (bool, error) {
tag, err := s.businessPool.Exec(ctx, `
INSERT INTO desktop_client_primary_leases (
tenant_id,
workspace_id,
primary_client_id,
lease_expires_at
)
VALUES ($1, $2, $3::uuid, NOW() + $4::interval)
ON CONFLICT (tenant_id, workspace_id) DO NOTHING
`, tenantID, workspaceID, installationID, leaseTTL)
if err != nil {
return false, response.ErrInternal(50041, "primary_lease_insert_failed", "failed to initialize monitoring desktop primary lease")
}
return tag.RowsAffected() > 0, nil
}
type heartbeatPrimaryLeaseState struct {
Found bool
PrimaryInstallationID string
CurrentIsPrimary bool
LeaseLive bool
RenewDue bool
PrimaryAvailable bool
}
func (s *MonitoringCallbackService) loadHeartbeatPrimaryLeaseState(ctx context.Context, tenantID, workspaceID int64, installationID string) (heartbeatPrimaryLeaseState, error) {
var state heartbeatPrimaryLeaseState
renewAfter := formatPgInterval(monitoringHeartbeatPrimaryRenewAfter)
if err := s.businessPool.QueryRow(ctx, `
SELECT
l.primary_client_id::text,
l.primary_client_id = $3::uuid AS current_is_primary,
l.lease_expires_at > NOW() AS lease_live,
l.updated_at <= NOW() - $4::interval AS renew_due,
dc.id IS NOT NULL AND dc.revoked_at IS NULL AS primary_available
FROM desktop_client_primary_leases l
LEFT JOIN desktop_clients dc ON dc.id = l.primary_client_id
WHERE l.tenant_id = $1
AND l.workspace_id = $2
`, tenantID, workspaceID, installationID, renewAfter).Scan(
&state.PrimaryInstallationID,
&state.CurrentIsPrimary,
&state.LeaseLive,
&state.RenewDue,
&state.PrimaryAvailable,
); err != nil {
if err == pgx.ErrNoRows { if err == pgx.ErrNoRows {
return false, nil return state, nil
} }
return false, response.ErrInternal(50041, "query_failed", "failed to inspect monitoring desktop client heartbeat") return state, response.ErrInternal(50041, "query_failed", "failed to inspect monitoring desktop primary lease")
}
state.Found = true
return state, nil
} }
if !lastSeenAt.Valid { type heartbeatPrimaryLeaseAction string
return false, nil
} const (
return time.Since(lastSeenAt.Time) <= monitoringOnlineDuration, nil heartbeatPrimaryLeaseActionPrimaryHit heartbeatPrimaryLeaseAction = "primary_hit"
heartbeatPrimaryLeaseActionNonPrimaryLive heartbeatPrimaryLeaseAction = "non_primary_live"
heartbeatPrimaryLeaseActionRenew heartbeatPrimaryLeaseAction = "renew"
heartbeatPrimaryLeaseActionClaimExpired heartbeatPrimaryLeaseAction = "claim_expired"
heartbeatPrimaryLeaseActionClaimUnavailable heartbeatPrimaryLeaseAction = "claim_unavailable"
)
type heartbeatPrimaryLeaseDecision struct {
Action heartbeatPrimaryLeaseAction
} }
func (s *MonitoringCallbackService) persistHeartbeatPrimaryInstallationID(ctx context.Context, tenantID int64, installation *monitoringInstallationIdentity) error { func decideHeartbeatPrimaryLease(state heartbeatPrimaryLeaseState) heartbeatPrimaryLeaseDecision {
if _, err := s.businessPool.Exec(ctx, ` if !state.Found {
INSERT INTO tenant_monitoring_quotas ( return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionNonPrimaryLive}
tenant_id, workspace_id, primary_client_id }
) if state.CurrentIsPrimary {
VALUES ( if !state.LeaseLive || !state.PrimaryAvailable || state.RenewDue {
$1, return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionRenew}
$2, }
$3::uuid return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionPrimaryHit}
) }
ON CONFLICT (tenant_id) DO UPDATE if !state.LeaseLive {
SET workspace_id = EXCLUDED.workspace_id, return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionClaimExpired}
primary_client_id = EXCLUDED.primary_client_id, }
if !state.PrimaryAvailable {
return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionClaimUnavailable}
}
return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionNonPrimaryLive}
}
func (s *MonitoringCallbackService) renewHeartbeatPrimaryLease(ctx context.Context, tenantID, workspaceID int64, installationID, leaseTTL string) (bool, error) {
tag, err := s.businessPool.Exec(ctx, `
UPDATE desktop_client_primary_leases
SET lease_expires_at = NOW() + $4::interval,
updated_at = NOW() updated_at = NOW()
`, tenantID, installation.WorkspaceID, installation.ID); err != nil { WHERE tenant_id = $1
return response.ErrInternal(50041, "update_failed", "failed to persist primary monitoring desktop client") AND workspace_id = $2
AND primary_client_id = $3::uuid
`, tenantID, workspaceID, installationID, leaseTTL)
if err != nil {
return false, response.ErrInternal(50041, "primary_lease_update_failed", "failed to renew monitoring desktop primary lease")
} }
return nil return tag.RowsAffected() > 0, nil
}
func (s *MonitoringCallbackService) claimExpiredHeartbeatPrimaryLease(
ctx context.Context,
tenantID, workspaceID int64,
installationID string,
previousPrimaryInstallationID string,
leaseTTL string,
action heartbeatPrimaryLeaseAction,
) (bool, error) {
claimPredicate := "lease_expires_at <= NOW()"
if action == heartbeatPrimaryLeaseActionClaimUnavailable {
claimPredicate = `NOT EXISTS (
SELECT 1
FROM desktop_clients dc
WHERE dc.id = desktop_client_primary_leases.primary_client_id
AND dc.revoked_at IS NULL
)`
}
query := `
UPDATE desktop_client_primary_leases
SET primary_client_id = $3::uuid,
lease_expires_at = NOW() + $5::interval,
updated_at = NOW()
WHERE tenant_id = $1
AND workspace_id = $2
AND primary_client_id = $4::uuid
AND ` + claimPredicate
tag, err := s.businessPool.Exec(ctx, query, tenantID, workspaceID, installationID, previousPrimaryInstallationID, leaseTTL)
if err != nil {
return false, response.ErrInternal(50041, "primary_lease_update_failed", "failed to claim monitoring desktop primary lease")
}
return tag.RowsAffected() > 0, nil
}
func (s *MonitoringCallbackService) resolveHeartbeatPrimaryAfterContention(ctx context.Context, tenantID, workspaceID int64, installationID string) (bool, error) {
state, err := s.loadHeartbeatPrimaryLeaseState(ctx, tenantID, workspaceID, installationID)
if err != nil {
return false, err
}
return state.CurrentIsPrimary && state.LeaseLive && state.PrimaryAvailable, nil
} }
func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantID, taskID int64) (*monitoringCollectTask, error) { func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantID, taskID int64) (*monitoringCollectTask, error) {
@@ -1464,20 +1727,7 @@ func (s *MonitoringCallbackService) markTaskResultReceived(ctx context.Context,
return nil return nil
} }
func (s *MonitoringCallbackService) upsertPlatformAccessSnapshot(ctx context.Context, tx pgx.Tx, installation *monitoringInstallationIdentity, businessDate string, platform MonitoringHeartbeatPlatform) (bool, error) { func (s *MonitoringCallbackService) upsertPlatformAccessSnapshot(ctx context.Context, tx pgx.Tx, installation *monitoringInstallationIdentity, businessDate string, platform MonitoringHeartbeatPlatform) error {
previous, err := s.loadPlatformAccessSnapshotState(ctx, tx, installation.TenantID, installation.ID, businessDate, platform.AIPlatformID)
if err != nil {
return false, err
}
nextState := monitoringPlatformAccessSnapshotState{
AccessStatus: monitoringStringValue(platform.AccessStatus),
Connected: derefBool(platform.Connected),
Accessible: derefBool(platform.Accessible),
ReasonCode: optionalString(monitoringStringValue(platform.ReasonCode)),
ReasonMessage: optionalString(monitoringStringValue(platform.ReasonMessage)),
}
if _, err := tx.Exec(ctx, ` if _, err := tx.Exec(ctx, `
INSERT INTO monitoring_platform_access_snapshots ( INSERT INTO monitoring_platform_access_snapshots (
tenant_id, client_id, ai_platform_id, business_date, access_status, tenant_id, client_id, ai_platform_id, business_date, access_status,
@@ -1498,44 +1748,101 @@ func (s *MonitoringCallbackService) upsertPlatformAccessSnapshot(ctx context.Con
installation.ID, installation.ID,
platform.AIPlatformID, platform.AIPlatformID,
businessDate, businessDate,
platform.AccessStatus, monitoringStringValue(platform.AccessStatus),
derefBool(platform.Connected), derefBool(platform.Connected),
derefBool(platform.Accessible), derefBool(platform.Accessible),
resolveHeartbeatDetectedAt(platform.DetectedAt), resolveHeartbeatDetectedAt(platform.DetectedAt),
nullableString(platform.ReasonCode), nullableString(platform.ReasonCode),
nullableString(platform.ReasonMessage), nullableString(platform.ReasonMessage),
); err != nil { ); err != nil {
return false, response.ErrInternal(50041, "access_snapshot_upsert_failed", "failed to persist monitoring platform access snapshot") return response.ErrInternal(50041, "access_snapshot_upsert_failed", "failed to persist monitoring platform access snapshot")
} }
return hasPlatformAccessSnapshotChanged(previous, nextState), nil return nil
} }
func (s *MonitoringCallbackService) loadPlatformAccessSnapshotState(ctx context.Context, tx pgx.Tx, tenantID int64, installationID string, businessDate, aiPlatformID string) (*monitoringPlatformAccessSnapshotState, error) { func (s *MonitoringCallbackService) loadPlatformAccessSnapshotStates(ctx context.Context, tenantID int64, installationID string, businessDate string, aiPlatformIDs []string) (map[string]*monitoringPlatformAccessSnapshotState, error) {
item := &monitoringPlatformAccessSnapshotState{} states := make(map[string]*monitoringPlatformAccessSnapshotState, len(aiPlatformIDs))
var reasonCode sql.NullString if len(aiPlatformIDs) == 0 {
var reasonMessage sql.NullString return states, nil
if err := tx.QueryRow(ctx, ` }
SELECT access_status, connected, accessible, reason_code, reason_message
rows, err := s.monitoringPool.Query(ctx, `
SELECT ai_platform_id, access_status, connected, accessible, reason_code, reason_message, updated_at
FROM monitoring_platform_access_snapshots FROM monitoring_platform_access_snapshots
WHERE tenant_id = $1 WHERE tenant_id = $1
AND client_id = $2::uuid AND client_id = $2::uuid
AND ai_platform_id = $3 AND business_date = $3::date
AND business_date = $4::date AND ai_platform_id = ANY($4::text[])
`, tenantID, installationID, aiPlatformID, businessDate).Scan( `, tenantID, installationID, businessDate, aiPlatformIDs)
if err != nil {
return nil, response.ErrInternal(50041, "access_snapshot_lookup_failed", "failed to inspect monitoring platform access snapshots")
}
defer rows.Close()
for rows.Next() {
item := &monitoringPlatformAccessSnapshotState{}
var platformID string
var reasonCode sql.NullString
var reasonMessage sql.NullString
if scanErr := rows.Scan(
&platformID,
&item.AccessStatus, &item.AccessStatus,
&item.Connected, &item.Connected,
&item.Accessible, &item.Accessible,
&reasonCode, &reasonCode,
&reasonMessage, &reasonMessage,
); err != nil { &item.UpdatedAt,
if err == pgx.ErrNoRows { ); scanErr != nil {
return nil, nil return nil, response.ErrInternal(50041, "access_snapshot_scan_failed", "failed to parse monitoring platform access snapshots")
}
return nil, response.ErrInternal(50041, "access_snapshot_lookup_failed", "failed to inspect monitoring platform access snapshot")
} }
if reasonCode.Valid {
item.ReasonCode = optionalString(reasonCode.String) item.ReasonCode = optionalString(reasonCode.String)
}
if reasonMessage.Valid {
item.ReasonMessage = optionalString(reasonMessage.String) item.ReasonMessage = optionalString(reasonMessage.String)
return item, nil }
states[platformID] = item
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "access_snapshot_scan_failed", "failed to iterate monitoring platform access snapshots")
}
return states, nil
}
func monitoringPlatformAccessSnapshotStateFromHeartbeat(platform MonitoringHeartbeatPlatform) monitoringPlatformAccessSnapshotState {
return monitoringPlatformAccessSnapshotState{
AccessStatus: monitoringStringValue(platform.AccessStatus),
Connected: derefBool(platform.Connected),
Accessible: derefBool(platform.Accessible),
ReasonCode: optionalString(monitoringStringValue(platform.ReasonCode)),
ReasonMessage: optionalString(monitoringStringValue(platform.ReasonMessage)),
}
}
func decideMonitoringHeartbeatSnapshot(previous *monitoringPlatformAccessSnapshotState, next monitoringPlatformAccessSnapshotState, now time.Time, isPrimary bool) monitoringHeartbeatSnapshotDecision {
stateChanged := hasPlatformAccessSnapshotChanged(previous, next)
refreshDue := false
if previous != nil && !stateChanged {
refreshDue = previous.UpdatedAt.IsZero() || !previous.UpdatedAt.After(now.Add(-monitoringHeartbeatAccessSnapshotRefreshAfter))
}
operation := "skip"
if previous == nil {
operation = "insert"
} else if stateChanged {
operation = "update"
} else if refreshDue {
operation = "refresh"
}
return monitoringHeartbeatSnapshotDecision{
WriteSnapshot: stateChanged || refreshDue,
StateChanged: stateChanged,
RefreshDue: refreshDue,
PendingReconcileCandidate: isPrimary && next.AccessStatus != "accessible",
SnapshotOperation: operation,
}
} }
func hasPlatformAccessSnapshotChanged(previous *monitoringPlatformAccessSnapshotState, next monitoringPlatformAccessSnapshotState) bool { func hasPlatformAccessSnapshotChanged(previous *monitoringPlatformAccessSnapshotState, next monitoringPlatformAccessSnapshotState) bool {
@@ -1558,6 +1865,41 @@ func hasPlatformAccessSnapshotChanged(previous *monitoringPlatformAccessSnapshot
return false return false
} }
func (s *MonitoringCallbackService) loadPendingMonitoringTaskPlatformIDs(ctx context.Context, tenantID int64, businessDate string, aiPlatformIDs []string) (map[string]struct{}, error) {
result := make(map[string]struct{}, len(aiPlatformIDs))
if len(aiPlatformIDs) == 0 {
return result, nil
}
rows, err := s.monitoringPool.Query(ctx, `
SELECT ai_platform_id
FROM monitoring_collect_tasks
WHERE tenant_id = $1
AND ai_platform_id = ANY($2::text[])
AND collector_type = $3
AND business_date = $4::date
AND status = 'pending'
GROUP BY ai_platform_id
`, tenantID, aiPlatformIDs, monitoringCollectorType, businessDate)
if err != nil {
return nil, response.ErrInternal(50041, "pending_task_platform_lookup_failed", "failed to inspect pending monitoring task platforms")
}
defer rows.Close()
for rows.Next() {
var platformID string
if scanErr := rows.Scan(&platformID); scanErr != nil {
return nil, response.ErrInternal(50041, "pending_task_platform_scan_failed", "failed to parse pending monitoring task platforms")
}
result[platformID] = struct{}{}
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "pending_task_platform_scan_failed", "failed to iterate pending monitoring task platforms")
}
return result, nil
}
func (s *MonitoringCallbackService) reconcilePendingTasksForPlatform(ctx context.Context, tx pgx.Tx, tenantID int64, businessDate, aiPlatformID string) (int64, error) { func (s *MonitoringCallbackService) reconcilePendingTasksForPlatform(ctx context.Context, tx pgx.Tx, tenantID int64, businessDate, aiPlatformID string) (int64, error) {
tag, err := tx.Exec(ctx, ` tag, err := tx.Exec(ctx, `
UPDATE monitoring_collect_tasks UPDATE monitoring_collect_tasks
@@ -2530,6 +2872,20 @@ func normalizeMonitoringLeasePlatformIDs(values []string) []string {
return result return result
} }
func monitoringHeartbeatPlatformIDs(platforms []MonitoringHeartbeatPlatform) []string {
if len(platforms) == 0 {
return nil
}
result := make([]string, 0, len(platforms))
for _, platform := range platforms {
platformID := strings.TrimSpace(platform.AIPlatformID)
if platformID != "" {
result = append(result, platformID)
}
}
return result
}
func normalizeHeartbeatPlatforms(items []MonitoringHeartbeatPlatform, fallbackDetectedAt time.Time) ([]MonitoringHeartbeatPlatform, error) { func normalizeHeartbeatPlatforms(items []MonitoringHeartbeatPlatform, fallbackDetectedAt time.Time) ([]MonitoringHeartbeatPlatform, error) {
if len(items) == 0 { if len(items) == 0 {
return nil, nil return nil, nil
@@ -2,9 +2,287 @@ package app
import ( import (
"encoding/json" "encoding/json"
"errors"
"strings"
"testing" "testing"
"time"
) )
func TestDecideHeartbeatPrimaryLease(t *testing.T) {
tests := []struct {
name string
state heartbeatPrimaryLeaseState
want heartbeatPrimaryLeaseAction
}{
{
name: "current primary fast path",
state: heartbeatPrimaryLeaseState{
Found: true,
CurrentIsPrimary: true,
LeaseLive: true,
PrimaryAvailable: true,
},
want: heartbeatPrimaryLeaseActionPrimaryHit,
},
{
name: "current primary renews when due",
state: heartbeatPrimaryLeaseState{
Found: true,
CurrentIsPrimary: true,
LeaseLive: true,
RenewDue: true,
PrimaryAvailable: true,
},
want: heartbeatPrimaryLeaseActionRenew,
},
{
name: "non primary does not steal live lease",
state: heartbeatPrimaryLeaseState{
Found: true,
CurrentIsPrimary: false,
LeaseLive: true,
PrimaryAvailable: true,
},
want: heartbeatPrimaryLeaseActionNonPrimaryLive,
},
{
name: "expired lease can be claimed",
state: heartbeatPrimaryLeaseState{
Found: true,
CurrentIsPrimary: false,
LeaseLive: false,
PrimaryAvailable: true,
},
want: heartbeatPrimaryLeaseActionClaimExpired,
},
{
name: "revoked primary can be replaced",
state: heartbeatPrimaryLeaseState{
Found: true,
CurrentIsPrimary: false,
LeaseLive: true,
PrimaryAvailable: false,
},
want: heartbeatPrimaryLeaseActionClaimUnavailable,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := decideHeartbeatPrimaryLease(tt.state)
if got.Action != tt.want {
t.Fatalf("decideHeartbeatPrimaryLease() = %v, want %v", got.Action, tt.want)
}
})
}
}
func TestMonitoringHeartbeatPrimaryLeaseMetrics(t *testing.T) {
resetMonitoringHeartbeatPrimaryLeaseMetricsForTest()
t.Cleanup(resetMonitoringHeartbeatPrimaryLeaseMetricsForTest)
recordHeartbeatPrimaryLeaseMetric("primary_hit", 20*time.Millisecond)
recordHeartbeatPrimaryLeaseMetric("contention", 10*time.Millisecond)
recordHeartbeatPrimaryLeaseWriteMetric("renew")
snapshot := MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue()
if snapshot.PrimaryHitTotal != 1 {
t.Fatalf("PrimaryHitTotal = %d, want 1", snapshot.PrimaryHitTotal)
}
if snapshot.ContentionTotal != 1 {
t.Fatalf("ContentionTotal = %d, want 1", snapshot.ContentionTotal)
}
if snapshot.DBRenewTotal != 1 {
t.Fatalf("DBRenewTotal = %d, want 1", snapshot.DBRenewTotal)
}
if snapshot.DurationCount != 2 {
t.Fatalf("DurationCount = %d, want 2", snapshot.DurationCount)
}
prometheus := MonitoringHeartbeatPrimaryLeaseMetricsPrometheus()
if !strings.Contains(prometheus, `monitoring_heartbeat_primary_lease_resolutions_total{outcome="primary_hit"} 1`) {
t.Fatalf("prometheus output missing primary_hit metric: %s", prometheus)
}
if !strings.Contains(prometheus, `monitoring_heartbeat_primary_lease_db_writes_total{operation="renew"} 1`) {
t.Fatalf("prometheus output missing renew write metric: %s", prometheus)
}
}
func TestDecideMonitoringHeartbeatSnapshot(t *testing.T) {
now := time.Date(2026, 4, 24, 3, 0, 0, 0, time.UTC)
base := monitoringPlatformAccessSnapshotState{
AccessStatus: "accessible",
Connected: true,
Accessible: true,
UpdatedAt: now.Add(-time.Minute),
}
tests := []struct {
name string
previous *monitoringPlatformAccessSnapshotState
next monitoringPlatformAccessSnapshotState
primary bool
want monitoringHeartbeatSnapshotDecision
}{
{
name: "missing snapshot inserts",
next: base,
primary: true,
want: monitoringHeartbeatSnapshotDecision{
WriteSnapshot: true,
StateChanged: true,
SnapshotOperation: "insert",
},
},
{
name: "fresh unchanged accessible snapshot skips write",
previous: &base,
next: base,
primary: true,
want: monitoringHeartbeatSnapshotDecision{
SnapshotOperation: "skip",
},
},
{
name: "stale unchanged snapshot refreshes",
previous: &monitoringPlatformAccessSnapshotState{
AccessStatus: "accessible",
Connected: true,
Accessible: true,
UpdatedAt: now.Add(-monitoringHeartbeatAccessSnapshotRefreshAfter - time.Second),
},
next: base,
primary: true,
want: monitoringHeartbeatSnapshotDecision{
WriteSnapshot: true,
RefreshDue: true,
SnapshotOperation: "refresh",
},
},
{
name: "changed snapshot updates",
previous: &base,
next: monitoringPlatformAccessSnapshotState{
AccessStatus: "unavailable",
Connected: true,
Accessible: false,
},
primary: true,
want: monitoringHeartbeatSnapshotDecision{
WriteSnapshot: true,
StateChanged: true,
PendingReconcileCandidate: true,
SnapshotOperation: "update",
},
},
{
name: "fresh unchanged primary inaccessible snapshot checks pending tasks without writing snapshot",
previous: &monitoringPlatformAccessSnapshotState{
AccessStatus: "not_logged_in",
Connected: false,
Accessible: false,
UpdatedAt: now.Add(-time.Minute),
},
next: monitoringPlatformAccessSnapshotState{
AccessStatus: "not_logged_in",
Connected: false,
Accessible: false,
},
primary: true,
want: monitoringHeartbeatSnapshotDecision{
PendingReconcileCandidate: true,
SnapshotOperation: "skip",
},
},
{
name: "non-primary inaccessible snapshot does not reconcile",
previous: &monitoringPlatformAccessSnapshotState{
AccessStatus: "not_logged_in",
Connected: false,
Accessible: false,
UpdatedAt: now.Add(-time.Minute),
},
next: monitoringPlatformAccessSnapshotState{
AccessStatus: "not_logged_in",
Connected: false,
Accessible: false,
},
primary: false,
want: monitoringHeartbeatSnapshotDecision{
SnapshotOperation: "skip",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := decideMonitoringHeartbeatSnapshot(tt.previous, tt.next, now, tt.primary)
if got.WriteSnapshot != tt.want.WriteSnapshot ||
got.StateChanged != tt.want.StateChanged ||
got.RefreshDue != tt.want.RefreshDue ||
got.PendingReconcileCandidate != tt.want.PendingReconcileCandidate ||
got.ReconcilePending != tt.want.ReconcilePending ||
got.SnapshotOperation != tt.want.SnapshotOperation {
t.Fatalf("decideMonitoringHeartbeatSnapshot() = %+v, want %+v", got, tt.want)
}
})
}
}
func TestMonitoringHeartbeatSnapshotMetrics(t *testing.T) {
resetMonitoringHeartbeatSnapshotMetricsForTest()
t.Cleanup(resetMonitoringHeartbeatSnapshotMetricsForTest)
recordMonitoringHeartbeatSnapshotMetrics(monitoringHeartbeatSnapshotMetricsEvent{
PlatformCount: 3,
TransactionOpened: true,
SnapshotInsertedCount: 1,
SnapshotSkippedCount: 2,
ReconciledSkippedTaskCount: 5,
ProjectionRebuildCount: 2,
Duration: 20 * time.Millisecond,
})
recordMonitoringHeartbeatSnapshotMetrics(monitoringHeartbeatSnapshotMetricsEvent{
PlatformCount: 1,
Duration: 10 * time.Millisecond,
Err: errors.New("snapshot failed"),
})
snapshot := MonitoringHeartbeatSnapshotMetricsSnapshotValue()
if snapshot.PlatformReportedTotal != 4 {
t.Fatalf("PlatformReportedTotal = %d, want 4", snapshot.PlatformReportedTotal)
}
if snapshot.TransactionOpenedTotal != 1 {
t.Fatalf("TransactionOpenedTotal = %d, want 1", snapshot.TransactionOpenedTotal)
}
if snapshot.SnapshotInsertedTotal != 1 {
t.Fatalf("SnapshotInsertedTotal = %d, want 1", snapshot.SnapshotInsertedTotal)
}
if snapshot.SnapshotSkippedTotal != 2 {
t.Fatalf("SnapshotSkippedTotal = %d, want 2", snapshot.SnapshotSkippedTotal)
}
if snapshot.ReconciledSkippedTaskTotal != 5 {
t.Fatalf("ReconciledSkippedTaskTotal = %d, want 5", snapshot.ReconciledSkippedTaskTotal)
}
if snapshot.ProjectionRebuildTotal != 2 {
t.Fatalf("ProjectionRebuildTotal = %d, want 2", snapshot.ProjectionRebuildTotal)
}
if snapshot.ErrorTotal != 1 {
t.Fatalf("ErrorTotal = %d, want 1", snapshot.ErrorTotal)
}
if snapshot.DurationCount != 2 {
t.Fatalf("DurationCount = %d, want 2", snapshot.DurationCount)
}
prometheus := MonitoringHeartbeatSnapshotMetricsPrometheus()
if !strings.Contains(prometheus, `monitoring_heartbeat_snapshot_writes_total{operation="insert"} 1`) {
t.Fatalf("prometheus output missing insert write metric: %s", prometheus)
}
if !strings.Contains(prometheus, "monitoring_heartbeat_snapshot_transactions_total 1") {
t.Fatalf("prometheus output missing transaction metric: %s", prometheus)
}
}
func TestBuildMonitoringRawPayloadKimiExcludesSearchResultsFromCitationInputs(t *testing.T) { func TestBuildMonitoringRawPayloadKimiExcludesSearchResultsFromCitationInputs(t *testing.T) {
citationTitle := "真实引用文章" citationTitle := "真实引用文章"
searchTitle := "搜索网页结果" searchTitle := "搜索网页结果"
@@ -0,0 +1,201 @@
package app
import (
"context"
"errors"
"fmt"
"strings"
"sync/atomic"
"time"
"github.com/geo-platform/tenant-api/internal/shared/response"
)
type MonitoringDailyTaskMetricsSnapshot struct {
RunTotal int64 `json:"run_total"`
RunCompletedTotal int64 `json:"run_completed_total"`
RunFailedTotal int64 `json:"run_failed_total"`
RunPartialFailureTotal int64 `json:"run_partial_failure_total"`
PlanFailureTotal int64 `json:"plan_failure_total"`
BrandFailureTotal int64 `json:"brand_failure_total"`
SkippedPlanTotal int64 `json:"skipped_plan_total"`
PlannedTaskTotal int64 `json:"planned_task_total"`
DueTaskTotal int64 `json:"due_task_total"`
CreatedTaskTotal int64 `json:"created_task_total"`
DesktopTaskTotal int64 `json:"desktop_task_total"`
DeferredTaskTotal int64 `json:"deferred_task_total"`
LastRunAt *time.Time `json:"last_run_at,omitempty"`
LastSuccessAt *time.Time `json:"last_success_at,omitempty"`
LastErrorAt *time.Time `json:"last_error_at,omitempty"`
LastDurationSeconds float64 `json:"last_duration_seconds"`
LastBusinessDate string `json:"last_business_date,omitempty"`
LastTenantCount int64 `json:"last_tenant_count"`
LastBrandCount int64 `json:"last_brand_count"`
LastErrorCode string `json:"last_error_code,omitempty"`
LastErrorMessage string `json:"last_error_message,omitempty"`
}
type monitoringDailyTaskMetricsRecorder struct {
runTotal atomic.Int64
runCompletedTotal atomic.Int64
runFailedTotal atomic.Int64
runPartialFailureTotal atomic.Int64
planFailureTotal atomic.Int64
brandFailureTotal atomic.Int64
skippedPlanTotal atomic.Int64
plannedTaskTotal atomic.Int64
dueTaskTotal atomic.Int64
createdTaskTotal atomic.Int64
desktopTaskTotal atomic.Int64
deferredTaskTotal atomic.Int64
lastRunAtUnixNano atomic.Int64
lastSuccessAtUnixNano atomic.Int64
lastErrorAtUnixNano atomic.Int64
lastDurationNanos atomic.Int64
lastTenantCount atomic.Int64
lastBrandCount atomic.Int64
lastBusinessDate atomic.Value
lastErrorCode atomic.Value
lastErrorMessage atomic.Value
}
var monitoringDailyTaskMetrics = newMonitoringDailyTaskMetricsRecorder()
func newMonitoringDailyTaskMetricsRecorder() *monitoringDailyTaskMetricsRecorder {
return &monitoringDailyTaskMetricsRecorder{}
}
func recordMonitoringDailyTaskMetrics(summary *MonitoringDailyTaskGenerationSummary, duration time.Duration, err error) {
monitoringDailyTaskMetrics.record(summary, duration, err)
}
func MonitoringDailyTaskMetricsSnapshotValue() MonitoringDailyTaskMetricsSnapshot {
return monitoringDailyTaskMetrics.snapshot()
}
func resetMonitoringDailyTaskMetricsForTest() {
monitoringDailyTaskMetrics = newMonitoringDailyTaskMetricsRecorder()
}
func (r *monitoringDailyTaskMetricsRecorder) record(summary *MonitoringDailyTaskGenerationSummary, duration time.Duration, err error) {
if r == nil {
return
}
now := time.Now().UTC()
if duration < 0 {
duration = 0
}
r.runTotal.Add(1)
r.lastRunAtUnixNano.Store(now.UnixNano())
r.lastDurationNanos.Store(duration.Nanoseconds())
if summary != nil {
r.lastBusinessDate.Store(summary.BusinessDate)
r.lastTenantCount.Store(int64(summary.TenantCount))
r.lastBrandCount.Store(int64(summary.BrandCount))
r.planFailureTotal.Add(int64(summary.FailedPlanCount))
r.brandFailureTotal.Add(int64(summary.FailedBrandCount))
r.skippedPlanTotal.Add(int64(summary.SkippedPlanCount))
r.plannedTaskTotal.Add(summary.PlannedTaskCount)
r.dueTaskTotal.Add(summary.DueTaskCount)
r.createdTaskTotal.Add(summary.CreatedTaskCount)
r.desktopTaskTotal.Add(summary.DesktopTaskCount)
r.deferredTaskTotal.Add(summary.DeferredTaskCount)
if summary.FailedPlanCount > 0 || summary.FailedBrandCount > 0 {
r.runPartialFailureTotal.Add(1)
}
}
if err != nil {
code, message := monitoringDailyMetricError(err)
r.runFailedTotal.Add(1)
r.lastErrorAtUnixNano.Store(now.UnixNano())
r.lastErrorCode.Store(code)
r.lastErrorMessage.Store(message)
return
}
r.runCompletedTotal.Add(1)
r.lastSuccessAtUnixNano.Store(now.UnixNano())
}
func (r *monitoringDailyTaskMetricsRecorder) snapshot() MonitoringDailyTaskMetricsSnapshot {
if r == nil {
return MonitoringDailyTaskMetricsSnapshot{}
}
return MonitoringDailyTaskMetricsSnapshot{
RunTotal: r.runTotal.Load(),
RunCompletedTotal: r.runCompletedTotal.Load(),
RunFailedTotal: r.runFailedTotal.Load(),
RunPartialFailureTotal: r.runPartialFailureTotal.Load(),
PlanFailureTotal: r.planFailureTotal.Load(),
BrandFailureTotal: r.brandFailureTotal.Load(),
SkippedPlanTotal: r.skippedPlanTotal.Load(),
PlannedTaskTotal: r.plannedTaskTotal.Load(),
DueTaskTotal: r.dueTaskTotal.Load(),
CreatedTaskTotal: r.createdTaskTotal.Load(),
DesktopTaskTotal: r.desktopTaskTotal.Load(),
DeferredTaskTotal: r.deferredTaskTotal.Load(),
LastRunAt: monitoringDailyUnixNanoTime(r.lastRunAtUnixNano.Load()),
LastSuccessAt: monitoringDailyUnixNanoTime(r.lastSuccessAtUnixNano.Load()),
LastErrorAt: monitoringDailyUnixNanoTime(r.lastErrorAtUnixNano.Load()),
LastDurationSeconds: float64(r.lastDurationNanos.Load()) / float64(time.Second),
LastBusinessDate: monitoringDailyLoadString(&r.lastBusinessDate),
LastTenantCount: r.lastTenantCount.Load(),
LastBrandCount: r.lastBrandCount.Load(),
LastErrorCode: monitoringDailyLoadString(&r.lastErrorCode),
LastErrorMessage: monitoringDailyLoadString(&r.lastErrorMessage),
}
}
func monitoringDailyMetricError(err error) (string, string) {
normalized := normalizeMonitoringWorkerError(err)
if normalized == nil {
return "", ""
}
switch {
case errors.Is(normalized, context.Canceled):
return "context_canceled", normalized.Error()
case errors.Is(normalized, context.DeadlineExceeded):
return "context_deadline_exceeded", normalized.Error()
}
var appErr *response.AppError
if errors.As(normalized, &appErr) {
if strings.TrimSpace(appErr.Message) != "" {
return appErr.Message, appErr.Error()
}
return "app_error", appErr.Error()
}
code := strings.TrimLeft(fmt.Sprintf("%T", normalized), "*")
code = strings.ReplaceAll(code, "/", "_")
code = strings.ReplaceAll(code, ".", "_")
if code == "" {
code = "error"
}
return code, normalized.Error()
}
func monitoringDailyUnixNanoTime(value int64) *time.Time {
if value <= 0 {
return nil
}
t := time.Unix(0, value).UTC()
return &t
}
func monitoringDailyLoadString(value *atomic.Value) string {
if value == nil {
return ""
}
raw := value.Load()
result, _ := raw.(string)
return result
}
func monitoringDailyTimeSeconds(value *time.Time) float64 {
if value == nil || value.IsZero() {
return 0
}
return float64(value.UnixNano()) / float64(time.Second)
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,337 @@
package app
import (
"context"
"net/http"
"net/http/httptest"
"runtime"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/geo-platform/tenant-api/internal/shared/config"
)
func TestBuildMonitoringDailyTaskCandidatesMaterializesAfterMidnight(t *testing.T) {
businessDay := time.Date(2026, 4, 24, 0, 0, 0, 0, monitoringBusinessLocation())
questions := []monitoringConfiguredQuestion{
{ID: 101, QuestionHash: []byte("q101")},
{ID: 102, QuestionHash: []byte("q102")},
}
platforms := []monitoringPlatformMetadata{
{ID: "qwen", Name: "Qwen"},
{ID: "doubao", Name: "Doubao"},
{ID: "deepseek", Name: "DeepSeek"},
}
candidates := buildMonitoringDailyTaskCandidates(7, 11, businessDay, questions, platforms, 4)
require.Len(t, candidates, 4)
start := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset).UTC()
end := start.Add(defaultMonitoringDailyMaterializeWindow)
for i, candidate := range candidates {
assert.False(t, candidate.MaterializeAt.Before(start))
assert.False(t, candidate.MaterializeAt.After(end))
assert.Equal(t, candidate.MaterializeAt, candidate.DispatchAfter)
if i > 0 {
assert.False(t, candidate.MaterializeAt.Before(candidates[i-1].MaterializeAt))
}
}
}
func TestSelectDueMonitoringDailyCandidatesSkipsExistingAndHonorsLimit(t *testing.T) {
businessDay := time.Date(2026, 4, 24, 0, 0, 0, 0, monitoringBusinessLocation())
candidates := buildMonitoringDailyTaskCandidates(
7,
11,
businessDay,
[]monitoringConfiguredQuestion{
{ID: 101, QuestionHash: []byte("q101")},
{ID: 102, QuestionHash: []byte("q102")},
},
[]monitoringPlatformMetadata{
{ID: "qwen"},
{ID: "doubao"},
{ID: "deepseek"},
},
6,
)
require.Len(t, candidates, 6)
existing := map[monitoringDailyTaskKey]struct{}{
{
QuestionID: candidates[0].Question.ID,
PlatformID: candidates[0].Platform.ID,
}: {},
}
now := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset + defaultMonitoringDailyMaterializeWindow)
due := selectDueMonitoringDailyCandidates(candidates, existing, now, defaultMonitoringDailyLookahead, 2)
require.Len(t, due, 2)
for _, candidate := range due {
_, exists := existing[monitoringDailyTaskKey{
QuestionID: candidate.Question.ID,
PlatformID: candidate.Platform.ID,
}]
assert.False(t, exists)
}
}
func TestMonitoringDailyRemainingKeepsFullCandidateHorizon(t *testing.T) {
businessDay := time.Date(2026, 4, 24, 0, 0, 0, 0, monitoringBusinessLocation())
hardCap := 6
remaining := monitoringDailyRemainingCapacity(hardCap, 4)
assert.Equal(t, 2, remaining)
candidates := buildMonitoringDailyTaskCandidates(
7,
11,
businessDay,
[]monitoringConfiguredQuestion{
{ID: 101, QuestionHash: []byte("q101")},
{ID: 102, QuestionHash: []byte("q102")},
{ID: 103, QuestionHash: []byte("q103")},
},
[]monitoringPlatformMetadata{
{ID: "qwen"},
{ID: "doubao"},
},
hardCap,
)
require.Len(t, candidates, hardCap)
existing := make(map[monitoringDailyTaskKey]struct{}, 4)
for _, candidate := range candidates[:4] {
existing[monitoringDailyTaskKey{
QuestionID: candidate.Question.ID,
PlatformID: candidate.Platform.ID,
}] = struct{}{}
}
now := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset + defaultMonitoringDailyMaterializeWindow)
due := selectDueMonitoringDailyCandidates(candidates, existing, now, defaultMonitoringDailyLookahead, remaining)
require.Len(t, due, remaining)
for _, candidate := range due {
_, exists := existing[monitoringDailyTaskKey{
QuestionID: candidate.Question.ID,
PlatformID: candidate.Platform.ID,
}]
assert.False(t, exists)
}
assert.Equal(t, 1, decrementMonitoringDailyRemaining(remaining, 1))
assert.Equal(t, 0, decrementMonitoringDailyRemaining(remaining, 2))
}
func TestMonitoringDailyQuestionLimitUsesBrandLibraryConfig(t *testing.T) {
assert.Equal(t, 25, monitoringDailyQuestionLimit(config.BrandLibraryConfig{
MaxKeywords: 5,
MaxQuestionsPerKeyword: 5,
}))
assert.Equal(t, 6, monitoringDailyQuestionLimit(config.BrandLibraryConfig{
MaxKeywords: 2,
MaxQuestionsPerKeyword: 3,
}))
questions := []monitoringConfiguredQuestion{
{ID: 1},
{ID: 2},
{ID: 3},
}
assert.Len(t, limitMonitoringDailyQuestions(questions, 2), 2)
assert.Len(t, limitMonitoringDailyQuestions(questions, 25), 3)
}
func TestDecodeDailyMonitoringEnabledPlatformsReportsMalformedJSON(t *testing.T) {
platforms, err := decodeDailyMonitoringEnabledPlatforms([]byte(`["qwen"`))
require.Error(t, err)
assert.Nil(t, platforms)
}
func TestDecodeDailyMonitoringEnabledPlatformsReconcilesSupportedPlatforms(t *testing.T) {
platforms, err := decodeDailyMonitoringEnabledPlatforms([]byte(`["qwen","unknown","doubao"]`))
require.NoError(t, err)
assert.Equal(t, []string{"doubao", "qwen"}, platforms)
}
func TestBuildDailyMonitoringPlansFromSelectedClientsRequiresPlatformSnapshot(t *testing.T) {
clientOne := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000001")
clientTwo := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000002")
candidates := []monitoringDailyPlanClientCandidate{
{
TenantID: 1,
WorkspaceID: 10,
PrimaryClientID: clientOne,
MaxBrands: 3,
MaxQuestionsPerBrand: 25,
},
{
TenantID: 1,
WorkspaceID: 11,
PrimaryClientID: clientTwo,
MaxBrands: 2,
MaxQuestionsPerBrand: 10,
},
}
enabledByClient := map[monitoringDailyPlanClientKey][]byte{
candidates[0].monitoringDailyPlanClientKey(): []byte(`["qwen"]`),
}
plans := buildDailyMonitoringPlansFromSelectedClients(candidates, enabledByClient)
require.Len(t, plans, 1)
assert.Equal(t, int64(1), plans[0].TenantID)
assert.Equal(t, int64(10), plans[0].WorkspaceID)
assert.Equal(t, 3, plans[0].MaxBrands)
assert.Equal(t, 25, plans[0].MaxQuestionsPerBrand)
require.NotNil(t, plans[0].PrimaryClientID)
assert.Equal(t, clientOne, *plans[0].PrimaryClientID)
assert.Equal(t, []byte(`["qwen"]`), plans[0].EnabledJSON)
}
func TestOrderDailyMonitoringPlansRotatesDeterministicallyByBucket(t *testing.T) {
plans := []monitoringDailyPlan{
{TenantID: 1},
{TenantID: 2},
{TenantID: 3},
}
now := time.Date(2026, 4, 24, 1, 0, 0, 0, time.UTC)
first := orderDailyMonitoringPlans(plans, "2026-04-24", now)
second := orderDailyMonitoringPlans(plans, "2026-04-24", now)
assert.Equal(t, first, second)
assert.Equal(t, []monitoringDailyPlan{{TenantID: 1}, {TenantID: 2}, {TenantID: 3}}, plans)
}
func TestResolveDailyMonitoringExecutionOwnerRequiresDesktopRolloutAndClient(t *testing.T) {
clientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000001")
disabled := &MonitoringService{desktopTasksRolloutPercentage: 0}
owner, ok, reason := disabled.resolveDailyMonitoringExecutionOwner(monitoringDailyPlan{
TenantID: 1,
PrimaryClientID: &clientID,
})
assert.False(t, ok)
assert.Empty(t, owner)
assert.Equal(t, "desktop_tasks_rollout_disabled", reason)
enabled := &MonitoringService{desktopTasksRolloutPercentage: 100}
owner, ok, reason = enabled.resolveDailyMonitoringExecutionOwner(monitoringDailyPlan{TenantID: 1})
assert.False(t, ok)
assert.Empty(t, owner)
assert.Equal(t, "missing_primary_client", reason)
owner, ok, reason = enabled.resolveDailyMonitoringExecutionOwner(monitoringDailyPlan{
TenantID: 1,
PrimaryClientID: &clientID,
})
assert.True(t, ok)
assert.Equal(t, "desktop_tasks", owner)
assert.Empty(t, reason)
}
func TestMonitoringDailyTaskMetricsRecordsSummaryCounters(t *testing.T) {
resetMonitoringDailyTaskMetricsForTest()
t.Cleanup(resetMonitoringDailyTaskMetricsForTest)
recordMonitoringDailyTaskMetrics(&MonitoringDailyTaskGenerationSummary{
BusinessDate: "2026-04-24",
TenantCount: 3,
BrandCount: 2,
FailedPlanCount: 1,
FailedBrandCount: 2,
PlannedTaskCount: 20,
DueTaskCount: 8,
CreatedTaskCount: 6,
DesktopTaskCount: 4,
DeferredTaskCount: 2,
SkippedPlanCount: 1,
}, 1500*time.Millisecond, nil)
snapshot := MonitoringDailyTaskMetricsSnapshotValue()
assert.Equal(t, int64(1), snapshot.RunTotal)
assert.Equal(t, int64(1), snapshot.RunCompletedTotal)
assert.Equal(t, int64(0), snapshot.RunFailedTotal)
assert.Equal(t, int64(1), snapshot.RunPartialFailureTotal)
assert.Equal(t, int64(1), snapshot.PlanFailureTotal)
assert.Equal(t, int64(2), snapshot.BrandFailureTotal)
assert.Equal(t, int64(1), snapshot.SkippedPlanTotal)
assert.Equal(t, int64(20), snapshot.PlannedTaskTotal)
assert.Equal(t, int64(8), snapshot.DueTaskTotal)
assert.Equal(t, int64(6), snapshot.CreatedTaskTotal)
assert.Equal(t, int64(4), snapshot.DesktopTaskTotal)
assert.Equal(t, int64(2), snapshot.DeferredTaskTotal)
assert.Equal(t, "2026-04-24", snapshot.LastBusinessDate)
assert.Equal(t, int64(3), snapshot.LastTenantCount)
assert.Equal(t, int64(2), snapshot.LastBrandCount)
assert.NotNil(t, snapshot.LastRunAt)
assert.NotNil(t, snapshot.LastSuccessAt)
prometheus := MonitoringDailyTaskMetricsPrometheus()
assert.Contains(t, prometheus, `monitoring_daily_task_worker_runs_total{status="completed"} 1`)
assert.Contains(t, prometheus, `monitoring_daily_task_generation_failures_total{scope="plan"} 1`)
assert.Contains(t, prometheus, `monitoring_daily_task_generation_tasks_total{stage="created"} 6`)
assert.Contains(t, prometheus, `monitoring_daily_task_worker_last_business_date_info{business_date="2026-04-24"} 1`)
}
func TestMonitoringDailyTaskMetricsRecordsFatalError(t *testing.T) {
resetMonitoringDailyTaskMetricsForTest()
t.Cleanup(resetMonitoringDailyTaskMetricsForTest)
recordMonitoringDailyTaskMetrics(nil, time.Second, context.DeadlineExceeded)
snapshot := MonitoringDailyTaskMetricsSnapshotValue()
assert.Equal(t, int64(1), snapshot.RunTotal)
assert.Equal(t, int64(0), snapshot.RunCompletedTotal)
assert.Equal(t, int64(1), snapshot.RunFailedTotal)
assert.Equal(t, "context_deadline_exceeded", snapshot.LastErrorCode)
assert.NotNil(t, snapshot.LastRunAt)
assert.NotNil(t, snapshot.LastErrorAt)
prometheus := MonitoringDailyTaskMetricsPrometheus()
assert.Contains(t, prometheus, `monitoring_daily_task_worker_runs_total{status="failed"} 1`)
assert.Contains(t, prometheus, `monitoring_daily_task_worker_last_error_info{code="context_deadline_exceeded"} 1`)
}
func TestMonitoringDailyTaskPrometheusEscapesCarriageReturnLabels(t *testing.T) {
resetMonitoringDailyTaskMetricsForTest()
t.Cleanup(resetMonitoringDailyTaskMetricsForTest)
monitoringDailyTaskMetrics.lastBusinessDate.Store("2026-04-24\rbad")
prometheus := MonitoringDailyTaskMetricsPrometheus()
assert.NotContains(t, prometheus, "\r")
assert.Contains(t, prometheus, `monitoring_daily_task_worker_last_business_date_info{business_date="2026-04-24\\rbad"} 1`)
}
func TestMonitoringSchedulerMetricsPrometheusHandlerIncludesRuntimeAndProcessCollectors(t *testing.T) {
resetMonitoringDailyTaskMetricsForTest()
t.Cleanup(resetMonitoringDailyTaskMetricsForTest)
recordMonitoringDailyTaskMetrics(&MonitoringDailyTaskGenerationSummary{CreatedTaskCount: 1}, time.Second, nil)
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodGet, "/metrics", nil)
MonitoringSchedulerMetricsPrometheusHandler().ServeHTTP(recorder, request)
body := recorder.Body.String()
assert.Equal(t, http.StatusOK, recorder.Code)
assert.Contains(t, body, `monitoring_daily_task_worker_runs_total{status="completed"} 1`)
assert.Contains(t, body, "go_goroutines")
if runtime.GOOS == "linux" || runtime.GOOS == "windows" {
assert.Contains(t, body, "process_cpu_seconds_total")
}
assert.True(t, strings.HasPrefix(recorder.Header().Get("Content-Type"), "text/plain"))
}
func mustParseDailyMonitoringUUID(t *testing.T, value string) uuid.UUID {
t.Helper()
id, err := uuid.Parse(value)
require.NoError(t, err)
return id
}
@@ -0,0 +1,141 @@
package app
import (
"sync/atomic"
"time"
)
type MonitoringHeartbeatPrimaryLeaseMetricsSnapshot struct {
PrimaryHitTotal int64 `json:"primary_hit_total"`
NonPrimaryLiveTotal int64 `json:"non_primary_live_total"`
InsertedTotal int64 `json:"inserted_total"`
RenewedTotal int64 `json:"renewed_total"`
ClaimedExpiredTotal int64 `json:"claimed_expired_total"`
ClaimedUnavailableTotal int64 `json:"claimed_unavailable_total"`
ContentionTotal int64 `json:"contention_total"`
ErrorTotal int64 `json:"error_total"`
DBInsertTotal int64 `json:"db_insert_total"`
DBRenewTotal int64 `json:"db_renew_total"`
DBClaimTotal int64 `json:"db_claim_total"`
DurationCount int64 `json:"duration_count"`
DurationSumSeconds float64 `json:"duration_sum_seconds"`
LastDurationSeconds float64 `json:"last_duration_seconds"`
LastOutcome string `json:"last_outcome,omitempty"`
}
type monitoringHeartbeatPrimaryLeaseMetricsRecorder struct {
primaryHitTotal atomic.Int64
nonPrimaryLiveTotal atomic.Int64
insertedTotal atomic.Int64
renewedTotal atomic.Int64
claimedExpiredTotal atomic.Int64
claimedUnavailableTotal atomic.Int64
contentionTotal atomic.Int64
errorTotal atomic.Int64
dbInsertTotal atomic.Int64
dbRenewTotal atomic.Int64
dbClaimTotal atomic.Int64
durationCount atomic.Int64
durationSumNanos atomic.Int64
lastDurationNanos atomic.Int64
lastOutcome atomic.Value
}
var monitoringHeartbeatPrimaryLeaseMetrics = newMonitoringHeartbeatPrimaryLeaseMetricsRecorder()
func newMonitoringHeartbeatPrimaryLeaseMetricsRecorder() *monitoringHeartbeatPrimaryLeaseMetricsRecorder {
return &monitoringHeartbeatPrimaryLeaseMetricsRecorder{}
}
func recordHeartbeatPrimaryLeaseMetric(outcome string, duration time.Duration) {
monitoringHeartbeatPrimaryLeaseMetrics.record(outcome, duration)
}
func recordHeartbeatPrimaryLeaseWriteMetric(operation string) {
monitoringHeartbeatPrimaryLeaseMetrics.recordWrite(operation)
}
func MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue() MonitoringHeartbeatPrimaryLeaseMetricsSnapshot {
return monitoringHeartbeatPrimaryLeaseMetrics.snapshot()
}
func resetMonitoringHeartbeatPrimaryLeaseMetricsForTest() {
monitoringHeartbeatPrimaryLeaseMetrics = newMonitoringHeartbeatPrimaryLeaseMetricsRecorder()
}
func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) record(outcome string, duration time.Duration) {
if r == nil {
return
}
if duration < 0 {
duration = 0
}
switch outcome {
case "primary_hit":
r.primaryHitTotal.Add(1)
case "non_primary_live":
r.nonPrimaryLiveTotal.Add(1)
case "inserted":
r.insertedTotal.Add(1)
case "renewed":
r.renewedTotal.Add(1)
case "claimed_expired":
r.claimedExpiredTotal.Add(1)
case "claimed_unavailable":
r.claimedUnavailableTotal.Add(1)
case "contention":
r.contentionTotal.Add(1)
case "error":
r.errorTotal.Add(1)
}
r.durationCount.Add(1)
r.durationSumNanos.Add(duration.Nanoseconds())
r.lastDurationNanos.Store(duration.Nanoseconds())
r.lastOutcome.Store(outcome)
}
func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) recordWrite(operation string) {
if r == nil {
return
}
switch operation {
case "insert":
r.dbInsertTotal.Add(1)
case "renew":
r.dbRenewTotal.Add(1)
case "claim":
r.dbClaimTotal.Add(1)
}
}
func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) snapshot() MonitoringHeartbeatPrimaryLeaseMetricsSnapshot {
if r == nil {
return MonitoringHeartbeatPrimaryLeaseMetricsSnapshot{}
}
return MonitoringHeartbeatPrimaryLeaseMetricsSnapshot{
PrimaryHitTotal: r.primaryHitTotal.Load(),
NonPrimaryLiveTotal: r.nonPrimaryLiveTotal.Load(),
InsertedTotal: r.insertedTotal.Load(),
RenewedTotal: r.renewedTotal.Load(),
ClaimedExpiredTotal: r.claimedExpiredTotal.Load(),
ClaimedUnavailableTotal: r.claimedUnavailableTotal.Load(),
ContentionTotal: r.contentionTotal.Load(),
ErrorTotal: r.errorTotal.Load(),
DBInsertTotal: r.dbInsertTotal.Load(),
DBRenewTotal: r.dbRenewTotal.Load(),
DBClaimTotal: r.dbClaimTotal.Load(),
DurationCount: r.durationCount.Load(),
DurationSumSeconds: float64(r.durationSumNanos.Load()) / float64(time.Second),
LastDurationSeconds: float64(r.lastDurationNanos.Load()) / float64(time.Second),
LastOutcome: heartbeatPrimaryMetricLoadString(&r.lastOutcome),
}
}
func heartbeatPrimaryMetricLoadString(value *atomic.Value) string {
if value == nil {
return ""
}
raw := value.Load()
result, _ := raw.(string)
return result
}
@@ -0,0 +1,123 @@
package app
import (
"sync/atomic"
"time"
)
type MonitoringHeartbeatSnapshotMetricsSnapshot struct {
PlatformReportedTotal int64 `json:"platform_reported_total"`
TransactionOpenedTotal int64 `json:"transaction_opened_total"`
SnapshotInsertedTotal int64 `json:"snapshot_inserted_total"`
SnapshotUpdatedTotal int64 `json:"snapshot_updated_total"`
SnapshotRefreshedTotal int64 `json:"snapshot_refreshed_total"`
SnapshotSkippedTotal int64 `json:"snapshot_skipped_total"`
ReconciledSkippedTaskTotal int64 `json:"reconciled_skipped_task_total"`
ProjectionRebuildTotal int64 `json:"projection_rebuild_total"`
ErrorTotal int64 `json:"error_total"`
DurationCount int64 `json:"duration_count"`
DurationSumSeconds float64 `json:"duration_sum_seconds"`
LastDurationSeconds float64 `json:"last_duration_seconds"`
LastErrorCode string `json:"last_error_code,omitempty"`
LastErrorMessage string `json:"last_error_message,omitempty"`
}
type monitoringHeartbeatSnapshotMetricsEvent struct {
Active bool
PlatformCount int
TransactionOpened bool
SnapshotInsertedCount int64
SnapshotUpdatedCount int64
SnapshotRefreshedCount int64
SnapshotSkippedCount int64
ReconciledSkippedTaskCount int64
ProjectionRebuildCount int64
Duration time.Duration
Err error
}
type monitoringHeartbeatSnapshotMetricsRecorder struct {
platformReportedTotal atomic.Int64
transactionOpenedTotal atomic.Int64
snapshotInsertedTotal atomic.Int64
snapshotUpdatedTotal atomic.Int64
snapshotRefreshedTotal atomic.Int64
snapshotSkippedTotal atomic.Int64
reconciledSkippedTaskTotal atomic.Int64
projectionRebuildTotal atomic.Int64
errorTotal atomic.Int64
durationCount atomic.Int64
durationSumNanos atomic.Int64
lastDurationNanos atomic.Int64
lastErrorCode atomic.Value
lastErrorMessage atomic.Value
}
var monitoringHeartbeatSnapshotMetrics = newMonitoringHeartbeatSnapshotMetricsRecorder()
func newMonitoringHeartbeatSnapshotMetricsRecorder() *monitoringHeartbeatSnapshotMetricsRecorder {
return &monitoringHeartbeatSnapshotMetricsRecorder{}
}
func recordMonitoringHeartbeatSnapshotMetrics(event monitoringHeartbeatSnapshotMetricsEvent) {
monitoringHeartbeatSnapshotMetrics.record(event)
}
func MonitoringHeartbeatSnapshotMetricsSnapshotValue() MonitoringHeartbeatSnapshotMetricsSnapshot {
return monitoringHeartbeatSnapshotMetrics.snapshot()
}
func resetMonitoringHeartbeatSnapshotMetricsForTest() {
monitoringHeartbeatSnapshotMetrics = newMonitoringHeartbeatSnapshotMetricsRecorder()
}
func (r *monitoringHeartbeatSnapshotMetricsRecorder) record(event monitoringHeartbeatSnapshotMetricsEvent) {
if r == nil {
return
}
if event.Duration < 0 {
event.Duration = 0
}
r.platformReportedTotal.Add(int64(event.PlatformCount))
if event.TransactionOpened {
r.transactionOpenedTotal.Add(1)
}
r.snapshotInsertedTotal.Add(event.SnapshotInsertedCount)
r.snapshotUpdatedTotal.Add(event.SnapshotUpdatedCount)
r.snapshotRefreshedTotal.Add(event.SnapshotRefreshedCount)
r.snapshotSkippedTotal.Add(event.SnapshotSkippedCount)
r.reconciledSkippedTaskTotal.Add(event.ReconciledSkippedTaskCount)
r.projectionRebuildTotal.Add(event.ProjectionRebuildCount)
r.durationCount.Add(1)
r.durationSumNanos.Add(event.Duration.Nanoseconds())
r.lastDurationNanos.Store(event.Duration.Nanoseconds())
if event.Err != nil {
code, message := monitoringDailyMetricError(event.Err)
r.errorTotal.Add(1)
r.lastErrorCode.Store(code)
r.lastErrorMessage.Store(message)
}
}
func (r *monitoringHeartbeatSnapshotMetricsRecorder) snapshot() MonitoringHeartbeatSnapshotMetricsSnapshot {
if r == nil {
return MonitoringHeartbeatSnapshotMetricsSnapshot{}
}
return MonitoringHeartbeatSnapshotMetricsSnapshot{
PlatformReportedTotal: r.platformReportedTotal.Load(),
TransactionOpenedTotal: r.transactionOpenedTotal.Load(),
SnapshotInsertedTotal: r.snapshotInsertedTotal.Load(),
SnapshotUpdatedTotal: r.snapshotUpdatedTotal.Load(),
SnapshotRefreshedTotal: r.snapshotRefreshedTotal.Load(),
SnapshotSkippedTotal: r.snapshotSkippedTotal.Load(),
ReconciledSkippedTaskTotal: r.reconciledSkippedTaskTotal.Load(),
ProjectionRebuildTotal: r.projectionRebuildTotal.Load(),
ErrorTotal: r.errorTotal.Load(),
DurationCount: r.durationCount.Load(),
DurationSumSeconds: float64(r.durationSumNanos.Load()) / float64(time.Second),
LastDurationSeconds: float64(r.lastDurationNanos.Load()) / float64(time.Second),
LastErrorCode: monitoringDailyLoadString(&r.lastErrorCode),
LastErrorMessage: monitoringDailyLoadString(&r.lastErrorMessage),
}
}
@@ -0,0 +1,195 @@
package app
import (
"bytes"
"net/http"
"strings"
"github.com/prometheus/client_golang/prometheus"
promcollectors "github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/prometheus/common/expfmt"
)
func MonitoringSchedulerMetricsPrometheusHandler() http.Handler {
return monitoringPrometheusHandler(true, monitoringDailyTaskMetricsCollector{})
}
func MonitoringDailyTaskMetricsPrometheus() string {
return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringDailyTaskMetricsCollector{}))
}
func MonitoringDailyTaskMetricsPrometheusHandler() http.Handler {
return monitoringPrometheusHandler(false, monitoringDailyTaskMetricsCollector{})
}
func MonitoringHeartbeatPrimaryLeaseMetricsPrometheus() string {
return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringHeartbeatPrimaryLeaseMetricsCollector{}))
}
func MonitoringHeartbeatPrimaryLeaseMetricsPrometheusHandler() http.Handler {
return monitoringPrometheusHandler(false, monitoringHeartbeatPrimaryLeaseMetricsCollector{})
}
func MonitoringHeartbeatSnapshotMetricsPrometheus() string {
return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringHeartbeatSnapshotMetricsCollector{}))
}
func MonitoringHeartbeatSnapshotMetricsPrometheusHandler() http.Handler {
return monitoringPrometheusHandler(false, monitoringHeartbeatSnapshotMetricsCollector{})
}
func monitoringPrometheusHandler(includeRuntime bool, metricCollectors ...prometheus.Collector) http.Handler {
return promhttp.HandlerFor(monitoringPrometheusRegistry(includeRuntime, metricCollectors...), promhttp.HandlerOpts{})
}
func monitoringPrometheusRegistry(includeRuntime bool, metricCollectors ...prometheus.Collector) *prometheus.Registry {
registry := prometheus.NewRegistry()
if includeRuntime {
registry.MustRegister(
promcollectors.NewGoCollector(),
promcollectors.NewProcessCollector(promcollectors.ProcessCollectorOpts{}),
)
}
for _, collector := range metricCollectors {
registry.MustRegister(collector)
}
return registry
}
func renderMonitoringPrometheus(gatherer prometheus.Gatherer) string {
metricFamilies, err := gatherer.Gather()
if err != nil {
return ""
}
var b bytes.Buffer
encoder := expfmt.NewEncoder(&b, expfmt.NewFormat(expfmt.TypeTextPlain))
for _, metricFamily := range metricFamilies {
if err := encoder.Encode(metricFamily); err != nil {
return ""
}
}
if closer, ok := encoder.(expfmt.Closer); ok {
_ = closer.Close()
}
return b.String()
}
type monitoringDailyTaskMetricsCollector struct{}
func (monitoringDailyTaskMetricsCollector) Describe(chan<- *prometheus.Desc) {}
func (monitoringDailyTaskMetricsCollector) Collect(ch chan<- prometheus.Metric) {
snapshot := MonitoringDailyTaskMetricsSnapshotValue()
monitoringCounter(ch, "monitoring_daily_task_worker_runs_total", "Total daily monitoring worker runs by terminal status.", float64(snapshot.RunCompletedTotal), []string{"status"}, "completed")
monitoringCounter(ch, "monitoring_daily_task_worker_runs_total", "Total daily monitoring worker runs by terminal status.", float64(snapshot.RunFailedTotal), []string{"status"}, "failed")
monitoringCounter(ch, "monitoring_daily_task_worker_partial_failure_runs_total", "Total completed daily monitoring worker runs with at least one plan or brand failure.", float64(snapshot.RunPartialFailureTotal), nil)
monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.PlanFailureTotal), []string{"scope"}, "plan")
monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.BrandFailureTotal), []string{"scope"}, "brand")
monitoringCounter(ch, "monitoring_daily_task_generation_skipped_plans_total", "Total daily monitoring plans skipped before generation.", float64(snapshot.SkippedPlanTotal), nil)
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.PlannedTaskTotal), []string{"stage"}, "planned")
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DueTaskTotal), []string{"stage"}, "due")
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.CreatedTaskTotal), []string{"stage"}, "created")
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DesktopTaskTotal), []string{"stage"}, "desktop")
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DeferredTaskTotal), []string{"stage"}, "deferred")
monitoringGauge(ch, "monitoring_daily_task_worker_last_run_timestamp_seconds", "Unix timestamp of the last daily monitoring worker run completion.", monitoringDailyTimeSeconds(snapshot.LastRunAt), nil)
monitoringGauge(ch, "monitoring_daily_task_worker_last_success_timestamp_seconds", "Unix timestamp of the last completed daily monitoring worker run.", monitoringDailyTimeSeconds(snapshot.LastSuccessAt), nil)
monitoringGauge(ch, "monitoring_daily_task_worker_last_error_timestamp_seconds", "Unix timestamp of the last failed daily monitoring worker run.", monitoringDailyTimeSeconds(snapshot.LastErrorAt), nil)
monitoringGauge(ch, "monitoring_daily_task_worker_last_duration_seconds", "Duration of the last daily monitoring worker run.", snapshot.LastDurationSeconds, nil)
monitoringGauge(ch, "monitoring_daily_task_worker_last_scope_count", "Tenant and brand counts observed in the last daily monitoring worker run.", float64(snapshot.LastTenantCount), []string{"scope"}, "tenant")
monitoringGauge(ch, "monitoring_daily_task_worker_last_scope_count", "Tenant and brand counts observed in the last daily monitoring worker run.", float64(snapshot.LastBrandCount), []string{"scope"}, "brand")
if snapshot.LastBusinessDate != "" {
monitoringGauge(ch, "monitoring_daily_task_worker_last_business_date_info", "Business date processed by the last daily monitoring worker run.", 1, []string{"business_date"}, snapshot.LastBusinessDate)
}
if snapshot.LastErrorCode != "" {
monitoringGauge(ch, "monitoring_daily_task_worker_last_error_info", "Last fatal daily monitoring worker error code.", 1, []string{"code"}, snapshot.LastErrorCode)
}
}
type monitoringHeartbeatPrimaryLeaseMetricsCollector struct{}
func (monitoringHeartbeatPrimaryLeaseMetricsCollector) Describe(chan<- *prometheus.Desc) {}
func (monitoringHeartbeatPrimaryLeaseMetricsCollector) Collect(ch chan<- prometheus.Metric) {
snapshot := MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue()
help := "Total monitoring heartbeat primary lease resolutions by outcome."
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.PrimaryHitTotal), []string{"outcome"}, "primary_hit")
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.NonPrimaryLiveTotal), []string{"outcome"}, "non_primary_live")
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.InsertedTotal), []string{"outcome"}, "inserted")
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.RenewedTotal), []string{"outcome"}, "renewed")
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ClaimedExpiredTotal), []string{"outcome"}, "claimed_expired")
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ClaimedUnavailableTotal), []string{"outcome"}, "claimed_unavailable")
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ContentionTotal), []string{"outcome"}, "contention")
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ErrorTotal), []string{"outcome"}, "error")
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBInsertTotal), []string{"operation"}, "insert")
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBRenewTotal), []string{"operation"}, "renew")
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBClaimTotal), []string{"operation"}, "claim")
monitoringSummary(ch, "monitoring_heartbeat_primary_lease_duration_seconds", "Monitoring heartbeat primary lease resolution duration summary.", snapshot.DurationCount, snapshot.DurationSumSeconds, nil)
monitoringGauge(ch, "monitoring_heartbeat_primary_lease_duration_seconds_last", "Last monitoring heartbeat primary lease resolution duration in seconds.", snapshot.LastDurationSeconds, nil)
if snapshot.LastOutcome != "" {
monitoringGauge(ch, "monitoring_heartbeat_primary_lease_last_outcome_info", "Last monitoring heartbeat primary lease resolution outcome.", 1, []string{"outcome"}, snapshot.LastOutcome)
}
}
type monitoringHeartbeatSnapshotMetricsCollector struct{}
func (monitoringHeartbeatSnapshotMetricsCollector) Describe(chan<- *prometheus.Desc) {}
func (monitoringHeartbeatSnapshotMetricsCollector) Collect(ch chan<- prometheus.Metric) {
snapshot := MonitoringHeartbeatSnapshotMetricsSnapshotValue()
monitoringCounter(ch, "monitoring_heartbeat_snapshot_platform_reports_total", "Total monitoring heartbeat platform reports received.", float64(snapshot.PlatformReportedTotal), nil)
monitoringCounter(ch, "monitoring_heartbeat_snapshot_transactions_total", "Total monitoring heartbeat snapshot transactions opened.", float64(snapshot.TransactionOpenedTotal), nil)
monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotInsertedTotal), []string{"operation"}, "insert")
monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotUpdatedTotal), []string{"operation"}, "update")
monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotRefreshedTotal), []string{"operation"}, "refresh")
monitoringCounter(ch, "monitoring_heartbeat_snapshot_skips_total", "Total monitoring heartbeat access snapshot reports skipped because state was fresh and unchanged.", float64(snapshot.SnapshotSkippedTotal), nil)
monitoringCounter(ch, "monitoring_heartbeat_snapshot_reconciled_tasks_total", "Total pending monitoring tasks skipped by heartbeat access reconciliation.", float64(snapshot.ReconciledSkippedTaskTotal), nil)
monitoringCounter(ch, "monitoring_heartbeat_snapshot_projection_rebuilds_total", "Total monitoring daily projection rebuilds triggered by heartbeat snapshots.", float64(snapshot.ProjectionRebuildTotal), nil)
monitoringCounter(ch, "monitoring_heartbeat_snapshot_errors_total", "Total monitoring heartbeat snapshot errors.", float64(snapshot.ErrorTotal), nil)
monitoringSummary(ch, "monitoring_heartbeat_snapshot_duration_seconds", "Monitoring heartbeat snapshot processing duration summary.", snapshot.DurationCount, snapshot.DurationSumSeconds, nil)
monitoringGauge(ch, "monitoring_heartbeat_snapshot_duration_seconds_last", "Last monitoring heartbeat snapshot processing duration in seconds.", snapshot.LastDurationSeconds, nil)
if snapshot.LastErrorCode != "" {
monitoringGauge(ch, "monitoring_heartbeat_snapshot_last_error_info", "Last monitoring heartbeat snapshot error code.", 1, []string{"code"}, snapshot.LastErrorCode)
}
}
func monitoringCounter(ch chan<- prometheus.Metric, name, help string, value float64, labelNames []string, labelValues ...string) {
monitoringMetric(ch, name, help, prometheus.CounterValue, value, labelNames, labelValues...)
}
func monitoringGauge(ch chan<- prometheus.Metric, name, help string, value float64, labelNames []string, labelValues ...string) {
monitoringMetric(ch, name, help, prometheus.GaugeValue, value, labelNames, labelValues...)
}
func monitoringMetric(ch chan<- prometheus.Metric, name, help string, valueType prometheus.ValueType, value float64, labelNames []string, labelValues ...string) {
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(name, help, labelNames, nil),
valueType,
value,
monitoringPrometheusLabelValues(labelValues)...,
)
}
func monitoringSummary(ch chan<- prometheus.Metric, name, help string, count int64, sum float64, labelNames []string, labelValues ...string) {
if count < 0 {
count = 0
}
ch <- prometheus.MustNewConstSummary(
prometheus.NewDesc(name, help, labelNames, nil),
uint64(count),
sum,
nil,
monitoringPrometheusLabelValues(labelValues)...,
)
}
func monitoringPrometheusLabelValues(values []string) []string {
if len(values) == 0 {
return values
}
result := make([]string, len(values))
for idx, value := range values {
result[idx] = strings.ReplaceAll(value, "\r", `\r`)
}
return result
}
@@ -223,8 +223,7 @@ func (s *MonitoringService) loadMonitorDesktopAccountMap(
rows, err := s.businessPool.Query(ctx, ` rows, err := s.businessPool.Query(ctx, `
SELECT DISTINCT ON (platform_id) SELECT DISTINCT ON (platform_id)
platform_id, platform_id,
desktop_id, desktop_id
health
FROM platform_accounts FROM platform_accounts
WHERE tenant_id = $1 WHERE tenant_id = $1
AND workspace_id = $2 AND workspace_id = $2
@@ -251,14 +250,10 @@ func (s *MonitoringService) loadMonitorDesktopAccountMap(
var ( var (
platformID string platformID string
desktopID uuid.UUID desktopID uuid.UUID
health string
) )
if scanErr := rows.Scan(&platformID, &desktopID, &health); scanErr != nil { if scanErr := rows.Scan(&platformID, &desktopID); scanErr != nil {
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to parse monitor desktop accounts") return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to parse monitor desktop accounts")
} }
if strings.TrimSpace(health) != "live" {
continue
}
result[platformID] = desktopID result[platformID] = desktopID
} }
if err := rows.Err(); err != nil { if err := rows.Err(); err != nil {
@@ -3,11 +3,9 @@ package app
import ( import (
"context" "context"
"database/sql" "database/sql"
"encoding/json"
"time" "time"
"github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
) )
type monitoringBrandDailyAggregate struct { type monitoringBrandDailyAggregate struct {
@@ -33,7 +31,7 @@ type monitoringPlatformDailyAggregate struct {
} }
func (s *MonitoringCallbackService) rebuildBrandDailyProjection(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDate time.Time) error { func (s *MonitoringCallbackService) rebuildBrandDailyProjection(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDate time.Time) error {
platforms, err := loadMonitoringProjectionPlatforms(ctx, s.businessPool, tenantID) platforms, err := loadMonitoringProjectionPlatforms(ctx, tx, tenantID, brandID, businessDate)
if err != nil { if err != nil {
return err return err
} }
@@ -133,29 +131,8 @@ func (s *MonitoringCallbackService) rebuildBrandDailyProjection(ctx context.Cont
return nil return nil
} }
func loadMonitoringProjectionPlatforms(ctx context.Context, businessPool *pgxpool.Pool, tenantID int64) ([]monitoringPlatformMetadata, error) { func loadMonitoringProjectionPlatforms(_ context.Context, _ pgx.Tx, _ int64, _ int64, _ time.Time) ([]monitoringPlatformMetadata, error) {
enabled := []string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"} return defaultMonitoringPlatformMetadata(), nil
var enabledJSON []byte
if err := businessPool.QueryRow(ctx, `
SELECT enabled_platforms
FROM tenant_monitoring_quotas
WHERE tenant_id = $1
`, tenantID).Scan(&enabledJSON); err != nil {
if err == pgx.ErrNoRows {
return resolveMonitoringPlatforms(enabled), nil
}
return nil, monitoringInternalError(50041, "quota_lookup_failed", "failed to load monitoring quota", err)
}
if len(enabledJSON) > 0 {
var configured []string
if jsonErr := json.Unmarshal(enabledJSON, &configured); jsonErr == nil && len(configured) > 0 {
enabled = monitoringPlatformIDs(defaultMonitoringPlatforms)
}
}
return resolveMonitoringPlatforms(enabled), nil
} }
func (s *MonitoringCallbackService) loadMonitoringEnabledQuestionCount(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDate time.Time) (int64, error) { func (s *MonitoringCallbackService) loadMonitoringEnabledQuestionCount(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDate time.Time) (int64, error) {
+233 -84
View File
@@ -30,6 +30,10 @@ const (
monitoringCollectorType = "desktop" monitoringCollectorType = "desktop"
monitoringOnlineDuration = 20 * time.Minute monitoringOnlineDuration = 20 * time.Minute
monitoringCollectNowQuestionLimit = 5 monitoringCollectNowQuestionLimit = 5
monitoringPlatformAuthorizationAuthorized = "authorized"
monitoringPlatformAuthorizationNoDesktopClient = "no_desktop_client"
monitoringPlatformAuthorizationNoAuthorizedPlatforms = "no_authorized_platforms"
) )
type MonitoringService struct { type MonitoringService struct {
@@ -38,12 +42,14 @@ type MonitoringService struct {
rabbitMQ *rabbitmq.Client rabbitMQ *rabbitmq.Client
logger *zap.Logger logger *zap.Logger
desktopTasksRolloutPercentage int desktopTasksRolloutPercentage int
brandLibraryConfig config.BrandLibraryConfig
} }
func NewMonitoringService( func NewMonitoringService(
businessPool, monitoringPool *pgxpool.Pool, businessPool, monitoringPool *pgxpool.Pool,
rabbitMQClient *rabbitmq.Client, rabbitMQClient *rabbitmq.Client,
dispatchCfg config.MonitoringDispatchConfig, dispatchCfg config.MonitoringDispatchConfig,
brandLibraryCfg config.BrandLibraryConfig,
logger *zap.Logger, logger *zap.Logger,
) *MonitoringService { ) *MonitoringService {
return &MonitoringService{ return &MonitoringService{
@@ -52,6 +58,7 @@ func NewMonitoringService(
rabbitMQ: rabbitMQClient, rabbitMQ: rabbitMQClient,
logger: logger, logger: logger,
desktopTasksRolloutPercentage: dispatchCfg.DesktopTasksRolloutPercentage, desktopTasksRolloutPercentage: dispatchCfg.DesktopTasksRolloutPercentage,
brandLibraryConfig: brandLibraryCfg,
} }
} }
@@ -66,6 +73,9 @@ type MonitoringDashboardCompositeResponse struct {
type MonitoringDashboardRuntime struct { type MonitoringDashboardRuntime struct {
CurrentUserClientOnline bool `json:"current_user_client_online"` CurrentUserClientOnline bool `json:"current_user_client_online"`
PlatformAuthorizationStatus string `json:"platform_authorization_status"`
AuthorizedMonitoringPlatformIDs []string `json:"authorized_monitoring_platform_ids"`
AuthorizedMonitoringPlatformCount int `json:"authorized_monitoring_platform_count"`
} }
type MonitoringOverview struct { type MonitoringOverview struct {
@@ -127,6 +137,7 @@ type MonitoringQuestionDetailResponse struct {
QuestionID int64 `json:"question_id"` QuestionID int64 `json:"question_id"`
QuestionHash string `json:"question_hash"` QuestionHash string `json:"question_hash"`
QuestionText string `json:"question_text"` QuestionText string `json:"question_text"`
PlatformAuthorizationStatus string `json:"platform_authorization_status"`
TimeWindow MonitoringQuestionDetailTimeWindow `json:"time_window"` TimeWindow MonitoringQuestionDetailTimeWindow `json:"time_window"`
Platforms []MonitoringQuestionDetailPlatform `json:"platforms"` Platforms []MonitoringQuestionDetailPlatform `json:"platforms"`
CitationAnalysis []MonitoringQuestionCitationStats `json:"citation_analysis"` CitationAnalysis []MonitoringQuestionCitationStats `json:"citation_analysis"`
@@ -216,6 +227,16 @@ type monitoringQuotaConfig struct {
EnabledPlatforms []string EnabledPlatforms []string
} }
func monitoringPlatformAuthorizationStatus(quota monitoringQuotaConfig) string {
if quota.PrimaryClientID == nil {
return monitoringPlatformAuthorizationNoDesktopClient
}
if len(quota.EnabledPlatforms) == 0 {
return monitoringPlatformAuthorizationNoAuthorizedPlatforms
}
return monitoringPlatformAuthorizationAuthorized
}
type monitoringBrandIdentity struct { type monitoringBrandIdentity struct {
ID int64 ID int64
Name string Name string
@@ -290,6 +311,7 @@ func (s *MonitoringService) DashboardComposite(
aiPlatformID *string, aiPlatformID *string,
) (*MonitoringDashboardCompositeResponse, error) { ) (*MonitoringDashboardCompositeResponse, error) {
actor := auth.MustActor(ctx) actor := auth.MustActor(ctx)
workspaceID := auth.CurrentWorkspaceID(ctx)
brand, err := s.resolveBrand(ctx, actor.TenantID, brandID) brand, err := s.resolveBrand(ctx, actor.TenantID, brandID)
if err != nil { if err != nil {
@@ -302,12 +324,12 @@ func (s *MonitoringService) DashboardComposite(
} }
} }
quota, err := s.loadQuota(ctx, actor.TenantID) quota, err := s.loadQuota(ctx, actor, workspaceID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
runtime, err := s.loadDashboardRuntime(ctx, actor) runtime, err := s.loadDashboardRuntime(ctx, actor, workspaceID, quota)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -318,7 +340,7 @@ func (s *MonitoringService) DashboardComposite(
} }
questionIDs := configuredQuestionIDs(configuredQuestions) questionIDs := configuredQuestionIDs(configuredQuestions)
platforms := resolveMonitoringPlatforms(quota.EnabledPlatforms) platforms := defaultMonitoringPlatformMetadata()
selectedPlatformID := normalizedOptionalMonitoringPlatformPointer(aiPlatformID) selectedPlatformID := normalizedOptionalMonitoringPlatformPointer(aiPlatformID)
filteredPlatforms := filterMonitoringPlatforms(platforms, selectedPlatformID) filteredPlatforms := filterMonitoringPlatforms(platforms, selectedPlatformID)
startDate, endDate, err := resolveDashboardDateWindow(days, businessDate) startDate, endDate, err := resolveDashboardDateWindow(days, businessDate)
@@ -334,7 +356,7 @@ func (s *MonitoringService) DashboardComposite(
return nil, err return nil, err
} }
accessStates, err := s.loadAccessStates(ctx, actor.TenantID, quota.PrimaryClientID, endDate) accessStates, err := s.loadAccessStates(ctx, actor.TenantID, workspaceID, quota.PrimaryClientID, endDate)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -369,19 +391,27 @@ func (s *MonitoringService) DashboardComposite(
}, nil }, nil
} }
func (s *MonitoringService) loadDashboardRuntime(ctx context.Context, actor auth.Actor) (MonitoringDashboardRuntime, error) { func (s *MonitoringService) loadDashboardRuntime(ctx context.Context, actor auth.Actor, workspaceID int64, quota monitoringQuotaConfig) (MonitoringDashboardRuntime, error) {
online, err := s.hasOnlineClientForUser(ctx, actor.TenantID, actor.PrimaryWorkspaceID, actor.UserID) online := false
if quota.PrimaryClientID != nil {
var err error
online, err = s.isClientOnline(ctx, actor.TenantID, workspaceID, *quota.PrimaryClientID)
if err != nil { if err != nil {
return MonitoringDashboardRuntime{}, err return MonitoringDashboardRuntime{}, err
} }
}
return MonitoringDashboardRuntime{ return MonitoringDashboardRuntime{
CurrentUserClientOnline: online, CurrentUserClientOnline: online,
PlatformAuthorizationStatus: monitoringPlatformAuthorizationStatus(quota),
AuthorizedMonitoringPlatformIDs: append([]string(nil), quota.EnabledPlatforms...),
AuthorizedMonitoringPlatformCount: len(quota.EnabledPlatforms),
}, nil }, nil
} }
func (s *MonitoringService) QuestionDetail(ctx context.Context, brandID, questionID int64, dateFrom, dateTo string, questionHash string, aiPlatformID *string) (*MonitoringQuestionDetailResponse, error) { func (s *MonitoringService) QuestionDetail(ctx context.Context, brandID, questionID int64, dateFrom, dateTo string, questionHash string, aiPlatformID *string) (*MonitoringQuestionDetailResponse, error) {
actor := auth.MustActor(ctx) actor := auth.MustActor(ctx)
workspaceID := auth.CurrentWorkspaceID(ctx)
brand, err := s.resolveBrand(ctx, actor.TenantID, brandID) brand, err := s.resolveBrand(ctx, actor.TenantID, brandID)
if err != nil { if err != nil {
@@ -393,12 +423,12 @@ func (s *MonitoringService) QuestionDetail(ctx context.Context, brandID, questio
return nil, err return nil, err
} }
quota, err := s.loadQuota(ctx, actor.TenantID) quota, err := s.loadQuota(ctx, actor, workspaceID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
platforms := resolveMonitoringPlatforms(quota.EnabledPlatforms)
selectedPlatformID := normalizedOptionalMonitoringPlatformPointer(aiPlatformID) selectedPlatformID := normalizedOptionalMonitoringPlatformPointer(aiPlatformID)
platforms := filterMonitoringPlatforms(defaultMonitoringPlatformMetadata(), selectedPlatformID)
questionText, err := s.loadQuestionText(ctx, actor.TenantID, brand.ID, questionID) questionText, err := s.loadQuestionText(ctx, actor.TenantID, brand.ID, questionID)
if err != nil { if err != nil {
@@ -418,7 +448,7 @@ func (s *MonitoringService) QuestionDetail(ctx context.Context, brandID, questio
} }
} }
accessStates, err := s.loadAccessStates(ctx, actor.TenantID, quota.PrimaryClientID, toDate) accessStates, err := s.loadAccessStates(ctx, actor.TenantID, workspaceID, quota.PrimaryClientID, toDate)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -484,6 +514,7 @@ func (s *MonitoringService) QuestionDetail(ctx context.Context, brandID, questio
QuestionID: questionID, QuestionID: questionID,
QuestionHash: encodeQuestionHash(hashBytes), QuestionHash: encodeQuestionHash(hashBytes),
QuestionText: questionText, QuestionText: questionText,
PlatformAuthorizationStatus: monitoringPlatformAuthorizationStatus(quota),
TimeWindow: MonitoringQuestionDetailTimeWindow{ TimeWindow: MonitoringQuestionDetailTimeWindow{
DateFrom: fromDate.Format("2006-01-02"), DateFrom: fromDate.Format("2006-01-02"),
DateTo: toDate.Format("2006-01-02"), DateTo: toDate.Format("2006-01-02"),
@@ -501,6 +532,7 @@ func (s *MonitoringService) CollectNow(
options MonitoringCollectNowOptions, options MonitoringCollectNowOptions,
) (*MonitoringCollectNowResponse, error) { ) (*MonitoringCollectNowResponse, error) {
actor := auth.MustActor(ctx) actor := auth.MustActor(ctx)
workspaceID := auth.CurrentWorkspaceID(ctx)
brand, err := s.resolveBrand(ctx, actor.TenantID, brandID) brand, err := s.resolveBrand(ctx, actor.TenantID, brandID)
if err != nil { if err != nil {
@@ -513,7 +545,7 @@ func (s *MonitoringService) CollectNow(
} }
} }
quota, err := s.loadQuota(ctx, actor.TenantID) quota, err := s.loadQuota(ctx, actor, workspaceID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -540,19 +572,7 @@ func (s *MonitoringService) CollectNow(
}, nil }, nil
} }
platforms := resolveMonitoringPlatforms(quota.EnabledPlatforms) clientID, err := s.resolveCollectNowClientID(ctx, actor.TenantID, workspaceID, actor.UserID, quota.PrimaryClientID, options.TargetClientID)
platforms, err = intersectMonitoringPlatforms(platforms, options.PlatformIDs)
if err != nil {
return nil, err
}
questionIDs := configuredQuestionIDs(configuredQuestions)
platformIDs := monitoringPlatformIDs(platforms)
executionOwner := "legacy"
if s.shouldUseDesktopTasksExecution(actor.TenantID) {
executionOwner = "desktop_tasks"
}
clientID, err := s.resolveCollectNowClientID(ctx, actor.TenantID, actor.PrimaryWorkspaceID, actor.UserID, options.TargetClientID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -560,10 +580,37 @@ func (s *MonitoringService) CollectNow(
return nil, response.ErrConflict(40941, "desktop_client_offline", "current user's monitoring desktop client is offline") return nil, response.ErrConflict(40941, "desktop_client_offline", "current user's monitoring desktop client is offline")
} }
if err := s.ensureClientOnline(ctx, actor.TenantID, actor.PrimaryWorkspaceID, *clientID); err != nil { if err := s.ensureClientOnline(ctx, actor.TenantID, workspaceID, *clientID); err != nil {
return nil, err return nil, err
} }
targetPlatformIDs, err := s.loadMonitoringClientPlatformIDs(ctx, actor.TenantID, workspaceID, *clientID)
if err != nil {
return nil, err
}
platforms := resolveMonitoringPlatforms(targetPlatformIDs)
platforms, err = intersectMonitoringPlatforms(platforms, options.PlatformIDs)
if err != nil {
return nil, err
}
if len(platforms) == 0 {
return &MonitoringCollectNowResponse{
CollectionMode: quota.CollectionMode,
RefreshedTaskCount: 0,
CreatedTaskCount: 0,
LeasedTaskCount: 0,
CompletedTaskCount: 0,
HasEffectiveSnapshot: true,
Message: "当前桌面端暂无已授权的 AI 平台,未创建采集任务",
}, nil
}
questionIDs := configuredQuestionIDs(configuredQuestions)
platformIDs := monitoringPlatformIDs(platforms)
executionOwner := "legacy"
if s.shouldUseDesktopTasksExecution(actor.TenantID) {
executionOwner = "desktop_tasks"
}
now := time.Now().UTC() now := time.Now().UTC()
todayTime, _ := monitoringBusinessDayAndDateAt(now) todayTime, _ := monitoringBusinessDayAndDateAt(now)
requestID := uuid.New() requestID := uuid.New()
@@ -584,7 +631,7 @@ func (s *MonitoringService) CollectNow(
ctx, ctx,
tx, tx,
actor.TenantID, actor.TenantID,
actor.PrimaryWorkspaceID, workspaceID,
brand.ID, brand.ID,
actor.UserID, actor.UserID,
scopeHash, scopeHash,
@@ -663,7 +710,7 @@ func (s *MonitoringService) CollectNow(
ctx, ctx,
tx, tx,
actor.TenantID, actor.TenantID,
actor.PrimaryWorkspaceID, workspaceID,
todayTime, todayTime,
questionIDs, questionIDs,
platformIDs, platformIDs,
@@ -722,7 +769,7 @@ func (s *MonitoringService) CollectNow(
$12, $13, $14, $12, $13, $14,
$15, $16 $15, $16
) )
`, requestID, actor.TenantID, actor.PrimaryWorkspaceID, brand.ID, nullableInt64(keywordID), actor.UserID, *clientID, scopeHash, requestScopeJSON, affectedTaskCount, affectedTaskCount, abortedQueuedCount, int64(len(interruptTaskIDs)), interruptGeneration, message, ttlExpiresAt); err != nil { `, requestID, actor.TenantID, workspaceID, brand.ID, nullableInt64(keywordID), actor.UserID, *clientID, scopeHash, requestScopeJSON, affectedTaskCount, affectedTaskCount, abortedQueuedCount, int64(len(interruptTaskIDs)), interruptGeneration, message, ttlExpiresAt); err != nil {
return nil, response.ErrInternal(50041, "request_create_failed", "failed to persist monitoring collect-now request") return nil, response.ErrInternal(50041, "request_create_failed", "failed to persist monitoring collect-now request")
} }
@@ -730,7 +777,7 @@ func (s *MonitoringService) CollectNow(
ctx, ctx,
tx, tx,
requestID, requestID,
actor.PrimaryWorkspaceID, workspaceID,
*clientID, *clientID,
affectedTaskCount, affectedTaskCount,
interruptGeneration, interruptGeneration,
@@ -748,8 +795,6 @@ func (s *MonitoringService) CollectNow(
return nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring collect-now") return nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring collect-now")
} }
_ = s.persistPrimaryClientID(ctx, actor.TenantID, actor.PrimaryWorkspaceID, *clientID)
phase2PublishedTasks := make([]*repository.DesktopTask, 0) phase2PublishedTasks := make([]*repository.DesktopTask, 0)
phase2DispatchReady := executionOwner == "desktop_tasks" phase2DispatchReady := executionOwner == "desktop_tasks"
if len(phase2TaskSpecs) > 0 { if len(phase2TaskSpecs) > 0 {
@@ -877,6 +922,7 @@ func (s *MonitoringService) CollectNow(
func (s *MonitoringService) resolveCollectNowClientID( func (s *MonitoringService) resolveCollectNowClientID(
ctx context.Context, ctx context.Context,
tenantID, workspaceID, userID int64, tenantID, workspaceID, userID int64,
defaultClientID *uuid.UUID,
preferredClientID *uuid.UUID, preferredClientID *uuid.UUID,
) (*uuid.UUID, error) { ) (*uuid.UUID, error) {
if preferredClientID != nil { if preferredClientID != nil {
@@ -885,7 +931,7 @@ func (s *MonitoringService) resolveCollectNowClientID(
} }
return preferredClientID, nil return preferredClientID, nil
} }
return s.findLatestOnlineClientForUser(ctx, tenantID, workspaceID, userID) return defaultClientID, nil
} }
func (s *MonitoringService) hasOnlineClientForUser(ctx context.Context, tenantID, workspaceID, userID int64) (bool, error) { func (s *MonitoringService) hasOnlineClientForUser(ctx context.Context, tenantID, workspaceID, userID int64) (bool, error) {
@@ -968,6 +1014,10 @@ func monitoringPlatformIDs(items []monitoringPlatformMetadata) []string {
return result return result
} }
func defaultMonitoringPlatformMetadata() []monitoringPlatformMetadata {
return append([]monitoringPlatformMetadata(nil), defaultMonitoringPlatforms...)
}
func normalizeMonitoringPlatformID(platformID string) string { func normalizeMonitoringPlatformID(platformID string) string {
return strings.ToLower(strings.TrimSpace(platformID)) return strings.ToLower(strings.TrimSpace(platformID))
} }
@@ -996,10 +1046,30 @@ func monitoringPlatformQueryIDs(value *string) []string {
} }
func reconcileEnabledMonitoringPlatforms(enabled []string) []string { func reconcileEnabledMonitoringPlatforms(enabled []string) []string {
// Old quota rows may still contain partial or aliased platform ids. Admin monitoring if len(enabled) == 0 {
// should always expose the full canonical desktop-supported set for the tenant. return nil
_ = enabled }
return monitoringPlatformIDs(defaultMonitoringPlatforms) requested := make(map[string]struct{}, len(enabled))
for _, item := range enabled {
id := normalizeMonitoringPlatformID(item)
if id == "" {
continue
}
if _, supported := monitoringPlatformNames[id]; !supported {
continue
}
requested[id] = struct{}{}
}
if len(requested) == 0 {
return nil
}
result := make([]string, 0, len(requested))
for _, platform := range defaultMonitoringPlatforms {
if _, ok := requested[platform.ID]; ok {
result = append(result, platform.ID)
}
}
return result
} }
func monitoringPlatformSortIndex(platformID string) int { func monitoringPlatformSortIndex(platformID string) int {
@@ -1054,26 +1124,6 @@ func (s *MonitoringService) findLatestOnlineClientForUser(ctx context.Context, t
return &clientID, nil return &clientID, nil
} }
func (s *MonitoringService) persistPrimaryClientID(ctx context.Context, tenantID, workspaceID int64, clientID uuid.UUID) error {
if _, err := s.businessPool.Exec(ctx, `
INSERT INTO tenant_monitoring_quotas (
tenant_id, workspace_id, primary_client_id
)
VALUES (
$1,
$2,
$3
)
ON CONFLICT (tenant_id) DO UPDATE
SET workspace_id = EXCLUDED.workspace_id,
primary_client_id = EXCLUDED.primary_client_id,
updated_at = NOW()
`, tenantID, workspaceID, clientID); err != nil {
return response.ErrInternal(50041, "update_failed", "failed to persist primary monitoring desktop client")
}
return nil
}
func formatPgInterval(duration time.Duration) string { func formatPgInterval(duration time.Duration) string {
return fmt.Sprintf("%d seconds", int64(duration/time.Second)) return fmt.Sprintf("%d seconds", int64(duration/time.Second))
} }
@@ -1431,40 +1481,117 @@ func (s *MonitoringService) deferQueuedNormalTasks(
return tag.RowsAffected(), nil return tag.RowsAffected(), nil
} }
func (s *MonitoringService) loadQuota(ctx context.Context, tenantID int64) (monitoringQuotaConfig, error) { func (s *MonitoringService) loadQuota(ctx context.Context, actor auth.Actor, workspaceID int64) (monitoringQuotaConfig, error) {
cfg := monitoringQuotaConfig{ cfg := monitoringQuotaConfig{
CollectionMode: monitoringCollectorType, CollectionMode: monitoringCollectorType,
EnabledPlatforms: reconcileEnabledMonitoringPlatforms(nil),
} }
var enabledJSON []byte clientID, err := s.findLatestRegisteredClientWithMonitoringAccountForUser(ctx, actor.TenantID, workspaceID, actor.UserID)
var clientID uuid.NullUUID if err != nil {
return cfg, err
}
if clientID == nil {
clientID, err = s.findLatestRegisteredClientForUser(ctx, actor.TenantID, workspaceID, actor.UserID)
if err != nil {
return cfg, err
}
}
cfg.PrimaryClientID = clientID
if clientID == nil {
return cfg, nil
}
platforms, err := s.loadMonitoringClientPlatformIDs(ctx, actor.TenantID, workspaceID, *clientID)
if err != nil {
return cfg, err
}
cfg.EnabledPlatforms = platforms
return cfg, nil
}
// findLatestRegisteredClientWithMonitoringAccountForUser returns the newest
// non-revoked desktop client that has at least one bound monitoring platform
// account. Platform account health is not authorization.
func (s *MonitoringService) findLatestRegisteredClientWithMonitoringAccountForUser(ctx context.Context, tenantID, workspaceID, userID int64) (*uuid.UUID, error) {
var clientID uuid.UUID
err := s.businessPool.QueryRow(ctx, ` err := s.businessPool.QueryRow(ctx, `
SELECT collection_mode, enabled_platforms, primary_client_id SELECT dc.id
FROM tenant_monitoring_quotas FROM desktop_clients dc
WHERE tenant_id = $1 WHERE dc.tenant_id = $1
`, tenantID).Scan(&cfg.CollectionMode, &enabledJSON, &clientID) AND dc.workspace_id = $2
AND dc.user_id = $3
AND dc.revoked_at IS NULL
AND EXISTS (
SELECT 1
FROM platform_accounts pa
WHERE pa.tenant_id = dc.tenant_id
AND pa.workspace_id = dc.workspace_id
AND pa.client_id = dc.id
AND pa.deleted_at IS NULL
AND pa.platform_id = ANY($4::text[])
)
ORDER BY dc.created_at DESC, dc.id DESC
LIMIT 1
`, tenantID, workspaceID, userID, monitoringPlatformIDs(defaultMonitoringPlatforms)).Scan(&clientID)
if err != nil { if err != nil {
if err == pgx.ErrNoRows { if err == pgx.ErrNoRows {
return cfg, nil return nil, nil
} }
return cfg, response.ErrInternal(50041, "query_failed", "failed to load monitoring quota") return nil, response.ErrInternal(50041, "query_failed", "failed to inspect monitoring desktop client authorization")
}
return &clientID, nil
} }
if len(enabledJSON) > 0 { // findLatestRegisteredClientForUser returns the newest non-revoked desktop client
var enabled []string // registered by creation order. It intentionally ignores last_seen_at and health;
if jsonErr := json.Unmarshal(enabledJSON, &enabled); jsonErr == nil { // use findLatestOnlineClientForUser when online presence is required.
cfg.EnabledPlatforms = reconcileEnabledMonitoringPlatforms(enabled) func (s *MonitoringService) findLatestRegisteredClientForUser(ctx context.Context, tenantID, workspaceID, userID int64) (*uuid.UUID, error) {
var clientID uuid.UUID
err := s.businessPool.QueryRow(ctx, `
SELECT id
FROM desktop_clients
WHERE tenant_id = $1
AND workspace_id = $2
AND user_id = $3
AND revoked_at IS NULL
ORDER BY created_at DESC, id DESC
LIMIT 1
`, tenantID, workspaceID, userID).Scan(&clientID)
if err != nil {
if err == pgx.ErrNoRows {
return nil, nil
} }
return nil, response.ErrInternal(50041, "query_failed", "failed to inspect monitoring desktop client")
} }
if clientID.Valid { return &clientID, nil
id := clientID.UUID
cfg.PrimaryClientID = &id
} }
if strings.TrimSpace(cfg.CollectionMode) == "" {
cfg.CollectionMode = monitoringCollectorType func (s *MonitoringService) loadMonitoringClientPlatformIDs(ctx context.Context, tenantID, workspaceID int64, clientID uuid.UUID) ([]string, error) {
rows, err := s.businessPool.Query(ctx, `
SELECT DISTINCT platform_id
FROM platform_accounts
WHERE tenant_id = $1
AND workspace_id = $2
AND client_id = $3
AND deleted_at IS NULL
AND platform_id = ANY($4::text[])
`, tenantID, workspaceID, clientID, monitoringPlatformIDs(defaultMonitoringPlatforms))
if err != nil {
return nil, response.ErrInternal(50041, "query_failed", "failed to load monitoring desktop platform accounts")
} }
return cfg, nil defer rows.Close()
platformIDs := make([]string, 0)
for rows.Next() {
var platformID string
if scanErr := rows.Scan(&platformID); scanErr != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse monitoring desktop platform accounts")
}
platformIDs = append(platformIDs, platformID)
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate monitoring desktop platform accounts")
}
return reconcileEnabledMonitoringPlatforms(platformIDs), nil
} }
func newMonitoringDerivedMetrics() monitoringDerivedMetrics { func newMonitoringDerivedMetrics() monitoringDerivedMetrics {
@@ -2021,11 +2148,18 @@ func applyDerivedRatesToOverviewSnapshot(snapshot *monitoringOverviewSnapshot, d
} }
} }
func (s *MonitoringService) loadAccessStates(ctx context.Context, tenantID int64, clientID *uuid.UUID, businessDate time.Time) (map[string]monitoringAccessState, error) { func (s *MonitoringService) loadAccessStates(ctx context.Context, tenantID, workspaceID int64, clientID *uuid.UUID, businessDate time.Time) (map[string]monitoringAccessState, error) {
states := map[string]monitoringAccessState{} states := map[string]monitoringAccessState{}
if clientID == nil { if clientID == nil {
return states, nil return states, nil
} }
belongs, err := s.monitoringClientBelongsToWorkspace(ctx, tenantID, workspaceID, *clientID)
if err != nil {
return nil, err
}
if !belongs {
return states, nil
}
rows, err := s.monitoringPool.Query(ctx, ` rows, err := s.monitoringPool.Query(ctx, `
SELECT ai_platform_id, access_status SELECT ai_platform_id, access_status
@@ -2054,10 +2188,30 @@ func (s *MonitoringService) loadAccessStates(ctx context.Context, tenantID int64
} }
states[platformID] = monitoringAccessState{AccessStatus: accessStatus} states[platformID] = monitoringAccessState{AccessStatus: accessStatus}
} }
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate platform access states")
}
return states, nil return states, nil
} }
func (s *MonitoringService) monitoringClientBelongsToWorkspace(ctx context.Context, tenantID, workspaceID int64, clientID uuid.UUID) (bool, error) {
var exists bool
if err := s.businessPool.QueryRow(ctx, `
SELECT EXISTS (
SELECT 1
FROM desktop_clients
WHERE id = $1
AND tenant_id = $2
AND workspace_id = $3
AND revoked_at IS NULL
)
`, clientID, tenantID, workspaceID).Scan(&exists); err != nil {
return false, response.ErrInternal(50041, "query_failed", "failed to validate monitoring desktop client workspace")
}
return exists, nil
}
func (s *MonitoringService) loadPlatformBreakdown(ctx context.Context, tenantID, brandID int64, businessDate time.Time, collectionMode string, platforms []monitoringPlatformMetadata, accessStates map[string]monitoringAccessState, derived monitoringDerivedMetrics) ([]MonitoringPlatformDaily, error) { func (s *MonitoringService) loadPlatformBreakdown(ctx context.Context, tenantID, brandID int64, businessDate time.Time, collectionMode string, platforms []monitoringPlatformMetadata, accessStates map[string]monitoringAccessState, derived monitoringDerivedMetrics) ([]MonitoringPlatformDaily, error) {
rows, err := s.monitoringPool.Query(ctx, ` rows, err := s.monitoringPool.Query(ctx, `
SELECT SELECT
@@ -3117,7 +3271,7 @@ func normalizeTrackingDays(days int) int {
func resolveMonitoringPlatforms(enabled []string) []monitoringPlatformMetadata { func resolveMonitoringPlatforms(enabled []string) []monitoringPlatformMetadata {
if len(enabled) == 0 { if len(enabled) == 0 {
return defaultMonitoringPlatforms return nil
} }
platforms := make([]monitoringPlatformMetadata, 0, len(enabled)) platforms := make([]monitoringPlatformMetadata, 0, len(enabled))
seen := make(map[string]struct{}, len(enabled)) seen := make(map[string]struct{}, len(enabled))
@@ -3139,7 +3293,7 @@ func resolveMonitoringPlatforms(enabled []string) []monitoringPlatformMetadata {
}) })
} }
if len(platforms) == 0 { if len(platforms) == 0 {
return defaultMonitoringPlatforms return nil
} }
return platforms return platforms
} }
@@ -3192,12 +3346,7 @@ func filterMonitoringPlatforms(platforms []monitoringPlatformMetadata, aiPlatfor
} }
} }
return []monitoringPlatformMetadata{ return nil
{
ID: platformID,
Name: platformDisplayName(platformID),
},
}
} }
func platformDisplayName(platformID string) string { func platformDisplayName(platformID string) string {
@@ -4,6 +4,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@@ -81,7 +82,7 @@ func TestFilterMonitoringPlatformsReturnsSelectedPlatform(t *testing.T) {
}, filtered) }, filtered)
} }
func TestFilterMonitoringPlatformsFallsBackToDisplayName(t *testing.T) { func TestFilterMonitoringPlatformsDropsUnauthorizedSelection(t *testing.T) {
platforms := []monitoringPlatformMetadata{ platforms := []monitoringPlatformMetadata{
{ID: "deepseek", Name: "DeepSeek"}, {ID: "deepseek", Name: "DeepSeek"},
} }
@@ -89,9 +90,7 @@ func TestFilterMonitoringPlatformsFallsBackToDisplayName(t *testing.T) {
filtered := filterMonitoringPlatforms(platforms, &selected) filtered := filterMonitoringPlatforms(platforms, &selected)
assert.Equal(t, []monitoringPlatformMetadata{ assert.Empty(t, filtered)
{ID: "doubao", Name: "豆包"},
}, filtered)
} }
func TestNormalizeMonitoringPlatformIDCanonicalizesLatestIDs(t *testing.T) { func TestNormalizeMonitoringPlatformIDCanonicalizesLatestIDs(t *testing.T) {
@@ -127,8 +126,40 @@ func TestIntersectMonitoringPlatformsAcceptsCanonicalIDs(t *testing.T) {
}, filtered) }, filtered)
} }
func TestReconcileEnabledMonitoringPlatformsBackfillsSupportedCatalog(t *testing.T) { func TestReconcileEnabledMonitoringPlatformsKeepsAuthorizedSubset(t *testing.T) {
assert.Equal(t, []string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"}, reconcileEnabledMonitoringPlatforms([]string{"deepseek", "qwen", "doubao"})) assert.Equal(t, []string{"deepseek", "doubao", "qwen"}, reconcileEnabledMonitoringPlatforms([]string{"deepseek", "qwen", "doubao"}))
assert.Empty(t, reconcileEnabledMonitoringPlatforms(nil))
assert.Empty(t, resolveMonitoringPlatforms(nil))
}
func TestMonitoringPlatformAuthorizationStatus(t *testing.T) {
clientID := uuid.New()
assert.Equal(t, monitoringPlatformAuthorizationNoDesktopClient, monitoringPlatformAuthorizationStatus(monitoringQuotaConfig{}))
assert.Equal(t, monitoringPlatformAuthorizationNoAuthorizedPlatforms, monitoringPlatformAuthorizationStatus(monitoringQuotaConfig{
PrimaryClientID: &clientID,
}))
assert.Equal(t, monitoringPlatformAuthorizationAuthorized, monitoringPlatformAuthorizationStatus(monitoringQuotaConfig{
PrimaryClientID: &clientID,
EnabledPlatforms: []string{"qwen"},
}))
}
func TestDefaultMonitoringPlatformMetadataReturnsFullCatalogCopy(t *testing.T) {
platforms := defaultMonitoringPlatformMetadata()
require.Len(t, platforms, 6)
assert.Equal(t, []string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"}, monitoringPlatformIDs(platforms))
platforms[0].ID = "mutated"
assert.Equal(t, "yuanbao", defaultMonitoringPlatformMetadata()[0].ID)
}
func TestLoadMonitoringProjectionPlatformsReturnsFullCatalog(t *testing.T) {
platforms, err := loadMonitoringProjectionPlatforms(nil, nil, 0, 0, time.Time{})
require.NoError(t, err)
assert.Equal(t, []string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"}, monitoringPlatformIDs(platforms))
} }
func TestApplyDerivedRatesToOverviewSnapshotUsesAllSamplesForMentionRate(t *testing.T) { func TestApplyDerivedRatesToOverviewSnapshotUsesAllSamplesForMentionRate(t *testing.T) {
@@ -0,0 +1,61 @@
package transport
import (
"strings"
"github.com/gin-gonic/gin"
"github.com/geo-platform/tenant-api/internal/bootstrap"
"github.com/geo-platform/tenant-api/internal/shared/response"
tenantapp "github.com/geo-platform/tenant-api/internal/tenant/app"
)
func registerInternalMetricsRoutes(app *bootstrap.App) {
token := strings.TrimSpace(app.Config.Scheduler.InternalMetricsToken)
if token == "" {
app.Logger.Warn("tenant-api internal metrics routes not registered: scheduler.internal_metrics_token is required")
return
}
metrics := app.Engine.Group("/api/internal/metrics")
metrics.Use(internalMetricsAuthMiddleware(token))
metrics.GET("/monitoring/heartbeat-primary-lease", func(c *gin.Context) {
response.Success(c, tenantapp.MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue())
})
primaryLeasePrometheusHandler := tenantapp.MonitoringHeartbeatPrimaryLeaseMetricsPrometheusHandler()
metrics.GET("/monitoring/heartbeat-primary-lease/prometheus", func(c *gin.Context) {
primaryLeasePrometheusHandler.ServeHTTP(c.Writer, c.Request)
})
metrics.GET("/monitoring/heartbeat-snapshots", func(c *gin.Context) {
response.Success(c, tenantapp.MonitoringHeartbeatSnapshotMetricsSnapshotValue())
})
snapshotPrometheusHandler := tenantapp.MonitoringHeartbeatSnapshotMetricsPrometheusHandler()
metrics.GET("/monitoring/heartbeat-snapshots/prometheus", func(c *gin.Context) {
snapshotPrometheusHandler.ServeHTTP(c.Writer, c.Request)
})
}
func internalMetricsAuthMiddleware(token string) gin.HandlerFunc {
expected := strings.TrimSpace(token)
return func(c *gin.Context) {
if expected == "" || !internalMetricsTokenMatches(c, expected) {
response.Error(c, response.ErrUnauthorized(40181, "internal_metrics_unauthorized", "valid internal metrics token is required"))
c.Abort()
return
}
c.Next()
}
}
func internalMetricsTokenMatches(c *gin.Context, expected string) bool {
if c == nil {
return false
}
header := strings.TrimSpace(c.GetHeader("Authorization"))
if strings.HasPrefix(strings.ToLower(header), "bearer ") {
if strings.TrimSpace(header[len("bearer "):]) == expected {
return true
}
}
return strings.TrimSpace(c.GetHeader("X-Internal-Metrics-Token")) == expected
}
@@ -26,7 +26,7 @@ type monitoringCollectNowRequest struct {
func NewMonitoringHandler(a *bootstrap.App) *MonitoringHandler { func NewMonitoringHandler(a *bootstrap.App) *MonitoringHandler {
return &MonitoringHandler{ return &MonitoringHandler{
svc: app.NewMonitoringService(a.DB, a.MonitoringDB, a.RabbitMQ, a.Config.MonitoringDispatch, a.Logger), svc: app.NewMonitoringService(a.DB, a.MonitoringDB, a.RabbitMQ, a.Config.MonitoringDispatch, a.Config.BrandLibrary, a.Logger),
} }
} }
@@ -8,6 +8,8 @@ import (
) )
func RegisterRoutes(app *bootstrap.App) { func RegisterRoutes(app *bootstrap.App) {
registerInternalMetricsRoutes(app)
authAPI := app.Engine.Group("/api/auth") authAPI := app.Engine.Group("/api/auth")
authHandler := NewAuthHandler(app) authHandler := NewAuthHandler(app)
authAPI.POST("/login", authHandler.Login) authAPI.POST("/login", authHandler.Login)
@@ -21,6 +23,7 @@ func RegisterRoutes(app *bootstrap.App) {
protected := app.Engine.Group("/api") protected := app.Engine.Group("/api")
protected.Use(auth.Middleware(app.JWT, app.Sessions)) protected.Use(auth.Middleware(app.JWT, app.Sessions))
protected.Use(middleware.TenantScope()) protected.Use(middleware.TenantScope())
protected.Use(middleware.WorkspaceScope(app.DB))
protected.GET("/auth/me", authHandler.Me) protected.GET("/auth/me", authHandler.Me)
protected.POST("/auth/logout", authHandler.Logout) protected.POST("/auth/logout", authHandler.Logout)
@@ -0,0 +1,4 @@
DROP INDEX IF EXISTS idx_desktop_clients_tenant_workspace_user_seen_active;
DROP INDEX IF EXISTS idx_desktop_clients_tenant_workspace_seen_active;
DROP INDEX IF EXISTS idx_desktop_client_primary_leases_primary_client;
DROP TABLE IF EXISTS desktop_client_primary_leases;
@@ -0,0 +1,52 @@
-- Backfilled primary leases use the same 90s TTL as
-- server/internal/tenant/app/desktop_presence.go:desktopClientPresenceTTL.
-- Keep these values aligned; desktop heartbeat interval must be < TTL/2 so
-- a single missed heartbeat does not let another client steal primary.
CREATE TABLE IF NOT EXISTS desktop_client_primary_leases (
tenant_id BIGINT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
workspace_id BIGINT NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE,
primary_client_id UUID NOT NULL REFERENCES desktop_clients(id) ON DELETE CASCADE,
lease_expires_at TIMESTAMPTZ NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (tenant_id, workspace_id)
);
CREATE INDEX IF NOT EXISTS idx_desktop_client_primary_leases_primary_client
ON desktop_client_primary_leases (primary_client_id);
CREATE INDEX IF NOT EXISTS idx_desktop_clients_tenant_workspace_seen_active
ON desktop_clients (tenant_id, workspace_id, last_seen_at DESC, created_at DESC, id DESC)
WHERE revoked_at IS NULL;
CREATE INDEX IF NOT EXISTS idx_desktop_clients_tenant_workspace_user_seen_active
ON desktop_clients (tenant_id, workspace_id, user_id, last_seen_at DESC, created_at DESC, id DESC)
WHERE revoked_at IS NULL;
INSERT INTO desktop_client_primary_leases (
tenant_id,
workspace_id,
primary_client_id,
lease_expires_at
)
SELECT
tenant_id,
workspace_id,
id,
COALESCE(last_seen_at, created_at) + INTERVAL '90 seconds'
FROM (
SELECT
dc.id,
dc.tenant_id,
dc.workspace_id,
dc.created_at,
dc.last_seen_at,
ROW_NUMBER() OVER (
PARTITION BY dc.tenant_id, dc.workspace_id
ORDER BY dc.last_seen_at DESC NULLS LAST, dc.created_at DESC, dc.id DESC
) AS rn
FROM desktop_clients dc
WHERE dc.revoked_at IS NULL
) ranked_clients
WHERE rn = 1
ON CONFLICT (tenant_id, workspace_id) DO NOTHING;
@@ -0,0 +1 @@
DROP INDEX IF EXISTS idx_collect_tasks_pending_platform_reconcile;
@@ -0,0 +1,4 @@
CREATE INDEX IF NOT EXISTS idx_collect_tasks_pending_platform_reconcile
ON monitoring_collect_tasks (tenant_id, business_date, ai_platform_id)
WHERE collector_type = 'desktop'
AND status = 'pending';