feat(monitoring): generate daily tasks and optimize heartbeat writes
Adds a MonitoringDailyTaskWorker that materializes and dispatches daily collect tasks from active subscription/desktop-client state, with per-run atomic metrics and a Prometheus collector. Dashboard and question-detail APIs now surface a platform_authorization_status so the UI can distinguish no-desktop-client, no-authorized-platforms, and authorized states; quota/access-state lookups resolve by request workspace instead of tenant. Hardens heartbeat: a desktop_client_primary_leases table gives per-(tenant, workspace) sticky primary selection, read-mostly lease resolution avoids a per-heartbeat transaction, and unchanged platform access reports skip snapshot upserts outside a 10-minute refresh window. Adds heartbeat primary/snapshot metrics exposed via token-protected tenant-api /api/internal/metrics endpoints plus Prometheus. Monitoring quota seed is removed from dev-seed since daily plans now derive from desktop bindings, and the projection uses the canonical six-platform catalog so platforms with zero samples still render. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -246,10 +246,6 @@ func seedBusinessData(ctx context.Context, pool *pgxpool.Pool, membershipCfg con
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := ensureMonitoringQuota(ctx, tx, state); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := tx.Commit(ctx); err != nil {
|
if err := tx.Commit(ctx); err != nil {
|
||||||
return nil, fmt.Errorf("commit business seed: %w", err)
|
return nil, fmt.Errorf("commit business seed: %w", err)
|
||||||
}
|
}
|
||||||
@@ -692,28 +688,6 @@ func ensureDesktopClient(ctx context.Context, tx pgx.Tx, tenantID, workspaceID,
|
|||||||
return ensuredID, nil
|
return ensuredID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func ensureMonitoringQuota(ctx context.Context, tx pgx.Tx, state *seedState) error {
|
|
||||||
enabledPlatforms, _ := json.Marshal([]string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"})
|
|
||||||
_, err := tx.Exec(ctx, `
|
|
||||||
INSERT INTO tenant_monitoring_quotas (
|
|
||||||
tenant_id, workspace_id, max_brands, collection_mode, question_selection_mode, collect_frequency,
|
|
||||||
enabled_platforms, primary_client_id, task_daily_hard_cap, plan_tier
|
|
||||||
)
|
|
||||||
VALUES ($1, $2, 1, 'desktop', 'best_effort_all', 'daily', $3::jsonb, $4, 36, 'free')
|
|
||||||
ON CONFLICT (tenant_id) DO UPDATE
|
|
||||||
SET workspace_id = EXCLUDED.workspace_id,
|
|
||||||
collection_mode = EXCLUDED.collection_mode,
|
|
||||||
question_selection_mode = EXCLUDED.question_selection_mode,
|
|
||||||
collect_frequency = EXCLUDED.collect_frequency,
|
|
||||||
enabled_platforms = EXCLUDED.enabled_platforms,
|
|
||||||
primary_client_id = EXCLUDED.primary_client_id,
|
|
||||||
task_daily_hard_cap = EXCLUDED.task_daily_hard_cap,
|
|
||||||
plan_tier = EXCLUDED.plan_tier,
|
|
||||||
updated_at = NOW()
|
|
||||||
`, state.TenantID, state.WorkspaceID, string(enabledPlatforms), state.DesktopClientID)
|
|
||||||
return wrapSeedErr("upsert monitoring quota", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func clearMonitoringData(ctx context.Context, tx pgx.Tx, state *seedState) error {
|
func clearMonitoringData(ctx context.Context, tx pgx.Tx, state *seedState) error {
|
||||||
pairs := []string{
|
pairs := []string{
|
||||||
`DELETE FROM monitoring_citation_facts WHERE tenant_id = $1 AND brand_id = $2`,
|
`DELETE FROM monitoring_citation_facts WHERE tenant_id = $1 AND brand_id = $2`,
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ func main() {
|
|||||||
app.DesktopTaskStreams.Run(workerCtx)
|
app.DesktopTaskStreams.Run(workerCtx)
|
||||||
app.DesktopDispatch.Run(workerCtx)
|
app.DesktopDispatch.Run(workerCtx)
|
||||||
|
|
||||||
monitoringService := tenantapp.NewMonitoringService(app.DB, app.MonitoringDB, app.RabbitMQ, app.Config.MonitoringDispatch, app.Logger)
|
monitoringService := tenantapp.NewMonitoringService(app.DB, app.MonitoringDB, app.RabbitMQ, app.Config.MonitoringDispatch, app.Config.BrandLibrary, app.Logger)
|
||||||
monitoringCallbackService := tenantapp.NewMonitoringCallbackService(app.DB, app.MonitoringDB, app.RabbitMQ, app.LLM, app.Logger)
|
monitoringCallbackService := tenantapp.NewMonitoringCallbackService(app.DB, app.MonitoringDB, app.RabbitMQ, app.LLM, app.Logger)
|
||||||
tenantapp.NewMonitoringCollectOutboxWorker(monitoringService, app.Logger).Start(workerCtx)
|
tenantapp.NewMonitoringCollectOutboxWorker(monitoringService, app.Logger).Start(workerCtx)
|
||||||
tenantapp.NewMonitoringResultIngestWorker(monitoringCallbackService, app.RabbitMQ, app.Logger, app.Config.MonitoringWorkers).Start(workerCtx)
|
tenantapp.NewMonitoringResultIngestWorker(monitoringCallbackService, app.RabbitMQ, app.Logger, app.Config.MonitoringWorkers).Start(workerCtx)
|
||||||
|
|||||||
@@ -14,6 +14,8 @@ require (
|
|||||||
github.com/jackc/pgx/v5 v5.5.5
|
github.com/jackc/pgx/v5 v5.5.5
|
||||||
github.com/ledongthuc/pdf v0.0.0-20240314090751-a2a84ec735c3
|
github.com/ledongthuc/pdf v0.0.0-20240314090751-a2a84ec735c3
|
||||||
github.com/minio/minio-go/v7 v7.0.83
|
github.com/minio/minio-go/v7 v7.0.83
|
||||||
|
github.com/prometheus/client_golang v1.20.5
|
||||||
|
github.com/prometheus/common v0.60.1
|
||||||
github.com/qdrant/go-client v1.13.0
|
github.com/qdrant/go-client v1.13.0
|
||||||
github.com/rabbitmq/amqp091-go v1.10.0
|
github.com/rabbitmq/amqp091-go v1.10.0
|
||||||
github.com/redis/go-redis/v9 v9.5.1
|
github.com/redis/go-redis/v9 v9.5.1
|
||||||
@@ -32,6 +34,7 @@ require (
|
|||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/beorn7/perks v1.0.1 // indirect
|
||||||
github.com/bytedance/sonic v1.9.1 // indirect
|
github.com/bytedance/sonic v1.9.1 // indirect
|
||||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
|
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
|
||||||
@@ -63,8 +66,11 @@ require (
|
|||||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
|
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
|
||||||
|
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||||
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
|
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||||
|
github.com/prometheus/client_model v0.6.1 // indirect
|
||||||
|
github.com/prometheus/procfs v0.15.1 // indirect
|
||||||
github.com/richardlehane/mscfb v1.0.4 // indirect
|
github.com/richardlehane/mscfb v1.0.4 // indirect
|
||||||
github.com/richardlehane/msoleps v1.0.3 // indirect
|
github.com/richardlehane/msoleps v1.0.3 // indirect
|
||||||
github.com/rs/xid v1.6.0 // indirect
|
github.com/rs/xid v1.6.0 // indirect
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ github.com/alicebob/miniredis/v2 v2.37.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj
|
|||||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g=
|
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g=
|
||||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
|
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
|
||||||
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
|
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
|
||||||
|
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||||
|
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||||
@@ -115,6 +117,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
|||||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||||
|
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||||
|
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||||
github.com/ledongthuc/pdf v0.0.0-20240314090751-a2a84ec735c3 h1:DDPccSesc/SJu8CiCEhHhWjK9C9U1fhrRKB3jRNw4f4=
|
github.com/ledongthuc/pdf v0.0.0-20240314090751-a2a84ec735c3 h1:DDPccSesc/SJu8CiCEhHhWjK9C9U1fhrRKB3jRNw4f4=
|
||||||
github.com/ledongthuc/pdf v0.0.0-20240314090751-a2a84ec735c3/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
|
github.com/ledongthuc/pdf v0.0.0-20240314090751-a2a84ec735c3/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
|
||||||
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
|
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
|
||||||
@@ -136,13 +140,23 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
|
|||||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
|
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
|
||||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
|
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
|
||||||
|
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||||
|
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||||
github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
|
github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
|
||||||
github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
|
github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
|
||||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
|
||||||
|
github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
|
||||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||||
|
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
|
||||||
|
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
|
||||||
|
github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc=
|
||||||
|
github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw=
|
||||||
|
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
|
||||||
|
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
|
||||||
github.com/qdrant/go-client v1.13.0 h1:qeWKCs1vxvfF2MLLFnP2qDG0R8wI18HyAoSfc7wJim8=
|
github.com/qdrant/go-client v1.13.0 h1:qeWKCs1vxvfF2MLLFnP2qDG0R8wI18HyAoSfc7wJim8=
|
||||||
github.com/qdrant/go-client v1.13.0/go.mod h1:iO8ts78jL4x6LDHFOViyYWELVtIBDTjOykBmiOTHLnQ=
|
github.com/qdrant/go-client v1.13.0/go.mod h1:iO8ts78jL4x6LDHFOViyYWELVtIBDTjOykBmiOTHLnQ=
|
||||||
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
|
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
|
||||||
|
|||||||
@@ -105,6 +105,9 @@ type RabbitMQConfig struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type SchedulerConfig struct {
|
type SchedulerConfig struct {
|
||||||
|
HTTPHost string `mapstructure:"http_host"`
|
||||||
|
HTTPPort int `mapstructure:"http_port"`
|
||||||
|
InternalMetricsToken string `mapstructure:"internal_metrics_token"`
|
||||||
DispatchInterval time.Duration `mapstructure:"dispatch_interval"`
|
DispatchInterval time.Duration `mapstructure:"dispatch_interval"`
|
||||||
DispatchTimeout time.Duration `mapstructure:"dispatch_timeout"`
|
DispatchTimeout time.Duration `mapstructure:"dispatch_timeout"`
|
||||||
DispatchBatchSize int `mapstructure:"dispatch_batch_size"`
|
DispatchBatchSize int `mapstructure:"dispatch_batch_size"`
|
||||||
@@ -427,6 +430,12 @@ func applyEnvOverrides(cfg *Config) {
|
|||||||
if model, ok := lookupFirstNonEmptyEnv("SILICONFLOW_RERANKER_MODEL", "RETRIEVAL_RERANKER_MODEL"); ok {
|
if model, ok := lookupFirstNonEmptyEnv("SILICONFLOW_RERANKER_MODEL", "RETRIEVAL_RERANKER_MODEL"); ok {
|
||||||
cfg.Retrieval.RerankerModel = model
|
cfg.Retrieval.RerankerModel = model
|
||||||
}
|
}
|
||||||
|
if host, ok := lookupNonEmptyEnv("SCHEDULER_HTTP_HOST"); ok {
|
||||||
|
cfg.Scheduler.HTTPHost = host
|
||||||
|
}
|
||||||
|
if token, ok := lookupFirstNonEmptyEnv("SCHEDULER_INTERNAL_METRICS_TOKEN", "SCHEDULER_METRICS_TOKEN"); ok {
|
||||||
|
cfg.Scheduler.InternalMetricsToken = token
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeRabbitMQConfig(cfg *RabbitMQConfig) {
|
func normalizeRabbitMQConfig(cfg *RabbitMQConfig) {
|
||||||
@@ -569,6 +578,14 @@ func normalizeSchedulerConfig(cfg *SchedulerConfig) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if cfg.HTTPPort == 0 {
|
||||||
|
cfg.HTTPPort = 8081
|
||||||
|
}
|
||||||
|
cfg.HTTPHost = strings.TrimSpace(cfg.HTTPHost)
|
||||||
|
if cfg.HTTPHost == "" {
|
||||||
|
cfg.HTTPHost = "127.0.0.1"
|
||||||
|
}
|
||||||
|
cfg.InternalMetricsToken = strings.TrimSpace(cfg.InternalMetricsToken)
|
||||||
if cfg.DispatchInterval <= 0 {
|
if cfg.DispatchInterval <= 0 {
|
||||||
cfg.DispatchInterval = 15 * time.Second
|
cfg.DispatchInterval = 15 * time.Second
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -114,6 +114,58 @@ brand_library: {}
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadAppliesSchedulerHTTPDefaultsAndDisableSentinel(t *testing.T) {
|
||||||
|
defaultConfigPath := writeTestConfig(t, `
|
||||||
|
scheduler: {}
|
||||||
|
`)
|
||||||
|
|
||||||
|
cfg, err := Load(defaultConfigPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() default error = %v", err)
|
||||||
|
}
|
||||||
|
if cfg.Scheduler.HTTPHost != "127.0.0.1" {
|
||||||
|
t.Fatalf("expected default scheduler http host 127.0.0.1, got %q", cfg.Scheduler.HTTPHost)
|
||||||
|
}
|
||||||
|
if cfg.Scheduler.HTTPPort != 8081 {
|
||||||
|
t.Fatalf("expected default scheduler http port 8081, got %d", cfg.Scheduler.HTTPPort)
|
||||||
|
}
|
||||||
|
|
||||||
|
disabledConfigPath := writeTestConfig(t, `
|
||||||
|
scheduler:
|
||||||
|
http_port: -1
|
||||||
|
`)
|
||||||
|
|
||||||
|
cfg, err = Load(disabledConfigPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() disabled error = %v", err)
|
||||||
|
}
|
||||||
|
if cfg.Scheduler.HTTPPort != -1 {
|
||||||
|
t.Fatalf("expected scheduler http port -1 to disable server, got %d", cfg.Scheduler.HTTPPort)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadPrefersEnvSchedulerMetricsSettings(t *testing.T) {
|
||||||
|
t.Setenv("SCHEDULER_HTTP_HOST", "0.0.0.0")
|
||||||
|
t.Setenv("SCHEDULER_INTERNAL_METRICS_TOKEN", "env-metrics-token")
|
||||||
|
|
||||||
|
configPath := writeTestConfig(t, `
|
||||||
|
scheduler:
|
||||||
|
http_host: 127.0.0.1
|
||||||
|
internal_metrics_token: config-token
|
||||||
|
`)
|
||||||
|
|
||||||
|
cfg, err := Load(configPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load() error = %v", err)
|
||||||
|
}
|
||||||
|
if cfg.Scheduler.HTTPHost != "0.0.0.0" {
|
||||||
|
t.Fatalf("expected env scheduler http host, got %q", cfg.Scheduler.HTTPHost)
|
||||||
|
}
|
||||||
|
if cfg.Scheduler.InternalMetricsToken != "env-metrics-token" {
|
||||||
|
t.Fatalf("expected env scheduler metrics token, got %q", cfg.Scheduler.InternalMetricsToken)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadAppliesMembershipDefaults(t *testing.T) {
|
func TestLoadAppliesMembershipDefaults(t *testing.T) {
|
||||||
configPath := writeTestConfig(t, `
|
configPath := writeTestConfig(t, `
|
||||||
membership: {}
|
membership: {}
|
||||||
|
|||||||
@@ -9,6 +9,9 @@ import (
|
|||||||
goredis "github.com/redis/go-redis/v9"
|
goredis "github.com/redis/go-redis/v9"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// desktopClientPresenceTTL is also used as the monitoring primary-client lease TTL.
|
||||||
|
// Desktop heartbeat interval must stay below TTL/2, so one missed heartbeat does
|
||||||
|
// not make another client eligible to steal a still-healthy primary lease.
|
||||||
const desktopClientPresenceTTL = 90 * time.Second
|
const desktopClientPresenceTTL = 90 * time.Second
|
||||||
const desktopAccountPresenceTTL = 90 * time.Second
|
const desktopAccountPresenceTTL = 90 * time.Second
|
||||||
|
|
||||||
|
|||||||
@@ -30,6 +30,14 @@ const (
|
|||||||
maxMonitoringLeaseLimit = 50
|
maxMonitoringLeaseLimit = 50
|
||||||
monitoringLeaseTTL = 15 * time.Minute
|
monitoringLeaseTTL = 15 * time.Minute
|
||||||
monitoringTaskResultQueueIOTimeout = 5 * time.Second
|
monitoringTaskResultQueueIOTimeout = 5 * time.Second
|
||||||
|
// The current primary renews the DB lease only after this interval. With the
|
||||||
|
// desktop client's 25s heartbeat and a 90s lease TTL, this keeps normal
|
||||||
|
// heartbeats read-only while still tolerating one missed heartbeat.
|
||||||
|
monitoringHeartbeatPrimaryRenewAfter = desktopClientPresenceTTL / 3
|
||||||
|
// Access snapshots are semantic state, not a liveness clock. Stable heartbeat
|
||||||
|
// reports skip writes and only refresh the row at this interval so dashboards
|
||||||
|
// do not depend on per-heartbeat database churn.
|
||||||
|
monitoringHeartbeatAccessSnapshotRefreshAfter = 10 * time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
type MonitoringCallbackService struct {
|
type MonitoringCallbackService struct {
|
||||||
@@ -205,6 +213,16 @@ type monitoringPlatformAccessSnapshotState struct {
|
|||||||
Accessible bool
|
Accessible bool
|
||||||
ReasonCode *string
|
ReasonCode *string
|
||||||
ReasonMessage *string
|
ReasonMessage *string
|
||||||
|
UpdatedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
type monitoringHeartbeatSnapshotDecision struct {
|
||||||
|
WriteSnapshot bool
|
||||||
|
StateChanged bool
|
||||||
|
RefreshDue bool
|
||||||
|
PendingReconcileCandidate bool
|
||||||
|
ReconcilePending bool
|
||||||
|
SnapshotOperation string
|
||||||
}
|
}
|
||||||
|
|
||||||
type monitoringCollectTask struct {
|
type monitoringCollectTask struct {
|
||||||
@@ -434,7 +452,18 @@ func (s *MonitoringCallbackService) handleTaskResultForInstallation(
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationID string, installationToken string, req MonitoringHeartbeatRequest) (*MonitoringHeartbeatResponse, error) {
|
func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationID string, installationToken string, req MonitoringHeartbeatRequest) (_ *MonitoringHeartbeatResponse, err error) {
|
||||||
|
startedAt := time.Now()
|
||||||
|
snapshotMetrics := monitoringHeartbeatSnapshotMetricsEvent{}
|
||||||
|
defer func() {
|
||||||
|
if !snapshotMetrics.Active {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
snapshotMetrics.Duration = time.Since(startedAt)
|
||||||
|
snapshotMetrics.Err = err
|
||||||
|
recordMonitoringHeartbeatSnapshotMetrics(snapshotMetrics)
|
||||||
|
}()
|
||||||
|
|
||||||
installation, err := s.authenticateInstallation(ctx, installationID, installationToken)
|
installation, err := s.authenticateInstallation(ctx, installationID, installationToken)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -451,28 +480,111 @@ func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationI
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
snapshotMetrics.Active = true
|
||||||
|
snapshotMetrics.PlatformCount = len(platforms)
|
||||||
|
|
||||||
|
if len(platforms) == 0 {
|
||||||
|
_ = s.touchInstallation(ctx, installation.ID)
|
||||||
|
return &MonitoringHeartbeatResponse{
|
||||||
|
InstallationID: installation.ID,
|
||||||
|
BusinessDate: businessDate,
|
||||||
|
UpdatedPlatformCount: 0,
|
||||||
|
ReconciledSkippedTaskCount: 0,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
platformIDs := monitoringHeartbeatPlatformIDs(platforms)
|
||||||
|
previousStates, err := s.loadPlatformAccessSnapshotStates(ctx, installation.TenantID, installation.ID, businessDate, platformIDs)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
type heartbeatPlatformAction struct {
|
||||||
|
platform MonitoringHeartbeatPlatform
|
||||||
|
decision monitoringHeartbeatSnapshotDecision
|
||||||
|
}
|
||||||
|
|
||||||
|
actions := make([]heartbeatPlatformAction, 0, len(platforms))
|
||||||
|
inaccessiblePlatformIDs := make([]string, 0, len(platforms))
|
||||||
|
needsTransaction := false
|
||||||
|
for _, platform := range platforms {
|
||||||
|
nextState := monitoringPlatformAccessSnapshotStateFromHeartbeat(platform)
|
||||||
|
decision := decideMonitoringHeartbeatSnapshot(previousStates[platform.AIPlatformID], nextState, now, isPrimaryInstallation)
|
||||||
|
if decision.WriteSnapshot {
|
||||||
|
needsTransaction = true
|
||||||
|
} else {
|
||||||
|
snapshotMetrics.SnapshotSkippedCount++
|
||||||
|
}
|
||||||
|
if decision.PendingReconcileCandidate {
|
||||||
|
inaccessiblePlatformIDs = append(inaccessiblePlatformIDs, platform.AIPlatformID)
|
||||||
|
}
|
||||||
|
actions = append(actions, heartbeatPlatformAction{
|
||||||
|
platform: platform,
|
||||||
|
decision: decision,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if isPrimaryInstallation && len(inaccessiblePlatformIDs) > 0 {
|
||||||
|
pendingPlatforms, pendingErr := s.loadPendingMonitoringTaskPlatformIDs(ctx, installation.TenantID, businessDate, inaccessiblePlatformIDs)
|
||||||
|
if pendingErr != nil {
|
||||||
|
return nil, pendingErr
|
||||||
|
}
|
||||||
|
for idx := range actions {
|
||||||
|
if !actions[idx].decision.PendingReconcileCandidate {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := pendingPlatforms[actions[idx].platform.AIPlatformID]; ok {
|
||||||
|
actions[idx].decision.ReconcilePending = true
|
||||||
|
needsTransaction = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !needsTransaction {
|
||||||
|
_ = s.touchInstallation(ctx, installation.ID)
|
||||||
|
return &MonitoringHeartbeatResponse{
|
||||||
|
InstallationID: installation.ID,
|
||||||
|
BusinessDate: businessDate,
|
||||||
|
UpdatedPlatformCount: len(platforms),
|
||||||
|
ReconciledSkippedTaskCount: 0,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
tx, err := s.monitoringPool.Begin(ctx)
|
tx, err := s.monitoringPool.Begin(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, response.ErrInternal(50041, "begin_failed", "failed to start monitoring heartbeat")
|
return nil, response.ErrInternal(50041, "begin_failed", "failed to start monitoring heartbeat")
|
||||||
}
|
}
|
||||||
|
snapshotMetrics.TransactionOpened = true
|
||||||
defer tx.Rollback(ctx)
|
defer tx.Rollback(ctx)
|
||||||
|
|
||||||
var reconciledSkippedTaskCount int64
|
var reconciledSkippedTaskCount int64
|
||||||
|
var snapshotInsertedCount int64
|
||||||
|
var snapshotUpdatedCount int64
|
||||||
|
var snapshotRefreshedCount int64
|
||||||
|
var projectionRebuildCount int64
|
||||||
affectedBrandIDs := make(map[int64]struct{})
|
affectedBrandIDs := make(map[int64]struct{})
|
||||||
for _, platform := range platforms {
|
for _, action := range actions {
|
||||||
accessChanged, err := s.upsertPlatformAccessSnapshot(ctx, tx, installation, businessDate, platform)
|
if action.decision.WriteSnapshot {
|
||||||
if err != nil {
|
if err := s.upsertPlatformAccessSnapshot(ctx, tx, installation, businessDate, action.platform); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
switch action.decision.SnapshotOperation {
|
||||||
|
case "insert":
|
||||||
|
snapshotInsertedCount++
|
||||||
|
case "update":
|
||||||
|
snapshotUpdatedCount++
|
||||||
|
case "refresh":
|
||||||
|
snapshotRefreshedCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if !isPrimaryInstallation {
|
if !isPrimaryInstallation {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
needsProjectionRefresh := accessChanged
|
needsProjectionRefresh := action.decision.StateChanged
|
||||||
if monitoringStringValue(platform.AccessStatus) != "accessible" {
|
if action.decision.ReconcilePending {
|
||||||
count, reconcileErr := s.reconcilePendingTasksForPlatform(ctx, tx, installation.TenantID, businessDate, platform.AIPlatformID)
|
count, reconcileErr := s.reconcilePendingTasksForPlatform(ctx, tx, installation.TenantID, businessDate, action.platform.AIPlatformID)
|
||||||
if reconcileErr != nil {
|
if reconcileErr != nil {
|
||||||
return nil, reconcileErr
|
return nil, reconcileErr
|
||||||
}
|
}
|
||||||
@@ -486,7 +598,7 @@ func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationI
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
brandIDs, loadErr := s.loadMonitoringAffectedBrandIDs(ctx, tx, installation.TenantID, businessDay, platform.AIPlatformID)
|
brandIDs, loadErr := s.loadMonitoringAffectedBrandIDs(ctx, tx, installation.TenantID, businessDay, action.platform.AIPlatformID)
|
||||||
if loadErr != nil {
|
if loadErr != nil {
|
||||||
return nil, loadErr
|
return nil, loadErr
|
||||||
}
|
}
|
||||||
@@ -500,6 +612,7 @@ func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationI
|
|||||||
if err := s.rebuildBrandDailyProjection(ctx, tx, installation.TenantID, brandID, businessDay); err != nil {
|
if err := s.rebuildBrandDailyProjection(ctx, tx, installation.TenantID, brandID, businessDay); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
projectionRebuildCount++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -507,6 +620,12 @@ func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationI
|
|||||||
return nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring heartbeat")
|
return nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring heartbeat")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
snapshotMetrics.SnapshotInsertedCount = snapshotInsertedCount
|
||||||
|
snapshotMetrics.SnapshotUpdatedCount = snapshotUpdatedCount
|
||||||
|
snapshotMetrics.SnapshotRefreshedCount = snapshotRefreshedCount
|
||||||
|
snapshotMetrics.ReconciledSkippedTaskCount = reconciledSkippedTaskCount
|
||||||
|
snapshotMetrics.ProjectionRebuildCount = projectionRebuildCount
|
||||||
|
|
||||||
_ = s.touchInstallation(ctx, installation.ID)
|
_ = s.touchInstallation(ctx, installation.ID)
|
||||||
if isPrimaryInstallation && len(affectedBrandIDs) > 0 {
|
if isPrimaryInstallation && len(affectedBrandIDs) > 0 {
|
||||||
s.dispatchMonitoringProjectionRebuilds(installation.TenantID, businessDay, affectedBrandIDs, "heartbeat")
|
s.dispatchMonitoringProjectionRebuilds(installation.TenantID, businessDay, affectedBrandIDs, "heartbeat")
|
||||||
@@ -963,80 +1082,224 @@ func (s *MonitoringCallbackService) authenticateInstallation(ctx context.Context
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *MonitoringCallbackService) resolveHeartbeatPrimaryInstallation(ctx context.Context, tenantID int64, installation *monitoringInstallationIdentity) (bool, error) {
|
func (s *MonitoringCallbackService) resolveHeartbeatPrimaryInstallation(ctx context.Context, tenantID int64, installation *monitoringInstallationIdentity) (bool, error) {
|
||||||
var preferred sql.NullString
|
if installation == nil || strings.TrimSpace(installation.ID) == "" {
|
||||||
err := s.businessPool.QueryRow(ctx, `
|
|
||||||
SELECT primary_client_id::text
|
|
||||||
FROM tenant_monitoring_quotas
|
|
||||||
WHERE tenant_id = $1
|
|
||||||
`, tenantID).Scan(&preferred)
|
|
||||||
if err != nil && err != pgx.ErrNoRows {
|
|
||||||
return false, response.ErrInternal(50041, "query_failed", "failed to inspect primary monitoring desktop client")
|
|
||||||
}
|
|
||||||
|
|
||||||
if !preferred.Valid || preferred.String == installation.ID {
|
|
||||||
if !preferred.Valid {
|
|
||||||
if err := s.persistHeartbeatPrimaryInstallationID(ctx, tenantID, installation); err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
online, err := s.isMonitoringInstallationRecentlyOnline(ctx, tenantID, installation.WorkspaceID, preferred.String)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
if online {
|
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
|
isPrimary, err := s.claimHeartbeatPrimaryLease(ctx, tenantID, installation.WorkspaceID, installation.ID)
|
||||||
if err := s.persistHeartbeatPrimaryInstallationID(ctx, tenantID, installation); err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
return true, nil
|
return isPrimary, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MonitoringCallbackService) isMonitoringInstallationRecentlyOnline(ctx context.Context, tenantID, workspaceID int64, installationID string) (bool, error) {
|
func (s *MonitoringCallbackService) claimHeartbeatPrimaryLease(ctx context.Context, tenantID, workspaceID int64, installationID string) (bool, error) {
|
||||||
var lastSeenAt sql.NullTime
|
startedAt := time.Now()
|
||||||
err := s.businessPool.QueryRow(ctx, `
|
installationID = strings.TrimSpace(installationID)
|
||||||
SELECT last_seen_at
|
leaseTTL := formatPgInterval(desktopClientPresenceTTL)
|
||||||
FROM desktop_clients
|
|
||||||
WHERE id = $1::uuid
|
state, err := s.loadHeartbeatPrimaryLeaseState(ctx, tenantID, workspaceID, installationID)
|
||||||
AND tenant_id = $2
|
|
||||||
AND workspace_id = $3
|
|
||||||
AND revoked_at IS NULL
|
|
||||||
`, strings.TrimSpace(installationID), tenantID, workspaceID).Scan(&lastSeenAt)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
recordHeartbeatPrimaryLeaseMetric("error", time.Since(startedAt))
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if !state.Found {
|
||||||
|
inserted, insertErr := s.tryInsertHeartbeatPrimaryLease(ctx, tenantID, workspaceID, installationID, leaseTTL)
|
||||||
|
if insertErr != nil {
|
||||||
|
recordHeartbeatPrimaryLeaseMetric("error", time.Since(startedAt))
|
||||||
|
return false, insertErr
|
||||||
|
}
|
||||||
|
if inserted {
|
||||||
|
recordHeartbeatPrimaryLeaseMetric("inserted", time.Since(startedAt))
|
||||||
|
recordHeartbeatPrimaryLeaseWriteMetric("insert")
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
recordHeartbeatPrimaryLeaseMetric("contention", time.Since(startedAt))
|
||||||
|
return s.resolveHeartbeatPrimaryAfterContention(ctx, tenantID, workspaceID, installationID)
|
||||||
|
}
|
||||||
|
|
||||||
|
decision := decideHeartbeatPrimaryLease(state)
|
||||||
|
switch decision.Action {
|
||||||
|
case heartbeatPrimaryLeaseActionPrimaryHit:
|
||||||
|
recordHeartbeatPrimaryLeaseMetric("primary_hit", time.Since(startedAt))
|
||||||
|
return true, nil
|
||||||
|
case heartbeatPrimaryLeaseActionNonPrimaryLive:
|
||||||
|
recordHeartbeatPrimaryLeaseMetric("non_primary_live", time.Since(startedAt))
|
||||||
|
return false, nil
|
||||||
|
case heartbeatPrimaryLeaseActionRenew:
|
||||||
|
updated, updateErr := s.renewHeartbeatPrimaryLease(ctx, tenantID, workspaceID, installationID, leaseTTL)
|
||||||
|
if updateErr != nil {
|
||||||
|
recordHeartbeatPrimaryLeaseMetric("error", time.Since(startedAt))
|
||||||
|
return false, updateErr
|
||||||
|
}
|
||||||
|
if updated {
|
||||||
|
recordHeartbeatPrimaryLeaseMetric("renewed", time.Since(startedAt))
|
||||||
|
recordHeartbeatPrimaryLeaseWriteMetric("renew")
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
case heartbeatPrimaryLeaseActionClaimExpired, heartbeatPrimaryLeaseActionClaimUnavailable:
|
||||||
|
updated, updateErr := s.claimExpiredHeartbeatPrimaryLease(ctx, tenantID, workspaceID, installationID, state.PrimaryInstallationID, leaseTTL, decision.Action)
|
||||||
|
if updateErr != nil {
|
||||||
|
recordHeartbeatPrimaryLeaseMetric("error", time.Since(startedAt))
|
||||||
|
return false, updateErr
|
||||||
|
}
|
||||||
|
if updated {
|
||||||
|
if decision.Action == heartbeatPrimaryLeaseActionClaimExpired {
|
||||||
|
recordHeartbeatPrimaryLeaseMetric("claimed_expired", time.Since(startedAt))
|
||||||
|
} else {
|
||||||
|
recordHeartbeatPrimaryLeaseMetric("claimed_unavailable", time.Since(startedAt))
|
||||||
|
}
|
||||||
|
recordHeartbeatPrimaryLeaseWriteMetric("claim")
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
recordHeartbeatPrimaryLeaseMetric("contention", time.Since(startedAt))
|
||||||
|
return s.resolveHeartbeatPrimaryAfterContention(ctx, tenantID, workspaceID, installationID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *MonitoringCallbackService) tryInsertHeartbeatPrimaryLease(ctx context.Context, tenantID, workspaceID int64, installationID, leaseTTL string) (bool, error) {
|
||||||
|
tag, err := s.businessPool.Exec(ctx, `
|
||||||
|
INSERT INTO desktop_client_primary_leases (
|
||||||
|
tenant_id,
|
||||||
|
workspace_id,
|
||||||
|
primary_client_id,
|
||||||
|
lease_expires_at
|
||||||
|
)
|
||||||
|
VALUES ($1, $2, $3::uuid, NOW() + $4::interval)
|
||||||
|
ON CONFLICT (tenant_id, workspace_id) DO NOTHING
|
||||||
|
`, tenantID, workspaceID, installationID, leaseTTL)
|
||||||
|
if err != nil {
|
||||||
|
return false, response.ErrInternal(50041, "primary_lease_insert_failed", "failed to initialize monitoring desktop primary lease")
|
||||||
|
}
|
||||||
|
return tag.RowsAffected() > 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type heartbeatPrimaryLeaseState struct {
|
||||||
|
Found bool
|
||||||
|
PrimaryInstallationID string
|
||||||
|
CurrentIsPrimary bool
|
||||||
|
LeaseLive bool
|
||||||
|
RenewDue bool
|
||||||
|
PrimaryAvailable bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *MonitoringCallbackService) loadHeartbeatPrimaryLeaseState(ctx context.Context, tenantID, workspaceID int64, installationID string) (heartbeatPrimaryLeaseState, error) {
|
||||||
|
var state heartbeatPrimaryLeaseState
|
||||||
|
renewAfter := formatPgInterval(monitoringHeartbeatPrimaryRenewAfter)
|
||||||
|
if err := s.businessPool.QueryRow(ctx, `
|
||||||
|
SELECT
|
||||||
|
l.primary_client_id::text,
|
||||||
|
l.primary_client_id = $3::uuid AS current_is_primary,
|
||||||
|
l.lease_expires_at > NOW() AS lease_live,
|
||||||
|
l.updated_at <= NOW() - $4::interval AS renew_due,
|
||||||
|
dc.id IS NOT NULL AND dc.revoked_at IS NULL AS primary_available
|
||||||
|
FROM desktop_client_primary_leases l
|
||||||
|
LEFT JOIN desktop_clients dc ON dc.id = l.primary_client_id
|
||||||
|
WHERE l.tenant_id = $1
|
||||||
|
AND l.workspace_id = $2
|
||||||
|
`, tenantID, workspaceID, installationID, renewAfter).Scan(
|
||||||
|
&state.PrimaryInstallationID,
|
||||||
|
&state.CurrentIsPrimary,
|
||||||
|
&state.LeaseLive,
|
||||||
|
&state.RenewDue,
|
||||||
|
&state.PrimaryAvailable,
|
||||||
|
); err != nil {
|
||||||
if err == pgx.ErrNoRows {
|
if err == pgx.ErrNoRows {
|
||||||
return false, nil
|
return state, nil
|
||||||
}
|
}
|
||||||
return false, response.ErrInternal(50041, "query_failed", "failed to inspect monitoring desktop client heartbeat")
|
return state, response.ErrInternal(50041, "query_failed", "failed to inspect monitoring desktop primary lease")
|
||||||
|
}
|
||||||
|
state.Found = true
|
||||||
|
return state, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if !lastSeenAt.Valid {
|
type heartbeatPrimaryLeaseAction string
|
||||||
return false, nil
|
|
||||||
}
|
const (
|
||||||
return time.Since(lastSeenAt.Time) <= monitoringOnlineDuration, nil
|
heartbeatPrimaryLeaseActionPrimaryHit heartbeatPrimaryLeaseAction = "primary_hit"
|
||||||
|
heartbeatPrimaryLeaseActionNonPrimaryLive heartbeatPrimaryLeaseAction = "non_primary_live"
|
||||||
|
heartbeatPrimaryLeaseActionRenew heartbeatPrimaryLeaseAction = "renew"
|
||||||
|
heartbeatPrimaryLeaseActionClaimExpired heartbeatPrimaryLeaseAction = "claim_expired"
|
||||||
|
heartbeatPrimaryLeaseActionClaimUnavailable heartbeatPrimaryLeaseAction = "claim_unavailable"
|
||||||
|
)
|
||||||
|
|
||||||
|
type heartbeatPrimaryLeaseDecision struct {
|
||||||
|
Action heartbeatPrimaryLeaseAction
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MonitoringCallbackService) persistHeartbeatPrimaryInstallationID(ctx context.Context, tenantID int64, installation *monitoringInstallationIdentity) error {
|
func decideHeartbeatPrimaryLease(state heartbeatPrimaryLeaseState) heartbeatPrimaryLeaseDecision {
|
||||||
if _, err := s.businessPool.Exec(ctx, `
|
if !state.Found {
|
||||||
INSERT INTO tenant_monitoring_quotas (
|
return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionNonPrimaryLive}
|
||||||
tenant_id, workspace_id, primary_client_id
|
}
|
||||||
)
|
if state.CurrentIsPrimary {
|
||||||
VALUES (
|
if !state.LeaseLive || !state.PrimaryAvailable || state.RenewDue {
|
||||||
$1,
|
return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionRenew}
|
||||||
$2,
|
}
|
||||||
$3::uuid
|
return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionPrimaryHit}
|
||||||
)
|
}
|
||||||
ON CONFLICT (tenant_id) DO UPDATE
|
if !state.LeaseLive {
|
||||||
SET workspace_id = EXCLUDED.workspace_id,
|
return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionClaimExpired}
|
||||||
primary_client_id = EXCLUDED.primary_client_id,
|
}
|
||||||
|
if !state.PrimaryAvailable {
|
||||||
|
return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionClaimUnavailable}
|
||||||
|
}
|
||||||
|
return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionNonPrimaryLive}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *MonitoringCallbackService) renewHeartbeatPrimaryLease(ctx context.Context, tenantID, workspaceID int64, installationID, leaseTTL string) (bool, error) {
|
||||||
|
tag, err := s.businessPool.Exec(ctx, `
|
||||||
|
UPDATE desktop_client_primary_leases
|
||||||
|
SET lease_expires_at = NOW() + $4::interval,
|
||||||
updated_at = NOW()
|
updated_at = NOW()
|
||||||
`, tenantID, installation.WorkspaceID, installation.ID); err != nil {
|
WHERE tenant_id = $1
|
||||||
return response.ErrInternal(50041, "update_failed", "failed to persist primary monitoring desktop client")
|
AND workspace_id = $2
|
||||||
|
AND primary_client_id = $3::uuid
|
||||||
|
`, tenantID, workspaceID, installationID, leaseTTL)
|
||||||
|
if err != nil {
|
||||||
|
return false, response.ErrInternal(50041, "primary_lease_update_failed", "failed to renew monitoring desktop primary lease")
|
||||||
}
|
}
|
||||||
return nil
|
return tag.RowsAffected() > 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *MonitoringCallbackService) claimExpiredHeartbeatPrimaryLease(
|
||||||
|
ctx context.Context,
|
||||||
|
tenantID, workspaceID int64,
|
||||||
|
installationID string,
|
||||||
|
previousPrimaryInstallationID string,
|
||||||
|
leaseTTL string,
|
||||||
|
action heartbeatPrimaryLeaseAction,
|
||||||
|
) (bool, error) {
|
||||||
|
claimPredicate := "lease_expires_at <= NOW()"
|
||||||
|
if action == heartbeatPrimaryLeaseActionClaimUnavailable {
|
||||||
|
claimPredicate = `NOT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM desktop_clients dc
|
||||||
|
WHERE dc.id = desktop_client_primary_leases.primary_client_id
|
||||||
|
AND dc.revoked_at IS NULL
|
||||||
|
)`
|
||||||
|
}
|
||||||
|
query := `
|
||||||
|
UPDATE desktop_client_primary_leases
|
||||||
|
SET primary_client_id = $3::uuid,
|
||||||
|
lease_expires_at = NOW() + $5::interval,
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE tenant_id = $1
|
||||||
|
AND workspace_id = $2
|
||||||
|
AND primary_client_id = $4::uuid
|
||||||
|
AND ` + claimPredicate
|
||||||
|
tag, err := s.businessPool.Exec(ctx, query, tenantID, workspaceID, installationID, previousPrimaryInstallationID, leaseTTL)
|
||||||
|
if err != nil {
|
||||||
|
return false, response.ErrInternal(50041, "primary_lease_update_failed", "failed to claim monitoring desktop primary lease")
|
||||||
|
}
|
||||||
|
return tag.RowsAffected() > 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *MonitoringCallbackService) resolveHeartbeatPrimaryAfterContention(ctx context.Context, tenantID, workspaceID int64, installationID string) (bool, error) {
|
||||||
|
state, err := s.loadHeartbeatPrimaryLeaseState(ctx, tenantID, workspaceID, installationID)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return state.CurrentIsPrimary && state.LeaseLive && state.PrimaryAvailable, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantID, taskID int64) (*monitoringCollectTask, error) {
|
func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantID, taskID int64) (*monitoringCollectTask, error) {
|
||||||
@@ -1464,20 +1727,7 @@ func (s *MonitoringCallbackService) markTaskResultReceived(ctx context.Context,
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MonitoringCallbackService) upsertPlatformAccessSnapshot(ctx context.Context, tx pgx.Tx, installation *monitoringInstallationIdentity, businessDate string, platform MonitoringHeartbeatPlatform) (bool, error) {
|
func (s *MonitoringCallbackService) upsertPlatformAccessSnapshot(ctx context.Context, tx pgx.Tx, installation *monitoringInstallationIdentity, businessDate string, platform MonitoringHeartbeatPlatform) error {
|
||||||
previous, err := s.loadPlatformAccessSnapshotState(ctx, tx, installation.TenantID, installation.ID, businessDate, platform.AIPlatformID)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
nextState := monitoringPlatformAccessSnapshotState{
|
|
||||||
AccessStatus: monitoringStringValue(platform.AccessStatus),
|
|
||||||
Connected: derefBool(platform.Connected),
|
|
||||||
Accessible: derefBool(platform.Accessible),
|
|
||||||
ReasonCode: optionalString(monitoringStringValue(platform.ReasonCode)),
|
|
||||||
ReasonMessage: optionalString(monitoringStringValue(platform.ReasonMessage)),
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := tx.Exec(ctx, `
|
if _, err := tx.Exec(ctx, `
|
||||||
INSERT INTO monitoring_platform_access_snapshots (
|
INSERT INTO monitoring_platform_access_snapshots (
|
||||||
tenant_id, client_id, ai_platform_id, business_date, access_status,
|
tenant_id, client_id, ai_platform_id, business_date, access_status,
|
||||||
@@ -1498,44 +1748,101 @@ func (s *MonitoringCallbackService) upsertPlatformAccessSnapshot(ctx context.Con
|
|||||||
installation.ID,
|
installation.ID,
|
||||||
platform.AIPlatformID,
|
platform.AIPlatformID,
|
||||||
businessDate,
|
businessDate,
|
||||||
platform.AccessStatus,
|
monitoringStringValue(platform.AccessStatus),
|
||||||
derefBool(platform.Connected),
|
derefBool(platform.Connected),
|
||||||
derefBool(platform.Accessible),
|
derefBool(platform.Accessible),
|
||||||
resolveHeartbeatDetectedAt(platform.DetectedAt),
|
resolveHeartbeatDetectedAt(platform.DetectedAt),
|
||||||
nullableString(platform.ReasonCode),
|
nullableString(platform.ReasonCode),
|
||||||
nullableString(platform.ReasonMessage),
|
nullableString(platform.ReasonMessage),
|
||||||
); err != nil {
|
); err != nil {
|
||||||
return false, response.ErrInternal(50041, "access_snapshot_upsert_failed", "failed to persist monitoring platform access snapshot")
|
return response.ErrInternal(50041, "access_snapshot_upsert_failed", "failed to persist monitoring platform access snapshot")
|
||||||
}
|
}
|
||||||
return hasPlatformAccessSnapshotChanged(previous, nextState), nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MonitoringCallbackService) loadPlatformAccessSnapshotState(ctx context.Context, tx pgx.Tx, tenantID int64, installationID string, businessDate, aiPlatformID string) (*monitoringPlatformAccessSnapshotState, error) {
|
func (s *MonitoringCallbackService) loadPlatformAccessSnapshotStates(ctx context.Context, tenantID int64, installationID string, businessDate string, aiPlatformIDs []string) (map[string]*monitoringPlatformAccessSnapshotState, error) {
|
||||||
item := &monitoringPlatformAccessSnapshotState{}
|
states := make(map[string]*monitoringPlatformAccessSnapshotState, len(aiPlatformIDs))
|
||||||
var reasonCode sql.NullString
|
if len(aiPlatformIDs) == 0 {
|
||||||
var reasonMessage sql.NullString
|
return states, nil
|
||||||
if err := tx.QueryRow(ctx, `
|
}
|
||||||
SELECT access_status, connected, accessible, reason_code, reason_message
|
|
||||||
|
rows, err := s.monitoringPool.Query(ctx, `
|
||||||
|
SELECT ai_platform_id, access_status, connected, accessible, reason_code, reason_message, updated_at
|
||||||
FROM monitoring_platform_access_snapshots
|
FROM monitoring_platform_access_snapshots
|
||||||
WHERE tenant_id = $1
|
WHERE tenant_id = $1
|
||||||
AND client_id = $2::uuid
|
AND client_id = $2::uuid
|
||||||
AND ai_platform_id = $3
|
AND business_date = $3::date
|
||||||
AND business_date = $4::date
|
AND ai_platform_id = ANY($4::text[])
|
||||||
`, tenantID, installationID, aiPlatformID, businessDate).Scan(
|
`, tenantID, installationID, businessDate, aiPlatformIDs)
|
||||||
|
if err != nil {
|
||||||
|
return nil, response.ErrInternal(50041, "access_snapshot_lookup_failed", "failed to inspect monitoring platform access snapshots")
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
for rows.Next() {
|
||||||
|
item := &monitoringPlatformAccessSnapshotState{}
|
||||||
|
var platformID string
|
||||||
|
var reasonCode sql.NullString
|
||||||
|
var reasonMessage sql.NullString
|
||||||
|
if scanErr := rows.Scan(
|
||||||
|
&platformID,
|
||||||
&item.AccessStatus,
|
&item.AccessStatus,
|
||||||
&item.Connected,
|
&item.Connected,
|
||||||
&item.Accessible,
|
&item.Accessible,
|
||||||
&reasonCode,
|
&reasonCode,
|
||||||
&reasonMessage,
|
&reasonMessage,
|
||||||
); err != nil {
|
&item.UpdatedAt,
|
||||||
if err == pgx.ErrNoRows {
|
); scanErr != nil {
|
||||||
return nil, nil
|
return nil, response.ErrInternal(50041, "access_snapshot_scan_failed", "failed to parse monitoring platform access snapshots")
|
||||||
}
|
|
||||||
return nil, response.ErrInternal(50041, "access_snapshot_lookup_failed", "failed to inspect monitoring platform access snapshot")
|
|
||||||
}
|
}
|
||||||
|
if reasonCode.Valid {
|
||||||
item.ReasonCode = optionalString(reasonCode.String)
|
item.ReasonCode = optionalString(reasonCode.String)
|
||||||
|
}
|
||||||
|
if reasonMessage.Valid {
|
||||||
item.ReasonMessage = optionalString(reasonMessage.String)
|
item.ReasonMessage = optionalString(reasonMessage.String)
|
||||||
return item, nil
|
}
|
||||||
|
states[platformID] = item
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, response.ErrInternal(50041, "access_snapshot_scan_failed", "failed to iterate monitoring platform access snapshots")
|
||||||
|
}
|
||||||
|
|
||||||
|
return states, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitoringPlatformAccessSnapshotStateFromHeartbeat(platform MonitoringHeartbeatPlatform) monitoringPlatformAccessSnapshotState {
|
||||||
|
return monitoringPlatformAccessSnapshotState{
|
||||||
|
AccessStatus: monitoringStringValue(platform.AccessStatus),
|
||||||
|
Connected: derefBool(platform.Connected),
|
||||||
|
Accessible: derefBool(platform.Accessible),
|
||||||
|
ReasonCode: optionalString(monitoringStringValue(platform.ReasonCode)),
|
||||||
|
ReasonMessage: optionalString(monitoringStringValue(platform.ReasonMessage)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func decideMonitoringHeartbeatSnapshot(previous *monitoringPlatformAccessSnapshotState, next monitoringPlatformAccessSnapshotState, now time.Time, isPrimary bool) monitoringHeartbeatSnapshotDecision {
|
||||||
|
stateChanged := hasPlatformAccessSnapshotChanged(previous, next)
|
||||||
|
refreshDue := false
|
||||||
|
if previous != nil && !stateChanged {
|
||||||
|
refreshDue = previous.UpdatedAt.IsZero() || !previous.UpdatedAt.After(now.Add(-monitoringHeartbeatAccessSnapshotRefreshAfter))
|
||||||
|
}
|
||||||
|
|
||||||
|
operation := "skip"
|
||||||
|
if previous == nil {
|
||||||
|
operation = "insert"
|
||||||
|
} else if stateChanged {
|
||||||
|
operation = "update"
|
||||||
|
} else if refreshDue {
|
||||||
|
operation = "refresh"
|
||||||
|
}
|
||||||
|
|
||||||
|
return monitoringHeartbeatSnapshotDecision{
|
||||||
|
WriteSnapshot: stateChanged || refreshDue,
|
||||||
|
StateChanged: stateChanged,
|
||||||
|
RefreshDue: refreshDue,
|
||||||
|
PendingReconcileCandidate: isPrimary && next.AccessStatus != "accessible",
|
||||||
|
SnapshotOperation: operation,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func hasPlatformAccessSnapshotChanged(previous *monitoringPlatformAccessSnapshotState, next monitoringPlatformAccessSnapshotState) bool {
|
func hasPlatformAccessSnapshotChanged(previous *monitoringPlatformAccessSnapshotState, next monitoringPlatformAccessSnapshotState) bool {
|
||||||
@@ -1558,6 +1865,41 @@ func hasPlatformAccessSnapshotChanged(previous *monitoringPlatformAccessSnapshot
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *MonitoringCallbackService) loadPendingMonitoringTaskPlatformIDs(ctx context.Context, tenantID int64, businessDate string, aiPlatformIDs []string) (map[string]struct{}, error) {
|
||||||
|
result := make(map[string]struct{}, len(aiPlatformIDs))
|
||||||
|
if len(aiPlatformIDs) == 0 {
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := s.monitoringPool.Query(ctx, `
|
||||||
|
SELECT ai_platform_id
|
||||||
|
FROM monitoring_collect_tasks
|
||||||
|
WHERE tenant_id = $1
|
||||||
|
AND ai_platform_id = ANY($2::text[])
|
||||||
|
AND collector_type = $3
|
||||||
|
AND business_date = $4::date
|
||||||
|
AND status = 'pending'
|
||||||
|
GROUP BY ai_platform_id
|
||||||
|
`, tenantID, aiPlatformIDs, monitoringCollectorType, businessDate)
|
||||||
|
if err != nil {
|
||||||
|
return nil, response.ErrInternal(50041, "pending_task_platform_lookup_failed", "failed to inspect pending monitoring task platforms")
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
for rows.Next() {
|
||||||
|
var platformID string
|
||||||
|
if scanErr := rows.Scan(&platformID); scanErr != nil {
|
||||||
|
return nil, response.ErrInternal(50041, "pending_task_platform_scan_failed", "failed to parse pending monitoring task platforms")
|
||||||
|
}
|
||||||
|
result[platformID] = struct{}{}
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, response.ErrInternal(50041, "pending_task_platform_scan_failed", "failed to iterate pending monitoring task platforms")
|
||||||
|
}
|
||||||
|
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *MonitoringCallbackService) reconcilePendingTasksForPlatform(ctx context.Context, tx pgx.Tx, tenantID int64, businessDate, aiPlatformID string) (int64, error) {
|
func (s *MonitoringCallbackService) reconcilePendingTasksForPlatform(ctx context.Context, tx pgx.Tx, tenantID int64, businessDate, aiPlatformID string) (int64, error) {
|
||||||
tag, err := tx.Exec(ctx, `
|
tag, err := tx.Exec(ctx, `
|
||||||
UPDATE monitoring_collect_tasks
|
UPDATE monitoring_collect_tasks
|
||||||
@@ -2530,6 +2872,20 @@ func normalizeMonitoringLeasePlatformIDs(values []string) []string {
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func monitoringHeartbeatPlatformIDs(platforms []MonitoringHeartbeatPlatform) []string {
|
||||||
|
if len(platforms) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
result := make([]string, 0, len(platforms))
|
||||||
|
for _, platform := range platforms {
|
||||||
|
platformID := strings.TrimSpace(platform.AIPlatformID)
|
||||||
|
if platformID != "" {
|
||||||
|
result = append(result, platformID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
func normalizeHeartbeatPlatforms(items []MonitoringHeartbeatPlatform, fallbackDetectedAt time.Time) ([]MonitoringHeartbeatPlatform, error) {
|
func normalizeHeartbeatPlatforms(items []MonitoringHeartbeatPlatform, fallbackDetectedAt time.Time) ([]MonitoringHeartbeatPlatform, error) {
|
||||||
if len(items) == 0 {
|
if len(items) == 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
|
|||||||
@@ -2,9 +2,287 @@ package app
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestDecideHeartbeatPrimaryLease(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
state heartbeatPrimaryLeaseState
|
||||||
|
want heartbeatPrimaryLeaseAction
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "current primary fast path",
|
||||||
|
state: heartbeatPrimaryLeaseState{
|
||||||
|
Found: true,
|
||||||
|
CurrentIsPrimary: true,
|
||||||
|
LeaseLive: true,
|
||||||
|
PrimaryAvailable: true,
|
||||||
|
},
|
||||||
|
want: heartbeatPrimaryLeaseActionPrimaryHit,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "current primary renews when due",
|
||||||
|
state: heartbeatPrimaryLeaseState{
|
||||||
|
Found: true,
|
||||||
|
CurrentIsPrimary: true,
|
||||||
|
LeaseLive: true,
|
||||||
|
RenewDue: true,
|
||||||
|
PrimaryAvailable: true,
|
||||||
|
},
|
||||||
|
want: heartbeatPrimaryLeaseActionRenew,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non primary does not steal live lease",
|
||||||
|
state: heartbeatPrimaryLeaseState{
|
||||||
|
Found: true,
|
||||||
|
CurrentIsPrimary: false,
|
||||||
|
LeaseLive: true,
|
||||||
|
PrimaryAvailable: true,
|
||||||
|
},
|
||||||
|
want: heartbeatPrimaryLeaseActionNonPrimaryLive,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "expired lease can be claimed",
|
||||||
|
state: heartbeatPrimaryLeaseState{
|
||||||
|
Found: true,
|
||||||
|
CurrentIsPrimary: false,
|
||||||
|
LeaseLive: false,
|
||||||
|
PrimaryAvailable: true,
|
||||||
|
},
|
||||||
|
want: heartbeatPrimaryLeaseActionClaimExpired,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "revoked primary can be replaced",
|
||||||
|
state: heartbeatPrimaryLeaseState{
|
||||||
|
Found: true,
|
||||||
|
CurrentIsPrimary: false,
|
||||||
|
LeaseLive: true,
|
||||||
|
PrimaryAvailable: false,
|
||||||
|
},
|
||||||
|
want: heartbeatPrimaryLeaseActionClaimUnavailable,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got := decideHeartbeatPrimaryLease(tt.state)
|
||||||
|
if got.Action != tt.want {
|
||||||
|
t.Fatalf("decideHeartbeatPrimaryLease() = %v, want %v", got.Action, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitoringHeartbeatPrimaryLeaseMetrics(t *testing.T) {
|
||||||
|
resetMonitoringHeartbeatPrimaryLeaseMetricsForTest()
|
||||||
|
t.Cleanup(resetMonitoringHeartbeatPrimaryLeaseMetricsForTest)
|
||||||
|
|
||||||
|
recordHeartbeatPrimaryLeaseMetric("primary_hit", 20*time.Millisecond)
|
||||||
|
recordHeartbeatPrimaryLeaseMetric("contention", 10*time.Millisecond)
|
||||||
|
recordHeartbeatPrimaryLeaseWriteMetric("renew")
|
||||||
|
|
||||||
|
snapshot := MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue()
|
||||||
|
if snapshot.PrimaryHitTotal != 1 {
|
||||||
|
t.Fatalf("PrimaryHitTotal = %d, want 1", snapshot.PrimaryHitTotal)
|
||||||
|
}
|
||||||
|
if snapshot.ContentionTotal != 1 {
|
||||||
|
t.Fatalf("ContentionTotal = %d, want 1", snapshot.ContentionTotal)
|
||||||
|
}
|
||||||
|
if snapshot.DBRenewTotal != 1 {
|
||||||
|
t.Fatalf("DBRenewTotal = %d, want 1", snapshot.DBRenewTotal)
|
||||||
|
}
|
||||||
|
if snapshot.DurationCount != 2 {
|
||||||
|
t.Fatalf("DurationCount = %d, want 2", snapshot.DurationCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
prometheus := MonitoringHeartbeatPrimaryLeaseMetricsPrometheus()
|
||||||
|
if !strings.Contains(prometheus, `monitoring_heartbeat_primary_lease_resolutions_total{outcome="primary_hit"} 1`) {
|
||||||
|
t.Fatalf("prometheus output missing primary_hit metric: %s", prometheus)
|
||||||
|
}
|
||||||
|
if !strings.Contains(prometheus, `monitoring_heartbeat_primary_lease_db_writes_total{operation="renew"} 1`) {
|
||||||
|
t.Fatalf("prometheus output missing renew write metric: %s", prometheus)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecideMonitoringHeartbeatSnapshot(t *testing.T) {
|
||||||
|
now := time.Date(2026, 4, 24, 3, 0, 0, 0, time.UTC)
|
||||||
|
base := monitoringPlatformAccessSnapshotState{
|
||||||
|
AccessStatus: "accessible",
|
||||||
|
Connected: true,
|
||||||
|
Accessible: true,
|
||||||
|
UpdatedAt: now.Add(-time.Minute),
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
previous *monitoringPlatformAccessSnapshotState
|
||||||
|
next monitoringPlatformAccessSnapshotState
|
||||||
|
primary bool
|
||||||
|
want monitoringHeartbeatSnapshotDecision
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "missing snapshot inserts",
|
||||||
|
next: base,
|
||||||
|
primary: true,
|
||||||
|
want: monitoringHeartbeatSnapshotDecision{
|
||||||
|
WriteSnapshot: true,
|
||||||
|
StateChanged: true,
|
||||||
|
SnapshotOperation: "insert",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fresh unchanged accessible snapshot skips write",
|
||||||
|
previous: &base,
|
||||||
|
next: base,
|
||||||
|
primary: true,
|
||||||
|
want: monitoringHeartbeatSnapshotDecision{
|
||||||
|
SnapshotOperation: "skip",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "stale unchanged snapshot refreshes",
|
||||||
|
previous: &monitoringPlatformAccessSnapshotState{
|
||||||
|
AccessStatus: "accessible",
|
||||||
|
Connected: true,
|
||||||
|
Accessible: true,
|
||||||
|
UpdatedAt: now.Add(-monitoringHeartbeatAccessSnapshotRefreshAfter - time.Second),
|
||||||
|
},
|
||||||
|
next: base,
|
||||||
|
primary: true,
|
||||||
|
want: monitoringHeartbeatSnapshotDecision{
|
||||||
|
WriteSnapshot: true,
|
||||||
|
RefreshDue: true,
|
||||||
|
SnapshotOperation: "refresh",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "changed snapshot updates",
|
||||||
|
previous: &base,
|
||||||
|
next: monitoringPlatformAccessSnapshotState{
|
||||||
|
AccessStatus: "unavailable",
|
||||||
|
Connected: true,
|
||||||
|
Accessible: false,
|
||||||
|
},
|
||||||
|
primary: true,
|
||||||
|
want: monitoringHeartbeatSnapshotDecision{
|
||||||
|
WriteSnapshot: true,
|
||||||
|
StateChanged: true,
|
||||||
|
PendingReconcileCandidate: true,
|
||||||
|
SnapshotOperation: "update",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fresh unchanged primary inaccessible snapshot checks pending tasks without writing snapshot",
|
||||||
|
previous: &monitoringPlatformAccessSnapshotState{
|
||||||
|
AccessStatus: "not_logged_in",
|
||||||
|
Connected: false,
|
||||||
|
Accessible: false,
|
||||||
|
UpdatedAt: now.Add(-time.Minute),
|
||||||
|
},
|
||||||
|
next: monitoringPlatformAccessSnapshotState{
|
||||||
|
AccessStatus: "not_logged_in",
|
||||||
|
Connected: false,
|
||||||
|
Accessible: false,
|
||||||
|
},
|
||||||
|
primary: true,
|
||||||
|
want: monitoringHeartbeatSnapshotDecision{
|
||||||
|
PendingReconcileCandidate: true,
|
||||||
|
SnapshotOperation: "skip",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non-primary inaccessible snapshot does not reconcile",
|
||||||
|
previous: &monitoringPlatformAccessSnapshotState{
|
||||||
|
AccessStatus: "not_logged_in",
|
||||||
|
Connected: false,
|
||||||
|
Accessible: false,
|
||||||
|
UpdatedAt: now.Add(-time.Minute),
|
||||||
|
},
|
||||||
|
next: monitoringPlatformAccessSnapshotState{
|
||||||
|
AccessStatus: "not_logged_in",
|
||||||
|
Connected: false,
|
||||||
|
Accessible: false,
|
||||||
|
},
|
||||||
|
primary: false,
|
||||||
|
want: monitoringHeartbeatSnapshotDecision{
|
||||||
|
SnapshotOperation: "skip",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got := decideMonitoringHeartbeatSnapshot(tt.previous, tt.next, now, tt.primary)
|
||||||
|
if got.WriteSnapshot != tt.want.WriteSnapshot ||
|
||||||
|
got.StateChanged != tt.want.StateChanged ||
|
||||||
|
got.RefreshDue != tt.want.RefreshDue ||
|
||||||
|
got.PendingReconcileCandidate != tt.want.PendingReconcileCandidate ||
|
||||||
|
got.ReconcilePending != tt.want.ReconcilePending ||
|
||||||
|
got.SnapshotOperation != tt.want.SnapshotOperation {
|
||||||
|
t.Fatalf("decideMonitoringHeartbeatSnapshot() = %+v, want %+v", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitoringHeartbeatSnapshotMetrics(t *testing.T) {
|
||||||
|
resetMonitoringHeartbeatSnapshotMetricsForTest()
|
||||||
|
t.Cleanup(resetMonitoringHeartbeatSnapshotMetricsForTest)
|
||||||
|
|
||||||
|
recordMonitoringHeartbeatSnapshotMetrics(monitoringHeartbeatSnapshotMetricsEvent{
|
||||||
|
PlatformCount: 3,
|
||||||
|
TransactionOpened: true,
|
||||||
|
SnapshotInsertedCount: 1,
|
||||||
|
SnapshotSkippedCount: 2,
|
||||||
|
ReconciledSkippedTaskCount: 5,
|
||||||
|
ProjectionRebuildCount: 2,
|
||||||
|
Duration: 20 * time.Millisecond,
|
||||||
|
})
|
||||||
|
recordMonitoringHeartbeatSnapshotMetrics(monitoringHeartbeatSnapshotMetricsEvent{
|
||||||
|
PlatformCount: 1,
|
||||||
|
Duration: 10 * time.Millisecond,
|
||||||
|
Err: errors.New("snapshot failed"),
|
||||||
|
})
|
||||||
|
|
||||||
|
snapshot := MonitoringHeartbeatSnapshotMetricsSnapshotValue()
|
||||||
|
if snapshot.PlatformReportedTotal != 4 {
|
||||||
|
t.Fatalf("PlatformReportedTotal = %d, want 4", snapshot.PlatformReportedTotal)
|
||||||
|
}
|
||||||
|
if snapshot.TransactionOpenedTotal != 1 {
|
||||||
|
t.Fatalf("TransactionOpenedTotal = %d, want 1", snapshot.TransactionOpenedTotal)
|
||||||
|
}
|
||||||
|
if snapshot.SnapshotInsertedTotal != 1 {
|
||||||
|
t.Fatalf("SnapshotInsertedTotal = %d, want 1", snapshot.SnapshotInsertedTotal)
|
||||||
|
}
|
||||||
|
if snapshot.SnapshotSkippedTotal != 2 {
|
||||||
|
t.Fatalf("SnapshotSkippedTotal = %d, want 2", snapshot.SnapshotSkippedTotal)
|
||||||
|
}
|
||||||
|
if snapshot.ReconciledSkippedTaskTotal != 5 {
|
||||||
|
t.Fatalf("ReconciledSkippedTaskTotal = %d, want 5", snapshot.ReconciledSkippedTaskTotal)
|
||||||
|
}
|
||||||
|
if snapshot.ProjectionRebuildTotal != 2 {
|
||||||
|
t.Fatalf("ProjectionRebuildTotal = %d, want 2", snapshot.ProjectionRebuildTotal)
|
||||||
|
}
|
||||||
|
if snapshot.ErrorTotal != 1 {
|
||||||
|
t.Fatalf("ErrorTotal = %d, want 1", snapshot.ErrorTotal)
|
||||||
|
}
|
||||||
|
if snapshot.DurationCount != 2 {
|
||||||
|
t.Fatalf("DurationCount = %d, want 2", snapshot.DurationCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
prometheus := MonitoringHeartbeatSnapshotMetricsPrometheus()
|
||||||
|
if !strings.Contains(prometheus, `monitoring_heartbeat_snapshot_writes_total{operation="insert"} 1`) {
|
||||||
|
t.Fatalf("prometheus output missing insert write metric: %s", prometheus)
|
||||||
|
}
|
||||||
|
if !strings.Contains(prometheus, "monitoring_heartbeat_snapshot_transactions_total 1") {
|
||||||
|
t.Fatalf("prometheus output missing transaction metric: %s", prometheus)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestBuildMonitoringRawPayloadKimiExcludesSearchResultsFromCitationInputs(t *testing.T) {
|
func TestBuildMonitoringRawPayloadKimiExcludesSearchResultsFromCitationInputs(t *testing.T) {
|
||||||
citationTitle := "真实引用文章"
|
citationTitle := "真实引用文章"
|
||||||
searchTitle := "搜索网页结果"
|
searchTitle := "搜索网页结果"
|
||||||
|
|||||||
@@ -0,0 +1,201 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/geo-platform/tenant-api/internal/shared/response"
|
||||||
|
)
|
||||||
|
|
||||||
|
type MonitoringDailyTaskMetricsSnapshot struct {
|
||||||
|
RunTotal int64 `json:"run_total"`
|
||||||
|
RunCompletedTotal int64 `json:"run_completed_total"`
|
||||||
|
RunFailedTotal int64 `json:"run_failed_total"`
|
||||||
|
RunPartialFailureTotal int64 `json:"run_partial_failure_total"`
|
||||||
|
PlanFailureTotal int64 `json:"plan_failure_total"`
|
||||||
|
BrandFailureTotal int64 `json:"brand_failure_total"`
|
||||||
|
SkippedPlanTotal int64 `json:"skipped_plan_total"`
|
||||||
|
PlannedTaskTotal int64 `json:"planned_task_total"`
|
||||||
|
DueTaskTotal int64 `json:"due_task_total"`
|
||||||
|
CreatedTaskTotal int64 `json:"created_task_total"`
|
||||||
|
DesktopTaskTotal int64 `json:"desktop_task_total"`
|
||||||
|
DeferredTaskTotal int64 `json:"deferred_task_total"`
|
||||||
|
LastRunAt *time.Time `json:"last_run_at,omitempty"`
|
||||||
|
LastSuccessAt *time.Time `json:"last_success_at,omitempty"`
|
||||||
|
LastErrorAt *time.Time `json:"last_error_at,omitempty"`
|
||||||
|
LastDurationSeconds float64 `json:"last_duration_seconds"`
|
||||||
|
LastBusinessDate string `json:"last_business_date,omitempty"`
|
||||||
|
LastTenantCount int64 `json:"last_tenant_count"`
|
||||||
|
LastBrandCount int64 `json:"last_brand_count"`
|
||||||
|
LastErrorCode string `json:"last_error_code,omitempty"`
|
||||||
|
LastErrorMessage string `json:"last_error_message,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type monitoringDailyTaskMetricsRecorder struct {
|
||||||
|
runTotal atomic.Int64
|
||||||
|
runCompletedTotal atomic.Int64
|
||||||
|
runFailedTotal atomic.Int64
|
||||||
|
runPartialFailureTotal atomic.Int64
|
||||||
|
planFailureTotal atomic.Int64
|
||||||
|
brandFailureTotal atomic.Int64
|
||||||
|
skippedPlanTotal atomic.Int64
|
||||||
|
plannedTaskTotal atomic.Int64
|
||||||
|
dueTaskTotal atomic.Int64
|
||||||
|
createdTaskTotal atomic.Int64
|
||||||
|
desktopTaskTotal atomic.Int64
|
||||||
|
deferredTaskTotal atomic.Int64
|
||||||
|
lastRunAtUnixNano atomic.Int64
|
||||||
|
lastSuccessAtUnixNano atomic.Int64
|
||||||
|
lastErrorAtUnixNano atomic.Int64
|
||||||
|
lastDurationNanos atomic.Int64
|
||||||
|
lastTenantCount atomic.Int64
|
||||||
|
lastBrandCount atomic.Int64
|
||||||
|
lastBusinessDate atomic.Value
|
||||||
|
lastErrorCode atomic.Value
|
||||||
|
lastErrorMessage atomic.Value
|
||||||
|
}
|
||||||
|
|
||||||
|
var monitoringDailyTaskMetrics = newMonitoringDailyTaskMetricsRecorder()
|
||||||
|
|
||||||
|
func newMonitoringDailyTaskMetricsRecorder() *monitoringDailyTaskMetricsRecorder {
|
||||||
|
return &monitoringDailyTaskMetricsRecorder{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func recordMonitoringDailyTaskMetrics(summary *MonitoringDailyTaskGenerationSummary, duration time.Duration, err error) {
|
||||||
|
monitoringDailyTaskMetrics.record(summary, duration, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func MonitoringDailyTaskMetricsSnapshotValue() MonitoringDailyTaskMetricsSnapshot {
|
||||||
|
return monitoringDailyTaskMetrics.snapshot()
|
||||||
|
}
|
||||||
|
|
||||||
|
func resetMonitoringDailyTaskMetricsForTest() {
|
||||||
|
monitoringDailyTaskMetrics = newMonitoringDailyTaskMetricsRecorder()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *monitoringDailyTaskMetricsRecorder) record(summary *MonitoringDailyTaskGenerationSummary, duration time.Duration, err error) {
|
||||||
|
if r == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
now := time.Now().UTC()
|
||||||
|
if duration < 0 {
|
||||||
|
duration = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
r.runTotal.Add(1)
|
||||||
|
r.lastRunAtUnixNano.Store(now.UnixNano())
|
||||||
|
r.lastDurationNanos.Store(duration.Nanoseconds())
|
||||||
|
|
||||||
|
if summary != nil {
|
||||||
|
r.lastBusinessDate.Store(summary.BusinessDate)
|
||||||
|
r.lastTenantCount.Store(int64(summary.TenantCount))
|
||||||
|
r.lastBrandCount.Store(int64(summary.BrandCount))
|
||||||
|
r.planFailureTotal.Add(int64(summary.FailedPlanCount))
|
||||||
|
r.brandFailureTotal.Add(int64(summary.FailedBrandCount))
|
||||||
|
r.skippedPlanTotal.Add(int64(summary.SkippedPlanCount))
|
||||||
|
r.plannedTaskTotal.Add(summary.PlannedTaskCount)
|
||||||
|
r.dueTaskTotal.Add(summary.DueTaskCount)
|
||||||
|
r.createdTaskTotal.Add(summary.CreatedTaskCount)
|
||||||
|
r.desktopTaskTotal.Add(summary.DesktopTaskCount)
|
||||||
|
r.deferredTaskTotal.Add(summary.DeferredTaskCount)
|
||||||
|
if summary.FailedPlanCount > 0 || summary.FailedBrandCount > 0 {
|
||||||
|
r.runPartialFailureTotal.Add(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
code, message := monitoringDailyMetricError(err)
|
||||||
|
r.runFailedTotal.Add(1)
|
||||||
|
r.lastErrorAtUnixNano.Store(now.UnixNano())
|
||||||
|
r.lastErrorCode.Store(code)
|
||||||
|
r.lastErrorMessage.Store(message)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
r.runCompletedTotal.Add(1)
|
||||||
|
r.lastSuccessAtUnixNano.Store(now.UnixNano())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *monitoringDailyTaskMetricsRecorder) snapshot() MonitoringDailyTaskMetricsSnapshot {
|
||||||
|
if r == nil {
|
||||||
|
return MonitoringDailyTaskMetricsSnapshot{}
|
||||||
|
}
|
||||||
|
return MonitoringDailyTaskMetricsSnapshot{
|
||||||
|
RunTotal: r.runTotal.Load(),
|
||||||
|
RunCompletedTotal: r.runCompletedTotal.Load(),
|
||||||
|
RunFailedTotal: r.runFailedTotal.Load(),
|
||||||
|
RunPartialFailureTotal: r.runPartialFailureTotal.Load(),
|
||||||
|
PlanFailureTotal: r.planFailureTotal.Load(),
|
||||||
|
BrandFailureTotal: r.brandFailureTotal.Load(),
|
||||||
|
SkippedPlanTotal: r.skippedPlanTotal.Load(),
|
||||||
|
PlannedTaskTotal: r.plannedTaskTotal.Load(),
|
||||||
|
DueTaskTotal: r.dueTaskTotal.Load(),
|
||||||
|
CreatedTaskTotal: r.createdTaskTotal.Load(),
|
||||||
|
DesktopTaskTotal: r.desktopTaskTotal.Load(),
|
||||||
|
DeferredTaskTotal: r.deferredTaskTotal.Load(),
|
||||||
|
LastRunAt: monitoringDailyUnixNanoTime(r.lastRunAtUnixNano.Load()),
|
||||||
|
LastSuccessAt: monitoringDailyUnixNanoTime(r.lastSuccessAtUnixNano.Load()),
|
||||||
|
LastErrorAt: monitoringDailyUnixNanoTime(r.lastErrorAtUnixNano.Load()),
|
||||||
|
LastDurationSeconds: float64(r.lastDurationNanos.Load()) / float64(time.Second),
|
||||||
|
LastBusinessDate: monitoringDailyLoadString(&r.lastBusinessDate),
|
||||||
|
LastTenantCount: r.lastTenantCount.Load(),
|
||||||
|
LastBrandCount: r.lastBrandCount.Load(),
|
||||||
|
LastErrorCode: monitoringDailyLoadString(&r.lastErrorCode),
|
||||||
|
LastErrorMessage: monitoringDailyLoadString(&r.lastErrorMessage),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitoringDailyMetricError(err error) (string, string) {
|
||||||
|
normalized := normalizeMonitoringWorkerError(err)
|
||||||
|
if normalized == nil {
|
||||||
|
return "", ""
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case errors.Is(normalized, context.Canceled):
|
||||||
|
return "context_canceled", normalized.Error()
|
||||||
|
case errors.Is(normalized, context.DeadlineExceeded):
|
||||||
|
return "context_deadline_exceeded", normalized.Error()
|
||||||
|
}
|
||||||
|
var appErr *response.AppError
|
||||||
|
if errors.As(normalized, &appErr) {
|
||||||
|
if strings.TrimSpace(appErr.Message) != "" {
|
||||||
|
return appErr.Message, appErr.Error()
|
||||||
|
}
|
||||||
|
return "app_error", appErr.Error()
|
||||||
|
}
|
||||||
|
code := strings.TrimLeft(fmt.Sprintf("%T", normalized), "*")
|
||||||
|
code = strings.ReplaceAll(code, "/", "_")
|
||||||
|
code = strings.ReplaceAll(code, ".", "_")
|
||||||
|
if code == "" {
|
||||||
|
code = "error"
|
||||||
|
}
|
||||||
|
return code, normalized.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitoringDailyUnixNanoTime(value int64) *time.Time {
|
||||||
|
if value <= 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
t := time.Unix(0, value).UTC()
|
||||||
|
return &t
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitoringDailyLoadString(value *atomic.Value) string {
|
||||||
|
if value == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
raw := value.Load()
|
||||||
|
result, _ := raw.(string)
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitoringDailyTimeSeconds(value *time.Time) float64 {
|
||||||
|
if value == nil || value.IsZero() {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return float64(value.UnixNano()) / float64(time.Second)
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,337 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"github.com/geo-platform/tenant-api/internal/shared/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBuildMonitoringDailyTaskCandidatesMaterializesAfterMidnight(t *testing.T) {
|
||||||
|
businessDay := time.Date(2026, 4, 24, 0, 0, 0, 0, monitoringBusinessLocation())
|
||||||
|
questions := []monitoringConfiguredQuestion{
|
||||||
|
{ID: 101, QuestionHash: []byte("q101")},
|
||||||
|
{ID: 102, QuestionHash: []byte("q102")},
|
||||||
|
}
|
||||||
|
platforms := []monitoringPlatformMetadata{
|
||||||
|
{ID: "qwen", Name: "Qwen"},
|
||||||
|
{ID: "doubao", Name: "Doubao"},
|
||||||
|
{ID: "deepseek", Name: "DeepSeek"},
|
||||||
|
}
|
||||||
|
|
||||||
|
candidates := buildMonitoringDailyTaskCandidates(7, 11, businessDay, questions, platforms, 4)
|
||||||
|
|
||||||
|
require.Len(t, candidates, 4)
|
||||||
|
start := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset).UTC()
|
||||||
|
end := start.Add(defaultMonitoringDailyMaterializeWindow)
|
||||||
|
for i, candidate := range candidates {
|
||||||
|
assert.False(t, candidate.MaterializeAt.Before(start))
|
||||||
|
assert.False(t, candidate.MaterializeAt.After(end))
|
||||||
|
assert.Equal(t, candidate.MaterializeAt, candidate.DispatchAfter)
|
||||||
|
if i > 0 {
|
||||||
|
assert.False(t, candidate.MaterializeAt.Before(candidates[i-1].MaterializeAt))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectDueMonitoringDailyCandidatesSkipsExistingAndHonorsLimit(t *testing.T) {
|
||||||
|
businessDay := time.Date(2026, 4, 24, 0, 0, 0, 0, monitoringBusinessLocation())
|
||||||
|
candidates := buildMonitoringDailyTaskCandidates(
|
||||||
|
7,
|
||||||
|
11,
|
||||||
|
businessDay,
|
||||||
|
[]monitoringConfiguredQuestion{
|
||||||
|
{ID: 101, QuestionHash: []byte("q101")},
|
||||||
|
{ID: 102, QuestionHash: []byte("q102")},
|
||||||
|
},
|
||||||
|
[]monitoringPlatformMetadata{
|
||||||
|
{ID: "qwen"},
|
||||||
|
{ID: "doubao"},
|
||||||
|
{ID: "deepseek"},
|
||||||
|
},
|
||||||
|
6,
|
||||||
|
)
|
||||||
|
require.Len(t, candidates, 6)
|
||||||
|
|
||||||
|
existing := map[monitoringDailyTaskKey]struct{}{
|
||||||
|
{
|
||||||
|
QuestionID: candidates[0].Question.ID,
|
||||||
|
PlatformID: candidates[0].Platform.ID,
|
||||||
|
}: {},
|
||||||
|
}
|
||||||
|
now := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset + defaultMonitoringDailyMaterializeWindow)
|
||||||
|
|
||||||
|
due := selectDueMonitoringDailyCandidates(candidates, existing, now, defaultMonitoringDailyLookahead, 2)
|
||||||
|
|
||||||
|
require.Len(t, due, 2)
|
||||||
|
for _, candidate := range due {
|
||||||
|
_, exists := existing[monitoringDailyTaskKey{
|
||||||
|
QuestionID: candidate.Question.ID,
|
||||||
|
PlatformID: candidate.Platform.ID,
|
||||||
|
}]
|
||||||
|
assert.False(t, exists)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitoringDailyRemainingKeepsFullCandidateHorizon(t *testing.T) {
|
||||||
|
businessDay := time.Date(2026, 4, 24, 0, 0, 0, 0, monitoringBusinessLocation())
|
||||||
|
hardCap := 6
|
||||||
|
remaining := monitoringDailyRemainingCapacity(hardCap, 4)
|
||||||
|
assert.Equal(t, 2, remaining)
|
||||||
|
|
||||||
|
candidates := buildMonitoringDailyTaskCandidates(
|
||||||
|
7,
|
||||||
|
11,
|
||||||
|
businessDay,
|
||||||
|
[]monitoringConfiguredQuestion{
|
||||||
|
{ID: 101, QuestionHash: []byte("q101")},
|
||||||
|
{ID: 102, QuestionHash: []byte("q102")},
|
||||||
|
{ID: 103, QuestionHash: []byte("q103")},
|
||||||
|
},
|
||||||
|
[]monitoringPlatformMetadata{
|
||||||
|
{ID: "qwen"},
|
||||||
|
{ID: "doubao"},
|
||||||
|
},
|
||||||
|
hardCap,
|
||||||
|
)
|
||||||
|
require.Len(t, candidates, hardCap)
|
||||||
|
|
||||||
|
existing := make(map[monitoringDailyTaskKey]struct{}, 4)
|
||||||
|
for _, candidate := range candidates[:4] {
|
||||||
|
existing[monitoringDailyTaskKey{
|
||||||
|
QuestionID: candidate.Question.ID,
|
||||||
|
PlatformID: candidate.Platform.ID,
|
||||||
|
}] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
now := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset + defaultMonitoringDailyMaterializeWindow)
|
||||||
|
due := selectDueMonitoringDailyCandidates(candidates, existing, now, defaultMonitoringDailyLookahead, remaining)
|
||||||
|
|
||||||
|
require.Len(t, due, remaining)
|
||||||
|
for _, candidate := range due {
|
||||||
|
_, exists := existing[monitoringDailyTaskKey{
|
||||||
|
QuestionID: candidate.Question.ID,
|
||||||
|
PlatformID: candidate.Platform.ID,
|
||||||
|
}]
|
||||||
|
assert.False(t, exists)
|
||||||
|
}
|
||||||
|
assert.Equal(t, 1, decrementMonitoringDailyRemaining(remaining, 1))
|
||||||
|
assert.Equal(t, 0, decrementMonitoringDailyRemaining(remaining, 2))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitoringDailyQuestionLimitUsesBrandLibraryConfig(t *testing.T) {
|
||||||
|
assert.Equal(t, 25, monitoringDailyQuestionLimit(config.BrandLibraryConfig{
|
||||||
|
MaxKeywords: 5,
|
||||||
|
MaxQuestionsPerKeyword: 5,
|
||||||
|
}))
|
||||||
|
assert.Equal(t, 6, monitoringDailyQuestionLimit(config.BrandLibraryConfig{
|
||||||
|
MaxKeywords: 2,
|
||||||
|
MaxQuestionsPerKeyword: 3,
|
||||||
|
}))
|
||||||
|
|
||||||
|
questions := []monitoringConfiguredQuestion{
|
||||||
|
{ID: 1},
|
||||||
|
{ID: 2},
|
||||||
|
{ID: 3},
|
||||||
|
}
|
||||||
|
assert.Len(t, limitMonitoringDailyQuestions(questions, 2), 2)
|
||||||
|
assert.Len(t, limitMonitoringDailyQuestions(questions, 25), 3)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecodeDailyMonitoringEnabledPlatformsReportsMalformedJSON(t *testing.T) {
|
||||||
|
platforms, err := decodeDailyMonitoringEnabledPlatforms([]byte(`["qwen"`))
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Nil(t, platforms)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecodeDailyMonitoringEnabledPlatformsReconcilesSupportedPlatforms(t *testing.T) {
|
||||||
|
platforms, err := decodeDailyMonitoringEnabledPlatforms([]byte(`["qwen","unknown","doubao"]`))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, []string{"doubao", "qwen"}, platforms)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildDailyMonitoringPlansFromSelectedClientsRequiresPlatformSnapshot(t *testing.T) {
|
||||||
|
clientOne := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000001")
|
||||||
|
clientTwo := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000002")
|
||||||
|
candidates := []monitoringDailyPlanClientCandidate{
|
||||||
|
{
|
||||||
|
TenantID: 1,
|
||||||
|
WorkspaceID: 10,
|
||||||
|
PrimaryClientID: clientOne,
|
||||||
|
MaxBrands: 3,
|
||||||
|
MaxQuestionsPerBrand: 25,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
TenantID: 1,
|
||||||
|
WorkspaceID: 11,
|
||||||
|
PrimaryClientID: clientTwo,
|
||||||
|
MaxBrands: 2,
|
||||||
|
MaxQuestionsPerBrand: 10,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
enabledByClient := map[monitoringDailyPlanClientKey][]byte{
|
||||||
|
candidates[0].monitoringDailyPlanClientKey(): []byte(`["qwen"]`),
|
||||||
|
}
|
||||||
|
|
||||||
|
plans := buildDailyMonitoringPlansFromSelectedClients(candidates, enabledByClient)
|
||||||
|
|
||||||
|
require.Len(t, plans, 1)
|
||||||
|
assert.Equal(t, int64(1), plans[0].TenantID)
|
||||||
|
assert.Equal(t, int64(10), plans[0].WorkspaceID)
|
||||||
|
assert.Equal(t, 3, plans[0].MaxBrands)
|
||||||
|
assert.Equal(t, 25, plans[0].MaxQuestionsPerBrand)
|
||||||
|
require.NotNil(t, plans[0].PrimaryClientID)
|
||||||
|
assert.Equal(t, clientOne, *plans[0].PrimaryClientID)
|
||||||
|
assert.Equal(t, []byte(`["qwen"]`), plans[0].EnabledJSON)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOrderDailyMonitoringPlansRotatesDeterministicallyByBucket(t *testing.T) {
|
||||||
|
plans := []monitoringDailyPlan{
|
||||||
|
{TenantID: 1},
|
||||||
|
{TenantID: 2},
|
||||||
|
{TenantID: 3},
|
||||||
|
}
|
||||||
|
now := time.Date(2026, 4, 24, 1, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
first := orderDailyMonitoringPlans(plans, "2026-04-24", now)
|
||||||
|
second := orderDailyMonitoringPlans(plans, "2026-04-24", now)
|
||||||
|
|
||||||
|
assert.Equal(t, first, second)
|
||||||
|
assert.Equal(t, []monitoringDailyPlan{{TenantID: 1}, {TenantID: 2}, {TenantID: 3}}, plans)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveDailyMonitoringExecutionOwnerRequiresDesktopRolloutAndClient(t *testing.T) {
|
||||||
|
clientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000001")
|
||||||
|
|
||||||
|
disabled := &MonitoringService{desktopTasksRolloutPercentage: 0}
|
||||||
|
owner, ok, reason := disabled.resolveDailyMonitoringExecutionOwner(monitoringDailyPlan{
|
||||||
|
TenantID: 1,
|
||||||
|
PrimaryClientID: &clientID,
|
||||||
|
})
|
||||||
|
assert.False(t, ok)
|
||||||
|
assert.Empty(t, owner)
|
||||||
|
assert.Equal(t, "desktop_tasks_rollout_disabled", reason)
|
||||||
|
|
||||||
|
enabled := &MonitoringService{desktopTasksRolloutPercentage: 100}
|
||||||
|
owner, ok, reason = enabled.resolveDailyMonitoringExecutionOwner(monitoringDailyPlan{TenantID: 1})
|
||||||
|
assert.False(t, ok)
|
||||||
|
assert.Empty(t, owner)
|
||||||
|
assert.Equal(t, "missing_primary_client", reason)
|
||||||
|
|
||||||
|
owner, ok, reason = enabled.resolveDailyMonitoringExecutionOwner(monitoringDailyPlan{
|
||||||
|
TenantID: 1,
|
||||||
|
PrimaryClientID: &clientID,
|
||||||
|
})
|
||||||
|
assert.True(t, ok)
|
||||||
|
assert.Equal(t, "desktop_tasks", owner)
|
||||||
|
assert.Empty(t, reason)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitoringDailyTaskMetricsRecordsSummaryCounters(t *testing.T) {
|
||||||
|
resetMonitoringDailyTaskMetricsForTest()
|
||||||
|
t.Cleanup(resetMonitoringDailyTaskMetricsForTest)
|
||||||
|
|
||||||
|
recordMonitoringDailyTaskMetrics(&MonitoringDailyTaskGenerationSummary{
|
||||||
|
BusinessDate: "2026-04-24",
|
||||||
|
TenantCount: 3,
|
||||||
|
BrandCount: 2,
|
||||||
|
FailedPlanCount: 1,
|
||||||
|
FailedBrandCount: 2,
|
||||||
|
PlannedTaskCount: 20,
|
||||||
|
DueTaskCount: 8,
|
||||||
|
CreatedTaskCount: 6,
|
||||||
|
DesktopTaskCount: 4,
|
||||||
|
DeferredTaskCount: 2,
|
||||||
|
SkippedPlanCount: 1,
|
||||||
|
}, 1500*time.Millisecond, nil)
|
||||||
|
|
||||||
|
snapshot := MonitoringDailyTaskMetricsSnapshotValue()
|
||||||
|
assert.Equal(t, int64(1), snapshot.RunTotal)
|
||||||
|
assert.Equal(t, int64(1), snapshot.RunCompletedTotal)
|
||||||
|
assert.Equal(t, int64(0), snapshot.RunFailedTotal)
|
||||||
|
assert.Equal(t, int64(1), snapshot.RunPartialFailureTotal)
|
||||||
|
assert.Equal(t, int64(1), snapshot.PlanFailureTotal)
|
||||||
|
assert.Equal(t, int64(2), snapshot.BrandFailureTotal)
|
||||||
|
assert.Equal(t, int64(1), snapshot.SkippedPlanTotal)
|
||||||
|
assert.Equal(t, int64(20), snapshot.PlannedTaskTotal)
|
||||||
|
assert.Equal(t, int64(8), snapshot.DueTaskTotal)
|
||||||
|
assert.Equal(t, int64(6), snapshot.CreatedTaskTotal)
|
||||||
|
assert.Equal(t, int64(4), snapshot.DesktopTaskTotal)
|
||||||
|
assert.Equal(t, int64(2), snapshot.DeferredTaskTotal)
|
||||||
|
assert.Equal(t, "2026-04-24", snapshot.LastBusinessDate)
|
||||||
|
assert.Equal(t, int64(3), snapshot.LastTenantCount)
|
||||||
|
assert.Equal(t, int64(2), snapshot.LastBrandCount)
|
||||||
|
assert.NotNil(t, snapshot.LastRunAt)
|
||||||
|
assert.NotNil(t, snapshot.LastSuccessAt)
|
||||||
|
|
||||||
|
prometheus := MonitoringDailyTaskMetricsPrometheus()
|
||||||
|
assert.Contains(t, prometheus, `monitoring_daily_task_worker_runs_total{status="completed"} 1`)
|
||||||
|
assert.Contains(t, prometheus, `monitoring_daily_task_generation_failures_total{scope="plan"} 1`)
|
||||||
|
assert.Contains(t, prometheus, `monitoring_daily_task_generation_tasks_total{stage="created"} 6`)
|
||||||
|
assert.Contains(t, prometheus, `monitoring_daily_task_worker_last_business_date_info{business_date="2026-04-24"} 1`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitoringDailyTaskMetricsRecordsFatalError(t *testing.T) {
|
||||||
|
resetMonitoringDailyTaskMetricsForTest()
|
||||||
|
t.Cleanup(resetMonitoringDailyTaskMetricsForTest)
|
||||||
|
|
||||||
|
recordMonitoringDailyTaskMetrics(nil, time.Second, context.DeadlineExceeded)
|
||||||
|
|
||||||
|
snapshot := MonitoringDailyTaskMetricsSnapshotValue()
|
||||||
|
assert.Equal(t, int64(1), snapshot.RunTotal)
|
||||||
|
assert.Equal(t, int64(0), snapshot.RunCompletedTotal)
|
||||||
|
assert.Equal(t, int64(1), snapshot.RunFailedTotal)
|
||||||
|
assert.Equal(t, "context_deadline_exceeded", snapshot.LastErrorCode)
|
||||||
|
assert.NotNil(t, snapshot.LastRunAt)
|
||||||
|
assert.NotNil(t, snapshot.LastErrorAt)
|
||||||
|
|
||||||
|
prometheus := MonitoringDailyTaskMetricsPrometheus()
|
||||||
|
assert.Contains(t, prometheus, `monitoring_daily_task_worker_runs_total{status="failed"} 1`)
|
||||||
|
assert.Contains(t, prometheus, `monitoring_daily_task_worker_last_error_info{code="context_deadline_exceeded"} 1`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitoringDailyTaskPrometheusEscapesCarriageReturnLabels(t *testing.T) {
|
||||||
|
resetMonitoringDailyTaskMetricsForTest()
|
||||||
|
t.Cleanup(resetMonitoringDailyTaskMetricsForTest)
|
||||||
|
|
||||||
|
monitoringDailyTaskMetrics.lastBusinessDate.Store("2026-04-24\rbad")
|
||||||
|
|
||||||
|
prometheus := MonitoringDailyTaskMetricsPrometheus()
|
||||||
|
assert.NotContains(t, prometheus, "\r")
|
||||||
|
assert.Contains(t, prometheus, `monitoring_daily_task_worker_last_business_date_info{business_date="2026-04-24\\rbad"} 1`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitoringSchedulerMetricsPrometheusHandlerIncludesRuntimeAndProcessCollectors(t *testing.T) {
|
||||||
|
resetMonitoringDailyTaskMetricsForTest()
|
||||||
|
t.Cleanup(resetMonitoringDailyTaskMetricsForTest)
|
||||||
|
recordMonitoringDailyTaskMetrics(&MonitoringDailyTaskGenerationSummary{CreatedTaskCount: 1}, time.Second, nil)
|
||||||
|
|
||||||
|
recorder := httptest.NewRecorder()
|
||||||
|
request := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||||
|
MonitoringSchedulerMetricsPrometheusHandler().ServeHTTP(recorder, request)
|
||||||
|
|
||||||
|
body := recorder.Body.String()
|
||||||
|
assert.Equal(t, http.StatusOK, recorder.Code)
|
||||||
|
assert.Contains(t, body, `monitoring_daily_task_worker_runs_total{status="completed"} 1`)
|
||||||
|
assert.Contains(t, body, "go_goroutines")
|
||||||
|
if runtime.GOOS == "linux" || runtime.GOOS == "windows" {
|
||||||
|
assert.Contains(t, body, "process_cpu_seconds_total")
|
||||||
|
}
|
||||||
|
assert.True(t, strings.HasPrefix(recorder.Header().Get("Content-Type"), "text/plain"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustParseDailyMonitoringUUID(t *testing.T, value string) uuid.UUID {
|
||||||
|
t.Helper()
|
||||||
|
id, err := uuid.Parse(value)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return id
|
||||||
|
}
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type MonitoringHeartbeatPrimaryLeaseMetricsSnapshot struct {
|
||||||
|
PrimaryHitTotal int64 `json:"primary_hit_total"`
|
||||||
|
NonPrimaryLiveTotal int64 `json:"non_primary_live_total"`
|
||||||
|
InsertedTotal int64 `json:"inserted_total"`
|
||||||
|
RenewedTotal int64 `json:"renewed_total"`
|
||||||
|
ClaimedExpiredTotal int64 `json:"claimed_expired_total"`
|
||||||
|
ClaimedUnavailableTotal int64 `json:"claimed_unavailable_total"`
|
||||||
|
ContentionTotal int64 `json:"contention_total"`
|
||||||
|
ErrorTotal int64 `json:"error_total"`
|
||||||
|
DBInsertTotal int64 `json:"db_insert_total"`
|
||||||
|
DBRenewTotal int64 `json:"db_renew_total"`
|
||||||
|
DBClaimTotal int64 `json:"db_claim_total"`
|
||||||
|
DurationCount int64 `json:"duration_count"`
|
||||||
|
DurationSumSeconds float64 `json:"duration_sum_seconds"`
|
||||||
|
LastDurationSeconds float64 `json:"last_duration_seconds"`
|
||||||
|
LastOutcome string `json:"last_outcome,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type monitoringHeartbeatPrimaryLeaseMetricsRecorder struct {
|
||||||
|
primaryHitTotal atomic.Int64
|
||||||
|
nonPrimaryLiveTotal atomic.Int64
|
||||||
|
insertedTotal atomic.Int64
|
||||||
|
renewedTotal atomic.Int64
|
||||||
|
claimedExpiredTotal atomic.Int64
|
||||||
|
claimedUnavailableTotal atomic.Int64
|
||||||
|
contentionTotal atomic.Int64
|
||||||
|
errorTotal atomic.Int64
|
||||||
|
dbInsertTotal atomic.Int64
|
||||||
|
dbRenewTotal atomic.Int64
|
||||||
|
dbClaimTotal atomic.Int64
|
||||||
|
durationCount atomic.Int64
|
||||||
|
durationSumNanos atomic.Int64
|
||||||
|
lastDurationNanos atomic.Int64
|
||||||
|
lastOutcome atomic.Value
|
||||||
|
}
|
||||||
|
|
||||||
|
var monitoringHeartbeatPrimaryLeaseMetrics = newMonitoringHeartbeatPrimaryLeaseMetricsRecorder()
|
||||||
|
|
||||||
|
func newMonitoringHeartbeatPrimaryLeaseMetricsRecorder() *monitoringHeartbeatPrimaryLeaseMetricsRecorder {
|
||||||
|
return &monitoringHeartbeatPrimaryLeaseMetricsRecorder{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func recordHeartbeatPrimaryLeaseMetric(outcome string, duration time.Duration) {
|
||||||
|
monitoringHeartbeatPrimaryLeaseMetrics.record(outcome, duration)
|
||||||
|
}
|
||||||
|
|
||||||
|
func recordHeartbeatPrimaryLeaseWriteMetric(operation string) {
|
||||||
|
monitoringHeartbeatPrimaryLeaseMetrics.recordWrite(operation)
|
||||||
|
}
|
||||||
|
|
||||||
|
func MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue() MonitoringHeartbeatPrimaryLeaseMetricsSnapshot {
|
||||||
|
return monitoringHeartbeatPrimaryLeaseMetrics.snapshot()
|
||||||
|
}
|
||||||
|
|
||||||
|
func resetMonitoringHeartbeatPrimaryLeaseMetricsForTest() {
|
||||||
|
monitoringHeartbeatPrimaryLeaseMetrics = newMonitoringHeartbeatPrimaryLeaseMetricsRecorder()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) record(outcome string, duration time.Duration) {
|
||||||
|
if r == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if duration < 0 {
|
||||||
|
duration = 0
|
||||||
|
}
|
||||||
|
switch outcome {
|
||||||
|
case "primary_hit":
|
||||||
|
r.primaryHitTotal.Add(1)
|
||||||
|
case "non_primary_live":
|
||||||
|
r.nonPrimaryLiveTotal.Add(1)
|
||||||
|
case "inserted":
|
||||||
|
r.insertedTotal.Add(1)
|
||||||
|
case "renewed":
|
||||||
|
r.renewedTotal.Add(1)
|
||||||
|
case "claimed_expired":
|
||||||
|
r.claimedExpiredTotal.Add(1)
|
||||||
|
case "claimed_unavailable":
|
||||||
|
r.claimedUnavailableTotal.Add(1)
|
||||||
|
case "contention":
|
||||||
|
r.contentionTotal.Add(1)
|
||||||
|
case "error":
|
||||||
|
r.errorTotal.Add(1)
|
||||||
|
}
|
||||||
|
r.durationCount.Add(1)
|
||||||
|
r.durationSumNanos.Add(duration.Nanoseconds())
|
||||||
|
r.lastDurationNanos.Store(duration.Nanoseconds())
|
||||||
|
r.lastOutcome.Store(outcome)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) recordWrite(operation string) {
|
||||||
|
if r == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch operation {
|
||||||
|
case "insert":
|
||||||
|
r.dbInsertTotal.Add(1)
|
||||||
|
case "renew":
|
||||||
|
r.dbRenewTotal.Add(1)
|
||||||
|
case "claim":
|
||||||
|
r.dbClaimTotal.Add(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) snapshot() MonitoringHeartbeatPrimaryLeaseMetricsSnapshot {
|
||||||
|
if r == nil {
|
||||||
|
return MonitoringHeartbeatPrimaryLeaseMetricsSnapshot{}
|
||||||
|
}
|
||||||
|
return MonitoringHeartbeatPrimaryLeaseMetricsSnapshot{
|
||||||
|
PrimaryHitTotal: r.primaryHitTotal.Load(),
|
||||||
|
NonPrimaryLiveTotal: r.nonPrimaryLiveTotal.Load(),
|
||||||
|
InsertedTotal: r.insertedTotal.Load(),
|
||||||
|
RenewedTotal: r.renewedTotal.Load(),
|
||||||
|
ClaimedExpiredTotal: r.claimedExpiredTotal.Load(),
|
||||||
|
ClaimedUnavailableTotal: r.claimedUnavailableTotal.Load(),
|
||||||
|
ContentionTotal: r.contentionTotal.Load(),
|
||||||
|
ErrorTotal: r.errorTotal.Load(),
|
||||||
|
DBInsertTotal: r.dbInsertTotal.Load(),
|
||||||
|
DBRenewTotal: r.dbRenewTotal.Load(),
|
||||||
|
DBClaimTotal: r.dbClaimTotal.Load(),
|
||||||
|
DurationCount: r.durationCount.Load(),
|
||||||
|
DurationSumSeconds: float64(r.durationSumNanos.Load()) / float64(time.Second),
|
||||||
|
LastDurationSeconds: float64(r.lastDurationNanos.Load()) / float64(time.Second),
|
||||||
|
LastOutcome: heartbeatPrimaryMetricLoadString(&r.lastOutcome),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func heartbeatPrimaryMetricLoadString(value *atomic.Value) string {
|
||||||
|
if value == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
raw := value.Load()
|
||||||
|
result, _ := raw.(string)
|
||||||
|
return result
|
||||||
|
}
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type MonitoringHeartbeatSnapshotMetricsSnapshot struct {
|
||||||
|
PlatformReportedTotal int64 `json:"platform_reported_total"`
|
||||||
|
TransactionOpenedTotal int64 `json:"transaction_opened_total"`
|
||||||
|
SnapshotInsertedTotal int64 `json:"snapshot_inserted_total"`
|
||||||
|
SnapshotUpdatedTotal int64 `json:"snapshot_updated_total"`
|
||||||
|
SnapshotRefreshedTotal int64 `json:"snapshot_refreshed_total"`
|
||||||
|
SnapshotSkippedTotal int64 `json:"snapshot_skipped_total"`
|
||||||
|
ReconciledSkippedTaskTotal int64 `json:"reconciled_skipped_task_total"`
|
||||||
|
ProjectionRebuildTotal int64 `json:"projection_rebuild_total"`
|
||||||
|
ErrorTotal int64 `json:"error_total"`
|
||||||
|
DurationCount int64 `json:"duration_count"`
|
||||||
|
DurationSumSeconds float64 `json:"duration_sum_seconds"`
|
||||||
|
LastDurationSeconds float64 `json:"last_duration_seconds"`
|
||||||
|
LastErrorCode string `json:"last_error_code,omitempty"`
|
||||||
|
LastErrorMessage string `json:"last_error_message,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type monitoringHeartbeatSnapshotMetricsEvent struct {
|
||||||
|
Active bool
|
||||||
|
PlatformCount int
|
||||||
|
TransactionOpened bool
|
||||||
|
SnapshotInsertedCount int64
|
||||||
|
SnapshotUpdatedCount int64
|
||||||
|
SnapshotRefreshedCount int64
|
||||||
|
SnapshotSkippedCount int64
|
||||||
|
ReconciledSkippedTaskCount int64
|
||||||
|
ProjectionRebuildCount int64
|
||||||
|
Duration time.Duration
|
||||||
|
Err error
|
||||||
|
}
|
||||||
|
|
||||||
|
type monitoringHeartbeatSnapshotMetricsRecorder struct {
|
||||||
|
platformReportedTotal atomic.Int64
|
||||||
|
transactionOpenedTotal atomic.Int64
|
||||||
|
snapshotInsertedTotal atomic.Int64
|
||||||
|
snapshotUpdatedTotal atomic.Int64
|
||||||
|
snapshotRefreshedTotal atomic.Int64
|
||||||
|
snapshotSkippedTotal atomic.Int64
|
||||||
|
reconciledSkippedTaskTotal atomic.Int64
|
||||||
|
projectionRebuildTotal atomic.Int64
|
||||||
|
errorTotal atomic.Int64
|
||||||
|
durationCount atomic.Int64
|
||||||
|
durationSumNanos atomic.Int64
|
||||||
|
lastDurationNanos atomic.Int64
|
||||||
|
lastErrorCode atomic.Value
|
||||||
|
lastErrorMessage atomic.Value
|
||||||
|
}
|
||||||
|
|
||||||
|
var monitoringHeartbeatSnapshotMetrics = newMonitoringHeartbeatSnapshotMetricsRecorder()
|
||||||
|
|
||||||
|
func newMonitoringHeartbeatSnapshotMetricsRecorder() *monitoringHeartbeatSnapshotMetricsRecorder {
|
||||||
|
return &monitoringHeartbeatSnapshotMetricsRecorder{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func recordMonitoringHeartbeatSnapshotMetrics(event monitoringHeartbeatSnapshotMetricsEvent) {
|
||||||
|
monitoringHeartbeatSnapshotMetrics.record(event)
|
||||||
|
}
|
||||||
|
|
||||||
|
func MonitoringHeartbeatSnapshotMetricsSnapshotValue() MonitoringHeartbeatSnapshotMetricsSnapshot {
|
||||||
|
return monitoringHeartbeatSnapshotMetrics.snapshot()
|
||||||
|
}
|
||||||
|
|
||||||
|
func resetMonitoringHeartbeatSnapshotMetricsForTest() {
|
||||||
|
monitoringHeartbeatSnapshotMetrics = newMonitoringHeartbeatSnapshotMetricsRecorder()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *monitoringHeartbeatSnapshotMetricsRecorder) record(event monitoringHeartbeatSnapshotMetricsEvent) {
|
||||||
|
if r == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if event.Duration < 0 {
|
||||||
|
event.Duration = 0
|
||||||
|
}
|
||||||
|
r.platformReportedTotal.Add(int64(event.PlatformCount))
|
||||||
|
if event.TransactionOpened {
|
||||||
|
r.transactionOpenedTotal.Add(1)
|
||||||
|
}
|
||||||
|
r.snapshotInsertedTotal.Add(event.SnapshotInsertedCount)
|
||||||
|
r.snapshotUpdatedTotal.Add(event.SnapshotUpdatedCount)
|
||||||
|
r.snapshotRefreshedTotal.Add(event.SnapshotRefreshedCount)
|
||||||
|
r.snapshotSkippedTotal.Add(event.SnapshotSkippedCount)
|
||||||
|
r.reconciledSkippedTaskTotal.Add(event.ReconciledSkippedTaskCount)
|
||||||
|
r.projectionRebuildTotal.Add(event.ProjectionRebuildCount)
|
||||||
|
r.durationCount.Add(1)
|
||||||
|
r.durationSumNanos.Add(event.Duration.Nanoseconds())
|
||||||
|
r.lastDurationNanos.Store(event.Duration.Nanoseconds())
|
||||||
|
|
||||||
|
if event.Err != nil {
|
||||||
|
code, message := monitoringDailyMetricError(event.Err)
|
||||||
|
r.errorTotal.Add(1)
|
||||||
|
r.lastErrorCode.Store(code)
|
||||||
|
r.lastErrorMessage.Store(message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *monitoringHeartbeatSnapshotMetricsRecorder) snapshot() MonitoringHeartbeatSnapshotMetricsSnapshot {
|
||||||
|
if r == nil {
|
||||||
|
return MonitoringHeartbeatSnapshotMetricsSnapshot{}
|
||||||
|
}
|
||||||
|
return MonitoringHeartbeatSnapshotMetricsSnapshot{
|
||||||
|
PlatformReportedTotal: r.platformReportedTotal.Load(),
|
||||||
|
TransactionOpenedTotal: r.transactionOpenedTotal.Load(),
|
||||||
|
SnapshotInsertedTotal: r.snapshotInsertedTotal.Load(),
|
||||||
|
SnapshotUpdatedTotal: r.snapshotUpdatedTotal.Load(),
|
||||||
|
SnapshotRefreshedTotal: r.snapshotRefreshedTotal.Load(),
|
||||||
|
SnapshotSkippedTotal: r.snapshotSkippedTotal.Load(),
|
||||||
|
ReconciledSkippedTaskTotal: r.reconciledSkippedTaskTotal.Load(),
|
||||||
|
ProjectionRebuildTotal: r.projectionRebuildTotal.Load(),
|
||||||
|
ErrorTotal: r.errorTotal.Load(),
|
||||||
|
DurationCount: r.durationCount.Load(),
|
||||||
|
DurationSumSeconds: float64(r.durationSumNanos.Load()) / float64(time.Second),
|
||||||
|
LastDurationSeconds: float64(r.lastDurationNanos.Load()) / float64(time.Second),
|
||||||
|
LastErrorCode: monitoringDailyLoadString(&r.lastErrorCode),
|
||||||
|
LastErrorMessage: monitoringDailyLoadString(&r.lastErrorMessage),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,195 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
promcollectors "github.com/prometheus/client_golang/prometheus/collectors"
|
||||||
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||||
|
"github.com/prometheus/common/expfmt"
|
||||||
|
)
|
||||||
|
|
||||||
|
func MonitoringSchedulerMetricsPrometheusHandler() http.Handler {
|
||||||
|
return monitoringPrometheusHandler(true, monitoringDailyTaskMetricsCollector{})
|
||||||
|
}
|
||||||
|
|
||||||
|
func MonitoringDailyTaskMetricsPrometheus() string {
|
||||||
|
return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringDailyTaskMetricsCollector{}))
|
||||||
|
}
|
||||||
|
|
||||||
|
func MonitoringDailyTaskMetricsPrometheusHandler() http.Handler {
|
||||||
|
return monitoringPrometheusHandler(false, monitoringDailyTaskMetricsCollector{})
|
||||||
|
}
|
||||||
|
|
||||||
|
func MonitoringHeartbeatPrimaryLeaseMetricsPrometheus() string {
|
||||||
|
return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringHeartbeatPrimaryLeaseMetricsCollector{}))
|
||||||
|
}
|
||||||
|
|
||||||
|
func MonitoringHeartbeatPrimaryLeaseMetricsPrometheusHandler() http.Handler {
|
||||||
|
return monitoringPrometheusHandler(false, monitoringHeartbeatPrimaryLeaseMetricsCollector{})
|
||||||
|
}
|
||||||
|
|
||||||
|
func MonitoringHeartbeatSnapshotMetricsPrometheus() string {
|
||||||
|
return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringHeartbeatSnapshotMetricsCollector{}))
|
||||||
|
}
|
||||||
|
|
||||||
|
func MonitoringHeartbeatSnapshotMetricsPrometheusHandler() http.Handler {
|
||||||
|
return monitoringPrometheusHandler(false, monitoringHeartbeatSnapshotMetricsCollector{})
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitoringPrometheusHandler(includeRuntime bool, metricCollectors ...prometheus.Collector) http.Handler {
|
||||||
|
return promhttp.HandlerFor(monitoringPrometheusRegistry(includeRuntime, metricCollectors...), promhttp.HandlerOpts{})
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitoringPrometheusRegistry(includeRuntime bool, metricCollectors ...prometheus.Collector) *prometheus.Registry {
|
||||||
|
registry := prometheus.NewRegistry()
|
||||||
|
if includeRuntime {
|
||||||
|
registry.MustRegister(
|
||||||
|
promcollectors.NewGoCollector(),
|
||||||
|
promcollectors.NewProcessCollector(promcollectors.ProcessCollectorOpts{}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
for _, collector := range metricCollectors {
|
||||||
|
registry.MustRegister(collector)
|
||||||
|
}
|
||||||
|
return registry
|
||||||
|
}
|
||||||
|
|
||||||
|
func renderMonitoringPrometheus(gatherer prometheus.Gatherer) string {
|
||||||
|
metricFamilies, err := gatherer.Gather()
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
var b bytes.Buffer
|
||||||
|
encoder := expfmt.NewEncoder(&b, expfmt.NewFormat(expfmt.TypeTextPlain))
|
||||||
|
for _, metricFamily := range metricFamilies {
|
||||||
|
if err := encoder.Encode(metricFamily); err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if closer, ok := encoder.(expfmt.Closer); ok {
|
||||||
|
_ = closer.Close()
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
type monitoringDailyTaskMetricsCollector struct{}
|
||||||
|
|
||||||
|
func (monitoringDailyTaskMetricsCollector) Describe(chan<- *prometheus.Desc) {}
|
||||||
|
|
||||||
|
func (monitoringDailyTaskMetricsCollector) Collect(ch chan<- prometheus.Metric) {
|
||||||
|
snapshot := MonitoringDailyTaskMetricsSnapshotValue()
|
||||||
|
monitoringCounter(ch, "monitoring_daily_task_worker_runs_total", "Total daily monitoring worker runs by terminal status.", float64(snapshot.RunCompletedTotal), []string{"status"}, "completed")
|
||||||
|
monitoringCounter(ch, "monitoring_daily_task_worker_runs_total", "Total daily monitoring worker runs by terminal status.", float64(snapshot.RunFailedTotal), []string{"status"}, "failed")
|
||||||
|
monitoringCounter(ch, "monitoring_daily_task_worker_partial_failure_runs_total", "Total completed daily monitoring worker runs with at least one plan or brand failure.", float64(snapshot.RunPartialFailureTotal), nil)
|
||||||
|
monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.PlanFailureTotal), []string{"scope"}, "plan")
|
||||||
|
monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.BrandFailureTotal), []string{"scope"}, "brand")
|
||||||
|
monitoringCounter(ch, "monitoring_daily_task_generation_skipped_plans_total", "Total daily monitoring plans skipped before generation.", float64(snapshot.SkippedPlanTotal), nil)
|
||||||
|
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.PlannedTaskTotal), []string{"stage"}, "planned")
|
||||||
|
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DueTaskTotal), []string{"stage"}, "due")
|
||||||
|
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.CreatedTaskTotal), []string{"stage"}, "created")
|
||||||
|
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DesktopTaskTotal), []string{"stage"}, "desktop")
|
||||||
|
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DeferredTaskTotal), []string{"stage"}, "deferred")
|
||||||
|
monitoringGauge(ch, "monitoring_daily_task_worker_last_run_timestamp_seconds", "Unix timestamp of the last daily monitoring worker run completion.", monitoringDailyTimeSeconds(snapshot.LastRunAt), nil)
|
||||||
|
monitoringGauge(ch, "monitoring_daily_task_worker_last_success_timestamp_seconds", "Unix timestamp of the last completed daily monitoring worker run.", monitoringDailyTimeSeconds(snapshot.LastSuccessAt), nil)
|
||||||
|
monitoringGauge(ch, "monitoring_daily_task_worker_last_error_timestamp_seconds", "Unix timestamp of the last failed daily monitoring worker run.", monitoringDailyTimeSeconds(snapshot.LastErrorAt), nil)
|
||||||
|
monitoringGauge(ch, "monitoring_daily_task_worker_last_duration_seconds", "Duration of the last daily monitoring worker run.", snapshot.LastDurationSeconds, nil)
|
||||||
|
monitoringGauge(ch, "monitoring_daily_task_worker_last_scope_count", "Tenant and brand counts observed in the last daily monitoring worker run.", float64(snapshot.LastTenantCount), []string{"scope"}, "tenant")
|
||||||
|
monitoringGauge(ch, "monitoring_daily_task_worker_last_scope_count", "Tenant and brand counts observed in the last daily monitoring worker run.", float64(snapshot.LastBrandCount), []string{"scope"}, "brand")
|
||||||
|
if snapshot.LastBusinessDate != "" {
|
||||||
|
monitoringGauge(ch, "monitoring_daily_task_worker_last_business_date_info", "Business date processed by the last daily monitoring worker run.", 1, []string{"business_date"}, snapshot.LastBusinessDate)
|
||||||
|
}
|
||||||
|
if snapshot.LastErrorCode != "" {
|
||||||
|
monitoringGauge(ch, "monitoring_daily_task_worker_last_error_info", "Last fatal daily monitoring worker error code.", 1, []string{"code"}, snapshot.LastErrorCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type monitoringHeartbeatPrimaryLeaseMetricsCollector struct{}
|
||||||
|
|
||||||
|
func (monitoringHeartbeatPrimaryLeaseMetricsCollector) Describe(chan<- *prometheus.Desc) {}
|
||||||
|
|
||||||
|
func (monitoringHeartbeatPrimaryLeaseMetricsCollector) Collect(ch chan<- prometheus.Metric) {
|
||||||
|
snapshot := MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue()
|
||||||
|
help := "Total monitoring heartbeat primary lease resolutions by outcome."
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.PrimaryHitTotal), []string{"outcome"}, "primary_hit")
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.NonPrimaryLiveTotal), []string{"outcome"}, "non_primary_live")
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.InsertedTotal), []string{"outcome"}, "inserted")
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.RenewedTotal), []string{"outcome"}, "renewed")
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ClaimedExpiredTotal), []string{"outcome"}, "claimed_expired")
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ClaimedUnavailableTotal), []string{"outcome"}, "claimed_unavailable")
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ContentionTotal), []string{"outcome"}, "contention")
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ErrorTotal), []string{"outcome"}, "error")
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBInsertTotal), []string{"operation"}, "insert")
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBRenewTotal), []string{"operation"}, "renew")
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBClaimTotal), []string{"operation"}, "claim")
|
||||||
|
monitoringSummary(ch, "monitoring_heartbeat_primary_lease_duration_seconds", "Monitoring heartbeat primary lease resolution duration summary.", snapshot.DurationCount, snapshot.DurationSumSeconds, nil)
|
||||||
|
monitoringGauge(ch, "monitoring_heartbeat_primary_lease_duration_seconds_last", "Last monitoring heartbeat primary lease resolution duration in seconds.", snapshot.LastDurationSeconds, nil)
|
||||||
|
if snapshot.LastOutcome != "" {
|
||||||
|
monitoringGauge(ch, "monitoring_heartbeat_primary_lease_last_outcome_info", "Last monitoring heartbeat primary lease resolution outcome.", 1, []string{"outcome"}, snapshot.LastOutcome)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type monitoringHeartbeatSnapshotMetricsCollector struct{}
|
||||||
|
|
||||||
|
func (monitoringHeartbeatSnapshotMetricsCollector) Describe(chan<- *prometheus.Desc) {}
|
||||||
|
|
||||||
|
func (monitoringHeartbeatSnapshotMetricsCollector) Collect(ch chan<- prometheus.Metric) {
|
||||||
|
snapshot := MonitoringHeartbeatSnapshotMetricsSnapshotValue()
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_snapshot_platform_reports_total", "Total monitoring heartbeat platform reports received.", float64(snapshot.PlatformReportedTotal), nil)
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_snapshot_transactions_total", "Total monitoring heartbeat snapshot transactions opened.", float64(snapshot.TransactionOpenedTotal), nil)
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotInsertedTotal), []string{"operation"}, "insert")
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotUpdatedTotal), []string{"operation"}, "update")
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotRefreshedTotal), []string{"operation"}, "refresh")
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_snapshot_skips_total", "Total monitoring heartbeat access snapshot reports skipped because state was fresh and unchanged.", float64(snapshot.SnapshotSkippedTotal), nil)
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_snapshot_reconciled_tasks_total", "Total pending monitoring tasks skipped by heartbeat access reconciliation.", float64(snapshot.ReconciledSkippedTaskTotal), nil)
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_snapshot_projection_rebuilds_total", "Total monitoring daily projection rebuilds triggered by heartbeat snapshots.", float64(snapshot.ProjectionRebuildTotal), nil)
|
||||||
|
monitoringCounter(ch, "monitoring_heartbeat_snapshot_errors_total", "Total monitoring heartbeat snapshot errors.", float64(snapshot.ErrorTotal), nil)
|
||||||
|
monitoringSummary(ch, "monitoring_heartbeat_snapshot_duration_seconds", "Monitoring heartbeat snapshot processing duration summary.", snapshot.DurationCount, snapshot.DurationSumSeconds, nil)
|
||||||
|
monitoringGauge(ch, "monitoring_heartbeat_snapshot_duration_seconds_last", "Last monitoring heartbeat snapshot processing duration in seconds.", snapshot.LastDurationSeconds, nil)
|
||||||
|
if snapshot.LastErrorCode != "" {
|
||||||
|
monitoringGauge(ch, "monitoring_heartbeat_snapshot_last_error_info", "Last monitoring heartbeat snapshot error code.", 1, []string{"code"}, snapshot.LastErrorCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitoringCounter(ch chan<- prometheus.Metric, name, help string, value float64, labelNames []string, labelValues ...string) {
|
||||||
|
monitoringMetric(ch, name, help, prometheus.CounterValue, value, labelNames, labelValues...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitoringGauge(ch chan<- prometheus.Metric, name, help string, value float64, labelNames []string, labelValues ...string) {
|
||||||
|
monitoringMetric(ch, name, help, prometheus.GaugeValue, value, labelNames, labelValues...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitoringMetric(ch chan<- prometheus.Metric, name, help string, valueType prometheus.ValueType, value float64, labelNames []string, labelValues ...string) {
|
||||||
|
ch <- prometheus.MustNewConstMetric(
|
||||||
|
prometheus.NewDesc(name, help, labelNames, nil),
|
||||||
|
valueType,
|
||||||
|
value,
|
||||||
|
monitoringPrometheusLabelValues(labelValues)...,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitoringSummary(ch chan<- prometheus.Metric, name, help string, count int64, sum float64, labelNames []string, labelValues ...string) {
|
||||||
|
if count < 0 {
|
||||||
|
count = 0
|
||||||
|
}
|
||||||
|
ch <- prometheus.MustNewConstSummary(
|
||||||
|
prometheus.NewDesc(name, help, labelNames, nil),
|
||||||
|
uint64(count),
|
||||||
|
sum,
|
||||||
|
nil,
|
||||||
|
monitoringPrometheusLabelValues(labelValues)...,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitoringPrometheusLabelValues(values []string) []string {
|
||||||
|
if len(values) == 0 {
|
||||||
|
return values
|
||||||
|
}
|
||||||
|
result := make([]string, len(values))
|
||||||
|
for idx, value := range values {
|
||||||
|
result[idx] = strings.ReplaceAll(value, "\r", `\r`)
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
@@ -223,8 +223,7 @@ func (s *MonitoringService) loadMonitorDesktopAccountMap(
|
|||||||
rows, err := s.businessPool.Query(ctx, `
|
rows, err := s.businessPool.Query(ctx, `
|
||||||
SELECT DISTINCT ON (platform_id)
|
SELECT DISTINCT ON (platform_id)
|
||||||
platform_id,
|
platform_id,
|
||||||
desktop_id,
|
desktop_id
|
||||||
health
|
|
||||||
FROM platform_accounts
|
FROM platform_accounts
|
||||||
WHERE tenant_id = $1
|
WHERE tenant_id = $1
|
||||||
AND workspace_id = $2
|
AND workspace_id = $2
|
||||||
@@ -251,14 +250,10 @@ func (s *MonitoringService) loadMonitorDesktopAccountMap(
|
|||||||
var (
|
var (
|
||||||
platformID string
|
platformID string
|
||||||
desktopID uuid.UUID
|
desktopID uuid.UUID
|
||||||
health string
|
|
||||||
)
|
)
|
||||||
if scanErr := rows.Scan(&platformID, &desktopID, &health); scanErr != nil {
|
if scanErr := rows.Scan(&platformID, &desktopID); scanErr != nil {
|
||||||
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to parse monitor desktop accounts")
|
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to parse monitor desktop accounts")
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(health) != "live" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
result[platformID] = desktopID
|
result[platformID] = desktopID
|
||||||
}
|
}
|
||||||
if err := rows.Err(); err != nil {
|
if err := rows.Err(); err != nil {
|
||||||
|
|||||||
@@ -3,11 +3,9 @@ package app
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"encoding/json"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/jackc/pgx/v5"
|
"github.com/jackc/pgx/v5"
|
||||||
"github.com/jackc/pgx/v5/pgxpool"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type monitoringBrandDailyAggregate struct {
|
type monitoringBrandDailyAggregate struct {
|
||||||
@@ -33,7 +31,7 @@ type monitoringPlatformDailyAggregate struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *MonitoringCallbackService) rebuildBrandDailyProjection(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDate time.Time) error {
|
func (s *MonitoringCallbackService) rebuildBrandDailyProjection(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDate time.Time) error {
|
||||||
platforms, err := loadMonitoringProjectionPlatforms(ctx, s.businessPool, tenantID)
|
platforms, err := loadMonitoringProjectionPlatforms(ctx, tx, tenantID, brandID, businessDate)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -133,29 +131,8 @@ func (s *MonitoringCallbackService) rebuildBrandDailyProjection(ctx context.Cont
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadMonitoringProjectionPlatforms(ctx context.Context, businessPool *pgxpool.Pool, tenantID int64) ([]monitoringPlatformMetadata, error) {
|
func loadMonitoringProjectionPlatforms(_ context.Context, _ pgx.Tx, _ int64, _ int64, _ time.Time) ([]monitoringPlatformMetadata, error) {
|
||||||
enabled := []string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"}
|
return defaultMonitoringPlatformMetadata(), nil
|
||||||
|
|
||||||
var enabledJSON []byte
|
|
||||||
if err := businessPool.QueryRow(ctx, `
|
|
||||||
SELECT enabled_platforms
|
|
||||||
FROM tenant_monitoring_quotas
|
|
||||||
WHERE tenant_id = $1
|
|
||||||
`, tenantID).Scan(&enabledJSON); err != nil {
|
|
||||||
if err == pgx.ErrNoRows {
|
|
||||||
return resolveMonitoringPlatforms(enabled), nil
|
|
||||||
}
|
|
||||||
return nil, monitoringInternalError(50041, "quota_lookup_failed", "failed to load monitoring quota", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(enabledJSON) > 0 {
|
|
||||||
var configured []string
|
|
||||||
if jsonErr := json.Unmarshal(enabledJSON, &configured); jsonErr == nil && len(configured) > 0 {
|
|
||||||
enabled = monitoringPlatformIDs(defaultMonitoringPlatforms)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return resolveMonitoringPlatforms(enabled), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MonitoringCallbackService) loadMonitoringEnabledQuestionCount(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDate time.Time) (int64, error) {
|
func (s *MonitoringCallbackService) loadMonitoringEnabledQuestionCount(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDate time.Time) (int64, error) {
|
||||||
|
|||||||
@@ -30,6 +30,10 @@ const (
|
|||||||
monitoringCollectorType = "desktop"
|
monitoringCollectorType = "desktop"
|
||||||
monitoringOnlineDuration = 20 * time.Minute
|
monitoringOnlineDuration = 20 * time.Minute
|
||||||
monitoringCollectNowQuestionLimit = 5
|
monitoringCollectNowQuestionLimit = 5
|
||||||
|
|
||||||
|
monitoringPlatformAuthorizationAuthorized = "authorized"
|
||||||
|
monitoringPlatformAuthorizationNoDesktopClient = "no_desktop_client"
|
||||||
|
monitoringPlatformAuthorizationNoAuthorizedPlatforms = "no_authorized_platforms"
|
||||||
)
|
)
|
||||||
|
|
||||||
type MonitoringService struct {
|
type MonitoringService struct {
|
||||||
@@ -38,12 +42,14 @@ type MonitoringService struct {
|
|||||||
rabbitMQ *rabbitmq.Client
|
rabbitMQ *rabbitmq.Client
|
||||||
logger *zap.Logger
|
logger *zap.Logger
|
||||||
desktopTasksRolloutPercentage int
|
desktopTasksRolloutPercentage int
|
||||||
|
brandLibraryConfig config.BrandLibraryConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewMonitoringService(
|
func NewMonitoringService(
|
||||||
businessPool, monitoringPool *pgxpool.Pool,
|
businessPool, monitoringPool *pgxpool.Pool,
|
||||||
rabbitMQClient *rabbitmq.Client,
|
rabbitMQClient *rabbitmq.Client,
|
||||||
dispatchCfg config.MonitoringDispatchConfig,
|
dispatchCfg config.MonitoringDispatchConfig,
|
||||||
|
brandLibraryCfg config.BrandLibraryConfig,
|
||||||
logger *zap.Logger,
|
logger *zap.Logger,
|
||||||
) *MonitoringService {
|
) *MonitoringService {
|
||||||
return &MonitoringService{
|
return &MonitoringService{
|
||||||
@@ -52,6 +58,7 @@ func NewMonitoringService(
|
|||||||
rabbitMQ: rabbitMQClient,
|
rabbitMQ: rabbitMQClient,
|
||||||
logger: logger,
|
logger: logger,
|
||||||
desktopTasksRolloutPercentage: dispatchCfg.DesktopTasksRolloutPercentage,
|
desktopTasksRolloutPercentage: dispatchCfg.DesktopTasksRolloutPercentage,
|
||||||
|
brandLibraryConfig: brandLibraryCfg,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -66,6 +73,9 @@ type MonitoringDashboardCompositeResponse struct {
|
|||||||
|
|
||||||
type MonitoringDashboardRuntime struct {
|
type MonitoringDashboardRuntime struct {
|
||||||
CurrentUserClientOnline bool `json:"current_user_client_online"`
|
CurrentUserClientOnline bool `json:"current_user_client_online"`
|
||||||
|
PlatformAuthorizationStatus string `json:"platform_authorization_status"`
|
||||||
|
AuthorizedMonitoringPlatformIDs []string `json:"authorized_monitoring_platform_ids"`
|
||||||
|
AuthorizedMonitoringPlatformCount int `json:"authorized_monitoring_platform_count"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type MonitoringOverview struct {
|
type MonitoringOverview struct {
|
||||||
@@ -127,6 +137,7 @@ type MonitoringQuestionDetailResponse struct {
|
|||||||
QuestionID int64 `json:"question_id"`
|
QuestionID int64 `json:"question_id"`
|
||||||
QuestionHash string `json:"question_hash"`
|
QuestionHash string `json:"question_hash"`
|
||||||
QuestionText string `json:"question_text"`
|
QuestionText string `json:"question_text"`
|
||||||
|
PlatformAuthorizationStatus string `json:"platform_authorization_status"`
|
||||||
TimeWindow MonitoringQuestionDetailTimeWindow `json:"time_window"`
|
TimeWindow MonitoringQuestionDetailTimeWindow `json:"time_window"`
|
||||||
Platforms []MonitoringQuestionDetailPlatform `json:"platforms"`
|
Platforms []MonitoringQuestionDetailPlatform `json:"platforms"`
|
||||||
CitationAnalysis []MonitoringQuestionCitationStats `json:"citation_analysis"`
|
CitationAnalysis []MonitoringQuestionCitationStats `json:"citation_analysis"`
|
||||||
@@ -216,6 +227,16 @@ type monitoringQuotaConfig struct {
|
|||||||
EnabledPlatforms []string
|
EnabledPlatforms []string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func monitoringPlatformAuthorizationStatus(quota monitoringQuotaConfig) string {
|
||||||
|
if quota.PrimaryClientID == nil {
|
||||||
|
return monitoringPlatformAuthorizationNoDesktopClient
|
||||||
|
}
|
||||||
|
if len(quota.EnabledPlatforms) == 0 {
|
||||||
|
return monitoringPlatformAuthorizationNoAuthorizedPlatforms
|
||||||
|
}
|
||||||
|
return monitoringPlatformAuthorizationAuthorized
|
||||||
|
}
|
||||||
|
|
||||||
type monitoringBrandIdentity struct {
|
type monitoringBrandIdentity struct {
|
||||||
ID int64
|
ID int64
|
||||||
Name string
|
Name string
|
||||||
@@ -290,6 +311,7 @@ func (s *MonitoringService) DashboardComposite(
|
|||||||
aiPlatformID *string,
|
aiPlatformID *string,
|
||||||
) (*MonitoringDashboardCompositeResponse, error) {
|
) (*MonitoringDashboardCompositeResponse, error) {
|
||||||
actor := auth.MustActor(ctx)
|
actor := auth.MustActor(ctx)
|
||||||
|
workspaceID := auth.CurrentWorkspaceID(ctx)
|
||||||
|
|
||||||
brand, err := s.resolveBrand(ctx, actor.TenantID, brandID)
|
brand, err := s.resolveBrand(ctx, actor.TenantID, brandID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -302,12 +324,12 @@ func (s *MonitoringService) DashboardComposite(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
quota, err := s.loadQuota(ctx, actor.TenantID)
|
quota, err := s.loadQuota(ctx, actor, workspaceID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
runtime, err := s.loadDashboardRuntime(ctx, actor)
|
runtime, err := s.loadDashboardRuntime(ctx, actor, workspaceID, quota)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -318,7 +340,7 @@ func (s *MonitoringService) DashboardComposite(
|
|||||||
}
|
}
|
||||||
questionIDs := configuredQuestionIDs(configuredQuestions)
|
questionIDs := configuredQuestionIDs(configuredQuestions)
|
||||||
|
|
||||||
platforms := resolveMonitoringPlatforms(quota.EnabledPlatforms)
|
platforms := defaultMonitoringPlatformMetadata()
|
||||||
selectedPlatformID := normalizedOptionalMonitoringPlatformPointer(aiPlatformID)
|
selectedPlatformID := normalizedOptionalMonitoringPlatformPointer(aiPlatformID)
|
||||||
filteredPlatforms := filterMonitoringPlatforms(platforms, selectedPlatformID)
|
filteredPlatforms := filterMonitoringPlatforms(platforms, selectedPlatformID)
|
||||||
startDate, endDate, err := resolveDashboardDateWindow(days, businessDate)
|
startDate, endDate, err := resolveDashboardDateWindow(days, businessDate)
|
||||||
@@ -334,7 +356,7 @@ func (s *MonitoringService) DashboardComposite(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
accessStates, err := s.loadAccessStates(ctx, actor.TenantID, quota.PrimaryClientID, endDate)
|
accessStates, err := s.loadAccessStates(ctx, actor.TenantID, workspaceID, quota.PrimaryClientID, endDate)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -369,19 +391,27 @@ func (s *MonitoringService) DashboardComposite(
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MonitoringService) loadDashboardRuntime(ctx context.Context, actor auth.Actor) (MonitoringDashboardRuntime, error) {
|
func (s *MonitoringService) loadDashboardRuntime(ctx context.Context, actor auth.Actor, workspaceID int64, quota monitoringQuotaConfig) (MonitoringDashboardRuntime, error) {
|
||||||
online, err := s.hasOnlineClientForUser(ctx, actor.TenantID, actor.PrimaryWorkspaceID, actor.UserID)
|
online := false
|
||||||
|
if quota.PrimaryClientID != nil {
|
||||||
|
var err error
|
||||||
|
online, err = s.isClientOnline(ctx, actor.TenantID, workspaceID, *quota.PrimaryClientID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return MonitoringDashboardRuntime{}, err
|
return MonitoringDashboardRuntime{}, err
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return MonitoringDashboardRuntime{
|
return MonitoringDashboardRuntime{
|
||||||
CurrentUserClientOnline: online,
|
CurrentUserClientOnline: online,
|
||||||
|
PlatformAuthorizationStatus: monitoringPlatformAuthorizationStatus(quota),
|
||||||
|
AuthorizedMonitoringPlatformIDs: append([]string(nil), quota.EnabledPlatforms...),
|
||||||
|
AuthorizedMonitoringPlatformCount: len(quota.EnabledPlatforms),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MonitoringService) QuestionDetail(ctx context.Context, brandID, questionID int64, dateFrom, dateTo string, questionHash string, aiPlatformID *string) (*MonitoringQuestionDetailResponse, error) {
|
func (s *MonitoringService) QuestionDetail(ctx context.Context, brandID, questionID int64, dateFrom, dateTo string, questionHash string, aiPlatformID *string) (*MonitoringQuestionDetailResponse, error) {
|
||||||
actor := auth.MustActor(ctx)
|
actor := auth.MustActor(ctx)
|
||||||
|
workspaceID := auth.CurrentWorkspaceID(ctx)
|
||||||
|
|
||||||
brand, err := s.resolveBrand(ctx, actor.TenantID, brandID)
|
brand, err := s.resolveBrand(ctx, actor.TenantID, brandID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -393,12 +423,12 @@ func (s *MonitoringService) QuestionDetail(ctx context.Context, brandID, questio
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
quota, err := s.loadQuota(ctx, actor.TenantID)
|
quota, err := s.loadQuota(ctx, actor, workspaceID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
platforms := resolveMonitoringPlatforms(quota.EnabledPlatforms)
|
|
||||||
selectedPlatformID := normalizedOptionalMonitoringPlatformPointer(aiPlatformID)
|
selectedPlatformID := normalizedOptionalMonitoringPlatformPointer(aiPlatformID)
|
||||||
|
platforms := filterMonitoringPlatforms(defaultMonitoringPlatformMetadata(), selectedPlatformID)
|
||||||
|
|
||||||
questionText, err := s.loadQuestionText(ctx, actor.TenantID, brand.ID, questionID)
|
questionText, err := s.loadQuestionText(ctx, actor.TenantID, brand.ID, questionID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -418,7 +448,7 @@ func (s *MonitoringService) QuestionDetail(ctx context.Context, brandID, questio
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
accessStates, err := s.loadAccessStates(ctx, actor.TenantID, quota.PrimaryClientID, toDate)
|
accessStates, err := s.loadAccessStates(ctx, actor.TenantID, workspaceID, quota.PrimaryClientID, toDate)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -484,6 +514,7 @@ func (s *MonitoringService) QuestionDetail(ctx context.Context, brandID, questio
|
|||||||
QuestionID: questionID,
|
QuestionID: questionID,
|
||||||
QuestionHash: encodeQuestionHash(hashBytes),
|
QuestionHash: encodeQuestionHash(hashBytes),
|
||||||
QuestionText: questionText,
|
QuestionText: questionText,
|
||||||
|
PlatformAuthorizationStatus: monitoringPlatformAuthorizationStatus(quota),
|
||||||
TimeWindow: MonitoringQuestionDetailTimeWindow{
|
TimeWindow: MonitoringQuestionDetailTimeWindow{
|
||||||
DateFrom: fromDate.Format("2006-01-02"),
|
DateFrom: fromDate.Format("2006-01-02"),
|
||||||
DateTo: toDate.Format("2006-01-02"),
|
DateTo: toDate.Format("2006-01-02"),
|
||||||
@@ -501,6 +532,7 @@ func (s *MonitoringService) CollectNow(
|
|||||||
options MonitoringCollectNowOptions,
|
options MonitoringCollectNowOptions,
|
||||||
) (*MonitoringCollectNowResponse, error) {
|
) (*MonitoringCollectNowResponse, error) {
|
||||||
actor := auth.MustActor(ctx)
|
actor := auth.MustActor(ctx)
|
||||||
|
workspaceID := auth.CurrentWorkspaceID(ctx)
|
||||||
|
|
||||||
brand, err := s.resolveBrand(ctx, actor.TenantID, brandID)
|
brand, err := s.resolveBrand(ctx, actor.TenantID, brandID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -513,7 +545,7 @@ func (s *MonitoringService) CollectNow(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
quota, err := s.loadQuota(ctx, actor.TenantID)
|
quota, err := s.loadQuota(ctx, actor, workspaceID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -540,19 +572,7 @@ func (s *MonitoringService) CollectNow(
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
platforms := resolveMonitoringPlatforms(quota.EnabledPlatforms)
|
clientID, err := s.resolveCollectNowClientID(ctx, actor.TenantID, workspaceID, actor.UserID, quota.PrimaryClientID, options.TargetClientID)
|
||||||
platforms, err = intersectMonitoringPlatforms(platforms, options.PlatformIDs)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
questionIDs := configuredQuestionIDs(configuredQuestions)
|
|
||||||
platformIDs := monitoringPlatformIDs(platforms)
|
|
||||||
executionOwner := "legacy"
|
|
||||||
if s.shouldUseDesktopTasksExecution(actor.TenantID) {
|
|
||||||
executionOwner = "desktop_tasks"
|
|
||||||
}
|
|
||||||
|
|
||||||
clientID, err := s.resolveCollectNowClientID(ctx, actor.TenantID, actor.PrimaryWorkspaceID, actor.UserID, options.TargetClientID)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -560,10 +580,37 @@ func (s *MonitoringService) CollectNow(
|
|||||||
return nil, response.ErrConflict(40941, "desktop_client_offline", "current user's monitoring desktop client is offline")
|
return nil, response.ErrConflict(40941, "desktop_client_offline", "current user's monitoring desktop client is offline")
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := s.ensureClientOnline(ctx, actor.TenantID, actor.PrimaryWorkspaceID, *clientID); err != nil {
|
if err := s.ensureClientOnline(ctx, actor.TenantID, workspaceID, *clientID); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
targetPlatformIDs, err := s.loadMonitoringClientPlatformIDs(ctx, actor.TenantID, workspaceID, *clientID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
platforms := resolveMonitoringPlatforms(targetPlatformIDs)
|
||||||
|
platforms, err = intersectMonitoringPlatforms(platforms, options.PlatformIDs)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(platforms) == 0 {
|
||||||
|
return &MonitoringCollectNowResponse{
|
||||||
|
CollectionMode: quota.CollectionMode,
|
||||||
|
RefreshedTaskCount: 0,
|
||||||
|
CreatedTaskCount: 0,
|
||||||
|
LeasedTaskCount: 0,
|
||||||
|
CompletedTaskCount: 0,
|
||||||
|
HasEffectiveSnapshot: true,
|
||||||
|
Message: "当前桌面端暂无已授权的 AI 平台,未创建采集任务",
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
questionIDs := configuredQuestionIDs(configuredQuestions)
|
||||||
|
platformIDs := monitoringPlatformIDs(platforms)
|
||||||
|
executionOwner := "legacy"
|
||||||
|
if s.shouldUseDesktopTasksExecution(actor.TenantID) {
|
||||||
|
executionOwner = "desktop_tasks"
|
||||||
|
}
|
||||||
|
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
todayTime, _ := monitoringBusinessDayAndDateAt(now)
|
todayTime, _ := monitoringBusinessDayAndDateAt(now)
|
||||||
requestID := uuid.New()
|
requestID := uuid.New()
|
||||||
@@ -584,7 +631,7 @@ func (s *MonitoringService) CollectNow(
|
|||||||
ctx,
|
ctx,
|
||||||
tx,
|
tx,
|
||||||
actor.TenantID,
|
actor.TenantID,
|
||||||
actor.PrimaryWorkspaceID,
|
workspaceID,
|
||||||
brand.ID,
|
brand.ID,
|
||||||
actor.UserID,
|
actor.UserID,
|
||||||
scopeHash,
|
scopeHash,
|
||||||
@@ -663,7 +710,7 @@ func (s *MonitoringService) CollectNow(
|
|||||||
ctx,
|
ctx,
|
||||||
tx,
|
tx,
|
||||||
actor.TenantID,
|
actor.TenantID,
|
||||||
actor.PrimaryWorkspaceID,
|
workspaceID,
|
||||||
todayTime,
|
todayTime,
|
||||||
questionIDs,
|
questionIDs,
|
||||||
platformIDs,
|
platformIDs,
|
||||||
@@ -722,7 +769,7 @@ func (s *MonitoringService) CollectNow(
|
|||||||
$12, $13, $14,
|
$12, $13, $14,
|
||||||
$15, $16
|
$15, $16
|
||||||
)
|
)
|
||||||
`, requestID, actor.TenantID, actor.PrimaryWorkspaceID, brand.ID, nullableInt64(keywordID), actor.UserID, *clientID, scopeHash, requestScopeJSON, affectedTaskCount, affectedTaskCount, abortedQueuedCount, int64(len(interruptTaskIDs)), interruptGeneration, message, ttlExpiresAt); err != nil {
|
`, requestID, actor.TenantID, workspaceID, brand.ID, nullableInt64(keywordID), actor.UserID, *clientID, scopeHash, requestScopeJSON, affectedTaskCount, affectedTaskCount, abortedQueuedCount, int64(len(interruptTaskIDs)), interruptGeneration, message, ttlExpiresAt); err != nil {
|
||||||
return nil, response.ErrInternal(50041, "request_create_failed", "failed to persist monitoring collect-now request")
|
return nil, response.ErrInternal(50041, "request_create_failed", "failed to persist monitoring collect-now request")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -730,7 +777,7 @@ func (s *MonitoringService) CollectNow(
|
|||||||
ctx,
|
ctx,
|
||||||
tx,
|
tx,
|
||||||
requestID,
|
requestID,
|
||||||
actor.PrimaryWorkspaceID,
|
workspaceID,
|
||||||
*clientID,
|
*clientID,
|
||||||
affectedTaskCount,
|
affectedTaskCount,
|
||||||
interruptGeneration,
|
interruptGeneration,
|
||||||
@@ -748,8 +795,6 @@ func (s *MonitoringService) CollectNow(
|
|||||||
return nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring collect-now")
|
return nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring collect-now")
|
||||||
}
|
}
|
||||||
|
|
||||||
_ = s.persistPrimaryClientID(ctx, actor.TenantID, actor.PrimaryWorkspaceID, *clientID)
|
|
||||||
|
|
||||||
phase2PublishedTasks := make([]*repository.DesktopTask, 0)
|
phase2PublishedTasks := make([]*repository.DesktopTask, 0)
|
||||||
phase2DispatchReady := executionOwner == "desktop_tasks"
|
phase2DispatchReady := executionOwner == "desktop_tasks"
|
||||||
if len(phase2TaskSpecs) > 0 {
|
if len(phase2TaskSpecs) > 0 {
|
||||||
@@ -877,6 +922,7 @@ func (s *MonitoringService) CollectNow(
|
|||||||
func (s *MonitoringService) resolveCollectNowClientID(
|
func (s *MonitoringService) resolveCollectNowClientID(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
tenantID, workspaceID, userID int64,
|
tenantID, workspaceID, userID int64,
|
||||||
|
defaultClientID *uuid.UUID,
|
||||||
preferredClientID *uuid.UUID,
|
preferredClientID *uuid.UUID,
|
||||||
) (*uuid.UUID, error) {
|
) (*uuid.UUID, error) {
|
||||||
if preferredClientID != nil {
|
if preferredClientID != nil {
|
||||||
@@ -885,7 +931,7 @@ func (s *MonitoringService) resolveCollectNowClientID(
|
|||||||
}
|
}
|
||||||
return preferredClientID, nil
|
return preferredClientID, nil
|
||||||
}
|
}
|
||||||
return s.findLatestOnlineClientForUser(ctx, tenantID, workspaceID, userID)
|
return defaultClientID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MonitoringService) hasOnlineClientForUser(ctx context.Context, tenantID, workspaceID, userID int64) (bool, error) {
|
func (s *MonitoringService) hasOnlineClientForUser(ctx context.Context, tenantID, workspaceID, userID int64) (bool, error) {
|
||||||
@@ -968,6 +1014,10 @@ func monitoringPlatformIDs(items []monitoringPlatformMetadata) []string {
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func defaultMonitoringPlatformMetadata() []monitoringPlatformMetadata {
|
||||||
|
return append([]monitoringPlatformMetadata(nil), defaultMonitoringPlatforms...)
|
||||||
|
}
|
||||||
|
|
||||||
func normalizeMonitoringPlatformID(platformID string) string {
|
func normalizeMonitoringPlatformID(platformID string) string {
|
||||||
return strings.ToLower(strings.TrimSpace(platformID))
|
return strings.ToLower(strings.TrimSpace(platformID))
|
||||||
}
|
}
|
||||||
@@ -996,10 +1046,30 @@ func monitoringPlatformQueryIDs(value *string) []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func reconcileEnabledMonitoringPlatforms(enabled []string) []string {
|
func reconcileEnabledMonitoringPlatforms(enabled []string) []string {
|
||||||
// Old quota rows may still contain partial or aliased platform ids. Admin monitoring
|
if len(enabled) == 0 {
|
||||||
// should always expose the full canonical desktop-supported set for the tenant.
|
return nil
|
||||||
_ = enabled
|
}
|
||||||
return monitoringPlatformIDs(defaultMonitoringPlatforms)
|
requested := make(map[string]struct{}, len(enabled))
|
||||||
|
for _, item := range enabled {
|
||||||
|
id := normalizeMonitoringPlatformID(item)
|
||||||
|
if id == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, supported := monitoringPlatformNames[id]; !supported {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
requested[id] = struct{}{}
|
||||||
|
}
|
||||||
|
if len(requested) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
result := make([]string, 0, len(requested))
|
||||||
|
for _, platform := range defaultMonitoringPlatforms {
|
||||||
|
if _, ok := requested[platform.ID]; ok {
|
||||||
|
result = append(result, platform.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
func monitoringPlatformSortIndex(platformID string) int {
|
func monitoringPlatformSortIndex(platformID string) int {
|
||||||
@@ -1054,26 +1124,6 @@ func (s *MonitoringService) findLatestOnlineClientForUser(ctx context.Context, t
|
|||||||
return &clientID, nil
|
return &clientID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MonitoringService) persistPrimaryClientID(ctx context.Context, tenantID, workspaceID int64, clientID uuid.UUID) error {
|
|
||||||
if _, err := s.businessPool.Exec(ctx, `
|
|
||||||
INSERT INTO tenant_monitoring_quotas (
|
|
||||||
tenant_id, workspace_id, primary_client_id
|
|
||||||
)
|
|
||||||
VALUES (
|
|
||||||
$1,
|
|
||||||
$2,
|
|
||||||
$3
|
|
||||||
)
|
|
||||||
ON CONFLICT (tenant_id) DO UPDATE
|
|
||||||
SET workspace_id = EXCLUDED.workspace_id,
|
|
||||||
primary_client_id = EXCLUDED.primary_client_id,
|
|
||||||
updated_at = NOW()
|
|
||||||
`, tenantID, workspaceID, clientID); err != nil {
|
|
||||||
return response.ErrInternal(50041, "update_failed", "failed to persist primary monitoring desktop client")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func formatPgInterval(duration time.Duration) string {
|
func formatPgInterval(duration time.Duration) string {
|
||||||
return fmt.Sprintf("%d seconds", int64(duration/time.Second))
|
return fmt.Sprintf("%d seconds", int64(duration/time.Second))
|
||||||
}
|
}
|
||||||
@@ -1431,40 +1481,117 @@ func (s *MonitoringService) deferQueuedNormalTasks(
|
|||||||
return tag.RowsAffected(), nil
|
return tag.RowsAffected(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MonitoringService) loadQuota(ctx context.Context, tenantID int64) (monitoringQuotaConfig, error) {
|
func (s *MonitoringService) loadQuota(ctx context.Context, actor auth.Actor, workspaceID int64) (monitoringQuotaConfig, error) {
|
||||||
cfg := monitoringQuotaConfig{
|
cfg := monitoringQuotaConfig{
|
||||||
CollectionMode: monitoringCollectorType,
|
CollectionMode: monitoringCollectorType,
|
||||||
EnabledPlatforms: reconcileEnabledMonitoringPlatforms(nil),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var enabledJSON []byte
|
clientID, err := s.findLatestRegisteredClientWithMonitoringAccountForUser(ctx, actor.TenantID, workspaceID, actor.UserID)
|
||||||
var clientID uuid.NullUUID
|
if err != nil {
|
||||||
|
return cfg, err
|
||||||
|
}
|
||||||
|
if clientID == nil {
|
||||||
|
clientID, err = s.findLatestRegisteredClientForUser(ctx, actor.TenantID, workspaceID, actor.UserID)
|
||||||
|
if err != nil {
|
||||||
|
return cfg, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cfg.PrimaryClientID = clientID
|
||||||
|
if clientID == nil {
|
||||||
|
return cfg, nil
|
||||||
|
}
|
||||||
|
platforms, err := s.loadMonitoringClientPlatformIDs(ctx, actor.TenantID, workspaceID, *clientID)
|
||||||
|
if err != nil {
|
||||||
|
return cfg, err
|
||||||
|
}
|
||||||
|
cfg.EnabledPlatforms = platforms
|
||||||
|
return cfg, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// findLatestRegisteredClientWithMonitoringAccountForUser returns the newest
|
||||||
|
// non-revoked desktop client that has at least one bound monitoring platform
|
||||||
|
// account. Platform account health is not authorization.
|
||||||
|
func (s *MonitoringService) findLatestRegisteredClientWithMonitoringAccountForUser(ctx context.Context, tenantID, workspaceID, userID int64) (*uuid.UUID, error) {
|
||||||
|
var clientID uuid.UUID
|
||||||
err := s.businessPool.QueryRow(ctx, `
|
err := s.businessPool.QueryRow(ctx, `
|
||||||
SELECT collection_mode, enabled_platforms, primary_client_id
|
SELECT dc.id
|
||||||
FROM tenant_monitoring_quotas
|
FROM desktop_clients dc
|
||||||
WHERE tenant_id = $1
|
WHERE dc.tenant_id = $1
|
||||||
`, tenantID).Scan(&cfg.CollectionMode, &enabledJSON, &clientID)
|
AND dc.workspace_id = $2
|
||||||
|
AND dc.user_id = $3
|
||||||
|
AND dc.revoked_at IS NULL
|
||||||
|
AND EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM platform_accounts pa
|
||||||
|
WHERE pa.tenant_id = dc.tenant_id
|
||||||
|
AND pa.workspace_id = dc.workspace_id
|
||||||
|
AND pa.client_id = dc.id
|
||||||
|
AND pa.deleted_at IS NULL
|
||||||
|
AND pa.platform_id = ANY($4::text[])
|
||||||
|
)
|
||||||
|
ORDER BY dc.created_at DESC, dc.id DESC
|
||||||
|
LIMIT 1
|
||||||
|
`, tenantID, workspaceID, userID, monitoringPlatformIDs(defaultMonitoringPlatforms)).Scan(&clientID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if err == pgx.ErrNoRows {
|
if err == pgx.ErrNoRows {
|
||||||
return cfg, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
return cfg, response.ErrInternal(50041, "query_failed", "failed to load monitoring quota")
|
return nil, response.ErrInternal(50041, "query_failed", "failed to inspect monitoring desktop client authorization")
|
||||||
|
}
|
||||||
|
return &clientID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(enabledJSON) > 0 {
|
// findLatestRegisteredClientForUser returns the newest non-revoked desktop client
|
||||||
var enabled []string
|
// registered by creation order. It intentionally ignores last_seen_at and health;
|
||||||
if jsonErr := json.Unmarshal(enabledJSON, &enabled); jsonErr == nil {
|
// use findLatestOnlineClientForUser when online presence is required.
|
||||||
cfg.EnabledPlatforms = reconcileEnabledMonitoringPlatforms(enabled)
|
func (s *MonitoringService) findLatestRegisteredClientForUser(ctx context.Context, tenantID, workspaceID, userID int64) (*uuid.UUID, error) {
|
||||||
|
var clientID uuid.UUID
|
||||||
|
err := s.businessPool.QueryRow(ctx, `
|
||||||
|
SELECT id
|
||||||
|
FROM desktop_clients
|
||||||
|
WHERE tenant_id = $1
|
||||||
|
AND workspace_id = $2
|
||||||
|
AND user_id = $3
|
||||||
|
AND revoked_at IS NULL
|
||||||
|
ORDER BY created_at DESC, id DESC
|
||||||
|
LIMIT 1
|
||||||
|
`, tenantID, workspaceID, userID).Scan(&clientID)
|
||||||
|
if err != nil {
|
||||||
|
if err == pgx.ErrNoRows {
|
||||||
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
return nil, response.ErrInternal(50041, "query_failed", "failed to inspect monitoring desktop client")
|
||||||
}
|
}
|
||||||
if clientID.Valid {
|
return &clientID, nil
|
||||||
id := clientID.UUID
|
|
||||||
cfg.PrimaryClientID = &id
|
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(cfg.CollectionMode) == "" {
|
|
||||||
cfg.CollectionMode = monitoringCollectorType
|
func (s *MonitoringService) loadMonitoringClientPlatformIDs(ctx context.Context, tenantID, workspaceID int64, clientID uuid.UUID) ([]string, error) {
|
||||||
|
rows, err := s.businessPool.Query(ctx, `
|
||||||
|
SELECT DISTINCT platform_id
|
||||||
|
FROM platform_accounts
|
||||||
|
WHERE tenant_id = $1
|
||||||
|
AND workspace_id = $2
|
||||||
|
AND client_id = $3
|
||||||
|
AND deleted_at IS NULL
|
||||||
|
AND platform_id = ANY($4::text[])
|
||||||
|
`, tenantID, workspaceID, clientID, monitoringPlatformIDs(defaultMonitoringPlatforms))
|
||||||
|
if err != nil {
|
||||||
|
return nil, response.ErrInternal(50041, "query_failed", "failed to load monitoring desktop platform accounts")
|
||||||
}
|
}
|
||||||
return cfg, nil
|
defer rows.Close()
|
||||||
|
|
||||||
|
platformIDs := make([]string, 0)
|
||||||
|
for rows.Next() {
|
||||||
|
var platformID string
|
||||||
|
if scanErr := rows.Scan(&platformID); scanErr != nil {
|
||||||
|
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse monitoring desktop platform accounts")
|
||||||
|
}
|
||||||
|
platformIDs = append(platformIDs, platformID)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate monitoring desktop platform accounts")
|
||||||
|
}
|
||||||
|
return reconcileEnabledMonitoringPlatforms(platformIDs), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func newMonitoringDerivedMetrics() monitoringDerivedMetrics {
|
func newMonitoringDerivedMetrics() monitoringDerivedMetrics {
|
||||||
@@ -2021,11 +2148,18 @@ func applyDerivedRatesToOverviewSnapshot(snapshot *monitoringOverviewSnapshot, d
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MonitoringService) loadAccessStates(ctx context.Context, tenantID int64, clientID *uuid.UUID, businessDate time.Time) (map[string]monitoringAccessState, error) {
|
func (s *MonitoringService) loadAccessStates(ctx context.Context, tenantID, workspaceID int64, clientID *uuid.UUID, businessDate time.Time) (map[string]monitoringAccessState, error) {
|
||||||
states := map[string]monitoringAccessState{}
|
states := map[string]monitoringAccessState{}
|
||||||
if clientID == nil {
|
if clientID == nil {
|
||||||
return states, nil
|
return states, nil
|
||||||
}
|
}
|
||||||
|
belongs, err := s.monitoringClientBelongsToWorkspace(ctx, tenantID, workspaceID, *clientID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !belongs {
|
||||||
|
return states, nil
|
||||||
|
}
|
||||||
|
|
||||||
rows, err := s.monitoringPool.Query(ctx, `
|
rows, err := s.monitoringPool.Query(ctx, `
|
||||||
SELECT ai_platform_id, access_status
|
SELECT ai_platform_id, access_status
|
||||||
@@ -2054,10 +2188,30 @@ func (s *MonitoringService) loadAccessStates(ctx context.Context, tenantID int64
|
|||||||
}
|
}
|
||||||
states[platformID] = monitoringAccessState{AccessStatus: accessStatus}
|
states[platformID] = monitoringAccessState{AccessStatus: accessStatus}
|
||||||
}
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate platform access states")
|
||||||
|
}
|
||||||
|
|
||||||
return states, nil
|
return states, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *MonitoringService) monitoringClientBelongsToWorkspace(ctx context.Context, tenantID, workspaceID int64, clientID uuid.UUID) (bool, error) {
|
||||||
|
var exists bool
|
||||||
|
if err := s.businessPool.QueryRow(ctx, `
|
||||||
|
SELECT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM desktop_clients
|
||||||
|
WHERE id = $1
|
||||||
|
AND tenant_id = $2
|
||||||
|
AND workspace_id = $3
|
||||||
|
AND revoked_at IS NULL
|
||||||
|
)
|
||||||
|
`, clientID, tenantID, workspaceID).Scan(&exists); err != nil {
|
||||||
|
return false, response.ErrInternal(50041, "query_failed", "failed to validate monitoring desktop client workspace")
|
||||||
|
}
|
||||||
|
return exists, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *MonitoringService) loadPlatformBreakdown(ctx context.Context, tenantID, brandID int64, businessDate time.Time, collectionMode string, platforms []monitoringPlatformMetadata, accessStates map[string]monitoringAccessState, derived monitoringDerivedMetrics) ([]MonitoringPlatformDaily, error) {
|
func (s *MonitoringService) loadPlatformBreakdown(ctx context.Context, tenantID, brandID int64, businessDate time.Time, collectionMode string, platforms []monitoringPlatformMetadata, accessStates map[string]monitoringAccessState, derived monitoringDerivedMetrics) ([]MonitoringPlatformDaily, error) {
|
||||||
rows, err := s.monitoringPool.Query(ctx, `
|
rows, err := s.monitoringPool.Query(ctx, `
|
||||||
SELECT
|
SELECT
|
||||||
@@ -3117,7 +3271,7 @@ func normalizeTrackingDays(days int) int {
|
|||||||
|
|
||||||
func resolveMonitoringPlatforms(enabled []string) []monitoringPlatformMetadata {
|
func resolveMonitoringPlatforms(enabled []string) []monitoringPlatformMetadata {
|
||||||
if len(enabled) == 0 {
|
if len(enabled) == 0 {
|
||||||
return defaultMonitoringPlatforms
|
return nil
|
||||||
}
|
}
|
||||||
platforms := make([]monitoringPlatformMetadata, 0, len(enabled))
|
platforms := make([]monitoringPlatformMetadata, 0, len(enabled))
|
||||||
seen := make(map[string]struct{}, len(enabled))
|
seen := make(map[string]struct{}, len(enabled))
|
||||||
@@ -3139,7 +3293,7 @@ func resolveMonitoringPlatforms(enabled []string) []monitoringPlatformMetadata {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
if len(platforms) == 0 {
|
if len(platforms) == 0 {
|
||||||
return defaultMonitoringPlatforms
|
return nil
|
||||||
}
|
}
|
||||||
return platforms
|
return platforms
|
||||||
}
|
}
|
||||||
@@ -3192,12 +3346,7 @@ func filterMonitoringPlatforms(platforms []monitoringPlatformMetadata, aiPlatfor
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return []monitoringPlatformMetadata{
|
return nil
|
||||||
{
|
|
||||||
ID: platformID,
|
|
||||||
Name: platformDisplayName(platformID),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func platformDisplayName(platformID string) string {
|
func platformDisplayName(platformID string) string {
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
@@ -81,7 +82,7 @@ func TestFilterMonitoringPlatformsReturnsSelectedPlatform(t *testing.T) {
|
|||||||
}, filtered)
|
}, filtered)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFilterMonitoringPlatformsFallsBackToDisplayName(t *testing.T) {
|
func TestFilterMonitoringPlatformsDropsUnauthorizedSelection(t *testing.T) {
|
||||||
platforms := []monitoringPlatformMetadata{
|
platforms := []monitoringPlatformMetadata{
|
||||||
{ID: "deepseek", Name: "DeepSeek"},
|
{ID: "deepseek", Name: "DeepSeek"},
|
||||||
}
|
}
|
||||||
@@ -89,9 +90,7 @@ func TestFilterMonitoringPlatformsFallsBackToDisplayName(t *testing.T) {
|
|||||||
|
|
||||||
filtered := filterMonitoringPlatforms(platforms, &selected)
|
filtered := filterMonitoringPlatforms(platforms, &selected)
|
||||||
|
|
||||||
assert.Equal(t, []monitoringPlatformMetadata{
|
assert.Empty(t, filtered)
|
||||||
{ID: "doubao", Name: "豆包"},
|
|
||||||
}, filtered)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNormalizeMonitoringPlatformIDCanonicalizesLatestIDs(t *testing.T) {
|
func TestNormalizeMonitoringPlatformIDCanonicalizesLatestIDs(t *testing.T) {
|
||||||
@@ -127,8 +126,40 @@ func TestIntersectMonitoringPlatformsAcceptsCanonicalIDs(t *testing.T) {
|
|||||||
}, filtered)
|
}, filtered)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReconcileEnabledMonitoringPlatformsBackfillsSupportedCatalog(t *testing.T) {
|
func TestReconcileEnabledMonitoringPlatformsKeepsAuthorizedSubset(t *testing.T) {
|
||||||
assert.Equal(t, []string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"}, reconcileEnabledMonitoringPlatforms([]string{"deepseek", "qwen", "doubao"}))
|
assert.Equal(t, []string{"deepseek", "doubao", "qwen"}, reconcileEnabledMonitoringPlatforms([]string{"deepseek", "qwen", "doubao"}))
|
||||||
|
assert.Empty(t, reconcileEnabledMonitoringPlatforms(nil))
|
||||||
|
assert.Empty(t, resolveMonitoringPlatforms(nil))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitoringPlatformAuthorizationStatus(t *testing.T) {
|
||||||
|
clientID := uuid.New()
|
||||||
|
|
||||||
|
assert.Equal(t, monitoringPlatformAuthorizationNoDesktopClient, monitoringPlatformAuthorizationStatus(monitoringQuotaConfig{}))
|
||||||
|
assert.Equal(t, monitoringPlatformAuthorizationNoAuthorizedPlatforms, monitoringPlatformAuthorizationStatus(monitoringQuotaConfig{
|
||||||
|
PrimaryClientID: &clientID,
|
||||||
|
}))
|
||||||
|
assert.Equal(t, monitoringPlatformAuthorizationAuthorized, monitoringPlatformAuthorizationStatus(monitoringQuotaConfig{
|
||||||
|
PrimaryClientID: &clientID,
|
||||||
|
EnabledPlatforms: []string{"qwen"},
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDefaultMonitoringPlatformMetadataReturnsFullCatalogCopy(t *testing.T) {
|
||||||
|
platforms := defaultMonitoringPlatformMetadata()
|
||||||
|
|
||||||
|
require.Len(t, platforms, 6)
|
||||||
|
assert.Equal(t, []string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"}, monitoringPlatformIDs(platforms))
|
||||||
|
|
||||||
|
platforms[0].ID = "mutated"
|
||||||
|
assert.Equal(t, "yuanbao", defaultMonitoringPlatformMetadata()[0].ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadMonitoringProjectionPlatformsReturnsFullCatalog(t *testing.T) {
|
||||||
|
platforms, err := loadMonitoringProjectionPlatforms(nil, nil, 0, 0, time.Time{})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
assert.Equal(t, []string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"}, monitoringPlatformIDs(platforms))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestApplyDerivedRatesToOverviewSnapshotUsesAllSamplesForMentionRate(t *testing.T) {
|
func TestApplyDerivedRatesToOverviewSnapshotUsesAllSamplesForMentionRate(t *testing.T) {
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
package transport
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/gin-gonic/gin"
|
||||||
|
|
||||||
|
"github.com/geo-platform/tenant-api/internal/bootstrap"
|
||||||
|
"github.com/geo-platform/tenant-api/internal/shared/response"
|
||||||
|
tenantapp "github.com/geo-platform/tenant-api/internal/tenant/app"
|
||||||
|
)
|
||||||
|
|
||||||
|
func registerInternalMetricsRoutes(app *bootstrap.App) {
|
||||||
|
token := strings.TrimSpace(app.Config.Scheduler.InternalMetricsToken)
|
||||||
|
if token == "" {
|
||||||
|
app.Logger.Warn("tenant-api internal metrics routes not registered: scheduler.internal_metrics_token is required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
metrics := app.Engine.Group("/api/internal/metrics")
|
||||||
|
metrics.Use(internalMetricsAuthMiddleware(token))
|
||||||
|
metrics.GET("/monitoring/heartbeat-primary-lease", func(c *gin.Context) {
|
||||||
|
response.Success(c, tenantapp.MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue())
|
||||||
|
})
|
||||||
|
primaryLeasePrometheusHandler := tenantapp.MonitoringHeartbeatPrimaryLeaseMetricsPrometheusHandler()
|
||||||
|
metrics.GET("/monitoring/heartbeat-primary-lease/prometheus", func(c *gin.Context) {
|
||||||
|
primaryLeasePrometheusHandler.ServeHTTP(c.Writer, c.Request)
|
||||||
|
})
|
||||||
|
metrics.GET("/monitoring/heartbeat-snapshots", func(c *gin.Context) {
|
||||||
|
response.Success(c, tenantapp.MonitoringHeartbeatSnapshotMetricsSnapshotValue())
|
||||||
|
})
|
||||||
|
snapshotPrometheusHandler := tenantapp.MonitoringHeartbeatSnapshotMetricsPrometheusHandler()
|
||||||
|
metrics.GET("/monitoring/heartbeat-snapshots/prometheus", func(c *gin.Context) {
|
||||||
|
snapshotPrometheusHandler.ServeHTTP(c.Writer, c.Request)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func internalMetricsAuthMiddleware(token string) gin.HandlerFunc {
|
||||||
|
expected := strings.TrimSpace(token)
|
||||||
|
return func(c *gin.Context) {
|
||||||
|
if expected == "" || !internalMetricsTokenMatches(c, expected) {
|
||||||
|
response.Error(c, response.ErrUnauthorized(40181, "internal_metrics_unauthorized", "valid internal metrics token is required"))
|
||||||
|
c.Abort()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.Next()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func internalMetricsTokenMatches(c *gin.Context, expected string) bool {
|
||||||
|
if c == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
header := strings.TrimSpace(c.GetHeader("Authorization"))
|
||||||
|
if strings.HasPrefix(strings.ToLower(header), "bearer ") {
|
||||||
|
if strings.TrimSpace(header[len("bearer "):]) == expected {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(c.GetHeader("X-Internal-Metrics-Token")) == expected
|
||||||
|
}
|
||||||
@@ -26,7 +26,7 @@ type monitoringCollectNowRequest struct {
|
|||||||
|
|
||||||
func NewMonitoringHandler(a *bootstrap.App) *MonitoringHandler {
|
func NewMonitoringHandler(a *bootstrap.App) *MonitoringHandler {
|
||||||
return &MonitoringHandler{
|
return &MonitoringHandler{
|
||||||
svc: app.NewMonitoringService(a.DB, a.MonitoringDB, a.RabbitMQ, a.Config.MonitoringDispatch, a.Logger),
|
svc: app.NewMonitoringService(a.DB, a.MonitoringDB, a.RabbitMQ, a.Config.MonitoringDispatch, a.Config.BrandLibrary, a.Logger),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func RegisterRoutes(app *bootstrap.App) {
|
func RegisterRoutes(app *bootstrap.App) {
|
||||||
|
registerInternalMetricsRoutes(app)
|
||||||
|
|
||||||
authAPI := app.Engine.Group("/api/auth")
|
authAPI := app.Engine.Group("/api/auth")
|
||||||
authHandler := NewAuthHandler(app)
|
authHandler := NewAuthHandler(app)
|
||||||
authAPI.POST("/login", authHandler.Login)
|
authAPI.POST("/login", authHandler.Login)
|
||||||
@@ -21,6 +23,7 @@ func RegisterRoutes(app *bootstrap.App) {
|
|||||||
protected := app.Engine.Group("/api")
|
protected := app.Engine.Group("/api")
|
||||||
protected.Use(auth.Middleware(app.JWT, app.Sessions))
|
protected.Use(auth.Middleware(app.JWT, app.Sessions))
|
||||||
protected.Use(middleware.TenantScope())
|
protected.Use(middleware.TenantScope())
|
||||||
|
protected.Use(middleware.WorkspaceScope(app.DB))
|
||||||
|
|
||||||
protected.GET("/auth/me", authHandler.Me)
|
protected.GET("/auth/me", authHandler.Me)
|
||||||
protected.POST("/auth/logout", authHandler.Logout)
|
protected.POST("/auth/logout", authHandler.Logout)
|
||||||
|
|||||||
@@ -0,0 +1,4 @@
|
|||||||
|
DROP INDEX IF EXISTS idx_desktop_clients_tenant_workspace_user_seen_active;
|
||||||
|
DROP INDEX IF EXISTS idx_desktop_clients_tenant_workspace_seen_active;
|
||||||
|
DROP INDEX IF EXISTS idx_desktop_client_primary_leases_primary_client;
|
||||||
|
DROP TABLE IF EXISTS desktop_client_primary_leases;
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
-- Backfilled primary leases use the same 90s TTL as
|
||||||
|
-- server/internal/tenant/app/desktop_presence.go:desktopClientPresenceTTL.
|
||||||
|
-- Keep these values aligned; desktop heartbeat interval must be < TTL/2 so
|
||||||
|
-- a single missed heartbeat does not let another client steal primary.
|
||||||
|
CREATE TABLE IF NOT EXISTS desktop_client_primary_leases (
|
||||||
|
tenant_id BIGINT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
|
||||||
|
workspace_id BIGINT NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE,
|
||||||
|
primary_client_id UUID NOT NULL REFERENCES desktop_clients(id) ON DELETE CASCADE,
|
||||||
|
lease_expires_at TIMESTAMPTZ NOT NULL,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||||
|
PRIMARY KEY (tenant_id, workspace_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_desktop_client_primary_leases_primary_client
|
||||||
|
ON desktop_client_primary_leases (primary_client_id);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_desktop_clients_tenant_workspace_seen_active
|
||||||
|
ON desktop_clients (tenant_id, workspace_id, last_seen_at DESC, created_at DESC, id DESC)
|
||||||
|
WHERE revoked_at IS NULL;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_desktop_clients_tenant_workspace_user_seen_active
|
||||||
|
ON desktop_clients (tenant_id, workspace_id, user_id, last_seen_at DESC, created_at DESC, id DESC)
|
||||||
|
WHERE revoked_at IS NULL;
|
||||||
|
|
||||||
|
INSERT INTO desktop_client_primary_leases (
|
||||||
|
tenant_id,
|
||||||
|
workspace_id,
|
||||||
|
primary_client_id,
|
||||||
|
lease_expires_at
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
tenant_id,
|
||||||
|
workspace_id,
|
||||||
|
id,
|
||||||
|
COALESCE(last_seen_at, created_at) + INTERVAL '90 seconds'
|
||||||
|
FROM (
|
||||||
|
SELECT
|
||||||
|
dc.id,
|
||||||
|
dc.tenant_id,
|
||||||
|
dc.workspace_id,
|
||||||
|
dc.created_at,
|
||||||
|
dc.last_seen_at,
|
||||||
|
ROW_NUMBER() OVER (
|
||||||
|
PARTITION BY dc.tenant_id, dc.workspace_id
|
||||||
|
ORDER BY dc.last_seen_at DESC NULLS LAST, dc.created_at DESC, dc.id DESC
|
||||||
|
) AS rn
|
||||||
|
FROM desktop_clients dc
|
||||||
|
WHERE dc.revoked_at IS NULL
|
||||||
|
) ranked_clients
|
||||||
|
WHERE rn = 1
|
||||||
|
ON CONFLICT (tenant_id, workspace_id) DO NOTHING;
|
||||||
+1
@@ -0,0 +1 @@
|
|||||||
|
DROP INDEX IF EXISTS idx_collect_tasks_pending_platform_reconcile;
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
CREATE INDEX IF NOT EXISTS idx_collect_tasks_pending_platform_reconcile
|
||||||
|
ON monitoring_collect_tasks (tenant_id, business_date, ai_platform_id)
|
||||||
|
WHERE collector_type = 'desktop'
|
||||||
|
AND status = 'pending';
|
||||||
Reference in New Issue
Block a user