feat(monitoring): generate daily tasks and optimize heartbeat writes
Adds a MonitoringDailyTaskWorker that materializes and dispatches daily collect tasks from active subscription/desktop-client state, with per-run atomic metrics and a Prometheus collector. Dashboard and question-detail APIs now surface a platform_authorization_status so the UI can distinguish no-desktop-client, no-authorized-platforms, and authorized states; quota/access-state lookups resolve by request workspace instead of tenant. Hardens heartbeat: a desktop_client_primary_leases table gives per-(tenant, workspace) sticky primary selection, read-mostly lease resolution avoids a per-heartbeat transaction, and unchanged platform access reports skip snapshot upserts outside a 10-minute refresh window. Adds heartbeat primary/snapshot metrics exposed via token-protected tenant-api /api/internal/metrics endpoints plus Prometheus. Monitoring quota seed is removed from dev-seed since daily plans now derive from desktop bindings, and the projection uses the canonical six-platform catalog so platforms with zero samples still render. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -246,10 +246,6 @@ func seedBusinessData(ctx context.Context, pool *pgxpool.Pool, membershipCfg con
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := ensureMonitoringQuota(ctx, tx, state); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, fmt.Errorf("commit business seed: %w", err)
|
||||
}
|
||||
@@ -692,28 +688,6 @@ func ensureDesktopClient(ctx context.Context, tx pgx.Tx, tenantID, workspaceID,
|
||||
return ensuredID, nil
|
||||
}
|
||||
|
||||
func ensureMonitoringQuota(ctx context.Context, tx pgx.Tx, state *seedState) error {
|
||||
enabledPlatforms, _ := json.Marshal([]string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"})
|
||||
_, err := tx.Exec(ctx, `
|
||||
INSERT INTO tenant_monitoring_quotas (
|
||||
tenant_id, workspace_id, max_brands, collection_mode, question_selection_mode, collect_frequency,
|
||||
enabled_platforms, primary_client_id, task_daily_hard_cap, plan_tier
|
||||
)
|
||||
VALUES ($1, $2, 1, 'desktop', 'best_effort_all', 'daily', $3::jsonb, $4, 36, 'free')
|
||||
ON CONFLICT (tenant_id) DO UPDATE
|
||||
SET workspace_id = EXCLUDED.workspace_id,
|
||||
collection_mode = EXCLUDED.collection_mode,
|
||||
question_selection_mode = EXCLUDED.question_selection_mode,
|
||||
collect_frequency = EXCLUDED.collect_frequency,
|
||||
enabled_platforms = EXCLUDED.enabled_platforms,
|
||||
primary_client_id = EXCLUDED.primary_client_id,
|
||||
task_daily_hard_cap = EXCLUDED.task_daily_hard_cap,
|
||||
plan_tier = EXCLUDED.plan_tier,
|
||||
updated_at = NOW()
|
||||
`, state.TenantID, state.WorkspaceID, string(enabledPlatforms), state.DesktopClientID)
|
||||
return wrapSeedErr("upsert monitoring quota", err)
|
||||
}
|
||||
|
||||
func clearMonitoringData(ctx context.Context, tx pgx.Tx, state *seedState) error {
|
||||
pairs := []string{
|
||||
`DELETE FROM monitoring_citation_facts WHERE tenant_id = $1 AND brand_id = $2`,
|
||||
|
||||
@@ -32,7 +32,7 @@ func main() {
|
||||
app.DesktopTaskStreams.Run(workerCtx)
|
||||
app.DesktopDispatch.Run(workerCtx)
|
||||
|
||||
monitoringService := tenantapp.NewMonitoringService(app.DB, app.MonitoringDB, app.RabbitMQ, app.Config.MonitoringDispatch, app.Logger)
|
||||
monitoringService := tenantapp.NewMonitoringService(app.DB, app.MonitoringDB, app.RabbitMQ, app.Config.MonitoringDispatch, app.Config.BrandLibrary, app.Logger)
|
||||
monitoringCallbackService := tenantapp.NewMonitoringCallbackService(app.DB, app.MonitoringDB, app.RabbitMQ, app.LLM, app.Logger)
|
||||
tenantapp.NewMonitoringCollectOutboxWorker(monitoringService, app.Logger).Start(workerCtx)
|
||||
tenantapp.NewMonitoringResultIngestWorker(monitoringCallbackService, app.RabbitMQ, app.Logger, app.Config.MonitoringWorkers).Start(workerCtx)
|
||||
|
||||
@@ -14,6 +14,8 @@ require (
|
||||
github.com/jackc/pgx/v5 v5.5.5
|
||||
github.com/ledongthuc/pdf v0.0.0-20240314090751-a2a84ec735c3
|
||||
github.com/minio/minio-go/v7 v7.0.83
|
||||
github.com/prometheus/client_golang v1.20.5
|
||||
github.com/prometheus/common v0.60.1
|
||||
github.com/qdrant/go-client v1.13.0
|
||||
github.com/rabbitmq/amqp091-go v1.10.0
|
||||
github.com/redis/go-redis/v9 v9.5.1
|
||||
@@ -32,6 +34,7 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bytedance/sonic v1.9.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
|
||||
@@ -63,8 +66,11 @@ require (
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/prometheus/client_model v0.6.1 // indirect
|
||||
github.com/prometheus/procfs v0.15.1 // indirect
|
||||
github.com/richardlehane/mscfb v1.0.4 // indirect
|
||||
github.com/richardlehane/msoleps v1.0.3 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
|
||||
@@ -5,6 +5,8 @@ github.com/alicebob/miniredis/v2 v2.37.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g=
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
|
||||
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
@@ -115,6 +117,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/ledongthuc/pdf v0.0.0-20240314090751-a2a84ec735c3 h1:DDPccSesc/SJu8CiCEhHhWjK9C9U1fhrRKB3jRNw4f4=
|
||||
github.com/ledongthuc/pdf v0.0.0-20240314090751-a2a84ec735c3/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
|
||||
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
|
||||
@@ -136,13 +140,23 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
|
||||
github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
|
||||
github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
|
||||
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
|
||||
github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc=
|
||||
github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw=
|
||||
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
|
||||
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
|
||||
github.com/qdrant/go-client v1.13.0 h1:qeWKCs1vxvfF2MLLFnP2qDG0R8wI18HyAoSfc7wJim8=
|
||||
github.com/qdrant/go-client v1.13.0/go.mod h1:iO8ts78jL4x6LDHFOViyYWELVtIBDTjOykBmiOTHLnQ=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
|
||||
|
||||
@@ -105,6 +105,9 @@ type RabbitMQConfig struct {
|
||||
}
|
||||
|
||||
type SchedulerConfig struct {
|
||||
HTTPHost string `mapstructure:"http_host"`
|
||||
HTTPPort int `mapstructure:"http_port"`
|
||||
InternalMetricsToken string `mapstructure:"internal_metrics_token"`
|
||||
DispatchInterval time.Duration `mapstructure:"dispatch_interval"`
|
||||
DispatchTimeout time.Duration `mapstructure:"dispatch_timeout"`
|
||||
DispatchBatchSize int `mapstructure:"dispatch_batch_size"`
|
||||
@@ -427,6 +430,12 @@ func applyEnvOverrides(cfg *Config) {
|
||||
if model, ok := lookupFirstNonEmptyEnv("SILICONFLOW_RERANKER_MODEL", "RETRIEVAL_RERANKER_MODEL"); ok {
|
||||
cfg.Retrieval.RerankerModel = model
|
||||
}
|
||||
if host, ok := lookupNonEmptyEnv("SCHEDULER_HTTP_HOST"); ok {
|
||||
cfg.Scheduler.HTTPHost = host
|
||||
}
|
||||
if token, ok := lookupFirstNonEmptyEnv("SCHEDULER_INTERNAL_METRICS_TOKEN", "SCHEDULER_METRICS_TOKEN"); ok {
|
||||
cfg.Scheduler.InternalMetricsToken = token
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeRabbitMQConfig(cfg *RabbitMQConfig) {
|
||||
@@ -569,6 +578,14 @@ func normalizeSchedulerConfig(cfg *SchedulerConfig) {
|
||||
return
|
||||
}
|
||||
|
||||
if cfg.HTTPPort == 0 {
|
||||
cfg.HTTPPort = 8081
|
||||
}
|
||||
cfg.HTTPHost = strings.TrimSpace(cfg.HTTPHost)
|
||||
if cfg.HTTPHost == "" {
|
||||
cfg.HTTPHost = "127.0.0.1"
|
||||
}
|
||||
cfg.InternalMetricsToken = strings.TrimSpace(cfg.InternalMetricsToken)
|
||||
if cfg.DispatchInterval <= 0 {
|
||||
cfg.DispatchInterval = 15 * time.Second
|
||||
}
|
||||
|
||||
@@ -114,6 +114,58 @@ brand_library: {}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAppliesSchedulerHTTPDefaultsAndDisableSentinel(t *testing.T) {
|
||||
defaultConfigPath := writeTestConfig(t, `
|
||||
scheduler: {}
|
||||
`)
|
||||
|
||||
cfg, err := Load(defaultConfigPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() default error = %v", err)
|
||||
}
|
||||
if cfg.Scheduler.HTTPHost != "127.0.0.1" {
|
||||
t.Fatalf("expected default scheduler http host 127.0.0.1, got %q", cfg.Scheduler.HTTPHost)
|
||||
}
|
||||
if cfg.Scheduler.HTTPPort != 8081 {
|
||||
t.Fatalf("expected default scheduler http port 8081, got %d", cfg.Scheduler.HTTPPort)
|
||||
}
|
||||
|
||||
disabledConfigPath := writeTestConfig(t, `
|
||||
scheduler:
|
||||
http_port: -1
|
||||
`)
|
||||
|
||||
cfg, err = Load(disabledConfigPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() disabled error = %v", err)
|
||||
}
|
||||
if cfg.Scheduler.HTTPPort != -1 {
|
||||
t.Fatalf("expected scheduler http port -1 to disable server, got %d", cfg.Scheduler.HTTPPort)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPrefersEnvSchedulerMetricsSettings(t *testing.T) {
|
||||
t.Setenv("SCHEDULER_HTTP_HOST", "0.0.0.0")
|
||||
t.Setenv("SCHEDULER_INTERNAL_METRICS_TOKEN", "env-metrics-token")
|
||||
|
||||
configPath := writeTestConfig(t, `
|
||||
scheduler:
|
||||
http_host: 127.0.0.1
|
||||
internal_metrics_token: config-token
|
||||
`)
|
||||
|
||||
cfg, err := Load(configPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
if cfg.Scheduler.HTTPHost != "0.0.0.0" {
|
||||
t.Fatalf("expected env scheduler http host, got %q", cfg.Scheduler.HTTPHost)
|
||||
}
|
||||
if cfg.Scheduler.InternalMetricsToken != "env-metrics-token" {
|
||||
t.Fatalf("expected env scheduler metrics token, got %q", cfg.Scheduler.InternalMetricsToken)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAppliesMembershipDefaults(t *testing.T) {
|
||||
configPath := writeTestConfig(t, `
|
||||
membership: {}
|
||||
|
||||
@@ -9,6 +9,9 @@ import (
|
||||
goredis "github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// desktopClientPresenceTTL is also used as the monitoring primary-client lease TTL.
|
||||
// Desktop heartbeat interval must stay below TTL/2, so one missed heartbeat does
|
||||
// not make another client eligible to steal a still-healthy primary lease.
|
||||
const desktopClientPresenceTTL = 90 * time.Second
|
||||
const desktopAccountPresenceTTL = 90 * time.Second
|
||||
|
||||
|
||||
@@ -30,6 +30,14 @@ const (
|
||||
maxMonitoringLeaseLimit = 50
|
||||
monitoringLeaseTTL = 15 * time.Minute
|
||||
monitoringTaskResultQueueIOTimeout = 5 * time.Second
|
||||
// The current primary renews the DB lease only after this interval. With the
|
||||
// desktop client's 25s heartbeat and a 90s lease TTL, this keeps normal
|
||||
// heartbeats read-only while still tolerating one missed heartbeat.
|
||||
monitoringHeartbeatPrimaryRenewAfter = desktopClientPresenceTTL / 3
|
||||
// Access snapshots are semantic state, not a liveness clock. Stable heartbeat
|
||||
// reports skip writes and only refresh the row at this interval so dashboards
|
||||
// do not depend on per-heartbeat database churn.
|
||||
monitoringHeartbeatAccessSnapshotRefreshAfter = 10 * time.Minute
|
||||
)
|
||||
|
||||
type MonitoringCallbackService struct {
|
||||
@@ -205,6 +213,16 @@ type monitoringPlatformAccessSnapshotState struct {
|
||||
Accessible bool
|
||||
ReasonCode *string
|
||||
ReasonMessage *string
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type monitoringHeartbeatSnapshotDecision struct {
|
||||
WriteSnapshot bool
|
||||
StateChanged bool
|
||||
RefreshDue bool
|
||||
PendingReconcileCandidate bool
|
||||
ReconcilePending bool
|
||||
SnapshotOperation string
|
||||
}
|
||||
|
||||
type monitoringCollectTask struct {
|
||||
@@ -434,7 +452,18 @@ func (s *MonitoringCallbackService) handleTaskResultForInstallation(
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationID string, installationToken string, req MonitoringHeartbeatRequest) (*MonitoringHeartbeatResponse, error) {
|
||||
func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationID string, installationToken string, req MonitoringHeartbeatRequest) (_ *MonitoringHeartbeatResponse, err error) {
|
||||
startedAt := time.Now()
|
||||
snapshotMetrics := monitoringHeartbeatSnapshotMetricsEvent{}
|
||||
defer func() {
|
||||
if !snapshotMetrics.Active {
|
||||
return
|
||||
}
|
||||
snapshotMetrics.Duration = time.Since(startedAt)
|
||||
snapshotMetrics.Err = err
|
||||
recordMonitoringHeartbeatSnapshotMetrics(snapshotMetrics)
|
||||
}()
|
||||
|
||||
installation, err := s.authenticateInstallation(ctx, installationID, installationToken)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -451,28 +480,111 @@ func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationI
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
snapshotMetrics.Active = true
|
||||
snapshotMetrics.PlatformCount = len(platforms)
|
||||
|
||||
if len(platforms) == 0 {
|
||||
_ = s.touchInstallation(ctx, installation.ID)
|
||||
return &MonitoringHeartbeatResponse{
|
||||
InstallationID: installation.ID,
|
||||
BusinessDate: businessDate,
|
||||
UpdatedPlatformCount: 0,
|
||||
ReconciledSkippedTaskCount: 0,
|
||||
}, nil
|
||||
}
|
||||
|
||||
platformIDs := monitoringHeartbeatPlatformIDs(platforms)
|
||||
previousStates, err := s.loadPlatformAccessSnapshotStates(ctx, installation.TenantID, installation.ID, businessDate, platformIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
type heartbeatPlatformAction struct {
|
||||
platform MonitoringHeartbeatPlatform
|
||||
decision monitoringHeartbeatSnapshotDecision
|
||||
}
|
||||
|
||||
actions := make([]heartbeatPlatformAction, 0, len(platforms))
|
||||
inaccessiblePlatformIDs := make([]string, 0, len(platforms))
|
||||
needsTransaction := false
|
||||
for _, platform := range platforms {
|
||||
nextState := monitoringPlatformAccessSnapshotStateFromHeartbeat(platform)
|
||||
decision := decideMonitoringHeartbeatSnapshot(previousStates[platform.AIPlatformID], nextState, now, isPrimaryInstallation)
|
||||
if decision.WriteSnapshot {
|
||||
needsTransaction = true
|
||||
} else {
|
||||
snapshotMetrics.SnapshotSkippedCount++
|
||||
}
|
||||
if decision.PendingReconcileCandidate {
|
||||
inaccessiblePlatformIDs = append(inaccessiblePlatformIDs, platform.AIPlatformID)
|
||||
}
|
||||
actions = append(actions, heartbeatPlatformAction{
|
||||
platform: platform,
|
||||
decision: decision,
|
||||
})
|
||||
}
|
||||
|
||||
if isPrimaryInstallation && len(inaccessiblePlatformIDs) > 0 {
|
||||
pendingPlatforms, pendingErr := s.loadPendingMonitoringTaskPlatformIDs(ctx, installation.TenantID, businessDate, inaccessiblePlatformIDs)
|
||||
if pendingErr != nil {
|
||||
return nil, pendingErr
|
||||
}
|
||||
for idx := range actions {
|
||||
if !actions[idx].decision.PendingReconcileCandidate {
|
||||
continue
|
||||
}
|
||||
if _, ok := pendingPlatforms[actions[idx].platform.AIPlatformID]; ok {
|
||||
actions[idx].decision.ReconcilePending = true
|
||||
needsTransaction = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !needsTransaction {
|
||||
_ = s.touchInstallation(ctx, installation.ID)
|
||||
return &MonitoringHeartbeatResponse{
|
||||
InstallationID: installation.ID,
|
||||
BusinessDate: businessDate,
|
||||
UpdatedPlatformCount: len(platforms),
|
||||
ReconciledSkippedTaskCount: 0,
|
||||
}, nil
|
||||
}
|
||||
|
||||
tx, err := s.monitoringPool.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(50041, "begin_failed", "failed to start monitoring heartbeat")
|
||||
}
|
||||
snapshotMetrics.TransactionOpened = true
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var reconciledSkippedTaskCount int64
|
||||
var snapshotInsertedCount int64
|
||||
var snapshotUpdatedCount int64
|
||||
var snapshotRefreshedCount int64
|
||||
var projectionRebuildCount int64
|
||||
affectedBrandIDs := make(map[int64]struct{})
|
||||
for _, platform := range platforms {
|
||||
accessChanged, err := s.upsertPlatformAccessSnapshot(ctx, tx, installation, businessDate, platform)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
for _, action := range actions {
|
||||
if action.decision.WriteSnapshot {
|
||||
if err := s.upsertPlatformAccessSnapshot(ctx, tx, installation, businessDate, action.platform); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch action.decision.SnapshotOperation {
|
||||
case "insert":
|
||||
snapshotInsertedCount++
|
||||
case "update":
|
||||
snapshotUpdatedCount++
|
||||
case "refresh":
|
||||
snapshotRefreshedCount++
|
||||
}
|
||||
}
|
||||
|
||||
if !isPrimaryInstallation {
|
||||
continue
|
||||
}
|
||||
|
||||
needsProjectionRefresh := accessChanged
|
||||
if monitoringStringValue(platform.AccessStatus) != "accessible" {
|
||||
count, reconcileErr := s.reconcilePendingTasksForPlatform(ctx, tx, installation.TenantID, businessDate, platform.AIPlatformID)
|
||||
needsProjectionRefresh := action.decision.StateChanged
|
||||
if action.decision.ReconcilePending {
|
||||
count, reconcileErr := s.reconcilePendingTasksForPlatform(ctx, tx, installation.TenantID, businessDate, action.platform.AIPlatformID)
|
||||
if reconcileErr != nil {
|
||||
return nil, reconcileErr
|
||||
}
|
||||
@@ -486,7 +598,7 @@ func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationI
|
||||
continue
|
||||
}
|
||||
|
||||
brandIDs, loadErr := s.loadMonitoringAffectedBrandIDs(ctx, tx, installation.TenantID, businessDay, platform.AIPlatformID)
|
||||
brandIDs, loadErr := s.loadMonitoringAffectedBrandIDs(ctx, tx, installation.TenantID, businessDay, action.platform.AIPlatformID)
|
||||
if loadErr != nil {
|
||||
return nil, loadErr
|
||||
}
|
||||
@@ -500,6 +612,7 @@ func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationI
|
||||
if err := s.rebuildBrandDailyProjection(ctx, tx, installation.TenantID, brandID, businessDay); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
projectionRebuildCount++
|
||||
}
|
||||
}
|
||||
|
||||
@@ -507,6 +620,12 @@ func (s *MonitoringCallbackService) Heartbeat(ctx context.Context, installationI
|
||||
return nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring heartbeat")
|
||||
}
|
||||
|
||||
snapshotMetrics.SnapshotInsertedCount = snapshotInsertedCount
|
||||
snapshotMetrics.SnapshotUpdatedCount = snapshotUpdatedCount
|
||||
snapshotMetrics.SnapshotRefreshedCount = snapshotRefreshedCount
|
||||
snapshotMetrics.ReconciledSkippedTaskCount = reconciledSkippedTaskCount
|
||||
snapshotMetrics.ProjectionRebuildCount = projectionRebuildCount
|
||||
|
||||
_ = s.touchInstallation(ctx, installation.ID)
|
||||
if isPrimaryInstallation && len(affectedBrandIDs) > 0 {
|
||||
s.dispatchMonitoringProjectionRebuilds(installation.TenantID, businessDay, affectedBrandIDs, "heartbeat")
|
||||
@@ -963,80 +1082,224 @@ func (s *MonitoringCallbackService) authenticateInstallation(ctx context.Context
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) resolveHeartbeatPrimaryInstallation(ctx context.Context, tenantID int64, installation *monitoringInstallationIdentity) (bool, error) {
|
||||
var preferred sql.NullString
|
||||
err := s.businessPool.QueryRow(ctx, `
|
||||
SELECT primary_client_id::text
|
||||
FROM tenant_monitoring_quotas
|
||||
WHERE tenant_id = $1
|
||||
`, tenantID).Scan(&preferred)
|
||||
if err != nil && err != pgx.ErrNoRows {
|
||||
return false, response.ErrInternal(50041, "query_failed", "failed to inspect primary monitoring desktop client")
|
||||
if installation == nil || strings.TrimSpace(installation.ID) == "" {
|
||||
return false, nil
|
||||
}
|
||||
isPrimary, err := s.claimHeartbeatPrimaryLease(ctx, tenantID, installation.WorkspaceID, installation.ID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return isPrimary, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) claimHeartbeatPrimaryLease(ctx context.Context, tenantID, workspaceID int64, installationID string) (bool, error) {
|
||||
startedAt := time.Now()
|
||||
installationID = strings.TrimSpace(installationID)
|
||||
leaseTTL := formatPgInterval(desktopClientPresenceTTL)
|
||||
|
||||
state, err := s.loadHeartbeatPrimaryLeaseState(ctx, tenantID, workspaceID, installationID)
|
||||
if err != nil {
|
||||
recordHeartbeatPrimaryLeaseMetric("error", time.Since(startedAt))
|
||||
return false, err
|
||||
}
|
||||
if !state.Found {
|
||||
inserted, insertErr := s.tryInsertHeartbeatPrimaryLease(ctx, tenantID, workspaceID, installationID, leaseTTL)
|
||||
if insertErr != nil {
|
||||
recordHeartbeatPrimaryLeaseMetric("error", time.Since(startedAt))
|
||||
return false, insertErr
|
||||
}
|
||||
if inserted {
|
||||
recordHeartbeatPrimaryLeaseMetric("inserted", time.Since(startedAt))
|
||||
recordHeartbeatPrimaryLeaseWriteMetric("insert")
|
||||
return true, nil
|
||||
}
|
||||
recordHeartbeatPrimaryLeaseMetric("contention", time.Since(startedAt))
|
||||
return s.resolveHeartbeatPrimaryAfterContention(ctx, tenantID, workspaceID, installationID)
|
||||
}
|
||||
|
||||
if !preferred.Valid || preferred.String == installation.ID {
|
||||
if !preferred.Valid {
|
||||
if err := s.persistHeartbeatPrimaryInstallationID(ctx, tenantID, installation); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
decision := decideHeartbeatPrimaryLease(state)
|
||||
switch decision.Action {
|
||||
case heartbeatPrimaryLeaseActionPrimaryHit:
|
||||
recordHeartbeatPrimaryLeaseMetric("primary_hit", time.Since(startedAt))
|
||||
return true, nil
|
||||
}
|
||||
|
||||
online, err := s.isMonitoringInstallationRecentlyOnline(ctx, tenantID, installation.WorkspaceID, preferred.String)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if online {
|
||||
case heartbeatPrimaryLeaseActionNonPrimaryLive:
|
||||
recordHeartbeatPrimaryLeaseMetric("non_primary_live", time.Since(startedAt))
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if err := s.persistHeartbeatPrimaryInstallationID(ctx, tenantID, installation); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) isMonitoringInstallationRecentlyOnline(ctx context.Context, tenantID, workspaceID int64, installationID string) (bool, error) {
|
||||
var lastSeenAt sql.NullTime
|
||||
err := s.businessPool.QueryRow(ctx, `
|
||||
SELECT last_seen_at
|
||||
FROM desktop_clients
|
||||
WHERE id = $1::uuid
|
||||
AND tenant_id = $2
|
||||
AND workspace_id = $3
|
||||
AND revoked_at IS NULL
|
||||
`, strings.TrimSpace(installationID), tenantID, workspaceID).Scan(&lastSeenAt)
|
||||
if err != nil {
|
||||
if err == pgx.ErrNoRows {
|
||||
return false, nil
|
||||
case heartbeatPrimaryLeaseActionRenew:
|
||||
updated, updateErr := s.renewHeartbeatPrimaryLease(ctx, tenantID, workspaceID, installationID, leaseTTL)
|
||||
if updateErr != nil {
|
||||
recordHeartbeatPrimaryLeaseMetric("error", time.Since(startedAt))
|
||||
return false, updateErr
|
||||
}
|
||||
if updated {
|
||||
recordHeartbeatPrimaryLeaseMetric("renewed", time.Since(startedAt))
|
||||
recordHeartbeatPrimaryLeaseWriteMetric("renew")
|
||||
return true, nil
|
||||
}
|
||||
case heartbeatPrimaryLeaseActionClaimExpired, heartbeatPrimaryLeaseActionClaimUnavailable:
|
||||
updated, updateErr := s.claimExpiredHeartbeatPrimaryLease(ctx, tenantID, workspaceID, installationID, state.PrimaryInstallationID, leaseTTL, decision.Action)
|
||||
if updateErr != nil {
|
||||
recordHeartbeatPrimaryLeaseMetric("error", time.Since(startedAt))
|
||||
return false, updateErr
|
||||
}
|
||||
if updated {
|
||||
if decision.Action == heartbeatPrimaryLeaseActionClaimExpired {
|
||||
recordHeartbeatPrimaryLeaseMetric("claimed_expired", time.Since(startedAt))
|
||||
} else {
|
||||
recordHeartbeatPrimaryLeaseMetric("claimed_unavailable", time.Since(startedAt))
|
||||
}
|
||||
recordHeartbeatPrimaryLeaseWriteMetric("claim")
|
||||
return true, nil
|
||||
}
|
||||
return false, response.ErrInternal(50041, "query_failed", "failed to inspect monitoring desktop client heartbeat")
|
||||
}
|
||||
|
||||
if !lastSeenAt.Valid {
|
||||
return false, nil
|
||||
}
|
||||
return time.Since(lastSeenAt.Time) <= monitoringOnlineDuration, nil
|
||||
recordHeartbeatPrimaryLeaseMetric("contention", time.Since(startedAt))
|
||||
return s.resolveHeartbeatPrimaryAfterContention(ctx, tenantID, workspaceID, installationID)
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) persistHeartbeatPrimaryInstallationID(ctx context.Context, tenantID int64, installation *monitoringInstallationIdentity) error {
|
||||
if _, err := s.businessPool.Exec(ctx, `
|
||||
INSERT INTO tenant_monitoring_quotas (
|
||||
tenant_id, workspace_id, primary_client_id
|
||||
func (s *MonitoringCallbackService) tryInsertHeartbeatPrimaryLease(ctx context.Context, tenantID, workspaceID int64, installationID, leaseTTL string) (bool, error) {
|
||||
tag, err := s.businessPool.Exec(ctx, `
|
||||
INSERT INTO desktop_client_primary_leases (
|
||||
tenant_id,
|
||||
workspace_id,
|
||||
primary_client_id,
|
||||
lease_expires_at
|
||||
)
|
||||
VALUES (
|
||||
$1,
|
||||
$2,
|
||||
$3::uuid
|
||||
)
|
||||
ON CONFLICT (tenant_id) DO UPDATE
|
||||
SET workspace_id = EXCLUDED.workspace_id,
|
||||
primary_client_id = EXCLUDED.primary_client_id,
|
||||
updated_at = NOW()
|
||||
`, tenantID, installation.WorkspaceID, installation.ID); err != nil {
|
||||
return response.ErrInternal(50041, "update_failed", "failed to persist primary monitoring desktop client")
|
||||
VALUES ($1, $2, $3::uuid, NOW() + $4::interval)
|
||||
ON CONFLICT (tenant_id, workspace_id) DO NOTHING
|
||||
`, tenantID, workspaceID, installationID, leaseTTL)
|
||||
if err != nil {
|
||||
return false, response.ErrInternal(50041, "primary_lease_insert_failed", "failed to initialize monitoring desktop primary lease")
|
||||
}
|
||||
return nil
|
||||
return tag.RowsAffected() > 0, nil
|
||||
}
|
||||
|
||||
type heartbeatPrimaryLeaseState struct {
|
||||
Found bool
|
||||
PrimaryInstallationID string
|
||||
CurrentIsPrimary bool
|
||||
LeaseLive bool
|
||||
RenewDue bool
|
||||
PrimaryAvailable bool
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) loadHeartbeatPrimaryLeaseState(ctx context.Context, tenantID, workspaceID int64, installationID string) (heartbeatPrimaryLeaseState, error) {
|
||||
var state heartbeatPrimaryLeaseState
|
||||
renewAfter := formatPgInterval(monitoringHeartbeatPrimaryRenewAfter)
|
||||
if err := s.businessPool.QueryRow(ctx, `
|
||||
SELECT
|
||||
l.primary_client_id::text,
|
||||
l.primary_client_id = $3::uuid AS current_is_primary,
|
||||
l.lease_expires_at > NOW() AS lease_live,
|
||||
l.updated_at <= NOW() - $4::interval AS renew_due,
|
||||
dc.id IS NOT NULL AND dc.revoked_at IS NULL AS primary_available
|
||||
FROM desktop_client_primary_leases l
|
||||
LEFT JOIN desktop_clients dc ON dc.id = l.primary_client_id
|
||||
WHERE l.tenant_id = $1
|
||||
AND l.workspace_id = $2
|
||||
`, tenantID, workspaceID, installationID, renewAfter).Scan(
|
||||
&state.PrimaryInstallationID,
|
||||
&state.CurrentIsPrimary,
|
||||
&state.LeaseLive,
|
||||
&state.RenewDue,
|
||||
&state.PrimaryAvailable,
|
||||
); err != nil {
|
||||
if err == pgx.ErrNoRows {
|
||||
return state, nil
|
||||
}
|
||||
return state, response.ErrInternal(50041, "query_failed", "failed to inspect monitoring desktop primary lease")
|
||||
}
|
||||
state.Found = true
|
||||
return state, nil
|
||||
}
|
||||
|
||||
type heartbeatPrimaryLeaseAction string
|
||||
|
||||
const (
|
||||
heartbeatPrimaryLeaseActionPrimaryHit heartbeatPrimaryLeaseAction = "primary_hit"
|
||||
heartbeatPrimaryLeaseActionNonPrimaryLive heartbeatPrimaryLeaseAction = "non_primary_live"
|
||||
heartbeatPrimaryLeaseActionRenew heartbeatPrimaryLeaseAction = "renew"
|
||||
heartbeatPrimaryLeaseActionClaimExpired heartbeatPrimaryLeaseAction = "claim_expired"
|
||||
heartbeatPrimaryLeaseActionClaimUnavailable heartbeatPrimaryLeaseAction = "claim_unavailable"
|
||||
)
|
||||
|
||||
type heartbeatPrimaryLeaseDecision struct {
|
||||
Action heartbeatPrimaryLeaseAction
|
||||
}
|
||||
|
||||
func decideHeartbeatPrimaryLease(state heartbeatPrimaryLeaseState) heartbeatPrimaryLeaseDecision {
|
||||
if !state.Found {
|
||||
return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionNonPrimaryLive}
|
||||
}
|
||||
if state.CurrentIsPrimary {
|
||||
if !state.LeaseLive || !state.PrimaryAvailable || state.RenewDue {
|
||||
return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionRenew}
|
||||
}
|
||||
return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionPrimaryHit}
|
||||
}
|
||||
if !state.LeaseLive {
|
||||
return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionClaimExpired}
|
||||
}
|
||||
if !state.PrimaryAvailable {
|
||||
return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionClaimUnavailable}
|
||||
}
|
||||
return heartbeatPrimaryLeaseDecision{Action: heartbeatPrimaryLeaseActionNonPrimaryLive}
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) renewHeartbeatPrimaryLease(ctx context.Context, tenantID, workspaceID int64, installationID, leaseTTL string) (bool, error) {
|
||||
tag, err := s.businessPool.Exec(ctx, `
|
||||
UPDATE desktop_client_primary_leases
|
||||
SET lease_expires_at = NOW() + $4::interval,
|
||||
updated_at = NOW()
|
||||
WHERE tenant_id = $1
|
||||
AND workspace_id = $2
|
||||
AND primary_client_id = $3::uuid
|
||||
`, tenantID, workspaceID, installationID, leaseTTL)
|
||||
if err != nil {
|
||||
return false, response.ErrInternal(50041, "primary_lease_update_failed", "failed to renew monitoring desktop primary lease")
|
||||
}
|
||||
return tag.RowsAffected() > 0, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) claimExpiredHeartbeatPrimaryLease(
|
||||
ctx context.Context,
|
||||
tenantID, workspaceID int64,
|
||||
installationID string,
|
||||
previousPrimaryInstallationID string,
|
||||
leaseTTL string,
|
||||
action heartbeatPrimaryLeaseAction,
|
||||
) (bool, error) {
|
||||
claimPredicate := "lease_expires_at <= NOW()"
|
||||
if action == heartbeatPrimaryLeaseActionClaimUnavailable {
|
||||
claimPredicate = `NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM desktop_clients dc
|
||||
WHERE dc.id = desktop_client_primary_leases.primary_client_id
|
||||
AND dc.revoked_at IS NULL
|
||||
)`
|
||||
}
|
||||
query := `
|
||||
UPDATE desktop_client_primary_leases
|
||||
SET primary_client_id = $3::uuid,
|
||||
lease_expires_at = NOW() + $5::interval,
|
||||
updated_at = NOW()
|
||||
WHERE tenant_id = $1
|
||||
AND workspace_id = $2
|
||||
AND primary_client_id = $4::uuid
|
||||
AND ` + claimPredicate
|
||||
tag, err := s.businessPool.Exec(ctx, query, tenantID, workspaceID, installationID, previousPrimaryInstallationID, leaseTTL)
|
||||
if err != nil {
|
||||
return false, response.ErrInternal(50041, "primary_lease_update_failed", "failed to claim monitoring desktop primary lease")
|
||||
}
|
||||
return tag.RowsAffected() > 0, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) resolveHeartbeatPrimaryAfterContention(ctx context.Context, tenantID, workspaceID int64, installationID string) (bool, error) {
|
||||
state, err := s.loadHeartbeatPrimaryLeaseState(ctx, tenantID, workspaceID, installationID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return state.CurrentIsPrimary && state.LeaseLive && state.PrimaryAvailable, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantID, taskID int64) (*monitoringCollectTask, error) {
|
||||
@@ -1464,20 +1727,7 @@ func (s *MonitoringCallbackService) markTaskResultReceived(ctx context.Context,
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) upsertPlatformAccessSnapshot(ctx context.Context, tx pgx.Tx, installation *monitoringInstallationIdentity, businessDate string, platform MonitoringHeartbeatPlatform) (bool, error) {
|
||||
previous, err := s.loadPlatformAccessSnapshotState(ctx, tx, installation.TenantID, installation.ID, businessDate, platform.AIPlatformID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
nextState := monitoringPlatformAccessSnapshotState{
|
||||
AccessStatus: monitoringStringValue(platform.AccessStatus),
|
||||
Connected: derefBool(platform.Connected),
|
||||
Accessible: derefBool(platform.Accessible),
|
||||
ReasonCode: optionalString(monitoringStringValue(platform.ReasonCode)),
|
||||
ReasonMessage: optionalString(monitoringStringValue(platform.ReasonMessage)),
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) upsertPlatformAccessSnapshot(ctx context.Context, tx pgx.Tx, installation *monitoringInstallationIdentity, businessDate string, platform MonitoringHeartbeatPlatform) error {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO monitoring_platform_access_snapshots (
|
||||
tenant_id, client_id, ai_platform_id, business_date, access_status,
|
||||
@@ -1498,44 +1748,101 @@ func (s *MonitoringCallbackService) upsertPlatformAccessSnapshot(ctx context.Con
|
||||
installation.ID,
|
||||
platform.AIPlatformID,
|
||||
businessDate,
|
||||
platform.AccessStatus,
|
||||
monitoringStringValue(platform.AccessStatus),
|
||||
derefBool(platform.Connected),
|
||||
derefBool(platform.Accessible),
|
||||
resolveHeartbeatDetectedAt(platform.DetectedAt),
|
||||
nullableString(platform.ReasonCode),
|
||||
nullableString(platform.ReasonMessage),
|
||||
); err != nil {
|
||||
return false, response.ErrInternal(50041, "access_snapshot_upsert_failed", "failed to persist monitoring platform access snapshot")
|
||||
return response.ErrInternal(50041, "access_snapshot_upsert_failed", "failed to persist monitoring platform access snapshot")
|
||||
}
|
||||
return hasPlatformAccessSnapshotChanged(previous, nextState), nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) loadPlatformAccessSnapshotState(ctx context.Context, tx pgx.Tx, tenantID int64, installationID string, businessDate, aiPlatformID string) (*monitoringPlatformAccessSnapshotState, error) {
|
||||
item := &monitoringPlatformAccessSnapshotState{}
|
||||
var reasonCode sql.NullString
|
||||
var reasonMessage sql.NullString
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT access_status, connected, accessible, reason_code, reason_message
|
||||
func (s *MonitoringCallbackService) loadPlatformAccessSnapshotStates(ctx context.Context, tenantID int64, installationID string, businessDate string, aiPlatformIDs []string) (map[string]*monitoringPlatformAccessSnapshotState, error) {
|
||||
states := make(map[string]*monitoringPlatformAccessSnapshotState, len(aiPlatformIDs))
|
||||
if len(aiPlatformIDs) == 0 {
|
||||
return states, nil
|
||||
}
|
||||
|
||||
rows, err := s.monitoringPool.Query(ctx, `
|
||||
SELECT ai_platform_id, access_status, connected, accessible, reason_code, reason_message, updated_at
|
||||
FROM monitoring_platform_access_snapshots
|
||||
WHERE tenant_id = $1
|
||||
AND client_id = $2::uuid
|
||||
AND ai_platform_id = $3
|
||||
AND business_date = $4::date
|
||||
`, tenantID, installationID, aiPlatformID, businessDate).Scan(
|
||||
&item.AccessStatus,
|
||||
&item.Connected,
|
||||
&item.Accessible,
|
||||
&reasonCode,
|
||||
&reasonMessage,
|
||||
); err != nil {
|
||||
if err == pgx.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, response.ErrInternal(50041, "access_snapshot_lookup_failed", "failed to inspect monitoring platform access snapshot")
|
||||
AND business_date = $3::date
|
||||
AND ai_platform_id = ANY($4::text[])
|
||||
`, tenantID, installationID, businessDate, aiPlatformIDs)
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(50041, "access_snapshot_lookup_failed", "failed to inspect monitoring platform access snapshots")
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
item := &monitoringPlatformAccessSnapshotState{}
|
||||
var platformID string
|
||||
var reasonCode sql.NullString
|
||||
var reasonMessage sql.NullString
|
||||
if scanErr := rows.Scan(
|
||||
&platformID,
|
||||
&item.AccessStatus,
|
||||
&item.Connected,
|
||||
&item.Accessible,
|
||||
&reasonCode,
|
||||
&reasonMessage,
|
||||
&item.UpdatedAt,
|
||||
); scanErr != nil {
|
||||
return nil, response.ErrInternal(50041, "access_snapshot_scan_failed", "failed to parse monitoring platform access snapshots")
|
||||
}
|
||||
if reasonCode.Valid {
|
||||
item.ReasonCode = optionalString(reasonCode.String)
|
||||
}
|
||||
if reasonMessage.Valid {
|
||||
item.ReasonMessage = optionalString(reasonMessage.String)
|
||||
}
|
||||
states[platformID] = item
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, response.ErrInternal(50041, "access_snapshot_scan_failed", "failed to iterate monitoring platform access snapshots")
|
||||
}
|
||||
|
||||
return states, nil
|
||||
}
|
||||
|
||||
func monitoringPlatformAccessSnapshotStateFromHeartbeat(platform MonitoringHeartbeatPlatform) monitoringPlatformAccessSnapshotState {
|
||||
return monitoringPlatformAccessSnapshotState{
|
||||
AccessStatus: monitoringStringValue(platform.AccessStatus),
|
||||
Connected: derefBool(platform.Connected),
|
||||
Accessible: derefBool(platform.Accessible),
|
||||
ReasonCode: optionalString(monitoringStringValue(platform.ReasonCode)),
|
||||
ReasonMessage: optionalString(monitoringStringValue(platform.ReasonMessage)),
|
||||
}
|
||||
}
|
||||
|
||||
func decideMonitoringHeartbeatSnapshot(previous *monitoringPlatformAccessSnapshotState, next monitoringPlatformAccessSnapshotState, now time.Time, isPrimary bool) monitoringHeartbeatSnapshotDecision {
|
||||
stateChanged := hasPlatformAccessSnapshotChanged(previous, next)
|
||||
refreshDue := false
|
||||
if previous != nil && !stateChanged {
|
||||
refreshDue = previous.UpdatedAt.IsZero() || !previous.UpdatedAt.After(now.Add(-monitoringHeartbeatAccessSnapshotRefreshAfter))
|
||||
}
|
||||
|
||||
operation := "skip"
|
||||
if previous == nil {
|
||||
operation = "insert"
|
||||
} else if stateChanged {
|
||||
operation = "update"
|
||||
} else if refreshDue {
|
||||
operation = "refresh"
|
||||
}
|
||||
|
||||
return monitoringHeartbeatSnapshotDecision{
|
||||
WriteSnapshot: stateChanged || refreshDue,
|
||||
StateChanged: stateChanged,
|
||||
RefreshDue: refreshDue,
|
||||
PendingReconcileCandidate: isPrimary && next.AccessStatus != "accessible",
|
||||
SnapshotOperation: operation,
|
||||
}
|
||||
item.ReasonCode = optionalString(reasonCode.String)
|
||||
item.ReasonMessage = optionalString(reasonMessage.String)
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func hasPlatformAccessSnapshotChanged(previous *monitoringPlatformAccessSnapshotState, next monitoringPlatformAccessSnapshotState) bool {
|
||||
@@ -1558,6 +1865,41 @@ func hasPlatformAccessSnapshotChanged(previous *monitoringPlatformAccessSnapshot
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) loadPendingMonitoringTaskPlatformIDs(ctx context.Context, tenantID int64, businessDate string, aiPlatformIDs []string) (map[string]struct{}, error) {
|
||||
result := make(map[string]struct{}, len(aiPlatformIDs))
|
||||
if len(aiPlatformIDs) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
rows, err := s.monitoringPool.Query(ctx, `
|
||||
SELECT ai_platform_id
|
||||
FROM monitoring_collect_tasks
|
||||
WHERE tenant_id = $1
|
||||
AND ai_platform_id = ANY($2::text[])
|
||||
AND collector_type = $3
|
||||
AND business_date = $4::date
|
||||
AND status = 'pending'
|
||||
GROUP BY ai_platform_id
|
||||
`, tenantID, aiPlatformIDs, monitoringCollectorType, businessDate)
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(50041, "pending_task_platform_lookup_failed", "failed to inspect pending monitoring task platforms")
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var platformID string
|
||||
if scanErr := rows.Scan(&platformID); scanErr != nil {
|
||||
return nil, response.ErrInternal(50041, "pending_task_platform_scan_failed", "failed to parse pending monitoring task platforms")
|
||||
}
|
||||
result[platformID] = struct{}{}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, response.ErrInternal(50041, "pending_task_platform_scan_failed", "failed to iterate pending monitoring task platforms")
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) reconcilePendingTasksForPlatform(ctx context.Context, tx pgx.Tx, tenantID int64, businessDate, aiPlatformID string) (int64, error) {
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE monitoring_collect_tasks
|
||||
@@ -2530,6 +2872,20 @@ func normalizeMonitoringLeasePlatformIDs(values []string) []string {
|
||||
return result
|
||||
}
|
||||
|
||||
func monitoringHeartbeatPlatformIDs(platforms []MonitoringHeartbeatPlatform) []string {
|
||||
if len(platforms) == 0 {
|
||||
return nil
|
||||
}
|
||||
result := make([]string, 0, len(platforms))
|
||||
for _, platform := range platforms {
|
||||
platformID := strings.TrimSpace(platform.AIPlatformID)
|
||||
if platformID != "" {
|
||||
result = append(result, platformID)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func normalizeHeartbeatPlatforms(items []MonitoringHeartbeatPlatform, fallbackDetectedAt time.Time) ([]MonitoringHeartbeatPlatform, error) {
|
||||
if len(items) == 0 {
|
||||
return nil, nil
|
||||
|
||||
@@ -2,9 +2,287 @@ package app
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestDecideHeartbeatPrimaryLease(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
state heartbeatPrimaryLeaseState
|
||||
want heartbeatPrimaryLeaseAction
|
||||
}{
|
||||
{
|
||||
name: "current primary fast path",
|
||||
state: heartbeatPrimaryLeaseState{
|
||||
Found: true,
|
||||
CurrentIsPrimary: true,
|
||||
LeaseLive: true,
|
||||
PrimaryAvailable: true,
|
||||
},
|
||||
want: heartbeatPrimaryLeaseActionPrimaryHit,
|
||||
},
|
||||
{
|
||||
name: "current primary renews when due",
|
||||
state: heartbeatPrimaryLeaseState{
|
||||
Found: true,
|
||||
CurrentIsPrimary: true,
|
||||
LeaseLive: true,
|
||||
RenewDue: true,
|
||||
PrimaryAvailable: true,
|
||||
},
|
||||
want: heartbeatPrimaryLeaseActionRenew,
|
||||
},
|
||||
{
|
||||
name: "non primary does not steal live lease",
|
||||
state: heartbeatPrimaryLeaseState{
|
||||
Found: true,
|
||||
CurrentIsPrimary: false,
|
||||
LeaseLive: true,
|
||||
PrimaryAvailable: true,
|
||||
},
|
||||
want: heartbeatPrimaryLeaseActionNonPrimaryLive,
|
||||
},
|
||||
{
|
||||
name: "expired lease can be claimed",
|
||||
state: heartbeatPrimaryLeaseState{
|
||||
Found: true,
|
||||
CurrentIsPrimary: false,
|
||||
LeaseLive: false,
|
||||
PrimaryAvailable: true,
|
||||
},
|
||||
want: heartbeatPrimaryLeaseActionClaimExpired,
|
||||
},
|
||||
{
|
||||
name: "revoked primary can be replaced",
|
||||
state: heartbeatPrimaryLeaseState{
|
||||
Found: true,
|
||||
CurrentIsPrimary: false,
|
||||
LeaseLive: true,
|
||||
PrimaryAvailable: false,
|
||||
},
|
||||
want: heartbeatPrimaryLeaseActionClaimUnavailable,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := decideHeartbeatPrimaryLease(tt.state)
|
||||
if got.Action != tt.want {
|
||||
t.Fatalf("decideHeartbeatPrimaryLease() = %v, want %v", got.Action, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitoringHeartbeatPrimaryLeaseMetrics(t *testing.T) {
|
||||
resetMonitoringHeartbeatPrimaryLeaseMetricsForTest()
|
||||
t.Cleanup(resetMonitoringHeartbeatPrimaryLeaseMetricsForTest)
|
||||
|
||||
recordHeartbeatPrimaryLeaseMetric("primary_hit", 20*time.Millisecond)
|
||||
recordHeartbeatPrimaryLeaseMetric("contention", 10*time.Millisecond)
|
||||
recordHeartbeatPrimaryLeaseWriteMetric("renew")
|
||||
|
||||
snapshot := MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue()
|
||||
if snapshot.PrimaryHitTotal != 1 {
|
||||
t.Fatalf("PrimaryHitTotal = %d, want 1", snapshot.PrimaryHitTotal)
|
||||
}
|
||||
if snapshot.ContentionTotal != 1 {
|
||||
t.Fatalf("ContentionTotal = %d, want 1", snapshot.ContentionTotal)
|
||||
}
|
||||
if snapshot.DBRenewTotal != 1 {
|
||||
t.Fatalf("DBRenewTotal = %d, want 1", snapshot.DBRenewTotal)
|
||||
}
|
||||
if snapshot.DurationCount != 2 {
|
||||
t.Fatalf("DurationCount = %d, want 2", snapshot.DurationCount)
|
||||
}
|
||||
|
||||
prometheus := MonitoringHeartbeatPrimaryLeaseMetricsPrometheus()
|
||||
if !strings.Contains(prometheus, `monitoring_heartbeat_primary_lease_resolutions_total{outcome="primary_hit"} 1`) {
|
||||
t.Fatalf("prometheus output missing primary_hit metric: %s", prometheus)
|
||||
}
|
||||
if !strings.Contains(prometheus, `monitoring_heartbeat_primary_lease_db_writes_total{operation="renew"} 1`) {
|
||||
t.Fatalf("prometheus output missing renew write metric: %s", prometheus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecideMonitoringHeartbeatSnapshot(t *testing.T) {
|
||||
now := time.Date(2026, 4, 24, 3, 0, 0, 0, time.UTC)
|
||||
base := monitoringPlatformAccessSnapshotState{
|
||||
AccessStatus: "accessible",
|
||||
Connected: true,
|
||||
Accessible: true,
|
||||
UpdatedAt: now.Add(-time.Minute),
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
previous *monitoringPlatformAccessSnapshotState
|
||||
next monitoringPlatformAccessSnapshotState
|
||||
primary bool
|
||||
want monitoringHeartbeatSnapshotDecision
|
||||
}{
|
||||
{
|
||||
name: "missing snapshot inserts",
|
||||
next: base,
|
||||
primary: true,
|
||||
want: monitoringHeartbeatSnapshotDecision{
|
||||
WriteSnapshot: true,
|
||||
StateChanged: true,
|
||||
SnapshotOperation: "insert",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "fresh unchanged accessible snapshot skips write",
|
||||
previous: &base,
|
||||
next: base,
|
||||
primary: true,
|
||||
want: monitoringHeartbeatSnapshotDecision{
|
||||
SnapshotOperation: "skip",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "stale unchanged snapshot refreshes",
|
||||
previous: &monitoringPlatformAccessSnapshotState{
|
||||
AccessStatus: "accessible",
|
||||
Connected: true,
|
||||
Accessible: true,
|
||||
UpdatedAt: now.Add(-monitoringHeartbeatAccessSnapshotRefreshAfter - time.Second),
|
||||
},
|
||||
next: base,
|
||||
primary: true,
|
||||
want: monitoringHeartbeatSnapshotDecision{
|
||||
WriteSnapshot: true,
|
||||
RefreshDue: true,
|
||||
SnapshotOperation: "refresh",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "changed snapshot updates",
|
||||
previous: &base,
|
||||
next: monitoringPlatformAccessSnapshotState{
|
||||
AccessStatus: "unavailable",
|
||||
Connected: true,
|
||||
Accessible: false,
|
||||
},
|
||||
primary: true,
|
||||
want: monitoringHeartbeatSnapshotDecision{
|
||||
WriteSnapshot: true,
|
||||
StateChanged: true,
|
||||
PendingReconcileCandidate: true,
|
||||
SnapshotOperation: "update",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "fresh unchanged primary inaccessible snapshot checks pending tasks without writing snapshot",
|
||||
previous: &monitoringPlatformAccessSnapshotState{
|
||||
AccessStatus: "not_logged_in",
|
||||
Connected: false,
|
||||
Accessible: false,
|
||||
UpdatedAt: now.Add(-time.Minute),
|
||||
},
|
||||
next: monitoringPlatformAccessSnapshotState{
|
||||
AccessStatus: "not_logged_in",
|
||||
Connected: false,
|
||||
Accessible: false,
|
||||
},
|
||||
primary: true,
|
||||
want: monitoringHeartbeatSnapshotDecision{
|
||||
PendingReconcileCandidate: true,
|
||||
SnapshotOperation: "skip",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "non-primary inaccessible snapshot does not reconcile",
|
||||
previous: &monitoringPlatformAccessSnapshotState{
|
||||
AccessStatus: "not_logged_in",
|
||||
Connected: false,
|
||||
Accessible: false,
|
||||
UpdatedAt: now.Add(-time.Minute),
|
||||
},
|
||||
next: monitoringPlatformAccessSnapshotState{
|
||||
AccessStatus: "not_logged_in",
|
||||
Connected: false,
|
||||
Accessible: false,
|
||||
},
|
||||
primary: false,
|
||||
want: monitoringHeartbeatSnapshotDecision{
|
||||
SnapshotOperation: "skip",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := decideMonitoringHeartbeatSnapshot(tt.previous, tt.next, now, tt.primary)
|
||||
if got.WriteSnapshot != tt.want.WriteSnapshot ||
|
||||
got.StateChanged != tt.want.StateChanged ||
|
||||
got.RefreshDue != tt.want.RefreshDue ||
|
||||
got.PendingReconcileCandidate != tt.want.PendingReconcileCandidate ||
|
||||
got.ReconcilePending != tt.want.ReconcilePending ||
|
||||
got.SnapshotOperation != tt.want.SnapshotOperation {
|
||||
t.Fatalf("decideMonitoringHeartbeatSnapshot() = %+v, want %+v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitoringHeartbeatSnapshotMetrics(t *testing.T) {
|
||||
resetMonitoringHeartbeatSnapshotMetricsForTest()
|
||||
t.Cleanup(resetMonitoringHeartbeatSnapshotMetricsForTest)
|
||||
|
||||
recordMonitoringHeartbeatSnapshotMetrics(monitoringHeartbeatSnapshotMetricsEvent{
|
||||
PlatformCount: 3,
|
||||
TransactionOpened: true,
|
||||
SnapshotInsertedCount: 1,
|
||||
SnapshotSkippedCount: 2,
|
||||
ReconciledSkippedTaskCount: 5,
|
||||
ProjectionRebuildCount: 2,
|
||||
Duration: 20 * time.Millisecond,
|
||||
})
|
||||
recordMonitoringHeartbeatSnapshotMetrics(monitoringHeartbeatSnapshotMetricsEvent{
|
||||
PlatformCount: 1,
|
||||
Duration: 10 * time.Millisecond,
|
||||
Err: errors.New("snapshot failed"),
|
||||
})
|
||||
|
||||
snapshot := MonitoringHeartbeatSnapshotMetricsSnapshotValue()
|
||||
if snapshot.PlatformReportedTotal != 4 {
|
||||
t.Fatalf("PlatformReportedTotal = %d, want 4", snapshot.PlatformReportedTotal)
|
||||
}
|
||||
if snapshot.TransactionOpenedTotal != 1 {
|
||||
t.Fatalf("TransactionOpenedTotal = %d, want 1", snapshot.TransactionOpenedTotal)
|
||||
}
|
||||
if snapshot.SnapshotInsertedTotal != 1 {
|
||||
t.Fatalf("SnapshotInsertedTotal = %d, want 1", snapshot.SnapshotInsertedTotal)
|
||||
}
|
||||
if snapshot.SnapshotSkippedTotal != 2 {
|
||||
t.Fatalf("SnapshotSkippedTotal = %d, want 2", snapshot.SnapshotSkippedTotal)
|
||||
}
|
||||
if snapshot.ReconciledSkippedTaskTotal != 5 {
|
||||
t.Fatalf("ReconciledSkippedTaskTotal = %d, want 5", snapshot.ReconciledSkippedTaskTotal)
|
||||
}
|
||||
if snapshot.ProjectionRebuildTotal != 2 {
|
||||
t.Fatalf("ProjectionRebuildTotal = %d, want 2", snapshot.ProjectionRebuildTotal)
|
||||
}
|
||||
if snapshot.ErrorTotal != 1 {
|
||||
t.Fatalf("ErrorTotal = %d, want 1", snapshot.ErrorTotal)
|
||||
}
|
||||
if snapshot.DurationCount != 2 {
|
||||
t.Fatalf("DurationCount = %d, want 2", snapshot.DurationCount)
|
||||
}
|
||||
|
||||
prometheus := MonitoringHeartbeatSnapshotMetricsPrometheus()
|
||||
if !strings.Contains(prometheus, `monitoring_heartbeat_snapshot_writes_total{operation="insert"} 1`) {
|
||||
t.Fatalf("prometheus output missing insert write metric: %s", prometheus)
|
||||
}
|
||||
if !strings.Contains(prometheus, "monitoring_heartbeat_snapshot_transactions_total 1") {
|
||||
t.Fatalf("prometheus output missing transaction metric: %s", prometheus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMonitoringRawPayloadKimiExcludesSearchResultsFromCitationInputs(t *testing.T) {
|
||||
citationTitle := "真实引用文章"
|
||||
searchTitle := "搜索网页结果"
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/response"
|
||||
)
|
||||
|
||||
type MonitoringDailyTaskMetricsSnapshot struct {
|
||||
RunTotal int64 `json:"run_total"`
|
||||
RunCompletedTotal int64 `json:"run_completed_total"`
|
||||
RunFailedTotal int64 `json:"run_failed_total"`
|
||||
RunPartialFailureTotal int64 `json:"run_partial_failure_total"`
|
||||
PlanFailureTotal int64 `json:"plan_failure_total"`
|
||||
BrandFailureTotal int64 `json:"brand_failure_total"`
|
||||
SkippedPlanTotal int64 `json:"skipped_plan_total"`
|
||||
PlannedTaskTotal int64 `json:"planned_task_total"`
|
||||
DueTaskTotal int64 `json:"due_task_total"`
|
||||
CreatedTaskTotal int64 `json:"created_task_total"`
|
||||
DesktopTaskTotal int64 `json:"desktop_task_total"`
|
||||
DeferredTaskTotal int64 `json:"deferred_task_total"`
|
||||
LastRunAt *time.Time `json:"last_run_at,omitempty"`
|
||||
LastSuccessAt *time.Time `json:"last_success_at,omitempty"`
|
||||
LastErrorAt *time.Time `json:"last_error_at,omitempty"`
|
||||
LastDurationSeconds float64 `json:"last_duration_seconds"`
|
||||
LastBusinessDate string `json:"last_business_date,omitempty"`
|
||||
LastTenantCount int64 `json:"last_tenant_count"`
|
||||
LastBrandCount int64 `json:"last_brand_count"`
|
||||
LastErrorCode string `json:"last_error_code,omitempty"`
|
||||
LastErrorMessage string `json:"last_error_message,omitempty"`
|
||||
}
|
||||
|
||||
type monitoringDailyTaskMetricsRecorder struct {
|
||||
runTotal atomic.Int64
|
||||
runCompletedTotal atomic.Int64
|
||||
runFailedTotal atomic.Int64
|
||||
runPartialFailureTotal atomic.Int64
|
||||
planFailureTotal atomic.Int64
|
||||
brandFailureTotal atomic.Int64
|
||||
skippedPlanTotal atomic.Int64
|
||||
plannedTaskTotal atomic.Int64
|
||||
dueTaskTotal atomic.Int64
|
||||
createdTaskTotal atomic.Int64
|
||||
desktopTaskTotal atomic.Int64
|
||||
deferredTaskTotal atomic.Int64
|
||||
lastRunAtUnixNano atomic.Int64
|
||||
lastSuccessAtUnixNano atomic.Int64
|
||||
lastErrorAtUnixNano atomic.Int64
|
||||
lastDurationNanos atomic.Int64
|
||||
lastTenantCount atomic.Int64
|
||||
lastBrandCount atomic.Int64
|
||||
lastBusinessDate atomic.Value
|
||||
lastErrorCode atomic.Value
|
||||
lastErrorMessage atomic.Value
|
||||
}
|
||||
|
||||
var monitoringDailyTaskMetrics = newMonitoringDailyTaskMetricsRecorder()
|
||||
|
||||
func newMonitoringDailyTaskMetricsRecorder() *monitoringDailyTaskMetricsRecorder {
|
||||
return &monitoringDailyTaskMetricsRecorder{}
|
||||
}
|
||||
|
||||
func recordMonitoringDailyTaskMetrics(summary *MonitoringDailyTaskGenerationSummary, duration time.Duration, err error) {
|
||||
monitoringDailyTaskMetrics.record(summary, duration, err)
|
||||
}
|
||||
|
||||
func MonitoringDailyTaskMetricsSnapshotValue() MonitoringDailyTaskMetricsSnapshot {
|
||||
return monitoringDailyTaskMetrics.snapshot()
|
||||
}
|
||||
|
||||
func resetMonitoringDailyTaskMetricsForTest() {
|
||||
monitoringDailyTaskMetrics = newMonitoringDailyTaskMetricsRecorder()
|
||||
}
|
||||
|
||||
func (r *monitoringDailyTaskMetricsRecorder) record(summary *MonitoringDailyTaskGenerationSummary, duration time.Duration, err error) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if duration < 0 {
|
||||
duration = 0
|
||||
}
|
||||
|
||||
r.runTotal.Add(1)
|
||||
r.lastRunAtUnixNano.Store(now.UnixNano())
|
||||
r.lastDurationNanos.Store(duration.Nanoseconds())
|
||||
|
||||
if summary != nil {
|
||||
r.lastBusinessDate.Store(summary.BusinessDate)
|
||||
r.lastTenantCount.Store(int64(summary.TenantCount))
|
||||
r.lastBrandCount.Store(int64(summary.BrandCount))
|
||||
r.planFailureTotal.Add(int64(summary.FailedPlanCount))
|
||||
r.brandFailureTotal.Add(int64(summary.FailedBrandCount))
|
||||
r.skippedPlanTotal.Add(int64(summary.SkippedPlanCount))
|
||||
r.plannedTaskTotal.Add(summary.PlannedTaskCount)
|
||||
r.dueTaskTotal.Add(summary.DueTaskCount)
|
||||
r.createdTaskTotal.Add(summary.CreatedTaskCount)
|
||||
r.desktopTaskTotal.Add(summary.DesktopTaskCount)
|
||||
r.deferredTaskTotal.Add(summary.DeferredTaskCount)
|
||||
if summary.FailedPlanCount > 0 || summary.FailedBrandCount > 0 {
|
||||
r.runPartialFailureTotal.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
code, message := monitoringDailyMetricError(err)
|
||||
r.runFailedTotal.Add(1)
|
||||
r.lastErrorAtUnixNano.Store(now.UnixNano())
|
||||
r.lastErrorCode.Store(code)
|
||||
r.lastErrorMessage.Store(message)
|
||||
return
|
||||
}
|
||||
|
||||
r.runCompletedTotal.Add(1)
|
||||
r.lastSuccessAtUnixNano.Store(now.UnixNano())
|
||||
}
|
||||
|
||||
func (r *monitoringDailyTaskMetricsRecorder) snapshot() MonitoringDailyTaskMetricsSnapshot {
|
||||
if r == nil {
|
||||
return MonitoringDailyTaskMetricsSnapshot{}
|
||||
}
|
||||
return MonitoringDailyTaskMetricsSnapshot{
|
||||
RunTotal: r.runTotal.Load(),
|
||||
RunCompletedTotal: r.runCompletedTotal.Load(),
|
||||
RunFailedTotal: r.runFailedTotal.Load(),
|
||||
RunPartialFailureTotal: r.runPartialFailureTotal.Load(),
|
||||
PlanFailureTotal: r.planFailureTotal.Load(),
|
||||
BrandFailureTotal: r.brandFailureTotal.Load(),
|
||||
SkippedPlanTotal: r.skippedPlanTotal.Load(),
|
||||
PlannedTaskTotal: r.plannedTaskTotal.Load(),
|
||||
DueTaskTotal: r.dueTaskTotal.Load(),
|
||||
CreatedTaskTotal: r.createdTaskTotal.Load(),
|
||||
DesktopTaskTotal: r.desktopTaskTotal.Load(),
|
||||
DeferredTaskTotal: r.deferredTaskTotal.Load(),
|
||||
LastRunAt: monitoringDailyUnixNanoTime(r.lastRunAtUnixNano.Load()),
|
||||
LastSuccessAt: monitoringDailyUnixNanoTime(r.lastSuccessAtUnixNano.Load()),
|
||||
LastErrorAt: monitoringDailyUnixNanoTime(r.lastErrorAtUnixNano.Load()),
|
||||
LastDurationSeconds: float64(r.lastDurationNanos.Load()) / float64(time.Second),
|
||||
LastBusinessDate: monitoringDailyLoadString(&r.lastBusinessDate),
|
||||
LastTenantCount: r.lastTenantCount.Load(),
|
||||
LastBrandCount: r.lastBrandCount.Load(),
|
||||
LastErrorCode: monitoringDailyLoadString(&r.lastErrorCode),
|
||||
LastErrorMessage: monitoringDailyLoadString(&r.lastErrorMessage),
|
||||
}
|
||||
}
|
||||
|
||||
func monitoringDailyMetricError(err error) (string, string) {
|
||||
normalized := normalizeMonitoringWorkerError(err)
|
||||
if normalized == nil {
|
||||
return "", ""
|
||||
}
|
||||
switch {
|
||||
case errors.Is(normalized, context.Canceled):
|
||||
return "context_canceled", normalized.Error()
|
||||
case errors.Is(normalized, context.DeadlineExceeded):
|
||||
return "context_deadline_exceeded", normalized.Error()
|
||||
}
|
||||
var appErr *response.AppError
|
||||
if errors.As(normalized, &appErr) {
|
||||
if strings.TrimSpace(appErr.Message) != "" {
|
||||
return appErr.Message, appErr.Error()
|
||||
}
|
||||
return "app_error", appErr.Error()
|
||||
}
|
||||
code := strings.TrimLeft(fmt.Sprintf("%T", normalized), "*")
|
||||
code = strings.ReplaceAll(code, "/", "_")
|
||||
code = strings.ReplaceAll(code, ".", "_")
|
||||
if code == "" {
|
||||
code = "error"
|
||||
}
|
||||
return code, normalized.Error()
|
||||
}
|
||||
|
||||
func monitoringDailyUnixNanoTime(value int64) *time.Time {
|
||||
if value <= 0 {
|
||||
return nil
|
||||
}
|
||||
t := time.Unix(0, value).UTC()
|
||||
return &t
|
||||
}
|
||||
|
||||
func monitoringDailyLoadString(value *atomic.Value) string {
|
||||
if value == nil {
|
||||
return ""
|
||||
}
|
||||
raw := value.Load()
|
||||
result, _ := raw.(string)
|
||||
return result
|
||||
}
|
||||
|
||||
func monitoringDailyTimeSeconds(value *time.Time) float64 {
|
||||
if value == nil || value.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return float64(value.UnixNano()) / float64(time.Second)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,337 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/config"
|
||||
)
|
||||
|
||||
func TestBuildMonitoringDailyTaskCandidatesMaterializesAfterMidnight(t *testing.T) {
|
||||
businessDay := time.Date(2026, 4, 24, 0, 0, 0, 0, monitoringBusinessLocation())
|
||||
questions := []monitoringConfiguredQuestion{
|
||||
{ID: 101, QuestionHash: []byte("q101")},
|
||||
{ID: 102, QuestionHash: []byte("q102")},
|
||||
}
|
||||
platforms := []monitoringPlatformMetadata{
|
||||
{ID: "qwen", Name: "Qwen"},
|
||||
{ID: "doubao", Name: "Doubao"},
|
||||
{ID: "deepseek", Name: "DeepSeek"},
|
||||
}
|
||||
|
||||
candidates := buildMonitoringDailyTaskCandidates(7, 11, businessDay, questions, platforms, 4)
|
||||
|
||||
require.Len(t, candidates, 4)
|
||||
start := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset).UTC()
|
||||
end := start.Add(defaultMonitoringDailyMaterializeWindow)
|
||||
for i, candidate := range candidates {
|
||||
assert.False(t, candidate.MaterializeAt.Before(start))
|
||||
assert.False(t, candidate.MaterializeAt.After(end))
|
||||
assert.Equal(t, candidate.MaterializeAt, candidate.DispatchAfter)
|
||||
if i > 0 {
|
||||
assert.False(t, candidate.MaterializeAt.Before(candidates[i-1].MaterializeAt))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectDueMonitoringDailyCandidatesSkipsExistingAndHonorsLimit(t *testing.T) {
|
||||
businessDay := time.Date(2026, 4, 24, 0, 0, 0, 0, monitoringBusinessLocation())
|
||||
candidates := buildMonitoringDailyTaskCandidates(
|
||||
7,
|
||||
11,
|
||||
businessDay,
|
||||
[]monitoringConfiguredQuestion{
|
||||
{ID: 101, QuestionHash: []byte("q101")},
|
||||
{ID: 102, QuestionHash: []byte("q102")},
|
||||
},
|
||||
[]monitoringPlatformMetadata{
|
||||
{ID: "qwen"},
|
||||
{ID: "doubao"},
|
||||
{ID: "deepseek"},
|
||||
},
|
||||
6,
|
||||
)
|
||||
require.Len(t, candidates, 6)
|
||||
|
||||
existing := map[monitoringDailyTaskKey]struct{}{
|
||||
{
|
||||
QuestionID: candidates[0].Question.ID,
|
||||
PlatformID: candidates[0].Platform.ID,
|
||||
}: {},
|
||||
}
|
||||
now := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset + defaultMonitoringDailyMaterializeWindow)
|
||||
|
||||
due := selectDueMonitoringDailyCandidates(candidates, existing, now, defaultMonitoringDailyLookahead, 2)
|
||||
|
||||
require.Len(t, due, 2)
|
||||
for _, candidate := range due {
|
||||
_, exists := existing[monitoringDailyTaskKey{
|
||||
QuestionID: candidate.Question.ID,
|
||||
PlatformID: candidate.Platform.ID,
|
||||
}]
|
||||
assert.False(t, exists)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitoringDailyRemainingKeepsFullCandidateHorizon(t *testing.T) {
|
||||
businessDay := time.Date(2026, 4, 24, 0, 0, 0, 0, monitoringBusinessLocation())
|
||||
hardCap := 6
|
||||
remaining := monitoringDailyRemainingCapacity(hardCap, 4)
|
||||
assert.Equal(t, 2, remaining)
|
||||
|
||||
candidates := buildMonitoringDailyTaskCandidates(
|
||||
7,
|
||||
11,
|
||||
businessDay,
|
||||
[]monitoringConfiguredQuestion{
|
||||
{ID: 101, QuestionHash: []byte("q101")},
|
||||
{ID: 102, QuestionHash: []byte("q102")},
|
||||
{ID: 103, QuestionHash: []byte("q103")},
|
||||
},
|
||||
[]monitoringPlatformMetadata{
|
||||
{ID: "qwen"},
|
||||
{ID: "doubao"},
|
||||
},
|
||||
hardCap,
|
||||
)
|
||||
require.Len(t, candidates, hardCap)
|
||||
|
||||
existing := make(map[monitoringDailyTaskKey]struct{}, 4)
|
||||
for _, candidate := range candidates[:4] {
|
||||
existing[monitoringDailyTaskKey{
|
||||
QuestionID: candidate.Question.ID,
|
||||
PlatformID: candidate.Platform.ID,
|
||||
}] = struct{}{}
|
||||
}
|
||||
|
||||
now := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset + defaultMonitoringDailyMaterializeWindow)
|
||||
due := selectDueMonitoringDailyCandidates(candidates, existing, now, defaultMonitoringDailyLookahead, remaining)
|
||||
|
||||
require.Len(t, due, remaining)
|
||||
for _, candidate := range due {
|
||||
_, exists := existing[monitoringDailyTaskKey{
|
||||
QuestionID: candidate.Question.ID,
|
||||
PlatformID: candidate.Platform.ID,
|
||||
}]
|
||||
assert.False(t, exists)
|
||||
}
|
||||
assert.Equal(t, 1, decrementMonitoringDailyRemaining(remaining, 1))
|
||||
assert.Equal(t, 0, decrementMonitoringDailyRemaining(remaining, 2))
|
||||
}
|
||||
|
||||
func TestMonitoringDailyQuestionLimitUsesBrandLibraryConfig(t *testing.T) {
|
||||
assert.Equal(t, 25, monitoringDailyQuestionLimit(config.BrandLibraryConfig{
|
||||
MaxKeywords: 5,
|
||||
MaxQuestionsPerKeyword: 5,
|
||||
}))
|
||||
assert.Equal(t, 6, monitoringDailyQuestionLimit(config.BrandLibraryConfig{
|
||||
MaxKeywords: 2,
|
||||
MaxQuestionsPerKeyword: 3,
|
||||
}))
|
||||
|
||||
questions := []monitoringConfiguredQuestion{
|
||||
{ID: 1},
|
||||
{ID: 2},
|
||||
{ID: 3},
|
||||
}
|
||||
assert.Len(t, limitMonitoringDailyQuestions(questions, 2), 2)
|
||||
assert.Len(t, limitMonitoringDailyQuestions(questions, 25), 3)
|
||||
}
|
||||
|
||||
func TestDecodeDailyMonitoringEnabledPlatformsReportsMalformedJSON(t *testing.T) {
|
||||
platforms, err := decodeDailyMonitoringEnabledPlatforms([]byte(`["qwen"`))
|
||||
require.Error(t, err)
|
||||
assert.Nil(t, platforms)
|
||||
}
|
||||
|
||||
func TestDecodeDailyMonitoringEnabledPlatformsReconcilesSupportedPlatforms(t *testing.T) {
|
||||
platforms, err := decodeDailyMonitoringEnabledPlatforms([]byte(`["qwen","unknown","doubao"]`))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"doubao", "qwen"}, platforms)
|
||||
}
|
||||
|
||||
func TestBuildDailyMonitoringPlansFromSelectedClientsRequiresPlatformSnapshot(t *testing.T) {
|
||||
clientOne := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000001")
|
||||
clientTwo := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000002")
|
||||
candidates := []monitoringDailyPlanClientCandidate{
|
||||
{
|
||||
TenantID: 1,
|
||||
WorkspaceID: 10,
|
||||
PrimaryClientID: clientOne,
|
||||
MaxBrands: 3,
|
||||
MaxQuestionsPerBrand: 25,
|
||||
},
|
||||
{
|
||||
TenantID: 1,
|
||||
WorkspaceID: 11,
|
||||
PrimaryClientID: clientTwo,
|
||||
MaxBrands: 2,
|
||||
MaxQuestionsPerBrand: 10,
|
||||
},
|
||||
}
|
||||
enabledByClient := map[monitoringDailyPlanClientKey][]byte{
|
||||
candidates[0].monitoringDailyPlanClientKey(): []byte(`["qwen"]`),
|
||||
}
|
||||
|
||||
plans := buildDailyMonitoringPlansFromSelectedClients(candidates, enabledByClient)
|
||||
|
||||
require.Len(t, plans, 1)
|
||||
assert.Equal(t, int64(1), plans[0].TenantID)
|
||||
assert.Equal(t, int64(10), plans[0].WorkspaceID)
|
||||
assert.Equal(t, 3, plans[0].MaxBrands)
|
||||
assert.Equal(t, 25, plans[0].MaxQuestionsPerBrand)
|
||||
require.NotNil(t, plans[0].PrimaryClientID)
|
||||
assert.Equal(t, clientOne, *plans[0].PrimaryClientID)
|
||||
assert.Equal(t, []byte(`["qwen"]`), plans[0].EnabledJSON)
|
||||
}
|
||||
|
||||
func TestOrderDailyMonitoringPlansRotatesDeterministicallyByBucket(t *testing.T) {
|
||||
plans := []monitoringDailyPlan{
|
||||
{TenantID: 1},
|
||||
{TenantID: 2},
|
||||
{TenantID: 3},
|
||||
}
|
||||
now := time.Date(2026, 4, 24, 1, 0, 0, 0, time.UTC)
|
||||
|
||||
first := orderDailyMonitoringPlans(plans, "2026-04-24", now)
|
||||
second := orderDailyMonitoringPlans(plans, "2026-04-24", now)
|
||||
|
||||
assert.Equal(t, first, second)
|
||||
assert.Equal(t, []monitoringDailyPlan{{TenantID: 1}, {TenantID: 2}, {TenantID: 3}}, plans)
|
||||
}
|
||||
|
||||
func TestResolveDailyMonitoringExecutionOwnerRequiresDesktopRolloutAndClient(t *testing.T) {
|
||||
clientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000001")
|
||||
|
||||
disabled := &MonitoringService{desktopTasksRolloutPercentage: 0}
|
||||
owner, ok, reason := disabled.resolveDailyMonitoringExecutionOwner(monitoringDailyPlan{
|
||||
TenantID: 1,
|
||||
PrimaryClientID: &clientID,
|
||||
})
|
||||
assert.False(t, ok)
|
||||
assert.Empty(t, owner)
|
||||
assert.Equal(t, "desktop_tasks_rollout_disabled", reason)
|
||||
|
||||
enabled := &MonitoringService{desktopTasksRolloutPercentage: 100}
|
||||
owner, ok, reason = enabled.resolveDailyMonitoringExecutionOwner(monitoringDailyPlan{TenantID: 1})
|
||||
assert.False(t, ok)
|
||||
assert.Empty(t, owner)
|
||||
assert.Equal(t, "missing_primary_client", reason)
|
||||
|
||||
owner, ok, reason = enabled.resolveDailyMonitoringExecutionOwner(monitoringDailyPlan{
|
||||
TenantID: 1,
|
||||
PrimaryClientID: &clientID,
|
||||
})
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "desktop_tasks", owner)
|
||||
assert.Empty(t, reason)
|
||||
}
|
||||
|
||||
func TestMonitoringDailyTaskMetricsRecordsSummaryCounters(t *testing.T) {
|
||||
resetMonitoringDailyTaskMetricsForTest()
|
||||
t.Cleanup(resetMonitoringDailyTaskMetricsForTest)
|
||||
|
||||
recordMonitoringDailyTaskMetrics(&MonitoringDailyTaskGenerationSummary{
|
||||
BusinessDate: "2026-04-24",
|
||||
TenantCount: 3,
|
||||
BrandCount: 2,
|
||||
FailedPlanCount: 1,
|
||||
FailedBrandCount: 2,
|
||||
PlannedTaskCount: 20,
|
||||
DueTaskCount: 8,
|
||||
CreatedTaskCount: 6,
|
||||
DesktopTaskCount: 4,
|
||||
DeferredTaskCount: 2,
|
||||
SkippedPlanCount: 1,
|
||||
}, 1500*time.Millisecond, nil)
|
||||
|
||||
snapshot := MonitoringDailyTaskMetricsSnapshotValue()
|
||||
assert.Equal(t, int64(1), snapshot.RunTotal)
|
||||
assert.Equal(t, int64(1), snapshot.RunCompletedTotal)
|
||||
assert.Equal(t, int64(0), snapshot.RunFailedTotal)
|
||||
assert.Equal(t, int64(1), snapshot.RunPartialFailureTotal)
|
||||
assert.Equal(t, int64(1), snapshot.PlanFailureTotal)
|
||||
assert.Equal(t, int64(2), snapshot.BrandFailureTotal)
|
||||
assert.Equal(t, int64(1), snapshot.SkippedPlanTotal)
|
||||
assert.Equal(t, int64(20), snapshot.PlannedTaskTotal)
|
||||
assert.Equal(t, int64(8), snapshot.DueTaskTotal)
|
||||
assert.Equal(t, int64(6), snapshot.CreatedTaskTotal)
|
||||
assert.Equal(t, int64(4), snapshot.DesktopTaskTotal)
|
||||
assert.Equal(t, int64(2), snapshot.DeferredTaskTotal)
|
||||
assert.Equal(t, "2026-04-24", snapshot.LastBusinessDate)
|
||||
assert.Equal(t, int64(3), snapshot.LastTenantCount)
|
||||
assert.Equal(t, int64(2), snapshot.LastBrandCount)
|
||||
assert.NotNil(t, snapshot.LastRunAt)
|
||||
assert.NotNil(t, snapshot.LastSuccessAt)
|
||||
|
||||
prometheus := MonitoringDailyTaskMetricsPrometheus()
|
||||
assert.Contains(t, prometheus, `monitoring_daily_task_worker_runs_total{status="completed"} 1`)
|
||||
assert.Contains(t, prometheus, `monitoring_daily_task_generation_failures_total{scope="plan"} 1`)
|
||||
assert.Contains(t, prometheus, `monitoring_daily_task_generation_tasks_total{stage="created"} 6`)
|
||||
assert.Contains(t, prometheus, `monitoring_daily_task_worker_last_business_date_info{business_date="2026-04-24"} 1`)
|
||||
}
|
||||
|
||||
func TestMonitoringDailyTaskMetricsRecordsFatalError(t *testing.T) {
|
||||
resetMonitoringDailyTaskMetricsForTest()
|
||||
t.Cleanup(resetMonitoringDailyTaskMetricsForTest)
|
||||
|
||||
recordMonitoringDailyTaskMetrics(nil, time.Second, context.DeadlineExceeded)
|
||||
|
||||
snapshot := MonitoringDailyTaskMetricsSnapshotValue()
|
||||
assert.Equal(t, int64(1), snapshot.RunTotal)
|
||||
assert.Equal(t, int64(0), snapshot.RunCompletedTotal)
|
||||
assert.Equal(t, int64(1), snapshot.RunFailedTotal)
|
||||
assert.Equal(t, "context_deadline_exceeded", snapshot.LastErrorCode)
|
||||
assert.NotNil(t, snapshot.LastRunAt)
|
||||
assert.NotNil(t, snapshot.LastErrorAt)
|
||||
|
||||
prometheus := MonitoringDailyTaskMetricsPrometheus()
|
||||
assert.Contains(t, prometheus, `monitoring_daily_task_worker_runs_total{status="failed"} 1`)
|
||||
assert.Contains(t, prometheus, `monitoring_daily_task_worker_last_error_info{code="context_deadline_exceeded"} 1`)
|
||||
}
|
||||
|
||||
func TestMonitoringDailyTaskPrometheusEscapesCarriageReturnLabels(t *testing.T) {
|
||||
resetMonitoringDailyTaskMetricsForTest()
|
||||
t.Cleanup(resetMonitoringDailyTaskMetricsForTest)
|
||||
|
||||
monitoringDailyTaskMetrics.lastBusinessDate.Store("2026-04-24\rbad")
|
||||
|
||||
prometheus := MonitoringDailyTaskMetricsPrometheus()
|
||||
assert.NotContains(t, prometheus, "\r")
|
||||
assert.Contains(t, prometheus, `monitoring_daily_task_worker_last_business_date_info{business_date="2026-04-24\\rbad"} 1`)
|
||||
}
|
||||
|
||||
func TestMonitoringSchedulerMetricsPrometheusHandlerIncludesRuntimeAndProcessCollectors(t *testing.T) {
|
||||
resetMonitoringDailyTaskMetricsForTest()
|
||||
t.Cleanup(resetMonitoringDailyTaskMetricsForTest)
|
||||
recordMonitoringDailyTaskMetrics(&MonitoringDailyTaskGenerationSummary{CreatedTaskCount: 1}, time.Second, nil)
|
||||
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
MonitoringSchedulerMetricsPrometheusHandler().ServeHTTP(recorder, request)
|
||||
|
||||
body := recorder.Body.String()
|
||||
assert.Equal(t, http.StatusOK, recorder.Code)
|
||||
assert.Contains(t, body, `monitoring_daily_task_worker_runs_total{status="completed"} 1`)
|
||||
assert.Contains(t, body, "go_goroutines")
|
||||
if runtime.GOOS == "linux" || runtime.GOOS == "windows" {
|
||||
assert.Contains(t, body, "process_cpu_seconds_total")
|
||||
}
|
||||
assert.True(t, strings.HasPrefix(recorder.Header().Get("Content-Type"), "text/plain"))
|
||||
}
|
||||
|
||||
func mustParseDailyMonitoringUUID(t *testing.T, value string) uuid.UUID {
|
||||
t.Helper()
|
||||
id, err := uuid.Parse(value)
|
||||
require.NoError(t, err)
|
||||
return id
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
type MonitoringHeartbeatPrimaryLeaseMetricsSnapshot struct {
|
||||
PrimaryHitTotal int64 `json:"primary_hit_total"`
|
||||
NonPrimaryLiveTotal int64 `json:"non_primary_live_total"`
|
||||
InsertedTotal int64 `json:"inserted_total"`
|
||||
RenewedTotal int64 `json:"renewed_total"`
|
||||
ClaimedExpiredTotal int64 `json:"claimed_expired_total"`
|
||||
ClaimedUnavailableTotal int64 `json:"claimed_unavailable_total"`
|
||||
ContentionTotal int64 `json:"contention_total"`
|
||||
ErrorTotal int64 `json:"error_total"`
|
||||
DBInsertTotal int64 `json:"db_insert_total"`
|
||||
DBRenewTotal int64 `json:"db_renew_total"`
|
||||
DBClaimTotal int64 `json:"db_claim_total"`
|
||||
DurationCount int64 `json:"duration_count"`
|
||||
DurationSumSeconds float64 `json:"duration_sum_seconds"`
|
||||
LastDurationSeconds float64 `json:"last_duration_seconds"`
|
||||
LastOutcome string `json:"last_outcome,omitempty"`
|
||||
}
|
||||
|
||||
type monitoringHeartbeatPrimaryLeaseMetricsRecorder struct {
|
||||
primaryHitTotal atomic.Int64
|
||||
nonPrimaryLiveTotal atomic.Int64
|
||||
insertedTotal atomic.Int64
|
||||
renewedTotal atomic.Int64
|
||||
claimedExpiredTotal atomic.Int64
|
||||
claimedUnavailableTotal atomic.Int64
|
||||
contentionTotal atomic.Int64
|
||||
errorTotal atomic.Int64
|
||||
dbInsertTotal atomic.Int64
|
||||
dbRenewTotal atomic.Int64
|
||||
dbClaimTotal atomic.Int64
|
||||
durationCount atomic.Int64
|
||||
durationSumNanos atomic.Int64
|
||||
lastDurationNanos atomic.Int64
|
||||
lastOutcome atomic.Value
|
||||
}
|
||||
|
||||
var monitoringHeartbeatPrimaryLeaseMetrics = newMonitoringHeartbeatPrimaryLeaseMetricsRecorder()
|
||||
|
||||
func newMonitoringHeartbeatPrimaryLeaseMetricsRecorder() *monitoringHeartbeatPrimaryLeaseMetricsRecorder {
|
||||
return &monitoringHeartbeatPrimaryLeaseMetricsRecorder{}
|
||||
}
|
||||
|
||||
func recordHeartbeatPrimaryLeaseMetric(outcome string, duration time.Duration) {
|
||||
monitoringHeartbeatPrimaryLeaseMetrics.record(outcome, duration)
|
||||
}
|
||||
|
||||
func recordHeartbeatPrimaryLeaseWriteMetric(operation string) {
|
||||
monitoringHeartbeatPrimaryLeaseMetrics.recordWrite(operation)
|
||||
}
|
||||
|
||||
func MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue() MonitoringHeartbeatPrimaryLeaseMetricsSnapshot {
|
||||
return monitoringHeartbeatPrimaryLeaseMetrics.snapshot()
|
||||
}
|
||||
|
||||
func resetMonitoringHeartbeatPrimaryLeaseMetricsForTest() {
|
||||
monitoringHeartbeatPrimaryLeaseMetrics = newMonitoringHeartbeatPrimaryLeaseMetricsRecorder()
|
||||
}
|
||||
|
||||
func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) record(outcome string, duration time.Duration) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
if duration < 0 {
|
||||
duration = 0
|
||||
}
|
||||
switch outcome {
|
||||
case "primary_hit":
|
||||
r.primaryHitTotal.Add(1)
|
||||
case "non_primary_live":
|
||||
r.nonPrimaryLiveTotal.Add(1)
|
||||
case "inserted":
|
||||
r.insertedTotal.Add(1)
|
||||
case "renewed":
|
||||
r.renewedTotal.Add(1)
|
||||
case "claimed_expired":
|
||||
r.claimedExpiredTotal.Add(1)
|
||||
case "claimed_unavailable":
|
||||
r.claimedUnavailableTotal.Add(1)
|
||||
case "contention":
|
||||
r.contentionTotal.Add(1)
|
||||
case "error":
|
||||
r.errorTotal.Add(1)
|
||||
}
|
||||
r.durationCount.Add(1)
|
||||
r.durationSumNanos.Add(duration.Nanoseconds())
|
||||
r.lastDurationNanos.Store(duration.Nanoseconds())
|
||||
r.lastOutcome.Store(outcome)
|
||||
}
|
||||
|
||||
func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) recordWrite(operation string) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
switch operation {
|
||||
case "insert":
|
||||
r.dbInsertTotal.Add(1)
|
||||
case "renew":
|
||||
r.dbRenewTotal.Add(1)
|
||||
case "claim":
|
||||
r.dbClaimTotal.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *monitoringHeartbeatPrimaryLeaseMetricsRecorder) snapshot() MonitoringHeartbeatPrimaryLeaseMetricsSnapshot {
|
||||
if r == nil {
|
||||
return MonitoringHeartbeatPrimaryLeaseMetricsSnapshot{}
|
||||
}
|
||||
return MonitoringHeartbeatPrimaryLeaseMetricsSnapshot{
|
||||
PrimaryHitTotal: r.primaryHitTotal.Load(),
|
||||
NonPrimaryLiveTotal: r.nonPrimaryLiveTotal.Load(),
|
||||
InsertedTotal: r.insertedTotal.Load(),
|
||||
RenewedTotal: r.renewedTotal.Load(),
|
||||
ClaimedExpiredTotal: r.claimedExpiredTotal.Load(),
|
||||
ClaimedUnavailableTotal: r.claimedUnavailableTotal.Load(),
|
||||
ContentionTotal: r.contentionTotal.Load(),
|
||||
ErrorTotal: r.errorTotal.Load(),
|
||||
DBInsertTotal: r.dbInsertTotal.Load(),
|
||||
DBRenewTotal: r.dbRenewTotal.Load(),
|
||||
DBClaimTotal: r.dbClaimTotal.Load(),
|
||||
DurationCount: r.durationCount.Load(),
|
||||
DurationSumSeconds: float64(r.durationSumNanos.Load()) / float64(time.Second),
|
||||
LastDurationSeconds: float64(r.lastDurationNanos.Load()) / float64(time.Second),
|
||||
LastOutcome: heartbeatPrimaryMetricLoadString(&r.lastOutcome),
|
||||
}
|
||||
}
|
||||
|
||||
func heartbeatPrimaryMetricLoadString(value *atomic.Value) string {
|
||||
if value == nil {
|
||||
return ""
|
||||
}
|
||||
raw := value.Load()
|
||||
result, _ := raw.(string)
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
type MonitoringHeartbeatSnapshotMetricsSnapshot struct {
|
||||
PlatformReportedTotal int64 `json:"platform_reported_total"`
|
||||
TransactionOpenedTotal int64 `json:"transaction_opened_total"`
|
||||
SnapshotInsertedTotal int64 `json:"snapshot_inserted_total"`
|
||||
SnapshotUpdatedTotal int64 `json:"snapshot_updated_total"`
|
||||
SnapshotRefreshedTotal int64 `json:"snapshot_refreshed_total"`
|
||||
SnapshotSkippedTotal int64 `json:"snapshot_skipped_total"`
|
||||
ReconciledSkippedTaskTotal int64 `json:"reconciled_skipped_task_total"`
|
||||
ProjectionRebuildTotal int64 `json:"projection_rebuild_total"`
|
||||
ErrorTotal int64 `json:"error_total"`
|
||||
DurationCount int64 `json:"duration_count"`
|
||||
DurationSumSeconds float64 `json:"duration_sum_seconds"`
|
||||
LastDurationSeconds float64 `json:"last_duration_seconds"`
|
||||
LastErrorCode string `json:"last_error_code,omitempty"`
|
||||
LastErrorMessage string `json:"last_error_message,omitempty"`
|
||||
}
|
||||
|
||||
type monitoringHeartbeatSnapshotMetricsEvent struct {
|
||||
Active bool
|
||||
PlatformCount int
|
||||
TransactionOpened bool
|
||||
SnapshotInsertedCount int64
|
||||
SnapshotUpdatedCount int64
|
||||
SnapshotRefreshedCount int64
|
||||
SnapshotSkippedCount int64
|
||||
ReconciledSkippedTaskCount int64
|
||||
ProjectionRebuildCount int64
|
||||
Duration time.Duration
|
||||
Err error
|
||||
}
|
||||
|
||||
type monitoringHeartbeatSnapshotMetricsRecorder struct {
|
||||
platformReportedTotal atomic.Int64
|
||||
transactionOpenedTotal atomic.Int64
|
||||
snapshotInsertedTotal atomic.Int64
|
||||
snapshotUpdatedTotal atomic.Int64
|
||||
snapshotRefreshedTotal atomic.Int64
|
||||
snapshotSkippedTotal atomic.Int64
|
||||
reconciledSkippedTaskTotal atomic.Int64
|
||||
projectionRebuildTotal atomic.Int64
|
||||
errorTotal atomic.Int64
|
||||
durationCount atomic.Int64
|
||||
durationSumNanos atomic.Int64
|
||||
lastDurationNanos atomic.Int64
|
||||
lastErrorCode atomic.Value
|
||||
lastErrorMessage atomic.Value
|
||||
}
|
||||
|
||||
var monitoringHeartbeatSnapshotMetrics = newMonitoringHeartbeatSnapshotMetricsRecorder()
|
||||
|
||||
func newMonitoringHeartbeatSnapshotMetricsRecorder() *monitoringHeartbeatSnapshotMetricsRecorder {
|
||||
return &monitoringHeartbeatSnapshotMetricsRecorder{}
|
||||
}
|
||||
|
||||
func recordMonitoringHeartbeatSnapshotMetrics(event monitoringHeartbeatSnapshotMetricsEvent) {
|
||||
monitoringHeartbeatSnapshotMetrics.record(event)
|
||||
}
|
||||
|
||||
func MonitoringHeartbeatSnapshotMetricsSnapshotValue() MonitoringHeartbeatSnapshotMetricsSnapshot {
|
||||
return monitoringHeartbeatSnapshotMetrics.snapshot()
|
||||
}
|
||||
|
||||
func resetMonitoringHeartbeatSnapshotMetricsForTest() {
|
||||
monitoringHeartbeatSnapshotMetrics = newMonitoringHeartbeatSnapshotMetricsRecorder()
|
||||
}
|
||||
|
||||
func (r *monitoringHeartbeatSnapshotMetricsRecorder) record(event monitoringHeartbeatSnapshotMetricsEvent) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
if event.Duration < 0 {
|
||||
event.Duration = 0
|
||||
}
|
||||
r.platformReportedTotal.Add(int64(event.PlatformCount))
|
||||
if event.TransactionOpened {
|
||||
r.transactionOpenedTotal.Add(1)
|
||||
}
|
||||
r.snapshotInsertedTotal.Add(event.SnapshotInsertedCount)
|
||||
r.snapshotUpdatedTotal.Add(event.SnapshotUpdatedCount)
|
||||
r.snapshotRefreshedTotal.Add(event.SnapshotRefreshedCount)
|
||||
r.snapshotSkippedTotal.Add(event.SnapshotSkippedCount)
|
||||
r.reconciledSkippedTaskTotal.Add(event.ReconciledSkippedTaskCount)
|
||||
r.projectionRebuildTotal.Add(event.ProjectionRebuildCount)
|
||||
r.durationCount.Add(1)
|
||||
r.durationSumNanos.Add(event.Duration.Nanoseconds())
|
||||
r.lastDurationNanos.Store(event.Duration.Nanoseconds())
|
||||
|
||||
if event.Err != nil {
|
||||
code, message := monitoringDailyMetricError(event.Err)
|
||||
r.errorTotal.Add(1)
|
||||
r.lastErrorCode.Store(code)
|
||||
r.lastErrorMessage.Store(message)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *monitoringHeartbeatSnapshotMetricsRecorder) snapshot() MonitoringHeartbeatSnapshotMetricsSnapshot {
|
||||
if r == nil {
|
||||
return MonitoringHeartbeatSnapshotMetricsSnapshot{}
|
||||
}
|
||||
return MonitoringHeartbeatSnapshotMetricsSnapshot{
|
||||
PlatformReportedTotal: r.platformReportedTotal.Load(),
|
||||
TransactionOpenedTotal: r.transactionOpenedTotal.Load(),
|
||||
SnapshotInsertedTotal: r.snapshotInsertedTotal.Load(),
|
||||
SnapshotUpdatedTotal: r.snapshotUpdatedTotal.Load(),
|
||||
SnapshotRefreshedTotal: r.snapshotRefreshedTotal.Load(),
|
||||
SnapshotSkippedTotal: r.snapshotSkippedTotal.Load(),
|
||||
ReconciledSkippedTaskTotal: r.reconciledSkippedTaskTotal.Load(),
|
||||
ProjectionRebuildTotal: r.projectionRebuildTotal.Load(),
|
||||
ErrorTotal: r.errorTotal.Load(),
|
||||
DurationCount: r.durationCount.Load(),
|
||||
DurationSumSeconds: float64(r.durationSumNanos.Load()) / float64(time.Second),
|
||||
LastDurationSeconds: float64(r.lastDurationNanos.Load()) / float64(time.Second),
|
||||
LastErrorCode: monitoringDailyLoadString(&r.lastErrorCode),
|
||||
LastErrorMessage: monitoringDailyLoadString(&r.lastErrorMessage),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
promcollectors "github.com/prometheus/client_golang/prometheus/collectors"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"github.com/prometheus/common/expfmt"
|
||||
)
|
||||
|
||||
func MonitoringSchedulerMetricsPrometheusHandler() http.Handler {
|
||||
return monitoringPrometheusHandler(true, monitoringDailyTaskMetricsCollector{})
|
||||
}
|
||||
|
||||
func MonitoringDailyTaskMetricsPrometheus() string {
|
||||
return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringDailyTaskMetricsCollector{}))
|
||||
}
|
||||
|
||||
func MonitoringDailyTaskMetricsPrometheusHandler() http.Handler {
|
||||
return monitoringPrometheusHandler(false, monitoringDailyTaskMetricsCollector{})
|
||||
}
|
||||
|
||||
func MonitoringHeartbeatPrimaryLeaseMetricsPrometheus() string {
|
||||
return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringHeartbeatPrimaryLeaseMetricsCollector{}))
|
||||
}
|
||||
|
||||
func MonitoringHeartbeatPrimaryLeaseMetricsPrometheusHandler() http.Handler {
|
||||
return monitoringPrometheusHandler(false, monitoringHeartbeatPrimaryLeaseMetricsCollector{})
|
||||
}
|
||||
|
||||
func MonitoringHeartbeatSnapshotMetricsPrometheus() string {
|
||||
return renderMonitoringPrometheus(monitoringPrometheusRegistry(false, monitoringHeartbeatSnapshotMetricsCollector{}))
|
||||
}
|
||||
|
||||
func MonitoringHeartbeatSnapshotMetricsPrometheusHandler() http.Handler {
|
||||
return monitoringPrometheusHandler(false, monitoringHeartbeatSnapshotMetricsCollector{})
|
||||
}
|
||||
|
||||
func monitoringPrometheusHandler(includeRuntime bool, metricCollectors ...prometheus.Collector) http.Handler {
|
||||
return promhttp.HandlerFor(monitoringPrometheusRegistry(includeRuntime, metricCollectors...), promhttp.HandlerOpts{})
|
||||
}
|
||||
|
||||
func monitoringPrometheusRegistry(includeRuntime bool, metricCollectors ...prometheus.Collector) *prometheus.Registry {
|
||||
registry := prometheus.NewRegistry()
|
||||
if includeRuntime {
|
||||
registry.MustRegister(
|
||||
promcollectors.NewGoCollector(),
|
||||
promcollectors.NewProcessCollector(promcollectors.ProcessCollectorOpts{}),
|
||||
)
|
||||
}
|
||||
for _, collector := range metricCollectors {
|
||||
registry.MustRegister(collector)
|
||||
}
|
||||
return registry
|
||||
}
|
||||
|
||||
func renderMonitoringPrometheus(gatherer prometheus.Gatherer) string {
|
||||
metricFamilies, err := gatherer.Gather()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
var b bytes.Buffer
|
||||
encoder := expfmt.NewEncoder(&b, expfmt.NewFormat(expfmt.TypeTextPlain))
|
||||
for _, metricFamily := range metricFamilies {
|
||||
if err := encoder.Encode(metricFamily); err != nil {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
if closer, ok := encoder.(expfmt.Closer); ok {
|
||||
_ = closer.Close()
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
type monitoringDailyTaskMetricsCollector struct{}
|
||||
|
||||
func (monitoringDailyTaskMetricsCollector) Describe(chan<- *prometheus.Desc) {}
|
||||
|
||||
func (monitoringDailyTaskMetricsCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
snapshot := MonitoringDailyTaskMetricsSnapshotValue()
|
||||
monitoringCounter(ch, "monitoring_daily_task_worker_runs_total", "Total daily monitoring worker runs by terminal status.", float64(snapshot.RunCompletedTotal), []string{"status"}, "completed")
|
||||
monitoringCounter(ch, "monitoring_daily_task_worker_runs_total", "Total daily monitoring worker runs by terminal status.", float64(snapshot.RunFailedTotal), []string{"status"}, "failed")
|
||||
monitoringCounter(ch, "monitoring_daily_task_worker_partial_failure_runs_total", "Total completed daily monitoring worker runs with at least one plan or brand failure.", float64(snapshot.RunPartialFailureTotal), nil)
|
||||
monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.PlanFailureTotal), []string{"scope"}, "plan")
|
||||
monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.BrandFailureTotal), []string{"scope"}, "brand")
|
||||
monitoringCounter(ch, "monitoring_daily_task_generation_skipped_plans_total", "Total daily monitoring plans skipped before generation.", float64(snapshot.SkippedPlanTotal), nil)
|
||||
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.PlannedTaskTotal), []string{"stage"}, "planned")
|
||||
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DueTaskTotal), []string{"stage"}, "due")
|
||||
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.CreatedTaskTotal), []string{"stage"}, "created")
|
||||
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DesktopTaskTotal), []string{"stage"}, "desktop")
|
||||
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DeferredTaskTotal), []string{"stage"}, "deferred")
|
||||
monitoringGauge(ch, "monitoring_daily_task_worker_last_run_timestamp_seconds", "Unix timestamp of the last daily monitoring worker run completion.", monitoringDailyTimeSeconds(snapshot.LastRunAt), nil)
|
||||
monitoringGauge(ch, "monitoring_daily_task_worker_last_success_timestamp_seconds", "Unix timestamp of the last completed daily monitoring worker run.", monitoringDailyTimeSeconds(snapshot.LastSuccessAt), nil)
|
||||
monitoringGauge(ch, "monitoring_daily_task_worker_last_error_timestamp_seconds", "Unix timestamp of the last failed daily monitoring worker run.", monitoringDailyTimeSeconds(snapshot.LastErrorAt), nil)
|
||||
monitoringGauge(ch, "monitoring_daily_task_worker_last_duration_seconds", "Duration of the last daily monitoring worker run.", snapshot.LastDurationSeconds, nil)
|
||||
monitoringGauge(ch, "monitoring_daily_task_worker_last_scope_count", "Tenant and brand counts observed in the last daily monitoring worker run.", float64(snapshot.LastTenantCount), []string{"scope"}, "tenant")
|
||||
monitoringGauge(ch, "monitoring_daily_task_worker_last_scope_count", "Tenant and brand counts observed in the last daily monitoring worker run.", float64(snapshot.LastBrandCount), []string{"scope"}, "brand")
|
||||
if snapshot.LastBusinessDate != "" {
|
||||
monitoringGauge(ch, "monitoring_daily_task_worker_last_business_date_info", "Business date processed by the last daily monitoring worker run.", 1, []string{"business_date"}, snapshot.LastBusinessDate)
|
||||
}
|
||||
if snapshot.LastErrorCode != "" {
|
||||
monitoringGauge(ch, "monitoring_daily_task_worker_last_error_info", "Last fatal daily monitoring worker error code.", 1, []string{"code"}, snapshot.LastErrorCode)
|
||||
}
|
||||
}
|
||||
|
||||
type monitoringHeartbeatPrimaryLeaseMetricsCollector struct{}
|
||||
|
||||
func (monitoringHeartbeatPrimaryLeaseMetricsCollector) Describe(chan<- *prometheus.Desc) {}
|
||||
|
||||
func (monitoringHeartbeatPrimaryLeaseMetricsCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
snapshot := MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue()
|
||||
help := "Total monitoring heartbeat primary lease resolutions by outcome."
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.PrimaryHitTotal), []string{"outcome"}, "primary_hit")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.NonPrimaryLiveTotal), []string{"outcome"}, "non_primary_live")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.InsertedTotal), []string{"outcome"}, "inserted")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.RenewedTotal), []string{"outcome"}, "renewed")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ClaimedExpiredTotal), []string{"outcome"}, "claimed_expired")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ClaimedUnavailableTotal), []string{"outcome"}, "claimed_unavailable")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ContentionTotal), []string{"outcome"}, "contention")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_resolutions_total", help, float64(snapshot.ErrorTotal), []string{"outcome"}, "error")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBInsertTotal), []string{"operation"}, "insert")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBRenewTotal), []string{"operation"}, "renew")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_primary_lease_db_writes_total", "Total monitoring heartbeat primary lease database writes by operation.", float64(snapshot.DBClaimTotal), []string{"operation"}, "claim")
|
||||
monitoringSummary(ch, "monitoring_heartbeat_primary_lease_duration_seconds", "Monitoring heartbeat primary lease resolution duration summary.", snapshot.DurationCount, snapshot.DurationSumSeconds, nil)
|
||||
monitoringGauge(ch, "monitoring_heartbeat_primary_lease_duration_seconds_last", "Last monitoring heartbeat primary lease resolution duration in seconds.", snapshot.LastDurationSeconds, nil)
|
||||
if snapshot.LastOutcome != "" {
|
||||
monitoringGauge(ch, "monitoring_heartbeat_primary_lease_last_outcome_info", "Last monitoring heartbeat primary lease resolution outcome.", 1, []string{"outcome"}, snapshot.LastOutcome)
|
||||
}
|
||||
}
|
||||
|
||||
type monitoringHeartbeatSnapshotMetricsCollector struct{}
|
||||
|
||||
func (monitoringHeartbeatSnapshotMetricsCollector) Describe(chan<- *prometheus.Desc) {}
|
||||
|
||||
func (monitoringHeartbeatSnapshotMetricsCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
snapshot := MonitoringHeartbeatSnapshotMetricsSnapshotValue()
|
||||
monitoringCounter(ch, "monitoring_heartbeat_snapshot_platform_reports_total", "Total monitoring heartbeat platform reports received.", float64(snapshot.PlatformReportedTotal), nil)
|
||||
monitoringCounter(ch, "monitoring_heartbeat_snapshot_transactions_total", "Total monitoring heartbeat snapshot transactions opened.", float64(snapshot.TransactionOpenedTotal), nil)
|
||||
monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotInsertedTotal), []string{"operation"}, "insert")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotUpdatedTotal), []string{"operation"}, "update")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_snapshot_writes_total", "Total monitoring heartbeat access snapshot writes by operation.", float64(snapshot.SnapshotRefreshedTotal), []string{"operation"}, "refresh")
|
||||
monitoringCounter(ch, "monitoring_heartbeat_snapshot_skips_total", "Total monitoring heartbeat access snapshot reports skipped because state was fresh and unchanged.", float64(snapshot.SnapshotSkippedTotal), nil)
|
||||
monitoringCounter(ch, "monitoring_heartbeat_snapshot_reconciled_tasks_total", "Total pending monitoring tasks skipped by heartbeat access reconciliation.", float64(snapshot.ReconciledSkippedTaskTotal), nil)
|
||||
monitoringCounter(ch, "monitoring_heartbeat_snapshot_projection_rebuilds_total", "Total monitoring daily projection rebuilds triggered by heartbeat snapshots.", float64(snapshot.ProjectionRebuildTotal), nil)
|
||||
monitoringCounter(ch, "monitoring_heartbeat_snapshot_errors_total", "Total monitoring heartbeat snapshot errors.", float64(snapshot.ErrorTotal), nil)
|
||||
monitoringSummary(ch, "monitoring_heartbeat_snapshot_duration_seconds", "Monitoring heartbeat snapshot processing duration summary.", snapshot.DurationCount, snapshot.DurationSumSeconds, nil)
|
||||
monitoringGauge(ch, "monitoring_heartbeat_snapshot_duration_seconds_last", "Last monitoring heartbeat snapshot processing duration in seconds.", snapshot.LastDurationSeconds, nil)
|
||||
if snapshot.LastErrorCode != "" {
|
||||
monitoringGauge(ch, "monitoring_heartbeat_snapshot_last_error_info", "Last monitoring heartbeat snapshot error code.", 1, []string{"code"}, snapshot.LastErrorCode)
|
||||
}
|
||||
}
|
||||
|
||||
func monitoringCounter(ch chan<- prometheus.Metric, name, help string, value float64, labelNames []string, labelValues ...string) {
|
||||
monitoringMetric(ch, name, help, prometheus.CounterValue, value, labelNames, labelValues...)
|
||||
}
|
||||
|
||||
func monitoringGauge(ch chan<- prometheus.Metric, name, help string, value float64, labelNames []string, labelValues ...string) {
|
||||
monitoringMetric(ch, name, help, prometheus.GaugeValue, value, labelNames, labelValues...)
|
||||
}
|
||||
|
||||
func monitoringMetric(ch chan<- prometheus.Metric, name, help string, valueType prometheus.ValueType, value float64, labelNames []string, labelValues ...string) {
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(name, help, labelNames, nil),
|
||||
valueType,
|
||||
value,
|
||||
monitoringPrometheusLabelValues(labelValues)...,
|
||||
)
|
||||
}
|
||||
|
||||
func monitoringSummary(ch chan<- prometheus.Metric, name, help string, count int64, sum float64, labelNames []string, labelValues ...string) {
|
||||
if count < 0 {
|
||||
count = 0
|
||||
}
|
||||
ch <- prometheus.MustNewConstSummary(
|
||||
prometheus.NewDesc(name, help, labelNames, nil),
|
||||
uint64(count),
|
||||
sum,
|
||||
nil,
|
||||
monitoringPrometheusLabelValues(labelValues)...,
|
||||
)
|
||||
}
|
||||
|
||||
func monitoringPrometheusLabelValues(values []string) []string {
|
||||
if len(values) == 0 {
|
||||
return values
|
||||
}
|
||||
result := make([]string, len(values))
|
||||
for idx, value := range values {
|
||||
result[idx] = strings.ReplaceAll(value, "\r", `\r`)
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -223,8 +223,7 @@ func (s *MonitoringService) loadMonitorDesktopAccountMap(
|
||||
rows, err := s.businessPool.Query(ctx, `
|
||||
SELECT DISTINCT ON (platform_id)
|
||||
platform_id,
|
||||
desktop_id,
|
||||
health
|
||||
desktop_id
|
||||
FROM platform_accounts
|
||||
WHERE tenant_id = $1
|
||||
AND workspace_id = $2
|
||||
@@ -251,14 +250,10 @@ func (s *MonitoringService) loadMonitorDesktopAccountMap(
|
||||
var (
|
||||
platformID string
|
||||
desktopID uuid.UUID
|
||||
health string
|
||||
)
|
||||
if scanErr := rows.Scan(&platformID, &desktopID, &health); scanErr != nil {
|
||||
if scanErr := rows.Scan(&platformID, &desktopID); scanErr != nil {
|
||||
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to parse monitor desktop accounts")
|
||||
}
|
||||
if strings.TrimSpace(health) != "live" {
|
||||
continue
|
||||
}
|
||||
result[platformID] = desktopID
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
|
||||
@@ -3,11 +3,9 @@ package app
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type monitoringBrandDailyAggregate struct {
|
||||
@@ -33,7 +31,7 @@ type monitoringPlatformDailyAggregate struct {
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) rebuildBrandDailyProjection(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDate time.Time) error {
|
||||
platforms, err := loadMonitoringProjectionPlatforms(ctx, s.businessPool, tenantID)
|
||||
platforms, err := loadMonitoringProjectionPlatforms(ctx, tx, tenantID, brandID, businessDate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -133,29 +131,8 @@ func (s *MonitoringCallbackService) rebuildBrandDailyProjection(ctx context.Cont
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadMonitoringProjectionPlatforms(ctx context.Context, businessPool *pgxpool.Pool, tenantID int64) ([]monitoringPlatformMetadata, error) {
|
||||
enabled := []string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"}
|
||||
|
||||
var enabledJSON []byte
|
||||
if err := businessPool.QueryRow(ctx, `
|
||||
SELECT enabled_platforms
|
||||
FROM tenant_monitoring_quotas
|
||||
WHERE tenant_id = $1
|
||||
`, tenantID).Scan(&enabledJSON); err != nil {
|
||||
if err == pgx.ErrNoRows {
|
||||
return resolveMonitoringPlatforms(enabled), nil
|
||||
}
|
||||
return nil, monitoringInternalError(50041, "quota_lookup_failed", "failed to load monitoring quota", err)
|
||||
}
|
||||
|
||||
if len(enabledJSON) > 0 {
|
||||
var configured []string
|
||||
if jsonErr := json.Unmarshal(enabledJSON, &configured); jsonErr == nil && len(configured) > 0 {
|
||||
enabled = monitoringPlatformIDs(defaultMonitoringPlatforms)
|
||||
}
|
||||
}
|
||||
|
||||
return resolveMonitoringPlatforms(enabled), nil
|
||||
func loadMonitoringProjectionPlatforms(_ context.Context, _ pgx.Tx, _ int64, _ int64, _ time.Time) ([]monitoringPlatformMetadata, error) {
|
||||
return defaultMonitoringPlatformMetadata(), nil
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) loadMonitoringEnabledQuestionCount(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDate time.Time) (int64, error) {
|
||||
|
||||
@@ -30,6 +30,10 @@ const (
|
||||
monitoringCollectorType = "desktop"
|
||||
monitoringOnlineDuration = 20 * time.Minute
|
||||
monitoringCollectNowQuestionLimit = 5
|
||||
|
||||
monitoringPlatformAuthorizationAuthorized = "authorized"
|
||||
monitoringPlatformAuthorizationNoDesktopClient = "no_desktop_client"
|
||||
monitoringPlatformAuthorizationNoAuthorizedPlatforms = "no_authorized_platforms"
|
||||
)
|
||||
|
||||
type MonitoringService struct {
|
||||
@@ -38,12 +42,14 @@ type MonitoringService struct {
|
||||
rabbitMQ *rabbitmq.Client
|
||||
logger *zap.Logger
|
||||
desktopTasksRolloutPercentage int
|
||||
brandLibraryConfig config.BrandLibraryConfig
|
||||
}
|
||||
|
||||
func NewMonitoringService(
|
||||
businessPool, monitoringPool *pgxpool.Pool,
|
||||
rabbitMQClient *rabbitmq.Client,
|
||||
dispatchCfg config.MonitoringDispatchConfig,
|
||||
brandLibraryCfg config.BrandLibraryConfig,
|
||||
logger *zap.Logger,
|
||||
) *MonitoringService {
|
||||
return &MonitoringService{
|
||||
@@ -52,6 +58,7 @@ func NewMonitoringService(
|
||||
rabbitMQ: rabbitMQClient,
|
||||
logger: logger,
|
||||
desktopTasksRolloutPercentage: dispatchCfg.DesktopTasksRolloutPercentage,
|
||||
brandLibraryConfig: brandLibraryCfg,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,7 +72,10 @@ type MonitoringDashboardCompositeResponse struct {
|
||||
}
|
||||
|
||||
type MonitoringDashboardRuntime struct {
|
||||
CurrentUserClientOnline bool `json:"current_user_client_online"`
|
||||
CurrentUserClientOnline bool `json:"current_user_client_online"`
|
||||
PlatformAuthorizationStatus string `json:"platform_authorization_status"`
|
||||
AuthorizedMonitoringPlatformIDs []string `json:"authorized_monitoring_platform_ids"`
|
||||
AuthorizedMonitoringPlatformCount int `json:"authorized_monitoring_platform_count"`
|
||||
}
|
||||
|
||||
type MonitoringOverview struct {
|
||||
@@ -124,13 +134,14 @@ type MonitoringCitedArticle struct {
|
||||
}
|
||||
|
||||
type MonitoringQuestionDetailResponse struct {
|
||||
QuestionID int64 `json:"question_id"`
|
||||
QuestionHash string `json:"question_hash"`
|
||||
QuestionText string `json:"question_text"`
|
||||
TimeWindow MonitoringQuestionDetailTimeWindow `json:"time_window"`
|
||||
Platforms []MonitoringQuestionDetailPlatform `json:"platforms"`
|
||||
CitationAnalysis []MonitoringQuestionCitationStats `json:"citation_analysis"`
|
||||
ContentCitations []MonitoringQuestionContentCitation `json:"content_citations"`
|
||||
QuestionID int64 `json:"question_id"`
|
||||
QuestionHash string `json:"question_hash"`
|
||||
QuestionText string `json:"question_text"`
|
||||
PlatformAuthorizationStatus string `json:"platform_authorization_status"`
|
||||
TimeWindow MonitoringQuestionDetailTimeWindow `json:"time_window"`
|
||||
Platforms []MonitoringQuestionDetailPlatform `json:"platforms"`
|
||||
CitationAnalysis []MonitoringQuestionCitationStats `json:"citation_analysis"`
|
||||
ContentCitations []MonitoringQuestionContentCitation `json:"content_citations"`
|
||||
}
|
||||
|
||||
type MonitoringQuestionDetailTimeWindow struct {
|
||||
@@ -216,6 +227,16 @@ type monitoringQuotaConfig struct {
|
||||
EnabledPlatforms []string
|
||||
}
|
||||
|
||||
func monitoringPlatformAuthorizationStatus(quota monitoringQuotaConfig) string {
|
||||
if quota.PrimaryClientID == nil {
|
||||
return monitoringPlatformAuthorizationNoDesktopClient
|
||||
}
|
||||
if len(quota.EnabledPlatforms) == 0 {
|
||||
return monitoringPlatformAuthorizationNoAuthorizedPlatforms
|
||||
}
|
||||
return monitoringPlatformAuthorizationAuthorized
|
||||
}
|
||||
|
||||
type monitoringBrandIdentity struct {
|
||||
ID int64
|
||||
Name string
|
||||
@@ -290,6 +311,7 @@ func (s *MonitoringService) DashboardComposite(
|
||||
aiPlatformID *string,
|
||||
) (*MonitoringDashboardCompositeResponse, error) {
|
||||
actor := auth.MustActor(ctx)
|
||||
workspaceID := auth.CurrentWorkspaceID(ctx)
|
||||
|
||||
brand, err := s.resolveBrand(ctx, actor.TenantID, brandID)
|
||||
if err != nil {
|
||||
@@ -302,12 +324,12 @@ func (s *MonitoringService) DashboardComposite(
|
||||
}
|
||||
}
|
||||
|
||||
quota, err := s.loadQuota(ctx, actor.TenantID)
|
||||
quota, err := s.loadQuota(ctx, actor, workspaceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
runtime, err := s.loadDashboardRuntime(ctx, actor)
|
||||
runtime, err := s.loadDashboardRuntime(ctx, actor, workspaceID, quota)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -318,7 +340,7 @@ func (s *MonitoringService) DashboardComposite(
|
||||
}
|
||||
questionIDs := configuredQuestionIDs(configuredQuestions)
|
||||
|
||||
platforms := resolveMonitoringPlatforms(quota.EnabledPlatforms)
|
||||
platforms := defaultMonitoringPlatformMetadata()
|
||||
selectedPlatformID := normalizedOptionalMonitoringPlatformPointer(aiPlatformID)
|
||||
filteredPlatforms := filterMonitoringPlatforms(platforms, selectedPlatformID)
|
||||
startDate, endDate, err := resolveDashboardDateWindow(days, businessDate)
|
||||
@@ -334,7 +356,7 @@ func (s *MonitoringService) DashboardComposite(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
accessStates, err := s.loadAccessStates(ctx, actor.TenantID, quota.PrimaryClientID, endDate)
|
||||
accessStates, err := s.loadAccessStates(ctx, actor.TenantID, workspaceID, quota.PrimaryClientID, endDate)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -369,19 +391,27 @@ func (s *MonitoringService) DashboardComposite(
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringService) loadDashboardRuntime(ctx context.Context, actor auth.Actor) (MonitoringDashboardRuntime, error) {
|
||||
online, err := s.hasOnlineClientForUser(ctx, actor.TenantID, actor.PrimaryWorkspaceID, actor.UserID)
|
||||
if err != nil {
|
||||
return MonitoringDashboardRuntime{}, err
|
||||
func (s *MonitoringService) loadDashboardRuntime(ctx context.Context, actor auth.Actor, workspaceID int64, quota monitoringQuotaConfig) (MonitoringDashboardRuntime, error) {
|
||||
online := false
|
||||
if quota.PrimaryClientID != nil {
|
||||
var err error
|
||||
online, err = s.isClientOnline(ctx, actor.TenantID, workspaceID, *quota.PrimaryClientID)
|
||||
if err != nil {
|
||||
return MonitoringDashboardRuntime{}, err
|
||||
}
|
||||
}
|
||||
|
||||
return MonitoringDashboardRuntime{
|
||||
CurrentUserClientOnline: online,
|
||||
CurrentUserClientOnline: online,
|
||||
PlatformAuthorizationStatus: monitoringPlatformAuthorizationStatus(quota),
|
||||
AuthorizedMonitoringPlatformIDs: append([]string(nil), quota.EnabledPlatforms...),
|
||||
AuthorizedMonitoringPlatformCount: len(quota.EnabledPlatforms),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringService) QuestionDetail(ctx context.Context, brandID, questionID int64, dateFrom, dateTo string, questionHash string, aiPlatformID *string) (*MonitoringQuestionDetailResponse, error) {
|
||||
actor := auth.MustActor(ctx)
|
||||
workspaceID := auth.CurrentWorkspaceID(ctx)
|
||||
|
||||
brand, err := s.resolveBrand(ctx, actor.TenantID, brandID)
|
||||
if err != nil {
|
||||
@@ -393,12 +423,12 @@ func (s *MonitoringService) QuestionDetail(ctx context.Context, brandID, questio
|
||||
return nil, err
|
||||
}
|
||||
|
||||
quota, err := s.loadQuota(ctx, actor.TenantID)
|
||||
quota, err := s.loadQuota(ctx, actor, workspaceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
platforms := resolveMonitoringPlatforms(quota.EnabledPlatforms)
|
||||
selectedPlatformID := normalizedOptionalMonitoringPlatformPointer(aiPlatformID)
|
||||
platforms := filterMonitoringPlatforms(defaultMonitoringPlatformMetadata(), selectedPlatformID)
|
||||
|
||||
questionText, err := s.loadQuestionText(ctx, actor.TenantID, brand.ID, questionID)
|
||||
if err != nil {
|
||||
@@ -418,7 +448,7 @@ func (s *MonitoringService) QuestionDetail(ctx context.Context, brandID, questio
|
||||
}
|
||||
}
|
||||
|
||||
accessStates, err := s.loadAccessStates(ctx, actor.TenantID, quota.PrimaryClientID, toDate)
|
||||
accessStates, err := s.loadAccessStates(ctx, actor.TenantID, workspaceID, quota.PrimaryClientID, toDate)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -481,9 +511,10 @@ func (s *MonitoringService) QuestionDetail(ctx context.Context, brandID, questio
|
||||
}
|
||||
|
||||
return &MonitoringQuestionDetailResponse{
|
||||
QuestionID: questionID,
|
||||
QuestionHash: encodeQuestionHash(hashBytes),
|
||||
QuestionText: questionText,
|
||||
QuestionID: questionID,
|
||||
QuestionHash: encodeQuestionHash(hashBytes),
|
||||
QuestionText: questionText,
|
||||
PlatformAuthorizationStatus: monitoringPlatformAuthorizationStatus(quota),
|
||||
TimeWindow: MonitoringQuestionDetailTimeWindow{
|
||||
DateFrom: fromDate.Format("2006-01-02"),
|
||||
DateTo: toDate.Format("2006-01-02"),
|
||||
@@ -501,6 +532,7 @@ func (s *MonitoringService) CollectNow(
|
||||
options MonitoringCollectNowOptions,
|
||||
) (*MonitoringCollectNowResponse, error) {
|
||||
actor := auth.MustActor(ctx)
|
||||
workspaceID := auth.CurrentWorkspaceID(ctx)
|
||||
|
||||
brand, err := s.resolveBrand(ctx, actor.TenantID, brandID)
|
||||
if err != nil {
|
||||
@@ -513,7 +545,7 @@ func (s *MonitoringService) CollectNow(
|
||||
}
|
||||
}
|
||||
|
||||
quota, err := s.loadQuota(ctx, actor.TenantID)
|
||||
quota, err := s.loadQuota(ctx, actor, workspaceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -540,19 +572,7 @@ func (s *MonitoringService) CollectNow(
|
||||
}, nil
|
||||
}
|
||||
|
||||
platforms := resolveMonitoringPlatforms(quota.EnabledPlatforms)
|
||||
platforms, err = intersectMonitoringPlatforms(platforms, options.PlatformIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
questionIDs := configuredQuestionIDs(configuredQuestions)
|
||||
platformIDs := monitoringPlatformIDs(platforms)
|
||||
executionOwner := "legacy"
|
||||
if s.shouldUseDesktopTasksExecution(actor.TenantID) {
|
||||
executionOwner = "desktop_tasks"
|
||||
}
|
||||
|
||||
clientID, err := s.resolveCollectNowClientID(ctx, actor.TenantID, actor.PrimaryWorkspaceID, actor.UserID, options.TargetClientID)
|
||||
clientID, err := s.resolveCollectNowClientID(ctx, actor.TenantID, workspaceID, actor.UserID, quota.PrimaryClientID, options.TargetClientID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -560,10 +580,37 @@ func (s *MonitoringService) CollectNow(
|
||||
return nil, response.ErrConflict(40941, "desktop_client_offline", "current user's monitoring desktop client is offline")
|
||||
}
|
||||
|
||||
if err := s.ensureClientOnline(ctx, actor.TenantID, actor.PrimaryWorkspaceID, *clientID); err != nil {
|
||||
if err := s.ensureClientOnline(ctx, actor.TenantID, workspaceID, *clientID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
targetPlatformIDs, err := s.loadMonitoringClientPlatformIDs(ctx, actor.TenantID, workspaceID, *clientID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
platforms := resolveMonitoringPlatforms(targetPlatformIDs)
|
||||
platforms, err = intersectMonitoringPlatforms(platforms, options.PlatformIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(platforms) == 0 {
|
||||
return &MonitoringCollectNowResponse{
|
||||
CollectionMode: quota.CollectionMode,
|
||||
RefreshedTaskCount: 0,
|
||||
CreatedTaskCount: 0,
|
||||
LeasedTaskCount: 0,
|
||||
CompletedTaskCount: 0,
|
||||
HasEffectiveSnapshot: true,
|
||||
Message: "当前桌面端暂无已授权的 AI 平台，未创建采集任务",
|
||||
}, nil
|
||||
}
|
||||
questionIDs := configuredQuestionIDs(configuredQuestions)
|
||||
platformIDs := monitoringPlatformIDs(platforms)
|
||||
executionOwner := "legacy"
|
||||
if s.shouldUseDesktopTasksExecution(actor.TenantID) {
|
||||
executionOwner = "desktop_tasks"
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
todayTime, _ := monitoringBusinessDayAndDateAt(now)
|
||||
requestID := uuid.New()
|
||||
@@ -584,7 +631,7 @@ func (s *MonitoringService) CollectNow(
|
||||
ctx,
|
||||
tx,
|
||||
actor.TenantID,
|
||||
actor.PrimaryWorkspaceID,
|
||||
workspaceID,
|
||||
brand.ID,
|
||||
actor.UserID,
|
||||
scopeHash,
|
||||
@@ -663,7 +710,7 @@ func (s *MonitoringService) CollectNow(
|
||||
ctx,
|
||||
tx,
|
||||
actor.TenantID,
|
||||
actor.PrimaryWorkspaceID,
|
||||
workspaceID,
|
||||
todayTime,
|
||||
questionIDs,
|
||||
platformIDs,
|
||||
@@ -722,7 +769,7 @@ func (s *MonitoringService) CollectNow(
|
||||
$12, $13, $14,
|
||||
$15, $16
|
||||
)
|
||||
`, requestID, actor.TenantID, actor.PrimaryWorkspaceID, brand.ID, nullableInt64(keywordID), actor.UserID, *clientID, scopeHash, requestScopeJSON, affectedTaskCount, affectedTaskCount, abortedQueuedCount, int64(len(interruptTaskIDs)), interruptGeneration, message, ttlExpiresAt); err != nil {
|
||||
`, requestID, actor.TenantID, workspaceID, brand.ID, nullableInt64(keywordID), actor.UserID, *clientID, scopeHash, requestScopeJSON, affectedTaskCount, affectedTaskCount, abortedQueuedCount, int64(len(interruptTaskIDs)), interruptGeneration, message, ttlExpiresAt); err != nil {
|
||||
return nil, response.ErrInternal(50041, "request_create_failed", "failed to persist monitoring collect-now request")
|
||||
}
|
||||
|
||||
@@ -730,7 +777,7 @@ func (s *MonitoringService) CollectNow(
|
||||
ctx,
|
||||
tx,
|
||||
requestID,
|
||||
actor.PrimaryWorkspaceID,
|
||||
workspaceID,
|
||||
*clientID,
|
||||
affectedTaskCount,
|
||||
interruptGeneration,
|
||||
@@ -748,8 +795,6 @@ func (s *MonitoringService) CollectNow(
|
||||
return nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring collect-now")
|
||||
}
|
||||
|
||||
_ = s.persistPrimaryClientID(ctx, actor.TenantID, actor.PrimaryWorkspaceID, *clientID)
|
||||
|
||||
phase2PublishedTasks := make([]*repository.DesktopTask, 0)
|
||||
phase2DispatchReady := executionOwner == "desktop_tasks"
|
||||
if len(phase2TaskSpecs) > 0 {
|
||||
@@ -877,6 +922,7 @@ func (s *MonitoringService) CollectNow(
|
||||
func (s *MonitoringService) resolveCollectNowClientID(
|
||||
ctx context.Context,
|
||||
tenantID, workspaceID, userID int64,
|
||||
defaultClientID *uuid.UUID,
|
||||
preferredClientID *uuid.UUID,
|
||||
) (*uuid.UUID, error) {
|
||||
if preferredClientID != nil {
|
||||
@@ -885,7 +931,7 @@ func (s *MonitoringService) resolveCollectNowClientID(
|
||||
}
|
||||
return preferredClientID, nil
|
||||
}
|
||||
return s.findLatestOnlineClientForUser(ctx, tenantID, workspaceID, userID)
|
||||
return defaultClientID, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringService) hasOnlineClientForUser(ctx context.Context, tenantID, workspaceID, userID int64) (bool, error) {
|
||||
@@ -968,6 +1014,10 @@ func monitoringPlatformIDs(items []monitoringPlatformMetadata) []string {
|
||||
return result
|
||||
}
|
||||
|
||||
func defaultMonitoringPlatformMetadata() []monitoringPlatformMetadata {
|
||||
return append([]monitoringPlatformMetadata(nil), defaultMonitoringPlatforms...)
|
||||
}
|
||||
|
||||
func normalizeMonitoringPlatformID(platformID string) string {
|
||||
return strings.ToLower(strings.TrimSpace(platformID))
|
||||
}
|
||||
@@ -996,10 +1046,30 @@ func monitoringPlatformQueryIDs(value *string) []string {
|
||||
}
|
||||
|
||||
func reconcileEnabledMonitoringPlatforms(enabled []string) []string {
|
||||
// Old quota rows may still contain partial or aliased platform ids. Admin monitoring
|
||||
// should always expose the full canonical desktop-supported set for the tenant.
|
||||
_ = enabled
|
||||
return monitoringPlatformIDs(defaultMonitoringPlatforms)
|
||||
if len(enabled) == 0 {
|
||||
return nil
|
||||
}
|
||||
requested := make(map[string]struct{}, len(enabled))
|
||||
for _, item := range enabled {
|
||||
id := normalizeMonitoringPlatformID(item)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, supported := monitoringPlatformNames[id]; !supported {
|
||||
continue
|
||||
}
|
||||
requested[id] = struct{}{}
|
||||
}
|
||||
if len(requested) == 0 {
|
||||
return nil
|
||||
}
|
||||
result := make([]string, 0, len(requested))
|
||||
for _, platform := range defaultMonitoringPlatforms {
|
||||
if _, ok := requested[platform.ID]; ok {
|
||||
result = append(result, platform.ID)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func monitoringPlatformSortIndex(platformID string) int {
|
||||
@@ -1054,26 +1124,6 @@ func (s *MonitoringService) findLatestOnlineClientForUser(ctx context.Context, t
|
||||
return &clientID, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringService) persistPrimaryClientID(ctx context.Context, tenantID, workspaceID int64, clientID uuid.UUID) error {
|
||||
if _, err := s.businessPool.Exec(ctx, `
|
||||
INSERT INTO tenant_monitoring_quotas (
|
||||
tenant_id, workspace_id, primary_client_id
|
||||
)
|
||||
VALUES (
|
||||
$1,
|
||||
$2,
|
||||
$3
|
||||
)
|
||||
ON CONFLICT (tenant_id) DO UPDATE
|
||||
SET workspace_id = EXCLUDED.workspace_id,
|
||||
primary_client_id = EXCLUDED.primary_client_id,
|
||||
updated_at = NOW()
|
||||
`, tenantID, workspaceID, clientID); err != nil {
|
||||
return response.ErrInternal(50041, "update_failed", "failed to persist primary monitoring desktop client")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func formatPgInterval(duration time.Duration) string {
|
||||
return fmt.Sprintf("%d seconds", int64(duration/time.Second))
|
||||
}
|
||||
@@ -1431,40 +1481,117 @@ func (s *MonitoringService) deferQueuedNormalTasks(
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
func (s *MonitoringService) loadQuota(ctx context.Context, tenantID int64) (monitoringQuotaConfig, error) {
|
||||
func (s *MonitoringService) loadQuota(ctx context.Context, actor auth.Actor, workspaceID int64) (monitoringQuotaConfig, error) {
|
||||
cfg := monitoringQuotaConfig{
|
||||
CollectionMode: monitoringCollectorType,
|
||||
EnabledPlatforms: reconcileEnabledMonitoringPlatforms(nil),
|
||||
CollectionMode: monitoringCollectorType,
|
||||
}
|
||||
|
||||
var enabledJSON []byte
|
||||
var clientID uuid.NullUUID
|
||||
clientID, err := s.findLatestRegisteredClientWithMonitoringAccountForUser(ctx, actor.TenantID, workspaceID, actor.UserID)
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
if clientID == nil {
|
||||
clientID, err = s.findLatestRegisteredClientForUser(ctx, actor.TenantID, workspaceID, actor.UserID)
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
}
|
||||
cfg.PrimaryClientID = clientID
|
||||
if clientID == nil {
|
||||
return cfg, nil
|
||||
}
|
||||
platforms, err := s.loadMonitoringClientPlatformIDs(ctx, actor.TenantID, workspaceID, *clientID)
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
cfg.EnabledPlatforms = platforms
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// findLatestRegisteredClientWithMonitoringAccountForUser returns the newest
|
||||
// non-revoked desktop client that has at least one bound monitoring platform
|
||||
// account. Platform account health is not authorization.
|
||||
func (s *MonitoringService) findLatestRegisteredClientWithMonitoringAccountForUser(ctx context.Context, tenantID, workspaceID, userID int64) (*uuid.UUID, error) {
|
||||
var clientID uuid.UUID
|
||||
err := s.businessPool.QueryRow(ctx, `
|
||||
SELECT collection_mode, enabled_platforms, primary_client_id
|
||||
FROM tenant_monitoring_quotas
|
||||
WHERE tenant_id = $1
|
||||
`, tenantID).Scan(&cfg.CollectionMode, &enabledJSON, &clientID)
|
||||
SELECT dc.id
|
||||
FROM desktop_clients dc
|
||||
WHERE dc.tenant_id = $1
|
||||
AND dc.workspace_id = $2
|
||||
AND dc.user_id = $3
|
||||
AND dc.revoked_at IS NULL
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM platform_accounts pa
|
||||
WHERE pa.tenant_id = dc.tenant_id
|
||||
AND pa.workspace_id = dc.workspace_id
|
||||
AND pa.client_id = dc.id
|
||||
AND pa.deleted_at IS NULL
|
||||
AND pa.platform_id = ANY($4::text[])
|
||||
)
|
||||
ORDER BY dc.created_at DESC, dc.id DESC
|
||||
LIMIT 1
|
||||
`, tenantID, workspaceID, userID, monitoringPlatformIDs(defaultMonitoringPlatforms)).Scan(&clientID)
|
||||
if err != nil {
|
||||
if err == pgx.ErrNoRows {
|
||||
return cfg, nil
|
||||
return nil, nil
|
||||
}
|
||||
return cfg, response.ErrInternal(50041, "query_failed", "failed to load monitoring quota")
|
||||
return nil, response.ErrInternal(50041, "query_failed", "failed to inspect monitoring desktop client authorization")
|
||||
}
|
||||
return &clientID, nil
|
||||
}
|
||||
|
||||
if len(enabledJSON) > 0 {
|
||||
var enabled []string
|
||||
if jsonErr := json.Unmarshal(enabledJSON, &enabled); jsonErr == nil {
|
||||
cfg.EnabledPlatforms = reconcileEnabledMonitoringPlatforms(enabled)
|
||||
// findLatestRegisteredClientForUser returns the newest non-revoked desktop client
|
||||
// registered by creation order. It intentionally ignores last_seen_at and health;
|
||||
// use findLatestOnlineClientForUser when online presence is required.
|
||||
func (s *MonitoringService) findLatestRegisteredClientForUser(ctx context.Context, tenantID, workspaceID, userID int64) (*uuid.UUID, error) {
|
||||
var clientID uuid.UUID
|
||||
err := s.businessPool.QueryRow(ctx, `
|
||||
SELECT id
|
||||
FROM desktop_clients
|
||||
WHERE tenant_id = $1
|
||||
AND workspace_id = $2
|
||||
AND user_id = $3
|
||||
AND revoked_at IS NULL
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT 1
|
||||
`, tenantID, workspaceID, userID).Scan(&clientID)
|
||||
if err != nil {
|
||||
if err == pgx.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, response.ErrInternal(50041, "query_failed", "failed to inspect monitoring desktop client")
|
||||
}
|
||||
if clientID.Valid {
|
||||
id := clientID.UUID
|
||||
cfg.PrimaryClientID = &id
|
||||
return &clientID, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringService) loadMonitoringClientPlatformIDs(ctx context.Context, tenantID, workspaceID int64, clientID uuid.UUID) ([]string, error) {
|
||||
rows, err := s.businessPool.Query(ctx, `
|
||||
SELECT DISTINCT platform_id
|
||||
FROM platform_accounts
|
||||
WHERE tenant_id = $1
|
||||
AND workspace_id = $2
|
||||
AND client_id = $3
|
||||
AND deleted_at IS NULL
|
||||
AND platform_id = ANY($4::text[])
|
||||
`, tenantID, workspaceID, clientID, monitoringPlatformIDs(defaultMonitoringPlatforms))
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(50041, "query_failed", "failed to load monitoring desktop platform accounts")
|
||||
}
|
||||
if strings.TrimSpace(cfg.CollectionMode) == "" {
|
||||
cfg.CollectionMode = monitoringCollectorType
|
||||
defer rows.Close()
|
||||
|
||||
platformIDs := make([]string, 0)
|
||||
for rows.Next() {
|
||||
var platformID string
|
||||
if scanErr := rows.Scan(&platformID); scanErr != nil {
|
||||
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse monitoring desktop platform accounts")
|
||||
}
|
||||
platformIDs = append(platformIDs, platformID)
|
||||
}
|
||||
return cfg, nil
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate monitoring desktop platform accounts")
|
||||
}
|
||||
return reconcileEnabledMonitoringPlatforms(platformIDs), nil
|
||||
}
|
||||
|
||||
func newMonitoringDerivedMetrics() monitoringDerivedMetrics {
|
||||
@@ -2021,11 +2148,18 @@ func applyDerivedRatesToOverviewSnapshot(snapshot *monitoringOverviewSnapshot, d
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MonitoringService) loadAccessStates(ctx context.Context, tenantID int64, clientID *uuid.UUID, businessDate time.Time) (map[string]monitoringAccessState, error) {
|
||||
func (s *MonitoringService) loadAccessStates(ctx context.Context, tenantID, workspaceID int64, clientID *uuid.UUID, businessDate time.Time) (map[string]monitoringAccessState, error) {
|
||||
states := map[string]monitoringAccessState{}
|
||||
if clientID == nil {
|
||||
return states, nil
|
||||
}
|
||||
belongs, err := s.monitoringClientBelongsToWorkspace(ctx, tenantID, workspaceID, *clientID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !belongs {
|
||||
return states, nil
|
||||
}
|
||||
|
||||
rows, err := s.monitoringPool.Query(ctx, `
|
||||
SELECT ai_platform_id, access_status
|
||||
@@ -2054,10 +2188,30 @@ func (s *MonitoringService) loadAccessStates(ctx context.Context, tenantID int64
|
||||
}
|
||||
states[platformID] = monitoringAccessState{AccessStatus: accessStatus}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate platform access states")
|
||||
}
|
||||
|
||||
return states, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringService) monitoringClientBelongsToWorkspace(ctx context.Context, tenantID, workspaceID int64, clientID uuid.UUID) (bool, error) {
|
||||
var exists bool
|
||||
if err := s.businessPool.QueryRow(ctx, `
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM desktop_clients
|
||||
WHERE id = $1
|
||||
AND tenant_id = $2
|
||||
AND workspace_id = $3
|
||||
AND revoked_at IS NULL
|
||||
)
|
||||
`, clientID, tenantID, workspaceID).Scan(&exists); err != nil {
|
||||
return false, response.ErrInternal(50041, "query_failed", "failed to validate monitoring desktop client workspace")
|
||||
}
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringService) loadPlatformBreakdown(ctx context.Context, tenantID, brandID int64, businessDate time.Time, collectionMode string, platforms []monitoringPlatformMetadata, accessStates map[string]monitoringAccessState, derived monitoringDerivedMetrics) ([]MonitoringPlatformDaily, error) {
|
||||
rows, err := s.monitoringPool.Query(ctx, `
|
||||
SELECT
|
||||
@@ -3117,7 +3271,7 @@ func normalizeTrackingDays(days int) int {
|
||||
|
||||
func resolveMonitoringPlatforms(enabled []string) []monitoringPlatformMetadata {
|
||||
if len(enabled) == 0 {
|
||||
return defaultMonitoringPlatforms
|
||||
return nil
|
||||
}
|
||||
platforms := make([]monitoringPlatformMetadata, 0, len(enabled))
|
||||
seen := make(map[string]struct{}, len(enabled))
|
||||
@@ -3139,7 +3293,7 @@ func resolveMonitoringPlatforms(enabled []string) []monitoringPlatformMetadata {
|
||||
})
|
||||
}
|
||||
if len(platforms) == 0 {
|
||||
return defaultMonitoringPlatforms
|
||||
return nil
|
||||
}
|
||||
return platforms
|
||||
}
|
||||
@@ -3192,12 +3346,7 @@ func filterMonitoringPlatforms(platforms []monitoringPlatformMetadata, aiPlatfor
|
||||
}
|
||||
}
|
||||
|
||||
return []monitoringPlatformMetadata{
|
||||
{
|
||||
ID: platformID,
|
||||
Name: platformDisplayName(platformID),
|
||||
},
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func platformDisplayName(platformID string) string {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
@@ -81,7 +82,7 @@ func TestFilterMonitoringPlatformsReturnsSelectedPlatform(t *testing.T) {
|
||||
}, filtered)
|
||||
}
|
||||
|
||||
func TestFilterMonitoringPlatformsFallsBackToDisplayName(t *testing.T) {
|
||||
func TestFilterMonitoringPlatformsDropsUnauthorizedSelection(t *testing.T) {
|
||||
platforms := []monitoringPlatformMetadata{
|
||||
{ID: "deepseek", Name: "DeepSeek"},
|
||||
}
|
||||
@@ -89,9 +90,7 @@ func TestFilterMonitoringPlatformsFallsBackToDisplayName(t *testing.T) {
|
||||
|
||||
filtered := filterMonitoringPlatforms(platforms, &selected)
|
||||
|
||||
assert.Equal(t, []monitoringPlatformMetadata{
|
||||
{ID: "doubao", Name: "豆包"},
|
||||
}, filtered)
|
||||
assert.Empty(t, filtered)
|
||||
}
|
||||
|
||||
func TestNormalizeMonitoringPlatformIDCanonicalizesLatestIDs(t *testing.T) {
|
||||
@@ -127,8 +126,40 @@ func TestIntersectMonitoringPlatformsAcceptsCanonicalIDs(t *testing.T) {
|
||||
}, filtered)
|
||||
}
|
||||
|
||||
func TestReconcileEnabledMonitoringPlatformsBackfillsSupportedCatalog(t *testing.T) {
|
||||
assert.Equal(t, []string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"}, reconcileEnabledMonitoringPlatforms([]string{"deepseek", "qwen", "doubao"}))
|
||||
func TestReconcileEnabledMonitoringPlatformsKeepsAuthorizedSubset(t *testing.T) {
|
||||
assert.Equal(t, []string{"deepseek", "doubao", "qwen"}, reconcileEnabledMonitoringPlatforms([]string{"deepseek", "qwen", "doubao"}))
|
||||
assert.Empty(t, reconcileEnabledMonitoringPlatforms(nil))
|
||||
assert.Empty(t, resolveMonitoringPlatforms(nil))
|
||||
}
|
||||
|
||||
func TestMonitoringPlatformAuthorizationStatus(t *testing.T) {
|
||||
clientID := uuid.New()
|
||||
|
||||
assert.Equal(t, monitoringPlatformAuthorizationNoDesktopClient, monitoringPlatformAuthorizationStatus(monitoringQuotaConfig{}))
|
||||
assert.Equal(t, monitoringPlatformAuthorizationNoAuthorizedPlatforms, monitoringPlatformAuthorizationStatus(monitoringQuotaConfig{
|
||||
PrimaryClientID: &clientID,
|
||||
}))
|
||||
assert.Equal(t, monitoringPlatformAuthorizationAuthorized, monitoringPlatformAuthorizationStatus(monitoringQuotaConfig{
|
||||
PrimaryClientID: &clientID,
|
||||
EnabledPlatforms: []string{"qwen"},
|
||||
}))
|
||||
}
|
||||
|
||||
func TestDefaultMonitoringPlatformMetadataReturnsFullCatalogCopy(t *testing.T) {
|
||||
platforms := defaultMonitoringPlatformMetadata()
|
||||
|
||||
require.Len(t, platforms, 6)
|
||||
assert.Equal(t, []string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"}, monitoringPlatformIDs(platforms))
|
||||
|
||||
platforms[0].ID = "mutated"
|
||||
assert.Equal(t, "yuanbao", defaultMonitoringPlatformMetadata()[0].ID)
|
||||
}
|
||||
|
||||
func TestLoadMonitoringProjectionPlatformsReturnsFullCatalog(t *testing.T) {
|
||||
platforms, err := loadMonitoringProjectionPlatforms(nil, nil, 0, 0, time.Time{})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, []string{"yuanbao", "kimi", "wenxin", "deepseek", "doubao", "qwen"}, monitoringPlatformIDs(platforms))
|
||||
}
|
||||
|
||||
func TestApplyDerivedRatesToOverviewSnapshotUsesAllSamplesForMentionRate(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package transport
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/bootstrap"
|
||||
"github.com/geo-platform/tenant-api/internal/shared/response"
|
||||
tenantapp "github.com/geo-platform/tenant-api/internal/tenant/app"
|
||||
)
|
||||
|
||||
func registerInternalMetricsRoutes(app *bootstrap.App) {
|
||||
token := strings.TrimSpace(app.Config.Scheduler.InternalMetricsToken)
|
||||
if token == "" {
|
||||
app.Logger.Warn("tenant-api internal metrics routes not registered: scheduler.internal_metrics_token is required")
|
||||
return
|
||||
}
|
||||
|
||||
metrics := app.Engine.Group("/api/internal/metrics")
|
||||
metrics.Use(internalMetricsAuthMiddleware(token))
|
||||
metrics.GET("/monitoring/heartbeat-primary-lease", func(c *gin.Context) {
|
||||
response.Success(c, tenantapp.MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue())
|
||||
})
|
||||
primaryLeasePrometheusHandler := tenantapp.MonitoringHeartbeatPrimaryLeaseMetricsPrometheusHandler()
|
||||
metrics.GET("/monitoring/heartbeat-primary-lease/prometheus", func(c *gin.Context) {
|
||||
primaryLeasePrometheusHandler.ServeHTTP(c.Writer, c.Request)
|
||||
})
|
||||
metrics.GET("/monitoring/heartbeat-snapshots", func(c *gin.Context) {
|
||||
response.Success(c, tenantapp.MonitoringHeartbeatSnapshotMetricsSnapshotValue())
|
||||
})
|
||||
snapshotPrometheusHandler := tenantapp.MonitoringHeartbeatSnapshotMetricsPrometheusHandler()
|
||||
metrics.GET("/monitoring/heartbeat-snapshots/prometheus", func(c *gin.Context) {
|
||||
snapshotPrometheusHandler.ServeHTTP(c.Writer, c.Request)
|
||||
})
|
||||
}
|
||||
|
||||
func internalMetricsAuthMiddleware(token string) gin.HandlerFunc {
|
||||
expected := strings.TrimSpace(token)
|
||||
return func(c *gin.Context) {
|
||||
if expected == "" || !internalMetricsTokenMatches(c, expected) {
|
||||
response.Error(c, response.ErrUnauthorized(40181, "internal_metrics_unauthorized", "valid internal metrics token is required"))
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
|
||||
func internalMetricsTokenMatches(c *gin.Context, expected string) bool {
|
||||
if c == nil {
|
||||
return false
|
||||
}
|
||||
header := strings.TrimSpace(c.GetHeader("Authorization"))
|
||||
if strings.HasPrefix(strings.ToLower(header), "bearer ") {
|
||||
if strings.TrimSpace(header[len("bearer "):]) == expected {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(c.GetHeader("X-Internal-Metrics-Token")) == expected
|
||||
}
|
||||
@@ -26,7 +26,7 @@ type monitoringCollectNowRequest struct {
|
||||
|
||||
func NewMonitoringHandler(a *bootstrap.App) *MonitoringHandler {
|
||||
return &MonitoringHandler{
|
||||
svc: app.NewMonitoringService(a.DB, a.MonitoringDB, a.RabbitMQ, a.Config.MonitoringDispatch, a.Logger),
|
||||
svc: app.NewMonitoringService(a.DB, a.MonitoringDB, a.RabbitMQ, a.Config.MonitoringDispatch, a.Config.BrandLibrary, a.Logger),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
)
|
||||
|
||||
func RegisterRoutes(app *bootstrap.App) {
|
||||
registerInternalMetricsRoutes(app)
|
||||
|
||||
authAPI := app.Engine.Group("/api/auth")
|
||||
authHandler := NewAuthHandler(app)
|
||||
authAPI.POST("/login", authHandler.Login)
|
||||
@@ -21,6 +23,7 @@ func RegisterRoutes(app *bootstrap.App) {
|
||||
protected := app.Engine.Group("/api")
|
||||
protected.Use(auth.Middleware(app.JWT, app.Sessions))
|
||||
protected.Use(middleware.TenantScope())
|
||||
protected.Use(middleware.WorkspaceScope(app.DB))
|
||||
|
||||
protected.GET("/auth/me", authHandler.Me)
|
||||
protected.POST("/auth/logout", authHandler.Logout)
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
DROP INDEX IF EXISTS idx_desktop_clients_tenant_workspace_user_seen_active;
|
||||
DROP INDEX IF EXISTS idx_desktop_clients_tenant_workspace_seen_active;
|
||||
DROP INDEX IF EXISTS idx_desktop_client_primary_leases_primary_client;
|
||||
DROP TABLE IF EXISTS desktop_client_primary_leases;
|
||||
@@ -0,0 +1,52 @@
|
||||
-- Backfilled primary leases use the same 90s TTL as
|
||||
-- server/internal/tenant/app/desktop_presence.go:desktopClientPresenceTTL.
|
||||
-- Keep these values aligned; desktop heartbeat interval must be < TTL/2 so
|
||||
-- a single missed heartbeat does not let another client steal primary.
|
||||
CREATE TABLE IF NOT EXISTS desktop_client_primary_leases (
|
||||
tenant_id BIGINT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
|
||||
workspace_id BIGINT NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE,
|
||||
primary_client_id UUID NOT NULL REFERENCES desktop_clients(id) ON DELETE CASCADE,
|
||||
lease_expires_at TIMESTAMPTZ NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (tenant_id, workspace_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_desktop_client_primary_leases_primary_client
|
||||
ON desktop_client_primary_leases (primary_client_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_desktop_clients_tenant_workspace_seen_active
|
||||
ON desktop_clients (tenant_id, workspace_id, last_seen_at DESC, created_at DESC, id DESC)
|
||||
WHERE revoked_at IS NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_desktop_clients_tenant_workspace_user_seen_active
|
||||
ON desktop_clients (tenant_id, workspace_id, user_id, last_seen_at DESC, created_at DESC, id DESC)
|
||||
WHERE revoked_at IS NULL;
|
||||
|
||||
INSERT INTO desktop_client_primary_leases (
|
||||
tenant_id,
|
||||
workspace_id,
|
||||
primary_client_id,
|
||||
lease_expires_at
|
||||
)
|
||||
SELECT
|
||||
tenant_id,
|
||||
workspace_id,
|
||||
id,
|
||||
COALESCE(last_seen_at, created_at) + INTERVAL '90 seconds'
|
||||
FROM (
|
||||
SELECT
|
||||
dc.id,
|
||||
dc.tenant_id,
|
||||
dc.workspace_id,
|
||||
dc.created_at,
|
||||
dc.last_seen_at,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY dc.tenant_id, dc.workspace_id
|
||||
ORDER BY dc.last_seen_at DESC NULLS LAST, dc.created_at DESC, dc.id DESC
|
||||
) AS rn
|
||||
FROM desktop_clients dc
|
||||
WHERE dc.revoked_at IS NULL
|
||||
) ranked_clients
|
||||
WHERE rn = 1
|
||||
ON CONFLICT (tenant_id, workspace_id) DO NOTHING;
|
||||
+1
@@ -0,0 +1 @@
|
||||
DROP INDEX IF EXISTS idx_collect_tasks_pending_platform_reconcile;
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
CREATE INDEX IF NOT EXISTS idx_collect_tasks_pending_platform_reconcile
|
||||
ON monitoring_collect_tasks (tenant_id, business_date, ai_platform_id)
|
||||
WHERE collector_type = 'desktop'
|
||||
AND status = 'pending';
|
||||
Reference in New Issue
Block a user