Compare commits

...

4 Commits

Author SHA1 Message Date
root 05d4266649 enable desktop monitoring rollout
Deployment Config CI / Deployment Config (push) Successful in 32s
Backend CI / Backend (push) Failing after 8m33s
2026-06-27 16:53:20 +08:00
root 11b8e3f5cd fix monitoring daily dispatch cap 2026-06-27 16:47:28 +08:00
root beb290a5a7 fix monitoring online dispatch retries 2026-06-27 16:17:02 +08:00
root 940ac23493 fix monitoring daily backpressure 2026-06-27 15:31:08 +08:00
21 changed files with 1218 additions and 267 deletions
+1 -1
View File
@@ -95,7 +95,7 @@ monitoring_workers:
projection_rebuild_concurrency: 2
monitoring_dispatch:
desktop_tasks_rollout_percentage: 5
desktop_tasks_rollout_percentage: 100
membership:
default_plan_code: free
+1 -1
View File
@@ -100,7 +100,7 @@ monitoring_workers:
projection_rebuild_concurrency: 2
monitoring_dispatch:
desktop_tasks_rollout_percentage: 5
desktop_tasks_rollout_percentage: 100
membership:
default_plan_code: free
+2 -13
View File
@@ -344,20 +344,9 @@ func shutdownSchedulerMetricsServer(server *http.Server, logger interface {
}
func monitoringDailyRunOptions(jobCtx internalscheduler.JobRunContext) tenantapp.MonitoringDailyTaskRunOptions {
batchSize := 0
if jobCtx.Job != nil && jobCtx.Job.BatchSize != nil {
batchSize = *jobCtx.Job.BatchSize
}
if batchSize <= 0 {
batchSize = internalscheduler.AsInt(jobCtx.Config, "max_materialize_per_run", 0)
}
if batchSize <= 0 {
batchSize = internalscheduler.AsInt(jobCtx.Config, "batch_size", 0)
}
return tenantapp.MonitoringDailyTaskRunOptions{
MaxMaterializePerRun: batchSize,
MaxMaterializePerPlan: internalscheduler.AsInt(jobCtx.Config, "max_materialize_per_plan", 0),
MaxMaterializePerBrand: internalscheduler.AsInt(jobCtx.Config, "max_materialize_per_brand", 0),
MaxDesktopDispatchPerRun: internalscheduler.AsInt(jobCtx.Config, "max_desktop_dispatch_per_run", 0),
MaxDesktopClientBacklog: internalscheduler.AsInt(jobCtx.Config, "max_desktop_client_backlog", 0),
}
}
+13 -5
View File
@@ -59,7 +59,7 @@ func TestSchedulerMetricsAuthMiddleware(t *testing.T) {
}
}
func TestMonitoringDailyRunOptionsUsesJobBatchSize(t *testing.T) {
func TestMonitoringDailyRunOptionsIgnoresJobBatchSizeForDailyCollection(t *testing.T) {
batchSize := 88
got := monitoringDailyRunOptions(internalscheduler.JobRunContext{
Job: &opsdomain.SchedulerJob{BatchSize: &batchSize},
@@ -67,14 +67,22 @@ func TestMonitoringDailyRunOptionsUsesJobBatchSize(t *testing.T) {
"max_materialize_per_run": 999,
"max_materialize_per_plan": 12,
"max_materialize_per_brand": 4,
"max_desktop_client_backlog": 6,
"batch_size": 77,
},
})
if got.MaxMaterializePerRun != 88 {
t.Fatalf("MaxMaterializePerRun = %d, want 88", got.MaxMaterializePerRun)
if got.MaxDesktopDispatchPerRun != 0 {
t.Fatalf("MaxDesktopDispatchPerRun = %d, want unlimited", got.MaxDesktopDispatchPerRun)
}
if got.MaxMaterializePerPlan != 12 || got.MaxMaterializePerBrand != 4 {
t.Fatalf("unexpected plan/brand options: %#v", got)
if got.MaxMaterializePerPlan != 0 || got.MaxMaterializePerBrand != 0 {
t.Fatalf("legacy plan/brand throttles should be ignored: %#v", got)
}
if got.MaxMaterializePerRun != 0 {
t.Fatalf("batch_size must not throttle daily materialization: %#v", got)
}
if got.MaxDesktopClientBacklog != 6 {
t.Fatalf("MaxDesktopClientBacklog = %d, want 6", got.MaxDesktopClientBacklog)
}
}
+1 -1
View File
@@ -100,7 +100,7 @@ monitoring_workers:
projection_rebuild_concurrency: 2
monitoring_dispatch:
desktop_tasks_rollout_percentage: 5
desktop_tasks_rollout_percentage: 100
membership:
default_plan_code: free
@@ -40,7 +40,9 @@ const (
// reports skip writes and only refresh the row at this interval so dashboards
// do not depend on per-heartbeat database churn.
monitoringHeartbeatAccessSnapshotRefreshAfter = 10 * time.Minute
monitoringRuntimeUnknownMaxDispatchAttempts = 3
monitoringRuntimeUnknownMaxDispatchAttempts = 2
monitoringAutomaticFailureMaxAttempts = 2
monitoringAutomaticFailureRetryAfter = 2 * time.Minute
)
var monitoringCitationMarkerPattern = regexp.MustCompile(`(?i)\[\s*citation\s*:\s*\d+\s*\]`)
@@ -261,6 +263,7 @@ type monitoringCollectTask struct {
QuestionText string
TargetClientID sql.NullString
DispatchAttempts int
AutomaticFailureCount int
}
type monitoringDesktopExecutionLease struct {
@@ -1062,6 +1065,32 @@ type monitoringSkipOutcome struct {
RetryAfter time.Duration
}
type monitoringFailedResultOutcome struct {
TaskStatus string
Requeue bool
RetryAfter time.Duration
}
func monitoringFailedResultOutcomeForTask(task *monitoringCollectTask, req MonitoringTaskResultRequest) monitoringFailedResultOutcome {
if task == nil {
return monitoringFailedResultOutcome{TaskStatus: "failed"}
}
if !strings.EqualFold(strings.TrimSpace(task.TriggerSource), "automatic") {
return monitoringFailedResultOutcome{TaskStatus: "failed"}
}
if monitoringTaskFailureDispositionFromRequest(req).NonRetryable {
return monitoringFailedResultOutcome{TaskStatus: "failed"}
}
if task.AutomaticFailureCount < monitoringAutomaticFailureMaxAttempts-1 {
return monitoringFailedResultOutcome{
TaskStatus: "pending",
Requeue: true,
RetryAfter: monitoringAutomaticFailureRetryAfter,
}
}
return monitoringFailedResultOutcome{TaskStatus: "failed"}
}
func monitoringSkipOutcomeForReason(skipReason string, dispatchAttempts int, triggerSource string) monitoringSkipOutcome {
switch strings.TrimSpace(skipReason) {
case "runtime_unknown":
@@ -1562,6 +1591,11 @@ func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantI
trigger_source,
COALESCE(dispatch_attempts, 0) AS dispatch_attempts,
COALESCE(target_client_id::text, '') AS target_client_id,
CASE
WHEN COALESCE(t.request_payload_json ->> 'automatic_failure_count', '') ~ '^[0-9]+$'
THEN (t.request_payload_json ->> 'automatic_failure_count')::int
ELSE 0
END AS automatic_failure_count,
COALESCE(NULLIF(t.question_text_snapshot, ''), (
SELECT question_text_snapshot
FROM monitoring_question_config_snapshots
@@ -1600,6 +1634,7 @@ func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantI
&item.TriggerSource,
&item.DispatchAttempts,
&item.TargetClientID,
&item.AutomaticFailureCount,
&item.QuestionText,
); err != nil {
if err == pgx.ErrNoRows {
@@ -1635,6 +1670,11 @@ func (s *MonitoringCallbackService) loadCollectTaskByID(ctx context.Context, tas
trigger_source,
COALESCE(dispatch_attempts, 0) AS dispatch_attempts,
COALESCE(target_client_id::text, '') AS target_client_id,
CASE
WHEN COALESCE(t.request_payload_json ->> 'automatic_failure_count', '') ~ '^[0-9]+$'
THEN (t.request_payload_json ->> 'automatic_failure_count')::int
ELSE 0
END AS automatic_failure_count,
COALESCE(NULLIF(t.question_text_snapshot, ''), (
SELECT question_text_snapshot
FROM monitoring_question_config_snapshots
@@ -1672,6 +1712,7 @@ func (s *MonitoringCallbackService) loadCollectTaskByID(ctx context.Context, tas
&item.TriggerSource,
&item.DispatchAttempts,
&item.TargetClientID,
&item.AutomaticFailureCount,
&item.QuestionText,
); err != nil {
if err == pgx.ErrNoRows {
@@ -2312,12 +2353,20 @@ func (s *MonitoringCallbackService) processTaskResult(ctx context.Context, task
}
defer tx.Rollback(ctx)
runID, err := s.upsertRun(ctx, tx, task, req, runStatus, completedAt, rawPayload)
var failedOutcome monitoringFailedResultOutcome
if runStatus == "failed" {
failedOutcome = monitoringFailedResultOutcomeForTask(task, req)
}
var runID int64
if !failedOutcome.Requeue {
runID, err = s.upsertRun(ctx, tx, task, req, runStatus, completedAt, rawPayload)
if err != nil {
return nil, err
}
}
if runStatus == "succeeded" {
if runStatus == "succeeded" && !failedOutcome.Requeue {
if err := s.upsertParseResult(ctx, tx, task, runID, parseReq, parseVersion); err != nil {
return nil, err
}
@@ -2325,10 +2374,12 @@ func (s *MonitoringCallbackService) processTaskResult(ctx context.Context, task
citationSourceCount := 0
contentCitationCount := 0
if !failedOutcome.Requeue {
if err := s.replaceCitationFacts(ctx, tx, task.TenantID, runID); err != nil {
return nil, err
}
if runStatus == "succeeded" {
}
if runStatus == "succeeded" && !failedOutcome.Requeue {
facts, buildErr := s.buildCitationFacts(ctx, tx, task.TenantID, task.AIPlatformID, sourceInputs)
if buildErr != nil {
return nil, buildErr
@@ -2346,12 +2397,16 @@ func (s *MonitoringCallbackService) processTaskResult(ctx context.Context, task
taskStatus := "completed"
if runStatus == "failed" {
taskStatus = "failed"
taskStatus = failedOutcome.TaskStatus
}
if err := s.updateCollectTaskStatus(ctx, tx, task.ID, taskStatus, completedAt, req.ErrorMessage); err != nil {
if failedOutcome.Requeue {
if err := s.requeueCollectTaskAfterFailedResult(ctx, tx, task.ID, failedOutcome.RetryAfter, req.ErrorMessage); err != nil {
return nil, err
}
if runStatus == "failed" {
} else if err := s.updateCollectTaskStatus(ctx, tx, task.ID, taskStatus, completedAt, req.ErrorMessage); err != nil {
return nil, err
}
if runStatus == "failed" && !failedOutcome.Requeue {
if _, err := s.failRemainingMonitoringTasksForAuthorizationFailure(ctx, tx, task, req, completedAt); err != nil {
return nil, err
}
@@ -2365,9 +2420,13 @@ func (s *MonitoringCallbackService) processTaskResult(ctx context.Context, task
return nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring result ingest")
}
var runIDPtr *int64
if !failedOutcome.Requeue {
runIDPtr = &runID
}
return &MonitoringTaskResultResponse{
TaskID: task.ID,
RunID: &runID,
RunID: runIDPtr,
TaskStatus: taskStatus,
CitationSourceCount: citationSourceCount,
ContentCitationCount: contentCitationCount,
@@ -3113,6 +3172,51 @@ func (s *MonitoringCallbackService) updateCollectTaskStatus(ctx context.Context,
return nil
}
func (s *MonitoringCallbackService) requeueCollectTaskAfterFailedResult(ctx context.Context, tx pgx.Tx, taskID int64, retryAfter time.Duration, errorMessage *string) error {
if retryAfter <= 0 {
retryAfter = monitoringAutomaticFailureRetryAfter
}
if _, err := tx.Exec(ctx, `
UPDATE monitoring_collect_tasks
SET status = 'pending',
lease_token_hash = NULL,
leased_to_executor = NULL,
leased_at = NULL,
lease_expires_at = NULL,
callback_received_at = NULL,
completed_at = NULL,
skip_reason = NULL,
last_dispatched_at = NULL,
dispatch_after = NOW() + $2::interval,
error_message = $3,
request_payload_json = (
CASE
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
THEN '{}'::jsonb
ELSE request_payload_json
END
) || jsonb_build_object(
'automatic_failure_count',
(
CASE
WHEN COALESCE(request_payload_json ->> 'automatic_failure_count', '') ~ '^[0-9]+$'
THEN (request_payload_json ->> 'automatic_failure_count')::int
ELSE 0
END
) + 1,
'automatic_failure_retry_after_seconds',
$4::int,
'automatic_failure_last_requeued_at',
to_jsonb(NOW())
),
updated_at = NOW()
WHERE id = $1
`, taskID, formatPgInterval(retryAfter), nullableString(errorMessage), int(retryAfter.Seconds())); err != nil {
return response.ErrInternal(50041, "task_update_failed", "failed to requeue monitoring task after failed result")
}
return nil
}
func (s *MonitoringCallbackService) failRemainingMonitoringTasksForAuthorizationFailure(
ctx context.Context,
tx pgx.Tx,
@@ -499,6 +499,73 @@ func TestMonitoringTaskFailureDispositionDetectsAuthorizationFailureFromMessage(
}
}
func TestMonitoringFailedResultOutcomeRequeuesAutomaticFirstFailure(t *testing.T) {
message := "模型调用超时"
got := monitoringFailedResultOutcomeForTask(&monitoringCollectTask{
TriggerSource: "automatic",
AutomaticFailureCount: 0,
}, MonitoringTaskResultRequest{
Status: "failed",
ErrorMessage: &message,
})
if got.TaskStatus != "pending" {
t.Fatalf("TaskStatus = %q, want pending", got.TaskStatus)
}
if !got.Requeue {
t.Fatal("Requeue = false, want true")
}
if got.RetryAfter != monitoringAutomaticFailureRetryAfter {
t.Fatalf("RetryAfter = %v, want %v", got.RetryAfter, monitoringAutomaticFailureRetryAfter)
}
}
func TestMonitoringFailedResultOutcomeFailsAutomaticSecondFailure(t *testing.T) {
got := monitoringFailedResultOutcomeForTask(&monitoringCollectTask{
TriggerSource: "automatic",
AutomaticFailureCount: monitoringAutomaticFailureMaxAttempts - 1,
}, MonitoringTaskResultRequest{Status: "failed"})
if got.TaskStatus != "failed" {
t.Fatalf("TaskStatus = %q, want failed", got.TaskStatus)
}
if got.Requeue {
t.Fatal("Requeue = true, want false")
}
}
func TestMonitoringFailedResultOutcomeDoesNotRetryAuthorizationFailure(t *testing.T) {
message := "登录态已失效,请重新授权后再试。"
got := monitoringFailedResultOutcomeForTask(&monitoringCollectTask{
TriggerSource: "automatic",
AutomaticFailureCount: 0,
}, MonitoringTaskResultRequest{
Status: "failed",
ErrorMessage: &message,
})
if got.TaskStatus != "failed" {
t.Fatalf("TaskStatus = %q, want failed", got.TaskStatus)
}
if got.Requeue {
t.Fatal("Requeue = true, want false")
}
}
func TestMonitoringFailedResultOutcomeDoesNotRetryManualTask(t *testing.T) {
got := monitoringFailedResultOutcomeForTask(&monitoringCollectTask{
TriggerSource: "manual",
AutomaticFailureCount: 0,
}, MonitoringTaskResultRequest{Status: "failed"})
if got.TaskStatus != "failed" {
t.Fatalf("TaskStatus = %q, want failed", got.TaskStatus)
}
if got.Requeue {
t.Fatal("Requeue = true, want false")
}
}
func TestMonitoringAuthorizationFailureTargetClientPrefersExplicitTargetClient(t *testing.T) {
targetClientID := uuid.New().String()
task := &monitoringCollectTask{
@@ -20,6 +20,7 @@ type MonitoringDailyTaskMetricsSnapshot struct {
BrandFailureTotal int64 `json:"brand_failure_total"`
SkippedPlanTotal int64 `json:"skipped_plan_total"`
PlannedTaskTotal int64 `json:"planned_task_total"`
EligibleTaskTotal int64 `json:"eligible_task_total"`
DueTaskTotal int64 `json:"due_task_total"`
CreatedTaskTotal int64 `json:"created_task_total"`
DesktopTaskTotal int64 `json:"desktop_task_total"`
@@ -44,6 +45,7 @@ type monitoringDailyTaskMetricsRecorder struct {
brandFailureTotal atomic.Int64
skippedPlanTotal atomic.Int64
plannedTaskTotal atomic.Int64
eligibleTaskTotal atomic.Int64
dueTaskTotal atomic.Int64
createdTaskTotal atomic.Int64
desktopTaskTotal atomic.Int64
@@ -98,6 +100,7 @@ func (r *monitoringDailyTaskMetricsRecorder) record(summary *MonitoringDailyTask
r.brandFailureTotal.Add(int64(summary.FailedBrandCount))
r.skippedPlanTotal.Add(int64(summary.SkippedPlanCount))
r.plannedTaskTotal.Add(summary.PlannedTaskCount)
r.eligibleTaskTotal.Add(summary.EligibleTaskCount)
r.dueTaskTotal.Add(summary.DueTaskCount)
r.createdTaskTotal.Add(summary.CreatedTaskCount)
r.desktopTaskTotal.Add(summary.DesktopTaskCount)
@@ -133,6 +136,7 @@ func (r *monitoringDailyTaskMetricsRecorder) snapshot() MonitoringDailyTaskMetri
BrandFailureTotal: r.brandFailureTotal.Load(),
SkippedPlanTotal: r.skippedPlanTotal.Load(),
PlannedTaskTotal: r.plannedTaskTotal.Load(),
EligibleTaskTotal: r.eligibleTaskTotal.Load(),
DueTaskTotal: r.dueTaskTotal.Load(),
CreatedTaskTotal: r.createdTaskTotal.Load(),
DesktopTaskTotal: r.desktopTaskTotal.Load(),
@@ -24,12 +24,13 @@ const (
defaultMonitoringDailyTaskHardCap = 36
defaultMonitoringDailyBrandLimit = 1
defaultMonitoringDailyLookahead = 10 * time.Minute
defaultMonitoringDailyMaterializeWindow = 2 * time.Hour
defaultMonitoringDailyDesktopRetryCooldown = 15 * time.Minute
defaultMonitoringDailyPublishTimeout = 5 * time.Second
defaultMonitoringDailyMaxMaterializePerRun = 64
defaultMonitoringDailyMaxMaterializePerPlan = 12
defaultMonitoringDailyMaxMaterializePerBrand = 4
defaultMonitoringDailyMaxMaterializePerRun = 0
defaultMonitoringDailyMaxMaterializePerPlan = 0
defaultMonitoringDailyMaxMaterializePerBrand = 0
defaultMonitoringDailyMaxDesktopDispatch = 0
defaultMonitoringDailyMaxDesktopClientBacklog = 12
monitoringDailyPlatformActiveWindow = 24 * time.Hour
monitoringDailyMaterializeStartOffset = 5 * time.Minute
)
@@ -49,6 +50,7 @@ type MonitoringDailyTaskGenerationSummary struct {
FailedPlanCount int
FailedBrandCount int
PlannedTaskCount int64
EligibleTaskCount int64
DueTaskCount int64
CreatedTaskCount int64
DesktopTaskCount int64
@@ -60,6 +62,8 @@ type MonitoringDailyTaskRunOptions struct {
MaxMaterializePerRun int
MaxMaterializePerPlan int
MaxMaterializePerBrand int
MaxDesktopDispatchPerRun int
MaxDesktopClientBacklog int
}
func normalizeMonitoringDailyRunOptions(options ...MonitoringDailyTaskRunOptions) MonitoringDailyTaskRunOptions {
@@ -67,6 +71,8 @@ func normalizeMonitoringDailyRunOptions(options ...MonitoringDailyTaskRunOptions
MaxMaterializePerRun: defaultMonitoringDailyMaxMaterializePerRun,
MaxMaterializePerPlan: defaultMonitoringDailyMaxMaterializePerPlan,
MaxMaterializePerBrand: defaultMonitoringDailyMaxMaterializePerBrand,
MaxDesktopDispatchPerRun: defaultMonitoringDailyMaxDesktopDispatch,
MaxDesktopClientBacklog: defaultMonitoringDailyMaxDesktopClientBacklog,
}
if len(options) > 0 {
in := options[0]
@@ -79,11 +85,17 @@ func normalizeMonitoringDailyRunOptions(options ...MonitoringDailyTaskRunOptions
if in.MaxMaterializePerBrand > 0 {
out.MaxMaterializePerBrand = in.MaxMaterializePerBrand
}
if in.MaxDesktopDispatchPerRun > 0 {
out.MaxDesktopDispatchPerRun = in.MaxDesktopDispatchPerRun
}
if out.MaxMaterializePerPlan > out.MaxMaterializePerRun {
if in.MaxDesktopClientBacklog > 0 {
out.MaxDesktopClientBacklog = in.MaxDesktopClientBacklog
}
}
if out.MaxMaterializePerRun > 0 && out.MaxMaterializePerPlan > out.MaxMaterializePerRun {
out.MaxMaterializePerPlan = out.MaxMaterializePerRun
}
if out.MaxMaterializePerBrand > out.MaxMaterializePerPlan {
if out.MaxMaterializePerPlan > 0 && out.MaxMaterializePerBrand > out.MaxMaterializePerPlan {
out.MaxMaterializePerBrand = out.MaxMaterializePerPlan
}
return out
@@ -227,6 +239,7 @@ func (w *MonitoringDailyTaskWorker) RunOnce(parent context.Context, options ...M
stats["failed_plan_count"] = summary.FailedPlanCount
stats["failed_brand_count"] = summary.FailedBrandCount
stats["planned_task_count"] = summary.PlannedTaskCount
stats["eligible_task_count"] = summary.EligibleTaskCount
stats["due_task_count"] = summary.DueTaskCount
stats["created_task_count"] = summary.CreatedTaskCount
stats["desktop_task_count"] = summary.DesktopTaskCount
@@ -236,6 +249,8 @@ func (w *MonitoringDailyTaskWorker) RunOnce(parent context.Context, options ...M
stats["max_materialize_per_run"] = runOptions.MaxMaterializePerRun
stats["max_materialize_per_plan"] = runOptions.MaxMaterializePerPlan
stats["max_materialize_per_brand"] = runOptions.MaxMaterializePerBrand
stats["max_desktop_dispatch_per_run"] = runOptions.MaxDesktopDispatchPerRun
stats["max_desktop_client_backlog"] = runOptions.MaxDesktopClientBacklog
if err != nil {
if w.logger != nil {
w.logger.Warn("monitoring daily task generation failed", MonitoringWorkerErrorFields(err)...)
@@ -267,10 +282,10 @@ func (s *MonitoringService) GenerateDailyMonitoringTasks(
TenantCount: len(plans),
}
options := normalizeMonitoringDailyRunOptions(runOptions...)
globalRemaining := options.MaxMaterializePerRun
globalMaterializeRemaining := options.MaxMaterializePerRun
for _, plan := range plans {
if globalRemaining <= 0 {
if monitoringDailyLimitReached(globalMaterializeRemaining, options.MaxMaterializePerRun) {
break
}
executionOwner, ok, skipReason := s.resolveDailyMonitoringExecutionOwner(plan)
@@ -308,10 +323,15 @@ func (s *MonitoringService) GenerateDailyMonitoringTasks(
continue
}
planRunRemaining := minMonitoringDailyInt(options.MaxMaterializePerPlan, globalRemaining)
planMaterializeRemaining := options.MaxMaterializePerPlan
if options.MaxMaterializePerRun > 0 && (planMaterializeRemaining <= 0 || globalMaterializeRemaining < planMaterializeRemaining) {
planMaterializeRemaining = globalMaterializeRemaining
}
planDesktopDispatchRemaining := monitoringDailyPlanDesktopDispatchBudget(options)
for _, brand := range brands {
if planRunRemaining <= 0 || globalRemaining <= 0 {
if monitoringDailyLimitReached(planMaterializeRemaining, options.MaxMaterializePerPlan) ||
monitoringDailyLimitReached(globalMaterializeRemaining, options.MaxMaterializePerRun) {
break
}
@@ -334,9 +354,14 @@ func (s *MonitoringService) GenerateDailyMonitoringTasks(
continue
}
dispatchLimit := minMonitoringDailyInt(options.MaxMaterializePerBrand, planRunRemaining, globalRemaining)
materializeLimit := dispatchLimit
createdCount, dueCount, desktopCount, deferredCount, err := s.generateDailyMonitoringTasksForBrand(
materializeLimit := monitoringDailyBoundedLimit(
len(candidates),
options.MaxMaterializePerBrand,
planMaterializeRemaining,
globalMaterializeRemaining,
)
dispatchLimit := monitoringDailyDispatchLimit(len(candidates), planDesktopDispatchRemaining, planDesktopDispatchRemaining)
createdCount, eligibleCount, dueCount, desktopCount, deferredCount, err := s.generateDailyMonitoringTasksForBrand(
ctx,
projectionService,
plan,
@@ -348,6 +373,7 @@ func (s *MonitoringService) GenerateDailyMonitoringTasks(
materializeLimit,
dispatchLimit,
executionOwner,
options.MaxDesktopClientBacklog,
)
if err != nil {
if abortErr := monitoringDailyAbortErr(ctx); abortErr != nil {
@@ -358,13 +384,18 @@ func (s *MonitoringService) GenerateDailyMonitoringTasks(
continue
}
processedCount := int(maxMonitoringDailyInt(dueCount, desktopCount+deferredCount))
if processedCount > 0 {
planRunRemaining -= processedCount
globalRemaining -= processedCount
if options.MaxMaterializePerPlan > 0 {
planMaterializeRemaining -= int(dueCount)
}
if options.MaxMaterializePerRun > 0 {
globalMaterializeRemaining -= int(dueCount)
}
if planDesktopDispatchRemaining > 0 {
planDesktopDispatchRemaining -= int(desktopCount + deferredCount)
}
summary.BrandCount++
summary.PlannedTaskCount += int64(len(candidates))
summary.EligibleTaskCount += eligibleCount
summary.DueTaskCount += dueCount
summary.CreatedTaskCount += createdCount
summary.DesktopTaskCount += desktopCount
@@ -431,89 +462,105 @@ func (s *MonitoringService) generateDailyMonitoringTasksForBrand(
materializeLimit int,
dispatchLimit int,
executionOwner string,
) (int64, int64, int64, int64, error) {
maxDesktopClientBacklog int,
) (int64, int64, int64, int64, int64, error) {
if materializeLimit <= 0 && dispatchLimit <= 0 {
return 0, 0, 0, 0, nil
return 0, 0, 0, 0, 0, nil
}
if executionOwner != "desktop_tasks" {
return 0, 0, 0, 0, nil
return 0, 0, 0, 0, 0, nil
}
var createdCount int64
var eligibleCount int64
var dueCount int64
if materializeLimit > 0 {
tx, err := s.monitoringPool.Begin(ctx)
if err != nil {
return 0, 0, 0, 0, response.ErrInternal(50041, "begin_failed", "failed to start monitoring daily task generation")
}
defer tx.Rollback(ctx)
locked, err := acquireMonitoringDailyBrandLock(ctx, tx, plan.TenantID, brand.BrandID, monitoringBusinessDateText(businessDay))
if err != nil {
return 0, 0, 0, 0, err
}
if locked {
existingKeys, err := s.loadDailyMonitoringTaskKeys(ctx, tx, plan.TenantID, brand.BrandID, businessDay)
if err != nil {
return 0, 0, 0, 0, err
}
dueCandidates := selectDueMonitoringDailyCandidates(candidates, existingKeys, now, defaultMonitoringDailyLookahead, materializeLimit)
dueCount = int64(len(dueCandidates))
if len(dueCandidates) > 0 {
if err := s.syncMonitoringQuestionSnapshots(ctx, tx, plan.TenantID, brand.BrandID, questions, true); err != nil {
return 0, 0, 0, 0, err
}
createdCount, err = s.ensureDailyMonitoringCollectTasks(ctx, tx, plan, brand, businessDay, dueCandidates, executionOwner)
if err != nil {
return 0, 0, 0, 0, err
}
if projectionService != nil && createdCount > 0 {
if err := projectionService.rebuildBrandDailyProjection(ctx, tx, plan.TenantID, brand.BrandID, businessDay); err != nil {
return 0, 0, 0, 0, err
}
}
}
}
if err := tx.Commit(ctx); err != nil {
return 0, 0, 0, 0, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring daily task generation")
}
}
specs := make([]monitorDesktopTaskSpec, 0)
if dispatchLimit <= 0 {
return createdCount, dueCount, 0, 0, nil
return createdCount, eligibleCount, dueCount, 0, 0, nil
}
dispatchablePlatformIDs, err := s.loadDailyMonitorDispatchablePlatformIDs(ctx, plan.TenantID, plan.WorkspaceID, candidates)
dispatchablePlatformIDs, err := s.loadDailyMonitorDispatchablePlatformIDs(ctx, plan.TenantID, plan.WorkspaceID, candidates, maxDesktopClientBacklog)
if err != nil {
return createdCount, dueCount, 0, 0, err
return createdCount, eligibleCount, dueCount, 0, 0, err
}
if len(dispatchablePlatformIDs) == 0 {
return createdCount, dueCount, 0, 0, nil
return createdCount, eligibleCount, dueCount, 0, 0, nil
}
specs, err := s.loadDueDailyMonitorDesktopTaskSpecs(
dispatchablePlatformSet := monitoringDailyPlatformSet(dispatchablePlatformIDs)
dispatchableCandidates := filterMonitoringDailyCandidatesByPlatform(candidates, dispatchablePlatformSet)
eligibleCount = int64(len(dispatchableCandidates))
retrySpecs, err := s.loadDueDailyMonitorDesktopTaskSpecs(
ctx,
s.monitoringPool,
plan.TenantID,
plan.WorkspaceID,
brand.BrandID,
businessDay,
now.Add(defaultMonitoringDailyLookahead),
now,
now.Add(-defaultMonitoringDailyDesktopRetryCooldown),
dispatchablePlatformIDs,
dispatchLimit,
)
if err != nil {
return createdCount, dueCount, 0, 0, err
return createdCount, eligibleCount, dueCount, 0, 0, err
}
if len(specs) == 0 {
return createdCount, dueCount, 0, 0, nil
if len(retrySpecs) > 0 {
specs = append(specs, retrySpecs...)
createdCount += int64(len(retrySpecs))
dueCount += int64(len(retrySpecs))
}
publishedTasks, deferredTaskIDs, err := s.dispatchMonitorDesktopTasks(ctx, specs)
remainingCreateLimit := dispatchLimit - len(specs)
if remainingCreateLimit <= 0 {
return s.dispatchDailyMonitoringDesktopSpecs(ctx, specs, maxDesktopClientBacklog, createdCount, eligibleCount, dueCount)
}
materializeLimit = monitoringDailyBoundedLimit(materializeLimit, dispatchLimit)
if materializeLimit > 0 && len(dispatchableCandidates) > 0 {
var materializeErr error
var newCreatedCount int64
var newDueCount int64
var newSpecs []monitorDesktopTaskSpec
newCreatedCount, newDueCount, newSpecs, materializeErr = s.createDispatchableDailyMonitoringTasksForBrand(
ctx,
projectionService,
plan,
brand,
businessDay,
now,
questions,
dispatchableCandidates,
monitoringDailyBoundedLimit(materializeLimit, remainingCreateLimit),
executionOwner,
)
if materializeErr != nil {
return createdCount, eligibleCount, dueCount, 0, 0, materializeErr
}
createdCount += newCreatedCount
dueCount += newDueCount
specs = append(specs, newSpecs...)
}
if len(specs) == 0 {
return createdCount, eligibleCount, dueCount, 0, 0, nil
}
return s.dispatchDailyMonitoringDesktopSpecs(ctx, specs, maxDesktopClientBacklog, createdCount, eligibleCount, dueCount)
}
func (s *MonitoringService) dispatchDailyMonitoringDesktopSpecs(
ctx context.Context,
specs []monitorDesktopTaskSpec,
maxDesktopClientBacklog int,
createdCount int64,
eligibleCount int64,
dueCount int64,
) (int64, int64, int64, int64, int64, error) {
if len(specs) == 0 {
return createdCount, eligibleCount, dueCount, 0, 0, nil
}
publishedTasks, deferredTaskIDs, err := s.dispatchMonitorDesktopTasks(ctx, specs, maxDesktopClientBacklog)
if err != nil {
if s.logger != nil {
s.logger.Warn("daily monitor desktop dispatch failed; pending tasks will be retried",
@@ -521,7 +568,7 @@ func (s *MonitoringService) generateDailyMonitoringTasksForBrand(
zap.Error(err),
)
}
return createdCount, dueCount, 0, 0, err
return createdCount, eligibleCount, dueCount, 0, 0, err
}
for _, task := range publishedTasks {
@@ -547,7 +594,7 @@ func (s *MonitoringService) generateDailyMonitoringTasksForBrand(
cancel()
}
return createdCount, dueCount, int64(len(publishedTasks)), int64(len(deferredTaskIDs)), nil
return createdCount, eligibleCount, dueCount, int64(len(publishedTasks)), int64(len(deferredTaskIDs)), nil
}
func (s *MonitoringService) loadDailyMonitoringPlans(ctx context.Context) ([]monitoringDailyPlan, error) {
@@ -872,10 +919,72 @@ func (s *MonitoringService) loadDailyMonitoringTaskKeys(ctx context.Context, q m
return keys, nil
}
func (s *MonitoringService) createDispatchableDailyMonitoringTasksForBrand(
ctx context.Context,
projectionService *MonitoringCallbackService,
plan monitoringDailyPlan,
brand monitoringDailyBrand,
businessDay time.Time,
now time.Time,
questions []monitoringConfiguredQuestion,
candidates []monitoringDailyTaskCandidate,
limit int,
executionOwner string,
) (int64, int64, []monitorDesktopTaskSpec, error) {
if limit <= 0 || len(candidates) == 0 {
return 0, 0, nil, nil
}
tx, err := s.monitoringPool.Begin(ctx)
if err != nil {
return 0, 0, nil, response.ErrInternal(50041, "begin_failed", "failed to start monitoring daily task generation")
}
defer tx.Rollback(ctx)
locked, err := acquireMonitoringDailyBrandLock(ctx, tx, plan.TenantID, brand.BrandID, monitoringBusinessDateText(businessDay))
if err != nil {
return 0, 0, nil, err
}
if !locked {
return 0, 0, nil, nil
}
existingKeys, err := s.loadDailyMonitoringTaskKeys(ctx, tx, plan.TenantID, brand.BrandID, businessDay)
if err != nil {
return 0, 0, nil, err
}
dueCandidates := selectDueMonitoringDailyCandidates(candidates, existingKeys, now, defaultMonitoringDailyLookahead, limit)
dueCount := int64(len(dueCandidates))
if len(dueCandidates) == 0 {
if err := tx.Commit(ctx); err != nil {
return 0, dueCount, nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring daily task generation")
}
return 0, dueCount, nil, nil
}
if err := s.syncMonitoringQuestionSnapshots(ctx, tx, plan.TenantID, brand.BrandID, questions, true); err != nil {
return 0, dueCount, nil, err
}
createdCount, specs, err := s.ensureDailyMonitoringCollectTasks(ctx, tx, plan, brand, businessDay, dueCandidates, executionOwner)
if err != nil {
return 0, dueCount, nil, err
}
if projectionService != nil && createdCount > 0 {
if err := projectionService.rebuildBrandDailyProjection(ctx, tx, plan.TenantID, brand.BrandID, businessDay); err != nil {
return 0, dueCount, nil, err
}
}
if err := tx.Commit(ctx); err != nil {
return 0, dueCount, nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring daily task generation")
}
return createdCount, dueCount, specs, nil
}
func (s *MonitoringService) loadDailyMonitorDispatchablePlatformIDs(
ctx context.Context,
tenantID, workspaceID int64,
candidates []monitoringDailyTaskCandidate,
maxDesktopClientBacklog int,
) ([]string, error) {
if len(candidates) == 0 {
return nil, nil
@@ -898,7 +1007,7 @@ func (s *MonitoringService) loadDailyMonitorDispatchablePlatformIDs(
return nil, nil
}
targets, err := s.loadMonitorDesktopTaskTargets(ctx, tenantID, workspaceID, 0, specs)
targets, err := s.loadIdleMonitorDesktopTaskTargets(ctx, tenantID, workspaceID, 0, specs, maxDesktopClientBacklog)
if err != nil {
return nil, err
}
@@ -912,6 +1021,30 @@ func (s *MonitoringService) loadDailyMonitorDispatchablePlatformIDs(
return reconcileEnabledMonitoringPlatforms(platformIDs), nil
}
func monitoringDailyPlatformSet(platformIDs []string) map[string]struct{} {
result := make(map[string]struct{}, len(platformIDs))
for _, platformID := range platformIDs {
platformID = normalizeMonitoringPlatformID(platformID)
if platformID != "" {
result[platformID] = struct{}{}
}
}
return result
}
func filterMonitoringDailyCandidatesByPlatform(candidates []monitoringDailyTaskCandidate, platforms map[string]struct{}) []monitoringDailyTaskCandidate {
if len(candidates) == 0 || len(platforms) == 0 {
return nil
}
result := make([]monitoringDailyTaskCandidate, 0, len(candidates))
for _, candidate := range candidates {
if _, ok := platforms[normalizeMonitoringPlatformID(candidate.Platform.ID)]; ok {
result = append(result, candidate)
}
}
return result
}
func (s *MonitoringService) ensureDailyMonitoringCollectTasks(
ctx context.Context,
tx pgx.Tx,
@@ -920,12 +1053,12 @@ func (s *MonitoringService) ensureDailyMonitoringCollectTasks(
businessDay time.Time,
candidates []monitoringDailyTaskCandidate,
executionOwner string,
) (int64, error) {
) (int64, []monitorDesktopTaskSpec, error) {
if len(candidates) == 0 {
return 0, nil
return 0, nil, nil
}
if strings.TrimSpace(executionOwner) != "desktop_tasks" {
return 0, response.ErrInternal(50041, "invalid_execution_owner", "daily monitoring tasks require desktop task execution")
return 0, nil, response.ErrInternal(50041, "invalid_execution_owner", "daily monitoring tasks require desktop task execution")
}
businessDate := monitoringBusinessDateText(businessDay)
@@ -940,7 +1073,7 @@ func (s *MonitoringService) ensureDailyMonitoringCollectTasks(
for _, candidate := range candidates {
questionText, textErr := monitoringQuestionTextSnapshot(candidate.Question)
if textErr != nil {
return 0, textErr
return 0, nil, textErr
}
dispatchAfter := candidate.DispatchAfter
if dispatchAfter.IsZero() {
@@ -953,7 +1086,18 @@ func (s *MonitoringService) ensureDailyMonitoringCollectTasks(
dispatchAfterValues = append(dispatchAfterValues, dispatchAfter)
}
tag, err := tx.Exec(ctx, `
rows, err := tx.Query(ctx, `
WITH input AS (
SELECT *
FROM unnest(
$7::bigint[],
$8::bytea[],
$9::text[],
$10::text[],
$11::timestamptz[]
) AS item(question_id, question_hash, question_text, ai_platform_id, dispatch_after)
),
upserted AS (
INSERT INTO monitoring_collect_tasks (
tenant_id, brand_id, question_id, question_hash, question_text_snapshot, ai_platform_id,
collector_type, trigger_source, run_mode, business_date, planned_at, status,
@@ -979,21 +1123,53 @@ func (s *MonitoringService) ensureDailyMonitoringCollectTasks(
input.dispatch_after,
$5,
$6
FROM unnest(
$7::bigint[],
$8::bytea[],
$9::text[],
$10::text[],
$11::timestamptz[]
) AS input(question_id, question_hash, question_text, ai_platform_id, dispatch_after)
FROM input
ON CONFLICT (
tenant_id, brand_id, question_id, ai_platform_id, collector_type, run_mode, business_date
)
DO UPDATE SET
question_hash = EXCLUDED.question_hash,
question_text_snapshot = EXCLUDED.question_text_snapshot,
dispatch_after = LEAST(
COALESCE(monitoring_collect_tasks.dispatch_after, EXCLUDED.dispatch_after),
EXCLUDED.dispatch_after
),
updated_at = NOW()
WHERE btrim(COALESCE(monitoring_collect_tasks.question_text_snapshot, '')) = ''
WHERE monitoring_collect_tasks.status = 'pending'
AND monitoring_collect_tasks.callback_received_at IS NULL
RETURNING *
),
claimed AS (
UPDATE monitoring_collect_tasks t
SET last_dispatched_at = NOW(),
dispatch_attempts = COALESCE(t.dispatch_attempts, 0) + 1,
updated_at = NOW()
FROM upserted u
WHERE t.id = u.id
AND t.status = 'pending'
AND t.callback_received_at IS NULL
AND (t.dispatch_after IS NULL OR t.dispatch_after <= NOW())
AND (
t.last_dispatched_at IS NULL
OR t.last_dispatched_at <= NOW() - $12::interval
)
RETURNING t.*
)
SELECT
t.id,
t.tenant_id,
t.ai_platform_id,
t.business_date,
t.trigger_source,
t.run_mode,
t.question_id,
t.question_hash,
t.dispatch_priority,
t.dispatch_lane,
t.interrupt_generation,
t.question_text_snapshot
FROM claimed t
ORDER BY t.dispatch_priority DESC, t.id ASC
`,
plan.TenantID,
brand.BrandID,
@@ -1006,12 +1182,50 @@ func (s *MonitoringService) ensureDailyMonitoringCollectTasks(
questionTexts,
platformIDs,
dispatchAfterValues,
formatPgInterval(defaultMonitoringDailyDesktopRetryCooldown),
)
if err != nil {
return 0, response.ErrInternal(50041, "insert_failed", "failed to create daily monitoring tasks")
return 0, nil, response.ErrInternal(50041, "insert_failed", "failed to create daily monitoring tasks")
}
defer rows.Close()
return tag.RowsAffected(), nil
specs := make([]monitorDesktopTaskSpec, 0, len(candidates))
for rows.Next() {
var (
spec monitorDesktopTaskSpec
questionHash []byte
businessDay time.Time
)
if scanErr := rows.Scan(
&spec.MonitorTaskID,
&spec.TenantID,
&spec.PlatformID,
&businessDay,
&spec.TriggerSource,
&spec.RunMode,
&spec.QuestionID,
&questionHash,
&spec.Priority,
&spec.Lane,
&spec.InterruptGeneration,
&spec.QuestionText,
); scanErr != nil {
return 0, nil, response.ErrInternal(50041, "scan_failed", "failed to parse created daily monitoring tasks")
}
spec.WorkspaceID = plan.WorkspaceID
spec.BusinessDate = businessDay.Format("2006-01-02")
spec.QuestionHash = encodeQuestionHash(questionHash)
spec.QuestionText = strings.TrimSpace(spec.QuestionText)
if spec.QuestionText == "" {
continue
}
spec.SchedulerGroupKey = monitoringSchedulerGroupKey(spec.QuestionHash, spec.QuestionID, spec.QuestionText)
specs = append(specs, spec)
}
if err := rows.Err(); err != nil {
return 0, nil, response.ErrInternal(50041, "scan_failed", "failed to iterate created daily monitoring tasks")
}
return int64(len(specs)), specs, nil
}
func (s *MonitoringService) loadDueDailyMonitorDesktopTaskSpecs(
@@ -1178,52 +1392,65 @@ func buildMonitoringDailyTaskCandidates(
if len(candidates) > limit {
candidates = candidates[:limit]
}
assignMonitoringDailyScheduleTimes(tenantID, brandID, businessDay, candidates)
assignMonitoringDailyScheduleTimes(businessDay, candidates)
return candidates
}
func assignMonitoringDailyScheduleTimes(tenantID, brandID int64, businessDay time.Time, candidates []monitoringDailyTaskCandidate) {
func assignMonitoringDailyScheduleTimes(businessDay time.Time, candidates []monitoringDailyTaskCandidate) {
if len(candidates) == 0 {
return
}
start := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset)
window := defaultMonitoringDailyMaterializeWindow
if window <= 0 {
window = time.Hour
}
slot := window / time.Duration(len(candidates))
if slot <= 0 {
slot = time.Second
}
latest := start.Add(window)
businessDate := monitoringBusinessDateText(businessDay)
dueAt := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset).UTC()
for i := range candidates {
jitter := time.Duration(monitoringDailyStableHash(
"dispatch_jitter",
int64Text(tenantID),
int64Text(brandID),
businessDate,
int64Text(candidates[i].Question.ID),
candidates[i].Platform.ID,
) % uint64(slot))
materializeAt := start.Add(time.Duration(i)*slot + jitter)
if materializeAt.After(latest) {
materializeAt = latest
}
candidates[i].MaterializeAt = materializeAt.UTC()
candidates[i].DispatchAfter = materializeAt.UTC()
candidates[i].MaterializeAt = dueAt
candidates[i].DispatchAfter = dueAt
}
sort.SliceStable(candidates, func(i, j int) bool {
if candidates[i].MaterializeAt.Equal(candidates[j].MaterializeAt) {
return candidates[i].sortKey < candidates[j].sortKey
if candidates[i].sortKey == candidates[j].sortKey {
if candidates[i].Question.ID == candidates[j].Question.ID {
return candidates[i].Platform.ID < candidates[j].Platform.ID
}
return candidates[i].MaterializeAt.Before(candidates[j].MaterializeAt)
return candidates[i].Question.ID < candidates[j].Question.ID
}
return candidates[i].sortKey < candidates[j].sortKey
})
}
func monitoringDailyLimitReached(remaining, configuredLimit int) bool {
return configuredLimit > 0 && remaining <= 0
}
func monitoringDailyBoundedLimit(limit int, caps ...int) int {
if limit <= 0 {
return 0
}
for _, cap := range caps {
if cap > 0 && cap < limit {
limit = cap
}
}
return limit
}
func monitoringDailyDispatchLimit(candidateCount, remaining, configuredLimit int) int {
if configuredLimit <= 0 {
return candidateCount
}
if remaining <= 0 {
return 0
}
return monitoringDailyBoundedLimit(candidateCount, remaining)
}
func monitoringDailyPlanDesktopDispatchBudget(options MonitoringDailyTaskRunOptions) int {
if options.MaxDesktopDispatchPerRun > 0 {
return options.MaxDesktopDispatchPerRun
}
return options.MaxDesktopClientBacklog
}
func selectDueMonitoringDailyCandidates(
candidates []monitoringDailyTaskCandidate,
existing map[monitoringDailyTaskKey]struct{},
@@ -16,7 +16,7 @@ import (
"github.com/stretchr/testify/require"
)
func TestBuildMonitoringDailyTaskCandidatesMaterializesAfterMidnight(t *testing.T) {
func TestBuildMonitoringDailyTaskCandidatesMaterializesWholeDayImmediately(t *testing.T) {
businessDay := time.Date(2026, 4, 24, 0, 0, 0, 0, monitoringBusinessLocation())
questions := []monitoringConfiguredQuestion{
{ID: 101, QuestionHash: []byte("q101")},
@@ -31,18 +31,96 @@ func TestBuildMonitoringDailyTaskCandidatesMaterializesAfterMidnight(t *testing.
candidates := buildMonitoringDailyTaskCandidates(7, 11, businessDay, questions, platforms, 4)
require.Len(t, candidates, 4)
start := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset).UTC()
end := start.Add(defaultMonitoringDailyMaterializeWindow)
dueAt := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset).UTC()
for i, candidate := range candidates {
assert.False(t, candidate.MaterializeAt.Before(start))
assert.False(t, candidate.MaterializeAt.After(end))
assert.Equal(t, dueAt, candidate.MaterializeAt)
assert.Equal(t, candidate.MaterializeAt, candidate.DispatchAfter)
if i > 0 {
assert.False(t, candidate.MaterializeAt.Before(candidates[i-1].MaterializeAt))
assert.LessOrEqual(t, candidates[i-1].sortKey, candidate.sortKey)
}
}
}
func TestNormalizeMonitoringDailyRunOptionsDefaultsDoNotThrottleByPlanOrBrand(t *testing.T) {
got := normalizeMonitoringDailyRunOptions()
assert.Equal(t, 0, got.MaxMaterializePerRun)
assert.Equal(t, 0, got.MaxMaterializePerPlan)
assert.Equal(t, 0, got.MaxMaterializePerBrand)
assert.Equal(t, 0, got.MaxDesktopDispatchPerRun)
}
func TestMonitoringDailyBoundedLimitTreatsZeroCapsAsUnlimited(t *testing.T) {
assert.Equal(t, 102, monitoringDailyBoundedLimit(102, 0, 0, 0))
assert.Equal(t, 64, monitoringDailyBoundedLimit(102, 0, 64, 0))
assert.Equal(t, 12, monitoringDailyBoundedLimit(102, 0, 64, 12))
}
func TestMonitoringDailyDispatchLimitStopsWhenRunCapacityIsExhausted(t *testing.T) {
assert.Equal(t, 102, monitoringDailyDispatchLimit(102, 0, 0))
assert.Equal(t, 12, monitoringDailyDispatchLimit(102, 12, 12))
assert.Equal(t, 0, monitoringDailyDispatchLimit(102, 0, 12))
}
func TestMonitoringDailyPlanDesktopDispatchBudgetUsesBacklogWhenUnconfigured(t *testing.T) {
assert.Equal(t, 12, monitoringDailyPlanDesktopDispatchBudget(MonitoringDailyTaskRunOptions{
MaxDesktopClientBacklog: 12,
}))
assert.Equal(t, 6, monitoringDailyPlanDesktopDispatchBudget(MonitoringDailyTaskRunOptions{
MaxDesktopDispatchPerRun: 6,
MaxDesktopClientBacklog: 12,
}))
}
func TestAutomaticDailySelectionIncludesAllUserQuestionsAndPlatforms(t *testing.T) {
businessDay := time.Date(2026, 6, 27, 0, 0, 0, 0, monitoringBusinessLocation())
questions := make([]monitoringConfiguredQuestion, 0, 17)
for id := int64(1); id <= 17; id++ {
questions = append(questions, monitoringConfiguredQuestion{
ID: id,
QuestionText: "用户问题 " + int64Text(id),
QuestionHash: []byte("q" + int64Text(id)),
})
}
platforms := []monitoringPlatformMetadata{
{ID: "yuanbao"},
{ID: "kimi"},
{ID: "wenxin"},
{ID: "doubao"},
{ID: "deepseek"},
{ID: "qwen"},
}
candidates := buildMonitoringDailyTaskCandidates(
7,
7,
businessDay,
questions,
platforms,
len(questions)*len(platforms),
)
limit := monitoringDailyBoundedLimit(len(candidates), 0, 0, 0)
due := selectDueMonitoringDailyCandidates(
candidates,
nil,
monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset),
defaultMonitoringDailyLookahead,
limit,
)
require.Len(t, candidates, 17*6)
require.Len(t, due, 17*6)
seen := make(map[monitoringDailyTaskKey]struct{}, len(due))
for _, candidate := range due {
seen[monitoringDailyTaskKey{
QuestionID: candidate.Question.ID,
PlatformID: candidate.Platform.ID,
}] = struct{}{}
}
assert.Contains(t, seen, monitoringDailyTaskKey{QuestionID: 17, PlatformID: "qwen"})
assert.Contains(t, seen, monitoringDailyTaskKey{QuestionID: 17, PlatformID: "deepseek"})
}
func TestSelectDueMonitoringDailyCandidatesSkipsExistingAndHonorsLimit(t *testing.T) {
businessDay := time.Date(2026, 4, 24, 0, 0, 0, 0, monitoringBusinessLocation())
candidates := buildMonitoringDailyTaskCandidates(
@@ -68,7 +146,7 @@ func TestSelectDueMonitoringDailyCandidatesSkipsExistingAndHonorsLimit(t *testin
PlatformID: candidates[0].Platform.ID,
}: {},
}
now := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset + defaultMonitoringDailyMaterializeWindow)
now := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset)
due := selectDueMonitoringDailyCandidates(candidates, existing, now, defaultMonitoringDailyLookahead, 2)
@@ -110,7 +188,7 @@ func TestSelectDueMonitoringDailyCandidatesContinuesPastExistingTasks(t *testing
}] = struct{}{}
}
now := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset + defaultMonitoringDailyMaterializeWindow)
now := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset)
due := selectDueMonitoringDailyCandidates(candidates, existing, now, defaultMonitoringDailyLookahead, 4)
require.Len(t, due, 2)
@@ -158,10 +236,10 @@ func TestEnsureDailyMonitoringCollectTasksPersistsQuestionTextSnapshot(t *testin
tx := &captureDailyMonitorTaskTx{commandTag: pgconn.NewCommandTag("INSERT 0 2")}
businessDay := time.Date(2026, 6, 22, 0, 0, 0, 0, time.UTC)
created, err := (&MonitoringService{}).ensureDailyMonitoringCollectTasks(
created, specs, err := (&MonitoringService{}).ensureDailyMonitoringCollectTasks(
ctx,
tx,
monitoringDailyPlan{TenantID: 4},
monitoringDailyPlan{TenantID: 4, WorkspaceID: 8},
monitoringDailyBrand{BrandID: 8},
businessDay,
[]monitoringDailyTaskCandidate{
@@ -186,23 +264,27 @@ func TestEnsureDailyMonitoringCollectTasksPersistsQuestionTextSnapshot(t *testin
)
require.NoError(t, err)
assert.Equal(t, int64(2), created)
assert.True(t, tx.execCalled)
normalizedSQL := normalizeSQLWhitespace(tx.execSQL)
assert.Equal(t, int64(0), created)
assert.Empty(t, specs)
assert.True(t, tx.queryCalled)
normalizedSQL := normalizeSQLWhitespace(tx.querySQL)
assert.Contains(t, normalizedSQL, "question_text_snapshot, ai_platform_id")
assert.Contains(t, normalizedSQL, "$10::text[]")
assert.Contains(t, normalizedSQL, "AS input(question_id, question_hash, question_text, ai_platform_id, dispatch_after)")
assert.Contains(t, normalizedSQL, "AS item(question_id, question_hash, question_text, ai_platform_id, dispatch_after)")
assert.Contains(t, normalizedSQL, "question_text_snapshot = EXCLUDED.question_text_snapshot")
require.Len(t, tx.execArgs, 11)
assert.Equal(t, []string{"幽灵门轨道长度定制", "幽灵门轨道长度定制"}, tx.execArgs[8])
assert.Equal(t, []string{"qwen", "doubao"}, tx.execArgs[9])
assert.Contains(t, normalizedSQL, "WHERE monitoring_collect_tasks.status = 'pending'")
assert.Contains(t, normalizedSQL, "t.dispatch_after IS NULL OR t.dispatch_after <= NOW()")
assert.Contains(t, normalizedSQL, "SET last_dispatched_at = NOW()")
require.Len(t, tx.queryArgs, 12)
assert.Equal(t, []string{"幽灵门轨道长度定制", "幽灵门轨道长度定制"}, tx.queryArgs[8])
assert.Equal(t, []string{"qwen", "doubao"}, tx.queryArgs[9])
}
func TestEnsureDailyMonitoringCollectTasksRejectsEmptyQuestionText(t *testing.T) {
ctx := context.Background()
tx := &captureDailyMonitorTaskTx{}
created, err := (&MonitoringService{}).ensureDailyMonitoringCollectTasks(
created, specs, err := (&MonitoringService{}).ensureDailyMonitoringCollectTasks(
ctx,
tx,
monitoringDailyPlan{TenantID: 4},
@@ -223,10 +305,26 @@ func TestEnsureDailyMonitoringCollectTasksRejectsEmptyQuestionText(t *testing.T)
require.Error(t, err)
assert.Equal(t, int64(0), created)
assert.Empty(t, specs)
assert.False(t, tx.execCalled)
assert.False(t, tx.queryCalled)
assert.Contains(t, err.Error(), "missing_question_text_snapshot")
}
func TestFilterMonitoringDailyCandidatesByPlatformKeepsOnlyOnlineDispatchablePlatforms(t *testing.T) {
candidates := []monitoringDailyTaskCandidate{
{Question: monitoringConfiguredQuestion{ID: 1}, Platform: monitoringPlatformMetadata{ID: "qwen"}},
{Question: monitoringConfiguredQuestion{ID: 1}, Platform: monitoringPlatformMetadata{ID: "doubao"}},
{Question: monitoringConfiguredQuestion{ID: 2}, Platform: monitoringPlatformMetadata{ID: "kimi"}},
}
filtered := filterMonitoringDailyCandidatesByPlatform(candidates, monitoringDailyPlatformSet([]string{"qwen", "kimi"}))
require.Len(t, filtered, 2)
assert.Equal(t, "qwen", filtered[0].Platform.ID)
assert.Equal(t, "kimi", filtered[1].Platform.ID)
}
func TestDecodeDailyMonitoringEnabledPlatformsReportsMalformedJSON(t *testing.T) {
platforms, err := decodeDailyMonitoringEnabledPlatforms([]byte(`["qwen"`))
require.Error(t, err)
@@ -332,6 +430,7 @@ func TestMonitoringDailyTaskMetricsRecordsSummaryCounters(t *testing.T) {
FailedPlanCount: 1,
FailedBrandCount: 2,
PlannedTaskCount: 20,
EligibleTaskCount: 12,
DueTaskCount: 8,
CreatedTaskCount: 6,
DesktopTaskCount: 4,
@@ -348,6 +447,7 @@ func TestMonitoringDailyTaskMetricsRecordsSummaryCounters(t *testing.T) {
assert.Equal(t, int64(2), snapshot.BrandFailureTotal)
assert.Equal(t, int64(1), snapshot.SkippedPlanTotal)
assert.Equal(t, int64(20), snapshot.PlannedTaskTotal)
assert.Equal(t, int64(12), snapshot.EligibleTaskTotal)
assert.Equal(t, int64(8), snapshot.DueTaskTotal)
assert.Equal(t, int64(6), snapshot.CreatedTaskTotal)
assert.Equal(t, int64(4), snapshot.DesktopTaskTotal)
@@ -361,6 +461,7 @@ func TestMonitoringDailyTaskMetricsRecordsSummaryCounters(t *testing.T) {
prometheus := MonitoringDailyTaskMetricsPrometheus()
assert.Contains(t, prometheus, `monitoring_daily_task_worker_runs_total{status="completed"} 1`)
assert.Contains(t, prometheus, `monitoring_daily_task_generation_failures_total{scope="plan"} 1`)
assert.Contains(t, prometheus, `monitoring_daily_task_generation_tasks_total{stage="eligible"} 12`)
assert.Contains(t, prometheus, `monitoring_daily_task_generation_tasks_total{stage="created"} 6`)
assert.Contains(t, prometheus, `monitoring_daily_task_worker_last_business_date_info{business_date="2026-04-24"} 1`)
}
@@ -506,6 +607,9 @@ type captureDailyMonitorTaskTx struct {
execArgs []any
execSQLs []string
execArgses [][]any
queryCalled bool
querySQL string
queryArgs []any
commandTag pgconn.CommandTag
}
@@ -530,7 +634,10 @@ func (tx *captureDailyMonitorTaskTx) Exec(_ context.Context, sql string, argumen
tx.execArgses = append(tx.execArgses, arguments)
return tx.commandTag, nil
}
func (tx *captureDailyMonitorTaskTx) Query(context.Context, string, ...any) (pgx.Rows, error) {
func (tx *captureDailyMonitorTaskTx) Query(_ context.Context, sql string, arguments ...any) (pgx.Rows, error) {
tx.queryCalled = true
tx.querySQL = sql
tx.queryArgs = arguments
return emptyDailyMonitorRows{}, nil
}
func (tx *captureDailyMonitorTaskTx) QueryRow(context.Context, string, ...any) pgx.Row {
@@ -98,6 +98,7 @@ func (monitoringDailyTaskMetricsCollector) Collect(ch chan<- prometheus.Metric)
monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.BrandFailureTotal), []string{"scope"}, "brand")
monitoringCounter(ch, "monitoring_daily_task_generation_skipped_plans_total", "Total daily monitoring plans skipped before generation.", float64(snapshot.SkippedPlanTotal), nil)
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.PlannedTaskTotal), []string{"stage"}, "planned")
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.EligibleTaskTotal), []string{"stage"}, "eligible")
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DueTaskTotal), []string{"stage"}, "due")
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.CreatedTaskTotal), []string{"stage"}, "created")
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DesktopTaskTotal), []string{"stage"}, "desktop")
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"time"
@@ -65,6 +66,11 @@ type monitorDesktopTaskTargetCandidate struct {
ClientOrder int
}
type monitorDesktopTaskTargetOptions struct {
MaxClientBacklog int
RequireIdleClient bool
}
func monitorDesktopTaskSpecIDs(specs []monitorDesktopTaskSpec) []int64 {
if len(specs) == 0 {
return nil
@@ -203,15 +209,34 @@ func (s *MonitoringService) loadMonitorDesktopTaskSpecs(
func (s *MonitoringService) dispatchMonitorDesktopTasks(
ctx context.Context,
specs []monitorDesktopTaskSpec,
maxClientBacklog ...int,
) ([]*repository.DesktopTask, []int64, error) {
if s == nil || s.businessPool == nil || len(specs) == 0 {
return nil, nil, nil
}
backlogLimit := 0
if len(maxClientBacklog) > 0 && maxClientBacklog[0] > 0 {
backlogLimit = maxClientBacklog[0]
}
var targets map[string]monitorDesktopTaskTarget
if monitorDesktopTaskSpecsNeedTargetLookup(specs) {
var err error
targets, err = s.loadMonitorDesktopTaskTargets(ctx, specs[0].TenantID, specs[0].WorkspaceID, specs[0].RequestedByUserID, specs)
targets, err = s.loadMonitorDesktopTaskTargets(ctx, specs[0].TenantID, specs[0].WorkspaceID, specs[0].RequestedByUserID, specs, backlogLimit)
if err != nil {
return nil, nil, err
}
}
clientBacklog := make(map[uuid.UUID]int)
if backlogLimit > 0 {
var err error
clientBacklog, err = s.loadMonitorDesktopClientBacklog(
ctx,
specs[0].TenantID,
specs[0].WorkspaceID,
monitorDesktopTaskDispatchClientIDs(specs, targets),
backlogLimit,
)
if err != nil {
return nil, nil, err
}
@@ -237,11 +262,22 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
spec.TargetClientID = target.ClientID
}
task, shouldPublish, fallbackLegacy, upsertErr := s.upsertMonitorDesktopTask(ctx, tx, repo, spec)
task, shouldPublish, shouldDefer, upsertErr := s.upsertMonitorDesktopTask(
ctx,
tx,
repo,
spec,
func(oldClientID, newClientID uuid.UUID) bool {
return reserveMonitorDesktopClientBacklog(oldClientID, newClientID, clientBacklog, backlogLimit)
},
func(oldClientID, newClientID uuid.UUID) (bool, error) {
return s.reserveMonitorDesktopClientBacklogTx(ctx, tx, spec.TenantID, spec.WorkspaceID, oldClientID, newClientID, backlogLimit)
},
)
if upsertErr != nil {
return nil, nil, upsertErr
}
if fallbackLegacy {
if shouldDefer {
deferredTaskIDs = append(deferredTaskIDs, spec.MonitorTaskID)
continue
}
@@ -262,6 +298,26 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
return publishableTasks, deferredTaskIDs, nil
}
func monitorDesktopTaskDispatchClientIDs(specs []monitorDesktopTaskSpec, targets map[string]monitorDesktopTaskTarget) []uuid.UUID {
if len(specs) == 0 {
return nil
}
result := make([]uuid.UUID, 0, len(specs)*2)
for _, spec := range specs {
if spec.TargetClientID != uuid.Nil {
result = append(result, spec.TargetClientID)
}
if targets == nil {
continue
}
target, ok := targets[normalizeMonitoringPlatformID(spec.PlatformID)]
if ok && target.ClientID != uuid.Nil {
result = append(result, target.ClientID)
}
}
return uniqueUUIDs(result)
}
func monitorDesktopTaskSpecsNeedTargetLookup(specs []monitorDesktopTaskSpec) bool {
for _, spec := range specs {
if spec.TargetAccountID == uuid.Nil || spec.TargetClientID == uuid.Nil {
@@ -279,6 +335,34 @@ func (s *MonitoringService) loadMonitorDesktopTaskTargets(
ctx context.Context,
tenantID, workspaceID, userID int64,
specs []monitorDesktopTaskSpec,
maxClientBacklog ...int,
) (map[string]monitorDesktopTaskTarget, error) {
backlogLimit := 0
if len(maxClientBacklog) > 0 && maxClientBacklog[0] > 0 {
backlogLimit = maxClientBacklog[0]
}
return s.loadMonitorDesktopTaskTargetsWithOptions(ctx, tenantID, workspaceID, userID, specs, monitorDesktopTaskTargetOptions{
MaxClientBacklog: backlogLimit,
})
}
func (s *MonitoringService) loadIdleMonitorDesktopTaskTargets(
ctx context.Context,
tenantID, workspaceID, userID int64,
specs []monitorDesktopTaskSpec,
maxClientBacklog int,
) (map[string]monitorDesktopTaskTarget, error) {
return s.loadMonitorDesktopTaskTargetsWithOptions(ctx, tenantID, workspaceID, userID, specs, monitorDesktopTaskTargetOptions{
MaxClientBacklog: maxClientBacklog,
RequireIdleClient: true,
})
}
func (s *MonitoringService) loadMonitorDesktopTaskTargetsWithOptions(
ctx context.Context,
tenantID, workspaceID, userID int64,
specs []monitorDesktopTaskSpec,
options monitorDesktopTaskTargetOptions,
) (map[string]monitorDesktopTaskTarget, error) {
platformIDs := uniqueMonitorDesktopTaskPlatformIDs(specs)
if len(platformIDs) == 0 {
@@ -373,7 +457,15 @@ func (s *MonitoringService) loadMonitorDesktopTaskTargets(
if err != nil {
return nil, err
}
return selectMonitorDesktopTaskTargets(candidates, accountPresence, onlineClients), nil
backlogLookupLimit := options.MaxClientBacklog
if options.RequireIdleClient && backlogLookupLimit <= 0 {
backlogLookupLimit = 1
}
activeBacklog, err := s.loadMonitorDesktopClientBacklog(ctx, tenantID, workspaceID, clientIDs, backlogLookupLimit)
if err != nil {
return nil, err
}
return selectMonitorDesktopTaskTargetsWithOptions(candidates, accountPresence, onlineClients, activeBacklog, options), nil
}
func uniqueMonitorDesktopTaskPlatformIDs(specs []monitorDesktopTaskSpec) []string {
@@ -463,13 +555,70 @@ func (s *MonitoringService) loadOnlineMonitorDesktopClients(ctx context.Context,
return result, nil
}
func (s *MonitoringService) loadMonitorDesktopClientBacklog(
ctx context.Context,
tenantID, workspaceID int64,
clientIDs []uuid.UUID,
limit int,
) (map[uuid.UUID]int, error) {
result := make(map[uuid.UUID]int)
clientIDs = uniqueUUIDs(clientIDs)
if s == nil || s.businessPool == nil || limit <= 0 || len(clientIDs) == 0 {
return result, nil
}
rows, err := s.businessPool.Query(ctx, `
SELECT target_client_id, COUNT(*)::int
FROM desktop_tasks
WHERE tenant_id = $1
AND workspace_id = $2
AND target_client_id = ANY($3::uuid[])
AND kind = 'monitor'
AND status IN ('queued', 'in_progress')
GROUP BY target_client_id
`, tenantID, workspaceID, clientIDs)
if err != nil {
return nil, response.ErrInternal(50117, "desktop_client_backlog_lookup_failed", "failed to resolve monitor desktop client backlog")
}
defer rows.Close()
for rows.Next() {
var (
clientID uuid.UUID
count int
)
if scanErr := rows.Scan(&clientID, &count); scanErr != nil {
return nil, response.ErrInternal(50117, "desktop_client_backlog_lookup_failed", "failed to parse monitor desktop client backlog")
}
result[clientID] = count
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50117, "desktop_client_backlog_lookup_failed", "failed to iterate monitor desktop client backlog")
}
return result, nil
}
func selectMonitorDesktopTaskTargets(
candidates []monitorDesktopAccountCandidate,
accountPresence map[uuid.UUID][]desktopAccountPresenceClient,
onlineClients map[uuid.UUID]bool,
clientBacklog map[uuid.UUID]int,
maxClientBacklog int,
) map[string]monitorDesktopTaskTarget {
return selectMonitorDesktopTaskTargetsWithOptions(candidates, accountPresence, onlineClients, clientBacklog, monitorDesktopTaskTargetOptions{
MaxClientBacklog: maxClientBacklog,
})
}
func selectMonitorDesktopTaskTargetsWithOptions(
candidates []monitorDesktopAccountCandidate,
accountPresence map[uuid.UUID][]desktopAccountPresenceClient,
onlineClients map[uuid.UUID]bool,
clientBacklog map[uuid.UUID]int,
options monitorDesktopTaskTargetOptions,
) map[string]monitorDesktopTaskTarget {
result := make(map[string]monitorDesktopTaskTarget)
bestByPlatform := make(map[string]monitorDesktopTaskTargetCandidate)
candidatesByPlatform := make(map[string][]monitorDesktopTaskTargetCandidate)
for _, candidate := range candidates {
platformID := normalizeMonitoringPlatformID(candidate.PlatformID)
if platformID == "" || candidate.AccountID == uuid.Nil {
@@ -495,9 +644,7 @@ func selectMonitorDesktopTaskTargets(
RecordOrder: candidate.RecordOrder,
ClientOrder: index,
}
if current, ok := bestByPlatform[platformID]; !ok || item.betterThan(current) {
bestByPlatform[platformID] = item
}
candidatesByPlatform[platformID] = append(candidatesByPlatform[platformID], item)
}
}
if candidate.DBClientID == nil || *candidate.DBClientID == uuid.Nil || !onlineClients[*candidate.DBClientID] {
@@ -516,16 +663,93 @@ func selectMonitorDesktopTaskTargets(
SourceRank: 1,
RecordOrder: candidate.RecordOrder,
}
if current, ok := bestByPlatform[platformID]; !ok || item.betterThan(current) {
bestByPlatform[platformID] = item
candidatesByPlatform[platformID] = append(candidatesByPlatform[platformID], item)
}
platformIDs := make([]string, 0, len(candidatesByPlatform))
for platformID := range candidatesByPlatform {
platformIDs = append(platformIDs, platformID)
}
sort.Strings(platformIDs)
for _, platformID := range platformIDs {
platformCandidates := candidatesByPlatform[platformID]
sort.SliceStable(platformCandidates, func(i, j int) bool {
return platformCandidates[i].betterThan(platformCandidates[j])
})
for _, candidate := range platformCandidates {
if monitorDesktopClientBacklogUnavailable(candidate.Target.ClientID, clientBacklog, options) {
continue
}
for platformID, candidate := range bestByPlatform {
result[platformID] = candidate.Target
if options.MaxClientBacklog > 0 {
clientBacklog[candidate.Target.ClientID]++
}
break
}
}
return result
}
func monitorDesktopClientBacklogUnavailable(clientID uuid.UUID, clientBacklog map[uuid.UUID]int, options monitorDesktopTaskTargetOptions) bool {
if clientID == uuid.Nil {
return false
}
if options.RequireIdleClient && clientBacklog[clientID] > 0 {
return true
}
return monitorDesktopClientBacklogFull(clientID, clientBacklog, options.MaxClientBacklog)
}
func monitorDesktopClientBacklogFull(clientID uuid.UUID, clientBacklog map[uuid.UUID]int, maxClientBacklog int) bool {
return maxClientBacklog > 0 && clientID != uuid.Nil && clientBacklog[clientID] >= maxClientBacklog
}
func reserveMonitorDesktopClientBacklog(oldClientID, newClientID uuid.UUID, clientBacklog map[uuid.UUID]int, maxClientBacklog int) bool {
if maxClientBacklog <= 0 || newClientID == uuid.Nil || oldClientID == newClientID {
return true
}
if clientBacklog == nil {
return true
}
if monitorDesktopClientBacklogFull(newClientID, clientBacklog, maxClientBacklog) {
return false
}
clientBacklog[newClientID]++
if oldClientID != uuid.Nil && clientBacklog[oldClientID] > 0 {
clientBacklog[oldClientID]--
}
return true
}
func (s *MonitoringService) reserveMonitorDesktopClientBacklogTx(
ctx context.Context,
tx pgx.Tx,
tenantID, workspaceID int64,
oldClientID, newClientID uuid.UUID,
maxClientBacklog int,
) (bool, error) {
if tx == nil || maxClientBacklog <= 0 || newClientID == uuid.Nil || oldClientID == newClientID {
return true, nil
}
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, monitorDesktopTaskClientSlotLockKey(newClientID)); err != nil {
return false, response.ErrInternal(50123, "desktop_client_backlog_lock_failed", "failed to lock monitor desktop client backlog")
}
var backlog int
if err := tx.QueryRow(ctx, `
SELECT COUNT(*)::int
FROM desktop_tasks
WHERE tenant_id = $1
AND workspace_id = $2
AND target_client_id = $3
AND kind = 'monitor'
AND status IN ('queued', 'in_progress')
`, tenantID, workspaceID, newClientID).Scan(&backlog); err != nil {
return false, response.ErrInternal(50124, "desktop_client_backlog_check_failed", "failed to check monitor desktop client backlog")
}
return backlog < maxClientBacklog, nil
}
func (candidate monitorDesktopTaskTargetCandidate) betterThan(current monitorDesktopTaskTargetCandidate) bool {
if candidate.HealthRank != current.HealthRank {
return candidate.HealthRank < current.HealthRank
@@ -559,14 +783,17 @@ func (s *MonitoringService) upsertMonitorDesktopTask(
tx pgx.Tx,
repo repository.DesktopTaskRepository,
spec monitorDesktopTaskSpec,
reserveClientBacklog func(oldClientID, newClientID uuid.UUID) bool,
reserveClientBacklogTx func(oldClientID, newClientID uuid.UUID) (bool, error),
) (*repository.DesktopTask, bool, bool, error) {
var (
existingDesktopID uuid.UUID
existingTargetClientID uuid.UUID
existingStatus string
existingLeaseExpiresAt pgtype.Timestamptz
existingActiveAttempt pgtype.UUID
)
err := tx.QueryRow(ctx, upsertMonitorDesktopTaskActiveLookupSQL(), spec.MonitorTaskID).Scan(&existingDesktopID, &existingStatus, &existingLeaseExpiresAt, &existingActiveAttempt)
err := tx.QueryRow(ctx, upsertMonitorDesktopTaskActiveLookupSQL(), spec.MonitorTaskID).Scan(&existingDesktopID, &existingTargetClientID, &existingStatus, &existingLeaseExpiresAt, &existingActiveAttempt)
switch err {
case nil:
task, getErr := repo.GetByDesktopID(ctx, existingDesktopID, spec.WorkspaceID)
@@ -583,6 +810,18 @@ func (s *MonitoringService) upsertMonitorDesktopTask(
if targetAccountID == uuid.Nil {
targetAccountID = task.TargetAccountID
}
if reserveClientBacklog != nil && !reserveClientBacklog(existingTargetClientID, spec.TargetClientID) {
return nil, false, true, nil
}
if reserveClientBacklogTx != nil {
allowed, reserveErr := reserveClientBacklogTx(existingTargetClientID, spec.TargetClientID)
if reserveErr != nil {
return nil, false, false, reserveErr
}
if !allowed {
return nil, false, true, nil
}
}
payloadJSON, payloadErr := marshalMonitorDesktopTaskPayload(spec)
if payloadErr != nil {
return nil, false, false, payloadErr
@@ -658,6 +897,18 @@ func (s *MonitoringService) upsertMonitorDesktopTask(
if targetAccountID == uuid.Nil || spec.TargetClientID == uuid.Nil {
return nil, false, true, nil
}
if reserveClientBacklog != nil && !reserveClientBacklog(uuid.Nil, spec.TargetClientID) {
return nil, false, true, nil
}
if reserveClientBacklogTx != nil {
allowed, reserveErr := reserveClientBacklogTx(uuid.Nil, spec.TargetClientID)
if reserveErr != nil {
return nil, false, false, reserveErr
}
if !allowed {
return nil, false, true, nil
}
}
payloadJSON, payloadErr := marshalMonitorDesktopTaskPayload(spec)
if payloadErr != nil {
return nil, false, false, payloadErr
@@ -718,7 +969,7 @@ func (s *MonitoringService) upsertMonitorDesktopTask(
func upsertMonitorDesktopTaskActiveLookupSQL() string {
return `
SELECT desktop_id, status, lease_expires_at, active_attempt_id
SELECT desktop_id, target_client_id, status, lease_expires_at, active_attempt_id
FROM desktop_tasks
WHERE kind = 'monitor'
AND monitor_task_id = $1
@@ -32,6 +32,8 @@ func TestSelectMonitorDesktopTaskTargetsPrefersPresentAccountClient(t *testing.T
map[uuid.UUID]bool{
liveClientID: true,
},
nil,
0,
)
assert.Equal(t, monitorDesktopTaskTarget{
@@ -56,6 +58,8 @@ func TestSelectMonitorDesktopTaskTargetsFallsBackToOnlineDBClient(t *testing.T)
map[uuid.UUID]bool{
clientID: true,
},
nil,
0,
)
assert.Equal(t, monitorDesktopTaskTarget{
@@ -78,6 +82,8 @@ func TestSelectMonitorDesktopTaskTargetsSkipsOfflineClients(t *testing.T) {
},
nil,
map[uuid.UUID]bool{},
nil,
0,
)
assert.Empty(t, targets)
@@ -111,6 +117,8 @@ func TestSelectMonitorDesktopTaskTargetsPrefersHealthierOnlineCandidate(t *testi
riskClientID: true,
liveClientID: true,
},
nil,
0,
)
assert.Equal(t, monitorDesktopTaskTarget{
@@ -144,6 +152,8 @@ func TestSelectMonitorDesktopTaskTargetsPrefersEarlierOnlineClientForSameAccount
earlierClientID: true,
laterClientID: true,
},
nil,
0,
)
assert.Equal(t, monitorDesktopTaskTarget{
@@ -152,6 +162,147 @@ func TestSelectMonitorDesktopTaskTargetsPrefersEarlierOnlineClientForSameAccount
}, targets["doubao"])
}
func TestSelectMonitorDesktopTaskTargetsAppliesClientBackpressure(t *testing.T) {
fullClientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000701")
availableClientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000702")
fullAccountID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000801")
availableAccountID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000802")
targets := selectMonitorDesktopTaskTargets(
[]monitorDesktopAccountCandidate{
{
PlatformID: "qwen",
AccountID: fullAccountID,
DBClientID: &fullClientID,
HealthRank: 0,
RecordOrder: 0,
},
{
PlatformID: "qwen",
AccountID: availableAccountID,
DBClientID: &availableClientID,
HealthRank: 1,
RecordOrder: 1,
},
},
nil,
map[uuid.UUID]bool{
fullClientID: true,
availableClientID: true,
},
map[uuid.UUID]int{
fullClientID: 12,
},
12,
)
assert.Equal(t, monitorDesktopTaskTarget{
AccountID: availableAccountID,
ClientID: availableClientID,
}, targets["qwen"])
}
func TestSelectMonitorDesktopTaskTargetsConsumesBackpressureWithinRun(t *testing.T) {
clientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000703")
qwenAccountID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000803")
doubaoAccountID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000804")
targets := selectMonitorDesktopTaskTargets(
[]monitorDesktopAccountCandidate{
{
PlatformID: "doubao",
AccountID: doubaoAccountID,
DBClientID: &clientID,
HealthRank: 0,
RecordOrder: 0,
},
{
PlatformID: "qwen",
AccountID: qwenAccountID,
DBClientID: &clientID,
HealthRank: 0,
RecordOrder: 1,
},
},
nil,
map[uuid.UUID]bool{
clientID: true,
},
map[uuid.UUID]int{
clientID: 11,
},
12,
)
assert.Len(t, targets, 1)
assert.Contains(t, targets, "doubao")
assert.NotContains(t, targets, "qwen")
}
func TestSelectMonitorDesktopTaskTargetsRequireIdleSkipsAnyActiveBacklog(t *testing.T) {
clientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000708")
accountID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000808")
targets := selectMonitorDesktopTaskTargetsWithOptions(
[]monitorDesktopAccountCandidate{
{
PlatformID: "qwen",
AccountID: accountID,
DBClientID: &clientID,
},
},
nil,
map[uuid.UUID]bool{
clientID: true,
},
map[uuid.UUID]int{
clientID: 1,
},
monitorDesktopTaskTargetOptions{
MaxClientBacklog: 12,
RequireIdleClient: true,
},
)
assert.Empty(t, targets)
}
func TestReserveMonitorDesktopClientBacklogAllowsExistingSameClientWithoutConsumingSlot(t *testing.T) {
clientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000704")
backlog := map[uuid.UUID]int{
clientID: 12,
}
ok := reserveMonitorDesktopClientBacklog(clientID, clientID, backlog, 12)
assert.True(t, ok)
assert.Equal(t, 12, backlog[clientID])
}
func TestReserveMonitorDesktopClientBacklogConsumesCapacityPerTask(t *testing.T) {
clientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000705")
backlog := map[uuid.UUID]int{
clientID: 11,
}
assert.True(t, reserveMonitorDesktopClientBacklog(uuid.Nil, clientID, backlog, 12))
assert.False(t, reserveMonitorDesktopClientBacklog(uuid.Nil, clientID, backlog, 12))
assert.Equal(t, 12, backlog[clientID])
}
func TestReserveMonitorDesktopClientBacklogMovesCapacityWhenRetargeting(t *testing.T) {
oldClientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000706")
newClientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000707")
backlog := map[uuid.UUID]int{
oldClientID: 4,
newClientID: 11,
}
assert.True(t, reserveMonitorDesktopClientBacklog(oldClientID, newClientID, backlog, 12))
assert.Equal(t, 3, backlog[oldClientID])
assert.Equal(t, 12, backlog[newClientID])
}
func TestMarshalMonitorDesktopTaskPayloadRequiresQuestionText(t *testing.T) {
payload, err := marshalMonitorDesktopTaskPayload(monitorDesktopTaskSpec{
PlatformID: "qwen",
@@ -93,7 +93,7 @@ INSERT INTO ops.scheduler_jobs
(job_key, display_name, category, description, enabled, schedule_type, interval_seconds, timezone, timeout_seconds, batch_size, config)
VALUES
('schedule_dispatch', '内容定时任务派发', 'content', '索引驱动的延迟队列派发,仅认领到期内容任务并投递生成队列。', true, 'interval', 15, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
('monitoring_daily_task', 'AI 监控每日任务生成', 'monitoring', '按业务日小批量物化监控采集任务并派发到桌面任务队列', true, 'interval', 300, 'Asia/Shanghai', 45, 64, '{"max_materialize_per_plan":12,"max_materialize_per_brand":4}'::jsonb),
('monitoring_daily_task', 'AI 监控每日任务生成', 'monitoring', '客户端在线时按业务日滚动创建并派发监控采集任务,离线不生成待采积压', true, 'interval', 300, 'Asia/Shanghai', 45, 12, '{"max_desktop_client_backlog":12}'::jsonb),
('monitoring_result_recovery', '监控结果恢复', 'monitoring', '按回调结果队列索引恢复未成功入队的监控结果。', true, 'interval', 60, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
('monitoring_lease_recovery', '监控租约回收', 'monitoring', '按过期租约索引限批回收超过租约时间仍未回调的监控任务。', true, 'interval', 1800, 'Asia/Shanghai', 30, 1000, '{}'::jsonb),
('monitoring_received_inspection', '监控 received 卡死检查', 'monitoring', '按 received watchdog 索引限批标记长时间未入队的监控任务。', true, 'interval', 300, 'Asia/Shanghai', 30, 100, '{"threshold":"15m"}'::jsonb),
@@ -11,10 +11,11 @@ SET description = '索引驱动的延迟队列派发,仅认领到期内容任
WHERE job_key = 'schedule_dispatch';
UPDATE ops.scheduler_jobs
SET description = '按业务日小批量物化监控采集任务并派发到桌面任务队列',
SET description = '客户端在线时按业务日滚动创建并派发监控采集任务,离线不生成待采积压',
timeout_seconds = 45,
batch_size = 64,
config = config || '{"max_materialize_per_plan":12,"max_materialize_per_brand":4}'::jsonb,
batch_size = 12,
config = (config - 'max_materialize_per_run' - 'max_materialize_per_plan' - 'max_materialize_per_brand')
|| '{"max_desktop_client_backlog":12}'::jsonb,
updated_at = NOW()
WHERE job_key = 'monitoring_daily_task';
@@ -0,0 +1,7 @@
UPDATE ops.scheduler_jobs
SET description = '按业务日小批量物化监控采集任务并派发到桌面任务队列。',
batch_size = 64,
config = (config - 'max_desktop_client_backlog')
|| '{"max_materialize_per_plan":12,"max_materialize_per_brand":4}'::jsonb,
updated_at = NOW()
WHERE job_key = 'monitoring_daily_task';
@@ -0,0 +1,7 @@
UPDATE ops.scheduler_jobs
SET description = '客户端在线时按业务日滚动创建并派发监控采集任务,离线不生成待采积压。',
batch_size = 12,
config = (config - 'max_materialize_per_run' - 'max_materialize_per_plan' - 'max_materialize_per_brand')
|| '{"max_desktop_client_backlog":12}'::jsonb,
updated_at = NOW()
WHERE job_key = 'monitoring_daily_task';
@@ -0,0 +1,6 @@
UPDATE ops.scheduler_jobs
SET description = '按业务日全量创建监控采集事实,按容量限速派发到桌面任务队列。',
batch_size = 12,
config = config || '{"max_desktop_client_backlog":12}'::jsonb,
updated_at = NOW()
WHERE job_key = 'monitoring_daily_task';
@@ -0,0 +1,7 @@
UPDATE ops.scheduler_jobs
SET description = '客户端在线时按业务日滚动创建并派发监控采集任务,离线不生成待采积压。',
batch_size = 12,
config = (config - 'max_materialize_per_run' - 'max_materialize_per_plan' - 'max_materialize_per_brand')
|| '{"max_desktop_client_backlog":12}'::jsonb,
updated_at = NOW()
WHERE job_key = 'monitoring_daily_task';
@@ -0,0 +1,7 @@
UPDATE ops.scheduler_jobs
SET description = '客户端在线时按业务日滚动创建并派发监控采集任务,离线不生成待采积压。',
batch_size = 12,
config = (config - 'max_desktop_dispatch_per_run' - 'max_materialize_per_run' - 'max_materialize_per_plan' - 'max_materialize_per_brand')
|| '{"max_desktop_client_backlog":12}'::jsonb,
updated_at = NOW()
WHERE job_key = 'monitoring_daily_task';
@@ -0,0 +1,7 @@
UPDATE ops.scheduler_jobs
SET description = '客户端在线时按业务日补齐当天监控采集任务,离线不生成待采积压;不设置全局派发上限,客户端侧由 backlog 控制。',
batch_size = NULL,
config = (config - 'batch_size' - 'max_desktop_dispatch_per_run' - 'max_materialize_per_run' - 'max_materialize_per_plan' - 'max_materialize_per_brand')
|| '{"max_desktop_client_backlog":12}'::jsonb,
updated_at = NOW()
WHERE job_key = 'monitoring_daily_task';