fix monitoring online dispatch retries

This commit is contained in:
2026-06-27 16:17:02 +08:00
parent 940ac23493
commit beb290a5a7
11 changed files with 555 additions and 153 deletions
@@ -40,7 +40,9 @@ const (
// reports skip writes and only refresh the row at this interval so dashboards // reports skip writes and only refresh the row at this interval so dashboards
// do not depend on per-heartbeat database churn. // do not depend on per-heartbeat database churn.
monitoringHeartbeatAccessSnapshotRefreshAfter = 10 * time.Minute monitoringHeartbeatAccessSnapshotRefreshAfter = 10 * time.Minute
monitoringRuntimeUnknownMaxDispatchAttempts = 3 monitoringRuntimeUnknownMaxDispatchAttempts = 2
monitoringAutomaticFailureMaxAttempts = 2
monitoringAutomaticFailureRetryAfter = 2 * time.Minute
) )
var monitoringCitationMarkerPattern = regexp.MustCompile(`(?i)\[\s*citation\s*:\s*\d+\s*\]`) var monitoringCitationMarkerPattern = regexp.MustCompile(`(?i)\[\s*citation\s*:\s*\d+\s*\]`)
@@ -240,27 +242,28 @@ type monitoringHeartbeatSnapshotDecision struct {
} }
type monitoringCollectTask struct { type monitoringCollectTask struct {
ID int64 ID int64
TenantID int64 TenantID int64
BrandID int64 BrandID int64
QuestionID int64 QuestionID int64
QuestionHash []byte QuestionHash []byte
AIPlatformID string AIPlatformID string
CollectorType string CollectorType string
RunMode string RunMode string
BusinessDate time.Time BusinessDate time.Time
Status string Status string
ExecutionOwner string ExecutionOwner string
LeaseTokenHash sql.NullString LeaseTokenHash sql.NullString
LeasedToExecutor sql.NullString LeasedToExecutor sql.NullString
LeaseExpiresAt sql.NullTime LeaseExpiresAt sql.NullTime
CallbackReceivedAt sql.NullTime CallbackReceivedAt sql.NullTime
CompletedAt sql.NullTime CompletedAt sql.NullTime
SkipReason sql.NullString SkipReason sql.NullString
TriggerSource string TriggerSource string
QuestionText string QuestionText string
TargetClientID sql.NullString TargetClientID sql.NullString
DispatchAttempts int DispatchAttempts int
AutomaticFailureCount int
} }
type monitoringDesktopExecutionLease struct { type monitoringDesktopExecutionLease struct {
@@ -1062,6 +1065,32 @@ type monitoringSkipOutcome struct {
RetryAfter time.Duration RetryAfter time.Duration
} }
type monitoringFailedResultOutcome struct {
TaskStatus string
Requeue bool
RetryAfter time.Duration
}
func monitoringFailedResultOutcomeForTask(task *monitoringCollectTask, req MonitoringTaskResultRequest) monitoringFailedResultOutcome {
if task == nil {
return monitoringFailedResultOutcome{TaskStatus: "failed"}
}
if !strings.EqualFold(strings.TrimSpace(task.TriggerSource), "automatic") {
return monitoringFailedResultOutcome{TaskStatus: "failed"}
}
if monitoringTaskFailureDispositionFromRequest(req).NonRetryable {
return monitoringFailedResultOutcome{TaskStatus: "failed"}
}
if task.AutomaticFailureCount < monitoringAutomaticFailureMaxAttempts-1 {
return monitoringFailedResultOutcome{
TaskStatus: "pending",
Requeue: true,
RetryAfter: monitoringAutomaticFailureRetryAfter,
}
}
return monitoringFailedResultOutcome{TaskStatus: "failed"}
}
func monitoringSkipOutcomeForReason(skipReason string, dispatchAttempts int, triggerSource string) monitoringSkipOutcome { func monitoringSkipOutcomeForReason(skipReason string, dispatchAttempts int, triggerSource string) monitoringSkipOutcome {
switch strings.TrimSpace(skipReason) { switch strings.TrimSpace(skipReason) {
case "runtime_unknown": case "runtime_unknown":
@@ -1562,6 +1591,11 @@ func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantI
trigger_source, trigger_source,
COALESCE(dispatch_attempts, 0) AS dispatch_attempts, COALESCE(dispatch_attempts, 0) AS dispatch_attempts,
COALESCE(target_client_id::text, '') AS target_client_id, COALESCE(target_client_id::text, '') AS target_client_id,
CASE
WHEN COALESCE(t.request_payload_json ->> 'automatic_failure_count', '') ~ '^[0-9]+$'
THEN (t.request_payload_json ->> 'automatic_failure_count')::int
ELSE 0
END AS automatic_failure_count,
COALESCE(NULLIF(t.question_text_snapshot, ''), ( COALESCE(NULLIF(t.question_text_snapshot, ''), (
SELECT question_text_snapshot SELECT question_text_snapshot
FROM monitoring_question_config_snapshots FROM monitoring_question_config_snapshots
@@ -1600,6 +1634,7 @@ func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantI
&item.TriggerSource, &item.TriggerSource,
&item.DispatchAttempts, &item.DispatchAttempts,
&item.TargetClientID, &item.TargetClientID,
&item.AutomaticFailureCount,
&item.QuestionText, &item.QuestionText,
); err != nil { ); err != nil {
if err == pgx.ErrNoRows { if err == pgx.ErrNoRows {
@@ -1635,6 +1670,11 @@ func (s *MonitoringCallbackService) loadCollectTaskByID(ctx context.Context, tas
trigger_source, trigger_source,
COALESCE(dispatch_attempts, 0) AS dispatch_attempts, COALESCE(dispatch_attempts, 0) AS dispatch_attempts,
COALESCE(target_client_id::text, '') AS target_client_id, COALESCE(target_client_id::text, '') AS target_client_id,
CASE
WHEN COALESCE(t.request_payload_json ->> 'automatic_failure_count', '') ~ '^[0-9]+$'
THEN (t.request_payload_json ->> 'automatic_failure_count')::int
ELSE 0
END AS automatic_failure_count,
COALESCE(NULLIF(t.question_text_snapshot, ''), ( COALESCE(NULLIF(t.question_text_snapshot, ''), (
SELECT question_text_snapshot SELECT question_text_snapshot
FROM monitoring_question_config_snapshots FROM monitoring_question_config_snapshots
@@ -1672,6 +1712,7 @@ func (s *MonitoringCallbackService) loadCollectTaskByID(ctx context.Context, tas
&item.TriggerSource, &item.TriggerSource,
&item.DispatchAttempts, &item.DispatchAttempts,
&item.TargetClientID, &item.TargetClientID,
&item.AutomaticFailureCount,
&item.QuestionText, &item.QuestionText,
); err != nil { ); err != nil {
if err == pgx.ErrNoRows { if err == pgx.ErrNoRows {
@@ -2312,12 +2353,20 @@ func (s *MonitoringCallbackService) processTaskResult(ctx context.Context, task
} }
defer tx.Rollback(ctx) defer tx.Rollback(ctx)
runID, err := s.upsertRun(ctx, tx, task, req, runStatus, completedAt, rawPayload) var failedOutcome monitoringFailedResultOutcome
if err != nil { if runStatus == "failed" {
return nil, err failedOutcome = monitoringFailedResultOutcomeForTask(task, req)
} }
if runStatus == "succeeded" { var runID int64
if !failedOutcome.Requeue {
runID, err = s.upsertRun(ctx, tx, task, req, runStatus, completedAt, rawPayload)
if err != nil {
return nil, err
}
}
if runStatus == "succeeded" && !failedOutcome.Requeue {
if err := s.upsertParseResult(ctx, tx, task, runID, parseReq, parseVersion); err != nil { if err := s.upsertParseResult(ctx, tx, task, runID, parseReq, parseVersion); err != nil {
return nil, err return nil, err
} }
@@ -2325,10 +2374,12 @@ func (s *MonitoringCallbackService) processTaskResult(ctx context.Context, task
citationSourceCount := 0 citationSourceCount := 0
contentCitationCount := 0 contentCitationCount := 0
if err := s.replaceCitationFacts(ctx, tx, task.TenantID, runID); err != nil { if !failedOutcome.Requeue {
return nil, err if err := s.replaceCitationFacts(ctx, tx, task.TenantID, runID); err != nil {
return nil, err
}
} }
if runStatus == "succeeded" { if runStatus == "succeeded" && !failedOutcome.Requeue {
facts, buildErr := s.buildCitationFacts(ctx, tx, task.TenantID, task.AIPlatformID, sourceInputs) facts, buildErr := s.buildCitationFacts(ctx, tx, task.TenantID, task.AIPlatformID, sourceInputs)
if buildErr != nil { if buildErr != nil {
return nil, buildErr return nil, buildErr
@@ -2346,12 +2397,16 @@ func (s *MonitoringCallbackService) processTaskResult(ctx context.Context, task
taskStatus := "completed" taskStatus := "completed"
if runStatus == "failed" { if runStatus == "failed" {
taskStatus = "failed" taskStatus = failedOutcome.TaskStatus
} }
if err := s.updateCollectTaskStatus(ctx, tx, task.ID, taskStatus, completedAt, req.ErrorMessage); err != nil { if failedOutcome.Requeue {
if err := s.requeueCollectTaskAfterFailedResult(ctx, tx, task.ID, failedOutcome.RetryAfter, req.ErrorMessage); err != nil {
return nil, err
}
} else if err := s.updateCollectTaskStatus(ctx, tx, task.ID, taskStatus, completedAt, req.ErrorMessage); err != nil {
return nil, err return nil, err
} }
if runStatus == "failed" { if runStatus == "failed" && !failedOutcome.Requeue {
if _, err := s.failRemainingMonitoringTasksForAuthorizationFailure(ctx, tx, task, req, completedAt); err != nil { if _, err := s.failRemainingMonitoringTasksForAuthorizationFailure(ctx, tx, task, req, completedAt); err != nil {
return nil, err return nil, err
} }
@@ -2365,9 +2420,13 @@ func (s *MonitoringCallbackService) processTaskResult(ctx context.Context, task
return nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring result ingest") return nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring result ingest")
} }
var runIDPtr *int64
if !failedOutcome.Requeue {
runIDPtr = &runID
}
return &MonitoringTaskResultResponse{ return &MonitoringTaskResultResponse{
TaskID: task.ID, TaskID: task.ID,
RunID: &runID, RunID: runIDPtr,
TaskStatus: taskStatus, TaskStatus: taskStatus,
CitationSourceCount: citationSourceCount, CitationSourceCount: citationSourceCount,
ContentCitationCount: contentCitationCount, ContentCitationCount: contentCitationCount,
@@ -3113,6 +3172,51 @@ func (s *MonitoringCallbackService) updateCollectTaskStatus(ctx context.Context,
return nil return nil
} }
func (s *MonitoringCallbackService) requeueCollectTaskAfterFailedResult(ctx context.Context, tx pgx.Tx, taskID int64, retryAfter time.Duration, errorMessage *string) error {
if retryAfter <= 0 {
retryAfter = monitoringAutomaticFailureRetryAfter
}
if _, err := tx.Exec(ctx, `
UPDATE monitoring_collect_tasks
SET status = 'pending',
lease_token_hash = NULL,
leased_to_executor = NULL,
leased_at = NULL,
lease_expires_at = NULL,
callback_received_at = NULL,
completed_at = NULL,
skip_reason = NULL,
last_dispatched_at = NULL,
dispatch_after = NOW() + $2::interval,
error_message = $3,
request_payload_json = (
CASE
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
THEN '{}'::jsonb
ELSE request_payload_json
END
) || jsonb_build_object(
'automatic_failure_count',
(
CASE
WHEN COALESCE(request_payload_json ->> 'automatic_failure_count', '') ~ '^[0-9]+$'
THEN (request_payload_json ->> 'automatic_failure_count')::int
ELSE 0
END
) + 1,
'automatic_failure_retry_after_seconds',
$4::int,
'automatic_failure_last_requeued_at',
to_jsonb(NOW())
),
updated_at = NOW()
WHERE id = $1
`, taskID, formatPgInterval(retryAfter), nullableString(errorMessage), int(retryAfter.Seconds())); err != nil {
return response.ErrInternal(50041, "task_update_failed", "failed to requeue monitoring task after failed result")
}
return nil
}
func (s *MonitoringCallbackService) failRemainingMonitoringTasksForAuthorizationFailure( func (s *MonitoringCallbackService) failRemainingMonitoringTasksForAuthorizationFailure(
ctx context.Context, ctx context.Context,
tx pgx.Tx, tx pgx.Tx,
@@ -499,6 +499,73 @@ func TestMonitoringTaskFailureDispositionDetectsAuthorizationFailureFromMessage(
} }
} }
func TestMonitoringFailedResultOutcomeRequeuesAutomaticFirstFailure(t *testing.T) {
message := "模型调用超时"
got := monitoringFailedResultOutcomeForTask(&monitoringCollectTask{
TriggerSource: "automatic",
AutomaticFailureCount: 0,
}, MonitoringTaskResultRequest{
Status: "failed",
ErrorMessage: &message,
})
if got.TaskStatus != "pending" {
t.Fatalf("TaskStatus = %q, want pending", got.TaskStatus)
}
if !got.Requeue {
t.Fatal("Requeue = false, want true")
}
if got.RetryAfter != monitoringAutomaticFailureRetryAfter {
t.Fatalf("RetryAfter = %v, want %v", got.RetryAfter, monitoringAutomaticFailureRetryAfter)
}
}
func TestMonitoringFailedResultOutcomeFailsAutomaticSecondFailure(t *testing.T) {
got := monitoringFailedResultOutcomeForTask(&monitoringCollectTask{
TriggerSource: "automatic",
AutomaticFailureCount: monitoringAutomaticFailureMaxAttempts - 1,
}, MonitoringTaskResultRequest{Status: "failed"})
if got.TaskStatus != "failed" {
t.Fatalf("TaskStatus = %q, want failed", got.TaskStatus)
}
if got.Requeue {
t.Fatal("Requeue = true, want false")
}
}
func TestMonitoringFailedResultOutcomeDoesNotRetryAuthorizationFailure(t *testing.T) {
message := "登录态已失效,请重新授权后再试。"
got := monitoringFailedResultOutcomeForTask(&monitoringCollectTask{
TriggerSource: "automatic",
AutomaticFailureCount: 0,
}, MonitoringTaskResultRequest{
Status: "failed",
ErrorMessage: &message,
})
if got.TaskStatus != "failed" {
t.Fatalf("TaskStatus = %q, want failed", got.TaskStatus)
}
if got.Requeue {
t.Fatal("Requeue = true, want false")
}
}
func TestMonitoringFailedResultOutcomeDoesNotRetryManualTask(t *testing.T) {
got := monitoringFailedResultOutcomeForTask(&monitoringCollectTask{
TriggerSource: "manual",
AutomaticFailureCount: 0,
}, MonitoringTaskResultRequest{Status: "failed"})
if got.TaskStatus != "failed" {
t.Fatalf("TaskStatus = %q, want failed", got.TaskStatus)
}
if got.Requeue {
t.Fatal("Requeue = true, want false")
}
}
func TestMonitoringAuthorizationFailureTargetClientPrefersExplicitTargetClient(t *testing.T) { func TestMonitoringAuthorizationFailureTargetClientPrefersExplicitTargetClient(t *testing.T) {
targetClientID := uuid.New().String() targetClientID := uuid.New().String()
task := &monitoringCollectTask{ task := &monitoringCollectTask{
@@ -20,6 +20,7 @@ type MonitoringDailyTaskMetricsSnapshot struct {
BrandFailureTotal int64 `json:"brand_failure_total"` BrandFailureTotal int64 `json:"brand_failure_total"`
SkippedPlanTotal int64 `json:"skipped_plan_total"` SkippedPlanTotal int64 `json:"skipped_plan_total"`
PlannedTaskTotal int64 `json:"planned_task_total"` PlannedTaskTotal int64 `json:"planned_task_total"`
EligibleTaskTotal int64 `json:"eligible_task_total"`
DueTaskTotal int64 `json:"due_task_total"` DueTaskTotal int64 `json:"due_task_total"`
CreatedTaskTotal int64 `json:"created_task_total"` CreatedTaskTotal int64 `json:"created_task_total"`
DesktopTaskTotal int64 `json:"desktop_task_total"` DesktopTaskTotal int64 `json:"desktop_task_total"`
@@ -44,6 +45,7 @@ type monitoringDailyTaskMetricsRecorder struct {
brandFailureTotal atomic.Int64 brandFailureTotal atomic.Int64
skippedPlanTotal atomic.Int64 skippedPlanTotal atomic.Int64
plannedTaskTotal atomic.Int64 plannedTaskTotal atomic.Int64
eligibleTaskTotal atomic.Int64
dueTaskTotal atomic.Int64 dueTaskTotal atomic.Int64
createdTaskTotal atomic.Int64 createdTaskTotal atomic.Int64
desktopTaskTotal atomic.Int64 desktopTaskTotal atomic.Int64
@@ -98,6 +100,7 @@ func (r *monitoringDailyTaskMetricsRecorder) record(summary *MonitoringDailyTask
r.brandFailureTotal.Add(int64(summary.FailedBrandCount)) r.brandFailureTotal.Add(int64(summary.FailedBrandCount))
r.skippedPlanTotal.Add(int64(summary.SkippedPlanCount)) r.skippedPlanTotal.Add(int64(summary.SkippedPlanCount))
r.plannedTaskTotal.Add(summary.PlannedTaskCount) r.plannedTaskTotal.Add(summary.PlannedTaskCount)
r.eligibleTaskTotal.Add(summary.EligibleTaskCount)
r.dueTaskTotal.Add(summary.DueTaskCount) r.dueTaskTotal.Add(summary.DueTaskCount)
r.createdTaskTotal.Add(summary.CreatedTaskCount) r.createdTaskTotal.Add(summary.CreatedTaskCount)
r.desktopTaskTotal.Add(summary.DesktopTaskCount) r.desktopTaskTotal.Add(summary.DesktopTaskCount)
@@ -133,6 +136,7 @@ func (r *monitoringDailyTaskMetricsRecorder) snapshot() MonitoringDailyTaskMetri
BrandFailureTotal: r.brandFailureTotal.Load(), BrandFailureTotal: r.brandFailureTotal.Load(),
SkippedPlanTotal: r.skippedPlanTotal.Load(), SkippedPlanTotal: r.skippedPlanTotal.Load(),
PlannedTaskTotal: r.plannedTaskTotal.Load(), PlannedTaskTotal: r.plannedTaskTotal.Load(),
EligibleTaskTotal: r.eligibleTaskTotal.Load(),
DueTaskTotal: r.dueTaskTotal.Load(), DueTaskTotal: r.dueTaskTotal.Load(),
CreatedTaskTotal: r.createdTaskTotal.Load(), CreatedTaskTotal: r.createdTaskTotal.Load(),
DesktopTaskTotal: r.desktopTaskTotal.Load(), DesktopTaskTotal: r.desktopTaskTotal.Load(),
@@ -50,6 +50,7 @@ type MonitoringDailyTaskGenerationSummary struct {
FailedPlanCount int FailedPlanCount int
FailedBrandCount int FailedBrandCount int
PlannedTaskCount int64 PlannedTaskCount int64
EligibleTaskCount int64
DueTaskCount int64 DueTaskCount int64
CreatedTaskCount int64 CreatedTaskCount int64
DesktopTaskCount int64 DesktopTaskCount int64
@@ -238,6 +239,7 @@ func (w *MonitoringDailyTaskWorker) RunOnce(parent context.Context, options ...M
stats["failed_plan_count"] = summary.FailedPlanCount stats["failed_plan_count"] = summary.FailedPlanCount
stats["failed_brand_count"] = summary.FailedBrandCount stats["failed_brand_count"] = summary.FailedBrandCount
stats["planned_task_count"] = summary.PlannedTaskCount stats["planned_task_count"] = summary.PlannedTaskCount
stats["eligible_task_count"] = summary.EligibleTaskCount
stats["due_task_count"] = summary.DueTaskCount stats["due_task_count"] = summary.DueTaskCount
stats["created_task_count"] = summary.CreatedTaskCount stats["created_task_count"] = summary.CreatedTaskCount
stats["desktop_task_count"] = summary.DesktopTaskCount stats["desktop_task_count"] = summary.DesktopTaskCount
@@ -359,7 +361,7 @@ func (s *MonitoringService) GenerateDailyMonitoringTasks(
globalMaterializeRemaining, globalMaterializeRemaining,
) )
dispatchLimit := monitoringDailyDispatchLimit(len(candidates), desktopDispatchRemaining, options.MaxDesktopDispatchPerRun) dispatchLimit := monitoringDailyDispatchLimit(len(candidates), desktopDispatchRemaining, options.MaxDesktopDispatchPerRun)
createdCount, dueCount, desktopCount, deferredCount, err := s.generateDailyMonitoringTasksForBrand( createdCount, eligibleCount, dueCount, desktopCount, deferredCount, err := s.generateDailyMonitoringTasksForBrand(
ctx, ctx,
projectionService, projectionService,
plan, plan,
@@ -393,6 +395,7 @@ func (s *MonitoringService) GenerateDailyMonitoringTasks(
} }
summary.BrandCount++ summary.BrandCount++
summary.PlannedTaskCount += int64(len(candidates)) summary.PlannedTaskCount += int64(len(candidates))
summary.EligibleTaskCount += eligibleCount
summary.DueTaskCount += dueCount summary.DueTaskCount += dueCount
summary.CreatedTaskCount += createdCount summary.CreatedTaskCount += createdCount
summary.DesktopTaskCount += desktopCount summary.DesktopTaskCount += desktopCount
@@ -460,86 +463,101 @@ func (s *MonitoringService) generateDailyMonitoringTasksForBrand(
dispatchLimit int, dispatchLimit int,
executionOwner string, executionOwner string,
maxDesktopClientBacklog int, maxDesktopClientBacklog int,
) (int64, int64, int64, int64, error) { ) (int64, int64, int64, int64, int64, error) {
if materializeLimit <= 0 && dispatchLimit <= 0 { if materializeLimit <= 0 && dispatchLimit <= 0 {
return 0, 0, 0, 0, nil return 0, 0, 0, 0, 0, nil
} }
if executionOwner != "desktop_tasks" { if executionOwner != "desktop_tasks" {
return 0, 0, 0, 0, nil return 0, 0, 0, 0, 0, nil
} }
var createdCount int64 var createdCount int64
var eligibleCount int64
var dueCount int64 var dueCount int64
if materializeLimit > 0 { specs := make([]monitorDesktopTaskSpec, 0)
tx, err := s.monitoringPool.Begin(ctx)
if err != nil {
return 0, 0, 0, 0, response.ErrInternal(50041, "begin_failed", "failed to start monitoring daily task generation")
}
defer tx.Rollback(ctx)
locked, err := acquireMonitoringDailyBrandLock(ctx, tx, plan.TenantID, brand.BrandID, monitoringBusinessDateText(businessDay))
if err != nil {
return 0, 0, 0, 0, err
}
if locked {
existingKeys, err := s.loadDailyMonitoringTaskKeys(ctx, tx, plan.TenantID, brand.BrandID, businessDay)
if err != nil {
return 0, 0, 0, 0, err
}
dueCandidates := selectDueMonitoringDailyCandidates(candidates, existingKeys, now, defaultMonitoringDailyLookahead, materializeLimit)
dueCount = int64(len(dueCandidates))
if len(dueCandidates) > 0 {
if err := s.syncMonitoringQuestionSnapshots(ctx, tx, plan.TenantID, brand.BrandID, questions, true); err != nil {
return 0, 0, 0, 0, err
}
createdCount, err = s.ensureDailyMonitoringCollectTasks(ctx, tx, plan, brand, businessDay, dueCandidates, executionOwner)
if err != nil {
return 0, 0, 0, 0, err
}
if projectionService != nil && createdCount > 0 {
if err := projectionService.rebuildBrandDailyProjection(ctx, tx, plan.TenantID, brand.BrandID, businessDay); err != nil {
return 0, 0, 0, 0, err
}
}
}
}
if err := tx.Commit(ctx); err != nil {
return 0, 0, 0, 0, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring daily task generation")
}
}
if dispatchLimit <= 0 { if dispatchLimit <= 0 {
return createdCount, dueCount, 0, 0, nil return createdCount, eligibleCount, dueCount, 0, 0, nil
} }
dispatchablePlatformIDs, err := s.loadDailyMonitorDispatchablePlatformIDs(ctx, plan.TenantID, plan.WorkspaceID, candidates) dispatchablePlatformIDs, err := s.loadDailyMonitorDispatchablePlatformIDs(ctx, plan.TenantID, plan.WorkspaceID, candidates)
if err != nil { if err != nil {
return createdCount, dueCount, 0, 0, err return createdCount, eligibleCount, dueCount, 0, 0, err
} }
if len(dispatchablePlatformIDs) == 0 { if len(dispatchablePlatformIDs) == 0 {
return createdCount, dueCount, 0, 0, nil return createdCount, eligibleCount, dueCount, 0, 0, nil
} }
specs, err := s.loadDueDailyMonitorDesktopTaskSpecs( dispatchablePlatformSet := monitoringDailyPlatformSet(dispatchablePlatformIDs)
dispatchableCandidates := filterMonitoringDailyCandidatesByPlatform(candidates, dispatchablePlatformSet)
eligibleCount = int64(len(dispatchableCandidates))
retrySpecs, err := s.loadDueDailyMonitorDesktopTaskSpecs(
ctx, ctx,
s.monitoringPool, s.monitoringPool,
plan.TenantID, plan.TenantID,
plan.WorkspaceID, plan.WorkspaceID,
brand.BrandID, brand.BrandID,
businessDay, businessDay,
now.Add(defaultMonitoringDailyLookahead), now,
now.Add(-defaultMonitoringDailyDesktopRetryCooldown), now.Add(-defaultMonitoringDailyDesktopRetryCooldown),
dispatchablePlatformIDs, dispatchablePlatformIDs,
dispatchLimit, dispatchLimit,
) )
if err != nil { if err != nil {
return createdCount, dueCount, 0, 0, err return createdCount, eligibleCount, dueCount, 0, 0, err
}
if len(retrySpecs) > 0 {
specs = append(specs, retrySpecs...)
createdCount += int64(len(retrySpecs))
dueCount += int64(len(retrySpecs))
}
remainingCreateLimit := dispatchLimit - len(specs)
if remainingCreateLimit <= 0 {
return s.dispatchDailyMonitoringDesktopSpecs(ctx, specs, maxDesktopClientBacklog, createdCount, eligibleCount, dueCount)
}
materializeLimit = monitoringDailyBoundedLimit(materializeLimit, dispatchLimit)
if materializeLimit > 0 && len(dispatchableCandidates) > 0 {
var materializeErr error
var newCreatedCount int64
var newDueCount int64
var newSpecs []monitorDesktopTaskSpec
newCreatedCount, newDueCount, newSpecs, materializeErr = s.createDispatchableDailyMonitoringTasksForBrand(
ctx,
projectionService,
plan,
brand,
businessDay,
now,
questions,
dispatchableCandidates,
monitoringDailyBoundedLimit(materializeLimit, remainingCreateLimit),
executionOwner,
)
if materializeErr != nil {
return createdCount, eligibleCount, dueCount, 0, 0, materializeErr
}
createdCount += newCreatedCount
dueCount += newDueCount
specs = append(specs, newSpecs...)
} }
if len(specs) == 0 { if len(specs) == 0 {
return createdCount, dueCount, 0, 0, nil return createdCount, eligibleCount, dueCount, 0, 0, nil
}
return s.dispatchDailyMonitoringDesktopSpecs(ctx, specs, maxDesktopClientBacklog, createdCount, eligibleCount, dueCount)
}
func (s *MonitoringService) dispatchDailyMonitoringDesktopSpecs(
ctx context.Context,
specs []monitorDesktopTaskSpec,
maxDesktopClientBacklog int,
createdCount int64,
eligibleCount int64,
dueCount int64,
) (int64, int64, int64, int64, int64, error) {
if len(specs) == 0 {
return createdCount, eligibleCount, dueCount, 0, 0, nil
} }
publishedTasks, deferredTaskIDs, err := s.dispatchMonitorDesktopTasks(ctx, specs, maxDesktopClientBacklog) publishedTasks, deferredTaskIDs, err := s.dispatchMonitorDesktopTasks(ctx, specs, maxDesktopClientBacklog)
@@ -550,7 +568,7 @@ func (s *MonitoringService) generateDailyMonitoringTasksForBrand(
zap.Error(err), zap.Error(err),
) )
} }
return createdCount, dueCount, 0, 0, err return createdCount, eligibleCount, dueCount, 0, 0, err
} }
for _, task := range publishedTasks { for _, task := range publishedTasks {
@@ -576,7 +594,7 @@ func (s *MonitoringService) generateDailyMonitoringTasksForBrand(
cancel() cancel()
} }
return createdCount, dueCount, int64(len(publishedTasks)), int64(len(deferredTaskIDs)), nil return createdCount, eligibleCount, dueCount, int64(len(publishedTasks)), int64(len(deferredTaskIDs)), nil
} }
func (s *MonitoringService) loadDailyMonitoringPlans(ctx context.Context) ([]monitoringDailyPlan, error) { func (s *MonitoringService) loadDailyMonitoringPlans(ctx context.Context) ([]monitoringDailyPlan, error) {
@@ -901,6 +919,67 @@ func (s *MonitoringService) loadDailyMonitoringTaskKeys(ctx context.Context, q m
return keys, nil return keys, nil
} }
func (s *MonitoringService) createDispatchableDailyMonitoringTasksForBrand(
ctx context.Context,
projectionService *MonitoringCallbackService,
plan monitoringDailyPlan,
brand monitoringDailyBrand,
businessDay time.Time,
now time.Time,
questions []monitoringConfiguredQuestion,
candidates []monitoringDailyTaskCandidate,
limit int,
executionOwner string,
) (int64, int64, []monitorDesktopTaskSpec, error) {
if limit <= 0 || len(candidates) == 0 {
return 0, 0, nil, nil
}
tx, err := s.monitoringPool.Begin(ctx)
if err != nil {
return 0, 0, nil, response.ErrInternal(50041, "begin_failed", "failed to start monitoring daily task generation")
}
defer tx.Rollback(ctx)
locked, err := acquireMonitoringDailyBrandLock(ctx, tx, plan.TenantID, brand.BrandID, monitoringBusinessDateText(businessDay))
if err != nil {
return 0, 0, nil, err
}
if !locked {
return 0, 0, nil, nil
}
existingKeys, err := s.loadDailyMonitoringTaskKeys(ctx, tx, plan.TenantID, brand.BrandID, businessDay)
if err != nil {
return 0, 0, nil, err
}
dueCandidates := selectDueMonitoringDailyCandidates(candidates, existingKeys, now, defaultMonitoringDailyLookahead, limit)
dueCount := int64(len(dueCandidates))
if len(dueCandidates) == 0 {
if err := tx.Commit(ctx); err != nil {
return 0, dueCount, nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring daily task generation")
}
return 0, dueCount, nil, nil
}
if err := s.syncMonitoringQuestionSnapshots(ctx, tx, plan.TenantID, brand.BrandID, questions, true); err != nil {
return 0, dueCount, nil, err
}
createdCount, specs, err := s.ensureDailyMonitoringCollectTasks(ctx, tx, plan, brand, businessDay, dueCandidates, executionOwner)
if err != nil {
return 0, dueCount, nil, err
}
if projectionService != nil && createdCount > 0 {
if err := projectionService.rebuildBrandDailyProjection(ctx, tx, plan.TenantID, brand.BrandID, businessDay); err != nil {
return 0, dueCount, nil, err
}
}
if err := tx.Commit(ctx); err != nil {
return 0, dueCount, nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring daily task generation")
}
return createdCount, dueCount, specs, nil
}
func (s *MonitoringService) loadDailyMonitorDispatchablePlatformIDs( func (s *MonitoringService) loadDailyMonitorDispatchablePlatformIDs(
ctx context.Context, ctx context.Context,
tenantID, workspaceID int64, tenantID, workspaceID int64,
@@ -941,6 +1020,30 @@ func (s *MonitoringService) loadDailyMonitorDispatchablePlatformIDs(
return reconcileEnabledMonitoringPlatforms(platformIDs), nil return reconcileEnabledMonitoringPlatforms(platformIDs), nil
} }
func monitoringDailyPlatformSet(platformIDs []string) map[string]struct{} {
result := make(map[string]struct{}, len(platformIDs))
for _, platformID := range platformIDs {
platformID = normalizeMonitoringPlatformID(platformID)
if platformID != "" {
result[platformID] = struct{}{}
}
}
return result
}
func filterMonitoringDailyCandidatesByPlatform(candidates []monitoringDailyTaskCandidate, platforms map[string]struct{}) []monitoringDailyTaskCandidate {
if len(candidates) == 0 || len(platforms) == 0 {
return nil
}
result := make([]monitoringDailyTaskCandidate, 0, len(candidates))
for _, candidate := range candidates {
if _, ok := platforms[normalizeMonitoringPlatformID(candidate.Platform.ID)]; ok {
result = append(result, candidate)
}
}
return result
}
func (s *MonitoringService) ensureDailyMonitoringCollectTasks( func (s *MonitoringService) ensureDailyMonitoringCollectTasks(
ctx context.Context, ctx context.Context,
tx pgx.Tx, tx pgx.Tx,
@@ -949,12 +1052,12 @@ func (s *MonitoringService) ensureDailyMonitoringCollectTasks(
businessDay time.Time, businessDay time.Time,
candidates []monitoringDailyTaskCandidate, candidates []monitoringDailyTaskCandidate,
executionOwner string, executionOwner string,
) (int64, error) { ) (int64, []monitorDesktopTaskSpec, error) {
if len(candidates) == 0 { if len(candidates) == 0 {
return 0, nil return 0, nil, nil
} }
if strings.TrimSpace(executionOwner) != "desktop_tasks" { if strings.TrimSpace(executionOwner) != "desktop_tasks" {
return 0, response.ErrInternal(50041, "invalid_execution_owner", "daily monitoring tasks require desktop task execution") return 0, nil, response.ErrInternal(50041, "invalid_execution_owner", "daily monitoring tasks require desktop task execution")
} }
businessDate := monitoringBusinessDateText(businessDay) businessDate := monitoringBusinessDateText(businessDay)
@@ -969,7 +1072,7 @@ func (s *MonitoringService) ensureDailyMonitoringCollectTasks(
for _, candidate := range candidates { for _, candidate := range candidates {
questionText, textErr := monitoringQuestionTextSnapshot(candidate.Question) questionText, textErr := monitoringQuestionTextSnapshot(candidate.Question)
if textErr != nil { if textErr != nil {
return 0, textErr return 0, nil, textErr
} }
dispatchAfter := candidate.DispatchAfter dispatchAfter := candidate.DispatchAfter
if dispatchAfter.IsZero() { if dispatchAfter.IsZero() {
@@ -982,47 +1085,90 @@ func (s *MonitoringService) ensureDailyMonitoringCollectTasks(
dispatchAfterValues = append(dispatchAfterValues, dispatchAfter) dispatchAfterValues = append(dispatchAfterValues, dispatchAfter)
} }
tag, err := tx.Exec(ctx, ` rows, err := tx.Query(ctx, `
INSERT INTO monitoring_collect_tasks ( WITH input AS (
tenant_id, brand_id, question_id, question_hash, question_text_snapshot, ai_platform_id, SELECT *
collector_type, trigger_source, run_mode, business_date, planned_at, status, FROM unnest(
dispatch_priority, dispatch_lane, target_client_id, dispatch_after, $7::bigint[],
ingest_shard_key, execution_owner $8::bytea[],
$9::text[],
$10::text[],
$11::timestamptz[]
) AS item(question_id, question_hash, question_text, ai_platform_id, dispatch_after)
),
upserted AS (
INSERT INTO monitoring_collect_tasks (
tenant_id, brand_id, question_id, question_hash, question_text_snapshot, ai_platform_id,
collector_type, trigger_source, run_mode, business_date, planned_at, status,
dispatch_priority, dispatch_lane, target_client_id, dispatch_after,
ingest_shard_key, execution_owner
)
SELECT
$1,
$2,
input.question_id,
input.question_hash,
input.question_text,
input.ai_platform_id,
$3,
'automatic',
'desktop_standard',
$4::date,
input.dispatch_after,
'pending',
100,
'normal',
NULL,
input.dispatch_after,
$5,
$6
FROM input
ON CONFLICT (
tenant_id, brand_id, question_id, ai_platform_id, collector_type, run_mode, business_date
)
DO UPDATE SET
question_hash = EXCLUDED.question_hash,
question_text_snapshot = EXCLUDED.question_text_snapshot,
dispatch_after = LEAST(
COALESCE(monitoring_collect_tasks.dispatch_after, EXCLUDED.dispatch_after),
EXCLUDED.dispatch_after
),
updated_at = NOW()
WHERE monitoring_collect_tasks.status = 'pending'
AND monitoring_collect_tasks.callback_received_at IS NULL
RETURNING *
),
claimed AS (
UPDATE monitoring_collect_tasks t
SET last_dispatched_at = NOW(),
dispatch_attempts = COALESCE(t.dispatch_attempts, 0) + 1,
updated_at = NOW()
FROM upserted u
WHERE t.id = u.id
AND t.status = 'pending'
AND t.callback_received_at IS NULL
AND (t.dispatch_after IS NULL OR t.dispatch_after <= NOW())
AND (
t.last_dispatched_at IS NULL
OR t.last_dispatched_at <= NOW() - $12::interval
)
RETURNING t.*
) )
SELECT SELECT
$1, t.id,
$2, t.tenant_id,
input.question_id, t.ai_platform_id,
input.question_hash, t.business_date,
input.question_text, t.trigger_source,
input.ai_platform_id, t.run_mode,
$3, t.question_id,
'automatic', t.question_hash,
'desktop_standard', t.dispatch_priority,
$4::date, t.dispatch_lane,
input.dispatch_after, t.interrupt_generation,
'pending', t.question_text_snapshot
100, FROM claimed t
'normal', ORDER BY t.dispatch_priority DESC, t.id ASC
NULL,
input.dispatch_after,
$5,
$6
FROM unnest(
$7::bigint[],
$8::bytea[],
$9::text[],
$10::text[],
$11::timestamptz[]
) AS input(question_id, question_hash, question_text, ai_platform_id, dispatch_after)
ON CONFLICT (
tenant_id, brand_id, question_id, ai_platform_id, collector_type, run_mode, business_date
)
DO UPDATE SET
question_hash = EXCLUDED.question_hash,
question_text_snapshot = EXCLUDED.question_text_snapshot,
updated_at = NOW()
WHERE btrim(COALESCE(monitoring_collect_tasks.question_text_snapshot, '')) = ''
`, `,
plan.TenantID, plan.TenantID,
brand.BrandID, brand.BrandID,
@@ -1035,12 +1181,50 @@ func (s *MonitoringService) ensureDailyMonitoringCollectTasks(
questionTexts, questionTexts,
platformIDs, platformIDs,
dispatchAfterValues, dispatchAfterValues,
formatPgInterval(defaultMonitoringDailyDesktopRetryCooldown),
) )
if err != nil { if err != nil {
return 0, response.ErrInternal(50041, "insert_failed", "failed to create daily monitoring tasks") return 0, nil, response.ErrInternal(50041, "insert_failed", "failed to create daily monitoring tasks")
} }
defer rows.Close()
return tag.RowsAffected(), nil specs := make([]monitorDesktopTaskSpec, 0, len(candidates))
for rows.Next() {
var (
spec monitorDesktopTaskSpec
questionHash []byte
businessDay time.Time
)
if scanErr := rows.Scan(
&spec.MonitorTaskID,
&spec.TenantID,
&spec.PlatformID,
&businessDay,
&spec.TriggerSource,
&spec.RunMode,
&spec.QuestionID,
&questionHash,
&spec.Priority,
&spec.Lane,
&spec.InterruptGeneration,
&spec.QuestionText,
); scanErr != nil {
return 0, nil, response.ErrInternal(50041, "scan_failed", "failed to parse created daily monitoring tasks")
}
spec.WorkspaceID = plan.WorkspaceID
spec.BusinessDate = businessDay.Format("2006-01-02")
spec.QuestionHash = encodeQuestionHash(questionHash)
spec.QuestionText = strings.TrimSpace(spec.QuestionText)
if spec.QuestionText == "" {
continue
}
spec.SchedulerGroupKey = monitoringSchedulerGroupKey(spec.QuestionHash, spec.QuestionID, spec.QuestionText)
specs = append(specs, spec)
}
if err := rows.Err(); err != nil {
return 0, nil, response.ErrInternal(50041, "scan_failed", "failed to iterate created daily monitoring tasks")
}
return int64(len(specs)), specs, nil
} }
func (s *MonitoringService) loadDueDailyMonitorDesktopTaskSpecs( func (s *MonitoringService) loadDueDailyMonitorDesktopTaskSpecs(
@@ -225,10 +225,10 @@ func TestEnsureDailyMonitoringCollectTasksPersistsQuestionTextSnapshot(t *testin
tx := &captureDailyMonitorTaskTx{commandTag: pgconn.NewCommandTag("INSERT 0 2")} tx := &captureDailyMonitorTaskTx{commandTag: pgconn.NewCommandTag("INSERT 0 2")}
businessDay := time.Date(2026, 6, 22, 0, 0, 0, 0, time.UTC) businessDay := time.Date(2026, 6, 22, 0, 0, 0, 0, time.UTC)
created, err := (&MonitoringService{}).ensureDailyMonitoringCollectTasks( created, specs, err := (&MonitoringService{}).ensureDailyMonitoringCollectTasks(
ctx, ctx,
tx, tx,
monitoringDailyPlan{TenantID: 4}, monitoringDailyPlan{TenantID: 4, WorkspaceID: 8},
monitoringDailyBrand{BrandID: 8}, monitoringDailyBrand{BrandID: 8},
businessDay, businessDay,
[]monitoringDailyTaskCandidate{ []monitoringDailyTaskCandidate{
@@ -253,23 +253,27 @@ func TestEnsureDailyMonitoringCollectTasksPersistsQuestionTextSnapshot(t *testin
) )
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, int64(2), created) assert.Equal(t, int64(0), created)
assert.True(t, tx.execCalled) assert.Empty(t, specs)
normalizedSQL := normalizeSQLWhitespace(tx.execSQL) assert.True(t, tx.queryCalled)
normalizedSQL := normalizeSQLWhitespace(tx.querySQL)
assert.Contains(t, normalizedSQL, "question_text_snapshot, ai_platform_id") assert.Contains(t, normalizedSQL, "question_text_snapshot, ai_platform_id")
assert.Contains(t, normalizedSQL, "$10::text[]") assert.Contains(t, normalizedSQL, "$10::text[]")
assert.Contains(t, normalizedSQL, "AS input(question_id, question_hash, question_text, ai_platform_id, dispatch_after)") assert.Contains(t, normalizedSQL, "AS item(question_id, question_hash, question_text, ai_platform_id, dispatch_after)")
assert.Contains(t, normalizedSQL, "question_text_snapshot = EXCLUDED.question_text_snapshot") assert.Contains(t, normalizedSQL, "question_text_snapshot = EXCLUDED.question_text_snapshot")
require.Len(t, tx.execArgs, 11) assert.Contains(t, normalizedSQL, "WHERE monitoring_collect_tasks.status = 'pending'")
assert.Equal(t, []string{"幽灵门轨道长度定制", "幽灵门轨道长度定制"}, tx.execArgs[8]) assert.Contains(t, normalizedSQL, "t.dispatch_after IS NULL OR t.dispatch_after <= NOW()")
assert.Equal(t, []string{"qwen", "doubao"}, tx.execArgs[9]) assert.Contains(t, normalizedSQL, "SET last_dispatched_at = NOW()")
require.Len(t, tx.queryArgs, 12)
assert.Equal(t, []string{"幽灵门轨道长度定制", "幽灵门轨道长度定制"}, tx.queryArgs[8])
assert.Equal(t, []string{"qwen", "doubao"}, tx.queryArgs[9])
} }
func TestEnsureDailyMonitoringCollectTasksRejectsEmptyQuestionText(t *testing.T) { func TestEnsureDailyMonitoringCollectTasksRejectsEmptyQuestionText(t *testing.T) {
ctx := context.Background() ctx := context.Background()
tx := &captureDailyMonitorTaskTx{} tx := &captureDailyMonitorTaskTx{}
created, err := (&MonitoringService{}).ensureDailyMonitoringCollectTasks( created, specs, err := (&MonitoringService{}).ensureDailyMonitoringCollectTasks(
ctx, ctx,
tx, tx,
monitoringDailyPlan{TenantID: 4}, monitoringDailyPlan{TenantID: 4},
@@ -290,10 +294,26 @@ func TestEnsureDailyMonitoringCollectTasksRejectsEmptyQuestionText(t *testing.T)
require.Error(t, err) require.Error(t, err)
assert.Equal(t, int64(0), created) assert.Equal(t, int64(0), created)
assert.Empty(t, specs)
assert.False(t, tx.execCalled) assert.False(t, tx.execCalled)
assert.False(t, tx.queryCalled)
assert.Contains(t, err.Error(), "missing_question_text_snapshot") assert.Contains(t, err.Error(), "missing_question_text_snapshot")
} }
func TestFilterMonitoringDailyCandidatesByPlatformKeepsOnlyOnlineDispatchablePlatforms(t *testing.T) {
candidates := []monitoringDailyTaskCandidate{
{Question: monitoringConfiguredQuestion{ID: 1}, Platform: monitoringPlatformMetadata{ID: "qwen"}},
{Question: monitoringConfiguredQuestion{ID: 1}, Platform: monitoringPlatformMetadata{ID: "doubao"}},
{Question: monitoringConfiguredQuestion{ID: 2}, Platform: monitoringPlatformMetadata{ID: "kimi"}},
}
filtered := filterMonitoringDailyCandidatesByPlatform(candidates, monitoringDailyPlatformSet([]string{"qwen", "kimi"}))
require.Len(t, filtered, 2)
assert.Equal(t, "qwen", filtered[0].Platform.ID)
assert.Equal(t, "kimi", filtered[1].Platform.ID)
}
func TestDecodeDailyMonitoringEnabledPlatformsReportsMalformedJSON(t *testing.T) { func TestDecodeDailyMonitoringEnabledPlatformsReportsMalformedJSON(t *testing.T) {
platforms, err := decodeDailyMonitoringEnabledPlatforms([]byte(`["qwen"`)) platforms, err := decodeDailyMonitoringEnabledPlatforms([]byte(`["qwen"`))
require.Error(t, err) require.Error(t, err)
@@ -399,6 +419,7 @@ func TestMonitoringDailyTaskMetricsRecordsSummaryCounters(t *testing.T) {
FailedPlanCount: 1, FailedPlanCount: 1,
FailedBrandCount: 2, FailedBrandCount: 2,
PlannedTaskCount: 20, PlannedTaskCount: 20,
EligibleTaskCount: 12,
DueTaskCount: 8, DueTaskCount: 8,
CreatedTaskCount: 6, CreatedTaskCount: 6,
DesktopTaskCount: 4, DesktopTaskCount: 4,
@@ -415,6 +436,7 @@ func TestMonitoringDailyTaskMetricsRecordsSummaryCounters(t *testing.T) {
assert.Equal(t, int64(2), snapshot.BrandFailureTotal) assert.Equal(t, int64(2), snapshot.BrandFailureTotal)
assert.Equal(t, int64(1), snapshot.SkippedPlanTotal) assert.Equal(t, int64(1), snapshot.SkippedPlanTotal)
assert.Equal(t, int64(20), snapshot.PlannedTaskTotal) assert.Equal(t, int64(20), snapshot.PlannedTaskTotal)
assert.Equal(t, int64(12), snapshot.EligibleTaskTotal)
assert.Equal(t, int64(8), snapshot.DueTaskTotal) assert.Equal(t, int64(8), snapshot.DueTaskTotal)
assert.Equal(t, int64(6), snapshot.CreatedTaskTotal) assert.Equal(t, int64(6), snapshot.CreatedTaskTotal)
assert.Equal(t, int64(4), snapshot.DesktopTaskTotal) assert.Equal(t, int64(4), snapshot.DesktopTaskTotal)
@@ -428,6 +450,7 @@ func TestMonitoringDailyTaskMetricsRecordsSummaryCounters(t *testing.T) {
prometheus := MonitoringDailyTaskMetricsPrometheus() prometheus := MonitoringDailyTaskMetricsPrometheus()
assert.Contains(t, prometheus, `monitoring_daily_task_worker_runs_total{status="completed"} 1`) assert.Contains(t, prometheus, `monitoring_daily_task_worker_runs_total{status="completed"} 1`)
assert.Contains(t, prometheus, `monitoring_daily_task_generation_failures_total{scope="plan"} 1`) assert.Contains(t, prometheus, `monitoring_daily_task_generation_failures_total{scope="plan"} 1`)
assert.Contains(t, prometheus, `monitoring_daily_task_generation_tasks_total{stage="eligible"} 12`)
assert.Contains(t, prometheus, `monitoring_daily_task_generation_tasks_total{stage="created"} 6`) assert.Contains(t, prometheus, `monitoring_daily_task_generation_tasks_total{stage="created"} 6`)
assert.Contains(t, prometheus, `monitoring_daily_task_worker_last_business_date_info{business_date="2026-04-24"} 1`) assert.Contains(t, prometheus, `monitoring_daily_task_worker_last_business_date_info{business_date="2026-04-24"} 1`)
} }
@@ -568,12 +591,15 @@ func (r emptyDailyMonitorRows) RawValues() [][]byte { r
func (r emptyDailyMonitorRows) Conn() *pgx.Conn { return nil } func (r emptyDailyMonitorRows) Conn() *pgx.Conn { return nil }
type captureDailyMonitorTaskTx struct { type captureDailyMonitorTaskTx struct {
execCalled bool execCalled bool
execSQL string execSQL string
execArgs []any execArgs []any
execSQLs []string execSQLs []string
execArgses [][]any execArgses [][]any
commandTag pgconn.CommandTag queryCalled bool
querySQL string
queryArgs []any
commandTag pgconn.CommandTag
} }
func (tx *captureDailyMonitorTaskTx) Begin(context.Context) (pgx.Tx, error) { return tx, nil } func (tx *captureDailyMonitorTaskTx) Begin(context.Context) (pgx.Tx, error) { return tx, nil }
@@ -597,7 +623,10 @@ func (tx *captureDailyMonitorTaskTx) Exec(_ context.Context, sql string, argumen
tx.execArgses = append(tx.execArgses, arguments) tx.execArgses = append(tx.execArgses, arguments)
return tx.commandTag, nil return tx.commandTag, nil
} }
func (tx *captureDailyMonitorTaskTx) Query(context.Context, string, ...any) (pgx.Rows, error) { func (tx *captureDailyMonitorTaskTx) Query(_ context.Context, sql string, arguments ...any) (pgx.Rows, error) {
tx.queryCalled = true
tx.querySQL = sql
tx.queryArgs = arguments
return emptyDailyMonitorRows{}, nil return emptyDailyMonitorRows{}, nil
} }
func (tx *captureDailyMonitorTaskTx) QueryRow(context.Context, string, ...any) pgx.Row { func (tx *captureDailyMonitorTaskTx) QueryRow(context.Context, string, ...any) pgx.Row {
@@ -98,6 +98,7 @@ func (monitoringDailyTaskMetricsCollector) Collect(ch chan<- prometheus.Metric)
monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.BrandFailureTotal), []string{"scope"}, "brand") monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.BrandFailureTotal), []string{"scope"}, "brand")
monitoringCounter(ch, "monitoring_daily_task_generation_skipped_plans_total", "Total daily monitoring plans skipped before generation.", float64(snapshot.SkippedPlanTotal), nil) monitoringCounter(ch, "monitoring_daily_task_generation_skipped_plans_total", "Total daily monitoring plans skipped before generation.", float64(snapshot.SkippedPlanTotal), nil)
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.PlannedTaskTotal), []string{"stage"}, "planned") monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.PlannedTaskTotal), []string{"stage"}, "planned")
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.EligibleTaskTotal), []string{"stage"}, "eligible")
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DueTaskTotal), []string{"stage"}, "due") monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DueTaskTotal), []string{"stage"}, "due")
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.CreatedTaskTotal), []string{"stage"}, "created") monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.CreatedTaskTotal), []string{"stage"}, "created")
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DesktopTaskTotal), []string{"stage"}, "desktop") monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DesktopTaskTotal), []string{"stage"}, "desktop")
@@ -93,7 +93,7 @@ INSERT INTO ops.scheduler_jobs
(job_key, display_name, category, description, enabled, schedule_type, interval_seconds, timezone, timeout_seconds, batch_size, config) (job_key, display_name, category, description, enabled, schedule_type, interval_seconds, timezone, timeout_seconds, batch_size, config)
VALUES VALUES
('schedule_dispatch', '内容定时任务派发', 'content', '索引驱动的延迟队列派发,仅认领到期内容任务并投递生成队列。', true, 'interval', 15, 'Asia/Shanghai', 30, 100, '{}'::jsonb), ('schedule_dispatch', '内容定时任务派发', 'content', '索引驱动的延迟队列派发,仅认领到期内容任务并投递生成队列。', true, 'interval', 15, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
('monitoring_daily_task', 'AI 监控每日任务生成', 'monitoring', '按业务日全量创建监控采集事实,按容量限速派发到桌面任务队列', true, 'interval', 300, 'Asia/Shanghai', 45, 12, '{"max_desktop_client_backlog":12}'::jsonb), ('monitoring_daily_task', 'AI 监控每日任务生成', 'monitoring', '客户端在线时按业务日滚动创建并派发监控采集任务,离线不生成待采积压', true, 'interval', 300, 'Asia/Shanghai', 45, 12, '{"max_desktop_client_backlog":12}'::jsonb),
('monitoring_result_recovery', '监控结果恢复', 'monitoring', '按回调结果队列索引恢复未成功入队的监控结果。', true, 'interval', 60, 'Asia/Shanghai', 30, 100, '{}'::jsonb), ('monitoring_result_recovery', '监控结果恢复', 'monitoring', '按回调结果队列索引恢复未成功入队的监控结果。', true, 'interval', 60, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
('monitoring_lease_recovery', '监控租约回收', 'monitoring', '按过期租约索引限批回收超过租约时间仍未回调的监控任务。', true, 'interval', 1800, 'Asia/Shanghai', 30, 1000, '{}'::jsonb), ('monitoring_lease_recovery', '监控租约回收', 'monitoring', '按过期租约索引限批回收超过租约时间仍未回调的监控任务。', true, 'interval', 1800, 'Asia/Shanghai', 30, 1000, '{}'::jsonb),
('monitoring_received_inspection', '监控 received 卡死检查', 'monitoring', '按 received watchdog 索引限批标记长时间未入队的监控任务。', true, 'interval', 300, 'Asia/Shanghai', 30, 100, '{"threshold":"15m"}'::jsonb), ('monitoring_received_inspection', '监控 received 卡死检查', 'monitoring', '按 received watchdog 索引限批标记长时间未入队的监控任务。', true, 'interval', 300, 'Asia/Shanghai', 30, 100, '{"threshold":"15m"}'::jsonb),
@@ -11,7 +11,7 @@ SET description = '索引驱动的延迟队列派发,仅认领到期内容任
WHERE job_key = 'schedule_dispatch'; WHERE job_key = 'schedule_dispatch';
UPDATE ops.scheduler_jobs UPDATE ops.scheduler_jobs
SET description = '按业务日全量创建监控采集事实,按容量限速派发到桌面任务队列', SET description = '客户端在线时按业务日滚动创建并派发监控采集任务,离线不生成待采积压',
timeout_seconds = 45, timeout_seconds = 45,
batch_size = 12, batch_size = 12,
config = (config - 'max_materialize_per_run' - 'max_materialize_per_plan' - 'max_materialize_per_brand') config = (config - 'max_materialize_per_run' - 'max_materialize_per_plan' - 'max_materialize_per_brand')
@@ -1,5 +1,5 @@
UPDATE ops.scheduler_jobs UPDATE ops.scheduler_jobs
SET description = '按业务日全量创建监控采集事实,按容量限速派发到桌面任务队列', SET description = '客户端在线时按业务日滚动创建并派发监控采集任务,离线不生成待采积压',
batch_size = 12, batch_size = 12,
config = (config - 'max_materialize_per_run' - 'max_materialize_per_plan' - 'max_materialize_per_brand') config = (config - 'max_materialize_per_run' - 'max_materialize_per_plan' - 'max_materialize_per_brand')
|| '{"max_desktop_client_backlog":12}'::jsonb, || '{"max_desktop_client_backlog":12}'::jsonb,
@@ -0,0 +1,6 @@
UPDATE ops.scheduler_jobs
SET description = '按业务日全量创建监控采集事实,按容量限速派发到桌面任务队列。',
batch_size = 12,
config = config || '{"max_desktop_client_backlog":12}'::jsonb,
updated_at = NOW()
WHERE job_key = 'monitoring_daily_task';
@@ -0,0 +1,7 @@
UPDATE ops.scheduler_jobs
SET description = '客户端在线时按业务日滚动创建并派发监控采集任务,离线不生成待采积压。',
batch_size = 12,
config = (config - 'max_materialize_per_run' - 'max_materialize_per_plan' - 'max_materialize_per_brand')
|| '{"max_desktop_client_backlog":12}'::jsonb,
updated_at = NOW()
WHERE job_key = 'monitoring_daily_task';