fix: throttle desktop monitoring dispatch
Desktop Client Build / Resolve Build Metadata (push) Successful in 27s
Backend CI / Backend (push) Successful in 15m57s
Desktop Client Build / Build Desktop Client (push) Successful in 22m57s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 27s

This commit is contained in:
2026-06-22 12:21:59 +08:00
parent 54c2f92e02
commit 8c6789dca6
23 changed files with 591 additions and 88 deletions
@@ -138,15 +138,18 @@ func (s *MonitoringService) loadMonitorDesktopTaskSpecs(
t.dispatch_priority,
t.dispatch_lane,
t.interrupt_generation,
COALESCE((
COALESCE(NULLIF(t.question_text_snapshot, ''), (
SELECT question_text_snapshot
FROM monitoring_question_config_snapshots s
WHERE s.tenant_id = t.tenant_id
AND s.brand_id = t.brand_id
AND s.question_id = t.question_id
AND s.question_hash = t.question_hash
AND s.deleted_at IS NULL
ORDER BY s.projected_at DESC, s.id DESC
AND btrim(s.question_text_snapshot) <> ''
ORDER BY
CASE WHEN s.deleted_at IS NULL THEN 0 ELSE 1 END,
s.projected_at DESC,
s.id DESC
LIMIT 1
), '') AS question_text_snapshot
FROM monitoring_collect_tasks t
@@ -193,6 +196,10 @@ func (s *MonitoringService) loadMonitorDesktopTaskSpecs(
spec.RequestedByUserID = requestedByUserID
spec.BusinessDate = businessDay.Format("2006-01-02")
spec.QuestionHash = encodeQuestionHash(questionHash)
spec.QuestionText = strings.TrimSpace(spec.QuestionText)
if spec.QuestionText == "" {
continue
}
spec.SchedulerGroupKey = monitoringSchedulerGroupKey(spec.QuestionHash, spec.QuestionID, spec.QuestionText)
result = append(result, spec)
}
@@ -210,9 +217,13 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
return nil, nil, nil
}
targets, err := s.loadMonitorDesktopTaskTargets(ctx, specs[0].TenantID, specs[0].WorkspaceID, specs[0].RequestedByUserID, specs)
if err != nil {
return nil, nil, err
var targets map[string]monitorDesktopTaskTarget
if monitorDesktopTaskSpecsNeedTargetLookup(specs) {
var err error
targets, err = s.loadMonitorDesktopTaskTargets(ctx, specs[0].TenantID, specs[0].WorkspaceID, specs[0].RequestedByUserID, specs)
if err != nil {
return nil, nil, err
}
}
tx, err := s.businessPool.Begin(ctx)
@@ -225,13 +236,23 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
publishableTasks := make([]*repository.DesktopTask, 0, len(specs))
fallbackLegacyTaskIDs := make([]int64, 0)
for _, spec := range specs {
target, ok := targets[normalizeMonitoringPlatformID(spec.PlatformID)]
if !ok || target.AccountID == uuid.Nil || target.ClientID == uuid.Nil {
fallbackLegacyTaskIDs = append(fallbackLegacyTaskIDs, spec.MonitorTaskID)
if spec.TargetAccountID == uuid.Nil || spec.TargetClientID == uuid.Nil {
target, ok := targets[normalizeMonitoringPlatformID(spec.PlatformID)]
if !ok || target.AccountID == uuid.Nil || target.ClientID == uuid.Nil {
fallbackLegacyTaskIDs = append(fallbackLegacyTaskIDs, spec.MonitorTaskID)
continue
}
spec.TargetAccountID = target.AccountID
spec.TargetClientID = target.ClientID
}
available, throttleErr := acquireMonitorDesktopTaskPlatformSlot(ctx, tx, spec)
if throttleErr != nil {
return nil, nil, throttleErr
}
if !available {
continue
}
spec.TargetAccountID = target.AccountID
spec.TargetClientID = target.ClientID
task, shouldPublish, fallbackLegacy, upsertErr := s.upsertMonitorDesktopTask(ctx, tx, repo, spec)
if upsertErr != nil {
@@ -252,6 +273,54 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
return publishableTasks, fallbackLegacyTaskIDs, nil
}
func monitorDesktopTaskSpecsNeedTargetLookup(specs []monitorDesktopTaskSpec) bool {
for _, spec := range specs {
if spec.TargetAccountID == uuid.Nil || spec.TargetClientID == uuid.Nil {
return true
}
}
return false
}
func acquireMonitorDesktopTaskPlatformSlot(ctx context.Context, tx pgx.Tx, spec monitorDesktopTaskSpec) (bool, error) {
if tx == nil || spec.TargetClientID == uuid.Nil || strings.TrimSpace(spec.PlatformID) == "" {
return false, nil
}
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, monitorDesktopTaskPlatformSlotLockKey(spec.TargetClientID, spec.PlatformID)); err != nil {
return false, response.ErrInternal(50123, "desktop_task_throttle_lock_failed", "failed to acquire phase2 monitor desktop platform slot")
}
var busy bool
if err := tx.QueryRow(ctx, monitorDesktopTaskPlatformSlotBusySQL(), spec.TenantID, spec.WorkspaceID, spec.TargetClientID, normalizeMonitoringPlatformID(spec.PlatformID), spec.MonitorTaskID).Scan(&busy); err != nil {
return false, response.ErrInternal(50124, "desktop_task_throttle_lookup_failed", "failed to inspect phase2 monitor desktop platform slot")
}
return !busy, nil
}
func monitorDesktopTaskPlatformSlotBusySQL() string {
return `
SELECT EXISTS (
SELECT 1
FROM desktop_tasks
WHERE tenant_id = $1
AND workspace_id = $2
AND kind = 'monitor'
AND target_client_id = $3
AND platform_id = $4
AND status = 'in_progress'
AND (
$5::bigint <= 0
OR monitor_task_id IS NULL
OR monitor_task_id <> $5
)
)
`
}
func monitorDesktopTaskPlatformSlotLockKey(clientID uuid.UUID, platformID string) int64 {
return int64(monitoringDailyStableHash("monitor_desktop_platform_slot", clientID.String(), normalizeMonitoringPlatformID(platformID)))
}
func (s *MonitoringService) loadMonitorDesktopTaskTargets(
ctx context.Context,
tenantID, workspaceID, userID int64,
@@ -721,6 +790,7 @@ func (s *MonitoringService) requestPhase2MonitorInterrupts(
ctx context.Context,
targetClientID uuid.UUID,
excludedMonitorTaskIDs []int64,
excludedPlatformIDs []string,
interruptGeneration int,
) ([]monitorDesktopInterruptTarget, error) {
if s == nil || s.businessPool == nil {
@@ -728,6 +798,7 @@ func (s *MonitoringService) requestPhase2MonitorInterrupts(
}
queryArgs := []any{targetClientID, interruptGeneration}
nextParam := 3
query := `
UPDATE desktop_tasks
SET control_flags = (
@@ -752,8 +823,14 @@ func (s *MonitoringService) requestPhase2MonitorInterrupts(
AND monitor_task_id IS NOT NULL
`
if len(excludedMonitorTaskIDs) > 0 {
query += ` AND NOT (monitor_task_id = ANY($3::bigint[]))`
query += fmt.Sprintf(` AND NOT (monitor_task_id = ANY($%d::bigint[]))`, nextParam)
queryArgs = append(queryArgs, excludedMonitorTaskIDs)
nextParam++
}
excludedPlatformIDs = reconcileEnabledMonitoringPlatforms(excludedPlatformIDs)
if len(excludedPlatformIDs) > 0 {
query += fmt.Sprintf(` AND NOT (platform_id = ANY($%d::text[]))`, nextParam)
queryArgs = append(queryArgs, excludedPlatformIDs)
}
query += `
RETURNING desktop_id::text, monitor_task_id, target_client_id::text, workspace_id, interrupt_generation
@@ -981,18 +1058,18 @@ func monitoringSchedulerGroupKey(questionHash string, questionID int64, question
}
func marshalMonitorDesktopTaskPayload(spec monitorDesktopTaskSpec) ([]byte, error) {
title := strings.TrimSpace(spec.QuestionText)
platformName := platformDisplayName(spec.PlatformID)
if title != "" {
title = fmt.Sprintf("%s · %s", platformName, title)
} else {
title = fmt.Sprintf("监控任务 · %s", platformName)
questionText := strings.TrimSpace(spec.QuestionText)
if questionText == "" {
return nil, response.ErrInternal(50041, "missing_question_text_snapshot", "monitor desktop task question text is empty")
}
title := questionText
platformName := platformDisplayName(spec.PlatformID)
title = fmt.Sprintf("%s · %s", platformName, title)
payload, err := json.Marshal(map[string]any{
"title": title,
"business_date": spec.BusinessDate,
"scheduler_group_key": spec.SchedulerGroupKey,
"question_text": spec.QuestionText,
"question_text": questionText,
"question_id": spec.QuestionID,
"question_hash": spec.QuestionHash,
"ai_platform_id": spec.PlatformID,