fix: harden desktop monitoring recovery
Desktop Client Build / Resolve Build Metadata (push) Successful in 37s
Backend CI / Backend (push) Failing after 9m26s
Desktop Client Build / Build Desktop Client (push) Successful in 22m29s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 35s

This commit is contained in:
2026-06-23 23:12:25 +08:00
parent b4ebf34728
commit 04bd3e42e0
16 changed files with 1329 additions and 164 deletions
@@ -225,12 +225,12 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
repo := repository.NewDesktopTaskRepository(tx)
publishableTasks := make([]*repository.DesktopTask, 0, len(specs))
fallbackLegacyTaskIDs := make([]int64, 0)
deferredTaskIDs := make([]int64, 0)
for _, spec := range specs {
if spec.TargetAccountID == uuid.Nil || spec.TargetClientID == uuid.Nil {
target, ok := targets[normalizeMonitoringPlatformID(spec.PlatformID)]
if !ok || target.AccountID == uuid.Nil || target.ClientID == uuid.Nil {
fallbackLegacyTaskIDs = append(fallbackLegacyTaskIDs, spec.MonitorTaskID)
deferredTaskIDs = append(deferredTaskIDs, spec.MonitorTaskID)
continue
}
spec.TargetAccountID = target.AccountID
@@ -242,7 +242,7 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
return nil, nil, upsertErr
}
if fallbackLegacy {
fallbackLegacyTaskIDs = append(fallbackLegacyTaskIDs, spec.MonitorTaskID)
deferredTaskIDs = append(deferredTaskIDs, spec.MonitorTaskID)
continue
}
if shouldPublish && task != nil {
@@ -250,10 +250,16 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
}
}
if len(deferredTaskIDs) > 0 {
if err := deferPhase2MonitorCollectTasks(ctx, tx, deferredTaskIDs, defaultMonitoringDailyDesktopRetryCooldown); err != nil {
return nil, nil, err
}
}
if err := tx.Commit(ctx); err != nil {
return nil, nil, response.ErrInternal(50116, "desktop_task_commit_failed", "failed to commit phase2 monitor desktop dispatch")
}
return publishableTasks, fallbackLegacyTaskIDs, nil
return publishableTasks, deferredTaskIDs, nil
}
func monitorDesktopTaskSpecsNeedTargetLookup(specs []monitorDesktopTaskSpec) bool {
@@ -738,6 +744,35 @@ func (s *MonitoringService) fallbackMonitorDesktopTasksToLegacy(ctx context.Cont
return nil
}
func deferPhase2MonitorCollectTasks(ctx context.Context, tx pgx.Tx, taskIDs []int64, delay time.Duration) error {
if tx == nil || len(taskIDs) == 0 {
return nil
}
if delay <= 0 {
delay = defaultMonitoringDailyDesktopRetryCooldown
}
if _, err := tx.Exec(ctx, `
UPDATE monitoring_collect_tasks
SET status = 'pending',
lease_token_hash = NULL,
leased_to_executor = NULL,
leased_at = NULL,
lease_expires_at = NULL,
callback_received_at = NULL,
completed_at = NULL,
skip_reason = NULL,
dispatch_after = NOW() + $2::interval,
error_message = NULL,
updated_at = NOW()
WHERE id = ANY($1::bigint[])
AND COALESCE(execution_owner, 'legacy') = 'desktop_tasks'
AND callback_received_at IS NULL
`, uniqueInt64s(taskIDs), formatPgInterval(delay)); err != nil {
return response.ErrInternal(50041, "task_defer_failed", "failed to defer phase2 monitor tasks until a desktop client is online")
}
return nil
}
func (s *MonitoringService) deferQueuedPhase2MonitorTasks(
ctx context.Context,
targetClientID uuid.UUID,