fix: harden desktop monitoring recovery
Desktop Client Build / Resolve Build Metadata (push) Successful in 37s
Backend CI / Backend (push) Failing after 9m26s
Desktop Client Build / Build Desktop Client (push) Successful in 22m29s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 35s
Desktop Client Build / Resolve Build Metadata (push) Successful in 37s
Backend CI / Backend (push) Failing after 9m26s
Desktop Client Build / Build Desktop Client (push) Successful in 22m29s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 35s
This commit is contained in:
@@ -225,12 +225,12 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
|
||||
|
||||
repo := repository.NewDesktopTaskRepository(tx)
|
||||
publishableTasks := make([]*repository.DesktopTask, 0, len(specs))
|
||||
fallbackLegacyTaskIDs := make([]int64, 0)
|
||||
deferredTaskIDs := make([]int64, 0)
|
||||
for _, spec := range specs {
|
||||
if spec.TargetAccountID == uuid.Nil || spec.TargetClientID == uuid.Nil {
|
||||
target, ok := targets[normalizeMonitoringPlatformID(spec.PlatformID)]
|
||||
if !ok || target.AccountID == uuid.Nil || target.ClientID == uuid.Nil {
|
||||
fallbackLegacyTaskIDs = append(fallbackLegacyTaskIDs, spec.MonitorTaskID)
|
||||
deferredTaskIDs = append(deferredTaskIDs, spec.MonitorTaskID)
|
||||
continue
|
||||
}
|
||||
spec.TargetAccountID = target.AccountID
|
||||
@@ -242,7 +242,7 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
|
||||
return nil, nil, upsertErr
|
||||
}
|
||||
if fallbackLegacy {
|
||||
fallbackLegacyTaskIDs = append(fallbackLegacyTaskIDs, spec.MonitorTaskID)
|
||||
deferredTaskIDs = append(deferredTaskIDs, spec.MonitorTaskID)
|
||||
continue
|
||||
}
|
||||
if shouldPublish && task != nil {
|
||||
@@ -250,10 +250,16 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
|
||||
}
|
||||
}
|
||||
|
||||
if len(deferredTaskIDs) > 0 {
|
||||
if err := deferPhase2MonitorCollectTasks(ctx, tx, deferredTaskIDs, defaultMonitoringDailyDesktopRetryCooldown); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, nil, response.ErrInternal(50116, "desktop_task_commit_failed", "failed to commit phase2 monitor desktop dispatch")
|
||||
}
|
||||
return publishableTasks, fallbackLegacyTaskIDs, nil
|
||||
return publishableTasks, deferredTaskIDs, nil
|
||||
}
|
||||
|
||||
func monitorDesktopTaskSpecsNeedTargetLookup(specs []monitorDesktopTaskSpec) bool {
|
||||
@@ -738,6 +744,35 @@ func (s *MonitoringService) fallbackMonitorDesktopTasksToLegacy(ctx context.Cont
|
||||
return nil
|
||||
}
|
||||
|
||||
func deferPhase2MonitorCollectTasks(ctx context.Context, tx pgx.Tx, taskIDs []int64, delay time.Duration) error {
|
||||
if tx == nil || len(taskIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
if delay <= 0 {
|
||||
delay = defaultMonitoringDailyDesktopRetryCooldown
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE monitoring_collect_tasks
|
||||
SET status = 'pending',
|
||||
lease_token_hash = NULL,
|
||||
leased_to_executor = NULL,
|
||||
leased_at = NULL,
|
||||
lease_expires_at = NULL,
|
||||
callback_received_at = NULL,
|
||||
completed_at = NULL,
|
||||
skip_reason = NULL,
|
||||
dispatch_after = NOW() + $2::interval,
|
||||
error_message = NULL,
|
||||
updated_at = NOW()
|
||||
WHERE id = ANY($1::bigint[])
|
||||
AND COALESCE(execution_owner, 'legacy') = 'desktop_tasks'
|
||||
AND callback_received_at IS NULL
|
||||
`, uniqueInt64s(taskIDs), formatPgInterval(delay)); err != nil {
|
||||
return response.ErrInternal(50041, "task_defer_failed", "failed to defer phase2 monitor tasks until a desktop client is online")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MonitoringService) deferQueuedPhase2MonitorTasks(
|
||||
ctx context.Context,
|
||||
targetClientID uuid.UUID,
|
||||
|
||||
Reference in New Issue
Block a user