fix: harden desktop monitoring recovery
Desktop Client Build / Resolve Build Metadata (push) Successful in 37s
Backend CI / Backend (push) Failing after 9m26s
Desktop Client Build / Build Desktop Client (push) Successful in 22m29s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 35s

This commit is contained in:
2026-06-23 23:12:25 +08:00
parent b4ebf34728
commit 04bd3e42e0
16 changed files with 1329 additions and 164 deletions
@@ -1901,7 +1901,8 @@ type PublishLeaseRecoveryResult struct {
}
type MonitorLeaseRecoveryResult struct {
Failed int `json:"failed"`
Requeued int `json:"requeued"`
Failed int `json:"failed"`
}
// resolvePublishRecoveryOutcome decides what to do with a publish task whose lease was lost.
@@ -1923,7 +1924,7 @@ func resolvePublishRecoveryOutcome(submitStarted bool, attempts int) (status str
}
func resolveMonitorRecoveryOutcome(mode desktopTaskRecoveryMode, attempts int) (status string, payloadKind string) {
if mode == desktopTaskRecoveryModeLeaseExpiry || attempts >= 2 {
if mode != desktopTaskRecoveryModeLeaseExpiry && attempts >= 2 {
return "failed", "monitor_final"
}
return "queued", "monitor"
@@ -2026,12 +2027,21 @@ func (s *DesktopTaskService) recoverClientTasks(
continue
}
}
} else if item.MonitorTaskID.Valid {
requeuedMonitorTaskIDs, requeueErr := s.requeueMonitorCollectTasksForDesktopRecovery(ctx, []int64{item.MonitorTaskID.Int64}, monitorErrorForTask)
if requeueErr != nil {
return requeueErr
}
if len(requeuedMonitorTaskIDs) == 0 {
continue
}
}
if _, execErr := tx.Exec(ctx, `
UPDATE desktop_tasks
SET status = $2,
error = $3,
active_attempt_id = NULL,
UPDATE desktop_tasks
SET status = $2,
error = $3,
result = CASE WHEN $2 = 'queued' THEN NULL ELSE result END,
active_attempt_id = NULL,
lease_token_hash = NULL,
lease_expires_at = NULL,
interrupted_at = COALESCE(interrupted_at, NOW()),
@@ -2355,7 +2365,7 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
limit = 1000
}
errorJSON, reason, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "monitor_final")
errorJSON, reason, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "monitor")
if err != nil {
return result, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode monitor desktop task recovery payload")
}
@@ -2390,7 +2400,7 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
s.logWarn("expired desktop monitor recovery scan failed", scanErr)
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to parse recovered desktop tasks")
}
item.Status = "failed"
item.Status = "queued"
item.ErrorJSON = errorJSON
recovered = append(recovered, item)
if item.MonitorTaskID.Valid {
@@ -2402,18 +2412,18 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to iterate recovered desktop tasks")
}
failedMonitorTaskIDs, err := s.failMonitorCollectTasksForDesktopRecovery(ctx, monitorTaskIDs, errorJSON)
requeuedMonitorTaskIDs, err := s.requeueMonitorCollectTasksForDesktopRecovery(ctx, monitorTaskIDs, errorJSON)
if err != nil {
return result, err
}
failedMonitorTaskSet := int64Set(failedMonitorTaskIDs)
requeuedMonitorTaskSet := int64Set(requeuedMonitorTaskIDs)
recoverableDesktopIDs := make([]uuid.UUID, 0, len(recovered))
recoverable := make([]recoveredDesktopTaskLease, 0, len(recovered))
for _, item := range recovered {
if !item.MonitorTaskID.Valid {
continue
}
if _, ok := failedMonitorTaskSet[item.MonitorTaskID.Int64]; !ok {
if _, ok := requeuedMonitorTaskSet[item.MonitorTaskID.Int64]; !ok {
continue
}
recoverableDesktopIDs = append(recoverableDesktopIDs, item.TaskID)
@@ -2423,7 +2433,7 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
return result, nil
}
updateRows, err := tx.Query(ctx, failExpiredMonitorDesktopTasksSQL(), recoverableDesktopIDs, errorJSON, reason)
updateRows, err := tx.Query(ctx, requeueExpiredMonitorDesktopTasksSQL(), recoverableDesktopIDs, errorJSON, reason)
if err != nil {
s.logWarn("expired desktop monitor recovery update failed", err)
return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
@@ -2462,7 +2472,7 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
TaskID: item.TaskID,
AttemptID: attemptID,
FinalStatus: literalStringPtr("failed"),
FinalStatus: literalStringPtr("aborted"),
Error: errorJSON,
}); finishErr != nil {
s.logWarn("expired desktop monitor recovery attempt finish failed", finishErr, zap.String("task_id", item.TaskID.String()))
@@ -2473,7 +2483,7 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
return result, response.ErrInternal(50199, "desktop_task_recovery_commit_failed", "failed to commit desktop task recovery")
}
result.Failed = len(updatedDesktopIDs)
result.Requeued = len(updatedDesktopIDs)
for _, item := range recoverable {
if _, ok := updatedDesktopIDs[item.TaskID]; !ok {
@@ -2484,12 +2494,12 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
s.logWarn("expired desktop monitor recovery reload after commit failed", getErr, zap.String("task_id", item.TaskID.String()))
continue
}
s.publishTaskEvent(ctx, task, "task_completed")
s.publishTaskEvent(ctx, task, "task_available")
}
if result.Failed > 0 && s.logger != nil {
if result.Requeued > 0 && s.logger != nil {
s.logger.Info("expired desktop monitor tasks recovered",
zap.Int("failed", result.Failed),
zap.Int("requeued", result.Requeued),
)
}
@@ -2519,16 +2529,18 @@ func expiredMonitorDesktopTasksCandidateSQL() string {
`
}
func failExpiredMonitorDesktopTasksSQL() string {
func requeueExpiredMonitorDesktopTasksSQL() string {
return `
UPDATE desktop_tasks
SET status = 'failed',
SET status = 'queued',
error = $2,
result = NULL,
active_attempt_id = NULL,
lease_token_hash = NULL,
lease_expires_at = NULL,
interrupted_at = COALESCE(interrupted_at, NOW()),
interrupt_reason = COALESCE(interrupt_reason, $3),
enqueued_at = NOW(),
updated_at = NOW()
WHERE desktop_id = ANY($1::uuid[])
AND kind = 'monitor'
@@ -2537,6 +2549,69 @@ func failExpiredMonitorDesktopTasksSQL() string {
`
}
func (s *DesktopTaskService) requeueMonitorCollectTasksForDesktopRecovery(ctx context.Context, monitorTaskIDs []int64, errorJSON []byte) ([]int64, error) {
if s == nil || s.monitoringPool == nil || len(monitorTaskIDs) == 0 {
return nil, nil
}
monitorTaskIDs = uniqueInt64s(monitorTaskIDs)
message := desktopTaskRecoveryMessage(errorJSON)
requestPayloadJSON, err := json.Marshal(map[string]any{
"runtime_error": json.RawMessage(errorJSON),
})
if err != nil {
return nil, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode monitor desktop task recovery payload")
}
rows, err := s.monitoringPool.Query(ctx, `
WITH input AS (
SELECT unnest($1::bigint[]) AS id
),
updated AS (
UPDATE monitoring_collect_tasks
SET status = 'pending',
lease_token_hash = NULL,
leased_to_executor = NULL,
leased_at = NULL,
lease_expires_at = NULL,
callback_received_at = NULL,
completed_at = NULL,
skip_reason = NULL,
dispatch_after = NOW() + interval '30 seconds',
error_message = NULLIF($2, ''),
request_payload_json = (
CASE
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
THEN '{}'::jsonb
ELSE request_payload_json
END
) || $3::jsonb,
updated_at = NOW()
WHERE id IN (SELECT id FROM input)
AND COALESCE(execution_owner, 'legacy') = 'desktop_tasks'
AND callback_received_at IS NULL
AND status IN ('pending', 'leased', 'expired')
RETURNING id
)
SELECT id FROM updated
`, monitorTaskIDs, message, requestPayloadJSON)
if err != nil {
return nil, response.ErrInternal(50041, "task_update_failed", "failed to requeue expired monitor desktop tasks")
}
defer rows.Close()
requeuedIDs := make([]int64, 0, len(monitorTaskIDs))
for rows.Next() {
var id int64
if scanErr := rows.Scan(&id); scanErr != nil {
return nil, response.ErrInternal(50041, "task_update_failed", "failed to parse requeued monitor desktop tasks")
}
requeuedIDs = append(requeuedIDs, id)
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "task_update_failed", "failed to iterate requeued monitor desktop tasks")
}
return requeuedIDs, nil
}
func (s *DesktopTaskService) failMonitorCollectTasksForDesktopRecovery(ctx context.Context, monitorTaskIDs []int64, errorJSON []byte) ([]int64, error) {
if s == nil || s.monitoringPool == nil || len(monitorTaskIDs) == 0 {
return nil, nil
@@ -2586,8 +2661,8 @@ func (s *DesktopTaskService) failMonitorCollectTasksForDesktopRecovery(ctx conte
AND t.status = 'failed'
AND (
t.error_message = $2
OR t.request_payload_json #>> '{runtime_error,source}' = 'desktop_task_lease_expiry'
OR t.request_payload_json #>> '{runtime_error,reason}' = 'lease_expired'
OR t.request_payload_json #>> '{runtime_error,source}' = 'desktop_task_recovery'
OR t.request_payload_json #>> '{runtime_error,source}' = 'desktop_client_offline'
)
)
SELECT id FROM updated
@@ -2595,7 +2670,7 @@ func (s *DesktopTaskService) failMonitorCollectTasksForDesktopRecovery(ctx conte
SELECT id FROM already_failed
`, monitorTaskIDs, message, requestPayloadJSON)
if err != nil {
return nil, response.ErrInternal(50041, "task_update_failed", "failed to fail expired monitor desktop tasks")
return nil, response.ErrInternal(50041, "task_update_failed", "failed to fail recovered monitor desktop tasks")
}
defer rows.Close()
@@ -2700,6 +2775,8 @@ func desktopTaskRecoveryPayload(mode desktopTaskRecoveryMode, kind string) ([]by
message = "desktop publish task may have already been submitted to the platform before the lease was lost; kept as unknown for manual reconcile to avoid duplicate publishing"
} else if isMonitorFinal {
message = "desktop monitor task lost its lease; task has been marked failed to avoid repeated platform retries"
} else if kind == "monitor" {
message = "desktop monitor task lost its lease; task has been re-queued"
}
switch mode {
@@ -2735,8 +2812,10 @@ func desktopTaskRecoveryPayload(mode desktopTaskRecoveryMode, kind string) ([]by
message = fmt.Sprintf("desktop task lease expired before publish completion; task exceeded %d attempts and has been marked failed", desktopPublishMaxAttempts)
} else if isPublishUncertain {
message = "desktop task lease expired while the publish task was mid-submit; kept as unknown for manual reconcile to avoid duplicate publishing"
} else if isMonitorFinal || kind == "monitor" {
} else if isMonitorFinal {
message = "desktop task lease expired before monitor completion; task has been marked failed"
} else if kind == "monitor" {
message = "desktop task lease expired before monitor completion; task has been re-queued"
} else {
message = "desktop task lease expired before publish completion; task has been re-queued for retry"
}
@@ -101,21 +101,21 @@ func TestDesktopTaskRecoveryPayloadPublishFinalFailsAfterMaxAttempts(t *testing.
}
}
func TestDesktopTaskRecoveryPayloadMonitorLeaseExpiryFails(t *testing.T) {
func TestDesktopTaskRecoveryPayloadMonitorLeaseExpiryRequeues(t *testing.T) {
t.Parallel()
payload, reason, message, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "monitor_final")
payload, reason, message, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "monitor")
if err != nil {
t.Fatalf("desktopTaskRecoveryPayload returned error: %v", err)
}
if reason != "lease_expired" {
t.Fatalf("reason = %q, want lease_expired", reason)
}
if !strings.Contains(message, "marked failed") {
t.Fatalf("message = %q, want marked failed", message)
if !strings.Contains(message, "re-queued") {
t.Fatalf("message = %q, want re-queued", message)
}
if strings.Contains(message, "re-queued") {
t.Fatalf("message must not requeue expired monitor tasks: %q", message)
if strings.Contains(message, "marked failed") {
t.Fatalf("message must not fail expired monitor tasks: %q", message)
}
if !strings.Contains(string(payload), "desktop_task_lease_expiry") {
t.Fatalf("payload missing source: %s", payload)
@@ -176,7 +176,8 @@ func TestResolveMonitorRecoveryOutcome(t *testing.T) {
{"disconnect first attempt requeues", desktopTaskRecoveryModeDisconnect, 1, "queued", "monitor"},
{"startup repeated recovery fails", desktopTaskRecoveryModeStartup, 2, "failed", "monitor_final"},
{"disconnect repeated recovery fails", desktopTaskRecoveryModeDisconnect, 2, "failed", "monitor_final"},
{"lease expiry fails immediately", desktopTaskRecoveryModeLeaseExpiry, 1, "failed", "monitor_final"},
{"lease expiry requeues first attempt", desktopTaskRecoveryModeLeaseExpiry, 1, "queued", "monitor"},
{"lease expiry requeues repeated attempt", desktopTaskRecoveryModeLeaseExpiry, 2, "queued", "monitor"},
}
for _, tc := range cases {
@@ -221,16 +222,17 @@ func TestExpiredMonitorDesktopTasksCandidateSQLSelectsOnlyExpiredInProgressMonit
}
}
func TestFailExpiredMonitorDesktopTasksSQLMarksTerminalFailure(t *testing.T) {
func TestRequeueExpiredMonitorDesktopTasksSQLReturnsTaskToQueue(t *testing.T) {
t.Parallel()
normalized := normalizeSQLForDesktopTaskTest(failExpiredMonitorDesktopTasksSQL())
normalized := normalizeSQLForDesktopTaskTest(requeueExpiredMonitorDesktopTasksSQL())
for _, fragment := range []string{
"SET status = 'failed'",
"SET status = 'queued'",
"active_attempt_id = NULL",
"lease_token_hash = NULL",
"lease_expires_at = NULL",
"interrupt_reason = COALESCE(interrupt_reason, $3)",
"enqueued_at = NOW()",
"WHERE desktop_id = ANY($1::uuid[])",
"AND kind = 'monitor'",
"AND status = 'in_progress'",
@@ -40,6 +40,7 @@ const (
// reports skip writes and only refresh the row at this interval so dashboards
// do not depend on per-heartbeat database churn.
monitoringHeartbeatAccessSnapshotRefreshAfter = 10 * time.Minute
monitoringRuntimeUnknownMaxDispatchAttempts = 3
)
var monitoringCitationMarkerPattern = regexp.MustCompile(`(?i)\[\s*citation\s*:\s*\d+\s*\]`)
@@ -194,6 +195,15 @@ type MonitoringCancelTaskResponse struct {
TaskStatus string `json:"task_status"`
}
func isMonitoringInfrastructureCancelReason(reason *string) bool {
switch strings.TrimSpace(optionalStringValue(reason)) {
case "desktop_infra_unavailable", "playwright_cdp_unavailable", "playwright_cdp_recovery":
return true
default:
return false
}
}
type monitoringInstallationIdentity struct {
ID string
TenantID int64
@@ -250,6 +260,7 @@ type monitoringCollectTask struct {
TriggerSource string
QuestionText string
TargetClientID sql.NullString
DispatchAttempts int
}
type monitoringDesktopExecutionLease struct {
@@ -258,6 +269,7 @@ type monitoringDesktopExecutionLease struct {
TargetClientID uuid.UUID
LeaseTokenHash []byte
InterruptGeneration int
ActiveAttemptID pgtype.UUID
}
type monitoringLeasedTaskRow struct {
@@ -945,7 +957,24 @@ func (s *MonitoringCallbackService) skipTaskForInstallation(
}
skipReason := normalizeSkipReason(req.SkipReason)
skipOutcome := monitoringSkipOutcomeForReason(skipReason)
skipOutcome := monitoringSkipOutcomeForReason(skipReason, task.DispatchAttempts, task.TriggerSource)
errorMessage := strings.TrimSpace(optionalStringValue(req.ErrorMessage))
requestPayloadJSON := []byte(`{}`)
if skipReason == "runtime_unknown" {
var marshalErr error
requestPayloadJSON, marshalErr = json.Marshal(map[string]any{
"runtime_unknown": map[string]any{
"reason": skipReason,
"message": errorMessage,
"dispatch_attempts": task.DispatchAttempts,
"max_attempts": monitoringRuntimeUnknownMaxDispatchAttempts,
"retryable": skipOutcome.Requeue,
},
})
if marshalErr != nil {
return nil, response.ErrInternal(50041, "payload_encode_failed", "failed to encode monitoring task skip payload")
}
}
tx, err := s.monitoringPool.Begin(ctx)
if err != nil {
@@ -953,16 +982,48 @@ func (s *MonitoringCallbackService) skipTaskForInstallation(
}
defer tx.Rollback(ctx)
if _, err := tx.Exec(ctx, `
UPDATE monitoring_collect_tasks
SET status = $2,
callback_received_at = NOW(),
completed_at = $3,
skip_reason = $4,
error_message = $5,
updated_at = NOW()
WHERE id = $1
`, task.ID, skipOutcome.TaskStatus, completedAt, nullableStringPtr(skipOutcome.StoredSkipReason), nullableString(req.ErrorMessage)); err != nil {
if skipOutcome.Requeue {
if _, err := tx.Exec(ctx, `
UPDATE monitoring_collect_tasks
SET status = 'pending',
lease_token_hash = NULL,
leased_to_executor = NULL,
leased_at = NULL,
lease_expires_at = NULL,
callback_received_at = NULL,
completed_at = NULL,
skip_reason = NULL,
dispatch_after = NOW() + $2::interval,
error_message = NULLIF($3, ''),
request_payload_json = (
CASE
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
THEN '{}'::jsonb
ELSE request_payload_json
END
) || $4::jsonb,
updated_at = NOW()
WHERE id = $1
`, task.ID, formatPgInterval(skipOutcome.RetryAfter), errorMessage, requestPayloadJSON); err != nil {
return nil, response.ErrInternal(50041, "task_update_failed", "failed to requeue monitoring task after runtime unknown")
}
} else if _, err := tx.Exec(ctx, `
UPDATE monitoring_collect_tasks
SET status = $2,
callback_received_at = NOW(),
completed_at = $3,
skip_reason = $4,
error_message = $5,
request_payload_json = (
CASE
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
THEN '{}'::jsonb
ELSE request_payload_json
END
) || $6::jsonb,
updated_at = NOW()
WHERE id = $1
`, task.ID, skipOutcome.TaskStatus, completedAt, nullableStringPtr(skipOutcome.StoredSkipReason), nullableString(req.ErrorMessage), requestPayloadJSON); err != nil {
return nil, response.ErrInternal(50041, "task_update_failed", "failed to update monitoring task skip status")
}
@@ -976,29 +1037,56 @@ func (s *MonitoringCallbackService) skipTaskForInstallation(
_ = s.touchInstallation(ctx, installation.ID)
completedAtText := completedAt.Format(time.RFC3339)
var completedAtText *string
if !skipOutcome.Requeue {
text := completedAt.Format(time.RFC3339)
completedAtText = &text
}
return &MonitoringSkipTaskResponse{
TaskID: task.ID,
TaskStatus: skipOutcome.TaskStatus,
SkipReason: skipReason,
CompletedAt: &completedAtText,
CompletedAt: completedAtText,
}, nil
}
type monitoringSkipOutcome struct {
TaskStatus string
StoredSkipReason string
Requeue bool
RetryAfter time.Duration
}
func monitoringSkipOutcomeForReason(skipReason string) monitoringSkipOutcome {
func monitoringSkipOutcomeForReason(skipReason string, dispatchAttempts int, triggerSource string) monitoringSkipOutcome {
switch strings.TrimSpace(skipReason) {
case "runtime_unknown":
if strings.EqualFold(strings.TrimSpace(triggerSource), "manual") {
return monitoringSkipOutcome{TaskStatus: "failed"}
}
if dispatchAttempts < monitoringRuntimeUnknownMaxDispatchAttempts {
return monitoringSkipOutcome{
TaskStatus: "pending",
Requeue: true,
RetryAfter: monitoringRuntimeUnknownRetryAfter(dispatchAttempts),
}
}
return monitoringSkipOutcome{TaskStatus: "failed"}
default:
return monitoringSkipOutcome{TaskStatus: "skipped", StoredSkipReason: strings.TrimSpace(skipReason)}
}
}
func monitoringRuntimeUnknownRetryAfter(dispatchAttempts int) time.Duration {
switch {
case dispatchAttempts <= 0:
return 30 * time.Second
case dispatchAttempts == 1:
return 2 * time.Minute
default:
return 5 * time.Minute
}
}
func nullableStringPtr(value string) *string {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
@@ -1045,15 +1133,23 @@ func (s *MonitoringCallbackService) CancelTaskForDesktopClient(
if marshalErr != nil {
return nil, response.ErrInternal(50041, "payload_encode_failed", "failed to encode phase2 desktop cancel payload")
}
taskRepo := repository.NewDesktopTaskRepository(s.businessPool)
canceledTask, cancelErr := taskRepo.CancelByLease(ctx, desktopExecution.DesktopTaskID, desktopExecution.LeaseTokenHash, errorJSON)
if cancelErr != nil {
if errors.Is(cancelErr, pgx.ErrNoRows) {
return nil, response.ErrConflict(40941, "task_status_invalid", "phase2 monitor desktop task is no longer active")
if isMonitoringInfrastructureCancelReason(req.Reason) {
requeuedTask, requeueErr := s.requeueDesktopMonitorTaskAfterInfrastructureCancel(ctx, desktopExecution, errorJSON)
if requeueErr != nil {
return nil, requeueErr
}
return nil, response.ErrInternal(50041, "desktop_task_cancel_failed", "failed to cancel phase2 monitor desktop task lease")
desktopTaskCanceled = requeuedTask
} else {
taskRepo := repository.NewDesktopTaskRepository(s.businessPool)
canceledTask, cancelErr := taskRepo.CancelByLease(ctx, desktopExecution.DesktopTaskID, desktopExecution.LeaseTokenHash, errorJSON)
if cancelErr != nil {
if errors.Is(cancelErr, pgx.ErrNoRows) {
return nil, response.ErrConflict(40941, "task_status_invalid", "phase2 monitor desktop task is no longer active")
}
return nil, response.ErrInternal(50041, "desktop_task_cancel_failed", "failed to cancel phase2 monitor desktop task lease")
}
desktopTaskCanceled = canceledTask
}
desktopTaskCanceled = canceledTask
} else if task.Status != "leased" {
return nil, response.ErrConflict(40941, "task_status_invalid", "task status does not allow lease cancel")
}
@@ -1066,11 +1162,11 @@ func (s *MonitoringCallbackService) CancelTaskForDesktopClient(
leased_at = NULL,
lease_expires_at = NULL,
callback_received_at = NULL,
dispatch_after = NOW(),
dispatch_after = CASE WHEN $3::boolean THEN NOW() + interval '30 seconds' ELSE NOW() END,
error_message = $2,
updated_at = NOW()
WHERE id = $1
`, task.ID, nullableString(req.Reason)); err != nil {
`, task.ID, nullableString(req.Reason), isMonitoringInfrastructureCancelReason(req.Reason)); err != nil {
return nil, response.ErrInternal(50041, "task_update_failed", "failed to cancel monitoring task lease")
}
@@ -1106,6 +1202,72 @@ func (s *MonitoringCallbackService) CancelTaskForDesktopClient(
}, nil
}
func (s *MonitoringCallbackService) requeueDesktopMonitorTaskAfterInfrastructureCancel(
ctx context.Context,
lease *monitoringDesktopExecutionLease,
errorJSON []byte,
) (*repository.DesktopTask, error) {
if s == nil || s.businessPool == nil || lease == nil {
return nil, response.ErrInternal(50041, "desktop_task_requeue_failed", "failed to requeue phase2 monitor desktop task lease")
}
tx, err := s.businessPool.Begin(ctx)
if err != nil {
return nil, response.ErrInternal(50041, "desktop_task_requeue_begin_failed", "failed to start phase2 monitor desktop task requeue")
}
defer tx.Rollback(ctx)
repo := repository.NewDesktopTaskRepository(tx)
row := tx.QueryRow(ctx, `
UPDATE desktop_tasks
SET status = 'queued',
error = $3,
result = NULL,
active_attempt_id = NULL,
lease_token_hash = NULL,
lease_expires_at = NULL,
interrupted_at = COALESCE(interrupted_at, NOW()),
interrupt_reason = COALESCE(interrupt_reason, 'desktop_infra_unavailable'),
enqueued_at = NOW(),
updated_at = NOW()
WHERE desktop_id = $1
AND lease_token_hash = $2
AND kind = 'monitor'
AND status = 'in_progress'
RETURNING `+desktopTaskRepositoryReturningColumns,
lease.DesktopTaskID,
lease.LeaseTokenHash,
errorJSON,
)
task, scanErr := scanRepositoryDesktopTask(row)
if scanErr != nil {
if errors.Is(scanErr, pgx.ErrNoRows) {
return nil, response.ErrConflict(40941, "task_status_invalid", "phase2 monitor desktop task is no longer active")
}
return nil, response.ErrInternal(50041, "desktop_task_requeue_failed", "failed to requeue phase2 monitor desktop task lease")
}
if lease.ActiveAttemptID.Valid {
attemptID, convErr := uuid.FromBytes(lease.ActiveAttemptID.Bytes[:])
if convErr != nil {
return nil, response.ErrInternal(50041, "desktop_task_attempt_decode_failed", "failed to decode phase2 monitor desktop task attempt")
}
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
TaskID: task.DesktopID,
AttemptID: attemptID,
FinalStatus: literalStringPtr("aborted"),
Error: errorJSON,
}); finishErr != nil {
return nil, response.ErrInternal(50041, "desktop_task_attempt_finish_failed", "failed to finish phase2 monitor desktop task attempt")
}
}
if err := tx.Commit(ctx); err != nil {
return nil, response.ErrInternal(50041, "desktop_task_requeue_commit_failed", "failed to commit phase2 monitor desktop task requeue")
}
return task, nil
}
func (s *MonitoringCallbackService) ProcessQueuedTaskResult(ctx context.Context, envelope monitoringTaskResultEnvelope) (*MonitoringTaskResultResponse, error) {
if envelope.TaskID <= 0 {
return nil, response.ErrBadRequest(40041, "invalid_task_id", "task id is required")
@@ -1393,6 +1555,7 @@ func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantI
completed_at,
skip_reason,
trigger_source,
COALESCE(dispatch_attempts, 0) AS dispatch_attempts,
COALESCE(target_client_id::text, '') AS target_client_id,
COALESCE(NULLIF(t.question_text_snapshot, ''), (
SELECT question_text_snapshot
@@ -1430,6 +1593,7 @@ func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantI
&item.CompletedAt,
&item.SkipReason,
&item.TriggerSource,
&item.DispatchAttempts,
&item.TargetClientID,
&item.QuestionText,
); err != nil {
@@ -1464,6 +1628,7 @@ func (s *MonitoringCallbackService) loadCollectTaskByID(ctx context.Context, tas
completed_at,
skip_reason,
trigger_source,
COALESCE(dispatch_attempts, 0) AS dispatch_attempts,
COALESCE(target_client_id::text, '') AS target_client_id,
COALESCE(NULLIF(t.question_text_snapshot, ''), (
SELECT question_text_snapshot
@@ -1500,6 +1665,7 @@ func (s *MonitoringCallbackService) loadCollectTaskByID(ctx context.Context, tas
&item.CompletedAt,
&item.SkipReason,
&item.TriggerSource,
&item.DispatchAttempts,
&item.TargetClientID,
&item.QuestionText,
); err != nil {
@@ -1518,23 +1684,25 @@ func (s *MonitoringCallbackService) loadActiveDesktopMonitorExecutionLease(
) (*monitoringDesktopExecutionLease, error) {
item := &monitoringDesktopExecutionLease{}
if err := s.businessPool.QueryRow(ctx, `
SELECT
desktop_id,
status,
target_client_id,
lease_token_hash,
interrupt_generation
SELECT
desktop_id,
status,
target_client_id,
lease_token_hash,
active_attempt_id,
interrupt_generation
FROM desktop_tasks
WHERE kind = 'monitor'
AND monitor_task_id = $1
AND status IN ('queued', 'in_progress')
ORDER BY created_at DESC, id DESC
LIMIT 1
`, monitorTaskID).Scan(
`, monitorTaskID).Scan(
&item.DesktopTaskID,
&item.Status,
&item.TargetClientID,
&item.LeaseTokenHash,
&item.ActiveAttemptID,
&item.InterruptGeneration,
); err != nil {
if err == pgx.ErrNoRows {
@@ -225,12 +225,12 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
repo := repository.NewDesktopTaskRepository(tx)
publishableTasks := make([]*repository.DesktopTask, 0, len(specs))
fallbackLegacyTaskIDs := make([]int64, 0)
deferredTaskIDs := make([]int64, 0)
for _, spec := range specs {
if spec.TargetAccountID == uuid.Nil || spec.TargetClientID == uuid.Nil {
target, ok := targets[normalizeMonitoringPlatformID(spec.PlatformID)]
if !ok || target.AccountID == uuid.Nil || target.ClientID == uuid.Nil {
fallbackLegacyTaskIDs = append(fallbackLegacyTaskIDs, spec.MonitorTaskID)
deferredTaskIDs = append(deferredTaskIDs, spec.MonitorTaskID)
continue
}
spec.TargetAccountID = target.AccountID
@@ -242,7 +242,7 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
return nil, nil, upsertErr
}
if fallbackLegacy {
fallbackLegacyTaskIDs = append(fallbackLegacyTaskIDs, spec.MonitorTaskID)
deferredTaskIDs = append(deferredTaskIDs, spec.MonitorTaskID)
continue
}
if shouldPublish && task != nil {
@@ -250,10 +250,16 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
}
}
if len(deferredTaskIDs) > 0 {
if err := deferPhase2MonitorCollectTasks(ctx, tx, deferredTaskIDs, defaultMonitoringDailyDesktopRetryCooldown); err != nil {
return nil, nil, err
}
}
if err := tx.Commit(ctx); err != nil {
return nil, nil, response.ErrInternal(50116, "desktop_task_commit_failed", "failed to commit phase2 monitor desktop dispatch")
}
return publishableTasks, fallbackLegacyTaskIDs, nil
return publishableTasks, deferredTaskIDs, nil
}
func monitorDesktopTaskSpecsNeedTargetLookup(specs []monitorDesktopTaskSpec) bool {
@@ -738,6 +744,35 @@ func (s *MonitoringService) fallbackMonitorDesktopTasksToLegacy(ctx context.Cont
return nil
}
func deferPhase2MonitorCollectTasks(ctx context.Context, tx pgx.Tx, taskIDs []int64, delay time.Duration) error {
if tx == nil || len(taskIDs) == 0 {
return nil
}
if delay <= 0 {
delay = defaultMonitoringDailyDesktopRetryCooldown
}
if _, err := tx.Exec(ctx, `
UPDATE monitoring_collect_tasks
SET status = 'pending',
lease_token_hash = NULL,
leased_to_executor = NULL,
leased_at = NULL,
lease_expires_at = NULL,
callback_received_at = NULL,
completed_at = NULL,
skip_reason = NULL,
dispatch_after = NOW() + $2::interval,
error_message = NULL,
updated_at = NOW()
WHERE id = ANY($1::bigint[])
AND COALESCE(execution_owner, 'legacy') = 'desktop_tasks'
AND callback_received_at IS NULL
`, uniqueInt64s(taskIDs), formatPgInterval(delay)); err != nil {
return response.ErrInternal(50041, "task_defer_failed", "failed to defer phase2 monitor tasks until a desktop client is online")
}
return nil
}
func (s *MonitoringService) deferQueuedPhase2MonitorTasks(
ctx context.Context,
targetClientID uuid.UUID,
@@ -4,6 +4,7 @@ import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
@@ -193,15 +194,37 @@ func TestCompleteLinkedDesktopMonitorTaskOnlyFinalizesInProgress(t *testing.T) {
assert.NotContains(t, query, "'unknown'")
}
func TestMonitoringSkipOutcomeForRuntimeUnknownFailsTask(t *testing.T) {
outcome := monitoringSkipOutcomeForReason("runtime_unknown")
func TestMonitoringSkipOutcomeForRuntimeUnknownRequeuesAutomaticTask(t *testing.T) {
outcome := monitoringSkipOutcomeForReason("runtime_unknown", 1, "automatic")
assert.Equal(t, "pending", outcome.TaskStatus)
assert.True(t, outcome.Requeue)
assert.Equal(t, 2*time.Minute, outcome.RetryAfter)
assert.Empty(t, outcome.StoredSkipReason)
}
func TestMonitoringSkipOutcomeForRuntimeUnknownFailsAfterMaxAttempts(t *testing.T) {
outcome := monitoringSkipOutcomeForReason(
"runtime_unknown",
monitoringRuntimeUnknownMaxDispatchAttempts,
"automatic",
)
assert.Equal(t, "failed", outcome.TaskStatus)
assert.False(t, outcome.Requeue)
assert.Empty(t, outcome.StoredSkipReason)
}
func TestMonitoringSkipOutcomeForManualRuntimeUnknownFailsTask(t *testing.T) {
outcome := monitoringSkipOutcomeForReason("runtime_unknown", 0, "manual")
assert.Equal(t, "failed", outcome.TaskStatus)
assert.False(t, outcome.Requeue)
assert.Empty(t, outcome.StoredSkipReason)
}
func TestMonitoringSkipOutcomeKeepsOrdinarySkip(t *testing.T) {
outcome := monitoringSkipOutcomeForReason("stale_business_date")
outcome := monitoringSkipOutcomeForReason("stale_business_date", 0, "automatic")
assert.Equal(t, "skipped", outcome.TaskStatus)
assert.Equal(t, "stale_business_date", outcome.StoredSkipReason)
@@ -993,24 +993,21 @@ func (s *MonitoringService) CollectNow(
phase2DispatchReady := executionOwner == "desktop_tasks"
if len(phase2TaskSpecs) > 0 {
phase2TaskCount := len(phase2TaskSpecs)
publishedTasks, fallbackLegacyTaskIDs, dispatchErr := s.dispatchMonitorDesktopTasks(ctx, phase2TaskSpecs)
publishedTasks, deferredTaskIDs, dispatchErr := s.dispatchMonitorDesktopTasks(ctx, phase2TaskSpecs)
if dispatchErr != nil {
if fallbackErr := s.fallbackMonitorDesktopTasksToLegacy(ctx, monitorDesktopTaskSpecIDs(phase2TaskSpecs)); fallbackErr != nil {
return nil, fallbackErr
}
phase2DispatchReady = false
phase2TaskSpecs = nil
if s.logger != nil {
s.logger.Warn("phase2 monitor desktop dispatch failed, falling back to legacy execution",
s.logger.Warn("phase2 monitor desktop dispatch failed; pending tasks will retry when a desktop client is online",
zap.Error(dispatchErr),
zap.Int("phase2_task_count", phase2TaskCount),
)
}
} else {
if len(fallbackLegacyTaskIDs) > 0 {
if fallbackErr := s.fallbackMonitorDesktopTasksToLegacy(ctx, fallbackLegacyTaskIDs); fallbackErr != nil {
return nil, fallbackErr
}
if len(deferredTaskIDs) > 0 && s.logger != nil {
s.logger.Info("phase2 monitor desktop tasks deferred until a desktop client is online",
zap.Int("deferred_task_count", len(deferredTaskIDs)),
)
}
phase2PublishedTasks = publishedTasks
}