fix: harden desktop monitoring recovery
Desktop Client Build / Resolve Build Metadata (push) Successful in 37s
Backend CI / Backend (push) Failing after 9m26s
Desktop Client Build / Build Desktop Client (push) Successful in 22m29s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 35s
Desktop Client Build / Resolve Build Metadata (push) Successful in 37s
Backend CI / Backend (push) Failing after 9m26s
Desktop Client Build / Build Desktop Client (push) Successful in 22m29s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 35s
This commit is contained in:
@@ -1901,7 +1901,8 @@ type PublishLeaseRecoveryResult struct {
|
||||
}
|
||||
|
||||
type MonitorLeaseRecoveryResult struct {
|
||||
Failed int `json:"failed"`
|
||||
Requeued int `json:"requeued"`
|
||||
Failed int `json:"failed"`
|
||||
}
|
||||
|
||||
// resolvePublishRecoveryOutcome decides what to do with a publish task whose lease was lost.
|
||||
@@ -1923,7 +1924,7 @@ func resolvePublishRecoveryOutcome(submitStarted bool, attempts int) (status str
|
||||
}
|
||||
|
||||
func resolveMonitorRecoveryOutcome(mode desktopTaskRecoveryMode, attempts int) (status string, payloadKind string) {
|
||||
if mode == desktopTaskRecoveryModeLeaseExpiry || attempts >= 2 {
|
||||
if mode != desktopTaskRecoveryModeLeaseExpiry && attempts >= 2 {
|
||||
return "failed", "monitor_final"
|
||||
}
|
||||
return "queued", "monitor"
|
||||
@@ -2026,12 +2027,21 @@ func (s *DesktopTaskService) recoverClientTasks(
|
||||
continue
|
||||
}
|
||||
}
|
||||
} else if item.MonitorTaskID.Valid {
|
||||
requeuedMonitorTaskIDs, requeueErr := s.requeueMonitorCollectTasksForDesktopRecovery(ctx, []int64{item.MonitorTaskID.Int64}, monitorErrorForTask)
|
||||
if requeueErr != nil {
|
||||
return requeueErr
|
||||
}
|
||||
if len(requeuedMonitorTaskIDs) == 0 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if _, execErr := tx.Exec(ctx, `
|
||||
UPDATE desktop_tasks
|
||||
SET status = $2,
|
||||
error = $3,
|
||||
active_attempt_id = NULL,
|
||||
UPDATE desktop_tasks
|
||||
SET status = $2,
|
||||
error = $3,
|
||||
result = CASE WHEN $2 = 'queued' THEN NULL ELSE result END,
|
||||
active_attempt_id = NULL,
|
||||
lease_token_hash = NULL,
|
||||
lease_expires_at = NULL,
|
||||
interrupted_at = COALESCE(interrupted_at, NOW()),
|
||||
@@ -2355,7 +2365,7 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
|
||||
limit = 1000
|
||||
}
|
||||
|
||||
errorJSON, reason, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "monitor_final")
|
||||
errorJSON, reason, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "monitor")
|
||||
if err != nil {
|
||||
return result, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode monitor desktop task recovery payload")
|
||||
}
|
||||
@@ -2390,7 +2400,7 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
|
||||
s.logWarn("expired desktop monitor recovery scan failed", scanErr)
|
||||
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to parse recovered desktop tasks")
|
||||
}
|
||||
item.Status = "failed"
|
||||
item.Status = "queued"
|
||||
item.ErrorJSON = errorJSON
|
||||
recovered = append(recovered, item)
|
||||
if item.MonitorTaskID.Valid {
|
||||
@@ -2402,18 +2412,18 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
|
||||
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to iterate recovered desktop tasks")
|
||||
}
|
||||
|
||||
failedMonitorTaskIDs, err := s.failMonitorCollectTasksForDesktopRecovery(ctx, monitorTaskIDs, errorJSON)
|
||||
requeuedMonitorTaskIDs, err := s.requeueMonitorCollectTasksForDesktopRecovery(ctx, monitorTaskIDs, errorJSON)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
failedMonitorTaskSet := int64Set(failedMonitorTaskIDs)
|
||||
requeuedMonitorTaskSet := int64Set(requeuedMonitorTaskIDs)
|
||||
recoverableDesktopIDs := make([]uuid.UUID, 0, len(recovered))
|
||||
recoverable := make([]recoveredDesktopTaskLease, 0, len(recovered))
|
||||
for _, item := range recovered {
|
||||
if !item.MonitorTaskID.Valid {
|
||||
continue
|
||||
}
|
||||
if _, ok := failedMonitorTaskSet[item.MonitorTaskID.Int64]; !ok {
|
||||
if _, ok := requeuedMonitorTaskSet[item.MonitorTaskID.Int64]; !ok {
|
||||
continue
|
||||
}
|
||||
recoverableDesktopIDs = append(recoverableDesktopIDs, item.TaskID)
|
||||
@@ -2423,7 +2433,7 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
|
||||
return result, nil
|
||||
}
|
||||
|
||||
updateRows, err := tx.Query(ctx, failExpiredMonitorDesktopTasksSQL(), recoverableDesktopIDs, errorJSON, reason)
|
||||
updateRows, err := tx.Query(ctx, requeueExpiredMonitorDesktopTasksSQL(), recoverableDesktopIDs, errorJSON, reason)
|
||||
if err != nil {
|
||||
s.logWarn("expired desktop monitor recovery update failed", err)
|
||||
return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
|
||||
@@ -2462,7 +2472,7 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
|
||||
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
|
||||
TaskID: item.TaskID,
|
||||
AttemptID: attemptID,
|
||||
FinalStatus: literalStringPtr("failed"),
|
||||
FinalStatus: literalStringPtr("aborted"),
|
||||
Error: errorJSON,
|
||||
}); finishErr != nil {
|
||||
s.logWarn("expired desktop monitor recovery attempt finish failed", finishErr, zap.String("task_id", item.TaskID.String()))
|
||||
@@ -2473,7 +2483,7 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
|
||||
return result, response.ErrInternal(50199, "desktop_task_recovery_commit_failed", "failed to commit desktop task recovery")
|
||||
}
|
||||
|
||||
result.Failed = len(updatedDesktopIDs)
|
||||
result.Requeued = len(updatedDesktopIDs)
|
||||
|
||||
for _, item := range recoverable {
|
||||
if _, ok := updatedDesktopIDs[item.TaskID]; !ok {
|
||||
@@ -2484,12 +2494,12 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
|
||||
s.logWarn("expired desktop monitor recovery reload after commit failed", getErr, zap.String("task_id", item.TaskID.String()))
|
||||
continue
|
||||
}
|
||||
s.publishTaskEvent(ctx, task, "task_completed")
|
||||
s.publishTaskEvent(ctx, task, "task_available")
|
||||
}
|
||||
|
||||
if result.Failed > 0 && s.logger != nil {
|
||||
if result.Requeued > 0 && s.logger != nil {
|
||||
s.logger.Info("expired desktop monitor tasks recovered",
|
||||
zap.Int("failed", result.Failed),
|
||||
zap.Int("requeued", result.Requeued),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -2519,16 +2529,18 @@ func expiredMonitorDesktopTasksCandidateSQL() string {
|
||||
`
|
||||
}
|
||||
|
||||
func failExpiredMonitorDesktopTasksSQL() string {
|
||||
func requeueExpiredMonitorDesktopTasksSQL() string {
|
||||
return `
|
||||
UPDATE desktop_tasks
|
||||
SET status = 'failed',
|
||||
SET status = 'queued',
|
||||
error = $2,
|
||||
result = NULL,
|
||||
active_attempt_id = NULL,
|
||||
lease_token_hash = NULL,
|
||||
lease_expires_at = NULL,
|
||||
interrupted_at = COALESCE(interrupted_at, NOW()),
|
||||
interrupt_reason = COALESCE(interrupt_reason, $3),
|
||||
enqueued_at = NOW(),
|
||||
updated_at = NOW()
|
||||
WHERE desktop_id = ANY($1::uuid[])
|
||||
AND kind = 'monitor'
|
||||
@@ -2537,6 +2549,69 @@ func failExpiredMonitorDesktopTasksSQL() string {
|
||||
`
|
||||
}
|
||||
|
||||
func (s *DesktopTaskService) requeueMonitorCollectTasksForDesktopRecovery(ctx context.Context, monitorTaskIDs []int64, errorJSON []byte) ([]int64, error) {
|
||||
if s == nil || s.monitoringPool == nil || len(monitorTaskIDs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
monitorTaskIDs = uniqueInt64s(monitorTaskIDs)
|
||||
message := desktopTaskRecoveryMessage(errorJSON)
|
||||
requestPayloadJSON, err := json.Marshal(map[string]any{
|
||||
"runtime_error": json.RawMessage(errorJSON),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode monitor desktop task recovery payload")
|
||||
}
|
||||
rows, err := s.monitoringPool.Query(ctx, `
|
||||
WITH input AS (
|
||||
SELECT unnest($1::bigint[]) AS id
|
||||
),
|
||||
updated AS (
|
||||
UPDATE monitoring_collect_tasks
|
||||
SET status = 'pending',
|
||||
lease_token_hash = NULL,
|
||||
leased_to_executor = NULL,
|
||||
leased_at = NULL,
|
||||
lease_expires_at = NULL,
|
||||
callback_received_at = NULL,
|
||||
completed_at = NULL,
|
||||
skip_reason = NULL,
|
||||
dispatch_after = NOW() + interval '30 seconds',
|
||||
error_message = NULLIF($2, ''),
|
||||
request_payload_json = (
|
||||
CASE
|
||||
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
|
||||
THEN '{}'::jsonb
|
||||
ELSE request_payload_json
|
||||
END
|
||||
) || $3::jsonb,
|
||||
updated_at = NOW()
|
||||
WHERE id IN (SELECT id FROM input)
|
||||
AND COALESCE(execution_owner, 'legacy') = 'desktop_tasks'
|
||||
AND callback_received_at IS NULL
|
||||
AND status IN ('pending', 'leased', 'expired')
|
||||
RETURNING id
|
||||
)
|
||||
SELECT id FROM updated
|
||||
`, monitorTaskIDs, message, requestPayloadJSON)
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(50041, "task_update_failed", "failed to requeue expired monitor desktop tasks")
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
requeuedIDs := make([]int64, 0, len(monitorTaskIDs))
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if scanErr := rows.Scan(&id); scanErr != nil {
|
||||
return nil, response.ErrInternal(50041, "task_update_failed", "failed to parse requeued monitor desktop tasks")
|
||||
}
|
||||
requeuedIDs = append(requeuedIDs, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, response.ErrInternal(50041, "task_update_failed", "failed to iterate requeued monitor desktop tasks")
|
||||
}
|
||||
return requeuedIDs, nil
|
||||
}
|
||||
|
||||
func (s *DesktopTaskService) failMonitorCollectTasksForDesktopRecovery(ctx context.Context, monitorTaskIDs []int64, errorJSON []byte) ([]int64, error) {
|
||||
if s == nil || s.monitoringPool == nil || len(monitorTaskIDs) == 0 {
|
||||
return nil, nil
|
||||
@@ -2586,8 +2661,8 @@ func (s *DesktopTaskService) failMonitorCollectTasksForDesktopRecovery(ctx conte
|
||||
AND t.status = 'failed'
|
||||
AND (
|
||||
t.error_message = $2
|
||||
OR t.request_payload_json #>> '{runtime_error,source}' = 'desktop_task_lease_expiry'
|
||||
OR t.request_payload_json #>> '{runtime_error,reason}' = 'lease_expired'
|
||||
OR t.request_payload_json #>> '{runtime_error,source}' = 'desktop_task_recovery'
|
||||
OR t.request_payload_json #>> '{runtime_error,source}' = 'desktop_client_offline'
|
||||
)
|
||||
)
|
||||
SELECT id FROM updated
|
||||
@@ -2595,7 +2670,7 @@ func (s *DesktopTaskService) failMonitorCollectTasksForDesktopRecovery(ctx conte
|
||||
SELECT id FROM already_failed
|
||||
`, monitorTaskIDs, message, requestPayloadJSON)
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(50041, "task_update_failed", "failed to fail expired monitor desktop tasks")
|
||||
return nil, response.ErrInternal(50041, "task_update_failed", "failed to fail recovered monitor desktop tasks")
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
@@ -2700,6 +2775,8 @@ func desktopTaskRecoveryPayload(mode desktopTaskRecoveryMode, kind string) ([]by
|
||||
message = "desktop publish task may have already been submitted to the platform before the lease was lost; kept as unknown for manual reconcile to avoid duplicate publishing"
|
||||
} else if isMonitorFinal {
|
||||
message = "desktop monitor task lost its lease; task has been marked failed to avoid repeated platform retries"
|
||||
} else if kind == "monitor" {
|
||||
message = "desktop monitor task lost its lease; task has been re-queued"
|
||||
}
|
||||
|
||||
switch mode {
|
||||
@@ -2735,8 +2812,10 @@ func desktopTaskRecoveryPayload(mode desktopTaskRecoveryMode, kind string) ([]by
|
||||
message = fmt.Sprintf("desktop task lease expired before publish completion; task exceeded %d attempts and has been marked failed", desktopPublishMaxAttempts)
|
||||
} else if isPublishUncertain {
|
||||
message = "desktop task lease expired while the publish task was mid-submit; kept as unknown for manual reconcile to avoid duplicate publishing"
|
||||
} else if isMonitorFinal || kind == "monitor" {
|
||||
} else if isMonitorFinal {
|
||||
message = "desktop task lease expired before monitor completion; task has been marked failed"
|
||||
} else if kind == "monitor" {
|
||||
message = "desktop task lease expired before monitor completion; task has been re-queued"
|
||||
} else {
|
||||
message = "desktop task lease expired before publish completion; task has been re-queued for retry"
|
||||
}
|
||||
|
||||
@@ -101,21 +101,21 @@ func TestDesktopTaskRecoveryPayloadPublishFinalFailsAfterMaxAttempts(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestDesktopTaskRecoveryPayloadMonitorLeaseExpiryFails(t *testing.T) {
|
||||
func TestDesktopTaskRecoveryPayloadMonitorLeaseExpiryRequeues(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
payload, reason, message, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "monitor_final")
|
||||
payload, reason, message, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "monitor")
|
||||
if err != nil {
|
||||
t.Fatalf("desktopTaskRecoveryPayload returned error: %v", err)
|
||||
}
|
||||
if reason != "lease_expired" {
|
||||
t.Fatalf("reason = %q, want lease_expired", reason)
|
||||
}
|
||||
if !strings.Contains(message, "marked failed") {
|
||||
t.Fatalf("message = %q, want marked failed", message)
|
||||
if !strings.Contains(message, "re-queued") {
|
||||
t.Fatalf("message = %q, want re-queued", message)
|
||||
}
|
||||
if strings.Contains(message, "re-queued") {
|
||||
t.Fatalf("message must not requeue expired monitor tasks: %q", message)
|
||||
if strings.Contains(message, "marked failed") {
|
||||
t.Fatalf("message must not fail expired monitor tasks: %q", message)
|
||||
}
|
||||
if !strings.Contains(string(payload), "desktop_task_lease_expiry") {
|
||||
t.Fatalf("payload missing source: %s", payload)
|
||||
@@ -176,7 +176,8 @@ func TestResolveMonitorRecoveryOutcome(t *testing.T) {
|
||||
{"disconnect first attempt requeues", desktopTaskRecoveryModeDisconnect, 1, "queued", "monitor"},
|
||||
{"startup repeated recovery fails", desktopTaskRecoveryModeStartup, 2, "failed", "monitor_final"},
|
||||
{"disconnect repeated recovery fails", desktopTaskRecoveryModeDisconnect, 2, "failed", "monitor_final"},
|
||||
{"lease expiry fails immediately", desktopTaskRecoveryModeLeaseExpiry, 1, "failed", "monitor_final"},
|
||||
{"lease expiry requeues first attempt", desktopTaskRecoveryModeLeaseExpiry, 1, "queued", "monitor"},
|
||||
{"lease expiry requeues repeated attempt", desktopTaskRecoveryModeLeaseExpiry, 2, "queued", "monitor"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
@@ -221,16 +222,17 @@ func TestExpiredMonitorDesktopTasksCandidateSQLSelectsOnlyExpiredInProgressMonit
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailExpiredMonitorDesktopTasksSQLMarksTerminalFailure(t *testing.T) {
|
||||
func TestRequeueExpiredMonitorDesktopTasksSQLReturnsTaskToQueue(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
normalized := normalizeSQLForDesktopTaskTest(failExpiredMonitorDesktopTasksSQL())
|
||||
normalized := normalizeSQLForDesktopTaskTest(requeueExpiredMonitorDesktopTasksSQL())
|
||||
for _, fragment := range []string{
|
||||
"SET status = 'failed'",
|
||||
"SET status = 'queued'",
|
||||
"active_attempt_id = NULL",
|
||||
"lease_token_hash = NULL",
|
||||
"lease_expires_at = NULL",
|
||||
"interrupt_reason = COALESCE(interrupt_reason, $3)",
|
||||
"enqueued_at = NOW()",
|
||||
"WHERE desktop_id = ANY($1::uuid[])",
|
||||
"AND kind = 'monitor'",
|
||||
"AND status = 'in_progress'",
|
||||
|
||||
@@ -40,6 +40,7 @@ const (
|
||||
// reports skip writes and only refresh the row at this interval so dashboards
|
||||
// do not depend on per-heartbeat database churn.
|
||||
monitoringHeartbeatAccessSnapshotRefreshAfter = 10 * time.Minute
|
||||
monitoringRuntimeUnknownMaxDispatchAttempts = 3
|
||||
)
|
||||
|
||||
var monitoringCitationMarkerPattern = regexp.MustCompile(`(?i)\[\s*citation\s*:\s*\d+\s*\]`)
|
||||
@@ -194,6 +195,15 @@ type MonitoringCancelTaskResponse struct {
|
||||
TaskStatus string `json:"task_status"`
|
||||
}
|
||||
|
||||
func isMonitoringInfrastructureCancelReason(reason *string) bool {
|
||||
switch strings.TrimSpace(optionalStringValue(reason)) {
|
||||
case "desktop_infra_unavailable", "playwright_cdp_unavailable", "playwright_cdp_recovery":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type monitoringInstallationIdentity struct {
|
||||
ID string
|
||||
TenantID int64
|
||||
@@ -250,6 +260,7 @@ type monitoringCollectTask struct {
|
||||
TriggerSource string
|
||||
QuestionText string
|
||||
TargetClientID sql.NullString
|
||||
DispatchAttempts int
|
||||
}
|
||||
|
||||
type monitoringDesktopExecutionLease struct {
|
||||
@@ -258,6 +269,7 @@ type monitoringDesktopExecutionLease struct {
|
||||
TargetClientID uuid.UUID
|
||||
LeaseTokenHash []byte
|
||||
InterruptGeneration int
|
||||
ActiveAttemptID pgtype.UUID
|
||||
}
|
||||
|
||||
type monitoringLeasedTaskRow struct {
|
||||
@@ -945,7 +957,24 @@ func (s *MonitoringCallbackService) skipTaskForInstallation(
|
||||
}
|
||||
|
||||
skipReason := normalizeSkipReason(req.SkipReason)
|
||||
skipOutcome := monitoringSkipOutcomeForReason(skipReason)
|
||||
skipOutcome := monitoringSkipOutcomeForReason(skipReason, task.DispatchAttempts, task.TriggerSource)
|
||||
errorMessage := strings.TrimSpace(optionalStringValue(req.ErrorMessage))
|
||||
requestPayloadJSON := []byte(`{}`)
|
||||
if skipReason == "runtime_unknown" {
|
||||
var marshalErr error
|
||||
requestPayloadJSON, marshalErr = json.Marshal(map[string]any{
|
||||
"runtime_unknown": map[string]any{
|
||||
"reason": skipReason,
|
||||
"message": errorMessage,
|
||||
"dispatch_attempts": task.DispatchAttempts,
|
||||
"max_attempts": monitoringRuntimeUnknownMaxDispatchAttempts,
|
||||
"retryable": skipOutcome.Requeue,
|
||||
},
|
||||
})
|
||||
if marshalErr != nil {
|
||||
return nil, response.ErrInternal(50041, "payload_encode_failed", "failed to encode monitoring task skip payload")
|
||||
}
|
||||
}
|
||||
|
||||
tx, err := s.monitoringPool.Begin(ctx)
|
||||
if err != nil {
|
||||
@@ -953,16 +982,48 @@ func (s *MonitoringCallbackService) skipTaskForInstallation(
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE monitoring_collect_tasks
|
||||
SET status = $2,
|
||||
callback_received_at = NOW(),
|
||||
completed_at = $3,
|
||||
skip_reason = $4,
|
||||
error_message = $5,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
`, task.ID, skipOutcome.TaskStatus, completedAt, nullableStringPtr(skipOutcome.StoredSkipReason), nullableString(req.ErrorMessage)); err != nil {
|
||||
if skipOutcome.Requeue {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE monitoring_collect_tasks
|
||||
SET status = 'pending',
|
||||
lease_token_hash = NULL,
|
||||
leased_to_executor = NULL,
|
||||
leased_at = NULL,
|
||||
lease_expires_at = NULL,
|
||||
callback_received_at = NULL,
|
||||
completed_at = NULL,
|
||||
skip_reason = NULL,
|
||||
dispatch_after = NOW() + $2::interval,
|
||||
error_message = NULLIF($3, ''),
|
||||
request_payload_json = (
|
||||
CASE
|
||||
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
|
||||
THEN '{}'::jsonb
|
||||
ELSE request_payload_json
|
||||
END
|
||||
) || $4::jsonb,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
`, task.ID, formatPgInterval(skipOutcome.RetryAfter), errorMessage, requestPayloadJSON); err != nil {
|
||||
return nil, response.ErrInternal(50041, "task_update_failed", "failed to requeue monitoring task after runtime unknown")
|
||||
}
|
||||
} else if _, err := tx.Exec(ctx, `
|
||||
UPDATE monitoring_collect_tasks
|
||||
SET status = $2,
|
||||
callback_received_at = NOW(),
|
||||
completed_at = $3,
|
||||
skip_reason = $4,
|
||||
error_message = $5,
|
||||
request_payload_json = (
|
||||
CASE
|
||||
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
|
||||
THEN '{}'::jsonb
|
||||
ELSE request_payload_json
|
||||
END
|
||||
) || $6::jsonb,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
`, task.ID, skipOutcome.TaskStatus, completedAt, nullableStringPtr(skipOutcome.StoredSkipReason), nullableString(req.ErrorMessage), requestPayloadJSON); err != nil {
|
||||
return nil, response.ErrInternal(50041, "task_update_failed", "failed to update monitoring task skip status")
|
||||
}
|
||||
|
||||
@@ -976,29 +1037,56 @@ func (s *MonitoringCallbackService) skipTaskForInstallation(
|
||||
|
||||
_ = s.touchInstallation(ctx, installation.ID)
|
||||
|
||||
completedAtText := completedAt.Format(time.RFC3339)
|
||||
var completedAtText *string
|
||||
if !skipOutcome.Requeue {
|
||||
text := completedAt.Format(time.RFC3339)
|
||||
completedAtText = &text
|
||||
}
|
||||
return &MonitoringSkipTaskResponse{
|
||||
TaskID: task.ID,
|
||||
TaskStatus: skipOutcome.TaskStatus,
|
||||
SkipReason: skipReason,
|
||||
CompletedAt: &completedAtText,
|
||||
CompletedAt: completedAtText,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type monitoringSkipOutcome struct {
|
||||
TaskStatus string
|
||||
StoredSkipReason string
|
||||
Requeue bool
|
||||
RetryAfter time.Duration
|
||||
}
|
||||
|
||||
func monitoringSkipOutcomeForReason(skipReason string) monitoringSkipOutcome {
|
||||
func monitoringSkipOutcomeForReason(skipReason string, dispatchAttempts int, triggerSource string) monitoringSkipOutcome {
|
||||
switch strings.TrimSpace(skipReason) {
|
||||
case "runtime_unknown":
|
||||
if strings.EqualFold(strings.TrimSpace(triggerSource), "manual") {
|
||||
return monitoringSkipOutcome{TaskStatus: "failed"}
|
||||
}
|
||||
if dispatchAttempts < monitoringRuntimeUnknownMaxDispatchAttempts {
|
||||
return monitoringSkipOutcome{
|
||||
TaskStatus: "pending",
|
||||
Requeue: true,
|
||||
RetryAfter: monitoringRuntimeUnknownRetryAfter(dispatchAttempts),
|
||||
}
|
||||
}
|
||||
return monitoringSkipOutcome{TaskStatus: "failed"}
|
||||
default:
|
||||
return monitoringSkipOutcome{TaskStatus: "skipped", StoredSkipReason: strings.TrimSpace(skipReason)}
|
||||
}
|
||||
}
|
||||
|
||||
func monitoringRuntimeUnknownRetryAfter(dispatchAttempts int) time.Duration {
|
||||
switch {
|
||||
case dispatchAttempts <= 0:
|
||||
return 30 * time.Second
|
||||
case dispatchAttempts == 1:
|
||||
return 2 * time.Minute
|
||||
default:
|
||||
return 5 * time.Minute
|
||||
}
|
||||
}
|
||||
|
||||
func nullableStringPtr(value string) *string {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "" {
|
||||
@@ -1045,15 +1133,23 @@ func (s *MonitoringCallbackService) CancelTaskForDesktopClient(
|
||||
if marshalErr != nil {
|
||||
return nil, response.ErrInternal(50041, "payload_encode_failed", "failed to encode phase2 desktop cancel payload")
|
||||
}
|
||||
taskRepo := repository.NewDesktopTaskRepository(s.businessPool)
|
||||
canceledTask, cancelErr := taskRepo.CancelByLease(ctx, desktopExecution.DesktopTaskID, desktopExecution.LeaseTokenHash, errorJSON)
|
||||
if cancelErr != nil {
|
||||
if errors.Is(cancelErr, pgx.ErrNoRows) {
|
||||
return nil, response.ErrConflict(40941, "task_status_invalid", "phase2 monitor desktop task is no longer active")
|
||||
if isMonitoringInfrastructureCancelReason(req.Reason) {
|
||||
requeuedTask, requeueErr := s.requeueDesktopMonitorTaskAfterInfrastructureCancel(ctx, desktopExecution, errorJSON)
|
||||
if requeueErr != nil {
|
||||
return nil, requeueErr
|
||||
}
|
||||
return nil, response.ErrInternal(50041, "desktop_task_cancel_failed", "failed to cancel phase2 monitor desktop task lease")
|
||||
desktopTaskCanceled = requeuedTask
|
||||
} else {
|
||||
taskRepo := repository.NewDesktopTaskRepository(s.businessPool)
|
||||
canceledTask, cancelErr := taskRepo.CancelByLease(ctx, desktopExecution.DesktopTaskID, desktopExecution.LeaseTokenHash, errorJSON)
|
||||
if cancelErr != nil {
|
||||
if errors.Is(cancelErr, pgx.ErrNoRows) {
|
||||
return nil, response.ErrConflict(40941, "task_status_invalid", "phase2 monitor desktop task is no longer active")
|
||||
}
|
||||
return nil, response.ErrInternal(50041, "desktop_task_cancel_failed", "failed to cancel phase2 monitor desktop task lease")
|
||||
}
|
||||
desktopTaskCanceled = canceledTask
|
||||
}
|
||||
desktopTaskCanceled = canceledTask
|
||||
} else if task.Status != "leased" {
|
||||
return nil, response.ErrConflict(40941, "task_status_invalid", "task status does not allow lease cancel")
|
||||
}
|
||||
@@ -1066,11 +1162,11 @@ func (s *MonitoringCallbackService) CancelTaskForDesktopClient(
|
||||
leased_at = NULL,
|
||||
lease_expires_at = NULL,
|
||||
callback_received_at = NULL,
|
||||
dispatch_after = NOW(),
|
||||
dispatch_after = CASE WHEN $3::boolean THEN NOW() + interval '30 seconds' ELSE NOW() END,
|
||||
error_message = $2,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
`, task.ID, nullableString(req.Reason)); err != nil {
|
||||
`, task.ID, nullableString(req.Reason), isMonitoringInfrastructureCancelReason(req.Reason)); err != nil {
|
||||
return nil, response.ErrInternal(50041, "task_update_failed", "failed to cancel monitoring task lease")
|
||||
}
|
||||
|
||||
@@ -1106,6 +1202,72 @@ func (s *MonitoringCallbackService) CancelTaskForDesktopClient(
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) requeueDesktopMonitorTaskAfterInfrastructureCancel(
|
||||
ctx context.Context,
|
||||
lease *monitoringDesktopExecutionLease,
|
||||
errorJSON []byte,
|
||||
) (*repository.DesktopTask, error) {
|
||||
if s == nil || s.businessPool == nil || lease == nil {
|
||||
return nil, response.ErrInternal(50041, "desktop_task_requeue_failed", "failed to requeue phase2 monitor desktop task lease")
|
||||
}
|
||||
|
||||
tx, err := s.businessPool.Begin(ctx)
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(50041, "desktop_task_requeue_begin_failed", "failed to start phase2 monitor desktop task requeue")
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
repo := repository.NewDesktopTaskRepository(tx)
|
||||
row := tx.QueryRow(ctx, `
|
||||
UPDATE desktop_tasks
|
||||
SET status = 'queued',
|
||||
error = $3,
|
||||
result = NULL,
|
||||
active_attempt_id = NULL,
|
||||
lease_token_hash = NULL,
|
||||
lease_expires_at = NULL,
|
||||
interrupted_at = COALESCE(interrupted_at, NOW()),
|
||||
interrupt_reason = COALESCE(interrupt_reason, 'desktop_infra_unavailable'),
|
||||
enqueued_at = NOW(),
|
||||
updated_at = NOW()
|
||||
WHERE desktop_id = $1
|
||||
AND lease_token_hash = $2
|
||||
AND kind = 'monitor'
|
||||
AND status = 'in_progress'
|
||||
RETURNING `+desktopTaskRepositoryReturningColumns,
|
||||
lease.DesktopTaskID,
|
||||
lease.LeaseTokenHash,
|
||||
errorJSON,
|
||||
)
|
||||
task, scanErr := scanRepositoryDesktopTask(row)
|
||||
if scanErr != nil {
|
||||
if errors.Is(scanErr, pgx.ErrNoRows) {
|
||||
return nil, response.ErrConflict(40941, "task_status_invalid", "phase2 monitor desktop task is no longer active")
|
||||
}
|
||||
return nil, response.ErrInternal(50041, "desktop_task_requeue_failed", "failed to requeue phase2 monitor desktop task lease")
|
||||
}
|
||||
|
||||
if lease.ActiveAttemptID.Valid {
|
||||
attemptID, convErr := uuid.FromBytes(lease.ActiveAttemptID.Bytes[:])
|
||||
if convErr != nil {
|
||||
return nil, response.ErrInternal(50041, "desktop_task_attempt_decode_failed", "failed to decode phase2 monitor desktop task attempt")
|
||||
}
|
||||
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
|
||||
TaskID: task.DesktopID,
|
||||
AttemptID: attemptID,
|
||||
FinalStatus: literalStringPtr("aborted"),
|
||||
Error: errorJSON,
|
||||
}); finishErr != nil {
|
||||
return nil, response.ErrInternal(50041, "desktop_task_attempt_finish_failed", "failed to finish phase2 monitor desktop task attempt")
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, response.ErrInternal(50041, "desktop_task_requeue_commit_failed", "failed to commit phase2 monitor desktop task requeue")
|
||||
}
|
||||
return task, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) ProcessQueuedTaskResult(ctx context.Context, envelope monitoringTaskResultEnvelope) (*MonitoringTaskResultResponse, error) {
|
||||
if envelope.TaskID <= 0 {
|
||||
return nil, response.ErrBadRequest(40041, "invalid_task_id", "task id is required")
|
||||
@@ -1393,6 +1555,7 @@ func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantI
|
||||
completed_at,
|
||||
skip_reason,
|
||||
trigger_source,
|
||||
COALESCE(dispatch_attempts, 0) AS dispatch_attempts,
|
||||
COALESCE(target_client_id::text, '') AS target_client_id,
|
||||
COALESCE(NULLIF(t.question_text_snapshot, ''), (
|
||||
SELECT question_text_snapshot
|
||||
@@ -1430,6 +1593,7 @@ func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantI
|
||||
&item.CompletedAt,
|
||||
&item.SkipReason,
|
||||
&item.TriggerSource,
|
||||
&item.DispatchAttempts,
|
||||
&item.TargetClientID,
|
||||
&item.QuestionText,
|
||||
); err != nil {
|
||||
@@ -1464,6 +1628,7 @@ func (s *MonitoringCallbackService) loadCollectTaskByID(ctx context.Context, tas
|
||||
completed_at,
|
||||
skip_reason,
|
||||
trigger_source,
|
||||
COALESCE(dispatch_attempts, 0) AS dispatch_attempts,
|
||||
COALESCE(target_client_id::text, '') AS target_client_id,
|
||||
COALESCE(NULLIF(t.question_text_snapshot, ''), (
|
||||
SELECT question_text_snapshot
|
||||
@@ -1500,6 +1665,7 @@ func (s *MonitoringCallbackService) loadCollectTaskByID(ctx context.Context, tas
|
||||
&item.CompletedAt,
|
||||
&item.SkipReason,
|
||||
&item.TriggerSource,
|
||||
&item.DispatchAttempts,
|
||||
&item.TargetClientID,
|
||||
&item.QuestionText,
|
||||
); err != nil {
|
||||
@@ -1518,23 +1684,25 @@ func (s *MonitoringCallbackService) loadActiveDesktopMonitorExecutionLease(
|
||||
) (*monitoringDesktopExecutionLease, error) {
|
||||
item := &monitoringDesktopExecutionLease{}
|
||||
if err := s.businessPool.QueryRow(ctx, `
|
||||
SELECT
|
||||
desktop_id,
|
||||
status,
|
||||
target_client_id,
|
||||
lease_token_hash,
|
||||
interrupt_generation
|
||||
SELECT
|
||||
desktop_id,
|
||||
status,
|
||||
target_client_id,
|
||||
lease_token_hash,
|
||||
active_attempt_id,
|
||||
interrupt_generation
|
||||
FROM desktop_tasks
|
||||
WHERE kind = 'monitor'
|
||||
AND monitor_task_id = $1
|
||||
AND status IN ('queued', 'in_progress')
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT 1
|
||||
`, monitorTaskID).Scan(
|
||||
`, monitorTaskID).Scan(
|
||||
&item.DesktopTaskID,
|
||||
&item.Status,
|
||||
&item.TargetClientID,
|
||||
&item.LeaseTokenHash,
|
||||
&item.ActiveAttemptID,
|
||||
&item.InterruptGeneration,
|
||||
); err != nil {
|
||||
if err == pgx.ErrNoRows {
|
||||
|
||||
@@ -225,12 +225,12 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
|
||||
|
||||
repo := repository.NewDesktopTaskRepository(tx)
|
||||
publishableTasks := make([]*repository.DesktopTask, 0, len(specs))
|
||||
fallbackLegacyTaskIDs := make([]int64, 0)
|
||||
deferredTaskIDs := make([]int64, 0)
|
||||
for _, spec := range specs {
|
||||
if spec.TargetAccountID == uuid.Nil || spec.TargetClientID == uuid.Nil {
|
||||
target, ok := targets[normalizeMonitoringPlatformID(spec.PlatformID)]
|
||||
if !ok || target.AccountID == uuid.Nil || target.ClientID == uuid.Nil {
|
||||
fallbackLegacyTaskIDs = append(fallbackLegacyTaskIDs, spec.MonitorTaskID)
|
||||
deferredTaskIDs = append(deferredTaskIDs, spec.MonitorTaskID)
|
||||
continue
|
||||
}
|
||||
spec.TargetAccountID = target.AccountID
|
||||
@@ -242,7 +242,7 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
|
||||
return nil, nil, upsertErr
|
||||
}
|
||||
if fallbackLegacy {
|
||||
fallbackLegacyTaskIDs = append(fallbackLegacyTaskIDs, spec.MonitorTaskID)
|
||||
deferredTaskIDs = append(deferredTaskIDs, spec.MonitorTaskID)
|
||||
continue
|
||||
}
|
||||
if shouldPublish && task != nil {
|
||||
@@ -250,10 +250,16 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
|
||||
}
|
||||
}
|
||||
|
||||
if len(deferredTaskIDs) > 0 {
|
||||
if err := deferPhase2MonitorCollectTasks(ctx, tx, deferredTaskIDs, defaultMonitoringDailyDesktopRetryCooldown); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, nil, response.ErrInternal(50116, "desktop_task_commit_failed", "failed to commit phase2 monitor desktop dispatch")
|
||||
}
|
||||
return publishableTasks, fallbackLegacyTaskIDs, nil
|
||||
return publishableTasks, deferredTaskIDs, nil
|
||||
}
|
||||
|
||||
func monitorDesktopTaskSpecsNeedTargetLookup(specs []monitorDesktopTaskSpec) bool {
|
||||
@@ -738,6 +744,35 @@ func (s *MonitoringService) fallbackMonitorDesktopTasksToLegacy(ctx context.Cont
|
||||
return nil
|
||||
}
|
||||
|
||||
func deferPhase2MonitorCollectTasks(ctx context.Context, tx pgx.Tx, taskIDs []int64, delay time.Duration) error {
|
||||
if tx == nil || len(taskIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
if delay <= 0 {
|
||||
delay = defaultMonitoringDailyDesktopRetryCooldown
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE monitoring_collect_tasks
|
||||
SET status = 'pending',
|
||||
lease_token_hash = NULL,
|
||||
leased_to_executor = NULL,
|
||||
leased_at = NULL,
|
||||
lease_expires_at = NULL,
|
||||
callback_received_at = NULL,
|
||||
completed_at = NULL,
|
||||
skip_reason = NULL,
|
||||
dispatch_after = NOW() + $2::interval,
|
||||
error_message = NULL,
|
||||
updated_at = NOW()
|
||||
WHERE id = ANY($1::bigint[])
|
||||
AND COALESCE(execution_owner, 'legacy') = 'desktop_tasks'
|
||||
AND callback_received_at IS NULL
|
||||
`, uniqueInt64s(taskIDs), formatPgInterval(delay)); err != nil {
|
||||
return response.ErrInternal(50041, "task_defer_failed", "failed to defer phase2 monitor tasks until a desktop client is online")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MonitoringService) deferQueuedPhase2MonitorTasks(
|
||||
ctx context.Context,
|
||||
targetClientID uuid.UUID,
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -193,15 +194,37 @@ func TestCompleteLinkedDesktopMonitorTaskOnlyFinalizesInProgress(t *testing.T) {
|
||||
assert.NotContains(t, query, "'unknown'")
|
||||
}
|
||||
|
||||
func TestMonitoringSkipOutcomeForRuntimeUnknownFailsTask(t *testing.T) {
|
||||
outcome := monitoringSkipOutcomeForReason("runtime_unknown")
|
||||
func TestMonitoringSkipOutcomeForRuntimeUnknownRequeuesAutomaticTask(t *testing.T) {
|
||||
outcome := monitoringSkipOutcomeForReason("runtime_unknown", 1, "automatic")
|
||||
|
||||
assert.Equal(t, "pending", outcome.TaskStatus)
|
||||
assert.True(t, outcome.Requeue)
|
||||
assert.Equal(t, 2*time.Minute, outcome.RetryAfter)
|
||||
assert.Empty(t, outcome.StoredSkipReason)
|
||||
}
|
||||
|
||||
func TestMonitoringSkipOutcomeForRuntimeUnknownFailsAfterMaxAttempts(t *testing.T) {
|
||||
outcome := monitoringSkipOutcomeForReason(
|
||||
"runtime_unknown",
|
||||
monitoringRuntimeUnknownMaxDispatchAttempts,
|
||||
"automatic",
|
||||
)
|
||||
|
||||
assert.Equal(t, "failed", outcome.TaskStatus)
|
||||
assert.False(t, outcome.Requeue)
|
||||
assert.Empty(t, outcome.StoredSkipReason)
|
||||
}
|
||||
|
||||
func TestMonitoringSkipOutcomeForManualRuntimeUnknownFailsTask(t *testing.T) {
|
||||
outcome := monitoringSkipOutcomeForReason("runtime_unknown", 0, "manual")
|
||||
|
||||
assert.Equal(t, "failed", outcome.TaskStatus)
|
||||
assert.False(t, outcome.Requeue)
|
||||
assert.Empty(t, outcome.StoredSkipReason)
|
||||
}
|
||||
|
||||
func TestMonitoringSkipOutcomeKeepsOrdinarySkip(t *testing.T) {
|
||||
outcome := monitoringSkipOutcomeForReason("stale_business_date")
|
||||
outcome := monitoringSkipOutcomeForReason("stale_business_date", 0, "automatic")
|
||||
|
||||
assert.Equal(t, "skipped", outcome.TaskStatus)
|
||||
assert.Equal(t, "stale_business_date", outcome.StoredSkipReason)
|
||||
|
||||
@@ -993,24 +993,21 @@ func (s *MonitoringService) CollectNow(
|
||||
phase2DispatchReady := executionOwner == "desktop_tasks"
|
||||
if len(phase2TaskSpecs) > 0 {
|
||||
phase2TaskCount := len(phase2TaskSpecs)
|
||||
publishedTasks, fallbackLegacyTaskIDs, dispatchErr := s.dispatchMonitorDesktopTasks(ctx, phase2TaskSpecs)
|
||||
publishedTasks, deferredTaskIDs, dispatchErr := s.dispatchMonitorDesktopTasks(ctx, phase2TaskSpecs)
|
||||
if dispatchErr != nil {
|
||||
if fallbackErr := s.fallbackMonitorDesktopTasksToLegacy(ctx, monitorDesktopTaskSpecIDs(phase2TaskSpecs)); fallbackErr != nil {
|
||||
return nil, fallbackErr
|
||||
}
|
||||
phase2DispatchReady = false
|
||||
phase2TaskSpecs = nil
|
||||
if s.logger != nil {
|
||||
s.logger.Warn("phase2 monitor desktop dispatch failed, falling back to legacy execution",
|
||||
s.logger.Warn("phase2 monitor desktop dispatch failed; pending tasks will retry when a desktop client is online",
|
||||
zap.Error(dispatchErr),
|
||||
zap.Int("phase2_task_count", phase2TaskCount),
|
||||
)
|
||||
}
|
||||
} else {
|
||||
if len(fallbackLegacyTaskIDs) > 0 {
|
||||
if fallbackErr := s.fallbackMonitorDesktopTasksToLegacy(ctx, fallbackLegacyTaskIDs); fallbackErr != nil {
|
||||
return nil, fallbackErr
|
||||
}
|
||||
if len(deferredTaskIDs) > 0 && s.logger != nil {
|
||||
s.logger.Info("phase2 monitor desktop tasks deferred until a desktop client is online",
|
||||
zap.Int("deferred_task_count", len(deferredTaskIDs)),
|
||||
)
|
||||
}
|
||||
phase2PublishedTasks = publishedTasks
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user