fix: fail expired monitor desktop leases
Frontend CI / Frontend (push) Successful in 2m26s
Backend CI / Backend (push) Failing after 8m42s

This commit is contained in:
2026-06-23 01:03:35 +08:00
parent 8e92737bc2
commit d458dd7477
8 changed files with 514 additions and 21 deletions
@@ -23,6 +23,7 @@ func TestDesktopTaskRecoverySelectQueryMatchesRecoveredLeaseScanOrder(t *testing
"active_attempt_id",
"attempts",
"publish_submit_started_at",
"monitor_task_id",
}
if len(gotColumns) != len(wantColumns) {
@@ -100,6 +101,27 @@ func TestDesktopTaskRecoveryPayloadPublishFinalFailsAfterMaxAttempts(t *testing.
}
}
func TestDesktopTaskRecoveryPayloadMonitorLeaseExpiryFails(t *testing.T) {
t.Parallel()
payload, reason, message, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "monitor_final")
if err != nil {
t.Fatalf("desktopTaskRecoveryPayload returned error: %v", err)
}
if reason != "lease_expired" {
t.Fatalf("reason = %q, want lease_expired", reason)
}
if !strings.Contains(message, "marked failed") {
t.Fatalf("message = %q, want marked failed", message)
}
if strings.Contains(message, "re-queued") {
t.Fatalf("message must not requeue expired monitor tasks: %q", message)
}
if !strings.Contains(string(payload), "desktop_task_lease_expiry") {
t.Fatalf("payload missing source: %s", payload)
}
}
func TestReconcileNonRetryPublishesTaskReconciled(t *testing.T) {
t.Parallel()
@@ -140,6 +162,35 @@ func TestResolvePublishRecoveryOutcome(t *testing.T) {
}
}
func TestResolveMonitorRecoveryOutcome(t *testing.T) {
t.Parallel()
cases := []struct {
name string
mode desktopTaskRecoveryMode
attempts int
wantStatus string
wantKind string
}{
{"startup first attempt requeues", desktopTaskRecoveryModeStartup, 1, "queued", "monitor"},
{"disconnect first attempt requeues", desktopTaskRecoveryModeDisconnect, 1, "queued", "monitor"},
{"startup repeated recovery fails", desktopTaskRecoveryModeStartup, 2, "failed", "monitor_final"},
{"disconnect repeated recovery fails", desktopTaskRecoveryModeDisconnect, 2, "failed", "monitor_final"},
{"lease expiry fails immediately", desktopTaskRecoveryModeLeaseExpiry, 1, "failed", "monitor_final"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
gotStatus, gotKind := resolveMonitorRecoveryOutcome(tc.mode, tc.attempts)
if gotStatus != tc.wantStatus || gotKind != tc.wantKind {
t.Fatalf("resolveMonitorRecoveryOutcome(%q, %d) = (%q, %q), want (%q, %q)",
tc.mode, tc.attempts, gotStatus, gotKind, tc.wantStatus, tc.wantKind)
}
})
}
}
func TestNormalizeDesktopTaskCompletionStatusForKindFailsMonitorUnknown(t *testing.T) {
t.Parallel()
@@ -151,6 +202,46 @@ func TestNormalizeDesktopTaskCompletionStatusForKindFailsMonitorUnknown(t *testi
}
}
func TestExpiredMonitorDesktopTasksCandidateSQLSelectsOnlyExpiredInProgressMonitorTasks(t *testing.T) {
t.Parallel()
normalized := normalizeSQLForDesktopTaskTest(expiredMonitorDesktopTasksCandidateSQL())
for _, fragment := range []string{
"t.kind = 'monitor'",
"t.status = 'in_progress'",
"t.monitor_task_id IS NOT NULL",
"t.lease_expires_at IS NOT NULL",
"t.lease_expires_at < NOW()",
"ORDER BY t.lease_expires_at ASC",
"FOR UPDATE OF t SKIP LOCKED",
} {
if !strings.Contains(normalized, fragment) {
t.Fatalf("expired monitor candidate query missing %q: %s", fragment, normalized)
}
}
}
func TestFailExpiredMonitorDesktopTasksSQLMarksTerminalFailure(t *testing.T) {
t.Parallel()
normalized := normalizeSQLForDesktopTaskTest(failExpiredMonitorDesktopTasksSQL())
for _, fragment := range []string{
"SET status = 'failed'",
"active_attempt_id = NULL",
"lease_token_hash = NULL",
"lease_expires_at = NULL",
"interrupt_reason = COALESCE(interrupt_reason, $3)",
"WHERE desktop_id = ANY($1::uuid[])",
"AND kind = 'monitor'",
"AND status = 'in_progress'",
"RETURNING desktop_id",
} {
if !strings.Contains(normalized, fragment) {
t.Fatalf("expired monitor update query missing %q: %s", fragment, normalized)
}
}
}
func TestClassifyLeaseErrorDefersQueuedMonitorTask(t *testing.T) {
t.Parallel()