fix(publish): prevent stuck publish queue and duplicate posting
Desktop Client Build / Resolve Build Metadata (push) Successful in 41s
Backend CI / Backend (push) Failing after 7m14s
Desktop Client Build / Build Desktop Client (push) Successful in 23m46s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 56s
Desktop Client Build / Resolve Build Metadata (push) Successful in 41s
Backend CI / Backend (push) Failing after 7m14s
Desktop Client Build / Build Desktop Client (push) Successful in 23m46s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 56s
The desktop publish queue could stall for a long time and end users assumed the software was broken. Root cause: a hung adapter held its execution slot with no wall-clock timeout while auto-renewing its lease forever, so the server never reclaimed it and every queued task behind it stayed 等待发布. Auto-recovery also risked silently re-posting a non-idempotent article. Client (Electron): - per-task wall-clock deadline + abort; progress-gated lease renewal that stops and aborts a stalled task instead of renewing it forever - decouple the concurrency cap from CDP-induced CPU/memory pressure (admission gate instead of self-throttling collapse); 15s watchdog pump - all adapter network I/O now has fetch timeouts and honors context.signal; bounded image-upload concurrency with per-image timeout - surface live adapter progress, elapsed time, queue position and a working-vs-queued distinction in the publish view Server (tenant-api): - publish lease-recovery worker (every 3m) + supporting index: reclaim expired in_progress publish leases, requeue (<3 attempts) or terminal-fail - 3-minute lease TTL with client-presence-gated extension; max 3 attempts Idempotency (production-grade core): - durable publish_submit_started_at marker, set before the irreversible platform submit POST; recovery and abort route a maybe-submitted task to unknown (manual reconcile, kept in the dedup set) instead of re-posting - desktop UI requires explicit confirmation before retrying a possibly- already-published task Verified: go build/vet/test (incl. resolvePublishRecoveryOutcome), vue-tsc, vitest 141/141, gofmt all clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -17,6 +17,8 @@ func TestDesktopTaskRecoverySelectQueryMatchesRecoveredLeaseScanOrder(t *testing
|
||||
"kind",
|
||||
"status",
|
||||
"active_attempt_id",
|
||||
"attempts",
|
||||
"publish_submit_started_at",
|
||||
}
|
||||
|
||||
if len(gotColumns) != len(wantColumns) {
|
||||
@@ -55,6 +57,45 @@ func TestReconcileRetryPublishesTaskAvailableForAllDesktopTaskKinds(t *testing.T
|
||||
}
|
||||
}
|
||||
|
||||
func TestDesktopTaskRecoveryPayloadPublishLeaseExpiryRequeuesInsteadOfUnknown(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
payload, reason, message, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "publish")
|
||||
if err != nil {
|
||||
t.Fatalf("desktopTaskRecoveryPayload returned error: %v", err)
|
||||
}
|
||||
if reason != "lease_expired" {
|
||||
t.Fatalf("reason = %q, want lease_expired", reason)
|
||||
}
|
||||
if !strings.Contains(message, "re-queued for retry") {
|
||||
t.Fatalf("message = %q, want re-queued for retry", message)
|
||||
}
|
||||
if strings.Contains(message, "unknown") {
|
||||
t.Fatalf("message must not move publish recovery to unknown: %q", message)
|
||||
}
|
||||
if !strings.Contains(string(payload), "desktop_task_lease_expiry") {
|
||||
t.Fatalf("payload missing source: %s", payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDesktopTaskRecoveryPayloadPublishFinalFailsAfterMaxAttempts(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
payload, _, message, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "publish_final")
|
||||
if err != nil {
|
||||
t.Fatalf("desktopTaskRecoveryPayload returned error: %v", err)
|
||||
}
|
||||
if !strings.Contains(message, "marked failed") {
|
||||
t.Fatalf("message = %q, want marked failed", message)
|
||||
}
|
||||
if !strings.Contains(message, "3 attempts") {
|
||||
t.Fatalf("message = %q, want max attempts", message)
|
||||
}
|
||||
if !strings.Contains(string(payload), "marked failed") {
|
||||
t.Fatalf("payload missing final failure message: %s", payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileNonRetryPublishesTaskReconciled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -63,6 +104,38 @@ func TestReconcileNonRetryPublishesTaskReconciled(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePublishRecoveryOutcome(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
submitStarted bool
|
||||
attempts int
|
||||
wantStatus string
|
||||
wantKind string
|
||||
}{
|
||||
{"pre-submit first attempt requeues", false, 1, "queued", "publish"},
|
||||
{"pre-submit just below cap requeues", false, desktopPublishMaxAttempts - 1, "queued", "publish"},
|
||||
{"pre-submit at cap fails terminally", false, desktopPublishMaxAttempts, "failed", "publish_final"},
|
||||
{"pre-submit above cap fails terminally", false, desktopPublishMaxAttempts + 1, "failed", "publish_final"},
|
||||
// Submit may have happened: never auto-requeue or silently fail — route to unknown
|
||||
// (manual reconcile) regardless of remaining attempts, to avoid duplicate publishing.
|
||||
{"submit started first attempt is uncertain", true, 1, "unknown", "publish_uncertain"},
|
||||
{"submit started at cap is still uncertain", true, desktopPublishMaxAttempts, "unknown", "publish_uncertain"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
gotStatus, gotKind := resolvePublishRecoveryOutcome(tc.submitStarted, tc.attempts)
|
||||
if gotStatus != tc.wantStatus || gotKind != tc.wantKind {
|
||||
t.Fatalf("resolvePublishRecoveryOutcome(%v, %d) = (%q, %q), want (%q, %q)",
|
||||
tc.submitStarted, tc.attempts, gotStatus, gotKind, tc.wantStatus, tc.wantKind)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func recoverDesktopTaskSelectColumns(query string) []string {
|
||||
re := regexp.MustCompile(`(?is)select\s+(.*?)\s+from\s+desktop_tasks`)
|
||||
match := re.FindStringSubmatch(query)
|
||||
|
||||
Reference in New Issue
Block a user