fix(publish): prevent stuck publish queue and duplicate posting
Desktop Client Build / Resolve Build Metadata (push) Successful in 41s
Backend CI / Backend (push) Failing after 7m14s
Desktop Client Build / Build Desktop Client (push) Successful in 23m46s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 56s
Desktop Client Build / Resolve Build Metadata (push) Successful in 41s
Backend CI / Backend (push) Failing after 7m14s
Desktop Client Build / Build Desktop Client (push) Successful in 23m46s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 56s
The desktop publish queue could stall for a long time and end users assumed the software was broken. Root cause: a hung adapter held its execution slot with no wall-clock timeout while auto-renewing its lease forever, so the server never reclaimed it and every queued task behind it stayed 等待发布. Auto-recovery also risked silently re-posting a non-idempotent article. Client (Electron): - per-task wall-clock deadline + abort; progress-gated lease renewal that stops and aborts a stalled task instead of renewing it forever - decouple the concurrency cap from CDP-induced CPU/memory pressure (admission gate instead of self-throttling collapse); 15s watchdog pump - all adapter network I/O now has fetch timeouts and honors context.signal; bounded image-upload concurrency with per-image timeout - surface live adapter progress, elapsed time, queue position and a working-vs-queued distinction in the publish view Server (tenant-api): - publish lease-recovery worker (every 3m) + supporting index: reclaim expired in_progress publish leases, requeue (<3 attempts) or terminal-fail - 3-minute lease TTL with client-presence-gated extension; max 3 attempts Idempotency (production-grade core): - durable publish_submit_started_at marker, set before the irreversible platform submit POST; recovery and abort route a maybe-submitted task to unknown (manual reconcile, kept in the dedup set) instead of re-posting - desktop UI requires explicit confirmation before retrying a possibly- already-published task Verified: go build/vet/test (incl. resolvePublishRecoveryOutcome), vue-tsc, vitest 141/141, gofmt all clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -24,6 +25,8 @@ import (
|
||||
"github.com/geo-platform/tenant-api/internal/tenant/repository"
|
||||
)
|
||||
|
||||
const desktopPublishMaxAttempts = 3
|
||||
|
||||
type DesktopTaskService struct {
|
||||
pool *pgxpool.Pool
|
||||
monitoringPool *pgxpool.Pool
|
||||
@@ -107,6 +110,10 @@ type ExtendDesktopTaskRequest struct {
|
||||
LeaseToken string `json:"lease_token" binding:"required"`
|
||||
}
|
||||
|
||||
type MarkPublishSubmitStartedRequest struct {
|
||||
LeaseToken string `json:"lease_token" binding:"required"`
|
||||
}
|
||||
|
||||
type CompleteDesktopTaskRequest struct {
|
||||
LeaseToken string `json:"lease_token" binding:"required"`
|
||||
Status string `json:"status" binding:"required,oneof=succeeded failed unknown"`
|
||||
@@ -200,8 +207,10 @@ func (s *DesktopTaskService) Lease(ctx context.Context, client *repository.Deskt
|
||||
task, err = s.leaseQueuedDesktopTaskByID(ctx, client, *taskID, leaseParams)
|
||||
case req.Kind != nil && strings.TrimSpace(*req.Kind) == "monitor":
|
||||
task, err = s.leaseNextQueuedMonitorTask(ctx, client, leaseParams)
|
||||
case req.Kind != nil && strings.TrimSpace(*req.Kind) == "publish":
|
||||
task, err = s.leaseNextQueuedPublishTask(ctx, client, leaseParams)
|
||||
default:
|
||||
task, err = s.repo.LeaseNextQueued(ctx, client.ID, req.Kind, leaseParams)
|
||||
task, err = s.leaseNextQueuedPublishTask(ctx, client, leaseParams)
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
@@ -438,6 +447,54 @@ func (s *DesktopTaskService) leaseNextQueuedMonitorTask(
|
||||
return scanRepositoryDesktopTask(row)
|
||||
}
|
||||
|
||||
func (s *DesktopTaskService) leaseNextQueuedPublishTask(
|
||||
ctx context.Context,
|
||||
client *repository.DesktopClient,
|
||||
params repository.DesktopTaskLeaseParams,
|
||||
) (*repository.DesktopTask, error) {
|
||||
row := s.pool.QueryRow(ctx, `
|
||||
WITH candidate AS (
|
||||
SELECT dt.desktop_id
|
||||
FROM desktop_tasks AS dt
|
||||
WHERE dt.target_client_id = $1
|
||||
AND dt.kind = 'publish'
|
||||
AND dt.status = 'queued'
|
||||
AND dt.attempts < $4
|
||||
AND (
|
||||
NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM desktop_publish_jobs AS j
|
||||
WHERE j.desktop_id = dt.job_id
|
||||
AND j.status <> 'queued'
|
||||
)
|
||||
)
|
||||
ORDER BY dt.lane_weight DESC,
|
||||
dt.priority DESC,
|
||||
COALESCE(dt.enqueued_at, dt.created_at) ASC,
|
||||
dt.created_at ASC,
|
||||
dt.desktop_id ASC
|
||||
LIMIT 1
|
||||
FOR UPDATE OF dt SKIP LOCKED
|
||||
)
|
||||
UPDATE desktop_tasks AS t
|
||||
SET active_attempt_id = $2,
|
||||
lease_token_hash = $3,
|
||||
lease_expires_at = now() + interval '3 minutes',
|
||||
status = 'in_progress',
|
||||
attempts = t.attempts + 1,
|
||||
started_at = COALESCE(t.started_at, now()),
|
||||
updated_at = now()
|
||||
FROM candidate
|
||||
WHERE t.desktop_id = candidate.desktop_id
|
||||
RETURNING `+desktopTaskRepositoryReturningColumns,
|
||||
client.ID,
|
||||
params.AttemptID,
|
||||
params.LeaseTokenHash,
|
||||
desktopPublishMaxAttempts,
|
||||
)
|
||||
return scanRepositoryDesktopTask(row)
|
||||
}
|
||||
|
||||
func (s *DesktopTaskService) leaseQueuedDesktopTaskByID(
|
||||
ctx context.Context,
|
||||
client *repository.DesktopClient,
|
||||
@@ -457,6 +514,7 @@ func (s *DesktopTaskService) leaseQueuedDesktopTaskByID(
|
||||
WHERE dt.desktop_id = $1
|
||||
AND dt.workspace_id = $2
|
||||
AND dt.status = 'queued'
|
||||
AND (dt.kind <> 'publish' OR dt.attempts < $9)
|
||||
AND (
|
||||
(
|
||||
dt.kind = 'monitor'
|
||||
@@ -478,9 +536,10 @@ func (s *DesktopTaskService) leaseQueuedDesktopTaskByID(
|
||||
SET target_client_id = CASE WHEN t.kind = 'monitor' THEN $4 ELSE t.target_client_id END,
|
||||
active_attempt_id = $7,
|
||||
lease_token_hash = $8,
|
||||
lease_expires_at = now() + interval '10 minutes',
|
||||
lease_expires_at = now() + CASE WHEN t.kind = 'publish' THEN interval '3 minutes' ELSE interval '10 minutes' END,
|
||||
status = 'in_progress',
|
||||
attempts = t.attempts + 1,
|
||||
started_at = COALESCE(t.started_at, now()),
|
||||
updated_at = now()
|
||||
FROM candidate
|
||||
WHERE t.desktop_id = candidate.desktop_id
|
||||
@@ -493,6 +552,7 @@ func (s *DesktopTaskService) leaseQueuedDesktopTaskByID(
|
||||
accountIDs,
|
||||
params.AttemptID,
|
||||
params.LeaseTokenHash,
|
||||
desktopPublishMaxAttempts,
|
||||
)
|
||||
return scanRepositoryDesktopTask(row)
|
||||
}
|
||||
@@ -638,6 +698,12 @@ func (s *DesktopTaskService) Extend(ctx context.Context, client *repository.Desk
|
||||
if err := s.dropStaleMonitorTasksForClient(ctx, client); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.redis != nil {
|
||||
presence := loadDesktopClientPresence(ctx, s.redis, []uuid.UUID{client.ID})
|
||||
if presence != nil && !presence[client.ID] {
|
||||
return nil, response.ErrConflict(40986, "desktop_client_offline", "desktop client presence has expired")
|
||||
}
|
||||
}
|
||||
|
||||
task, err := s.repo.ExtendLease(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(req.LeaseToken)))
|
||||
if err != nil {
|
||||
@@ -653,6 +719,34 @@ func (s *DesktopTaskService) Extend(ctx context.Context, client *repository.Desk
|
||||
return &view, nil
|
||||
}
|
||||
|
||||
// MarkPublishSubmitStarted records a durable intent marker right before the client performs the
|
||||
// irreversible platform submit POST. Lease-recovery uses it to avoid silently re-posting a
|
||||
// non-idempotent article: once submit may have started, recovery routes the task to 'unknown'
|
||||
// (manual reconcile) instead of auto-requeueing it. Best-effort — a stale/lost lease is a no-op
|
||||
// and must never block the in-flight publish.
|
||||
func (s *DesktopTaskService) MarkPublishSubmitStarted(ctx context.Context, client *repository.DesktopClient, taskID uuid.UUID, leaseToken string) error {
|
||||
if s == nil || s.pool == nil || client == nil {
|
||||
return nil
|
||||
}
|
||||
leaseToken = strings.TrimSpace(leaseToken)
|
||||
if leaseToken == "" {
|
||||
return response.ErrBadRequest(40001, "invalid_params", "lease_token is required")
|
||||
}
|
||||
if _, err := s.pool.Exec(ctx, `
|
||||
UPDATE desktop_tasks
|
||||
SET publish_submit_started_at = COALESCE(publish_submit_started_at, NOW()),
|
||||
updated_at = NOW()
|
||||
WHERE desktop_id = $1
|
||||
AND target_client_id = $2
|
||||
AND kind = 'publish'
|
||||
AND status = 'in_progress'
|
||||
AND lease_token_hash = $3
|
||||
`, taskID, client.ID, HashDesktopClientToken(leaseToken)); err != nil {
|
||||
return response.ErrInternal(50200, "desktop_task_submit_marker_failed", "failed to mark publish submit started")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DesktopTaskService) Complete(ctx context.Context, client *repository.DesktopClient, desktopID uuid.UUID, req CompleteDesktopTaskRequest) (*DesktopTaskView, error) {
|
||||
if client == nil {
|
||||
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
|
||||
@@ -1320,6 +1414,33 @@ type recoveredDesktopTaskLease struct {
|
||||
Kind string
|
||||
Status string
|
||||
ActiveAttempt pgtype.UUID
|
||||
Attempts int
|
||||
SubmitStarted bool
|
||||
ErrorJSON []byte
|
||||
}
|
||||
|
||||
type PublishLeaseRecoveryResult struct {
|
||||
Requeued int `json:"requeued"`
|
||||
Failed int `json:"failed"`
|
||||
Uncertain int `json:"uncertain"`
|
||||
}
|
||||
|
||||
// resolvePublishRecoveryOutcome decides what to do with a publish task whose lease was lost.
|
||||
//
|
||||
// If the client had already entered the irreversible submit phase (publish_submit_started_at
|
||||
// is set), the article may already exist on the platform. Auto-requeueing would silently
|
||||
// re-post a non-idempotent article, so we route the task to 'unknown' (kept in the dedup
|
||||
// active set, surfaced for manual reconcile) instead. Tasks that never reached submit are
|
||||
// safe to auto-requeue while attempts remain, and give up to 'failed' once exhausted.
|
||||
func resolvePublishRecoveryOutcome(submitStarted bool, attempts int) (status string, payloadKind string) {
|
||||
switch {
|
||||
case submitStarted:
|
||||
return "unknown", "publish_uncertain"
|
||||
case attempts >= desktopPublishMaxAttempts:
|
||||
return "failed", "publish_final"
|
||||
default:
|
||||
return "queued", "publish"
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DesktopTaskService) recoverClientTasksOnStartup(ctx context.Context, client *repository.DesktopClient) error {
|
||||
@@ -1347,6 +1468,14 @@ func (s *DesktopTaskService) recoverClientTasks(
|
||||
if err != nil {
|
||||
return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
|
||||
}
|
||||
publishFinalErrorJSON, _, _, err := desktopTaskRecoveryPayload(mode, "publish_final")
|
||||
if err != nil {
|
||||
return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
|
||||
}
|
||||
publishUncertainErrorJSON, _, _, err := desktopTaskRecoveryPayload(mode, "publish_uncertain")
|
||||
if err != nil {
|
||||
return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
|
||||
}
|
||||
monitorErrorJSON, monitorReason, _, err := desktopTaskRecoveryPayload(mode, "monitor")
|
||||
if err != nil {
|
||||
return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode monitor desktop task recovery payload")
|
||||
@@ -1378,6 +1507,8 @@ func (s *DesktopTaskService) recoverClientTasks(
|
||||
&item.Kind,
|
||||
&item.Status,
|
||||
&item.ActiveAttempt,
|
||||
&item.Attempts,
|
||||
&item.SubmitStarted,
|
||||
); scanErr != nil {
|
||||
s.logWarn("desktop task recovery scan failed", scanErr, zap.String("client_id", client.ID.String()), zap.String("mode", string(mode)))
|
||||
return response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to parse recovered desktop tasks")
|
||||
@@ -1389,6 +1520,7 @@ func (s *DesktopTaskService) recoverClientTasks(
|
||||
return response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to iterate recovered desktop tasks")
|
||||
}
|
||||
|
||||
publishOutcomes := make([]*desktopPublishSyncOutcome, 0)
|
||||
for index := range recovered {
|
||||
item := &recovered[index]
|
||||
if item.Kind == "monitor" {
|
||||
@@ -1412,22 +1544,49 @@ func (s *DesktopTaskService) recoverClientTasks(
|
||||
continue
|
||||
}
|
||||
|
||||
finalStatus, payloadKind := resolvePublishRecoveryOutcome(item.SubmitStarted, item.Attempts)
|
||||
nextErrorJSON := publishErrorJSON
|
||||
switch payloadKind {
|
||||
case "publish_final":
|
||||
nextErrorJSON = publishFinalErrorJSON
|
||||
case "publish_uncertain":
|
||||
nextErrorJSON = publishUncertainErrorJSON
|
||||
}
|
||||
if _, execErr := tx.Exec(ctx, `
|
||||
UPDATE desktop_tasks
|
||||
SET status = 'unknown',
|
||||
error = $2,
|
||||
SET status = $2,
|
||||
error = $3,
|
||||
result = CASE WHEN $2 = 'queued' THEN NULL ELSE result END,
|
||||
active_attempt_id = NULL,
|
||||
lease_token_hash = NULL,
|
||||
lease_expires_at = NULL,
|
||||
interrupted_at = COALESCE(interrupted_at, NOW()),
|
||||
interrupt_reason = COALESCE(interrupt_reason, $3),
|
||||
interrupt_reason = COALESCE(interrupt_reason, $4),
|
||||
enqueued_at = CASE WHEN $2 = 'queued' THEN NOW() ELSE enqueued_at END,
|
||||
updated_at = NOW()
|
||||
WHERE desktop_id = $1
|
||||
`, item.TaskID, publishErrorJSON, publishReason); execErr != nil {
|
||||
`, item.TaskID, finalStatus, nextErrorJSON, publishReason); execErr != nil {
|
||||
s.logWarn("desktop publish task recovery update failed", execErr, zap.String("task_id", item.TaskID.String()), zap.String("mode", string(mode)))
|
||||
return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
|
||||
}
|
||||
item.Status = "unknown"
|
||||
item.Status = finalStatus
|
||||
item.ErrorJSON = nextErrorJSON
|
||||
// Only terminal 'failed' tasks finalize the publish_record; 'unknown' (may already be
|
||||
// published) is left in the dedup active set for manual reconcile, 'queued' is requeued.
|
||||
if finalStatus == "failed" {
|
||||
task, getErr := repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID)
|
||||
if getErr != nil {
|
||||
s.logWarn("desktop publish task recovery reload failed", getErr, zap.String("task_id", item.TaskID.String()), zap.String("mode", string(mode)))
|
||||
return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
|
||||
}
|
||||
publishOutcome, syncErr := syncDesktopPublishTaskState(ctx, tx, task)
|
||||
if syncErr != nil {
|
||||
return syncErr
|
||||
}
|
||||
if publishOutcome != nil {
|
||||
publishOutcomes = append(publishOutcomes, publishOutcome)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, item := range recovered {
|
||||
@@ -1442,7 +1601,10 @@ func (s *DesktopTaskService) recoverClientTasks(
|
||||
}
|
||||
|
||||
finalStatus := literalStringPtr(item.Status)
|
||||
errorJSON := publishErrorJSON
|
||||
errorJSON := item.ErrorJSON
|
||||
if len(errorJSON) == 0 {
|
||||
errorJSON = publishErrorJSON
|
||||
}
|
||||
if item.Kind == "monitor" {
|
||||
finalStatus = literalStringPtr("aborted")
|
||||
errorJSON = monitorErrorJSON
|
||||
@@ -1462,6 +1624,8 @@ func (s *DesktopTaskService) recoverClientTasks(
|
||||
return response.ErrInternal(50199, "desktop_task_recovery_commit_failed", "failed to commit desktop task recovery")
|
||||
}
|
||||
|
||||
s.afterRecoveredPublishOutcomes(ctx, publishOutcomes)
|
||||
|
||||
for _, item := range recovered {
|
||||
task, getErr := s.repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID)
|
||||
if getErr != nil {
|
||||
@@ -1470,7 +1634,7 @@ func (s *DesktopTaskService) recoverClientTasks(
|
||||
}
|
||||
|
||||
eventType := "task_completed"
|
||||
if item.Kind == "monitor" {
|
||||
if item.Status == "queued" {
|
||||
eventType = "task_available"
|
||||
}
|
||||
s.publishTaskEvent(ctx, task, eventType)
|
||||
@@ -1488,15 +1652,199 @@ func (s *DesktopTaskService) recoverClientTasks(
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DesktopTaskService) RecoverExpiredPublishTasks(ctx context.Context, limit int) (PublishLeaseRecoveryResult, error) {
|
||||
var result PublishLeaseRecoveryResult
|
||||
if s == nil || s.pool == nil {
|
||||
return result, nil
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 1000
|
||||
}
|
||||
|
||||
publishErrorJSON, publishReason, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "publish")
|
||||
if err != nil {
|
||||
return result, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
|
||||
}
|
||||
publishFinalErrorJSON, _, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "publish_final")
|
||||
if err != nil {
|
||||
return result, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
|
||||
}
|
||||
publishUncertainErrorJSON, _, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "publish_uncertain")
|
||||
if err != nil {
|
||||
return result, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
|
||||
}
|
||||
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return result, response.ErrInternal(50196, "desktop_task_recovery_begin_failed", "failed to start desktop task recovery")
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT
|
||||
t.desktop_id,
|
||||
t.workspace_id,
|
||||
t.kind,
|
||||
t.status,
|
||||
t.active_attempt_id,
|
||||
t.attempts,
|
||||
(t.publish_submit_started_at IS NOT NULL) AS submit_started
|
||||
FROM desktop_tasks AS t
|
||||
WHERE t.kind = 'publish'
|
||||
AND t.status = 'in_progress'
|
||||
AND t.lease_expires_at IS NOT NULL
|
||||
AND t.lease_expires_at < NOW()
|
||||
ORDER BY t.lease_expires_at ASC, t.updated_at ASC, t.desktop_id ASC
|
||||
LIMIT $1
|
||||
FOR UPDATE OF t SKIP LOCKED
|
||||
`, limit)
|
||||
if err != nil {
|
||||
s.logWarn("expired desktop publish recovery query failed", err)
|
||||
return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
recovered := make([]recoveredDesktopTaskLease, 0)
|
||||
for rows.Next() {
|
||||
var item recoveredDesktopTaskLease
|
||||
if scanErr := rows.Scan(
|
||||
&item.TaskID,
|
||||
&item.WorkspaceID,
|
||||
&item.Kind,
|
||||
&item.Status,
|
||||
&item.ActiveAttempt,
|
||||
&item.Attempts,
|
||||
&item.SubmitStarted,
|
||||
); scanErr != nil {
|
||||
s.logWarn("expired desktop publish recovery scan failed", scanErr)
|
||||
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to parse recovered desktop tasks")
|
||||
}
|
||||
recovered = append(recovered, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
s.logWarn("expired desktop publish recovery rows failed", err)
|
||||
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to iterate recovered desktop tasks")
|
||||
}
|
||||
|
||||
repo := repository.NewDesktopTaskRepository(tx)
|
||||
publishOutcomes := make([]*desktopPublishSyncOutcome, 0)
|
||||
for index := range recovered {
|
||||
item := &recovered[index]
|
||||
finalStatus, payloadKind := resolvePublishRecoveryOutcome(item.SubmitStarted, item.Attempts)
|
||||
nextErrorJSON := publishErrorJSON
|
||||
switch payloadKind {
|
||||
case "publish_final":
|
||||
nextErrorJSON = publishFinalErrorJSON
|
||||
case "publish_uncertain":
|
||||
nextErrorJSON = publishUncertainErrorJSON
|
||||
}
|
||||
if _, execErr := tx.Exec(ctx, `
|
||||
UPDATE desktop_tasks
|
||||
SET status = $2,
|
||||
error = $3,
|
||||
result = CASE WHEN $2 = 'queued' THEN NULL ELSE result END,
|
||||
active_attempt_id = NULL,
|
||||
lease_token_hash = NULL,
|
||||
lease_expires_at = NULL,
|
||||
interrupted_at = COALESCE(interrupted_at, NOW()),
|
||||
interrupt_reason = COALESCE(interrupt_reason, $4),
|
||||
enqueued_at = CASE WHEN $2 = 'queued' THEN NOW() ELSE enqueued_at END,
|
||||
updated_at = NOW()
|
||||
WHERE desktop_id = $1
|
||||
AND status = 'in_progress'
|
||||
`, item.TaskID, finalStatus, nextErrorJSON, publishReason); execErr != nil {
|
||||
s.logWarn("expired desktop publish recovery update failed", execErr, zap.String("task_id", item.TaskID.String()))
|
||||
return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
|
||||
}
|
||||
|
||||
item.Status = finalStatus
|
||||
item.ErrorJSON = nextErrorJSON
|
||||
switch finalStatus {
|
||||
case "failed":
|
||||
result.Failed++
|
||||
task, getErr := repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID)
|
||||
if getErr != nil {
|
||||
s.logWarn("expired desktop publish recovery reload failed", getErr, zap.String("task_id", item.TaskID.String()))
|
||||
return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
|
||||
}
|
||||
publishOutcome, syncErr := syncDesktopPublishTaskState(ctx, tx, task)
|
||||
if syncErr != nil {
|
||||
return result, syncErr
|
||||
}
|
||||
if publishOutcome != nil {
|
||||
publishOutcomes = append(publishOutcomes, publishOutcome)
|
||||
}
|
||||
case "unknown":
|
||||
// May already be published — keep as 'unknown' (still in the dedup active set) for
|
||||
// manual reconcile. Do NOT sync to a terminal publish_record state and do NOT requeue.
|
||||
result.Uncertain++
|
||||
default:
|
||||
result.Requeued++
|
||||
}
|
||||
}
|
||||
|
||||
for _, item := range recovered {
|
||||
if !item.ActiveAttempt.Valid {
|
||||
continue
|
||||
}
|
||||
|
||||
attemptID, convErr := uuid.FromBytes(item.ActiveAttempt.Bytes[:])
|
||||
if convErr != nil {
|
||||
s.logWarn("expired desktop publish recovery attempt id decode failed", convErr, zap.String("task_id", item.TaskID.String()))
|
||||
continue
|
||||
}
|
||||
|
||||
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
|
||||
TaskID: item.TaskID,
|
||||
AttemptID: attemptID,
|
||||
FinalStatus: literalStringPtr(item.Status),
|
||||
Error: item.ErrorJSON,
|
||||
}); finishErr != nil {
|
||||
s.logWarn("expired desktop publish recovery attempt finish failed", finishErr, zap.String("task_id", item.TaskID.String()))
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return result, response.ErrInternal(50199, "desktop_task_recovery_commit_failed", "failed to commit desktop task recovery")
|
||||
}
|
||||
|
||||
s.afterRecoveredPublishOutcomes(ctx, publishOutcomes)
|
||||
|
||||
for _, item := range recovered {
|
||||
task, getErr := s.repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID)
|
||||
if getErr != nil {
|
||||
s.logWarn("expired desktop publish recovery reload after commit failed", getErr, zap.String("task_id", item.TaskID.String()))
|
||||
continue
|
||||
}
|
||||
eventType := "task_completed"
|
||||
if item.Status == "queued" {
|
||||
eventType = "task_available"
|
||||
}
|
||||
s.publishTaskEvent(ctx, task, eventType)
|
||||
}
|
||||
|
||||
if len(recovered) > 0 && s.logger != nil {
|
||||
s.logger.Info("expired desktop publish tasks recovered",
|
||||
zap.Int("requeued", result.Requeued),
|
||||
zap.Int("failed", result.Failed),
|
||||
zap.Int("uncertain", result.Uncertain),
|
||||
)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func desktopTaskRecoverySelectQuery(mode desktopTaskRecoveryMode) string {
|
||||
query := `
|
||||
SELECT
|
||||
desktop_id,
|
||||
workspace_id,
|
||||
kind,
|
||||
status,
|
||||
active_attempt_id
|
||||
FROM desktop_tasks
|
||||
workspace_id,
|
||||
kind,
|
||||
status,
|
||||
active_attempt_id,
|
||||
attempts,
|
||||
publish_submit_started_at IS NOT NULL AS submit_started
|
||||
FROM desktop_tasks
|
||||
WHERE target_client_id = $1
|
||||
AND workspace_id = $2
|
||||
AND status = 'in_progress'
|
||||
@@ -1517,36 +1865,62 @@ func desktopTaskRecoveryPayload(mode desktopTaskRecoveryMode, kind string) ([]by
|
||||
reason := strings.TrimSpace(string(mode))
|
||||
message := "desktop task was recovered after the client lost the active lease"
|
||||
source := "desktop_task_recovery"
|
||||
isPublishFinal := kind == "publish_final"
|
||||
isPublishUncertain := kind == "publish_uncertain"
|
||||
|
||||
if isPublishFinal {
|
||||
message = fmt.Sprintf("desktop publish task exceeded %d attempts during recovery; task has been marked failed", desktopPublishMaxAttempts)
|
||||
} else if isPublishUncertain {
|
||||
message = "desktop publish task may have already been submitted to the platform before the lease was lost; kept as unknown for manual reconcile to avoid duplicate publishing"
|
||||
}
|
||||
|
||||
switch mode {
|
||||
case desktopTaskRecoveryModeStartup:
|
||||
source = "desktop_client_startup"
|
||||
if kind == "monitor" {
|
||||
if isPublishFinal {
|
||||
message = fmt.Sprintf("desktop client restarted while the publish task was in progress; task exceeded %d attempts and has been marked failed", desktopPublishMaxAttempts)
|
||||
} else if isPublishUncertain {
|
||||
message = "desktop client restarted while the publish task was mid-submit; kept as unknown for manual reconcile to avoid duplicate publishing"
|
||||
} else if kind == "monitor" {
|
||||
message = "desktop client restarted while the monitor task was in progress; task has been re-queued"
|
||||
} else {
|
||||
message = "desktop client restarted while the publish task was in progress; task has been moved to unknown for manual reconcile"
|
||||
message = "desktop client restarted while the publish task was in progress; task has been re-queued for retry"
|
||||
}
|
||||
case desktopTaskRecoveryModeDisconnect:
|
||||
source = "desktop_client_offline"
|
||||
if kind == "monitor" {
|
||||
if isPublishFinal {
|
||||
message = fmt.Sprintf("desktop client went offline while the publish task was in progress; task exceeded %d attempts and has been marked failed", desktopPublishMaxAttempts)
|
||||
} else if isPublishUncertain {
|
||||
message = "desktop client went offline while the publish task was mid-submit; kept as unknown for manual reconcile to avoid duplicate publishing"
|
||||
} else if kind == "monitor" {
|
||||
message = "desktop client went offline while the monitor task was in progress; task has been re-queued"
|
||||
} else {
|
||||
message = "desktop client went offline while the publish task was in progress; task has been moved to unknown for manual reconcile"
|
||||
message = "desktop client went offline while the publish task was in progress; task has been re-queued for retry"
|
||||
}
|
||||
case desktopTaskRecoveryModeLeaseExpiry:
|
||||
source = "desktop_task_lease_expiry"
|
||||
if kind == "monitor" {
|
||||
if isPublishFinal {
|
||||
message = fmt.Sprintf("desktop task lease expired before publish completion; task exceeded %d attempts and has been marked failed", desktopPublishMaxAttempts)
|
||||
} else if isPublishUncertain {
|
||||
message = "desktop task lease expired while the publish task was mid-submit; kept as unknown for manual reconcile to avoid duplicate publishing"
|
||||
} else if kind == "monitor" {
|
||||
message = "desktop task lease expired before monitor completion; task has been re-queued"
|
||||
} else {
|
||||
message = "desktop task lease expired before publish completion; task has been moved to unknown for manual reconcile"
|
||||
message = "desktop task lease expired before publish completion; task has been re-queued for retry"
|
||||
}
|
||||
}
|
||||
|
||||
payload, err := marshalOptionalJSON(map[string]any{
|
||||
fields := map[string]any{
|
||||
"reason": reason,
|
||||
"message": message,
|
||||
"source": source,
|
||||
})
|
||||
}
|
||||
if isPublishUncertain {
|
||||
// Surfaced to the desktop UI so a manual retry of a maybe-already-published task asks for
|
||||
// explicit confirmation instead of silently re-posting.
|
||||
fields["publish_submit_uncertain"] = true
|
||||
}
|
||||
payload, err := marshalOptionalJSON(fields)
|
||||
if err != nil {
|
||||
return nil, "", "", err
|
||||
}
|
||||
@@ -1781,6 +2155,20 @@ func (s *DesktopTaskService) publishTaskEvent(ctx context.Context, task *reposit
|
||||
publishDesktopTaskLifecycleEvent(ctx, s.messaging, s.logger, task, eventType)
|
||||
}
|
||||
|
||||
func (s *DesktopTaskService) afterRecoveredPublishOutcomes(ctx context.Context, outcomes []*desktopPublishSyncOutcome) {
|
||||
for _, outcome := range outcomes {
|
||||
if outcome == nil {
|
||||
continue
|
||||
}
|
||||
invalidateArticleCaches(ctx, s.cache, outcome.TenantID, &outcome.ArticleID)
|
||||
if outcome.ArticleAlias != nil {
|
||||
if syncErr := s.syncMonitoringArticleAlias(ctx, outcome.ArticleAlias); syncErr != nil {
|
||||
s.logWarn("monitoring article alias sync failed after publish recovery", syncErr, zap.Int64("article_id", outcome.ArticleID))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func reconcileDesktopTaskEventType(_ string, status string) string {
|
||||
if status == "retry" {
|
||||
return "task_available"
|
||||
|
||||
@@ -17,6 +17,8 @@ func TestDesktopTaskRecoverySelectQueryMatchesRecoveredLeaseScanOrder(t *testing
|
||||
"kind",
|
||||
"status",
|
||||
"active_attempt_id",
|
||||
"attempts",
|
||||
"publish_submit_started_at",
|
||||
}
|
||||
|
||||
if len(gotColumns) != len(wantColumns) {
|
||||
@@ -55,6 +57,45 @@ func TestReconcileRetryPublishesTaskAvailableForAllDesktopTaskKinds(t *testing.T
|
||||
}
|
||||
}
|
||||
|
||||
func TestDesktopTaskRecoveryPayloadPublishLeaseExpiryRequeuesInsteadOfUnknown(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
payload, reason, message, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "publish")
|
||||
if err != nil {
|
||||
t.Fatalf("desktopTaskRecoveryPayload returned error: %v", err)
|
||||
}
|
||||
if reason != "lease_expired" {
|
||||
t.Fatalf("reason = %q, want lease_expired", reason)
|
||||
}
|
||||
if !strings.Contains(message, "re-queued for retry") {
|
||||
t.Fatalf("message = %q, want re-queued for retry", message)
|
||||
}
|
||||
if strings.Contains(message, "unknown") {
|
||||
t.Fatalf("message must not move publish recovery to unknown: %q", message)
|
||||
}
|
||||
if !strings.Contains(string(payload), "desktop_task_lease_expiry") {
|
||||
t.Fatalf("payload missing source: %s", payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDesktopTaskRecoveryPayloadPublishFinalFailsAfterMaxAttempts(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
payload, _, message, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "publish_final")
|
||||
if err != nil {
|
||||
t.Fatalf("desktopTaskRecoveryPayload returned error: %v", err)
|
||||
}
|
||||
if !strings.Contains(message, "marked failed") {
|
||||
t.Fatalf("message = %q, want marked failed", message)
|
||||
}
|
||||
if !strings.Contains(message, "3 attempts") {
|
||||
t.Fatalf("message = %q, want max attempts", message)
|
||||
}
|
||||
if !strings.Contains(string(payload), "marked failed") {
|
||||
t.Fatalf("payload missing final failure message: %s", payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileNonRetryPublishesTaskReconciled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -63,6 +104,38 @@ func TestReconcileNonRetryPublishesTaskReconciled(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePublishRecoveryOutcome(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
submitStarted bool
|
||||
attempts int
|
||||
wantStatus string
|
||||
wantKind string
|
||||
}{
|
||||
{"pre-submit first attempt requeues", false, 1, "queued", "publish"},
|
||||
{"pre-submit just below cap requeues", false, desktopPublishMaxAttempts - 1, "queued", "publish"},
|
||||
{"pre-submit at cap fails terminally", false, desktopPublishMaxAttempts, "failed", "publish_final"},
|
||||
{"pre-submit above cap fails terminally", false, desktopPublishMaxAttempts + 1, "failed", "publish_final"},
|
||||
// Submit may have happened: never auto-requeue or silently fail — route to unknown
|
||||
// (manual reconcile) regardless of remaining attempts, to avoid duplicate publishing.
|
||||
{"submit started first attempt is uncertain", true, 1, "unknown", "publish_uncertain"},
|
||||
{"submit started at cap is still uncertain", true, desktopPublishMaxAttempts, "unknown", "publish_uncertain"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
gotStatus, gotKind := resolvePublishRecoveryOutcome(tc.submitStarted, tc.attempts)
|
||||
if gotStatus != tc.wantStatus || gotKind != tc.wantKind {
|
||||
t.Fatalf("resolvePublishRecoveryOutcome(%v, %d) = (%q, %q), want (%q, %q)",
|
||||
tc.submitStarted, tc.attempts, gotStatus, gotKind, tc.wantStatus, tc.wantKind)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func recoverDesktopTaskSelectColumns(query string) []string {
|
||||
re := regexp.MustCompile(`(?is)select\s+(.*?)\s+from\s+desktop_tasks`)
|
||||
match := re.FindStringSubmatch(query)
|
||||
|
||||
Reference in New Issue
Block a user