feat(publish): cancel stale queued publish tasks after queue timeout

Add CancelStaleQueuedPublishTasks to abort publish tasks that sit in the
queue past a configurable timeout (default 3 days) without being claimed by
a desktop client, so they no longer linger indefinitely as 'queued'. The
lease recovery worker runs the cleanup each cycle and reports a
cancelled_queued count; the timeout is configurable via job run config.
Aborted tasks carry a publish_queue_timeout error payload with a readable
Chinese duration, and the admin-web publish summary surfaces it as an
auto-cancelled queue-timeout message.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

server/internal/scheduler/publish_lease_recovery_worker.go

@@ -13,26 +13,36 @@ const (
 	defaultPublishLeaseRecoveryInterval = 3 * time.Minute
 	defaultPublishLeaseRecoveryTimeout  = 30 * time.Second
 	defaultPublishLeaseRecoveryLimit    = 1000
+	defaultPublishQueuedTaskTimeout     = 3 * 24 * time.Hour
 )
 
 type PublishLeaseRecoveryWorker struct {
-	service  *tenantapp.DesktopTaskService
-	logger   *zap.Logger
-	interval time.Duration
-	timeout  time.Duration
-	limit    int
+	service           *tenantapp.DesktopTaskService
+	logger            *zap.Logger
+	interval          time.Duration
+	timeout           time.Duration
+	limit             int
+	queuedTaskTimeout time.Duration
 }
 
 func NewPublishLeaseRecoveryWorker(service *tenantapp.DesktopTaskService, logger *zap.Logger) *PublishLeaseRecoveryWorker {
 	return &PublishLeaseRecoveryWorker{
-		service:  service,
-		logger:   logger,
-		interval: defaultPublishLeaseRecoveryInterval,
-		timeout:  defaultPublishLeaseRecoveryTimeout,
-		limit:    defaultPublishLeaseRecoveryLimit,
+		service:           service,
+		logger:            logger,
+		interval:          defaultPublishLeaseRecoveryInterval,
+		timeout:           defaultPublishLeaseRecoveryTimeout,
+		limit:             defaultPublishLeaseRecoveryLimit,
+		queuedTaskTimeout: defaultPublishQueuedTaskTimeout,
 	}
 }
 
+func (w *PublishLeaseRecoveryWorker) WithQueuedTaskTimeout(timeout time.Duration) *PublishLeaseRecoveryWorker {
+	if w != nil && timeout > 0 {
+		w.queuedTaskTimeout = timeout
+	}
+	return w
+}
+
 func (w *PublishLeaseRecoveryWorker) Start(ctx context.Context) {
 	go w.Run(ctx)
 }
@@ -72,15 +82,40 @@ func (w *PublishLeaseRecoveryWorker) runOnce(parent context.Context) {
 		return
 	}
 
-	if (result.Requeued > 0 || result.Failed > 0 || result.Uncertain > 0) && w.logger != nil {
+	queuedTaskTimeout := w.queuedTaskTimeout
+	if queuedTaskTimeout <= 0 {
+		queuedTaskTimeout = defaultPublishQueuedTaskTimeout
+	}
+	cancelledQueued, err := w.service.CancelStaleQueuedPublishTasks(ctx, queuedTaskTimeout, w.limit)
+	if err != nil {
+		if w.logger != nil {
+			w.logger.Warn("publish queued task timeout cleanup failed", zap.Error(err))
+		}
+		return
+	}
+	result.CancelledQueued = cancelledQueued
+
+	if (result.Requeued > 0 || result.Failed > 0 || result.Uncertain > 0 || result.CancelledQueued > 0) && w.logger != nil {
 		w.logger.Info("publish lease recovery completed",
 			zap.Int("requeued_task_count", result.Requeued),
 			zap.Int("failed_task_count", result.Failed),
 			zap.Int("uncertain_task_count", result.Uncertain),
+			zap.Int("cancelled_queued_task_count", result.CancelledQueued),
 		)
 	}
 }
 
+func publishQueuedTaskTimeoutFromRun(run JobRunContext) time.Duration {
+	timeout := AsDuration(run.Config, "queued_task_timeout", defaultPublishQueuedTaskTimeout)
+	if seconds := AsInt(run.Config, "queued_task_timeout_seconds", 0); seconds > 0 {
+		timeout = time.Duration(seconds) * time.Second
+	}
+	if timeout <= 0 {
+		return defaultPublishQueuedTaskTimeout
+	}
+	return timeout
+}
+
 func (w *PublishLeaseRecoveryWorker) RunOnce(parent context.Context, run JobRunContext) (map[string]any, error) {
 	if w == nil || w.service == nil {
 		return map[string]any{"skipped": true, "reason": "worker_not_ready"}, nil
@@ -104,11 +139,19 @@ func (w *PublishLeaseRecoveryWorker) RunOnce(parent context.Context, run JobRunC
 	if err != nil {
 		return map[string]any{"stage": "recover"}, err
 	}
+	queuedTimeout := publishQueuedTaskTimeoutFromRun(run)
+	cancelledQueued, err := w.service.CancelStaleQueuedPublishTasks(ctx, queuedTimeout, limit)
+	if err != nil {
+		return map[string]any{"stage": "cancel_stale_queued"}, err
+	}
+	result.CancelledQueued = cancelledQueued
 	return map[string]any{
-		"requeued_task_count":  result.Requeued,
-		"failed_task_count":    result.Failed,
-		"uncertain_task_count": result.Uncertain,
-		"duration_ms":          time.Since(startedAt).Milliseconds(),
-		"batch_size":           limit,
+		"requeued_task_count":         result.Requeued,
+		"failed_task_count":           result.Failed,
+		"uncertain_task_count":        result.Uncertain,
+		"cancelled_queued_task_count": result.CancelledQueued,
+		"queued_task_timeout_seconds": int64(queuedTimeout.Seconds()),
+		"duration_ms":                 time.Since(startedAt).Milliseconds(),
+		"batch_size":                  limit,
 	}, nil
 }

server/internal/scheduler/publish_lease_recovery_worker_test.go

@@ -0,0 +1,42 @@
+package scheduler
+
+import (
+	"testing"
+	"time"
+)
+
+func TestPublishQueuedTaskTimeoutFromRunDefaultsToThreeDays(t *testing.T) {
+	t.Parallel()
+
+	got := publishQueuedTaskTimeoutFromRun(JobRunContext{})
+	if got != 3*24*time.Hour {
+		t.Fatalf("publishQueuedTaskTimeoutFromRun() = %s, want 72h", got)
+	}
+}
+
+func TestPublishQueuedTaskTimeoutFromRunReadsDurationConfig(t *testing.T) {
+	t.Parallel()
+
+	got := publishQueuedTaskTimeoutFromRun(JobRunContext{
+		Config: map[string]any{
+			"queued_task_timeout": "72h",
+		},
+	})
+	if got != 72*time.Hour {
+		t.Fatalf("publishQueuedTaskTimeoutFromRun(duration) = %s, want 72h", got)
+	}
+}
+
+func TestPublishQueuedTaskTimeoutFromRunReadsSecondsConfig(t *testing.T) {
+	t.Parallel()
+
+	got := publishQueuedTaskTimeoutFromRun(JobRunContext{
+		Config: map[string]any{
+			"queued_task_timeout":         "72h",
+			"queued_task_timeout_seconds": 3600,
+		},
+	})
+	if got != time.Hour {
+		t.Fatalf("publishQueuedTaskTimeoutFromRun(seconds) = %s, want 1h", got)
+	}
+}