fix(publish): prevent stuck publish queue and duplicate posting
Desktop Client Build / Resolve Build Metadata (push) Successful in 41s
Backend CI / Backend (push) Failing after 7m14s
Desktop Client Build / Build Desktop Client (push) Successful in 23m46s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 56s

The desktop publish queue could stall for a long time and end users assumed
the software was broken. Root cause: a hung adapter held its execution slot
with no wall-clock timeout while auto-renewing its lease forever, so the
server never reclaimed it and every queued task behind it stayed 等待发布.
Auto-recovery also risked silently re-posting a non-idempotent article.

Client (Electron):
- per-task wall-clock deadline + abort; progress-gated lease renewal that
  stops and aborts a stalled task instead of renewing it forever
- decouple the concurrency cap from CDP-induced CPU/memory pressure
  (admission gate instead of self-throttling collapse); 15s watchdog pump
- all adapter network I/O now has fetch timeouts and honors context.signal;
  bounded image-upload concurrency with per-image timeout
- surface live adapter progress, elapsed time, queue position and a
  working-vs-queued distinction in the publish view

Server (tenant-api):
- publish lease-recovery worker (every 3m) + supporting index: reclaim
  expired in_progress publish leases, requeue (<3 attempts) or terminal-fail
- 3-minute lease TTL with client-presence-gated extension; max 3 attempts

Idempotency (production-grade core):
- durable publish_submit_started_at marker, set before the irreversible
  platform submit POST; recovery and abort route a maybe-submitted task to
  unknown (manual reconcile, kept in the dedup set) instead of re-posting
- desktop UI requires explicit confirmation before retrying a possibly-
  already-published task

Verified: go build/vet/test (incl. resolvePublishRecoveryOutcome), vue-tsc,
vitest 141/141, gofmt all clean.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-30 19:52:17 +08:00
parent 9d6181260a
commit fa51a3455f
41 changed files with 2124 additions and 358 deletions
@@ -118,6 +118,7 @@ import {
leaseDesktopTask,
leaseMonitoringTasks,
listDesktopAccounts,
markDesktopTaskPublishSubmitStarted,
noteTransportHeartbeat,
noteTransportPull,
offlineDesktopClient,
@@ -148,7 +149,11 @@ type MonitorExecutionPhase =
const heartbeatIntervalMs = 15_000
const pullIntervalMs = 60_000
const pumpWatchdogIntervalMs = 15_000
const leaseExtendIntervalMs = 60_000
const defaultPublishTaskTimeoutMs = 5 * 60_000
const publishTaskStaleProgressMs = 2 * 60_000
const publishLongRunningNotificationMs = 15 * 60_000
const maxActivityItems = 60
const monitorBusinessTimeZone = 'Asia/Shanghai'
const authRequiredMonitorPlatforms = new Set(aiPlatformCatalog.map((platform) => platform.id))
@@ -171,6 +176,7 @@ interface RuntimeTaskRecord {
status: RuntimeTaskStatus
routing: RuntimeTaskRouting
leaseExpiresAt: number | null
startedAt: number | null
updatedAt: number
summary: string
payload: Record<string, JsonValue>
@@ -226,6 +232,16 @@ interface ActiveRuntimeExecution {
interruptGeneration: number
interruptReason: string | null
lastSafePointAt: number | null
startedAt: number
lastProgressAt: number
lastProgressSummary: string | null
deadlineAt: number | null
deadlineHandle: ReturnType<typeof setTimeout> | null
longRunningNotifiedAt: number | null
// Set the moment a publish adapter enters its irreversible submit/save_draft stage. Once set,
// an interrupted publish is completed as 'unknown' (manual reconcile) instead of 'failed', so a
// possibly-already-published article is never silently re-posted by a retry.
submitStartedAt: number | null
}
interface RuntimeState {
@@ -241,6 +257,7 @@ interface RuntimeState {
publishFallbackBackoffUntil: number
heartbeatTimer: ReturnType<typeof setInterval> | null
pullTimer: ReturnType<typeof setInterval> | null
pumpWatchdogTimer: ReturnType<typeof setInterval> | null
accountHealthReportTimer: ReturnType<typeof setTimeout> | null
accountHealthReportQueue: Map<string, DesktopAccountHealthReport>
accountHealthReportBadAccounts: Set<string>
@@ -271,6 +288,7 @@ const state: RuntimeState = {
publishFallbackBackoffUntil: 0,
heartbeatTimer: null,
pullTimer: null,
pumpWatchdogTimer: null,
accountHealthReportTimer: null,
accountHealthReportQueue: new Map<string, DesktopAccountHealthReport>(),
accountHealthReportBadAccounts: new Set<string>(),
@@ -659,7 +677,10 @@ export function getRuntimeControllerSnapshot(): RuntimeControllerSnapshot {
),
tasks: [...state.tasks.values()]
.sort((left, right) => right.updatedAt - left.updatedAt)
.map(({ leaseToken: _leaseToken, ...task }) => ({ ...task })),
.map(({ leaseToken: _leaseToken, ...task }) => ({
...task,
startedAt: state.activeExecutions.get(task.id)?.startedAt ?? task.startedAt,
})),
activity: [...state.activity],
lastHeartbeatAt: state.lastHeartbeatAt,
lastHeartbeatStatus: state.lastHeartbeatStatus,
@@ -690,6 +711,7 @@ function startRuntime(): void {
connectDispatchWs()
scheduleHeartbeatLoop()
schedulePullLoop()
schedulePumpWatchdogLoop()
void sendHeartbeat('startup')
void syncAccounts('startup')
@@ -714,6 +736,10 @@ function stopRuntime(): void {
state.running = false
state.leaseInFlightKinds.clear()
for (const execution of state.activeExecutions.values()) {
if (execution.deadlineHandle) {
clearTimeout(execution.deadlineHandle)
execution.deadlineHandle = null
}
execution.abortController.abort()
}
state.activeExecutions.clear()
@@ -726,6 +752,10 @@ function stopRuntime(): void {
clearInterval(state.pullTimer)
state.pullTimer = null
}
if (state.pumpWatchdogTimer) {
clearInterval(state.pumpWatchdogTimer)
state.pumpWatchdogTimer = null
}
if (state.accountHealthReportTimer) {
clearTimeout(state.accountHealthReportTimer)
state.accountHealthReportTimer = null
@@ -793,6 +823,17 @@ function schedulePullLoop(): void {
}, pullIntervalMs)
}
function schedulePumpWatchdogLoop(): void {
if (state.pumpWatchdogTimer) {
clearInterval(state.pumpWatchdogTimer)
}
state.pumpWatchdogTimer = setInterval(() => {
notifyLongRunningPublishTasks()
pumpExecutionLoop()
}, pumpWatchdogIntervalMs)
}
function connectDispatchWs(): void {
if (!canRunLive(state.session)) {
return
@@ -909,6 +950,7 @@ function handleTaskEvent(event: DesktopTaskEventMessage): void {
status: event.status,
routing: existing?.routing ?? 'rabbitmq-primary',
leaseExpiresAt: event.status === 'in_progress' ? (existing?.leaseExpiresAt ?? null) : null,
startedAt: event.status === 'in_progress' ? (existing?.startedAt ?? eventUpdatedAt) : null,
updatedAt: eventUpdatedAt,
summary: summaryFromEventType(event.type, event.status),
payload: existing?.payload ?? {},
@@ -1577,6 +1619,114 @@ function enqueuePublishTask(taskId: string, platform: string, routing: RuntimeTa
syncSchedulerSurface()
}
function createActiveExecution(
taskRecord: RuntimeTaskRecord,
abortController: AbortController,
): ActiveRuntimeExecution {
const now = Date.now()
return {
taskId: taskRecord.id,
kind: taskRecord.kind,
platform: taskRecord.platform,
accountId: taskRecord.accountId,
abortController,
executionPhase: taskRecord.kind === 'monitor' ? 'PREPARING' : null,
interruptRequested:
taskRecord.kind === 'monitor' && Boolean(taskRecord.payload.interrupt_requested),
interruptGeneration:
taskRecord.kind === 'monitor'
? toInterruptGeneration(taskRecord.payload.interrupt_generation)
: 0,
interruptReason: taskRecord.kind === 'monitor' ? interruptReasonFromTask(taskRecord) : null,
lastSafePointAt: taskRecord.kind === 'monitor' ? now : null,
startedAt: now,
lastProgressAt: now,
lastProgressSummary: taskRecord.summary,
deadlineAt: null,
deadlineHandle: null,
longRunningNotifiedAt: null,
submitStartedAt: null,
}
}
function armPublishTaskDeadline(
taskRecord: RuntimeTaskRecord,
active: ActiveRuntimeExecution,
): void {
if (taskRecord.kind !== 'publish') {
return
}
const taskDeadlineMs = resolvePublishTaskTimeoutMs(taskRecord.platform)
active.deadlineAt = active.startedAt + taskDeadlineMs
active.deadlineHandle = setTimeout(() => {
if (active.abortController.signal.aborted) {
return
}
const timeoutError = new Error('publish_task_timeout')
recordActivity(
'warn',
'任务超时',
`${taskRecord.title} 执行超过 ${Math.round(taskDeadlineMs / 1000)}s,已中止并释放。`,
)
updateTaskProgress(taskRecord.id, `${taskRecord.title} 执行超时,正在释放任务。`)
active.abortController.abort(timeoutError)
}, taskDeadlineMs)
}
function resolvePublishTaskTimeoutMs(platform: string): number {
const envKey = `GEO_DESKTOP_PUBLISH_TIMEOUT_${platform.toUpperCase().replace(/[^A-Z0-9]/g, '_')}_MS`
return resolvePositiveEnvInteger(envKey) ?? resolvePositiveEnvInteger('GEO_DESKTOP_PUBLISH_TIMEOUT_MS') ?? defaultPublishTaskTimeoutMs
}
function resolvePositiveEnvInteger(key: string): number | null {
const raw = process.env[key]?.trim()
if (!raw) {
return null
}
const parsed = Number.parseInt(raw, 10)
return Number.isFinite(parsed) && parsed > 0 ? parsed : null
}
// Stages emitted by an adapter right before it performs an irreversible platform write (creating
// the article/draft). After this point the article may exist on the platform regardless of whether
// the local task records it, so an interrupted publish must not be auto-retried.
function isIrreversiblePublishStage(stage: string): boolean {
const normalized = stage.trim().toLowerCase()
return (
normalized.endsWith('.submit') ||
normalized.endsWith('.save_draft') ||
normalized === 'submit' ||
normalized === 'save_draft'
)
}
// Persist a durable "submit started" marker (server-side, before/at the irreversible POST) and a
// local flag the abort path reads. Fires at most once per execution and never blocks publishing.
function markPublishSubmitStartedIfNeeded(taskId: string, stage: string): void {
if (!isIrreversiblePublishStage(stage)) {
return
}
const active = state.activeExecutions.get(taskId)
if (!active || active.kind !== 'publish' || active.submitStartedAt !== null) {
return
}
active.submitStartedAt = Date.now()
state.activeExecutions.set(taskId, active)
const leaseToken = state.tasks.get(taskId)?.leaseToken
if (!leaseToken) {
return
}
void markDesktopTaskPublishSubmitStarted(taskId, leaseToken).catch((error) => {
console.warn('[desktop-runtime] mark publish submit started failed', {
taskId,
message: error instanceof Error ? error.message : String(error),
})
})
}
function pumpExecutionLoop(): void {
if (!state.running) {
return
@@ -1837,6 +1987,7 @@ function enqueueLeasedMonitoringTasks(
status: 'queued',
routing,
leaseExpiresAt: parseTimestamp(task.lease_expires_at),
startedAt: null,
updatedAt: now,
summary: `已领取监控租约,等待本地调度执行(${describeRouting(routing)})。`,
payload: {
@@ -1920,18 +2071,7 @@ async function executeLeasedMonitoringTask(
recordActivity('info', '监控任务开始执行', `${taskRecord.title} 已进入本地执行队列。`)
const abortController = new AbortController()
state.activeExecutions.set(taskRecord.id, {
taskId: taskRecord.id,
kind: 'monitor',
platform: taskRecord.platform,
accountId: taskRecord.accountId,
abortController,
executionPhase: 'PREPARING',
interruptRequested: Boolean(taskRecord.payload.interrupt_requested),
interruptGeneration: toInterruptGeneration(taskRecord.payload.interrupt_generation),
interruptReason: interruptReasonFromTask(taskRecord),
lastSafePointAt: Date.now(),
})
state.activeExecutions.set(taskRecord.id, createActiveExecution(taskRecord, abortController))
noteMonitorTaskActivated(taskId)
syncSchedulerSurface()
queueMicrotask(() => pumpExecutionLoop())
@@ -2382,6 +2522,7 @@ function noteMonitorExecutionSafePoint(taskId: string, phase: MonitorExecutionPh
active.executionPhase = phase
}
active.lastSafePointAt = Date.now()
active.lastProgressAt = Date.now()
state.activeExecutions.set(taskId, active)
const taskRecord = state.tasks.get(taskId)
@@ -2406,6 +2547,7 @@ function requestMonitorStaleBusinessDateDrop(taskId: string, staleBusinessDate:
active.interruptReason = 'stale_business_date'
active.lastSafePointAt = Date.now()
active.lastProgressAt = Date.now()
state.activeExecutions.set(taskId, active)
const existing = state.tasks.get(taskId)
@@ -2523,22 +2665,9 @@ async function executeLeasedTask(
)
const abortController = new AbortController()
state.activeExecutions.set(taskRecord.id, {
taskId: taskRecord.id,
kind: taskRecord.kind,
platform: taskRecord.platform,
accountId: taskRecord.accountId,
abortController,
executionPhase: taskRecord.kind === 'monitor' ? 'PREPARING' : null,
interruptRequested:
taskRecord.kind === 'monitor' && Boolean(taskRecord.payload.interrupt_requested),
interruptGeneration:
taskRecord.kind === 'monitor'
? toInterruptGeneration(taskRecord.payload.interrupt_generation)
: 0,
interruptReason: taskRecord.kind === 'monitor' ? interruptReasonFromTask(taskRecord) : null,
lastSafePointAt: taskRecord.kind === 'monitor' ? Date.now() : null,
})
const activeExecution = createActiveExecution(taskRecord, abortController)
armPublishTaskDeadline(taskRecord, activeExecution)
state.activeExecutions.set(taskRecord.id, activeExecution)
if (task.kind === 'monitor') {
noteMonitorTaskLeased(task, routing)
@@ -2598,7 +2727,23 @@ async function executeLeasedTask(
const structuredError = toStructuredError(error)
const accountIdentity = accountIdentityFromTask(taskRecord)
if (accountIdentity) {
// If a publish task was interrupted AFTER it may have entered the irreversible submit phase,
// the article might already be live on the platform. Completing as a clean 'failed' would drop
// it from the server dedup set and let a retry double-post. Complete as 'unknown' instead so it
// stays for manual reconcile. (A pre-submit interruption is a true failure and stays 'failed'.)
const submitMaybeStarted =
taskRecord.kind === 'publish' && activeExecution.submitStartedAt !== null
const terminalStatus: 'failed' | 'unknown' = submitMaybeStarted ? 'unknown' : 'failed'
const interruptedSummary = '发布可能已提交到平台,结果待人工确认,已避免自动重复发布。'
// Tag the persisted error so the desktop UI can require explicit confirmation before a manual
// retry of a possibly-already-published task (mirrors the server recovery payload marker).
const terminalError = submitMaybeStarted
? { ...structuredError, publish_submit_uncertain: true }
: structuredError
// A submit-phase interruption is typically a network/timeout stall, not an auth problem — don't
// wrongly degrade account health for it.
if (accountIdentity && !submitMaybeStarted) {
await reportAccountFailure(accountIdentity, {
summary: failureMessage,
error: structuredError,
@@ -2615,22 +2760,24 @@ async function executeLeasedTask(
try {
const completed = await completeDesktopTask(taskRecord.id, {
lease_token: leased.lease_token,
status: 'failed',
error: structuredError,
status: terminalStatus,
error: terminalError,
})
upsertTaskFromInfo(completed, {
routing,
summary: failureMessage || '桌面任务执行失败。',
summary: submitMaybeStarted ? interruptedSummary : failureMessage || '桌面任务执行失败。',
attemptId: null,
leaseToken: null,
})
} catch (completeError) {
const existing = state.tasks.get(taskRecord.id)
if (existing) {
existing.status = 'failed'
existing.error = structuredError
existing.summary = `${failureMessage || '桌面任务执行失败。'}result 回写失败:${errorMessage(completeError)}`
existing.status = terminalStatus
existing.error = terminalError
existing.summary = submitMaybeStarted
? `${interruptedSummary}result 回写失败:${errorMessage(completeError)}`
: `${failureMessage || '桌面任务执行失败。'}result 回写失败:${errorMessage(completeError)}`
existing.updatedAt = Date.now()
state.tasks.set(taskRecord.id, existing)
}
@@ -2638,12 +2785,18 @@ async function executeLeasedTask(
noteLeaseReleased('failed', taskRecord.id)
recordActivity(
'danger',
'任务执行失败',
`${taskRecord.title} 执行失败:${failureMessage || '未知错误'}`,
submitMaybeStarted ? 'warn' : 'danger',
submitMaybeStarted ? '发布结果待确认' : '任务执行失败',
submitMaybeStarted
? `${taskRecord.title} 进入提交阶段后中断,可能已发布,已转人工确认以避免重复发文。`
: `${taskRecord.title} 执行失败:${failureMessage || '未知错误'}`,
)
} finally {
clearInterval(extendHandle)
if (activeExecution.deadlineHandle) {
clearTimeout(activeExecution.deadlineHandle)
activeExecution.deadlineHandle = null
}
if (taskRecord.kind === 'publish') {
notePublishTaskCompleted(taskRecord.id, taskRecord.platform)
}
@@ -2656,6 +2809,7 @@ async function executeLeasedTask(
async function extendActiveLease(taskId: string, leaseToken: string): Promise<void> {
const existing = state.tasks.get(taskId)
const active = state.activeExecutions.get(taskId)
if (existing?.kind === 'monitor') {
const staleBusinessDate = resolveStaleMonitoringBusinessDate(existing.payload)
if (staleBusinessDate) {
@@ -2663,6 +2817,28 @@ async function extendActiveLease(taskId: string, leaseToken: string): Promise<vo
return
}
}
if (existing?.kind === 'publish' && active) {
const idleMs = Date.now() - active.lastProgressAt
if (idleMs >= publishTaskStaleProgressMs) {
const detail = active.lastProgressSummary ? `,最后进度:${active.lastProgressSummary}` : ''
existing.summary = `发布任务超过 ${Math.round(idleMs / 1000)} 秒没有新进度,已中止并释放,准备由服务端重排${detail}`
existing.updatedAt = Date.now()
state.tasks.set(taskId, existing)
recordActivity(
'warn',
'发布任务疑似卡住',
`${existing.title} ${existing.summary}`,
)
// Abort the local execution instead of merely letting the lease lapse: this
// releases the slot immediately and, crucially, closes the window where the
// server could re-queue the task while a stalled adapter is still mid-publish
// (which would double-post a non-idempotent article).
if (!active.abortController.signal.aborted) {
active.abortController.abort(new Error('publish_stale_progress'))
}
return
}
}
try {
const task = await extendDesktopTask(taskId, { lease_token: leaseToken })
@@ -2673,7 +2849,10 @@ async function extendActiveLease(taskId: string, leaseToken: string): Promise<vo
if (existing) {
existing.leaseExpiresAt = parseTimestamp(task.lease_expires_at) ?? existing.leaseExpiresAt
existing.updatedAt = Date.now()
existing.summary = '租约已自动续期,任务仍在执行。'
existing.summary =
existing.kind === 'publish'
? (active?.lastProgressSummary ?? '租约已自动续期,发布任务仍在执行。')
: '租约已自动续期,任务仍在执行。'
state.tasks.set(taskId, existing)
}
} catch (error) {
@@ -2738,6 +2917,7 @@ async function executeTaskAdapter(
reportProgress(stage: string) {
noteMonitorExecutionSafePoint(task.id, null)
updateTaskProgress(task.id, `publish adapter progress: ${stage}`)
markPublishSubmitStartedIfNeeded(task.id, stage)
},
},
payload,
@@ -3077,6 +3257,17 @@ function updateTaskProgress(taskId: string, summary: string): void {
existing.summary = summary
existing.updatedAt = Date.now()
state.tasks.set(taskId, existing)
const active = state.activeExecutions.get(taskId)
if (active) {
active.lastProgressAt = Date.now()
active.lastProgressSummary = summary
state.activeExecutions.set(taskId, active)
}
if (existing.kind === 'publish') {
emitRuntimeInvalidated('publish-task-progress', { taskId })
}
}
function upsertTaskFromInfo(
@@ -3102,6 +3293,7 @@ function upsertTaskFromInfo(
status: task.status,
routing: options.routing ?? existing?.routing ?? 'db-recovery',
leaseExpiresAt: parseTimestamp(task.lease_expires_at) ?? null,
startedAt: task.status === 'in_progress' ? (existing?.startedAt ?? Date.now()) : null,
updatedAt: parseTimestamp(task.updated_at) ?? Date.now(),
summary: options.summary ?? existing?.summary ?? summaryFromTaskStatus(task.status),
payload,
@@ -3149,6 +3341,42 @@ function syncSchedulerSurface(): void {
setSchedulerCurrentTask(primaryActiveTaskId())
}
function notifyLongRunningPublishTasks(): void {
const now = Date.now()
for (const active of state.activeExecutions.values()) {
if (active.kind !== 'publish' || active.longRunningNotifiedAt) {
continue
}
if (now - active.startedAt < publishLongRunningNotificationMs) {
continue
}
active.longRunningNotifiedAt = now
state.activeExecutions.set(active.taskId, active)
const task = state.tasks.get(active.taskId)
if (task) {
task.summary = `${task.summary || '发布任务仍在执行。'}(已执行 ${formatDurationText(now - active.startedAt)},如网络异常可取消后重试。)`
task.updatedAt = now
state.tasks.set(task.id, task)
recordActivity('warn', '发布等待较久', `${task.title} 已执行 ${formatDurationText(now - active.startedAt)},可能存在网络异常。`)
emitRuntimeInvalidated('publish-task-progress', { taskId: task.id })
}
}
}
function formatDurationText(ms: number): string {
const seconds = Math.max(0, Math.floor(ms / 1000))
const minutes = Math.floor(seconds / 60)
const restSeconds = seconds % 60
if (minutes <= 0) {
return `${restSeconds}`
}
if (restSeconds === 0) {
return `${minutes} 分钟`
}
return `${minutes}${restSeconds}`
}
function localQueueDepth(): number {
const publish = getPublishSchedulerSnapshot()
const monitor = getMonitorSchedulerSnapshot()
@@ -3229,7 +3457,10 @@ function activeAdmissionCount(): number {
}
function hasTotalCapacityForNewExecution(): boolean {
return activeAdmissionCount() < resolveAdaptiveTotalConcurrency()
return (
activeAdmissionCount() < resolveAdaptiveTotalConcurrency() &&
shouldAdmitNewExecutionUnderRuntimePressure()
)
}
function canStartAnotherPublishExecution(): boolean {
@@ -3355,9 +3586,13 @@ function resolveHardwareTotalConcurrency(): number {
}
function resolveRuntimeHealthTotalConcurrency(hardwareCap: number): number {
return hardwareCap
}
function shouldAdmitNewExecutionUnderRuntimePressure(): boolean {
const metrics = getProcessMetricsSnapshot().latestSample
if (!metrics) {
return Math.min(hardwareCap, minimumForegroundTotalConcurrency)
return true
}
const totalCPUUsage = metrics.processes.reduce((sum, item) => sum + item.percentCPUUsage, 0)
@@ -3371,19 +3606,10 @@ function resolveRuntimeHealthTotalConcurrency(hardwareCap: number): number {
mainPrivateBytesKB >= 1_000_000 ||
processCount >= 40
) {
return minimumForegroundTotalConcurrency
return activeAdmissionCount() < minimumForegroundTotalConcurrency
}
if (
totalCPUUsage >= 120 ||
totalPrivateBytesKB >= 1_800_000 ||
mainPrivateBytesKB >= 750_000 ||
processCount >= 32
) {
return Math.min(hardwareCap, 3)
}
return hardwareCap
return true
}
function resolveFeatureFlagTotalConcurrency(): number | null {