fix(publish): prevent stuck publish queue and duplicate posting
Desktop Client Build / Resolve Build Metadata (push) Successful in 41s
Backend CI / Backend (push) Failing after 7m14s
Desktop Client Build / Build Desktop Client (push) Successful in 23m46s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 56s
Desktop Client Build / Resolve Build Metadata (push) Successful in 41s
Backend CI / Backend (push) Failing after 7m14s
Desktop Client Build / Build Desktop Client (push) Successful in 23m46s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 56s
The desktop publish queue could stall for a long time and end users assumed the software was broken. Root cause: a hung adapter held its execution slot with no wall-clock timeout while auto-renewing its lease forever, so the server never reclaimed it and every queued task behind it stayed 等待发布. Auto-recovery also risked silently re-posting a non-idempotent article. Client (Electron): - per-task wall-clock deadline + abort; progress-gated lease renewal that stops and aborts a stalled task instead of renewing it forever - decouple the concurrency cap from CDP-induced CPU/memory pressure (admission gate instead of self-throttling collapse); 15s watchdog pump - all adapter network I/O now has fetch timeouts and honors context.signal; bounded image-upload concurrency with per-image timeout - surface live adapter progress, elapsed time, queue position and a working-vs-queued distinction in the publish view Server (tenant-api): - publish lease-recovery worker (every 3m) + supporting index: reclaim expired in_progress publish leases, requeue (<3 attempts) or terminal-fail - 3-minute lease TTL with client-presence-gated extension; max 3 attempts Idempotency (production-grade core): - durable publish_submit_started_at marker, set before the irreversible platform submit POST; recovery and abort route a maybe-submitted task to unknown (manual reconcile, kept in the dedup set) instead of re-posting - desktop UI requires explicit confirmation before retrying a possibly- already-published task Verified: go build/vet/test (incl. resolvePublishRecoveryOutcome), vue-tsc, vitest 141/141, gofmt all clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -118,6 +118,7 @@ import {
|
||||
leaseDesktopTask,
|
||||
leaseMonitoringTasks,
|
||||
listDesktopAccounts,
|
||||
markDesktopTaskPublishSubmitStarted,
|
||||
noteTransportHeartbeat,
|
||||
noteTransportPull,
|
||||
offlineDesktopClient,
|
||||
@@ -148,7 +149,11 @@ type MonitorExecutionPhase =
|
||||
|
||||
const heartbeatIntervalMs = 15_000
|
||||
const pullIntervalMs = 60_000
|
||||
const pumpWatchdogIntervalMs = 15_000
|
||||
const leaseExtendIntervalMs = 60_000
|
||||
const defaultPublishTaskTimeoutMs = 5 * 60_000
|
||||
const publishTaskStaleProgressMs = 2 * 60_000
|
||||
const publishLongRunningNotificationMs = 15 * 60_000
|
||||
const maxActivityItems = 60
|
||||
const monitorBusinessTimeZone = 'Asia/Shanghai'
|
||||
const authRequiredMonitorPlatforms = new Set(aiPlatformCatalog.map((platform) => platform.id))
|
||||
@@ -171,6 +176,7 @@ interface RuntimeTaskRecord {
|
||||
status: RuntimeTaskStatus
|
||||
routing: RuntimeTaskRouting
|
||||
leaseExpiresAt: number | null
|
||||
startedAt: number | null
|
||||
updatedAt: number
|
||||
summary: string
|
||||
payload: Record<string, JsonValue>
|
||||
@@ -226,6 +232,16 @@ interface ActiveRuntimeExecution {
|
||||
interruptGeneration: number
|
||||
interruptReason: string | null
|
||||
lastSafePointAt: number | null
|
||||
startedAt: number
|
||||
lastProgressAt: number
|
||||
lastProgressSummary: string | null
|
||||
deadlineAt: number | null
|
||||
deadlineHandle: ReturnType<typeof setTimeout> | null
|
||||
longRunningNotifiedAt: number | null
|
||||
// Set the moment a publish adapter enters its irreversible submit/save_draft stage. Once set,
|
||||
// an interrupted publish is completed as 'unknown' (manual reconcile) instead of 'failed', so a
|
||||
// possibly-already-published article is never silently re-posted by a retry.
|
||||
submitStartedAt: number | null
|
||||
}
|
||||
|
||||
interface RuntimeState {
|
||||
@@ -241,6 +257,7 @@ interface RuntimeState {
|
||||
publishFallbackBackoffUntil: number
|
||||
heartbeatTimer: ReturnType<typeof setInterval> | null
|
||||
pullTimer: ReturnType<typeof setInterval> | null
|
||||
pumpWatchdogTimer: ReturnType<typeof setInterval> | null
|
||||
accountHealthReportTimer: ReturnType<typeof setTimeout> | null
|
||||
accountHealthReportQueue: Map<string, DesktopAccountHealthReport>
|
||||
accountHealthReportBadAccounts: Set<string>
|
||||
@@ -271,6 +288,7 @@ const state: RuntimeState = {
|
||||
publishFallbackBackoffUntil: 0,
|
||||
heartbeatTimer: null,
|
||||
pullTimer: null,
|
||||
pumpWatchdogTimer: null,
|
||||
accountHealthReportTimer: null,
|
||||
accountHealthReportQueue: new Map<string, DesktopAccountHealthReport>(),
|
||||
accountHealthReportBadAccounts: new Set<string>(),
|
||||
@@ -659,7 +677,10 @@ export function getRuntimeControllerSnapshot(): RuntimeControllerSnapshot {
|
||||
),
|
||||
tasks: [...state.tasks.values()]
|
||||
.sort((left, right) => right.updatedAt - left.updatedAt)
|
||||
.map(({ leaseToken: _leaseToken, ...task }) => ({ ...task })),
|
||||
.map(({ leaseToken: _leaseToken, ...task }) => ({
|
||||
...task,
|
||||
startedAt: state.activeExecutions.get(task.id)?.startedAt ?? task.startedAt,
|
||||
})),
|
||||
activity: [...state.activity],
|
||||
lastHeartbeatAt: state.lastHeartbeatAt,
|
||||
lastHeartbeatStatus: state.lastHeartbeatStatus,
|
||||
@@ -690,6 +711,7 @@ function startRuntime(): void {
|
||||
connectDispatchWs()
|
||||
scheduleHeartbeatLoop()
|
||||
schedulePullLoop()
|
||||
schedulePumpWatchdogLoop()
|
||||
|
||||
void sendHeartbeat('startup')
|
||||
void syncAccounts('startup')
|
||||
@@ -714,6 +736,10 @@ function stopRuntime(): void {
|
||||
state.running = false
|
||||
state.leaseInFlightKinds.clear()
|
||||
for (const execution of state.activeExecutions.values()) {
|
||||
if (execution.deadlineHandle) {
|
||||
clearTimeout(execution.deadlineHandle)
|
||||
execution.deadlineHandle = null
|
||||
}
|
||||
execution.abortController.abort()
|
||||
}
|
||||
state.activeExecutions.clear()
|
||||
@@ -726,6 +752,10 @@ function stopRuntime(): void {
|
||||
clearInterval(state.pullTimer)
|
||||
state.pullTimer = null
|
||||
}
|
||||
if (state.pumpWatchdogTimer) {
|
||||
clearInterval(state.pumpWatchdogTimer)
|
||||
state.pumpWatchdogTimer = null
|
||||
}
|
||||
if (state.accountHealthReportTimer) {
|
||||
clearTimeout(state.accountHealthReportTimer)
|
||||
state.accountHealthReportTimer = null
|
||||
@@ -793,6 +823,17 @@ function schedulePullLoop(): void {
|
||||
}, pullIntervalMs)
|
||||
}
|
||||
|
||||
function schedulePumpWatchdogLoop(): void {
|
||||
if (state.pumpWatchdogTimer) {
|
||||
clearInterval(state.pumpWatchdogTimer)
|
||||
}
|
||||
|
||||
state.pumpWatchdogTimer = setInterval(() => {
|
||||
notifyLongRunningPublishTasks()
|
||||
pumpExecutionLoop()
|
||||
}, pumpWatchdogIntervalMs)
|
||||
}
|
||||
|
||||
function connectDispatchWs(): void {
|
||||
if (!canRunLive(state.session)) {
|
||||
return
|
||||
@@ -909,6 +950,7 @@ function handleTaskEvent(event: DesktopTaskEventMessage): void {
|
||||
status: event.status,
|
||||
routing: existing?.routing ?? 'rabbitmq-primary',
|
||||
leaseExpiresAt: event.status === 'in_progress' ? (existing?.leaseExpiresAt ?? null) : null,
|
||||
startedAt: event.status === 'in_progress' ? (existing?.startedAt ?? eventUpdatedAt) : null,
|
||||
updatedAt: eventUpdatedAt,
|
||||
summary: summaryFromEventType(event.type, event.status),
|
||||
payload: existing?.payload ?? {},
|
||||
@@ -1577,6 +1619,114 @@ function enqueuePublishTask(taskId: string, platform: string, routing: RuntimeTa
|
||||
syncSchedulerSurface()
|
||||
}
|
||||
|
||||
function createActiveExecution(
|
||||
taskRecord: RuntimeTaskRecord,
|
||||
abortController: AbortController,
|
||||
): ActiveRuntimeExecution {
|
||||
const now = Date.now()
|
||||
return {
|
||||
taskId: taskRecord.id,
|
||||
kind: taskRecord.kind,
|
||||
platform: taskRecord.platform,
|
||||
accountId: taskRecord.accountId,
|
||||
abortController,
|
||||
executionPhase: taskRecord.kind === 'monitor' ? 'PREPARING' : null,
|
||||
interruptRequested:
|
||||
taskRecord.kind === 'monitor' && Boolean(taskRecord.payload.interrupt_requested),
|
||||
interruptGeneration:
|
||||
taskRecord.kind === 'monitor'
|
||||
? toInterruptGeneration(taskRecord.payload.interrupt_generation)
|
||||
: 0,
|
||||
interruptReason: taskRecord.kind === 'monitor' ? interruptReasonFromTask(taskRecord) : null,
|
||||
lastSafePointAt: taskRecord.kind === 'monitor' ? now : null,
|
||||
startedAt: now,
|
||||
lastProgressAt: now,
|
||||
lastProgressSummary: taskRecord.summary,
|
||||
deadlineAt: null,
|
||||
deadlineHandle: null,
|
||||
longRunningNotifiedAt: null,
|
||||
submitStartedAt: null,
|
||||
}
|
||||
}
|
||||
|
||||
function armPublishTaskDeadline(
|
||||
taskRecord: RuntimeTaskRecord,
|
||||
active: ActiveRuntimeExecution,
|
||||
): void {
|
||||
if (taskRecord.kind !== 'publish') {
|
||||
return
|
||||
}
|
||||
|
||||
const taskDeadlineMs = resolvePublishTaskTimeoutMs(taskRecord.platform)
|
||||
active.deadlineAt = active.startedAt + taskDeadlineMs
|
||||
active.deadlineHandle = setTimeout(() => {
|
||||
if (active.abortController.signal.aborted) {
|
||||
return
|
||||
}
|
||||
|
||||
const timeoutError = new Error('publish_task_timeout')
|
||||
recordActivity(
|
||||
'warn',
|
||||
'任务超时',
|
||||
`${taskRecord.title} 执行超过 ${Math.round(taskDeadlineMs / 1000)}s,已中止并释放。`,
|
||||
)
|
||||
updateTaskProgress(taskRecord.id, `${taskRecord.title} 执行超时,正在释放任务。`)
|
||||
active.abortController.abort(timeoutError)
|
||||
}, taskDeadlineMs)
|
||||
}
|
||||
|
||||
function resolvePublishTaskTimeoutMs(platform: string): number {
|
||||
const envKey = `GEO_DESKTOP_PUBLISH_TIMEOUT_${platform.toUpperCase().replace(/[^A-Z0-9]/g, '_')}_MS`
|
||||
return resolvePositiveEnvInteger(envKey) ?? resolvePositiveEnvInteger('GEO_DESKTOP_PUBLISH_TIMEOUT_MS') ?? defaultPublishTaskTimeoutMs
|
||||
}
|
||||
|
||||
function resolvePositiveEnvInteger(key: string): number | null {
|
||||
const raw = process.env[key]?.trim()
|
||||
if (!raw) {
|
||||
return null
|
||||
}
|
||||
const parsed = Number.parseInt(raw, 10)
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : null
|
||||
}
|
||||
|
||||
// Stages emitted by an adapter right before it performs an irreversible platform write (creating
|
||||
// the article/draft). After this point the article may exist on the platform regardless of whether
|
||||
// the local task records it, so an interrupted publish must not be auto-retried.
|
||||
function isIrreversiblePublishStage(stage: string): boolean {
|
||||
const normalized = stage.trim().toLowerCase()
|
||||
return (
|
||||
normalized.endsWith('.submit') ||
|
||||
normalized.endsWith('.save_draft') ||
|
||||
normalized === 'submit' ||
|
||||
normalized === 'save_draft'
|
||||
)
|
||||
}
|
||||
|
||||
// Persist a durable "submit started" marker (server-side, before/at the irreversible POST) and a
|
||||
// local flag the abort path reads. Fires at most once per execution and never blocks publishing.
|
||||
function markPublishSubmitStartedIfNeeded(taskId: string, stage: string): void {
|
||||
if (!isIrreversiblePublishStage(stage)) {
|
||||
return
|
||||
}
|
||||
const active = state.activeExecutions.get(taskId)
|
||||
if (!active || active.kind !== 'publish' || active.submitStartedAt !== null) {
|
||||
return
|
||||
}
|
||||
active.submitStartedAt = Date.now()
|
||||
state.activeExecutions.set(taskId, active)
|
||||
|
||||
const leaseToken = state.tasks.get(taskId)?.leaseToken
|
||||
if (!leaseToken) {
|
||||
return
|
||||
}
|
||||
void markDesktopTaskPublishSubmitStarted(taskId, leaseToken).catch((error) => {
|
||||
console.warn('[desktop-runtime] mark publish submit started failed', {
|
||||
taskId,
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function pumpExecutionLoop(): void {
|
||||
if (!state.running) {
|
||||
return
|
||||
@@ -1837,6 +1987,7 @@ function enqueueLeasedMonitoringTasks(
|
||||
status: 'queued',
|
||||
routing,
|
||||
leaseExpiresAt: parseTimestamp(task.lease_expires_at),
|
||||
startedAt: null,
|
||||
updatedAt: now,
|
||||
summary: `已领取监控租约,等待本地调度执行(${describeRouting(routing)})。`,
|
||||
payload: {
|
||||
@@ -1920,18 +2071,7 @@ async function executeLeasedMonitoringTask(
|
||||
recordActivity('info', '监控任务开始执行', `${taskRecord.title} 已进入本地执行队列。`)
|
||||
|
||||
const abortController = new AbortController()
|
||||
state.activeExecutions.set(taskRecord.id, {
|
||||
taskId: taskRecord.id,
|
||||
kind: 'monitor',
|
||||
platform: taskRecord.platform,
|
||||
accountId: taskRecord.accountId,
|
||||
abortController,
|
||||
executionPhase: 'PREPARING',
|
||||
interruptRequested: Boolean(taskRecord.payload.interrupt_requested),
|
||||
interruptGeneration: toInterruptGeneration(taskRecord.payload.interrupt_generation),
|
||||
interruptReason: interruptReasonFromTask(taskRecord),
|
||||
lastSafePointAt: Date.now(),
|
||||
})
|
||||
state.activeExecutions.set(taskRecord.id, createActiveExecution(taskRecord, abortController))
|
||||
noteMonitorTaskActivated(taskId)
|
||||
syncSchedulerSurface()
|
||||
queueMicrotask(() => pumpExecutionLoop())
|
||||
@@ -2382,6 +2522,7 @@ function noteMonitorExecutionSafePoint(taskId: string, phase: MonitorExecutionPh
|
||||
active.executionPhase = phase
|
||||
}
|
||||
active.lastSafePointAt = Date.now()
|
||||
active.lastProgressAt = Date.now()
|
||||
state.activeExecutions.set(taskId, active)
|
||||
|
||||
const taskRecord = state.tasks.get(taskId)
|
||||
@@ -2406,6 +2547,7 @@ function requestMonitorStaleBusinessDateDrop(taskId: string, staleBusinessDate:
|
||||
|
||||
active.interruptReason = 'stale_business_date'
|
||||
active.lastSafePointAt = Date.now()
|
||||
active.lastProgressAt = Date.now()
|
||||
state.activeExecutions.set(taskId, active)
|
||||
|
||||
const existing = state.tasks.get(taskId)
|
||||
@@ -2523,22 +2665,9 @@ async function executeLeasedTask(
|
||||
)
|
||||
|
||||
const abortController = new AbortController()
|
||||
state.activeExecutions.set(taskRecord.id, {
|
||||
taskId: taskRecord.id,
|
||||
kind: taskRecord.kind,
|
||||
platform: taskRecord.platform,
|
||||
accountId: taskRecord.accountId,
|
||||
abortController,
|
||||
executionPhase: taskRecord.kind === 'monitor' ? 'PREPARING' : null,
|
||||
interruptRequested:
|
||||
taskRecord.kind === 'monitor' && Boolean(taskRecord.payload.interrupt_requested),
|
||||
interruptGeneration:
|
||||
taskRecord.kind === 'monitor'
|
||||
? toInterruptGeneration(taskRecord.payload.interrupt_generation)
|
||||
: 0,
|
||||
interruptReason: taskRecord.kind === 'monitor' ? interruptReasonFromTask(taskRecord) : null,
|
||||
lastSafePointAt: taskRecord.kind === 'monitor' ? Date.now() : null,
|
||||
})
|
||||
const activeExecution = createActiveExecution(taskRecord, abortController)
|
||||
armPublishTaskDeadline(taskRecord, activeExecution)
|
||||
state.activeExecutions.set(taskRecord.id, activeExecution)
|
||||
|
||||
if (task.kind === 'monitor') {
|
||||
noteMonitorTaskLeased(task, routing)
|
||||
@@ -2598,7 +2727,23 @@ async function executeLeasedTask(
|
||||
const structuredError = toStructuredError(error)
|
||||
const accountIdentity = accountIdentityFromTask(taskRecord)
|
||||
|
||||
if (accountIdentity) {
|
||||
// If a publish task was interrupted AFTER it may have entered the irreversible submit phase,
|
||||
// the article might already be live on the platform. Completing as a clean 'failed' would drop
|
||||
// it from the server dedup set and let a retry double-post. Complete as 'unknown' instead so it
|
||||
// stays for manual reconcile. (A pre-submit interruption is a true failure and stays 'failed'.)
|
||||
const submitMaybeStarted =
|
||||
taskRecord.kind === 'publish' && activeExecution.submitStartedAt !== null
|
||||
const terminalStatus: 'failed' | 'unknown' = submitMaybeStarted ? 'unknown' : 'failed'
|
||||
const interruptedSummary = '发布可能已提交到平台,结果待人工确认,已避免自动重复发布。'
|
||||
// Tag the persisted error so the desktop UI can require explicit confirmation before a manual
|
||||
// retry of a possibly-already-published task (mirrors the server recovery payload marker).
|
||||
const terminalError = submitMaybeStarted
|
||||
? { ...structuredError, publish_submit_uncertain: true }
|
||||
: structuredError
|
||||
|
||||
// A submit-phase interruption is typically a network/timeout stall, not an auth problem — don't
|
||||
// wrongly degrade account health for it.
|
||||
if (accountIdentity && !submitMaybeStarted) {
|
||||
await reportAccountFailure(accountIdentity, {
|
||||
summary: failureMessage,
|
||||
error: structuredError,
|
||||
@@ -2615,22 +2760,24 @@ async function executeLeasedTask(
|
||||
try {
|
||||
const completed = await completeDesktopTask(taskRecord.id, {
|
||||
lease_token: leased.lease_token,
|
||||
status: 'failed',
|
||||
error: structuredError,
|
||||
status: terminalStatus,
|
||||
error: terminalError,
|
||||
})
|
||||
|
||||
upsertTaskFromInfo(completed, {
|
||||
routing,
|
||||
summary: failureMessage || '桌面任务执行失败。',
|
||||
summary: submitMaybeStarted ? interruptedSummary : failureMessage || '桌面任务执行失败。',
|
||||
attemptId: null,
|
||||
leaseToken: null,
|
||||
})
|
||||
} catch (completeError) {
|
||||
const existing = state.tasks.get(taskRecord.id)
|
||||
if (existing) {
|
||||
existing.status = 'failed'
|
||||
existing.error = structuredError
|
||||
existing.summary = `${failureMessage || '桌面任务执行失败。'}(result 回写失败:${errorMessage(completeError)})`
|
||||
existing.status = terminalStatus
|
||||
existing.error = terminalError
|
||||
existing.summary = submitMaybeStarted
|
||||
? `${interruptedSummary}(result 回写失败:${errorMessage(completeError)})`
|
||||
: `${failureMessage || '桌面任务执行失败。'}(result 回写失败:${errorMessage(completeError)})`
|
||||
existing.updatedAt = Date.now()
|
||||
state.tasks.set(taskRecord.id, existing)
|
||||
}
|
||||
@@ -2638,12 +2785,18 @@ async function executeLeasedTask(
|
||||
|
||||
noteLeaseReleased('failed', taskRecord.id)
|
||||
recordActivity(
|
||||
'danger',
|
||||
'任务执行失败',
|
||||
`${taskRecord.title} 执行失败:${failureMessage || '未知错误'}`,
|
||||
submitMaybeStarted ? 'warn' : 'danger',
|
||||
submitMaybeStarted ? '发布结果待确认' : '任务执行失败',
|
||||
submitMaybeStarted
|
||||
? `${taskRecord.title} 进入提交阶段后中断,可能已发布,已转人工确认以避免重复发文。`
|
||||
: `${taskRecord.title} 执行失败:${failureMessage || '未知错误'}`,
|
||||
)
|
||||
} finally {
|
||||
clearInterval(extendHandle)
|
||||
if (activeExecution.deadlineHandle) {
|
||||
clearTimeout(activeExecution.deadlineHandle)
|
||||
activeExecution.deadlineHandle = null
|
||||
}
|
||||
if (taskRecord.kind === 'publish') {
|
||||
notePublishTaskCompleted(taskRecord.id, taskRecord.platform)
|
||||
}
|
||||
@@ -2656,6 +2809,7 @@ async function executeLeasedTask(
|
||||
|
||||
async function extendActiveLease(taskId: string, leaseToken: string): Promise<void> {
|
||||
const existing = state.tasks.get(taskId)
|
||||
const active = state.activeExecutions.get(taskId)
|
||||
if (existing?.kind === 'monitor') {
|
||||
const staleBusinessDate = resolveStaleMonitoringBusinessDate(existing.payload)
|
||||
if (staleBusinessDate) {
|
||||
@@ -2663,6 +2817,28 @@ async function extendActiveLease(taskId: string, leaseToken: string): Promise<vo
|
||||
return
|
||||
}
|
||||
}
|
||||
if (existing?.kind === 'publish' && active) {
|
||||
const idleMs = Date.now() - active.lastProgressAt
|
||||
if (idleMs >= publishTaskStaleProgressMs) {
|
||||
const detail = active.lastProgressSummary ? `,最后进度:${active.lastProgressSummary}` : ''
|
||||
existing.summary = `发布任务超过 ${Math.round(idleMs / 1000)} 秒没有新进度,已中止并释放,准备由服务端重排${detail}。`
|
||||
existing.updatedAt = Date.now()
|
||||
state.tasks.set(taskId, existing)
|
||||
recordActivity(
|
||||
'warn',
|
||||
'发布任务疑似卡住',
|
||||
`${existing.title} ${existing.summary}`,
|
||||
)
|
||||
// Abort the local execution instead of merely letting the lease lapse: this
|
||||
// releases the slot immediately and, crucially, closes the window where the
|
||||
// server could re-queue the task while a stalled adapter is still mid-publish
|
||||
// (which would double-post a non-idempotent article).
|
||||
if (!active.abortController.signal.aborted) {
|
||||
active.abortController.abort(new Error('publish_stale_progress'))
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const task = await extendDesktopTask(taskId, { lease_token: leaseToken })
|
||||
@@ -2673,7 +2849,10 @@ async function extendActiveLease(taskId: string, leaseToken: string): Promise<vo
|
||||
if (existing) {
|
||||
existing.leaseExpiresAt = parseTimestamp(task.lease_expires_at) ?? existing.leaseExpiresAt
|
||||
existing.updatedAt = Date.now()
|
||||
existing.summary = '租约已自动续期,任务仍在执行。'
|
||||
existing.summary =
|
||||
existing.kind === 'publish'
|
||||
? (active?.lastProgressSummary ?? '租约已自动续期,发布任务仍在执行。')
|
||||
: '租约已自动续期,任务仍在执行。'
|
||||
state.tasks.set(taskId, existing)
|
||||
}
|
||||
} catch (error) {
|
||||
@@ -2738,6 +2917,7 @@ async function executeTaskAdapter(
|
||||
reportProgress(stage: string) {
|
||||
noteMonitorExecutionSafePoint(task.id, null)
|
||||
updateTaskProgress(task.id, `publish adapter progress: ${stage}`)
|
||||
markPublishSubmitStartedIfNeeded(task.id, stage)
|
||||
},
|
||||
},
|
||||
payload,
|
||||
@@ -3077,6 +3257,17 @@ function updateTaskProgress(taskId: string, summary: string): void {
|
||||
existing.summary = summary
|
||||
existing.updatedAt = Date.now()
|
||||
state.tasks.set(taskId, existing)
|
||||
|
||||
const active = state.activeExecutions.get(taskId)
|
||||
if (active) {
|
||||
active.lastProgressAt = Date.now()
|
||||
active.lastProgressSummary = summary
|
||||
state.activeExecutions.set(taskId, active)
|
||||
}
|
||||
|
||||
if (existing.kind === 'publish') {
|
||||
emitRuntimeInvalidated('publish-task-progress', { taskId })
|
||||
}
|
||||
}
|
||||
|
||||
function upsertTaskFromInfo(
|
||||
@@ -3102,6 +3293,7 @@ function upsertTaskFromInfo(
|
||||
status: task.status,
|
||||
routing: options.routing ?? existing?.routing ?? 'db-recovery',
|
||||
leaseExpiresAt: parseTimestamp(task.lease_expires_at) ?? null,
|
||||
startedAt: task.status === 'in_progress' ? (existing?.startedAt ?? Date.now()) : null,
|
||||
updatedAt: parseTimestamp(task.updated_at) ?? Date.now(),
|
||||
summary: options.summary ?? existing?.summary ?? summaryFromTaskStatus(task.status),
|
||||
payload,
|
||||
@@ -3149,6 +3341,42 @@ function syncSchedulerSurface(): void {
|
||||
setSchedulerCurrentTask(primaryActiveTaskId())
|
||||
}
|
||||
|
||||
function notifyLongRunningPublishTasks(): void {
|
||||
const now = Date.now()
|
||||
for (const active of state.activeExecutions.values()) {
|
||||
if (active.kind !== 'publish' || active.longRunningNotifiedAt) {
|
||||
continue
|
||||
}
|
||||
if (now - active.startedAt < publishLongRunningNotificationMs) {
|
||||
continue
|
||||
}
|
||||
active.longRunningNotifiedAt = now
|
||||
state.activeExecutions.set(active.taskId, active)
|
||||
|
||||
const task = state.tasks.get(active.taskId)
|
||||
if (task) {
|
||||
task.summary = `${task.summary || '发布任务仍在执行。'}(已执行 ${formatDurationText(now - active.startedAt)},如网络异常可取消后重试。)`
|
||||
task.updatedAt = now
|
||||
state.tasks.set(task.id, task)
|
||||
recordActivity('warn', '发布等待较久', `${task.title} 已执行 ${formatDurationText(now - active.startedAt)},可能存在网络异常。`)
|
||||
emitRuntimeInvalidated('publish-task-progress', { taskId: task.id })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function formatDurationText(ms: number): string {
|
||||
const seconds = Math.max(0, Math.floor(ms / 1000))
|
||||
const minutes = Math.floor(seconds / 60)
|
||||
const restSeconds = seconds % 60
|
||||
if (minutes <= 0) {
|
||||
return `${restSeconds} 秒`
|
||||
}
|
||||
if (restSeconds === 0) {
|
||||
return `${minutes} 分钟`
|
||||
}
|
||||
return `${minutes} 分 ${restSeconds} 秒`
|
||||
}
|
||||
|
||||
function localQueueDepth(): number {
|
||||
const publish = getPublishSchedulerSnapshot()
|
||||
const monitor = getMonitorSchedulerSnapshot()
|
||||
@@ -3229,7 +3457,10 @@ function activeAdmissionCount(): number {
|
||||
}
|
||||
|
||||
function hasTotalCapacityForNewExecution(): boolean {
|
||||
return activeAdmissionCount() < resolveAdaptiveTotalConcurrency()
|
||||
return (
|
||||
activeAdmissionCount() < resolveAdaptiveTotalConcurrency() &&
|
||||
shouldAdmitNewExecutionUnderRuntimePressure()
|
||||
)
|
||||
}
|
||||
|
||||
function canStartAnotherPublishExecution(): boolean {
|
||||
@@ -3355,9 +3586,13 @@ function resolveHardwareTotalConcurrency(): number {
|
||||
}
|
||||
|
||||
function resolveRuntimeHealthTotalConcurrency(hardwareCap: number): number {
|
||||
return hardwareCap
|
||||
}
|
||||
|
||||
function shouldAdmitNewExecutionUnderRuntimePressure(): boolean {
|
||||
const metrics = getProcessMetricsSnapshot().latestSample
|
||||
if (!metrics) {
|
||||
return Math.min(hardwareCap, minimumForegroundTotalConcurrency)
|
||||
return true
|
||||
}
|
||||
|
||||
const totalCPUUsage = metrics.processes.reduce((sum, item) => sum + item.percentCPUUsage, 0)
|
||||
@@ -3371,19 +3606,10 @@ function resolveRuntimeHealthTotalConcurrency(hardwareCap: number): number {
|
||||
mainPrivateBytesKB >= 1_000_000 ||
|
||||
processCount >= 40
|
||||
) {
|
||||
return minimumForegroundTotalConcurrency
|
||||
return activeAdmissionCount() < minimumForegroundTotalConcurrency
|
||||
}
|
||||
|
||||
if (
|
||||
totalCPUUsage >= 120 ||
|
||||
totalPrivateBytesKB >= 1_800_000 ||
|
||||
mainPrivateBytesKB >= 750_000 ||
|
||||
processCount >= 32
|
||||
) {
|
||||
return Math.min(hardwareCap, 3)
|
||||
}
|
||||
|
||||
return hardwareCap
|
||||
return true
|
||||
}
|
||||
|
||||
function resolveFeatureFlagTotalConcurrency(): number | null {
|
||||
|
||||
Reference in New Issue
Block a user