fix: harden desktop monitoring recovery
Desktop Client Build / Resolve Build Metadata (push) Successful in 37s
Backend CI / Backend (push) Failing after 9m26s
Desktop Client Build / Build Desktop Client (push) Successful in 22m29s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 35s

This commit is contained in:
2026-06-23 23:12:25 +08:00
parent b4ebf34728
commit 04bd3e42e0
16 changed files with 1329 additions and 164 deletions
@@ -12,6 +12,7 @@ const {
isNonCitationAssetUrl,
parseDoubaoStreamBody,
resolveDoubaoSubmissionAnswer,
shouldRetryDoubaoPageAttempt,
} = __doubaoTestUtils
describe('doubao adapter helpers', () => {
@@ -28,6 +29,69 @@ describe('doubao adapter helpers', () => {
expect(isDoubaoRecoverablePageError('doubao_login_expired')).toBe(false)
})
it('retries once when Doubao is stuck on the history surface without an answer', () => {
const streamSummary = parseDoubaoStreamBody(
['event: SSE_REPLY_END', 'data: {"message_id":"msg_1"}', '', ''].join('\n'),
)
expect(
shouldRetryDoubaoPageAttempt(
{
ok: false,
error: 'doubao_query_timeout',
detail: '历史对话 主对话 折叠门联动配件明细',
url: 'https://www.doubao.com/chat/local_4995117315817271',
title: '豆包',
modelLabel: '豆包',
modeLabel: '思考',
thinkingModeApplied: true,
deepThinkEnabled: true,
domAnswer: '折叠门联动配件明细',
domReasoning: null,
domLinks: [],
submitted: true,
questionAnchorFound: true,
historySurfaceDetected: true,
},
streamSummary,
),
).toBe(true)
})
it('does not retry a recoverable Doubao page error after SSE already yielded an answer', () => {
const streamSummary = parseDoubaoStreamBody(
[
'event: STREAM_CHUNK',
'data: {"text_block":{"text":"这是一段完整的豆包回答。"}}',
'',
'',
].join('\n'),
)
expect(
shouldRetryDoubaoPageAttempt(
{
ok: false,
error: 'doubao_query_timeout',
detail: 'timeout',
url: 'https://www.doubao.com/chat/local_4995117315817271',
title: '豆包',
modelLabel: '豆包',
modeLabel: '思考',
thinkingModeApplied: true,
deepThinkEnabled: true,
domAnswer: null,
domReasoning: null,
domLinks: [],
submitted: true,
questionAnchorFound: true,
historySurfaceDetected: true,
},
streamSummary,
),
).toBe(false)
})
it('maps the new expert/deep-thinking mode label to expert mode', () => {
expect(doubaoModeKindFromLabelText('专家')).toBe('expert')
expect(doubaoModeKindFromLabelText('专家 深度思考 研究级智能模型')).toBe('expert')
+208 -33
View File
@@ -9,7 +9,7 @@ const DOUBAO_PAGE_READY_TIMEOUT_MS = 20_000
const DOUBAO_QUERY_TIMEOUT_MS = 120_000
const DOUBAO_CAPTURE_LIMIT = 64
const DOUBAO_CAPTURE_BODY_LIMIT = 8_000_000
const DOUBAO_CAPTURE_FLUSH_TIMEOUT_MS = DOUBAO_QUERY_TIMEOUT_MS
const DOUBAO_CAPTURE_FLUSH_TIMEOUT_MS = 10_000
const DOUBAO_STREAM_URL_FRAGMENT = '/chat/completion'
const DOUBAO_NON_CITATION_ASSET_DOMAINS = new Set(['byteimg.com', 'bytednsdoc.com'])
@@ -116,10 +116,22 @@ type DoubaoPageQueryFailureResult = {
domAnswer: string | null
domReasoning: string | null
domLinks: DoubaoDomLink[]
submitted: boolean | null
questionAnchorFound: boolean | null
historySurfaceDetected: boolean | null
}
type DoubaoPageQueryResult = DoubaoPageQuerySuccessResult | DoubaoPageQueryFailureResult
type DoubaoQueryAttemptResult = {
attempt: number
pageResult: DoubaoPageQueryResult
captures: DoubaoCaptureRecord[]
streamSummary: DoubaoStreamSummary
captureDebug: JsonValue[]
pendingCaptureCount: number
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value)
}
@@ -1039,7 +1051,12 @@ function isDoubaoExecutionContextNavigationError(error: unknown): boolean {
}
function isDoubaoRecoverablePageError(error: string): boolean {
return error === 'doubao_query_timeout' || error === 'doubao_page_navigation_interrupted'
return (
error === 'doubao_query_timeout' ||
error === 'doubao_page_navigation_interrupted' ||
error === 'doubao_submission_not_observed' ||
error === 'doubao_history_surface_timeout'
)
}
function buildDoubaoPageNavigationResult(
@@ -1059,9 +1076,30 @@ function buildDoubaoPageNavigationResult(
domAnswer: null,
domReasoning: null,
domLinks: [],
submitted: null,
questionAnchorFound: null,
historySurfaceDetected: null,
}
}
function shouldRetryDoubaoPageAttempt(
pageResult: DoubaoPageQueryResult,
streamSummary: DoubaoStreamSummary,
): boolean {
if (pageResult.ok || !isDoubaoRecoverablePageError(pageResult.error)) {
return false
}
if (isDoubaoSSESubmissionCandidate(streamSummary.answer)) {
return false
}
return (
pageResult.error === 'doubao_submission_not_observed' ||
pageResult.error === 'doubao_history_surface_timeout' ||
pageResult.historySurfaceDetected === true ||
pageResult.questionAnchorFound === false
)
}
function buildDoubaoUnknownResult(
code: string,
summary: string,
@@ -1597,6 +1635,8 @@ const doubaoQueryInPage = async (
const pollIntervalMs = 1_500
const incompleteAnswerGraceMs = 30_000
const minimumAnswerSettleMs = 10_000
const submissionObserveTimeoutMs = 12_000
const staleHistoryTimeoutMs = 18_000
const wait = (timeMs: number) =>
new Promise<void>((resolve) => {
@@ -2684,6 +2724,39 @@ const doubaoQueryInPage = async (
line,
))
const looksLikeHistoryTitleList = (text: string | null): boolean => {
if (!text) {
return false
}
const normalizedText = text.replace(/\s+/g, ' ').trim()
if (
/搜索…\s*Ctrl\s*K\s*豆包|新对话\s*Ctrl\s*Shift\s*K|历史对话\s*主对话/.test(
normalizedText,
)
) {
return true
}
const lines = text
.split(/\n+/)
.map((line) => line.trim())
.filter(Boolean)
if (lines.length < 6) {
return false
}
const shortTitleLines = lines.filter(
(line) =>
line.length <= 36 &&
!/[。!?!?]/.test(line) &&
/门|锁|轨|五金|导轨|合页|配件|工厂|厂家|供货|定制|折叠|幽灵|口袋/.test(line),
).length
const navHits = lines.filter((line) =>
/^(搜索|豆包|新对话|新办公任务|AI 创作|AI创作|云盘|收起|发现智能体|历史对话|主对话)$/.test(
line,
),
).length
return shortTitleLines >= 6 && (navHits >= 2 || shortTitleLines / lines.length >= 0.65)
}
// Tokens that only appear on Doubao's empty-chat surface (prompt templates, nav, feature chips).
const EMPTY_CHAT_TOKENS = [
'快速',
@@ -2721,6 +2794,9 @@ const doubaoQueryInPage = async (
return hits >= 3 || (hits >= 2 && lines.length <= 6)
}
const looksLikeNonAnswerSurface = (text: string | null): boolean =>
looksLikeEmptyChatSurface(text) || looksLikeHistoryTitleList(text)
const stripKnownNoise = (text: string | null): string | null => {
if (!text) {
return null
@@ -2870,7 +2946,7 @@ const doubaoQueryInPage = async (
if (!text && links.length === 0) {
continue
}
if (rawText && looksLikeEmptyChatSurface(rawText)) {
if (rawText && looksLikeNonAnswerSurface(rawText)) {
continue
}
if (text && text.length < 12 && links.length === 0) {
@@ -3033,6 +3109,11 @@ const doubaoQueryInPage = async (
applied: null,
},
snapshotOverride: ConversationSnapshot | null = null,
diagnostics: {
submitted?: boolean | null
questionAnchorFound?: boolean | null
historySurfaceDetected?: boolean | null
} = {},
): DoubaoPageQueryFailureResult => {
const snapshot = snapshotOverride ?? snapshotConversation(composerTop)
const composer = findComposer()
@@ -3057,6 +3138,9 @@ const doubaoQueryInPage = async (
domAnswer: snapshot.answerText,
domReasoning: snapshot.reasoningText,
domLinks: snapshot.links,
submitted: diagnostics.submitted ?? null,
questionAnchorFound: diagnostics.questionAnchorFound ?? null,
historySurfaceDetected: diagnostics.historySurfaceDetected ?? null,
}
}
@@ -3113,6 +3197,8 @@ const doubaoQueryInPage = async (
}
const initialSnapshot = snapshotConversation(composerTop)
let questionAnchorFoundAfterSubmit = Boolean(findQuestionAnchor())
let historySurfaceDetected = looksLikeHistoryTitleList(bodyText())
let bestSnapshot = initialSnapshot
let bestContentSnapshot: ConversationSnapshot | null = hasObservedContent(initialSnapshot)
? initialSnapshot
@@ -3121,10 +3207,13 @@ const doubaoQueryInPage = async (
let stableCount = submitQuestion ? 0 : hasObservedContent(initialSnapshot) ? 1 : 0
let sawChange = !submitQuestion && hasObservedContent(initialSnapshot)
let firstChangedContentAt: number | null = sawChange ? Date.now() : null
const submittedAt = Date.now()
const deadline = Date.now() + timeoutMs
while (Date.now() <= deadline) {
const snapshot = snapshotConversation(composerTop)
questionAnchorFoundAfterSubmit = questionAnchorFoundAfterSubmit || Boolean(findQuestionAnchor())
historySurfaceDetected = historySurfaceDetected || looksLikeHistoryTitleList(bodyText())
bestSnapshot = pickBetterSnapshot(bestSnapshot, snapshot)
if (hasObservedContent(snapshot)) {
bestContentSnapshot = pickBetterSnapshot(bestContentSnapshot, snapshot)
@@ -3152,6 +3241,53 @@ const doubaoQueryInPage = async (
composerTop,
answerMode,
bestContentSnapshot ?? bestSnapshot,
{
submitted: submitQuestion,
questionAnchorFound: questionAnchorFoundAfterSubmit,
historySurfaceDetected,
},
)
}
const elapsedSinceSubmit = Date.now() - submittedAt
if (
submitQuestion &&
!questionAnchorFoundAfterSubmit &&
!busy &&
elapsedSinceSubmit >= submissionObserveTimeoutMs
) {
return fail(
'doubao_submission_not_observed',
bodyText(),
composerTop,
answerMode,
bestContentSnapshot ?? bestSnapshot,
{
submitted: true,
questionAnchorFound: false,
historySurfaceDetected,
},
)
}
if (
submitQuestion &&
questionAnchorFoundAfterSubmit &&
historySurfaceDetected &&
!hasContent &&
!busy &&
elapsedSinceSubmit >= staleHistoryTimeoutMs
) {
return fail(
'doubao_history_surface_timeout',
bodyText(),
composerTop,
answerMode,
bestContentSnapshot ?? bestSnapshot,
{
submitted: true,
questionAnchorFound: true,
historySurfaceDetected: true,
},
)
}
@@ -3196,6 +3332,11 @@ const doubaoQueryInPage = async (
composerTop,
answerMode,
bestContentSnapshot ?? bestSnapshot,
{
submitted: submitQuestion,
questionAnchorFound: questionAnchorFoundAfterSubmit,
historySurfaceDetected,
},
)
}
@@ -3214,40 +3355,66 @@ export const doubaoAdapter: MonitorAdapter = {
const questionText = extractQuestionText(payload)
const page = context.playwright.page
const runAttempt = async (attempt: number): Promise<DoubaoQueryAttemptResult> => {
context.reportProgress(attempt === 1 ? 'doubao.query' : 'doubao.query_retry')
const capture = attachDoubaoResponseCapture(page)
let pageResult: DoubaoPageQueryResult
try {
try {
pageResult = await page.evaluate(doubaoQueryInPage, {
questionText,
timeoutMs: DOUBAO_QUERY_TIMEOUT_MS,
readyTimeoutMs: DOUBAO_PAGE_READY_TIMEOUT_MS,
enableDeepThink: true,
})
} catch (error) {
if (!isDoubaoExecutionContextNavigationError(error)) {
throw error
}
await page
.waitForLoadState('domcontentloaded', { timeout: DOUBAO_PAGE_READY_TIMEOUT_MS })
.catch(() => undefined)
await page.waitForLoadState('networkidle', { timeout: 3_000 }).catch(() => undefined)
pageResult = buildDoubaoPageNavigationResult(page, formatUnknownError(error))
}
await sleep(600, context.signal).catch(() => undefined)
await capture.flush(DOUBAO_CAPTURE_FLUSH_TIMEOUT_MS)
} finally {
capture.dispose()
}
const streamSummary = parseDoubaoCaptures(capture.captures)
return {
attempt,
pageResult,
captures: [...capture.captures],
streamSummary,
captureDebug: summarizeCapturesForDebug(capture.captures),
pendingCaptureCount: capture.pendingCount(),
}
}
context.reportProgress('doubao.page_ready')
await ensurePageOnDoubaoChat(page, context.signal)
const capture = attachDoubaoResponseCapture(page)
let pageResult: DoubaoPageQueryResult
try {
context.reportProgress('doubao.query')
try {
pageResult = await page.evaluate(doubaoQueryInPage, {
questionText,
timeoutMs: DOUBAO_QUERY_TIMEOUT_MS,
readyTimeoutMs: DOUBAO_PAGE_READY_TIMEOUT_MS,
enableDeepThink: true,
})
} catch (error) {
if (!isDoubaoExecutionContextNavigationError(error)) {
throw error
}
await page
.waitForLoadState('domcontentloaded', { timeout: DOUBAO_PAGE_READY_TIMEOUT_MS })
.catch(() => undefined)
await page.waitForLoadState('networkidle', { timeout: 3_000 }).catch(() => undefined)
pageResult = buildDoubaoPageNavigationResult(page, formatUnknownError(error))
}
await sleep(600, context.signal).catch(() => undefined)
await capture.flush(DOUBAO_CAPTURE_FLUSH_TIMEOUT_MS)
} finally {
capture.dispose()
let attemptResult = await runAttempt(1)
if (
attemptResult.pageResult.ok === false &&
shouldRetryDoubaoPageAttempt(attemptResult.pageResult, attemptResult.streamSummary)
) {
context.reportProgress('doubao.recover_new_chat')
await resetDoubaoPageAfterRecoverableFailure(
page,
attemptResult.pageResult.error,
context.signal,
)
await ensurePageOnDoubaoChat(page, context.signal)
attemptResult = await runAttempt(2)
}
const streamSummary = parseDoubaoCaptures(capture.captures)
const captureDebug = summarizeCapturesForDebug(capture.captures)
const pendingCaptureCount = capture.pendingCount()
const pageResult = attemptResult.pageResult
const streamSummary = attemptResult.streamSummary
const captureDebug = attemptResult.captureDebug
const pendingCaptureCount = attemptResult.pendingCaptureCount
const domSearchResults = dedupeSourceItems(
pageResult.domLinks
.map((item) =>
@@ -3316,6 +3483,11 @@ export const doubaoAdapter: MonitorAdapter = {
reply_end_event_count: streamSummary.reply_end_event_count,
answer_finish_event_count: streamSummary.answer_finish_event_count,
stream_event_count: streamSummary.event_count,
submitted: failed.submitted,
question_anchor_found: failed.questionAnchorFound,
history_surface_detected: failed.historySurfaceDetected,
query_attempt: attemptResult.attempt,
captures: captureDebug,
}),
}
}
@@ -3426,11 +3598,12 @@ export const doubaoAdapter: MonitorAdapter = {
platform: 'doubao',
page_url: pageResult.url,
page_title: pageResult.title,
query_attempt: attemptResult.attempt,
model_label: pageResult.modelLabel,
mode_label: pageResult.modeLabel,
thinking_mode_applied: pageResult.thinkingModeApplied,
deep_think_enabled: pageResult.deepThinkEnabled,
capture_count: capture.captures.length,
capture_count: attemptResult.captures.length,
capture_pending_after_flush: pendingCaptureCount,
event_count: streamSummary.event_count,
full_message_event_count: streamSummary.full_message_event_count,
@@ -3464,6 +3637,8 @@ export const __doubaoTestUtils = {
isNonCitationAssetDomain,
isNonCitationAssetUrl,
isDoubaoRecoverablePageError,
isDoubaoDOMSubmissionCandidate,
shouldRetryDoubaoPageAttempt,
parseDoubaoStreamBody,
resolveDoubaoSubmissionAnswer,
}
+71 -6
View File
@@ -53,7 +53,7 @@ import {
installObservedGlobalFetch,
registerObservedRequestRendererTarget,
} from './network-observer'
import { getPlaywrightCDPPort, startHiddenPlaywrightReaper } from './playwright-cdp'
import { getPlaywrightCDPPort, preparePlaywrightCDPPort, startHiddenPlaywrightReaper } from './playwright-cdp'
import { initProcessMetricsSampler } from './process-metrics'
import { registerRendererDevtoolsProxyTarget } from './renderer-devtools-proxy'
import {
@@ -112,8 +112,8 @@ app.commandLine.appendSwitch(
)
app.commandLine.appendSwitch('disable-quic')
app.commandLine.appendSwitch('disable-background-networking')
// CDP is required by the hidden Playwright execution pool in packaged builds.
app.commandLine.appendSwitch('remote-debugging-port', String(getPlaywrightCDPPort()))
// CDP is required by the hidden Playwright execution pool. Electron only reads
// this switch during early startup, so select the port before app.whenReady().
app.userAgentFallback = STANDARD_USER_AGENT
// Windows 10+ routes tray.displayBalloon through the Toast Notification API,
@@ -151,6 +151,7 @@ console.info('[desktop-main] boot', {
chrome: process.versions.chrome,
node: process.versions.node,
ua: STANDARD_USER_AGENT,
cdpPort: getPlaywrightCDPPort(),
})
process.on('unhandledRejection', (reason) => {
@@ -166,6 +167,7 @@ let loginWindow: ElectronBrowserWindow | null = null
let settingsWindow: ElectronBrowserWindow | null = null
let mainRendererContents: ElectronWebContents | null = null
let quitReleaseInFlight = false
let cdpRecoveryRelaunchInFlight = false
let quitAfterSaasLogoutPromise: Promise<void> | null = null
let mainWindowCreatePromise: Promise<ElectronBrowserWindow> | null = null
let loginWindowCreatePromise: Promise<ElectronBrowserWindow> | null = null
@@ -176,6 +178,10 @@ let canHandleDesktopDeepLinks = false
const pendingDesktopDeepLinks: string[] = []
const forceCloseWindows = new WeakSet<ElectronBrowserWindow>()
const appWindowModes = new WeakMap<ElectronBrowserWindow, RendererWindowMode>()
const maxCDPRecoveryRelaunchesPerWindow = 3
const cdpRecoveryRelaunchWindowMs = 10 * 60_000
const cdpRecoveryRelaunchDelayMs = 1_500
const cdpFatalArg = '--geo-cdp-fatal'
type WindowMode = 'login' | 'main'
type RendererWindowMode = WindowMode | 'settings'
@@ -547,6 +553,50 @@ function quitAfterClearingSaasSession(): void {
})
}
function requestPlaywrightCDPRecoveryRelaunch(message: string): void {
if (cdpRecoveryRelaunchInFlight) {
return
}
const now = Date.now()
const recentRelaunches = process.argv
.filter((arg) => arg.startsWith('--geo-cdp-recovery-at='))
.map((arg) => Number.parseInt(arg.split('=')[1] ?? '', 10))
.filter((timestamp) => Number.isFinite(timestamp) && now - timestamp < cdpRecoveryRelaunchWindowMs)
if (recentRelaunches.length >= maxCDPRecoveryRelaunchesPerWindow) {
console.error('[desktop-main] playwright cdp recovery relaunch suppressed', {
message,
recentRelaunches: recentRelaunches.length,
})
showTrayBalloon(
'浏览器自动化通道异常',
'客户端已停止采集。请重启客户端或检查安全软件/端口占用。',
)
return
}
cdpRecoveryRelaunchInFlight = true
console.warn('[desktop-main] scheduling playwright cdp recovery relaunch', {
message,
cdpPort: getPlaywrightCDPPort(),
})
showTrayBalloon('正在恢复浏览器自动化通道', '客户端将自动重启后台进程,登录状态会保留。')
setTimeout(() => {
quitReleaseInFlight = true
app.relaunch({
args: [
...process.argv.slice(1).filter((arg) => !arg.startsWith('--geo-cdp-recovery-at=')),
...recentRelaunches.map((timestamp) => `--geo-cdp-recovery-at=${timestamp}`),
`--geo-cdp-recovery-at=${now}`,
...(recentRelaunches.length + 1 >= maxCDPRecoveryRelaunchesPerWindow ? [cdpFatalArg] : []),
],
})
app.exit(0)
}, cdpRecoveryRelaunchDelayMs)
}
function windowFromIpcEvent(event: IpcMainInvokeEvent): ElectronBrowserWindow | null {
const window = BrowserWindow.fromWebContents(event.sender)
if (!window || window.isDestroyed()) {
@@ -1197,9 +1247,15 @@ if (!hasSingleInstanceLock) {
console.info('[desktop-main] another instance is already running; exiting')
app.exit(0)
} else {
app
.whenReady()
preparePlaywrightCDPPort()
.then((selectedPlaywrightCDPPort) => {
app.commandLine.appendSwitch('remote-debugging-port', String(selectedPlaywrightCDPPort))
return app.whenReady()
})
.then(async () => {
console.info('[desktop-main] playwright cdp port selected', {
cdpPort: getPlaywrightCDPPort(),
})
suppressNativeWindowMenu()
registerDesktopProtocolClient()
currentWindowMode = readPersistedWindowMode()
@@ -1215,6 +1271,15 @@ if (!hasSingleInstanceLock) {
startHiddenPlaywrightReaper()
onRuntimeInvalidated((event) => {
syncDesktopHealthIndicator(currentMainWindow())
if (event.reason === 'playwright-cdp-fatal') {
showTrayBalloon(
'浏览器自动化组件不可用',
event.message ?? '客户端已停止采集,请重启客户端或检查安全软件/端口占用。',
)
}
if (event.reason === 'playwright-cdp-recovery') {
requestPlaywrightCDPRecoveryRelaunch(event.message ?? 'playwright_cdp_endpoint_unavailable')
}
if (!mainRendererContents || mainRendererContents.isDestroyed()) {
return
}
@@ -1234,7 +1299,7 @@ if (!hasSingleInstanceLock) {
flushPendingDesktopDeepLinks()
})
.catch((error) => {
console.error('[desktop-main] app.whenReady failed', error)
console.error('[desktop-main] startup failed', error)
})
app.on('activate', () => {
@@ -64,6 +64,7 @@ interface MonitorSchedulerSelectionOptions {
maxConcurrency: number
activePlatforms: ReadonlySet<string>
activeQuestionKeys: ReadonlySet<string>
blockedPlatforms?: ReadonlySet<string>
activeCount: number
}
@@ -434,6 +435,22 @@ export function noteMonitorTaskLeaseMiss(taskId: string): void {
})
}
export function noteMonitorTaskDeferred(taskId: string, delayMs: number): void {
const now = Date.now()
const availableAt = now + Math.max(0, delayMs)
mutatePersistedState((draft) => {
const existing = draft.tasks[taskId]
if (!existing) {
return
}
existing.state = 'queued'
existing.availableAt = Math.max(existing.availableAt, availableAt)
existing.lastSeenAt = now
})
}
export function noteMonitorTaskCompleted(taskId: string): void {
const now = Date.now()
@@ -488,6 +505,10 @@ export function selectNextMonitorTask(
})
for (const candidate of candidates) {
if (options.blockedPlatforms?.has(candidate.platform)) {
continue
}
if (options.activePlatforms.has(candidate.platform)) {
continue
}
+210 -24
View File
@@ -1,3 +1,5 @@
import { createServer } from 'node:net'
import type { BrowserWindow as ElectronBrowserWindow, Session } from 'electron/main'
import { BrowserWindow, nativeTheme } from 'electron/main'
import type { Browser as PlaywrightBrowser, Page as PlaywrightPage } from 'playwright-core'
@@ -13,9 +15,23 @@ const hiddenPlaywrightReaperIntervalMs = 60_000
const defaultConnectTimeoutMs = 10_000
const defaultPageTimeoutMs = 45_000
const hiddenBootstrapMetaName = 'geo-hidden-playwright-token'
const cdpPort = Number.parseInt(process.env.GEO_ELECTRON_CDP_PORT ?? '9339', 10)
const cdpEndpointURL = `http://127.0.0.1:${cdpPort}`
const cdpTargetsListURL = `${cdpEndpointURL}/json/list`
const preferredCDPPort = normalizeCDPPort(process.env.GEO_ELECTRON_CDP_PORT, 9339)
const dynamicCDPPortMin = normalizeCDPPort(process.env.GEO_ELECTRON_CDP_PORT_MIN, 9339)
const dynamicCDPPortMax = normalizeCDPPort(process.env.GEO_ELECTRON_CDP_PORT_MAX, 20000)
const cdpCircuitOpenTTLms = 60_000
const cdpCircuitProbeTimeoutMs = 2_000
let cdpPort = preferredCDPPort
let cdpEndpointURL = endpointURLForPort(cdpPort)
let cdpTargetsListURL = `${cdpEndpointURL}/json/list`
interface PlaywrightCDPHealthState {
status: 'unknown' | 'healthy' | 'unavailable'
checkedAt: number | null
unavailableSince: number | null
nextRetryAt: number | null
lastError: string | null
}
interface HiddenPlaywrightRecord {
key: string
@@ -41,11 +57,95 @@ const records = new Map<string, HiddenPlaywrightRecord>()
let reaperHandle: ReturnType<typeof setInterval> | null = null
let connectPromise: Promise<PlaywrightBrowser> | null = null
let connectedBrowser: PlaywrightBrowser | null = null
let cdpHealthState: PlaywrightCDPHealthState = {
status: 'unknown',
checkedAt: null,
unavailableSince: null,
nextRetryAt: null,
lastError: null,
}
export function getPlaywrightCDPPort(): number {
return cdpPort
}
export async function preparePlaywrightCDPPort(): Promise<number> {
const port = await findAvailableCDPPort()
setPlaywrightCDPPort(port)
return port
}
export function isPlaywrightCDPUnavailableError(error: unknown): boolean {
if (!(error instanceof Error)) {
return false
}
return (
error.message === 'playwright_cdp_endpoint_unavailable' ||
error.message === 'playwright_cdp_port_unavailable' ||
error.message.includes('playwright_cdp_endpoint_unavailable') ||
error.message.includes('connectOverCDP')
)
}
export async function ensurePlaywrightCDPReady(options: { force?: boolean } = {}): Promise<void> {
const now = Date.now()
if (
!options.force &&
cdpHealthState.status === 'healthy' &&
cdpHealthState.checkedAt &&
now - cdpHealthState.checkedAt < 15_000
) {
return
}
if (
!options.force &&
cdpHealthState.status === 'unavailable' &&
cdpHealthState.nextRetryAt &&
cdpHealthState.nextRetryAt > now
) {
throw new Error(cdpHealthState.lastError || 'playwright_cdp_endpoint_unavailable')
}
try {
await waitForCDPEndpoint(cdpCircuitProbeTimeoutMs)
markPlaywrightCDPHealthy()
} catch (error) {
markPlaywrightCDPUnavailable(error)
throw error
}
}
export function isPlaywrightCDPExecutionAvailable(now = Date.now()): boolean {
if (connectedBrowser?.isConnected()) {
return true
}
if (cdpHealthState.status !== 'unavailable') {
return true
}
return Boolean(cdpHealthState.nextRetryAt && cdpHealthState.nextRetryAt <= now)
}
export function getHiddenPlaywrightSnapshot() {
return {
cdpPort,
cdpEndpointURL,
connected: Boolean(connectedBrowser?.isConnected()),
pageCount: records.size,
idleTTLms: hiddenPlaywrightIdleTTLms,
health: { ...cdpHealthState },
records: [...records.values()].map((record) => ({
accountId: record.accountId,
title: record.title,
targetURL: record.targetURL,
retainCount: record.retainCount,
activatedAt: record.activatedAt,
lastReleasedAt: record.lastReleasedAt,
windowDestroyed: record.window.isDestroyed(),
pageClosed: record.page.isClosed(),
})),
}
}
export function startHiddenPlaywrightReaper(): void {
if (reaperHandle) {
return
@@ -155,26 +255,6 @@ export function reapIdleHiddenPlaywrightPages(
return reclaimed
}
export function getHiddenPlaywrightSnapshot() {
return {
cdpPort,
cdpEndpointURL,
connected: Boolean(connectedBrowser?.isConnected()),
pageCount: records.size,
idleTTLms: hiddenPlaywrightIdleTTLms,
records: [...records.values()].map((record) => ({
accountId: record.accountId,
title: record.title,
targetURL: record.targetURL,
retainCount: record.retainCount,
activatedAt: record.activatedAt,
lastReleasedAt: record.lastReleasedAt,
windowDestroyed: record.window.isDestroyed(),
pageClosed: record.page.isClosed(),
})),
}
}
function createLease(record: HiddenPlaywrightRecord): HiddenPlaywrightLease {
return {
accountId: record.accountId,
@@ -271,6 +351,7 @@ async function connectBrowserOverCDP(): Promise<PlaywrightBrowser> {
connectPromise = (async () => {
await waitForCDPEndpoint()
markPlaywrightCDPHealthy()
const { chromium } = await import('playwright-core')
await pruneStaleCDPTargets()
@@ -292,6 +373,9 @@ async function connectBrowserOverCDP(): Promise<PlaywrightBrowser> {
try {
return await connectPromise
} catch (error) {
markPlaywrightCDPUnavailable(error)
throw error
} finally {
if (!connectedBrowser?.isConnected()) {
connectPromise = null
@@ -314,7 +398,7 @@ async function waitForCDPEndpoint(timeoutMs = defaultConnectTimeoutMs): Promise<
await wait(200)
}
throw new Error('playwright_cdp_endpoint_unavailable')
throw new Error(`playwright_cdp_endpoint_unavailable:${cdpEndpointURL}`)
}
async function openBrowserOverCDP(
@@ -330,6 +414,7 @@ async function openBrowserOverCDP(
})
connectedBrowser = browser
markPlaywrightCDPHealthy()
return browser
}
@@ -468,3 +553,104 @@ function wait(timeMs: number): Promise<void> {
setTimeout(resolve, timeMs)
})
}
function normalizeCDPPort(value: string | undefined, fallback: number): number {
const parsed = Number.parseInt(value ?? '', 10)
if (!Number.isFinite(parsed) || parsed <= 0 || parsed > 65_535) {
return fallback
}
return parsed
}
function endpointURLForPort(port: number): string {
return `http://127.0.0.1:${port}`
}
function setPlaywrightCDPPort(port: number): void {
cdpPort = port
cdpEndpointURL = endpointURLForPort(port)
cdpTargetsListURL = `${cdpEndpointURL}/json/list`
}
async function findAvailableCDPPort(): Promise<number> {
if (await canBindPort(preferredCDPPort)) {
return preferredCDPPort
}
const min = Math.min(dynamicCDPPortMin, dynamicCDPPortMax)
const max = Math.max(dynamicCDPPortMin, dynamicCDPPortMax)
const start = Math.max(min, preferredCDPPort)
for (let port = start; port <= max; port += 1) {
if (port === preferredCDPPort) {
continue
}
if (await canBindPort(port)) {
return port
}
}
for (let port = min; port < start; port += 1) {
if (port === preferredCDPPort) {
continue
}
if (await canBindPort(port)) {
return port
}
}
const reserved = await reserveEphemeralPort()
return reserved
}
function canBindPort(port: number): Promise<boolean> {
return new Promise((resolve) => {
const server = createServer()
server.once('error', () => {
resolve(false)
})
server.listen({ host: '127.0.0.1', port }, () => {
server.close(() => {
resolve(true)
})
})
})
}
function reserveEphemeralPort(): Promise<number> {
return new Promise((resolve, reject) => {
const server = createServer()
server.once('error', reject)
server.listen({ host: '127.0.0.1', port: 0 }, () => {
const address = server.address()
const port = typeof address === 'object' && address ? address.port : 0
server.close(() => {
if (port > 0) {
resolve(port)
} else {
reject(new Error('playwright_cdp_port_unavailable'))
}
})
})
})
}
function markPlaywrightCDPHealthy(): void {
cdpHealthState = {
status: 'healthy',
checkedAt: Date.now(),
unavailableSince: null,
nextRetryAt: null,
lastError: null,
}
}
function markPlaywrightCDPUnavailable(error: unknown): void {
const now = Date.now()
cdpHealthState = {
status: 'unavailable',
checkedAt: now,
unavailableSince: cdpHealthState.unavailableSince ?? now,
nextRetryAt: now + cdpCircuitOpenTTLms,
lastError: error instanceof Error ? error.message : String(error),
}
}
@@ -32,6 +32,7 @@ interface PublishSchedulerSelectionOptions {
now?: number
maxConcurrency: number
activePlatforms?: ReadonlySet<string>
blockedPlatforms?: ReadonlySet<string>
activeCount: number
}
@@ -97,6 +98,10 @@ export function selectNextPublishTask(
})
for (const candidate of candidates) {
if (options.blockedPlatforms?.has(candidate.platform)) {
continue
}
if (activePlatforms.has(candidate.platform)) {
continue
}
@@ -143,6 +148,19 @@ export function notePublishTaskLeaseMiss(taskId: string): void {
tasks.delete(taskId)
}
export function notePublishTaskDeferred(taskId: string): void {
const existing = tasks.get(taskId)
if (!existing) {
return
}
tasks.set(taskId, {
...existing,
state: 'queued',
updatedAt: Date.now(),
})
}
export function notePublishTaskCompleted(taskId: string, platform?: string): void {
const existing = tasks.get(taskId)
const resolvedPlatform = (platform ?? existing?.platform ?? '').trim()
@@ -68,10 +68,17 @@ import {
noteMonitorTaskActivated,
noteMonitorTaskCanceled,
noteMonitorTaskCompleted,
noteMonitorTaskDeferred,
noteMonitorTaskLeased,
selectNextMonitorTask,
} from './monitor-scheduler'
import { retainHiddenPlaywrightPage, startHiddenPlaywrightReaper } from './playwright-cdp'
import {
ensurePlaywrightCDPReady,
isPlaywrightCDPExecutionAvailable,
isPlaywrightCDPUnavailableError,
retainHiddenPlaywrightPage,
startHiddenPlaywrightReaper,
} from './playwright-cdp'
import { getProcessMetricsSnapshot } from './process-metrics'
import {
enqueuePublishTask as enqueuePublishSchedulerTask,
@@ -80,6 +87,7 @@ import {
notePublishTaskActivated,
notePublishTaskCanceled,
notePublishTaskCompleted,
notePublishTaskDeferred,
notePublishTaskLeaseMiss,
selectNextPublishTask,
} from './publish-scheduler'
@@ -102,6 +110,7 @@ import {
clearSessionHandle,
createSessionHandle,
hasLocalSessionPartition,
type SessionHandle,
} from './session-registry'
import {
cancelDesktopTask,
@@ -148,6 +157,8 @@ const heartbeatIntervalMs = 15_000
const pullIntervalMs = 60_000
const publishPullIntervalMs = 30_000
const monitorPullFallbackIntervalMs = 10_000
const playwrightCDPPreflightDeferMs = 15_000
const playwrightCDPRecoveryActivityCooldownMs = 60_000
const accountSyncIntervalMs = 30_000
const pumpWatchdogIntervalMs = 30_000
const leaseExtendIntervalMs = 60_000
@@ -163,6 +174,8 @@ const maximumRuntimeTotalConcurrency = 4
const BAIJIAHAO_RISK_CONTROL_PROMPT = '您触发平台风控,请手动发稿一篇并完成验证,方可继续使用'
const accountHealthReportDebounceMs = 1_000
const accountHealthReportMaxBatchSize = 100
const maxPlaywrightCDPSoftRecoveries = 3
const playwrightCDPFatalMarker = '--geo-cdp-fatal'
interface RuntimeTaskRecord {
id: string
@@ -312,6 +325,18 @@ const state: RuntimeState = {
activitySeq: 0,
}
let accountHealthReportBridgeUnsubscribe: (() => void) | null = null
let lastPlaywrightCDPRecoveryActivityAt = 0
let playwrightCDPRecoveryAttemptCount = process.argv.filter((arg) =>
arg.startsWith('--geo-cdp-recovery-at='),
).length
let playwrightCDPFatal = process.argv.includes(playwrightCDPFatalMarker)
class PlaywrightCDPInfrastructureError extends Error {
constructor(message: string) {
super(message)
this.name = 'PlaywrightCDPInfrastructureError'
}
}
function accountIdentityFromDesktopAccount(account: DesktopAccountInfo): PublishAccountIdentity {
return {
@@ -703,6 +728,17 @@ function startRuntime(): void {
if (!canRunLive(state.session) || state.running) {
return
}
if (playwrightCDPFatal) {
const message = '浏览器自动化通道不可用,客户端已停止采集。请重启客户端或检查安全软件/端口占用。'
state.lastError = message
setSchedulerPhase('paused')
setSchedulerError(message)
recordActivity('danger', '浏览器自动化组件不可用', message)
emitRuntimeInvalidated('playwright-cdp-fatal', {
message,
})
return
}
initScheduler()
initMonitorScheduler()
@@ -1850,17 +1886,40 @@ function asSubmitUncertainExecution(result: AdapterExecutionResult): AdapterExec
}
}
async function retainPlaywrightPageOrThrow(
task: RuntimeTaskRecord,
payload: Record<string, JsonValue>,
sessionHandle: SessionHandle,
) {
try {
await ensurePlaywrightCDPReady()
return await retainHiddenPlaywrightPage({
accountId: task.accountId || task.id,
session: sessionHandle.session,
targetURL: resolvePlaywrightTargetURL(task.platform, payload),
title: `${task.title} · Hidden Playwright`,
})
} catch (error) {
if (isPlaywrightCDPUnavailableError(error)) {
throw new PlaywrightCDPInfrastructureError(errorMessage(error))
}
throw error
}
}
function pumpExecutionLoop(): void {
if (!state.running) {
if (!state.running || playwrightCDPFatal) {
return
}
syncSchedulerSurface()
const blockedPlaywrightPlatforms = blockedPlaywrightExecutionPlatforms()
if (canStartAnotherPublishExecution()) {
const selected = selectNextPublishTask({
maxConcurrency: resolveAdaptiveTotalConcurrency(),
activePlatforms: activePublishPlatforms(),
blockedPlatforms: blockedPlaywrightPlatforms,
activeCount: activePublishCount(),
})
if (selected) {
@@ -1883,6 +1942,7 @@ function pumpExecutionLoop(): void {
maxConcurrency: resolveAdaptiveMonitorConcurrency(),
activePlatforms: activeMonitorPlatforms(),
activeQuestionKeys: activeMonitorQuestionKeys(),
blockedPlatforms: blockedPlaywrightPlatforms,
activeCount: activeMonitorCount(),
})
if (selected) {
@@ -1912,6 +1972,10 @@ async function leaseSpecificTask(
return
}
if (!(await ensurePlaywrightCDPAdmissionForRequest(expectedKind, request))) {
return
}
state.leaseInFlightKinds.add(expectedKind)
try {
@@ -1967,7 +2031,11 @@ async function leaseSpecificTask(
async function pullNextTask(kind: 'publish' | 'monitor'): Promise<void> {
if (kind === 'monitor') {
if (!canRunLive(state.session) || isLeaseInFlight('monitor')) {
if (!canRunLive(state.session) || isLeaseInFlight('monitor') || playwrightCDPFatal) {
return
}
if (!(await ensurePlaywrightCDPAdmissionForFallback('monitor'))) {
return
}
@@ -2002,7 +2070,11 @@ async function pullNextTask(kind: 'publish' | 'monitor'): Promise<void> {
return
}
if (!canRunLive(state.session) || isLeaseInFlight('publish')) {
if (!canRunLive(state.session) || isLeaseInFlight('publish') || playwrightCDPFatal) {
return
}
if (!(await ensurePlaywrightCDPAdmissionForFallback('publish'))) {
return
}
@@ -2046,7 +2118,7 @@ async function pullNextTask(kind: 'publish' | 'monitor'): Promise<void> {
async function pullMonitoringTasks(routing: RuntimeTaskRouting): Promise<void> {
const leaseLimit = resolveLegacyMonitoringLeaseLimit()
if (!canRunLive(state.session) || isLeaseInFlight('monitor') || leaseLimit <= 0) {
if (!canRunLive(state.session) || isLeaseInFlight('monitor') || leaseLimit <= 0 || playwrightCDPFatal) {
return
}
@@ -2270,6 +2342,15 @@ async function executeLeasedMonitoringTask(
`${taskRecord.title} ${execution.summary}`,
)
} catch (error) {
if (isPlaywrightCDPInfrastructureError(error)) {
await deferMonitoringLeaseAfterInfrastructureFailure(
taskRecord,
monitoringTaskID,
taskRecord.leaseToken,
)
return
}
if (isMonitorAbortReason(taskRecord, error, 'stale_business_date')) {
let dropSummary = staleMonitoringDropSummary(taskRecord)
try {
@@ -2531,6 +2612,11 @@ async function executeLeasedDesktopMonitorTask(
`${taskRecord.title} ${execution.summary}`,
)
} catch (error) {
if (isPlaywrightCDPInfrastructureError(error)) {
await deferDesktopMonitorTaskAfterInfrastructureFailure(taskRecord, leased, monitoringTaskID)
return
}
if (isMonitorAbortReason(taskRecord, error, 'stale_business_date')) {
let dropSummary = staleMonitoringDropSummary(taskRecord)
try {
@@ -3024,12 +3110,7 @@ async function executeTaskAdapter(
: retainHotView(task.accountId || task.id)
const playwrightLease =
adapter.executionMode === 'playwright'
? await retainHiddenPlaywrightPage({
accountId: task.accountId || task.id,
session: sessionHandle.session,
targetURL: resolvePlaywrightTargetURL(task.platform, payload),
title: `${task.title} · Hidden Playwright`,
})
? await retainPlaywrightPageOrThrow(task, payload, sessionHandle)
: null
try {
@@ -3098,12 +3179,7 @@ async function executeTaskAdapter(
: retainHotView(task.accountId || task.id)
const playwrightLease =
adapter.executionMode === 'playwright'
? await retainHiddenPlaywrightPage({
accountId: task.accountId || task.id,
session: sessionHandle.session,
targetURL: resolvePlaywrightTargetURL(task.platform, payload),
title: `${task.title} · Hidden Playwright`,
})
? await retainPlaywrightPageOrThrow(task, payload, sessionHandle)
: null
try {
@@ -3534,6 +3610,238 @@ function isLeaseInFlight(kind: 'publish' | 'monitor'): boolean {
return state.leaseInFlightKinds.has(kind)
}
function blockedPlaywrightExecutionPlatforms(): Set<string> {
if (isPlaywrightCDPExecutionAvailable()) {
return new Set()
}
return new Set([
...state.accounts
.map((account) => account.platform)
.filter((platform) => adapterRequiresPlaywright('monitor', platform)),
...getPublishSchedulerSnapshot()
.tasks.map((task) => task.platform)
.filter((platform) => adapterRequiresPlaywright('publish', platform)),
...getMonitorSchedulerSnapshot()
.tasks.map((task) => task.platform)
.filter((platform) => adapterRequiresPlaywright('monitor', platform)),
])
}
async function ensurePlaywrightCDPAdmissionForRequest(
kind: 'publish' | 'monitor',
request: PendingTaskRequest,
): Promise<boolean> {
if (!request.platform || !adapterRequiresPlaywright(kind, request.platform)) {
return true
}
const ready = await ensurePlaywrightCDPAdmission()
if (!ready) {
deferPendingTaskRequest(kind, request, playwrightCDPPreflightDeferMs)
}
return ready
}
async function ensurePlaywrightCDPAdmissionForFallback(kind: 'publish' | 'monitor'): Promise<boolean> {
if (kind === 'publish') {
const queuedPlaywrightPublish = getPublishSchedulerSnapshot().tasks.some(
(task) => task.state === 'queued' && adapterRequiresPlaywright('publish', task.platform),
)
if (!queuedPlaywrightPublish) {
return true
}
return ensurePlaywrightCDPAdmission()
}
const hasNonPlaywrightMonitorAccount = state.accounts.some(
(account) =>
isAIPlatformId(account.platform) &&
!adapterRequiresPlaywright('monitor', account.platform) &&
getProjectedAccountHealth({
accountId: account.id,
platform: account.platform,
health: account.health,
verifiedAt: account.verified_at,
}).health === 'live',
)
if (hasNonPlaywrightMonitorAccount) {
return true
}
return ensurePlaywrightCDPAdmission()
}
async function ensurePlaywrightCDPAdmission(): Promise<boolean> {
try {
await ensurePlaywrightCDPReady()
return true
} catch (error) {
handlePlaywrightCDPUnavailable(error)
return false
}
}
function isPlaywrightCDPInfrastructureError(error: unknown): error is PlaywrightCDPInfrastructureError {
return error instanceof PlaywrightCDPInfrastructureError || isPlaywrightCDPUnavailableError(error)
}
function deferPendingTaskRequest(
kind: 'publish' | 'monitor',
request: PendingTaskRequest,
delayMs: number,
): void {
if (kind === 'publish') {
notePublishTaskDeferred(request.taskId)
state.publishFallbackBackoffUntil = Date.now() + Math.max(delayMs, publishPullIntervalMs)
return
}
noteMonitorTaskDeferred(request.taskId, delayMs)
state.monitorFallbackBackoffUntil = Date.now() + Math.max(delayMs, monitorPullFallbackIntervalMs)
}
function adapterRequiresPlaywright(kind: 'publish' | 'monitor', platform: string): boolean {
const adapter = kind === 'publish' ? selectPublishAdapter(platform) : selectMonitorAdapter(platform)
return adapter?.executionMode === 'playwright'
}
function handlePlaywrightCDPUnavailable(error: unknown, taskId?: string): void {
const message = errorMessage(error) || 'playwright_cdp_endpoint_unavailable'
state.lastError = message
setSchedulerError(message)
state.monitorFallbackBackoffUntil = Date.now() + monitorPullFallbackIntervalMs
state.publishFallbackBackoffUntil = Date.now() + publishPullIntervalMs
if (playwrightCDPRecoveryAttemptCount >= maxPlaywrightCDPSoftRecoveries) {
markPlaywrightCDPFatal(message, taskId)
return
}
playwrightCDPRecoveryAttemptCount += 1
const now = Date.now()
if (now - lastPlaywrightCDPRecoveryActivityAt >= playwrightCDPRecoveryActivityCooldownMs) {
lastPlaywrightCDPRecoveryActivityAt = now
recordActivity(
'warn',
'浏览器自动化通道异常',
'已暂停领取 Playwright 平台任务,客户端将自动恢复 CDP 后继续采集。',
)
}
emitRuntimeInvalidated('playwright-cdp-recovery', {
taskId,
message,
})
syncSchedulerSurface()
}
function markPlaywrightCDPFatal(message: string, taskId?: string): void {
if (playwrightCDPFatal) {
return
}
playwrightCDPFatal = true
const fatalMessage =
message ||
'浏览器自动化通道不可用,客户端已停止采集。请重启客户端或检查安全软件/端口占用。'
state.lastError = fatalMessage
setSchedulerPhase('paused')
setSchedulerError(fatalMessage)
setTransportDispatchState('idle')
state.dispatchWsConnected = false
state.dispatchWsClient?.stop()
state.dispatchWsClient = null
state.leaseInFlightKinds.clear()
recordActivity('danger', '浏览器自动化组件不可用', fatalMessage)
void offlineDesktopClient().catch((offlineError) => {
console.warn('[desktop-runtime] failed to mark client offline after cdp fatal', {
message: errorMessage(offlineError),
})
})
emitRuntimeInvalidated('playwright-cdp-fatal', {
taskId,
message: fatalMessage,
})
syncSchedulerSurface()
}
async function deferMonitoringLeaseAfterInfrastructureFailure(
task: RuntimeTaskRecord,
monitoringTaskID: number,
leaseToken: string,
reason = 'playwright_cdp_unavailable',
): Promise<void> {
try {
await cancelMonitoringTask(monitoringTaskID, {
lease_token: leaseToken,
reason,
})
} catch (cancelError) {
recordActivity(
'warn',
'监控租约释放失败',
`${task.title} 基础设施异常后释放租约失败:${errorMessage(cancelError)}`,
)
}
const existing = state.tasks.get(task.id)
if (existing) {
existing.status = 'queued'
existing.summary = '浏览器自动化通道异常,已释放租约并等待客户端自动恢复后重试。'
existing.error = {
code: reason,
message: 'playwright cdp is unavailable; task deferred without marking collection failed',
}
existing.updatedAt = Date.now()
existing.attemptId = null
existing.leaseToken = null
state.tasks.set(task.id, existing)
}
noteLeaseReleased('completed', task.id)
noteMonitorTaskDeferred(task.id, playwrightCDPPreflightDeferMs)
handlePlaywrightCDPUnavailable(new Error(reason), task.id)
}
async function deferDesktopMonitorTaskAfterInfrastructureFailure(
task: RuntimeTaskRecord,
leased: LeaseDesktopTaskResponse,
monitoringTaskID: number,
reason = 'playwright_cdp_unavailable',
): Promise<void> {
try {
await cancelMonitoringTask(monitoringTaskID, {
lease_token: leased.lease_token as string,
reason,
})
} catch (cancelError) {
recordActivity(
'warn',
'监控租约释放失败',
`${task.title} 基础设施异常后释放 phase2 租约失败:${errorMessage(cancelError)}`,
)
}
const existing = state.tasks.get(task.id)
if (existing) {
existing.status = 'queued'
existing.summary = '浏览器自动化通道异常,任务已回队等待客户端自动恢复。'
existing.error = {
code: reason,
message: 'playwright cdp is unavailable; desktop monitor task requeued',
}
existing.updatedAt = Date.now()
existing.attemptId = null
existing.leaseToken = null
state.tasks.set(task.id, existing)
}
noteLeaseReleased('completed', task.id)
noteMonitorTaskDeferred(task.id, playwrightCDPPreflightDeferMs)
handlePlaywrightCDPUnavailable(new Error(reason), task.id)
}
function activePublishCount(): number {
return [...state.activeExecutions.values()].filter((item) => item.kind === 'publish').length
}
@@ -1,17 +1,23 @@
type RuntimeInvalidationReason = 'account-health' | 'publish-task-lease' | 'publish-task-progress'
type RuntimeInvalidationReason =
| 'account-health'
| 'publish-task-lease'
| 'publish-task-progress'
| 'playwright-cdp-recovery'
| 'playwright-cdp-fatal'
type RuntimeInvalidationListener = (event: {
reason: RuntimeInvalidationReason
at: number
accountId?: string
taskId?: string
message?: string
}) => void
const listeners = new Set<RuntimeInvalidationListener>()
export function emitRuntimeInvalidated(
reason: RuntimeInvalidationReason,
details: { accountId?: string; taskId?: string } = {},
details: { accountId?: string; taskId?: string; message?: string } = {},
): void {
const event = {
reason,
+14 -2
View File
@@ -249,19 +249,31 @@ const desktopBridge = {
ipcRenderer.invoke('desktop:set-window-mode', mode, request) as Promise<null>,
onRuntimeInvalidated: (
listener: (event: {
reason: 'account-health' | 'publish-task-lease' | 'publish-task-progress'
reason:
| 'account-health'
| 'publish-task-lease'
| 'publish-task-progress'
| 'playwright-cdp-recovery'
| 'playwright-cdp-fatal'
at: number
accountId?: string
taskId?: string
message?: string
}) => void,
) => {
const wrapped = (
_event: IpcRendererEvent,
payload: {
reason: 'account-health' | 'publish-task-lease' | 'publish-task-progress'
reason:
| 'account-health'
| 'publish-task-lease'
| 'publish-task-progress'
| 'playwright-cdp-recovery'
| 'playwright-cdp-fatal'
at: number
accountId?: string
taskId?: string
message?: string
},
) => {
listener(payload)
+7 -1
View File
@@ -142,10 +142,16 @@ declare global {
): Promise<null>
onRuntimeInvalidated(
listener: (event: {
reason: 'account-health' | 'publish-task-lease' | 'publish-task-progress'
reason:
| 'account-health'
| 'publish-task-lease'
| 'publish-task-progress'
| 'playwright-cdp-recovery'
| 'playwright-cdp-fatal'
at: number
accountId?: string
taskId?: string
message?: string
}) => void,
): () => void
onNetworkObserved(listener: (event: DesktopObservedRequest) => void): () => void