fix: harden desktop monitoring recovery
Desktop Client Build / Resolve Build Metadata (push) Successful in 37s
Backend CI / Backend (push) Failing after 9m26s
Desktop Client Build / Build Desktop Client (push) Successful in 22m29s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 35s
Desktop Client Build / Resolve Build Metadata (push) Successful in 37s
Backend CI / Backend (push) Failing after 9m26s
Desktop Client Build / Build Desktop Client (push) Successful in 22m29s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 35s
This commit is contained in:
@@ -12,6 +12,7 @@ const {
|
||||
isNonCitationAssetUrl,
|
||||
parseDoubaoStreamBody,
|
||||
resolveDoubaoSubmissionAnswer,
|
||||
shouldRetryDoubaoPageAttempt,
|
||||
} = __doubaoTestUtils
|
||||
|
||||
describe('doubao adapter helpers', () => {
|
||||
@@ -28,6 +29,69 @@ describe('doubao adapter helpers', () => {
|
||||
expect(isDoubaoRecoverablePageError('doubao_login_expired')).toBe(false)
|
||||
})
|
||||
|
||||
it('retries once when Doubao is stuck on the history surface without an answer', () => {
|
||||
const streamSummary = parseDoubaoStreamBody(
|
||||
['event: SSE_REPLY_END', 'data: {"message_id":"msg_1"}', '', ''].join('\n'),
|
||||
)
|
||||
|
||||
expect(
|
||||
shouldRetryDoubaoPageAttempt(
|
||||
{
|
||||
ok: false,
|
||||
error: 'doubao_query_timeout',
|
||||
detail: '历史对话 主对话 折叠门联动配件明细',
|
||||
url: 'https://www.doubao.com/chat/local_4995117315817271',
|
||||
title: '豆包',
|
||||
modelLabel: '豆包',
|
||||
modeLabel: '思考',
|
||||
thinkingModeApplied: true,
|
||||
deepThinkEnabled: true,
|
||||
domAnswer: '折叠门联动配件明细',
|
||||
domReasoning: null,
|
||||
domLinks: [],
|
||||
submitted: true,
|
||||
questionAnchorFound: true,
|
||||
historySurfaceDetected: true,
|
||||
},
|
||||
streamSummary,
|
||||
),
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('does not retry a recoverable Doubao page error after SSE already yielded an answer', () => {
|
||||
const streamSummary = parseDoubaoStreamBody(
|
||||
[
|
||||
'event: STREAM_CHUNK',
|
||||
'data: {"text_block":{"text":"这是一段完整的豆包回答。"}}',
|
||||
'',
|
||||
'',
|
||||
].join('\n'),
|
||||
)
|
||||
|
||||
expect(
|
||||
shouldRetryDoubaoPageAttempt(
|
||||
{
|
||||
ok: false,
|
||||
error: 'doubao_query_timeout',
|
||||
detail: 'timeout',
|
||||
url: 'https://www.doubao.com/chat/local_4995117315817271',
|
||||
title: '豆包',
|
||||
modelLabel: '豆包',
|
||||
modeLabel: '思考',
|
||||
thinkingModeApplied: true,
|
||||
deepThinkEnabled: true,
|
||||
domAnswer: null,
|
||||
domReasoning: null,
|
||||
domLinks: [],
|
||||
submitted: true,
|
||||
questionAnchorFound: true,
|
||||
historySurfaceDetected: true,
|
||||
},
|
||||
streamSummary,
|
||||
),
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('maps the new expert/deep-thinking mode label to expert mode', () => {
|
||||
expect(doubaoModeKindFromLabelText('专家')).toBe('expert')
|
||||
expect(doubaoModeKindFromLabelText('专家 深度思考 研究级智能模型')).toBe('expert')
|
||||
|
||||
@@ -9,7 +9,7 @@ const DOUBAO_PAGE_READY_TIMEOUT_MS = 20_000
|
||||
const DOUBAO_QUERY_TIMEOUT_MS = 120_000
|
||||
const DOUBAO_CAPTURE_LIMIT = 64
|
||||
const DOUBAO_CAPTURE_BODY_LIMIT = 8_000_000
|
||||
const DOUBAO_CAPTURE_FLUSH_TIMEOUT_MS = DOUBAO_QUERY_TIMEOUT_MS
|
||||
const DOUBAO_CAPTURE_FLUSH_TIMEOUT_MS = 10_000
|
||||
const DOUBAO_STREAM_URL_FRAGMENT = '/chat/completion'
|
||||
const DOUBAO_NON_CITATION_ASSET_DOMAINS = new Set(['byteimg.com', 'bytednsdoc.com'])
|
||||
|
||||
@@ -116,10 +116,22 @@ type DoubaoPageQueryFailureResult = {
|
||||
domAnswer: string | null
|
||||
domReasoning: string | null
|
||||
domLinks: DoubaoDomLink[]
|
||||
submitted: boolean | null
|
||||
questionAnchorFound: boolean | null
|
||||
historySurfaceDetected: boolean | null
|
||||
}
|
||||
|
||||
type DoubaoPageQueryResult = DoubaoPageQuerySuccessResult | DoubaoPageQueryFailureResult
|
||||
|
||||
type DoubaoQueryAttemptResult = {
|
||||
attempt: number
|
||||
pageResult: DoubaoPageQueryResult
|
||||
captures: DoubaoCaptureRecord[]
|
||||
streamSummary: DoubaoStreamSummary
|
||||
captureDebug: JsonValue[]
|
||||
pendingCaptureCount: number
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||
}
|
||||
@@ -1039,7 +1051,12 @@ function isDoubaoExecutionContextNavigationError(error: unknown): boolean {
|
||||
}
|
||||
|
||||
function isDoubaoRecoverablePageError(error: string): boolean {
|
||||
return error === 'doubao_query_timeout' || error === 'doubao_page_navigation_interrupted'
|
||||
return (
|
||||
error === 'doubao_query_timeout' ||
|
||||
error === 'doubao_page_navigation_interrupted' ||
|
||||
error === 'doubao_submission_not_observed' ||
|
||||
error === 'doubao_history_surface_timeout'
|
||||
)
|
||||
}
|
||||
|
||||
function buildDoubaoPageNavigationResult(
|
||||
@@ -1059,9 +1076,30 @@ function buildDoubaoPageNavigationResult(
|
||||
domAnswer: null,
|
||||
domReasoning: null,
|
||||
domLinks: [],
|
||||
submitted: null,
|
||||
questionAnchorFound: null,
|
||||
historySurfaceDetected: null,
|
||||
}
|
||||
}
|
||||
|
||||
function shouldRetryDoubaoPageAttempt(
|
||||
pageResult: DoubaoPageQueryResult,
|
||||
streamSummary: DoubaoStreamSummary,
|
||||
): boolean {
|
||||
if (pageResult.ok || !isDoubaoRecoverablePageError(pageResult.error)) {
|
||||
return false
|
||||
}
|
||||
if (isDoubaoSSESubmissionCandidate(streamSummary.answer)) {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
pageResult.error === 'doubao_submission_not_observed' ||
|
||||
pageResult.error === 'doubao_history_surface_timeout' ||
|
||||
pageResult.historySurfaceDetected === true ||
|
||||
pageResult.questionAnchorFound === false
|
||||
)
|
||||
}
|
||||
|
||||
function buildDoubaoUnknownResult(
|
||||
code: string,
|
||||
summary: string,
|
||||
@@ -1597,6 +1635,8 @@ const doubaoQueryInPage = async (
|
||||
const pollIntervalMs = 1_500
|
||||
const incompleteAnswerGraceMs = 30_000
|
||||
const minimumAnswerSettleMs = 10_000
|
||||
const submissionObserveTimeoutMs = 12_000
|
||||
const staleHistoryTimeoutMs = 18_000
|
||||
|
||||
const wait = (timeMs: number) =>
|
||||
new Promise<void>((resolve) => {
|
||||
@@ -2684,6 +2724,39 @@ const doubaoQueryInPage = async (
|
||||
line,
|
||||
))
|
||||
|
||||
const looksLikeHistoryTitleList = (text: string | null): boolean => {
|
||||
if (!text) {
|
||||
return false
|
||||
}
|
||||
const normalizedText = text.replace(/\s+/g, ' ').trim()
|
||||
if (
|
||||
/搜索…\s*Ctrl\s*K\s*豆包|新对话\s*Ctrl\s*Shift\s*K|历史对话\s*主对话/.test(
|
||||
normalizedText,
|
||||
)
|
||||
) {
|
||||
return true
|
||||
}
|
||||
const lines = text
|
||||
.split(/\n+/)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
if (lines.length < 6) {
|
||||
return false
|
||||
}
|
||||
const shortTitleLines = lines.filter(
|
||||
(line) =>
|
||||
line.length <= 36 &&
|
||||
!/[。!?!?]/.test(line) &&
|
||||
/门|锁|轨|五金|导轨|合页|配件|工厂|厂家|供货|定制|折叠|幽灵|口袋/.test(line),
|
||||
).length
|
||||
const navHits = lines.filter((line) =>
|
||||
/^(搜索|豆包|新对话|新办公任务|AI 创作|AI创作|云盘|收起|发现智能体|历史对话|主对话)$/.test(
|
||||
line,
|
||||
),
|
||||
).length
|
||||
return shortTitleLines >= 6 && (navHits >= 2 || shortTitleLines / lines.length >= 0.65)
|
||||
}
|
||||
|
||||
// Tokens that only appear on Doubao's empty-chat surface (prompt templates, nav, feature chips).
|
||||
const EMPTY_CHAT_TOKENS = [
|
||||
'快速',
|
||||
@@ -2721,6 +2794,9 @@ const doubaoQueryInPage = async (
|
||||
return hits >= 3 || (hits >= 2 && lines.length <= 6)
|
||||
}
|
||||
|
||||
const looksLikeNonAnswerSurface = (text: string | null): boolean =>
|
||||
looksLikeEmptyChatSurface(text) || looksLikeHistoryTitleList(text)
|
||||
|
||||
const stripKnownNoise = (text: string | null): string | null => {
|
||||
if (!text) {
|
||||
return null
|
||||
@@ -2870,7 +2946,7 @@ const doubaoQueryInPage = async (
|
||||
if (!text && links.length === 0) {
|
||||
continue
|
||||
}
|
||||
if (rawText && looksLikeEmptyChatSurface(rawText)) {
|
||||
if (rawText && looksLikeNonAnswerSurface(rawText)) {
|
||||
continue
|
||||
}
|
||||
if (text && text.length < 12 && links.length === 0) {
|
||||
@@ -3033,6 +3109,11 @@ const doubaoQueryInPage = async (
|
||||
applied: null,
|
||||
},
|
||||
snapshotOverride: ConversationSnapshot | null = null,
|
||||
diagnostics: {
|
||||
submitted?: boolean | null
|
||||
questionAnchorFound?: boolean | null
|
||||
historySurfaceDetected?: boolean | null
|
||||
} = {},
|
||||
): DoubaoPageQueryFailureResult => {
|
||||
const snapshot = snapshotOverride ?? snapshotConversation(composerTop)
|
||||
const composer = findComposer()
|
||||
@@ -3057,6 +3138,9 @@ const doubaoQueryInPage = async (
|
||||
domAnswer: snapshot.answerText,
|
||||
domReasoning: snapshot.reasoningText,
|
||||
domLinks: snapshot.links,
|
||||
submitted: diagnostics.submitted ?? null,
|
||||
questionAnchorFound: diagnostics.questionAnchorFound ?? null,
|
||||
historySurfaceDetected: diagnostics.historySurfaceDetected ?? null,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3113,6 +3197,8 @@ const doubaoQueryInPage = async (
|
||||
}
|
||||
|
||||
const initialSnapshot = snapshotConversation(composerTop)
|
||||
let questionAnchorFoundAfterSubmit = Boolean(findQuestionAnchor())
|
||||
let historySurfaceDetected = looksLikeHistoryTitleList(bodyText())
|
||||
let bestSnapshot = initialSnapshot
|
||||
let bestContentSnapshot: ConversationSnapshot | null = hasObservedContent(initialSnapshot)
|
||||
? initialSnapshot
|
||||
@@ -3121,10 +3207,13 @@ const doubaoQueryInPage = async (
|
||||
let stableCount = submitQuestion ? 0 : hasObservedContent(initialSnapshot) ? 1 : 0
|
||||
let sawChange = !submitQuestion && hasObservedContent(initialSnapshot)
|
||||
let firstChangedContentAt: number | null = sawChange ? Date.now() : null
|
||||
const submittedAt = Date.now()
|
||||
const deadline = Date.now() + timeoutMs
|
||||
|
||||
while (Date.now() <= deadline) {
|
||||
const snapshot = snapshotConversation(composerTop)
|
||||
questionAnchorFoundAfterSubmit = questionAnchorFoundAfterSubmit || Boolean(findQuestionAnchor())
|
||||
historySurfaceDetected = historySurfaceDetected || looksLikeHistoryTitleList(bodyText())
|
||||
bestSnapshot = pickBetterSnapshot(bestSnapshot, snapshot)
|
||||
if (hasObservedContent(snapshot)) {
|
||||
bestContentSnapshot = pickBetterSnapshot(bestContentSnapshot, snapshot)
|
||||
@@ -3152,6 +3241,53 @@ const doubaoQueryInPage = async (
|
||||
composerTop,
|
||||
answerMode,
|
||||
bestContentSnapshot ?? bestSnapshot,
|
||||
{
|
||||
submitted: submitQuestion,
|
||||
questionAnchorFound: questionAnchorFoundAfterSubmit,
|
||||
historySurfaceDetected,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
const elapsedSinceSubmit = Date.now() - submittedAt
|
||||
if (
|
||||
submitQuestion &&
|
||||
!questionAnchorFoundAfterSubmit &&
|
||||
!busy &&
|
||||
elapsedSinceSubmit >= submissionObserveTimeoutMs
|
||||
) {
|
||||
return fail(
|
||||
'doubao_submission_not_observed',
|
||||
bodyText(),
|
||||
composerTop,
|
||||
answerMode,
|
||||
bestContentSnapshot ?? bestSnapshot,
|
||||
{
|
||||
submitted: true,
|
||||
questionAnchorFound: false,
|
||||
historySurfaceDetected,
|
||||
},
|
||||
)
|
||||
}
|
||||
if (
|
||||
submitQuestion &&
|
||||
questionAnchorFoundAfterSubmit &&
|
||||
historySurfaceDetected &&
|
||||
!hasContent &&
|
||||
!busy &&
|
||||
elapsedSinceSubmit >= staleHistoryTimeoutMs
|
||||
) {
|
||||
return fail(
|
||||
'doubao_history_surface_timeout',
|
||||
bodyText(),
|
||||
composerTop,
|
||||
answerMode,
|
||||
bestContentSnapshot ?? bestSnapshot,
|
||||
{
|
||||
submitted: true,
|
||||
questionAnchorFound: true,
|
||||
historySurfaceDetected: true,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -3196,6 +3332,11 @@ const doubaoQueryInPage = async (
|
||||
composerTop,
|
||||
answerMode,
|
||||
bestContentSnapshot ?? bestSnapshot,
|
||||
{
|
||||
submitted: submitQuestion,
|
||||
questionAnchorFound: questionAnchorFoundAfterSubmit,
|
||||
historySurfaceDetected,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -3214,40 +3355,66 @@ export const doubaoAdapter: MonitorAdapter = {
|
||||
const questionText = extractQuestionText(payload)
|
||||
const page = context.playwright.page
|
||||
|
||||
const runAttempt = async (attempt: number): Promise<DoubaoQueryAttemptResult> => {
|
||||
context.reportProgress(attempt === 1 ? 'doubao.query' : 'doubao.query_retry')
|
||||
const capture = attachDoubaoResponseCapture(page)
|
||||
let pageResult: DoubaoPageQueryResult
|
||||
try {
|
||||
try {
|
||||
pageResult = await page.evaluate(doubaoQueryInPage, {
|
||||
questionText,
|
||||
timeoutMs: DOUBAO_QUERY_TIMEOUT_MS,
|
||||
readyTimeoutMs: DOUBAO_PAGE_READY_TIMEOUT_MS,
|
||||
enableDeepThink: true,
|
||||
})
|
||||
} catch (error) {
|
||||
if (!isDoubaoExecutionContextNavigationError(error)) {
|
||||
throw error
|
||||
}
|
||||
await page
|
||||
.waitForLoadState('domcontentloaded', { timeout: DOUBAO_PAGE_READY_TIMEOUT_MS })
|
||||
.catch(() => undefined)
|
||||
await page.waitForLoadState('networkidle', { timeout: 3_000 }).catch(() => undefined)
|
||||
pageResult = buildDoubaoPageNavigationResult(page, formatUnknownError(error))
|
||||
}
|
||||
await sleep(600, context.signal).catch(() => undefined)
|
||||
await capture.flush(DOUBAO_CAPTURE_FLUSH_TIMEOUT_MS)
|
||||
} finally {
|
||||
capture.dispose()
|
||||
}
|
||||
const streamSummary = parseDoubaoCaptures(capture.captures)
|
||||
return {
|
||||
attempt,
|
||||
pageResult,
|
||||
captures: [...capture.captures],
|
||||
streamSummary,
|
||||
captureDebug: summarizeCapturesForDebug(capture.captures),
|
||||
pendingCaptureCount: capture.pendingCount(),
|
||||
}
|
||||
}
|
||||
|
||||
context.reportProgress('doubao.page_ready')
|
||||
await ensurePageOnDoubaoChat(page, context.signal)
|
||||
|
||||
const capture = attachDoubaoResponseCapture(page)
|
||||
let pageResult: DoubaoPageQueryResult
|
||||
|
||||
try {
|
||||
context.reportProgress('doubao.query')
|
||||
try {
|
||||
pageResult = await page.evaluate(doubaoQueryInPage, {
|
||||
questionText,
|
||||
timeoutMs: DOUBAO_QUERY_TIMEOUT_MS,
|
||||
readyTimeoutMs: DOUBAO_PAGE_READY_TIMEOUT_MS,
|
||||
enableDeepThink: true,
|
||||
})
|
||||
} catch (error) {
|
||||
if (!isDoubaoExecutionContextNavigationError(error)) {
|
||||
throw error
|
||||
}
|
||||
await page
|
||||
.waitForLoadState('domcontentloaded', { timeout: DOUBAO_PAGE_READY_TIMEOUT_MS })
|
||||
.catch(() => undefined)
|
||||
await page.waitForLoadState('networkidle', { timeout: 3_000 }).catch(() => undefined)
|
||||
pageResult = buildDoubaoPageNavigationResult(page, formatUnknownError(error))
|
||||
}
|
||||
await sleep(600, context.signal).catch(() => undefined)
|
||||
await capture.flush(DOUBAO_CAPTURE_FLUSH_TIMEOUT_MS)
|
||||
} finally {
|
||||
capture.dispose()
|
||||
let attemptResult = await runAttempt(1)
|
||||
if (
|
||||
attemptResult.pageResult.ok === false &&
|
||||
shouldRetryDoubaoPageAttempt(attemptResult.pageResult, attemptResult.streamSummary)
|
||||
) {
|
||||
context.reportProgress('doubao.recover_new_chat')
|
||||
await resetDoubaoPageAfterRecoverableFailure(
|
||||
page,
|
||||
attemptResult.pageResult.error,
|
||||
context.signal,
|
||||
)
|
||||
await ensurePageOnDoubaoChat(page, context.signal)
|
||||
attemptResult = await runAttempt(2)
|
||||
}
|
||||
|
||||
const streamSummary = parseDoubaoCaptures(capture.captures)
|
||||
const captureDebug = summarizeCapturesForDebug(capture.captures)
|
||||
const pendingCaptureCount = capture.pendingCount()
|
||||
const pageResult = attemptResult.pageResult
|
||||
const streamSummary = attemptResult.streamSummary
|
||||
const captureDebug = attemptResult.captureDebug
|
||||
const pendingCaptureCount = attemptResult.pendingCaptureCount
|
||||
const domSearchResults = dedupeSourceItems(
|
||||
pageResult.domLinks
|
||||
.map((item) =>
|
||||
@@ -3316,6 +3483,11 @@ export const doubaoAdapter: MonitorAdapter = {
|
||||
reply_end_event_count: streamSummary.reply_end_event_count,
|
||||
answer_finish_event_count: streamSummary.answer_finish_event_count,
|
||||
stream_event_count: streamSummary.event_count,
|
||||
submitted: failed.submitted,
|
||||
question_anchor_found: failed.questionAnchorFound,
|
||||
history_surface_detected: failed.historySurfaceDetected,
|
||||
query_attempt: attemptResult.attempt,
|
||||
captures: captureDebug,
|
||||
}),
|
||||
}
|
||||
}
|
||||
@@ -3426,11 +3598,12 @@ export const doubaoAdapter: MonitorAdapter = {
|
||||
platform: 'doubao',
|
||||
page_url: pageResult.url,
|
||||
page_title: pageResult.title,
|
||||
query_attempt: attemptResult.attempt,
|
||||
model_label: pageResult.modelLabel,
|
||||
mode_label: pageResult.modeLabel,
|
||||
thinking_mode_applied: pageResult.thinkingModeApplied,
|
||||
deep_think_enabled: pageResult.deepThinkEnabled,
|
||||
capture_count: capture.captures.length,
|
||||
capture_count: attemptResult.captures.length,
|
||||
capture_pending_after_flush: pendingCaptureCount,
|
||||
event_count: streamSummary.event_count,
|
||||
full_message_event_count: streamSummary.full_message_event_count,
|
||||
@@ -3464,6 +3637,8 @@ export const __doubaoTestUtils = {
|
||||
isNonCitationAssetDomain,
|
||||
isNonCitationAssetUrl,
|
||||
isDoubaoRecoverablePageError,
|
||||
isDoubaoDOMSubmissionCandidate,
|
||||
shouldRetryDoubaoPageAttempt,
|
||||
parseDoubaoStreamBody,
|
||||
resolveDoubaoSubmissionAnswer,
|
||||
}
|
||||
|
||||
@@ -53,7 +53,7 @@ import {
|
||||
installObservedGlobalFetch,
|
||||
registerObservedRequestRendererTarget,
|
||||
} from './network-observer'
|
||||
import { getPlaywrightCDPPort, startHiddenPlaywrightReaper } from './playwright-cdp'
|
||||
import { getPlaywrightCDPPort, preparePlaywrightCDPPort, startHiddenPlaywrightReaper } from './playwright-cdp'
|
||||
import { initProcessMetricsSampler } from './process-metrics'
|
||||
import { registerRendererDevtoolsProxyTarget } from './renderer-devtools-proxy'
|
||||
import {
|
||||
@@ -112,8 +112,8 @@ app.commandLine.appendSwitch(
|
||||
)
|
||||
app.commandLine.appendSwitch('disable-quic')
|
||||
app.commandLine.appendSwitch('disable-background-networking')
|
||||
// CDP is required by the hidden Playwright execution pool in packaged builds.
|
||||
app.commandLine.appendSwitch('remote-debugging-port', String(getPlaywrightCDPPort()))
|
||||
// CDP is required by the hidden Playwright execution pool. Electron only reads
|
||||
// this switch during early startup, so select the port before app.whenReady().
|
||||
app.userAgentFallback = STANDARD_USER_AGENT
|
||||
|
||||
// Windows 10+ routes tray.displayBalloon through the Toast Notification API,
|
||||
@@ -151,6 +151,7 @@ console.info('[desktop-main] boot', {
|
||||
chrome: process.versions.chrome,
|
||||
node: process.versions.node,
|
||||
ua: STANDARD_USER_AGENT,
|
||||
cdpPort: getPlaywrightCDPPort(),
|
||||
})
|
||||
|
||||
process.on('unhandledRejection', (reason) => {
|
||||
@@ -166,6 +167,7 @@ let loginWindow: ElectronBrowserWindow | null = null
|
||||
let settingsWindow: ElectronBrowserWindow | null = null
|
||||
let mainRendererContents: ElectronWebContents | null = null
|
||||
let quitReleaseInFlight = false
|
||||
let cdpRecoveryRelaunchInFlight = false
|
||||
let quitAfterSaasLogoutPromise: Promise<void> | null = null
|
||||
let mainWindowCreatePromise: Promise<ElectronBrowserWindow> | null = null
|
||||
let loginWindowCreatePromise: Promise<ElectronBrowserWindow> | null = null
|
||||
@@ -176,6 +178,10 @@ let canHandleDesktopDeepLinks = false
|
||||
const pendingDesktopDeepLinks: string[] = []
|
||||
const forceCloseWindows = new WeakSet<ElectronBrowserWindow>()
|
||||
const appWindowModes = new WeakMap<ElectronBrowserWindow, RendererWindowMode>()
|
||||
const maxCDPRecoveryRelaunchesPerWindow = 3
|
||||
const cdpRecoveryRelaunchWindowMs = 10 * 60_000
|
||||
const cdpRecoveryRelaunchDelayMs = 1_500
|
||||
const cdpFatalArg = '--geo-cdp-fatal'
|
||||
|
||||
type WindowMode = 'login' | 'main'
|
||||
type RendererWindowMode = WindowMode | 'settings'
|
||||
@@ -547,6 +553,50 @@ function quitAfterClearingSaasSession(): void {
|
||||
})
|
||||
}
|
||||
|
||||
function requestPlaywrightCDPRecoveryRelaunch(message: string): void {
|
||||
if (cdpRecoveryRelaunchInFlight) {
|
||||
return
|
||||
}
|
||||
|
||||
const now = Date.now()
|
||||
const recentRelaunches = process.argv
|
||||
.filter((arg) => arg.startsWith('--geo-cdp-recovery-at='))
|
||||
.map((arg) => Number.parseInt(arg.split('=')[1] ?? '', 10))
|
||||
.filter((timestamp) => Number.isFinite(timestamp) && now - timestamp < cdpRecoveryRelaunchWindowMs)
|
||||
|
||||
if (recentRelaunches.length >= maxCDPRecoveryRelaunchesPerWindow) {
|
||||
console.error('[desktop-main] playwright cdp recovery relaunch suppressed', {
|
||||
message,
|
||||
recentRelaunches: recentRelaunches.length,
|
||||
})
|
||||
showTrayBalloon(
|
||||
'浏览器自动化通道异常',
|
||||
'客户端已停止采集。请重启客户端或检查安全软件/端口占用。',
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
cdpRecoveryRelaunchInFlight = true
|
||||
console.warn('[desktop-main] scheduling playwright cdp recovery relaunch', {
|
||||
message,
|
||||
cdpPort: getPlaywrightCDPPort(),
|
||||
})
|
||||
showTrayBalloon('正在恢复浏览器自动化通道', '客户端将自动重启后台进程,登录状态会保留。')
|
||||
|
||||
setTimeout(() => {
|
||||
quitReleaseInFlight = true
|
||||
app.relaunch({
|
||||
args: [
|
||||
...process.argv.slice(1).filter((arg) => !arg.startsWith('--geo-cdp-recovery-at=')),
|
||||
...recentRelaunches.map((timestamp) => `--geo-cdp-recovery-at=${timestamp}`),
|
||||
`--geo-cdp-recovery-at=${now}`,
|
||||
...(recentRelaunches.length + 1 >= maxCDPRecoveryRelaunchesPerWindow ? [cdpFatalArg] : []),
|
||||
],
|
||||
})
|
||||
app.exit(0)
|
||||
}, cdpRecoveryRelaunchDelayMs)
|
||||
}
|
||||
|
||||
function windowFromIpcEvent(event: IpcMainInvokeEvent): ElectronBrowserWindow | null {
|
||||
const window = BrowserWindow.fromWebContents(event.sender)
|
||||
if (!window || window.isDestroyed()) {
|
||||
@@ -1197,9 +1247,15 @@ if (!hasSingleInstanceLock) {
|
||||
console.info('[desktop-main] another instance is already running; exiting')
|
||||
app.exit(0)
|
||||
} else {
|
||||
app
|
||||
.whenReady()
|
||||
preparePlaywrightCDPPort()
|
||||
.then((selectedPlaywrightCDPPort) => {
|
||||
app.commandLine.appendSwitch('remote-debugging-port', String(selectedPlaywrightCDPPort))
|
||||
return app.whenReady()
|
||||
})
|
||||
.then(async () => {
|
||||
console.info('[desktop-main] playwright cdp port selected', {
|
||||
cdpPort: getPlaywrightCDPPort(),
|
||||
})
|
||||
suppressNativeWindowMenu()
|
||||
registerDesktopProtocolClient()
|
||||
currentWindowMode = readPersistedWindowMode()
|
||||
@@ -1215,6 +1271,15 @@ if (!hasSingleInstanceLock) {
|
||||
startHiddenPlaywrightReaper()
|
||||
onRuntimeInvalidated((event) => {
|
||||
syncDesktopHealthIndicator(currentMainWindow())
|
||||
if (event.reason === 'playwright-cdp-fatal') {
|
||||
showTrayBalloon(
|
||||
'浏览器自动化组件不可用',
|
||||
event.message ?? '客户端已停止采集,请重启客户端或检查安全软件/端口占用。',
|
||||
)
|
||||
}
|
||||
if (event.reason === 'playwright-cdp-recovery') {
|
||||
requestPlaywrightCDPRecoveryRelaunch(event.message ?? 'playwright_cdp_endpoint_unavailable')
|
||||
}
|
||||
if (!mainRendererContents || mainRendererContents.isDestroyed()) {
|
||||
return
|
||||
}
|
||||
@@ -1234,7 +1299,7 @@ if (!hasSingleInstanceLock) {
|
||||
flushPendingDesktopDeepLinks()
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error('[desktop-main] app.whenReady failed', error)
|
||||
console.error('[desktop-main] startup failed', error)
|
||||
})
|
||||
|
||||
app.on('activate', () => {
|
||||
|
||||
@@ -64,6 +64,7 @@ interface MonitorSchedulerSelectionOptions {
|
||||
maxConcurrency: number
|
||||
activePlatforms: ReadonlySet<string>
|
||||
activeQuestionKeys: ReadonlySet<string>
|
||||
blockedPlatforms?: ReadonlySet<string>
|
||||
activeCount: number
|
||||
}
|
||||
|
||||
@@ -434,6 +435,22 @@ export function noteMonitorTaskLeaseMiss(taskId: string): void {
|
||||
})
|
||||
}
|
||||
|
||||
export function noteMonitorTaskDeferred(taskId: string, delayMs: number): void {
|
||||
const now = Date.now()
|
||||
const availableAt = now + Math.max(0, delayMs)
|
||||
|
||||
mutatePersistedState((draft) => {
|
||||
const existing = draft.tasks[taskId]
|
||||
if (!existing) {
|
||||
return
|
||||
}
|
||||
|
||||
existing.state = 'queued'
|
||||
existing.availableAt = Math.max(existing.availableAt, availableAt)
|
||||
existing.lastSeenAt = now
|
||||
})
|
||||
}
|
||||
|
||||
export function noteMonitorTaskCompleted(taskId: string): void {
|
||||
const now = Date.now()
|
||||
|
||||
@@ -488,6 +505,10 @@ export function selectNextMonitorTask(
|
||||
})
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (options.blockedPlatforms?.has(candidate.platform)) {
|
||||
continue
|
||||
}
|
||||
|
||||
if (options.activePlatforms.has(candidate.platform)) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { createServer } from 'node:net'
|
||||
|
||||
import type { BrowserWindow as ElectronBrowserWindow, Session } from 'electron/main'
|
||||
import { BrowserWindow, nativeTheme } from 'electron/main'
|
||||
import type { Browser as PlaywrightBrowser, Page as PlaywrightPage } from 'playwright-core'
|
||||
@@ -13,9 +15,23 @@ const hiddenPlaywrightReaperIntervalMs = 60_000
|
||||
const defaultConnectTimeoutMs = 10_000
|
||||
const defaultPageTimeoutMs = 45_000
|
||||
const hiddenBootstrapMetaName = 'geo-hidden-playwright-token'
|
||||
const cdpPort = Number.parseInt(process.env.GEO_ELECTRON_CDP_PORT ?? '9339', 10)
|
||||
const cdpEndpointURL = `http://127.0.0.1:${cdpPort}`
|
||||
const cdpTargetsListURL = `${cdpEndpointURL}/json/list`
|
||||
const preferredCDPPort = normalizeCDPPort(process.env.GEO_ELECTRON_CDP_PORT, 9339)
|
||||
const dynamicCDPPortMin = normalizeCDPPort(process.env.GEO_ELECTRON_CDP_PORT_MIN, 9339)
|
||||
const dynamicCDPPortMax = normalizeCDPPort(process.env.GEO_ELECTRON_CDP_PORT_MAX, 20000)
|
||||
const cdpCircuitOpenTTLms = 60_000
|
||||
const cdpCircuitProbeTimeoutMs = 2_000
|
||||
|
||||
let cdpPort = preferredCDPPort
|
||||
let cdpEndpointURL = endpointURLForPort(cdpPort)
|
||||
let cdpTargetsListURL = `${cdpEndpointURL}/json/list`
|
||||
|
||||
interface PlaywrightCDPHealthState {
|
||||
status: 'unknown' | 'healthy' | 'unavailable'
|
||||
checkedAt: number | null
|
||||
unavailableSince: number | null
|
||||
nextRetryAt: number | null
|
||||
lastError: string | null
|
||||
}
|
||||
|
||||
interface HiddenPlaywrightRecord {
|
||||
key: string
|
||||
@@ -41,11 +57,95 @@ const records = new Map<string, HiddenPlaywrightRecord>()
|
||||
let reaperHandle: ReturnType<typeof setInterval> | null = null
|
||||
let connectPromise: Promise<PlaywrightBrowser> | null = null
|
||||
let connectedBrowser: PlaywrightBrowser | null = null
|
||||
let cdpHealthState: PlaywrightCDPHealthState = {
|
||||
status: 'unknown',
|
||||
checkedAt: null,
|
||||
unavailableSince: null,
|
||||
nextRetryAt: null,
|
||||
lastError: null,
|
||||
}
|
||||
|
||||
export function getPlaywrightCDPPort(): number {
|
||||
return cdpPort
|
||||
}
|
||||
|
||||
export async function preparePlaywrightCDPPort(): Promise<number> {
|
||||
const port = await findAvailableCDPPort()
|
||||
setPlaywrightCDPPort(port)
|
||||
return port
|
||||
}
|
||||
|
||||
export function isPlaywrightCDPUnavailableError(error: unknown): boolean {
|
||||
if (!(error instanceof Error)) {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
error.message === 'playwright_cdp_endpoint_unavailable' ||
|
||||
error.message === 'playwright_cdp_port_unavailable' ||
|
||||
error.message.includes('playwright_cdp_endpoint_unavailable') ||
|
||||
error.message.includes('connectOverCDP')
|
||||
)
|
||||
}
|
||||
|
||||
export async function ensurePlaywrightCDPReady(options: { force?: boolean } = {}): Promise<void> {
|
||||
const now = Date.now()
|
||||
if (
|
||||
!options.force &&
|
||||
cdpHealthState.status === 'healthy' &&
|
||||
cdpHealthState.checkedAt &&
|
||||
now - cdpHealthState.checkedAt < 15_000
|
||||
) {
|
||||
return
|
||||
}
|
||||
if (
|
||||
!options.force &&
|
||||
cdpHealthState.status === 'unavailable' &&
|
||||
cdpHealthState.nextRetryAt &&
|
||||
cdpHealthState.nextRetryAt > now
|
||||
) {
|
||||
throw new Error(cdpHealthState.lastError || 'playwright_cdp_endpoint_unavailable')
|
||||
}
|
||||
|
||||
try {
|
||||
await waitForCDPEndpoint(cdpCircuitProbeTimeoutMs)
|
||||
markPlaywrightCDPHealthy()
|
||||
} catch (error) {
|
||||
markPlaywrightCDPUnavailable(error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export function isPlaywrightCDPExecutionAvailable(now = Date.now()): boolean {
|
||||
if (connectedBrowser?.isConnected()) {
|
||||
return true
|
||||
}
|
||||
if (cdpHealthState.status !== 'unavailable') {
|
||||
return true
|
||||
}
|
||||
return Boolean(cdpHealthState.nextRetryAt && cdpHealthState.nextRetryAt <= now)
|
||||
}
|
||||
|
||||
export function getHiddenPlaywrightSnapshot() {
|
||||
return {
|
||||
cdpPort,
|
||||
cdpEndpointURL,
|
||||
connected: Boolean(connectedBrowser?.isConnected()),
|
||||
pageCount: records.size,
|
||||
idleTTLms: hiddenPlaywrightIdleTTLms,
|
||||
health: { ...cdpHealthState },
|
||||
records: [...records.values()].map((record) => ({
|
||||
accountId: record.accountId,
|
||||
title: record.title,
|
||||
targetURL: record.targetURL,
|
||||
retainCount: record.retainCount,
|
||||
activatedAt: record.activatedAt,
|
||||
lastReleasedAt: record.lastReleasedAt,
|
||||
windowDestroyed: record.window.isDestroyed(),
|
||||
pageClosed: record.page.isClosed(),
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
export function startHiddenPlaywrightReaper(): void {
|
||||
if (reaperHandle) {
|
||||
return
|
||||
@@ -155,26 +255,6 @@ export function reapIdleHiddenPlaywrightPages(
|
||||
return reclaimed
|
||||
}
|
||||
|
||||
export function getHiddenPlaywrightSnapshot() {
|
||||
return {
|
||||
cdpPort,
|
||||
cdpEndpointURL,
|
||||
connected: Boolean(connectedBrowser?.isConnected()),
|
||||
pageCount: records.size,
|
||||
idleTTLms: hiddenPlaywrightIdleTTLms,
|
||||
records: [...records.values()].map((record) => ({
|
||||
accountId: record.accountId,
|
||||
title: record.title,
|
||||
targetURL: record.targetURL,
|
||||
retainCount: record.retainCount,
|
||||
activatedAt: record.activatedAt,
|
||||
lastReleasedAt: record.lastReleasedAt,
|
||||
windowDestroyed: record.window.isDestroyed(),
|
||||
pageClosed: record.page.isClosed(),
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
function createLease(record: HiddenPlaywrightRecord): HiddenPlaywrightLease {
|
||||
return {
|
||||
accountId: record.accountId,
|
||||
@@ -271,6 +351,7 @@ async function connectBrowserOverCDP(): Promise<PlaywrightBrowser> {
|
||||
|
||||
connectPromise = (async () => {
|
||||
await waitForCDPEndpoint()
|
||||
markPlaywrightCDPHealthy()
|
||||
const { chromium } = await import('playwright-core')
|
||||
await pruneStaleCDPTargets()
|
||||
|
||||
@@ -292,6 +373,9 @@ async function connectBrowserOverCDP(): Promise<PlaywrightBrowser> {
|
||||
|
||||
try {
|
||||
return await connectPromise
|
||||
} catch (error) {
|
||||
markPlaywrightCDPUnavailable(error)
|
||||
throw error
|
||||
} finally {
|
||||
if (!connectedBrowser?.isConnected()) {
|
||||
connectPromise = null
|
||||
@@ -314,7 +398,7 @@ async function waitForCDPEndpoint(timeoutMs = defaultConnectTimeoutMs): Promise<
|
||||
await wait(200)
|
||||
}
|
||||
|
||||
throw new Error('playwright_cdp_endpoint_unavailable')
|
||||
throw new Error(`playwright_cdp_endpoint_unavailable:${cdpEndpointURL}`)
|
||||
}
|
||||
|
||||
async function openBrowserOverCDP(
|
||||
@@ -330,6 +414,7 @@ async function openBrowserOverCDP(
|
||||
})
|
||||
|
||||
connectedBrowser = browser
|
||||
markPlaywrightCDPHealthy()
|
||||
return browser
|
||||
}
|
||||
|
||||
@@ -468,3 +553,104 @@ function wait(timeMs: number): Promise<void> {
|
||||
setTimeout(resolve, timeMs)
|
||||
})
|
||||
}
|
||||
|
||||
function normalizeCDPPort(value: string | undefined, fallback: number): number {
|
||||
const parsed = Number.parseInt(value ?? '', 10)
|
||||
if (!Number.isFinite(parsed) || parsed <= 0 || parsed > 65_535) {
|
||||
return fallback
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
function endpointURLForPort(port: number): string {
|
||||
return `http://127.0.0.1:${port}`
|
||||
}
|
||||
|
||||
function setPlaywrightCDPPort(port: number): void {
|
||||
cdpPort = port
|
||||
cdpEndpointURL = endpointURLForPort(port)
|
||||
cdpTargetsListURL = `${cdpEndpointURL}/json/list`
|
||||
}
|
||||
|
||||
async function findAvailableCDPPort(): Promise<number> {
|
||||
if (await canBindPort(preferredCDPPort)) {
|
||||
return preferredCDPPort
|
||||
}
|
||||
|
||||
const min = Math.min(dynamicCDPPortMin, dynamicCDPPortMax)
|
||||
const max = Math.max(dynamicCDPPortMin, dynamicCDPPortMax)
|
||||
const start = Math.max(min, preferredCDPPort)
|
||||
for (let port = start; port <= max; port += 1) {
|
||||
if (port === preferredCDPPort) {
|
||||
continue
|
||||
}
|
||||
if (await canBindPort(port)) {
|
||||
return port
|
||||
}
|
||||
}
|
||||
|
||||
for (let port = min; port < start; port += 1) {
|
||||
if (port === preferredCDPPort) {
|
||||
continue
|
||||
}
|
||||
if (await canBindPort(port)) {
|
||||
return port
|
||||
}
|
||||
}
|
||||
|
||||
const reserved = await reserveEphemeralPort()
|
||||
return reserved
|
||||
}
|
||||
|
||||
function canBindPort(port: number): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
const server = createServer()
|
||||
server.once('error', () => {
|
||||
resolve(false)
|
||||
})
|
||||
server.listen({ host: '127.0.0.1', port }, () => {
|
||||
server.close(() => {
|
||||
resolve(true)
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function reserveEphemeralPort(): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = createServer()
|
||||
server.once('error', reject)
|
||||
server.listen({ host: '127.0.0.1', port: 0 }, () => {
|
||||
const address = server.address()
|
||||
const port = typeof address === 'object' && address ? address.port : 0
|
||||
server.close(() => {
|
||||
if (port > 0) {
|
||||
resolve(port)
|
||||
} else {
|
||||
reject(new Error('playwright_cdp_port_unavailable'))
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function markPlaywrightCDPHealthy(): void {
|
||||
cdpHealthState = {
|
||||
status: 'healthy',
|
||||
checkedAt: Date.now(),
|
||||
unavailableSince: null,
|
||||
nextRetryAt: null,
|
||||
lastError: null,
|
||||
}
|
||||
}
|
||||
|
||||
function markPlaywrightCDPUnavailable(error: unknown): void {
|
||||
const now = Date.now()
|
||||
cdpHealthState = {
|
||||
status: 'unavailable',
|
||||
checkedAt: now,
|
||||
unavailableSince: cdpHealthState.unavailableSince ?? now,
|
||||
nextRetryAt: now + cdpCircuitOpenTTLms,
|
||||
lastError: error instanceof Error ? error.message : String(error),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ interface PublishSchedulerSelectionOptions {
|
||||
now?: number
|
||||
maxConcurrency: number
|
||||
activePlatforms?: ReadonlySet<string>
|
||||
blockedPlatforms?: ReadonlySet<string>
|
||||
activeCount: number
|
||||
}
|
||||
|
||||
@@ -97,6 +98,10 @@ export function selectNextPublishTask(
|
||||
})
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (options.blockedPlatforms?.has(candidate.platform)) {
|
||||
continue
|
||||
}
|
||||
|
||||
if (activePlatforms.has(candidate.platform)) {
|
||||
continue
|
||||
}
|
||||
@@ -143,6 +148,19 @@ export function notePublishTaskLeaseMiss(taskId: string): void {
|
||||
tasks.delete(taskId)
|
||||
}
|
||||
|
||||
export function notePublishTaskDeferred(taskId: string): void {
|
||||
const existing = tasks.get(taskId)
|
||||
if (!existing) {
|
||||
return
|
||||
}
|
||||
|
||||
tasks.set(taskId, {
|
||||
...existing,
|
||||
state: 'queued',
|
||||
updatedAt: Date.now(),
|
||||
})
|
||||
}
|
||||
|
||||
export function notePublishTaskCompleted(taskId: string, platform?: string): void {
|
||||
const existing = tasks.get(taskId)
|
||||
const resolvedPlatform = (platform ?? existing?.platform ?? '').trim()
|
||||
|
||||
@@ -68,10 +68,17 @@ import {
|
||||
noteMonitorTaskActivated,
|
||||
noteMonitorTaskCanceled,
|
||||
noteMonitorTaskCompleted,
|
||||
noteMonitorTaskDeferred,
|
||||
noteMonitorTaskLeased,
|
||||
selectNextMonitorTask,
|
||||
} from './monitor-scheduler'
|
||||
import { retainHiddenPlaywrightPage, startHiddenPlaywrightReaper } from './playwright-cdp'
|
||||
import {
|
||||
ensurePlaywrightCDPReady,
|
||||
isPlaywrightCDPExecutionAvailable,
|
||||
isPlaywrightCDPUnavailableError,
|
||||
retainHiddenPlaywrightPage,
|
||||
startHiddenPlaywrightReaper,
|
||||
} from './playwright-cdp'
|
||||
import { getProcessMetricsSnapshot } from './process-metrics'
|
||||
import {
|
||||
enqueuePublishTask as enqueuePublishSchedulerTask,
|
||||
@@ -80,6 +87,7 @@ import {
|
||||
notePublishTaskActivated,
|
||||
notePublishTaskCanceled,
|
||||
notePublishTaskCompleted,
|
||||
notePublishTaskDeferred,
|
||||
notePublishTaskLeaseMiss,
|
||||
selectNextPublishTask,
|
||||
} from './publish-scheduler'
|
||||
@@ -102,6 +110,7 @@ import {
|
||||
clearSessionHandle,
|
||||
createSessionHandle,
|
||||
hasLocalSessionPartition,
|
||||
type SessionHandle,
|
||||
} from './session-registry'
|
||||
import {
|
||||
cancelDesktopTask,
|
||||
@@ -148,6 +157,8 @@ const heartbeatIntervalMs = 15_000
|
||||
const pullIntervalMs = 60_000
|
||||
const publishPullIntervalMs = 30_000
|
||||
const monitorPullFallbackIntervalMs = 10_000
|
||||
const playwrightCDPPreflightDeferMs = 15_000
|
||||
const playwrightCDPRecoveryActivityCooldownMs = 60_000
|
||||
const accountSyncIntervalMs = 30_000
|
||||
const pumpWatchdogIntervalMs = 30_000
|
||||
const leaseExtendIntervalMs = 60_000
|
||||
@@ -163,6 +174,8 @@ const maximumRuntimeTotalConcurrency = 4
|
||||
const BAIJIAHAO_RISK_CONTROL_PROMPT = '您触发平台风控,请手动发稿一篇并完成验证,方可继续使用'
|
||||
const accountHealthReportDebounceMs = 1_000
|
||||
const accountHealthReportMaxBatchSize = 100
|
||||
const maxPlaywrightCDPSoftRecoveries = 3
|
||||
const playwrightCDPFatalMarker = '--geo-cdp-fatal'
|
||||
|
||||
interface RuntimeTaskRecord {
|
||||
id: string
|
||||
@@ -312,6 +325,18 @@ const state: RuntimeState = {
|
||||
activitySeq: 0,
|
||||
}
|
||||
let accountHealthReportBridgeUnsubscribe: (() => void) | null = null
|
||||
let lastPlaywrightCDPRecoveryActivityAt = 0
|
||||
let playwrightCDPRecoveryAttemptCount = process.argv.filter((arg) =>
|
||||
arg.startsWith('--geo-cdp-recovery-at='),
|
||||
).length
|
||||
let playwrightCDPFatal = process.argv.includes(playwrightCDPFatalMarker)
|
||||
|
||||
class PlaywrightCDPInfrastructureError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message)
|
||||
this.name = 'PlaywrightCDPInfrastructureError'
|
||||
}
|
||||
}
|
||||
|
||||
function accountIdentityFromDesktopAccount(account: DesktopAccountInfo): PublishAccountIdentity {
|
||||
return {
|
||||
@@ -703,6 +728,17 @@ function startRuntime(): void {
|
||||
if (!canRunLive(state.session) || state.running) {
|
||||
return
|
||||
}
|
||||
if (playwrightCDPFatal) {
|
||||
const message = '浏览器自动化通道不可用,客户端已停止采集。请重启客户端或检查安全软件/端口占用。'
|
||||
state.lastError = message
|
||||
setSchedulerPhase('paused')
|
||||
setSchedulerError(message)
|
||||
recordActivity('danger', '浏览器自动化组件不可用', message)
|
||||
emitRuntimeInvalidated('playwright-cdp-fatal', {
|
||||
message,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
initScheduler()
|
||||
initMonitorScheduler()
|
||||
@@ -1850,17 +1886,40 @@ function asSubmitUncertainExecution(result: AdapterExecutionResult): AdapterExec
|
||||
}
|
||||
}
|
||||
|
||||
async function retainPlaywrightPageOrThrow(
|
||||
task: RuntimeTaskRecord,
|
||||
payload: Record<string, JsonValue>,
|
||||
sessionHandle: SessionHandle,
|
||||
) {
|
||||
try {
|
||||
await ensurePlaywrightCDPReady()
|
||||
return await retainHiddenPlaywrightPage({
|
||||
accountId: task.accountId || task.id,
|
||||
session: sessionHandle.session,
|
||||
targetURL: resolvePlaywrightTargetURL(task.platform, payload),
|
||||
title: `${task.title} · Hidden Playwright`,
|
||||
})
|
||||
} catch (error) {
|
||||
if (isPlaywrightCDPUnavailableError(error)) {
|
||||
throw new PlaywrightCDPInfrastructureError(errorMessage(error))
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
function pumpExecutionLoop(): void {
|
||||
if (!state.running) {
|
||||
if (!state.running || playwrightCDPFatal) {
|
||||
return
|
||||
}
|
||||
|
||||
syncSchedulerSurface()
|
||||
const blockedPlaywrightPlatforms = blockedPlaywrightExecutionPlatforms()
|
||||
|
||||
if (canStartAnotherPublishExecution()) {
|
||||
const selected = selectNextPublishTask({
|
||||
maxConcurrency: resolveAdaptiveTotalConcurrency(),
|
||||
activePlatforms: activePublishPlatforms(),
|
||||
blockedPlatforms: blockedPlaywrightPlatforms,
|
||||
activeCount: activePublishCount(),
|
||||
})
|
||||
if (selected) {
|
||||
@@ -1883,6 +1942,7 @@ function pumpExecutionLoop(): void {
|
||||
maxConcurrency: resolveAdaptiveMonitorConcurrency(),
|
||||
activePlatforms: activeMonitorPlatforms(),
|
||||
activeQuestionKeys: activeMonitorQuestionKeys(),
|
||||
blockedPlatforms: blockedPlaywrightPlatforms,
|
||||
activeCount: activeMonitorCount(),
|
||||
})
|
||||
if (selected) {
|
||||
@@ -1912,6 +1972,10 @@ async function leaseSpecificTask(
|
||||
return
|
||||
}
|
||||
|
||||
if (!(await ensurePlaywrightCDPAdmissionForRequest(expectedKind, request))) {
|
||||
return
|
||||
}
|
||||
|
||||
state.leaseInFlightKinds.add(expectedKind)
|
||||
|
||||
try {
|
||||
@@ -1967,7 +2031,11 @@ async function leaseSpecificTask(
|
||||
|
||||
async function pullNextTask(kind: 'publish' | 'monitor'): Promise<void> {
|
||||
if (kind === 'monitor') {
|
||||
if (!canRunLive(state.session) || isLeaseInFlight('monitor')) {
|
||||
if (!canRunLive(state.session) || isLeaseInFlight('monitor') || playwrightCDPFatal) {
|
||||
return
|
||||
}
|
||||
|
||||
if (!(await ensurePlaywrightCDPAdmissionForFallback('monitor'))) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -2002,7 +2070,11 @@ async function pullNextTask(kind: 'publish' | 'monitor'): Promise<void> {
|
||||
return
|
||||
}
|
||||
|
||||
if (!canRunLive(state.session) || isLeaseInFlight('publish')) {
|
||||
if (!canRunLive(state.session) || isLeaseInFlight('publish') || playwrightCDPFatal) {
|
||||
return
|
||||
}
|
||||
|
||||
if (!(await ensurePlaywrightCDPAdmissionForFallback('publish'))) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -2046,7 +2118,7 @@ async function pullNextTask(kind: 'publish' | 'monitor'): Promise<void> {
|
||||
|
||||
async function pullMonitoringTasks(routing: RuntimeTaskRouting): Promise<void> {
|
||||
const leaseLimit = resolveLegacyMonitoringLeaseLimit()
|
||||
if (!canRunLive(state.session) || isLeaseInFlight('monitor') || leaseLimit <= 0) {
|
||||
if (!canRunLive(state.session) || isLeaseInFlight('monitor') || leaseLimit <= 0 || playwrightCDPFatal) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -2270,6 +2342,15 @@ async function executeLeasedMonitoringTask(
|
||||
`${taskRecord.title} ${execution.summary}`,
|
||||
)
|
||||
} catch (error) {
|
||||
if (isPlaywrightCDPInfrastructureError(error)) {
|
||||
await deferMonitoringLeaseAfterInfrastructureFailure(
|
||||
taskRecord,
|
||||
monitoringTaskID,
|
||||
taskRecord.leaseToken,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
if (isMonitorAbortReason(taskRecord, error, 'stale_business_date')) {
|
||||
let dropSummary = staleMonitoringDropSummary(taskRecord)
|
||||
try {
|
||||
@@ -2531,6 +2612,11 @@ async function executeLeasedDesktopMonitorTask(
|
||||
`${taskRecord.title} ${execution.summary}`,
|
||||
)
|
||||
} catch (error) {
|
||||
if (isPlaywrightCDPInfrastructureError(error)) {
|
||||
await deferDesktopMonitorTaskAfterInfrastructureFailure(taskRecord, leased, monitoringTaskID)
|
||||
return
|
||||
}
|
||||
|
||||
if (isMonitorAbortReason(taskRecord, error, 'stale_business_date')) {
|
||||
let dropSummary = staleMonitoringDropSummary(taskRecord)
|
||||
try {
|
||||
@@ -3024,12 +3110,7 @@ async function executeTaskAdapter(
|
||||
: retainHotView(task.accountId || task.id)
|
||||
const playwrightLease =
|
||||
adapter.executionMode === 'playwright'
|
||||
? await retainHiddenPlaywrightPage({
|
||||
accountId: task.accountId || task.id,
|
||||
session: sessionHandle.session,
|
||||
targetURL: resolvePlaywrightTargetURL(task.platform, payload),
|
||||
title: `${task.title} · Hidden Playwright`,
|
||||
})
|
||||
? await retainPlaywrightPageOrThrow(task, payload, sessionHandle)
|
||||
: null
|
||||
|
||||
try {
|
||||
@@ -3098,12 +3179,7 @@ async function executeTaskAdapter(
|
||||
: retainHotView(task.accountId || task.id)
|
||||
const playwrightLease =
|
||||
adapter.executionMode === 'playwright'
|
||||
? await retainHiddenPlaywrightPage({
|
||||
accountId: task.accountId || task.id,
|
||||
session: sessionHandle.session,
|
||||
targetURL: resolvePlaywrightTargetURL(task.platform, payload),
|
||||
title: `${task.title} · Hidden Playwright`,
|
||||
})
|
||||
? await retainPlaywrightPageOrThrow(task, payload, sessionHandle)
|
||||
: null
|
||||
|
||||
try {
|
||||
@@ -3534,6 +3610,238 @@ function isLeaseInFlight(kind: 'publish' | 'monitor'): boolean {
|
||||
return state.leaseInFlightKinds.has(kind)
|
||||
}
|
||||
|
||||
function blockedPlaywrightExecutionPlatforms(): Set<string> {
|
||||
if (isPlaywrightCDPExecutionAvailable()) {
|
||||
return new Set()
|
||||
}
|
||||
|
||||
return new Set([
|
||||
...state.accounts
|
||||
.map((account) => account.platform)
|
||||
.filter((platform) => adapterRequiresPlaywright('monitor', platform)),
|
||||
...getPublishSchedulerSnapshot()
|
||||
.tasks.map((task) => task.platform)
|
||||
.filter((platform) => adapterRequiresPlaywright('publish', platform)),
|
||||
...getMonitorSchedulerSnapshot()
|
||||
.tasks.map((task) => task.platform)
|
||||
.filter((platform) => adapterRequiresPlaywright('monitor', platform)),
|
||||
])
|
||||
}
|
||||
|
||||
async function ensurePlaywrightCDPAdmissionForRequest(
|
||||
kind: 'publish' | 'monitor',
|
||||
request: PendingTaskRequest,
|
||||
): Promise<boolean> {
|
||||
if (!request.platform || !adapterRequiresPlaywright(kind, request.platform)) {
|
||||
return true
|
||||
}
|
||||
|
||||
const ready = await ensurePlaywrightCDPAdmission()
|
||||
if (!ready) {
|
||||
deferPendingTaskRequest(kind, request, playwrightCDPPreflightDeferMs)
|
||||
}
|
||||
return ready
|
||||
}
|
||||
|
||||
async function ensurePlaywrightCDPAdmissionForFallback(kind: 'publish' | 'monitor'): Promise<boolean> {
|
||||
if (kind === 'publish') {
|
||||
const queuedPlaywrightPublish = getPublishSchedulerSnapshot().tasks.some(
|
||||
(task) => task.state === 'queued' && adapterRequiresPlaywright('publish', task.platform),
|
||||
)
|
||||
if (!queuedPlaywrightPublish) {
|
||||
return true
|
||||
}
|
||||
return ensurePlaywrightCDPAdmission()
|
||||
}
|
||||
|
||||
const hasNonPlaywrightMonitorAccount = state.accounts.some(
|
||||
(account) =>
|
||||
isAIPlatformId(account.platform) &&
|
||||
!adapterRequiresPlaywright('monitor', account.platform) &&
|
||||
getProjectedAccountHealth({
|
||||
accountId: account.id,
|
||||
platform: account.platform,
|
||||
health: account.health,
|
||||
verifiedAt: account.verified_at,
|
||||
}).health === 'live',
|
||||
)
|
||||
if (hasNonPlaywrightMonitorAccount) {
|
||||
return true
|
||||
}
|
||||
|
||||
return ensurePlaywrightCDPAdmission()
|
||||
}
|
||||
|
||||
async function ensurePlaywrightCDPAdmission(): Promise<boolean> {
|
||||
try {
|
||||
await ensurePlaywrightCDPReady()
|
||||
return true
|
||||
} catch (error) {
|
||||
handlePlaywrightCDPUnavailable(error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function isPlaywrightCDPInfrastructureError(error: unknown): error is PlaywrightCDPInfrastructureError {
|
||||
return error instanceof PlaywrightCDPInfrastructureError || isPlaywrightCDPUnavailableError(error)
|
||||
}
|
||||
|
||||
function deferPendingTaskRequest(
|
||||
kind: 'publish' | 'monitor',
|
||||
request: PendingTaskRequest,
|
||||
delayMs: number,
|
||||
): void {
|
||||
if (kind === 'publish') {
|
||||
notePublishTaskDeferred(request.taskId)
|
||||
state.publishFallbackBackoffUntil = Date.now() + Math.max(delayMs, publishPullIntervalMs)
|
||||
return
|
||||
}
|
||||
|
||||
noteMonitorTaskDeferred(request.taskId, delayMs)
|
||||
state.monitorFallbackBackoffUntil = Date.now() + Math.max(delayMs, monitorPullFallbackIntervalMs)
|
||||
}
|
||||
|
||||
function adapterRequiresPlaywright(kind: 'publish' | 'monitor', platform: string): boolean {
|
||||
const adapter = kind === 'publish' ? selectPublishAdapter(platform) : selectMonitorAdapter(platform)
|
||||
return adapter?.executionMode === 'playwright'
|
||||
}
|
||||
|
||||
function handlePlaywrightCDPUnavailable(error: unknown, taskId?: string): void {
|
||||
const message = errorMessage(error) || 'playwright_cdp_endpoint_unavailable'
|
||||
state.lastError = message
|
||||
setSchedulerError(message)
|
||||
state.monitorFallbackBackoffUntil = Date.now() + monitorPullFallbackIntervalMs
|
||||
state.publishFallbackBackoffUntil = Date.now() + publishPullIntervalMs
|
||||
|
||||
if (playwrightCDPRecoveryAttemptCount >= maxPlaywrightCDPSoftRecoveries) {
|
||||
markPlaywrightCDPFatal(message, taskId)
|
||||
return
|
||||
}
|
||||
playwrightCDPRecoveryAttemptCount += 1
|
||||
|
||||
const now = Date.now()
|
||||
if (now - lastPlaywrightCDPRecoveryActivityAt >= playwrightCDPRecoveryActivityCooldownMs) {
|
||||
lastPlaywrightCDPRecoveryActivityAt = now
|
||||
recordActivity(
|
||||
'warn',
|
||||
'浏览器自动化通道异常',
|
||||
'已暂停领取 Playwright 平台任务,客户端将自动恢复 CDP 后继续采集。',
|
||||
)
|
||||
}
|
||||
|
||||
emitRuntimeInvalidated('playwright-cdp-recovery', {
|
||||
taskId,
|
||||
message,
|
||||
})
|
||||
syncSchedulerSurface()
|
||||
}
|
||||
|
||||
function markPlaywrightCDPFatal(message: string, taskId?: string): void {
|
||||
if (playwrightCDPFatal) {
|
||||
return
|
||||
}
|
||||
playwrightCDPFatal = true
|
||||
|
||||
const fatalMessage =
|
||||
message ||
|
||||
'浏览器自动化通道不可用,客户端已停止采集。请重启客户端或检查安全软件/端口占用。'
|
||||
state.lastError = fatalMessage
|
||||
setSchedulerPhase('paused')
|
||||
setSchedulerError(fatalMessage)
|
||||
setTransportDispatchState('idle')
|
||||
state.dispatchWsConnected = false
|
||||
state.dispatchWsClient?.stop()
|
||||
state.dispatchWsClient = null
|
||||
state.leaseInFlightKinds.clear()
|
||||
recordActivity('danger', '浏览器自动化组件不可用', fatalMessage)
|
||||
|
||||
void offlineDesktopClient().catch((offlineError) => {
|
||||
console.warn('[desktop-runtime] failed to mark client offline after cdp fatal', {
|
||||
message: errorMessage(offlineError),
|
||||
})
|
||||
})
|
||||
|
||||
emitRuntimeInvalidated('playwright-cdp-fatal', {
|
||||
taskId,
|
||||
message: fatalMessage,
|
||||
})
|
||||
syncSchedulerSurface()
|
||||
}
|
||||
|
||||
async function deferMonitoringLeaseAfterInfrastructureFailure(
|
||||
task: RuntimeTaskRecord,
|
||||
monitoringTaskID: number,
|
||||
leaseToken: string,
|
||||
reason = 'playwright_cdp_unavailable',
|
||||
): Promise<void> {
|
||||
try {
|
||||
await cancelMonitoringTask(monitoringTaskID, {
|
||||
lease_token: leaseToken,
|
||||
reason,
|
||||
})
|
||||
} catch (cancelError) {
|
||||
recordActivity(
|
||||
'warn',
|
||||
'监控租约释放失败',
|
||||
`${task.title} 基础设施异常后释放租约失败:${errorMessage(cancelError)}`,
|
||||
)
|
||||
}
|
||||
|
||||
const existing = state.tasks.get(task.id)
|
||||
if (existing) {
|
||||
existing.status = 'queued'
|
||||
existing.summary = '浏览器自动化通道异常,已释放租约并等待客户端自动恢复后重试。'
|
||||
existing.error = {
|
||||
code: reason,
|
||||
message: 'playwright cdp is unavailable; task deferred without marking collection failed',
|
||||
}
|
||||
existing.updatedAt = Date.now()
|
||||
existing.attemptId = null
|
||||
existing.leaseToken = null
|
||||
state.tasks.set(task.id, existing)
|
||||
}
|
||||
noteLeaseReleased('completed', task.id)
|
||||
noteMonitorTaskDeferred(task.id, playwrightCDPPreflightDeferMs)
|
||||
handlePlaywrightCDPUnavailable(new Error(reason), task.id)
|
||||
}
|
||||
|
||||
async function deferDesktopMonitorTaskAfterInfrastructureFailure(
|
||||
task: RuntimeTaskRecord,
|
||||
leased: LeaseDesktopTaskResponse,
|
||||
monitoringTaskID: number,
|
||||
reason = 'playwright_cdp_unavailable',
|
||||
): Promise<void> {
|
||||
try {
|
||||
await cancelMonitoringTask(monitoringTaskID, {
|
||||
lease_token: leased.lease_token as string,
|
||||
reason,
|
||||
})
|
||||
} catch (cancelError) {
|
||||
recordActivity(
|
||||
'warn',
|
||||
'监控租约释放失败',
|
||||
`${task.title} 基础设施异常后释放 phase2 租约失败:${errorMessage(cancelError)}`,
|
||||
)
|
||||
}
|
||||
|
||||
const existing = state.tasks.get(task.id)
|
||||
if (existing) {
|
||||
existing.status = 'queued'
|
||||
existing.summary = '浏览器自动化通道异常,任务已回队等待客户端自动恢复。'
|
||||
existing.error = {
|
||||
code: reason,
|
||||
message: 'playwright cdp is unavailable; desktop monitor task requeued',
|
||||
}
|
||||
existing.updatedAt = Date.now()
|
||||
existing.attemptId = null
|
||||
existing.leaseToken = null
|
||||
state.tasks.set(task.id, existing)
|
||||
}
|
||||
noteLeaseReleased('completed', task.id)
|
||||
noteMonitorTaskDeferred(task.id, playwrightCDPPreflightDeferMs)
|
||||
handlePlaywrightCDPUnavailable(new Error(reason), task.id)
|
||||
}
|
||||
|
||||
function activePublishCount(): number {
|
||||
return [...state.activeExecutions.values()].filter((item) => item.kind === 'publish').length
|
||||
}
|
||||
|
||||
@@ -1,17 +1,23 @@
|
||||
type RuntimeInvalidationReason = 'account-health' | 'publish-task-lease' | 'publish-task-progress'
|
||||
type RuntimeInvalidationReason =
|
||||
| 'account-health'
|
||||
| 'publish-task-lease'
|
||||
| 'publish-task-progress'
|
||||
| 'playwright-cdp-recovery'
|
||||
| 'playwright-cdp-fatal'
|
||||
|
||||
type RuntimeInvalidationListener = (event: {
|
||||
reason: RuntimeInvalidationReason
|
||||
at: number
|
||||
accountId?: string
|
||||
taskId?: string
|
||||
message?: string
|
||||
}) => void
|
||||
|
||||
const listeners = new Set<RuntimeInvalidationListener>()
|
||||
|
||||
export function emitRuntimeInvalidated(
|
||||
reason: RuntimeInvalidationReason,
|
||||
details: { accountId?: string; taskId?: string } = {},
|
||||
details: { accountId?: string; taskId?: string; message?: string } = {},
|
||||
): void {
|
||||
const event = {
|
||||
reason,
|
||||
|
||||
@@ -249,19 +249,31 @@ const desktopBridge = {
|
||||
ipcRenderer.invoke('desktop:set-window-mode', mode, request) as Promise<null>,
|
||||
onRuntimeInvalidated: (
|
||||
listener: (event: {
|
||||
reason: 'account-health' | 'publish-task-lease' | 'publish-task-progress'
|
||||
reason:
|
||||
| 'account-health'
|
||||
| 'publish-task-lease'
|
||||
| 'publish-task-progress'
|
||||
| 'playwright-cdp-recovery'
|
||||
| 'playwright-cdp-fatal'
|
||||
at: number
|
||||
accountId?: string
|
||||
taskId?: string
|
||||
message?: string
|
||||
}) => void,
|
||||
) => {
|
||||
const wrapped = (
|
||||
_event: IpcRendererEvent,
|
||||
payload: {
|
||||
reason: 'account-health' | 'publish-task-lease' | 'publish-task-progress'
|
||||
reason:
|
||||
| 'account-health'
|
||||
| 'publish-task-lease'
|
||||
| 'publish-task-progress'
|
||||
| 'playwright-cdp-recovery'
|
||||
| 'playwright-cdp-fatal'
|
||||
at: number
|
||||
accountId?: string
|
||||
taskId?: string
|
||||
message?: string
|
||||
},
|
||||
) => {
|
||||
listener(payload)
|
||||
|
||||
+7
-1
@@ -142,10 +142,16 @@ declare global {
|
||||
): Promise<null>
|
||||
onRuntimeInvalidated(
|
||||
listener: (event: {
|
||||
reason: 'account-health' | 'publish-task-lease' | 'publish-task-progress'
|
||||
reason:
|
||||
| 'account-health'
|
||||
| 'publish-task-lease'
|
||||
| 'publish-task-progress'
|
||||
| 'playwright-cdp-recovery'
|
||||
| 'playwright-cdp-fatal'
|
||||
at: number
|
||||
accountId?: string
|
||||
taskId?: string
|
||||
message?: string
|
||||
}) => void,
|
||||
): () => void
|
||||
onNetworkObserved(listener: (event: DesktopObservedRequest) => void): () => void
|
||||
|
||||
Reference in New Issue
Block a user