fix: harden desktop monitoring recovery
Desktop Client Build / Resolve Build Metadata (push) Successful in 37s
Backend CI / Backend (push) Failing after 9m26s
Desktop Client Build / Build Desktop Client (push) Successful in 22m29s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 35s
Desktop Client Build / Resolve Build Metadata (push) Successful in 37s
Backend CI / Backend (push) Failing after 9m26s
Desktop Client Build / Build Desktop Client (push) Successful in 22m29s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 35s
This commit is contained in:
@@ -12,6 +12,7 @@ const {
|
|||||||
isNonCitationAssetUrl,
|
isNonCitationAssetUrl,
|
||||||
parseDoubaoStreamBody,
|
parseDoubaoStreamBody,
|
||||||
resolveDoubaoSubmissionAnswer,
|
resolveDoubaoSubmissionAnswer,
|
||||||
|
shouldRetryDoubaoPageAttempt,
|
||||||
} = __doubaoTestUtils
|
} = __doubaoTestUtils
|
||||||
|
|
||||||
describe('doubao adapter helpers', () => {
|
describe('doubao adapter helpers', () => {
|
||||||
@@ -28,6 +29,69 @@ describe('doubao adapter helpers', () => {
|
|||||||
expect(isDoubaoRecoverablePageError('doubao_login_expired')).toBe(false)
|
expect(isDoubaoRecoverablePageError('doubao_login_expired')).toBe(false)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('retries once when Doubao is stuck on the history surface without an answer', () => {
|
||||||
|
const streamSummary = parseDoubaoStreamBody(
|
||||||
|
['event: SSE_REPLY_END', 'data: {"message_id":"msg_1"}', '', ''].join('\n'),
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(
|
||||||
|
shouldRetryDoubaoPageAttempt(
|
||||||
|
{
|
||||||
|
ok: false,
|
||||||
|
error: 'doubao_query_timeout',
|
||||||
|
detail: '历史对话 主对话 折叠门联动配件明细',
|
||||||
|
url: 'https://www.doubao.com/chat/local_4995117315817271',
|
||||||
|
title: '豆包',
|
||||||
|
modelLabel: '豆包',
|
||||||
|
modeLabel: '思考',
|
||||||
|
thinkingModeApplied: true,
|
||||||
|
deepThinkEnabled: true,
|
||||||
|
domAnswer: '折叠门联动配件明细',
|
||||||
|
domReasoning: null,
|
||||||
|
domLinks: [],
|
||||||
|
submitted: true,
|
||||||
|
questionAnchorFound: true,
|
||||||
|
historySurfaceDetected: true,
|
||||||
|
},
|
||||||
|
streamSummary,
|
||||||
|
),
|
||||||
|
).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('does not retry a recoverable Doubao page error after SSE already yielded an answer', () => {
|
||||||
|
const streamSummary = parseDoubaoStreamBody(
|
||||||
|
[
|
||||||
|
'event: STREAM_CHUNK',
|
||||||
|
'data: {"text_block":{"text":"这是一段完整的豆包回答。"}}',
|
||||||
|
'',
|
||||||
|
'',
|
||||||
|
].join('\n'),
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(
|
||||||
|
shouldRetryDoubaoPageAttempt(
|
||||||
|
{
|
||||||
|
ok: false,
|
||||||
|
error: 'doubao_query_timeout',
|
||||||
|
detail: 'timeout',
|
||||||
|
url: 'https://www.doubao.com/chat/local_4995117315817271',
|
||||||
|
title: '豆包',
|
||||||
|
modelLabel: '豆包',
|
||||||
|
modeLabel: '思考',
|
||||||
|
thinkingModeApplied: true,
|
||||||
|
deepThinkEnabled: true,
|
||||||
|
domAnswer: null,
|
||||||
|
domReasoning: null,
|
||||||
|
domLinks: [],
|
||||||
|
submitted: true,
|
||||||
|
questionAnchorFound: true,
|
||||||
|
historySurfaceDetected: true,
|
||||||
|
},
|
||||||
|
streamSummary,
|
||||||
|
),
|
||||||
|
).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
it('maps the new expert/deep-thinking mode label to expert mode', () => {
|
it('maps the new expert/deep-thinking mode label to expert mode', () => {
|
||||||
expect(doubaoModeKindFromLabelText('专家')).toBe('expert')
|
expect(doubaoModeKindFromLabelText('专家')).toBe('expert')
|
||||||
expect(doubaoModeKindFromLabelText('专家 深度思考 研究级智能模型')).toBe('expert')
|
expect(doubaoModeKindFromLabelText('专家 深度思考 研究级智能模型')).toBe('expert')
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ const DOUBAO_PAGE_READY_TIMEOUT_MS = 20_000
|
|||||||
const DOUBAO_QUERY_TIMEOUT_MS = 120_000
|
const DOUBAO_QUERY_TIMEOUT_MS = 120_000
|
||||||
const DOUBAO_CAPTURE_LIMIT = 64
|
const DOUBAO_CAPTURE_LIMIT = 64
|
||||||
const DOUBAO_CAPTURE_BODY_LIMIT = 8_000_000
|
const DOUBAO_CAPTURE_BODY_LIMIT = 8_000_000
|
||||||
const DOUBAO_CAPTURE_FLUSH_TIMEOUT_MS = DOUBAO_QUERY_TIMEOUT_MS
|
const DOUBAO_CAPTURE_FLUSH_TIMEOUT_MS = 10_000
|
||||||
const DOUBAO_STREAM_URL_FRAGMENT = '/chat/completion'
|
const DOUBAO_STREAM_URL_FRAGMENT = '/chat/completion'
|
||||||
const DOUBAO_NON_CITATION_ASSET_DOMAINS = new Set(['byteimg.com', 'bytednsdoc.com'])
|
const DOUBAO_NON_CITATION_ASSET_DOMAINS = new Set(['byteimg.com', 'bytednsdoc.com'])
|
||||||
|
|
||||||
@@ -116,10 +116,22 @@ type DoubaoPageQueryFailureResult = {
|
|||||||
domAnswer: string | null
|
domAnswer: string | null
|
||||||
domReasoning: string | null
|
domReasoning: string | null
|
||||||
domLinks: DoubaoDomLink[]
|
domLinks: DoubaoDomLink[]
|
||||||
|
submitted: boolean | null
|
||||||
|
questionAnchorFound: boolean | null
|
||||||
|
historySurfaceDetected: boolean | null
|
||||||
}
|
}
|
||||||
|
|
||||||
type DoubaoPageQueryResult = DoubaoPageQuerySuccessResult | DoubaoPageQueryFailureResult
|
type DoubaoPageQueryResult = DoubaoPageQuerySuccessResult | DoubaoPageQueryFailureResult
|
||||||
|
|
||||||
|
type DoubaoQueryAttemptResult = {
|
||||||
|
attempt: number
|
||||||
|
pageResult: DoubaoPageQueryResult
|
||||||
|
captures: DoubaoCaptureRecord[]
|
||||||
|
streamSummary: DoubaoStreamSummary
|
||||||
|
captureDebug: JsonValue[]
|
||||||
|
pendingCaptureCount: number
|
||||||
|
}
|
||||||
|
|
||||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||||
}
|
}
|
||||||
@@ -1039,7 +1051,12 @@ function isDoubaoExecutionContextNavigationError(error: unknown): boolean {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function isDoubaoRecoverablePageError(error: string): boolean {
|
function isDoubaoRecoverablePageError(error: string): boolean {
|
||||||
return error === 'doubao_query_timeout' || error === 'doubao_page_navigation_interrupted'
|
return (
|
||||||
|
error === 'doubao_query_timeout' ||
|
||||||
|
error === 'doubao_page_navigation_interrupted' ||
|
||||||
|
error === 'doubao_submission_not_observed' ||
|
||||||
|
error === 'doubao_history_surface_timeout'
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildDoubaoPageNavigationResult(
|
function buildDoubaoPageNavigationResult(
|
||||||
@@ -1059,9 +1076,30 @@ function buildDoubaoPageNavigationResult(
|
|||||||
domAnswer: null,
|
domAnswer: null,
|
||||||
domReasoning: null,
|
domReasoning: null,
|
||||||
domLinks: [],
|
domLinks: [],
|
||||||
|
submitted: null,
|
||||||
|
questionAnchorFound: null,
|
||||||
|
historySurfaceDetected: null,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function shouldRetryDoubaoPageAttempt(
|
||||||
|
pageResult: DoubaoPageQueryResult,
|
||||||
|
streamSummary: DoubaoStreamSummary,
|
||||||
|
): boolean {
|
||||||
|
if (pageResult.ok || !isDoubaoRecoverablePageError(pageResult.error)) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if (isDoubaoSSESubmissionCandidate(streamSummary.answer)) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
pageResult.error === 'doubao_submission_not_observed' ||
|
||||||
|
pageResult.error === 'doubao_history_surface_timeout' ||
|
||||||
|
pageResult.historySurfaceDetected === true ||
|
||||||
|
pageResult.questionAnchorFound === false
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
function buildDoubaoUnknownResult(
|
function buildDoubaoUnknownResult(
|
||||||
code: string,
|
code: string,
|
||||||
summary: string,
|
summary: string,
|
||||||
@@ -1597,6 +1635,8 @@ const doubaoQueryInPage = async (
|
|||||||
const pollIntervalMs = 1_500
|
const pollIntervalMs = 1_500
|
||||||
const incompleteAnswerGraceMs = 30_000
|
const incompleteAnswerGraceMs = 30_000
|
||||||
const minimumAnswerSettleMs = 10_000
|
const minimumAnswerSettleMs = 10_000
|
||||||
|
const submissionObserveTimeoutMs = 12_000
|
||||||
|
const staleHistoryTimeoutMs = 18_000
|
||||||
|
|
||||||
const wait = (timeMs: number) =>
|
const wait = (timeMs: number) =>
|
||||||
new Promise<void>((resolve) => {
|
new Promise<void>((resolve) => {
|
||||||
@@ -2684,6 +2724,39 @@ const doubaoQueryInPage = async (
|
|||||||
line,
|
line,
|
||||||
))
|
))
|
||||||
|
|
||||||
|
const looksLikeHistoryTitleList = (text: string | null): boolean => {
|
||||||
|
if (!text) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
const normalizedText = text.replace(/\s+/g, ' ').trim()
|
||||||
|
if (
|
||||||
|
/搜索…\s*Ctrl\s*K\s*豆包|新对话\s*Ctrl\s*Shift\s*K|历史对话\s*主对话/.test(
|
||||||
|
normalizedText,
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
const lines = text
|
||||||
|
.split(/\n+/)
|
||||||
|
.map((line) => line.trim())
|
||||||
|
.filter(Boolean)
|
||||||
|
if (lines.length < 6) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
const shortTitleLines = lines.filter(
|
||||||
|
(line) =>
|
||||||
|
line.length <= 36 &&
|
||||||
|
!/[。!?!?]/.test(line) &&
|
||||||
|
/门|锁|轨|五金|导轨|合页|配件|工厂|厂家|供货|定制|折叠|幽灵|口袋/.test(line),
|
||||||
|
).length
|
||||||
|
const navHits = lines.filter((line) =>
|
||||||
|
/^(搜索|豆包|新对话|新办公任务|AI 创作|AI创作|云盘|收起|发现智能体|历史对话|主对话)$/.test(
|
||||||
|
line,
|
||||||
|
),
|
||||||
|
).length
|
||||||
|
return shortTitleLines >= 6 && (navHits >= 2 || shortTitleLines / lines.length >= 0.65)
|
||||||
|
}
|
||||||
|
|
||||||
// Tokens that only appear on Doubao's empty-chat surface (prompt templates, nav, feature chips).
|
// Tokens that only appear on Doubao's empty-chat surface (prompt templates, nav, feature chips).
|
||||||
const EMPTY_CHAT_TOKENS = [
|
const EMPTY_CHAT_TOKENS = [
|
||||||
'快速',
|
'快速',
|
||||||
@@ -2721,6 +2794,9 @@ const doubaoQueryInPage = async (
|
|||||||
return hits >= 3 || (hits >= 2 && lines.length <= 6)
|
return hits >= 3 || (hits >= 2 && lines.length <= 6)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const looksLikeNonAnswerSurface = (text: string | null): boolean =>
|
||||||
|
looksLikeEmptyChatSurface(text) || looksLikeHistoryTitleList(text)
|
||||||
|
|
||||||
const stripKnownNoise = (text: string | null): string | null => {
|
const stripKnownNoise = (text: string | null): string | null => {
|
||||||
if (!text) {
|
if (!text) {
|
||||||
return null
|
return null
|
||||||
@@ -2870,7 +2946,7 @@ const doubaoQueryInPage = async (
|
|||||||
if (!text && links.length === 0) {
|
if (!text && links.length === 0) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if (rawText && looksLikeEmptyChatSurface(rawText)) {
|
if (rawText && looksLikeNonAnswerSurface(rawText)) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if (text && text.length < 12 && links.length === 0) {
|
if (text && text.length < 12 && links.length === 0) {
|
||||||
@@ -3033,6 +3109,11 @@ const doubaoQueryInPage = async (
|
|||||||
applied: null,
|
applied: null,
|
||||||
},
|
},
|
||||||
snapshotOverride: ConversationSnapshot | null = null,
|
snapshotOverride: ConversationSnapshot | null = null,
|
||||||
|
diagnostics: {
|
||||||
|
submitted?: boolean | null
|
||||||
|
questionAnchorFound?: boolean | null
|
||||||
|
historySurfaceDetected?: boolean | null
|
||||||
|
} = {},
|
||||||
): DoubaoPageQueryFailureResult => {
|
): DoubaoPageQueryFailureResult => {
|
||||||
const snapshot = snapshotOverride ?? snapshotConversation(composerTop)
|
const snapshot = snapshotOverride ?? snapshotConversation(composerTop)
|
||||||
const composer = findComposer()
|
const composer = findComposer()
|
||||||
@@ -3057,6 +3138,9 @@ const doubaoQueryInPage = async (
|
|||||||
domAnswer: snapshot.answerText,
|
domAnswer: snapshot.answerText,
|
||||||
domReasoning: snapshot.reasoningText,
|
domReasoning: snapshot.reasoningText,
|
||||||
domLinks: snapshot.links,
|
domLinks: snapshot.links,
|
||||||
|
submitted: diagnostics.submitted ?? null,
|
||||||
|
questionAnchorFound: diagnostics.questionAnchorFound ?? null,
|
||||||
|
historySurfaceDetected: diagnostics.historySurfaceDetected ?? null,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3113,6 +3197,8 @@ const doubaoQueryInPage = async (
|
|||||||
}
|
}
|
||||||
|
|
||||||
const initialSnapshot = snapshotConversation(composerTop)
|
const initialSnapshot = snapshotConversation(composerTop)
|
||||||
|
let questionAnchorFoundAfterSubmit = Boolean(findQuestionAnchor())
|
||||||
|
let historySurfaceDetected = looksLikeHistoryTitleList(bodyText())
|
||||||
let bestSnapshot = initialSnapshot
|
let bestSnapshot = initialSnapshot
|
||||||
let bestContentSnapshot: ConversationSnapshot | null = hasObservedContent(initialSnapshot)
|
let bestContentSnapshot: ConversationSnapshot | null = hasObservedContent(initialSnapshot)
|
||||||
? initialSnapshot
|
? initialSnapshot
|
||||||
@@ -3121,10 +3207,13 @@ const doubaoQueryInPage = async (
|
|||||||
let stableCount = submitQuestion ? 0 : hasObservedContent(initialSnapshot) ? 1 : 0
|
let stableCount = submitQuestion ? 0 : hasObservedContent(initialSnapshot) ? 1 : 0
|
||||||
let sawChange = !submitQuestion && hasObservedContent(initialSnapshot)
|
let sawChange = !submitQuestion && hasObservedContent(initialSnapshot)
|
||||||
let firstChangedContentAt: number | null = sawChange ? Date.now() : null
|
let firstChangedContentAt: number | null = sawChange ? Date.now() : null
|
||||||
|
const submittedAt = Date.now()
|
||||||
const deadline = Date.now() + timeoutMs
|
const deadline = Date.now() + timeoutMs
|
||||||
|
|
||||||
while (Date.now() <= deadline) {
|
while (Date.now() <= deadline) {
|
||||||
const snapshot = snapshotConversation(composerTop)
|
const snapshot = snapshotConversation(composerTop)
|
||||||
|
questionAnchorFoundAfterSubmit = questionAnchorFoundAfterSubmit || Boolean(findQuestionAnchor())
|
||||||
|
historySurfaceDetected = historySurfaceDetected || looksLikeHistoryTitleList(bodyText())
|
||||||
bestSnapshot = pickBetterSnapshot(bestSnapshot, snapshot)
|
bestSnapshot = pickBetterSnapshot(bestSnapshot, snapshot)
|
||||||
if (hasObservedContent(snapshot)) {
|
if (hasObservedContent(snapshot)) {
|
||||||
bestContentSnapshot = pickBetterSnapshot(bestContentSnapshot, snapshot)
|
bestContentSnapshot = pickBetterSnapshot(bestContentSnapshot, snapshot)
|
||||||
@@ -3152,6 +3241,53 @@ const doubaoQueryInPage = async (
|
|||||||
composerTop,
|
composerTop,
|
||||||
answerMode,
|
answerMode,
|
||||||
bestContentSnapshot ?? bestSnapshot,
|
bestContentSnapshot ?? bestSnapshot,
|
||||||
|
{
|
||||||
|
submitted: submitQuestion,
|
||||||
|
questionAnchorFound: questionAnchorFoundAfterSubmit,
|
||||||
|
historySurfaceDetected,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const elapsedSinceSubmit = Date.now() - submittedAt
|
||||||
|
if (
|
||||||
|
submitQuestion &&
|
||||||
|
!questionAnchorFoundAfterSubmit &&
|
||||||
|
!busy &&
|
||||||
|
elapsedSinceSubmit >= submissionObserveTimeoutMs
|
||||||
|
) {
|
||||||
|
return fail(
|
||||||
|
'doubao_submission_not_observed',
|
||||||
|
bodyText(),
|
||||||
|
composerTop,
|
||||||
|
answerMode,
|
||||||
|
bestContentSnapshot ?? bestSnapshot,
|
||||||
|
{
|
||||||
|
submitted: true,
|
||||||
|
questionAnchorFound: false,
|
||||||
|
historySurfaceDetected,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
submitQuestion &&
|
||||||
|
questionAnchorFoundAfterSubmit &&
|
||||||
|
historySurfaceDetected &&
|
||||||
|
!hasContent &&
|
||||||
|
!busy &&
|
||||||
|
elapsedSinceSubmit >= staleHistoryTimeoutMs
|
||||||
|
) {
|
||||||
|
return fail(
|
||||||
|
'doubao_history_surface_timeout',
|
||||||
|
bodyText(),
|
||||||
|
composerTop,
|
||||||
|
answerMode,
|
||||||
|
bestContentSnapshot ?? bestSnapshot,
|
||||||
|
{
|
||||||
|
submitted: true,
|
||||||
|
questionAnchorFound: true,
|
||||||
|
historySurfaceDetected: true,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3196,6 +3332,11 @@ const doubaoQueryInPage = async (
|
|||||||
composerTop,
|
composerTop,
|
||||||
answerMode,
|
answerMode,
|
||||||
bestContentSnapshot ?? bestSnapshot,
|
bestContentSnapshot ?? bestSnapshot,
|
||||||
|
{
|
||||||
|
submitted: submitQuestion,
|
||||||
|
questionAnchorFound: questionAnchorFoundAfterSubmit,
|
||||||
|
historySurfaceDetected,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3214,14 +3355,11 @@ export const doubaoAdapter: MonitorAdapter = {
|
|||||||
const questionText = extractQuestionText(payload)
|
const questionText = extractQuestionText(payload)
|
||||||
const page = context.playwright.page
|
const page = context.playwright.page
|
||||||
|
|
||||||
context.reportProgress('doubao.page_ready')
|
const runAttempt = async (attempt: number): Promise<DoubaoQueryAttemptResult> => {
|
||||||
await ensurePageOnDoubaoChat(page, context.signal)
|
context.reportProgress(attempt === 1 ? 'doubao.query' : 'doubao.query_retry')
|
||||||
|
|
||||||
const capture = attachDoubaoResponseCapture(page)
|
const capture = attachDoubaoResponseCapture(page)
|
||||||
let pageResult: DoubaoPageQueryResult
|
let pageResult: DoubaoPageQueryResult
|
||||||
|
|
||||||
try {
|
try {
|
||||||
context.reportProgress('doubao.query')
|
|
||||||
try {
|
try {
|
||||||
pageResult = await page.evaluate(doubaoQueryInPage, {
|
pageResult = await page.evaluate(doubaoQueryInPage, {
|
||||||
questionText,
|
questionText,
|
||||||
@@ -3244,10 +3382,39 @@ export const doubaoAdapter: MonitorAdapter = {
|
|||||||
} finally {
|
} finally {
|
||||||
capture.dispose()
|
capture.dispose()
|
||||||
}
|
}
|
||||||
|
|
||||||
const streamSummary = parseDoubaoCaptures(capture.captures)
|
const streamSummary = parseDoubaoCaptures(capture.captures)
|
||||||
const captureDebug = summarizeCapturesForDebug(capture.captures)
|
return {
|
||||||
const pendingCaptureCount = capture.pendingCount()
|
attempt,
|
||||||
|
pageResult,
|
||||||
|
captures: [...capture.captures],
|
||||||
|
streamSummary,
|
||||||
|
captureDebug: summarizeCapturesForDebug(capture.captures),
|
||||||
|
pendingCaptureCount: capture.pendingCount(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
context.reportProgress('doubao.page_ready')
|
||||||
|
await ensurePageOnDoubaoChat(page, context.signal)
|
||||||
|
|
||||||
|
let attemptResult = await runAttempt(1)
|
||||||
|
if (
|
||||||
|
attemptResult.pageResult.ok === false &&
|
||||||
|
shouldRetryDoubaoPageAttempt(attemptResult.pageResult, attemptResult.streamSummary)
|
||||||
|
) {
|
||||||
|
context.reportProgress('doubao.recover_new_chat')
|
||||||
|
await resetDoubaoPageAfterRecoverableFailure(
|
||||||
|
page,
|
||||||
|
attemptResult.pageResult.error,
|
||||||
|
context.signal,
|
||||||
|
)
|
||||||
|
await ensurePageOnDoubaoChat(page, context.signal)
|
||||||
|
attemptResult = await runAttempt(2)
|
||||||
|
}
|
||||||
|
|
||||||
|
const pageResult = attemptResult.pageResult
|
||||||
|
const streamSummary = attemptResult.streamSummary
|
||||||
|
const captureDebug = attemptResult.captureDebug
|
||||||
|
const pendingCaptureCount = attemptResult.pendingCaptureCount
|
||||||
const domSearchResults = dedupeSourceItems(
|
const domSearchResults = dedupeSourceItems(
|
||||||
pageResult.domLinks
|
pageResult.domLinks
|
||||||
.map((item) =>
|
.map((item) =>
|
||||||
@@ -3316,6 +3483,11 @@ export const doubaoAdapter: MonitorAdapter = {
|
|||||||
reply_end_event_count: streamSummary.reply_end_event_count,
|
reply_end_event_count: streamSummary.reply_end_event_count,
|
||||||
answer_finish_event_count: streamSummary.answer_finish_event_count,
|
answer_finish_event_count: streamSummary.answer_finish_event_count,
|
||||||
stream_event_count: streamSummary.event_count,
|
stream_event_count: streamSummary.event_count,
|
||||||
|
submitted: failed.submitted,
|
||||||
|
question_anchor_found: failed.questionAnchorFound,
|
||||||
|
history_surface_detected: failed.historySurfaceDetected,
|
||||||
|
query_attempt: attemptResult.attempt,
|
||||||
|
captures: captureDebug,
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3426,11 +3598,12 @@ export const doubaoAdapter: MonitorAdapter = {
|
|||||||
platform: 'doubao',
|
platform: 'doubao',
|
||||||
page_url: pageResult.url,
|
page_url: pageResult.url,
|
||||||
page_title: pageResult.title,
|
page_title: pageResult.title,
|
||||||
|
query_attempt: attemptResult.attempt,
|
||||||
model_label: pageResult.modelLabel,
|
model_label: pageResult.modelLabel,
|
||||||
mode_label: pageResult.modeLabel,
|
mode_label: pageResult.modeLabel,
|
||||||
thinking_mode_applied: pageResult.thinkingModeApplied,
|
thinking_mode_applied: pageResult.thinkingModeApplied,
|
||||||
deep_think_enabled: pageResult.deepThinkEnabled,
|
deep_think_enabled: pageResult.deepThinkEnabled,
|
||||||
capture_count: capture.captures.length,
|
capture_count: attemptResult.captures.length,
|
||||||
capture_pending_after_flush: pendingCaptureCount,
|
capture_pending_after_flush: pendingCaptureCount,
|
||||||
event_count: streamSummary.event_count,
|
event_count: streamSummary.event_count,
|
||||||
full_message_event_count: streamSummary.full_message_event_count,
|
full_message_event_count: streamSummary.full_message_event_count,
|
||||||
@@ -3464,6 +3637,8 @@ export const __doubaoTestUtils = {
|
|||||||
isNonCitationAssetDomain,
|
isNonCitationAssetDomain,
|
||||||
isNonCitationAssetUrl,
|
isNonCitationAssetUrl,
|
||||||
isDoubaoRecoverablePageError,
|
isDoubaoRecoverablePageError,
|
||||||
|
isDoubaoDOMSubmissionCandidate,
|
||||||
|
shouldRetryDoubaoPageAttempt,
|
||||||
parseDoubaoStreamBody,
|
parseDoubaoStreamBody,
|
||||||
resolveDoubaoSubmissionAnswer,
|
resolveDoubaoSubmissionAnswer,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ import {
|
|||||||
installObservedGlobalFetch,
|
installObservedGlobalFetch,
|
||||||
registerObservedRequestRendererTarget,
|
registerObservedRequestRendererTarget,
|
||||||
} from './network-observer'
|
} from './network-observer'
|
||||||
import { getPlaywrightCDPPort, startHiddenPlaywrightReaper } from './playwright-cdp'
|
import { getPlaywrightCDPPort, preparePlaywrightCDPPort, startHiddenPlaywrightReaper } from './playwright-cdp'
|
||||||
import { initProcessMetricsSampler } from './process-metrics'
|
import { initProcessMetricsSampler } from './process-metrics'
|
||||||
import { registerRendererDevtoolsProxyTarget } from './renderer-devtools-proxy'
|
import { registerRendererDevtoolsProxyTarget } from './renderer-devtools-proxy'
|
||||||
import {
|
import {
|
||||||
@@ -112,8 +112,8 @@ app.commandLine.appendSwitch(
|
|||||||
)
|
)
|
||||||
app.commandLine.appendSwitch('disable-quic')
|
app.commandLine.appendSwitch('disable-quic')
|
||||||
app.commandLine.appendSwitch('disable-background-networking')
|
app.commandLine.appendSwitch('disable-background-networking')
|
||||||
// CDP is required by the hidden Playwright execution pool in packaged builds.
|
// CDP is required by the hidden Playwright execution pool. Electron only reads
|
||||||
app.commandLine.appendSwitch('remote-debugging-port', String(getPlaywrightCDPPort()))
|
// this switch during early startup, so select the port before app.whenReady().
|
||||||
app.userAgentFallback = STANDARD_USER_AGENT
|
app.userAgentFallback = STANDARD_USER_AGENT
|
||||||
|
|
||||||
// Windows 10+ routes tray.displayBalloon through the Toast Notification API,
|
// Windows 10+ routes tray.displayBalloon through the Toast Notification API,
|
||||||
@@ -151,6 +151,7 @@ console.info('[desktop-main] boot', {
|
|||||||
chrome: process.versions.chrome,
|
chrome: process.versions.chrome,
|
||||||
node: process.versions.node,
|
node: process.versions.node,
|
||||||
ua: STANDARD_USER_AGENT,
|
ua: STANDARD_USER_AGENT,
|
||||||
|
cdpPort: getPlaywrightCDPPort(),
|
||||||
})
|
})
|
||||||
|
|
||||||
process.on('unhandledRejection', (reason) => {
|
process.on('unhandledRejection', (reason) => {
|
||||||
@@ -166,6 +167,7 @@ let loginWindow: ElectronBrowserWindow | null = null
|
|||||||
let settingsWindow: ElectronBrowserWindow | null = null
|
let settingsWindow: ElectronBrowserWindow | null = null
|
||||||
let mainRendererContents: ElectronWebContents | null = null
|
let mainRendererContents: ElectronWebContents | null = null
|
||||||
let quitReleaseInFlight = false
|
let quitReleaseInFlight = false
|
||||||
|
let cdpRecoveryRelaunchInFlight = false
|
||||||
let quitAfterSaasLogoutPromise: Promise<void> | null = null
|
let quitAfterSaasLogoutPromise: Promise<void> | null = null
|
||||||
let mainWindowCreatePromise: Promise<ElectronBrowserWindow> | null = null
|
let mainWindowCreatePromise: Promise<ElectronBrowserWindow> | null = null
|
||||||
let loginWindowCreatePromise: Promise<ElectronBrowserWindow> | null = null
|
let loginWindowCreatePromise: Promise<ElectronBrowserWindow> | null = null
|
||||||
@@ -176,6 +178,10 @@ let canHandleDesktopDeepLinks = false
|
|||||||
const pendingDesktopDeepLinks: string[] = []
|
const pendingDesktopDeepLinks: string[] = []
|
||||||
const forceCloseWindows = new WeakSet<ElectronBrowserWindow>()
|
const forceCloseWindows = new WeakSet<ElectronBrowserWindow>()
|
||||||
const appWindowModes = new WeakMap<ElectronBrowserWindow, RendererWindowMode>()
|
const appWindowModes = new WeakMap<ElectronBrowserWindow, RendererWindowMode>()
|
||||||
|
const maxCDPRecoveryRelaunchesPerWindow = 3
|
||||||
|
const cdpRecoveryRelaunchWindowMs = 10 * 60_000
|
||||||
|
const cdpRecoveryRelaunchDelayMs = 1_500
|
||||||
|
const cdpFatalArg = '--geo-cdp-fatal'
|
||||||
|
|
||||||
type WindowMode = 'login' | 'main'
|
type WindowMode = 'login' | 'main'
|
||||||
type RendererWindowMode = WindowMode | 'settings'
|
type RendererWindowMode = WindowMode | 'settings'
|
||||||
@@ -547,6 +553,50 @@ function quitAfterClearingSaasSession(): void {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function requestPlaywrightCDPRecoveryRelaunch(message: string): void {
|
||||||
|
if (cdpRecoveryRelaunchInFlight) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const now = Date.now()
|
||||||
|
const recentRelaunches = process.argv
|
||||||
|
.filter((arg) => arg.startsWith('--geo-cdp-recovery-at='))
|
||||||
|
.map((arg) => Number.parseInt(arg.split('=')[1] ?? '', 10))
|
||||||
|
.filter((timestamp) => Number.isFinite(timestamp) && now - timestamp < cdpRecoveryRelaunchWindowMs)
|
||||||
|
|
||||||
|
if (recentRelaunches.length >= maxCDPRecoveryRelaunchesPerWindow) {
|
||||||
|
console.error('[desktop-main] playwright cdp recovery relaunch suppressed', {
|
||||||
|
message,
|
||||||
|
recentRelaunches: recentRelaunches.length,
|
||||||
|
})
|
||||||
|
showTrayBalloon(
|
||||||
|
'浏览器自动化通道异常',
|
||||||
|
'客户端已停止采集。请重启客户端或检查安全软件/端口占用。',
|
||||||
|
)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cdpRecoveryRelaunchInFlight = true
|
||||||
|
console.warn('[desktop-main] scheduling playwright cdp recovery relaunch', {
|
||||||
|
message,
|
||||||
|
cdpPort: getPlaywrightCDPPort(),
|
||||||
|
})
|
||||||
|
showTrayBalloon('正在恢复浏览器自动化通道', '客户端将自动重启后台进程,登录状态会保留。')
|
||||||
|
|
||||||
|
setTimeout(() => {
|
||||||
|
quitReleaseInFlight = true
|
||||||
|
app.relaunch({
|
||||||
|
args: [
|
||||||
|
...process.argv.slice(1).filter((arg) => !arg.startsWith('--geo-cdp-recovery-at=')),
|
||||||
|
...recentRelaunches.map((timestamp) => `--geo-cdp-recovery-at=${timestamp}`),
|
||||||
|
`--geo-cdp-recovery-at=${now}`,
|
||||||
|
...(recentRelaunches.length + 1 >= maxCDPRecoveryRelaunchesPerWindow ? [cdpFatalArg] : []),
|
||||||
|
],
|
||||||
|
})
|
||||||
|
app.exit(0)
|
||||||
|
}, cdpRecoveryRelaunchDelayMs)
|
||||||
|
}
|
||||||
|
|
||||||
function windowFromIpcEvent(event: IpcMainInvokeEvent): ElectronBrowserWindow | null {
|
function windowFromIpcEvent(event: IpcMainInvokeEvent): ElectronBrowserWindow | null {
|
||||||
const window = BrowserWindow.fromWebContents(event.sender)
|
const window = BrowserWindow.fromWebContents(event.sender)
|
||||||
if (!window || window.isDestroyed()) {
|
if (!window || window.isDestroyed()) {
|
||||||
@@ -1197,9 +1247,15 @@ if (!hasSingleInstanceLock) {
|
|||||||
console.info('[desktop-main] another instance is already running; exiting')
|
console.info('[desktop-main] another instance is already running; exiting')
|
||||||
app.exit(0)
|
app.exit(0)
|
||||||
} else {
|
} else {
|
||||||
app
|
preparePlaywrightCDPPort()
|
||||||
.whenReady()
|
.then((selectedPlaywrightCDPPort) => {
|
||||||
|
app.commandLine.appendSwitch('remote-debugging-port', String(selectedPlaywrightCDPPort))
|
||||||
|
return app.whenReady()
|
||||||
|
})
|
||||||
.then(async () => {
|
.then(async () => {
|
||||||
|
console.info('[desktop-main] playwright cdp port selected', {
|
||||||
|
cdpPort: getPlaywrightCDPPort(),
|
||||||
|
})
|
||||||
suppressNativeWindowMenu()
|
suppressNativeWindowMenu()
|
||||||
registerDesktopProtocolClient()
|
registerDesktopProtocolClient()
|
||||||
currentWindowMode = readPersistedWindowMode()
|
currentWindowMode = readPersistedWindowMode()
|
||||||
@@ -1215,6 +1271,15 @@ if (!hasSingleInstanceLock) {
|
|||||||
startHiddenPlaywrightReaper()
|
startHiddenPlaywrightReaper()
|
||||||
onRuntimeInvalidated((event) => {
|
onRuntimeInvalidated((event) => {
|
||||||
syncDesktopHealthIndicator(currentMainWindow())
|
syncDesktopHealthIndicator(currentMainWindow())
|
||||||
|
if (event.reason === 'playwright-cdp-fatal') {
|
||||||
|
showTrayBalloon(
|
||||||
|
'浏览器自动化组件不可用',
|
||||||
|
event.message ?? '客户端已停止采集,请重启客户端或检查安全软件/端口占用。',
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if (event.reason === 'playwright-cdp-recovery') {
|
||||||
|
requestPlaywrightCDPRecoveryRelaunch(event.message ?? 'playwright_cdp_endpoint_unavailable')
|
||||||
|
}
|
||||||
if (!mainRendererContents || mainRendererContents.isDestroyed()) {
|
if (!mainRendererContents || mainRendererContents.isDestroyed()) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -1234,7 +1299,7 @@ if (!hasSingleInstanceLock) {
|
|||||||
flushPendingDesktopDeepLinks()
|
flushPendingDesktopDeepLinks()
|
||||||
})
|
})
|
||||||
.catch((error) => {
|
.catch((error) => {
|
||||||
console.error('[desktop-main] app.whenReady failed', error)
|
console.error('[desktop-main] startup failed', error)
|
||||||
})
|
})
|
||||||
|
|
||||||
app.on('activate', () => {
|
app.on('activate', () => {
|
||||||
|
|||||||
@@ -64,6 +64,7 @@ interface MonitorSchedulerSelectionOptions {
|
|||||||
maxConcurrency: number
|
maxConcurrency: number
|
||||||
activePlatforms: ReadonlySet<string>
|
activePlatforms: ReadonlySet<string>
|
||||||
activeQuestionKeys: ReadonlySet<string>
|
activeQuestionKeys: ReadonlySet<string>
|
||||||
|
blockedPlatforms?: ReadonlySet<string>
|
||||||
activeCount: number
|
activeCount: number
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -434,6 +435,22 @@ export function noteMonitorTaskLeaseMiss(taskId: string): void {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function noteMonitorTaskDeferred(taskId: string, delayMs: number): void {
|
||||||
|
const now = Date.now()
|
||||||
|
const availableAt = now + Math.max(0, delayMs)
|
||||||
|
|
||||||
|
mutatePersistedState((draft) => {
|
||||||
|
const existing = draft.tasks[taskId]
|
||||||
|
if (!existing) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
existing.state = 'queued'
|
||||||
|
existing.availableAt = Math.max(existing.availableAt, availableAt)
|
||||||
|
existing.lastSeenAt = now
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
export function noteMonitorTaskCompleted(taskId: string): void {
|
export function noteMonitorTaskCompleted(taskId: string): void {
|
||||||
const now = Date.now()
|
const now = Date.now()
|
||||||
|
|
||||||
@@ -488,6 +505,10 @@ export function selectNextMonitorTask(
|
|||||||
})
|
})
|
||||||
|
|
||||||
for (const candidate of candidates) {
|
for (const candidate of candidates) {
|
||||||
|
if (options.blockedPlatforms?.has(candidate.platform)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
if (options.activePlatforms.has(candidate.platform)) {
|
if (options.activePlatforms.has(candidate.platform)) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import { createServer } from 'node:net'
|
||||||
|
|
||||||
import type { BrowserWindow as ElectronBrowserWindow, Session } from 'electron/main'
|
import type { BrowserWindow as ElectronBrowserWindow, Session } from 'electron/main'
|
||||||
import { BrowserWindow, nativeTheme } from 'electron/main'
|
import { BrowserWindow, nativeTheme } from 'electron/main'
|
||||||
import type { Browser as PlaywrightBrowser, Page as PlaywrightPage } from 'playwright-core'
|
import type { Browser as PlaywrightBrowser, Page as PlaywrightPage } from 'playwright-core'
|
||||||
@@ -13,9 +15,23 @@ const hiddenPlaywrightReaperIntervalMs = 60_000
|
|||||||
const defaultConnectTimeoutMs = 10_000
|
const defaultConnectTimeoutMs = 10_000
|
||||||
const defaultPageTimeoutMs = 45_000
|
const defaultPageTimeoutMs = 45_000
|
||||||
const hiddenBootstrapMetaName = 'geo-hidden-playwright-token'
|
const hiddenBootstrapMetaName = 'geo-hidden-playwright-token'
|
||||||
const cdpPort = Number.parseInt(process.env.GEO_ELECTRON_CDP_PORT ?? '9339', 10)
|
const preferredCDPPort = normalizeCDPPort(process.env.GEO_ELECTRON_CDP_PORT, 9339)
|
||||||
const cdpEndpointURL = `http://127.0.0.1:${cdpPort}`
|
const dynamicCDPPortMin = normalizeCDPPort(process.env.GEO_ELECTRON_CDP_PORT_MIN, 9339)
|
||||||
const cdpTargetsListURL = `${cdpEndpointURL}/json/list`
|
const dynamicCDPPortMax = normalizeCDPPort(process.env.GEO_ELECTRON_CDP_PORT_MAX, 20000)
|
||||||
|
const cdpCircuitOpenTTLms = 60_000
|
||||||
|
const cdpCircuitProbeTimeoutMs = 2_000
|
||||||
|
|
||||||
|
let cdpPort = preferredCDPPort
|
||||||
|
let cdpEndpointURL = endpointURLForPort(cdpPort)
|
||||||
|
let cdpTargetsListURL = `${cdpEndpointURL}/json/list`
|
||||||
|
|
||||||
|
interface PlaywrightCDPHealthState {
|
||||||
|
status: 'unknown' | 'healthy' | 'unavailable'
|
||||||
|
checkedAt: number | null
|
||||||
|
unavailableSince: number | null
|
||||||
|
nextRetryAt: number | null
|
||||||
|
lastError: string | null
|
||||||
|
}
|
||||||
|
|
||||||
interface HiddenPlaywrightRecord {
|
interface HiddenPlaywrightRecord {
|
||||||
key: string
|
key: string
|
||||||
@@ -41,11 +57,95 @@ const records = new Map<string, HiddenPlaywrightRecord>()
|
|||||||
let reaperHandle: ReturnType<typeof setInterval> | null = null
|
let reaperHandle: ReturnType<typeof setInterval> | null = null
|
||||||
let connectPromise: Promise<PlaywrightBrowser> | null = null
|
let connectPromise: Promise<PlaywrightBrowser> | null = null
|
||||||
let connectedBrowser: PlaywrightBrowser | null = null
|
let connectedBrowser: PlaywrightBrowser | null = null
|
||||||
|
let cdpHealthState: PlaywrightCDPHealthState = {
|
||||||
|
status: 'unknown',
|
||||||
|
checkedAt: null,
|
||||||
|
unavailableSince: null,
|
||||||
|
nextRetryAt: null,
|
||||||
|
lastError: null,
|
||||||
|
}
|
||||||
|
|
||||||
export function getPlaywrightCDPPort(): number {
|
export function getPlaywrightCDPPort(): number {
|
||||||
return cdpPort
|
return cdpPort
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function preparePlaywrightCDPPort(): Promise<number> {
|
||||||
|
const port = await findAvailableCDPPort()
|
||||||
|
setPlaywrightCDPPort(port)
|
||||||
|
return port
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isPlaywrightCDPUnavailableError(error: unknown): boolean {
|
||||||
|
if (!(error instanceof Error)) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
error.message === 'playwright_cdp_endpoint_unavailable' ||
|
||||||
|
error.message === 'playwright_cdp_port_unavailable' ||
|
||||||
|
error.message.includes('playwright_cdp_endpoint_unavailable') ||
|
||||||
|
error.message.includes('connectOverCDP')
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function ensurePlaywrightCDPReady(options: { force?: boolean } = {}): Promise<void> {
|
||||||
|
const now = Date.now()
|
||||||
|
if (
|
||||||
|
!options.force &&
|
||||||
|
cdpHealthState.status === 'healthy' &&
|
||||||
|
cdpHealthState.checkedAt &&
|
||||||
|
now - cdpHealthState.checkedAt < 15_000
|
||||||
|
) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
!options.force &&
|
||||||
|
cdpHealthState.status === 'unavailable' &&
|
||||||
|
cdpHealthState.nextRetryAt &&
|
||||||
|
cdpHealthState.nextRetryAt > now
|
||||||
|
) {
|
||||||
|
throw new Error(cdpHealthState.lastError || 'playwright_cdp_endpoint_unavailable')
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await waitForCDPEndpoint(cdpCircuitProbeTimeoutMs)
|
||||||
|
markPlaywrightCDPHealthy()
|
||||||
|
} catch (error) {
|
||||||
|
markPlaywrightCDPUnavailable(error)
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isPlaywrightCDPExecutionAvailable(now = Date.now()): boolean {
|
||||||
|
if (connectedBrowser?.isConnected()) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if (cdpHealthState.status !== 'unavailable') {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return Boolean(cdpHealthState.nextRetryAt && cdpHealthState.nextRetryAt <= now)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getHiddenPlaywrightSnapshot() {
|
||||||
|
return {
|
||||||
|
cdpPort,
|
||||||
|
cdpEndpointURL,
|
||||||
|
connected: Boolean(connectedBrowser?.isConnected()),
|
||||||
|
pageCount: records.size,
|
||||||
|
idleTTLms: hiddenPlaywrightIdleTTLms,
|
||||||
|
health: { ...cdpHealthState },
|
||||||
|
records: [...records.values()].map((record) => ({
|
||||||
|
accountId: record.accountId,
|
||||||
|
title: record.title,
|
||||||
|
targetURL: record.targetURL,
|
||||||
|
retainCount: record.retainCount,
|
||||||
|
activatedAt: record.activatedAt,
|
||||||
|
lastReleasedAt: record.lastReleasedAt,
|
||||||
|
windowDestroyed: record.window.isDestroyed(),
|
||||||
|
pageClosed: record.page.isClosed(),
|
||||||
|
})),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export function startHiddenPlaywrightReaper(): void {
|
export function startHiddenPlaywrightReaper(): void {
|
||||||
if (reaperHandle) {
|
if (reaperHandle) {
|
||||||
return
|
return
|
||||||
@@ -155,26 +255,6 @@ export function reapIdleHiddenPlaywrightPages(
|
|||||||
return reclaimed
|
return reclaimed
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getHiddenPlaywrightSnapshot() {
|
|
||||||
return {
|
|
||||||
cdpPort,
|
|
||||||
cdpEndpointURL,
|
|
||||||
connected: Boolean(connectedBrowser?.isConnected()),
|
|
||||||
pageCount: records.size,
|
|
||||||
idleTTLms: hiddenPlaywrightIdleTTLms,
|
|
||||||
records: [...records.values()].map((record) => ({
|
|
||||||
accountId: record.accountId,
|
|
||||||
title: record.title,
|
|
||||||
targetURL: record.targetURL,
|
|
||||||
retainCount: record.retainCount,
|
|
||||||
activatedAt: record.activatedAt,
|
|
||||||
lastReleasedAt: record.lastReleasedAt,
|
|
||||||
windowDestroyed: record.window.isDestroyed(),
|
|
||||||
pageClosed: record.page.isClosed(),
|
|
||||||
})),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function createLease(record: HiddenPlaywrightRecord): HiddenPlaywrightLease {
|
function createLease(record: HiddenPlaywrightRecord): HiddenPlaywrightLease {
|
||||||
return {
|
return {
|
||||||
accountId: record.accountId,
|
accountId: record.accountId,
|
||||||
@@ -271,6 +351,7 @@ async function connectBrowserOverCDP(): Promise<PlaywrightBrowser> {
|
|||||||
|
|
||||||
connectPromise = (async () => {
|
connectPromise = (async () => {
|
||||||
await waitForCDPEndpoint()
|
await waitForCDPEndpoint()
|
||||||
|
markPlaywrightCDPHealthy()
|
||||||
const { chromium } = await import('playwright-core')
|
const { chromium } = await import('playwright-core')
|
||||||
await pruneStaleCDPTargets()
|
await pruneStaleCDPTargets()
|
||||||
|
|
||||||
@@ -292,6 +373,9 @@ async function connectBrowserOverCDP(): Promise<PlaywrightBrowser> {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
return await connectPromise
|
return await connectPromise
|
||||||
|
} catch (error) {
|
||||||
|
markPlaywrightCDPUnavailable(error)
|
||||||
|
throw error
|
||||||
} finally {
|
} finally {
|
||||||
if (!connectedBrowser?.isConnected()) {
|
if (!connectedBrowser?.isConnected()) {
|
||||||
connectPromise = null
|
connectPromise = null
|
||||||
@@ -314,7 +398,7 @@ async function waitForCDPEndpoint(timeoutMs = defaultConnectTimeoutMs): Promise<
|
|||||||
await wait(200)
|
await wait(200)
|
||||||
}
|
}
|
||||||
|
|
||||||
throw new Error('playwright_cdp_endpoint_unavailable')
|
throw new Error(`playwright_cdp_endpoint_unavailable:${cdpEndpointURL}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
async function openBrowserOverCDP(
|
async function openBrowserOverCDP(
|
||||||
@@ -330,6 +414,7 @@ async function openBrowserOverCDP(
|
|||||||
})
|
})
|
||||||
|
|
||||||
connectedBrowser = browser
|
connectedBrowser = browser
|
||||||
|
markPlaywrightCDPHealthy()
|
||||||
return browser
|
return browser
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -468,3 +553,104 @@ function wait(timeMs: number): Promise<void> {
|
|||||||
setTimeout(resolve, timeMs)
|
setTimeout(resolve, timeMs)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function normalizeCDPPort(value: string | undefined, fallback: number): number {
|
||||||
|
const parsed = Number.parseInt(value ?? '', 10)
|
||||||
|
if (!Number.isFinite(parsed) || parsed <= 0 || parsed > 65_535) {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return parsed
|
||||||
|
}
|
||||||
|
|
||||||
|
function endpointURLForPort(port: number): string {
|
||||||
|
return `http://127.0.0.1:${port}`
|
||||||
|
}
|
||||||
|
|
||||||
|
function setPlaywrightCDPPort(port: number): void {
|
||||||
|
cdpPort = port
|
||||||
|
cdpEndpointURL = endpointURLForPort(port)
|
||||||
|
cdpTargetsListURL = `${cdpEndpointURL}/json/list`
|
||||||
|
}
|
||||||
|
|
||||||
|
async function findAvailableCDPPort(): Promise<number> {
|
||||||
|
if (await canBindPort(preferredCDPPort)) {
|
||||||
|
return preferredCDPPort
|
||||||
|
}
|
||||||
|
|
||||||
|
const min = Math.min(dynamicCDPPortMin, dynamicCDPPortMax)
|
||||||
|
const max = Math.max(dynamicCDPPortMin, dynamicCDPPortMax)
|
||||||
|
const start = Math.max(min, preferredCDPPort)
|
||||||
|
for (let port = start; port <= max; port += 1) {
|
||||||
|
if (port === preferredCDPPort) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (await canBindPort(port)) {
|
||||||
|
return port
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let port = min; port < start; port += 1) {
|
||||||
|
if (port === preferredCDPPort) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (await canBindPort(port)) {
|
||||||
|
return port
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const reserved = await reserveEphemeralPort()
|
||||||
|
return reserved
|
||||||
|
}
|
||||||
|
|
||||||
|
function canBindPort(port: number): Promise<boolean> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const server = createServer()
|
||||||
|
server.once('error', () => {
|
||||||
|
resolve(false)
|
||||||
|
})
|
||||||
|
server.listen({ host: '127.0.0.1', port }, () => {
|
||||||
|
server.close(() => {
|
||||||
|
resolve(true)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
function reserveEphemeralPort(): Promise<number> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const server = createServer()
|
||||||
|
server.once('error', reject)
|
||||||
|
server.listen({ host: '127.0.0.1', port: 0 }, () => {
|
||||||
|
const address = server.address()
|
||||||
|
const port = typeof address === 'object' && address ? address.port : 0
|
||||||
|
server.close(() => {
|
||||||
|
if (port > 0) {
|
||||||
|
resolve(port)
|
||||||
|
} else {
|
||||||
|
reject(new Error('playwright_cdp_port_unavailable'))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
function markPlaywrightCDPHealthy(): void {
|
||||||
|
cdpHealthState = {
|
||||||
|
status: 'healthy',
|
||||||
|
checkedAt: Date.now(),
|
||||||
|
unavailableSince: null,
|
||||||
|
nextRetryAt: null,
|
||||||
|
lastError: null,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function markPlaywrightCDPUnavailable(error: unknown): void {
|
||||||
|
const now = Date.now()
|
||||||
|
cdpHealthState = {
|
||||||
|
status: 'unavailable',
|
||||||
|
checkedAt: now,
|
||||||
|
unavailableSince: cdpHealthState.unavailableSince ?? now,
|
||||||
|
nextRetryAt: now + cdpCircuitOpenTTLms,
|
||||||
|
lastError: error instanceof Error ? error.message : String(error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ interface PublishSchedulerSelectionOptions {
|
|||||||
now?: number
|
now?: number
|
||||||
maxConcurrency: number
|
maxConcurrency: number
|
||||||
activePlatforms?: ReadonlySet<string>
|
activePlatforms?: ReadonlySet<string>
|
||||||
|
blockedPlatforms?: ReadonlySet<string>
|
||||||
activeCount: number
|
activeCount: number
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -97,6 +98,10 @@ export function selectNextPublishTask(
|
|||||||
})
|
})
|
||||||
|
|
||||||
for (const candidate of candidates) {
|
for (const candidate of candidates) {
|
||||||
|
if (options.blockedPlatforms?.has(candidate.platform)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
if (activePlatforms.has(candidate.platform)) {
|
if (activePlatforms.has(candidate.platform)) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -143,6 +148,19 @@ export function notePublishTaskLeaseMiss(taskId: string): void {
|
|||||||
tasks.delete(taskId)
|
tasks.delete(taskId)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function notePublishTaskDeferred(taskId: string): void {
|
||||||
|
const existing = tasks.get(taskId)
|
||||||
|
if (!existing) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
tasks.set(taskId, {
|
||||||
|
...existing,
|
||||||
|
state: 'queued',
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
export function notePublishTaskCompleted(taskId: string, platform?: string): void {
|
export function notePublishTaskCompleted(taskId: string, platform?: string): void {
|
||||||
const existing = tasks.get(taskId)
|
const existing = tasks.get(taskId)
|
||||||
const resolvedPlatform = (platform ?? existing?.platform ?? '').trim()
|
const resolvedPlatform = (platform ?? existing?.platform ?? '').trim()
|
||||||
|
|||||||
@@ -68,10 +68,17 @@ import {
|
|||||||
noteMonitorTaskActivated,
|
noteMonitorTaskActivated,
|
||||||
noteMonitorTaskCanceled,
|
noteMonitorTaskCanceled,
|
||||||
noteMonitorTaskCompleted,
|
noteMonitorTaskCompleted,
|
||||||
|
noteMonitorTaskDeferred,
|
||||||
noteMonitorTaskLeased,
|
noteMonitorTaskLeased,
|
||||||
selectNextMonitorTask,
|
selectNextMonitorTask,
|
||||||
} from './monitor-scheduler'
|
} from './monitor-scheduler'
|
||||||
import { retainHiddenPlaywrightPage, startHiddenPlaywrightReaper } from './playwright-cdp'
|
import {
|
||||||
|
ensurePlaywrightCDPReady,
|
||||||
|
isPlaywrightCDPExecutionAvailable,
|
||||||
|
isPlaywrightCDPUnavailableError,
|
||||||
|
retainHiddenPlaywrightPage,
|
||||||
|
startHiddenPlaywrightReaper,
|
||||||
|
} from './playwright-cdp'
|
||||||
import { getProcessMetricsSnapshot } from './process-metrics'
|
import { getProcessMetricsSnapshot } from './process-metrics'
|
||||||
import {
|
import {
|
||||||
enqueuePublishTask as enqueuePublishSchedulerTask,
|
enqueuePublishTask as enqueuePublishSchedulerTask,
|
||||||
@@ -80,6 +87,7 @@ import {
|
|||||||
notePublishTaskActivated,
|
notePublishTaskActivated,
|
||||||
notePublishTaskCanceled,
|
notePublishTaskCanceled,
|
||||||
notePublishTaskCompleted,
|
notePublishTaskCompleted,
|
||||||
|
notePublishTaskDeferred,
|
||||||
notePublishTaskLeaseMiss,
|
notePublishTaskLeaseMiss,
|
||||||
selectNextPublishTask,
|
selectNextPublishTask,
|
||||||
} from './publish-scheduler'
|
} from './publish-scheduler'
|
||||||
@@ -102,6 +110,7 @@ import {
|
|||||||
clearSessionHandle,
|
clearSessionHandle,
|
||||||
createSessionHandle,
|
createSessionHandle,
|
||||||
hasLocalSessionPartition,
|
hasLocalSessionPartition,
|
||||||
|
type SessionHandle,
|
||||||
} from './session-registry'
|
} from './session-registry'
|
||||||
import {
|
import {
|
||||||
cancelDesktopTask,
|
cancelDesktopTask,
|
||||||
@@ -148,6 +157,8 @@ const heartbeatIntervalMs = 15_000
|
|||||||
const pullIntervalMs = 60_000
|
const pullIntervalMs = 60_000
|
||||||
const publishPullIntervalMs = 30_000
|
const publishPullIntervalMs = 30_000
|
||||||
const monitorPullFallbackIntervalMs = 10_000
|
const monitorPullFallbackIntervalMs = 10_000
|
||||||
|
const playwrightCDPPreflightDeferMs = 15_000
|
||||||
|
const playwrightCDPRecoveryActivityCooldownMs = 60_000
|
||||||
const accountSyncIntervalMs = 30_000
|
const accountSyncIntervalMs = 30_000
|
||||||
const pumpWatchdogIntervalMs = 30_000
|
const pumpWatchdogIntervalMs = 30_000
|
||||||
const leaseExtendIntervalMs = 60_000
|
const leaseExtendIntervalMs = 60_000
|
||||||
@@ -163,6 +174,8 @@ const maximumRuntimeTotalConcurrency = 4
|
|||||||
const BAIJIAHAO_RISK_CONTROL_PROMPT = '您触发平台风控,请手动发稿一篇并完成验证,方可继续使用'
|
const BAIJIAHAO_RISK_CONTROL_PROMPT = '您触发平台风控,请手动发稿一篇并完成验证,方可继续使用'
|
||||||
const accountHealthReportDebounceMs = 1_000
|
const accountHealthReportDebounceMs = 1_000
|
||||||
const accountHealthReportMaxBatchSize = 100
|
const accountHealthReportMaxBatchSize = 100
|
||||||
|
const maxPlaywrightCDPSoftRecoveries = 3
|
||||||
|
const playwrightCDPFatalMarker = '--geo-cdp-fatal'
|
||||||
|
|
||||||
interface RuntimeTaskRecord {
|
interface RuntimeTaskRecord {
|
||||||
id: string
|
id: string
|
||||||
@@ -312,6 +325,18 @@ const state: RuntimeState = {
|
|||||||
activitySeq: 0,
|
activitySeq: 0,
|
||||||
}
|
}
|
||||||
let accountHealthReportBridgeUnsubscribe: (() => void) | null = null
|
let accountHealthReportBridgeUnsubscribe: (() => void) | null = null
|
||||||
|
let lastPlaywrightCDPRecoveryActivityAt = 0
|
||||||
|
let playwrightCDPRecoveryAttemptCount = process.argv.filter((arg) =>
|
||||||
|
arg.startsWith('--geo-cdp-recovery-at='),
|
||||||
|
).length
|
||||||
|
let playwrightCDPFatal = process.argv.includes(playwrightCDPFatalMarker)
|
||||||
|
|
||||||
|
class PlaywrightCDPInfrastructureError extends Error {
|
||||||
|
constructor(message: string) {
|
||||||
|
super(message)
|
||||||
|
this.name = 'PlaywrightCDPInfrastructureError'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function accountIdentityFromDesktopAccount(account: DesktopAccountInfo): PublishAccountIdentity {
|
function accountIdentityFromDesktopAccount(account: DesktopAccountInfo): PublishAccountIdentity {
|
||||||
return {
|
return {
|
||||||
@@ -703,6 +728,17 @@ function startRuntime(): void {
|
|||||||
if (!canRunLive(state.session) || state.running) {
|
if (!canRunLive(state.session) || state.running) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if (playwrightCDPFatal) {
|
||||||
|
const message = '浏览器自动化通道不可用,客户端已停止采集。请重启客户端或检查安全软件/端口占用。'
|
||||||
|
state.lastError = message
|
||||||
|
setSchedulerPhase('paused')
|
||||||
|
setSchedulerError(message)
|
||||||
|
recordActivity('danger', '浏览器自动化组件不可用', message)
|
||||||
|
emitRuntimeInvalidated('playwright-cdp-fatal', {
|
||||||
|
message,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
initScheduler()
|
initScheduler()
|
||||||
initMonitorScheduler()
|
initMonitorScheduler()
|
||||||
@@ -1850,17 +1886,40 @@ function asSubmitUncertainExecution(result: AdapterExecutionResult): AdapterExec
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function retainPlaywrightPageOrThrow(
|
||||||
|
task: RuntimeTaskRecord,
|
||||||
|
payload: Record<string, JsonValue>,
|
||||||
|
sessionHandle: SessionHandle,
|
||||||
|
) {
|
||||||
|
try {
|
||||||
|
await ensurePlaywrightCDPReady()
|
||||||
|
return await retainHiddenPlaywrightPage({
|
||||||
|
accountId: task.accountId || task.id,
|
||||||
|
session: sessionHandle.session,
|
||||||
|
targetURL: resolvePlaywrightTargetURL(task.platform, payload),
|
||||||
|
title: `${task.title} · Hidden Playwright`,
|
||||||
|
})
|
||||||
|
} catch (error) {
|
||||||
|
if (isPlaywrightCDPUnavailableError(error)) {
|
||||||
|
throw new PlaywrightCDPInfrastructureError(errorMessage(error))
|
||||||
|
}
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function pumpExecutionLoop(): void {
|
function pumpExecutionLoop(): void {
|
||||||
if (!state.running) {
|
if (!state.running || playwrightCDPFatal) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
syncSchedulerSurface()
|
syncSchedulerSurface()
|
||||||
|
const blockedPlaywrightPlatforms = blockedPlaywrightExecutionPlatforms()
|
||||||
|
|
||||||
if (canStartAnotherPublishExecution()) {
|
if (canStartAnotherPublishExecution()) {
|
||||||
const selected = selectNextPublishTask({
|
const selected = selectNextPublishTask({
|
||||||
maxConcurrency: resolveAdaptiveTotalConcurrency(),
|
maxConcurrency: resolveAdaptiveTotalConcurrency(),
|
||||||
activePlatforms: activePublishPlatforms(),
|
activePlatforms: activePublishPlatforms(),
|
||||||
|
blockedPlatforms: blockedPlaywrightPlatforms,
|
||||||
activeCount: activePublishCount(),
|
activeCount: activePublishCount(),
|
||||||
})
|
})
|
||||||
if (selected) {
|
if (selected) {
|
||||||
@@ -1883,6 +1942,7 @@ function pumpExecutionLoop(): void {
|
|||||||
maxConcurrency: resolveAdaptiveMonitorConcurrency(),
|
maxConcurrency: resolveAdaptiveMonitorConcurrency(),
|
||||||
activePlatforms: activeMonitorPlatforms(),
|
activePlatforms: activeMonitorPlatforms(),
|
||||||
activeQuestionKeys: activeMonitorQuestionKeys(),
|
activeQuestionKeys: activeMonitorQuestionKeys(),
|
||||||
|
blockedPlatforms: blockedPlaywrightPlatforms,
|
||||||
activeCount: activeMonitorCount(),
|
activeCount: activeMonitorCount(),
|
||||||
})
|
})
|
||||||
if (selected) {
|
if (selected) {
|
||||||
@@ -1912,6 +1972,10 @@ async function leaseSpecificTask(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!(await ensurePlaywrightCDPAdmissionForRequest(expectedKind, request))) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
state.leaseInFlightKinds.add(expectedKind)
|
state.leaseInFlightKinds.add(expectedKind)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -1967,7 +2031,11 @@ async function leaseSpecificTask(
|
|||||||
|
|
||||||
async function pullNextTask(kind: 'publish' | 'monitor'): Promise<void> {
|
async function pullNextTask(kind: 'publish' | 'monitor'): Promise<void> {
|
||||||
if (kind === 'monitor') {
|
if (kind === 'monitor') {
|
||||||
if (!canRunLive(state.session) || isLeaseInFlight('monitor')) {
|
if (!canRunLive(state.session) || isLeaseInFlight('monitor') || playwrightCDPFatal) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!(await ensurePlaywrightCDPAdmissionForFallback('monitor'))) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2002,7 +2070,11 @@ async function pullNextTask(kind: 'publish' | 'monitor'): Promise<void> {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!canRunLive(state.session) || isLeaseInFlight('publish')) {
|
if (!canRunLive(state.session) || isLeaseInFlight('publish') || playwrightCDPFatal) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!(await ensurePlaywrightCDPAdmissionForFallback('publish'))) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2046,7 +2118,7 @@ async function pullNextTask(kind: 'publish' | 'monitor'): Promise<void> {
|
|||||||
|
|
||||||
async function pullMonitoringTasks(routing: RuntimeTaskRouting): Promise<void> {
|
async function pullMonitoringTasks(routing: RuntimeTaskRouting): Promise<void> {
|
||||||
const leaseLimit = resolveLegacyMonitoringLeaseLimit()
|
const leaseLimit = resolveLegacyMonitoringLeaseLimit()
|
||||||
if (!canRunLive(state.session) || isLeaseInFlight('monitor') || leaseLimit <= 0) {
|
if (!canRunLive(state.session) || isLeaseInFlight('monitor') || leaseLimit <= 0 || playwrightCDPFatal) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2270,6 +2342,15 @@ async function executeLeasedMonitoringTask(
|
|||||||
`${taskRecord.title} ${execution.summary}`,
|
`${taskRecord.title} ${execution.summary}`,
|
||||||
)
|
)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (isPlaywrightCDPInfrastructureError(error)) {
|
||||||
|
await deferMonitoringLeaseAfterInfrastructureFailure(
|
||||||
|
taskRecord,
|
||||||
|
monitoringTaskID,
|
||||||
|
taskRecord.leaseToken,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if (isMonitorAbortReason(taskRecord, error, 'stale_business_date')) {
|
if (isMonitorAbortReason(taskRecord, error, 'stale_business_date')) {
|
||||||
let dropSummary = staleMonitoringDropSummary(taskRecord)
|
let dropSummary = staleMonitoringDropSummary(taskRecord)
|
||||||
try {
|
try {
|
||||||
@@ -2531,6 +2612,11 @@ async function executeLeasedDesktopMonitorTask(
|
|||||||
`${taskRecord.title} ${execution.summary}`,
|
`${taskRecord.title} ${execution.summary}`,
|
||||||
)
|
)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
if (isPlaywrightCDPInfrastructureError(error)) {
|
||||||
|
await deferDesktopMonitorTaskAfterInfrastructureFailure(taskRecord, leased, monitoringTaskID)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if (isMonitorAbortReason(taskRecord, error, 'stale_business_date')) {
|
if (isMonitorAbortReason(taskRecord, error, 'stale_business_date')) {
|
||||||
let dropSummary = staleMonitoringDropSummary(taskRecord)
|
let dropSummary = staleMonitoringDropSummary(taskRecord)
|
||||||
try {
|
try {
|
||||||
@@ -3024,12 +3110,7 @@ async function executeTaskAdapter(
|
|||||||
: retainHotView(task.accountId || task.id)
|
: retainHotView(task.accountId || task.id)
|
||||||
const playwrightLease =
|
const playwrightLease =
|
||||||
adapter.executionMode === 'playwright'
|
adapter.executionMode === 'playwright'
|
||||||
? await retainHiddenPlaywrightPage({
|
? await retainPlaywrightPageOrThrow(task, payload, sessionHandle)
|
||||||
accountId: task.accountId || task.id,
|
|
||||||
session: sessionHandle.session,
|
|
||||||
targetURL: resolvePlaywrightTargetURL(task.platform, payload),
|
|
||||||
title: `${task.title} · Hidden Playwright`,
|
|
||||||
})
|
|
||||||
: null
|
: null
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -3098,12 +3179,7 @@ async function executeTaskAdapter(
|
|||||||
: retainHotView(task.accountId || task.id)
|
: retainHotView(task.accountId || task.id)
|
||||||
const playwrightLease =
|
const playwrightLease =
|
||||||
adapter.executionMode === 'playwright'
|
adapter.executionMode === 'playwright'
|
||||||
? await retainHiddenPlaywrightPage({
|
? await retainPlaywrightPageOrThrow(task, payload, sessionHandle)
|
||||||
accountId: task.accountId || task.id,
|
|
||||||
session: sessionHandle.session,
|
|
||||||
targetURL: resolvePlaywrightTargetURL(task.platform, payload),
|
|
||||||
title: `${task.title} · Hidden Playwright`,
|
|
||||||
})
|
|
||||||
: null
|
: null
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -3534,6 +3610,238 @@ function isLeaseInFlight(kind: 'publish' | 'monitor'): boolean {
|
|||||||
return state.leaseInFlightKinds.has(kind)
|
return state.leaseInFlightKinds.has(kind)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function blockedPlaywrightExecutionPlatforms(): Set<string> {
|
||||||
|
if (isPlaywrightCDPExecutionAvailable()) {
|
||||||
|
return new Set()
|
||||||
|
}
|
||||||
|
|
||||||
|
return new Set([
|
||||||
|
...state.accounts
|
||||||
|
.map((account) => account.platform)
|
||||||
|
.filter((platform) => adapterRequiresPlaywright('monitor', platform)),
|
||||||
|
...getPublishSchedulerSnapshot()
|
||||||
|
.tasks.map((task) => task.platform)
|
||||||
|
.filter((platform) => adapterRequiresPlaywright('publish', platform)),
|
||||||
|
...getMonitorSchedulerSnapshot()
|
||||||
|
.tasks.map((task) => task.platform)
|
||||||
|
.filter((platform) => adapterRequiresPlaywright('monitor', platform)),
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
|
async function ensurePlaywrightCDPAdmissionForRequest(
|
||||||
|
kind: 'publish' | 'monitor',
|
||||||
|
request: PendingTaskRequest,
|
||||||
|
): Promise<boolean> {
|
||||||
|
if (!request.platform || !adapterRequiresPlaywright(kind, request.platform)) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
const ready = await ensurePlaywrightCDPAdmission()
|
||||||
|
if (!ready) {
|
||||||
|
deferPendingTaskRequest(kind, request, playwrightCDPPreflightDeferMs)
|
||||||
|
}
|
||||||
|
return ready
|
||||||
|
}
|
||||||
|
|
||||||
|
async function ensurePlaywrightCDPAdmissionForFallback(kind: 'publish' | 'monitor'): Promise<boolean> {
|
||||||
|
if (kind === 'publish') {
|
||||||
|
const queuedPlaywrightPublish = getPublishSchedulerSnapshot().tasks.some(
|
||||||
|
(task) => task.state === 'queued' && adapterRequiresPlaywright('publish', task.platform),
|
||||||
|
)
|
||||||
|
if (!queuedPlaywrightPublish) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return ensurePlaywrightCDPAdmission()
|
||||||
|
}
|
||||||
|
|
||||||
|
const hasNonPlaywrightMonitorAccount = state.accounts.some(
|
||||||
|
(account) =>
|
||||||
|
isAIPlatformId(account.platform) &&
|
||||||
|
!adapterRequiresPlaywright('monitor', account.platform) &&
|
||||||
|
getProjectedAccountHealth({
|
||||||
|
accountId: account.id,
|
||||||
|
platform: account.platform,
|
||||||
|
health: account.health,
|
||||||
|
verifiedAt: account.verified_at,
|
||||||
|
}).health === 'live',
|
||||||
|
)
|
||||||
|
if (hasNonPlaywrightMonitorAccount) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
return ensurePlaywrightCDPAdmission()
|
||||||
|
}
|
||||||
|
|
||||||
|
async function ensurePlaywrightCDPAdmission(): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
await ensurePlaywrightCDPReady()
|
||||||
|
return true
|
||||||
|
} catch (error) {
|
||||||
|
handlePlaywrightCDPUnavailable(error)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPlaywrightCDPInfrastructureError(error: unknown): error is PlaywrightCDPInfrastructureError {
|
||||||
|
return error instanceof PlaywrightCDPInfrastructureError || isPlaywrightCDPUnavailableError(error)
|
||||||
|
}
|
||||||
|
|
||||||
|
function deferPendingTaskRequest(
|
||||||
|
kind: 'publish' | 'monitor',
|
||||||
|
request: PendingTaskRequest,
|
||||||
|
delayMs: number,
|
||||||
|
): void {
|
||||||
|
if (kind === 'publish') {
|
||||||
|
notePublishTaskDeferred(request.taskId)
|
||||||
|
state.publishFallbackBackoffUntil = Date.now() + Math.max(delayMs, publishPullIntervalMs)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
noteMonitorTaskDeferred(request.taskId, delayMs)
|
||||||
|
state.monitorFallbackBackoffUntil = Date.now() + Math.max(delayMs, monitorPullFallbackIntervalMs)
|
||||||
|
}
|
||||||
|
|
||||||
|
function adapterRequiresPlaywright(kind: 'publish' | 'monitor', platform: string): boolean {
|
||||||
|
const adapter = kind === 'publish' ? selectPublishAdapter(platform) : selectMonitorAdapter(platform)
|
||||||
|
return adapter?.executionMode === 'playwright'
|
||||||
|
}
|
||||||
|
|
||||||
|
function handlePlaywrightCDPUnavailable(error: unknown, taskId?: string): void {
|
||||||
|
const message = errorMessage(error) || 'playwright_cdp_endpoint_unavailable'
|
||||||
|
state.lastError = message
|
||||||
|
setSchedulerError(message)
|
||||||
|
state.monitorFallbackBackoffUntil = Date.now() + monitorPullFallbackIntervalMs
|
||||||
|
state.publishFallbackBackoffUntil = Date.now() + publishPullIntervalMs
|
||||||
|
|
||||||
|
if (playwrightCDPRecoveryAttemptCount >= maxPlaywrightCDPSoftRecoveries) {
|
||||||
|
markPlaywrightCDPFatal(message, taskId)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
playwrightCDPRecoveryAttemptCount += 1
|
||||||
|
|
||||||
|
const now = Date.now()
|
||||||
|
if (now - lastPlaywrightCDPRecoveryActivityAt >= playwrightCDPRecoveryActivityCooldownMs) {
|
||||||
|
lastPlaywrightCDPRecoveryActivityAt = now
|
||||||
|
recordActivity(
|
||||||
|
'warn',
|
||||||
|
'浏览器自动化通道异常',
|
||||||
|
'已暂停领取 Playwright 平台任务,客户端将自动恢复 CDP 后继续采集。',
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
emitRuntimeInvalidated('playwright-cdp-recovery', {
|
||||||
|
taskId,
|
||||||
|
message,
|
||||||
|
})
|
||||||
|
syncSchedulerSurface()
|
||||||
|
}
|
||||||
|
|
||||||
|
function markPlaywrightCDPFatal(message: string, taskId?: string): void {
|
||||||
|
if (playwrightCDPFatal) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
playwrightCDPFatal = true
|
||||||
|
|
||||||
|
const fatalMessage =
|
||||||
|
message ||
|
||||||
|
'浏览器自动化通道不可用,客户端已停止采集。请重启客户端或检查安全软件/端口占用。'
|
||||||
|
state.lastError = fatalMessage
|
||||||
|
setSchedulerPhase('paused')
|
||||||
|
setSchedulerError(fatalMessage)
|
||||||
|
setTransportDispatchState('idle')
|
||||||
|
state.dispatchWsConnected = false
|
||||||
|
state.dispatchWsClient?.stop()
|
||||||
|
state.dispatchWsClient = null
|
||||||
|
state.leaseInFlightKinds.clear()
|
||||||
|
recordActivity('danger', '浏览器自动化组件不可用', fatalMessage)
|
||||||
|
|
||||||
|
void offlineDesktopClient().catch((offlineError) => {
|
||||||
|
console.warn('[desktop-runtime] failed to mark client offline after cdp fatal', {
|
||||||
|
message: errorMessage(offlineError),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
emitRuntimeInvalidated('playwright-cdp-fatal', {
|
||||||
|
taskId,
|
||||||
|
message: fatalMessage,
|
||||||
|
})
|
||||||
|
syncSchedulerSurface()
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deferMonitoringLeaseAfterInfrastructureFailure(
|
||||||
|
task: RuntimeTaskRecord,
|
||||||
|
monitoringTaskID: number,
|
||||||
|
leaseToken: string,
|
||||||
|
reason = 'playwright_cdp_unavailable',
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
await cancelMonitoringTask(monitoringTaskID, {
|
||||||
|
lease_token: leaseToken,
|
||||||
|
reason,
|
||||||
|
})
|
||||||
|
} catch (cancelError) {
|
||||||
|
recordActivity(
|
||||||
|
'warn',
|
||||||
|
'监控租约释放失败',
|
||||||
|
`${task.title} 基础设施异常后释放租约失败:${errorMessage(cancelError)}`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const existing = state.tasks.get(task.id)
|
||||||
|
if (existing) {
|
||||||
|
existing.status = 'queued'
|
||||||
|
existing.summary = '浏览器自动化通道异常,已释放租约并等待客户端自动恢复后重试。'
|
||||||
|
existing.error = {
|
||||||
|
code: reason,
|
||||||
|
message: 'playwright cdp is unavailable; task deferred without marking collection failed',
|
||||||
|
}
|
||||||
|
existing.updatedAt = Date.now()
|
||||||
|
existing.attemptId = null
|
||||||
|
existing.leaseToken = null
|
||||||
|
state.tasks.set(task.id, existing)
|
||||||
|
}
|
||||||
|
noteLeaseReleased('completed', task.id)
|
||||||
|
noteMonitorTaskDeferred(task.id, playwrightCDPPreflightDeferMs)
|
||||||
|
handlePlaywrightCDPUnavailable(new Error(reason), task.id)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deferDesktopMonitorTaskAfterInfrastructureFailure(
|
||||||
|
task: RuntimeTaskRecord,
|
||||||
|
leased: LeaseDesktopTaskResponse,
|
||||||
|
monitoringTaskID: number,
|
||||||
|
reason = 'playwright_cdp_unavailable',
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
await cancelMonitoringTask(monitoringTaskID, {
|
||||||
|
lease_token: leased.lease_token as string,
|
||||||
|
reason,
|
||||||
|
})
|
||||||
|
} catch (cancelError) {
|
||||||
|
recordActivity(
|
||||||
|
'warn',
|
||||||
|
'监控租约释放失败',
|
||||||
|
`${task.title} 基础设施异常后释放 phase2 租约失败:${errorMessage(cancelError)}`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const existing = state.tasks.get(task.id)
|
||||||
|
if (existing) {
|
||||||
|
existing.status = 'queued'
|
||||||
|
existing.summary = '浏览器自动化通道异常,任务已回队等待客户端自动恢复。'
|
||||||
|
existing.error = {
|
||||||
|
code: reason,
|
||||||
|
message: 'playwright cdp is unavailable; desktop monitor task requeued',
|
||||||
|
}
|
||||||
|
existing.updatedAt = Date.now()
|
||||||
|
existing.attemptId = null
|
||||||
|
existing.leaseToken = null
|
||||||
|
state.tasks.set(task.id, existing)
|
||||||
|
}
|
||||||
|
noteLeaseReleased('completed', task.id)
|
||||||
|
noteMonitorTaskDeferred(task.id, playwrightCDPPreflightDeferMs)
|
||||||
|
handlePlaywrightCDPUnavailable(new Error(reason), task.id)
|
||||||
|
}
|
||||||
|
|
||||||
function activePublishCount(): number {
|
function activePublishCount(): number {
|
||||||
return [...state.activeExecutions.values()].filter((item) => item.kind === 'publish').length
|
return [...state.activeExecutions.values()].filter((item) => item.kind === 'publish').length
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,17 +1,23 @@
|
|||||||
type RuntimeInvalidationReason = 'account-health' | 'publish-task-lease' | 'publish-task-progress'
|
type RuntimeInvalidationReason =
|
||||||
|
| 'account-health'
|
||||||
|
| 'publish-task-lease'
|
||||||
|
| 'publish-task-progress'
|
||||||
|
| 'playwright-cdp-recovery'
|
||||||
|
| 'playwright-cdp-fatal'
|
||||||
|
|
||||||
type RuntimeInvalidationListener = (event: {
|
type RuntimeInvalidationListener = (event: {
|
||||||
reason: RuntimeInvalidationReason
|
reason: RuntimeInvalidationReason
|
||||||
at: number
|
at: number
|
||||||
accountId?: string
|
accountId?: string
|
||||||
taskId?: string
|
taskId?: string
|
||||||
|
message?: string
|
||||||
}) => void
|
}) => void
|
||||||
|
|
||||||
const listeners = new Set<RuntimeInvalidationListener>()
|
const listeners = new Set<RuntimeInvalidationListener>()
|
||||||
|
|
||||||
export function emitRuntimeInvalidated(
|
export function emitRuntimeInvalidated(
|
||||||
reason: RuntimeInvalidationReason,
|
reason: RuntimeInvalidationReason,
|
||||||
details: { accountId?: string; taskId?: string } = {},
|
details: { accountId?: string; taskId?: string; message?: string } = {},
|
||||||
): void {
|
): void {
|
||||||
const event = {
|
const event = {
|
||||||
reason,
|
reason,
|
||||||
|
|||||||
@@ -249,19 +249,31 @@ const desktopBridge = {
|
|||||||
ipcRenderer.invoke('desktop:set-window-mode', mode, request) as Promise<null>,
|
ipcRenderer.invoke('desktop:set-window-mode', mode, request) as Promise<null>,
|
||||||
onRuntimeInvalidated: (
|
onRuntimeInvalidated: (
|
||||||
listener: (event: {
|
listener: (event: {
|
||||||
reason: 'account-health' | 'publish-task-lease' | 'publish-task-progress'
|
reason:
|
||||||
|
| 'account-health'
|
||||||
|
| 'publish-task-lease'
|
||||||
|
| 'publish-task-progress'
|
||||||
|
| 'playwright-cdp-recovery'
|
||||||
|
| 'playwright-cdp-fatal'
|
||||||
at: number
|
at: number
|
||||||
accountId?: string
|
accountId?: string
|
||||||
taskId?: string
|
taskId?: string
|
||||||
|
message?: string
|
||||||
}) => void,
|
}) => void,
|
||||||
) => {
|
) => {
|
||||||
const wrapped = (
|
const wrapped = (
|
||||||
_event: IpcRendererEvent,
|
_event: IpcRendererEvent,
|
||||||
payload: {
|
payload: {
|
||||||
reason: 'account-health' | 'publish-task-lease' | 'publish-task-progress'
|
reason:
|
||||||
|
| 'account-health'
|
||||||
|
| 'publish-task-lease'
|
||||||
|
| 'publish-task-progress'
|
||||||
|
| 'playwright-cdp-recovery'
|
||||||
|
| 'playwright-cdp-fatal'
|
||||||
at: number
|
at: number
|
||||||
accountId?: string
|
accountId?: string
|
||||||
taskId?: string
|
taskId?: string
|
||||||
|
message?: string
|
||||||
},
|
},
|
||||||
) => {
|
) => {
|
||||||
listener(payload)
|
listener(payload)
|
||||||
|
|||||||
+7
-1
@@ -142,10 +142,16 @@ declare global {
|
|||||||
): Promise<null>
|
): Promise<null>
|
||||||
onRuntimeInvalidated(
|
onRuntimeInvalidated(
|
||||||
listener: (event: {
|
listener: (event: {
|
||||||
reason: 'account-health' | 'publish-task-lease' | 'publish-task-progress'
|
reason:
|
||||||
|
| 'account-health'
|
||||||
|
| 'publish-task-lease'
|
||||||
|
| 'publish-task-progress'
|
||||||
|
| 'playwright-cdp-recovery'
|
||||||
|
| 'playwright-cdp-fatal'
|
||||||
at: number
|
at: number
|
||||||
accountId?: string
|
accountId?: string
|
||||||
taskId?: string
|
taskId?: string
|
||||||
|
message?: string
|
||||||
}) => void,
|
}) => void,
|
||||||
): () => void
|
): () => void
|
||||||
onNetworkObserved(listener: (event: DesktopObservedRequest) => void): () => void
|
onNetworkObserved(listener: (event: DesktopObservedRequest) => void): () => void
|
||||||
|
|||||||
@@ -1901,6 +1901,7 @@ type PublishLeaseRecoveryResult struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type MonitorLeaseRecoveryResult struct {
|
type MonitorLeaseRecoveryResult struct {
|
||||||
|
Requeued int `json:"requeued"`
|
||||||
Failed int `json:"failed"`
|
Failed int `json:"failed"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1923,7 +1924,7 @@ func resolvePublishRecoveryOutcome(submitStarted bool, attempts int) (status str
|
|||||||
}
|
}
|
||||||
|
|
||||||
func resolveMonitorRecoveryOutcome(mode desktopTaskRecoveryMode, attempts int) (status string, payloadKind string) {
|
func resolveMonitorRecoveryOutcome(mode desktopTaskRecoveryMode, attempts int) (status string, payloadKind string) {
|
||||||
if mode == desktopTaskRecoveryModeLeaseExpiry || attempts >= 2 {
|
if mode != desktopTaskRecoveryModeLeaseExpiry && attempts >= 2 {
|
||||||
return "failed", "monitor_final"
|
return "failed", "monitor_final"
|
||||||
}
|
}
|
||||||
return "queued", "monitor"
|
return "queued", "monitor"
|
||||||
@@ -2026,11 +2027,20 @@ func (s *DesktopTaskService) recoverClientTasks(
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} else if item.MonitorTaskID.Valid {
|
||||||
|
requeuedMonitorTaskIDs, requeueErr := s.requeueMonitorCollectTasksForDesktopRecovery(ctx, []int64{item.MonitorTaskID.Int64}, monitorErrorForTask)
|
||||||
|
if requeueErr != nil {
|
||||||
|
return requeueErr
|
||||||
|
}
|
||||||
|
if len(requeuedMonitorTaskIDs) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if _, execErr := tx.Exec(ctx, `
|
if _, execErr := tx.Exec(ctx, `
|
||||||
UPDATE desktop_tasks
|
UPDATE desktop_tasks
|
||||||
SET status = $2,
|
SET status = $2,
|
||||||
error = $3,
|
error = $3,
|
||||||
|
result = CASE WHEN $2 = 'queued' THEN NULL ELSE result END,
|
||||||
active_attempt_id = NULL,
|
active_attempt_id = NULL,
|
||||||
lease_token_hash = NULL,
|
lease_token_hash = NULL,
|
||||||
lease_expires_at = NULL,
|
lease_expires_at = NULL,
|
||||||
@@ -2355,7 +2365,7 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
|
|||||||
limit = 1000
|
limit = 1000
|
||||||
}
|
}
|
||||||
|
|
||||||
errorJSON, reason, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "monitor_final")
|
errorJSON, reason, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "monitor")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode monitor desktop task recovery payload")
|
return result, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode monitor desktop task recovery payload")
|
||||||
}
|
}
|
||||||
@@ -2390,7 +2400,7 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
|
|||||||
s.logWarn("expired desktop monitor recovery scan failed", scanErr)
|
s.logWarn("expired desktop monitor recovery scan failed", scanErr)
|
||||||
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to parse recovered desktop tasks")
|
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to parse recovered desktop tasks")
|
||||||
}
|
}
|
||||||
item.Status = "failed"
|
item.Status = "queued"
|
||||||
item.ErrorJSON = errorJSON
|
item.ErrorJSON = errorJSON
|
||||||
recovered = append(recovered, item)
|
recovered = append(recovered, item)
|
||||||
if item.MonitorTaskID.Valid {
|
if item.MonitorTaskID.Valid {
|
||||||
@@ -2402,18 +2412,18 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
|
|||||||
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to iterate recovered desktop tasks")
|
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to iterate recovered desktop tasks")
|
||||||
}
|
}
|
||||||
|
|
||||||
failedMonitorTaskIDs, err := s.failMonitorCollectTasksForDesktopRecovery(ctx, monitorTaskIDs, errorJSON)
|
requeuedMonitorTaskIDs, err := s.requeueMonitorCollectTasksForDesktopRecovery(ctx, monitorTaskIDs, errorJSON)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
failedMonitorTaskSet := int64Set(failedMonitorTaskIDs)
|
requeuedMonitorTaskSet := int64Set(requeuedMonitorTaskIDs)
|
||||||
recoverableDesktopIDs := make([]uuid.UUID, 0, len(recovered))
|
recoverableDesktopIDs := make([]uuid.UUID, 0, len(recovered))
|
||||||
recoverable := make([]recoveredDesktopTaskLease, 0, len(recovered))
|
recoverable := make([]recoveredDesktopTaskLease, 0, len(recovered))
|
||||||
for _, item := range recovered {
|
for _, item := range recovered {
|
||||||
if !item.MonitorTaskID.Valid {
|
if !item.MonitorTaskID.Valid {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if _, ok := failedMonitorTaskSet[item.MonitorTaskID.Int64]; !ok {
|
if _, ok := requeuedMonitorTaskSet[item.MonitorTaskID.Int64]; !ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
recoverableDesktopIDs = append(recoverableDesktopIDs, item.TaskID)
|
recoverableDesktopIDs = append(recoverableDesktopIDs, item.TaskID)
|
||||||
@@ -2423,7 +2433,7 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
updateRows, err := tx.Query(ctx, failExpiredMonitorDesktopTasksSQL(), recoverableDesktopIDs, errorJSON, reason)
|
updateRows, err := tx.Query(ctx, requeueExpiredMonitorDesktopTasksSQL(), recoverableDesktopIDs, errorJSON, reason)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logWarn("expired desktop monitor recovery update failed", err)
|
s.logWarn("expired desktop monitor recovery update failed", err)
|
||||||
return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
|
return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
|
||||||
@@ -2462,7 +2472,7 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
|
|||||||
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
|
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
|
||||||
TaskID: item.TaskID,
|
TaskID: item.TaskID,
|
||||||
AttemptID: attemptID,
|
AttemptID: attemptID,
|
||||||
FinalStatus: literalStringPtr("failed"),
|
FinalStatus: literalStringPtr("aborted"),
|
||||||
Error: errorJSON,
|
Error: errorJSON,
|
||||||
}); finishErr != nil {
|
}); finishErr != nil {
|
||||||
s.logWarn("expired desktop monitor recovery attempt finish failed", finishErr, zap.String("task_id", item.TaskID.String()))
|
s.logWarn("expired desktop monitor recovery attempt finish failed", finishErr, zap.String("task_id", item.TaskID.String()))
|
||||||
@@ -2473,7 +2483,7 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
|
|||||||
return result, response.ErrInternal(50199, "desktop_task_recovery_commit_failed", "failed to commit desktop task recovery")
|
return result, response.ErrInternal(50199, "desktop_task_recovery_commit_failed", "failed to commit desktop task recovery")
|
||||||
}
|
}
|
||||||
|
|
||||||
result.Failed = len(updatedDesktopIDs)
|
result.Requeued = len(updatedDesktopIDs)
|
||||||
|
|
||||||
for _, item := range recoverable {
|
for _, item := range recoverable {
|
||||||
if _, ok := updatedDesktopIDs[item.TaskID]; !ok {
|
if _, ok := updatedDesktopIDs[item.TaskID]; !ok {
|
||||||
@@ -2484,12 +2494,12 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
|
|||||||
s.logWarn("expired desktop monitor recovery reload after commit failed", getErr, zap.String("task_id", item.TaskID.String()))
|
s.logWarn("expired desktop monitor recovery reload after commit failed", getErr, zap.String("task_id", item.TaskID.String()))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
s.publishTaskEvent(ctx, task, "task_completed")
|
s.publishTaskEvent(ctx, task, "task_available")
|
||||||
}
|
}
|
||||||
|
|
||||||
if result.Failed > 0 && s.logger != nil {
|
if result.Requeued > 0 && s.logger != nil {
|
||||||
s.logger.Info("expired desktop monitor tasks recovered",
|
s.logger.Info("expired desktop monitor tasks recovered",
|
||||||
zap.Int("failed", result.Failed),
|
zap.Int("requeued", result.Requeued),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2519,16 +2529,18 @@ func expiredMonitorDesktopTasksCandidateSQL() string {
|
|||||||
`
|
`
|
||||||
}
|
}
|
||||||
|
|
||||||
func failExpiredMonitorDesktopTasksSQL() string {
|
func requeueExpiredMonitorDesktopTasksSQL() string {
|
||||||
return `
|
return `
|
||||||
UPDATE desktop_tasks
|
UPDATE desktop_tasks
|
||||||
SET status = 'failed',
|
SET status = 'queued',
|
||||||
error = $2,
|
error = $2,
|
||||||
|
result = NULL,
|
||||||
active_attempt_id = NULL,
|
active_attempt_id = NULL,
|
||||||
lease_token_hash = NULL,
|
lease_token_hash = NULL,
|
||||||
lease_expires_at = NULL,
|
lease_expires_at = NULL,
|
||||||
interrupted_at = COALESCE(interrupted_at, NOW()),
|
interrupted_at = COALESCE(interrupted_at, NOW()),
|
||||||
interrupt_reason = COALESCE(interrupt_reason, $3),
|
interrupt_reason = COALESCE(interrupt_reason, $3),
|
||||||
|
enqueued_at = NOW(),
|
||||||
updated_at = NOW()
|
updated_at = NOW()
|
||||||
WHERE desktop_id = ANY($1::uuid[])
|
WHERE desktop_id = ANY($1::uuid[])
|
||||||
AND kind = 'monitor'
|
AND kind = 'monitor'
|
||||||
@@ -2537,6 +2549,69 @@ func failExpiredMonitorDesktopTasksSQL() string {
|
|||||||
`
|
`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *DesktopTaskService) requeueMonitorCollectTasksForDesktopRecovery(ctx context.Context, monitorTaskIDs []int64, errorJSON []byte) ([]int64, error) {
|
||||||
|
if s == nil || s.monitoringPool == nil || len(monitorTaskIDs) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
monitorTaskIDs = uniqueInt64s(monitorTaskIDs)
|
||||||
|
message := desktopTaskRecoveryMessage(errorJSON)
|
||||||
|
requestPayloadJSON, err := json.Marshal(map[string]any{
|
||||||
|
"runtime_error": json.RawMessage(errorJSON),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode monitor desktop task recovery payload")
|
||||||
|
}
|
||||||
|
rows, err := s.monitoringPool.Query(ctx, `
|
||||||
|
WITH input AS (
|
||||||
|
SELECT unnest($1::bigint[]) AS id
|
||||||
|
),
|
||||||
|
updated AS (
|
||||||
|
UPDATE monitoring_collect_tasks
|
||||||
|
SET status = 'pending',
|
||||||
|
lease_token_hash = NULL,
|
||||||
|
leased_to_executor = NULL,
|
||||||
|
leased_at = NULL,
|
||||||
|
lease_expires_at = NULL,
|
||||||
|
callback_received_at = NULL,
|
||||||
|
completed_at = NULL,
|
||||||
|
skip_reason = NULL,
|
||||||
|
dispatch_after = NOW() + interval '30 seconds',
|
||||||
|
error_message = NULLIF($2, ''),
|
||||||
|
request_payload_json = (
|
||||||
|
CASE
|
||||||
|
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
|
||||||
|
THEN '{}'::jsonb
|
||||||
|
ELSE request_payload_json
|
||||||
|
END
|
||||||
|
) || $3::jsonb,
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id IN (SELECT id FROM input)
|
||||||
|
AND COALESCE(execution_owner, 'legacy') = 'desktop_tasks'
|
||||||
|
AND callback_received_at IS NULL
|
||||||
|
AND status IN ('pending', 'leased', 'expired')
|
||||||
|
RETURNING id
|
||||||
|
)
|
||||||
|
SELECT id FROM updated
|
||||||
|
`, monitorTaskIDs, message, requestPayloadJSON)
|
||||||
|
if err != nil {
|
||||||
|
return nil, response.ErrInternal(50041, "task_update_failed", "failed to requeue expired monitor desktop tasks")
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
requeuedIDs := make([]int64, 0, len(monitorTaskIDs))
|
||||||
|
for rows.Next() {
|
||||||
|
var id int64
|
||||||
|
if scanErr := rows.Scan(&id); scanErr != nil {
|
||||||
|
return nil, response.ErrInternal(50041, "task_update_failed", "failed to parse requeued monitor desktop tasks")
|
||||||
|
}
|
||||||
|
requeuedIDs = append(requeuedIDs, id)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, response.ErrInternal(50041, "task_update_failed", "failed to iterate requeued monitor desktop tasks")
|
||||||
|
}
|
||||||
|
return requeuedIDs, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *DesktopTaskService) failMonitorCollectTasksForDesktopRecovery(ctx context.Context, monitorTaskIDs []int64, errorJSON []byte) ([]int64, error) {
|
func (s *DesktopTaskService) failMonitorCollectTasksForDesktopRecovery(ctx context.Context, monitorTaskIDs []int64, errorJSON []byte) ([]int64, error) {
|
||||||
if s == nil || s.monitoringPool == nil || len(monitorTaskIDs) == 0 {
|
if s == nil || s.monitoringPool == nil || len(monitorTaskIDs) == 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
@@ -2586,8 +2661,8 @@ func (s *DesktopTaskService) failMonitorCollectTasksForDesktopRecovery(ctx conte
|
|||||||
AND t.status = 'failed'
|
AND t.status = 'failed'
|
||||||
AND (
|
AND (
|
||||||
t.error_message = $2
|
t.error_message = $2
|
||||||
OR t.request_payload_json #>> '{runtime_error,source}' = 'desktop_task_lease_expiry'
|
OR t.request_payload_json #>> '{runtime_error,source}' = 'desktop_task_recovery'
|
||||||
OR t.request_payload_json #>> '{runtime_error,reason}' = 'lease_expired'
|
OR t.request_payload_json #>> '{runtime_error,source}' = 'desktop_client_offline'
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
SELECT id FROM updated
|
SELECT id FROM updated
|
||||||
@@ -2595,7 +2670,7 @@ func (s *DesktopTaskService) failMonitorCollectTasksForDesktopRecovery(ctx conte
|
|||||||
SELECT id FROM already_failed
|
SELECT id FROM already_failed
|
||||||
`, monitorTaskIDs, message, requestPayloadJSON)
|
`, monitorTaskIDs, message, requestPayloadJSON)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, response.ErrInternal(50041, "task_update_failed", "failed to fail expired monitor desktop tasks")
|
return nil, response.ErrInternal(50041, "task_update_failed", "failed to fail recovered monitor desktop tasks")
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
defer rows.Close()
|
||||||
|
|
||||||
@@ -2700,6 +2775,8 @@ func desktopTaskRecoveryPayload(mode desktopTaskRecoveryMode, kind string) ([]by
|
|||||||
message = "desktop publish task may have already been submitted to the platform before the lease was lost; kept as unknown for manual reconcile to avoid duplicate publishing"
|
message = "desktop publish task may have already been submitted to the platform before the lease was lost; kept as unknown for manual reconcile to avoid duplicate publishing"
|
||||||
} else if isMonitorFinal {
|
} else if isMonitorFinal {
|
||||||
message = "desktop monitor task lost its lease; task has been marked failed to avoid repeated platform retries"
|
message = "desktop monitor task lost its lease; task has been marked failed to avoid repeated platform retries"
|
||||||
|
} else if kind == "monitor" {
|
||||||
|
message = "desktop monitor task lost its lease; task has been re-queued"
|
||||||
}
|
}
|
||||||
|
|
||||||
switch mode {
|
switch mode {
|
||||||
@@ -2735,8 +2812,10 @@ func desktopTaskRecoveryPayload(mode desktopTaskRecoveryMode, kind string) ([]by
|
|||||||
message = fmt.Sprintf("desktop task lease expired before publish completion; task exceeded %d attempts and has been marked failed", desktopPublishMaxAttempts)
|
message = fmt.Sprintf("desktop task lease expired before publish completion; task exceeded %d attempts and has been marked failed", desktopPublishMaxAttempts)
|
||||||
} else if isPublishUncertain {
|
} else if isPublishUncertain {
|
||||||
message = "desktop task lease expired while the publish task was mid-submit; kept as unknown for manual reconcile to avoid duplicate publishing"
|
message = "desktop task lease expired while the publish task was mid-submit; kept as unknown for manual reconcile to avoid duplicate publishing"
|
||||||
} else if isMonitorFinal || kind == "monitor" {
|
} else if isMonitorFinal {
|
||||||
message = "desktop task lease expired before monitor completion; task has been marked failed"
|
message = "desktop task lease expired before monitor completion; task has been marked failed"
|
||||||
|
} else if kind == "monitor" {
|
||||||
|
message = "desktop task lease expired before monitor completion; task has been re-queued"
|
||||||
} else {
|
} else {
|
||||||
message = "desktop task lease expired before publish completion; task has been re-queued for retry"
|
message = "desktop task lease expired before publish completion; task has been re-queued for retry"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -101,21 +101,21 @@ func TestDesktopTaskRecoveryPayloadPublishFinalFailsAfterMaxAttempts(t *testing.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDesktopTaskRecoveryPayloadMonitorLeaseExpiryFails(t *testing.T) {
|
func TestDesktopTaskRecoveryPayloadMonitorLeaseExpiryRequeues(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
payload, reason, message, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "monitor_final")
|
payload, reason, message, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "monitor")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("desktopTaskRecoveryPayload returned error: %v", err)
|
t.Fatalf("desktopTaskRecoveryPayload returned error: %v", err)
|
||||||
}
|
}
|
||||||
if reason != "lease_expired" {
|
if reason != "lease_expired" {
|
||||||
t.Fatalf("reason = %q, want lease_expired", reason)
|
t.Fatalf("reason = %q, want lease_expired", reason)
|
||||||
}
|
}
|
||||||
if !strings.Contains(message, "marked failed") {
|
if !strings.Contains(message, "re-queued") {
|
||||||
t.Fatalf("message = %q, want marked failed", message)
|
t.Fatalf("message = %q, want re-queued", message)
|
||||||
}
|
}
|
||||||
if strings.Contains(message, "re-queued") {
|
if strings.Contains(message, "marked failed") {
|
||||||
t.Fatalf("message must not requeue expired monitor tasks: %q", message)
|
t.Fatalf("message must not fail expired monitor tasks: %q", message)
|
||||||
}
|
}
|
||||||
if !strings.Contains(string(payload), "desktop_task_lease_expiry") {
|
if !strings.Contains(string(payload), "desktop_task_lease_expiry") {
|
||||||
t.Fatalf("payload missing source: %s", payload)
|
t.Fatalf("payload missing source: %s", payload)
|
||||||
@@ -176,7 +176,8 @@ func TestResolveMonitorRecoveryOutcome(t *testing.T) {
|
|||||||
{"disconnect first attempt requeues", desktopTaskRecoveryModeDisconnect, 1, "queued", "monitor"},
|
{"disconnect first attempt requeues", desktopTaskRecoveryModeDisconnect, 1, "queued", "monitor"},
|
||||||
{"startup repeated recovery fails", desktopTaskRecoveryModeStartup, 2, "failed", "monitor_final"},
|
{"startup repeated recovery fails", desktopTaskRecoveryModeStartup, 2, "failed", "monitor_final"},
|
||||||
{"disconnect repeated recovery fails", desktopTaskRecoveryModeDisconnect, 2, "failed", "monitor_final"},
|
{"disconnect repeated recovery fails", desktopTaskRecoveryModeDisconnect, 2, "failed", "monitor_final"},
|
||||||
{"lease expiry fails immediately", desktopTaskRecoveryModeLeaseExpiry, 1, "failed", "monitor_final"},
|
{"lease expiry requeues first attempt", desktopTaskRecoveryModeLeaseExpiry, 1, "queued", "monitor"},
|
||||||
|
{"lease expiry requeues repeated attempt", desktopTaskRecoveryModeLeaseExpiry, 2, "queued", "monitor"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
@@ -221,16 +222,17 @@ func TestExpiredMonitorDesktopTasksCandidateSQLSelectsOnlyExpiredInProgressMonit
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFailExpiredMonitorDesktopTasksSQLMarksTerminalFailure(t *testing.T) {
|
func TestRequeueExpiredMonitorDesktopTasksSQLReturnsTaskToQueue(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
normalized := normalizeSQLForDesktopTaskTest(failExpiredMonitorDesktopTasksSQL())
|
normalized := normalizeSQLForDesktopTaskTest(requeueExpiredMonitorDesktopTasksSQL())
|
||||||
for _, fragment := range []string{
|
for _, fragment := range []string{
|
||||||
"SET status = 'failed'",
|
"SET status = 'queued'",
|
||||||
"active_attempt_id = NULL",
|
"active_attempt_id = NULL",
|
||||||
"lease_token_hash = NULL",
|
"lease_token_hash = NULL",
|
||||||
"lease_expires_at = NULL",
|
"lease_expires_at = NULL",
|
||||||
"interrupt_reason = COALESCE(interrupt_reason, $3)",
|
"interrupt_reason = COALESCE(interrupt_reason, $3)",
|
||||||
|
"enqueued_at = NOW()",
|
||||||
"WHERE desktop_id = ANY($1::uuid[])",
|
"WHERE desktop_id = ANY($1::uuid[])",
|
||||||
"AND kind = 'monitor'",
|
"AND kind = 'monitor'",
|
||||||
"AND status = 'in_progress'",
|
"AND status = 'in_progress'",
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ const (
|
|||||||
// reports skip writes and only refresh the row at this interval so dashboards
|
// reports skip writes and only refresh the row at this interval so dashboards
|
||||||
// do not depend on per-heartbeat database churn.
|
// do not depend on per-heartbeat database churn.
|
||||||
monitoringHeartbeatAccessSnapshotRefreshAfter = 10 * time.Minute
|
monitoringHeartbeatAccessSnapshotRefreshAfter = 10 * time.Minute
|
||||||
|
monitoringRuntimeUnknownMaxDispatchAttempts = 3
|
||||||
)
|
)
|
||||||
|
|
||||||
var monitoringCitationMarkerPattern = regexp.MustCompile(`(?i)\[\s*citation\s*:\s*\d+\s*\]`)
|
var monitoringCitationMarkerPattern = regexp.MustCompile(`(?i)\[\s*citation\s*:\s*\d+\s*\]`)
|
||||||
@@ -194,6 +195,15 @@ type MonitoringCancelTaskResponse struct {
|
|||||||
TaskStatus string `json:"task_status"`
|
TaskStatus string `json:"task_status"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isMonitoringInfrastructureCancelReason(reason *string) bool {
|
||||||
|
switch strings.TrimSpace(optionalStringValue(reason)) {
|
||||||
|
case "desktop_infra_unavailable", "playwright_cdp_unavailable", "playwright_cdp_recovery":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type monitoringInstallationIdentity struct {
|
type monitoringInstallationIdentity struct {
|
||||||
ID string
|
ID string
|
||||||
TenantID int64
|
TenantID int64
|
||||||
@@ -250,6 +260,7 @@ type monitoringCollectTask struct {
|
|||||||
TriggerSource string
|
TriggerSource string
|
||||||
QuestionText string
|
QuestionText string
|
||||||
TargetClientID sql.NullString
|
TargetClientID sql.NullString
|
||||||
|
DispatchAttempts int
|
||||||
}
|
}
|
||||||
|
|
||||||
type monitoringDesktopExecutionLease struct {
|
type monitoringDesktopExecutionLease struct {
|
||||||
@@ -258,6 +269,7 @@ type monitoringDesktopExecutionLease struct {
|
|||||||
TargetClientID uuid.UUID
|
TargetClientID uuid.UUID
|
||||||
LeaseTokenHash []byte
|
LeaseTokenHash []byte
|
||||||
InterruptGeneration int
|
InterruptGeneration int
|
||||||
|
ActiveAttemptID pgtype.UUID
|
||||||
}
|
}
|
||||||
|
|
||||||
type monitoringLeasedTaskRow struct {
|
type monitoringLeasedTaskRow struct {
|
||||||
@@ -945,7 +957,24 @@ func (s *MonitoringCallbackService) skipTaskForInstallation(
|
|||||||
}
|
}
|
||||||
|
|
||||||
skipReason := normalizeSkipReason(req.SkipReason)
|
skipReason := normalizeSkipReason(req.SkipReason)
|
||||||
skipOutcome := monitoringSkipOutcomeForReason(skipReason)
|
skipOutcome := monitoringSkipOutcomeForReason(skipReason, task.DispatchAttempts, task.TriggerSource)
|
||||||
|
errorMessage := strings.TrimSpace(optionalStringValue(req.ErrorMessage))
|
||||||
|
requestPayloadJSON := []byte(`{}`)
|
||||||
|
if skipReason == "runtime_unknown" {
|
||||||
|
var marshalErr error
|
||||||
|
requestPayloadJSON, marshalErr = json.Marshal(map[string]any{
|
||||||
|
"runtime_unknown": map[string]any{
|
||||||
|
"reason": skipReason,
|
||||||
|
"message": errorMessage,
|
||||||
|
"dispatch_attempts": task.DispatchAttempts,
|
||||||
|
"max_attempts": monitoringRuntimeUnknownMaxDispatchAttempts,
|
||||||
|
"retryable": skipOutcome.Requeue,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if marshalErr != nil {
|
||||||
|
return nil, response.ErrInternal(50041, "payload_encode_failed", "failed to encode monitoring task skip payload")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
tx, err := s.monitoringPool.Begin(ctx)
|
tx, err := s.monitoringPool.Begin(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -953,16 +982,48 @@ func (s *MonitoringCallbackService) skipTaskForInstallation(
|
|||||||
}
|
}
|
||||||
defer tx.Rollback(ctx)
|
defer tx.Rollback(ctx)
|
||||||
|
|
||||||
|
if skipOutcome.Requeue {
|
||||||
if _, err := tx.Exec(ctx, `
|
if _, err := tx.Exec(ctx, `
|
||||||
|
UPDATE monitoring_collect_tasks
|
||||||
|
SET status = 'pending',
|
||||||
|
lease_token_hash = NULL,
|
||||||
|
leased_to_executor = NULL,
|
||||||
|
leased_at = NULL,
|
||||||
|
lease_expires_at = NULL,
|
||||||
|
callback_received_at = NULL,
|
||||||
|
completed_at = NULL,
|
||||||
|
skip_reason = NULL,
|
||||||
|
dispatch_after = NOW() + $2::interval,
|
||||||
|
error_message = NULLIF($3, ''),
|
||||||
|
request_payload_json = (
|
||||||
|
CASE
|
||||||
|
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
|
||||||
|
THEN '{}'::jsonb
|
||||||
|
ELSE request_payload_json
|
||||||
|
END
|
||||||
|
) || $4::jsonb,
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id = $1
|
||||||
|
`, task.ID, formatPgInterval(skipOutcome.RetryAfter), errorMessage, requestPayloadJSON); err != nil {
|
||||||
|
return nil, response.ErrInternal(50041, "task_update_failed", "failed to requeue monitoring task after runtime unknown")
|
||||||
|
}
|
||||||
|
} else if _, err := tx.Exec(ctx, `
|
||||||
UPDATE monitoring_collect_tasks
|
UPDATE monitoring_collect_tasks
|
||||||
SET status = $2,
|
SET status = $2,
|
||||||
callback_received_at = NOW(),
|
callback_received_at = NOW(),
|
||||||
completed_at = $3,
|
completed_at = $3,
|
||||||
skip_reason = $4,
|
skip_reason = $4,
|
||||||
error_message = $5,
|
error_message = $5,
|
||||||
|
request_payload_json = (
|
||||||
|
CASE
|
||||||
|
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
|
||||||
|
THEN '{}'::jsonb
|
||||||
|
ELSE request_payload_json
|
||||||
|
END
|
||||||
|
) || $6::jsonb,
|
||||||
updated_at = NOW()
|
updated_at = NOW()
|
||||||
WHERE id = $1
|
WHERE id = $1
|
||||||
`, task.ID, skipOutcome.TaskStatus, completedAt, nullableStringPtr(skipOutcome.StoredSkipReason), nullableString(req.ErrorMessage)); err != nil {
|
`, task.ID, skipOutcome.TaskStatus, completedAt, nullableStringPtr(skipOutcome.StoredSkipReason), nullableString(req.ErrorMessage), requestPayloadJSON); err != nil {
|
||||||
return nil, response.ErrInternal(50041, "task_update_failed", "failed to update monitoring task skip status")
|
return nil, response.ErrInternal(50041, "task_update_failed", "failed to update monitoring task skip status")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -976,29 +1037,56 @@ func (s *MonitoringCallbackService) skipTaskForInstallation(
|
|||||||
|
|
||||||
_ = s.touchInstallation(ctx, installation.ID)
|
_ = s.touchInstallation(ctx, installation.ID)
|
||||||
|
|
||||||
completedAtText := completedAt.Format(time.RFC3339)
|
var completedAtText *string
|
||||||
|
if !skipOutcome.Requeue {
|
||||||
|
text := completedAt.Format(time.RFC3339)
|
||||||
|
completedAtText = &text
|
||||||
|
}
|
||||||
return &MonitoringSkipTaskResponse{
|
return &MonitoringSkipTaskResponse{
|
||||||
TaskID: task.ID,
|
TaskID: task.ID,
|
||||||
TaskStatus: skipOutcome.TaskStatus,
|
TaskStatus: skipOutcome.TaskStatus,
|
||||||
SkipReason: skipReason,
|
SkipReason: skipReason,
|
||||||
CompletedAt: &completedAtText,
|
CompletedAt: completedAtText,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type monitoringSkipOutcome struct {
|
type monitoringSkipOutcome struct {
|
||||||
TaskStatus string
|
TaskStatus string
|
||||||
StoredSkipReason string
|
StoredSkipReason string
|
||||||
|
Requeue bool
|
||||||
|
RetryAfter time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
func monitoringSkipOutcomeForReason(skipReason string) monitoringSkipOutcome {
|
func monitoringSkipOutcomeForReason(skipReason string, dispatchAttempts int, triggerSource string) monitoringSkipOutcome {
|
||||||
switch strings.TrimSpace(skipReason) {
|
switch strings.TrimSpace(skipReason) {
|
||||||
case "runtime_unknown":
|
case "runtime_unknown":
|
||||||
|
if strings.EqualFold(strings.TrimSpace(triggerSource), "manual") {
|
||||||
|
return monitoringSkipOutcome{TaskStatus: "failed"}
|
||||||
|
}
|
||||||
|
if dispatchAttempts < monitoringRuntimeUnknownMaxDispatchAttempts {
|
||||||
|
return monitoringSkipOutcome{
|
||||||
|
TaskStatus: "pending",
|
||||||
|
Requeue: true,
|
||||||
|
RetryAfter: monitoringRuntimeUnknownRetryAfter(dispatchAttempts),
|
||||||
|
}
|
||||||
|
}
|
||||||
return monitoringSkipOutcome{TaskStatus: "failed"}
|
return monitoringSkipOutcome{TaskStatus: "failed"}
|
||||||
default:
|
default:
|
||||||
return monitoringSkipOutcome{TaskStatus: "skipped", StoredSkipReason: strings.TrimSpace(skipReason)}
|
return monitoringSkipOutcome{TaskStatus: "skipped", StoredSkipReason: strings.TrimSpace(skipReason)}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func monitoringRuntimeUnknownRetryAfter(dispatchAttempts int) time.Duration {
|
||||||
|
switch {
|
||||||
|
case dispatchAttempts <= 0:
|
||||||
|
return 30 * time.Second
|
||||||
|
case dispatchAttempts == 1:
|
||||||
|
return 2 * time.Minute
|
||||||
|
default:
|
||||||
|
return 5 * time.Minute
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func nullableStringPtr(value string) *string {
|
func nullableStringPtr(value string) *string {
|
||||||
trimmed := strings.TrimSpace(value)
|
trimmed := strings.TrimSpace(value)
|
||||||
if trimmed == "" {
|
if trimmed == "" {
|
||||||
@@ -1045,6 +1133,13 @@ func (s *MonitoringCallbackService) CancelTaskForDesktopClient(
|
|||||||
if marshalErr != nil {
|
if marshalErr != nil {
|
||||||
return nil, response.ErrInternal(50041, "payload_encode_failed", "failed to encode phase2 desktop cancel payload")
|
return nil, response.ErrInternal(50041, "payload_encode_failed", "failed to encode phase2 desktop cancel payload")
|
||||||
}
|
}
|
||||||
|
if isMonitoringInfrastructureCancelReason(req.Reason) {
|
||||||
|
requeuedTask, requeueErr := s.requeueDesktopMonitorTaskAfterInfrastructureCancel(ctx, desktopExecution, errorJSON)
|
||||||
|
if requeueErr != nil {
|
||||||
|
return nil, requeueErr
|
||||||
|
}
|
||||||
|
desktopTaskCanceled = requeuedTask
|
||||||
|
} else {
|
||||||
taskRepo := repository.NewDesktopTaskRepository(s.businessPool)
|
taskRepo := repository.NewDesktopTaskRepository(s.businessPool)
|
||||||
canceledTask, cancelErr := taskRepo.CancelByLease(ctx, desktopExecution.DesktopTaskID, desktopExecution.LeaseTokenHash, errorJSON)
|
canceledTask, cancelErr := taskRepo.CancelByLease(ctx, desktopExecution.DesktopTaskID, desktopExecution.LeaseTokenHash, errorJSON)
|
||||||
if cancelErr != nil {
|
if cancelErr != nil {
|
||||||
@@ -1054,6 +1149,7 @@ func (s *MonitoringCallbackService) CancelTaskForDesktopClient(
|
|||||||
return nil, response.ErrInternal(50041, "desktop_task_cancel_failed", "failed to cancel phase2 monitor desktop task lease")
|
return nil, response.ErrInternal(50041, "desktop_task_cancel_failed", "failed to cancel phase2 monitor desktop task lease")
|
||||||
}
|
}
|
||||||
desktopTaskCanceled = canceledTask
|
desktopTaskCanceled = canceledTask
|
||||||
|
}
|
||||||
} else if task.Status != "leased" {
|
} else if task.Status != "leased" {
|
||||||
return nil, response.ErrConflict(40941, "task_status_invalid", "task status does not allow lease cancel")
|
return nil, response.ErrConflict(40941, "task_status_invalid", "task status does not allow lease cancel")
|
||||||
}
|
}
|
||||||
@@ -1066,11 +1162,11 @@ func (s *MonitoringCallbackService) CancelTaskForDesktopClient(
|
|||||||
leased_at = NULL,
|
leased_at = NULL,
|
||||||
lease_expires_at = NULL,
|
lease_expires_at = NULL,
|
||||||
callback_received_at = NULL,
|
callback_received_at = NULL,
|
||||||
dispatch_after = NOW(),
|
dispatch_after = CASE WHEN $3::boolean THEN NOW() + interval '30 seconds' ELSE NOW() END,
|
||||||
error_message = $2,
|
error_message = $2,
|
||||||
updated_at = NOW()
|
updated_at = NOW()
|
||||||
WHERE id = $1
|
WHERE id = $1
|
||||||
`, task.ID, nullableString(req.Reason)); err != nil {
|
`, task.ID, nullableString(req.Reason), isMonitoringInfrastructureCancelReason(req.Reason)); err != nil {
|
||||||
return nil, response.ErrInternal(50041, "task_update_failed", "failed to cancel monitoring task lease")
|
return nil, response.ErrInternal(50041, "task_update_failed", "failed to cancel monitoring task lease")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1106,6 +1202,72 @@ func (s *MonitoringCallbackService) CancelTaskForDesktopClient(
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *MonitoringCallbackService) requeueDesktopMonitorTaskAfterInfrastructureCancel(
|
||||||
|
ctx context.Context,
|
||||||
|
lease *monitoringDesktopExecutionLease,
|
||||||
|
errorJSON []byte,
|
||||||
|
) (*repository.DesktopTask, error) {
|
||||||
|
if s == nil || s.businessPool == nil || lease == nil {
|
||||||
|
return nil, response.ErrInternal(50041, "desktop_task_requeue_failed", "failed to requeue phase2 monitor desktop task lease")
|
||||||
|
}
|
||||||
|
|
||||||
|
tx, err := s.businessPool.Begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, response.ErrInternal(50041, "desktop_task_requeue_begin_failed", "failed to start phase2 monitor desktop task requeue")
|
||||||
|
}
|
||||||
|
defer tx.Rollback(ctx)
|
||||||
|
|
||||||
|
repo := repository.NewDesktopTaskRepository(tx)
|
||||||
|
row := tx.QueryRow(ctx, `
|
||||||
|
UPDATE desktop_tasks
|
||||||
|
SET status = 'queued',
|
||||||
|
error = $3,
|
||||||
|
result = NULL,
|
||||||
|
active_attempt_id = NULL,
|
||||||
|
lease_token_hash = NULL,
|
||||||
|
lease_expires_at = NULL,
|
||||||
|
interrupted_at = COALESCE(interrupted_at, NOW()),
|
||||||
|
interrupt_reason = COALESCE(interrupt_reason, 'desktop_infra_unavailable'),
|
||||||
|
enqueued_at = NOW(),
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE desktop_id = $1
|
||||||
|
AND lease_token_hash = $2
|
||||||
|
AND kind = 'monitor'
|
||||||
|
AND status = 'in_progress'
|
||||||
|
RETURNING `+desktopTaskRepositoryReturningColumns,
|
||||||
|
lease.DesktopTaskID,
|
||||||
|
lease.LeaseTokenHash,
|
||||||
|
errorJSON,
|
||||||
|
)
|
||||||
|
task, scanErr := scanRepositoryDesktopTask(row)
|
||||||
|
if scanErr != nil {
|
||||||
|
if errors.Is(scanErr, pgx.ErrNoRows) {
|
||||||
|
return nil, response.ErrConflict(40941, "task_status_invalid", "phase2 monitor desktop task is no longer active")
|
||||||
|
}
|
||||||
|
return nil, response.ErrInternal(50041, "desktop_task_requeue_failed", "failed to requeue phase2 monitor desktop task lease")
|
||||||
|
}
|
||||||
|
|
||||||
|
if lease.ActiveAttemptID.Valid {
|
||||||
|
attemptID, convErr := uuid.FromBytes(lease.ActiveAttemptID.Bytes[:])
|
||||||
|
if convErr != nil {
|
||||||
|
return nil, response.ErrInternal(50041, "desktop_task_attempt_decode_failed", "failed to decode phase2 monitor desktop task attempt")
|
||||||
|
}
|
||||||
|
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
|
||||||
|
TaskID: task.DesktopID,
|
||||||
|
AttemptID: attemptID,
|
||||||
|
FinalStatus: literalStringPtr("aborted"),
|
||||||
|
Error: errorJSON,
|
||||||
|
}); finishErr != nil {
|
||||||
|
return nil, response.ErrInternal(50041, "desktop_task_attempt_finish_failed", "failed to finish phase2 monitor desktop task attempt")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tx.Commit(ctx); err != nil {
|
||||||
|
return nil, response.ErrInternal(50041, "desktop_task_requeue_commit_failed", "failed to commit phase2 monitor desktop task requeue")
|
||||||
|
}
|
||||||
|
return task, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *MonitoringCallbackService) ProcessQueuedTaskResult(ctx context.Context, envelope monitoringTaskResultEnvelope) (*MonitoringTaskResultResponse, error) {
|
func (s *MonitoringCallbackService) ProcessQueuedTaskResult(ctx context.Context, envelope monitoringTaskResultEnvelope) (*MonitoringTaskResultResponse, error) {
|
||||||
if envelope.TaskID <= 0 {
|
if envelope.TaskID <= 0 {
|
||||||
return nil, response.ErrBadRequest(40041, "invalid_task_id", "task id is required")
|
return nil, response.ErrBadRequest(40041, "invalid_task_id", "task id is required")
|
||||||
@@ -1393,6 +1555,7 @@ func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantI
|
|||||||
completed_at,
|
completed_at,
|
||||||
skip_reason,
|
skip_reason,
|
||||||
trigger_source,
|
trigger_source,
|
||||||
|
COALESCE(dispatch_attempts, 0) AS dispatch_attempts,
|
||||||
COALESCE(target_client_id::text, '') AS target_client_id,
|
COALESCE(target_client_id::text, '') AS target_client_id,
|
||||||
COALESCE(NULLIF(t.question_text_snapshot, ''), (
|
COALESCE(NULLIF(t.question_text_snapshot, ''), (
|
||||||
SELECT question_text_snapshot
|
SELECT question_text_snapshot
|
||||||
@@ -1430,6 +1593,7 @@ func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantI
|
|||||||
&item.CompletedAt,
|
&item.CompletedAt,
|
||||||
&item.SkipReason,
|
&item.SkipReason,
|
||||||
&item.TriggerSource,
|
&item.TriggerSource,
|
||||||
|
&item.DispatchAttempts,
|
||||||
&item.TargetClientID,
|
&item.TargetClientID,
|
||||||
&item.QuestionText,
|
&item.QuestionText,
|
||||||
); err != nil {
|
); err != nil {
|
||||||
@@ -1464,6 +1628,7 @@ func (s *MonitoringCallbackService) loadCollectTaskByID(ctx context.Context, tas
|
|||||||
completed_at,
|
completed_at,
|
||||||
skip_reason,
|
skip_reason,
|
||||||
trigger_source,
|
trigger_source,
|
||||||
|
COALESCE(dispatch_attempts, 0) AS dispatch_attempts,
|
||||||
COALESCE(target_client_id::text, '') AS target_client_id,
|
COALESCE(target_client_id::text, '') AS target_client_id,
|
||||||
COALESCE(NULLIF(t.question_text_snapshot, ''), (
|
COALESCE(NULLIF(t.question_text_snapshot, ''), (
|
||||||
SELECT question_text_snapshot
|
SELECT question_text_snapshot
|
||||||
@@ -1500,6 +1665,7 @@ func (s *MonitoringCallbackService) loadCollectTaskByID(ctx context.Context, tas
|
|||||||
&item.CompletedAt,
|
&item.CompletedAt,
|
||||||
&item.SkipReason,
|
&item.SkipReason,
|
||||||
&item.TriggerSource,
|
&item.TriggerSource,
|
||||||
|
&item.DispatchAttempts,
|
||||||
&item.TargetClientID,
|
&item.TargetClientID,
|
||||||
&item.QuestionText,
|
&item.QuestionText,
|
||||||
); err != nil {
|
); err != nil {
|
||||||
@@ -1523,6 +1689,7 @@ func (s *MonitoringCallbackService) loadActiveDesktopMonitorExecutionLease(
|
|||||||
status,
|
status,
|
||||||
target_client_id,
|
target_client_id,
|
||||||
lease_token_hash,
|
lease_token_hash,
|
||||||
|
active_attempt_id,
|
||||||
interrupt_generation
|
interrupt_generation
|
||||||
FROM desktop_tasks
|
FROM desktop_tasks
|
||||||
WHERE kind = 'monitor'
|
WHERE kind = 'monitor'
|
||||||
@@ -1535,6 +1702,7 @@ func (s *MonitoringCallbackService) loadActiveDesktopMonitorExecutionLease(
|
|||||||
&item.Status,
|
&item.Status,
|
||||||
&item.TargetClientID,
|
&item.TargetClientID,
|
||||||
&item.LeaseTokenHash,
|
&item.LeaseTokenHash,
|
||||||
|
&item.ActiveAttemptID,
|
||||||
&item.InterruptGeneration,
|
&item.InterruptGeneration,
|
||||||
); err != nil {
|
); err != nil {
|
||||||
if err == pgx.ErrNoRows {
|
if err == pgx.ErrNoRows {
|
||||||
|
|||||||
@@ -225,12 +225,12 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
|
|||||||
|
|
||||||
repo := repository.NewDesktopTaskRepository(tx)
|
repo := repository.NewDesktopTaskRepository(tx)
|
||||||
publishableTasks := make([]*repository.DesktopTask, 0, len(specs))
|
publishableTasks := make([]*repository.DesktopTask, 0, len(specs))
|
||||||
fallbackLegacyTaskIDs := make([]int64, 0)
|
deferredTaskIDs := make([]int64, 0)
|
||||||
for _, spec := range specs {
|
for _, spec := range specs {
|
||||||
if spec.TargetAccountID == uuid.Nil || spec.TargetClientID == uuid.Nil {
|
if spec.TargetAccountID == uuid.Nil || spec.TargetClientID == uuid.Nil {
|
||||||
target, ok := targets[normalizeMonitoringPlatformID(spec.PlatformID)]
|
target, ok := targets[normalizeMonitoringPlatformID(spec.PlatformID)]
|
||||||
if !ok || target.AccountID == uuid.Nil || target.ClientID == uuid.Nil {
|
if !ok || target.AccountID == uuid.Nil || target.ClientID == uuid.Nil {
|
||||||
fallbackLegacyTaskIDs = append(fallbackLegacyTaskIDs, spec.MonitorTaskID)
|
deferredTaskIDs = append(deferredTaskIDs, spec.MonitorTaskID)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
spec.TargetAccountID = target.AccountID
|
spec.TargetAccountID = target.AccountID
|
||||||
@@ -242,7 +242,7 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
|
|||||||
return nil, nil, upsertErr
|
return nil, nil, upsertErr
|
||||||
}
|
}
|
||||||
if fallbackLegacy {
|
if fallbackLegacy {
|
||||||
fallbackLegacyTaskIDs = append(fallbackLegacyTaskIDs, spec.MonitorTaskID)
|
deferredTaskIDs = append(deferredTaskIDs, spec.MonitorTaskID)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if shouldPublish && task != nil {
|
if shouldPublish && task != nil {
|
||||||
@@ -250,10 +250,16 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(deferredTaskIDs) > 0 {
|
||||||
|
if err := deferPhase2MonitorCollectTasks(ctx, tx, deferredTaskIDs, defaultMonitoringDailyDesktopRetryCooldown); err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if err := tx.Commit(ctx); err != nil {
|
if err := tx.Commit(ctx); err != nil {
|
||||||
return nil, nil, response.ErrInternal(50116, "desktop_task_commit_failed", "failed to commit phase2 monitor desktop dispatch")
|
return nil, nil, response.ErrInternal(50116, "desktop_task_commit_failed", "failed to commit phase2 monitor desktop dispatch")
|
||||||
}
|
}
|
||||||
return publishableTasks, fallbackLegacyTaskIDs, nil
|
return publishableTasks, deferredTaskIDs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func monitorDesktopTaskSpecsNeedTargetLookup(specs []monitorDesktopTaskSpec) bool {
|
func monitorDesktopTaskSpecsNeedTargetLookup(specs []monitorDesktopTaskSpec) bool {
|
||||||
@@ -738,6 +744,35 @@ func (s *MonitoringService) fallbackMonitorDesktopTasksToLegacy(ctx context.Cont
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func deferPhase2MonitorCollectTasks(ctx context.Context, tx pgx.Tx, taskIDs []int64, delay time.Duration) error {
|
||||||
|
if tx == nil || len(taskIDs) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if delay <= 0 {
|
||||||
|
delay = defaultMonitoringDailyDesktopRetryCooldown
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(ctx, `
|
||||||
|
UPDATE monitoring_collect_tasks
|
||||||
|
SET status = 'pending',
|
||||||
|
lease_token_hash = NULL,
|
||||||
|
leased_to_executor = NULL,
|
||||||
|
leased_at = NULL,
|
||||||
|
lease_expires_at = NULL,
|
||||||
|
callback_received_at = NULL,
|
||||||
|
completed_at = NULL,
|
||||||
|
skip_reason = NULL,
|
||||||
|
dispatch_after = NOW() + $2::interval,
|
||||||
|
error_message = NULL,
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id = ANY($1::bigint[])
|
||||||
|
AND COALESCE(execution_owner, 'legacy') = 'desktop_tasks'
|
||||||
|
AND callback_received_at IS NULL
|
||||||
|
`, uniqueInt64s(taskIDs), formatPgInterval(delay)); err != nil {
|
||||||
|
return response.ErrInternal(50041, "task_defer_failed", "failed to defer phase2 monitor tasks until a desktop client is online")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *MonitoringService) deferQueuedPhase2MonitorTasks(
|
func (s *MonitoringService) deferQueuedPhase2MonitorTasks(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
targetClientID uuid.UUID,
|
targetClientID uuid.UUID,
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
@@ -193,15 +194,37 @@ func TestCompleteLinkedDesktopMonitorTaskOnlyFinalizesInProgress(t *testing.T) {
|
|||||||
assert.NotContains(t, query, "'unknown'")
|
assert.NotContains(t, query, "'unknown'")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMonitoringSkipOutcomeForRuntimeUnknownFailsTask(t *testing.T) {
|
func TestMonitoringSkipOutcomeForRuntimeUnknownRequeuesAutomaticTask(t *testing.T) {
|
||||||
outcome := monitoringSkipOutcomeForReason("runtime_unknown")
|
outcome := monitoringSkipOutcomeForReason("runtime_unknown", 1, "automatic")
|
||||||
|
|
||||||
|
assert.Equal(t, "pending", outcome.TaskStatus)
|
||||||
|
assert.True(t, outcome.Requeue)
|
||||||
|
assert.Equal(t, 2*time.Minute, outcome.RetryAfter)
|
||||||
|
assert.Empty(t, outcome.StoredSkipReason)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitoringSkipOutcomeForRuntimeUnknownFailsAfterMaxAttempts(t *testing.T) {
|
||||||
|
outcome := monitoringSkipOutcomeForReason(
|
||||||
|
"runtime_unknown",
|
||||||
|
monitoringRuntimeUnknownMaxDispatchAttempts,
|
||||||
|
"automatic",
|
||||||
|
)
|
||||||
|
|
||||||
assert.Equal(t, "failed", outcome.TaskStatus)
|
assert.Equal(t, "failed", outcome.TaskStatus)
|
||||||
|
assert.False(t, outcome.Requeue)
|
||||||
|
assert.Empty(t, outcome.StoredSkipReason)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitoringSkipOutcomeForManualRuntimeUnknownFailsTask(t *testing.T) {
|
||||||
|
outcome := monitoringSkipOutcomeForReason("runtime_unknown", 0, "manual")
|
||||||
|
|
||||||
|
assert.Equal(t, "failed", outcome.TaskStatus)
|
||||||
|
assert.False(t, outcome.Requeue)
|
||||||
assert.Empty(t, outcome.StoredSkipReason)
|
assert.Empty(t, outcome.StoredSkipReason)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMonitoringSkipOutcomeKeepsOrdinarySkip(t *testing.T) {
|
func TestMonitoringSkipOutcomeKeepsOrdinarySkip(t *testing.T) {
|
||||||
outcome := monitoringSkipOutcomeForReason("stale_business_date")
|
outcome := monitoringSkipOutcomeForReason("stale_business_date", 0, "automatic")
|
||||||
|
|
||||||
assert.Equal(t, "skipped", outcome.TaskStatus)
|
assert.Equal(t, "skipped", outcome.TaskStatus)
|
||||||
assert.Equal(t, "stale_business_date", outcome.StoredSkipReason)
|
assert.Equal(t, "stale_business_date", outcome.StoredSkipReason)
|
||||||
|
|||||||
@@ -993,24 +993,21 @@ func (s *MonitoringService) CollectNow(
|
|||||||
phase2DispatchReady := executionOwner == "desktop_tasks"
|
phase2DispatchReady := executionOwner == "desktop_tasks"
|
||||||
if len(phase2TaskSpecs) > 0 {
|
if len(phase2TaskSpecs) > 0 {
|
||||||
phase2TaskCount := len(phase2TaskSpecs)
|
phase2TaskCount := len(phase2TaskSpecs)
|
||||||
publishedTasks, fallbackLegacyTaskIDs, dispatchErr := s.dispatchMonitorDesktopTasks(ctx, phase2TaskSpecs)
|
publishedTasks, deferredTaskIDs, dispatchErr := s.dispatchMonitorDesktopTasks(ctx, phase2TaskSpecs)
|
||||||
if dispatchErr != nil {
|
if dispatchErr != nil {
|
||||||
if fallbackErr := s.fallbackMonitorDesktopTasksToLegacy(ctx, monitorDesktopTaskSpecIDs(phase2TaskSpecs)); fallbackErr != nil {
|
|
||||||
return nil, fallbackErr
|
|
||||||
}
|
|
||||||
phase2DispatchReady = false
|
phase2DispatchReady = false
|
||||||
phase2TaskSpecs = nil
|
phase2TaskSpecs = nil
|
||||||
if s.logger != nil {
|
if s.logger != nil {
|
||||||
s.logger.Warn("phase2 monitor desktop dispatch failed, falling back to legacy execution",
|
s.logger.Warn("phase2 monitor desktop dispatch failed; pending tasks will retry when a desktop client is online",
|
||||||
zap.Error(dispatchErr),
|
zap.Error(dispatchErr),
|
||||||
zap.Int("phase2_task_count", phase2TaskCount),
|
zap.Int("phase2_task_count", phase2TaskCount),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if len(fallbackLegacyTaskIDs) > 0 {
|
if len(deferredTaskIDs) > 0 && s.logger != nil {
|
||||||
if fallbackErr := s.fallbackMonitorDesktopTasksToLegacy(ctx, fallbackLegacyTaskIDs); fallbackErr != nil {
|
s.logger.Info("phase2 monitor desktop tasks deferred until a desktop client is online",
|
||||||
return nil, fallbackErr
|
zap.Int("deferred_task_count", len(deferredTaskIDs)),
|
||||||
}
|
)
|
||||||
}
|
}
|
||||||
phase2PublishedTasks = publishedTasks
|
phase2PublishedTasks = publishedTasks
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user