root
9fa091c150
Introduce a desktop.task.dispatch topic exchange with per-client routing keys. Tenant-api publishes a task_available frame keyed by target client ID when a publish job is created; every instance binds its own transient queue and forwards to the matching WebSocket (/api/desktop/dispatch) it owns. Desktop-client now prefers this push channel and only falls back to HTTP /lease polling when the socket is down. Also drop admin-web monitoring-plugin remnants and scope the publish modal account list to publish platforms. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1953 lines
56 KiB
TypeScript
1953 lines
56 KiB
TypeScript
import { hostname } from "node:os";
|
||
|
||
import { ApiClientError } from "@geo/http-client";
|
||
import type {
|
||
DesktopArticleContent,
|
||
DesktopAccountInfo,
|
||
DesktopClientInfo,
|
||
DesktopRuntimeSessionSyncRequest,
|
||
DesktopTaskEventMessage,
|
||
DesktopTaskInfo,
|
||
JsonValue,
|
||
LeaseDesktopTaskResponse,
|
||
} from "@geo/shared-types";
|
||
import { getAIPlatformCatalogItem, isAIPlatformId } from "@geo/shared-types";
|
||
|
||
import {
|
||
doubaoAdapter,
|
||
qwenAdapter,
|
||
toutiaoAdapter,
|
||
zhihuAdapter,
|
||
type AdapterExecutionResult,
|
||
type MonitorAdapter,
|
||
type PublishAdapter,
|
||
} from "./adapters";
|
||
import {
|
||
type PublishAccountIdentity,
|
||
type PublishAccountProfile,
|
||
} from "./account-binder";
|
||
import {
|
||
ensureAccountReady,
|
||
forgetTrackedAccountHealth,
|
||
getProjectedAccountHealth,
|
||
probeTrackedAccount,
|
||
reportAccountFailure,
|
||
syncTrackedAccounts,
|
||
} from "./account-health";
|
||
import {
|
||
buildAccountIdentityKey,
|
||
normalizeAccountPlatformUid,
|
||
sameAccountPlatformUid,
|
||
} from "../shared/account-identity";
|
||
import { releaseHotView, retainHotView } from "./view-pool";
|
||
import { getLeaseManagerSnapshot, noteLeaseExtended, noteLeaseReleased, setActiveLease } from "./lease-manager";
|
||
import {
|
||
enqueueMonitorTaskFromEvent,
|
||
getMonitorSchedulerSnapshot,
|
||
initMonitorScheduler,
|
||
noteMonitorTaskCanceled,
|
||
noteMonitorTaskCompleted,
|
||
noteMonitorTaskLeaseMiss,
|
||
noteMonitorTaskLeased,
|
||
selectNextMonitorTask,
|
||
} from "./monitor-scheduler";
|
||
import {
|
||
retainHiddenPlaywrightPage,
|
||
startHiddenPlaywrightReaper,
|
||
} from "./playwright-cdp";
|
||
import { getProcessMetricsSnapshot } from "./process-metrics";
|
||
import {
|
||
initScheduler,
|
||
noteSchedulerAccountsSync,
|
||
noteSchedulerHeartbeat,
|
||
noteSchedulerPull,
|
||
noteSchedulerSseEvent,
|
||
noteSchedulerTaskFinished,
|
||
setSchedulerCurrentTask,
|
||
setSchedulerError,
|
||
setSchedulerNextHeartbeat,
|
||
setSchedulerNextPull,
|
||
setSchedulerPhase,
|
||
setSchedulerQueueDepth,
|
||
} from "./scheduler";
|
||
import { clearSessionHandle, createSessionHandle } from "./session-registry";
|
||
import {
|
||
completeDesktopTask,
|
||
configureTransport,
|
||
deleteDesktopAccount,
|
||
extendDesktopTask,
|
||
getDesktopArticleContent,
|
||
heartbeatDesktopClient,
|
||
leaseDesktopTask,
|
||
listDesktopAccounts,
|
||
noteTransportHeartbeat,
|
||
noteTransportPull,
|
||
offlineDesktopClient,
|
||
revokeDesktopClient,
|
||
setTransportAuthState,
|
||
setTransportSseState,
|
||
upsertDesktopAccount,
|
||
} from "./transport/api-client";
|
||
import { SseClient, SseClientError } from "./transport/sse-client";
|
||
import { DispatchWsClient } from "./transport/dispatch-ws-client";
|
||
|
||
type RuntimeTone = "info" | "success" | "warn" | "danger";
|
||
type RuntimeTaskStatus =
|
||
| "queued"
|
||
| "in_progress"
|
||
| "succeeded"
|
||
| "failed"
|
||
| "unknown"
|
||
| "aborted";
|
||
type RuntimeTaskRouting = "rabbitmq-primary" | "db-recovery";
|
||
|
||
const heartbeatIntervalMs = 25_000;
|
||
const pullIntervalMs = 60_000;
|
||
const leaseExtendIntervalMs = 60_000;
|
||
const maxActivityItems = 60;
|
||
const monitorBusinessTimeZone = "Asia/Shanghai";
|
||
const authRequiredMonitorPlatforms = new Set(["yuanbao", "kimi", "deepseek"]);
|
||
|
||
interface RuntimeTaskRecord {
|
||
id: string;
|
||
jobId: string;
|
||
title: string;
|
||
kind: "publish" | "monitor";
|
||
platform: string;
|
||
accountId: string;
|
||
accountName: string;
|
||
clientId: string;
|
||
status: RuntimeTaskStatus;
|
||
routing: RuntimeTaskRouting;
|
||
leaseExpiresAt: number | null;
|
||
updatedAt: number;
|
||
summary: string;
|
||
payload: Record<string, JsonValue>;
|
||
error: Record<string, JsonValue> | null;
|
||
result: Record<string, JsonValue> | null;
|
||
attemptId: string | null;
|
||
leaseToken: string | null;
|
||
}
|
||
|
||
export interface RuntimeControllerActivity {
|
||
id: string;
|
||
severity: RuntimeTone;
|
||
title: string;
|
||
detail: string;
|
||
at: number;
|
||
}
|
||
|
||
export interface RuntimeControllerSnapshot {
|
||
session: DesktopRuntimeSessionSyncRequest | null;
|
||
client: DesktopClientInfo | null;
|
||
running: boolean;
|
||
sseConnected: boolean;
|
||
queueDepth: number;
|
||
accounts: DesktopAccountInfo[];
|
||
accountProfiles: Record<string, PublishAccountProfile>;
|
||
tasks: Array<Omit<RuntimeTaskRecord, "leaseToken">>;
|
||
activity: RuntimeControllerActivity[];
|
||
lastHeartbeatAt: number;
|
||
lastHeartbeatStatus: "idle" | "success" | "failed";
|
||
lastPullAt: number;
|
||
lastPullStatus: "idle" | "success" | "failed";
|
||
lastAccountsSyncAt: number;
|
||
lastServerTime: string | null;
|
||
lastError: string | null;
|
||
}
|
||
|
||
interface PendingTaskRequest {
|
||
taskId: string;
|
||
routing: RuntimeTaskRouting;
|
||
}
|
||
|
||
interface ActiveRuntimeExecution {
|
||
taskId: string;
|
||
kind: "publish" | "monitor";
|
||
platform: string;
|
||
accountId: string;
|
||
abortController: AbortController;
|
||
}
|
||
|
||
interface RuntimeState {
|
||
session: DesktopRuntimeSessionSyncRequest | null;
|
||
client: DesktopClientInfo | null;
|
||
running: boolean;
|
||
sseConnected: boolean;
|
||
publishQueue: PendingTaskRequest[];
|
||
queuedPublishTaskIds: Set<string>;
|
||
tasks: Map<string, RuntimeTaskRecord>;
|
||
accounts: DesktopAccountInfo[];
|
||
accountProfiles: Map<string, PublishAccountProfile>;
|
||
activity: RuntimeControllerActivity[];
|
||
activeExecutions: Map<string, ActiveRuntimeExecution>;
|
||
leaseInFlight: boolean;
|
||
heartbeatTimer: ReturnType<typeof setInterval> | null;
|
||
pullTimer: ReturnType<typeof setInterval> | null;
|
||
sseClient: SseClient | null;
|
||
dispatchWsClient: DispatchWsClient | null;
|
||
dispatchWsConnected: boolean;
|
||
lastHeartbeatAt: number;
|
||
lastHeartbeatStatus: "idle" | "success" | "failed";
|
||
lastPullAt: number;
|
||
lastPullStatus: "idle" | "success" | "failed";
|
||
lastAccountsSyncAt: number;
|
||
lastServerTime: string | null;
|
||
lastError: string | null;
|
||
activitySeq: number;
|
||
}
|
||
|
||
const state: RuntimeState = {
|
||
session: null,
|
||
client: null,
|
||
running: false,
|
||
sseConnected: false,
|
||
publishQueue: [],
|
||
queuedPublishTaskIds: new Set<string>(),
|
||
tasks: new Map<string, RuntimeTaskRecord>(),
|
||
accounts: [],
|
||
accountProfiles: new Map<string, PublishAccountProfile>(),
|
||
activity: [],
|
||
activeExecutions: new Map<string, ActiveRuntimeExecution>(),
|
||
leaseInFlight: false,
|
||
heartbeatTimer: null,
|
||
pullTimer: null,
|
||
sseClient: null,
|
||
dispatchWsClient: null,
|
||
dispatchWsConnected: false,
|
||
lastHeartbeatAt: 0,
|
||
lastHeartbeatStatus: "idle",
|
||
lastPullAt: 0,
|
||
lastPullStatus: "idle",
|
||
lastAccountsSyncAt: 0,
|
||
lastServerTime: null,
|
||
lastError: null,
|
||
activitySeq: 0,
|
||
};
|
||
|
||
function accountIdentityFromDesktopAccount(account: DesktopAccountInfo): PublishAccountIdentity {
|
||
return {
|
||
id: account.id,
|
||
platform: account.platform,
|
||
platformUid: normalizeAccountPlatformUid(account.platform_uid) || account.platform_uid,
|
||
displayName: account.display_name,
|
||
};
|
||
}
|
||
|
||
function accountIdentityFromTask(task: RuntimeTaskRecord): PublishAccountIdentity | null {
|
||
if (!task.accountId) {
|
||
return null;
|
||
}
|
||
|
||
const matched = state.accounts.find((account) => account.id === task.accountId) ?? null;
|
||
if (matched) {
|
||
return accountIdentityFromDesktopAccount(matched);
|
||
}
|
||
|
||
return {
|
||
id: task.accountId,
|
||
platform: task.platform,
|
||
platformUid: "",
|
||
displayName: task.accountName || "待同步账号",
|
||
};
|
||
}
|
||
|
||
function isDefinitiveAuthState(authState: string): boolean {
|
||
return ["active", "expired", "challenge_required", "revoked"].includes(authState);
|
||
}
|
||
|
||
function isoFromTimestamp(value: number | null): string | null {
|
||
return value ? new Date(value).toISOString() : null;
|
||
}
|
||
|
||
export function syncRuntimeSession(next: DesktopRuntimeSessionSyncRequest | null): void {
|
||
const normalized = normalizeSession(next);
|
||
const changed = !sameSession(state.session, normalized);
|
||
|
||
state.session = normalized;
|
||
state.client = normalized?.desktop_client ?? null;
|
||
|
||
if (!canRunLive(normalized)) {
|
||
stopRuntime();
|
||
clearRuntimeState();
|
||
configureTransport(null);
|
||
setTransportAuthState("pending");
|
||
return;
|
||
}
|
||
|
||
configureTransport(normalized);
|
||
setTransportAuthState("registered");
|
||
|
||
if (changed) {
|
||
stopRuntime();
|
||
clearRuntimeState();
|
||
recordActivity(
|
||
"info",
|
||
"任务消费端已就绪",
|
||
`客户端 ${normalized.desktop_client?.device_name ?? normalized.desktop_client?.id ?? "未知客户端"} 已准备消费桌面任务。`,
|
||
);
|
||
}
|
||
|
||
if (!state.running) {
|
||
startRuntime();
|
||
}
|
||
}
|
||
|
||
export async function releaseRuntimeSession(options: { revoke?: boolean } = {}): Promise<void> {
|
||
const currentSession = state.session;
|
||
const shouldReleaseRemote = canRunLive(currentSession);
|
||
|
||
if (shouldReleaseRemote) {
|
||
try {
|
||
if (options.revoke) {
|
||
await revokeDesktopClient();
|
||
} else {
|
||
await offlineDesktopClient();
|
||
}
|
||
} catch (error) {
|
||
console.warn("[desktop-runtime] release runtime session failed", {
|
||
revoke: Boolean(options.revoke),
|
||
message: errorMessage(error),
|
||
});
|
||
}
|
||
}
|
||
|
||
stopRuntime();
|
||
clearRuntimeState();
|
||
configureTransport(null);
|
||
setTransportAuthState("pending");
|
||
state.session = null;
|
||
state.client = null;
|
||
}
|
||
|
||
export function getRuntimeControllerSnapshot(): RuntimeControllerSnapshot {
|
||
return {
|
||
session: state.session ? { ...state.session } : null,
|
||
client: state.client ? { ...state.client } : null,
|
||
running: state.running,
|
||
sseConnected: state.sseConnected,
|
||
queueDepth: localQueueDepth(),
|
||
accounts: state.accounts.map((item) => ({ ...item })),
|
||
accountProfiles: Object.fromEntries(
|
||
[...state.accountProfiles.entries()].map(([accountId, profile]) => [accountId, { ...profile }]),
|
||
),
|
||
tasks: [...state.tasks.values()]
|
||
.sort((left, right) => right.updatedAt - left.updatedAt)
|
||
.map(({ leaseToken: _leaseToken, ...task }) => ({ ...task })),
|
||
activity: [...state.activity],
|
||
lastHeartbeatAt: state.lastHeartbeatAt,
|
||
lastHeartbeatStatus: state.lastHeartbeatStatus,
|
||
lastPullAt: state.lastPullAt,
|
||
lastPullStatus: state.lastPullStatus,
|
||
lastAccountsSyncAt: state.lastAccountsSyncAt,
|
||
lastServerTime: state.lastServerTime,
|
||
lastError: state.lastError,
|
||
};
|
||
}
|
||
|
||
function startRuntime(): void {
|
||
if (!canRunLive(state.session) || state.running) {
|
||
return;
|
||
}
|
||
|
||
initScheduler();
|
||
initMonitorScheduler();
|
||
startHiddenPlaywrightReaper();
|
||
state.running = true;
|
||
setSchedulerPhase("running");
|
||
setSchedulerError(null);
|
||
recordActivity("success", "调度节点已启动", "已启动 SSE、心跳上报和兜底拉取。");
|
||
|
||
connectEventStream();
|
||
connectDispatchWs();
|
||
scheduleHeartbeatLoop();
|
||
schedulePullLoop();
|
||
|
||
void sendHeartbeat("startup");
|
||
void syncAccounts("startup");
|
||
pumpExecutionLoop();
|
||
}
|
||
|
||
function stopRuntime(): void {
|
||
state.running = false;
|
||
state.sseConnected = false;
|
||
state.leaseInFlight = false;
|
||
for (const execution of state.activeExecutions.values()) {
|
||
execution.abortController.abort();
|
||
}
|
||
state.activeExecutions.clear();
|
||
|
||
if (state.heartbeatTimer) {
|
||
clearInterval(state.heartbeatTimer);
|
||
state.heartbeatTimer = null;
|
||
}
|
||
if (state.pullTimer) {
|
||
clearInterval(state.pullTimer);
|
||
state.pullTimer = null;
|
||
}
|
||
if (state.sseClient) {
|
||
state.sseClient.stop();
|
||
state.sseClient = null;
|
||
}
|
||
if (state.dispatchWsClient) {
|
||
state.dispatchWsClient.stop();
|
||
state.dispatchWsClient = null;
|
||
}
|
||
state.dispatchWsConnected = false;
|
||
|
||
setTransportSseState("idle");
|
||
setSchedulerPhase("idle");
|
||
setSchedulerCurrentTask(null);
|
||
setSchedulerQueueDepth(localQueueDepth());
|
||
setSchedulerNextHeartbeat(null);
|
||
setSchedulerNextPull(null);
|
||
|
||
if (getLeaseManagerSnapshot().activeTaskId) {
|
||
noteLeaseReleased("cleared");
|
||
}
|
||
}
|
||
|
||
function clearRuntimeState(): void {
|
||
state.publishQueue = [];
|
||
state.queuedPublishTaskIds.clear();
|
||
state.tasks.clear();
|
||
state.accounts = [];
|
||
state.accountProfiles.clear();
|
||
state.activeExecutions.clear();
|
||
state.lastHeartbeatAt = 0;
|
||
state.lastHeartbeatStatus = "idle";
|
||
state.lastPullAt = 0;
|
||
state.lastPullStatus = "idle";
|
||
state.lastAccountsSyncAt = 0;
|
||
state.lastServerTime = null;
|
||
state.lastError = null;
|
||
setSchedulerQueueDepth(localQueueDepth());
|
||
}
|
||
|
||
function scheduleHeartbeatLoop(): void {
|
||
if (state.heartbeatTimer) {
|
||
clearInterval(state.heartbeatTimer);
|
||
}
|
||
|
||
setSchedulerNextHeartbeat(Date.now() + heartbeatIntervalMs);
|
||
state.heartbeatTimer = setInterval(() => {
|
||
setSchedulerNextHeartbeat(Date.now() + heartbeatIntervalMs);
|
||
void sendHeartbeat("timer");
|
||
}, heartbeatIntervalMs);
|
||
}
|
||
|
||
function schedulePullLoop(): void {
|
||
if (state.pullTimer) {
|
||
clearInterval(state.pullTimer);
|
||
}
|
||
|
||
setSchedulerNextPull(Date.now() + pullIntervalMs);
|
||
state.pullTimer = setInterval(() => {
|
||
setSchedulerNextPull(Date.now() + pullIntervalMs);
|
||
pumpExecutionLoop();
|
||
}, pullIntervalMs);
|
||
}
|
||
|
||
function connectEventStream(): void {
|
||
if (!canRunLive(state.session)) {
|
||
return;
|
||
}
|
||
|
||
state.sseClient?.stop();
|
||
|
||
const sseClient = new SseClient({
|
||
url: buildApiUrl(state.session.api_base_url, "/api/desktop/events"),
|
||
headers: {
|
||
Authorization: `Bearer ${state.session.client_token}`,
|
||
},
|
||
});
|
||
|
||
sseClient.on("state", (payload) => {
|
||
if (payload === "streaming") {
|
||
setTransportSseState("streaming");
|
||
return;
|
||
}
|
||
if (payload === "connecting") {
|
||
setTransportSseState("connecting");
|
||
return;
|
||
}
|
||
setTransportSseState("idle");
|
||
});
|
||
|
||
sseClient.on("connected", (payload) => {
|
||
state.sseConnected = true;
|
||
noteSchedulerSseEvent();
|
||
setSchedulerError(null);
|
||
|
||
if (isConnectedEvent(payload)) {
|
||
state.lastServerTime = payload.server_time;
|
||
}
|
||
|
||
recordActivity("success", "单点事件流已连接", "服务端已建立定向任务事件流。");
|
||
});
|
||
|
||
sseClient.on("ping", () => {
|
||
state.sseConnected = true;
|
||
noteSchedulerSseEvent();
|
||
});
|
||
|
||
const taskEventHandler = (payload: unknown) => {
|
||
if (!isDesktopTaskEvent(payload)) {
|
||
return;
|
||
}
|
||
state.sseConnected = true;
|
||
noteSchedulerSseEvent();
|
||
handleTaskEvent(payload);
|
||
};
|
||
|
||
sseClient.on("task_available", taskEventHandler);
|
||
sseClient.on("task_leased", taskEventHandler);
|
||
sseClient.on("task_extended", taskEventHandler);
|
||
sseClient.on("task_completed", taskEventHandler);
|
||
sseClient.on("task_canceled", taskEventHandler);
|
||
sseClient.on("task_reconciled", taskEventHandler);
|
||
|
||
sseClient.on("error", (payload) => {
|
||
state.sseConnected = false;
|
||
|
||
if (payload instanceof SseClientError && payload.status === 401) {
|
||
handleAuthExpired(payload);
|
||
return;
|
||
}
|
||
|
||
const message = errorMessage(payload);
|
||
state.lastError = message;
|
||
setSchedulerError(message);
|
||
recordActivity("warn", "单点事件流重连中", message);
|
||
});
|
||
|
||
sseClient.on("reconnect", (payload) => {
|
||
if (!isReconnectPayload(payload)) {
|
||
return;
|
||
}
|
||
recordActivity("info", "单点事件流退避等待", `将在 ${payload.delayMs}ms 后重连 SSE。`);
|
||
});
|
||
|
||
state.sseClient = sseClient;
|
||
sseClient.start();
|
||
}
|
||
|
||
function connectDispatchWs(): void {
|
||
if (!canRunLive(state.session)) {
|
||
return;
|
||
}
|
||
|
||
state.dispatchWsClient?.stop();
|
||
|
||
const wsUrl = buildApiUrl(state.session.api_base_url, "/api/desktop/dispatch")
|
||
.replace(/^http:\/\//, "ws://")
|
||
.replace(/^https:\/\//, "wss://");
|
||
|
||
const client = new DispatchWsClient({
|
||
url: wsUrl,
|
||
headers: {
|
||
Authorization: `Bearer ${state.session.client_token}`,
|
||
},
|
||
});
|
||
|
||
client.on("open", () => {
|
||
state.dispatchWsConnected = true;
|
||
setSchedulerError(null);
|
||
recordActivity("success", "任务分发通道已连接", "AMQP 直推 WebSocket 已就绪,发布任务将以低延迟到达。");
|
||
});
|
||
|
||
client.on("connected", () => {
|
||
state.dispatchWsConnected = true;
|
||
});
|
||
|
||
client.on("task_available", (payload) => {
|
||
if (!isDesktopTaskEvent(payload)) {
|
||
return;
|
||
}
|
||
state.dispatchWsConnected = true;
|
||
handleTaskEvent(payload);
|
||
});
|
||
|
||
client.on("error", (payload) => {
|
||
state.dispatchWsConnected = false;
|
||
const message = errorMessage(payload);
|
||
recordActivity("warn", "任务分发通道异常", message);
|
||
});
|
||
|
||
client.on("close", () => {
|
||
state.dispatchWsConnected = false;
|
||
});
|
||
|
||
client.on("reconnect", (payload) => {
|
||
if (!isReconnectPayload(payload)) {
|
||
return;
|
||
}
|
||
recordActivity("info", "任务分发通道退避等待", `将在 ${payload.delayMs}ms 后重连 WebSocket。`);
|
||
});
|
||
|
||
state.dispatchWsClient = client;
|
||
client.start();
|
||
}
|
||
|
||
function handleTaskEvent(event: DesktopTaskEventMessage): void {
|
||
const existing = state.tasks.get(event.task_id);
|
||
const next: RuntimeTaskRecord = {
|
||
id: event.task_id,
|
||
jobId: event.job_id,
|
||
title: existing?.title ?? defaultTaskTitle(event.kind, event.platform || existing?.platform),
|
||
kind: event.kind,
|
||
platform: event.platform || existing?.platform || inferPlatformFromAccount(event.target_account_id),
|
||
accountId: event.target_account_id || existing?.accountId || "",
|
||
accountName: existing?.accountName ?? "待同步账号",
|
||
clientId: event.target_client_id,
|
||
status: event.status,
|
||
routing: existing?.routing ?? "rabbitmq-primary",
|
||
leaseExpiresAt: event.status === "in_progress" ? existing?.leaseExpiresAt ?? null : null,
|
||
updatedAt: parseTimestamp(event.updated_at) ?? Date.now(),
|
||
summary: summaryFromEventType(event.type, event.status),
|
||
payload: existing?.payload ?? {},
|
||
error: existing?.error ?? null,
|
||
result: existing?.result ?? null,
|
||
attemptId: event.status === "in_progress" ? existing?.attemptId ?? null : null,
|
||
leaseToken: event.status === "in_progress" ? existing?.leaseToken ?? null : null,
|
||
};
|
||
|
||
state.tasks.set(event.task_id, next);
|
||
|
||
if (event.kind === "monitor") {
|
||
if (event.type === "task_available") {
|
||
enqueueMonitorTaskFromEvent(event, "rabbitmq-primary");
|
||
}
|
||
if (event.type === "task_completed" || event.type === "task_reconciled" || event.type === "task_canceled") {
|
||
noteMonitorTaskCanceled(event.task_id);
|
||
}
|
||
}
|
||
|
||
if (event.type === "task_available") {
|
||
if (event.kind === "publish") {
|
||
enqueuePublishTask(event.task_id, "rabbitmq-primary");
|
||
}
|
||
pumpExecutionLoop();
|
||
}
|
||
}
|
||
|
||
async function sendHeartbeat(source: "startup" | "timer"): Promise<void> {
|
||
if (!canRunLive(state.session)) {
|
||
return;
|
||
}
|
||
|
||
const previousStatus = state.lastHeartbeatStatus;
|
||
|
||
try {
|
||
const response = await heartbeatDesktopClient({
|
||
device_name: hostname() || state.client?.device_name || `Desktop ${process.platform}`,
|
||
os: process.platform,
|
||
cpu_arch: process.arch,
|
||
client_version: state.client?.client_version ?? "0.1.0-dev",
|
||
channel: state.client?.channel ?? "dev",
|
||
account_ids: state.accounts
|
||
.filter((account) =>
|
||
getProjectedAccountHealth({
|
||
accountId: account.id,
|
||
platform: account.platform,
|
||
health: account.health,
|
||
verifiedAt: account.verified_at,
|
||
}).health === "live",
|
||
)
|
||
.map((account) => account.id),
|
||
});
|
||
|
||
state.client = response.client;
|
||
state.lastHeartbeatAt = Date.now();
|
||
state.lastHeartbeatStatus = "success";
|
||
state.lastServerTime = response.server_time;
|
||
state.lastError = null;
|
||
|
||
noteTransportHeartbeat(true);
|
||
noteSchedulerHeartbeat();
|
||
setSchedulerError(null);
|
||
|
||
if (source === "startup" || previousStatus === "failed") {
|
||
recordActivity("success", "心跳连接正常", "客户端心跳已回写到服务端。");
|
||
}
|
||
} catch (error) {
|
||
state.lastHeartbeatAt = Date.now();
|
||
state.lastHeartbeatStatus = "failed";
|
||
state.lastError = errorMessage(error);
|
||
|
||
noteTransportHeartbeat(false);
|
||
setSchedulerError(state.lastError);
|
||
|
||
if (isApiClientError(error, 401)) {
|
||
handleAuthExpired(error);
|
||
return;
|
||
}
|
||
|
||
if (source === "startup" || previousStatus !== "failed") {
|
||
recordActivity("danger", "心跳连接失败", state.lastError);
|
||
}
|
||
}
|
||
}
|
||
|
||
async function syncAccounts(source: "startup" | "manual"): Promise<void> {
|
||
if (!canRunLive(state.session)) {
|
||
return;
|
||
}
|
||
|
||
try {
|
||
const accounts = await listDesktopAccounts();
|
||
const currentClientId = state.client?.id ?? state.session?.desktop_client?.id ?? null;
|
||
state.lastAccountsSyncAt = Date.now();
|
||
state.lastError = null;
|
||
|
||
noteSchedulerAccountsSync();
|
||
setSchedulerError(null);
|
||
|
||
const identities = accounts.map((account) => accountIdentityFromDesktopAccount(account));
|
||
syncTrackedAccounts(identities);
|
||
if (source === "manual") {
|
||
await Promise.allSettled(
|
||
identities.map((account) =>
|
||
probeTrackedAccount(account, {
|
||
trigger: "manual",
|
||
allowSilentRefresh: true,
|
||
}),
|
||
),
|
||
);
|
||
}
|
||
|
||
state.accountProfiles.clear();
|
||
const syncedAccounts: Array<DesktopAccountInfo | null> = await Promise.all(accounts.map(async (account) => {
|
||
const normalizedPlatformUid = normalizeAccountPlatformUid(account.platform_uid) || account.platform_uid;
|
||
const isAIPlatform = isAIPlatformId(account.platform);
|
||
const identity = accountIdentityFromDesktopAccount(account);
|
||
const projected = getProjectedAccountHealth({
|
||
accountId: account.id,
|
||
platform: account.platform,
|
||
health: account.health,
|
||
verifiedAt: account.verified_at,
|
||
});
|
||
|
||
if (projected.profile?.subjectUid && projected.profile.displayName) {
|
||
state.accountProfiles.set(account.id, {
|
||
platformUid: projected.profile.subjectUid,
|
||
displayName: projected.profile.displayName,
|
||
avatarUrl: projected.profile.avatarUrl,
|
||
});
|
||
}
|
||
|
||
const normalizedLocalPlatformUid = projected.profile?.subjectUid
|
||
? (normalizeAccountPlatformUid(projected.profile.subjectUid) || projected.profile.subjectUid)
|
||
: null;
|
||
const hasPlatformUidMismatch = Boolean(
|
||
normalizedLocalPlatformUid && !sameAccountPlatformUid(normalizedLocalPlatformUid, normalizedPlatformUid),
|
||
);
|
||
|
||
if (hasPlatformUidMismatch && !isAIPlatform) {
|
||
return {
|
||
...account,
|
||
platform_uid: normalizedPlatformUid,
|
||
health: "expired" as const,
|
||
};
|
||
}
|
||
|
||
let resolvedAccount: DesktopAccountInfo = {
|
||
...account,
|
||
platform_uid: normalizedPlatformUid,
|
||
display_name: projected.profile?.displayName ?? account.display_name,
|
||
avatar_url: projected.profile?.avatarUrl ?? account.avatar_url,
|
||
health: projected.health,
|
||
verified_at: isoFromTimestamp(projected.lastVerifiedAt) ?? account.verified_at,
|
||
};
|
||
|
||
const shouldRelinkClient = Boolean(currentClientId && resolvedAccount.client_id !== currentClientId);
|
||
const shouldMirrorHealth =
|
||
resolvedAccount.health !== account.health
|
||
&& isDefinitiveAuthState(projected.authState);
|
||
|
||
if (shouldRelinkClient || shouldMirrorHealth) {
|
||
try {
|
||
const updated = await upsertDesktopAccount({
|
||
platform: resolvedAccount.platform,
|
||
platform_uid: normalizedPlatformUid,
|
||
display_name: projected.profile?.displayName ?? resolvedAccount.display_name,
|
||
avatar_url: projected.profile?.avatarUrl ?? resolvedAccount.avatar_url,
|
||
account_fingerprint: resolvedAccount.account_fingerprint ?? normalizedPlatformUid,
|
||
health: resolvedAccount.health,
|
||
verified_at: isoFromTimestamp(projected.lastVerifiedAt) ?? resolvedAccount.verified_at,
|
||
tags: resolvedAccount.tags,
|
||
});
|
||
|
||
resolvedAccount = {
|
||
...updated,
|
||
platform_uid: normalizeAccountPlatformUid(updated.platform_uid) || updated.platform_uid,
|
||
health: resolvedAccount.health,
|
||
};
|
||
} catch (error) {
|
||
console.warn("[desktop-runtime] account relink failed", {
|
||
accountId: resolvedAccount.id,
|
||
platform: resolvedAccount.platform,
|
||
platformUid: resolvedAccount.platform_uid,
|
||
message: errorMessage(error),
|
||
});
|
||
}
|
||
}
|
||
|
||
return resolvedAccount;
|
||
}));
|
||
const duplicateAccounts: DesktopAccountInfo[] = [];
|
||
const dedupedAccounts: DesktopAccountInfo[] = [];
|
||
const seenAccountKeys = new Set<string>();
|
||
|
||
for (const account of syncedAccounts) {
|
||
if (!account) {
|
||
continue;
|
||
}
|
||
|
||
const identityKey = buildAccountIdentityKey({
|
||
platform: account.platform,
|
||
platformUid: account.platform_uid,
|
||
displayName: account.display_name,
|
||
});
|
||
if (seenAccountKeys.has(identityKey)) {
|
||
duplicateAccounts.push(account);
|
||
continue;
|
||
}
|
||
|
||
seenAccountKeys.add(identityKey);
|
||
dedupedAccounts.push(account);
|
||
}
|
||
|
||
state.accounts = dedupedAccounts;
|
||
|
||
if (duplicateAccounts.length > 0) {
|
||
void cleanupDuplicateDesktopAccounts(duplicateAccounts);
|
||
}
|
||
|
||
if (source === "startup") {
|
||
recordActivity("info", "本地账号同步", `已同步 ${state.accounts.length} 个桌面账号。`);
|
||
}
|
||
|
||
refreshAccountNames();
|
||
void sendHeartbeat("timer");
|
||
} catch (error) {
|
||
const message = errorMessage(error);
|
||
state.lastError = message;
|
||
setSchedulerError(message);
|
||
|
||
if (isApiClientError(error, 401)) {
|
||
handleAuthExpired(error);
|
||
return;
|
||
}
|
||
|
||
recordActivity("warn", "账号同步失败", message);
|
||
}
|
||
}
|
||
|
||
async function cleanupDuplicateDesktopAccounts(accounts: DesktopAccountInfo[]): Promise<void> {
|
||
for (const account of accounts) {
|
||
try {
|
||
await deleteDesktopAccount(account.id, account.sync_version);
|
||
} catch (error) {
|
||
console.warn("[desktop-runtime] duplicate desktop account cleanup failed", {
|
||
accountId: account.id,
|
||
platform: account.platform,
|
||
platformUid: account.platform_uid,
|
||
message: errorMessage(error),
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
export async function refreshRuntimeAccounts(): Promise<void> {
|
||
await syncAccounts("manual");
|
||
}
|
||
|
||
export async function unbindRuntimeAccount(accountId: string, syncVersion: number): Promise<void> {
|
||
const matchedAccount = state.accounts.find((item) => item.id === accountId) ?? null;
|
||
|
||
await deleteDesktopAccount(accountId, syncVersion);
|
||
await clearSessionHandle(accountId);
|
||
forgetTrackedAccountHealth(accountId);
|
||
|
||
state.accounts = state.accounts.filter((item) => item.id !== accountId);
|
||
state.accountProfiles.delete(accountId);
|
||
refreshAccountNames();
|
||
|
||
if (matchedAccount) {
|
||
recordActivity(
|
||
"info",
|
||
"账号已解绑",
|
||
`${matchedAccount.display_name} 已解绑，并清理了当前机器上的本地会话缓存。`,
|
||
);
|
||
}
|
||
|
||
await syncAccounts("manual");
|
||
}
|
||
|
||
function enqueuePublishTask(taskId: string, routing: RuntimeTaskRouting): void {
|
||
if (!taskId || state.activeExecutions.has(taskId) || state.queuedPublishTaskIds.has(taskId)) {
|
||
return;
|
||
}
|
||
|
||
state.publishQueue.push({ taskId, routing });
|
||
state.queuedPublishTaskIds.add(taskId);
|
||
syncSchedulerSurface();
|
||
}
|
||
|
||
function dequeuePublishTask(): PendingTaskRequest | null {
|
||
const next = state.publishQueue.shift() ?? null;
|
||
if (next) {
|
||
state.queuedPublishTaskIds.delete(next.taskId);
|
||
}
|
||
syncSchedulerSurface();
|
||
return next;
|
||
}
|
||
|
||
function pumpExecutionLoop(): void {
|
||
if (!state.running || state.leaseInFlight) {
|
||
return;
|
||
}
|
||
|
||
syncSchedulerSurface();
|
||
|
||
if (hasActivePublishExecution()) {
|
||
return;
|
||
}
|
||
|
||
if (state.publishQueue.length > 0) {
|
||
if (state.activeExecutions.size === 0) {
|
||
const queuedPublish = dequeuePublishTask();
|
||
if (queuedPublish) {
|
||
void leaseSpecificTask(queuedPublish, "publish");
|
||
}
|
||
}
|
||
return;
|
||
}
|
||
|
||
if (canStartAnotherMonitorExecution()) {
|
||
const selected = selectNextMonitorTask({
|
||
maxConcurrency: resolveAdaptiveMonitorConcurrency(),
|
||
activePlatforms: activeMonitorPlatforms(),
|
||
activeQuestionKeys: activeMonitorQuestionKeys(),
|
||
activeCount: activeMonitorCount(),
|
||
});
|
||
if (selected) {
|
||
void leaseSpecificTask(selected, "monitor");
|
||
return;
|
||
}
|
||
}
|
||
|
||
if (state.activeExecutions.size === 0) {
|
||
// Publish task_available is push-delivered over the dispatch WebSocket. We
|
||
// only fall back to the HTTP pull when the push channel is down, otherwise
|
||
// every scheduler tick would wake the /lease endpoint for no reason.
|
||
if (!state.dispatchWsConnected) {
|
||
void pullNextTask("publish");
|
||
}
|
||
return;
|
||
}
|
||
|
||
if (canStartAnotherMonitorExecution()) {
|
||
// Monitor has not been migrated to the dispatch channel yet, so it still
|
||
// relies on the periodic pull as its primary signal.
|
||
void pullNextTask("monitor");
|
||
}
|
||
}
|
||
|
||
async function leaseSpecificTask(
|
||
request: PendingTaskRequest,
|
||
expectedKind: "publish" | "monitor",
|
||
): Promise<void> {
|
||
if (!canRunLive(state.session) || state.leaseInFlight) {
|
||
return;
|
||
}
|
||
|
||
state.leaseInFlight = true;
|
||
|
||
try {
|
||
const leased = await leaseDesktopTask({ task_id: request.taskId });
|
||
if (!leased.task) {
|
||
if (expectedKind === "monitor") {
|
||
noteMonitorTaskLeaseMiss(request.taskId);
|
||
}
|
||
return;
|
||
}
|
||
state.leaseInFlight = false;
|
||
await executeLeasedTask(leased, request.routing);
|
||
} catch (error) {
|
||
if (expectedKind === "monitor") {
|
||
noteMonitorTaskLeaseMiss(request.taskId);
|
||
}
|
||
|
||
if (isApiClientError(error, 401)) {
|
||
handleAuthExpired(error);
|
||
return;
|
||
}
|
||
|
||
state.lastError = errorMessage(error);
|
||
setSchedulerError(state.lastError);
|
||
recordActivity("warn", "指定任务领取失败", `${request.taskId} 未能成功领取：${state.lastError}`);
|
||
} finally {
|
||
if (state.leaseInFlight) {
|
||
state.leaseInFlight = false;
|
||
syncSchedulerSurface();
|
||
if (state.publishQueue.length > 0 || localQueueDepth() > 0) {
|
||
pumpExecutionLoop();
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
async function pullNextTask(kind: "publish" | "monitor"): Promise<void> {
|
||
if (!canRunLive(state.session) || state.leaseInFlight) {
|
||
return;
|
||
}
|
||
|
||
state.leaseInFlight = true;
|
||
let leasedTask = false;
|
||
|
||
try {
|
||
const leased = await leaseDesktopTask({ kind });
|
||
state.lastPullAt = Date.now();
|
||
state.lastPullStatus = "success";
|
||
noteTransportPull(true);
|
||
noteSchedulerPull();
|
||
|
||
if (!leased.task) {
|
||
return;
|
||
}
|
||
|
||
leasedTask = true;
|
||
state.leaseInFlight = false;
|
||
await executeLeasedTask(leased, "db-recovery");
|
||
} catch (error) {
|
||
state.lastPullAt = Date.now();
|
||
state.lastPullStatus = "failed";
|
||
state.lastError = errorMessage(error);
|
||
|
||
noteTransportPull(false);
|
||
setSchedulerError(state.lastError);
|
||
|
||
if (isApiClientError(error, 401)) {
|
||
handleAuthExpired(error);
|
||
return;
|
||
}
|
||
|
||
recordActivity("warn", "兜底拉取失败", state.lastError);
|
||
} finally {
|
||
state.leaseInFlight = false;
|
||
syncSchedulerSurface();
|
||
|
||
if (!leasedTask && kind === "publish" && canStartAnotherMonitorExecution()) {
|
||
void pullNextTask("monitor");
|
||
}
|
||
}
|
||
}
|
||
|
||
async function executeLeasedTask(
|
||
leased: LeaseDesktopTaskResponse,
|
||
routing: RuntimeTaskRouting,
|
||
): Promise<void> {
|
||
if (!leased.task || !leased.lease_token) {
|
||
return;
|
||
}
|
||
|
||
const task = leased.task;
|
||
|
||
setSchedulerPhase("running");
|
||
|
||
setActiveLease({
|
||
taskId: task.id,
|
||
attemptId: leased.attempt_id ?? null,
|
||
leaseExpiresAt: leased.lease_expires_at ?? task.lease_expires_at ?? null,
|
||
});
|
||
|
||
const taskRecord = upsertTaskFromInfo(task, {
|
||
routing,
|
||
attemptId: leased.attempt_id ?? null,
|
||
leaseToken: leased.lease_token,
|
||
summary: `已领取租约，开始执行 ${routing === "rabbitmq-primary" ? "MQ 唤醒" : "兜底拉取"} 消费。`,
|
||
});
|
||
|
||
recordActivity(
|
||
"info",
|
||
"任务已领取",
|
||
`${taskRecord.title} 已由当前客户端领取，路由：${describeRouting(routing)}。`,
|
||
);
|
||
|
||
const abortController = new AbortController();
|
||
state.activeExecutions.set(taskRecord.id, {
|
||
taskId: taskRecord.id,
|
||
kind: taskRecord.kind,
|
||
platform: taskRecord.platform,
|
||
accountId: taskRecord.accountId,
|
||
abortController,
|
||
});
|
||
syncSchedulerSurface();
|
||
|
||
if (task.kind === "monitor") {
|
||
noteMonitorTaskLeased(task, routing);
|
||
}
|
||
|
||
const extendHandle = setInterval(() => {
|
||
void extendActiveLease(taskRecord.id, leased.lease_token as string);
|
||
}, leaseExtendIntervalMs);
|
||
|
||
try {
|
||
const execution = await executeTaskAdapter(taskRecord, abortController.signal);
|
||
const completed = await completeDesktopTask(taskRecord.id, {
|
||
lease_token: leased.lease_token,
|
||
status: execution.status,
|
||
payload: execution.payload,
|
||
error: execution.error,
|
||
});
|
||
|
||
noteLeaseReleased(execution.status === "failed" ? "failed" : "completed", taskRecord.id);
|
||
upsertTaskFromInfo(completed, {
|
||
routing,
|
||
summary: execution.summary,
|
||
attemptId: null,
|
||
leaseToken: null,
|
||
});
|
||
|
||
recordActivity(
|
||
execution.status === "failed" ? "danger" : execution.status === "unknown" ? "warn" : "success",
|
||
execution.status === "failed"
|
||
? "任务执行失败"
|
||
: execution.status === "unknown"
|
||
? "任务执行结果待确认"
|
||
: "任务执行成功",
|
||
`${taskRecord.title} ${execution.summary}`,
|
||
);
|
||
|
||
if (task.kind === "monitor") {
|
||
noteMonitorTaskCompleted(taskRecord.id);
|
||
}
|
||
} catch (error) {
|
||
const failureMessage = errorMessage(error);
|
||
const structuredError = toStructuredError(error);
|
||
const accountIdentity = accountIdentityFromTask(taskRecord);
|
||
|
||
if (accountIdentity) {
|
||
await reportAccountFailure(accountIdentity, {
|
||
summary: failureMessage,
|
||
error: structuredError,
|
||
}).catch((reportError) => {
|
||
console.warn("[desktop-runtime] thrown failure report failed", {
|
||
taskId: taskRecord.id,
|
||
accountId: accountIdentity.id,
|
||
platform: accountIdentity.platform,
|
||
message: reportError instanceof Error ? reportError.message : String(reportError),
|
||
});
|
||
});
|
||
}
|
||
|
||
try {
|
||
const completed = await completeDesktopTask(taskRecord.id, {
|
||
lease_token: leased.lease_token,
|
||
status: "failed",
|
||
error: structuredError,
|
||
});
|
||
|
||
upsertTaskFromInfo(completed, {
|
||
routing,
|
||
summary: failureMessage || "桌面任务执行失败。",
|
||
attemptId: null,
|
||
leaseToken: null,
|
||
});
|
||
} catch (completeError) {
|
||
const existing = state.tasks.get(taskRecord.id);
|
||
if (existing) {
|
||
existing.status = "failed";
|
||
existing.error = structuredError;
|
||
existing.summary = `${failureMessage || "桌面任务执行失败。"}（result 回写失败：${errorMessage(completeError)}）`;
|
||
existing.updatedAt = Date.now();
|
||
state.tasks.set(taskRecord.id, existing);
|
||
}
|
||
}
|
||
|
||
noteLeaseReleased("failed", taskRecord.id);
|
||
recordActivity("danger", "任务执行失败", `${taskRecord.title} 执行失败：${failureMessage || "未知错误"}`);
|
||
|
||
if (task.kind === "monitor") {
|
||
noteMonitorTaskCompleted(taskRecord.id);
|
||
}
|
||
} finally {
|
||
clearInterval(extendHandle);
|
||
state.activeExecutions.delete(taskRecord.id);
|
||
syncSchedulerSurface();
|
||
noteSchedulerTaskFinished();
|
||
pumpExecutionLoop();
|
||
}
|
||
}
|
||
|
||
async function extendActiveLease(taskId: string, leaseToken: string): Promise<void> {
|
||
try {
|
||
const task = await extendDesktopTask(taskId, { lease_token: leaseToken });
|
||
|
||
noteLeaseExtended(task.lease_expires_at ?? null, taskId);
|
||
|
||
const existing = state.tasks.get(taskId);
|
||
if (existing) {
|
||
existing.leaseExpiresAt = parseTimestamp(task.lease_expires_at) ?? existing.leaseExpiresAt;
|
||
existing.updatedAt = Date.now();
|
||
existing.summary = "租约已自动续期，任务仍在执行。";
|
||
state.tasks.set(taskId, existing);
|
||
}
|
||
} catch (error) {
|
||
const message = errorMessage(error);
|
||
state.lastError = message;
|
||
setSchedulerError(message);
|
||
recordActivity("warn", "租约续期失败", `${taskId} 续租失败：${message}`);
|
||
}
|
||
}
|
||
|
||
async function executeTaskAdapter(
|
||
task: RuntimeTaskRecord,
|
||
signal: AbortSignal,
|
||
): Promise<AdapterExecutionResult> {
|
||
const accountIdentity = accountIdentityFromTask(task);
|
||
if (accountIdentity && shouldEnforceActiveAccount(task)) {
|
||
const readiness = await ensureAccountReady(accountIdentity);
|
||
if (readiness.authState !== "active") {
|
||
return buildAccountBlockedResult(task, readiness);
|
||
}
|
||
}
|
||
|
||
const sessionHandle = createSessionHandle(task.accountId || task.id);
|
||
const payload = task.payload;
|
||
|
||
if (task.kind === "publish") {
|
||
const adapter = selectPublishAdapter(task.platform);
|
||
if (!adapter) {
|
||
return buildScaffoldResult(task, payload, "publish adapter is not implemented yet");
|
||
}
|
||
|
||
const viewHandle =
|
||
adapter.executionMode === "session" || adapter.executionMode === "playwright"
|
||
? null
|
||
: retainHotView(task.accountId || task.id);
|
||
const playwrightLease =
|
||
adapter.executionMode === "playwright"
|
||
? await retainHiddenPlaywrightPage({
|
||
accountId: task.accountId || task.id,
|
||
session: sessionHandle.session,
|
||
targetURL: resolvePlaywrightTargetURL(task.platform, payload),
|
||
title: `${task.title} · Hidden Playwright`,
|
||
})
|
||
: null;
|
||
|
||
try {
|
||
const result = await adapter.publish(
|
||
{
|
||
taskId: task.id,
|
||
accountId: task.accountId || task.id,
|
||
session: sessionHandle.session,
|
||
view: viewHandle?.view ?? null,
|
||
playwright: playwrightLease
|
||
? {
|
||
browser: playwrightLease.browser,
|
||
page: playwrightLease.page,
|
||
}
|
||
: null,
|
||
signal,
|
||
phase: "initial",
|
||
article: await loadPublishArticle(task),
|
||
reportProgress(stage: string) {
|
||
updateTaskProgress(task.id, `publish adapter progress: ${stage}`);
|
||
},
|
||
},
|
||
payload,
|
||
);
|
||
await maybeReportTaskAuthFailure(task, result, accountIdentity);
|
||
return result;
|
||
} finally {
|
||
await playwrightLease?.release();
|
||
if (viewHandle) {
|
||
releaseHotView(viewHandle.accountId);
|
||
}
|
||
}
|
||
}
|
||
|
||
const staleBusinessDate = resolveStaleMonitoringBusinessDate(payload);
|
||
if (staleBusinessDate) {
|
||
return {
|
||
status: "unknown",
|
||
payload: {
|
||
...payload,
|
||
dropped_by_client: true,
|
||
dropped_reason: "stale_business_date",
|
||
},
|
||
error: {
|
||
code: "desktop_monitor_task_stale",
|
||
message: "monitor task is stale and has been dropped by desktop client",
|
||
business_date: staleBusinessDate,
|
||
},
|
||
summary: `${task.title} 已过业务日 ${staleBusinessDate}，按漏采策略直接丢弃。`,
|
||
};
|
||
}
|
||
|
||
const adapter = selectMonitorAdapter(task.platform);
|
||
if (!adapter) {
|
||
return buildScaffoldResult(task, payload, "monitor adapter is not implemented yet");
|
||
}
|
||
|
||
const viewHandle =
|
||
adapter.executionMode === "session" || adapter.executionMode === "playwright"
|
||
? null
|
||
: retainHotView(task.accountId || task.id);
|
||
const playwrightLease =
|
||
adapter.executionMode === "playwright"
|
||
? await retainHiddenPlaywrightPage({
|
||
accountId: task.accountId || task.id,
|
||
session: sessionHandle.session,
|
||
targetURL: resolvePlaywrightTargetURL(task.platform, payload),
|
||
title: `${task.title} · Hidden Playwright`,
|
||
})
|
||
: null;
|
||
|
||
try {
|
||
const result = await adapter.query(
|
||
{
|
||
taskId: task.id,
|
||
accountId: task.accountId || task.id,
|
||
session: sessionHandle.session,
|
||
view: viewHandle?.view ?? null,
|
||
playwright: playwrightLease
|
||
? {
|
||
browser: playwrightLease.browser,
|
||
page: playwrightLease.page,
|
||
}
|
||
: null,
|
||
signal,
|
||
phase: "initial",
|
||
reportProgress(stage: string) {
|
||
updateTaskProgress(task.id, `monitor adapter progress: ${stage}`);
|
||
},
|
||
},
|
||
payload,
|
||
);
|
||
await maybeReportTaskAuthFailure(task, result, accountIdentity);
|
||
return result;
|
||
} finally {
|
||
await playwrightLease?.release();
|
||
if (viewHandle) {
|
||
releaseHotView(viewHandle.accountId);
|
||
}
|
||
}
|
||
}
|
||
|
||
function shouldEnforceActiveAccount(task: RuntimeTaskRecord): boolean {
|
||
if (task.kind === "publish") {
|
||
return true;
|
||
}
|
||
|
||
if (!isAIPlatformId(task.platform)) {
|
||
return true;
|
||
}
|
||
|
||
return authRequiredMonitorPlatforms.has(task.platform);
|
||
}
|
||
|
||
function buildAccountBlockedResult(
|
||
task: RuntimeTaskRecord,
|
||
readiness: Awaited<ReturnType<typeof ensureAccountReady>>,
|
||
): AdapterExecutionResult {
|
||
if (readiness.authState === "challenge_required") {
|
||
return {
|
||
status: "failed",
|
||
summary: `${task.accountName} 需要完成人机验证后才能继续执行。`,
|
||
error: {
|
||
code: "desktop_account_challenge_required",
|
||
message: "desktop account challenge required",
|
||
},
|
||
};
|
||
}
|
||
|
||
if (readiness.authState === "expired" || readiness.authState === "revoked") {
|
||
return {
|
||
status: "failed",
|
||
summary: `${task.accountName} 登录态已失效，请重新授权后再试。`,
|
||
error: {
|
||
code: "desktop_account_auth_expired",
|
||
message: "desktop account auth expired",
|
||
},
|
||
};
|
||
}
|
||
|
||
return {
|
||
status: "unknown",
|
||
summary: `${task.accountName} 最近校验失败，已暂缓执行任务。`,
|
||
error: {
|
||
code: "desktop_account_probe_pending",
|
||
message: "desktop account readiness pending",
|
||
},
|
||
};
|
||
}
|
||
|
||
async function maybeReportTaskAuthFailure(
|
||
task: RuntimeTaskRecord,
|
||
result: AdapterExecutionResult,
|
||
accountIdentity: PublishAccountIdentity | null,
|
||
): Promise<void> {
|
||
if (!accountIdentity) {
|
||
return;
|
||
}
|
||
|
||
await reportAccountFailure(accountIdentity, {
|
||
summary: result.summary,
|
||
error: result.error ?? null,
|
||
}).catch((error) => {
|
||
console.warn("[desktop-runtime] account failure report failed", {
|
||
taskId: task.id,
|
||
accountId: accountIdentity.id,
|
||
platform: accountIdentity.platform,
|
||
message: error instanceof Error ? error.message : String(error),
|
||
});
|
||
});
|
||
}
|
||
|
||
function resolvePlaywrightTargetURL(
|
||
platform: string,
|
||
payload: Record<string, JsonValue>,
|
||
): string {
|
||
const payloadURL = resolveStringField(payload, ["target_url", "targetUrl", "console_url", "consoleUrl", "url"]);
|
||
if (payloadURL) {
|
||
return payloadURL;
|
||
}
|
||
|
||
const aiPlatform = getAIPlatformCatalogItem(platform);
|
||
if (aiPlatform?.consoleUrl) {
|
||
return aiPlatform.consoleUrl;
|
||
}
|
||
|
||
return "about:blank";
|
||
}
|
||
|
||
function buildScaffoldResult(
|
||
task: RuntimeTaskRecord,
|
||
adapterPayload: Record<string, JsonValue>,
|
||
detail: string,
|
||
): AdapterExecutionResult {
|
||
return {
|
||
status: "failed",
|
||
payload: {
|
||
...adapterPayload,
|
||
runtime_mode: "desktop-scaffold",
|
||
task_kind: task.kind,
|
||
task_platform: task.platform,
|
||
},
|
||
error: {
|
||
code: "desktop_task_adapter_missing",
|
||
message: "desktop runtime adapter is not implemented for this platform",
|
||
detail,
|
||
},
|
||
summary: `${task.title} 执行失败：当前平台的 desktop ${task.kind === "publish" ? "发布" : "监测"}适配器尚未实现。`,
|
||
};
|
||
}
|
||
|
||
async function loadPublishArticle(
|
||
task: RuntimeTaskRecord,
|
||
): Promise<DesktopArticleContent> {
|
||
const articleId = resolveArticleId(task.payload);
|
||
if (articleId === null) {
|
||
throw new Error("desktop_publish_article_id_missing");
|
||
}
|
||
|
||
const article = await getDesktopArticleContent(articleId);
|
||
return {
|
||
...article,
|
||
title: article.title?.trim() || task.title,
|
||
};
|
||
}
|
||
|
||
function resolveArticleId(payload: Record<string, JsonValue>): number | null {
|
||
const direct = toPositiveInt(payload.article_id);
|
||
if (direct !== null) {
|
||
return direct;
|
||
}
|
||
|
||
const contentRef = payload.content_ref;
|
||
if (!contentRef || typeof contentRef !== "object" || Array.isArray(contentRef)) {
|
||
return null;
|
||
}
|
||
|
||
const typed = contentRef as Record<string, JsonValue>;
|
||
return toPositiveInt(typed.id ?? typed.article_id ?? null);
|
||
}
|
||
|
||
function toPositiveInt(value: JsonValue | null): number | null {
|
||
if (typeof value === "number" && Number.isInteger(value) && value > 0) {
|
||
return value;
|
||
}
|
||
if (typeof value === "string" && /^\d+$/.test(value.trim())) {
|
||
const parsed = Number.parseInt(value.trim(), 10);
|
||
return parsed > 0 ? parsed : null;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function resolveStringField(
|
||
payload: Record<string, JsonValue>,
|
||
keys: string[],
|
||
): string | null {
|
||
for (const key of keys) {
|
||
const value = payload[key];
|
||
if (typeof value !== "string") {
|
||
continue;
|
||
}
|
||
const normalized = value.trim();
|
||
if (normalized) {
|
||
return normalized;
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function resolveStaleMonitoringBusinessDate(payload: Record<string, JsonValue>): string | null {
|
||
const businessDate = resolveMonitoringBusinessDate(payload);
|
||
if (!businessDate) {
|
||
return null;
|
||
}
|
||
|
||
return businessDate < currentMonitoringBusinessDate() ? businessDate : null;
|
||
}
|
||
|
||
function resolveMonitoringBusinessDate(payload: Record<string, JsonValue>): string | null {
|
||
const candidates = [
|
||
payload.business_date,
|
||
payload.businessDate,
|
||
payload.metric_date,
|
||
payload.date,
|
||
];
|
||
|
||
for (const candidate of candidates) {
|
||
if (typeof candidate !== "string") {
|
||
continue;
|
||
}
|
||
const normalized = candidate.trim();
|
||
if (/^\d{4}-\d{2}-\d{2}$/.test(normalized)) {
|
||
return normalized;
|
||
}
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
function currentMonitoringBusinessDate(now = new Date()): string {
|
||
const parts = new Intl.DateTimeFormat("en-US", {
|
||
timeZone: monitorBusinessTimeZone,
|
||
year: "numeric",
|
||
month: "2-digit",
|
||
day: "2-digit",
|
||
}).formatToParts(now);
|
||
|
||
const year = parts.find((part) => part.type === "year")?.value ?? "";
|
||
const month = parts.find((part) => part.type === "month")?.value ?? "";
|
||
const day = parts.find((part) => part.type === "day")?.value ?? "";
|
||
return `${year}-${month}-${day}`;
|
||
}
|
||
|
||
function updateTaskProgress(taskId: string, summary: string): void {
|
||
const existing = state.tasks.get(taskId);
|
||
if (!existing) {
|
||
return;
|
||
}
|
||
|
||
existing.summary = summary;
|
||
existing.updatedAt = Date.now();
|
||
state.tasks.set(taskId, existing);
|
||
}
|
||
|
||
function upsertTaskFromInfo(
|
||
task: DesktopTaskInfo,
|
||
options: {
|
||
routing?: RuntimeTaskRouting;
|
||
summary?: string;
|
||
attemptId?: string | null;
|
||
leaseToken?: string | null;
|
||
} = {},
|
||
): RuntimeTaskRecord {
|
||
const existing = state.tasks.get(task.id);
|
||
const payload = normalizeJsonObject(task.payload);
|
||
const record: RuntimeTaskRecord = {
|
||
id: task.id,
|
||
jobId: task.job_id,
|
||
title: resolveTaskTitle(task, payload, existing),
|
||
kind: task.kind,
|
||
platform: task.platform,
|
||
accountId: task.target_account_id,
|
||
accountName: resolveAccountName(task.target_account_id, existing?.accountName),
|
||
clientId: task.target_client_id,
|
||
status: task.status,
|
||
routing: options.routing ?? existing?.routing ?? "db-recovery",
|
||
leaseExpiresAt: parseTimestamp(task.lease_expires_at) ?? null,
|
||
updatedAt: parseTimestamp(task.updated_at) ?? Date.now(),
|
||
summary: options.summary ?? existing?.summary ?? summaryFromTaskStatus(task.status),
|
||
payload,
|
||
error: normalizeJsonObjectOrNull(task.error),
|
||
result: normalizeJsonObjectOrNull(task.result),
|
||
attemptId:
|
||
task.status === "in_progress"
|
||
? options.attemptId ?? existing?.attemptId ?? task.active_attempt_id ?? null
|
||
: null,
|
||
leaseToken: task.status === "in_progress" ? options.leaseToken ?? existing?.leaseToken ?? null : null,
|
||
};
|
||
|
||
state.tasks.set(task.id, record);
|
||
return record;
|
||
}
|
||
|
||
function refreshAccountNames(): void {
|
||
for (const [taskId, task] of state.tasks.entries()) {
|
||
const nextName = resolveAccountName(task.accountId, task.accountName);
|
||
if (nextName === task.accountName) {
|
||
continue;
|
||
}
|
||
state.tasks.set(taskId, { ...task, accountName: nextName });
|
||
}
|
||
}
|
||
|
||
function recordActivity(
|
||
severity: RuntimeTone,
|
||
title: string,
|
||
detail: string,
|
||
): void {
|
||
const next: RuntimeControllerActivity = {
|
||
id: `rt-${Date.now()}-${state.activitySeq += 1}`,
|
||
severity,
|
||
title,
|
||
detail,
|
||
at: Date.now(),
|
||
};
|
||
|
||
state.activity.unshift(next);
|
||
if (state.activity.length > maxActivityItems) {
|
||
state.activity.length = maxActivityItems;
|
||
}
|
||
}
|
||
|
||
function syncSchedulerSurface(): void {
|
||
setSchedulerQueueDepth(localQueueDepth());
|
||
setSchedulerCurrentTask(primaryActiveTaskId());
|
||
}
|
||
|
||
function localQueueDepth(): number {
|
||
const snapshot = getMonitorSchedulerSnapshot();
|
||
return state.publishQueue.length + snapshot.queuedCount + snapshot.leasingCount;
|
||
}
|
||
|
||
function primaryActiveTaskId(): string | null {
|
||
const active = [...state.activeExecutions.values()];
|
||
const publish = active.find((item) => item.kind === "publish") ?? null;
|
||
return publish?.taskId ?? active[0]?.taskId ?? null;
|
||
}
|
||
|
||
function hasActivePublishExecution(): boolean {
|
||
return [...state.activeExecutions.values()].some((item) => item.kind === "publish");
|
||
}
|
||
|
||
function activeMonitorCount(): number {
|
||
return [...state.activeExecutions.values()].filter((item) => item.kind === "monitor").length;
|
||
}
|
||
|
||
function activeMonitorPlatforms(): Set<string> {
|
||
return new Set(
|
||
[...state.activeExecutions.values()]
|
||
.filter((item) => item.kind === "monitor")
|
||
.map((item) => item.platform),
|
||
);
|
||
}
|
||
|
||
function activeMonitorQuestionKeys(): Set<string> {
|
||
return new Set(
|
||
getMonitorSchedulerSnapshot().tasks
|
||
.filter((task) => task.state === "active" && Boolean(task.questionKey))
|
||
.map((task) => task.questionKey as string),
|
||
);
|
||
}
|
||
|
||
function canStartAnotherMonitorExecution(): boolean {
|
||
if (state.publishQueue.length > 0 || hasActivePublishExecution()) {
|
||
return false;
|
||
}
|
||
const limit = resolveAdaptiveMonitorConcurrency();
|
||
return limit > 0 && activeMonitorCount() < limit;
|
||
}
|
||
|
||
function resolveAdaptiveMonitorConcurrency(): number {
|
||
const monitorAccountCount = state.accounts.filter((account) =>
|
||
isAIPlatformId(account.platform)
|
||
&& getProjectedAccountHealth({
|
||
accountId: account.id,
|
||
platform: account.platform,
|
||
health: account.health,
|
||
verifiedAt: account.verified_at,
|
||
}).health === "live",
|
||
).length;
|
||
if (monitorAccountCount <= 1) {
|
||
return monitorAccountCount;
|
||
}
|
||
|
||
const metrics = getProcessMetricsSnapshot().latestSample;
|
||
if (!metrics) {
|
||
return 1;
|
||
}
|
||
|
||
const totalCPUUsage = metrics.processes.reduce((sum, item) => sum + item.percentCPUUsage, 0);
|
||
const totalPrivateBytesKB = metrics.totals.privateBytes;
|
||
const processCount = metrics.totals.processCount;
|
||
const mainPrivateBytesKB = metrics.mainProcess.memory?.private ?? 0;
|
||
|
||
if (
|
||
totalCPUUsage >= 180
|
||
|| totalPrivateBytesKB >= 2_500_000
|
||
|| mainPrivateBytesKB >= 1_000_000
|
||
|| processCount >= 40
|
||
) {
|
||
return 1;
|
||
}
|
||
|
||
return 2;
|
||
}
|
||
|
||
function handleAuthExpired(error: unknown): void {
|
||
const message = errorMessage(error);
|
||
state.lastError = message;
|
||
setTransportAuthState("expired");
|
||
recordActivity("danger", "客户端令牌已过期", message);
|
||
stopRuntime();
|
||
}
|
||
|
||
function selectPublishAdapter(platform: string): PublishAdapter | null {
|
||
if (platform === toutiaoAdapter.platform) {
|
||
return toutiaoAdapter;
|
||
}
|
||
if (platform === zhihuAdapter.platform) {
|
||
return zhihuAdapter;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function selectMonitorAdapter(platform: string): MonitorAdapter | null {
|
||
if (platform === doubaoAdapter.provider) {
|
||
return doubaoAdapter;
|
||
}
|
||
if (platform === qwenAdapter.provider) {
|
||
return qwenAdapter;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function normalizeSession(
|
||
session: DesktopRuntimeSessionSyncRequest | null,
|
||
): DesktopRuntimeSessionSyncRequest | null {
|
||
if (!session) {
|
||
return null;
|
||
}
|
||
|
||
return {
|
||
api_base_url: session.api_base_url.trim() || "http://localhost:8080",
|
||
mode: session.mode,
|
||
client_token: session.client_token?.trim() || null,
|
||
desktop_client: session.desktop_client ? { ...session.desktop_client } : null,
|
||
};
|
||
}
|
||
|
||
function sameSession(
|
||
left: DesktopRuntimeSessionSyncRequest | null,
|
||
right: DesktopRuntimeSessionSyncRequest | null,
|
||
): boolean {
|
||
if (!left && !right) {
|
||
return true;
|
||
}
|
||
if (!left || !right) {
|
||
return false;
|
||
}
|
||
|
||
return (
|
||
left.api_base_url === right.api_base_url
|
||
&& left.mode === right.mode
|
||
&& left.client_token === right.client_token
|
||
&& left.desktop_client?.id === right.desktop_client?.id
|
||
);
|
||
}
|
||
|
||
function canRunLive(session: DesktopRuntimeSessionSyncRequest | null): session is DesktopRuntimeSessionSyncRequest {
|
||
return Boolean(
|
||
session
|
||
&& session.mode === "authenticated"
|
||
&& session.client_token
|
||
&& session.desktop_client,
|
||
);
|
||
}
|
||
|
||
function resolveAccountName(accountId: string, fallback = "待同步账号"): string {
|
||
const account = state.accounts.find((item) => item.id === accountId);
|
||
return account?.display_name ?? fallback;
|
||
}
|
||
|
||
function resolveTaskTitle(
|
||
task: DesktopTaskInfo,
|
||
payload: Record<string, JsonValue>,
|
||
existing?: RuntimeTaskRecord,
|
||
): string {
|
||
const title = typeof payload.title === "string" ? payload.title.trim() : "";
|
||
if (title) {
|
||
return title;
|
||
}
|
||
return existing?.title ?? defaultTaskTitle(task.kind, task.platform);
|
||
}
|
||
|
||
function describeRouting(routing: RuntimeTaskRouting): string {
|
||
return routing === "rabbitmq-primary" ? "主消息队列通道" : "数据库兜底拉取";
|
||
}
|
||
|
||
function defaultTaskTitle(kind: "publish" | "monitor", platform?: string): string {
|
||
return `${kind === "publish" ? "发布任务" : "监控任务"}${platform ? ` · ${platform}` : ""}`;
|
||
}
|
||
|
||
function summaryFromTaskStatus(status: RuntimeTaskStatus): string {
|
||
switch (status) {
|
||
case "queued":
|
||
return "任务已排队，等待客户端领取租约。";
|
||
case "in_progress":
|
||
return "任务正在执行，结果回写需要携带有效租约凭证。";
|
||
case "succeeded":
|
||
return "任务执行成功。";
|
||
case "failed":
|
||
return "任务执行失败。";
|
||
case "aborted":
|
||
return "任务已被取消。";
|
||
default:
|
||
return "任务执行异常，已按失败处理。";
|
||
}
|
||
}
|
||
|
||
function summaryFromEventType(type: DesktopTaskEventMessage["type"], status: RuntimeTaskStatus): string {
|
||
switch (type) {
|
||
case "task_available":
|
||
return "主消息队列事件已送达当前客户端，等待领取租约。";
|
||
case "task_leased":
|
||
return "服务端已确认租约归属。";
|
||
case "task_extended":
|
||
return "任务租约已续期。";
|
||
case "task_canceled":
|
||
return "任务已被取消。";
|
||
case "task_reconciled":
|
||
return "任务已完成人工对账并回写。";
|
||
default:
|
||
return summaryFromTaskStatus(status);
|
||
}
|
||
}
|
||
|
||
function inferPlatformFromAccount(accountId: string | null): string {
|
||
if (!accountId) {
|
||
return "desktop";
|
||
}
|
||
return state.accounts.find((item) => item.id === accountId)?.platform ?? "desktop";
|
||
}
|
||
|
||
function buildApiUrl(baseURL: string, pathname: string): string {
|
||
const normalized = baseURL.endsWith("/") ? baseURL : `${baseURL}/`;
|
||
return new URL(pathname.replace(/^\//, ""), normalized).toString();
|
||
}
|
||
|
||
function normalizeJsonObject(value: Record<string, JsonValue> | null | undefined): Record<string, JsonValue> {
|
||
if (!value) {
|
||
return {};
|
||
}
|
||
return { ...value };
|
||
}
|
||
|
||
function normalizeJsonObjectOrNull(
|
||
value: Record<string, JsonValue> | null | undefined,
|
||
): Record<string, JsonValue> | null {
|
||
if (!value) {
|
||
return null;
|
||
}
|
||
return { ...value };
|
||
}
|
||
|
||
function toStructuredError(error: unknown): Record<string, JsonValue> {
|
||
if (error instanceof ApiClientError) {
|
||
return {
|
||
code: error.code,
|
||
message: error.message,
|
||
detail: error.detail ?? null,
|
||
request_id: error.requestId ?? null,
|
||
status: error.status ?? null,
|
||
};
|
||
}
|
||
|
||
if (error instanceof Error) {
|
||
return {
|
||
code: "desktop_runtime_error",
|
||
message: error.message,
|
||
name: error.name,
|
||
};
|
||
}
|
||
|
||
return {
|
||
code: "desktop_runtime_error",
|
||
message: String(error),
|
||
};
|
||
}
|
||
|
||
function errorMessage(error: unknown): string {
|
||
if (error instanceof ApiClientError) {
|
||
return error.detail ? `${error.message}: ${error.detail}` : error.message;
|
||
}
|
||
if (error instanceof Error) {
|
||
return error.message;
|
||
}
|
||
return String(error);
|
||
}
|
||
|
||
function parseTimestamp(value: string | null | undefined): number | null {
|
||
if (!value) {
|
||
return null;
|
||
}
|
||
|
||
const timestamp = Date.parse(value);
|
||
return Number.isNaN(timestamp) ? null : timestamp;
|
||
}
|
||
|
||
function isApiClientError(error: unknown, status: number): error is ApiClientError {
|
||
return error instanceof ApiClientError && error.status === status;
|
||
}
|
||
|
||
function isDesktopTaskEvent(value: unknown): value is DesktopTaskEventMessage {
|
||
if (typeof value !== "object" || value === null) {
|
||
return false;
|
||
}
|
||
|
||
const event = value as Partial<DesktopTaskEventMessage>;
|
||
return Boolean(
|
||
typeof event.type === "string"
|
||
&& typeof event.task_id === "string"
|
||
&& typeof event.job_id === "string"
|
||
&& typeof event.target_account_id === "string"
|
||
&& typeof event.target_client_id === "string"
|
||
&& typeof event.platform === "string"
|
||
&& typeof event.kind === "string"
|
||
&& typeof event.status === "string"
|
||
&& typeof event.updated_at === "string",
|
||
);
|
||
}
|
||
|
||
function isConnectedEvent(value: unknown): value is { server_time: string } {
|
||
if (typeof value !== "object" || value === null) {
|
||
return false;
|
||
}
|
||
return typeof (value as { server_time?: string }).server_time === "string";
|
||
}
|
||
|
||
function isReconnectPayload(value: unknown): value is { delayMs: number } {
|
||
if (typeof value !== "object" || value === null) {
|
||
return false;
|
||
}
|
||
return typeof (value as { delayMs?: number }).delayMs === "number";
|
||
}
|