feat(desktop): add foreground publish admission scheduler
Publish is now treated as the runtime foreground channel and AI monitoring as background. A new in-memory publish-scheduler owns queue selection, per-platform active locks, and 3-6s post-completion platform cooldowns. The runtime controller tracks lease-in-flight per kind so a monitor lease no longer blocks a publish lease, derives total concurrency from hardware class, current process health, and an optional GEO_DESKTOP_MAX_TOTAL_CONCURRENCY override, and reserves a publish slot when budgeting monitor capacity. Monitor admission returns zero whenever publish has backlog or an in-flight lease, so a cooldown-blocked publish queue still prevents monitor from consuming the foreground reserve. Runtime diagnostics expose publishScheduler alongside monitorScheduler. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,53 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
enqueuePublishTask,
|
||||
getPublishSchedulerSnapshot,
|
||||
initPublishScheduler,
|
||||
notePublishTaskActivated,
|
||||
notePublishTaskCompleted,
|
||||
selectNextPublishTask,
|
||||
} from "./publish-scheduler";
|
||||
|
||||
describe("publish scheduler", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-04-24T00:00:00.000Z"));
|
||||
initPublishScheduler();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
initPublishScheduler();
|
||||
});
|
||||
|
||||
it("skips same-platform queued tasks while selecting an allowed platform", () => {
|
||||
enqueuePublishTask({ taskId: "toutiao-active", platform: "toutiao", routing: "rabbitmq-primary" });
|
||||
notePublishTaskActivated("toutiao-active", "toutiao");
|
||||
enqueuePublishTask({ taskId: "toutiao-next", platform: "toutiao", routing: "rabbitmq-primary" });
|
||||
enqueuePublishTask({ taskId: "zhihu-next", platform: "zhihu", routing: "rabbitmq-primary" });
|
||||
|
||||
const selected = selectNextPublishTask({
|
||||
maxConcurrency: 2,
|
||||
activeCount: 1,
|
||||
});
|
||||
|
||||
expect(selected?.taskId).toBe("zhihu-next");
|
||||
expect(getPublishSchedulerSnapshot().leasingCount).toBe(1);
|
||||
});
|
||||
|
||||
it("holds same-platform tasks during completion cooldown", () => {
|
||||
enqueuePublishTask({ taskId: "first", platform: "toutiao", routing: "rabbitmq-primary" });
|
||||
expect(selectNextPublishTask({ maxConcurrency: 2, activeCount: 0 })?.taskId).toBe("first");
|
||||
notePublishTaskActivated("first", "toutiao");
|
||||
notePublishTaskCompleted("first", "toutiao");
|
||||
|
||||
enqueuePublishTask({ taskId: "second", platform: "toutiao", routing: "rabbitmq-primary" });
|
||||
|
||||
expect(selectNextPublishTask({ maxConcurrency: 2, activeCount: 0 })).toBeNull();
|
||||
|
||||
vi.advanceTimersByTime(6_001);
|
||||
|
||||
expect(selectNextPublishTask({ maxConcurrency: 2, activeCount: 0 })?.taskId).toBe("second");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,202 @@
|
||||
export type PublishSchedulerRouting = "rabbitmq-primary" | "db-recovery";
|
||||
|
||||
type PublishTaskState = "queued" | "leasing" | "active";
|
||||
|
||||
export interface PublishSchedulerTaskRecord {
|
||||
taskId: string;
|
||||
platform: string;
|
||||
routing: PublishSchedulerRouting;
|
||||
state: PublishTaskState;
|
||||
enqueuedAt: number;
|
||||
updatedAt: number;
|
||||
lastStartedAt: number | null;
|
||||
}
|
||||
|
||||
export interface PublishSchedulerSelection {
|
||||
taskId: string;
|
||||
platform: string;
|
||||
routing: PublishSchedulerRouting;
|
||||
}
|
||||
|
||||
export interface PublishSchedulerSnapshot {
|
||||
queueDepth: number;
|
||||
queuedCount: number;
|
||||
leasingCount: number;
|
||||
activeCount: number;
|
||||
cooldownCount: number;
|
||||
tasks: PublishSchedulerTaskRecord[];
|
||||
cooldownUntil: Record<string, number>;
|
||||
}
|
||||
|
||||
interface PublishSchedulerSelectionOptions {
|
||||
now?: number;
|
||||
maxConcurrency: number;
|
||||
activePlatforms?: ReadonlySet<string>;
|
||||
activeCount: number;
|
||||
}
|
||||
|
||||
const publishPlatformCooldownMinMs = 3_000;
|
||||
const publishPlatformCooldownMaxMs = 6_000;
|
||||
|
||||
const tasks = new Map<string, PublishSchedulerTaskRecord>();
|
||||
const cooldownUntil = new Map<string, number>();
|
||||
|
||||
export function initPublishScheduler(): void {
|
||||
tasks.clear();
|
||||
cooldownUntil.clear();
|
||||
}
|
||||
|
||||
export function enqueuePublishTask(input: {
|
||||
taskId: string;
|
||||
platform: string;
|
||||
routing: PublishSchedulerRouting;
|
||||
}): void {
|
||||
const taskId = input.taskId.trim();
|
||||
const platform = input.platform.trim();
|
||||
if (!taskId || !platform) {
|
||||
return;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const existing = tasks.get(taskId);
|
||||
tasks.set(taskId, {
|
||||
taskId,
|
||||
platform,
|
||||
routing: input.routing,
|
||||
state: existing?.state === "active" ? "active" : "queued",
|
||||
enqueuedAt: existing?.enqueuedAt ?? now,
|
||||
updatedAt: now,
|
||||
lastStartedAt: existing?.lastStartedAt ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
export function selectNextPublishTask(
|
||||
options: PublishSchedulerSelectionOptions,
|
||||
): PublishSchedulerSelection | null {
|
||||
const now = options.now ?? Date.now();
|
||||
pruneCooldowns(now);
|
||||
|
||||
if (options.activeCount >= options.maxConcurrency) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const activePlatforms = new Set(options.activePlatforms ?? []);
|
||||
for (const task of tasks.values()) {
|
||||
if (task.state === "active" || task.state === "leasing") {
|
||||
activePlatforms.add(task.platform);
|
||||
}
|
||||
}
|
||||
|
||||
const candidates = [...tasks.values()]
|
||||
.filter((task) => task.state === "queued")
|
||||
.sort((left, right) => {
|
||||
if (left.enqueuedAt !== right.enqueuedAt) {
|
||||
return left.enqueuedAt - right.enqueuedAt;
|
||||
}
|
||||
return left.updatedAt - right.updatedAt;
|
||||
});
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (activePlatforms.has(candidate.platform)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const platformCooldown = cooldownUntil.get(candidate.platform) ?? 0;
|
||||
if (platformCooldown > now) {
|
||||
continue;
|
||||
}
|
||||
|
||||
candidate.state = "leasing";
|
||||
candidate.updatedAt = now;
|
||||
tasks.set(candidate.taskId, candidate);
|
||||
return {
|
||||
taskId: candidate.taskId,
|
||||
platform: candidate.platform,
|
||||
routing: candidate.routing,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function notePublishTaskActivated(taskId: string, platform: string): void {
|
||||
const normalizedTaskId = taskId.trim();
|
||||
const normalizedPlatform = platform.trim();
|
||||
if (!normalizedTaskId || !normalizedPlatform) {
|
||||
return;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const existing = tasks.get(normalizedTaskId);
|
||||
tasks.set(normalizedTaskId, {
|
||||
taskId: normalizedTaskId,
|
||||
platform: normalizedPlatform,
|
||||
routing: existing?.routing ?? "db-recovery",
|
||||
state: "active",
|
||||
enqueuedAt: existing?.enqueuedAt ?? now,
|
||||
updatedAt: now,
|
||||
lastStartedAt: now,
|
||||
});
|
||||
}
|
||||
|
||||
export function notePublishTaskLeaseMiss(taskId: string): void {
|
||||
tasks.delete(taskId);
|
||||
}
|
||||
|
||||
export function notePublishTaskCompleted(taskId: string, platform?: string): void {
|
||||
const existing = tasks.get(taskId);
|
||||
const resolvedPlatform = (platform ?? existing?.platform ?? "").trim();
|
||||
tasks.delete(taskId);
|
||||
|
||||
if (resolvedPlatform) {
|
||||
cooldownUntil.set(resolvedPlatform, Date.now() + randomPublishCooldownMs());
|
||||
}
|
||||
}
|
||||
|
||||
export function notePublishTaskCanceled(taskId: string): void {
|
||||
tasks.delete(taskId);
|
||||
}
|
||||
|
||||
export function getPublishSchedulerSnapshot(): PublishSchedulerSnapshot {
|
||||
const now = Date.now();
|
||||
pruneCooldowns(now);
|
||||
const sortedTasks = [...tasks.values()].sort((left, right) => {
|
||||
if (left.state !== right.state) {
|
||||
return stateRank(left.state) - stateRank(right.state);
|
||||
}
|
||||
return left.enqueuedAt - right.enqueuedAt;
|
||||
});
|
||||
|
||||
return {
|
||||
queueDepth: sortedTasks.filter((task) => task.state === "queued" || task.state === "leasing").length,
|
||||
queuedCount: sortedTasks.filter((task) => task.state === "queued").length,
|
||||
leasingCount: sortedTasks.filter((task) => task.state === "leasing").length,
|
||||
activeCount: sortedTasks.filter((task) => task.state === "active").length,
|
||||
cooldownCount: cooldownUntil.size,
|
||||
tasks: sortedTasks.map((task) => ({ ...task })),
|
||||
cooldownUntil: Object.fromEntries(cooldownUntil.entries()),
|
||||
};
|
||||
}
|
||||
|
||||
function pruneCooldowns(now: number): void {
|
||||
for (const [platform, until] of cooldownUntil.entries()) {
|
||||
if (until <= now) {
|
||||
cooldownUntil.delete(platform);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function randomPublishCooldownMs(): number {
|
||||
const spread = publishPlatformCooldownMaxMs - publishPlatformCooldownMinMs;
|
||||
return publishPlatformCooldownMinMs + Math.floor(Math.random() * (spread + 1));
|
||||
}
|
||||
|
||||
function stateRank(state: PublishTaskState): number {
|
||||
if (state === "active") {
|
||||
return 0;
|
||||
}
|
||||
if (state === "leasing") {
|
||||
return 1;
|
||||
}
|
||||
return 2;
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { hostname } from "node:os";
|
||||
import { cpus, hostname, totalmem } from "node:os";
|
||||
|
||||
import { ApiClientError } from "@geo/http-client";
|
||||
import type {
|
||||
@@ -56,6 +56,16 @@ import {
|
||||
noteMonitorTaskLeased,
|
||||
selectNextMonitorTask,
|
||||
} from "./monitor-scheduler";
|
||||
import {
|
||||
enqueuePublishTask as enqueuePublishSchedulerTask,
|
||||
getPublishSchedulerSnapshot,
|
||||
initPublishScheduler,
|
||||
notePublishTaskActivated,
|
||||
notePublishTaskCanceled,
|
||||
notePublishTaskCompleted,
|
||||
notePublishTaskLeaseMiss,
|
||||
selectNextPublishTask,
|
||||
} from "./publish-scheduler";
|
||||
import {
|
||||
retainHiddenPlaywrightPage,
|
||||
startHiddenPlaywrightReaper,
|
||||
@@ -126,6 +136,9 @@ const leaseExtendIntervalMs = 60_000;
|
||||
const maxActivityItems = 60;
|
||||
const monitorBusinessTimeZone = "Asia/Shanghai";
|
||||
const authRequiredMonitorPlatforms = new Set(["yuanbao", "kimi", "deepseek", "doubao", "wenxin"]);
|
||||
const publishReservedSlots = 1;
|
||||
const minimumForegroundTotalConcurrency = 2;
|
||||
const maximumRuntimeTotalConcurrency = 4;
|
||||
|
||||
interface RuntimeTaskRecord {
|
||||
id: string;
|
||||
@@ -178,6 +191,7 @@ export interface RuntimeControllerSnapshot {
|
||||
interface PendingTaskRequest {
|
||||
taskId: string;
|
||||
routing: RuntimeTaskRouting;
|
||||
platform?: string;
|
||||
source?: "desktop_task" | "monitoring_lease";
|
||||
}
|
||||
|
||||
@@ -198,14 +212,13 @@ interface RuntimeState {
|
||||
session: DesktopRuntimeSessionSyncRequest | null;
|
||||
client: DesktopClientInfo | null;
|
||||
running: boolean;
|
||||
publishQueue: PendingTaskRequest[];
|
||||
queuedPublishTaskIds: Set<string>;
|
||||
tasks: Map<string, RuntimeTaskRecord>;
|
||||
accounts: DesktopAccountInfo[];
|
||||
accountProfiles: Map<string, PublishAccountProfile>;
|
||||
activity: RuntimeControllerActivity[];
|
||||
activeExecutions: Map<string, ActiveRuntimeExecution>;
|
||||
leaseInFlight: boolean;
|
||||
leaseInFlightKinds: Set<"publish" | "monitor">;
|
||||
publishFallbackBackoffUntil: number;
|
||||
heartbeatTimer: ReturnType<typeof setInterval> | null;
|
||||
pullTimer: ReturnType<typeof setInterval> | null;
|
||||
dispatchWsClient: DispatchWsClient | null;
|
||||
@@ -224,14 +237,13 @@ const state: RuntimeState = {
|
||||
session: null,
|
||||
client: null,
|
||||
running: false,
|
||||
publishQueue: [],
|
||||
queuedPublishTaskIds: new Set<string>(),
|
||||
tasks: new Map<string, RuntimeTaskRecord>(),
|
||||
accounts: [],
|
||||
accountProfiles: new Map<string, PublishAccountProfile>(),
|
||||
activity: [],
|
||||
activeExecutions: new Map<string, ActiveRuntimeExecution>(),
|
||||
leaseInFlight: false,
|
||||
leaseInFlightKinds: new Set<"publish" | "monitor">(),
|
||||
publishFallbackBackoffUntil: 0,
|
||||
heartbeatTimer: null,
|
||||
pullTimer: null,
|
||||
dispatchWsClient: null,
|
||||
@@ -420,6 +432,7 @@ function startRuntime(): void {
|
||||
|
||||
initScheduler();
|
||||
initMonitorScheduler();
|
||||
initPublishScheduler();
|
||||
startHiddenPlaywrightReaper();
|
||||
state.running = true;
|
||||
setSchedulerPhase("running");
|
||||
@@ -438,7 +451,7 @@ function startRuntime(): void {
|
||||
|
||||
function stopRuntime(): void {
|
||||
state.running = false;
|
||||
state.leaseInFlight = false;
|
||||
state.leaseInFlightKinds.clear();
|
||||
for (const execution of state.activeExecutions.values()) {
|
||||
execution.abortController.abort();
|
||||
}
|
||||
@@ -471,8 +484,7 @@ function stopRuntime(): void {
|
||||
}
|
||||
|
||||
function clearRuntimeState(): void {
|
||||
state.publishQueue = [];
|
||||
state.queuedPublishTaskIds.clear();
|
||||
initPublishScheduler();
|
||||
state.tasks.clear();
|
||||
state.accounts = [];
|
||||
state.accountProfiles.clear();
|
||||
@@ -484,6 +496,7 @@ function clearRuntimeState(): void {
|
||||
state.lastAccountsSyncAt = 0;
|
||||
state.lastServerTime = null;
|
||||
state.lastError = null;
|
||||
state.publishFallbackBackoffUntil = 0;
|
||||
setSchedulerQueueDepth(localQueueDepth());
|
||||
}
|
||||
|
||||
@@ -646,9 +659,16 @@ function handleTaskEvent(event: DesktopTaskEventMessage): void {
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
event.kind === "publish"
|
||||
&& (event.type === "task_completed" || event.type === "task_reconciled" || event.type === "task_canceled")
|
||||
) {
|
||||
notePublishTaskCanceled(event.task_id);
|
||||
}
|
||||
|
||||
if (event.type === "task_available") {
|
||||
if (event.kind === "publish") {
|
||||
enqueuePublishTask(event.task_id, "rabbitmq-primary");
|
||||
enqueuePublishTask(next.id, next.platform, "rabbitmq-primary");
|
||||
}
|
||||
pumpExecutionLoop();
|
||||
}
|
||||
@@ -1036,43 +1056,40 @@ export async function unbindRuntimeAccount(accountId: string, syncVersion: numbe
|
||||
await syncAccounts("manual");
|
||||
}
|
||||
|
||||
function enqueuePublishTask(taskId: string, routing: RuntimeTaskRouting): void {
|
||||
if (!taskId || state.activeExecutions.has(taskId) || state.queuedPublishTaskIds.has(taskId)) {
|
||||
function enqueuePublishTask(taskId: string, platform: string, routing: RuntimeTaskRouting): void {
|
||||
if (!taskId || state.activeExecutions.has(taskId)) {
|
||||
return;
|
||||
}
|
||||
|
||||
state.publishQueue.push({ taskId, routing });
|
||||
state.queuedPublishTaskIds.add(taskId);
|
||||
enqueuePublishSchedulerTask({ taskId, platform, routing });
|
||||
syncSchedulerSurface();
|
||||
}
|
||||
|
||||
function dequeuePublishTask(): PendingTaskRequest | null {
|
||||
const next = state.publishQueue.shift() ?? null;
|
||||
if (next) {
|
||||
state.queuedPublishTaskIds.delete(next.taskId);
|
||||
}
|
||||
syncSchedulerSurface();
|
||||
return next;
|
||||
}
|
||||
|
||||
function pumpExecutionLoop(): void {
|
||||
if (!state.running || state.leaseInFlight) {
|
||||
if (!state.running) {
|
||||
return;
|
||||
}
|
||||
|
||||
syncSchedulerSurface();
|
||||
|
||||
if (hasActivePublishExecution()) {
|
||||
if (canStartAnotherPublishExecution()) {
|
||||
const selected = selectNextPublishTask({
|
||||
maxConcurrency: resolveAdaptiveTotalConcurrency(),
|
||||
activePlatforms: activePublishPlatforms(),
|
||||
activeCount: activePublishCount(),
|
||||
});
|
||||
if (selected) {
|
||||
void leaseSpecificTask(selected, "publish");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (shouldPullPublishFallback()) {
|
||||
void pullNextTask("publish");
|
||||
return;
|
||||
}
|
||||
|
||||
if (state.publishQueue.length > 0) {
|
||||
if (state.activeExecutions.size === 0) {
|
||||
const queuedPublish = dequeuePublishTask();
|
||||
if (queuedPublish) {
|
||||
void leaseSpecificTask(queuedPublish, "publish");
|
||||
}
|
||||
}
|
||||
if (hasPublishBacklog()) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1089,30 +1106,22 @@ function pumpExecutionLoop(): void {
|
||||
}
|
||||
}
|
||||
|
||||
if (state.activeExecutions.size === 0) {
|
||||
// Publish task_available is push-delivered over the dispatch WebSocket. We
|
||||
// only fall back to the HTTP pull when the push channel is down, otherwise
|
||||
// every scheduler tick would wake the /lease endpoint for no reason.
|
||||
if (!state.dispatchWsConnected) {
|
||||
void pullNextTask("publish");
|
||||
}
|
||||
return;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
async function leaseSpecificTask(
|
||||
request: PendingTaskRequest,
|
||||
expectedKind: "publish" | "monitor",
|
||||
): Promise<void> {
|
||||
if (!canRunLive(state.session) || state.leaseInFlight) {
|
||||
if (!canRunLive(state.session) || isLeaseInFlight(expectedKind)) {
|
||||
return;
|
||||
}
|
||||
|
||||
state.leaseInFlight = true;
|
||||
state.leaseInFlightKinds.add(expectedKind);
|
||||
|
||||
try {
|
||||
if (expectedKind === "monitor" && request.source === "monitoring_lease") {
|
||||
state.leaseInFlight = false;
|
||||
state.leaseInFlightKinds.delete(expectedKind);
|
||||
await executeLeasedMonitoringTask(request.taskId, request.routing);
|
||||
return;
|
||||
}
|
||||
@@ -1121,10 +1130,12 @@ async function leaseSpecificTask(
|
||||
if (!leased.task) {
|
||||
if (expectedKind === "monitor") {
|
||||
noteMonitorTaskLeaseMiss(request.taskId);
|
||||
} else {
|
||||
notePublishTaskLeaseMiss(request.taskId);
|
||||
}
|
||||
return;
|
||||
}
|
||||
state.leaseInFlight = false;
|
||||
state.leaseInFlightKinds.delete(expectedKind);
|
||||
await executeLeasedTask(leased, request.routing);
|
||||
} catch (error) {
|
||||
if (expectedKind === "monitor") {
|
||||
@@ -1141,6 +1152,8 @@ async function leaseSpecificTask(
|
||||
} else {
|
||||
noteMonitorTaskLeaseMiss(request.taskId);
|
||||
}
|
||||
} else {
|
||||
notePublishTaskLeaseMiss(request.taskId);
|
||||
}
|
||||
|
||||
if (isApiClientError(error, 401)) {
|
||||
@@ -1152,10 +1165,10 @@ async function leaseSpecificTask(
|
||||
setSchedulerError(state.lastError);
|
||||
recordActivity("warn", "指定任务领取失败", `${request.taskId} 未能成功领取:${state.lastError}`);
|
||||
} finally {
|
||||
if (state.leaseInFlight) {
|
||||
state.leaseInFlight = false;
|
||||
if (state.leaseInFlightKinds.has(expectedKind)) {
|
||||
state.leaseInFlightKinds.delete(expectedKind);
|
||||
syncSchedulerSurface();
|
||||
if (state.publishQueue.length > 0 || localQueueDepth() > 0) {
|
||||
if (localQueueDepth() > 0) {
|
||||
pumpExecutionLoop();
|
||||
}
|
||||
}
|
||||
@@ -1164,11 +1177,11 @@ async function leaseSpecificTask(
|
||||
|
||||
async function pullNextTask(kind: "publish" | "monitor"): Promise<void> {
|
||||
if (kind === "monitor") {
|
||||
if (!canRunLive(state.session) || state.leaseInFlight) {
|
||||
if (!canRunLive(state.session) || isLeaseInFlight("monitor")) {
|
||||
return;
|
||||
}
|
||||
|
||||
state.leaseInFlight = true;
|
||||
state.leaseInFlightKinds.add("monitor");
|
||||
try {
|
||||
const leased = await leaseDesktopTask({ kind: "monitor" });
|
||||
state.lastPullAt = Date.now();
|
||||
@@ -1177,7 +1190,7 @@ async function pullNextTask(kind: "publish" | "monitor"): Promise<void> {
|
||||
noteSchedulerPull();
|
||||
|
||||
if (leased.task) {
|
||||
state.leaseInFlight = false;
|
||||
state.leaseInFlightKinds.delete("monitor");
|
||||
await executeLeasedTask(leased, "rabbitmq-primary");
|
||||
return;
|
||||
}
|
||||
@@ -1193,7 +1206,7 @@ async function pullNextTask(kind: "publish" | "monitor"): Promise<void> {
|
||||
return;
|
||||
}
|
||||
} finally {
|
||||
state.leaseInFlight = false;
|
||||
state.leaseInFlightKinds.delete("monitor");
|
||||
syncSchedulerSurface();
|
||||
}
|
||||
|
||||
@@ -1201,11 +1214,12 @@ async function pullNextTask(kind: "publish" | "monitor"): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!canRunLive(state.session) || state.leaseInFlight) {
|
||||
if (!canRunLive(state.session) || isLeaseInFlight("publish")) {
|
||||
return;
|
||||
}
|
||||
|
||||
state.leaseInFlight = true;
|
||||
let leasedPublishTask = false;
|
||||
state.leaseInFlightKinds.add("publish");
|
||||
|
||||
try {
|
||||
const leased = await leaseDesktopTask({ kind });
|
||||
@@ -1215,15 +1229,19 @@ async function pullNextTask(kind: "publish" | "monitor"): Promise<void> {
|
||||
noteSchedulerPull();
|
||||
|
||||
if (!leased.task) {
|
||||
state.publishFallbackBackoffUntil = Date.now() + pullIntervalMs;
|
||||
return;
|
||||
}
|
||||
|
||||
state.leaseInFlight = false;
|
||||
state.publishFallbackBackoffUntil = 0;
|
||||
leasedPublishTask = true;
|
||||
state.leaseInFlightKinds.delete("publish");
|
||||
await executeLeasedTask(leased, "db-recovery");
|
||||
} catch (error) {
|
||||
state.lastPullAt = Date.now();
|
||||
state.lastPullStatus = "failed";
|
||||
state.lastError = errorMessage(error);
|
||||
state.publishFallbackBackoffUntil = Date.now() + pullIntervalMs;
|
||||
|
||||
noteTransportPull(false);
|
||||
setSchedulerError(state.lastError);
|
||||
@@ -1235,18 +1253,21 @@ async function pullNextTask(kind: "publish" | "monitor"): Promise<void> {
|
||||
|
||||
recordActivity("warn", "兜底拉取失败", state.lastError);
|
||||
} finally {
|
||||
state.leaseInFlight = false;
|
||||
state.leaseInFlightKinds.delete("publish");
|
||||
syncSchedulerSurface();
|
||||
if (!leasedPublishTask && state.running) {
|
||||
queueMicrotask(() => pumpExecutionLoop());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function pullMonitoringTasks(routing: RuntimeTaskRouting): Promise<void> {
|
||||
const leaseLimit = resolveLegacyMonitoringLeaseLimit();
|
||||
if (!canRunLive(state.session) || state.leaseInFlight || leaseLimit <= 0) {
|
||||
if (!canRunLive(state.session) || isLeaseInFlight("monitor") || leaseLimit <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
state.leaseInFlight = true;
|
||||
state.leaseInFlightKinds.add("monitor");
|
||||
|
||||
try {
|
||||
const leased = await leaseMonitoringTasks({
|
||||
@@ -1272,7 +1293,7 @@ async function pullMonitoringTasks(routing: RuntimeTaskRouting): Promise<void> {
|
||||
|
||||
recordActivity("warn", "监控任务拉取失败", state.lastError);
|
||||
} finally {
|
||||
state.leaseInFlight = false;
|
||||
state.leaseInFlightKinds.delete("monitor");
|
||||
syncSchedulerSurface();
|
||||
}
|
||||
}
|
||||
@@ -1417,6 +1438,7 @@ async function executeLeasedMonitoringTask(
|
||||
});
|
||||
noteMonitorTaskActivated(taskId);
|
||||
syncSchedulerSurface();
|
||||
queueMicrotask(() => pumpExecutionLoop());
|
||||
|
||||
try {
|
||||
const execution = await executeTaskAdapter(taskRecord, abortController.signal);
|
||||
@@ -1971,11 +1993,14 @@ async function executeLeasedTask(
|
||||
interruptReason: taskRecord.kind === "monitor" ? interruptReasonFromTask(taskRecord) : null,
|
||||
lastSafePointAt: taskRecord.kind === "monitor" ? Date.now() : null,
|
||||
});
|
||||
syncSchedulerSurface();
|
||||
|
||||
if (task.kind === "monitor") {
|
||||
noteMonitorTaskLeased(task, routing);
|
||||
} else {
|
||||
notePublishTaskActivated(taskRecord.id, taskRecord.platform);
|
||||
}
|
||||
syncSchedulerSurface();
|
||||
queueMicrotask(() => pumpExecutionLoop());
|
||||
|
||||
const extendHandle = setInterval(() => {
|
||||
void extendActiveLease(taskRecord.id, leased.lease_token as string);
|
||||
@@ -2061,6 +2086,9 @@ async function executeLeasedTask(
|
||||
|
||||
} finally {
|
||||
clearInterval(extendHandle);
|
||||
if (taskRecord.kind === "publish") {
|
||||
notePublishTaskCompleted(taskRecord.id, taskRecord.platform);
|
||||
}
|
||||
state.activeExecutions.delete(taskRecord.id);
|
||||
syncSchedulerSurface();
|
||||
noteSchedulerTaskFinished();
|
||||
@@ -2551,8 +2579,9 @@ function syncSchedulerSurface(): void {
|
||||
}
|
||||
|
||||
function localQueueDepth(): number {
|
||||
const snapshot = getMonitorSchedulerSnapshot();
|
||||
return state.publishQueue.length + snapshot.queuedCount + snapshot.leasingCount;
|
||||
const publish = getPublishSchedulerSnapshot();
|
||||
const monitor = getMonitorSchedulerSnapshot();
|
||||
return publish.queueDepth + monitor.queuedCount + monitor.leasingCount;
|
||||
}
|
||||
|
||||
function primaryActiveTaskId(): string | null {
|
||||
@@ -2561,8 +2590,26 @@ function primaryActiveTaskId(): string | null {
|
||||
return publish?.taskId ?? active[0]?.taskId ?? null;
|
||||
}
|
||||
|
||||
function hasActivePublishExecution(): boolean {
|
||||
return [...state.activeExecutions.values()].some((item) => item.kind === "publish");
|
||||
function isLeaseInFlight(kind: "publish" | "monitor"): boolean {
|
||||
return state.leaseInFlightKinds.has(kind);
|
||||
}
|
||||
|
||||
function activePublishCount(): number {
|
||||
return [...state.activeExecutions.values()].filter((item) => item.kind === "publish").length;
|
||||
}
|
||||
|
||||
function activePublishPlatforms(): Set<string> {
|
||||
const platforms = new Set(
|
||||
[...state.activeExecutions.values()]
|
||||
.filter((item) => item.kind === "publish")
|
||||
.map((item) => item.platform),
|
||||
);
|
||||
for (const task of getPublishSchedulerSnapshot().tasks) {
|
||||
if (task.state === "active" || task.state === "leasing") {
|
||||
platforms.add(task.platform);
|
||||
}
|
||||
}
|
||||
return platforms;
|
||||
}
|
||||
|
||||
function activeMonitorCount(): number {
|
||||
@@ -2591,8 +2638,42 @@ function activeMonitorQuestionKeys(): Set<string> {
|
||||
);
|
||||
}
|
||||
|
||||
function hasPublishBacklog(): boolean {
|
||||
const snapshot = getPublishSchedulerSnapshot();
|
||||
return snapshot.queuedCount + snapshot.leasingCount > 0;
|
||||
}
|
||||
|
||||
function activeAdmissionCount(): number {
|
||||
const publish = getPublishSchedulerSnapshot();
|
||||
const monitor = getMonitorSchedulerSnapshot();
|
||||
return state.activeExecutions.size
|
||||
+ publish.leasingCount
|
||||
+ monitor.leasingCount
|
||||
+ state.leaseInFlightKinds.size;
|
||||
}
|
||||
|
||||
function hasTotalCapacityForNewExecution(): boolean {
|
||||
return activeAdmissionCount() < resolveAdaptiveTotalConcurrency();
|
||||
}
|
||||
|
||||
function canStartAnotherPublishExecution(): boolean {
|
||||
const snapshot = getPublishSchedulerSnapshot();
|
||||
return snapshot.queuedCount > 0
|
||||
&& !isLeaseInFlight("publish")
|
||||
&& hasTotalCapacityForNewExecution();
|
||||
}
|
||||
|
||||
function shouldPullPublishFallback(): boolean {
|
||||
return !state.dispatchWsConnected
|
||||
&& Date.now() >= state.publishFallbackBackoffUntil
|
||||
&& !isLeaseInFlight("publish")
|
||||
&& !hasPublishBacklog()
|
||||
&& activePublishCount() === 0
|
||||
&& hasTotalCapacityForNewExecution();
|
||||
}
|
||||
|
||||
function canStartAnotherMonitorExecution(): boolean {
|
||||
if (state.publishQueue.length > 0 || hasActivePublishExecution()) {
|
||||
if (hasPublishBacklog() || isLeaseInFlight("publish") || isLeaseInFlight("monitor")) {
|
||||
return false;
|
||||
}
|
||||
const limit = resolveAdaptiveMonitorConcurrency();
|
||||
@@ -2600,7 +2681,8 @@ function canStartAnotherMonitorExecution(): boolean {
|
||||
return false;
|
||||
}
|
||||
const scheduler = getMonitorSchedulerSnapshot();
|
||||
return activeMonitorCount() + scheduler.leasingCount < limit;
|
||||
return activeMonitorCount() + scheduler.leasingCount < limit
|
||||
&& hasTotalCapacityForNewExecution();
|
||||
}
|
||||
|
||||
function resolveLegacyMonitoringLeaseLimit(): number {
|
||||
@@ -2623,6 +2705,11 @@ function resolveLegacyMonitoringLeaseLimit(): number {
|
||||
}
|
||||
|
||||
function resolveAdaptiveMonitorConcurrency(): number {
|
||||
const backgroundBudget = resolveMonitorBackgroundBudget();
|
||||
if (backgroundBudget <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const monitorAccountCount = state.accounts.filter((account) =>
|
||||
isAIPlatformId(account.platform)
|
||||
&& getProjectedAccountHealth({
|
||||
@@ -2633,12 +2720,12 @@ function resolveAdaptiveMonitorConcurrency(): number {
|
||||
}).health === "live",
|
||||
).length;
|
||||
if (monitorAccountCount <= 1) {
|
||||
return monitorAccountCount;
|
||||
return Math.min(monitorAccountCount, backgroundBudget);
|
||||
}
|
||||
|
||||
const metrics = getProcessMetricsSnapshot().latestSample;
|
||||
if (!metrics) {
|
||||
return 1;
|
||||
return Math.min(1, backgroundBudget);
|
||||
}
|
||||
|
||||
const totalCPUUsage = metrics.processes.reduce((sum, item) => sum + item.percentCPUUsage, 0);
|
||||
@@ -2652,12 +2739,92 @@ function resolveAdaptiveMonitorConcurrency(): number {
|
||||
|| mainPrivateBytesKB >= 1_000_000
|
||||
|| processCount >= 40
|
||||
) {
|
||||
return 1;
|
||||
return Math.min(1, backgroundBudget);
|
||||
}
|
||||
|
||||
return Math.min(2, backgroundBudget);
|
||||
}
|
||||
|
||||
function resolveMonitorBackgroundBudget(): number {
|
||||
if (hasPublishBacklog() || isLeaseInFlight("publish")) {
|
||||
return 0;
|
||||
}
|
||||
return Math.max(0, resolveAdaptiveTotalConcurrency() - publishReservedSlots);
|
||||
}
|
||||
|
||||
function resolveAdaptiveTotalConcurrency(): number {
|
||||
const hardwareCap = resolveHardwareTotalConcurrency();
|
||||
const runtimeCap = resolveRuntimeHealthTotalConcurrency(hardwareCap);
|
||||
const featureFlagCap = resolveFeatureFlagTotalConcurrency() ?? maximumRuntimeTotalConcurrency;
|
||||
return clampInteger(
|
||||
Math.min(hardwareCap, runtimeCap, featureFlagCap),
|
||||
featureFlagCap <= 1 ? 1 : minimumForegroundTotalConcurrency,
|
||||
maximumRuntimeTotalConcurrency,
|
||||
);
|
||||
}
|
||||
|
||||
function resolveHardwareTotalConcurrency(): number {
|
||||
const cpuCount = Math.max(1, cpus().length);
|
||||
const memoryGB = totalmem() / 1024 / 1024 / 1024;
|
||||
|
||||
if (cpuCount >= 10 && memoryGB >= 32) {
|
||||
return 4;
|
||||
}
|
||||
if (cpuCount >= 6 && memoryGB >= 16) {
|
||||
return 3;
|
||||
}
|
||||
return 2;
|
||||
}
|
||||
|
||||
function resolveRuntimeHealthTotalConcurrency(hardwareCap: number): number {
|
||||
const metrics = getProcessMetricsSnapshot().latestSample;
|
||||
if (!metrics) {
|
||||
return Math.min(hardwareCap, minimumForegroundTotalConcurrency);
|
||||
}
|
||||
|
||||
const totalCPUUsage = metrics.processes.reduce((sum, item) => sum + item.percentCPUUsage, 0);
|
||||
const totalPrivateBytesKB = metrics.totals.privateBytes;
|
||||
const processCount = metrics.totals.processCount;
|
||||
const mainPrivateBytesKB = metrics.mainProcess.memory?.private ?? 0;
|
||||
|
||||
if (
|
||||
totalCPUUsage >= 180
|
||||
|| totalPrivateBytesKB >= 2_500_000
|
||||
|| mainPrivateBytesKB >= 1_000_000
|
||||
|| processCount >= 40
|
||||
) {
|
||||
return minimumForegroundTotalConcurrency;
|
||||
}
|
||||
|
||||
if (
|
||||
totalCPUUsage >= 120
|
||||
|| totalPrivateBytesKB >= 1_800_000
|
||||
|| mainPrivateBytesKB >= 750_000
|
||||
|| processCount >= 32
|
||||
) {
|
||||
return Math.min(hardwareCap, 3);
|
||||
}
|
||||
|
||||
return hardwareCap;
|
||||
}
|
||||
|
||||
function resolveFeatureFlagTotalConcurrency(): number | null {
|
||||
const raw = process.env.GEO_DESKTOP_MAX_TOTAL_CONCURRENCY?.trim();
|
||||
if (!raw) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parsed = Number.parseInt(raw, 10);
|
||||
if (!Number.isFinite(parsed)) {
|
||||
return null;
|
||||
}
|
||||
return clampInteger(parsed, 1, maximumRuntimeTotalConcurrency);
|
||||
}
|
||||
|
||||
function clampInteger(value: number, min: number, max: number): number {
|
||||
return Math.min(max, Math.max(min, Math.floor(value)));
|
||||
}
|
||||
|
||||
function handleAuthExpired(error: unknown): void {
|
||||
const message = errorMessage(error);
|
||||
state.lastError = message;
|
||||
|
||||
@@ -7,6 +7,7 @@ import { collectRecoverySnapshot } from "./crash-recovery";
|
||||
import { getKeepAlivePlan } from "./keep-alive";
|
||||
import { getLeaseManagerSnapshot } from "./lease-manager";
|
||||
import { getMonitorSchedulerSnapshot } from "./monitor-scheduler";
|
||||
import { getPublishSchedulerSnapshot } from "./publish-scheduler";
|
||||
import { getObservedRequestSnapshot } from "./network-observer";
|
||||
import { getHiddenPlaywrightSnapshot } from "./playwright-cdp";
|
||||
import { getRateLimiterSnapshot } from "./rate-limiter";
|
||||
@@ -169,6 +170,7 @@ function createLiveRuntimeSnapshot(controller: ReturnType<typeof getRuntimeContr
|
||||
lastError: controller.lastError,
|
||||
},
|
||||
scheduler,
|
||||
publishScheduler: getPublishSchedulerSnapshot(),
|
||||
monitorScheduler: getMonitorSchedulerSnapshot(),
|
||||
sessions,
|
||||
hotViews,
|
||||
|
||||
Reference in New Issue
Block a user