feat(desktop): add foreground publish admission scheduler
Publish is now treated as the runtime foreground channel and AI monitoring as background. A new in-memory publish-scheduler owns queue selection, per-platform active locks, and 3-6s post-completion platform cooldowns. The runtime controller tracks lease-in-flight per kind so a monitor lease no longer blocks a publish lease, derives total concurrency from hardware class, current process health, and an optional GEO_DESKTOP_MAX_TOTAL_CONCURRENCY override, and reserves a publish slot when budgeting monitor capacity. Monitor admission returns zero whenever publish has backlog or an in-flight lease, so a cooldown-blocked publish queue still prevents monitor from consuming the foreground reserve. Runtime diagnostics expose publishScheduler alongside monitorScheduler. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,53 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
|
||||||
|
import {
|
||||||
|
enqueuePublishTask,
|
||||||
|
getPublishSchedulerSnapshot,
|
||||||
|
initPublishScheduler,
|
||||||
|
notePublishTaskActivated,
|
||||||
|
notePublishTaskCompleted,
|
||||||
|
selectNextPublishTask,
|
||||||
|
} from "./publish-scheduler";
|
||||||
|
|
||||||
|
describe("publish scheduler", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
vi.setSystemTime(new Date("2026-04-24T00:00:00.000Z"));
|
||||||
|
initPublishScheduler();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
initPublishScheduler();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips same-platform queued tasks while selecting an allowed platform", () => {
|
||||||
|
enqueuePublishTask({ taskId: "toutiao-active", platform: "toutiao", routing: "rabbitmq-primary" });
|
||||||
|
notePublishTaskActivated("toutiao-active", "toutiao");
|
||||||
|
enqueuePublishTask({ taskId: "toutiao-next", platform: "toutiao", routing: "rabbitmq-primary" });
|
||||||
|
enqueuePublishTask({ taskId: "zhihu-next", platform: "zhihu", routing: "rabbitmq-primary" });
|
||||||
|
|
||||||
|
const selected = selectNextPublishTask({
|
||||||
|
maxConcurrency: 2,
|
||||||
|
activeCount: 1,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(selected?.taskId).toBe("zhihu-next");
|
||||||
|
expect(getPublishSchedulerSnapshot().leasingCount).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("holds same-platform tasks during completion cooldown", () => {
|
||||||
|
enqueuePublishTask({ taskId: "first", platform: "toutiao", routing: "rabbitmq-primary" });
|
||||||
|
expect(selectNextPublishTask({ maxConcurrency: 2, activeCount: 0 })?.taskId).toBe("first");
|
||||||
|
notePublishTaskActivated("first", "toutiao");
|
||||||
|
notePublishTaskCompleted("first", "toutiao");
|
||||||
|
|
||||||
|
enqueuePublishTask({ taskId: "second", platform: "toutiao", routing: "rabbitmq-primary" });
|
||||||
|
|
||||||
|
expect(selectNextPublishTask({ maxConcurrency: 2, activeCount: 0 })).toBeNull();
|
||||||
|
|
||||||
|
vi.advanceTimersByTime(6_001);
|
||||||
|
|
||||||
|
expect(selectNextPublishTask({ maxConcurrency: 2, activeCount: 0 })?.taskId).toBe("second");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,202 @@
|
|||||||
|
export type PublishSchedulerRouting = "rabbitmq-primary" | "db-recovery";
|
||||||
|
|
||||||
|
type PublishTaskState = "queued" | "leasing" | "active";
|
||||||
|
|
||||||
|
export interface PublishSchedulerTaskRecord {
|
||||||
|
taskId: string;
|
||||||
|
platform: string;
|
||||||
|
routing: PublishSchedulerRouting;
|
||||||
|
state: PublishTaskState;
|
||||||
|
enqueuedAt: number;
|
||||||
|
updatedAt: number;
|
||||||
|
lastStartedAt: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PublishSchedulerSelection {
|
||||||
|
taskId: string;
|
||||||
|
platform: string;
|
||||||
|
routing: PublishSchedulerRouting;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PublishSchedulerSnapshot {
|
||||||
|
queueDepth: number;
|
||||||
|
queuedCount: number;
|
||||||
|
leasingCount: number;
|
||||||
|
activeCount: number;
|
||||||
|
cooldownCount: number;
|
||||||
|
tasks: PublishSchedulerTaskRecord[];
|
||||||
|
cooldownUntil: Record<string, number>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface PublishSchedulerSelectionOptions {
|
||||||
|
now?: number;
|
||||||
|
maxConcurrency: number;
|
||||||
|
activePlatforms?: ReadonlySet<string>;
|
||||||
|
activeCount: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const publishPlatformCooldownMinMs = 3_000;
|
||||||
|
const publishPlatformCooldownMaxMs = 6_000;
|
||||||
|
|
||||||
|
const tasks = new Map<string, PublishSchedulerTaskRecord>();
|
||||||
|
const cooldownUntil = new Map<string, number>();
|
||||||
|
|
||||||
|
export function initPublishScheduler(): void {
|
||||||
|
tasks.clear();
|
||||||
|
cooldownUntil.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
export function enqueuePublishTask(input: {
|
||||||
|
taskId: string;
|
||||||
|
platform: string;
|
||||||
|
routing: PublishSchedulerRouting;
|
||||||
|
}): void {
|
||||||
|
const taskId = input.taskId.trim();
|
||||||
|
const platform = input.platform.trim();
|
||||||
|
if (!taskId || !platform) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const now = Date.now();
|
||||||
|
const existing = tasks.get(taskId);
|
||||||
|
tasks.set(taskId, {
|
||||||
|
taskId,
|
||||||
|
platform,
|
||||||
|
routing: input.routing,
|
||||||
|
state: existing?.state === "active" ? "active" : "queued",
|
||||||
|
enqueuedAt: existing?.enqueuedAt ?? now,
|
||||||
|
updatedAt: now,
|
||||||
|
lastStartedAt: existing?.lastStartedAt ?? null,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function selectNextPublishTask(
|
||||||
|
options: PublishSchedulerSelectionOptions,
|
||||||
|
): PublishSchedulerSelection | null {
|
||||||
|
const now = options.now ?? Date.now();
|
||||||
|
pruneCooldowns(now);
|
||||||
|
|
||||||
|
if (options.activeCount >= options.maxConcurrency) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const activePlatforms = new Set(options.activePlatforms ?? []);
|
||||||
|
for (const task of tasks.values()) {
|
||||||
|
if (task.state === "active" || task.state === "leasing") {
|
||||||
|
activePlatforms.add(task.platform);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const candidates = [...tasks.values()]
|
||||||
|
.filter((task) => task.state === "queued")
|
||||||
|
.sort((left, right) => {
|
||||||
|
if (left.enqueuedAt !== right.enqueuedAt) {
|
||||||
|
return left.enqueuedAt - right.enqueuedAt;
|
||||||
|
}
|
||||||
|
return left.updatedAt - right.updatedAt;
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const candidate of candidates) {
|
||||||
|
if (activePlatforms.has(candidate.platform)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const platformCooldown = cooldownUntil.get(candidate.platform) ?? 0;
|
||||||
|
if (platformCooldown > now) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
candidate.state = "leasing";
|
||||||
|
candidate.updatedAt = now;
|
||||||
|
tasks.set(candidate.taskId, candidate);
|
||||||
|
return {
|
||||||
|
taskId: candidate.taskId,
|
||||||
|
platform: candidate.platform,
|
||||||
|
routing: candidate.routing,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function notePublishTaskActivated(taskId: string, platform: string): void {
|
||||||
|
const normalizedTaskId = taskId.trim();
|
||||||
|
const normalizedPlatform = platform.trim();
|
||||||
|
if (!normalizedTaskId || !normalizedPlatform) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const now = Date.now();
|
||||||
|
const existing = tasks.get(normalizedTaskId);
|
||||||
|
tasks.set(normalizedTaskId, {
|
||||||
|
taskId: normalizedTaskId,
|
||||||
|
platform: normalizedPlatform,
|
||||||
|
routing: existing?.routing ?? "db-recovery",
|
||||||
|
state: "active",
|
||||||
|
enqueuedAt: existing?.enqueuedAt ?? now,
|
||||||
|
updatedAt: now,
|
||||||
|
lastStartedAt: now,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function notePublishTaskLeaseMiss(taskId: string): void {
|
||||||
|
tasks.delete(taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function notePublishTaskCompleted(taskId: string, platform?: string): void {
|
||||||
|
const existing = tasks.get(taskId);
|
||||||
|
const resolvedPlatform = (platform ?? existing?.platform ?? "").trim();
|
||||||
|
tasks.delete(taskId);
|
||||||
|
|
||||||
|
if (resolvedPlatform) {
|
||||||
|
cooldownUntil.set(resolvedPlatform, Date.now() + randomPublishCooldownMs());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function notePublishTaskCanceled(taskId: string): void {
|
||||||
|
tasks.delete(taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getPublishSchedulerSnapshot(): PublishSchedulerSnapshot {
|
||||||
|
const now = Date.now();
|
||||||
|
pruneCooldowns(now);
|
||||||
|
const sortedTasks = [...tasks.values()].sort((left, right) => {
|
||||||
|
if (left.state !== right.state) {
|
||||||
|
return stateRank(left.state) - stateRank(right.state);
|
||||||
|
}
|
||||||
|
return left.enqueuedAt - right.enqueuedAt;
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
queueDepth: sortedTasks.filter((task) => task.state === "queued" || task.state === "leasing").length,
|
||||||
|
queuedCount: sortedTasks.filter((task) => task.state === "queued").length,
|
||||||
|
leasingCount: sortedTasks.filter((task) => task.state === "leasing").length,
|
||||||
|
activeCount: sortedTasks.filter((task) => task.state === "active").length,
|
||||||
|
cooldownCount: cooldownUntil.size,
|
||||||
|
tasks: sortedTasks.map((task) => ({ ...task })),
|
||||||
|
cooldownUntil: Object.fromEntries(cooldownUntil.entries()),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function pruneCooldowns(now: number): void {
|
||||||
|
for (const [platform, until] of cooldownUntil.entries()) {
|
||||||
|
if (until <= now) {
|
||||||
|
cooldownUntil.delete(platform);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function randomPublishCooldownMs(): number {
|
||||||
|
const spread = publishPlatformCooldownMaxMs - publishPlatformCooldownMinMs;
|
||||||
|
return publishPlatformCooldownMinMs + Math.floor(Math.random() * (spread + 1));
|
||||||
|
}
|
||||||
|
|
||||||
|
function stateRank(state: PublishTaskState): number {
|
||||||
|
if (state === "active") {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (state === "leasing") {
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
return 2;
|
||||||
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
import { hostname } from "node:os";
|
import { cpus, hostname, totalmem } from "node:os";
|
||||||
|
|
||||||
import { ApiClientError } from "@geo/http-client";
|
import { ApiClientError } from "@geo/http-client";
|
||||||
import type {
|
import type {
|
||||||
@@ -56,6 +56,16 @@ import {
|
|||||||
noteMonitorTaskLeased,
|
noteMonitorTaskLeased,
|
||||||
selectNextMonitorTask,
|
selectNextMonitorTask,
|
||||||
} from "./monitor-scheduler";
|
} from "./monitor-scheduler";
|
||||||
|
import {
|
||||||
|
enqueuePublishTask as enqueuePublishSchedulerTask,
|
||||||
|
getPublishSchedulerSnapshot,
|
||||||
|
initPublishScheduler,
|
||||||
|
notePublishTaskActivated,
|
||||||
|
notePublishTaskCanceled,
|
||||||
|
notePublishTaskCompleted,
|
||||||
|
notePublishTaskLeaseMiss,
|
||||||
|
selectNextPublishTask,
|
||||||
|
} from "./publish-scheduler";
|
||||||
import {
|
import {
|
||||||
retainHiddenPlaywrightPage,
|
retainHiddenPlaywrightPage,
|
||||||
startHiddenPlaywrightReaper,
|
startHiddenPlaywrightReaper,
|
||||||
@@ -126,6 +136,9 @@ const leaseExtendIntervalMs = 60_000;
|
|||||||
const maxActivityItems = 60;
|
const maxActivityItems = 60;
|
||||||
const monitorBusinessTimeZone = "Asia/Shanghai";
|
const monitorBusinessTimeZone = "Asia/Shanghai";
|
||||||
const authRequiredMonitorPlatforms = new Set(["yuanbao", "kimi", "deepseek", "doubao", "wenxin"]);
|
const authRequiredMonitorPlatforms = new Set(["yuanbao", "kimi", "deepseek", "doubao", "wenxin"]);
|
||||||
|
const publishReservedSlots = 1;
|
||||||
|
const minimumForegroundTotalConcurrency = 2;
|
||||||
|
const maximumRuntimeTotalConcurrency = 4;
|
||||||
|
|
||||||
interface RuntimeTaskRecord {
|
interface RuntimeTaskRecord {
|
||||||
id: string;
|
id: string;
|
||||||
@@ -178,6 +191,7 @@ export interface RuntimeControllerSnapshot {
|
|||||||
interface PendingTaskRequest {
|
interface PendingTaskRequest {
|
||||||
taskId: string;
|
taskId: string;
|
||||||
routing: RuntimeTaskRouting;
|
routing: RuntimeTaskRouting;
|
||||||
|
platform?: string;
|
||||||
source?: "desktop_task" | "monitoring_lease";
|
source?: "desktop_task" | "monitoring_lease";
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -198,14 +212,13 @@ interface RuntimeState {
|
|||||||
session: DesktopRuntimeSessionSyncRequest | null;
|
session: DesktopRuntimeSessionSyncRequest | null;
|
||||||
client: DesktopClientInfo | null;
|
client: DesktopClientInfo | null;
|
||||||
running: boolean;
|
running: boolean;
|
||||||
publishQueue: PendingTaskRequest[];
|
|
||||||
queuedPublishTaskIds: Set<string>;
|
|
||||||
tasks: Map<string, RuntimeTaskRecord>;
|
tasks: Map<string, RuntimeTaskRecord>;
|
||||||
accounts: DesktopAccountInfo[];
|
accounts: DesktopAccountInfo[];
|
||||||
accountProfiles: Map<string, PublishAccountProfile>;
|
accountProfiles: Map<string, PublishAccountProfile>;
|
||||||
activity: RuntimeControllerActivity[];
|
activity: RuntimeControllerActivity[];
|
||||||
activeExecutions: Map<string, ActiveRuntimeExecution>;
|
activeExecutions: Map<string, ActiveRuntimeExecution>;
|
||||||
leaseInFlight: boolean;
|
leaseInFlightKinds: Set<"publish" | "monitor">;
|
||||||
|
publishFallbackBackoffUntil: number;
|
||||||
heartbeatTimer: ReturnType<typeof setInterval> | null;
|
heartbeatTimer: ReturnType<typeof setInterval> | null;
|
||||||
pullTimer: ReturnType<typeof setInterval> | null;
|
pullTimer: ReturnType<typeof setInterval> | null;
|
||||||
dispatchWsClient: DispatchWsClient | null;
|
dispatchWsClient: DispatchWsClient | null;
|
||||||
@@ -224,14 +237,13 @@ const state: RuntimeState = {
|
|||||||
session: null,
|
session: null,
|
||||||
client: null,
|
client: null,
|
||||||
running: false,
|
running: false,
|
||||||
publishQueue: [],
|
|
||||||
queuedPublishTaskIds: new Set<string>(),
|
|
||||||
tasks: new Map<string, RuntimeTaskRecord>(),
|
tasks: new Map<string, RuntimeTaskRecord>(),
|
||||||
accounts: [],
|
accounts: [],
|
||||||
accountProfiles: new Map<string, PublishAccountProfile>(),
|
accountProfiles: new Map<string, PublishAccountProfile>(),
|
||||||
activity: [],
|
activity: [],
|
||||||
activeExecutions: new Map<string, ActiveRuntimeExecution>(),
|
activeExecutions: new Map<string, ActiveRuntimeExecution>(),
|
||||||
leaseInFlight: false,
|
leaseInFlightKinds: new Set<"publish" | "monitor">(),
|
||||||
|
publishFallbackBackoffUntil: 0,
|
||||||
heartbeatTimer: null,
|
heartbeatTimer: null,
|
||||||
pullTimer: null,
|
pullTimer: null,
|
||||||
dispatchWsClient: null,
|
dispatchWsClient: null,
|
||||||
@@ -420,6 +432,7 @@ function startRuntime(): void {
|
|||||||
|
|
||||||
initScheduler();
|
initScheduler();
|
||||||
initMonitorScheduler();
|
initMonitorScheduler();
|
||||||
|
initPublishScheduler();
|
||||||
startHiddenPlaywrightReaper();
|
startHiddenPlaywrightReaper();
|
||||||
state.running = true;
|
state.running = true;
|
||||||
setSchedulerPhase("running");
|
setSchedulerPhase("running");
|
||||||
@@ -438,7 +451,7 @@ function startRuntime(): void {
|
|||||||
|
|
||||||
function stopRuntime(): void {
|
function stopRuntime(): void {
|
||||||
state.running = false;
|
state.running = false;
|
||||||
state.leaseInFlight = false;
|
state.leaseInFlightKinds.clear();
|
||||||
for (const execution of state.activeExecutions.values()) {
|
for (const execution of state.activeExecutions.values()) {
|
||||||
execution.abortController.abort();
|
execution.abortController.abort();
|
||||||
}
|
}
|
||||||
@@ -471,8 +484,7 @@ function stopRuntime(): void {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function clearRuntimeState(): void {
|
function clearRuntimeState(): void {
|
||||||
state.publishQueue = [];
|
initPublishScheduler();
|
||||||
state.queuedPublishTaskIds.clear();
|
|
||||||
state.tasks.clear();
|
state.tasks.clear();
|
||||||
state.accounts = [];
|
state.accounts = [];
|
||||||
state.accountProfiles.clear();
|
state.accountProfiles.clear();
|
||||||
@@ -484,6 +496,7 @@ function clearRuntimeState(): void {
|
|||||||
state.lastAccountsSyncAt = 0;
|
state.lastAccountsSyncAt = 0;
|
||||||
state.lastServerTime = null;
|
state.lastServerTime = null;
|
||||||
state.lastError = null;
|
state.lastError = null;
|
||||||
|
state.publishFallbackBackoffUntil = 0;
|
||||||
setSchedulerQueueDepth(localQueueDepth());
|
setSchedulerQueueDepth(localQueueDepth());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -646,9 +659,16 @@ function handleTaskEvent(event: DesktopTaskEventMessage): void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
event.kind === "publish"
|
||||||
|
&& (event.type === "task_completed" || event.type === "task_reconciled" || event.type === "task_canceled")
|
||||||
|
) {
|
||||||
|
notePublishTaskCanceled(event.task_id);
|
||||||
|
}
|
||||||
|
|
||||||
if (event.type === "task_available") {
|
if (event.type === "task_available") {
|
||||||
if (event.kind === "publish") {
|
if (event.kind === "publish") {
|
||||||
enqueuePublishTask(event.task_id, "rabbitmq-primary");
|
enqueuePublishTask(next.id, next.platform, "rabbitmq-primary");
|
||||||
}
|
}
|
||||||
pumpExecutionLoop();
|
pumpExecutionLoop();
|
||||||
}
|
}
|
||||||
@@ -1036,43 +1056,40 @@ export async function unbindRuntimeAccount(accountId: string, syncVersion: numbe
|
|||||||
await syncAccounts("manual");
|
await syncAccounts("manual");
|
||||||
}
|
}
|
||||||
|
|
||||||
function enqueuePublishTask(taskId: string, routing: RuntimeTaskRouting): void {
|
function enqueuePublishTask(taskId: string, platform: string, routing: RuntimeTaskRouting): void {
|
||||||
if (!taskId || state.activeExecutions.has(taskId) || state.queuedPublishTaskIds.has(taskId)) {
|
if (!taskId || state.activeExecutions.has(taskId)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
state.publishQueue.push({ taskId, routing });
|
enqueuePublishSchedulerTask({ taskId, platform, routing });
|
||||||
state.queuedPublishTaskIds.add(taskId);
|
|
||||||
syncSchedulerSurface();
|
syncSchedulerSurface();
|
||||||
}
|
}
|
||||||
|
|
||||||
function dequeuePublishTask(): PendingTaskRequest | null {
|
|
||||||
const next = state.publishQueue.shift() ?? null;
|
|
||||||
if (next) {
|
|
||||||
state.queuedPublishTaskIds.delete(next.taskId);
|
|
||||||
}
|
|
||||||
syncSchedulerSurface();
|
|
||||||
return next;
|
|
||||||
}
|
|
||||||
|
|
||||||
function pumpExecutionLoop(): void {
|
function pumpExecutionLoop(): void {
|
||||||
if (!state.running || state.leaseInFlight) {
|
if (!state.running) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
syncSchedulerSurface();
|
syncSchedulerSurface();
|
||||||
|
|
||||||
if (hasActivePublishExecution()) {
|
if (canStartAnotherPublishExecution()) {
|
||||||
|
const selected = selectNextPublishTask({
|
||||||
|
maxConcurrency: resolveAdaptiveTotalConcurrency(),
|
||||||
|
activePlatforms: activePublishPlatforms(),
|
||||||
|
activeCount: activePublishCount(),
|
||||||
|
});
|
||||||
|
if (selected) {
|
||||||
|
void leaseSpecificTask(selected, "publish");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (shouldPullPublishFallback()) {
|
||||||
|
void pullNextTask("publish");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (state.publishQueue.length > 0) {
|
if (hasPublishBacklog()) {
|
||||||
if (state.activeExecutions.size === 0) {
|
|
||||||
const queuedPublish = dequeuePublishTask();
|
|
||||||
if (queuedPublish) {
|
|
||||||
void leaseSpecificTask(queuedPublish, "publish");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1089,30 +1106,22 @@ function pumpExecutionLoop(): void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (state.activeExecutions.size === 0) {
|
return;
|
||||||
// Publish task_available is push-delivered over the dispatch WebSocket. We
|
|
||||||
// only fall back to the HTTP pull when the push channel is down, otherwise
|
|
||||||
// every scheduler tick would wake the /lease endpoint for no reason.
|
|
||||||
if (!state.dispatchWsConnected) {
|
|
||||||
void pullNextTask("publish");
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function leaseSpecificTask(
|
async function leaseSpecificTask(
|
||||||
request: PendingTaskRequest,
|
request: PendingTaskRequest,
|
||||||
expectedKind: "publish" | "monitor",
|
expectedKind: "publish" | "monitor",
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
if (!canRunLive(state.session) || state.leaseInFlight) {
|
if (!canRunLive(state.session) || isLeaseInFlight(expectedKind)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
state.leaseInFlight = true;
|
state.leaseInFlightKinds.add(expectedKind);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (expectedKind === "monitor" && request.source === "monitoring_lease") {
|
if (expectedKind === "monitor" && request.source === "monitoring_lease") {
|
||||||
state.leaseInFlight = false;
|
state.leaseInFlightKinds.delete(expectedKind);
|
||||||
await executeLeasedMonitoringTask(request.taskId, request.routing);
|
await executeLeasedMonitoringTask(request.taskId, request.routing);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -1121,10 +1130,12 @@ async function leaseSpecificTask(
|
|||||||
if (!leased.task) {
|
if (!leased.task) {
|
||||||
if (expectedKind === "monitor") {
|
if (expectedKind === "monitor") {
|
||||||
noteMonitorTaskLeaseMiss(request.taskId);
|
noteMonitorTaskLeaseMiss(request.taskId);
|
||||||
|
} else {
|
||||||
|
notePublishTaskLeaseMiss(request.taskId);
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
state.leaseInFlight = false;
|
state.leaseInFlightKinds.delete(expectedKind);
|
||||||
await executeLeasedTask(leased, request.routing);
|
await executeLeasedTask(leased, request.routing);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (expectedKind === "monitor") {
|
if (expectedKind === "monitor") {
|
||||||
@@ -1141,6 +1152,8 @@ async function leaseSpecificTask(
|
|||||||
} else {
|
} else {
|
||||||
noteMonitorTaskLeaseMiss(request.taskId);
|
noteMonitorTaskLeaseMiss(request.taskId);
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
notePublishTaskLeaseMiss(request.taskId);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isApiClientError(error, 401)) {
|
if (isApiClientError(error, 401)) {
|
||||||
@@ -1152,10 +1165,10 @@ async function leaseSpecificTask(
|
|||||||
setSchedulerError(state.lastError);
|
setSchedulerError(state.lastError);
|
||||||
recordActivity("warn", "指定任务领取失败", `${request.taskId} 未能成功领取:${state.lastError}`);
|
recordActivity("warn", "指定任务领取失败", `${request.taskId} 未能成功领取:${state.lastError}`);
|
||||||
} finally {
|
} finally {
|
||||||
if (state.leaseInFlight) {
|
if (state.leaseInFlightKinds.has(expectedKind)) {
|
||||||
state.leaseInFlight = false;
|
state.leaseInFlightKinds.delete(expectedKind);
|
||||||
syncSchedulerSurface();
|
syncSchedulerSurface();
|
||||||
if (state.publishQueue.length > 0 || localQueueDepth() > 0) {
|
if (localQueueDepth() > 0) {
|
||||||
pumpExecutionLoop();
|
pumpExecutionLoop();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1164,11 +1177,11 @@ async function leaseSpecificTask(
|
|||||||
|
|
||||||
async function pullNextTask(kind: "publish" | "monitor"): Promise<void> {
|
async function pullNextTask(kind: "publish" | "monitor"): Promise<void> {
|
||||||
if (kind === "monitor") {
|
if (kind === "monitor") {
|
||||||
if (!canRunLive(state.session) || state.leaseInFlight) {
|
if (!canRunLive(state.session) || isLeaseInFlight("monitor")) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
state.leaseInFlight = true;
|
state.leaseInFlightKinds.add("monitor");
|
||||||
try {
|
try {
|
||||||
const leased = await leaseDesktopTask({ kind: "monitor" });
|
const leased = await leaseDesktopTask({ kind: "monitor" });
|
||||||
state.lastPullAt = Date.now();
|
state.lastPullAt = Date.now();
|
||||||
@@ -1177,7 +1190,7 @@ async function pullNextTask(kind: "publish" | "monitor"): Promise<void> {
|
|||||||
noteSchedulerPull();
|
noteSchedulerPull();
|
||||||
|
|
||||||
if (leased.task) {
|
if (leased.task) {
|
||||||
state.leaseInFlight = false;
|
state.leaseInFlightKinds.delete("monitor");
|
||||||
await executeLeasedTask(leased, "rabbitmq-primary");
|
await executeLeasedTask(leased, "rabbitmq-primary");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -1193,7 +1206,7 @@ async function pullNextTask(kind: "publish" | "monitor"): Promise<void> {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
state.leaseInFlight = false;
|
state.leaseInFlightKinds.delete("monitor");
|
||||||
syncSchedulerSurface();
|
syncSchedulerSurface();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1201,11 +1214,12 @@ async function pullNextTask(kind: "publish" | "monitor"): Promise<void> {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!canRunLive(state.session) || state.leaseInFlight) {
|
if (!canRunLive(state.session) || isLeaseInFlight("publish")) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
state.leaseInFlight = true;
|
let leasedPublishTask = false;
|
||||||
|
state.leaseInFlightKinds.add("publish");
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const leased = await leaseDesktopTask({ kind });
|
const leased = await leaseDesktopTask({ kind });
|
||||||
@@ -1215,15 +1229,19 @@ async function pullNextTask(kind: "publish" | "monitor"): Promise<void> {
|
|||||||
noteSchedulerPull();
|
noteSchedulerPull();
|
||||||
|
|
||||||
if (!leased.task) {
|
if (!leased.task) {
|
||||||
|
state.publishFallbackBackoffUntil = Date.now() + pullIntervalMs;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
state.leaseInFlight = false;
|
state.publishFallbackBackoffUntil = 0;
|
||||||
|
leasedPublishTask = true;
|
||||||
|
state.leaseInFlightKinds.delete("publish");
|
||||||
await executeLeasedTask(leased, "db-recovery");
|
await executeLeasedTask(leased, "db-recovery");
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
state.lastPullAt = Date.now();
|
state.lastPullAt = Date.now();
|
||||||
state.lastPullStatus = "failed";
|
state.lastPullStatus = "failed";
|
||||||
state.lastError = errorMessage(error);
|
state.lastError = errorMessage(error);
|
||||||
|
state.publishFallbackBackoffUntil = Date.now() + pullIntervalMs;
|
||||||
|
|
||||||
noteTransportPull(false);
|
noteTransportPull(false);
|
||||||
setSchedulerError(state.lastError);
|
setSchedulerError(state.lastError);
|
||||||
@@ -1235,18 +1253,21 @@ async function pullNextTask(kind: "publish" | "monitor"): Promise<void> {
|
|||||||
|
|
||||||
recordActivity("warn", "兜底拉取失败", state.lastError);
|
recordActivity("warn", "兜底拉取失败", state.lastError);
|
||||||
} finally {
|
} finally {
|
||||||
state.leaseInFlight = false;
|
state.leaseInFlightKinds.delete("publish");
|
||||||
syncSchedulerSurface();
|
syncSchedulerSurface();
|
||||||
|
if (!leasedPublishTask && state.running) {
|
||||||
|
queueMicrotask(() => pumpExecutionLoop());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function pullMonitoringTasks(routing: RuntimeTaskRouting): Promise<void> {
|
async function pullMonitoringTasks(routing: RuntimeTaskRouting): Promise<void> {
|
||||||
const leaseLimit = resolveLegacyMonitoringLeaseLimit();
|
const leaseLimit = resolveLegacyMonitoringLeaseLimit();
|
||||||
if (!canRunLive(state.session) || state.leaseInFlight || leaseLimit <= 0) {
|
if (!canRunLive(state.session) || isLeaseInFlight("monitor") || leaseLimit <= 0) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
state.leaseInFlight = true;
|
state.leaseInFlightKinds.add("monitor");
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const leased = await leaseMonitoringTasks({
|
const leased = await leaseMonitoringTasks({
|
||||||
@@ -1272,7 +1293,7 @@ async function pullMonitoringTasks(routing: RuntimeTaskRouting): Promise<void> {
|
|||||||
|
|
||||||
recordActivity("warn", "监控任务拉取失败", state.lastError);
|
recordActivity("warn", "监控任务拉取失败", state.lastError);
|
||||||
} finally {
|
} finally {
|
||||||
state.leaseInFlight = false;
|
state.leaseInFlightKinds.delete("monitor");
|
||||||
syncSchedulerSurface();
|
syncSchedulerSurface();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1417,6 +1438,7 @@ async function executeLeasedMonitoringTask(
|
|||||||
});
|
});
|
||||||
noteMonitorTaskActivated(taskId);
|
noteMonitorTaskActivated(taskId);
|
||||||
syncSchedulerSurface();
|
syncSchedulerSurface();
|
||||||
|
queueMicrotask(() => pumpExecutionLoop());
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const execution = await executeTaskAdapter(taskRecord, abortController.signal);
|
const execution = await executeTaskAdapter(taskRecord, abortController.signal);
|
||||||
@@ -1971,11 +1993,14 @@ async function executeLeasedTask(
|
|||||||
interruptReason: taskRecord.kind === "monitor" ? interruptReasonFromTask(taskRecord) : null,
|
interruptReason: taskRecord.kind === "monitor" ? interruptReasonFromTask(taskRecord) : null,
|
||||||
lastSafePointAt: taskRecord.kind === "monitor" ? Date.now() : null,
|
lastSafePointAt: taskRecord.kind === "monitor" ? Date.now() : null,
|
||||||
});
|
});
|
||||||
syncSchedulerSurface();
|
|
||||||
|
|
||||||
if (task.kind === "monitor") {
|
if (task.kind === "monitor") {
|
||||||
noteMonitorTaskLeased(task, routing);
|
noteMonitorTaskLeased(task, routing);
|
||||||
|
} else {
|
||||||
|
notePublishTaskActivated(taskRecord.id, taskRecord.platform);
|
||||||
}
|
}
|
||||||
|
syncSchedulerSurface();
|
||||||
|
queueMicrotask(() => pumpExecutionLoop());
|
||||||
|
|
||||||
const extendHandle = setInterval(() => {
|
const extendHandle = setInterval(() => {
|
||||||
void extendActiveLease(taskRecord.id, leased.lease_token as string);
|
void extendActiveLease(taskRecord.id, leased.lease_token as string);
|
||||||
@@ -2061,6 +2086,9 @@ async function executeLeasedTask(
|
|||||||
|
|
||||||
} finally {
|
} finally {
|
||||||
clearInterval(extendHandle);
|
clearInterval(extendHandle);
|
||||||
|
if (taskRecord.kind === "publish") {
|
||||||
|
notePublishTaskCompleted(taskRecord.id, taskRecord.platform);
|
||||||
|
}
|
||||||
state.activeExecutions.delete(taskRecord.id);
|
state.activeExecutions.delete(taskRecord.id);
|
||||||
syncSchedulerSurface();
|
syncSchedulerSurface();
|
||||||
noteSchedulerTaskFinished();
|
noteSchedulerTaskFinished();
|
||||||
@@ -2551,8 +2579,9 @@ function syncSchedulerSurface(): void {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function localQueueDepth(): number {
|
function localQueueDepth(): number {
|
||||||
const snapshot = getMonitorSchedulerSnapshot();
|
const publish = getPublishSchedulerSnapshot();
|
||||||
return state.publishQueue.length + snapshot.queuedCount + snapshot.leasingCount;
|
const monitor = getMonitorSchedulerSnapshot();
|
||||||
|
return publish.queueDepth + monitor.queuedCount + monitor.leasingCount;
|
||||||
}
|
}
|
||||||
|
|
||||||
function primaryActiveTaskId(): string | null {
|
function primaryActiveTaskId(): string | null {
|
||||||
@@ -2561,8 +2590,26 @@ function primaryActiveTaskId(): string | null {
|
|||||||
return publish?.taskId ?? active[0]?.taskId ?? null;
|
return publish?.taskId ?? active[0]?.taskId ?? null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function hasActivePublishExecution(): boolean {
|
function isLeaseInFlight(kind: "publish" | "monitor"): boolean {
|
||||||
return [...state.activeExecutions.values()].some((item) => item.kind === "publish");
|
return state.leaseInFlightKinds.has(kind);
|
||||||
|
}
|
||||||
|
|
||||||
|
function activePublishCount(): number {
|
||||||
|
return [...state.activeExecutions.values()].filter((item) => item.kind === "publish").length;
|
||||||
|
}
|
||||||
|
|
||||||
|
function activePublishPlatforms(): Set<string> {
|
||||||
|
const platforms = new Set(
|
||||||
|
[...state.activeExecutions.values()]
|
||||||
|
.filter((item) => item.kind === "publish")
|
||||||
|
.map((item) => item.platform),
|
||||||
|
);
|
||||||
|
for (const task of getPublishSchedulerSnapshot().tasks) {
|
||||||
|
if (task.state === "active" || task.state === "leasing") {
|
||||||
|
platforms.add(task.platform);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return platforms;
|
||||||
}
|
}
|
||||||
|
|
||||||
function activeMonitorCount(): number {
|
function activeMonitorCount(): number {
|
||||||
@@ -2591,8 +2638,42 @@ function activeMonitorQuestionKeys(): Set<string> {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function hasPublishBacklog(): boolean {
|
||||||
|
const snapshot = getPublishSchedulerSnapshot();
|
||||||
|
return snapshot.queuedCount + snapshot.leasingCount > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
function activeAdmissionCount(): number {
|
||||||
|
const publish = getPublishSchedulerSnapshot();
|
||||||
|
const monitor = getMonitorSchedulerSnapshot();
|
||||||
|
return state.activeExecutions.size
|
||||||
|
+ publish.leasingCount
|
||||||
|
+ monitor.leasingCount
|
||||||
|
+ state.leaseInFlightKinds.size;
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasTotalCapacityForNewExecution(): boolean {
|
||||||
|
return activeAdmissionCount() < resolveAdaptiveTotalConcurrency();
|
||||||
|
}
|
||||||
|
|
||||||
|
function canStartAnotherPublishExecution(): boolean {
|
||||||
|
const snapshot = getPublishSchedulerSnapshot();
|
||||||
|
return snapshot.queuedCount > 0
|
||||||
|
&& !isLeaseInFlight("publish")
|
||||||
|
&& hasTotalCapacityForNewExecution();
|
||||||
|
}
|
||||||
|
|
||||||
|
function shouldPullPublishFallback(): boolean {
|
||||||
|
return !state.dispatchWsConnected
|
||||||
|
&& Date.now() >= state.publishFallbackBackoffUntil
|
||||||
|
&& !isLeaseInFlight("publish")
|
||||||
|
&& !hasPublishBacklog()
|
||||||
|
&& activePublishCount() === 0
|
||||||
|
&& hasTotalCapacityForNewExecution();
|
||||||
|
}
|
||||||
|
|
||||||
function canStartAnotherMonitorExecution(): boolean {
|
function canStartAnotherMonitorExecution(): boolean {
|
||||||
if (state.publishQueue.length > 0 || hasActivePublishExecution()) {
|
if (hasPublishBacklog() || isLeaseInFlight("publish") || isLeaseInFlight("monitor")) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
const limit = resolveAdaptiveMonitorConcurrency();
|
const limit = resolveAdaptiveMonitorConcurrency();
|
||||||
@@ -2600,7 +2681,8 @@ function canStartAnotherMonitorExecution(): boolean {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
const scheduler = getMonitorSchedulerSnapshot();
|
const scheduler = getMonitorSchedulerSnapshot();
|
||||||
return activeMonitorCount() + scheduler.leasingCount < limit;
|
return activeMonitorCount() + scheduler.leasingCount < limit
|
||||||
|
&& hasTotalCapacityForNewExecution();
|
||||||
}
|
}
|
||||||
|
|
||||||
function resolveLegacyMonitoringLeaseLimit(): number {
|
function resolveLegacyMonitoringLeaseLimit(): number {
|
||||||
@@ -2623,6 +2705,11 @@ function resolveLegacyMonitoringLeaseLimit(): number {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function resolveAdaptiveMonitorConcurrency(): number {
|
function resolveAdaptiveMonitorConcurrency(): number {
|
||||||
|
const backgroundBudget = resolveMonitorBackgroundBudget();
|
||||||
|
if (backgroundBudget <= 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
const monitorAccountCount = state.accounts.filter((account) =>
|
const monitorAccountCount = state.accounts.filter((account) =>
|
||||||
isAIPlatformId(account.platform)
|
isAIPlatformId(account.platform)
|
||||||
&& getProjectedAccountHealth({
|
&& getProjectedAccountHealth({
|
||||||
@@ -2633,12 +2720,12 @@ function resolveAdaptiveMonitorConcurrency(): number {
|
|||||||
}).health === "live",
|
}).health === "live",
|
||||||
).length;
|
).length;
|
||||||
if (monitorAccountCount <= 1) {
|
if (monitorAccountCount <= 1) {
|
||||||
return monitorAccountCount;
|
return Math.min(monitorAccountCount, backgroundBudget);
|
||||||
}
|
}
|
||||||
|
|
||||||
const metrics = getProcessMetricsSnapshot().latestSample;
|
const metrics = getProcessMetricsSnapshot().latestSample;
|
||||||
if (!metrics) {
|
if (!metrics) {
|
||||||
return 1;
|
return Math.min(1, backgroundBudget);
|
||||||
}
|
}
|
||||||
|
|
||||||
const totalCPUUsage = metrics.processes.reduce((sum, item) => sum + item.percentCPUUsage, 0);
|
const totalCPUUsage = metrics.processes.reduce((sum, item) => sum + item.percentCPUUsage, 0);
|
||||||
@@ -2652,12 +2739,92 @@ function resolveAdaptiveMonitorConcurrency(): number {
|
|||||||
|| mainPrivateBytesKB >= 1_000_000
|
|| mainPrivateBytesKB >= 1_000_000
|
||||||
|| processCount >= 40
|
|| processCount >= 40
|
||||||
) {
|
) {
|
||||||
return 1;
|
return Math.min(1, backgroundBudget);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return Math.min(2, backgroundBudget);
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveMonitorBackgroundBudget(): number {
|
||||||
|
if (hasPublishBacklog() || isLeaseInFlight("publish")) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return Math.max(0, resolveAdaptiveTotalConcurrency() - publishReservedSlots);
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveAdaptiveTotalConcurrency(): number {
|
||||||
|
const hardwareCap = resolveHardwareTotalConcurrency();
|
||||||
|
const runtimeCap = resolveRuntimeHealthTotalConcurrency(hardwareCap);
|
||||||
|
const featureFlagCap = resolveFeatureFlagTotalConcurrency() ?? maximumRuntimeTotalConcurrency;
|
||||||
|
return clampInteger(
|
||||||
|
Math.min(hardwareCap, runtimeCap, featureFlagCap),
|
||||||
|
featureFlagCap <= 1 ? 1 : minimumForegroundTotalConcurrency,
|
||||||
|
maximumRuntimeTotalConcurrency,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveHardwareTotalConcurrency(): number {
|
||||||
|
const cpuCount = Math.max(1, cpus().length);
|
||||||
|
const memoryGB = totalmem() / 1024 / 1024 / 1024;
|
||||||
|
|
||||||
|
if (cpuCount >= 10 && memoryGB >= 32) {
|
||||||
|
return 4;
|
||||||
|
}
|
||||||
|
if (cpuCount >= 6 && memoryGB >= 16) {
|
||||||
|
return 3;
|
||||||
|
}
|
||||||
return 2;
|
return 2;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function resolveRuntimeHealthTotalConcurrency(hardwareCap: number): number {
|
||||||
|
const metrics = getProcessMetricsSnapshot().latestSample;
|
||||||
|
if (!metrics) {
|
||||||
|
return Math.min(hardwareCap, minimumForegroundTotalConcurrency);
|
||||||
|
}
|
||||||
|
|
||||||
|
const totalCPUUsage = metrics.processes.reduce((sum, item) => sum + item.percentCPUUsage, 0);
|
||||||
|
const totalPrivateBytesKB = metrics.totals.privateBytes;
|
||||||
|
const processCount = metrics.totals.processCount;
|
||||||
|
const mainPrivateBytesKB = metrics.mainProcess.memory?.private ?? 0;
|
||||||
|
|
||||||
|
if (
|
||||||
|
totalCPUUsage >= 180
|
||||||
|
|| totalPrivateBytesKB >= 2_500_000
|
||||||
|
|| mainPrivateBytesKB >= 1_000_000
|
||||||
|
|| processCount >= 40
|
||||||
|
) {
|
||||||
|
return minimumForegroundTotalConcurrency;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
totalCPUUsage >= 120
|
||||||
|
|| totalPrivateBytesKB >= 1_800_000
|
||||||
|
|| mainPrivateBytesKB >= 750_000
|
||||||
|
|| processCount >= 32
|
||||||
|
) {
|
||||||
|
return Math.min(hardwareCap, 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
return hardwareCap;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveFeatureFlagTotalConcurrency(): number | null {
|
||||||
|
const raw = process.env.GEO_DESKTOP_MAX_TOTAL_CONCURRENCY?.trim();
|
||||||
|
if (!raw) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const parsed = Number.parseInt(raw, 10);
|
||||||
|
if (!Number.isFinite(parsed)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return clampInteger(parsed, 1, maximumRuntimeTotalConcurrency);
|
||||||
|
}
|
||||||
|
|
||||||
|
function clampInteger(value: number, min: number, max: number): number {
|
||||||
|
return Math.min(max, Math.max(min, Math.floor(value)));
|
||||||
|
}
|
||||||
|
|
||||||
function handleAuthExpired(error: unknown): void {
|
function handleAuthExpired(error: unknown): void {
|
||||||
const message = errorMessage(error);
|
const message = errorMessage(error);
|
||||||
state.lastError = message;
|
state.lastError = message;
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import { collectRecoverySnapshot } from "./crash-recovery";
|
|||||||
import { getKeepAlivePlan } from "./keep-alive";
|
import { getKeepAlivePlan } from "./keep-alive";
|
||||||
import { getLeaseManagerSnapshot } from "./lease-manager";
|
import { getLeaseManagerSnapshot } from "./lease-manager";
|
||||||
import { getMonitorSchedulerSnapshot } from "./monitor-scheduler";
|
import { getMonitorSchedulerSnapshot } from "./monitor-scheduler";
|
||||||
|
import { getPublishSchedulerSnapshot } from "./publish-scheduler";
|
||||||
import { getObservedRequestSnapshot } from "./network-observer";
|
import { getObservedRequestSnapshot } from "./network-observer";
|
||||||
import { getHiddenPlaywrightSnapshot } from "./playwright-cdp";
|
import { getHiddenPlaywrightSnapshot } from "./playwright-cdp";
|
||||||
import { getRateLimiterSnapshot } from "./rate-limiter";
|
import { getRateLimiterSnapshot } from "./rate-limiter";
|
||||||
@@ -169,6 +170,7 @@ function createLiveRuntimeSnapshot(controller: ReturnType<typeof getRuntimeContr
|
|||||||
lastError: controller.lastError,
|
lastError: controller.lastError,
|
||||||
},
|
},
|
||||||
scheduler,
|
scheduler,
|
||||||
|
publishScheduler: getPublishSchedulerSnapshot(),
|
||||||
monitorScheduler: getMonitorSchedulerSnapshot(),
|
monitorScheduler: getMonitorSchedulerSnapshot(),
|
||||||
sessions,
|
sessions,
|
||||||
hotViews,
|
hotViews,
|
||||||
|
|||||||
Reference in New Issue
Block a user