feat(monitoring): dispatch monitoring tasks to desktop via AMQP outbox

Replace SSE /desktop/events with priority AMQP dispatch for monitoring runs, and add phase1/phase2 infrastructure so desktop workers can lease, resume, report, skip, and cancel monitoring tasks over the existing dispatch WebSocket. - Add monitoring collect outbox worker and phase2 desktop task fields - Add /desktop/monitoring/tasks/{lease,resume,result,skip,cancel} routes - Introduce execution-devtools and network-observer on desktop runtime - Refactor runtime-controller, LoginView, and doubao adapter for the new flow - Add channelName column to tracking views Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-22 00:24:21 +08:00
parent 749b6b99cd
commit 4142c53fa6
57 changed files with 7897 additions and 985 deletions
@@ -7,6 +7,7 @@ import type { DesktopTaskEventMessage, DesktopTaskInfo, JsonValue } from "@geo/s

 export type MonitorSchedulerRouting = "rabbitmq-primary" | "db-recovery";
 type MonitorTaskState = "queued" | "leasing" | "active";
+export type MonitorTaskSource = "desktop_task" | "monitoring_lease";

 export interface MonitorSchedulerTaskRecord {
  taskId: string;
@@ -15,7 +16,11 @@ export interface MonitorSchedulerTaskRecord {
  clientId: string;
  platform: string;
  routing: MonitorSchedulerRouting;
+  source: MonitorTaskSource;
  state: MonitorTaskState;
+  priority: number;
+  lane: string | null;
+  laneWeight: number;
  title: string | null;
  businessDate: string | null;
  questionKey: string | null;
@@ -51,6 +56,7 @@ export interface MonitorSchedulerSnapshot {
 export interface MonitorSchedulerSelection {
  taskId: string;
  routing: MonitorSchedulerRouting;
+  source: MonitorTaskSource;
 }

 interface MonitorSchedulerSelectionOptions {
@@ -68,7 +74,7 @@ interface MonitorTaskMetadata {
  questionText: string | null;
 }

-const schedulerStateVersion = 1;
+const schedulerStateVersion = 5;
 const schedulerQuestionCooldownMs = 45_000;
 const schedulerPlatformCooldownMs = 5_000;
 const schedulerRestartRecoveryDelayMs = 90_000;
@@ -141,7 +147,15 @@ function normalizePersistedState(input: PersistedMonitorSchedulerState | null |
    return next;
  }

-  next.version = typeof input.version === "number" ? input.version : schedulerStateVersion;
+  // The monitor scheduler is only a local optimization layer. When its on-disk
+  // shape changes, start from a clean slate and let the server-side queue/resume
+  // APIs repopulate state. Keeping old records risks reviving already-finished
+  // monitor desktop_tasks and blocking legacy fallback pulls.
+  if (input.version !== schedulerStateVersion) {
+    return next;
+  }
+
+  next.version = schedulerStateVersion;
  next.lastHydratedAt = typeof input.lastHydratedAt === "number" ? input.lastHydratedAt : 0;

  if (input.tasks && typeof input.tasks === "object") {
@@ -162,7 +176,11 @@ function normalizePersistedState(input: PersistedMonitorSchedulerState | null |
        clientId: typeof task.clientId === "string" ? task.clientId : "",
        platform: task.platform,
        routing: task.routing === "db-recovery" ? "db-recovery" : "rabbitmq-primary",
+        source: task.source === "monitoring_lease" ? "monitoring_lease" : "desktop_task",
        state: task.state === "active" || task.state === "leasing" ? task.state : "queued",
+        priority: typeof task.priority === "number" ? task.priority : 100,
+        lane: normalizeOptionalString(task.lane),
+        laneWeight: typeof task.laneWeight === "number" ? task.laneWeight : laneWeightFromLane(task.lane),
        title: typeof task.title === "string" ? task.title : null,
        businessDate: normalizeBusinessDate(task.businessDate),
        questionKey: normalizeOptionalString(task.questionKey),
@@ -234,6 +252,11 @@ export function initMonitorScheduler(): void {
  const now = Date.now();
  const next = clonePersistedState(readPersistedState());
  prunePersistedState(next, now);
+  for (const [taskId, task] of Object.entries(next.tasks)) {
+    if (task.source === "monitoring_lease") {
+      delete next.tasks[taskId];
+    }
+  }
  next.lastHydratedAt = now;
  writePersistedState(next);
 }
@@ -254,7 +277,11 @@ export function enqueueMonitorTaskFromEvent(
      clientId: event.target_client_id,
      platform: event.platform,
      routing,
+      source: "desktop_task",
      state: existing?.state === "active" ? "active" : "queued",
+      priority: normalizePriority(event.priority, existing?.priority),
+      lane: normalizeLane(event.lane, existing?.lane),
+      laneWeight: laneWeightFromLane(event.lane ?? existing?.lane ?? null),
      title: metadata.title ?? existing?.title ?? null,
      businessDate: metadata.businessDate ?? existing?.businessDate ?? null,
      questionKey: metadata.questionKey ?? existing?.questionKey ?? null,
@@ -286,7 +313,11 @@ export function noteMonitorTaskLeased(
      clientId: task.target_client_id,
      platform: task.platform,
      routing,
+      source: "desktop_task",
      state: "active",
+      priority: normalizePriority(payloadNumber(task.payload ?? null, "priority"), existing?.priority),
+      lane: normalizeLane(firstString(task.payload ?? null, ["lane"]), existing?.lane),
+      laneWeight: laneWeightFromLane(firstString(task.payload ?? null, ["lane"]) ?? existing?.lane ?? null),
      title: metadata.title ?? existing?.title ?? null,
      businessDate: metadata.businessDate ?? existing?.businessDate ?? null,
      questionKey: metadata.questionKey ?? existing?.questionKey ?? null,
@@ -302,6 +333,67 @@ export function noteMonitorTaskLeased(
  });
 }

+export function enqueueMonitorLeaseTask(input: {
+  taskId: string;
+  jobId?: string;
+  accountId?: string;
+  clientId: string;
+  platform: string;
+  routing: MonitorSchedulerRouting;
+  priority?: number;
+  lane?: string | null;
+  title?: string | null;
+  businessDate?: string | null;
+  questionKey?: string | null;
+  questionText?: string | null;
+}): void {
+  const now = Date.now();
+
+  mutatePersistedState((draft) => {
+    const existing = draft.tasks[input.taskId];
+    draft.tasks[input.taskId] = {
+      taskId: input.taskId,
+      jobId: input.jobId ?? input.taskId,
+      accountId: input.accountId ?? existing?.accountId ?? "",
+      clientId: input.clientId,
+      platform: input.platform,
+      routing: input.routing,
+      source: "monitoring_lease",
+      state: existing?.state === "active" ? "active" : "queued",
+      priority: normalizePriority(input.priority, existing?.priority),
+      lane: normalizeLane(input.lane, existing?.lane),
+      laneWeight: laneWeightFromLane(input.lane ?? existing?.lane ?? null),
+      title: normalizeOptionalString(input.title) ?? existing?.title ?? null,
+      businessDate: normalizeBusinessDate(input.businessDate) ?? existing?.businessDate ?? null,
+      questionKey: normalizeOptionalString(input.questionKey) ?? existing?.questionKey ?? null,
+      questionText: normalizeOptionalString(input.questionText) ?? existing?.questionText ?? null,
+      updatedAt: now,
+      enqueuedAt: existing?.enqueuedAt ?? now,
+      availableAt: existing?.availableAt ?? now,
+      lastSeenAt: now,
+      lastLeaseAttemptAt: existing?.lastLeaseAttemptAt ?? null,
+      lastStartedAt: existing?.lastStartedAt ?? null,
+      leaseMisses: existing?.leaseMisses ?? 0,
+    };
+  });
+}
+
+export function noteMonitorTaskActivated(taskId: string): void {
+  const now = Date.now();
+
+  mutatePersistedState((draft) => {
+    const existing = draft.tasks[taskId];
+    if (!existing) {
+      return;
+    }
+    existing.state = "active";
+    existing.lastLeaseAttemptAt = now;
+    existing.lastStartedAt = now;
+    existing.lastSeenAt = now;
+    existing.availableAt = now;
+  });
+}
+
 export function noteMonitorTaskLeaseMiss(taskId: string): void {
  const now = Date.now();

@@ -363,6 +455,12 @@ export function selectNextMonitorTask(
  const candidates = Object.values(draft.tasks)
    .filter((task) => task.state === "queued" && task.availableAt <= now)
    .sort((left, right) => {
+      if (left.laneWeight !== right.laneWeight) {
+        return right.laneWeight - left.laneWeight;
+      }
+      if (left.priority !== right.priority) {
+        return right.priority - left.priority;
+      }
      if (left.availableAt !== right.availableAt) {
        return left.availableAt - right.availableAt;
      }
@@ -377,8 +475,10 @@ export function selectNextMonitorTask(
      continue;
    }

+    const bypassCooldowns = candidate.lane === "high";
+
    const platformCooldown = draft.platformCooldowns[candidate.platform] ?? 0;
-    if (platformCooldown > now) {
+    if (!bypassCooldowns && platformCooldown > now) {
      continue;
    }

@@ -391,7 +491,7 @@ export function selectNextMonitorTask(
        continue;
      }
      const questionCooldown = draft.questionCooldowns[candidate.questionKey] ?? 0;
-      if (questionCooldown > now) {
+      if (!bypassCooldowns && questionCooldown > now) {
        continue;
      }
    }
@@ -403,6 +503,7 @@ export function selectNextMonitorTask(
    return {
      taskId: candidate.taskId,
      routing: candidate.routing,
+      source: candidate.source,
    };
  }

@@ -412,7 +513,15 @@ export function selectNextMonitorTask(

 export function getMonitorSchedulerSnapshot(): MonitorSchedulerSnapshot {
  const state = readPersistedState();
-  const tasks = Object.values(state.tasks).sort((left, right) => left.enqueuedAt - right.enqueuedAt);
+  const tasks = Object.values(state.tasks).sort((left, right) => {
+    if (left.laneWeight !== right.laneWeight) {
+      return right.laneWeight - left.laneWeight;
+    }
+    if (left.priority !== right.priority) {
+      return right.priority - left.priority;
+    }
+    return left.enqueuedAt - right.enqueuedAt;
+  });
  return {
    hydratedAt: state.lastHydratedAt || null,
    queueDepth: tasks.length,
@@ -473,9 +582,12 @@ function metadataFromPayload(payload: Record<string, JsonValue> | null): Monitor
 }

 function firstString(
-  payload: Record<string, JsonValue>,
+  payload: Record<string, JsonValue> | null,
  keys: string[],
 ): string | null {
+  if (!payload) {
+    return null;
+  }
  for (const key of keys) {
    const value = payload[key];
    if (typeof value === "string") {
@@ -492,6 +604,26 @@ function firstString(
  return null;
 }

+function payloadNumber(
+  payload: Record<string, JsonValue> | null,
+  key: string,
+): number | null {
+  if (!payload) {
+    return null;
+  }
+  const value = payload[key];
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+  if (typeof value === "string" && value.trim()) {
+    const parsed = Number.parseInt(value.trim(), 10);
+    if (Number.isFinite(parsed)) {
+      return parsed;
+    }
+  }
+  return null;
+}
+
 function currentMonitoringBusinessDate(now: number): string {
  const parts = new Intl.DateTimeFormat("en-US", {
    timeZone: "Asia/Shanghai",
@@ -522,6 +654,32 @@ function normalizeOptionalString(value: unknown): string | null {
  return normalized || null;
 }

+function normalizePriority(value: unknown, fallback = 100): number {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+  return fallback;
+}
+
+function normalizeLane(value: unknown, fallback: string | null = null): string | null {
+  return normalizeOptionalString(value) ?? fallback;
+}
+
+function laneWeightFromLane(lane: unknown): number {
+  switch (normalizeOptionalString(lane)) {
+    case "high":
+      return 70;
+    case "normal_boosted":
+      return 50;
+    case "normal":
+      return 40;
+    case "retry":
+      return 20;
+    default:
+      return 0;
+  }
+}
+
 function digestText(value: string): string {
  return createHash("sha1").update(value).digest("hex");
 }