feat(desktop): add monitor scheduler, Playwright CDP layer, and account health

Move monitor-task scheduling authority onto the client with a durable file-backed queue that survives restart, drops stale cross-day tasks, enforces per-platform serialism, and adapts global concurrency from Electron process metrics. Publish tasks keep their existing FIFO. Add a hidden Playwright CDP manager that attaches to Electron Chromium on account session partitions, lets adapters opt into `executionMode: "playwright"`, and leaves the existing hidden WebContentsView path in place for current adapters. Introduce an account-health subsystem with silent probes, projected health/auth states, and IPC invalidation events so the renderer can show accurate auth/probe status and verification timestamps. Server-side, derive and forward title/business_date/scheduler_group_key/ question_text metadata on desktop task events so the local scheduler can defer same-question fan-out before leasing.
2026-04-20 17:16:15 +08:00
parent 07881a66d0
commit 55e1c2e74b
26 changed files with 3001 additions and 153 deletions
@@ -0,0 +1,638 @@
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+
+import { app } from "electron/main";
+import type { DesktopAccountInfo } from "@geo/shared-types";
+
+import type {
+  AccountAuthReason,
+  AccountAuthState,
+  AccountHealthProfile,
+  AccountHealthSnapshot,
+  AccountProbeState,
+  AuthProbeResult,
+  PlatformFailureClassification,
+} from "./auth-types";
+import { emitRuntimeInvalidated } from "./runtime-events";
+import { getPersistedPartition, getSessionHandle } from "./session-registry";
+import {
+  getPlatformAdapter,
+  type PlatformFailureInput,
+} from "./platform-auth-adapters";
+import type { PublishAccountIdentity } from "./account-binder";
+
+const PROBE_TICK_MS = 10_000;
+const FIRST_PROBE_MIN_MS = 5_000;
+const FIRST_PROBE_MAX_MS = 15_000;
+const REGULAR_PROBE_MIN_MS = 30 * 60_000;
+const REGULAR_PROBE_MAX_MS = 60 * 60_000;
+const CONFIRM_BACKOFF_STEPS_MS = [30_000, 120_000] as const;
+const NETWORK_RETRY_MS = 2 * 60_000;
+const STALE_AFTER_MS = 45 * 60_000;
+const PREFLIGHT_MAX_AGE_MS = 15 * 60_000;
+
+interface PersistedAccountHealthRecord {
+  accountId: string;
+  platform: string;
+  lastVerifiedAt: number | null;
+  lastAuthFailureAt: number | null;
+  profile: AccountHealthProfile | null;
+  sessionFingerprint: string | null;
+}
+
+interface AccountHealthRecord {
+  accountId: string;
+  platform: string;
+  authState: AccountAuthState;
+  probeState: AccountProbeState;
+  authReason: AccountAuthReason;
+  lastVerifiedAt: number | null;
+  lastProbeAt: number | null;
+  nextProbeAt: number | null;
+  lastAuthFailureAt: number | null;
+  profile: AccountHealthProfile | null;
+  sessionFingerprint: string | null;
+  suspectedExpiredAt: number | null;
+  confirmFailureCount: number;
+}
+
+const trackedAccounts = new Map<string, PublishAccountIdentity>();
+const records = new Map<string, AccountHealthRecord>();
+const accountLocks = new Map<string, Promise<void>>();
+let initialized = false;
+let probeTimer: ReturnType<typeof setInterval> | null = null;
+
+function randomInt(min: number, max: number): number {
+  if (max <= min) {
+    return min;
+  }
+  return min + Math.floor(Math.random() * (max - min));
+}
+
+function persistedHealthPath(): string {
+  return join(app.getPath("userData"), "desktop-account-health.json");
+}
+
+function resolvePartition(accountId: string): string {
+  return getSessionHandle(accountId)?.partition ?? getPersistedPartition(accountId) ?? `persist:acc-${accountId}`;
+}
+
+function nextInitialProbeAt(now = Date.now()): number {
+  return now + randomInt(FIRST_PROBE_MIN_MS, FIRST_PROBE_MAX_MS);
+}
+
+function nextRegularProbeAt(now = Date.now()): number {
+  return now + randomInt(REGULAR_PROBE_MIN_MS, REGULAR_PROBE_MAX_MS);
+}
+
+function normalizePersisted(
+  record: PersistedAccountHealthRecord,
+): AccountHealthRecord {
+  return {
+    accountId: record.accountId,
+    platform: record.platform,
+    authState: "unknown",
+    probeState: "idle",
+    authReason: null,
+    lastVerifiedAt: record.lastVerifiedAt ?? null,
+    lastProbeAt: null,
+    nextProbeAt: nextInitialProbeAt(),
+    lastAuthFailureAt: record.lastAuthFailureAt ?? null,
+    profile: record.profile ?? null,
+    sessionFingerprint: record.sessionFingerprint ?? null,
+    suspectedExpiredAt: null,
+    confirmFailureCount: 0,
+  };
+}
+
+function loadPersistedRecords(): void {
+  try {
+    const raw = JSON.parse(readFileSync(persistedHealthPath(), "utf8")) as PersistedAccountHealthRecord[];
+    for (const item of raw) {
+      if (!item?.accountId || !item?.platform) {
+        continue;
+      }
+      records.set(item.accountId, normalizePersisted(item));
+    }
+  } catch {
+    records.clear();
+  }
+}
+
+function persistRecords(): void {
+  const payload: PersistedAccountHealthRecord[] = [...records.values()].map((record) => ({
+    accountId: record.accountId,
+    platform: record.platform,
+    lastVerifiedAt: record.lastVerifiedAt,
+    lastAuthFailureAt: record.lastAuthFailureAt,
+    profile: record.profile,
+    sessionFingerprint: record.sessionFingerprint,
+  }));
+
+  const target = persistedHealthPath();
+  mkdirSync(dirname(target), { recursive: true });
+  writeFileSync(target, JSON.stringify(payload, null, 2), "utf8");
+}
+
+function ensureInitialized(): void {
+  if (initialized) {
+    return;
+  }
+
+  initialized = true;
+  loadPersistedRecords();
+}
+
+function ensureProbeTimer(): void {
+  ensureInitialized();
+  if (probeTimer) {
+    return;
+  }
+
+  probeTimer = setInterval(() => {
+    void runDueProbes();
+  }, PROBE_TICK_MS);
+}
+
+function coerceDerivedAuthState(record: AccountHealthRecord, now = Date.now()): AccountAuthState {
+  if (
+    record.authState === "active"
+    && record.lastVerifiedAt
+    && now - record.lastVerifiedAt >= STALE_AFTER_MS
+  ) {
+    return "expiring_soon";
+  }
+
+  return record.authState;
+}
+
+function toPublicSnapshot(record: AccountHealthRecord): AccountHealthSnapshot {
+  return {
+    accountId: record.accountId,
+    platform: record.platform,
+    partition: resolvePartition(record.accountId),
+    authState: coerceDerivedAuthState(record),
+    probeState: record.probeState,
+    authReason: record.authReason,
+    lastVerifiedAt: record.lastVerifiedAt,
+    lastProbeAt: record.lastProbeAt,
+    nextProbeAt: record.nextProbeAt,
+    lastAuthFailureAt: record.lastAuthFailureAt,
+    profile: record.profile,
+    sessionFingerprint: record.sessionFingerprint,
+  };
+}
+
+function snapshotSignature(record: AccountHealthRecord): string {
+  return JSON.stringify(toPublicSnapshot(record));
+}
+
+function commitRecord(record: AccountHealthRecord, previousSignature: string | null): AccountHealthSnapshot {
+  records.set(record.accountId, record);
+  persistRecords();
+
+  const nextSignature = snapshotSignature(record);
+  if (previousSignature !== nextSignature) {
+    emitRuntimeInvalidated("account-health");
+  }
+
+  return toPublicSnapshot(record);
+}
+
+function ensureRecord(account: PublishAccountIdentity): AccountHealthRecord {
+  ensureInitialized();
+
+  const existing = records.get(account.id);
+  if (existing) {
+    existing.platform = account.platform;
+    return existing;
+  }
+
+  const created: AccountHealthRecord = {
+    accountId: account.id,
+    platform: account.platform,
+    authState: "unknown",
+    probeState: "idle",
+    authReason: null,
+    lastVerifiedAt: null,
+    lastProbeAt: null,
+    nextProbeAt: nextInitialProbeAt(),
+    lastAuthFailureAt: null,
+    profile: null,
+    sessionFingerprint: null,
+    suspectedExpiredAt: null,
+    confirmFailureCount: 0,
+  };
+
+  records.set(account.id, created);
+  persistRecords();
+  return created;
+}
+
+async function withAccountLock<T>(accountId: string, work: () => Promise<T>): Promise<T> {
+  const previous = accountLocks.get(accountId) ?? Promise.resolve();
+  let release!: () => void;
+  const current = new Promise<void>((resolve) => {
+    release = resolve;
+  });
+
+  accountLocks.set(accountId, current);
+  await previous.catch(() => undefined);
+
+  try {
+    return await work();
+  } finally {
+    release();
+    if (accountLocks.get(accountId) === current) {
+      accountLocks.delete(accountId);
+    }
+  }
+}
+
+function confirmBackoffMs(failureCount: number): number {
+  const index = Math.min(Math.max(failureCount - 1, 0), CONFIRM_BACKOFF_STEPS_MS.length - 1);
+  return CONFIRM_BACKOFF_STEPS_MS[index] ?? CONFIRM_BACKOFF_STEPS_MS[CONFIRM_BACKOFF_STEPS_MS.length - 1];
+}
+
+function applyActiveResult(
+  record: AccountHealthRecord,
+  result: AuthProbeResult,
+  now: number,
+): AccountHealthSnapshot {
+  const previousSignature = snapshotSignature(record);
+
+  record.authState = "active";
+  record.probeState = "idle";
+  record.authReason = result.reason ?? "probe_success";
+  record.lastVerifiedAt = now;
+  record.lastProbeAt = now;
+  record.nextProbeAt = nextRegularProbeAt(now);
+  record.profile = result.profile ?? record.profile;
+  record.sessionFingerprint = result.sessionFingerprint ?? record.sessionFingerprint;
+  record.suspectedExpiredAt = null;
+  record.confirmFailureCount = 0;
+
+  return commitRecord(record, previousSignature);
+}
+
+function applyExpiredResult(
+  record: AccountHealthRecord,
+  result: AuthProbeResult,
+  now: number,
+): AccountHealthSnapshot {
+  const previousSignature = snapshotSignature(record);
+
+  record.authState = "expired";
+  record.probeState = "idle";
+  record.authReason = result.reason ?? "login_redirect";
+  record.lastProbeAt = now;
+  record.nextProbeAt = nextRegularProbeAt(now);
+  record.lastAuthFailureAt = now;
+  record.suspectedExpiredAt = null;
+  record.confirmFailureCount = 0;
+
+  return commitRecord(record, previousSignature);
+}
+
+function applyChallengeResult(
+  record: AccountHealthRecord,
+  result: AuthProbeResult,
+  now: number,
+): AccountHealthSnapshot {
+  const previousSignature = snapshotSignature(record);
+
+  record.authState = "challenge_required";
+  record.probeState = "idle";
+  record.authReason = result.reason ?? "captcha_gate";
+  record.lastProbeAt = now;
+  record.nextProbeAt = nextRegularProbeAt(now);
+  record.lastAuthFailureAt = now;
+  record.suspectedExpiredAt = null;
+  record.confirmFailureCount = 0;
+
+  return commitRecord(record, previousSignature);
+}
+
+function applyNetworkResult(
+  record: AccountHealthRecord,
+  result: AuthProbeResult,
+  now: number,
+  trigger: "scheduled" | "manual" | "preflight" | "confirm",
+): AccountHealthSnapshot {
+  const previousSignature = snapshotSignature(record);
+  const hadSuspicion = Boolean(record.suspectedExpiredAt);
+
+  record.probeState = "network_error";
+  record.authReason = result.reason ?? "network_error";
+  record.lastProbeAt = now;
+  if (hadSuspicion && trigger === "confirm") {
+    record.confirmFailureCount += 1;
+    record.authState = record.authState === "unknown" ? "unknown" : "expiring_soon";
+    record.nextProbeAt = now + confirmBackoffMs(record.confirmFailureCount);
+  } else {
+    record.nextProbeAt = now + NETWORK_RETRY_MS;
+    if (record.authState === "active") {
+      record.authState = "expiring_soon";
+    }
+  }
+
+  return commitRecord(record, previousSignature);
+}
+
+async function performProbeLocked(
+  account: PublishAccountIdentity,
+  options: {
+    trigger: "scheduled" | "manual" | "preflight" | "confirm";
+    allowSilentRefresh: boolean;
+  },
+): Promise<AccountHealthSnapshot> {
+  const record = ensureRecord(account);
+  const adapter = getPlatformAdapter(account.platform);
+  const previousSignature = snapshotSignature(record);
+
+  record.probeState = "probing";
+  commitRecord(record, previousSignature);
+
+  const partition = resolvePartition(account.id);
+
+  if (options.allowSilentRefresh) {
+    const refreshed = await adapter.silentRefresh(account, partition).catch(() => false);
+    if (!refreshed) {
+      const currentSignature = snapshotSignature(record);
+      record.authReason = "silent_refresh_failed";
+      commitRecord(record, currentSignature);
+    }
+  }
+
+  const result = await adapter.probe(account, partition).catch<AuthProbeResult>(() => ({
+    verdict: "network_error",
+    reason: "network_error",
+    profile: null,
+    sessionFingerprint: null,
+  }));
+
+  const now = Date.now();
+  switch (result.verdict) {
+    case "active":
+      return applyActiveResult(record, result, now);
+    case "expired":
+      return applyExpiredResult(record, result, now);
+    case "challenge_required":
+      return applyChallengeResult(record, result, now);
+    default:
+      return applyNetworkResult(record, result, now, options.trigger);
+  }
+}
+
+async function runDueProbes(): Promise<void> {
+  const now = Date.now();
+  const dueAccounts = [...trackedAccounts.values()].filter((account) => {
+    const record = records.get(account.id);
+    return Boolean(record?.nextProbeAt && record.nextProbeAt <= now);
+  });
+
+  for (const account of dueAccounts) {
+    const record = records.get(account.id);
+    if (!record) {
+      continue;
+    }
+
+    const trigger = record.suspectedExpiredAt ? "confirm" : "scheduled";
+    void withAccountLock(account.id, async () => {
+      await performProbeLocked(account, {
+        trigger,
+        allowSilentRefresh: false,
+      });
+    });
+  }
+}
+
+function parseVerifiedAt(value: string | null | undefined): number | null {
+  if (!value) {
+    return null;
+  }
+
+  const timestamp = Date.parse(value);
+  return Number.isNaN(timestamp) ? null : timestamp;
+}
+
+function fallbackAuthState(
+  health: DesktopAccountInfo["health"] | null | undefined,
+): AccountAuthState {
+  switch (health) {
+    case "live":
+      return "unknown";
+    case "captcha":
+      return "challenge_required";
+    case "expired":
+      return "expired";
+    default:
+      return "unknown";
+  }
+}
+
+export function initAccountHealth(): void {
+  ensureInitialized();
+  ensureProbeTimer();
+}
+
+export function syncTrackedAccounts(accounts: PublishAccountIdentity[]): void {
+  syncTrackedAccountsInternal(accounts, true);
+}
+
+function syncTrackedAccountsInternal(
+  accounts: PublishAccountIdentity[],
+  pruneMissing: boolean,
+): void {
+  initAccountHealth();
+  let removed = false;
+
+  if (pruneMissing) {
+    const nextIds = new Set(accounts.map((account) => account.id));
+    for (const [accountId] of trackedAccounts.entries()) {
+      if (!nextIds.has(accountId)) {
+        trackedAccounts.delete(accountId);
+        records.delete(accountId);
+        removed = true;
+      }
+    }
+  }
+
+  for (const account of accounts) {
+    trackedAccounts.set(account.id, account);
+    ensureRecord(account);
+  }
+
+  persistRecords();
+  if (removed) {
+    emitRuntimeInvalidated("account-health");
+  }
+}
+
+export function forgetTrackedAccountHealth(
+  accountId: string,
+): void {
+  trackedAccounts.delete(accountId);
+  if (records.delete(accountId)) {
+    persistRecords();
+    emitRuntimeInvalidated("account-health");
+  }
+}
+
+export function getAccountHealthSnapshot(accountId: string): AccountHealthSnapshot | null {
+  ensureInitialized();
+  const record = records.get(accountId);
+  return record ? toPublicSnapshot(record) : null;
+}
+
+export function getProjectedAccountHealth(input: {
+  accountId: string;
+  platform: string;
+  health?: DesktopAccountInfo["health"];
+  verifiedAt?: string | null;
+}) {
+  const snapshot = getAccountHealthSnapshot(input.accountId);
+  if (!snapshot) {
+    return {
+      health: input.health ?? "risk",
+      authState: fallbackAuthState(input.health),
+      probeState: "idle" as AccountProbeState,
+      authReason: null as AccountAuthReason,
+      lastVerifiedAt: parseVerifiedAt(input.verifiedAt),
+      lastProbeAt: null,
+      nextProbeAt: null,
+      lastAuthFailureAt: null,
+      profile: null,
+      sessionFingerprint: null,
+      partition: resolvePartition(input.accountId),
+    };
+  }
+
+  const health =
+    snapshot.authState === "active"
+      ? ("live" as const)
+      : snapshot.authState === "challenge_required"
+        ? ("captcha" as const)
+        : snapshot.authState === "expired" || snapshot.authState === "revoked"
+          ? ("expired" as const)
+          : ("risk" as const);
+
+  return {
+    ...snapshot,
+    health,
+  };
+}
+
+export async function probeTrackedAccount(
+  account: PublishAccountIdentity,
+  options: {
+    trigger?: "scheduled" | "manual" | "preflight" | "confirm";
+    allowSilentRefresh?: boolean;
+  } = {},
+): Promise<AccountHealthSnapshot> {
+  syncTrackedAccountsInternal([account], false);
+  return await withAccountLock(account.id, async () => {
+    return await performProbeLocked(account, {
+      trigger: options.trigger ?? "manual",
+      allowSilentRefresh: Boolean(options.allowSilentRefresh),
+    });
+  });
+}
+
+export async function ensureAccountReady(
+  account: PublishAccountIdentity,
+): Promise<AccountHealthSnapshot> {
+  syncTrackedAccountsInternal([account], false);
+  const snapshot = getAccountHealthSnapshot(account.id);
+  const isFresh =
+    snapshot?.lastVerifiedAt
+    ? Date.now() - snapshot.lastVerifiedAt < PREFLIGHT_MAX_AGE_MS
+    : false;
+
+  if (
+    snapshot
+    && snapshot.authState === "active"
+    && snapshot.probeState !== "network_error"
+    && isFresh
+  ) {
+    return snapshot;
+  }
+
+  if (snapshot && ["expired", "challenge_required", "revoked"].includes(snapshot.authState)) {
+    return snapshot;
+  }
+
+  return await withAccountLock(account.id, async () => {
+    const lockedSnapshot = getAccountHealthSnapshot(account.id);
+    const lockedFresh =
+      lockedSnapshot?.lastVerifiedAt
+      ? Date.now() - lockedSnapshot.lastVerifiedAt < PREFLIGHT_MAX_AGE_MS
+      : false;
+
+    if (
+      lockedSnapshot
+      && lockedSnapshot.authState === "active"
+      && lockedSnapshot.probeState !== "network_error"
+      && lockedFresh
+    ) {
+      return lockedSnapshot;
+    }
+
+    return await performProbeLocked(account, {
+      trigger: "preflight",
+      allowSilentRefresh: true,
+    });
+  });
+}
+
+export async function reportAccountFailure(
+  account: PublishAccountIdentity,
+  input: PlatformFailureInput,
+): Promise<AccountHealthSnapshot | null> {
+  syncTrackedAccountsInternal([account], false);
+  const adapter = getPlatformAdapter(account.platform);
+  const classification = adapter.classifyFailure(input);
+
+  if (classification === "ok") {
+    return getAccountHealthSnapshot(account.id);
+  }
+
+  return await withAccountLock(account.id, async () => {
+    const record = ensureRecord(account);
+    const previousSignature = snapshotSignature(record);
+    const now = Date.now();
+
+    if (classification === "challenge") {
+      record.authState = "challenge_required";
+      record.probeState = "idle";
+      record.authReason = "captcha_gate";
+      record.lastAuthFailureAt = now;
+      record.lastProbeAt = now;
+      record.nextProbeAt = nextRegularProbeAt(now);
+      record.suspectedExpiredAt = null;
+      record.confirmFailureCount = 0;
+      return commitRecord(record, previousSignature);
+    }
+
+    if (classification === "network") {
+      record.probeState = "network_error";
+      record.authReason = "network_error";
+      record.lastProbeAt = now;
+      record.nextProbeAt = now + NETWORK_RETRY_MS;
+      return commitRecord(record, previousSignature);
+    }
+
+    record.lastAuthFailureAt = now;
+    record.suspectedExpiredAt = record.suspectedExpiredAt ?? now;
+    if (record.authState === "active") {
+      record.authState = "expiring_soon";
+    }
+    record.nextProbeAt = now;
+    commitRecord(record, previousSignature);
+
+    return await performProbeLocked(account, {
+      trigger: "confirm",
+      allowSilentRefresh: false,
+    });
+  });
+}