Compare commits
3 Commits
eb4d870081
...
4dece0f26a
| Author | SHA1 | Date | |
|---|---|---|---|
| 4dece0f26a | |||
| 15a820712e | |||
| 283e8f6b46 |
@@ -179,6 +179,7 @@ import {
|
||||
markStoredMembershipBlocked,
|
||||
readStoredSession,
|
||||
setStoredTokens,
|
||||
type SessionSnapshot,
|
||||
} from './session'
|
||||
|
||||
const rawBaseURL = import.meta.env.VITE_API_BASE_URL ?? ''
|
||||
@@ -265,6 +266,64 @@ export function shouldRefreshAccessToken(
|
||||
return !expiresAt || expiresAt * 1000 - Date.now() <= minRemainingMs
|
||||
}
|
||||
|
||||
const authRefreshLockName = 'geo.admin-web.auth-refresh'
|
||||
|
||||
async function withCrossTabRefreshLock<T>(task: () => Promise<T>): Promise<T> {
|
||||
const locks = typeof navigator !== 'undefined' ? navigator.locks : undefined
|
||||
if (!locks?.request) {
|
||||
return task()
|
||||
}
|
||||
return (await locks.request(authRefreshLockName, task)) as T
|
||||
}
|
||||
|
||||
function storedTokensFromSnapshot(snapshot: SessionSnapshot): AuthTokens | null {
|
||||
if (!snapshot.accessToken || !snapshot.refreshToken) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
accessToken: snapshot.accessToken,
|
||||
refreshToken: snapshot.refreshToken,
|
||||
expiresAt: snapshot.accessTokenExpiresAt ?? undefined,
|
||||
}
|
||||
}
|
||||
|
||||
async function performStoredAuthRefresh(initialRefreshToken: string): Promise<AuthTokens> {
|
||||
// Another tab may have rotated the token while we waited for the lock:
|
||||
// reuse its result instead of racing the one-time-use refresh token.
|
||||
const current = readStoredSession()
|
||||
const currentTokens = storedTokensFromSnapshot(current)
|
||||
if (
|
||||
currentTokens &&
|
||||
currentTokens.refreshToken !== initialRefreshToken &&
|
||||
!shouldRefreshAccessToken(current.accessTokenExpiresAt)
|
||||
) {
|
||||
return currentTokens
|
||||
}
|
||||
|
||||
const requestedRefreshToken = currentTokens?.refreshToken ?? initialRefreshToken
|
||||
|
||||
try {
|
||||
const data = await publicClient.post<RefreshResponse, { refresh_token: string }>(
|
||||
'/api/auth/refresh',
|
||||
{ refresh_token: requestedRefreshToken },
|
||||
)
|
||||
const tokens = toAuthTokens(data)
|
||||
setStoredTokens(tokens)
|
||||
return tokens
|
||||
} catch (error) {
|
||||
if (isAuthSessionExpiredError(error)) {
|
||||
const snapshot = readStoredSession()
|
||||
const fallbackTokens = storedTokensFromSnapshot(snapshot)
|
||||
if (fallbackTokens && fallbackTokens.refreshToken !== requestedRefreshToken) {
|
||||
return fallbackTokens
|
||||
}
|
||||
|
||||
expireStoredSession()
|
||||
}
|
||||
throw markAuthErrorHandled(error)
|
||||
}
|
||||
}
|
||||
|
||||
export async function refreshStoredAuthSession(
|
||||
refreshToken = getStoredRefreshToken(),
|
||||
): Promise<AuthTokens> {
|
||||
@@ -273,40 +332,10 @@ export async function refreshStoredAuthSession(
|
||||
throw authRequiredError()
|
||||
}
|
||||
|
||||
const requestedRefreshToken = refreshToken
|
||||
|
||||
if (!storedRefreshPromise) {
|
||||
storedRefreshPromise = publicClient
|
||||
.post<RefreshResponse, { refresh_token: string }>('/api/auth/refresh', {
|
||||
refresh_token: requestedRefreshToken,
|
||||
})
|
||||
.then((data) => {
|
||||
const tokens = toAuthTokens(data)
|
||||
setStoredTokens(tokens)
|
||||
return tokens
|
||||
})
|
||||
.catch((error) => {
|
||||
if (isAuthSessionExpiredError(error)) {
|
||||
const current = readStoredSession()
|
||||
if (
|
||||
current.accessToken &&
|
||||
current.refreshToken &&
|
||||
current.refreshToken !== requestedRefreshToken
|
||||
) {
|
||||
const tokens = {
|
||||
accessToken: current.accessToken,
|
||||
refreshToken: current.refreshToken,
|
||||
expiresAt: current.accessTokenExpiresAt ?? undefined,
|
||||
}
|
||||
setStoredTokens(tokens)
|
||||
return tokens
|
||||
}
|
||||
|
||||
expireStoredSession()
|
||||
}
|
||||
throw markAuthErrorHandled(error)
|
||||
})
|
||||
.finally(() => {
|
||||
storedRefreshPromise = withCrossTabRefreshLock(() =>
|
||||
performStoredAuthRefresh(refreshToken),
|
||||
).finally(() => {
|
||||
storedRefreshPromise = null
|
||||
})
|
||||
}
|
||||
|
||||
@@ -5,10 +5,10 @@
|
||||
| 项目 | 内容 |
|
||||
| --- | --- |
|
||||
| 文档名称 | GEO SaaS MCP Agent Publishing PRD V1 |
|
||||
| 文档版本 | V1.1 |
|
||||
| 文档版本 | V1.2 |
|
||||
| 文档状态 | 待评审 |
|
||||
| 创建日期 | 2026-06-09 |
|
||||
| 最近修订 | 2026-06-10 |
|
||||
| 最近修订 | 2026-07-02 |
|
||||
| 适用阶段 | MCP 智能体接入 / 企业内容生成 / 多平台发布 / AI 可见度监测 / 桌面端采集 |
|
||||
| 关联文档 | `docs/geo-platform-prd-v1.md` |
|
||||
| 关联文档 | `docs/qdrant-rag-architecture-v1.md` |
|
||||
@@ -17,7 +17,9 @@
|
||||
| 关联文档 | `docs/ai-brand-monitoring-tech-design-v5.md` |
|
||||
| 关联文档 | `docs/worklogs/2026-04-20-legacy-plugin-design-status.md` |
|
||||
|
||||
### 1.0 V1.0 → V1.1 修订记录
|
||||
### 1.0 修订记录
|
||||
|
||||
#### V1.0 → V1.1
|
||||
|
||||
| 修订点 | V1.0 | V1.1 |
|
||||
| --- | --- | --- |
|
||||
@@ -29,6 +31,25 @@
|
||||
| 错误模型 | 含搜狐专用错误码 | 平台无关错误码,新增配额、限流、平台能力不支持、站点发布、监测采集等错误码 |
|
||||
| 工具与提示词命名 | 含 sohu 专用命名 | 全部平台参数化命名 |
|
||||
|
||||
#### V1.1 → V1.2(评审修订,对齐代码实态与协议缺口)
|
||||
|
||||
| 修订点 | V1.1 | V1.2 |
|
||||
| --- | --- | --- |
|
||||
| 批次部分成功口径 | 引用 `partial_failed` | 对齐代码实际枚举 `partial_success` |
|
||||
| 监测调度通道 | HIGH/NORMAL/LOW | 对齐实际通道 `high` / `normal`(重试走独立 `retry` 通道),无 `low` |
|
||||
| 定时发布语义 | 笼统"复用 schedules" | 拆分为一次性定时发布(桌面复用发布任务 `scheduled_at`;企业站点为新增服务端延迟执行)与循环定时生成计划(ScheduleTask,daily/weekly)两条链路,新增 `schedule_plan_create` / `schedule_cancel` |
|
||||
| 长任务模型 | generate 同步描述与"一律可轮询"矛盾 | 统一异步口径,新增 `run_status` 统一轮询工具,逐工具标注同步/异步 |
|
||||
| 用户选择恢复 | 状态机有挂起态但无恢复入口 | 新增 `agent_run_continue`(P0),定义挂起 TTL 与过期语义;resume 自 P1 提前 |
|
||||
| Confirmation token | 绑定单一 version + hash | 升级为逐目标 (platform, account/site, version, content hash) 元组,覆盖多平台变体 |
|
||||
| 工具命名 | `geo.` 点号前缀 | 去点号改下划线命名(主流 LLM API 工具名约束 `[a-zA-Z0-9_-]` 不含点号);本地桥工具改 `local_` 前缀 |
|
||||
| 幂等性 | 仅发布 version+target dedup | 创建型工具统一支持 `idempotency_key`,站点发布强制 |
|
||||
| 信任边界与数据披露 | 未写明 | §17 新增 agent 代述确认信任边界、可选带外强确认、按 scope 数据外发披露 |
|
||||
| 限流与配额 | 共享 token bucket 一句话 | §16 增加保活保底/collect-now 上限仲裁规则;§11 定义中途配额耗尽行为 |
|
||||
| 部署与会话 | 未涉及多副本 | §2 补 `Mcp-Session-Id` 会话亲和、elicitation 超时回退 |
|
||||
| 错误码 | — | 新增 `article_not_found` / `article_version_stale` / `site_not_found` / `agent_run_not_found` / `agent_run_expired` / `idempotency_conflict`;message 统一中文 |
|
||||
| 交付计划 | 10 项无排序 | 补第 11 项租户后台 Agent 集成管理页;标注内部交付顺序(P0 范围不变) |
|
||||
| 指标 | 仅可靠性指标 | §22 补采用度指标与网关非功能指标 |
|
||||
|
||||
### 1.1 文档目标
|
||||
|
||||
本文档定义 GEO SaaS 面向外部智能体(小龙虾、Hermes 等)的 MCP 能力规划,使智能体可以通过标准工具调用完成以下闭环:
|
||||
@@ -100,7 +121,7 @@ MCP 暴露的能力域与现有系统能力的对应关系(这是"全量暴露
|
||||
| 图片素材 | 图片库、文件夹、引用关系 | Tools(选图排序)+ Resources(读);上传 P1 | P0 |
|
||||
| 文章与生成 | 提示词规则生成、模板生成、仿写、重新生成、版本 | Tools + Resources | P0 |
|
||||
| 多平台发布 | 13 个桌面端平台适配器、PublishBatch、发布记录、重试 | Tools + Resources | P0 |
|
||||
| 定时发布 | schedules + 调度器 | Tools + Resources | P0 |
|
||||
| 定时发布 | 一次性定时(桌面复用发布任务 `scheduled_at`;站点延迟执行为新增)+ 循环定时生成计划(ScheduleTask,daily/weekly) | Tools + Resources | P0 |
|
||||
| 企业站点发布 | WordPress / PBootCMS 服务端适配器 | Tools + Resources | P0 |
|
||||
| 账号健康 | 桌面端保活探测、健康上报 | Tools(session check)+ Resources | P0 |
|
||||
| AI 品牌监测 | 6 个 AI 平台适配器、监测任务、看板、引用源 | Tools(读 + collect-now)+ Resources | P0 |
|
||||
@@ -121,7 +142,7 @@ MCP 暴露的能力域与现有系统能力的对应关系(这是"全量暴露
|
||||
4. 信息足够时调用知识库检索与文章生成,生成企业软文草稿;需要时按目标平台风格改写出平台变体。
|
||||
5. 自动选择或建议封面图、正文图,并校验目标平台的图片要求(由平台能力目录给出)。
|
||||
6. 发布前对每个目标账号调用本地 session 检查,确认账号处于可发布状态。
|
||||
7. 在用户确认或授权策略允许时创建发布任务:桌面平台走 PublishBatch + 桌面任务;企业站点走服务端 CMS 适配器;定时需求走 schedules。
|
||||
7. 在用户确认或授权策略允许时创建发布任务:桌面平台走 PublishBatch + 桌面任务;企业站点走服务端 CMS 适配器;定时需求走一次性定时或循环生成计划两条链路(见 §13)。
|
||||
8. 桌面客户端复用本地平台 session 执行图片上传、正文提交和结果回写;多个目标账号并行分发。
|
||||
9. 智能体返回每个目标的发布状态、外部文章 ID、管理链接、公开链接、失败原因或下一步建议;批量发布支持部分成功。
|
||||
10. 需要监测时,智能体读取监测看板(提及率、排名、引用源),或触发一次 collect-now 即时采集,由客户端绑定的 AI 账号执行。
|
||||
@@ -291,8 +312,9 @@ The product will support two MCP entry points:
|
||||
部署形态:
|
||||
|
||||
- Gateway 以独立进程 `server/cmd/mcp-gateway` 交付,与 tenant-api 同 Go module、复用 `internal/tenant` 应用服务,独立部署独立扩缩容(沿用 ops-api 的同模块独立部署先例)。
|
||||
- 传输协议采用 MCP Streamable HTTP;长任务(生成、发布、采集)一律返回可轮询的 run/task ID,不依赖长连接存活。
|
||||
- 用户选择决策优先使用 MCP elicitation 能力(客户端支持时),否则回退为本文档定义的机器可读 `needs_user_choice` 结构化输出。两条路径必须语义等价。
|
||||
- 传输协议采用 MCP Streamable HTTP。多副本扩缩容下 `Mcp-Session-Id` 采用网关层会话亲和(按 session ID 哈希路由);业务状态全部落库、MCP 会话本身不承载业务数据,副本失联时客户端按协议重建会话即可,不丢任务。
|
||||
- 长任务(生成、编排、发布执行、采集)一律异步:返回可轮询的 run/batch ID(经 `run_status` / `publish_status` 跟进),不依赖长连接存活;同步/异步口径逐工具标注,见 §6。
|
||||
- 用户选择决策优先使用 MCP elicitation 能力(客户端支持时),否则回退为本文档定义的机器可读 `needs_user_choice` 结构化输出。elicitation 等待设超时(默认 120 秒),超时同样回退为挂起 run。两条路径必须语义等价:同一 decisions schema,统一经 `agent_run_continue` 恢复(见 §7)。
|
||||
|
||||
P0 chooses SaaS MCP Gateway first because the existing SaaS already owns tenant auth, knowledge retrieval, article generation, compliance, publish job creation, audit logs and desktop dispatch. The desktop bridge is a controlled extension for "agent directly operates local client" scenarios.
|
||||
|
||||
@@ -328,10 +350,14 @@ Recommended scopes:
|
||||
|
||||
Agent credentials must be revocable, rate limited and visible in platform audit logs. SaaS user web JWT must not be reused as a long-lived agent credential. P0 采用平台签发的 agent token(哈希入库、可吊销、带 scope 与过期时间);OAuth 2.1 动态注册为 P1。
|
||||
|
||||
Agent token 的签发、scope 勾选、限流配置与吊销通过租户后台 **Agent 集成管理页**自助完成(P0 交付项 11,见 Further Notes);授权页按 scope 展示数据外发披露(见 §17)。每个 agent 有默认限流基线:工具调用 QPS、生成类每日次数、collect-now 每小时次数三档(默认值由运营配置、租户可下调),超限返回 `rate_limited`。
|
||||
|
||||
### 4. MCP Resources
|
||||
|
||||
Resources expose read-only context. They must never expose platform cookies, local browser storage, raw desktop vault contents, model API keys, CMS credentials or internal secrets.
|
||||
|
||||
Resources 定位原则:部分 MCP 宿主对 Resources 支持较弱,P0 的关键上下文与状态读取必须同时有工具可达(`workspace_context`、`publish_status`、`run_status` 与监测读工具),Resources 作为增强镜像供支持良好的宿主使用。`geo://images` 中的预览 URL 为短时效签名 URL。
|
||||
|
||||
Initial resource set:
|
||||
|
||||
| Resource URI | Description |
|
||||
@@ -374,61 +400,69 @@ Prompt output must treat knowledge snippets as reference material, not as higher
|
||||
|
||||
### 6. MCP Tools
|
||||
|
||||
命名规则:工具名与提示词名仅使用小写字母、数字与下划线(`[a-z0-9_]`),不使用点号——主流 LLM API 的工具名约束为 `[a-zA-Z0-9_-]`,点号会在部分 MCP 宿主把工具映射进模型工具列表时被改写或拒绝;MCP server 本身即命名空间,无需 `geo.` 前缀。每个工具的 description 必须写明"何时调用",宿主模型据此选择工具。
|
||||
|
||||
同步/异步口径:读类与轻量校验工具同步返回;生成、编排、发布执行、采集一律异步(返回可轮询 ID,经 `run_status` / `publish_status` 跟进),不受宿主工具调用超时(普遍 60 秒量级)限制。创建型工具统一接受可选 `idempotency_key`:同 key 重放返回首次创建的资源、不重复创建;同 key 不同请求体返回 `idempotency_conflict`。
|
||||
|
||||
P0 tools(按能力域分组):
|
||||
|
||||
#### 上下文与资产
|
||||
|
||||
| Tool | Purpose | Notes |
|
||||
| --- | --- | --- |
|
||||
| `geo.workspace_context` | Return workspace, default brand, knowledge groups, image summary, all bound media/AI accounts with health, enterprise sites, desktop status, quota snapshot | Used at the start of most agent workflows |
|
||||
| `geo.knowledge_search` | Search selected knowledge groups using a user brief or generated query | Wraps existing knowledge RAG, returns structured snippets with precise facts |
|
||||
| `geo.image_select` | Select or rank image assets for cover and inline usage, validated against each target platform's image requirements | P0 ranks existing assets; generated images are P1 |
|
||||
| `workspace_context` | Return workspace, default brand, knowledge groups, image summary, all bound media/AI accounts with health, enterprise sites, desktop status, quota snapshot | Used at the start of most agent workflows |
|
||||
| `knowledge_search` | Search selected knowledge groups using a user brief or generated query | Wraps existing knowledge RAG, returns structured snippets with precise facts |
|
||||
| `image_select` | Select or rank image assets for cover and inline usage, validated against each target platform's image requirements | P0 ranks existing assets; generated images are P1 |
|
||||
|
||||
#### 生成
|
||||
|
||||
| Tool | Purpose | Notes |
|
||||
| --- | --- | --- |
|
||||
| `geo.article_plan` | Produce title, angle and outline options without generating full article | 复用模板 title/outline 任务链路 |
|
||||
| `geo.article_generate` | Generate and save an article draft using brand, knowledge, image and user constraints | 收敛到现有生成入口(prompt rule / template / imitation),返回 article ID、version ID、预览与缺失决策 |
|
||||
| `geo.article_rewrite` | Rewrite a saved draft for a target platform or target reader | 多平台分发的平台变体生成;P0 |
|
||||
| `article_plan` | Produce title, angle and outline options without generating full article | 异步:复用模板 title/outline 任务链路,返回 run ID 经 `run_status` 轮询 |
|
||||
| `article_generate` | Generate and save an article draft using brand, knowledge, image and user constraints | 异步:创建生成 run 返回 run ID(草稿就绪目标 90 秒中位数,超出宿主同步等待预算,必须异步);完成后经 `run_status` 提供 article ID、version ID、预览与缺失决策。收敛到现有生成入口(prompt rule / template / imitation)。接受 `idempotency_key` |
|
||||
| `article_rewrite` | Rewrite a saved draft for a target platform or target reader | 异步。多平台分发的平台变体生成,产生新文章版本;P0。接受 `idempotency_key` |
|
||||
|
||||
#### 发布与调度
|
||||
|
||||
| Tool | Purpose | Notes |
|
||||
| --- | --- | --- |
|
||||
| `geo.desktop_session_check` | Check selected local account health through desktop runtime(媒体账号与 AI 账号通用) | 返回 §14 的标准状态;客户端离线时只返回 last-known + stale 标记 |
|
||||
| `geo.article_publish` | Create a publish batch for one article version to 1..N target accounts across 1..N platforms | Requires confirmation token unless auto-publish is authorized; maps to existing PublishBatch |
|
||||
| `geo.site_publish` | Publish an article to an enterprise site (WordPress / PBootCMS) with category mapping | Server-side adapter; no desktop dependency |
|
||||
| `geo.schedule_publish` | Create a scheduled publish (desktop platforms or sites) | Reuses existing schedules + dispatch worker |
|
||||
| `geo.publish_status` | Read publish batch / per-target record / schedule / site publish status | Used for long-running follow-up |
|
||||
| `geo.publish_retry` | Retry failed targets of a batch through the existing retry flow | Only for failed targets; never retries `unknown` submissions |
|
||||
| `desktop_session_check` | Check selected local account health through desktop runtime(媒体账号与 AI 账号通用) | 同步。返回 §14 的标准状态;客户端离线时只返回 last-known + stale 标记 |
|
||||
| `article_publish` | Create a publish batch to 1..N target accounts across 1..N platforms, per-target article version binding | 异步:创建 batch 返回 batch ID。Requires confirmation token unless auto-publish is authorized; maps to existing PublishBatch。多平台变体时逐目标绑定版本与哈希(见 §10/§11);version+target dedup 天然幂等 |
|
||||
| `site_publish` | Publish an article to an enterprise site (WordPress / PBootCMS) with category mapping | 异步。Server-side adapter; no desktop dependency。**必须携带 `idempotency_key`**:CMS API 非幂等,超时重试会产生第二篇文章 |
|
||||
| `schedule_publish` | Create a **one-shot** scheduled publish of an existing article version to desktop/site targets at a future time | 同步创建。桌面目标复用发布任务 `scheduled_at` 延迟派发(既有链路);企业站点一次性定时为 P0 新增的服务端延迟执行链路(见 §13/§18)。接受 `idempotency_key` |
|
||||
| `schedule_plan_create` | Create a **recurring** generation + publish plan (daily / weekly) bound to brand and prompt rules | 同步创建。映射既有 ScheduleTask(定时生成 + 可选自动发布);与一次性定时是两条不同链路,语义不可混用 |
|
||||
| `schedule_cancel` | Cancel a pending one-shot scheduled publish or a recurring plan | 同步。落实 `schedule:write` scope 承诺的 cancel 能力 |
|
||||
| `publish_status` | Read publish batch / per-target record / schedule / site publish status; supports query by batch ID, article ID or time window | 同步读。发布域跟进入口;生成/编排/采集 run 的轮询走 `run_status` |
|
||||
| `publish_retry` | Retry failed targets of a batch through the existing retry flow | 同步受理。Only for failed targets; never retries `unknown` submissions |
|
||||
|
||||
#### 监测
|
||||
|
||||
| Tool | Purpose | Notes |
|
||||
| --- | --- | --- |
|
||||
| `geo.monitoring_overview` | Read composite dashboard: mention rate, mention position, first-recommended, coverage, by AI platform and time range | Wraps existing dashboard composite API |
|
||||
| `geo.monitoring_question_detail` | Read per-question monitoring detail including answers and matched brand terms | |
|
||||
| `geo.monitoring_citations` | Read citation facts and correlate cited URLs with own publish records | |
|
||||
| `geo.monitoring_collect_now` | Trigger an immediate collection run for brand questions on selected AI platforms | Creates standard `monitor` desktop tasks on HIGH priority lane; respects account rate budget |
|
||||
| `monitoring_overview` | Read composite dashboard: mention rate, mention position, first-recommended, coverage, by AI platform and time range | Wraps existing dashboard composite API |
|
||||
| `monitoring_question_detail` | Read per-question monitoring detail including answers and matched brand terms | |
|
||||
| `monitoring_citations` | Read citation facts and correlate cited URLs with own publish records | |
|
||||
| `monitoring_collect_now` | Trigger an immediate collection run for brand questions on selected AI platforms | 异步:创建标准 `monitor` 桌面任务(`high` 调度通道),返回 collect run ID 经 `run_status` 轮询;respects account rate budget(仲裁规则见 §16) |
|
||||
|
||||
#### 合规与编排
|
||||
|
||||
| Tool | Purpose | Notes |
|
||||
| --- | --- | --- |
|
||||
| `geo.compliance_precheck` | Run compliance check on a draft before publish | 复用现有 compliance check |
|
||||
| `geo.compliance_ack` | Acknowledge a `needs_ack` compliance result, bound to article version + content hash + policy fingerprint | 必须由用户显式确认后调用 |
|
||||
| `geo.agent_publish_campaign` | High-level orchestration tool for one natural-language request | Calls planning, generation, rewrite, session check, publish/schedule internally |
|
||||
| `compliance_precheck` | Run compliance check on a draft before publish | 同步(LLM 判定超出同步预算时转异步返回 run ID)。复用现有 compliance check |
|
||||
| `compliance_ack` | Acknowledge a `needs_ack` compliance result, bound to article version + content hash + policy fingerprint | 同步。必须由用户显式确认后调用(信任边界见 §17) |
|
||||
| `agent_publish_campaign` | High-level orchestration tool for one natural-language request | 异步:创建 agent run 返回 run ID。Calls planning, generation, rewrite, session check, publish/schedule internally |
|
||||
| `run_status` | Poll any agent run or async operation: campaign run, generation run, collect-now run | 同步读。统一长任务轮询入口,返回 §7 状态 + 时间线摘要 + 挂起时的待决策项;发布批次细节仍走 `publish_status` |
|
||||
| `agent_run_continue` | Resume a run suspended in `needs_user_choice` / `awaiting_publish_confirmation` with the user's decisions | 同步受理。携带 run ID + decisions[](品牌 / 平台账号集 / 标题 / 图片 / 发布时间 / confirmation token / 合规 ack 等);elicitation 与结构化输出两条路径的用户决策统一汇入此入口。权限沿用被恢复 run 所需 scopes |
|
||||
|
||||
P1 tools:
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `geo.desktop_data_collect` | Create and dispatch deep data collection tasks(content list 对账、平台指标、账号资料刷新)using local session |
|
||||
| `geo.knowledge_item_add` | Add text / URL knowledge items into a group |
|
||||
| `geo.image_import` | Import an image asset by URL |
|
||||
| `geo.agent_run_cancel` | Cancel a waiting or queued agent run |
|
||||
| `geo.local_bridge_pair` | Pair a local desktop MCP bridge with the SaaS workspace |
|
||||
| `desktop_data_collect` | Create and dispatch deep data collection tasks(content list 对账、平台指标、账号资料刷新)using local session |
|
||||
| `knowledge_item_add` | Add text / URL knowledge items into a group |
|
||||
| `image_import` | Import an image asset by URL |
|
||||
| `agent_run_cancel` | Cancel a waiting or queued agent run |
|
||||
| `local_bridge_pair` | Pair a local desktop MCP bridge with the SaaS workspace |
|
||||
|
||||
### 7. Orchestration State Machine
|
||||
|
||||
@@ -447,11 +481,17 @@ Every high-level agent workflow creates an agent run with the following states:
|
||||
| `publishing` | Desktop client leased and executing, or CMS adapter submitting |
|
||||
| `collecting` | Monitoring collection or post-publish collection in progress |
|
||||
| `succeeded` | Workflow completed; all targets succeeded |
|
||||
| `partially_succeeded` | Batch finished with some targets failed(对齐 PublishBatch `partial_failed`) |
|
||||
| `partially_succeeded` | Batch finished with some targets failed(对齐 PublishBatch 实际枚举 `partial_success`) |
|
||||
| `failed` | Workflow failed with normalized error |
|
||||
| `canceled` | User or agent canceled before completion |
|
||||
|
||||
The run timeline must record tool name, input summary, output summary, error code, article ID, publish batch ID, schedule ID, desktop client ID, target platform keys and target account IDs where applicable.
|
||||
The run timeline must record tool name, input summary, output summary, error code, article ID, publish batch ID, schedule ID, desktop client ID, target platform keys and target account IDs where applicable. Input/output summary 遵守 §21 审计脱敏规范。
|
||||
|
||||
挂起与恢复:
|
||||
|
||||
- `needs_user_choice` 与 `awaiting_publish_confirmation` 是**挂起态**:run 与已产出资产(草稿、版本、候选项、confirmation 请求)在服务端持久化,通过 `agent_run_continue` 携带用户决策恢复执行。没有恢复入口的状态机是空转的,此工具为 P0 必备。
|
||||
- 挂起有 TTL(默认 24 小时,工作区可配);过期转 `canceled`,释放预留配额,已产出草稿保留。过期后调用 continue 返回 `agent_run_expired`。
|
||||
- 客户端支持 MCP elicitation 时,同一决策集可在工具调用期间即时收集(超时回退为挂起,见 §2);两条路径的 decisions schema 完全一致。
|
||||
|
||||
### 8. Guided Mode
|
||||
|
||||
@@ -491,21 +531,19 @@ Automatic mode may proceed without additional user choices only when all conditi
|
||||
- Generated article satisfies minimum quality checks.
|
||||
- Quota check passes(AI 积分足够完成本次工作流).
|
||||
|
||||
Even in automatic mode, the publish tool must be idempotent and tied to article ID, article version ID, the full target set and content hash.
|
||||
Even in automatic mode, the publish tool must be idempotent and tied to article ID, the full target set and per-target article version + content hash(多平台变体时逐目标绑定,见 §10).
|
||||
|
||||
### 10. Confirmation Token
|
||||
|
||||
Publishing is irreversible. When confirmation is required, `geo.article_generate` or `geo.agent_publish_campaign` returns a confirmation request with:
|
||||
Publishing is irreversible. When confirmation is required, `article_generate` or `agent_publish_campaign` returns a confirmation request with:
|
||||
|
||||
- article ID
|
||||
- article version ID
|
||||
- content hash
|
||||
- target set: list of (platform key, account ID) and/or (site ID, category), and/or schedule time
|
||||
- publish type (draft / publish / scheduled)
|
||||
- preview summary
|
||||
- 逐目标条目(per-target entries):每项绑定 `(platform_key, account_id 或 site_id+category, article_version_id, content_hash, publish_type, scheduled_at?)`——多平台变体(§11)下不同目标对应不同文章版本与内容哈希,token 必须逐目标锁定各自版本;单一 version + hash 无法覆盖变体场景
|
||||
- 全量目标集摘要哈希(防止确认后增删目标)
|
||||
- preview summary(按目标或共享)
|
||||
- expiration time
|
||||
|
||||
`geo.article_publish` / `geo.site_publish` / `geo.schedule_publish` must reject confirmation tokens that are expired, consumed, bound to different content, bound to a different target set or bound to a different publish type. 部分成功后的重试不得复用原 token 之外新增目标。
|
||||
`article_publish` / `site_publish` / `schedule_publish` must reject confirmation tokens that are expired, consumed, bound to a different target set or bound to a different publish type;任一目标条目的 article version 或 content hash 与实际待发内容不符时整体拒绝(`publish_confirmation_invalid`;确认后草稿被编辑的场景返回 `article_version_stale`,引导重新确认)。部分成功后的重试不得复用原 token 之外新增目标。
|
||||
|
||||
### 11. Knowledge and Content Generation
|
||||
|
||||
@@ -513,7 +551,7 @@ The MCP generation path must reuse the existing article generation and knowledge
|
||||
|
||||
Decisions:
|
||||
|
||||
- `geo.article_generate` 收敛到现有三类生成入口:提示词规则生成、模板生成、仿写生成;由编排器根据用户意图选择入口,不新建第四条生成管线。
|
||||
- `article_generate` 收敛到现有三类生成入口:提示词规则生成、模板生成、仿写生成;由编排器根据用户意图选择入口,不新建第四条生成管线。
|
||||
- 生成所用 LLM 模型由服务端生成栈配置决定(与现状一致),MCP 不暴露模型选择;用量日志记录实际模型。客户端绑定的 AI 平台账号仅用于监测采集,与服务端生成模型解耦。
|
||||
- Knowledge retrieval uses current tenant and selected knowledge groups.
|
||||
- Generated articles are saved as normal SaaS articles and versions.
|
||||
@@ -522,8 +560,9 @@ Decisions:
|
||||
- The agent may generate an article plan before full generation(复用模板 title/outline 任务)。
|
||||
- Soft article defaults include title, summary, section headings, body, CTA and optional image placement suggestions.
|
||||
- The final draft must be valid rich-text or Markdown convertible to each target platform's article format(HTML 转换由各桌面适配器完成,与现状一致)。
|
||||
- 多平台分发时,平台变体通过 `geo.article_rewrite` 产生新文章版本,发布时逐目标绑定对应版本。
|
||||
- 多平台分发时,平台变体通过 `article_rewrite` 产生新文章版本,发布时逐目标绑定对应版本。
|
||||
- 生成、改写、合规 LLM 判定全部走现有 quota reservation + AI point usage 流程,失败退款逻辑不变。
|
||||
- campaign 等多步工作流按整体预估一次性预留配额(生成 + 平台变体改写 + 合规 LLM 判定);若预估偏差导致中途耗尽,保留已产出草稿与版本,run 以 `quota_exhausted` 失败,配额补充后可基于已有资产续接,不重复扣费、不静默丢弃。
|
||||
|
||||
### 12. Image Handling
|
||||
|
||||
@@ -560,7 +599,7 @@ P1 can add AI image generation, image import and image editing, but that is outs
|
||||
8. Desktop platform adapter uploads images per platform requirements.
|
||||
9. Desktop platform adapter submits article as draft or publish(平台支持度由能力目录声明).
|
||||
10. Desktop client reports progress and final result per task(succeeded / failed / unknown).
|
||||
11. SaaS syncs publish records and batch status(含 partial_failed).
|
||||
11. SaaS syncs publish records and batch status(含 `partial_success`).
|
||||
12. MCP tool returns per-target results or a pollable batch ID.
|
||||
|
||||
#### 企业站点(wordpress / pbootcms)
|
||||
@@ -574,7 +613,8 @@ P1 can add AI image generation, image import and image editing, but that is outs
|
||||
|
||||
- The SaaS service must not call media platform APIs directly with server-side cookies(企业站点 CMS API 除外,凭证走既有密文存储)。
|
||||
- MCP Gateway 不得包含任何平台专用逻辑;平台差异 100% 留在桌面适配器、CMS 适配器与平台目录。
|
||||
- 定时发布走既有 schedules + 调度器;到点后进入与上面相同的发布链路。
|
||||
- 定时发布是两条链路,语义不可混用:**一次性定时发布**(已有文章版本 + 目标集 + 未来时间)——桌面目标复用发布任务 `scheduled_at` 延迟派发(既有能力);企业站点一次性定时为 P0 新增的服务端延迟执行。**循环定时生成计划**(daily / weekly)——映射既有 ScheduleTask(定时生成 + 可选自动发布,绑定提示词规则)。到点后均进入与上面相同的发布链路。
|
||||
- 创建型发布入口(publish batch / site publish / schedule)支持 `idempotency_key` 去重;站点发布强制携带(CMS API 非幂等,重复提交产生重复文章)。
|
||||
|
||||
### 14. Local Session Check and Reuse
|
||||
|
||||
@@ -615,19 +655,21 @@ Initial local bridge tools:
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `desktop.session_check` | Check local account health(媒体与 AI 账号) |
|
||||
| `desktop.publish_article` | Execute a SaaS-authorized publish task or local approved draft |
|
||||
| `desktop.collect_data` | Collect platform-side data using local session |
|
||||
| `desktop.open_account_console` | Open the platform console for human login or verification |
|
||||
| `local_session_check` | Check local account health(媒体与 AI 账号) |
|
||||
| `local_publish_article` | Execute a SaaS-authorized publish task or local approved draft |
|
||||
| `local_collect_data` | Collect platform-side data using local session |
|
||||
| `local_open_account_console` | Open the platform console for human login or verification |
|
||||
|
||||
本地桥工具以 `local_` 前缀命名,与 SaaS 网关工具区分,避免宿主同时连接两个 MCP server 时产生工具名歧义。
|
||||
|
||||
### 16. Monitoring and Data Collection
|
||||
|
||||
#### AI 品牌监测(P0,读 + collect-now)
|
||||
|
||||
监测复用既有体系:`monitor` 类型桌面任务、6 个 AI 平台适配器、HIGH/NORMAL/LOW 优先级通道、监测结果与引用源数据模型。MCP 暴露:
|
||||
监测复用既有体系:`monitor` 类型桌面任务、6 个 AI 平台适配器、`high` / `normal` 双调度通道(重试任务走独立 `retry` 通道,无 `low` 通道)、监测结果与引用源数据模型。MCP 暴露:
|
||||
|
||||
- 读看板:composite dashboard、question detail、citation summary(纯读,零新增采集链路)。
|
||||
- collect-now:等价于现有"立即采集"入口,创建 HIGH lane 的 monitor 任务,由客户端绑定 AI 账号执行。必须遵守账号级速率预算(与保活、常规监测共享 token bucket),MCP 不得绕过。
|
||||
- collect-now:等价于现有"立即采集"入口,创建 `high` 通道的 monitor 任务,由客户端绑定 AI 账号执行,返回 collect run ID 供 `run_status` 轮询。速率预算仲裁:与保活探测、常规监测共享账号级 token bucket,其中**保活探测享有保底预算、不可被 collect-now 挤占**(保活是账号健康的生命线);collect-now 另设独立上限(每账号每小时 + 每工作区每日双阈值,运营可配),超限返回 `rate_limited` 并附下次可用时间。MCP 不得绕过。
|
||||
- 采集结果字段沿用现有 payload:answer、citations、search_results、brand_mentioned、brand_mention_position、first_recommended、sentiment_label、matched_brand_terms。
|
||||
- AI 账号不可用(expired / challenge_required / 客户端离线)时返回对应标准错误,引导用户修复,不静默跳过。
|
||||
|
||||
@@ -652,7 +694,7 @@ Agent publishing must follow existing compliance policy:
|
||||
- MCP generation does not bypass article compliance checks.
|
||||
- MCP publish (desktop platforms, sites, schedules) does not bypass publish gate checks.
|
||||
- Compliance `block` stops publish.
|
||||
- Compliance `needs_ack` requires an acknowledgement flow(`geo.compliance_ack`,需用户显式确认).
|
||||
- Compliance `needs_ack` requires an acknowledgement flow(`compliance_ack`,需用户显式确认).
|
||||
- The acknowledgement must bind to article version, content hash and policy fingerprint.
|
||||
- MCP runs must store actor information and agent client information.
|
||||
|
||||
@@ -665,6 +707,12 @@ Additional safety rules:
|
||||
- Do not expose hidden system prompts, credentials, cookies, CMS credentials or platform request headers to the agent.
|
||||
- collect-now 频率受账号级与 agent 级双重限流,防止智能体高频采集触发平台风控。
|
||||
|
||||
信任边界与数据外发:
|
||||
|
||||
- **Agent 代述确认是显式接受的残余风险**:confirmation token 与 compliance ack 的"用户已确认"由 agent 转述,平台无法直接验证终端用户的真实操作;一个失控的持权 agent 可以自问自答。租户为 agent 授予 `article:publish` 即表示信任其如实转述用户决策;此项写入威胁模型,以 scope 最小化、完整审计时间线与即时吊销兜底。
|
||||
- 工作区策略可选**带外强确认**(默认关闭):开启后正式发布(非草稿)须经 Web 后台或桌面客户端的推送确认,confirmation token 仅在带外确认完成后生效;草稿与定时草稿豁免。供高敏租户选择。
|
||||
- **数据外发披露**:Agent 授权页必须按 scope 展示该 agent 可读取的数据类别(知识库片段、品牌资料、监测数据、账号与站点元数据),并明示知识库内容将流向 agent 厂商服务器;`geo://images` 预览 URL 为短时效签名 URL。
|
||||
|
||||
### 18. Data Model Additions
|
||||
|
||||
Reuse existing domain tables for articles, article versions, knowledge, platform accounts, desktop clients, desktop tasks, publish batches, publish records, schedules, enterprise site connections, monitoring tasks/runs and citation facts.
|
||||
@@ -677,10 +725,13 @@ Add MCP-specific records:
|
||||
| Agent token | Stores token hash, expiry and binding |
|
||||
| Agent run | Stores one high-level orchestration workflow |
|
||||
| Agent tool call | Stores individual MCP tool calls, status, input summary and output summary |
|
||||
| Agent confirmation | Stores publish confirmation tokens bound to article content and full target set |
|
||||
| Agent confirmation | Stores publish confirmation tokens bound to per-target (platform, account/site, article version, content hash) tuples and the full target set |
|
||||
| Agent run event | Stores timeline events for streaming and support diagnosis |
|
||||
| Agent idempotency record | Stores idempotency keys of create-type tool calls with the created resource ID, for replay-safe retries |
|
||||
|
||||
平台目录补充(非新表,扩展 `media_platforms` 元数据):为每个平台登记 MCP 所需能力字段——是否支持草稿、封面是否必需、图片数量约束、是否支持定时、发布通道(desktop/server)、绑定方式(cookie/oauth/api)、适配器就绪度。
|
||||
平台目录补充(非新表,扩展 `media_platforms` 元数据):为每个平台登记 MCP 所需能力字段——是否支持草稿、封面是否必需、图片数量约束、是否支持定时、发布通道(desktop/server)、绑定方式(cookie/oauth/api)、适配器就绪度。注意:现有 `media_platforms` 仅有展示元数据,能力字段为实打实的新增工作(13 平台 + 2 CMS 的数据梳理量需计入排期)。
|
||||
|
||||
企业站点一次性定时发布:扩展站点发布记录,增加 `scheduled_at` 与待执行状态,由服务端延迟执行(P0 新增能力,见 §13;桌面侧无需新增,复用发布任务 `scheduled_at`)。
|
||||
|
||||
No MCP table should store local platform cookies, desktop vault contents or CMS credentials.
|
||||
|
||||
@@ -696,7 +747,7 @@ Service boundaries:
|
||||
| Article Generation Service | Generate and persist drafts using existing generation stack (prompt rule / template / imitation) |
|
||||
| Knowledge Service | Resolve RAG context and render traceable knowledge snippets |
|
||||
| Publish Job Service | Compliance, dedup, publish batch creation, task dispatch, retry |
|
||||
| Schedule Service | Scheduled publish creation and dispatch(复用现有 scheduler) |
|
||||
| Schedule Service | 一次性定时发布(桌面复用发布任务 `scheduled_at`;企业站点为新增服务端延迟执行)与循环定时生成计划(复用既有 ScheduleTask 调度器) |
|
||||
| Enterprise Site Publisher | CMS adapter publish (WordPress / PBootCMS) |
|
||||
| Monitoring Service | Dashboard reads, collect-now task creation, citation correlation |
|
||||
| Desktop Runtime | Local session check, task lease, adapter execution, progress and result callback |
|
||||
@@ -716,6 +767,8 @@ MCP tool errors should be normalized for agents(平台无关;`platform` 作
|
||||
| `quota_exhausted` | AI points or generation quota insufficient |
|
||||
| `workspace_context_missing` | No tenant or workspace context |
|
||||
| `brand_not_found` | Requested brand is not available |
|
||||
| `article_not_found` | Article or article version not found in this workspace |
|
||||
| `article_version_stale` | Article was edited after confirmation; content hash no longer matches the confirmed version |
|
||||
| `platform_not_supported` | Platform key not enabled in catalog or no adapter readiness |
|
||||
| `platform_capability_unsupported` | Requested capability (e.g. draft, schedule, text-only) not supported by target platform |
|
||||
| `account_not_found` | Target account not bound in this workspace |
|
||||
@@ -734,16 +787,20 @@ MCP tool errors should be normalized for agents(平台无关;`platform` 作
|
||||
| `publish_duplicate` | Same article version already submitted to same target account |
|
||||
| `platform_publish_failed` | Desktop adapter submission failed(携带 platform key 与平台原始原因) |
|
||||
| `publish_result_unknown` | Submission started but final state unknown; manual reconciliation required |
|
||||
| `site_not_found` | Enterprise site connection not found in this workspace |
|
||||
| `site_unreachable` | Enterprise site ping failed |
|
||||
| `site_publish_failed` | CMS adapter submission failed |
|
||||
| `schedule_invalid` | Scheduled time invalid or scheduler rejected |
|
||||
| `agent_run_not_found` | Agent run not found or not visible to this agent |
|
||||
| `agent_run_expired` | Suspended run expired (TTL) before user decisions arrived |
|
||||
| `idempotency_conflict` | Same idempotency key reused with a different request payload |
|
||||
| `monitoring_no_account` | No healthy AI account for requested platform |
|
||||
| `monitoring_collect_failed` | Collection run failed |
|
||||
| `data_collection_failed` | Deep data collection failed(P1) |
|
||||
|
||||
Every error should include:
|
||||
|
||||
- human-readable message
|
||||
- human-readable message(中文文案,面向最终用户可直接展示;程序分支只依赖 code,不解析 message)
|
||||
- retryable boolean
|
||||
- suggested next action
|
||||
- related resource IDs and platform key where safe
|
||||
@@ -766,9 +823,12 @@ Required logs and metrics:
|
||||
- Auto-publish usage by workspace and agent.
|
||||
- AI point consumption by agent and usage type.
|
||||
- Data collection success and stale rate(P1).
|
||||
- Gateway 非功能观测:并发 MCP 会话数、工具调用 P95/P99 延迟、elicitation 使用率与回退率、挂起 run 恢复率与过期率、幂等重放命中数。
|
||||
|
||||
Support must be able to inspect a single agent run timeline without reading raw private article content unless the support role is authorized.
|
||||
|
||||
审计脱敏规范:agent tool call 的 input/output summary 不得包含知识库片段原文或文章正文全文,仅记录资源 ID、内容哈希、长度与白名单字段(标题、平台 key、账号 ID、错误码);正文可读性由支持角色授权单独控制(上一条)。
|
||||
|
||||
### 22. Product Metrics
|
||||
|
||||
P0 success metrics:
|
||||
@@ -784,6 +844,17 @@ P0 success metrics:
|
||||
- MCP tool calls with complete audit trail = 100%.
|
||||
- Cloud-stored platform cookie count = 0(媒体平台与 AI 平台均为 0).
|
||||
|
||||
P0 adoption metrics(采用度,回答"有没有人用"):
|
||||
|
||||
- 接入至少 1 个 agent 并完成首次 MCP 发布的工作区数(周观测)。
|
||||
- 周活跃 agent run 数;MCP 渠道发布量占总发布量比例。
|
||||
- needs_user_choice 平均决策轮数 <= 2(引导效率);挂起 run 恢复率 >= 70%。
|
||||
|
||||
Gateway non-functional targets:
|
||||
|
||||
- Gateway 可用性 >= 99.9%(月度)。
|
||||
- 工具调用 P95 延迟 <= 800ms(不含异步任务执行时间)。
|
||||
|
||||
P1 success metrics:
|
||||
|
||||
- Local bridge paired successfully on first attempt >= 90%.
|
||||
@@ -810,7 +881,7 @@ Use the following highest-level seams:
|
||||
| Enterprise site boundary | Site ping, category mapping, publish, credential redaction |
|
||||
| Desktop runtime boundary | Session check, task lease, progress reporting, success/failure/unknown callback |
|
||||
| Platform adapter conformance | 同一套契约测试参数化跑全部 13 个适配器:login detection, image upload, submit, error normalization, progress stages |
|
||||
| Monitoring boundary | Dashboard reads, collect-now task creation on HIGH lane, AI account health gating, citation correlation |
|
||||
| Monitoring boundary | Dashboard reads, collect-now task creation on the `high` dispatch lane, AI account health gating, citation correlation |
|
||||
| Audit boundary | Every MCP run and tool call produces a traceable audit event |
|
||||
|
||||
### 2. MCP Contract Tests
|
||||
@@ -825,6 +896,9 @@ Add contract tests with a fake MCP client:
|
||||
- Returns confirmation-required responses for publish when policy requires.
|
||||
- Returns pollable IDs for long-running operations.
|
||||
- Returns `quota_exhausted` / `rate_limited` 标准错误。
|
||||
- Same `idempotency_key` replay returns the original resource without duplicate creation; same key with a different payload returns `idempotency_conflict`.
|
||||
- Elicitation-capable 与不支持 elicitation 的客户端获得语义等价的决策流(同一 decisions schema;elicitation 超时回退为挂起 run,经 `agent_run_continue` 恢复)。
|
||||
- 工具名与提示词名全部符合 `[a-z0-9_]` 命名约束(宿主兼容性检查)。
|
||||
|
||||
### 3. Orchestration Tests
|
||||
|
||||
@@ -839,10 +913,13 @@ Use service-level tests with fake dependencies:
|
||||
- No image: target platform requires cover → returns `image_missing` choices; target platform allows text-only → can continue.
|
||||
- Expired account stops that target before batch creation; other targets proceed with user confirmation.
|
||||
- Offline desktop client stops desktop targets; enterprise site targets still proceed.
|
||||
- "明早 9 点发" creates a schedule instead of immediate batch.
|
||||
- "明早 9 点发" creates a one-shot scheduled publish instead of an immediate batch(桌面目标绑定 `scheduled_at`;企业站点目标走服务端延迟执行)。
|
||||
- "每周一早上自动写一篇发布" creates a recurring ScheduleTask plan(daily/weekly);一次性定时与循环生成计划不得混淆。
|
||||
- Compliance `block` stops publish; `needs_ack` returns acknowledgement requirement; ack 后可继续。
|
||||
- Quota exhausted before generation returns `quota_exhausted` without creating generation task.
|
||||
- Monitoring: "豆包上我们品牌表现如何" reads dashboard without creating tasks; "现在采集一次" creates HIGH lane monitor tasks only for healthy AI accounts.
|
||||
- Campaign 中途配额耗尽:保留已产出草稿与版本,run 以 `quota_exhausted` 结束;补充配额后可基于已有资产续接且不重复扣费。
|
||||
- `needs_user_choice` 挂起后经 `agent_run_continue` 携带 decisions 恢复执行;挂起过期转 `canceled` 并释放预留配额,此后 continue 返回 `agent_run_expired`。
|
||||
- Monitoring: "豆包上我们品牌表现如何" reads dashboard without creating tasks; "现在采集一次" creates `high`-lane monitor tasks only for healthy AI accounts.
|
||||
|
||||
### 4. Article Generation Tests
|
||||
|
||||
@@ -865,9 +942,11 @@ Cover:
|
||||
- Non-active account cannot be included as a publish target.
|
||||
- Dedup prevents same article version + target account duplicate publish.
|
||||
- Existing batch is returned instead of creating a duplicate.
|
||||
- Publish record syncs from desktop task completion; batch reflects `partial_failed`.
|
||||
- Unknown publish state is retained after possible submit-start failure; `geo.publish_retry` refuses `unknown` targets.
|
||||
- Publish record syncs from desktop task completion; batch reflects `partial_success`.
|
||||
- Unknown publish state is retained after possible submit-start failure; `publish_retry` refuses `unknown` targets.
|
||||
- Confirmation token bound to target set rejects any target-set mutation.
|
||||
- 逐目标版本绑定:token 内任一目标的 article version 或 content hash 与实际待发内容不符时整体拒绝;确认后草稿被编辑的场景返回 `article_version_stale`。
|
||||
- Site publish idempotency:same `idempotency_key` 重放不产生第二篇 CMS 文章。
|
||||
|
||||
### 6. Desktop and Adapter Tests
|
||||
|
||||
@@ -887,7 +966,7 @@ Manual smoke tests with real test accounts remain required before release for ea
|
||||
|
||||
- Dashboard reads respect tenant/workspace isolation and time-range filters.
|
||||
- collect-now creates `monitor` tasks only for selected platforms with healthy accounts; unhealthy ones return `monitoring_no_account`.
|
||||
- collect-now respects account-level rate budget; exceeding returns `rate_limited`.
|
||||
- collect-now respects account-level rate budget; exceeding returns `rate_limited`;且不得挤占保活探测的保底预算(预算仲裁规则见 §16)。
|
||||
- Collection results flow into existing monitoring projections; citation facts correlate publish records.
|
||||
- 客户端离线时 collect-now 返回 `desktop_client_offline` 并保留任务可选项(排队 vs 取消)。
|
||||
- P1: deep collection task is read-only and retryable; expired session returns session error, not collection success; local bridge collection creates SaaS audit events.
|
||||
@@ -937,6 +1016,9 @@ The following are out of scope for P0:
|
||||
8. Enterprise site publish(WordPress / PBootCMS)。
|
||||
9. Monitoring overview / question detail / citations / collect-now。
|
||||
10. Ops visibility for MCP run timelines, per-platform publish health and failures.
|
||||
11. 租户后台 Agent 集成管理页:创建/吊销 agent token、scope 勾选、限流配置、按 agent 的用量与积分归因、审计入口、按 scope 的数据外发披露文案。缺少此页面,功能无法自助开通(对应租户管理 user stories 全部 9 条)。
|
||||
|
||||
内部交付顺序(P0 范围与全平台全能力口径不变):第 1-6 项构成可内测的最小发布闭环(网关 + 能力目录 + 上下文 + 生成 + 确认/合规 + 会话检查,含 `run_status` / `agent_run_continue`);第 7-8 项接入批量发布与站点发布;第 9 项监测;第 10-11 项与第 7-9 项并行。每段设可验收里程碑,网关压测目标见 §22 非功能指标。
|
||||
|
||||
### P1 Delivery Plan
|
||||
|
||||
@@ -944,7 +1026,7 @@ The following are out of scope for P0:
|
||||
2. Deep data collection tasks(publish_status 回查、content_list 对账、article_metrics、account_profile)。
|
||||
3. Knowledge item add / image import write tools.
|
||||
4. Image ranking improvements and optional image generation.
|
||||
5. Agent run cancel and resume; OAuth 2.1 agent onboarding.
|
||||
5. Agent run cancel(resume 已作为 `agent_run_continue` 提前至 P0); OAuth 2.1 agent onboarding.
|
||||
6. DedeCMS / PHPCMS site adapters via catalog.
|
||||
|
||||
### Default Assumptions
|
||||
@@ -967,14 +1049,14 @@ The following are out of scope for P0:
|
||||
|
||||
Agent expected behavior:
|
||||
|
||||
1. Calls `geo.workspace_context`;发现默认品牌、知识分组、图片、3 个目标平台各自的健康账号。
|
||||
2. Calls `geo.article_plan`(引导模式)→ 用户选定标题与角度。
|
||||
3. Calls `geo.knowledge_search` + `geo.article_generate`;按平台目录校验百家号封面要求,调用 `geo.image_select`。
|
||||
4. Calls `geo.article_rewrite` 生成公众号风格变体(如需要)。
|
||||
5. Calls `geo.desktop_session_check` 覆盖 3 个账号;全部 `active`。
|
||||
1. Calls `workspace_context`;发现默认品牌、知识分组、图片、3 个目标平台各自的健康账号。
|
||||
2. Calls `article_plan`(引导模式)→ 用户选定标题与角度。
|
||||
3. Calls `knowledge_search` + `article_generate`;按平台目录校验百家号封面要求,调用 `image_select`。
|
||||
4. Calls `article_rewrite` 生成公众号风格变体(如需要)。
|
||||
5. Calls `desktop_session_check` 覆盖 3 个账号;全部 `active`。
|
||||
6. 返回预览与完整目标清单:"发布到搜狐号账号 A、百家号账号 B(正式发布),公众号账号 C(草稿)?"
|
||||
7. After user confirms, calls `geo.article_publish`(一个 batch,3 个目标)。
|
||||
8. Polls `geo.publish_status`;百家号失败时报告 `partially_succeeded`,给出 `geo.publish_retry` 选项。
|
||||
7. After user confirms, calls `article_publish`(一个 batch,3 个目标)。
|
||||
8. Polls `publish_status`;百家号失败时报告 `partially_succeeded`,给出 `publish_retry` 选项。
|
||||
9. Returns per-target管理链接与公开链接。
|
||||
|
||||
#### AI 可见度监测
|
||||
@@ -985,7 +1067,7 @@ Agent expected behavior:
|
||||
|
||||
Agent expected behavior:
|
||||
|
||||
1. Calls `geo.monitoring_overview`(platforms=doubao,deepseek, range=7d)→ 返回提及率、平均提及位次、首推率。
|
||||
2. Calls `geo.monitoring_citations` → 引用源 Top 列表,标记命中自有发布记录的 URL。
|
||||
3. Calls `geo.monitoring_collect_now`(HIGH lane)→ 豆包账号 `active` 创建任务;DeepSeek 账号 `challenge_required` 返回修复引导。
|
||||
4. Polls `geo.publish_status`(run 维度)→ 汇总新一轮采集结果并对比上周。
|
||||
1. Calls `monitoring_overview`(platforms=doubao,deepseek, range=7d)→ 返回提及率、平均提及位次、首推率。
|
||||
2. Calls `monitoring_citations` → 引用源 Top 列表,标记命中自有发布记录的 URL。
|
||||
3. Calls `monitoring_collect_now`(`high` 通道)→ 豆包账号 `active` 创建任务并返回 collect run ID;DeepSeek 账号 `challenge_required` 返回修复引导。
|
||||
4. Polls `run_status`(collect run)→ 汇总新一轮采集结果并对比上周。
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
DROP VIEW IF EXISTS geo_monitoring_archive.daily_model_site_citation_rank;
|
||||
|
||||
CREATE VIEW geo_monitoring_archive.daily_model_site_citation_rank AS
|
||||
SELECT
|
||||
business_date,
|
||||
model,
|
||||
ai_platform_id,
|
||||
site,
|
||||
hosts,
|
||||
citation_count,
|
||||
cited_run_count,
|
||||
unique_url_count,
|
||||
brand_count,
|
||||
rank() OVER (
|
||||
PARTITION BY business_date, model
|
||||
ORDER BY citation_count DESC, cited_run_count DESC, unique_url_count DESC, site ASC
|
||||
) AS site_rank
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
cf.business_date AS business_date,
|
||||
ifNull(nullIf(r.provider_model, ''), cf.ai_platform_id) AS model,
|
||||
cf.ai_platform_id AS ai_platform_id,
|
||||
cutToFirstSignificantSubdomain(cf.host) AS site,
|
||||
groupUniqArray(cf.host) AS hosts,
|
||||
count() AS citation_count,
|
||||
uniqExact(cf.run_id) AS cited_run_count,
|
||||
uniqExact(cf.normalized_url) AS unique_url_count,
|
||||
uniqExact(cf.brand_id) AS brand_count
|
||||
FROM geo_monitoring_archive.monitoring_citation_facts AS cf
|
||||
LEFT JOIN geo_monitoring_archive.question_monitor_runs AS r
|
||||
ON cf.tenant_id = r.tenant_id
|
||||
AND cf.run_id = r.id
|
||||
GROUP BY
|
||||
cf.business_date,
|
||||
model,
|
||||
cf.ai_platform_id,
|
||||
site
|
||||
);
|
||||
@@ -0,0 +1,6 @@
|
||||
TZ=Asia/Shanghai
|
||||
SHELL=/bin/bash
|
||||
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
|
||||
# Archive yesterday's PostgreSQL monitoring data into ClickHouse once per day.
|
||||
30 1 * * * root /vol1/1000/application/ClickHouse/scripts/pg_to_clickhouse_archive.sh previous-day >> /vol1/1000/application/ClickHouse/logs/pg_to_clickhouse_archive.cron.log 2>&1
|
||||
@@ -0,0 +1,609 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
trap 'echo "Interrupted"; exit 130' INT TERM
|
||||
|
||||
BASE_DIR="${BASE_DIR:-/vol1/1000/application/ClickHouse}"
|
||||
ENV_FILE="${ENV_FILE:-$BASE_DIR/.env}"
|
||||
PG_PASSWORD_FILE="${PG_PASSWORD_FILE:-$BASE_DIR/pg/geo.password}"
|
||||
STATE_DIR="${STATE_DIR:-$BASE_DIR/state}"
|
||||
LOG_DIR="${LOG_DIR:-$BASE_DIR/logs}"
|
||||
|
||||
CLICKHOUSE_CONTAINER="${CLICKHOUSE_CONTAINER:-geo-rankly-clickhouse}"
|
||||
ARCHIVE_DB="${ARCHIVE_DB:-geo_monitoring_archive}"
|
||||
|
||||
PG_HOST="${PG_HOST:-10.77.0.1}"
|
||||
PG_PORT="${PG_PORT:-15432}"
|
||||
PG_DB="${PG_DB:-geo_monitoring}"
|
||||
PG_USER="${PG_USER:-geo}"
|
||||
TIMEZONE="${TIMEZONE:-Asia/Shanghai}"
|
||||
FULL_IMPORT_CHUNK_DAYS="${FULL_IMPORT_CHUNK_DAYS:-31}"
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
pg_to_clickhouse_archive.sh setup
|
||||
pg_to_clickhouse_archive.sh initial-full [--force]
|
||||
pg_to_clickhouse_archive.sh previous-day
|
||||
pg_to_clickhouse_archive.sh date YYYY-MM-DD
|
||||
pg_to_clickhouse_archive.sh status
|
||||
|
||||
Modes:
|
||||
setup Create archive database and tables only.
|
||||
initial-full Import all current PostgreSQL rows once. Refuses to run again unless --force is provided.
|
||||
previous-day Import yesterday in Asia/Shanghai.
|
||||
date Import one explicit business day/date partition, e.g. 2026-06-30.
|
||||
status Show row counts in ClickHouse archive tables.
|
||||
USAGE
|
||||
}
|
||||
|
||||
die() {
|
||||
echo "ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
log() {
|
||||
printf '[%s] %s\n' "$(TZ="$TIMEZONE" date '+%F %T %Z')" "$*"
|
||||
}
|
||||
|
||||
require_root_or_docker_access() {
|
||||
if docker inspect "$CLICKHOUSE_CONTAINER" >/dev/null 2>&1; then
|
||||
return
|
||||
fi
|
||||
die "Cannot access Docker. Run with sudo/root, for example: sudo $0 $*"
|
||||
}
|
||||
|
||||
load_config() {
|
||||
[[ -f "$ENV_FILE" ]] || die "Missing ClickHouse env file: $ENV_FILE"
|
||||
[[ -f "$PG_PASSWORD_FILE" ]] || die "Missing PostgreSQL password file: $PG_PASSWORD_FILE"
|
||||
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
[[ -n "${CLICKHOUSE_USER:-}" ]] || die "CLICKHOUSE_USER is not set in $ENV_FILE"
|
||||
[[ -n "${CLICKHOUSE_PASSWORD:-}" ]] || die "CLICKHOUSE_PASSWORD is not set in $ENV_FILE"
|
||||
|
||||
PG_PASSWORD="$(tr -d '\r\n' < "$PG_PASSWORD_FILE")"
|
||||
[[ -n "$PG_PASSWORD" ]] || die "PostgreSQL password file is empty: $PG_PASSWORD_FILE"
|
||||
|
||||
mkdir -p "$STATE_DIR" "$LOG_DIR"
|
||||
}
|
||||
|
||||
sql_quote() {
|
||||
sed "s/'/''/g" <<<"$1"
|
||||
}
|
||||
|
||||
ch() {
|
||||
docker exec -i "$CLICKHOUSE_CONTAINER" clickhouse-client \
|
||||
--user "$CLICKHOUSE_USER" \
|
||||
--password "$CLICKHOUSE_PASSWORD" \
|
||||
"$@"
|
||||
}
|
||||
|
||||
ch_query() {
|
||||
local query="$1"
|
||||
ch --query "$query"
|
||||
}
|
||||
|
||||
ch_multi() {
|
||||
ch --multiquery
|
||||
}
|
||||
|
||||
pg_source() {
|
||||
local table="$1"
|
||||
local pg_password_sql
|
||||
pg_password_sql="$(sql_quote "$PG_PASSWORD")"
|
||||
printf "postgresql('%s:%s', '%s', '%s', '%s', '%s')" \
|
||||
"$PG_HOST" "$PG_PORT" "$PG_DB" "$table" "$PG_USER" "$pg_password_sql"
|
||||
}
|
||||
|
||||
psql_query() {
|
||||
local query="$1"
|
||||
PGPASSWORD="$PG_PASSWORD" PGCONNECT_TIMEOUT=15 PGTZ="$TIMEZONE" psql \
|
||||
-h "$PG_HOST" \
|
||||
-p "$PG_PORT" \
|
||||
-U "$PG_USER" \
|
||||
-d "$PG_DB" \
|
||||
-v ON_ERROR_STOP=1 \
|
||||
-At \
|
||||
-F $'\t' \
|
||||
-c "${query}"
|
||||
}
|
||||
|
||||
psql_copy_to_stage() {
|
||||
local table="$1"
|
||||
local stage="$2"
|
||||
local pg_where="$3"
|
||||
local select_list
|
||||
select_list="$(pg_select_list "$table")"
|
||||
|
||||
PGPASSWORD="$PG_PASSWORD" PGCONNECT_TIMEOUT=15 PGTZ="$TIMEZONE" psql \
|
||||
-h "$PG_HOST" \
|
||||
-p "$PG_PORT" \
|
||||
-U "$PG_USER" \
|
||||
-d "$PG_DB" \
|
||||
-v ON_ERROR_STOP=1 \
|
||||
-q \
|
||||
-c "\\copy (SELECT ${select_list} FROM public.${table} WHERE ${pg_where}) TO STDOUT WITH (FORMAT CSV, HEADER true)" \
|
||||
| ch --query "INSERT INTO ${ARCHIVE_DB}.${stage} FORMAT CSVWithNames"
|
||||
}
|
||||
|
||||
pg_select_list() {
|
||||
case "$1" in
|
||||
question_monitor_parse_results)
|
||||
cat <<'SQL'
|
||||
id,
|
||||
tenant_id,
|
||||
run_id,
|
||||
brand_id,
|
||||
question_id,
|
||||
question_hash,
|
||||
ai_platform_id,
|
||||
business_date,
|
||||
CASE WHEN brand_mentioned THEN 1 WHEN brand_mentioned IS FALSE THEN 0 ELSE NULL END AS brand_mentioned,
|
||||
brand_mention_position,
|
||||
CASE WHEN first_recommended THEN 1 WHEN first_recommended IS FALSE THEN 0 ELSE NULL END AS first_recommended,
|
||||
sentiment_label,
|
||||
matched_brand_terms,
|
||||
parse_version,
|
||||
parse_status,
|
||||
created_at,
|
||||
updated_at
|
||||
SQL
|
||||
;;
|
||||
monitoring_platform_access_snapshots)
|
||||
cat <<'SQL'
|
||||
id,
|
||||
tenant_id,
|
||||
client_id,
|
||||
ai_platform_id,
|
||||
business_date,
|
||||
access_status,
|
||||
CASE WHEN connected THEN 1 ELSE 0 END AS connected,
|
||||
CASE WHEN accessible THEN 1 ELSE 0 END AS accessible,
|
||||
detected_at,
|
||||
reason_code,
|
||||
reason_message,
|
||||
created_at,
|
||||
updated_at
|
||||
SQL
|
||||
;;
|
||||
monitoring_question_config_snapshots)
|
||||
cat <<'SQL'
|
||||
id,
|
||||
tenant_id,
|
||||
brand_id,
|
||||
question_id,
|
||||
keyword_id,
|
||||
question_hash,
|
||||
question_text_snapshot,
|
||||
CASE WHEN monitor_enabled THEN 1 ELSE 0 END AS monitor_enabled,
|
||||
superseded_at,
|
||||
deleted_at,
|
||||
projected_at,
|
||||
created_at,
|
||||
updated_at
|
||||
SQL
|
||||
;;
|
||||
*)
|
||||
echo "*"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
declare -a TABLES=(
|
||||
monitoring_collect_tasks
|
||||
question_monitor_runs
|
||||
question_monitor_parse_results
|
||||
monitoring_citation_facts
|
||||
monitoring_brand_daily
|
||||
monitoring_brand_platform_daily
|
||||
monitoring_platform_access_snapshots
|
||||
monitoring_collect_requests
|
||||
monitoring_collect_dispatch_outbox
|
||||
monitoring_question_config_snapshots
|
||||
monitoring_article_url_aliases
|
||||
monitoring_user_marked_articles
|
||||
)
|
||||
|
||||
date_expr() {
|
||||
case "$1" in
|
||||
monitoring_collect_tasks|question_monitor_runs|question_monitor_parse_results|monitoring_citation_facts|monitoring_brand_daily|monitoring_brand_platform_daily|monitoring_platform_access_snapshots)
|
||||
echo "business_date"
|
||||
;;
|
||||
monitoring_user_marked_articles)
|
||||
echo "toDate(marked_at, '$TIMEZONE')"
|
||||
;;
|
||||
monitoring_collect_dispatch_outbox|monitoring_collect_requests|monitoring_question_config_snapshots|monitoring_article_url_aliases)
|
||||
echo "toDate(created_at, '$TIMEZONE')"
|
||||
;;
|
||||
*)
|
||||
die "No date expression configured for table: $1"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
order_by_expr() {
|
||||
case "$1" in
|
||||
monitoring_collect_dispatch_outbox)
|
||||
echo "(toDate(created_at, '$TIMEZONE'), workspace_id, id)"
|
||||
;;
|
||||
monitoring_user_marked_articles)
|
||||
echo "(toDate(marked_at, '$TIMEZONE'), tenant_id, id)"
|
||||
;;
|
||||
monitoring_collect_requests|monitoring_question_config_snapshots|monitoring_article_url_aliases)
|
||||
echo "(toDate(created_at, '$TIMEZONE'), tenant_id, id)"
|
||||
;;
|
||||
*)
|
||||
echo "(business_date, tenant_id, id)"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
where_for_date() {
|
||||
local table="$1"
|
||||
local day="$2"
|
||||
local next_day
|
||||
next_day="$(TZ="$TIMEZONE" date -d "$day +1 day" +%F)"
|
||||
|
||||
case "$table" in
|
||||
monitoring_collect_tasks|question_monitor_runs|question_monitor_parse_results|monitoring_citation_facts|monitoring_brand_daily|monitoring_brand_platform_daily|monitoring_platform_access_snapshots)
|
||||
echo "business_date = toDate32('$day')"
|
||||
;;
|
||||
monitoring_user_marked_articles)
|
||||
echo "marked_at >= toDateTime64('$day 00:00:00', 6, '$TIMEZONE') AND marked_at < toDateTime64('$next_day 00:00:00', 6, '$TIMEZONE')"
|
||||
;;
|
||||
monitoring_collect_dispatch_outbox|monitoring_collect_requests|monitoring_question_config_snapshots|monitoring_article_url_aliases)
|
||||
echo "created_at >= toDateTime64('$day 00:00:00', 6, '$TIMEZONE') AND created_at < toDateTime64('$next_day 00:00:00', 6, '$TIMEZONE')"
|
||||
;;
|
||||
*)
|
||||
die "No date filter configured for table: $table"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
where_for_range() {
|
||||
local table="$1"
|
||||
local start_day="$2"
|
||||
local end_day="$3"
|
||||
|
||||
case "$table" in
|
||||
monitoring_collect_tasks|question_monitor_runs|question_monitor_parse_results|monitoring_citation_facts|monitoring_brand_daily|monitoring_brand_platform_daily|monitoring_platform_access_snapshots)
|
||||
echo "business_date >= toDate32('$start_day') AND business_date < toDate32('$end_day')"
|
||||
;;
|
||||
monitoring_user_marked_articles)
|
||||
echo "marked_at >= toDateTime64('$start_day 00:00:00', 6, '$TIMEZONE') AND marked_at < toDateTime64('$end_day 00:00:00', 6, '$TIMEZONE')"
|
||||
;;
|
||||
monitoring_collect_dispatch_outbox|monitoring_collect_requests|monitoring_question_config_snapshots|monitoring_article_url_aliases)
|
||||
echo "created_at >= toDateTime64('$start_day 00:00:00', 6, '$TIMEZONE') AND created_at < toDateTime64('$end_day 00:00:00', 6, '$TIMEZONE')"
|
||||
;;
|
||||
*)
|
||||
die "No date range filter configured for table: $table"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
pg_date_expr() {
|
||||
case "$1" in
|
||||
monitoring_collect_tasks|question_monitor_runs|question_monitor_parse_results|monitoring_citation_facts|monitoring_brand_daily|monitoring_brand_platform_daily|monitoring_platform_access_snapshots)
|
||||
echo "business_date"
|
||||
;;
|
||||
monitoring_user_marked_articles)
|
||||
echo "(marked_at AT TIME ZONE '$TIMEZONE')::date"
|
||||
;;
|
||||
monitoring_collect_dispatch_outbox|monitoring_collect_requests|monitoring_question_config_snapshots|monitoring_article_url_aliases)
|
||||
echo "(created_at AT TIME ZONE '$TIMEZONE')::date"
|
||||
;;
|
||||
*)
|
||||
die "No PostgreSQL date expression configured for table: $1"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
pg_where_for_date() {
|
||||
local table="$1"
|
||||
local day="$2"
|
||||
local next_day
|
||||
next_day="$(TZ="$TIMEZONE" date -d "$day +1 day" +%F)"
|
||||
|
||||
case "$table" in
|
||||
monitoring_collect_tasks|question_monitor_runs|question_monitor_parse_results|monitoring_citation_facts|monitoring_brand_daily|monitoring_brand_platform_daily|monitoring_platform_access_snapshots)
|
||||
echo "business_date = DATE '$day'"
|
||||
;;
|
||||
monitoring_user_marked_articles)
|
||||
echo "marked_at >= (TIMESTAMP '$day 00:00:00' AT TIME ZONE '$TIMEZONE') AND marked_at < (TIMESTAMP '$next_day 00:00:00' AT TIME ZONE '$TIMEZONE')"
|
||||
;;
|
||||
monitoring_collect_dispatch_outbox|monitoring_collect_requests|monitoring_question_config_snapshots|monitoring_article_url_aliases)
|
||||
echo "created_at >= (TIMESTAMP '$day 00:00:00' AT TIME ZONE '$TIMEZONE') AND created_at < (TIMESTAMP '$next_day 00:00:00' AT TIME ZONE '$TIMEZONE')"
|
||||
;;
|
||||
*)
|
||||
die "No PostgreSQL date filter configured for table: $table"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
pg_where_for_range() {
|
||||
local table="$1"
|
||||
local start_day="$2"
|
||||
local end_day="$3"
|
||||
|
||||
case "$table" in
|
||||
monitoring_collect_tasks|question_monitor_runs|question_monitor_parse_results|monitoring_citation_facts|monitoring_brand_daily|monitoring_brand_platform_daily|monitoring_platform_access_snapshots)
|
||||
echo "business_date >= DATE '$start_day' AND business_date < DATE '$end_day'"
|
||||
;;
|
||||
monitoring_user_marked_articles)
|
||||
echo "marked_at >= (TIMESTAMP '$start_day 00:00:00' AT TIME ZONE '$TIMEZONE') AND marked_at < (TIMESTAMP '$end_day 00:00:00' AT TIME ZONE '$TIMEZONE')"
|
||||
;;
|
||||
monitoring_collect_dispatch_outbox|monitoring_collect_requests|monitoring_question_config_snapshots|monitoring_article_url_aliases)
|
||||
echo "created_at >= (TIMESTAMP '$start_day 00:00:00' AT TIME ZONE '$TIMEZONE') AND created_at < (TIMESTAMP '$end_day 00:00:00' AT TIME ZONE '$TIMEZONE')"
|
||||
;;
|
||||
*)
|
||||
die "No PostgreSQL date range filter configured for table: $table"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
validate_day() {
|
||||
local day="$1"
|
||||
[[ "$day" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] || die "Invalid date: $day"
|
||||
TZ="$TIMEZONE" date -d "$day" +%F >/dev/null || die "Invalid date: $day"
|
||||
}
|
||||
|
||||
create_table_from_source() {
|
||||
local table="$1"
|
||||
local partition_expr order_expr source
|
||||
partition_expr="$(date_expr "$table")"
|
||||
order_expr="$(order_by_expr "$table")"
|
||||
source="$(pg_source "$table")"
|
||||
|
||||
ch_multi <<SQL
|
||||
CREATE DATABASE IF NOT EXISTS ${ARCHIVE_DB};
|
||||
CREATE TABLE IF NOT EXISTS ${ARCHIVE_DB}.${table}
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY ${partition_expr}
|
||||
ORDER BY ${order_expr}
|
||||
AS
|
||||
SELECT *
|
||||
FROM ${source}
|
||||
WHERE 0;
|
||||
SQL
|
||||
}
|
||||
|
||||
create_stage_table() {
|
||||
local table="$1"
|
||||
local stage="$2"
|
||||
local select_where="$3"
|
||||
|
||||
ch_multi <<SQL
|
||||
DROP TABLE IF EXISTS ${ARCHIVE_DB}.${stage};
|
||||
CREATE TABLE ${ARCHIVE_DB}.${stage} AS ${ARCHIVE_DB}.${table};
|
||||
SQL
|
||||
|
||||
psql_copy_to_stage "$table" "$stage" "$select_where"
|
||||
}
|
||||
|
||||
setup_tables() {
|
||||
log "Creating archive database and tables in ${ARCHIVE_DB}"
|
||||
for table in "${TABLES[@]}"; do
|
||||
log "setup ${table}"
|
||||
create_table_from_source "$table"
|
||||
done
|
||||
log "setup complete"
|
||||
}
|
||||
|
||||
cleanup_work_tables() {
|
||||
local table
|
||||
while IFS= read -r table; do
|
||||
[[ -n "$table" ]] || continue
|
||||
log "cleanup work table ${table}"
|
||||
ch_query "DROP TABLE IF EXISTS ${ARCHIVE_DB}.${table}"
|
||||
done < <(ch_query "SHOW TABLES FROM ${ARCHIVE_DB} LIKE '__stage_%' FORMAT TabSeparatedRaw")
|
||||
|
||||
while IFS= read -r table; do
|
||||
[[ -n "$table" ]] || continue
|
||||
log "cleanup old table ${table}"
|
||||
ch_query "DROP TABLE IF EXISTS ${ARCHIVE_DB}.${table}"
|
||||
done < <(ch_query "SHOW TABLES FROM ${ARCHIVE_DB} LIKE '__old_%' FORMAT TabSeparatedRaw")
|
||||
}
|
||||
|
||||
stage_count() {
|
||||
local table="$1"
|
||||
ch_query "SELECT count() FROM ${ARCHIVE_DB}.${table} FORMAT TabSeparatedRaw"
|
||||
}
|
||||
|
||||
stage_count_where() {
|
||||
local table="$1"
|
||||
local where_clause="$2"
|
||||
ch_query "SELECT count() FROM ${ARCHIVE_DB}.${table} WHERE ${where_clause} FORMAT TabSeparatedRaw"
|
||||
}
|
||||
|
||||
source_count() {
|
||||
local table="$1"
|
||||
local where_clause="${2:-1}"
|
||||
ch_query "SELECT count() FROM $(pg_source "$table") WHERE ${where_clause} FORMAT TabSeparatedRaw"
|
||||
}
|
||||
|
||||
import_full_table() {
|
||||
local table="$1"
|
||||
local bounds min_day max_day chunk_start chunk_end after_max where_clause stage total_days
|
||||
|
||||
bounds="$(source_date_bounds "$table")"
|
||||
min_day="${bounds%%$'\t'*}"
|
||||
max_day="${bounds##*$'\t'}"
|
||||
|
||||
if [[ -z "$min_day" || -z "$max_day" ]]; then
|
||||
log "full import ${table}: source has no rows"
|
||||
return
|
||||
fi
|
||||
|
||||
log "full import ${table}: day range ${min_day}..${max_day}"
|
||||
after_max="$(TZ="$TIMEZONE" date -d "$max_day +1 day" +%F)"
|
||||
chunk_start="$min_day"
|
||||
total_days=0
|
||||
|
||||
while [[ "$chunk_start" < "$after_max" ]]; do
|
||||
chunk_end="$(TZ="$TIMEZONE" date -d "$chunk_start +${FULL_IMPORT_CHUNK_DAYS} day" +%F)"
|
||||
if [[ "$chunk_end" > "$after_max" ]]; then
|
||||
chunk_end="$after_max"
|
||||
fi
|
||||
|
||||
where_clause="$(pg_where_for_range "$table" "$chunk_start" "$chunk_end")"
|
||||
stage="__stage_full_${chunk_start//-/}_${table}_$(TZ="$TIMEZONE" date +%Y%m%d%H%M%S)_$$"
|
||||
|
||||
log "full import staging ${table}: ${chunk_start}..<${chunk_end}"
|
||||
create_stage_table "$table" "$stage" "$where_clause"
|
||||
replace_partitions_from_stage "$table" "$stage" "$chunk_start" "$chunk_end"
|
||||
|
||||
total_days=$((total_days + ($(TZ="$TIMEZONE" date -d "$chunk_end" +%s) - $(TZ="$TIMEZONE" date -d "$chunk_start" +%s)) / 86400))
|
||||
chunk_start="$chunk_end"
|
||||
done
|
||||
log "full import ${table}: imported ${total_days} day partitions"
|
||||
}
|
||||
|
||||
source_date_bounds() {
|
||||
local table="$1"
|
||||
local expr
|
||||
expr="$(pg_date_expr "$table")"
|
||||
|
||||
psql_query "SELECT COALESCE(min(${expr})::text, ''), COALESCE(max(${expr})::text, '') FROM public.${table};"
|
||||
}
|
||||
|
||||
replace_partitions_from_stage() {
|
||||
local table="$1"
|
||||
local stage="$2"
|
||||
local start_day="$3"
|
||||
local end_day="$4"
|
||||
local day rows where_clause
|
||||
|
||||
day="$start_day"
|
||||
while [[ "$day" < "$end_day" ]]; do
|
||||
where_clause="$(where_for_date "$table" "$day")"
|
||||
rows="$(stage_count_where "$stage" "$where_clause")"
|
||||
|
||||
if [[ "$rows" == "0" ]]; then
|
||||
log "full import ${table} day=${day} rows=0; dropping target partition after successful staging"
|
||||
ch_query "ALTER TABLE ${ARCHIVE_DB}.${table} DROP PARTITION '${day}'"
|
||||
else
|
||||
log "full import replacing partition ${day} in ${table}, rows=${rows}"
|
||||
ch_query "ALTER TABLE ${ARCHIVE_DB}.${table} REPLACE PARTITION '${day}' FROM ${ARCHIVE_DB}.${stage}"
|
||||
fi
|
||||
|
||||
day="$(TZ="$TIMEZONE" date -d "$day +1 day" +%F)"
|
||||
done
|
||||
|
||||
ch_query "DROP TABLE IF EXISTS ${ARCHIVE_DB}.${stage}"
|
||||
}
|
||||
|
||||
initial_full() {
|
||||
local force="${1:-}"
|
||||
local marker="$STATE_DIR/initial_full_import.completed"
|
||||
|
||||
if [[ -f "$marker" && "$force" != "--force" ]]; then
|
||||
die "Initial full import already completed at $(cat "$marker"). Refusing to replace permanent archive. Use --force only if you are intentionally rebuilding from PostgreSQL's current retention window."
|
||||
fi
|
||||
|
||||
setup_tables
|
||||
cleanup_work_tables
|
||||
log "Starting initial full import"
|
||||
for table in "${TABLES[@]}"; do
|
||||
import_full_table "$table"
|
||||
done
|
||||
|
||||
TZ="$TIMEZONE" date '+%F %T %Z' > "$marker"
|
||||
log "Initial full import complete. Marker: $marker"
|
||||
}
|
||||
|
||||
import_date_table() {
|
||||
local table="$1"
|
||||
local day="$2"
|
||||
local where_clause stage rows partition
|
||||
|
||||
where_clause="$(pg_where_for_date "$table" "$day")"
|
||||
stage="__stage_${day//-/}_${table}_$(TZ="$TIMEZONE" date +%Y%m%d%H%M%S)_$$"
|
||||
partition="$day"
|
||||
|
||||
log "date import staging ${table} day=${day}"
|
||||
create_stage_table "$table" "$stage" "$where_clause"
|
||||
rows="$(stage_count "$stage")"
|
||||
|
||||
if [[ "$rows" == "0" ]]; then
|
||||
log "date import ${table} day=${day} rows=0; dropping target partition only after successful empty staging"
|
||||
ch_multi <<SQL
|
||||
ALTER TABLE ${ARCHIVE_DB}.${table} DROP PARTITION '${partition}';
|
||||
DROP TABLE ${ARCHIVE_DB}.${stage};
|
||||
SQL
|
||||
return
|
||||
fi
|
||||
|
||||
log "date import replacing partition ${partition} in ${table}, rows=${rows}"
|
||||
ch_multi <<SQL
|
||||
ALTER TABLE ${ARCHIVE_DB}.${table} REPLACE PARTITION '${partition}' FROM ${ARCHIVE_DB}.${stage};
|
||||
DROP TABLE ${ARCHIVE_DB}.${stage};
|
||||
SQL
|
||||
}
|
||||
|
||||
import_date() {
|
||||
local day="$1"
|
||||
local marker="$STATE_DIR/sync_${day}.completed"
|
||||
|
||||
validate_day "$day"
|
||||
setup_tables
|
||||
cleanup_work_tables
|
||||
|
||||
log "Starting date import day=${day}"
|
||||
for table in "${TABLES[@]}"; do
|
||||
import_date_table "$table" "$day"
|
||||
done
|
||||
|
||||
TZ="$TIMEZONE" date '+%F %T %Z' > "$marker"
|
||||
log "Date import complete day=${day}. Marker: $marker"
|
||||
}
|
||||
|
||||
previous_day() {
|
||||
TZ="$TIMEZONE" date -d yesterday +%F
|
||||
}
|
||||
|
||||
status() {
|
||||
setup_tables >/dev/null
|
||||
log "Archive table row counts"
|
||||
for table in "${TABLES[@]}"; do
|
||||
printf '%s\t%s\n' "$table" "$(stage_count "$table")"
|
||||
done
|
||||
}
|
||||
|
||||
main() {
|
||||
local mode="${1:-}"
|
||||
[[ -n "$mode" ]] || {
|
||||
usage
|
||||
exit 2
|
||||
}
|
||||
|
||||
require_root_or_docker_access "$@"
|
||||
load_config
|
||||
|
||||
case "$mode" in
|
||||
setup)
|
||||
setup_tables
|
||||
;;
|
||||
initial-full)
|
||||
initial_full "${2:-}"
|
||||
;;
|
||||
previous-day)
|
||||
import_date "$(previous_day)"
|
||||
;;
|
||||
date)
|
||||
[[ -n "${2:-}" ]] || die "date mode requires YYYY-MM-DD"
|
||||
import_date "$2"
|
||||
;;
|
||||
status)
|
||||
status
|
||||
;;
|
||||
-h|--help|help)
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -21,12 +22,16 @@ local current = redis.call("GET", KEYS[1])
|
||||
if not current then
|
||||
return 0
|
||||
end
|
||||
if current ~= ARGV[1] then
|
||||
return -1
|
||||
if current == ARGV[1] then
|
||||
redis.call("PSETEX", KEYS[1], ARGV[4], "grace:" .. ARGV[1])
|
||||
redis.call("PSETEX", KEYS[2], ARGV[3], ARGV[2])
|
||||
return 1
|
||||
end
|
||||
redis.call("DEL", KEYS[1])
|
||||
redis.call("PSETEX", KEYS[2], ARGV[3], ARGV[2])
|
||||
return 1
|
||||
if current == "grace:" .. ARGV[1] then
|
||||
redis.call("PSETEX", KEYS[2], ARGV[3], ARGV[2])
|
||||
return 1
|
||||
end
|
||||
return -1
|
||||
`)
|
||||
|
||||
var setMaxSessionVersionScript = redis.NewScript(`
|
||||
@@ -42,6 +47,13 @@ return next_number
|
||||
|
||||
const sessionVersionFallbackTTL = 30 * 24 * time.Hour
|
||||
|
||||
// refreshRotationGraceTTL keeps a rotated refresh token usable for a short
|
||||
// window so concurrent refreshes (e.g. multiple browser tabs racing with the
|
||||
// same token) each obtain a valid new pair instead of being logged out.
|
||||
const refreshRotationGraceTTL = 60 * time.Second
|
||||
|
||||
const rotationGracePrefix = "grace:"
|
||||
|
||||
type SessionStore struct {
|
||||
rdb *redis.Client
|
||||
fallback *memorySessionStore
|
||||
@@ -71,10 +83,20 @@ func (s *SessionStore) GetRefresh(ctx context.Context, jti string) (string, erro
|
||||
if s.rdb != nil {
|
||||
value, err := s.rdb.Get(ctx, key).Result()
|
||||
if err == nil {
|
||||
if strings.HasPrefix(value, rotationGracePrefix) {
|
||||
return "", ErrRefreshSessionNotFound
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
}
|
||||
return s.fallback.get(key)
|
||||
value, err := s.fallback.get(key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if strings.HasPrefix(value, rotationGracePrefix) {
|
||||
return "", ErrRefreshSessionNotFound
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func (s *SessionStore) DeleteRefresh(ctx context.Context, jti string) error {
|
||||
@@ -104,6 +126,7 @@ func (s *SessionStore) RotateRefresh(ctx context.Context, oldJTI, oldHash, newJT
|
||||
oldHash,
|
||||
newHash,
|
||||
ttl.Milliseconds(),
|
||||
refreshRotationGraceTTL.Milliseconds(),
|
||||
).Int()
|
||||
if err != nil {
|
||||
return s.fallback.rotate(oldKey, oldHash, newKey, newHash, ttl)
|
||||
@@ -298,10 +321,17 @@ func (s *memorySessionStore) rotate(oldKey, oldHash, newKey, newHash string, ttl
|
||||
delete(s.items, oldKey)
|
||||
return ErrRefreshSessionNotFound
|
||||
}
|
||||
if item.value != oldHash {
|
||||
switch item.value {
|
||||
case oldHash:
|
||||
s.items[oldKey] = memorySessionItem{
|
||||
value: rotationGracePrefix + oldHash,
|
||||
expiresAt: time.Now().Add(refreshRotationGraceTTL),
|
||||
}
|
||||
case rotationGracePrefix + oldHash:
|
||||
// Already rotated within the grace window; keep the marker as-is.
|
||||
default:
|
||||
return ErrRefreshTokenMismatch
|
||||
}
|
||||
delete(s.items, oldKey)
|
||||
if ttl > 0 {
|
||||
s.items[newKey] = memorySessionItem{
|
||||
value: newHash,
|
||||
|
||||
@@ -43,6 +43,62 @@ func TestSessionStoreRotateRefreshSuccess(t *testing.T) {
|
||||
assert.Equal(t, "new-hash", value)
|
||||
}
|
||||
|
||||
func TestSessionStoreRotateRefreshAllowsConcurrentReuseWithinGrace(t *testing.T) {
|
||||
mr := miniredis.RunT(t)
|
||||
client := goredis.NewClient(&goredis.Options{Addr: mr.Addr()})
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
|
||||
store := NewSessionStore(client)
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, store.SaveRefresh(ctx, "old-jti", "old-hash", time.Minute))
|
||||
|
||||
// Tab A rotates first, tab B retries with the same old token within the
|
||||
// grace window: both must obtain valid new sessions.
|
||||
require.NoError(t, store.RotateRefresh(ctx, "old-jti", "old-hash", "new-jti-a", "new-hash-a", 2*time.Minute))
|
||||
require.NoError(t, store.RotateRefresh(ctx, "old-jti", "old-hash", "new-jti-b", "new-hash-b", 2*time.Minute))
|
||||
|
||||
valueA, err := store.GetRefresh(ctx, "new-jti-a")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "new-hash-a", valueA)
|
||||
|
||||
valueB, err := store.GetRefresh(ctx, "new-jti-b")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "new-hash-b", valueB)
|
||||
|
||||
// A different (wrong) old hash must still be rejected during grace.
|
||||
err = store.RotateRefresh(ctx, "old-jti", "wrong-hash", "new-jti-c", "new-hash-c", 2*time.Minute)
|
||||
require.ErrorIs(t, err, ErrRefreshTokenMismatch)
|
||||
|
||||
// The rotated old token stays hidden from session lookups during grace.
|
||||
_, err = store.GetRefresh(ctx, "old-jti")
|
||||
require.ErrorIs(t, err, ErrRefreshSessionNotFound)
|
||||
}
|
||||
|
||||
func TestMemorySessionStoreRotateAllowsConcurrentReuseWithinGrace(t *testing.T) {
|
||||
store := NewSessionStore(nil)
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, store.SaveRefresh(ctx, "old-jti", "old-hash", time.Minute))
|
||||
|
||||
require.NoError(t, store.RotateRefresh(ctx, "old-jti", "old-hash", "new-jti-a", "new-hash-a", 2*time.Minute))
|
||||
require.NoError(t, store.RotateRefresh(ctx, "old-jti", "old-hash", "new-jti-b", "new-hash-b", 2*time.Minute))
|
||||
|
||||
_, err := store.GetRefresh(ctx, "old-jti")
|
||||
assert.Error(t, err)
|
||||
|
||||
valueA, err := store.GetRefresh(ctx, "new-jti-a")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "new-hash-a", valueA)
|
||||
|
||||
valueB, err := store.GetRefresh(ctx, "new-jti-b")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "new-hash-b", valueB)
|
||||
|
||||
err = store.RotateRefresh(ctx, "old-jti", "wrong-hash", "new-jti-c", "new-hash-c", 2*time.Minute)
|
||||
require.ErrorIs(t, err, ErrRefreshTokenMismatch)
|
||||
}
|
||||
|
||||
func TestSessionStoreRotateRefreshRejectsMismatch(t *testing.T) {
|
||||
mr := miniredis.RunT(t)
|
||||
client := goredis.NewClient(&goredis.Options{Addr: mr.Addr()})
|
||||
|
||||
Reference in New Issue
Block a user