root/geo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(scheduler): harden jobs for scalable execution

Browse Source
This commit is contained in:
liangxu
2026-05-20 12:19:27 +08:00
parent 4b09a34748
commit 5fb9d0b0dd
23 changed files with 887 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/migrations_ops/20260520120000_create_scheduler_control_tables.up.sql
+12 -7
View File
@@ -63,6 +63,10 @@ CREATE INDEX IF NOT EXISTS idx_scheduler_job_runs_job_started
CREATE INDEX IF NOT EXISTS idx_scheduler_job_runs_status_started
ON ops.scheduler_job_runs (status, started_at DESC);
CREATE INDEX IF NOT EXISTS idx_scheduler_job_runs_auto_started
ON ops.scheduler_job_runs (job_key, started_at DESC, id DESC)
WHERE trigger_type = 'auto';
CREATE TABLE IF NOT EXISTS ops.scheduler_job_triggers (
id BIGSERIAL PRIMARY KEY,
job_key VARCHAR(96) NOT NULL REFERENCES ops.scheduler_jobs(job_key),
@@ -88,16 +92,17 @@ CREATE INDEX IF NOT EXISTS idx_scheduler_job_triggers_pending
INSERT INTO ops.scheduler_jobs
(job_key, display_name, category, description, enabled, schedule_type, interval_seconds, timezone, timeout_seconds, batch_size, config)
VALUES
('schedule_dispatch', '内容定时任务派发', 'content', '扫描内容定时任务并投递生成队列。', true, 'interval', 15, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
('monitoring_daily_task', 'AI 监控每日任务生成', 'monitoring', '为品牌问题按业务日生成 AI 监控采集任务。', true, 'interval', 300, 'Asia/Shanghai', 45, 64, '{}'::jsonb),
('monitoring_result_recovery', '监控结果恢复', 'monitoring', '恢复已回调但未成功进入结果队列的监控采集结果。', true, 'interval', 60, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
('monitoring_lease_recovery', '监控租约回收', 'monitoring', '回收超过租约时间仍未回调的监控采集任务。', true, 'interval', 1800, 'Asia/Shanghai', 30, NULL, '{}'::jsonb),
('monitoring_received_inspection', '监控 received 卡死检查', 'monitoring', '检查长时间停留在 received 状态的监控任务。', true, 'interval', 300, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
('generation_state_check', '生成任务状态巡检', 'generation', '低频巡检文章生成任务与文章状态一致性。', true, 'interval', 300, 'Asia/Shanghai', 10, 100, '{}'::jsonb),
('monitoring_retention_cleanup', '监控历史保留清理', 'monitoring', '按业务日期清理最近 30 天窗口以前的监控历史数据。', true, 'interval', 86400, 'Asia/Shanghai', 900, 5000,
('schedule_dispatch', '内容定时任务派发', 'content', '索引驱动的延迟队列派发，仅认领到期内容任务并投递生成队列。', true, 'interval', 15, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
('monitoring_daily_task', 'AI 监控每日任务生成', 'monitoring', '按业务日小批量物化监控采集任务并派发到桌面任务队列。', true, 'interval', 300, 'Asia/Shanghai', 45, 64, '{"max_materialize_per_plan":12,"max_materialize_per_brand":4}'::jsonb),
('monitoring_result_recovery', '监控结果恢复', 'monitoring', '按回调结果队列索引恢复未成功入队的监控结果。', true, 'interval', 60, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
('monitoring_lease_recovery', '监控租约回收', 'monitoring', '按过期租约索引限批回收超过租约时间仍未回调的监控任务。', true, 'interval', 1800, 'Asia/Shanghai', 30, 1000, '{}'::jsonb),
('monitoring_received_inspection', '监控 received 卡死检查', 'monitoring', '按 received watchdog 索引限批标记长时间未入队的监控任务。', true, 'interval', 300, 'Asia/Shanghai', 30, 100, '{"threshold":"15m"}'::jsonb),
('generation_state_check', '生成任务状态巡检', 'generation', '低频、限批检查文章生成任务与文章状态一致性。', true, 'interval', 300, 'Asia/Shanghai', 10, 100, '{"lookback":"24h"}'::jsonb),
('monitoring_retention_cleanup', '监控历史保留清理', 'monitoring', '按保留窗口和索引限批删除最近 30 天以前的监控历史数据。', true, 'interval', 86400, 'Asia/Shanghai', 900, 5000,
'{"retention_days":30,"batch_size":5000,"max_batches_per_run":200,"dry_run":false,"statement_timeout":"5s","run_at_local":"02:30"}'::jsonb)
ON CONFLICT (job_key) DO UPDATE
SET display_name = EXCLUDED.display_name,
category = EXCLUDED.category,
description = EXCLUDED.description,
config = ops.scheduler_jobs.config || EXCLUDED.config,
updated_at = NOW();
server/migrations_ops/20260520143000_harden_scheduler_jobs_for_k3s.down.sql
+1
View File
@@ -0,0 +1 @@
DROP INDEX IF EXISTS ops.idx_scheduler_job_runs_auto_started;
server/migrations_ops/20260520143000_harden_scheduler_jobs_for_k3s.up.sql
+59
View File
@@ -0,0 +1,59 @@
CREATE INDEX IF NOT EXISTS idx_scheduler_job_runs_auto_started
ON ops.scheduler_job_runs (job_key, started_at DESC, id DESC)
WHERE trigger_type = 'auto';
UPDATE ops.scheduler_jobs
SET description = '索引驱动的延迟队列派发，仅认领到期内容任务并投递生成队列。',
timeout_seconds = 30,
batch_size = 100,
config = config || '{}'::jsonb,
updated_at = NOW()
WHERE job_key = 'schedule_dispatch';
UPDATE ops.scheduler_jobs
SET description = '按业务日小批量物化监控采集任务并派发到桌面任务队列。',
timeout_seconds = 45,
batch_size = 64,
config = config || '{"max_materialize_per_plan":12,"max_materialize_per_brand":4}'::jsonb,
updated_at = NOW()
WHERE job_key = 'monitoring_daily_task';
UPDATE ops.scheduler_jobs
SET description = '按回调结果队列索引恢复未成功入队的监控结果。',
timeout_seconds = 30,
batch_size = 100,
config = config || '{"claim_ttl":"5m"}'::jsonb,
updated_at = NOW()
WHERE job_key = 'monitoring_result_recovery';
UPDATE ops.scheduler_jobs
SET description = '按过期租约索引限批回收超过租约时间仍未回调的监控任务。',
timeout_seconds = 30,
batch_size = 1000,
config = config || '{}'::jsonb,
updated_at = NOW()
WHERE job_key = 'monitoring_lease_recovery';
UPDATE ops.scheduler_jobs
SET description = '按 received watchdog 索引限批标记长时间未入队的监控任务。',
timeout_seconds = 30,
batch_size = 100,
config = config || '{"threshold":"15m"}'::jsonb,
updated_at = NOW()
WHERE job_key = 'monitoring_received_inspection';
UPDATE ops.scheduler_jobs
SET description = '低频、限批检查文章生成任务与文章状态一致性。',
timeout_seconds = 10,
batch_size = 100,
config = config || '{"lookback":"24h"}'::jsonb,
updated_at = NOW()
WHERE job_key = 'generation_state_check';
UPDATE ops.scheduler_jobs
SET description = '按保留窗口和索引限批删除最近 30 天以前的监控历史数据。',
timeout_seconds = 900,
batch_size = 5000,
config = config || '{"retention_days":30,"batch_size":5000,"max_batches_per_run":200,"dry_run":false,"statement_timeout":"5s","run_at_local":"02:30"}'::jsonb,
updated_at = NOW()
WHERE job_key = 'monitoring_retention_cleanup';