feat: add ops scheduler control center
This commit is contained in:
@@ -49,6 +49,7 @@
|
||||
import {
|
||||
AuditOutlined,
|
||||
BookOutlined,
|
||||
ClockCircleOutlined,
|
||||
ControlOutlined,
|
||||
CrownOutlined,
|
||||
FileSearchOutlined,
|
||||
@@ -89,6 +90,7 @@ const menuLeaves: MenuLeaf[] = [
|
||||
{ key: '/compliance/stats', label: '合规统计', path: '/compliance/stats' },
|
||||
{ key: '/accounts', label: '操作员管理', path: '/accounts' },
|
||||
{ key: '/jobs', label: '任务中心', path: '/jobs' },
|
||||
{ key: '/scheduler', label: '调度中心', path: '/scheduler' },
|
||||
{ key: '/audits', label: '审计日志', path: '/audits' },
|
||||
]
|
||||
|
||||
@@ -155,6 +157,11 @@ const menuItems = computed<ItemType[]>(() => [
|
||||
label: '任务中心',
|
||||
icon: () => h(PartitionOutlined),
|
||||
},
|
||||
{
|
||||
key: '/scheduler',
|
||||
label: '调度中心',
|
||||
icon: () => h(ClockCircleOutlined),
|
||||
},
|
||||
{
|
||||
key: '/audits',
|
||||
label: '审计日志',
|
||||
|
||||
@@ -49,6 +49,12 @@ export const router = createRouter({
|
||||
component: () => import('@/views/JobsView.vue'),
|
||||
meta: { title: '任务中心' },
|
||||
},
|
||||
{
|
||||
path: 'scheduler',
|
||||
name: 'scheduler',
|
||||
component: () => import('@/views/SchedulerView.vue'),
|
||||
meta: { title: '调度中心' },
|
||||
},
|
||||
{
|
||||
path: 'site-domain-mappings',
|
||||
name: 'site-domain-mappings',
|
||||
|
||||
@@ -0,0 +1,731 @@
|
||||
<template>
|
||||
<div class="scheduler-page">
|
||||
<div class="scheduler-header">
|
||||
<div>
|
||||
<h2 class="ops-page-title">调度中心</h2>
|
||||
<div class="scheduler-header__meta">系统周期任务启停、配置、手动触发与运行追踪</div>
|
||||
</div>
|
||||
<a-space>
|
||||
<a-button :loading="instancesLoading" @click="loadInstances">
|
||||
<template #icon><ClusterOutlined /></template>
|
||||
实例
|
||||
</a-button>
|
||||
<a-button type="primary" :loading="loading" @click="reload">
|
||||
<template #icon><ReloadOutlined /></template>
|
||||
刷新
|
||||
</a-button>
|
||||
</a-space>
|
||||
</div>
|
||||
|
||||
<div class="scheduler-summary">
|
||||
<div class="summary-tile">
|
||||
<span>任务总数</span>
|
||||
<strong>{{ total }}</strong>
|
||||
</div>
|
||||
<div class="summary-tile summary-tile--green">
|
||||
<span>已启用</span>
|
||||
<strong>{{ enabledCount }}</strong>
|
||||
</div>
|
||||
<div class="summary-tile summary-tile--gold">
|
||||
<span>运行中</span>
|
||||
<strong>{{ runningCount }}</strong>
|
||||
</div>
|
||||
<div class="summary-tile summary-tile--red">
|
||||
<span>最近失败</span>
|
||||
<strong>{{ failedCount }}</strong>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="ops-card">
|
||||
<div class="ops-toolbar scheduler-toolbar">
|
||||
<a-input
|
||||
v-model:value="filter.keyword"
|
||||
placeholder="搜索任务名 / key"
|
||||
allow-clear
|
||||
style="width: 260px"
|
||||
@press-enter="resetAndReload"
|
||||
/>
|
||||
<a-select
|
||||
v-model:value="filter.category"
|
||||
:options="categoryOptions"
|
||||
allow-clear
|
||||
placeholder="分类"
|
||||
style="width: 160px"
|
||||
@change="resetAndReload"
|
||||
/>
|
||||
<a-select
|
||||
v-model:value="filter.enabled"
|
||||
:options="enabledOptions"
|
||||
allow-clear
|
||||
placeholder="启用状态"
|
||||
style="width: 140px"
|
||||
@change="resetAndReload"
|
||||
/>
|
||||
<a-button @click="resetAndReload">查询</a-button>
|
||||
</div>
|
||||
|
||||
<a-table
|
||||
:columns="columns"
|
||||
:data-source="rows"
|
||||
:loading="loading"
|
||||
:pagination="pagination"
|
||||
row-key="job_key"
|
||||
:scroll="{ x: 1320 }"
|
||||
@change="onTableChange"
|
||||
>
|
||||
<template #bodyCell="{ column, record }">
|
||||
<template v-if="column.key === 'job'">
|
||||
<div class="job-cell">
|
||||
<div class="job-title">
|
||||
<span>{{ record.display_name }}</span>
|
||||
<a-tag :color="categoryColor(record.category)">{{ categoryLabel(record.category) }}</a-tag>
|
||||
</div>
|
||||
<code>{{ record.job_key }}</code>
|
||||
<div class="muted">{{ record.description || '—' }}</div>
|
||||
</div>
|
||||
</template>
|
||||
<template v-else-if="column.key === 'enabled'">
|
||||
<a-switch
|
||||
:checked="record.enabled"
|
||||
:loading="savingKey === record.job_key"
|
||||
checked-children="启用"
|
||||
un-checked-children="暂停"
|
||||
@change="(checked: boolean) => toggleJob(record, checked)"
|
||||
/>
|
||||
</template>
|
||||
<template v-else-if="column.key === 'schedule'">
|
||||
<div>{{ scheduleText(record) }}</div>
|
||||
<div class="muted">超时 {{ record.timeout_seconds }}s</div>
|
||||
</template>
|
||||
<template v-else-if="column.key === 'runtime'">
|
||||
<a-tag v-if="record.running_run" color="processing">运行中</a-tag>
|
||||
<a-tag v-else :color="runStatusColor(record.last_run?.status)">
|
||||
{{ runStatusLabel(record.last_run?.status) }}
|
||||
</a-tag>
|
||||
<div class="muted">
|
||||
{{ record.last_run ? formatDate(record.last_run.started_at) : '暂无运行记录' }}
|
||||
</div>
|
||||
</template>
|
||||
<template v-else-if="column.key === 'stats'">
|
||||
<div class="mono-line">{{ statsSummary(record.last_run?.stats) }}</div>
|
||||
<div v-if="record.pending_triggers" class="muted">待触发 {{ record.pending_triggers }}</div>
|
||||
</template>
|
||||
<template v-else-if="column.key === 'actions'">
|
||||
<div class="table-actions-row">
|
||||
<a-tooltip title="编辑配置">
|
||||
<a-button type="text" class="action-btn action-edit" @click="openEdit(record)">
|
||||
<SettingOutlined />
|
||||
</a-button>
|
||||
</a-tooltip>
|
||||
<a-tooltip title="立即运行">
|
||||
<a-popconfirm title="确认立即运行该调度任务？" ok-text="运行" cancel-text="取消" @confirm="triggerJob(record, 'run')">
|
||||
<a-button type="text" class="action-btn action-play" :loading="triggeringKey === `${record.job_key}:run`">
|
||||
<PlayCircleOutlined />
|
||||
</a-button>
|
||||
</a-popconfirm>
|
||||
</a-tooltip>
|
||||
<a-tooltip title="Dry-run">
|
||||
<a-button type="text" class="action-btn action-warn" :loading="triggeringKey === `${record.job_key}:dry-run`" @click="triggerJob(record, 'dry-run')">
|
||||
<ExperimentOutlined />
|
||||
</a-button>
|
||||
</a-tooltip>
|
||||
<a-tooltip title="运行记录">
|
||||
<a-button type="text" class="action-btn" @click="openRuns(record)">
|
||||
<HistoryOutlined />
|
||||
</a-button>
|
||||
</a-tooltip>
|
||||
</div>
|
||||
</template>
|
||||
</template>
|
||||
</a-table>
|
||||
</div>
|
||||
|
||||
<a-drawer v-model:open="editOpen" width="620" :title="editTitle" :destroy-on-close="true">
|
||||
<a-form v-if="editing" layout="vertical">
|
||||
<a-form-item label="启用状态">
|
||||
<a-switch v-model:checked="editForm.enabled" checked-children="启用" un-checked-children="暂停" />
|
||||
</a-form-item>
|
||||
<a-form-item label="调度类型">
|
||||
<a-segmented v-model:value="editForm.schedule_type" :options="scheduleTypeOptions" />
|
||||
</a-form-item>
|
||||
<a-form-item v-if="editForm.schedule_type === 'cron'" label="Cron 表达式">
|
||||
<a-input v-model:value="editForm.cron_expr" placeholder="例如 0 30 2 * * *" />
|
||||
</a-form-item>
|
||||
<a-form-item label="间隔秒数">
|
||||
<a-input-number v-model:value="editForm.interval_seconds" :min="1" :max="86400" style="width: 100%" />
|
||||
</a-form-item>
|
||||
<a-form-item label="超时秒数">
|
||||
<a-input-number v-model:value="editForm.timeout_seconds" :min="1" :max="3600" style="width: 100%" />
|
||||
</a-form-item>
|
||||
<a-form-item label="批大小">
|
||||
<a-input-number v-model:value="editForm.batch_size" :min="1" :max="100000" style="width: 100%" />
|
||||
</a-form-item>
|
||||
<a-form-item label="时区">
|
||||
<a-input v-model:value="editForm.timezone" />
|
||||
</a-form-item>
|
||||
<a-form-item label="高级配置 JSON">
|
||||
<a-textarea v-model:value="editConfigText" :rows="12" spellcheck="false" class="config-editor" />
|
||||
</a-form-item>
|
||||
<div class="drawer-actions">
|
||||
<a-button @click="editOpen = false">取消</a-button>
|
||||
<a-button type="primary" :loading="savingKey === editing.job_key" @click="saveEdit">保存</a-button>
|
||||
</div>
|
||||
</a-form>
|
||||
</a-drawer>
|
||||
|
||||
<a-drawer v-model:open="runsOpen" width="820" :title="runsTitle" :destroy-on-close="true">
|
||||
<a-table
|
||||
:columns="runColumns"
|
||||
:data-source="runRows"
|
||||
:loading="runsLoading"
|
||||
:pagination="runPagination"
|
||||
row-key="id"
|
||||
:scroll="{ x: 900 }"
|
||||
@change="onRunTableChange"
|
||||
>
|
||||
<template #bodyCell="{ column, record }">
|
||||
<template v-if="column.key === 'status'">
|
||||
<a-tag :color="runStatusColor(record.status)">{{ runStatusLabel(record.status) }}</a-tag>
|
||||
<div class="muted">{{ record.trigger_type }}</div>
|
||||
</template>
|
||||
<template v-else-if="column.key === 'time'">
|
||||
<div>{{ formatDate(record.started_at) }}</div>
|
||||
<div class="muted">{{ formatMs(record.duration_ms) }}</div>
|
||||
</template>
|
||||
<template v-else-if="column.key === 'stats'">
|
||||
<pre class="run-json">{{ formatJSON(record.stats) }}</pre>
|
||||
</template>
|
||||
<template v-else-if="column.key === 'error'">
|
||||
<span class="error-text">{{ record.error_message || '—' }}</span>
|
||||
</template>
|
||||
</template>
|
||||
</a-table>
|
||||
</a-drawer>
|
||||
|
||||
<a-drawer v-model:open="instancesOpen" width="720" title="Scheduler 实例" :destroy-on-close="true">
|
||||
<a-table :columns="instanceColumns" :data-source="instances" :loading="instancesLoading" row-key="instance_id" :pagination="false">
|
||||
<template #bodyCell="{ column, record }">
|
||||
<template v-if="column.key === 'instance'">
|
||||
<div class="mono-line">{{ record.instance_id }}</div>
|
||||
<div class="muted">{{ record.hostname }} · {{ record.process }}</div>
|
||||
</template>
|
||||
<template v-else-if="column.key === 'seen'">
|
||||
<div>{{ formatDate(record.last_seen_at) }}</div>
|
||||
<div class="muted">启动 {{ formatDate(record.started_at) }}</div>
|
||||
</template>
|
||||
</template>
|
||||
</a-table>
|
||||
</a-drawer>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import {
|
||||
ClusterOutlined,
|
||||
ExperimentOutlined,
|
||||
HistoryOutlined,
|
||||
PlayCircleOutlined,
|
||||
ReloadOutlined,
|
||||
SettingOutlined,
|
||||
} from '@ant-design/icons-vue'
|
||||
import type { TablePaginationConfig } from 'ant-design-vue'
|
||||
import { message } from 'ant-design-vue'
|
||||
import dayjs from 'dayjs'
|
||||
import { computed, onMounted, reactive, ref } from 'vue'
|
||||
|
||||
import { showOpsError } from '@/lib/errors'
|
||||
import { http } from '@/lib/http'
|
||||
|
||||
interface SchedulerRun {
|
||||
id: number
|
||||
job_key: string
|
||||
trigger_type: string
|
||||
status: string
|
||||
config_version: number
|
||||
stats: Record<string, unknown>
|
||||
error_message?: string | null
|
||||
started_at: string
|
||||
finished_at?: string | null
|
||||
duration_ms?: number | null
|
||||
}
|
||||
|
||||
interface SchedulerJob {
|
||||
job_key: string
|
||||
display_name: string
|
||||
category: string
|
||||
description?: string | null
|
||||
enabled: boolean
|
||||
schedule_type: string
|
||||
interval_seconds: number
|
||||
cron_expr?: string | null
|
||||
timezone: string
|
||||
timeout_seconds: number
|
||||
batch_size?: number | null
|
||||
max_concurrency: number
|
||||
config: Record<string, unknown>
|
||||
version: number
|
||||
updated_at: string
|
||||
last_run?: SchedulerRun | null
|
||||
running_run?: SchedulerRun | null
|
||||
pending_triggers: number
|
||||
}
|
||||
|
||||
interface SchedulerInstance {
|
||||
instance_id: string
|
||||
hostname: string
|
||||
process: string
|
||||
build_version?: string | null
|
||||
started_at: string
|
||||
last_seen_at: string
|
||||
metadata: Record<string, unknown>
|
||||
}
|
||||
|
||||
interface ListResult<T> {
|
||||
items: T[]
|
||||
total: number
|
||||
page: number
|
||||
size: number
|
||||
}
|
||||
|
||||
const categoryOptions = [
|
||||
{ label: '监控', value: 'monitoring' },
|
||||
{ label: '内容', value: 'content' },
|
||||
{ label: '知识库', value: 'knowledge' },
|
||||
{ label: '生成', value: 'generation' },
|
||||
]
|
||||
|
||||
const enabledOptions = [
|
||||
{ label: '启用', value: 'true' },
|
||||
{ label: '暂停', value: 'false' },
|
||||
]
|
||||
|
||||
const scheduleTypeOptions = [
|
||||
{ label: '间隔', value: 'interval' },
|
||||
{ label: 'Cron', value: 'cron' },
|
||||
{ label: '手动', value: 'manual' },
|
||||
]
|
||||
|
||||
const columns = [
|
||||
{ title: '任务', key: 'job', width: 360, fixed: 'left' },
|
||||
{ title: '开关', key: 'enabled', width: 120 },
|
||||
{ title: '调度', key: 'schedule', width: 180 },
|
||||
{ title: '运行态', key: 'runtime', width: 190 },
|
||||
{ title: '最近结果', key: 'stats', width: 280 },
|
||||
{ title: '操作', key: 'actions', width: 172, fixed: 'right' },
|
||||
]
|
||||
|
||||
const runColumns = [
|
||||
{ title: '状态', key: 'status', width: 130 },
|
||||
{ title: '时间', key: 'time', width: 190 },
|
||||
{ title: '统计', key: 'stats', width: 360 },
|
||||
{ title: '错误', key: 'error', width: 220 },
|
||||
]
|
||||
|
||||
const instanceColumns = [
|
||||
{ title: '实例', key: 'instance' },
|
||||
{ title: '心跳', key: 'seen', width: 220 },
|
||||
]
|
||||
|
||||
const filter = reactive({
|
||||
keyword: '',
|
||||
category: undefined as string | undefined,
|
||||
enabled: undefined as string | undefined,
|
||||
})
|
||||
const rows = ref<SchedulerJob[]>([])
|
||||
const total = ref(0)
|
||||
const page = ref(1)
|
||||
const size = ref(50)
|
||||
const loading = ref(false)
|
||||
const savingKey = ref('')
|
||||
const triggeringKey = ref('')
|
||||
const editOpen = ref(false)
|
||||
const editing = ref<SchedulerJob | null>(null)
|
||||
const editConfigText = ref('')
|
||||
const editForm = reactive({
|
||||
enabled: true,
|
||||
schedule_type: 'interval',
|
||||
interval_seconds: 300,
|
||||
cron_expr: '',
|
||||
timeout_seconds: 30,
|
||||
batch_size: null as number | null,
|
||||
timezone: 'Asia/Shanghai',
|
||||
})
|
||||
const runsOpen = ref(false)
|
||||
const runsJob = ref<SchedulerJob | null>(null)
|
||||
const runRows = ref<SchedulerRun[]>([])
|
||||
const runsLoading = ref(false)
|
||||
const runPage = ref(1)
|
||||
const runSize = ref(20)
|
||||
const runTotal = ref(0)
|
||||
const instancesOpen = ref(false)
|
||||
const instances = ref<SchedulerInstance[]>([])
|
||||
const instancesLoading = ref(false)
|
||||
|
||||
const pagination = computed<TablePaginationConfig>(() => ({
|
||||
current: page.value,
|
||||
pageSize: size.value,
|
||||
total: total.value,
|
||||
showSizeChanger: true,
|
||||
pageSizeOptions: ['20', '50', '100', '200'],
|
||||
}))
|
||||
|
||||
const runPagination = computed<TablePaginationConfig>(() => ({
|
||||
current: runPage.value,
|
||||
pageSize: runSize.value,
|
||||
total: runTotal.value,
|
||||
showSizeChanger: true,
|
||||
}))
|
||||
|
||||
const enabledCount = computed(() => rows.value.filter((row) => row.enabled).length)
|
||||
const runningCount = computed(() => rows.value.filter((row) => row.running_run).length)
|
||||
const failedCount = computed(() => rows.value.filter((row) => row.last_run?.status === 'failed').length)
|
||||
const editTitle = computed(() => (editing.value ? `编辑 · ${editing.value.display_name}` : '编辑调度任务'))
|
||||
const runsTitle = computed(() => (runsJob.value ? `运行记录 · ${runsJob.value.display_name}` : '运行记录'))
|
||||
|
||||
async function reload() {
|
||||
loading.value = true
|
||||
try {
|
||||
const params: Record<string, unknown> = { page: page.value, size: size.value }
|
||||
if (filter.keyword.trim()) params.keyword = filter.keyword.trim()
|
||||
if (filter.category) params.category = filter.category
|
||||
if (filter.enabled) params.enabled = filter.enabled
|
||||
const result = await http.get<ListResult<SchedulerJob>>('/scheduler/jobs', params)
|
||||
rows.value = result.items
|
||||
total.value = result.total
|
||||
} catch (error) {
|
||||
showOpsError(error)
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function resetAndReload() {
|
||||
page.value = 1
|
||||
void reload()
|
||||
}
|
||||
|
||||
function onTableChange(pag: TablePaginationConfig) {
|
||||
page.value = pag.current ?? 1
|
||||
size.value = pag.pageSize ?? 50
|
||||
void reload()
|
||||
}
|
||||
|
||||
async function toggleJob(row: SchedulerJob, enabled: boolean) {
|
||||
savingKey.value = row.job_key
|
||||
try {
|
||||
const action = enabled ? 'enable' : 'disable'
|
||||
const updated = await http.post<SchedulerJob>(`/scheduler/jobs/${row.job_key}/${action}`)
|
||||
patchJob(updated)
|
||||
message.success(enabled ? '已启用' : '已暂停')
|
||||
} catch (error) {
|
||||
showOpsError(error)
|
||||
} finally {
|
||||
savingKey.value = ''
|
||||
}
|
||||
}
|
||||
|
||||
function openEdit(row: SchedulerJob) {
|
||||
editing.value = row
|
||||
editForm.enabled = row.enabled
|
||||
editForm.schedule_type = row.schedule_type
|
||||
editForm.interval_seconds = row.interval_seconds
|
||||
editForm.cron_expr = row.cron_expr ?? ''
|
||||
editForm.timeout_seconds = row.timeout_seconds
|
||||
editForm.batch_size = row.batch_size ?? null
|
||||
editForm.timezone = row.timezone
|
||||
editConfigText.value = JSON.stringify(row.config ?? {}, null, 2)
|
||||
editOpen.value = true
|
||||
}
|
||||
|
||||
async function saveEdit() {
|
||||
if (!editing.value) return
|
||||
let config: Record<string, unknown>
|
||||
try {
|
||||
config = JSON.parse(editConfigText.value || '{}') as Record<string, unknown>
|
||||
} catch {
|
||||
message.warning('高级配置必须是合法 JSON')
|
||||
return
|
||||
}
|
||||
savingKey.value = editing.value.job_key
|
||||
try {
|
||||
const updated = await http.patch<SchedulerJob>(`/scheduler/jobs/${editing.value.job_key}`, {
|
||||
enabled: editForm.enabled,
|
||||
schedule_type: editForm.schedule_type,
|
||||
interval_seconds: editForm.interval_seconds,
|
||||
cron_expr: editForm.cron_expr,
|
||||
timeout_seconds: editForm.timeout_seconds,
|
||||
batch_size: editForm.batch_size,
|
||||
timezone: editForm.timezone,
|
||||
config,
|
||||
})
|
||||
patchJob(updated)
|
||||
editOpen.value = false
|
||||
message.success('已保存')
|
||||
} catch (error) {
|
||||
showOpsError(error)
|
||||
} finally {
|
||||
savingKey.value = ''
|
||||
}
|
||||
}
|
||||
|
||||
async function triggerJob(row: SchedulerJob, action: 'run' | 'dry-run') {
|
||||
triggeringKey.value = `${row.job_key}:${action}`
|
||||
try {
|
||||
await http.post(`/scheduler/jobs/${row.job_key}/${action}`, {
|
||||
reason: action === 'dry-run' ? 'ops console dry-run' : 'ops console manual run',
|
||||
})
|
||||
message.success(action === 'dry-run' ? '已提交 dry-run' : '已提交运行')
|
||||
await reload()
|
||||
} catch (error) {
|
||||
showOpsError(error)
|
||||
} finally {
|
||||
triggeringKey.value = ''
|
||||
}
|
||||
}
|
||||
|
||||
function openRuns(row: SchedulerJob) {
|
||||
runsJob.value = row
|
||||
runPage.value = 1
|
||||
runsOpen.value = true
|
||||
void loadRuns()
|
||||
}
|
||||
|
||||
async function loadRuns() {
|
||||
if (!runsJob.value) return
|
||||
runsLoading.value = true
|
||||
try {
|
||||
const result = await http.get<ListResult<SchedulerRun>>(`/scheduler/jobs/${runsJob.value.job_key}/runs`, {
|
||||
page: runPage.value,
|
||||
size: runSize.value,
|
||||
})
|
||||
runRows.value = result.items
|
||||
runTotal.value = result.total
|
||||
} catch (error) {
|
||||
showOpsError(error)
|
||||
} finally {
|
||||
runsLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function onRunTableChange(pag: TablePaginationConfig) {
|
||||
runPage.value = pag.current ?? 1
|
||||
runSize.value = pag.pageSize ?? 20
|
||||
void loadRuns()
|
||||
}
|
||||
|
||||
async function loadInstances() {
|
||||
instancesOpen.value = true
|
||||
instancesLoading.value = true
|
||||
try {
|
||||
const result = await http.get<{ items: SchedulerInstance[] }>('/scheduler/instances')
|
||||
instances.value = result.items
|
||||
} catch (error) {
|
||||
showOpsError(error)
|
||||
} finally {
|
||||
instancesLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function patchJob(item: SchedulerJob) {
|
||||
const index = rows.value.findIndex((row) => row.job_key === item.job_key)
|
||||
if (index >= 0) rows.value[index] = { ...rows.value[index], ...item }
|
||||
}
|
||||
|
||||
function scheduleText(row: SchedulerJob): string {
|
||||
const runAt = row.config?.run_at_local
|
||||
if (typeof runAt === 'string' && runAt) return `每日 ${runAt}`
|
||||
if (row.schedule_type === 'manual') return '手动'
|
||||
if (row.schedule_type === 'cron') return row.cron_expr || 'Cron'
|
||||
return `每 ${formatSeconds(row.interval_seconds)}`
|
||||
}
|
||||
|
||||
function categoryLabel(value: string): string {
|
||||
return categoryOptions.find((item) => item.value === value)?.label ?? value
|
||||
}
|
||||
|
||||
function categoryColor(value: string): string {
|
||||
if (value === 'monitoring') return 'blue'
|
||||
if (value === 'generation') return 'purple'
|
||||
if (value === 'knowledge') return 'green'
|
||||
if (value === 'content') return 'geekblue'
|
||||
return 'default'
|
||||
}
|
||||
|
||||
function runStatusLabel(value?: string | null): string {
|
||||
if (!value) return '未运行'
|
||||
const labels: Record<string, string> = {
|
||||
running: '运行中',
|
||||
success: '成功',
|
||||
failed: '失败',
|
||||
skipped: '跳过',
|
||||
}
|
||||
return labels[value] ?? value
|
||||
}
|
||||
|
||||
function runStatusColor(value?: string | null): string {
|
||||
if (value === 'success') return 'green'
|
||||
if (value === 'failed') return 'red'
|
||||
if (value === 'running') return 'processing'
|
||||
if (value === 'skipped') return 'default'
|
||||
return 'default'
|
||||
}
|
||||
|
||||
function statsSummary(stats?: Record<string, unknown> | null): string {
|
||||
if (!stats) return '—'
|
||||
const pairs = Object.entries(stats)
|
||||
.filter(([key]) => ['deleted_count', 'candidate_count', 'created_task_count', 'claimed_count', 'expired_task_count', 'anomaly_count'].includes(key))
|
||||
.map(([key, value]) => `${key}=${String(value)}`)
|
||||
return pairs.length ? pairs.join(' · ') : '—'
|
||||
}
|
||||
|
||||
function formatDate(value?: string | null): string {
|
||||
if (!value) return '—'
|
||||
return dayjs(value).format('YYYY-MM-DD HH:mm:ss')
|
||||
}
|
||||
|
||||
function formatSeconds(seconds: number): string {
|
||||
if (seconds < 60) return `${seconds}s`
|
||||
if (seconds < 3600) return `${Math.floor(seconds / 60)}m`
|
||||
return `${Math.floor(seconds / 3600)}h`
|
||||
}
|
||||
|
||||
function formatMs(value?: number | null): string {
|
||||
if (value == null) return '—'
|
||||
if (value < 1000) return `${value}ms`
|
||||
return `${Math.round(value / 100) / 10}s`
|
||||
}
|
||||
|
||||
function formatJSON(value: unknown): string {
|
||||
try {
|
||||
return JSON.stringify(value ?? {}, null, 2)
|
||||
} catch {
|
||||
return String(value)
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
void reload()
|
||||
})
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.scheduler-page {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 18px;
|
||||
}
|
||||
|
||||
.scheduler-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
align-items: flex-start;
|
||||
}
|
||||
|
||||
.scheduler-header__meta,
|
||||
.muted {
|
||||
color: #64748b;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.scheduler-summary {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.summary-tile {
|
||||
border: 1px solid #dbe4ef;
|
||||
border-radius: 6px;
|
||||
background: #fff;
|
||||
padding: 16px 18px;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.summary-tile strong {
|
||||
font-size: 26px;
|
||||
line-height: 1;
|
||||
color: #0f172a;
|
||||
}
|
||||
|
||||
.summary-tile--green strong {
|
||||
color: #15803d;
|
||||
}
|
||||
|
||||
.summary-tile--gold strong {
|
||||
color: #b45309;
|
||||
}
|
||||
|
||||
.summary-tile--red strong {
|
||||
color: #b91c1c;
|
||||
}
|
||||
|
||||
.scheduler-toolbar {
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.job-cell {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.job-title {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
align-items: center;
|
||||
font-weight: 700;
|
||||
color: #0f172a;
|
||||
}
|
||||
|
||||
.mono-line,
|
||||
.job-cell code {
|
||||
font-family: 'SFMono-Regular', Consolas, monospace;
|
||||
font-size: 12px;
|
||||
color: #334155;
|
||||
}
|
||||
|
||||
.table-actions-row {
|
||||
display: flex;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.config-editor,
|
||||
.run-json {
|
||||
font-family: 'SFMono-Regular', Consolas, monospace;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.run-json {
|
||||
max-height: 140px;
|
||||
overflow: auto;
|
||||
margin: 0;
|
||||
background: #f8fafc;
|
||||
border: 1px solid #e2e8f0;
|
||||
border-radius: 6px;
|
||||
padding: 8px;
|
||||
}
|
||||
|
||||
.drawer-actions {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
gap: 8px;
|
||||
margin-top: 18px;
|
||||
}
|
||||
|
||||
.error-text {
|
||||
color: #b91c1c;
|
||||
}
|
||||
|
||||
@media (max-width: 900px) {
|
||||
.scheduler-header {
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.scheduler-summary {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -83,6 +83,7 @@ func main() {
|
||||
kolSubscriptionsRepo := repository.NewKolSubscriptionRepository(pool)
|
||||
auditsRepo := repository.NewAuditRepository(pool)
|
||||
siteDomainMappingsRepo := repository.NewSiteDomainMappingRepository(monitoringPool)
|
||||
schedulerRepo := repository.NewSchedulerRepository(pool)
|
||||
|
||||
ipRegionResolver, err := ipregion.NewResolver(cfg.IPRegion.V4XDBPath, cfg.IPRegion.V6XDBPath, logger)
|
||||
if err != nil {
|
||||
@@ -118,6 +119,7 @@ func main() {
|
||||
kolSubscriptionSvc := app.NewKolSubscriptionService(kolSubscriptionsRepo, auditSvc).WithCache(appCache)
|
||||
siteDomainMappingSvc := app.NewSiteDomainMappingService(siteDomainMappingsRepo, auditSvc)
|
||||
complianceSvc := app.NewComplianceService(pool, auditSvc, logger)
|
||||
schedulerSvc := app.NewSchedulerService(schedulerRepo, auditSvc)
|
||||
|
||||
if err := app.SeedDefaultAdmin(ctx, accountsRepo, app.DefaultAdminSeed{
|
||||
Username: cfg.DefaultAdmin.Username,
|
||||
@@ -165,6 +167,7 @@ func main() {
|
||||
Audits: auditSvc,
|
||||
SiteDomains: siteDomainMappingSvc,
|
||||
Compliance: complianceSvc,
|
||||
Scheduler: schedulerSvc,
|
||||
})
|
||||
|
||||
addr := fmt.Sprintf(":%d", cfg.Server.Port)
|
||||
|
||||
@@ -95,14 +95,59 @@ func main() {
|
||||
generationStateCheckWorker := generateworker.NewGenerationTaskStateCheckWorker(app.DB, app.Logger, app.Config().Generation).
|
||||
WithConfigProvider(app.ConfigStore).
|
||||
WithCache(app.Cache)
|
||||
monitoringRetentionWorker := internalscheduler.NewMonitoringRetentionWorker(app.MonitoringDB, app.Logger)
|
||||
|
||||
startSchedulerWorker(ctx, &workerWG, "schedule_dispatch", app.Logger.Sugar(), scheduleDispatchWorker.Run)
|
||||
startSchedulerWorker(ctx, &workerWG, "monitoring_daily_task", app.Logger.Sugar(), monitoringDailyTaskWorker.Run)
|
||||
startSchedulerWorker(ctx, &workerWG, "monitoring_result_recovery", app.Logger.Sugar(), monitoringResultRecoveryWorker.Run)
|
||||
startSchedulerWorker(ctx, &workerWG, "monitoring_lease_recovery", app.Logger.Sugar(), monitoringLeaseRecoveryWorker.Run)
|
||||
startSchedulerWorker(ctx, &workerWG, "monitoring_received_inspection", app.Logger.Sugar(), monitoringReceivedInspectionWorker.Run)
|
||||
startSchedulerWorker(ctx, &workerWG, "knowledge_deleted_cleanup", app.Logger.Sugar(), knowledgeDeletedCleanupWorker.Run)
|
||||
startSchedulerWorker(ctx, &workerWG, "generation_state_check", app.Logger.Sugar(), generationStateCheckWorker.Run)
|
||||
controlPlane := internalscheduler.NewControlPlane(app.DB, app.Logger, []internalscheduler.ControlledJob{
|
||||
{
|
||||
Key: "schedule_dispatch",
|
||||
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
|
||||
return scheduleDispatchWorker.RunOnce(runCtx)
|
||||
},
|
||||
},
|
||||
{
|
||||
Key: "monitoring_daily_task",
|
||||
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
|
||||
return monitoringDailyTaskWorker.RunOnce(runCtx)
|
||||
},
|
||||
},
|
||||
{
|
||||
Key: "monitoring_result_recovery",
|
||||
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
|
||||
return monitoringResultRecoveryWorker.RunOnce(runCtx)
|
||||
},
|
||||
},
|
||||
{
|
||||
Key: "monitoring_lease_recovery",
|
||||
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
|
||||
return monitoringLeaseRecoveryWorker.RunOnce(runCtx)
|
||||
},
|
||||
},
|
||||
{
|
||||
Key: "monitoring_received_inspection",
|
||||
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
|
||||
return monitoringReceivedInspectionWorker.RunOnce(runCtx)
|
||||
},
|
||||
},
|
||||
{
|
||||
Key: "knowledge_deleted_cleanup",
|
||||
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
|
||||
return knowledgeDeletedCleanupWorker.RunOnce(runCtx)
|
||||
},
|
||||
},
|
||||
{
|
||||
Key: "generation_state_check",
|
||||
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
|
||||
return generationStateCheckWorker.RunOnce(runCtx)
|
||||
},
|
||||
},
|
||||
{
|
||||
Key: "monitoring_retention_cleanup",
|
||||
Run: func(runCtx context.Context, jobCtx internalscheduler.JobRunContext) (map[string]any, error) {
|
||||
return monitoringRetentionWorker.RunOnce(runCtx, jobCtx)
|
||||
},
|
||||
},
|
||||
})
|
||||
startSchedulerWorker(ctx, &workerWG, "control_plane", app.Logger.Sugar(), controlPlane.Run)
|
||||
|
||||
var metricsServer *http.Server
|
||||
if app.Config().Scheduler.HTTPPort > 0 {
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/ops/domain"
|
||||
"github.com/geo-platform/tenant-api/internal/ops/repository"
|
||||
"github.com/geo-platform/tenant-api/internal/shared/response"
|
||||
)
|
||||
|
||||
const (
|
||||
ActionSchedulerJobUpdate = "scheduler_job.update"
|
||||
ActionSchedulerJobEnable = "scheduler_job.enable"
|
||||
ActionSchedulerJobDisable = "scheduler_job.disable"
|
||||
ActionSchedulerJobRun = "scheduler_job.run"
|
||||
ActionSchedulerJobDryRun = "scheduler_job.dry_run"
|
||||
)
|
||||
|
||||
type SchedulerService struct {
|
||||
repo *repository.SchedulerRepository
|
||||
audits *AuditService
|
||||
}
|
||||
|
||||
func NewSchedulerService(repo *repository.SchedulerRepository, audits *AuditService) *SchedulerService {
|
||||
return &SchedulerService{repo: repo, audits: audits}
|
||||
}
|
||||
|
||||
type SchedulerJobListInput struct {
|
||||
Category string
|
||||
Keyword string
|
||||
Enabled *bool
|
||||
Page int
|
||||
Size int
|
||||
}
|
||||
|
||||
type SchedulerJobListResult struct {
|
||||
Items []domain.SchedulerJob `json:"items"`
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
Size int `json:"size"`
|
||||
}
|
||||
|
||||
type SchedulerRunListResult struct {
|
||||
Items []domain.SchedulerRun `json:"items"`
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
Size int `json:"size"`
|
||||
}
|
||||
|
||||
type SchedulerJobUpdateInput struct {
|
||||
Enabled *bool `json:"enabled"`
|
||||
ScheduleType *string `json:"schedule_type"`
|
||||
IntervalSeconds *int `json:"interval_seconds"`
|
||||
CronExpr *string `json:"cron_expr"`
|
||||
Timezone *string `json:"timezone"`
|
||||
TimeoutSeconds *int `json:"timeout_seconds"`
|
||||
BatchSize *int `json:"batch_size"`
|
||||
MaxConcurrency *int `json:"max_concurrency"`
|
||||
Config map[string]any `json:"config"`
|
||||
}
|
||||
|
||||
type SchedulerTriggerInput struct {
|
||||
Reason string `json:"reason"`
|
||||
ConfigOverride map[string]any `json:"config_override"`
|
||||
}
|
||||
|
||||
func (s *SchedulerService) ListJobs(ctx context.Context, in SchedulerJobListInput) (*SchedulerJobListResult, error) {
|
||||
page := in.Page
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
size := in.Size
|
||||
if size <= 0 || size > 200 {
|
||||
size = 50
|
||||
}
|
||||
items, total, err := s.repo.ListJobs(ctx, repository.SchedulerJobFilter{
|
||||
Category: strings.TrimSpace(in.Category),
|
||||
Keyword: strings.TrimSpace(in.Keyword),
|
||||
Enabled: in.Enabled,
|
||||
Limit: size,
|
||||
Offset: (page - 1) * size,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(53001, "scheduler_jobs_query_failed", "failed to list scheduler jobs")
|
||||
}
|
||||
return &SchedulerJobListResult{Items: items, Total: total, Page: page, Size: size}, nil
|
||||
}
|
||||
|
||||
func (s *SchedulerService) GetJob(ctx context.Context, key string) (*domain.SchedulerJob, error) {
|
||||
item, err := s.repo.GetJob(ctx, normalizeSchedulerJobKey(key))
|
||||
if err != nil {
|
||||
return nil, mapSchedulerJobError(err)
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (s *SchedulerService) UpdateJob(ctx context.Context, actor *Actor, key string, in SchedulerJobUpdateInput) (*domain.SchedulerJob, error) {
|
||||
normalized, err := normalizeSchedulerJobUpdate(in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
updatedBy := actorID(actor)
|
||||
item, err := s.repo.UpdateJob(ctx, normalizeSchedulerJobKey(key), repository.SchedulerJobUpdate{
|
||||
Enabled: normalized.Enabled,
|
||||
ScheduleType: normalized.ScheduleType,
|
||||
IntervalSeconds: normalized.IntervalSeconds,
|
||||
CronExpr: normalized.CronExpr,
|
||||
Timezone: normalized.Timezone,
|
||||
TimeoutSeconds: normalized.TimeoutSeconds,
|
||||
BatchSize: normalized.BatchSize,
|
||||
MaxConcurrency: normalized.MaxConcurrency,
|
||||
Config: normalized.Config,
|
||||
UpdatedBy: updatedBy,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, mapSchedulerJobError(err)
|
||||
}
|
||||
s.auditScheduler(ctx, actor, ActionSchedulerJobUpdate, item.JobKey, map[string]any{
|
||||
"version": item.Version,
|
||||
})
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (s *SchedulerService) SetEnabled(ctx context.Context, actor *Actor, key string, enabled bool) (*domain.SchedulerJob, error) {
|
||||
item, err := s.repo.SetJobEnabled(ctx, normalizeSchedulerJobKey(key), enabled, actorID(actor))
|
||||
if err != nil {
|
||||
return nil, mapSchedulerJobError(err)
|
||||
}
|
||||
action := ActionSchedulerJobDisable
|
||||
if enabled {
|
||||
action = ActionSchedulerJobEnable
|
||||
}
|
||||
s.auditScheduler(ctx, actor, action, item.JobKey, map[string]any{"version": item.Version})
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (s *SchedulerService) Trigger(ctx context.Context, actor *Actor, key string, triggerType string, in SchedulerTriggerInput) (*domain.SchedulerTrigger, error) {
|
||||
key = normalizeSchedulerJobKey(key)
|
||||
if _, err := s.repo.GetJobForRuntime(ctx, key); err != nil {
|
||||
return nil, mapSchedulerJobError(err)
|
||||
}
|
||||
triggerType = strings.TrimSpace(triggerType)
|
||||
if triggerType != domain.SchedulerTriggerManual && triggerType != domain.SchedulerTriggerDryRun {
|
||||
return nil, response.ErrBadRequest(40083, "invalid_trigger_type", "trigger_type 不合法")
|
||||
}
|
||||
trigger, err := s.repo.CreateTrigger(ctx, repository.SchedulerTriggerCreate{
|
||||
JobKey: key,
|
||||
TriggerType: triggerType,
|
||||
RequestedBy: actorID(actor),
|
||||
Reason: strings.TrimSpace(in.Reason),
|
||||
ConfigOverride: sanitizeSchedulerConfig(in.ConfigOverride),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(53004, "scheduler_trigger_create_failed", "failed to create scheduler trigger")
|
||||
}
|
||||
action := ActionSchedulerJobRun
|
||||
if triggerType == domain.SchedulerTriggerDryRun {
|
||||
action = ActionSchedulerJobDryRun
|
||||
}
|
||||
s.auditScheduler(ctx, actor, action, key, map[string]any{
|
||||
"trigger_id": trigger.ID,
|
||||
"reason": strings.TrimSpace(in.Reason),
|
||||
})
|
||||
return trigger, nil
|
||||
}
|
||||
|
||||
func (s *SchedulerService) ListRuns(ctx context.Context, key, status string, page, size int) (*SchedulerRunListResult, error) {
|
||||
page = maxInt(page, 1)
|
||||
if size <= 0 || size > 200 {
|
||||
size = 50
|
||||
}
|
||||
items, total, err := s.repo.ListRuns(ctx, repository.SchedulerRunFilter{
|
||||
JobKey: normalizeSchedulerJobKey(key),
|
||||
Status: strings.TrimSpace(status),
|
||||
Limit: size,
|
||||
Offset: (page - 1) * size,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(53005, "scheduler_runs_query_failed", "failed to list scheduler runs")
|
||||
}
|
||||
return &SchedulerRunListResult{Items: items, Total: total, Page: page, Size: size}, nil
|
||||
}
|
||||
|
||||
func (s *SchedulerService) ListInstances(ctx context.Context) ([]domain.SchedulerInstance, error) {
|
||||
items, err := s.repo.ListInstances(ctx)
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(53006, "scheduler_instances_query_failed", "failed to list scheduler instances")
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func normalizeSchedulerJobUpdate(in SchedulerJobUpdateInput) (SchedulerJobUpdateInput, error) {
|
||||
out := in
|
||||
if out.ScheduleType != nil {
|
||||
value := strings.TrimSpace(*out.ScheduleType)
|
||||
if value != "interval" && value != "cron" && value != "manual" {
|
||||
return out, response.ErrBadRequest(40080, "invalid_schedule_type", "schedule_type 不合法")
|
||||
}
|
||||
out.ScheduleType = &value
|
||||
}
|
||||
if out.IntervalSeconds != nil && *out.IntervalSeconds <= 0 {
|
||||
return out, response.ErrBadRequest(40081, "invalid_interval_seconds", "interval_seconds 必须大于 0")
|
||||
}
|
||||
if out.TimeoutSeconds != nil && *out.TimeoutSeconds <= 0 {
|
||||
return out, response.ErrBadRequest(40082, "invalid_timeout_seconds", "timeout_seconds 必须大于 0")
|
||||
}
|
||||
if out.BatchSize != nil && *out.BatchSize <= 0 {
|
||||
return out, response.ErrBadRequest(40084, "invalid_batch_size", "batch_size 必须大于 0")
|
||||
}
|
||||
if out.MaxConcurrency != nil && *out.MaxConcurrency <= 0 {
|
||||
return out, response.ErrBadRequest(40085, "invalid_max_concurrency", "max_concurrency 必须大于 0")
|
||||
}
|
||||
if out.Timezone != nil {
|
||||
value := strings.TrimSpace(*out.Timezone)
|
||||
if value == "" {
|
||||
value = "Asia/Shanghai"
|
||||
}
|
||||
if _, err := time.LoadLocation(value); err != nil {
|
||||
return out, response.ErrBadRequest(40086, "invalid_timezone", "timezone 不合法")
|
||||
}
|
||||
out.Timezone = &value
|
||||
}
|
||||
if out.CronExpr != nil {
|
||||
value := strings.TrimSpace(*out.CronExpr)
|
||||
out.CronExpr = &value
|
||||
}
|
||||
out.Config = sanitizeSchedulerConfig(out.Config)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func sanitizeSchedulerConfig(config map[string]any) map[string]any {
|
||||
if config == nil {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(config))
|
||||
for key, value := range config {
|
||||
trimmed := strings.TrimSpace(key)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
out[trimmed] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func normalizeSchedulerJobKey(key string) string {
|
||||
return strings.TrimSpace(key)
|
||||
}
|
||||
|
||||
func mapSchedulerJobError(err error) error {
|
||||
if errors.Is(err, repository.ErrSchedulerJobNotFound) {
|
||||
return response.ErrNotFound(40480, "scheduler_job_not_found", "调度任务不存在")
|
||||
}
|
||||
return response.ErrInternal(53002, "scheduler_job_query_failed", "failed to load scheduler job")
|
||||
}
|
||||
|
||||
func actorID(actor *Actor) *int64 {
|
||||
if actor == nil || actor.OperatorID <= 0 {
|
||||
return nil
|
||||
}
|
||||
id := actor.OperatorID
|
||||
return &id
|
||||
}
|
||||
|
||||
func (s *SchedulerService) auditScheduler(ctx context.Context, actor *Actor, action, key string, metadata map[string]any) {
|
||||
if actor == nil || s.audits == nil {
|
||||
return
|
||||
}
|
||||
id := actor.OperatorID
|
||||
event := domain.AuditEvent{
|
||||
OperatorID: &id,
|
||||
OperatorName: actor.DisplayName,
|
||||
Action: action,
|
||||
TargetType: "scheduler_job",
|
||||
TargetID: key,
|
||||
Metadata: metadata,
|
||||
IP: actor.IP,
|
||||
UserAgent: actor.UserAgent,
|
||||
RequestID: actor.RequestID,
|
||||
}
|
||||
_ = s.audits.Append(ctx, event)
|
||||
}
|
||||
|
||||
func maxInt(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func SchedulerJobConfigAsString(config map[string]any, key string) string {
|
||||
value, ok := config[key]
|
||||
if !ok || value == nil {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprint(value)
|
||||
}
|
||||
@@ -37,19 +37,19 @@ type SchedulerJob struct {
|
||||
}
|
||||
|
||||
type SchedulerRun struct {
|
||||
ID int64 `json:"id"`
|
||||
JobKey string `json:"job_key"`
|
||||
TriggerID *int64 `json:"trigger_id,omitempty"`
|
||||
SchedulerInstanceID *string `json:"scheduler_instance_id,omitempty"`
|
||||
TriggerType string `json:"trigger_type"`
|
||||
Status string `json:"status"`
|
||||
ConfigVersion int `json:"config_version"`
|
||||
ConfigSnapshot map[string]any `json:"config_snapshot"`
|
||||
Stats map[string]any `json:"stats"`
|
||||
ErrorMessage *string `json:"error_message,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
DurationMilliseconds *int64 `json:"duration_ms,omitempty"`
|
||||
ID int64 `json:"id"`
|
||||
JobKey string `json:"job_key"`
|
||||
TriggerID *int64 `json:"trigger_id,omitempty"`
|
||||
SchedulerInstanceID *string `json:"scheduler_instance_id,omitempty"`
|
||||
TriggerType string `json:"trigger_type"`
|
||||
Status string `json:"status"`
|
||||
ConfigVersion int `json:"config_version"`
|
||||
ConfigSnapshot map[string]any `json:"config_snapshot"`
|
||||
Stats map[string]any `json:"stats"`
|
||||
ErrorMessage *string `json:"error_message,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
DurationMilliseconds *int64 `json:"duration_ms,omitempty"`
|
||||
}
|
||||
|
||||
type SchedulerInstance struct {
|
||||
|
||||
@@ -0,0 +1,603 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/ops/domain"
|
||||
)
|
||||
|
||||
var ErrSchedulerJobNotFound = errors.New("scheduler job not found")
|
||||
|
||||
type SchedulerRepository struct {
|
||||
pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func NewSchedulerRepository(pool *pgxpool.Pool) *SchedulerRepository {
|
||||
return &SchedulerRepository{pool: pool}
|
||||
}
|
||||
|
||||
type SchedulerJobFilter struct {
|
||||
Category string
|
||||
Keyword string
|
||||
Enabled *bool
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type SchedulerJobUpdate struct {
|
||||
Enabled *bool
|
||||
ScheduleType *string
|
||||
IntervalSeconds *int
|
||||
CronExpr *string
|
||||
Timezone *string
|
||||
TimeoutSeconds *int
|
||||
BatchSize *int
|
||||
MaxConcurrency *int
|
||||
Config map[string]any
|
||||
UpdatedBy *int64
|
||||
}
|
||||
|
||||
type SchedulerRunFilter struct {
|
||||
JobKey string
|
||||
Status string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type SchedulerRunStart struct {
|
||||
JobKey string
|
||||
TriggerID *int64
|
||||
SchedulerInstanceID string
|
||||
TriggerType string
|
||||
ConfigVersion int
|
||||
ConfigSnapshot map[string]any
|
||||
}
|
||||
|
||||
type SchedulerRunFinish struct {
|
||||
RunID int64
|
||||
Status string
|
||||
Stats map[string]any
|
||||
ErrorMessage *string
|
||||
}
|
||||
|
||||
type SchedulerTriggerCreate struct {
|
||||
JobKey string
|
||||
TriggerType string
|
||||
RequestedBy *int64
|
||||
Reason string
|
||||
ConfigOverride map[string]any
|
||||
}
|
||||
|
||||
type SchedulerTriggerClaim struct {
|
||||
ID int64
|
||||
JobKey string
|
||||
TriggerType string
|
||||
ConfigOverride map[string]any
|
||||
}
|
||||
|
||||
const schedulerJobSelect = `
|
||||
SELECT
|
||||
j.job_key,
|
||||
j.display_name,
|
||||
j.category,
|
||||
j.description,
|
||||
j.enabled,
|
||||
j.schedule_type,
|
||||
j.interval_seconds,
|
||||
j.cron_expr,
|
||||
j.timezone,
|
||||
j.timeout_seconds,
|
||||
j.batch_size,
|
||||
j.max_concurrency,
|
||||
j.config,
|
||||
j.version,
|
||||
j.updated_by,
|
||||
j.updated_at,
|
||||
j.created_at,
|
||||
COALESCE(pending.pending_count, 0)
|
||||
FROM ops.scheduler_jobs j
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT COUNT(*)::int AS pending_count
|
||||
FROM ops.scheduler_job_triggers t
|
||||
WHERE t.job_key = j.job_key AND t.status = 'pending'
|
||||
) pending ON true
|
||||
`
|
||||
|
||||
func (r *SchedulerRepository) ListJobs(ctx context.Context, f SchedulerJobFilter) ([]domain.SchedulerJob, int64, error) {
|
||||
args := []any{}
|
||||
where := "1=1"
|
||||
if f.Category != "" {
|
||||
args = append(args, f.Category)
|
||||
where += fmt.Sprintf(" AND j.category = $%d", len(args))
|
||||
}
|
||||
if f.Keyword != "" {
|
||||
args = append(args, "%"+f.Keyword+"%")
|
||||
where += fmt.Sprintf(" AND (j.job_key ILIKE $%d OR j.display_name ILIKE $%d OR COALESCE(j.description, '') ILIKE $%d)", len(args), len(args), len(args))
|
||||
}
|
||||
if f.Enabled != nil {
|
||||
args = append(args, *f.Enabled)
|
||||
where += fmt.Sprintf(" AND j.enabled = $%d", len(args))
|
||||
}
|
||||
|
||||
var total int64
|
||||
if err := r.pool.QueryRow(ctx, "SELECT COUNT(*) FROM ops.scheduler_jobs j WHERE "+where, args...).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
limit := f.Limit
|
||||
if limit <= 0 || limit > 200 {
|
||||
limit = 50
|
||||
}
|
||||
offset := f.Offset
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
args = append(args, limit, offset)
|
||||
limitArg := len(args) - 1
|
||||
offsetArg := len(args)
|
||||
|
||||
rows, err := r.pool.Query(ctx, schedulerJobSelect+fmt.Sprintf(`
|
||||
WHERE %s
|
||||
ORDER BY j.category ASC, j.job_key ASC
|
||||
LIMIT $%d OFFSET $%d`, where, limitArg, offsetArg), args...)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
items := make([]domain.SchedulerJob, 0, limit)
|
||||
for rows.Next() {
|
||||
item, scanErr := scanSchedulerJob(rows)
|
||||
if scanErr != nil {
|
||||
return nil, 0, scanErr
|
||||
}
|
||||
items = append(items, *item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
if err := r.attachRunSnapshots(ctx, items); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return items, total, nil
|
||||
}
|
||||
|
||||
func (r *SchedulerRepository) GetJob(ctx context.Context, key string) (*domain.SchedulerJob, error) {
|
||||
item, err := scanSchedulerJob(r.pool.QueryRow(ctx, schedulerJobSelect+`
|
||||
WHERE j.job_key = $1`, key))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items := []domain.SchedulerJob{*item}
|
||||
if err := r.attachRunSnapshots(ctx, items); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
*item = items[0]
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (r *SchedulerRepository) GetJobForRuntime(ctx context.Context, key string) (*domain.SchedulerJob, error) {
|
||||
return scanSchedulerJob(r.pool.QueryRow(ctx, schedulerJobSelect+`
|
||||
WHERE j.job_key = $1`, key))
|
||||
}
|
||||
|
||||
func (r *SchedulerRepository) UpdateJob(ctx context.Context, key string, in SchedulerJobUpdate) (*domain.SchedulerJob, error) {
|
||||
configJSON, err := marshalSchedulerJSONMap(in.Config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanSchedulerJob(r.pool.QueryRow(ctx, `
|
||||
UPDATE ops.scheduler_jobs
|
||||
SET enabled = COALESCE($2, enabled),
|
||||
schedule_type = COALESCE(NULLIF($3, ''), schedule_type),
|
||||
interval_seconds = COALESCE($4, interval_seconds),
|
||||
cron_expr = COALESCE($5, cron_expr),
|
||||
timezone = COALESCE(NULLIF($6, ''), timezone),
|
||||
timeout_seconds = COALESCE($7, timeout_seconds),
|
||||
batch_size = COALESCE($8, batch_size),
|
||||
max_concurrency = COALESCE($9, max_concurrency),
|
||||
config = COALESCE($10::jsonb, config),
|
||||
updated_by = COALESCE($11, updated_by),
|
||||
version = version + 1,
|
||||
updated_at = NOW()
|
||||
WHERE job_key = $1
|
||||
RETURNING job_key, display_name, category, description, enabled, schedule_type,
|
||||
interval_seconds, cron_expr, timezone, timeout_seconds, batch_size,
|
||||
max_concurrency, config, version, updated_by, updated_at, created_at, 0::int
|
||||
`, key, in.Enabled, nullableStringPtr(in.ScheduleType), in.IntervalSeconds, in.CronExpr,
|
||||
nullableStringPtr(in.Timezone), in.TimeoutSeconds, in.BatchSize, in.MaxConcurrency, configJSON, in.UpdatedBy))
|
||||
}
|
||||
|
||||
func (r *SchedulerRepository) SetJobEnabled(ctx context.Context, key string, enabled bool, updatedBy *int64) (*domain.SchedulerJob, error) {
|
||||
return r.UpdateJob(ctx, key, SchedulerJobUpdate{
|
||||
Enabled: &enabled,
|
||||
UpdatedBy: updatedBy,
|
||||
})
|
||||
}
|
||||
|
||||
func (r *SchedulerRepository) CreateTrigger(ctx context.Context, in SchedulerTriggerCreate) (*domain.SchedulerTrigger, error) {
|
||||
configJSON, err := marshalSchedulerJSONMap(in.ConfigOverride)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reason := nullableString(in.Reason)
|
||||
return scanSchedulerTrigger(r.pool.QueryRow(ctx, `
|
||||
INSERT INTO ops.scheduler_job_triggers (job_key, trigger_type, requested_by, reason, config_override)
|
||||
VALUES ($1, $2, $3, $4, COALESCE($5::jsonb, '{}'::jsonb))
|
||||
RETURNING id, job_key, trigger_type, status, requested_by, reason, config_override,
|
||||
scheduler_instance_id, run_id, requested_at, claimed_at, finished_at, error_message
|
||||
`, in.JobKey, in.TriggerType, in.RequestedBy, reason, configJSON))
|
||||
}
|
||||
|
||||
func (r *SchedulerRepository) ClaimPendingTrigger(ctx context.Context, jobKey, instanceID string) (*SchedulerTriggerClaim, bool, error) {
|
||||
var claim SchedulerTriggerClaim
|
||||
var configRaw []byte
|
||||
err := r.pool.QueryRow(ctx, `
|
||||
UPDATE ops.scheduler_job_triggers t
|
||||
SET status = 'claimed',
|
||||
claimed_at = NOW(),
|
||||
scheduler_instance_id = $2
|
||||
WHERE t.id = (
|
||||
SELECT id
|
||||
FROM ops.scheduler_job_triggers
|
||||
WHERE job_key = $1 AND status = 'pending'
|
||||
ORDER BY requested_at ASC, id ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING id, job_key, trigger_type, config_override
|
||||
`, jobKey, instanceID).Scan(&claim.ID, &claim.JobKey, &claim.TriggerType, &configRaw)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, false, nil
|
||||
}
|
||||
return nil, false, err
|
||||
}
|
||||
claim.ConfigOverride = decodeSchedulerJSONMap(configRaw)
|
||||
return &claim, true, nil
|
||||
}
|
||||
|
||||
func (r *SchedulerRepository) FinishTrigger(ctx context.Context, triggerID int64, runID *int64, status string, errorMessage *string) error {
|
||||
_, err := r.pool.Exec(ctx, `
|
||||
UPDATE ops.scheduler_job_triggers
|
||||
SET status = $3,
|
||||
run_id = $2,
|
||||
finished_at = NOW(),
|
||||
error_message = $4
|
||||
WHERE id = $1
|
||||
`, triggerID, runID, status, errorMessage)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *SchedulerRepository) StartRun(ctx context.Context, in SchedulerRunStart) (*domain.SchedulerRun, error) {
|
||||
configJSON, err := marshalSchedulerJSONMap(in.ConfigSnapshot)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanSchedulerRun(r.pool.QueryRow(ctx, `
|
||||
INSERT INTO ops.scheduler_job_runs (
|
||||
job_key, trigger_id, scheduler_instance_id, trigger_type,
|
||||
status, config_version, config_snapshot, started_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, 'running', $5, COALESCE($6::jsonb, '{}'::jsonb), NOW())
|
||||
RETURNING id, job_key, trigger_id, scheduler_instance_id, trigger_type, status,
|
||||
config_version, config_snapshot, stats, error_message, started_at,
|
||||
finished_at, duration_ms
|
||||
`, in.JobKey, in.TriggerID, nullableString(in.SchedulerInstanceID), in.TriggerType, in.ConfigVersion, configJSON))
|
||||
}
|
||||
|
||||
func (r *SchedulerRepository) FinishRun(ctx context.Context, in SchedulerRunFinish) (*domain.SchedulerRun, error) {
|
||||
statsJSON, err := marshalSchedulerJSONMap(in.Stats)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanSchedulerRun(r.pool.QueryRow(ctx, `
|
||||
UPDATE ops.scheduler_job_runs
|
||||
SET status = $2,
|
||||
stats = COALESCE($3::jsonb, '{}'::jsonb),
|
||||
error_message = $4,
|
||||
finished_at = NOW(),
|
||||
duration_ms = GREATEST(0, FLOOR(EXTRACT(EPOCH FROM (NOW() - started_at)) * 1000)::bigint)
|
||||
WHERE id = $1
|
||||
RETURNING id, job_key, trigger_id, scheduler_instance_id, trigger_type, status,
|
||||
config_version, config_snapshot, stats, error_message, started_at,
|
||||
finished_at, duration_ms
|
||||
`, in.RunID, in.Status, statsJSON, in.ErrorMessage))
|
||||
}
|
||||
|
||||
func (r *SchedulerRepository) ListRuns(ctx context.Context, f SchedulerRunFilter) ([]domain.SchedulerRun, int64, error) {
|
||||
args := []any{}
|
||||
where := "1=1"
|
||||
if f.JobKey != "" {
|
||||
args = append(args, f.JobKey)
|
||||
where += fmt.Sprintf(" AND job_key = $%d", len(args))
|
||||
}
|
||||
if f.Status != "" {
|
||||
args = append(args, f.Status)
|
||||
where += fmt.Sprintf(" AND status = $%d", len(args))
|
||||
}
|
||||
var total int64
|
||||
if err := r.pool.QueryRow(ctx, "SELECT COUNT(*) FROM ops.scheduler_job_runs WHERE "+where, args...).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
limit := f.Limit
|
||||
if limit <= 0 || limit > 200 {
|
||||
limit = 50
|
||||
}
|
||||
offset := f.Offset
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
args = append(args, limit, offset)
|
||||
limitArg := len(args) - 1
|
||||
offsetArg := len(args)
|
||||
rows, err := r.pool.Query(ctx, fmt.Sprintf(`
|
||||
SELECT id, job_key, trigger_id, scheduler_instance_id, trigger_type, status,
|
||||
config_version, config_snapshot, stats, error_message, started_at,
|
||||
finished_at, duration_ms
|
||||
FROM ops.scheduler_job_runs
|
||||
WHERE %s
|
||||
ORDER BY started_at DESC, id DESC
|
||||
LIMIT $%d OFFSET $%d
|
||||
`, where, limitArg, offsetArg), args...)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]domain.SchedulerRun, 0, limit)
|
||||
for rows.Next() {
|
||||
item, scanErr := scanSchedulerRun(rows)
|
||||
if scanErr != nil {
|
||||
return nil, 0, scanErr
|
||||
}
|
||||
items = append(items, *item)
|
||||
}
|
||||
return items, total, rows.Err()
|
||||
}
|
||||
|
||||
func (r *SchedulerRepository) UpsertInstance(ctx context.Context, instance domain.SchedulerInstance) error {
|
||||
metadataJSON, err := marshalSchedulerJSONMap(instance.Metadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = r.pool.Exec(ctx, `
|
||||
INSERT INTO ops.scheduler_instances (instance_id, hostname, process, build_version, metadata, started_at, last_seen_at)
|
||||
VALUES ($1, $2, $3, $4, COALESCE($5::jsonb, '{}'::jsonb), $6, NOW())
|
||||
ON CONFLICT (instance_id) DO UPDATE
|
||||
SET hostname = EXCLUDED.hostname,
|
||||
process = EXCLUDED.process,
|
||||
build_version = EXCLUDED.build_version,
|
||||
metadata = EXCLUDED.metadata,
|
||||
last_seen_at = NOW()
|
||||
`, instance.InstanceID, instance.Hostname, instance.Process, instance.BuildVersion, metadataJSON, instance.StartedAt)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *SchedulerRepository) ListInstances(ctx context.Context) ([]domain.SchedulerInstance, error) {
|
||||
rows, err := r.pool.Query(ctx, `
|
||||
SELECT instance_id, hostname, process, build_version, started_at, last_seen_at, metadata
|
||||
FROM ops.scheduler_instances
|
||||
ORDER BY last_seen_at DESC
|
||||
LIMIT 200
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]domain.SchedulerInstance, 0, 16)
|
||||
for rows.Next() {
|
||||
item, scanErr := scanSchedulerInstance(rows)
|
||||
if scanErr != nil {
|
||||
return nil, scanErr
|
||||
}
|
||||
items = append(items, *item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (r *SchedulerRepository) attachRunSnapshots(ctx context.Context, items []domain.SchedulerJob) error {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
keys := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
keys = append(keys, item.JobKey)
|
||||
}
|
||||
rows, err := r.pool.Query(ctx, `
|
||||
WITH ranked AS (
|
||||
SELECT
|
||||
id, job_key, trigger_id, scheduler_instance_id, trigger_type, status,
|
||||
config_version, config_snapshot, stats, error_message, started_at,
|
||||
finished_at, duration_ms,
|
||||
ROW_NUMBER() OVER (PARTITION BY job_key ORDER BY started_at DESC, id DESC) AS rn
|
||||
FROM ops.scheduler_job_runs
|
||||
WHERE job_key = ANY($1)
|
||||
)
|
||||
SELECT id, job_key, trigger_id, scheduler_instance_id, trigger_type, status,
|
||||
config_version, config_snapshot, stats, error_message, started_at,
|
||||
finished_at, duration_ms
|
||||
FROM ranked
|
||||
WHERE rn = 1
|
||||
`, keys)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
lastRuns := map[string]*domain.SchedulerRun{}
|
||||
for rows.Next() {
|
||||
run, scanErr := scanSchedulerRun(rows)
|
||||
if scanErr != nil {
|
||||
return scanErr
|
||||
}
|
||||
lastRuns[run.JobKey] = run
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
runningRows, err := r.pool.Query(ctx, `
|
||||
SELECT DISTINCT ON (job_key)
|
||||
id, job_key, trigger_id, scheduler_instance_id, trigger_type, status,
|
||||
config_version, config_snapshot, stats, error_message, started_at,
|
||||
finished_at, duration_ms
|
||||
FROM ops.scheduler_job_runs
|
||||
WHERE job_key = ANY($1) AND status = 'running'
|
||||
ORDER BY job_key, started_at DESC, id DESC
|
||||
`, keys)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer runningRows.Close()
|
||||
runningRuns := map[string]*domain.SchedulerRun{}
|
||||
for runningRows.Next() {
|
||||
run, scanErr := scanSchedulerRun(runningRows)
|
||||
if scanErr != nil {
|
||||
return scanErr
|
||||
}
|
||||
runningRuns[run.JobKey] = run
|
||||
}
|
||||
if err := runningRows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i := range items {
|
||||
items[i].LastRun = lastRuns[items[i].JobKey]
|
||||
items[i].RunningRun = runningRuns[items[i].JobKey]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func scanSchedulerJob(row pgx.Row) (*domain.SchedulerJob, error) {
|
||||
var item domain.SchedulerJob
|
||||
var configRaw []byte
|
||||
err := row.Scan(
|
||||
&item.JobKey,
|
||||
&item.DisplayName,
|
||||
&item.Category,
|
||||
&item.Description,
|
||||
&item.Enabled,
|
||||
&item.ScheduleType,
|
||||
&item.IntervalSeconds,
|
||||
&item.CronExpr,
|
||||
&item.Timezone,
|
||||
&item.TimeoutSeconds,
|
||||
&item.BatchSize,
|
||||
&item.MaxConcurrency,
|
||||
&configRaw,
|
||||
&item.Version,
|
||||
&item.UpdatedBy,
|
||||
&item.UpdatedAt,
|
||||
&item.CreatedAt,
|
||||
&item.PendingTriggers,
|
||||
)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrSchedulerJobNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
item.Config = decodeSchedulerJSONMap(configRaw)
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
func scanSchedulerRun(row pgx.Row) (*domain.SchedulerRun, error) {
|
||||
var item domain.SchedulerRun
|
||||
var configRaw, statsRaw []byte
|
||||
err := row.Scan(
|
||||
&item.ID,
|
||||
&item.JobKey,
|
||||
&item.TriggerID,
|
||||
&item.SchedulerInstanceID,
|
||||
&item.TriggerType,
|
||||
&item.Status,
|
||||
&item.ConfigVersion,
|
||||
&configRaw,
|
||||
&statsRaw,
|
||||
&item.ErrorMessage,
|
||||
&item.StartedAt,
|
||||
&item.FinishedAt,
|
||||
&item.DurationMilliseconds,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.ConfigSnapshot = decodeSchedulerJSONMap(configRaw)
|
||||
item.Stats = decodeSchedulerJSONMap(statsRaw)
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
func scanSchedulerTrigger(row pgx.Row) (*domain.SchedulerTrigger, error) {
|
||||
var item domain.SchedulerTrigger
|
||||
var configRaw []byte
|
||||
err := row.Scan(
|
||||
&item.ID,
|
||||
&item.JobKey,
|
||||
&item.TriggerType,
|
||||
&item.Status,
|
||||
&item.RequestedBy,
|
||||
&item.Reason,
|
||||
&configRaw,
|
||||
&item.SchedulerInstanceID,
|
||||
&item.RunID,
|
||||
&item.RequestedAt,
|
||||
&item.ClaimedAt,
|
||||
&item.FinishedAt,
|
||||
&item.ErrorMessage,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.ConfigOverride = decodeSchedulerJSONMap(configRaw)
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
func scanSchedulerInstance(row pgx.Row) (*domain.SchedulerInstance, error) {
|
||||
var item domain.SchedulerInstance
|
||||
var metadataRaw []byte
|
||||
if err := row.Scan(
|
||||
&item.InstanceID,
|
||||
&item.Hostname,
|
||||
&item.Process,
|
||||
&item.BuildVersion,
|
||||
&item.StartedAt,
|
||||
&item.LastSeenAt,
|
||||
&metadataRaw,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.Metadata = decodeSchedulerJSONMap(metadataRaw)
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
func marshalSchedulerJSONMap(value map[string]any) ([]byte, error) {
|
||||
if value == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return json.Marshal(value)
|
||||
}
|
||||
|
||||
func decodeSchedulerJSONMap(raw []byte) map[string]any {
|
||||
if len(raw) == 0 {
|
||||
return map[string]any{}
|
||||
}
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal(raw, &out); err != nil || out == nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func nullableStringPtr(value *string) *string {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
trimmed := *value
|
||||
return &trimmed
|
||||
}
|
||||
@@ -26,6 +26,7 @@ type Deps struct {
|
||||
Audits *app.AuditService
|
||||
SiteDomains *app.SiteDomainMappingService
|
||||
Compliance *app.ComplianceService
|
||||
Scheduler *app.SchedulerService
|
||||
}
|
||||
|
||||
func (d Deps) ServerAllowedOrigins() []string {
|
||||
@@ -115,6 +116,16 @@ func RegisterRoutes(d Deps) {
|
||||
authed.POST("/jobs/:source/:id/retry", retryJobHandler(d.Jobs))
|
||||
authed.POST("/jobs/:source/:id/cancel", cancelJobHandler(d.Jobs))
|
||||
|
||||
authed.GET("/scheduler/jobs", listSchedulerJobsHandler(d.Scheduler))
|
||||
authed.GET("/scheduler/jobs/:key", getSchedulerJobHandler(d.Scheduler))
|
||||
authed.PATCH("/scheduler/jobs/:key", updateSchedulerJobHandler(d.Scheduler))
|
||||
authed.POST("/scheduler/jobs/:key/enable", enableSchedulerJobHandler(d.Scheduler, true))
|
||||
authed.POST("/scheduler/jobs/:key/disable", enableSchedulerJobHandler(d.Scheduler, false))
|
||||
authed.POST("/scheduler/jobs/:key/run", triggerSchedulerJobHandler(d.Scheduler, "manual"))
|
||||
authed.POST("/scheduler/jobs/:key/dry-run", triggerSchedulerJobHandler(d.Scheduler, "dry_run"))
|
||||
authed.GET("/scheduler/jobs/:key/runs", listSchedulerRunsHandler(d.Scheduler))
|
||||
authed.GET("/scheduler/instances", listSchedulerInstancesHandler(d.Scheduler))
|
||||
|
||||
authed.GET("/site-domain-mappings", listSiteDomainMappingsHandler(d.SiteDomains))
|
||||
authed.POST("/site-domain-mappings", createSiteDomainMappingHandler(d.SiteDomains))
|
||||
authed.PATCH("/site-domain-mappings/:id", updateSiteDomainMappingHandler(d.SiteDomains))
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package transport
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/ops/app"
|
||||
"github.com/geo-platform/tenant-api/internal/shared/response"
|
||||
)
|
||||
|
||||
func listSchedulerJobsHandler(svc *app.SchedulerService) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
size, _ := strconv.Atoi(c.DefaultQuery("size", "50"))
|
||||
var enabled *bool
|
||||
if raw := strings.TrimSpace(c.Query("enabled")); raw != "" {
|
||||
value, err := strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
response.Error(c, response.ErrBadRequest(40087, "invalid_enabled", "enabled 必须是布尔值"))
|
||||
return
|
||||
}
|
||||
enabled = &value
|
||||
}
|
||||
result, err := svc.ListJobs(c.Request.Context(), app.SchedulerJobListInput{
|
||||
Category: strings.TrimSpace(c.Query("category")),
|
||||
Keyword: strings.TrimSpace(c.Query("keyword")),
|
||||
Enabled: enabled,
|
||||
Page: page,
|
||||
Size: size,
|
||||
})
|
||||
if err != nil {
|
||||
response.Error(c, err)
|
||||
return
|
||||
}
|
||||
response.Success(c, result)
|
||||
}
|
||||
}
|
||||
|
||||
func getSchedulerJobHandler(svc *app.SchedulerService) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
item, err := svc.GetJob(c.Request.Context(), c.Param("key"))
|
||||
if err != nil {
|
||||
response.Error(c, err)
|
||||
return
|
||||
}
|
||||
response.Success(c, item)
|
||||
}
|
||||
}
|
||||
|
||||
func updateSchedulerJobHandler(svc *app.SchedulerService) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
var req app.SchedulerJobUpdateInput
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
response.Error(c, response.ErrBadRequest(40088, "invalid_payload", err.Error()))
|
||||
return
|
||||
}
|
||||
item, err := svc.UpdateJob(c.Request.Context(), actorFromGin(c), c.Param("key"), req)
|
||||
if err != nil {
|
||||
response.Error(c, err)
|
||||
return
|
||||
}
|
||||
response.Success(c, item)
|
||||
}
|
||||
}
|
||||
|
||||
func enableSchedulerJobHandler(svc *app.SchedulerService, enabled bool) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
item, err := svc.SetEnabled(c.Request.Context(), actorFromGin(c), c.Param("key"), enabled)
|
||||
if err != nil {
|
||||
response.Error(c, err)
|
||||
return
|
||||
}
|
||||
response.Success(c, item)
|
||||
}
|
||||
}
|
||||
|
||||
func triggerSchedulerJobHandler(svc *app.SchedulerService, triggerType string) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
var req app.SchedulerTriggerInput
|
||||
if c.Request.Body != nil {
|
||||
if err := c.ShouldBindJSON(&req); err != nil && !errors.Is(err, io.EOF) {
|
||||
response.Error(c, response.ErrBadRequest(40089, "invalid_payload", err.Error()))
|
||||
return
|
||||
}
|
||||
}
|
||||
item, err := svc.Trigger(c.Request.Context(), actorFromGin(c), c.Param("key"), triggerType, req)
|
||||
if err != nil {
|
||||
response.Error(c, err)
|
||||
return
|
||||
}
|
||||
response.Success(c, item)
|
||||
}
|
||||
}
|
||||
|
||||
func listSchedulerRunsHandler(svc *app.SchedulerService) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
size, _ := strconv.Atoi(c.DefaultQuery("size", "50"))
|
||||
result, err := svc.ListRuns(c.Request.Context(), c.Param("key"), strings.TrimSpace(c.Query("status")), page, size)
|
||||
if err != nil {
|
||||
response.Error(c, err)
|
||||
return
|
||||
}
|
||||
response.Success(c, result)
|
||||
}
|
||||
}
|
||||
|
||||
func listSchedulerInstancesHandler(svc *app.SchedulerService) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
items, err := svc.ListInstances(c.Request.Context())
|
||||
if err != nil {
|
||||
response.Error(c, err)
|
||||
return
|
||||
}
|
||||
response.Success(c, gin.H{"items": items})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,498 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"go.uber.org/zap"
|
||||
|
||||
opsdomain "github.com/geo-platform/tenant-api/internal/ops/domain"
|
||||
opsrepo "github.com/geo-platform/tenant-api/internal/ops/repository"
|
||||
sharedschedule "github.com/geo-platform/tenant-api/internal/shared/schedule"
|
||||
)
|
||||
|
||||
type ControlledJob struct {
|
||||
Key string
|
||||
DefaultName string
|
||||
Run func(context.Context, JobRunContext) (map[string]any, error)
|
||||
}
|
||||
|
||||
type JobRunContext struct {
|
||||
Job *opsdomain.SchedulerJob
|
||||
TriggerID *int64
|
||||
Trigger string
|
||||
DryRun bool
|
||||
Config map[string]any
|
||||
}
|
||||
|
||||
type ControlPlane struct {
|
||||
pool *pgxpool.Pool
|
||||
repo *opsrepo.SchedulerRepository
|
||||
logger *zap.Logger
|
||||
instanceID string
|
||||
hostname string
|
||||
startedAt time.Time
|
||||
jobs map[string]ControlledJob
|
||||
}
|
||||
|
||||
func NewControlPlane(pool *pgxpool.Pool, logger *zap.Logger, jobs []ControlledJob) *ControlPlane {
|
||||
hostname, _ := os.Hostname()
|
||||
if hostname == "" {
|
||||
hostname = "unknown"
|
||||
}
|
||||
instanceID := hostname + ":" + strconv.Itoa(os.Getpid()) + ":" + strconv.FormatInt(time.Now().UnixNano(), 36)
|
||||
byKey := make(map[string]ControlledJob, len(jobs))
|
||||
for _, job := range jobs {
|
||||
if job.Key == "" || job.Run == nil {
|
||||
continue
|
||||
}
|
||||
byKey[job.Key] = job
|
||||
}
|
||||
return &ControlPlane{
|
||||
pool: pool,
|
||||
repo: opsrepo.NewSchedulerRepository(pool),
|
||||
logger: logger,
|
||||
instanceID: instanceID,
|
||||
hostname: hostname,
|
||||
startedAt: time.Now().UTC(),
|
||||
jobs: byKey,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ControlPlane) Run(ctx context.Context) {
|
||||
if p == nil || p.pool == nil || len(p.jobs) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
p.heartbeat(context.Background())
|
||||
heartbeatCtx, heartbeatCancel := context.WithCancel(ctx)
|
||||
defer heartbeatCancel()
|
||||
go p.runHeartbeat(heartbeatCtx)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for _, job := range p.jobs {
|
||||
job := job
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
p.runJobLoop(ctx, job)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func (p *ControlPlane) InstanceID() string {
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
return p.instanceID
|
||||
}
|
||||
|
||||
func (p *ControlPlane) runHeartbeat(ctx context.Context) {
|
||||
ticker := time.NewTicker(15 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
p.heartbeat(context.Background())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ControlPlane) heartbeat(ctx context.Context) {
|
||||
if p == nil || p.repo == nil {
|
||||
return
|
||||
}
|
||||
buildVersion := os.Getenv("BUILD_VERSION")
|
||||
if buildVersion == "" {
|
||||
buildVersion = os.Getenv("GIT_SHA")
|
||||
}
|
||||
var buildVersionPtr *string
|
||||
if buildVersion != "" {
|
||||
buildVersionPtr = &buildVersion
|
||||
}
|
||||
if err := p.repo.UpsertInstance(ctx, opsdomain.SchedulerInstance{
|
||||
InstanceID: p.instanceID,
|
||||
Hostname: p.hostname,
|
||||
Process: "scheduler",
|
||||
BuildVersion: buildVersionPtr,
|
||||
StartedAt: p.startedAt,
|
||||
LastSeenAt: time.Now().UTC(),
|
||||
Metadata: map[string]any{
|
||||
"pid": os.Getpid(),
|
||||
},
|
||||
}); err != nil && p.logger != nil {
|
||||
p.logger.Warn("scheduler control heartbeat failed", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ControlPlane) runJobLoop(ctx context.Context, job ControlledJob) {
|
||||
nextAutoAt := p.nextAutoAt(context.Background(), job.Key, time.Now(), true)
|
||||
|
||||
for {
|
||||
p.drainManualTriggers(context.Background(), job)
|
||||
now := time.Now()
|
||||
if !now.Before(nextAutoAt) {
|
||||
p.runJobOnce(context.Background(), job, opsdomain.SchedulerTriggerAuto, nil, nil)
|
||||
nextAutoAt = p.nextAutoAt(context.Background(), job.Key, time.Now(), false)
|
||||
}
|
||||
sleepFor := time.Until(nextAutoAt)
|
||||
if sleepFor > 15*time.Second {
|
||||
sleepFor = 15 * time.Second
|
||||
}
|
||||
if sleepFor <= 0 {
|
||||
sleepFor = time.Second
|
||||
}
|
||||
timer := time.NewTimer(sleepFor)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return
|
||||
case <-timer.C:
|
||||
p.drainManualTriggers(context.Background(), job)
|
||||
p.runJobOnce(context.Background(), job, opsdomain.SchedulerTriggerAuto, nil, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ControlPlane) drainManualTriggers(parent context.Context, job ControlledJob) {
|
||||
for {
|
||||
trigger, ok, err := p.repo.ClaimPendingTrigger(parent, job.Key, p.instanceID)
|
||||
if err != nil {
|
||||
if p.logger != nil {
|
||||
p.logger.Warn("scheduler trigger claim failed", zap.String("job_key", job.Key), zap.Error(err))
|
||||
}
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
triggerID := trigger.ID
|
||||
p.runJobOnce(parent, job, trigger.TriggerType, &triggerID, trigger.ConfigOverride)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ControlPlane) nextAutoAt(ctx context.Context, key string, now time.Time, initial bool) time.Time {
|
||||
job, err := p.repo.GetJobForRuntime(ctx, key)
|
||||
if err != nil || job == nil {
|
||||
return now.Add(30 * time.Second)
|
||||
}
|
||||
if job.ScheduleType == "manual" {
|
||||
return now.Add(15 * time.Second)
|
||||
}
|
||||
if job.ScheduleType == "cron" && job.CronExpr != nil {
|
||||
loc, locErr := time.LoadLocation(job.Timezone)
|
||||
if locErr != nil {
|
||||
loc = time.Local
|
||||
}
|
||||
schedule, parseErr := sharedschedule.Parse(*job.CronExpr)
|
||||
if parseErr != nil {
|
||||
if p.logger != nil {
|
||||
p.logger.Warn("scheduler controlled job has invalid cron",
|
||||
zap.String("job_key", key),
|
||||
zap.String("cron_expr", *job.CronExpr),
|
||||
zap.Error(parseErr),
|
||||
)
|
||||
}
|
||||
return now.Add(30 * time.Second)
|
||||
}
|
||||
base := now.In(loc)
|
||||
return schedule.Next(base)
|
||||
}
|
||||
if runAt := parseRunAtLocal(job.Config); runAt != "" {
|
||||
loc, locErr := time.LoadLocation(job.Timezone)
|
||||
if locErr != nil {
|
||||
loc = time.Local
|
||||
}
|
||||
hour, minute, ok := parseHourMinute(runAt)
|
||||
if ok {
|
||||
localNow := now.In(loc)
|
||||
candidate := time.Date(localNow.Year(), localNow.Month(), localNow.Day(), hour, minute, 0, 0, loc)
|
||||
if !initial || !localNow.Before(candidate) {
|
||||
candidate = candidate.AddDate(0, 0, 1)
|
||||
}
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
interval := time.Duration(job.IntervalSeconds) * time.Second
|
||||
if interval <= 0 {
|
||||
interval = 30 * time.Second
|
||||
}
|
||||
if initial {
|
||||
return now
|
||||
}
|
||||
return now.Add(interval)
|
||||
}
|
||||
|
||||
func (p *ControlPlane) runJobOnce(parent context.Context, job ControlledJob, triggerType string, triggerID *int64, configOverride map[string]any) {
|
||||
cfg, err := p.repo.GetJobForRuntime(parent, job.Key)
|
||||
if err != nil {
|
||||
if p.logger != nil {
|
||||
p.logger.Warn("scheduler job config load failed", zap.String("job_key", job.Key), zap.Error(err))
|
||||
}
|
||||
return
|
||||
}
|
||||
isManual := triggerType == opsdomain.SchedulerTriggerManual || triggerType == opsdomain.SchedulerTriggerDryRun
|
||||
if !cfg.Enabled && !isManual {
|
||||
return
|
||||
}
|
||||
if cfg.ScheduleType == "manual" && !isManual {
|
||||
return
|
||||
}
|
||||
|
||||
mergedConfig := cloneMap(cfg.Config)
|
||||
for key, value := range configOverride {
|
||||
mergedConfig[key] = value
|
||||
}
|
||||
dryRun := triggerType == opsdomain.SchedulerTriggerDryRun || boolFromMap(mergedConfig, "dry_run")
|
||||
if dryRun {
|
||||
mergedConfig["dry_run"] = true
|
||||
}
|
||||
|
||||
lockKey := advisoryLockKey(job.Key)
|
||||
lockConn, locked := p.tryAdvisoryLock(parent, lockKey)
|
||||
if !locked {
|
||||
if p.logger != nil {
|
||||
p.logger.Info("scheduler job skipped because another instance owns lock", zap.String("job_key", job.Key))
|
||||
}
|
||||
if triggerID != nil {
|
||||
_ = p.repo.FinishTrigger(parent, *triggerID, nil, opsdomain.SchedulerRunSkipped, controlStringPtr("another scheduler instance is running this job"))
|
||||
}
|
||||
return
|
||||
}
|
||||
defer p.advisoryUnlock(context.Background(), lockConn, lockKey)
|
||||
|
||||
run, err := p.repo.StartRun(parent, opsrepo.SchedulerRunStart{
|
||||
JobKey: job.Key,
|
||||
TriggerID: triggerID,
|
||||
SchedulerInstanceID: p.instanceID,
|
||||
TriggerType: triggerType,
|
||||
ConfigVersion: cfg.Version,
|
||||
ConfigSnapshot: schedulerConfigSnapshot(cfg, mergedConfig),
|
||||
})
|
||||
if err != nil {
|
||||
if p.logger != nil {
|
||||
p.logger.Warn("scheduler run start failed", zap.String("job_key", job.Key), zap.Error(err))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
timeout := time.Duration(cfg.TimeoutSeconds) * time.Second
|
||||
if timeout <= 0 {
|
||||
timeout = 30 * time.Second
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(parent, timeout)
|
||||
defer cancel()
|
||||
|
||||
stats, runErr := job.Run(ctx, JobRunContext{
|
||||
Job: cfg,
|
||||
TriggerID: triggerID,
|
||||
Trigger: triggerType,
|
||||
DryRun: dryRun,
|
||||
Config: mergedConfig,
|
||||
})
|
||||
status := opsdomain.SchedulerRunSuccess
|
||||
var errMsg *string
|
||||
if runErr != nil {
|
||||
status = opsdomain.SchedulerRunFailed
|
||||
message := runErr.Error()
|
||||
errMsg = &message
|
||||
}
|
||||
if stats == nil {
|
||||
stats = map[string]any{}
|
||||
}
|
||||
stats["dry_run"] = dryRun
|
||||
|
||||
finished, finishErr := p.repo.FinishRun(parent, opsrepo.SchedulerRunFinish{
|
||||
RunID: run.ID,
|
||||
Status: status,
|
||||
Stats: stats,
|
||||
ErrorMessage: errMsg,
|
||||
})
|
||||
if finishErr != nil && p.logger != nil {
|
||||
p.logger.Warn("scheduler run finish failed", zap.String("job_key", job.Key), zap.Int64("run_id", run.ID), zap.Error(finishErr))
|
||||
}
|
||||
if triggerID != nil {
|
||||
triggerStatus := status
|
||||
if finishErr != nil {
|
||||
triggerStatus = opsdomain.SchedulerRunFailed
|
||||
}
|
||||
runID := run.ID
|
||||
_ = p.repo.FinishTrigger(parent, *triggerID, &runID, triggerStatus, errMsg)
|
||||
}
|
||||
if runErr != nil {
|
||||
if p.logger != nil {
|
||||
p.logger.Warn("scheduler controlled job failed", zap.String("job_key", job.Key), zap.Error(runErr))
|
||||
}
|
||||
return
|
||||
}
|
||||
if p.logger != nil {
|
||||
duration := int64(0)
|
||||
if finished != nil && finished.DurationMilliseconds != nil {
|
||||
duration = *finished.DurationMilliseconds
|
||||
}
|
||||
p.logger.Info("scheduler controlled job completed", zap.String("job_key", job.Key), zap.Int64("duration_ms", duration))
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ControlPlane) tryAdvisoryLock(ctx context.Context, key int64) (*pgxpool.Conn, bool) {
|
||||
conn, err := p.pool.Acquire(ctx)
|
||||
if err != nil {
|
||||
if p.logger != nil {
|
||||
p.logger.Warn("scheduler advisory lock acquire failed", zap.Int64("lock_key", key), zap.Error(err))
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
var ok bool
|
||||
if err := conn.QueryRow(ctx, `SELECT pg_try_advisory_lock($1)`, key).Scan(&ok); err != nil {
|
||||
conn.Release()
|
||||
if p.logger != nil {
|
||||
p.logger.Warn("scheduler advisory lock failed", zap.Int64("lock_key", key), zap.Error(err))
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
if !ok {
|
||||
conn.Release()
|
||||
return nil, false
|
||||
}
|
||||
return conn, true
|
||||
}
|
||||
|
||||
func (p *ControlPlane) advisoryUnlock(ctx context.Context, conn *pgxpool.Conn, key int64) {
|
||||
if conn == nil {
|
||||
return
|
||||
}
|
||||
defer conn.Release()
|
||||
_, _ = conn.Exec(ctx, `SELECT pg_advisory_unlock($1)`, key)
|
||||
}
|
||||
|
||||
func advisoryLockKey(value string) int64 {
|
||||
hash := fnv.New64a()
|
||||
_, _ = hash.Write([]byte("scheduler:" + value))
|
||||
return int64(hash.Sum64() & 0x7fffffffffffffff)
|
||||
}
|
||||
|
||||
func schedulerConfigSnapshot(job *opsdomain.SchedulerJob, config map[string]any) map[string]any {
|
||||
out := map[string]any{
|
||||
"enabled": job.Enabled,
|
||||
"schedule_type": job.ScheduleType,
|
||||
"interval_seconds": job.IntervalSeconds,
|
||||
"timezone": job.Timezone,
|
||||
"timeout_seconds": job.TimeoutSeconds,
|
||||
"max_concurrency": job.MaxConcurrency,
|
||||
"config": config,
|
||||
}
|
||||
if job.BatchSize != nil {
|
||||
out["batch_size"] = *job.BatchSize
|
||||
}
|
||||
if job.CronExpr != nil {
|
||||
out["cron_expr"] = *job.CronExpr
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneMap(in map[string]any) map[string]any {
|
||||
out := make(map[string]any, len(in))
|
||||
for key, value := range in {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func boolFromMap(in map[string]any, key string) bool {
|
||||
value, ok := in[key]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case bool:
|
||||
return typed
|
||||
case string:
|
||||
parsed, _ := strconv.ParseBool(typed)
|
||||
return parsed
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func parseRunAtLocal(config map[string]any) string {
|
||||
value, ok := config["run_at_local"]
|
||||
if !ok || value == nil {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprint(value)
|
||||
}
|
||||
|
||||
func parseHourMinute(value string) (int, int, bool) {
|
||||
parts := strings.Split(strings.TrimSpace(value), ":")
|
||||
if len(parts) != 2 {
|
||||
return 0, 0, false
|
||||
}
|
||||
hour, hourErr := strconv.Atoi(parts[0])
|
||||
minute, minuteErr := strconv.Atoi(parts[1])
|
||||
if hourErr != nil || minuteErr != nil || hour < 0 || hour > 23 || minute < 0 || minute > 59 {
|
||||
return 0, 0, false
|
||||
}
|
||||
return hour, minute, true
|
||||
}
|
||||
|
||||
func controlStringPtr(value string) *string {
|
||||
return &value
|
||||
}
|
||||
|
||||
func AsInt(config map[string]any, key string, fallback int) int {
|
||||
value, ok := config[key]
|
||||
if !ok || value == nil {
|
||||
return fallback
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case int:
|
||||
return typed
|
||||
case int64:
|
||||
return int(typed)
|
||||
case float64:
|
||||
return int(typed)
|
||||
case string:
|
||||
parsed, err := strconv.Atoi(typed)
|
||||
if err == nil {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func AsDuration(config map[string]any, key string, fallback time.Duration) time.Duration {
|
||||
value, ok := config[key]
|
||||
if !ok || value == nil {
|
||||
return fallback
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case time.Duration:
|
||||
return typed
|
||||
case string:
|
||||
parsed, err := time.ParseDuration(typed)
|
||||
if err == nil {
|
||||
return parsed
|
||||
}
|
||||
case float64:
|
||||
return time.Duration(typed) * time.Second
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func ErrorFromContext(ctx context.Context) error {
|
||||
if err := ctx.Err(); err != nil && !errors.Is(err, context.Canceled) {
|
||||
return fmt.Errorf("scheduler job context finished: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -45,3 +45,14 @@ func (w *KnowledgeDeletedCleanupWorker) run(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *KnowledgeDeletedCleanupWorker) RunOnce(ctx context.Context) (map[string]any, error) {
|
||||
if w == nil || w.service == nil {
|
||||
return map[string]any{"skipped": true, "reason": "worker_not_ready"}, nil
|
||||
}
|
||||
startedAt := time.Now()
|
||||
w.service.RunDeletedCleanupSweep()
|
||||
return map[string]any{
|
||||
"duration_ms": time.Since(startedAt).Milliseconds(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -90,3 +90,30 @@ func (w *MonitoringLeaseRecoveryWorker) runOnce(parent context.Context) {
|
||||
w.logger.Info("monitoring lease recovery completed", zap.Int64("expired_task_count", expiredCount))
|
||||
}
|
||||
}
|
||||
|
||||
func (w *MonitoringLeaseRecoveryWorker) RunOnce(parent context.Context) (map[string]any, error) {
|
||||
if w == nil || w.monitoringPool == nil {
|
||||
return map[string]any{"skipped": true, "reason": "worker_not_ready"}, nil
|
||||
}
|
||||
startedAt := time.Now()
|
||||
ctx, cancel := context.WithTimeout(parent, w.timeout)
|
||||
defer cancel()
|
||||
|
||||
tx, err := w.monitoringPool.Begin(ctx)
|
||||
if err != nil {
|
||||
return map[string]any{"stage": "begin"}, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
expiredCount, err := tenantapp.ExpireMonitoringLeasedTasks(ctx, tx, nil, time.Now().UTC())
|
||||
if err != nil {
|
||||
return map[string]any{"stage": "expire"}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return map[string]any{"stage": "commit"}, err
|
||||
}
|
||||
return map[string]any{
|
||||
"expired_task_count": expiredCount,
|
||||
"duration_ms": time.Since(startedAt).Milliseconds(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -116,3 +116,32 @@ func (w *MonitoringReceivedInspectionWorker) runOnce(parent context.Context) {
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *MonitoringReceivedInspectionWorker) RunOnce(parent context.Context) (map[string]any, error) {
|
||||
if w == nil || w.monitoringPool == nil {
|
||||
return map[string]any{"skipped": true, "reason": "worker_not_ready"}, nil
|
||||
}
|
||||
startedAt := time.Now()
|
||||
ctx, cancel := context.WithTimeout(parent, w.timeout)
|
||||
defer cancel()
|
||||
|
||||
tx, err := w.monitoringPool.Begin(ctx)
|
||||
if err != nil {
|
||||
return map[string]any{"stage": "begin"}, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
now := time.Now().UTC()
|
||||
items, err := tenantapp.MarkMonitoringStaleReceivedTasks(ctx, tx, now, w.threshold, w.limit)
|
||||
if err != nil {
|
||||
return map[string]any{"stage": "inspect"}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return map[string]any{"stage": "commit"}, err
|
||||
}
|
||||
return map[string]any{
|
||||
"overdue_task_count": len(items),
|
||||
"threshold_seconds": int64(w.threshold.Seconds()),
|
||||
"duration_ms": time.Since(startedAt).Milliseconds(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -118,6 +118,41 @@ func (w *MonitoringResultRecoveryWorker) runOnce(parent context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
func (w *MonitoringResultRecoveryWorker) RunOnce(parent context.Context) (map[string]any, error) {
|
||||
if w == nil || w.monitoringPool == nil || w.service == nil {
|
||||
return map[string]any{"skipped": true, "reason": "worker_not_ready"}, nil
|
||||
}
|
||||
startedAt := time.Now()
|
||||
ctx, cancel := context.WithTimeout(parent, w.timeout)
|
||||
defer cancel()
|
||||
|
||||
tx, err := w.monitoringPool.Begin(ctx)
|
||||
if err != nil {
|
||||
return map[string]any{"stage": "begin"}, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
items, err := loadMonitoringRecoverableTaskResults(ctx, tx, w.limit)
|
||||
if err != nil {
|
||||
return map[string]any{"stage": "load"}, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return map[string]any{"stage": "commit"}, err
|
||||
}
|
||||
|
||||
republishedCount := 0
|
||||
for _, item := range items {
|
||||
if w.tryRepublish(parent, item) {
|
||||
republishedCount++
|
||||
}
|
||||
}
|
||||
return map[string]any{
|
||||
"recoverable_count": len(items),
|
||||
"republished_count": republishedCount,
|
||||
"duration_ms": time.Since(startedAt).Milliseconds(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (w *MonitoringResultRecoveryWorker) tryRepublish(parent context.Context, item monitoringRecoverableTaskResult) bool {
|
||||
if err := tenantapp.ValidateMonitoringTaskResultEnvelope(item.RequestPayloadJSON); err != nil {
|
||||
w.service.PatchTaskResultQueueMeta(item.ID, item.CallbackReceivedAt, tenantapp.MonitoringTaskResultQueueMetaPatch{
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultMonitoringRetentionDays = 30
|
||||
defaultMonitoringRetentionBatch = 5000
|
||||
defaultMonitoringRetentionBatches = 200
|
||||
)
|
||||
|
||||
type MonitoringRetentionWorker struct {
|
||||
monitoringPool *pgxpool.Pool
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
type retentionTarget struct {
|
||||
Name string
|
||||
DeleteSQL string
|
||||
CountSQL string
|
||||
}
|
||||
|
||||
func NewMonitoringRetentionWorker(monitoringPool *pgxpool.Pool, logger *zap.Logger) *MonitoringRetentionWorker {
|
||||
return &MonitoringRetentionWorker{monitoringPool: monitoringPool, logger: logger}
|
||||
}
|
||||
|
||||
func (w *MonitoringRetentionWorker) RunOnce(ctx context.Context, run JobRunContext) (map[string]any, error) {
|
||||
if w == nil || w.monitoringPool == nil {
|
||||
return map[string]any{"skipped": true, "reason": "worker_not_ready"}, nil
|
||||
}
|
||||
startedAt := time.Now()
|
||||
retentionDays := AsInt(run.Config, "retention_days", defaultMonitoringRetentionDays)
|
||||
if retentionDays < defaultMonitoringRetentionDays {
|
||||
retentionDays = defaultMonitoringRetentionDays
|
||||
}
|
||||
batchSize := AsInt(run.Config, "batch_size", defaultMonitoringRetentionBatch)
|
||||
if run.Job != nil && run.Job.BatchSize != nil {
|
||||
batchSize = *run.Job.BatchSize
|
||||
}
|
||||
if batchSize <= 0 {
|
||||
batchSize = defaultMonitoringRetentionBatch
|
||||
}
|
||||
maxBatches := AsInt(run.Config, "max_batches_per_run", defaultMonitoringRetentionBatches)
|
||||
if maxBatches <= 0 {
|
||||
maxBatches = defaultMonitoringRetentionBatches
|
||||
}
|
||||
statementTimeout := AsDuration(run.Config, "statement_timeout", 5*time.Second)
|
||||
cutoffDate := monitoringRetentionCutoffDate(time.Now(), retentionDays)
|
||||
dryRun := run.DryRun
|
||||
|
||||
stats := map[string]any{
|
||||
"retention_days": retentionDays,
|
||||
"cutoff_date": cutoffDate,
|
||||
"batch_size": batchSize,
|
||||
"max_batches_per_run": maxBatches,
|
||||
"dry_run": dryRun,
|
||||
}
|
||||
|
||||
conn, err := w.monitoringPool.Acquire(ctx)
|
||||
if err != nil {
|
||||
return stats, fmt.Errorf("acquire monitoring connection: %w", err)
|
||||
}
|
||||
defer conn.Release()
|
||||
|
||||
if statementTimeout > 0 {
|
||||
if _, err := conn.Exec(ctx, `SET statement_timeout = $1`, int(statementTimeout.Milliseconds())); err != nil {
|
||||
return stats, fmt.Errorf("set statement timeout: %w", err)
|
||||
}
|
||||
defer conn.Exec(context.Background(), `RESET statement_timeout`)
|
||||
}
|
||||
|
||||
totalDeleted := int64(0)
|
||||
totalCandidates := int64(0)
|
||||
perTable := map[string]any{}
|
||||
for _, target := range retentionTargets() {
|
||||
candidateCount, err := w.countTarget(ctx, conn, target, cutoffDate)
|
||||
if err != nil {
|
||||
return stats, fmt.Errorf("count %s: %w", target.Name, err)
|
||||
}
|
||||
totalCandidates += candidateCount
|
||||
tableStats := map[string]any{
|
||||
"candidate_count": candidateCount,
|
||||
}
|
||||
if !dryRun && candidateCount > 0 {
|
||||
deleted, batches, err := w.deleteTarget(ctx, conn, target, cutoffDate, batchSize, maxBatches)
|
||||
if err != nil {
|
||||
return stats, fmt.Errorf("delete %s: %w", target.Name, err)
|
||||
}
|
||||
tableStats["deleted_count"] = deleted
|
||||
tableStats["batch_count"] = batches
|
||||
totalDeleted += deleted
|
||||
}
|
||||
perTable[target.Name] = tableStats
|
||||
}
|
||||
|
||||
stats["candidate_count"] = totalCandidates
|
||||
stats["deleted_count"] = totalDeleted
|
||||
stats["tables"] = perTable
|
||||
stats["duration_ms"] = time.Since(startedAt).Milliseconds()
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func monitoringRetentionCutoffDate(now time.Time, retentionDays int) string {
|
||||
loc, err := time.LoadLocation("Asia/Shanghai")
|
||||
if err != nil {
|
||||
loc = time.FixedZone("UTC+8", 8*60*60)
|
||||
}
|
||||
localNow := now.In(loc)
|
||||
today := time.Date(localNow.Year(), localNow.Month(), localNow.Day(), 0, 0, 0, 0, loc)
|
||||
cutoff := today.AddDate(0, 0, -(retentionDays - 1))
|
||||
return cutoff.Format("2006-01-02")
|
||||
}
|
||||
|
||||
func (w *MonitoringRetentionWorker) countTarget(ctx context.Context, conn *pgxpool.Conn, target retentionTarget, cutoffDate string) (int64, error) {
|
||||
var count int64
|
||||
if err := conn.QueryRow(ctx, target.CountSQL, cutoffDate).Scan(&count); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func (w *MonitoringRetentionWorker) deleteTarget(ctx context.Context, conn *pgxpool.Conn, target retentionTarget, cutoffDate string, batchSize, maxBatches int) (int64, int, error) {
|
||||
total := int64(0)
|
||||
batches := 0
|
||||
for batches < maxBatches {
|
||||
tag, err := conn.Exec(ctx, target.DeleteSQL, cutoffDate, batchSize)
|
||||
if err != nil {
|
||||
return total, batches, err
|
||||
}
|
||||
affected := tag.RowsAffected()
|
||||
if affected == 0 {
|
||||
break
|
||||
}
|
||||
total += affected
|
||||
batches++
|
||||
if affected < int64(batchSize) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return total, batches, nil
|
||||
}
|
||||
|
||||
func retentionTargets() []retentionTarget {
|
||||
return []retentionTarget{
|
||||
{
|
||||
Name: "monitoring_collect_dispatch_outbox",
|
||||
CountSQL: `SELECT COUNT(*) FROM monitoring_collect_dispatch_outbox WHERE created_at < $1::date`,
|
||||
DeleteSQL: batchDeleteSQL("monitoring_collect_dispatch_outbox", "id", "created_at < $1::date"),
|
||||
},
|
||||
{
|
||||
Name: "monitoring_collect_requests",
|
||||
CountSQL: `SELECT COUNT(*) FROM monitoring_collect_requests WHERE created_at < $1::date`,
|
||||
DeleteSQL: batchDeleteSQL("monitoring_collect_requests", "id", "created_at < $1::date"),
|
||||
},
|
||||
{
|
||||
Name: "monitoring_brand_platform_daily",
|
||||
CountSQL: `SELECT COUNT(*) FROM monitoring_brand_platform_daily WHERE business_date < $1::date`,
|
||||
DeleteSQL: batchDeleteSQL("monitoring_brand_platform_daily", "id", "business_date < $1::date"),
|
||||
},
|
||||
{
|
||||
Name: "monitoring_brand_daily",
|
||||
CountSQL: `SELECT COUNT(*) FROM monitoring_brand_daily WHERE business_date < $1::date`,
|
||||
DeleteSQL: batchDeleteSQL("monitoring_brand_daily", "id", "business_date < $1::date"),
|
||||
},
|
||||
{
|
||||
Name: "monitoring_platform_access_snapshots",
|
||||
CountSQL: `SELECT COUNT(*) FROM monitoring_platform_access_snapshots WHERE business_date < $1::date`,
|
||||
DeleteSQL: batchDeleteSQL("monitoring_platform_access_snapshots", "id", "business_date < $1::date"),
|
||||
},
|
||||
{
|
||||
Name: "monitoring_collect_tasks",
|
||||
CountSQL: `SELECT COUNT(*) FROM monitoring_collect_tasks WHERE business_date < $1::date`,
|
||||
DeleteSQL: batchDeleteSQL("monitoring_collect_tasks", "id", "business_date < $1::date"),
|
||||
},
|
||||
{
|
||||
Name: "question_monitor_runs",
|
||||
CountSQL: `SELECT COUNT(*) FROM question_monitor_runs WHERE business_date < $1::date`,
|
||||
DeleteSQL: batchDeleteSQL("question_monitor_runs", "id", "business_date < $1::date"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func batchDeleteSQL(table, idColumn, predicate string) string {
|
||||
parts := []string{
|
||||
"WITH doomed AS (",
|
||||
"SELECT " + idColumn,
|
||||
"FROM " + table,
|
||||
"WHERE " + predicate,
|
||||
"ORDER BY " + idColumn + " ASC",
|
||||
"LIMIT $2",
|
||||
")",
|
||||
"DELETE FROM " + table,
|
||||
"WHERE " + idColumn + " IN (SELECT " + idColumn + " FROM doomed)",
|
||||
}
|
||||
return strings.Join(parts, "\n")
|
||||
}
|
||||
@@ -173,6 +173,57 @@ func (w *ScheduleDispatchWorker) runOnce(parent context.Context) {
|
||||
w.dispatchBatch(parent, tasks)
|
||||
}
|
||||
|
||||
func (w *ScheduleDispatchWorker) RunOnce(ctx context.Context) (map[string]any, error) {
|
||||
if w == nil || w.pool == nil || w.promptRuleService == nil {
|
||||
return map[string]any{"skipped": true, "reason": "worker_not_ready"}, nil
|
||||
}
|
||||
startedAt := time.Now()
|
||||
|
||||
stageCtx, cancel := w.stageContext(ctx)
|
||||
hydratedCount, err := w.hydrateMissingNextRuns(stageCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return map[string]any{
|
||||
"stage": "hydrate_missing_next_runs",
|
||||
}, err
|
||||
}
|
||||
|
||||
stageCtx, cancel = w.stageContext(ctx)
|
||||
paused, stats, err := w.shouldPauseDispatch(stageCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return map[string]any{
|
||||
"stage": "queue_backpressure",
|
||||
"hydrated_count": hydratedCount,
|
||||
}, err
|
||||
}
|
||||
if paused {
|
||||
return map[string]any{
|
||||
"hydrated_count": hydratedCount,
|
||||
"paused": true,
|
||||
"queue_depth": stats.Messages,
|
||||
"queue_consumers": stats.Consumers,
|
||||
"duration_ms": time.Since(startedAt).Milliseconds(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
stageCtx, cancel = w.stageContext(ctx)
|
||||
tasks, err := w.claimDueSchedules(stageCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return map[string]any{
|
||||
"stage": "claim_due_schedules",
|
||||
"hydrated_count": hydratedCount,
|
||||
}, err
|
||||
}
|
||||
w.dispatchBatch(ctx, tasks)
|
||||
return map[string]any{
|
||||
"hydrated_count": hydratedCount,
|
||||
"claimed_count": len(tasks),
|
||||
"duration_ms": time.Since(startedAt).Milliseconds(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (w *ScheduleDispatchWorker) hydrateMissingNextRuns(ctx context.Context) (int, error) {
|
||||
runtimeCfg := w.runtimeConfig()
|
||||
tx, err := w.pool.Begin(ctx)
|
||||
|
||||
@@ -174,6 +174,42 @@ func (w *MonitoringDailyTaskWorker) runOnce(parent context.Context) {
|
||||
)
|
||||
}
|
||||
|
||||
func (w *MonitoringDailyTaskWorker) RunOnce(parent context.Context) (map[string]any, error) {
|
||||
if w == nil || w.service == nil || w.service.businessPool == nil || w.service.monitoringPool == nil {
|
||||
return map[string]any{"skipped": true, "reason": "worker_not_ready"}, nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(parent, w.timeout)
|
||||
defer cancel()
|
||||
|
||||
startedAt := time.Now()
|
||||
summary, err := w.service.GenerateDailyMonitoringTasks(ctx, w.projectionService, time.Now().UTC())
|
||||
duration := time.Since(startedAt)
|
||||
recordMonitoringDailyTaskMetrics(summary, duration, err)
|
||||
stats := map[string]any{
|
||||
"duration_ms": duration.Milliseconds(),
|
||||
}
|
||||
if summary != nil {
|
||||
stats["business_date"] = summary.BusinessDate
|
||||
stats["tenant_count"] = summary.TenantCount
|
||||
stats["brand_count"] = summary.BrandCount
|
||||
stats["failed_plan_count"] = summary.FailedPlanCount
|
||||
stats["failed_brand_count"] = summary.FailedBrandCount
|
||||
stats["planned_task_count"] = summary.PlannedTaskCount
|
||||
stats["due_task_count"] = summary.DueTaskCount
|
||||
stats["created_task_count"] = summary.CreatedTaskCount
|
||||
stats["desktop_task_count"] = summary.DesktopTaskCount
|
||||
stats["deferred_task_count"] = summary.DeferredTaskCount
|
||||
stats["skipped_plan_count"] = summary.SkippedPlanCount
|
||||
}
|
||||
if err != nil {
|
||||
if w.logger != nil {
|
||||
w.logger.Warn("monitoring daily task generation failed", MonitoringWorkerErrorFields(err)...)
|
||||
}
|
||||
return stats, err
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringService) GenerateDailyMonitoringTasks(
|
||||
ctx context.Context,
|
||||
projectionService *MonitoringCallbackService,
|
||||
|
||||
@@ -92,6 +92,34 @@ func (w *GenerationTaskStateCheckWorker) runOnce(parent context.Context) {
|
||||
}, zap.Int("anomaly_count", count))
|
||||
}
|
||||
|
||||
func (w *GenerationTaskStateCheckWorker) RunOnce(parent context.Context) (map[string]any, error) {
|
||||
if w == nil || w.pool == nil {
|
||||
return map[string]any{"skipped": true, "reason": "worker_not_ready"}, nil
|
||||
}
|
||||
cfg := w.runtimeConfig()
|
||||
startedAt := time.Now()
|
||||
count, err := CheckGenerationTaskStateConsistency(parent, w.pool, w.logger, w.cache, cfg)
|
||||
duration := time.Since(startedAt)
|
||||
stats := map[string]any{
|
||||
"anomaly_count": count,
|
||||
"duration_ms": duration.Milliseconds(),
|
||||
"batch_size": cfg.BatchSize,
|
||||
"lookback_sec": int64(cfg.Lookback.Seconds()),
|
||||
}
|
||||
if err != nil {
|
||||
logCtx := tenantapp.GenerationTaskLogContext{
|
||||
Stage: "state_check",
|
||||
Result: tenantapp.GenerationTaskResultFailure,
|
||||
Severity: "warning",
|
||||
Duration: duration,
|
||||
}
|
||||
tenantapp.RecordGenerationTaskEvent(tenantapp.GenerationTaskEventStateCheckFailure, logCtx)
|
||||
tenantapp.LogGenerationTaskWarn(w.logger, "article generation state consistency check failed", logCtx, zap.Error(err))
|
||||
return stats, err
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (w *GenerationTaskStateCheckWorker) runtimeConfig() generationTaskStateCheckConfig {
|
||||
if w != nil && w.provider != nil {
|
||||
if cfg := w.provider.Current(); cfg != nil {
|
||||
|
||||
Reference in New Issue
Block a user