root/geo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(scheduler): harden jobs for scalable execution

Browse Source
This commit is contained in:
liangxu
2026-05-20 12:19:27 +08:00
parent 4b09a34748
commit 5fb9d0b0dd
23 changed files with 887 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/cmd/scheduler/main.go
+61 -12
View File
@@ -99,38 +99,47 @@ func main() {
controlPlane := internalscheduler.NewControlPlane(app.DB, app.Logger, []internalscheduler.ControlledJob{
{
Key: "schedule_dispatch",
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
return scheduleDispatchWorker.RunOnce(runCtx)
Run: func(runCtx context.Context, jobCtx internalscheduler.JobRunContext) (map[string]any, error) {
return scheduleDispatchWorker.RunOnce(runCtx, jobCtx)
},
},
{
Key: "monitoring_daily_task",
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
return monitoringDailyTaskWorker.RunOnce(runCtx)
Run: func(runCtx context.Context, jobCtx internalscheduler.JobRunContext) (map[string]any, error) {
timeout := 45 * time.Second
if jobCtx.Job != nil && jobCtx.Job.TimeoutSeconds > 0 {
timeout = time.Duration(jobCtx.Job.TimeoutSeconds) * time.Second
}
if timeout <= 0 {
timeout = 45 * time.Second
}
taskCtx, cancel := context.WithTimeout(runCtx, timeout)
defer cancel()
return monitoringDailyTaskWorker.RunOnce(taskCtx, monitoringDailyRunOptions(jobCtx))
},
},
{
Key: "monitoring_result_recovery",
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
return monitoringResultRecoveryWorker.RunOnce(runCtx)
Run: func(runCtx context.Context, jobCtx internalscheduler.JobRunContext) (map[string]any, error) {
return monitoringResultRecoveryWorker.RunOnce(runCtx, jobCtx)
},
},
{
Key: "monitoring_lease_recovery",
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
return monitoringLeaseRecoveryWorker.RunOnce(runCtx)
Run: func(runCtx context.Context, jobCtx internalscheduler.JobRunContext) (map[string]any, error) {
return monitoringLeaseRecoveryWorker.RunOnce(runCtx, jobCtx)
},
},
{
Key: "monitoring_received_inspection",
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
return monitoringReceivedInspectionWorker.RunOnce(runCtx)
Run: func(runCtx context.Context, jobCtx internalscheduler.JobRunContext) (map[string]any, error) {
return monitoringReceivedInspectionWorker.RunOnce(runCtx, jobCtx)
},
},
{
Key: "generation_state_check",
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
return generationStateCheckWorker.RunOnce(runCtx)
Run: func(runCtx context.Context, jobCtx internalscheduler.JobRunContext) (map[string]any, error) {
return generationStateCheckWorker.RunOnce(runCtx, generationStateCheckRunOptions(jobCtx))
},
},
{
@@ -260,6 +269,46 @@ func shutdownSchedulerMetricsServer(server *http.Server, logger interface {
logger.Infof("scheduler metrics http server stopped")
}
func monitoringDailyRunOptions(jobCtx internalscheduler.JobRunContext) tenantapp.MonitoringDailyTaskRunOptions {
batchSize := 0
if jobCtx.Job != nil && jobCtx.Job.BatchSize != nil {
batchSize = *jobCtx.Job.BatchSize
}
if batchSize <= 0 {
batchSize = internalscheduler.AsInt(jobCtx.Config, "max_materialize_per_run", 0)
}
if batchSize <= 0 {
batchSize = internalscheduler.AsInt(jobCtx.Config, "batch_size", 0)
}
return tenantapp.MonitoringDailyTaskRunOptions{
MaxMaterializePerRun: batchSize,
MaxMaterializePerPlan: internalscheduler.AsInt(jobCtx.Config, "max_materialize_per_plan", 0),
MaxMaterializePerBrand: internalscheduler.AsInt(jobCtx.Config, "max_materialize_per_brand", 0),
}
}
func generationStateCheckRunOptions(jobCtx internalscheduler.JobRunContext) generateworker.GenerationTaskStateCheckRunOptions {
batchSize := 0
if jobCtx.Job != nil && jobCtx.Job.BatchSize != nil {
batchSize = *jobCtx.Job.BatchSize
}
if batchSize <= 0 {
batchSize = internalscheduler.AsInt(jobCtx.Config, "batch_size", 0)
}
timeout := time.Duration(0)
if jobCtx.Job != nil && jobCtx.Job.TimeoutSeconds > 0 {
timeout = time.Duration(jobCtx.Job.TimeoutSeconds) * time.Second
}
if timeout <= 0 {
timeout = internalscheduler.AsDuration(jobCtx.Config, "timeout", 0)
}
return generateworker.GenerationTaskStateCheckRunOptions{
Timeout: timeout,
BatchSize: batchSize,
Lookback: internalscheduler.AsDuration(jobCtx.Config, "lookback", 0),
}
}
func schedulerHTTPAddr(host string, port int) string {
host = strings.TrimSpace(host)
if host == "" {