root/geo
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

perf(scheduler): harden jobs for scalable execution

Browse Source
This commit is contained in:
liangxu
2026-05-20 12:19:27 +08:00
parent 4b09a34748
commit 5fb9d0b0dd
23 changed files with 887 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/scheduler/schedule_dispatch_worker.go
+53 -21
View File
@@ -130,8 +130,9 @@ func (w *ScheduleDispatchWorker) run(ctx context.Context) {
}
func (w *ScheduleDispatchWorker) runOnce(parent context.Context) {
ctx, cancel := w.stageContext(parent)
count, err := w.hydrateMissingNextRuns(ctx)
runtimeCfg := w.runtimeConfig()
ctx, cancel := w.stageContext(parent, runtimeCfg)
count, err := w.hydrateMissingNextRuns(ctx, runtimeCfg)
cancel()
if err != nil {
if w.logger != nil {
@@ -141,8 +142,8 @@ func (w *ScheduleDispatchWorker) runOnce(parent context.Context) {
w.logger.Info("schedule dispatch hydrated next_run_at", zap.Int("task_count", count))
}
ctx, cancel = w.stageContext(parent)
paused, stats, err := w.shouldPauseDispatch(ctx)
ctx, cancel = w.stageContext(parent, runtimeCfg)
paused, stats, err := w.shouldPauseDispatch(ctx, runtimeCfg)
cancel()
if err != nil {
if w.logger != nil {
@@ -160,8 +161,8 @@ func (w *ScheduleDispatchWorker) runOnce(parent context.Context) {
return
}
ctx, cancel = w.stageContext(parent)
tasks, err := w.claimDueSchedules(ctx)
ctx, cancel = w.stageContext(parent, runtimeCfg)
tasks, err := w.claimDueSchedules(ctx, runtimeCfg)
cancel()
if err != nil {
if w.logger != nil {
@@ -173,14 +174,16 @@ func (w *ScheduleDispatchWorker) runOnce(parent context.Context) {
w.dispatchBatch(parent, tasks)
}
func (w *ScheduleDispatchWorker) RunOnce(ctx context.Context) (map[string]any, error) {
func (w *ScheduleDispatchWorker) RunOnce(ctx context.Context, run JobRunContext) (map[string]any, error) {
if w == nil || w.pool == nil || w.promptRuleService == nil {
return map[string]any{"skipped": true, "reason": "worker_not_ready"}, nil
}
startedAt := time.Now()
runtimeCfg := w.runtimeConfig()
runtimeCfg.ApplyJobRun(run)
stageCtx, cancel := w.stageContext(ctx)
hydratedCount, err := w.hydrateMissingNextRuns(stageCtx)
stageCtx, cancel := w.stageContext(ctx, runtimeCfg)
hydratedCount, err := w.hydrateMissingNextRuns(stageCtx, runtimeCfg)
cancel()
if err != nil {
return map[string]any{
@@ -188,8 +191,8 @@ func (w *ScheduleDispatchWorker) RunOnce(ctx context.Context) (map[string]any, e
}, err
}
stageCtx, cancel = w.stageContext(ctx)
paused, stats, err := w.shouldPauseDispatch(stageCtx)
stageCtx, cancel = w.stageContext(ctx, runtimeCfg)
paused, stats, err := w.shouldPauseDispatch(stageCtx, runtimeCfg)
cancel()
if err != nil {
return map[string]any{
@@ -204,11 +207,12 @@ func (w *ScheduleDispatchWorker) RunOnce(ctx context.Context) (map[string]any, e
"queue_depth": stats.Messages,
"queue_consumers": stats.Consumers,
"duration_ms": time.Since(startedAt).Milliseconds(),
"batch_size": runtimeCfg.BatchSize,
}, nil
}
stageCtx, cancel = w.stageContext(ctx)
tasks, err := w.claimDueSchedules(stageCtx)
stageCtx, cancel = w.stageContext(ctx, runtimeCfg)
tasks, err := w.claimDueSchedules(stageCtx, runtimeCfg)
cancel()
if err != nil {
return map[string]any{
@@ -221,11 +225,11 @@ func (w *ScheduleDispatchWorker) RunOnce(ctx context.Context) (map[string]any, e
"hydrated_count": hydratedCount,
"claimed_count": len(tasks),
"duration_ms": time.Since(startedAt).Milliseconds(),
"batch_size": runtimeCfg.BatchSize,
}, nil
}
func (w *ScheduleDispatchWorker) hydrateMissingNextRuns(ctx context.Context) (int, error) {
runtimeCfg := w.runtimeConfig()
func (w *ScheduleDispatchWorker) hydrateMissingNextRuns(ctx context.Context, runtimeCfg scheduleDispatchRuntimeConfig) (int, error) {
tx, err := w.pool.Begin(ctx)
if err != nil {
return 0, err
@@ -318,8 +322,7 @@ func (w *ScheduleDispatchWorker) hydrateMissingNextRuns(ctx context.Context) (in
return count, nil
}
func (w *ScheduleDispatchWorker) claimDueSchedules(ctx context.Context) ([]dueScheduleTask, error) {
runtimeCfg := w.runtimeConfig()
func (w *ScheduleDispatchWorker) claimDueSchedules(ctx context.Context, runtimeCfg scheduleDispatchRuntimeConfig) ([]dueScheduleTask, error) {
tx, err := w.pool.Begin(ctx)
if err != nil {
return nil, err
@@ -539,11 +542,10 @@ func (w *ScheduleDispatchWorker) dispatchBatch(parent context.Context, tasks []d
wg.Wait()
}
func (w *ScheduleDispatchWorker) shouldPauseDispatch(ctx context.Context) (bool, rabbitmq.QueueStats, error) {
func (w *ScheduleDispatchWorker) shouldPauseDispatch(ctx context.Context, runtimeCfg scheduleDispatchRuntimeConfig) (bool, rabbitmq.QueueStats, error) {
if w == nil || w.rabbitMQ == nil {
return false, rabbitmq.QueueStats{}, nil
}
runtimeCfg := w.runtimeConfig()
if runtimeCfg.QueueBackpressure <= 0 {
return false, rabbitmq.QueueStats{}, nil
}
@@ -565,6 +567,32 @@ func scheduleDispatchRuntimeFromConfig(cfg config.SchedulerConfig) scheduleDispa
}
}
func (cfg *scheduleDispatchRuntimeConfig) ApplyJobRun(run JobRunContext) {
if cfg == nil {
return
}
if run.Job != nil {
if run.Job.BatchSize != nil && *run.Job.BatchSize > 0 {
cfg.BatchSize = *run.Job.BatchSize
}
if run.Job.TimeoutSeconds > 0 {
cfg.Timeout = time.Duration(run.Job.TimeoutSeconds) * time.Second
}
}
if value := AsInt(run.Config, "dispatch_workers", 0); value > 0 {
cfg.DispatchWorkers = value
}
if value := AsInt(run.Config, "queue_backpressure", 0); value > 0 {
cfg.QueueBackpressure = value
}
if value := AsInt(run.Config, "batch_size", 0); value > 0 && (run.Job == nil || run.Job.BatchSize == nil) {
cfg.BatchSize = value
}
if timeout := AsDuration(run.Config, "timeout", 0); timeout > 0 && (run.Job == nil || run.Job.TimeoutSeconds <= 0) {
cfg.Timeout = timeout
}
}
func schedulerDispatchInterval(cfg config.SchedulerConfig) time.Duration {
if cfg.DispatchInterval > 0 {
return cfg.DispatchInterval
@@ -593,11 +621,15 @@ func schedulerDispatchConcurrency(cfg config.SchedulerConfig) int {
return 1
}
func (w *ScheduleDispatchWorker) stageContext(parent context.Context) (context.Context, context.CancelFunc) {
func (w *ScheduleDispatchWorker) stageContext(parent context.Context, runtimeCfg ...scheduleDispatchRuntimeConfig) (context.Context, context.CancelFunc) {
if parent == nil {
parent = context.Background()
}
return context.WithTimeout(parent, w.runtimeConfig().Timeout)
cfg := w.runtimeConfig()
if len(runtimeCfg) > 0 {
cfg = runtimeCfg[0]
}
return context.WithTimeout(parent, cfg.Timeout)
}
func (w *ScheduleDispatchWorker) invalidateScheduleTaskCaches(updates []scheduleTaskCacheUpdate) {