perf(scheduler): harden jobs for scalable execution

server/internal/scheduler/control_plane.go

@@ -33,6 +33,13 @@ type JobRunContext struct {
 	Config    map[string]any
 }
 
+const (
+	schedulerManualPollInterval     = 15 * time.Second
+	schedulerConfigRetryInterval    = 30 * time.Second
+	schedulerFinishTimeout          = 10 * time.Second
+	schedulerMaxManualTriggersDrain = 100
+)
+
 type ControlPlane struct {
 	pool       *pgxpool.Pool
 	repo       *opsrepo.SchedulerRepository
@@ -137,36 +144,35 @@ func (p *ControlPlane) heartbeat(ctx context.Context) {
 }
 
 func (p *ControlPlane) runJobLoop(ctx context.Context, job ControlledJob) {
-	nextAutoAt := p.nextAutoAt(context.Background(), job.Key, time.Now(), true)
+	nextAutoAt := p.nextAutoAt(ctx, job.Key, time.Now(), true)
 
 	for {
-		p.drainManualTriggers(context.Background(), job)
+		if ctx.Err() != nil {
+			return
+		}
+		p.drainManualTriggers(ctx, job)
 		now := time.Now()
 		if !now.Before(nextAutoAt) {
-			p.runJobOnce(context.Background(), job, opsdomain.SchedulerTriggerAuto, nil, nil)
-			nextAutoAt = p.nextAutoAt(context.Background(), job.Key, time.Now(), false)
-		}
-		sleepFor := time.Until(nextAutoAt)
-		if sleepFor > 15*time.Second {
-			sleepFor = 15 * time.Second
-		}
-		if sleepFor <= 0 {
-			sleepFor = time.Second
+			p.runJobOnce(ctx, job, opsdomain.SchedulerTriggerAuto, nil, nil)
+			nextAutoAt = p.nextAutoAt(ctx, job.Key, time.Now(), false)
+			continue
 		}
+		sleepFor := schedulerLoopSleep(time.Now(), nextAutoAt)
 		timer := time.NewTimer(sleepFor)
 		select {
 		case <-ctx.Done():
 			timer.Stop()
 			return
 		case <-timer.C:
-			p.drainManualTriggers(context.Background(), job)
-			p.runJobOnce(context.Background(), job, opsdomain.SchedulerTriggerAuto, nil, nil)
 		}
 	}
 }
 
 func (p *ControlPlane) drainManualTriggers(parent context.Context, job ControlledJob) {
-	for {
+	for claimed := 0; claimed < schedulerMaxManualTriggersDrain; claimed++ {
+		if parent.Err() != nil {
+			return
+		}
 		trigger, ok, err := p.repo.ClaimPendingTrigger(parent, job.Key, p.instanceID)
 		if err != nil {
 			if p.logger != nil {
@@ -180,15 +186,21 @@ func (p *ControlPlane) drainManualTriggers(parent context.Context, job Controlle
 		triggerID := trigger.ID
 		p.runJobOnce(parent, job, trigger.TriggerType, &triggerID, trigger.ConfigOverride)
 	}
+	if p.logger != nil {
+		p.logger.Warn("scheduler manual trigger drain reached batch limit",
+			zap.String("job_key", job.Key),
+			zap.Int("limit", schedulerMaxManualTriggersDrain),
+		)
+	}
 }
 
 func (p *ControlPlane) nextAutoAt(ctx context.Context, key string, now time.Time, initial bool) time.Time {
 	job, err := p.repo.GetJobForRuntime(ctx, key)
 	if err != nil || job == nil {
-		return now.Add(30 * time.Second)
+		return now.Add(schedulerConfigRetryInterval)
 	}
 	if job.ScheduleType == "manual" {
-		return now.Add(15 * time.Second)
+		return now.Add(schedulerManualPollInterval)
 	}
 	if job.ScheduleType == "cron" && job.CronExpr != nil {
 		loc, locErr := time.LoadLocation(job.Timezone)
@@ -204,7 +216,7 @@ func (p *ControlPlane) nextAutoAt(ctx context.Context, key string, now time.Time
 					zap.Error(parseErr),
 				)
 			}
-			return now.Add(30 * time.Second)
+			return now.Add(schedulerConfigRetryInterval)
 		}
 		base := now.In(loc)
 		return schedule.Next(base)
@@ -226,12 +238,23 @@ func (p *ControlPlane) nextAutoAt(ctx context.Context, key string, now time.Time
 	}
 	interval := time.Duration(job.IntervalSeconds) * time.Second
 	if interval <= 0 {
-		interval = 30 * time.Second
+		interval = schedulerConfigRetryInterval
 	}
+	var lastStartedAt *time.Time
 	if initial {
-		return now
+		startedAt, ok, lastErr := p.repo.LastAutoRunStartedAt(ctx, key)
+		if lastErr != nil {
+			if p.logger != nil {
+				p.logger.Warn("scheduler last auto run lookup failed",
+					zap.String("job_key", key),
+					zap.Error(lastErr),
+				)
+			}
+		} else if ok {
+			lastStartedAt = &startedAt
+		}
 	}
-	return now.Add(interval)
+	return nextIntervalAutoAt(now, interval, initial, lastStartedAt)
 }
 
 func (p *ControlPlane) runJobOnce(parent context.Context, job ControlledJob, triggerType string, triggerID *int64, configOverride map[string]any) {
@@ -313,7 +336,9 @@ func (p *ControlPlane) runJobOnce(parent context.Context, job ControlledJob, tri
 	}
 	stats["dry_run"] = dryRun
 
-	finished, finishErr := p.repo.FinishRun(parent, opsrepo.SchedulerRunFinish{
+	finishCtx, finishCancel := context.WithTimeout(context.Background(), schedulerFinishTimeout)
+	defer finishCancel()
+	finished, finishErr := p.repo.FinishRun(finishCtx, opsrepo.SchedulerRunFinish{
 		RunID:        run.ID,
 		Status:       status,
 		Stats:        stats,
@@ -328,7 +353,7 @@ func (p *ControlPlane) runJobOnce(parent context.Context, job ControlledJob, tri
 			triggerStatus = opsdomain.SchedulerRunFailed
 		}
 		runID := run.ID
-		_ = p.repo.FinishTrigger(parent, *triggerID, &runID, triggerStatus, errMsg)
+		_ = p.repo.FinishTrigger(finishCtx, *triggerID, &runID, triggerStatus, errMsg)
 	}
 	if runErr != nil {
 		if p.logger != nil {
@@ -382,6 +407,33 @@ func advisoryLockKey(value string) int64 {
 	return int64(hash.Sum64() & 0x7fffffffffffffff)
 }
 
+func schedulerLoopSleep(now, nextAutoAt time.Time) time.Duration {
+	sleepFor := nextAutoAt.Sub(now)
+	if sleepFor > schedulerManualPollInterval {
+		sleepFor = schedulerManualPollInterval
+	}
+	if sleepFor <= 0 {
+		sleepFor = time.Second
+	}
+	return sleepFor
+}
+
+func nextIntervalAutoAt(now time.Time, interval time.Duration, initial bool, lastStartedAt *time.Time) time.Time {
+	if interval <= 0 {
+		interval = schedulerConfigRetryInterval
+	}
+	if initial {
+		if lastStartedAt != nil && !lastStartedAt.IsZero() {
+			dueAt := lastStartedAt.Add(interval)
+			if dueAt.After(now) {
+				return dueAt
+			}
+		}
+		return now
+	}
+	return now.Add(interval)
+}
+
 func schedulerConfigSnapshot(job *opsdomain.SchedulerJob, config map[string]any) map[string]any {
 	out := map[string]any{
 		"enabled":          job.Enabled,