perf(scheduler): harden jobs for scalable execution

server/internal/scheduler/monitoring_lease_recovery_worker.go

@@ -13,6 +13,7 @@ import (
 const (
 	defaultMonitoringLeaseRecoveryInterval = 30 * time.Minute
 	defaultMonitoringLeaseRecoveryTimeout  = 30 * time.Second
+	defaultMonitoringLeaseRecoveryLimit    = 1000
 )
 
 type MonitoringLeaseRecoveryWorker struct {
@@ -20,6 +21,7 @@ type MonitoringLeaseRecoveryWorker struct {
 	logger         *zap.Logger
 	interval       time.Duration
 	timeout        time.Duration
+	limit          int
 }
 
 func NewMonitoringLeaseRecoveryWorker(monitoringPool *pgxpool.Pool, logger *zap.Logger) *MonitoringLeaseRecoveryWorker {
@@ -28,6 +30,7 @@ func NewMonitoringLeaseRecoveryWorker(monitoringPool *pgxpool.Pool, logger *zap.
 		logger:         logger,
 		interval:       defaultMonitoringLeaseRecoveryInterval,
 		timeout:        defaultMonitoringLeaseRecoveryTimeout,
+		limit:          defaultMonitoringLeaseRecoveryLimit,
 	}
 }
 
@@ -71,7 +74,7 @@ func (w *MonitoringLeaseRecoveryWorker) runOnce(parent context.Context) {
 	}
 	defer tx.Rollback(ctx)
 
-	expiredCount, err := tenantapp.ExpireMonitoringLeasedTasks(ctx, tx, nil, time.Now().UTC())
+	expiredCount, err := tenantapp.ExpireMonitoringLeasedTasks(ctx, tx, nil, time.Now().UTC(), w.limit)
 	if err != nil {
 		if w.logger != nil {
 			w.logger.Warn("monitoring lease recovery failed", tenantapp.MonitoringWorkerErrorFields(err)...)
@@ -91,12 +94,23 @@ func (w *MonitoringLeaseRecoveryWorker) runOnce(parent context.Context) {
 	}
 }
 
-func (w *MonitoringLeaseRecoveryWorker) RunOnce(parent context.Context) (map[string]any, error) {
+func (w *MonitoringLeaseRecoveryWorker) RunOnce(parent context.Context, run JobRunContext) (map[string]any, error) {
 	if w == nil || w.monitoringPool == nil {
 		return map[string]any{"skipped": true, "reason": "worker_not_ready"}, nil
 	}
 	startedAt := time.Now()
-	ctx, cancel := context.WithTimeout(parent, w.timeout)
+	timeout := w.timeout
+	if run.Job != nil && run.Job.TimeoutSeconds > 0 {
+		timeout = time.Duration(run.Job.TimeoutSeconds) * time.Second
+	}
+	limit := w.limit
+	if run.Job != nil && run.Job.BatchSize != nil && *run.Job.BatchSize > 0 {
+		limit = *run.Job.BatchSize
+	}
+	if value := AsInt(run.Config, "batch_size", 0); value > 0 && (run.Job == nil || run.Job.BatchSize == nil) {
+		limit = value
+	}
+	ctx, cancel := context.WithTimeout(parent, timeout)
 	defer cancel()
 
 	tx, err := w.monitoringPool.Begin(ctx)
@@ -105,7 +119,7 @@ func (w *MonitoringLeaseRecoveryWorker) RunOnce(parent context.Context) (map[str
 	}
 	defer tx.Rollback(ctx)
 
-	expiredCount, err := tenantapp.ExpireMonitoringLeasedTasks(ctx, tx, nil, time.Now().UTC())
+	expiredCount, err := tenantapp.ExpireMonitoringLeasedTasks(ctx, tx, nil, time.Now().UTC(), limit)
 	if err != nil {
 		return map[string]any{"stage": "expire"}, err
 	}
@@ -115,5 +129,6 @@ func (w *MonitoringLeaseRecoveryWorker) RunOnce(parent context.Context) (map[str
 	return map[string]any{
 		"expired_task_count": expiredCount,
 		"duration_ms":        time.Since(startedAt).Milliseconds(),
+		"batch_size":         limit,
 	}, nil
 }