root/geo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(scheduler): harden jobs for scalable execution

Browse Source
This commit is contained in:
liangxu
2026-05-20 12:19:27 +08:00
parent 4b09a34748
commit 5fb9d0b0dd
23 changed files with 887 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/cmd/scheduler/main.go
+61 -12
View File
@@ -99,38 +99,47 @@ func main() {
controlPlane := internalscheduler.NewControlPlane(app.DB, app.Logger, []internalscheduler.ControlledJob{
{
Key: "schedule_dispatch",
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
return scheduleDispatchWorker.RunOnce(runCtx)
Run: func(runCtx context.Context, jobCtx internalscheduler.JobRunContext) (map[string]any, error) {
return scheduleDispatchWorker.RunOnce(runCtx, jobCtx)
},
},
{
Key: "monitoring_daily_task",
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
return monitoringDailyTaskWorker.RunOnce(runCtx)
Run: func(runCtx context.Context, jobCtx internalscheduler.JobRunContext) (map[string]any, error) {
timeout := 45 * time.Second
if jobCtx.Job != nil && jobCtx.Job.TimeoutSeconds > 0 {
timeout = time.Duration(jobCtx.Job.TimeoutSeconds) * time.Second
}
if timeout <= 0 {
timeout = 45 * time.Second
}
taskCtx, cancel := context.WithTimeout(runCtx, timeout)
defer cancel()
return monitoringDailyTaskWorker.RunOnce(taskCtx, monitoringDailyRunOptions(jobCtx))
},
},
{
Key: "monitoring_result_recovery",
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
return monitoringResultRecoveryWorker.RunOnce(runCtx)
Run: func(runCtx context.Context, jobCtx internalscheduler.JobRunContext) (map[string]any, error) {
return monitoringResultRecoveryWorker.RunOnce(runCtx, jobCtx)
},
},
{
Key: "monitoring_lease_recovery",
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
return monitoringLeaseRecoveryWorker.RunOnce(runCtx)
Run: func(runCtx context.Context, jobCtx internalscheduler.JobRunContext) (map[string]any, error) {
return monitoringLeaseRecoveryWorker.RunOnce(runCtx, jobCtx)
},
},
{
Key: "monitoring_received_inspection",
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
return monitoringReceivedInspectionWorker.RunOnce(runCtx)
Run: func(runCtx context.Context, jobCtx internalscheduler.JobRunContext) (map[string]any, error) {
return monitoringReceivedInspectionWorker.RunOnce(runCtx, jobCtx)
},
},
{
Key: "generation_state_check",
Run: func(runCtx context.Context, _ internalscheduler.JobRunContext) (map[string]any, error) {
return generationStateCheckWorker.RunOnce(runCtx)
Run: func(runCtx context.Context, jobCtx internalscheduler.JobRunContext) (map[string]any, error) {
return generationStateCheckWorker.RunOnce(runCtx, generationStateCheckRunOptions(jobCtx))
},
},
{
@@ -260,6 +269,46 @@ func shutdownSchedulerMetricsServer(server *http.Server, logger interface {
logger.Infof("scheduler metrics http server stopped")
}
func monitoringDailyRunOptions(jobCtx internalscheduler.JobRunContext) tenantapp.MonitoringDailyTaskRunOptions {
batchSize := 0
if jobCtx.Job != nil && jobCtx.Job.BatchSize != nil {
batchSize = *jobCtx.Job.BatchSize
}
if batchSize <= 0 {
batchSize = internalscheduler.AsInt(jobCtx.Config, "max_materialize_per_run", 0)
}
if batchSize <= 0 {
batchSize = internalscheduler.AsInt(jobCtx.Config, "batch_size", 0)
}
return tenantapp.MonitoringDailyTaskRunOptions{
MaxMaterializePerRun: batchSize,
MaxMaterializePerPlan: internalscheduler.AsInt(jobCtx.Config, "max_materialize_per_plan", 0),
MaxMaterializePerBrand: internalscheduler.AsInt(jobCtx.Config, "max_materialize_per_brand", 0),
}
}
func generationStateCheckRunOptions(jobCtx internalscheduler.JobRunContext) generateworker.GenerationTaskStateCheckRunOptions {
batchSize := 0
if jobCtx.Job != nil && jobCtx.Job.BatchSize != nil {
batchSize = *jobCtx.Job.BatchSize
}
if batchSize <= 0 {
batchSize = internalscheduler.AsInt(jobCtx.Config, "batch_size", 0)
}
timeout := time.Duration(0)
if jobCtx.Job != nil && jobCtx.Job.TimeoutSeconds > 0 {
timeout = time.Duration(jobCtx.Job.TimeoutSeconds) * time.Second
}
if timeout <= 0 {
timeout = internalscheduler.AsDuration(jobCtx.Config, "timeout", 0)
}
return generateworker.GenerationTaskStateCheckRunOptions{
Timeout: timeout,
BatchSize: batchSize,
Lookback: internalscheduler.AsDuration(jobCtx.Config, "lookback", 0),
}
}
func schedulerHTTPAddr(host string, port int) string {
host = strings.TrimSpace(host)
if host == "" {
server/cmd/scheduler/main_test.go
+45
View File
@@ -9,6 +9,9 @@ import (
"time"
"github.com/gin-gonic/gin"
opsdomain "github.com/geo-platform/tenant-api/internal/ops/domain"
internalscheduler "github.com/geo-platform/tenant-api/internal/scheduler"
)
func TestSchedulerHTTPAddrDefaultsToLoopback(t *testing.T) {
@@ -56,6 +59,48 @@ func TestSchedulerMetricsAuthMiddleware(t *testing.T) {
}
}
func TestMonitoringDailyRunOptionsUsesJobBatchSize(t *testing.T) {
batchSize := 88
got := monitoringDailyRunOptions(internalscheduler.JobRunContext{
Job: &opsdomain.SchedulerJob{BatchSize: &batchSize},
Config: map[string]any{
"max_materialize_per_run": 999,
"max_materialize_per_plan": 12,
"max_materialize_per_brand": 4,
},
})
if got.MaxMaterializePerRun != 88 {
t.Fatalf("MaxMaterializePerRun = %d, want 88", got.MaxMaterializePerRun)
}
if got.MaxMaterializePerPlan != 12 || got.MaxMaterializePerBrand != 4 {
t.Fatalf("unexpected plan/brand options: %#v", got)
}
}
func TestGenerationStateCheckRunOptionsUsesOpsConfig(t *testing.T) {
batchSize := 50
got := generationStateCheckRunOptions(internalscheduler.JobRunContext{
Job: &opsdomain.SchedulerJob{
BatchSize: &batchSize,
TimeoutSeconds: 7,
},
Config: map[string]any{
"lookback": "2h",
},
})
if got.BatchSize != 50 {
t.Fatalf("BatchSize = %d, want 50", got.BatchSize)
}
if got.Timeout != 7*time.Second {
t.Fatalf("Timeout = %s, want 7s", got.Timeout)
}
if got.Lookback != 2*time.Hour {
t.Fatalf("Lookback = %s, want 2h", got.Lookback)
}
}
func TestStartSchedulerWorkerWaitsForRunToReturn(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()