feat(scheduler): random run-window scheduling and deduped manual triggers
Support a random daily run window (random_window_start/end) where the next auto-run time is derived from a stable per-job hash, avoiding thundering-herd starts while keeping one run per day. Honors the last auto-run on initial scheduling so restarts don't double-fire. Manual triggers can opt into dedupe (dedupe_manual_triggers): an advisory-locked path collapses concurrent requests onto an existing pending/running job instead of queueing duplicates. Scheduler error messages are localized to Chinese. Register the media_supply_cache_sync job (dry-run aware) and add the ops migration that seeds it.
This commit is contained in:
@@ -222,6 +222,27 @@ func (p *ControlPlane) nextAutoAt(ctx context.Context, key string, now time.Time
|
||||
base := now.In(loc)
|
||||
return schedule.Next(base)
|
||||
}
|
||||
if window, ok := parseRandomRunWindowLocal(job.Config); ok {
|
||||
loc, locErr := time.LoadLocation(job.Timezone)
|
||||
if locErr != nil {
|
||||
loc = time.Local
|
||||
}
|
||||
var lastStartedAt *time.Time
|
||||
if initial {
|
||||
startedAt, ok, lastErr := p.repo.LastAutoRunStartedAt(ctx, key)
|
||||
if lastErr != nil {
|
||||
if p.logger != nil {
|
||||
p.logger.Warn("scheduler last auto run lookup failed",
|
||||
zap.String("job_key", key),
|
||||
zap.Error(lastErr),
|
||||
)
|
||||
}
|
||||
} else if ok {
|
||||
lastStartedAt = &startedAt
|
||||
}
|
||||
}
|
||||
return nextRandomWindowAutoAt(key, now, loc, window, initial, lastStartedAt)
|
||||
}
|
||||
if runAt := parseRunAtLocal(job.Config); runAt != "" {
|
||||
loc, locErr := time.LoadLocation(job.Timezone)
|
||||
if locErr != nil {
|
||||
@@ -629,6 +650,93 @@ func parseRunAtLocal(config map[string]any) string {
|
||||
return fmt.Sprint(value)
|
||||
}
|
||||
|
||||
type randomRunWindowLocal struct {
|
||||
StartHour int
|
||||
StartMinute int
|
||||
EndHour int
|
||||
EndMinute int
|
||||
}
|
||||
|
||||
func parseRandomRunWindowLocal(config map[string]any) (randomRunWindowLocal, bool) {
|
||||
start := firstConfigString(config, "random_window_start", "random_window_start_local", "random_run_window_start")
|
||||
end := firstConfigString(config, "random_window_end", "random_window_end_local", "random_run_window_end")
|
||||
if start == "" || end == "" {
|
||||
return randomRunWindowLocal{}, false
|
||||
}
|
||||
startHour, startMinute, startOK := parseHourMinute(start)
|
||||
endHour, endMinute, endOK := parseHourMinute(end)
|
||||
startTotal := startHour*60 + startMinute
|
||||
endTotal := endHour*60 + endMinute
|
||||
if !startOK || !endOK || endTotal <= startTotal {
|
||||
return randomRunWindowLocal{}, false
|
||||
}
|
||||
return randomRunWindowLocal{
|
||||
StartHour: startHour,
|
||||
StartMinute: startMinute,
|
||||
EndHour: endHour,
|
||||
EndMinute: endMinute,
|
||||
}, true
|
||||
}
|
||||
|
||||
func nextRandomWindowAutoAt(key string, now time.Time, loc *time.Location, window randomRunWindowLocal, initial bool, lastStartedAt *time.Time) time.Time {
|
||||
if loc == nil {
|
||||
loc = time.Local
|
||||
}
|
||||
localNow := now.In(loc)
|
||||
todayStart := time.Date(localNow.Year(), localNow.Month(), localNow.Day(), window.StartHour, window.StartMinute, 0, 0, loc)
|
||||
todayEnd := time.Date(localNow.Year(), localNow.Month(), localNow.Day(), window.EndHour, window.EndMinute, 0, 0, loc)
|
||||
if !todayEnd.After(todayStart) {
|
||||
return now.Add(schedulerConfigRetryInterval)
|
||||
}
|
||||
todayCandidate := randomWindowCandidate(key, todayStart, todayEnd)
|
||||
if !initial {
|
||||
tomorrowStart := todayStart.AddDate(0, 0, 1)
|
||||
return randomWindowCandidate(key, tomorrowStart, tomorrowStart.Add(todayEnd.Sub(todayStart)))
|
||||
}
|
||||
if lastStartedAt != nil && sameLocalDate((*lastStartedAt).In(loc), localNow) {
|
||||
tomorrowStart := todayStart.AddDate(0, 0, 1)
|
||||
return randomWindowCandidate(key, tomorrowStart, tomorrowStart.Add(todayEnd.Sub(todayStart)))
|
||||
}
|
||||
if localNow.Before(todayCandidate) {
|
||||
return todayCandidate
|
||||
}
|
||||
if localNow.Before(todayEnd) {
|
||||
return now
|
||||
}
|
||||
tomorrowStart := todayStart.AddDate(0, 0, 1)
|
||||
return randomWindowCandidate(key, tomorrowStart, tomorrowStart.Add(todayEnd.Sub(todayStart)))
|
||||
}
|
||||
|
||||
func randomWindowCandidate(key string, start, end time.Time) time.Time {
|
||||
seconds := int64(end.Sub(start).Seconds())
|
||||
if seconds <= 0 {
|
||||
return start
|
||||
}
|
||||
hash := fnv.New64a()
|
||||
_, _ = hash.Write([]byte("scheduler-random-window:" + key + ":" + start.Format("2006-01-02") + ":" + start.Format("15:04") + ":" + end.Format("15:04")))
|
||||
offset := int64(hash.Sum64() % uint64(seconds))
|
||||
return start.Add(time.Duration(offset) * time.Second)
|
||||
}
|
||||
|
||||
func sameLocalDate(a, b time.Time) bool {
|
||||
ay, am, ad := a.Date()
|
||||
by, bm, bd := b.Date()
|
||||
return ay == by && am == bm && ad == bd
|
||||
}
|
||||
|
||||
func firstConfigString(config map[string]any, keys ...string) string {
|
||||
for _, key := range keys {
|
||||
value, ok := config[key]
|
||||
if !ok || value == nil {
|
||||
continue
|
||||
}
|
||||
if text := strings.TrimSpace(fmt.Sprint(value)); text != "" {
|
||||
return text
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func parseHourMinute(value string) (int, int, bool) {
|
||||
parts := strings.Split(strings.TrimSpace(value), ":")
|
||||
if len(parts) != 2 {
|
||||
|
||||
@@ -46,6 +46,46 @@ func TestNextIntervalAutoAtSchedulesAfterCurrentCompletion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNextRandomWindowAutoAtIsStableInsideWindow(t *testing.T) {
|
||||
loc := time.FixedZone("CST", 8*60*60)
|
||||
window, ok := parseRandomRunWindowLocal(map[string]any{
|
||||
"random_window_start": "01:00",
|
||||
"random_window_end": "04:00",
|
||||
})
|
||||
if !ok {
|
||||
t.Fatal("expected random window config to parse")
|
||||
}
|
||||
now := time.Date(2026, 6, 2, 0, 30, 0, 0, loc)
|
||||
|
||||
got := nextRandomWindowAutoAt("media_supply_cache_sync", now, loc, window, true, nil)
|
||||
gotAgain := nextRandomWindowAutoAt("media_supply_cache_sync", now.Add(10*time.Minute), loc, window, true, nil)
|
||||
start := time.Date(2026, 6, 2, 1, 0, 0, 0, loc)
|
||||
end := time.Date(2026, 6, 2, 4, 0, 0, 0, loc)
|
||||
if got.Before(start) || !got.Before(end) {
|
||||
t.Fatalf("nextRandomWindowAutoAt() = %s, want inside [%s, %s)", got, start, end)
|
||||
}
|
||||
if !got.Equal(gotAgain) {
|
||||
t.Fatalf("expected stable candidate, got %s and %s", got, gotAgain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNextRandomWindowAutoAtSkipsTodayAfterAutoRun(t *testing.T) {
|
||||
loc := time.FixedZone("CST", 8*60*60)
|
||||
window, _ := parseRandomRunWindowLocal(map[string]any{
|
||||
"random_window_start": "01:00",
|
||||
"random_window_end": "04:00",
|
||||
})
|
||||
now := time.Date(2026, 6, 2, 2, 0, 0, 0, loc)
|
||||
last := time.Date(2026, 6, 2, 1, 30, 0, 0, loc)
|
||||
|
||||
got := nextRandomWindowAutoAt("media_supply_cache_sync", now, loc, window, true, &last)
|
||||
wantStart := time.Date(2026, 6, 3, 1, 0, 0, 0, loc)
|
||||
wantEnd := time.Date(2026, 6, 3, 4, 0, 0, 0, loc)
|
||||
if got.Before(wantStart) || !got.Before(wantEnd) {
|
||||
t.Fatalf("nextRandomWindowAutoAt() = %s, want inside tomorrow window [%s, %s)", got, wantStart, wantEnd)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsEmptySuccessRunRequiresConfiguredZeroMetrics(t *testing.T) {
|
||||
stats := map[string]any{
|
||||
"hydrated_count": 0,
|
||||
|
||||
Reference in New Issue
Block a user