Reduce scheduler polling noise

2026-05-21 00:56:43 +08:00
parent 509f9e0c71
commit 49b1ccc961
10 changed files with 615 additions and 21 deletions
@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"hash/fnv"
+	"math"
 	"os"
 	"strconv"
 	"strings"
@@ -281,6 +282,12 @@ func (p *ControlPlane) runJobOnce(parent context.Context, job ControlledJob, tri
 	if dryRun {
 		mergedConfig["dry_run"] = true
 	}
+	// Empty-success suppression is only for high-frequency automatic polling jobs.
+	// Suppressed runs intentionally do not create running rows and do not advance
+	// LastAutoRunStartedAt, so process restarts may run them immediately once as
+	// overdue. Manual, dry-run, failed, and non-empty executions still get full
+	// audit rows below.
+	suppressEmptySuccessRun := shouldSuppressEmptySuccessRun(triggerType, dryRun, mergedConfig)

 	lockKey := advisoryLockKey(job.Key)
 	lockConn, locked := p.tryAdvisoryLock(parent, lockKey)
@@ -295,19 +302,25 @@ func (p *ControlPlane) runJobOnce(parent context.Context, job ControlledJob, tri
 	}
 	defer p.advisoryUnlock(context.Background(), lockConn, lockKey)

-	run, err := p.repo.StartRun(parent, opsrepo.SchedulerRunStart{
-		JobKey:              job.Key,
-		TriggerID:           triggerID,
-		SchedulerInstanceID: p.instanceID,
-		TriggerType:         triggerType,
-		ConfigVersion:       cfg.Version,
-		ConfigSnapshot:      schedulerConfigSnapshot(cfg, mergedConfig),
-	})
-	if err != nil {
-		if p.logger != nil {
-			p.logger.Warn("scheduler run start failed", zap.String("job_key", job.Key), zap.Error(err))
+	configSnapshot := schedulerConfigSnapshot(cfg, mergedConfig)
+	startedAt := time.Now()
+	var run *opsdomain.SchedulerRun
+	if !suppressEmptySuccessRun {
+		run, err = p.repo.StartRun(parent, opsrepo.SchedulerRunStart{
+			JobKey:              job.Key,
+			TriggerID:           triggerID,
+			SchedulerInstanceID: p.instanceID,
+			TriggerType:         triggerType,
+			ConfigVersion:       cfg.Version,
+			ConfigSnapshot:      configSnapshot,
+		})
+		if err != nil {
+			if p.logger != nil {
+				p.logger.Warn("scheduler run start failed", zap.String("job_key", job.Key), zap.Error(err))
+			}
+			return
 		}
-		return
+		startedAt = run.StartedAt
 	}

 	timeout := time.Duration(cfg.TimeoutSeconds) * time.Second
@@ -335,25 +348,61 @@ func (p *ControlPlane) runJobOnce(parent context.Context, job ControlledJob, tri
 		stats = map[string]any{}
 	}
 	stats["dry_run"] = dryRun
+	finishedAt := time.Now()
+	if suppressEmptySuccessRun && runErr == nil && isEmptySuccessRun(stats, mergedConfig) {
+		if p.logger != nil {
+			p.logger.Info("scheduler empty success run suppressed", zap.String("job_key", job.Key))
+		}
+		return
+	}

 	finishCtx, finishCancel := context.WithTimeout(context.Background(), schedulerFinishTimeout)
 	defer finishCancel()
-	finished, finishErr := p.repo.FinishRun(finishCtx, opsrepo.SchedulerRunFinish{
-		RunID:        run.ID,
-		Status:       status,
-		Stats:        stats,
-		ErrorMessage: errMsg,
-	})
+	var finished *opsdomain.SchedulerRun
+	var finishErr error
+	if run == nil {
+		finished, finishErr = p.repo.RecordRun(finishCtx, opsrepo.SchedulerRunRecord{
+			JobKey:              job.Key,
+			TriggerID:           triggerID,
+			SchedulerInstanceID: p.instanceID,
+			TriggerType:         triggerType,
+			ConfigVersion:       cfg.Version,
+			ConfigSnapshot:      configSnapshot,
+			Status:              status,
+			Stats:               stats,
+			ErrorMessage:        errMsg,
+			StartedAt:           startedAt,
+			FinishedAt:          finishedAt,
+		})
+	} else {
+		finished, finishErr = p.repo.FinishRun(finishCtx, opsrepo.SchedulerRunFinish{
+			RunID:        run.ID,
+			Status:       status,
+			Stats:        stats,
+			ErrorMessage: errMsg,
+		})
+	}
 	if finishErr != nil && p.logger != nil {
-		p.logger.Warn("scheduler run finish failed", zap.String("job_key", job.Key), zap.Int64("run_id", run.ID), zap.Error(finishErr))
+		fields := []zap.Field{zap.String("job_key", job.Key), zap.Error(finishErr)}
+		if run != nil {
+			fields = append(fields, zap.Int64("run_id", run.ID))
+		}
+		if finished != nil {
+			fields = append(fields, zap.Int64("finished_run_id", finished.ID))
+		}
+		p.logger.Warn("scheduler run finish failed", fields...)
 	}
 	if triggerID != nil {
 		triggerStatus := status
 		if finishErr != nil {
 			triggerStatus = opsdomain.SchedulerRunFailed
 		}
-		runID := run.ID
-		_ = p.repo.FinishTrigger(finishCtx, *triggerID, &runID, triggerStatus, errMsg)
+		var runID *int64
+		if finished != nil {
+			value := finished.ID
+			runID = &value
+		}
+		_ = p.repo.FinishTrigger(finishCtx, *triggerID, runID, triggerStatus, errMsg)
 	}
 	if runErr != nil {
 		if p.logger != nil {
@@ -477,6 +526,101 @@ func boolFromMap(in map[string]any, key string) bool {
 	}
 }

+func shouldSuppressEmptySuccessRun(triggerType string, dryRun bool, config map[string]any) bool {
+	if dryRun || triggerType != opsdomain.SchedulerTriggerAuto {
+		return false
+	}
+	return boolFromMap(config, "suppress_empty_success_runs")
+}
+
+func isEmptySuccessRun(stats map[string]any, config map[string]any) bool {
+	if stats == nil || !boolFromMap(config, "suppress_empty_success_runs") {
+		return false
+	}
+	keys := emptySuccessMetricKeys(config)
+	if len(keys) == 0 {
+		return false
+	}
+	for _, key := range keys {
+		value, ok := stats[key]
+		if !ok {
+			return false
+		}
+		if numericValue(value) != 0 {
+			return false
+		}
+	}
+	return true
+}
+
+func emptySuccessMetricKeys(config map[string]any) []string {
+	value, ok := config["empty_success_metric_keys"]
+	if !ok || value == nil {
+		return nil
+	}
+	switch typed := value.(type) {
+	case []string:
+		return typed
+	case []any:
+		out := make([]string, 0, len(typed))
+		for _, item := range typed {
+			if key := strings.TrimSpace(fmt.Sprint(item)); key != "" {
+				out = append(out, key)
+			}
+		}
+		return out
+	case string:
+		parts := strings.Split(typed, ",")
+		out := make([]string, 0, len(parts))
+		for _, part := range parts {
+			if key := strings.TrimSpace(part); key != "" {
+				out = append(out, key)
+			}
+		}
+		return out
+	default:
+		return nil
+	}
+}
+
+func numericValue(value any) float64 {
+	switch typed := value.(type) {
+	case int:
+		return float64(typed)
+	case int8:
+		return float64(typed)
+	case int16:
+		return float64(typed)
+	case int32:
+		return float64(typed)
+	case int64:
+		return float64(typed)
+	case uint:
+		return float64(typed)
+	case uint8:
+		return float64(typed)
+	case uint16:
+		return float64(typed)
+	case uint32:
+		return float64(typed)
+	case uint64:
+		return float64(typed)
+	case float32:
+		return float64(typed)
+	case float64:
+		if math.IsNaN(typed) {
+			return 0
+		}
+		return typed
+	case string:
+		parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
+		if err == nil {
+			return parsed
+		}
+	}
+	return 0
+}
+
 func parseRunAtLocal(config map[string]any) string {
 	value, ok := config["run_at_local"]
 	if !ok || value == nil {