root/geo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(scheduler): harden jobs for scalable execution

Browse Source
This commit is contained in:
liangxu
2026-05-20 12:19:27 +08:00
parent 4b09a34748
commit 5fb9d0b0dd
23 changed files with 887 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/scheduler/monitoring_retention_worker.go
+54 -32
View File
@@ -24,7 +24,7 @@ type MonitoringRetentionWorker struct {
type retentionTarget struct {
Name string
DeleteSQL string
CountSQL string
ProbeSQL string
}
func NewMonitoringRetentionWorker(monitoringPool *pgxpool.Pool, logger *zap.Logger) *MonitoringRetentionWorker {
@@ -77,18 +77,19 @@ func (w *MonitoringRetentionWorker) RunOnce(ctx context.Context, run JobRunConte
}
totalDeleted := int64(0)
totalCandidates := int64(0)
totalCandidateProbe := int64(0)
perTable := map[string]any{}
for _, target := range retentionTargets() {
candidateCount, err := w.countTarget(ctx, conn, target, cutoffDate)
if err != nil {
return stats, fmt.Errorf("count %s: %w", target.Name, err)
}
totalCandidates += candidateCount
tableStats := map[string]any{
"candidate_count": candidateCount,
}
if !dryRun && candidateCount > 0 {
tableStats := map[string]any{}
if dryRun {
candidateProbe, err := w.probeTarget(ctx, conn, target, cutoffDate, batchSize)
if err != nil {
return stats, fmt.Errorf("probe %s: %w", target.Name, err)
}
tableStats["candidate_probe_count"] = candidateProbe
tableStats["candidate_probe_limited"] = candidateProbe >= int64(batchSize)
totalCandidateProbe += candidateProbe
} else {
deleted, batches, err := w.deleteTarget(ctx, conn, target, cutoffDate, batchSize, maxBatches)
if err != nil {
return stats, fmt.Errorf("delete %s: %w", target.Name, err)
@@ -100,7 +101,9 @@ func (w *MonitoringRetentionWorker) RunOnce(ctx context.Context, run JobRunConte
perTable[target.Name] = tableStats
}
stats["candidate_count"] = totalCandidates
if dryRun {
stats["candidate_probe_count"] = totalCandidateProbe
}
stats["deleted_count"] = totalDeleted
stats["tables"] = perTable
stats["duration_ms"] = time.Since(startedAt).Milliseconds()
@@ -118,9 +121,12 @@ func monitoringRetentionCutoffDate(now time.Time, retentionDays int) string {
return cutoff.Format("2006-01-02")
}
func (w *MonitoringRetentionWorker) countTarget(ctx context.Context, conn *pgxpool.Conn, target retentionTarget, cutoffDate string) (int64, error) {
func (w *MonitoringRetentionWorker) probeTarget(ctx context.Context, conn *pgxpool.Conn, target retentionTarget, cutoffDate string, limit int) (int64, error) {
var count int64
if err := conn.QueryRow(ctx, target.CountSQL, cutoffDate).Scan(&count); err != nil {
if limit <= 0 {
limit = defaultMonitoringRetentionBatch
}
if err := conn.QueryRow(ctx, target.ProbeSQL, cutoffDate, limit).Scan(&count); err != nil {
return 0, err
}
return count, nil
@@ -151,53 +157,69 @@ func retentionTargets() []retentionTarget {
return []retentionTarget{
{
Name: "monitoring_collect_dispatch_outbox",
CountSQL: `SELECT COUNT(*) FROM monitoring_collect_dispatch_outbox WHERE created_at < $1::date`,
DeleteSQL: batchDeleteSQL("monitoring_collect_dispatch_outbox", "id", "created_at < $1::date"),
ProbeSQL: batchProbeSQL("monitoring_collect_dispatch_outbox", "id", "created_at < $1::date", "created_at ASC, id ASC"),
DeleteSQL: batchDeleteSQL("monitoring_collect_dispatch_outbox", "id", "created_at < $1::date", "created_at ASC, id ASC"),
},
{
Name: "monitoring_collect_requests",
CountSQL: `SELECT COUNT(*) FROM monitoring_collect_requests WHERE created_at < $1::date`,
DeleteSQL: batchDeleteSQL("monitoring_collect_requests", "id", "created_at < $1::date"),
ProbeSQL: batchProbeSQL("monitoring_collect_requests", "id", "created_at < $1::date", "created_at ASC, id ASC"),
DeleteSQL: batchDeleteSQL("monitoring_collect_requests", "id", "created_at < $1::date", "created_at ASC, id ASC"),
},
{
Name: "monitoring_brand_platform_daily",
CountSQL: `SELECT COUNT(*) FROM monitoring_brand_platform_daily WHERE business_date < $1::date`,
DeleteSQL: batchDeleteSQL("monitoring_brand_platform_daily", "id", "business_date < $1::date"),
ProbeSQL: batchProbeSQL("monitoring_brand_platform_daily", "id", "business_date < $1::date", "business_date ASC, id ASC"),
DeleteSQL: batchDeleteSQL("monitoring_brand_platform_daily", "id", "business_date < $1::date", "business_date ASC, id ASC"),
},
{
Name: "monitoring_brand_daily",
CountSQL: `SELECT COUNT(*) FROM monitoring_brand_daily WHERE business_date < $1::date`,
DeleteSQL: batchDeleteSQL("monitoring_brand_daily", "id", "business_date < $1::date"),
ProbeSQL: batchProbeSQL("monitoring_brand_daily", "id", "business_date < $1::date", "business_date ASC, id ASC"),
DeleteSQL: batchDeleteSQL("monitoring_brand_daily", "id", "business_date < $1::date", "business_date ASC, id ASC"),
},
{
Name: "monitoring_platform_access_snapshots",
CountSQL: `SELECT COUNT(*) FROM monitoring_platform_access_snapshots WHERE business_date < $1::date`,
DeleteSQL: batchDeleteSQL("monitoring_platform_access_snapshots", "id", "business_date < $1::date"),
ProbeSQL: batchProbeSQL("monitoring_platform_access_snapshots", "id", "business_date < $1::date", "business_date ASC, id ASC"),
DeleteSQL: batchDeleteSQL("monitoring_platform_access_snapshots", "id", "business_date < $1::date", "business_date ASC, id ASC"),
},
{
Name: "monitoring_collect_tasks",
CountSQL: `SELECT COUNT(*) FROM monitoring_collect_tasks WHERE business_date < $1::date`,
DeleteSQL: batchDeleteSQL("monitoring_collect_tasks", "id", "business_date < $1::date"),
ProbeSQL: batchProbeSQL("monitoring_collect_tasks", "id", "business_date < $1::date", "business_date ASC, id ASC"),
DeleteSQL: batchDeleteSQL("monitoring_collect_tasks", "id", "business_date < $1::date", "business_date ASC, id ASC"),
},
{
Name: "question_monitor_runs",
CountSQL: `SELECT COUNT(*) FROM question_monitor_runs WHERE business_date < $1::date`,
DeleteSQL: batchDeleteSQL("question_monitor_runs", "id", "business_date < $1::date"),
ProbeSQL: batchProbeSQL("question_monitor_runs", "id", "business_date < $1::date", "business_date ASC, id ASC"),
DeleteSQL: batchDeleteSQL("question_monitor_runs", "id", "business_date < $1::date", "business_date ASC, id ASC"),
},
}
}
func batchDeleteSQL(table, idColumn, predicate string) string {
func batchProbeSQL(table, idColumn, predicate, orderBy string) string {
parts := []string{
"SELECT COUNT(*)::bigint",
"FROM (",
"SELECT " + idColumn,
"FROM " + table,
"WHERE " + predicate,
"ORDER BY " + orderBy,
"LIMIT $2",
") candidates",
}
return strings.Join(parts, "\n")
}
func batchDeleteSQL(table, idColumn, predicate, orderBy string) string {
parts := []string{
"WITH doomed AS (",
"SELECT " + idColumn,
"FROM " + table,
"WHERE " + predicate,
"ORDER BY " + idColumn + " ASC",
"ORDER BY " + orderBy,
"LIMIT $2",
"FOR UPDATE SKIP LOCKED",
")",
"DELETE FROM " + table,
"WHERE " + idColumn + " IN (SELECT " + idColumn + " FROM doomed)",
"DELETE FROM " + table + " t",
"USING doomed",
"WHERE t." + idColumn + " = doomed." + idColumn,
}
return strings.Join(parts, "\n")
}