root/geo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(scheduler): harden jobs for scalable execution

Browse Source
This commit is contained in:
liangxu
2026-05-20 12:19:27 +08:00
parent 4b09a34748
commit 5fb9d0b0dd
23 changed files with 887 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/scheduler/monitoring_result_recovery_worker.go
+74 -22
View File
@@ -2,6 +2,7 @@ package scheduler
import (
"context"
"fmt"
"time"
"github.com/jackc/pgx/v5"
@@ -15,6 +16,7 @@ const (
defaultMonitoringResultRecoveryInterval = 1 * time.Minute
defaultMonitoringResultRecoveryTimeout = 30 * time.Second
defaultMonitoringResultRecoveryLimit = 100
defaultMonitoringResultRecoveryClaimTTL = 5 * time.Minute
)
type MonitoringResultRecoveryWorker struct {
@@ -87,7 +89,7 @@ func (w *MonitoringResultRecoveryWorker) runOnce(parent context.Context) {
}
defer tx.Rollback(ctx)
items, err := loadMonitoringRecoverableTaskResults(ctx, tx, w.limit)
items, err := loadMonitoringRecoverableTaskResults(ctx, tx, w.limit, defaultMonitoringResultRecoveryClaimTTL)
if err != nil {
if w.logger != nil {
w.logger.Warn("monitoring result recovery failed", tenantapp.MonitoringWorkerErrorFields(err)...)
@@ -118,12 +120,23 @@ func (w *MonitoringResultRecoveryWorker) runOnce(parent context.Context) {
}
}
func (w *MonitoringResultRecoveryWorker) RunOnce(parent context.Context) (map[string]any, error) {
func (w *MonitoringResultRecoveryWorker) RunOnce(parent context.Context, run JobRunContext) (map[string]any, error) {
if w == nil || w.monitoringPool == nil || w.service == nil {
return map[string]any{"skipped": true, "reason": "worker_not_ready"}, nil
}
startedAt := time.Now()
ctx, cancel := context.WithTimeout(parent, w.timeout)
timeout := w.timeout
if run.Job != nil && run.Job.TimeoutSeconds > 0 {
timeout = time.Duration(run.Job.TimeoutSeconds) * time.Second
}
limit := w.limit
if run.Job != nil && run.Job.BatchSize != nil && *run.Job.BatchSize > 0 {
limit = *run.Job.BatchSize
}
if value := AsInt(run.Config, "batch_size", 0); value > 0 && (run.Job == nil || run.Job.BatchSize == nil) {
limit = value
}
ctx, cancel := context.WithTimeout(parent, timeout)
defer cancel()
tx, err := w.monitoringPool.Begin(ctx)
@@ -132,7 +145,8 @@ func (w *MonitoringResultRecoveryWorker) RunOnce(parent context.Context) (map[st
}
defer tx.Rollback(ctx)
items, err := loadMonitoringRecoverableTaskResults(ctx, tx, w.limit)
claimTTL := AsDuration(run.Config, "claim_ttl", defaultMonitoringResultRecoveryClaimTTL)
items, err := loadMonitoringRecoverableTaskResults(ctx, tx, limit, claimTTL)
if err != nil {
return map[string]any{"stage": "load"}, err
}
@@ -150,6 +164,8 @@ func (w *MonitoringResultRecoveryWorker) RunOnce(parent context.Context) (map[st
"recoverable_count": len(items),
"republished_count": republishedCount,
"duration_ms": time.Since(startedAt).Milliseconds(),
"batch_size": limit,
"claim_ttl_seconds": int64(claimTTL.Seconds()),
}, nil
}
@@ -194,34 +210,70 @@ func (w *MonitoringResultRecoveryWorker) tryRepublish(parent context.Context, it
return true
}
func loadMonitoringRecoverableTaskResults(ctx context.Context, tx pgx.Tx, limit int) ([]monitoringRecoverableTaskResult, error) {
func loadMonitoringRecoverableTaskResults(ctx context.Context, tx pgx.Tx, limit int, claimTTL time.Duration) ([]monitoringRecoverableTaskResult, error) {
if limit <= 0 {
limit = defaultMonitoringResultRecoveryLimit
}
if claimTTL <= 0 {
claimTTL = defaultMonitoringResultRecoveryClaimTTL
}
rows, err := tx.Query(ctx, `
WITH candidates AS (
SELECT
id,
callback_received_at,
request_payload_json
FROM monitoring_collect_tasks
WHERE collector_type = $1
AND (
status = 'received'
OR (
COALESCE(execution_owner, 'legacy') = 'desktop_tasks'
AND status = 'pending'
AND callback_received_at IS NOT NULL
)
)
AND callback_received_at IS NOT NULL
AND request_payload_json IS NOT NULL
AND jsonb_typeof(request_payload_json) = 'object'
AND (
COALESCE(NULLIF(request_payload_json ->> 'queue_status', ''), 'pending') IN ('pending', 'publish_failed')
OR (
COALESCE(NULLIF(request_payload_json ->> 'queue_status', ''), 'pending') = 'claiming'
AND COALESCE(NULLIF(request_payload_json ->> 'queue_claimed_at', ''), '') <> ''
AND NULLIF(request_payload_json ->> 'queue_claimed_at', '')::timestamptz < NOW() - $3::interval
)
)
ORDER BY callback_received_at ASC, id ASC
FOR UPDATE SKIP LOCKED
LIMIT $2
),
claimed AS (
UPDATE monitoring_collect_tasks t
SET request_payload_json = (
CASE
WHEN t.request_payload_json IS NULL OR jsonb_typeof(t.request_payload_json) <> 'object'
THEN '{}'::jsonb
ELSE t.request_payload_json
END
) || jsonb_build_object(
'queue_status', 'claiming',
'queue_claimed_at', NOW()::text
),
updated_at = NOW()
FROM candidates c
WHERE t.id = c.id
AND t.callback_received_at = c.callback_received_at
RETURNING c.id, c.callback_received_at, c.request_payload_json
)
SELECT
id,
callback_received_at,
request_payload_json
FROM monitoring_collect_tasks
WHERE collector_type = $1
AND (
status = 'received'
OR (
COALESCE(execution_owner, 'legacy') = 'desktop_tasks'
AND status = 'pending'
AND callback_received_at IS NOT NULL
)
)
AND callback_received_at IS NOT NULL
AND request_payload_json IS NOT NULL
AND jsonb_typeof(request_payload_json) = 'object'
AND COALESCE(NULLIF(request_payload_json ->> 'queue_status', ''), 'pending') IN ('pending', 'publish_failed')
FROM claimed
ORDER BY callback_received_at ASC, id ASC
FOR UPDATE SKIP LOCKED
LIMIT $2
`, "plugin", limit)
`, "desktop", limit, fmt.Sprintf("%d milliseconds", claimTTL.Milliseconds()))
if err != nil {
return nil, tenantapp.InternalMonitoringError(50041, "result_recovery_query_failed", "failed to load recoverable monitoring results", err)
}