root/geo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(scheduler): harden jobs for scalable execution

Browse Source
This commit is contained in:
liangxu
2026-05-20 12:19:27 +08:00
parent 4b09a34748
commit 5fb9d0b0dd
23 changed files with 887 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/tenant/app/monitoring_lease_recovery.go
+31 -15
View File
@@ -14,16 +14,35 @@ const (
monitoringStaleTaskDropError = "monitoring task dropped after business day rollover"
)
func expireMonitoringLeasedTasks(ctx context.Context, tx pgx.Tx, tenantID *int64, now time.Time) (int64, error) {
func expireMonitoringLeasedTasks(ctx context.Context, tx pgx.Tx, tenantID *int64, now time.Time, limits ...int) (int64, error) {
limit := 0
if len(limits) > 0 {
limit = limits[0]
}
if limit <= 0 {
limit = 1000
}
tag, err := tx.Exec(ctx, `
WITH candidates AS (
SELECT id
FROM monitoring_collect_tasks
WHERE collector_type = $2
AND status = 'leased'
AND lease_expires_at IS NOT NULL
AND lease_expires_at < $3
AND ($4::bigint IS NULL OR tenant_id = $4)
ORDER BY lease_expires_at ASC, id ASC
FOR UPDATE SKIP LOCKED
LIMIT $5
)
UPDATE monitoring_collect_tasks
SET status = 'expired',
lease_token_hash = NULL,
leased_to_executor = NULL,
leased_at = NULL,
lease_expires_at = NULL,
error_message = COALESCE(NULLIF(error_message, ''), $1),
request_payload_json = (
lease_token_hash = NULL,
leased_to_executor = NULL,
leased_at = NULL,
lease_expires_at = NULL,
error_message = COALESCE(NULLIF(error_message, ''), $1),
request_payload_json = (
CASE
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
THEN '{}'::jsonb
@@ -40,12 +59,9 @@ func expireMonitoringLeasedTasks(ctx context.Context, tx pgx.Tx, tenantID *int64
) + 1
),
updated_at = NOW()
WHERE collector_type = $2
AND status = 'leased'
AND lease_expires_at IS NOT NULL
AND lease_expires_at < $3
AND ($4::bigint IS NULL OR tenant_id = $4)
`, monitoringLeaseExpiredErrPrefix, monitoringCollectorType, now.UTC(), nullableInt64(tenantID))
FROM candidates c
WHERE monitoring_collect_tasks.id = c.id
`, monitoringLeaseExpiredErrPrefix, monitoringCollectorType, now.UTC(), nullableInt64(tenantID), limit)
if err != nil {
return 0, monitoringInternalError(50041, "lease_expire_failed", "failed to expire overdue monitoring lease tasks", err)
}
@@ -118,6 +134,6 @@ func skipMonitoringTasksBeforeBusinessDate(
return tag.RowsAffected(), nil
}
func ExpireMonitoringLeasedTasks(ctx context.Context, tx pgx.Tx, tenantID *int64, now time.Time) (int64, error) {
return expireMonitoringLeasedTasks(ctx, tx, tenantID, now)
func ExpireMonitoringLeasedTasks(ctx context.Context, tx pgx.Tx, tenantID *int64, now time.Time, limit int) (int64, error) {
return expireMonitoringLeasedTasks(ctx, tx, tenantID, now, limit)
}