root/geo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(scheduler): harden jobs for scalable execution

Browse Source
This commit is contained in:
liangxu
2026-05-20 12:19:27 +08:00
parent 4b09a34748
commit 5fb9d0b0dd
23 changed files with 887 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/tenant/app/monitoring_daily_task_worker.go
+44 -5
View File
@@ -56,6 +56,39 @@ type MonitoringDailyTaskGenerationSummary struct {
SkippedPlanCount int
}
type MonitoringDailyTaskRunOptions struct {
MaxMaterializePerRun int
MaxMaterializePerPlan int
MaxMaterializePerBrand int
}
func normalizeMonitoringDailyRunOptions(options ...MonitoringDailyTaskRunOptions) MonitoringDailyTaskRunOptions {
out := MonitoringDailyTaskRunOptions{
MaxMaterializePerRun: defaultMonitoringDailyMaxMaterializePerRun,
MaxMaterializePerPlan: defaultMonitoringDailyMaxMaterializePerPlan,
MaxMaterializePerBrand: defaultMonitoringDailyMaxMaterializePerBrand,
}
if len(options) > 0 {
in := options[0]
if in.MaxMaterializePerRun > 0 {
out.MaxMaterializePerRun = in.MaxMaterializePerRun
}
if in.MaxMaterializePerPlan > 0 {
out.MaxMaterializePerPlan = in.MaxMaterializePerPlan
}
if in.MaxMaterializePerBrand > 0 {
out.MaxMaterializePerBrand = in.MaxMaterializePerBrand
}
}
if out.MaxMaterializePerPlan > out.MaxMaterializePerRun {
out.MaxMaterializePerPlan = out.MaxMaterializePerRun
}
if out.MaxMaterializePerBrand > out.MaxMaterializePerPlan {
out.MaxMaterializePerBrand = out.MaxMaterializePerPlan
}
return out
}
type monitoringDailyPlan struct {
TenantID int64
WorkspaceID int64
@@ -174,15 +207,16 @@ func (w *MonitoringDailyTaskWorker) runOnce(parent context.Context) {
)
}
func (w *MonitoringDailyTaskWorker) RunOnce(parent context.Context) (map[string]any, error) {
func (w *MonitoringDailyTaskWorker) RunOnce(parent context.Context, options ...MonitoringDailyTaskRunOptions) (map[string]any, error) {
if w == nil || w.service == nil || w.service.businessPool == nil || w.service.monitoringPool == nil {
return map[string]any{"skipped": true, "reason": "worker_not_ready"}, nil
}
runOptions := normalizeMonitoringDailyRunOptions(options...)
ctx, cancel := context.WithTimeout(parent, w.timeout)
defer cancel()
startedAt := time.Now()
summary, err := w.service.GenerateDailyMonitoringTasks(ctx, w.projectionService, time.Now().UTC())
summary, err := w.service.GenerateDailyMonitoringTasks(ctx, w.projectionService, time.Now().UTC(), runOptions)
duration := time.Since(startedAt)
recordMonitoringDailyTaskMetrics(summary, duration, err)
stats := map[string]any{
@@ -201,6 +235,9 @@ func (w *MonitoringDailyTaskWorker) RunOnce(parent context.Context) (map[string]
stats["deferred_task_count"] = summary.DeferredTaskCount
stats["skipped_plan_count"] = summary.SkippedPlanCount
}
stats["max_materialize_per_run"] = runOptions.MaxMaterializePerRun
stats["max_materialize_per_plan"] = runOptions.MaxMaterializePerPlan
stats["max_materialize_per_brand"] = runOptions.MaxMaterializePerBrand
if err != nil {
if w.logger != nil {
w.logger.Warn("monitoring daily task generation failed", MonitoringWorkerErrorFields(err)...)
@@ -214,6 +251,7 @@ func (s *MonitoringService) GenerateDailyMonitoringTasks(
ctx context.Context,
projectionService *MonitoringCallbackService,
now time.Time,
runOptions ...MonitoringDailyTaskRunOptions,
) (*MonitoringDailyTaskGenerationSummary, error) {
if s == nil || s.businessPool == nil || s.monitoringPool == nil {
return nil, nil
@@ -230,7 +268,8 @@ func (s *MonitoringService) GenerateDailyMonitoringTasks(
BusinessDate: businessDate,
TenantCount: len(plans),
}
globalRemaining := defaultMonitoringDailyMaxMaterializePerRun
options := normalizeMonitoringDailyRunOptions(runOptions...)
globalRemaining := options.MaxMaterializePerRun
for _, plan := range plans {
if globalRemaining <= 0 {
@@ -271,7 +310,7 @@ func (s *MonitoringService) GenerateDailyMonitoringTasks(
continue
}
planRunRemaining := minMonitoringDailyInt(defaultMonitoringDailyMaxMaterializePerPlan, globalRemaining)
planRunRemaining := minMonitoringDailyInt(options.MaxMaterializePerPlan, globalRemaining)
for _, brand := range brands {
if planRunRemaining <= 0 || globalRemaining <= 0 {
@@ -308,7 +347,7 @@ func (s *MonitoringService) GenerateDailyMonitoringTasks(
continue
}
dispatchLimit := minMonitoringDailyInt(defaultMonitoringDailyMaxMaterializePerBrand, planRunRemaining, globalRemaining)
dispatchLimit := minMonitoringDailyInt(options.MaxMaterializePerBrand, planRunRemaining, globalRemaining)
materializeLimit := minMonitoringDailyInt(dispatchLimit, dailyRemaining)
createdCount, dueCount, desktopCount, deferredCount, err := s.generateDailyMonitoringTasksForBrand(
ctx,
server/internal/tenant/app/monitoring_lease_recovery.go
+31 -15
View File
@@ -14,16 +14,35 @@ const (
monitoringStaleTaskDropError = "monitoring task dropped after business day rollover"
)
func expireMonitoringLeasedTasks(ctx context.Context, tx pgx.Tx, tenantID *int64, now time.Time) (int64, error) {
func expireMonitoringLeasedTasks(ctx context.Context, tx pgx.Tx, tenantID *int64, now time.Time, limits ...int) (int64, error) {
limit := 0
if len(limits) > 0 {
limit = limits[0]
}
if limit <= 0 {
limit = 1000
}
tag, err := tx.Exec(ctx, `
WITH candidates AS (
SELECT id
FROM monitoring_collect_tasks
WHERE collector_type = $2
AND status = 'leased'
AND lease_expires_at IS NOT NULL
AND lease_expires_at < $3
AND ($4::bigint IS NULL OR tenant_id = $4)
ORDER BY lease_expires_at ASC, id ASC
FOR UPDATE SKIP LOCKED
LIMIT $5
)
UPDATE monitoring_collect_tasks
SET status = 'expired',
lease_token_hash = NULL,
leased_to_executor = NULL,
leased_at = NULL,
lease_expires_at = NULL,
error_message = COALESCE(NULLIF(error_message, ''), $1),
request_payload_json = (
lease_token_hash = NULL,
leased_to_executor = NULL,
leased_at = NULL,
lease_expires_at = NULL,
error_message = COALESCE(NULLIF(error_message, ''), $1),
request_payload_json = (
CASE
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
THEN '{}'::jsonb
@@ -40,12 +59,9 @@ func expireMonitoringLeasedTasks(ctx context.Context, tx pgx.Tx, tenantID *int64
) + 1
),
updated_at = NOW()
WHERE collector_type = $2
AND status = 'leased'
AND lease_expires_at IS NOT NULL
AND lease_expires_at < $3
AND ($4::bigint IS NULL OR tenant_id = $4)
`, monitoringLeaseExpiredErrPrefix, monitoringCollectorType, now.UTC(), nullableInt64(tenantID))
FROM candidates c
WHERE monitoring_collect_tasks.id = c.id
`, monitoringLeaseExpiredErrPrefix, monitoringCollectorType, now.UTC(), nullableInt64(tenantID), limit)
if err != nil {
return 0, monitoringInternalError(50041, "lease_expire_failed", "failed to expire overdue monitoring lease tasks", err)
}
@@ -118,6 +134,6 @@ func skipMonitoringTasksBeforeBusinessDate(
return tag.RowsAffected(), nil
}
func ExpireMonitoringLeasedTasks(ctx context.Context, tx pgx.Tx, tenantID *int64, now time.Time) (int64, error) {
return expireMonitoringLeasedTasks(ctx, tx, tenantID, now)
func ExpireMonitoringLeasedTasks(ctx context.Context, tx pgx.Tx, tenantID *int64, now time.Time, limit int) (int64, error) {
return expireMonitoringLeasedTasks(ctx, tx, tenantID, now, limit)
}
server/internal/tenant/app/monitoring_received_inspection.go
+1 -1
View File
@@ -56,7 +56,7 @@ func markMonitoringStaleReceivedTasks(ctx context.Context, tx pgx.Tx, now time.T
AND (
request_payload_json IS NULL
OR jsonb_typeof(request_payload_json) <> 'object'
OR COALESCE(NULLIF(request_payload_json ->> 'received_alerted_at', ''), '') = ''
OR NOT (request_payload_json ? 'received_alerted_at')
)
ORDER BY callback_received_at ASC, id ASC
FOR UPDATE SKIP LOCKED