root/geo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix monitor collect-now queue fairness

Browse Source
This commit is contained in:
liangxu
2026-06-22 21:47:26 +08:00
parent 5a32926009
commit 082f91a6a9
11 changed files with 222 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/tenant/app/monitoring_service.go
+13 -79
View File
@@ -882,7 +882,7 @@ func (s *MonitoringService) CollectNow(
return nil, err
}
refreshedCount, createdCount, leasedCount, completedCount, interruptTaskIDs, err := s.ensureCollectNowTasks(
refreshedCount, createdCount, leasedCount, completedCount, err := s.ensureCollectNowTasks(
ctx,
tx,
actor.TenantID,
@@ -969,7 +969,7 @@ func (s *MonitoringService) CollectNow(
$12, $13, $14,
$15, $16
)
`, requestID, actor.TenantID, workspaceID, brand.ID, nullableInt64(keywordID), actor.UserID, *clientID, scopeHash, requestScopeJSON, affectedTaskCount, affectedTaskCount, abortedQueuedCount, int64(len(interruptTaskIDs)), interruptGeneration, message, ttlExpiresAt); err != nil {
`, requestID, actor.TenantID, workspaceID, brand.ID, nullableInt64(keywordID), actor.UserID, *clientID, scopeHash, requestScopeJSON, affectedTaskCount, affectedTaskCount, abortedQueuedCount, int64(0), interruptGeneration, message, ttlExpiresAt); err != nil {
return nil, response.ErrInternal(50041, "request_create_failed", "failed to persist monitoring collect-now request")
}
@@ -981,12 +981,6 @@ func (s *MonitoringService) CollectNow(
dispatchTargetClientIDs,
affectedTaskCount,
interruptGeneration,
func() []int64 {
if !options.Preempt {
return nil
}
return collectNowInterruptTaskIDs(interruptTaskIDs)
}(),
); err != nil {
return nil, err
}
@@ -1022,14 +1016,11 @@ func (s *MonitoringService) CollectNow(
}
}
phase2InterruptTargets := make([]monitorDesktopInterruptTarget, 0)
phase2DeferredTasks := make([]*repository.DesktopTask, 0)
if options.Preempt && phase2DispatchReady {
excludedMonitorTaskIDs := make([]int64, 0, len(phase2TaskSpecs))
excludedPlatformIDs := make([]string, 0, len(phase2TaskSpecs))
for _, spec := range phase2TaskSpecs {
excludedMonitorTaskIDs = append(excludedMonitorTaskIDs, spec.MonitorTaskID)
excludedPlatformIDs = append(excludedPlatformIDs, spec.PlatformID)
}
phase2TargetClientIDs := desktopTaskClientIDs(phase2PublishedTasks)
if len(phase2TargetClientIDs) == 0 {
@@ -1041,28 +1032,9 @@ func (s *MonitoringService) CollectNow(
return nil, deferErr
}
phase2DeferredTasks = append(phase2DeferredTasks, deferredTasks...)
interruptTargets, interruptErr := s.requestPhase2MonitorInterrupts(ctx, phase2TargetClientID, excludedMonitorTaskIDs, excludedPlatformIDs, interruptGeneration)
if interruptErr != nil {
return nil, interruptErr
}
phase2InterruptTargets = append(phase2InterruptTargets, interruptTargets...)
}
if len(phase2DeferredTasks) > 0 {
abortedQueuedCount += int64(len(phase2DeferredTasks))
}
if len(phase2InterruptTargets) > 0 {
// Keep collect-now request counters in sync after per-platform dispatch
// fan-out chooses more than one desktop client.
if _, updateErr := s.monitoringPool.Exec(ctx, `
UPDATE monitoring_collect_requests
SET aborted_queued_count = aborted_queued_count + $2,
interrupt_requested_count = interrupt_requested_count + $3,
updated_at = NOW()
WHERE request_id = $1
`, requestID, len(phase2DeferredTasks), len(phase2InterruptTargets)); updateErr != nil {
return nil, response.ErrInternal(50041, "request_update_failed", "failed to persist phase2 preempt counts")
}
} else if len(phase2DeferredTasks) > 0 {
if _, updateErr := s.monitoringPool.Exec(ctx, `
UPDATE monitoring_collect_requests
SET aborted_queued_count = aborted_queued_count + $2,
@@ -1107,12 +1079,8 @@ func (s *MonitoringService) CollectNow(
)
}
}
for _, target := range phase2InterruptTargets {
publishPhase2MonitorControlEvent(context.Background(), s.rabbitMQ, s.logger, target)
}
hasEffectiveSnapshot := leasedCount > 0 || completedCount > 0
totalInterruptRequestedCount := int64(len(interruptTaskIDs) + len(phase2InterruptTargets))
responseData := &MonitoringCollectNowResponse{
CollectionMode: quota.CollectionMode,
RefreshedTaskCount: refreshedCount,
@@ -1125,7 +1093,7 @@ func (s *MonitoringService) CollectNow(
AffectedTaskCount: affectedTaskCount,
PromotedTaskCount: affectedTaskCount,
AbortedQueuedCount: abortedQueuedCount,
InterruptRequestedCount: totalInterruptRequestedCount,
InterruptRequestedCount: 0,
InterruptGeneration: interruptGeneration,
TTLExpiresAt: ttlExpiresAt.Format(time.RFC3339),
Message: message,
@@ -1698,7 +1666,7 @@ func (s *MonitoringService) ensureCollectNowTasks(
targetClientID uuid.UUID,
interruptGeneration int,
executionOwner string,
) (int64, int64, int64, int64, []int64, error) {
) (int64, int64, int64, int64, error) {
const runMode = "plugin_standard"
if strings.TrimSpace(executionOwner) == "" {
executionOwner = "legacy"
@@ -1707,12 +1675,11 @@ func (s *MonitoringService) ensureCollectNowTasks(
dateText := businessDate.Format("2006-01-02")
var refreshedCount int64
var createdCount int64
interruptTaskIDs := make([]int64, 0)
for _, question := range questions {
questionText, textErr := monitoringQuestionTextSnapshot(question)
if textErr != nil {
return 0, 0, 0, 0, nil, textErr
return 0, 0, 0, 0, textErr
}
for _, platform := range platforms {
platformID := normalizeMonitoringPlatformID(platform.ID)
@@ -1762,11 +1729,11 @@ func (s *MonitoringService) ensureCollectNowTasks(
AND status IN ('pending', 'expired', 'failed', 'completed', 'skipped')
`, tenantID, brandID, question.ID, platformID, monitoringCollectorType, runMode, question.QuestionHash, dateText, platformTargetClientID, interruptGeneration, executionOwner, questionText)
if err != nil {
return 0, 0, 0, 0, nil, response.ErrInternal(50041, "update_failed", "failed to refresh monitoring tasks")
return 0, 0, 0, 0, response.ErrInternal(50041, "update_failed", "failed to refresh monitoring tasks")
}
refreshedCount += tag.RowsAffected()
rows, err := tx.Query(ctx, `
tag, err = tx.Exec(ctx, `
UPDATE monitoring_collect_tasks
SET question_hash = $7,
question_text_snapshot = $11,
@@ -1793,24 +1760,10 @@ func (s *MonitoringService) ensureCollectNowTasks(
AND business_date = $8::date
AND status = 'leased'
AND COALESCE(execution_owner, 'legacy') = 'legacy'
RETURNING id
`, tenantID, brandID, question.ID, platformID, monitoringCollectorType, runMode, question.QuestionHash, dateText, platformTargetClientID, interruptGeneration, questionText)
if err != nil {
return 0, 0, 0, 0, nil, response.ErrInternal(50041, "update_failed", "failed to promote leased monitoring tasks")
return 0, 0, 0, 0, response.ErrInternal(50041, "update_failed", "failed to promote leased monitoring tasks")
}
for rows.Next() {
var taskID int64
if scanErr := rows.Scan(&taskID); scanErr != nil {
rows.Close()
return 0, 0, 0, 0, nil, response.ErrInternal(50041, "scan_failed", "failed to parse promoted leased monitoring task")
}
interruptTaskIDs = append(interruptTaskIDs, taskID)
}
if err := rows.Err(); err != nil {
rows.Close()
return 0, 0, 0, 0, nil, response.ErrInternal(50041, "scan_failed", "failed to iterate promoted leased monitoring tasks")
}
rows.Close()
tag, err = tx.Exec(ctx, `
INSERT INTO monitoring_collect_tasks (
@@ -1831,7 +1784,7 @@ func (s *MonitoringService) ensureCollectNowTasks(
DO NOTHING
`, tenantID, brandID, question.ID, question.QuestionHash, platformID, monitoringCollectorType, runMode, dateText, platformTargetClientID, interruptGeneration, fmt.Sprintf("%d:%d:%s", tenantID, brandID, dateText), executionOwner, questionText)
if err != nil {
return 0, 0, 0, 0, nil, response.ErrInternal(50041, "insert_failed", "failed to create monitoring tasks")
return 0, 0, 0, 0, response.ErrInternal(50041, "insert_failed", "failed to create monitoring tasks")
}
createdCount += tag.RowsAffected()
}
@@ -1847,13 +1800,13 @@ func (s *MonitoringService) ensureCollectNowTasks(
WHERE tenant_id = $1
AND brand_id = $2
AND collector_type = $3
AND business_date = $4::date
AND question_id = ANY($5)
AND business_date = $4::date
AND question_id = ANY($5)
`, tenantID, brandID, monitoringCollectorType, dateText, configuredQuestionIDs(questions)).Scan(&leasedCount, &completedCount); err != nil {
return 0, 0, 0, 0, nil, response.ErrInternal(50041, "query_failed", "failed to inspect monitoring tasks")
return 0, 0, 0, 0, response.ErrInternal(50041, "query_failed", "failed to inspect monitoring tasks")
}
return refreshedCount, createdCount, leasedCount, completedCount, interruptTaskIDs, nil
return refreshedCount, createdCount, leasedCount, completedCount, nil
}
func (s *MonitoringService) deferQueuedNormalTasks(
@@ -2255,25 +2208,6 @@ func uuidStrings(values []uuid.UUID) []string {
return result
}
func collectNowInterruptTaskIDs(taskIDs []int64) []int64 {
if len(taskIDs) == 0 {
return nil
}
seen := make(map[int64]struct{}, len(taskIDs))
result := make([]int64, 0, len(taskIDs))
for _, taskID := range taskIDs {
if taskID <= 0 {
continue
}
if _, ok := seen[taskID]; ok {
continue
}
seen[taskID] = struct{}{}
result = append(result, taskID)
}
return result
}
func desktopTaskClientIDs(tasks []*repository.DesktopTask) []uuid.UUID {
if len(tasks) == 0 {
return nil