Fix monitor collect-now queue fairness

This commit is contained in:
2026-06-22 21:47:26 +08:00
parent 5a32926009
commit 082f91a6a9
11 changed files with 222 additions and 258 deletions
@@ -12,7 +12,6 @@ import (
"github.com/jackc/pgx/v5/pgtype"
"go.uber.org/zap"
"github.com/geo-platform/tenant-api/internal/shared/messaging/rabbitmq"
"github.com/geo-platform/tenant-api/internal/shared/response"
"github.com/geo-platform/tenant-api/internal/shared/stream"
"github.com/geo-platform/tenant-api/internal/tenant/repository"
@@ -66,14 +65,6 @@ type monitorDesktopTaskTargetCandidate struct {
ClientOrder int
}
type monitorDesktopInterruptTarget struct {
TaskID string
MonitorTaskID int64
TargetClientID string
WorkspaceID int64
InterruptGeneration int
}
func monitorDesktopTaskSpecIDs(specs []monitorDesktopTaskSpec) []int64 {
if len(specs) == 0 {
return nil
@@ -747,84 +738,6 @@ func (s *MonitoringService) fallbackMonitorDesktopTasksToLegacy(ctx context.Cont
return nil
}
func (s *MonitoringService) requestPhase2MonitorInterrupts(
ctx context.Context,
targetClientID uuid.UUID,
excludedMonitorTaskIDs []int64,
excludedPlatformIDs []string,
interruptGeneration int,
) ([]monitorDesktopInterruptTarget, error) {
if s == nil || s.businessPool == nil {
return nil, nil
}
queryArgs := []any{targetClientID, interruptGeneration}
nextParam := 3
query := `
UPDATE desktop_tasks
SET control_flags = (
CASE
WHEN control_flags IS NULL OR jsonb_typeof(control_flags) <> 'object'
THEN '{}'::jsonb
ELSE control_flags
END
) || jsonb_build_object(
'interrupt_requested', true,
'interrupt_generation', $2::integer,
'interrupt_reason', 'collect_now_preempt'
),
interrupt_generation = GREATEST(interrupt_generation, $2::integer),
interrupted_at = COALESCE(interrupted_at, NOW()),
interrupt_reason = 'collect_now_preempt',
updated_at = NOW()
WHERE kind = 'monitor'
AND target_client_id = $1
AND status = 'in_progress'
AND lane IN ('normal', 'normal_boosted', 'retry')
AND monitor_task_id IS NOT NULL
`
if len(excludedMonitorTaskIDs) > 0 {
query += fmt.Sprintf(` AND NOT (monitor_task_id = ANY($%d::bigint[]))`, nextParam)
queryArgs = append(queryArgs, excludedMonitorTaskIDs)
nextParam++
}
excludedPlatformIDs = reconcileEnabledMonitoringPlatforms(excludedPlatformIDs)
if len(excludedPlatformIDs) > 0 {
query += fmt.Sprintf(` AND NOT (platform_id = ANY($%d::text[]))`, nextParam)
queryArgs = append(queryArgs, excludedPlatformIDs)
}
query += `
RETURNING desktop_id::text, monitor_task_id, target_client_id::text, workspace_id, interrupt_generation
`
rows, err := s.businessPool.Query(ctx, query, queryArgs...)
if err != nil {
if s.logger != nil {
s.logger.Error("phase2 monitor interrupt query failed",
zap.Error(err),
zap.String("target_client_id", targetClientID.String()),
zap.Int("interrupt_generation", interruptGeneration),
zap.Int("excluded_monitor_task_count", len(excludedMonitorTaskIDs)),
)
}
return nil, response.ErrInternal(50121, "desktop_task_interrupt_failed", "failed to mark active phase2 monitor desktop tasks for interrupt")
}
defer rows.Close()
result := make([]monitorDesktopInterruptTarget, 0)
for rows.Next() {
var item monitorDesktopInterruptTarget
if scanErr := rows.Scan(&item.TaskID, &item.MonitorTaskID, &item.TargetClientID, &item.WorkspaceID, &item.InterruptGeneration); scanErr != nil {
return nil, response.ErrInternal(50121, "desktop_task_interrupt_failed", "failed to parse active phase2 monitor interrupt targets")
}
result = append(result, item)
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50121, "desktop_task_interrupt_failed", "failed to iterate active phase2 monitor interrupt targets")
}
return result, nil
}
func (s *MonitoringService) deferQueuedPhase2MonitorTasks(
ctx context.Context,
targetClientID uuid.UUID,
@@ -953,36 +866,6 @@ func publishMonitorDesktopTaskAvailable(
return nil
}
func publishPhase2MonitorControlEvent(
ctx context.Context,
client *rabbitmq.Client,
logger *zap.Logger,
target monitorDesktopInterruptTarget,
) {
if client == nil || strings.TrimSpace(target.TargetClientID) == "" || strings.TrimSpace(target.TaskID) == "" {
return
}
if err := publishDesktopDispatchEvent(ctx, client, logger, stream.DesktopDispatchEvent{
Type: "task_control",
TaskID: target.TaskID,
WorkspaceID: target.WorkspaceID,
TargetClientID: target.TargetClientID,
Status: "in_progress",
Kind: "monitor",
Lane: "high",
InterruptGeneration: target.InterruptGeneration,
Control: "interrupt_requested",
Reason: "collect_now_preempt",
UpdatedAt: time.Now().UTC(),
}); err != nil && logger != nil {
logger.Warn("phase2 monitor control event publish failed",
zap.String("task_id", target.TaskID),
zap.Int64("monitor_task_id", target.MonitorTaskID),
zap.Error(err),
)
}
}
func monitoringLaneWeight(lane string) int {
switch strings.TrimSpace(lane) {
case "high":