Fix monitor collect-now queue fairness
This commit is contained in:
@@ -12,7 +12,6 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/messaging/rabbitmq"
|
||||
"github.com/geo-platform/tenant-api/internal/shared/response"
|
||||
"github.com/geo-platform/tenant-api/internal/shared/stream"
|
||||
"github.com/geo-platform/tenant-api/internal/tenant/repository"
|
||||
@@ -66,14 +65,6 @@ type monitorDesktopTaskTargetCandidate struct {
|
||||
ClientOrder int
|
||||
}
|
||||
|
||||
type monitorDesktopInterruptTarget struct {
|
||||
TaskID string
|
||||
MonitorTaskID int64
|
||||
TargetClientID string
|
||||
WorkspaceID int64
|
||||
InterruptGeneration int
|
||||
}
|
||||
|
||||
func monitorDesktopTaskSpecIDs(specs []monitorDesktopTaskSpec) []int64 {
|
||||
if len(specs) == 0 {
|
||||
return nil
|
||||
@@ -747,84 +738,6 @@ func (s *MonitoringService) fallbackMonitorDesktopTasksToLegacy(ctx context.Cont
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MonitoringService) requestPhase2MonitorInterrupts(
|
||||
ctx context.Context,
|
||||
targetClientID uuid.UUID,
|
||||
excludedMonitorTaskIDs []int64,
|
||||
excludedPlatformIDs []string,
|
||||
interruptGeneration int,
|
||||
) ([]monitorDesktopInterruptTarget, error) {
|
||||
if s == nil || s.businessPool == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
queryArgs := []any{targetClientID, interruptGeneration}
|
||||
nextParam := 3
|
||||
query := `
|
||||
UPDATE desktop_tasks
|
||||
SET control_flags = (
|
||||
CASE
|
||||
WHEN control_flags IS NULL OR jsonb_typeof(control_flags) <> 'object'
|
||||
THEN '{}'::jsonb
|
||||
ELSE control_flags
|
||||
END
|
||||
) || jsonb_build_object(
|
||||
'interrupt_requested', true,
|
||||
'interrupt_generation', $2::integer,
|
||||
'interrupt_reason', 'collect_now_preempt'
|
||||
),
|
||||
interrupt_generation = GREATEST(interrupt_generation, $2::integer),
|
||||
interrupted_at = COALESCE(interrupted_at, NOW()),
|
||||
interrupt_reason = 'collect_now_preempt',
|
||||
updated_at = NOW()
|
||||
WHERE kind = 'monitor'
|
||||
AND target_client_id = $1
|
||||
AND status = 'in_progress'
|
||||
AND lane IN ('normal', 'normal_boosted', 'retry')
|
||||
AND monitor_task_id IS NOT NULL
|
||||
`
|
||||
if len(excludedMonitorTaskIDs) > 0 {
|
||||
query += fmt.Sprintf(` AND NOT (monitor_task_id = ANY($%d::bigint[]))`, nextParam)
|
||||
queryArgs = append(queryArgs, excludedMonitorTaskIDs)
|
||||
nextParam++
|
||||
}
|
||||
excludedPlatformIDs = reconcileEnabledMonitoringPlatforms(excludedPlatformIDs)
|
||||
if len(excludedPlatformIDs) > 0 {
|
||||
query += fmt.Sprintf(` AND NOT (platform_id = ANY($%d::text[]))`, nextParam)
|
||||
queryArgs = append(queryArgs, excludedPlatformIDs)
|
||||
}
|
||||
query += `
|
||||
RETURNING desktop_id::text, monitor_task_id, target_client_id::text, workspace_id, interrupt_generation
|
||||
`
|
||||
|
||||
rows, err := s.businessPool.Query(ctx, query, queryArgs...)
|
||||
if err != nil {
|
||||
if s.logger != nil {
|
||||
s.logger.Error("phase2 monitor interrupt query failed",
|
||||
zap.Error(err),
|
||||
zap.String("target_client_id", targetClientID.String()),
|
||||
zap.Int("interrupt_generation", interruptGeneration),
|
||||
zap.Int("excluded_monitor_task_count", len(excludedMonitorTaskIDs)),
|
||||
)
|
||||
}
|
||||
return nil, response.ErrInternal(50121, "desktop_task_interrupt_failed", "failed to mark active phase2 monitor desktop tasks for interrupt")
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
result := make([]monitorDesktopInterruptTarget, 0)
|
||||
for rows.Next() {
|
||||
var item monitorDesktopInterruptTarget
|
||||
if scanErr := rows.Scan(&item.TaskID, &item.MonitorTaskID, &item.TargetClientID, &item.WorkspaceID, &item.InterruptGeneration); scanErr != nil {
|
||||
return nil, response.ErrInternal(50121, "desktop_task_interrupt_failed", "failed to parse active phase2 monitor interrupt targets")
|
||||
}
|
||||
result = append(result, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, response.ErrInternal(50121, "desktop_task_interrupt_failed", "failed to iterate active phase2 monitor interrupt targets")
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringService) deferQueuedPhase2MonitorTasks(
|
||||
ctx context.Context,
|
||||
targetClientID uuid.UUID,
|
||||
@@ -953,36 +866,6 @@ func publishMonitorDesktopTaskAvailable(
|
||||
return nil
|
||||
}
|
||||
|
||||
func publishPhase2MonitorControlEvent(
|
||||
ctx context.Context,
|
||||
client *rabbitmq.Client,
|
||||
logger *zap.Logger,
|
||||
target monitorDesktopInterruptTarget,
|
||||
) {
|
||||
if client == nil || strings.TrimSpace(target.TargetClientID) == "" || strings.TrimSpace(target.TaskID) == "" {
|
||||
return
|
||||
}
|
||||
if err := publishDesktopDispatchEvent(ctx, client, logger, stream.DesktopDispatchEvent{
|
||||
Type: "task_control",
|
||||
TaskID: target.TaskID,
|
||||
WorkspaceID: target.WorkspaceID,
|
||||
TargetClientID: target.TargetClientID,
|
||||
Status: "in_progress",
|
||||
Kind: "monitor",
|
||||
Lane: "high",
|
||||
InterruptGeneration: target.InterruptGeneration,
|
||||
Control: "interrupt_requested",
|
||||
Reason: "collect_now_preempt",
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
}); err != nil && logger != nil {
|
||||
logger.Warn("phase2 monitor control event publish failed",
|
||||
zap.String("task_id", target.TaskID),
|
||||
zap.Int64("monitor_task_id", target.MonitorTaskID),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func monitoringLaneWeight(lane string) int {
|
||||
switch strings.TrimSpace(lane) {
|
||||
case "high":
|
||||
|
||||
Reference in New Issue
Block a user