feat(monitoring): dispatch monitoring tasks to desktop via AMQP outbox

Replace SSE /desktop/events with priority AMQP dispatch for monitoring runs, and add phase1/phase2 infrastructure so desktop workers can lease, resume, report, skip, and cancel monitoring tasks over the existing dispatch WebSocket. - Add monitoring collect outbox worker and phase2 desktop task fields - Add /desktop/monitoring/tasks/{lease,resume,result,skip,cancel} routes - Introduce execution-devtools and network-observer on desktop runtime - Refactor runtime-controller, LoginView, and doubao adapter for the new flow - Add channelName column to tracking views Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-22 00:24:21 +08:00
parent 749b6b99cd
commit 4142c53fa6
57 changed files with 7897 additions and 985 deletions
@@ -17,23 +17,26 @@ import (
 	sharedcache "github.com/geo-platform/tenant-api/internal/shared/cache"
 	"github.com/geo-platform/tenant-api/internal/shared/messaging/rabbitmq"
 	"github.com/geo-platform/tenant-api/internal/shared/response"
+	"github.com/geo-platform/tenant-api/internal/shared/stream"
 	"github.com/geo-platform/tenant-api/internal/tenant/repository"
 )

 type DesktopTaskService struct {
-	pool      *pgxpool.Pool
-	repo      repository.DesktopTaskRepository
-	cache     sharedcache.Cache
-	messaging *rabbitmq.Client
-	logger    *zap.Logger
+	pool           *pgxpool.Pool
+	monitoringPool *pgxpool.Pool
+	repo           repository.DesktopTaskRepository
+	cache          sharedcache.Cache
+	messaging      *rabbitmq.Client
+	logger         *zap.Logger
 }

-func NewDesktopTaskService(pool *pgxpool.Pool, messaging *rabbitmq.Client, logger *zap.Logger) *DesktopTaskService {
+func NewDesktopTaskService(pool *pgxpool.Pool, monitoringPool *pgxpool.Pool, messaging *rabbitmq.Client, logger *zap.Logger) *DesktopTaskService {
 	return &DesktopTaskService{
-		pool:      pool,
-		repo:      repository.NewDesktopTaskRepository(pool),
-		messaging: messaging,
-		logger:    logger,
+		pool:           pool,
+		monitoringPool: monitoringPool,
+		repo:           repository.NewDesktopTaskRepository(pool),
+		messaging:      messaging,
+		logger:         logger,
 	}
 }

@@ -126,6 +129,10 @@ func (s *DesktopTaskService) Lease(ctx context.Context, client *repository.Deskt
 		taskID = &parsed
 	}

+	if err := s.dropStaleMonitorTasksForClient(ctx, client); err != nil {
+		return nil, err
+	}
+
 	rawToken, tokenHash, err := newDesktopClientToken()
 	if err != nil {
 		return nil, response.ErrInternal(50087, "desktop_task_lease_token_failed", "failed to generate desktop task lease token")
@@ -137,6 +144,19 @@ func (s *DesktopTaskService) Lease(ctx context.Context, client *repository.Deskt
 		LeaseTokenHash: tokenHash,
 	}

+	if s.logger != nil {
+		fields := []zap.Field{
+			zap.String("client_id", client.ID.String()),
+		}
+		if req.Kind != nil {
+			fields = append(fields, zap.String("requested_kind", strings.TrimSpace(*req.Kind)))
+		}
+		if taskID != nil {
+			fields = append(fields, zap.String("task_id", taskID.String()))
+		}
+		s.logger.Debug("desktop task lease requested", fields...)
+	}
+
 	var task *repository.DesktopTask
 	switch {
 	case taskID != nil:
@@ -146,6 +166,18 @@ func (s *DesktopTaskService) Lease(ctx context.Context, client *repository.Deskt
 	}
 	if err != nil {
 		if errors.Is(err, pgx.ErrNoRows) {
+			if s.logger != nil {
+				fields := []zap.Field{
+					zap.String("client_id", client.ID.String()),
+				}
+				if req.Kind != nil {
+					fields = append(fields, zap.String("requested_kind", strings.TrimSpace(*req.Kind)))
+				}
+				if taskID != nil {
+					fields = append(fields, zap.String("task_id", taskID.String()))
+				}
+				s.logger.Debug("desktop task lease found no matching queued task", fields...)
+			}
 			if taskID == nil {
 				return &LeaseDesktopTaskResponse{}, nil
 			}
@@ -165,6 +197,15 @@ func (s *DesktopTaskService) Lease(ctx context.Context, client *repository.Deskt

 	s.publishTaskEvent(ctx, task, "task_leased")

+	if s.logger != nil {
+		s.logger.Info("desktop task leased",
+			zap.String("client_id", client.ID.String()),
+			zap.String("task_id", task.DesktopID.String()),
+			zap.String("kind", task.Kind),
+			zap.String("status", task.Status),
+		)
+	}
+
 	view := buildDesktopTaskView(task)
 	attemptIDText := attemptID.String()
 	return &LeaseDesktopTaskResponse{
@@ -180,6 +221,10 @@ func (s *DesktopTaskService) Extend(ctx context.Context, client *repository.Desk
 		return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
 	}

+	if err := s.dropStaleMonitorTasksForClient(ctx, client); err != nil {
+		return nil, err
+	}
+
 	task, err := s.repo.ExtendLease(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(req.LeaseToken)))
 	if err != nil {
 		if errors.Is(err, pgx.ErrNoRows) {
@@ -262,9 +307,6 @@ func (s *DesktopTaskService) Cancel(ctx context.Context, client *repository.Desk
 		return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
 	}

-	previousTask, _ := s.repo.GetByDesktopID(ctx, desktopID, client.WorkspaceID)
-	previousAttemptID := activeAttemptID(previousTask)
-
 	errorJSON, err := marshalOptionalJSON(map[string]any{
 		"reason": optionalStringValue(req.Reason),
 		"source": "desktop_client",
@@ -273,11 +315,21 @@ func (s *DesktopTaskService) Cancel(ctx context.Context, client *repository.Desk
 		return nil, response.ErrBadRequest(40088, "invalid_desktop_task_cancel", "cancel payload must be serializable")
 	}

+	tx, err := s.pool.Begin(ctx)
+	if err != nil {
+		return nil, response.ErrInternal(50107, "desktop_task_cancel_begin_failed", "failed to start desktop task cancel")
+	}
+	defer tx.Rollback(ctx)
+
+	taskRepo := repository.NewDesktopTaskRepository(tx)
+	previousTask, _ := taskRepo.GetByDesktopID(ctx, desktopID, client.WorkspaceID)
+	previousAttemptID := activeAttemptID(previousTask)
+
 	var task *repository.DesktopTask
 	if req.LeaseToken != nil && strings.TrimSpace(*req.LeaseToken) != "" {
-		task, err = s.repo.CancelByLease(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(*req.LeaseToken)), errorJSON)
+		task, err = taskRepo.CancelByLease(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(*req.LeaseToken)), errorJSON)
 	} else {
-		task, err = s.repo.CancelByClient(ctx, desktopID, client.ID, errorJSON)
+		task, err = taskRepo.CancelByClient(ctx, desktopID, client.ID, errorJSON)
 	}
 	if err != nil {
 		if errors.Is(err, pgx.ErrNoRows) {
@@ -287,7 +339,7 @@ func (s *DesktopTaskService) Cancel(ctx context.Context, client *repository.Desk
 	}

 	if previousAttemptID != nil && req.LeaseToken != nil && strings.TrimSpace(*req.LeaseToken) != "" {
-		if finishErr := s.repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
+		if finishErr := taskRepo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
 			TaskID:      task.DesktopID,
 			AttemptID:   *previousAttemptID,
 			FinalStatus: literalStringPtr("aborted"),
@@ -297,7 +349,22 @@ func (s *DesktopTaskService) Cancel(ctx context.Context, client *repository.Desk
 		}
 	}

+	var replacementTask *repository.DesktopTask
+	if shouldRequeuePreemptedMonitorTask(task, req.Reason) {
+		replacementTask, err = requeuePreemptedMonitorTask(ctx, tx, taskRepo, task)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if err := tx.Commit(ctx); err != nil {
+		return nil, response.ErrInternal(50108, "desktop_task_cancel_commit_failed", "failed to commit desktop task cancel")
+	}
+
 	s.publishTaskEvent(ctx, task, "task_canceled")
+	if replacementTask != nil {
+		s.publishTaskEvent(ctx, replacementTask, "task_available")
+	}

 	view := buildDesktopTaskView(task)
 	return &view, nil
@@ -372,7 +439,11 @@ func (s *DesktopTaskService) Reconcile(ctx context.Context, actor auth.Actor, de
 		invalidateArticleCaches(ctx, s.cache, publishOutcome.TenantID, &publishOutcome.ArticleID)
 	}

-	s.publishTaskEvent(ctx, task, "task_reconciled")
+	eventType := "task_reconciled"
+	if task.Kind == "monitor" && req.Status == "retry" {
+		eventType = "task_available"
+	}
+	s.publishTaskEvent(ctx, task, eventType)

 	view := buildDesktopTaskView(task)
 	return &view, nil
@@ -712,6 +783,143 @@ func normalizeDesktopTaskTerminalStatus(status string) string {
 	}
 }

+func (s *DesktopTaskService) dropStaleMonitorTasksForClient(ctx context.Context, client *repository.DesktopClient) error {
+	if s == nil || client == nil || s.pool == nil || s.monitoringPool == nil {
+		return nil
+	}
+
+	_, currentBusinessDate := monitoringBusinessDayAndDateAt(time.Now())
+	errorJSON, err := marshalOptionalJSON(map[string]any{
+		"reason":                monitoringStaleTaskDropReason,
+		"message":               monitoringStaleTaskDropError,
+		"current_business_date": currentBusinessDate,
+		"source":                "business_day_rollover",
+	})
+	if err != nil {
+		return response.ErrInternal(50190, "desktop_task_stale_payload_failed", "failed to encode stale monitor desktop task payload")
+	}
+
+	tx, err := s.pool.Begin(ctx)
+	if err != nil {
+		return response.ErrInternal(50191, "desktop_task_stale_begin_failed", "failed to start stale monitor desktop cleanup")
+	}
+	defer tx.Rollback(ctx)
+
+	rows, err := tx.Query(ctx, `
+		WITH stale AS (
+			SELECT
+				desktop_id,
+				active_attempt_id
+			FROM desktop_tasks
+			WHERE target_client_id = $1
+			  AND kind = 'monitor'
+			  AND status IN ('queued', 'in_progress', 'unknown')
+			  AND COALESCE(
+				NULLIF(payload ->> 'business_date', ''),
+				NULLIF(payload ->> 'businessDate', ''),
+				NULLIF(payload ->> 'metric_date', ''),
+				NULLIF(payload ->> 'date', '')
+			  ) < $2
+			FOR UPDATE
+		),
+		updated AS (
+			UPDATE desktop_tasks AS t
+			SET status = 'aborted',
+			    error = $3,
+			    active_attempt_id = NULL,
+			    lease_token_hash = NULL,
+			    lease_expires_at = NULL,
+			    interrupted_at = COALESCE(interrupted_at, NOW()),
+			    interrupt_reason = $4,
+			    updated_at = NOW()
+			FROM stale
+			WHERE t.desktop_id = stale.desktop_id
+			RETURNING t.desktop_id
+		)
+		SELECT stale.desktop_id, stale.active_attempt_id
+		FROM stale
+		INNER JOIN updated ON updated.desktop_id = stale.desktop_id
+	`, client.ID, currentBusinessDate, errorJSON, monitoringStaleTaskDropReason)
+	if err != nil {
+		return response.ErrInternal(50192, "desktop_task_stale_query_failed", "failed to select stale monitor desktop tasks")
+	}
+	defer rows.Close()
+
+	repo := repository.NewDesktopTaskRepository(tx)
+	droppedTaskIDs := make([]uuid.UUID, 0)
+	for rows.Next() {
+		var (
+			taskID    uuid.UUID
+			attemptID pgtype.UUID
+		)
+		if scanErr := rows.Scan(&taskID, &attemptID); scanErr != nil {
+			return response.ErrInternal(50193, "desktop_task_stale_scan_failed", "failed to parse stale monitor desktop tasks")
+		}
+
+		droppedTaskIDs = append(droppedTaskIDs, taskID)
+
+		if !attemptID.Valid {
+			continue
+		}
+
+		resolvedAttemptID, convErr := uuid.FromBytes(attemptID.Bytes[:])
+		if convErr != nil {
+			s.logWarn("desktop stale monitor attempt id decode failed", convErr, zap.String("task_id", taskID.String()))
+			continue
+		}
+
+		if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
+			TaskID:      taskID,
+			AttemptID:   resolvedAttemptID,
+			FinalStatus: literalStringPtr("aborted"),
+			Error:       errorJSON,
+		}); finishErr != nil {
+			s.logWarn("desktop stale monitor attempt finish failed", finishErr, zap.String("task_id", taskID.String()))
+		}
+	}
+	if err := rows.Err(); err != nil {
+		return response.ErrInternal(50193, "desktop_task_stale_scan_failed", "failed to iterate stale monitor desktop tasks")
+	}
+
+	if err := tx.Commit(ctx); err != nil {
+		return response.ErrInternal(50194, "desktop_task_stale_commit_failed", "failed to commit stale monitor desktop cleanup")
+	}
+
+	for _, taskID := range droppedTaskIDs {
+		task, getErr := s.repo.GetByDesktopID(ctx, taskID, client.WorkspaceID)
+		if getErr != nil {
+			s.logWarn("desktop stale monitor reload failed", getErr, zap.String("task_id", taskID.String()))
+			continue
+		}
+		s.publishTaskEvent(ctx, task, "task_canceled")
+	}
+
+	if _, err := s.monitoringPool.Exec(ctx, `
+		UPDATE monitoring_collect_tasks
+		SET status = 'skipped',
+		    lease_token_hash = NULL,
+		    leased_to_executor = NULL,
+		    leased_at = NULL,
+		    lease_expires_at = NULL,
+		    callback_received_at = COALESCE(callback_received_at, NOW()),
+		    completed_at = COALESCE(completed_at, NOW()),
+		    skip_reason = $4,
+		    error_message = COALESCE(NULLIF(error_message, ''), $5),
+		    updated_at = NOW()
+		WHERE tenant_id = $1
+		  AND collector_type = $2
+		  AND COALESCE(execution_owner, 'legacy') = 'desktop_tasks'
+		  AND target_client_id = $3
+		  AND business_date < $6::date
+		  AND callback_received_at IS NULL
+		  AND status IN ('pending', 'leased', 'expired')
+	`, client.TenantID, monitoringCollectorType, client.ID, monitoringStaleTaskDropReason, monitoringStaleTaskDropError, currentBusinessDate); err != nil {
+		return response.ErrInternal(50041, "task_skip_failed", "failed to drop stale phase2 monitoring tasks")
+	}
+
+	return nil
+}
+
 func activeAttemptID(task *repository.DesktopTask) *uuid.UUID {
 	if task == nil || task.ActiveAttemptID == nil {
 		return nil
@@ -720,31 +928,136 @@ func activeAttemptID(task *repository.DesktopTask) *uuid.UUID {
 	return &value
 }

+func shouldRequeuePreemptedMonitorTask(task *repository.DesktopTask, reason *string) bool {
+	if task == nil || task.Kind != "monitor" {
+		return false
+	}
+	return strings.EqualFold(strings.TrimSpace(optionalStringValue(reason)), "collect_now_preempt")
+}
+
+func requeuePreemptedMonitorTask(
+	ctx context.Context,
+	tx pgx.Tx,
+	repo repository.DesktopTaskRepository,
+	task *repository.DesktopTask,
+) (*repository.DesktopTask, error) {
+	if task == nil {
+		return nil, nil
+	}
+	replacementID := uuid.New()
+	replacementJobID := uuid.New()
+	if _, err := tx.Exec(ctx, `
+		INSERT INTO desktop_tasks (
+			desktop_id,
+			job_id,
+			tenant_id,
+			workspace_id,
+			target_account_id,
+			target_client_id,
+			platform_id,
+			kind,
+			payload,
+			status,
+			priority,
+			lane,
+			lane_weight,
+			source,
+			scheduler_group_key,
+			monitor_task_id,
+			supersedes_task_id,
+			control_flags,
+			interrupt_generation,
+			enqueued_at
+		)
+		VALUES (
+			$1,
+			$2,
+			$3,
+			$4,
+			$5,
+			$6,
+			$7,
+			'monitor',
+			$8,
+			'queued',
+			50,
+			'retry',
+			20,
+			'retry_recovery',
+			$9,
+			$10,
+			$11,
+			jsonb_build_object(
+				'interrupt_requested', false,
+				'requeued_at', NOW()::text,
+				'requeued_from_task_id', $11::text,
+				'requeued_reason', 'collect_now_preempt'
+			),
+			$12,
+			NOW()
+		)
+	`, replacementID, replacementJobID, task.TenantID, task.WorkspaceID, task.TargetAccountID, task.TargetClientID, task.Platform, task.Payload, nullableString(task.SchedulerGroupKey), nullableInt64(task.MonitorTaskID), task.DesktopID, task.InterruptGeneration); err != nil {
+		return nil, response.ErrInternal(50111, "desktop_task_requeue_failed", "failed to create replacement retry monitor desktop task")
+	}
+
+	next, err := repo.GetByDesktopID(ctx, replacementID, task.WorkspaceID)
+	if err != nil {
+		return nil, response.ErrInternal(50112, "desktop_task_lookup_failed", "failed to reload replacement retry monitor desktop task")
+	}
+	return next, nil
+}
+
 func (s *DesktopTaskService) publishTaskEvent(ctx context.Context, task *repository.DesktopTask, eventType string) {
 	if task == nil {
 		return
 	}

 	title, businessDate, schedulerGroupKey, questionText := deriveDesktopTaskEventMetadataFromPayload(task.Kind, task.Payload)
-	err := publishDesktopTaskEvent(ctx, s.messaging, DesktopTaskEvent{
-		Type:              eventType,
-		TaskID:            task.DesktopID.String(),
-		JobID:             task.JobID.String(),
-		WorkspaceID:       task.WorkspaceID,
-		TargetAccountID:   task.TargetAccountID.String(),
-		TargetClientID:    task.TargetClientID.String(),
-		Platform:          task.Platform,
-		Title:             title,
-		BusinessDate:      businessDate,
-		SchedulerGroupKey: schedulerGroupKey,
-		QuestionText:      questionText,
-		Status:            task.Status,
-		Kind:              task.Kind,
-		UpdatedAt:         task.UpdatedAt,
-	})
+	event := DesktopTaskEvent{
+		Type:                eventType,
+		TaskID:              task.DesktopID.String(),
+		JobID:               task.JobID.String(),
+		WorkspaceID:         task.WorkspaceID,
+		TargetAccountID:     task.TargetAccountID.String(),
+		TargetClientID:      task.TargetClientID.String(),
+		Platform:            task.Platform,
+		Title:               title,
+		BusinessDate:        businessDate,
+		SchedulerGroupKey:   schedulerGroupKey,
+		QuestionText:        questionText,
+		Status:              task.Status,
+		Kind:                task.Kind,
+		Priority:            task.Priority,
+		Lane:                task.Lane,
+		InterruptGeneration: task.InterruptGeneration,
+		UpdatedAt:           task.UpdatedAt,
+	}
+	err := publishDesktopTaskEvent(ctx, s.messaging, event)
 	if err != nil {
 		s.logWarn("desktop task event publish failed", err, zap.String("task_id", task.DesktopID.String()), zap.String("event_type", eventType))
 	}
+
+	if dispatchErr := publishDesktopDispatchEvent(ctx, s.messaging, s.logger, stream.DesktopDispatchEvent{
+		Type:                event.Type,
+		TaskID:              event.TaskID,
+		JobID:               event.JobID,
+		WorkspaceID:         event.WorkspaceID,
+		TargetAccountID:     event.TargetAccountID,
+		TargetClientID:      event.TargetClientID,
+		Platform:            event.Platform,
+		Title:               event.Title,
+		BusinessDate:        event.BusinessDate,
+		SchedulerGroupKey:   event.SchedulerGroupKey,
+		QuestionText:        event.QuestionText,
+		Status:              event.Status,
+		Kind:                event.Kind,
+		Priority:            event.Priority,
+		Lane:                event.Lane,
+		InterruptGeneration: event.InterruptGeneration,
+		UpdatedAt:           event.UpdatedAt,
+	}); dispatchErr != nil {
+		s.logWarn("desktop dispatch mirror publish failed", dispatchErr, zap.String("task_id", task.DesktopID.String()), zap.String("event_type", eventType))
+	}
 }

 func (s *DesktopTaskService) logWarn(message string, err error, fields ...zap.Field) {