feat(monitoring): dispatch monitoring tasks to desktop via AMQP outbox
Replace SSE /desktop/events with priority AMQP dispatch for monitoring
runs, and add phase1/phase2 infrastructure so desktop workers can lease,
resume, report, skip, and cancel monitoring tasks over the existing
dispatch WebSocket.
- Add monitoring collect outbox worker and phase2 desktop task fields
- Add /desktop/monitoring/tasks/{lease,resume,result,skip,cancel} routes
- Introduce execution-devtools and network-observer on desktop runtime
- Refactor runtime-controller, LoginView, and doubao adapter for the new flow
- Add channelName column to tracking views
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,14 +1,17 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"go.uber.org/zap"
|
||||
@@ -18,6 +21,7 @@ import (
|
||||
sharedllm "github.com/geo-platform/tenant-api/internal/shared/llm"
|
||||
"github.com/geo-platform/tenant-api/internal/shared/messaging/rabbitmq"
|
||||
"github.com/geo-platform/tenant-api/internal/shared/response"
|
||||
"github.com/geo-platform/tenant-api/internal/tenant/repository"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -139,6 +143,8 @@ type MonitoringLeaseTask struct {
|
||||
RunMode string `json:"run_mode"`
|
||||
TriggerSource string `json:"trigger_source"`
|
||||
BusinessDate string `json:"business_date"`
|
||||
Priority int `json:"priority,omitempty"`
|
||||
Lane string `json:"lane,omitempty"`
|
||||
LeaseToken string `json:"lease_token"`
|
||||
LeaseExpiresAt string `json:"lease_expires_at"`
|
||||
}
|
||||
@@ -165,12 +171,33 @@ type MonitoringSkipTaskResponse struct {
|
||||
CompletedAt *string `json:"completed_at"`
|
||||
}
|
||||
|
||||
type MonitoringCancelTaskRequest struct {
|
||||
LeaseToken string `json:"lease_token" binding:"required"`
|
||||
Reason *string `json:"reason"`
|
||||
}
|
||||
|
||||
type MonitoringCancelTaskResponse struct {
|
||||
TaskID int64 `json:"task_id"`
|
||||
TaskStatus string `json:"task_status"`
|
||||
}
|
||||
|
||||
type monitoringInstallationIdentity struct {
|
||||
ID string
|
||||
TenantID int64
|
||||
WorkspaceID int64
|
||||
}
|
||||
|
||||
func monitoringInstallationFromDesktopClient(client *repository.DesktopClient) *monitoringInstallationIdentity {
|
||||
if client == nil {
|
||||
return nil
|
||||
}
|
||||
return &monitoringInstallationIdentity{
|
||||
ID: client.ID.String(),
|
||||
TenantID: client.TenantID,
|
||||
WorkspaceID: client.WorkspaceID,
|
||||
}
|
||||
}
|
||||
|
||||
type monitoringPlatformAccessSnapshotState struct {
|
||||
AccessStatus string
|
||||
Connected bool
|
||||
@@ -180,23 +207,33 @@ type monitoringPlatformAccessSnapshotState struct {
|
||||
}
|
||||
|
||||
type monitoringCollectTask struct {
|
||||
ID int64
|
||||
TenantID int64
|
||||
BrandID int64
|
||||
QuestionID int64
|
||||
QuestionHash []byte
|
||||
AIPlatformID string
|
||||
CollectorType string
|
||||
RunMode string
|
||||
BusinessDate time.Time
|
||||
Status string
|
||||
LeaseTokenHash sql.NullString
|
||||
LeasedToExecutor sql.NullString
|
||||
LeaseExpiresAt sql.NullTime
|
||||
CompletedAt sql.NullTime
|
||||
SkipReason sql.NullString
|
||||
TriggerSource string
|
||||
QuestionText string
|
||||
ID int64
|
||||
TenantID int64
|
||||
BrandID int64
|
||||
QuestionID int64
|
||||
QuestionHash []byte
|
||||
AIPlatformID string
|
||||
CollectorType string
|
||||
RunMode string
|
||||
BusinessDate time.Time
|
||||
Status string
|
||||
ExecutionOwner string
|
||||
LeaseTokenHash sql.NullString
|
||||
LeasedToExecutor sql.NullString
|
||||
LeaseExpiresAt sql.NullTime
|
||||
CallbackReceivedAt sql.NullTime
|
||||
CompletedAt sql.NullTime
|
||||
SkipReason sql.NullString
|
||||
TriggerSource string
|
||||
QuestionText string
|
||||
}
|
||||
|
||||
type monitoringDesktopExecutionLease struct {
|
||||
DesktopTaskID uuid.UUID
|
||||
Status string
|
||||
TargetClientID uuid.UUID
|
||||
LeaseTokenHash []byte
|
||||
InterruptGeneration int
|
||||
}
|
||||
|
||||
type monitoringLeasedTaskRow struct {
|
||||
@@ -209,6 +246,8 @@ type monitoringLeasedTaskRow struct {
|
||||
RunMode string
|
||||
TriggerSource string
|
||||
BusinessDate time.Time
|
||||
Priority int
|
||||
Lane string
|
||||
QuestionText string
|
||||
}
|
||||
|
||||
@@ -263,6 +302,27 @@ func (s *MonitoringCallbackService) HandleTaskResult(ctx context.Context, instal
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s.handleTaskResultForInstallation(ctx, installation, taskID, req)
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) HandleTaskResultForDesktopClient(
|
||||
ctx context.Context,
|
||||
client *repository.DesktopClient,
|
||||
taskID int64,
|
||||
req MonitoringTaskResultRequest,
|
||||
) (*MonitoringTaskResultResponse, error) {
|
||||
if client == nil {
|
||||
return nil, response.ErrUnauthorized(40141, "desktop_client_missing", "desktop client context is required")
|
||||
}
|
||||
return s.handleTaskResultForInstallation(ctx, monitoringInstallationFromDesktopClient(client), taskID, req)
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) handleTaskResultForInstallation(
|
||||
ctx context.Context,
|
||||
installation *monitoringInstallationIdentity,
|
||||
taskID int64,
|
||||
req MonitoringTaskResultRequest,
|
||||
) (*MonitoringTaskResultResponse, error) {
|
||||
task, err := s.loadCollectTask(ctx, installation.TenantID, taskID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -272,18 +332,6 @@ func (s *MonitoringCallbackService) HandleTaskResult(ctx context.Context, instal
|
||||
return nil, response.ErrConflict(40941, "collector_type_mismatch", "task collector type is not desktop")
|
||||
}
|
||||
|
||||
if task.LeasedToExecutor.Valid && strings.TrimSpace(task.LeasedToExecutor.String) != "" && task.LeasedToExecutor.String != installation.ID {
|
||||
return nil, response.ErrUnauthorized(40141, "lease_owner_mismatch", "task is leased to another installation")
|
||||
}
|
||||
|
||||
if strings.TrimSpace(task.LeaseTokenHash.String) == "" {
|
||||
return nil, response.ErrConflict(40941, "lease_token_missing", "task does not have an active lease token")
|
||||
}
|
||||
|
||||
if sharedauth.HashToken(strings.TrimSpace(req.LeaseToken)) != task.LeaseTokenHash.String {
|
||||
return nil, response.ErrUnauthorized(40141, "invalid_lease_token", "lease token is invalid")
|
||||
}
|
||||
|
||||
if task.Status == "completed" {
|
||||
runID, citationSourceCount, contentCitationCount, loadErr := s.loadExistingRunSummary(ctx, task)
|
||||
if loadErr != nil {
|
||||
@@ -299,6 +347,24 @@ func (s *MonitoringCallbackService) HandleTaskResult(ctx context.Context, instal
|
||||
}, nil
|
||||
}
|
||||
|
||||
if task.ExecutionOwner == "desktop_tasks" {
|
||||
if err := s.validateDesktopMonitorExecutionLease(ctx, task, installation.ID, req.LeaseToken); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
if task.LeasedToExecutor.Valid && strings.TrimSpace(task.LeasedToExecutor.String) != "" && task.LeasedToExecutor.String != installation.ID {
|
||||
return nil, response.ErrUnauthorized(40141, "lease_owner_mismatch", "task is leased to another installation")
|
||||
}
|
||||
|
||||
if strings.TrimSpace(task.LeaseTokenHash.String) == "" {
|
||||
return nil, response.ErrConflict(40941, "lease_token_missing", "task does not have an active lease token")
|
||||
}
|
||||
|
||||
if sharedauth.HashToken(strings.TrimSpace(req.LeaseToken)) != task.LeaseTokenHash.String {
|
||||
return nil, response.ErrUnauthorized(40141, "invalid_lease_token", "lease token is invalid")
|
||||
}
|
||||
}
|
||||
|
||||
switch task.Status {
|
||||
case "leased", "received", "failed", "expired", "pending":
|
||||
default:
|
||||
@@ -322,7 +388,7 @@ func (s *MonitoringCallbackService) HandleTaskResult(ctx context.Context, instal
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
if err := s.markTaskResultReceived(ctx, tx, task.ID, receivedAt, requestPayloadJSON); err != nil {
|
||||
if err := s.markTaskResultReceived(ctx, tx, task, receivedAt, requestPayloadJSON); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -449,6 +515,21 @@ func (s *MonitoringCallbackService) LeaseTasks(ctx context.Context, installation
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s.leaseTasksForInstallation(ctx, installation, req)
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) LeaseTasksForDesktopClient(ctx context.Context, client *repository.DesktopClient, req MonitoringLeaseTasksRequest) (*MonitoringLeaseTasksResponse, error) {
|
||||
if client == nil {
|
||||
return nil, response.ErrUnauthorized(40141, "desktop_client_missing", "desktop client context is required")
|
||||
}
|
||||
return s.leaseTasksForInstallation(ctx, monitoringInstallationFromDesktopClient(client), req)
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) leaseTasksForInstallation(
|
||||
ctx context.Context,
|
||||
installation *monitoringInstallationIdentity,
|
||||
req MonitoringLeaseTasksRequest,
|
||||
) (*MonitoringLeaseTasksResponse, error) {
|
||||
limit := normalizeMonitoringLeaseLimit(req.Limit)
|
||||
eligiblePlatformIDs := normalizeMonitoringLeasePlatformIDs(req.AIPlatformIDs)
|
||||
now := time.Now().UTC()
|
||||
@@ -467,8 +548,11 @@ func (s *MonitoringCallbackService) LeaseTasks(ctx context.Context, installation
|
||||
if _, err := requeueMonitoringExpiredTasks(ctx, tx, installation.TenantID, businessDay); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := skipMonitoringTasksBeforeBusinessDate(ctx, tx, installation.TenantID, businessDay, "legacy"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rows, err := s.selectPendingLeaseTasks(ctx, tx, installation.TenantID, businessDate, limit, eligiblePlatformIDs)
|
||||
rows, err := s.selectPendingLeaseTasks(ctx, tx, installation, businessDate, limit, eligiblePlatformIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -505,6 +589,8 @@ func (s *MonitoringCallbackService) LeaseTasks(ctx context.Context, installation
|
||||
RunMode: row.RunMode,
|
||||
TriggerSource: row.TriggerSource,
|
||||
BusinessDate: row.BusinessDate.Format("2006-01-02"),
|
||||
Priority: row.Priority,
|
||||
Lane: row.Lane,
|
||||
LeaseToken: leaseToken,
|
||||
LeaseExpiresAt: leaseExpiresAt.Format(time.RFC3339),
|
||||
})
|
||||
@@ -529,6 +615,21 @@ func (s *MonitoringCallbackService) ResumeTasks(ctx context.Context, installatio
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s.resumeTasksForInstallation(ctx, installation, req)
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) ResumeTasksForDesktopClient(ctx context.Context, client *repository.DesktopClient, req MonitoringResumeTasksRequest) (*MonitoringResumeTasksResponse, error) {
|
||||
if client == nil {
|
||||
return nil, response.ErrUnauthorized(40141, "desktop_client_missing", "desktop client context is required")
|
||||
}
|
||||
return s.resumeTasksForInstallation(ctx, monitoringInstallationFromDesktopClient(client), req)
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) resumeTasksForInstallation(
|
||||
ctx context.Context,
|
||||
installation *monitoringInstallationIdentity,
|
||||
req MonitoringResumeTasksRequest,
|
||||
) (*MonitoringResumeTasksResponse, error) {
|
||||
eligiblePlatformIDs := normalizeMonitoringLeasePlatformIDs(req.AIPlatformIDs)
|
||||
now := time.Now().UTC()
|
||||
businessDay, businessDate := monitoringBusinessDayAndDateAt(now)
|
||||
@@ -546,6 +647,9 @@ func (s *MonitoringCallbackService) ResumeTasks(ctx context.Context, installatio
|
||||
if _, err := requeueMonitoringExpiredTasks(ctx, tx, installation.TenantID, businessDay); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := skipMonitoringTasksBeforeBusinessDate(ctx, tx, installation.TenantID, businessDay, "legacy"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rows, err := s.selectLeasedTasksForInstallation(ctx, tx, installation.TenantID, installation.ID, businessDate, eligiblePlatformIDs)
|
||||
if err != nil {
|
||||
@@ -584,6 +688,8 @@ func (s *MonitoringCallbackService) ResumeTasks(ctx context.Context, installatio
|
||||
RunMode: row.RunMode,
|
||||
TriggerSource: row.TriggerSource,
|
||||
BusinessDate: row.BusinessDate.Format("2006-01-02"),
|
||||
Priority: row.Priority,
|
||||
Lane: row.Lane,
|
||||
LeaseToken: leaseToken,
|
||||
LeaseExpiresAt: leaseExpiresAt.Format(time.RFC3339),
|
||||
})
|
||||
@@ -608,6 +714,27 @@ func (s *MonitoringCallbackService) SkipTask(ctx context.Context, installationID
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s.skipTaskForInstallation(ctx, installation, taskID, req)
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) SkipTaskForDesktopClient(
|
||||
ctx context.Context,
|
||||
client *repository.DesktopClient,
|
||||
taskID int64,
|
||||
req MonitoringSkipTaskRequest,
|
||||
) (*MonitoringSkipTaskResponse, error) {
|
||||
if client == nil {
|
||||
return nil, response.ErrUnauthorized(40141, "desktop_client_missing", "desktop client context is required")
|
||||
}
|
||||
return s.skipTaskForInstallation(ctx, monitoringInstallationFromDesktopClient(client), taskID, req)
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) skipTaskForInstallation(
|
||||
ctx context.Context,
|
||||
installation *monitoringInstallationIdentity,
|
||||
taskID int64,
|
||||
req MonitoringSkipTaskRequest,
|
||||
) (*MonitoringSkipTaskResponse, error) {
|
||||
task, err := s.loadCollectTask(ctx, installation.TenantID, taskID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -616,7 +743,7 @@ func (s *MonitoringCallbackService) SkipTask(ctx context.Context, installationID
|
||||
if task.CollectorType != monitoringCollectorType {
|
||||
return nil, response.ErrConflict(40941, "collector_type_mismatch", "task collector type is not plugin")
|
||||
}
|
||||
if err := validateTaskLease(task, installation.ID, req.LeaseToken); err != nil {
|
||||
if err := s.validateCollectTaskExecution(ctx, task, installation.ID, req.LeaseToken); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -681,6 +808,105 @@ func (s *MonitoringCallbackService) SkipTask(ctx context.Context, installationID
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) CancelTaskForDesktopClient(
|
||||
ctx context.Context,
|
||||
client *repository.DesktopClient,
|
||||
taskID int64,
|
||||
req MonitoringCancelTaskRequest,
|
||||
) (*MonitoringCancelTaskResponse, error) {
|
||||
if client == nil {
|
||||
return nil, response.ErrUnauthorized(40141, "desktop_client_missing", "desktop client context is required")
|
||||
}
|
||||
|
||||
installation := monitoringInstallationFromDesktopClient(client)
|
||||
task, err := s.loadCollectTask(ctx, installation.TenantID, taskID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if task.CollectorType != monitoringCollectorType {
|
||||
return nil, response.ErrConflict(40941, "collector_type_mismatch", "task collector type is not desktop")
|
||||
}
|
||||
if err := s.validateCollectTaskExecution(ctx, task, installation.ID, req.LeaseToken); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var desktopTaskCanceled *repository.DesktopTask
|
||||
if task.ExecutionOwner == "desktop_tasks" {
|
||||
desktopExecution, leaseErr := s.loadActiveDesktopMonitorExecutionLease(ctx, task.ID)
|
||||
if leaseErr != nil {
|
||||
return nil, leaseErr
|
||||
}
|
||||
if desktopExecution == nil || desktopExecution.Status != "in_progress" {
|
||||
return nil, response.ErrConflict(40941, "task_status_invalid", "task status does not allow lease cancel")
|
||||
}
|
||||
errorJSON, marshalErr := json.Marshal(map[string]any{
|
||||
"reason": optionalStringValue(req.Reason),
|
||||
"source": "legacy_monitor_cancel",
|
||||
})
|
||||
if marshalErr != nil {
|
||||
return nil, response.ErrInternal(50041, "payload_encode_failed", "failed to encode phase2 desktop cancel payload")
|
||||
}
|
||||
taskRepo := repository.NewDesktopTaskRepository(s.businessPool)
|
||||
canceledTask, cancelErr := taskRepo.CancelByLease(ctx, desktopExecution.DesktopTaskID, desktopExecution.LeaseTokenHash, errorJSON)
|
||||
if cancelErr != nil {
|
||||
if errors.Is(cancelErr, pgx.ErrNoRows) {
|
||||
return nil, response.ErrConflict(40941, "task_status_invalid", "phase2 monitor desktop task is no longer active")
|
||||
}
|
||||
return nil, response.ErrInternal(50041, "desktop_task_cancel_failed", "failed to cancel phase2 monitor desktop task lease")
|
||||
}
|
||||
desktopTaskCanceled = canceledTask
|
||||
} else if task.Status != "leased" {
|
||||
return nil, response.ErrConflict(40941, "task_status_invalid", "task status does not allow lease cancel")
|
||||
}
|
||||
|
||||
if _, err := s.monitoringPool.Exec(ctx, `
|
||||
UPDATE monitoring_collect_tasks
|
||||
SET status = 'pending',
|
||||
lease_token_hash = NULL,
|
||||
leased_to_executor = NULL,
|
||||
leased_at = NULL,
|
||||
lease_expires_at = NULL,
|
||||
callback_received_at = NULL,
|
||||
dispatch_after = NOW(),
|
||||
error_message = $2,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
`, task.ID, nullableString(req.Reason)); err != nil {
|
||||
return nil, response.ErrInternal(50041, "task_update_failed", "failed to cancel monitoring task lease")
|
||||
}
|
||||
|
||||
if desktopTaskCanceled != nil {
|
||||
title, businessDate, schedulerGroupKey, questionText := deriveDesktopTaskEventMetadataFromPayload(
|
||||
desktopTaskCanceled.Kind,
|
||||
desktopTaskCanceled.Payload,
|
||||
)
|
||||
publishDesktopTaskEvent(ctx, s.rabbitMQ, DesktopTaskEvent{
|
||||
Type: "task_canceled",
|
||||
TaskID: desktopTaskCanceled.DesktopID.String(),
|
||||
JobID: desktopTaskCanceled.JobID.String(),
|
||||
WorkspaceID: desktopTaskCanceled.WorkspaceID,
|
||||
TargetAccountID: desktopTaskCanceled.TargetAccountID.String(),
|
||||
TargetClientID: desktopTaskCanceled.TargetClientID.String(),
|
||||
Platform: desktopTaskCanceled.Platform,
|
||||
Title: title,
|
||||
BusinessDate: businessDate,
|
||||
SchedulerGroupKey: schedulerGroupKey,
|
||||
QuestionText: questionText,
|
||||
Status: desktopTaskCanceled.Status,
|
||||
Kind: desktopTaskCanceled.Kind,
|
||||
Priority: desktopTaskCanceled.Priority,
|
||||
Lane: desktopTaskCanceled.Lane,
|
||||
InterruptGeneration: desktopTaskCanceled.InterruptGeneration,
|
||||
UpdatedAt: desktopTaskCanceled.UpdatedAt,
|
||||
})
|
||||
}
|
||||
|
||||
return &MonitoringCancelTaskResponse{
|
||||
TaskID: task.ID,
|
||||
TaskStatus: "pending",
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) ProcessQueuedTaskResult(ctx context.Context, envelope monitoringTaskResultEnvelope) (*MonitoringTaskResultResponse, error) {
|
||||
if envelope.TaskID <= 0 {
|
||||
return nil, response.ErrBadRequest(40041, "invalid_task_id", "task id is required")
|
||||
@@ -698,7 +924,7 @@ func (s *MonitoringCallbackService) ProcessQueuedTaskResult(ctx context.Context,
|
||||
return nil, response.ErrConflict(40941, "task_tenant_mismatch", "task tenant does not match queued monitoring result")
|
||||
}
|
||||
|
||||
if task.Status != "received" {
|
||||
if !monitoringTaskReadyForQueuedResult(task) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@@ -816,9 +1042,11 @@ func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantI
|
||||
run_mode,
|
||||
business_date,
|
||||
status,
|
||||
COALESCE(execution_owner, 'legacy') AS execution_owner,
|
||||
lease_token_hash,
|
||||
leased_to_executor,
|
||||
lease_expires_at,
|
||||
callback_received_at,
|
||||
completed_at,
|
||||
skip_reason,
|
||||
trigger_source,
|
||||
@@ -847,9 +1075,11 @@ func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantI
|
||||
&item.RunMode,
|
||||
&item.BusinessDate,
|
||||
&item.Status,
|
||||
&item.ExecutionOwner,
|
||||
&item.LeaseTokenHash,
|
||||
&item.LeasedToExecutor,
|
||||
&item.LeaseExpiresAt,
|
||||
&item.CallbackReceivedAt,
|
||||
&item.CompletedAt,
|
||||
&item.SkipReason,
|
||||
&item.TriggerSource,
|
||||
@@ -878,9 +1108,11 @@ func (s *MonitoringCallbackService) loadCollectTaskByID(ctx context.Context, tas
|
||||
run_mode,
|
||||
business_date,
|
||||
status,
|
||||
COALESCE(execution_owner, 'legacy') AS execution_owner,
|
||||
lease_token_hash,
|
||||
leased_to_executor,
|
||||
lease_expires_at,
|
||||
callback_received_at,
|
||||
completed_at,
|
||||
skip_reason,
|
||||
trigger_source,
|
||||
@@ -908,9 +1140,11 @@ func (s *MonitoringCallbackService) loadCollectTaskByID(ctx context.Context, tas
|
||||
&item.RunMode,
|
||||
&item.BusinessDate,
|
||||
&item.Status,
|
||||
&item.ExecutionOwner,
|
||||
&item.LeaseTokenHash,
|
||||
&item.LeasedToExecutor,
|
||||
&item.LeaseExpiresAt,
|
||||
&item.CallbackReceivedAt,
|
||||
&item.CompletedAt,
|
||||
&item.SkipReason,
|
||||
&item.TriggerSource,
|
||||
@@ -925,15 +1159,48 @@ func (s *MonitoringCallbackService) loadCollectTaskByID(ctx context.Context, tas
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) loadActiveDesktopMonitorExecutionLease(
|
||||
ctx context.Context,
|
||||
monitorTaskID int64,
|
||||
) (*monitoringDesktopExecutionLease, error) {
|
||||
item := &monitoringDesktopExecutionLease{}
|
||||
if err := s.businessPool.QueryRow(ctx, `
|
||||
SELECT
|
||||
desktop_id,
|
||||
status,
|
||||
target_client_id,
|
||||
lease_token_hash,
|
||||
interrupt_generation
|
||||
FROM desktop_tasks
|
||||
WHERE kind = 'monitor'
|
||||
AND monitor_task_id = $1
|
||||
AND status IN ('queued', 'in_progress', 'unknown')
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT 1
|
||||
`, monitorTaskID).Scan(
|
||||
&item.DesktopTaskID,
|
||||
&item.Status,
|
||||
&item.TargetClientID,
|
||||
&item.LeaseTokenHash,
|
||||
&item.InterruptGeneration,
|
||||
); err != nil {
|
||||
if err == pgx.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, response.ErrInternal(50041, "query_failed", "failed to inspect phase2 monitor desktop execution lease")
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) selectPendingLeaseTasks(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
tenantID int64,
|
||||
installation *monitoringInstallationIdentity,
|
||||
businessDate string,
|
||||
limit int,
|
||||
eligiblePlatformIDs []string,
|
||||
) ([]monitoringLeasedTaskRow, error) {
|
||||
queryArgs := []interface{}{tenantID, monitoringCollectorType, businessDate, limit}
|
||||
queryArgs := []interface{}{installation.TenantID, monitoringCollectorType, businessDate, limit, installation.ID}
|
||||
query := `
|
||||
SELECT
|
||||
t.id,
|
||||
@@ -945,6 +1212,8 @@ func (s *MonitoringCallbackService) selectPendingLeaseTasks(
|
||||
t.run_mode,
|
||||
t.trigger_source,
|
||||
t.business_date,
|
||||
COALESCE(t.dispatch_priority, 100) AS dispatch_priority,
|
||||
COALESCE(t.dispatch_lane, 'normal') AS dispatch_lane,
|
||||
COALESCE((
|
||||
SELECT question_text_snapshot
|
||||
FROM monitoring_question_config_snapshots
|
||||
@@ -961,6 +1230,12 @@ func (s *MonitoringCallbackService) selectPendingLeaseTasks(
|
||||
AND t.collector_type = $2
|
||||
AND t.status = 'pending'
|
||||
AND t.business_date = $3::date
|
||||
AND COALESCE(t.execution_owner, 'legacy') = 'legacy'
|
||||
AND (t.dispatch_after IS NULL OR t.dispatch_after <= NOW())
|
||||
AND (
|
||||
t.target_client_id IS NULL
|
||||
OR t.target_client_id = $5::uuid
|
||||
)
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
FROM monitoring_question_config_snapshots s
|
||||
@@ -975,12 +1250,22 @@ func (s *MonitoringCallbackService) selectPendingLeaseTasks(
|
||||
`
|
||||
if len(eligiblePlatformIDs) > 0 {
|
||||
query += `
|
||||
AND t.ai_platform_id = ANY($5::text[])
|
||||
AND t.ai_platform_id = ANY($6::text[])
|
||||
`
|
||||
queryArgs = append(queryArgs, eligiblePlatformIDs)
|
||||
}
|
||||
query += `
|
||||
ORDER BY t.planned_at ASC, t.id ASC
|
||||
ORDER BY CASE t.dispatch_lane
|
||||
WHEN 'high' THEN 70
|
||||
WHEN 'normal_boosted' THEN 50
|
||||
WHEN 'normal' THEN 40
|
||||
WHEN 'retry' THEN 20
|
||||
ELSE 0
|
||||
END DESC,
|
||||
t.dispatch_priority DESC,
|
||||
t.dispatch_after ASC NULLS FIRST,
|
||||
t.planned_at ASC,
|
||||
t.id ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT $4
|
||||
`
|
||||
@@ -1004,6 +1289,8 @@ func (s *MonitoringCallbackService) selectPendingLeaseTasks(
|
||||
&item.RunMode,
|
||||
&item.TriggerSource,
|
||||
&item.BusinessDate,
|
||||
&item.Priority,
|
||||
&item.Lane,
|
||||
&item.QuestionText,
|
||||
); scanErr != nil {
|
||||
return nil, response.ErrInternal(50041, "lease_scan_failed", "failed to parse monitoring lease task")
|
||||
@@ -1037,6 +1324,8 @@ func (s *MonitoringCallbackService) selectLeasedTasksForInstallation(
|
||||
t.run_mode,
|
||||
t.trigger_source,
|
||||
t.business_date,
|
||||
COALESCE(t.dispatch_priority, 100) AS dispatch_priority,
|
||||
COALESCE(t.dispatch_lane, 'normal') AS dispatch_lane,
|
||||
COALESCE((
|
||||
SELECT question_text_snapshot
|
||||
FROM monitoring_question_config_snapshots
|
||||
@@ -1053,6 +1342,7 @@ func (s *MonitoringCallbackService) selectLeasedTasksForInstallation(
|
||||
AND t.collector_type = $2
|
||||
AND t.status = 'leased'
|
||||
AND t.business_date = $3::date
|
||||
AND COALESCE(t.execution_owner, 'legacy') = 'legacy'
|
||||
AND t.leased_to_executor = $4
|
||||
AND EXISTS (
|
||||
SELECT 1
|
||||
@@ -1096,6 +1386,8 @@ func (s *MonitoringCallbackService) selectLeasedTasksForInstallation(
|
||||
&item.RunMode,
|
||||
&item.TriggerSource,
|
||||
&item.BusinessDate,
|
||||
&item.Priority,
|
||||
&item.Lane,
|
||||
&item.QuestionText,
|
||||
); scanErr != nil {
|
||||
return nil, response.ErrInternal(50041, "resume_scan_failed", "failed to parse monitoring resumed task")
|
||||
@@ -1117,6 +1409,8 @@ func (s *MonitoringCallbackService) markTaskLeased(ctx context.Context, tx pgx.T
|
||||
leased_to_executor = $3,
|
||||
leased_at = $4,
|
||||
lease_expires_at = $5,
|
||||
last_dispatched_at = NOW(),
|
||||
dispatch_attempts = COALESCE(dispatch_attempts, 0) + 1,
|
||||
skip_reason = NULL,
|
||||
error_message = NULL,
|
||||
updated_at = NOW()
|
||||
@@ -1127,11 +1421,20 @@ func (s *MonitoringCallbackService) markTaskLeased(ctx context.Context, tx pgx.T
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) markTaskResultReceived(ctx context.Context, tx pgx.Tx, taskID int64, receivedAt time.Time, requestPayloadJSON []byte) error {
|
||||
func (s *MonitoringCallbackService) markTaskResultReceived(ctx context.Context, tx pgx.Tx, task *monitoringCollectTask, receivedAt time.Time, requestPayloadJSON []byte) error {
|
||||
if task == nil {
|
||||
return response.ErrBadRequest(40041, "invalid_task_id", "monitoring task is required")
|
||||
}
|
||||
|
||||
nextStatus := "received"
|
||||
if task.ExecutionOwner == "desktop_tasks" {
|
||||
nextStatus = "pending"
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE monitoring_collect_tasks
|
||||
SET status = 'received',
|
||||
callback_received_at = $2,
|
||||
SET status = $2,
|
||||
callback_received_at = $3,
|
||||
completed_at = NULL,
|
||||
skip_reason = NULL,
|
||||
error_message = NULL,
|
||||
@@ -1141,10 +1444,10 @@ func (s *MonitoringCallbackService) markTaskResultReceived(ctx context.Context,
|
||||
THEN '{}'::jsonb
|
||||
ELSE request_payload_json
|
||||
END
|
||||
) || $3::jsonb,
|
||||
) || $4::jsonb,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
`, taskID, receivedAt, requestPayloadJSON); err != nil {
|
||||
`, task.ID, nextStatus, receivedAt, requestPayloadJSON); err != nil {
|
||||
return response.ErrInternal(50041, "task_receive_failed", "failed to persist monitoring callback payload")
|
||||
}
|
||||
return nil
|
||||
@@ -2091,6 +2394,49 @@ func normalizeMonitoringAccessStatus(accessStatus *string, connected *bool, acce
|
||||
return "", false, false, response.ErrBadRequest(40041, "invalid_access_status", "access status requires access_status, connected, or accessible")
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) validateCollectTaskExecution(
|
||||
ctx context.Context,
|
||||
task *monitoringCollectTask,
|
||||
installationID string,
|
||||
leaseToken string,
|
||||
) error {
|
||||
if task == nil {
|
||||
return response.ErrBadRequest(40041, "invalid_task_id", "monitoring task is required")
|
||||
}
|
||||
if task.ExecutionOwner == "desktop_tasks" {
|
||||
return s.validateDesktopMonitorExecutionLease(ctx, task, installationID, leaseToken)
|
||||
}
|
||||
return validateTaskLease(task, installationID, leaseToken)
|
||||
}
|
||||
|
||||
func (s *MonitoringCallbackService) validateDesktopMonitorExecutionLease(
|
||||
ctx context.Context,
|
||||
task *monitoringCollectTask,
|
||||
installationID string,
|
||||
leaseToken string,
|
||||
) error {
|
||||
desktopExecution, err := s.loadActiveDesktopMonitorExecutionLease(ctx, task.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if desktopExecution == nil {
|
||||
return response.ErrConflict(40941, "desktop_task_missing", "phase2 monitor desktop task is not active")
|
||||
}
|
||||
if desktopExecution.TargetClientID.String() != installationID {
|
||||
return response.ErrUnauthorized(40141, "lease_owner_mismatch", "task is leased to another installation")
|
||||
}
|
||||
if desktopExecution.Status != "in_progress" {
|
||||
return response.ErrConflict(40941, "task_status_invalid", "phase2 monitor desktop task is not currently in progress")
|
||||
}
|
||||
if len(desktopExecution.LeaseTokenHash) == 0 {
|
||||
return response.ErrConflict(40941, "lease_token_missing", "phase2 monitor desktop task does not have an active lease token")
|
||||
}
|
||||
if !bytes.Equal(HashDesktopClientToken(strings.TrimSpace(leaseToken)), desktopExecution.LeaseTokenHash) {
|
||||
return response.ErrUnauthorized(40141, "invalid_lease_token", "lease token is invalid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateTaskLease(task *monitoringCollectTask, installationID string, leaseToken string) error {
|
||||
if task.LeasedToExecutor.Valid && strings.TrimSpace(task.LeasedToExecutor.String) != "" && task.LeasedToExecutor.String != installationID {
|
||||
return response.ErrUnauthorized(40141, "lease_owner_mismatch", "task is leased to another installation")
|
||||
@@ -2107,6 +2453,16 @@ func validateTaskLease(task *monitoringCollectTask, installationID string, lease
|
||||
return nil
|
||||
}
|
||||
|
||||
func monitoringTaskReadyForQueuedResult(task *monitoringCollectTask) bool {
|
||||
if task == nil {
|
||||
return false
|
||||
}
|
||||
if task.ExecutionOwner == "desktop_tasks" {
|
||||
return task.Status == "pending" && task.CallbackReceivedAt.Valid
|
||||
}
|
||||
return task.Status == "received"
|
||||
}
|
||||
|
||||
func normalizeSkipReason(value *string) string {
|
||||
if value == nil {
|
||||
return "plugin_skipped"
|
||||
|
||||
Reference in New Issue
Block a user