package app import ( "context" "encoding/json" "errors" "fmt" "strings" "time" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgxpool" goredis "github.com/redis/go-redis/v9" "go.uber.org/zap" "github.com/geo-platform/tenant-api/internal/shared/auth" sharedcache "github.com/geo-platform/tenant-api/internal/shared/cache" sharedcompliance "github.com/geo-platform/tenant-api/internal/shared/compliance" "github.com/geo-platform/tenant-api/internal/shared/config" "github.com/geo-platform/tenant-api/internal/shared/messaging/rabbitmq" "github.com/geo-platform/tenant-api/internal/shared/response" tenantcompliance "github.com/geo-platform/tenant-api/internal/tenant/compliance" "github.com/geo-platform/tenant-api/internal/tenant/repository" ) const desktopPublishMaxAttempts = 3 type DesktopTaskService struct { pool *pgxpool.Pool monitoringPool *pgxpool.Pool repo repository.DesktopTaskRepository cache sharedcache.Cache redis *goredis.Client messaging *rabbitmq.Client logger *zap.Logger compliance *tenantcompliance.Service } func NewDesktopTaskService(pool *pgxpool.Pool, monitoringPool *pgxpool.Pool, messaging *rabbitmq.Client, logger *zap.Logger) *DesktopTaskService { return &DesktopTaskService{ pool: pool, monitoringPool: monitoringPool, repo: repository.NewDesktopTaskRepository(pool), messaging: messaging, logger: logger, compliance: tenantcompliance.NewService(pool, config.NewStaticProvider(&config.Config{ Compliance: config.ComplianceConfig{Enabled: true}, }), logger).WithRabbitMQ(messaging), } } func NewDesktopTaskServiceWithConfig(pool *pgxpool.Pool, monitoringPool *pgxpool.Pool, messaging *rabbitmq.Client, logger *zap.Logger, cfg config.Provider) *DesktopTaskService { svc := NewDesktopTaskService(pool, monitoringPool, messaging, logger) svc.compliance = tenantcompliance.NewService(pool, cfg, logger).WithRabbitMQ(messaging) return svc } func (s *DesktopTaskService) WithCache(c sharedcache.Cache) *DesktopTaskService { s.cache = c return s } func (s *DesktopTaskService) WithRedis(redis *goredis.Client) *DesktopTaskService { if s != nil { s.redis = redis } return s } type DesktopTaskView struct { ID string `json:"id"` JobID string `json:"job_id"` TenantID int64 `json:"tenant_id"` WorkspaceID int64 `json:"workspace_id"` TargetAccountID string `json:"target_account_id"` TargetClientID string `json:"target_client_id"` Platform string `json:"platform"` Kind string `json:"kind"` Payload json.RawMessage `json:"payload"` Status string `json:"status"` PublishJobStatus *string `json:"publish_job_status,omitempty"` ComplianceBlockedRecordID *int64 `json:"compliance_blocked_record_id,omitempty"` ComplianceBlockedAt *time.Time `json:"compliance_blocked_at,omitempty"` ComplianceBlockedReason *string `json:"compliance_blocked_reason,omitempty"` DedupKey *string `json:"dedup_key"` ActiveAttemptID *string `json:"active_attempt_id"` LeaseExpiresAt *time.Time `json:"lease_expires_at"` Attempts int `json:"attempts"` Result json.RawMessage `json:"result"` Error json.RawMessage `json:"error"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` } type LeaseDesktopTaskRequest struct { TaskID *string `json:"task_id"` Kind *string `json:"kind" binding:"omitempty,oneof=publish monitor"` } type LeaseDesktopTaskResponse struct { Task *DesktopTaskView `json:"task"` AttemptID *string `json:"attempt_id,omitempty"` LeaseToken *string `json:"lease_token,omitempty"` LeaseExpiresAt *time.Time `json:"lease_expires_at,omitempty"` } type ExtendDesktopTaskRequest struct { LeaseToken string `json:"lease_token" binding:"required"` } type MarkPublishSubmitStartedRequest struct { LeaseToken string `json:"lease_token" binding:"required"` } type CompleteDesktopTaskRequest struct { LeaseToken string `json:"lease_token" binding:"required"` Status string `json:"status" binding:"required,oneof=succeeded failed unknown"` Payload map[string]any `json:"payload"` Error map[string]any `json:"error"` } type CancelDesktopTaskRequest struct { LeaseToken *string `json:"lease_token"` Reason *string `json:"reason"` } type ReconcileDesktopTaskRequest struct { Status string `json:"status" binding:"required,oneof=succeeded failed aborted retry"` Result map[string]any `json:"result"` Error map[string]any `json:"error"` } type ListPublishTasksRequest struct { Page int `form:"page"` PageSize int `form:"page_size"` Title string `form:"title"` } type DesktopPublishTaskList struct { Items []DesktopTaskView `json:"items"` Page int `json:"page"` PageSize int `json:"page_size"` Total int `json:"total"` PendingCount int `json:"pending_count"` HistoryTotal int `json:"history_total"` } type publishTaskListOwner struct { TenantID int64 WorkspaceID int64 UserID int64 } func (s *DesktopTaskService) Lease(ctx context.Context, client *repository.DesktopClient, req LeaseDesktopTaskRequest, routeTaskID *uuid.UUID) (*LeaseDesktopTaskResponse, error) { if client == nil { return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required") } markDesktopTaskConsumerPresent(ctx, s.redis, client.ID) taskID := routeTaskID if taskID == nil && req.TaskID != nil && strings.TrimSpace(*req.TaskID) != "" { parsed, err := uuid.Parse(strings.TrimSpace(*req.TaskID)) if err != nil { return nil, response.ErrBadRequest(40084, "invalid_desktop_task_id", "task id must be a uuid") } taskID = &parsed } if err := s.dropStaleMonitorTasksForClient(ctx, client); err != nil { return nil, err } if err := s.recoverExpiredClientTasks(ctx, client); err != nil { return nil, err } if err := s.recheckQueuedPublishTasksForClient(ctx, client); err != nil { return nil, err } rawToken, tokenHash, err := newDesktopClientToken() if err != nil { return nil, response.ErrInternal(50087, "desktop_task_lease_token_failed", "failed to generate desktop task lease token") } attemptID := uuid.New() leaseParams := repository.DesktopTaskLeaseParams{ AttemptID: attemptID, LeaseTokenHash: tokenHash, } if s.logger != nil { fields := []zap.Field{ zap.String("client_id", client.ID.String()), } if req.Kind != nil { fields = append(fields, zap.String("requested_kind", strings.TrimSpace(*req.Kind))) } if taskID != nil { fields = append(fields, zap.String("task_id", taskID.String())) } s.logger.Debug("desktop task lease requested", fields...) } var task *repository.DesktopTask switch { case taskID != nil: task, err = s.leaseQueuedDesktopTaskByID(ctx, client, *taskID, leaseParams) case req.Kind != nil && strings.TrimSpace(*req.Kind) == "monitor": task, err = s.leaseNextQueuedMonitorTask(ctx, client, leaseParams) case req.Kind != nil && strings.TrimSpace(*req.Kind) == "publish": task, err = s.leaseNextQueuedPublishTask(ctx, client, leaseParams) default: task, err = s.leaseNextQueuedPublishTask(ctx, client, leaseParams) } if err != nil { if errors.Is(err, pgx.ErrNoRows) { if s.logger != nil { fields := []zap.Field{ zap.String("client_id", client.ID.String()), } if req.Kind != nil { fields = append(fields, zap.String("requested_kind", strings.TrimSpace(*req.Kind))) } if taskID != nil { fields = append(fields, zap.String("task_id", taskID.String())) } s.logger.Debug("desktop task lease found no matching queued task", fields...) } if taskID == nil { return &LeaseDesktopTaskResponse{}, nil } return nil, s.classifyLeaseError(ctx, client, taskID) } fields := []zap.Field{ zap.String("client_id", client.ID.String()), } if req.Kind != nil { fields = append(fields, zap.String("requested_kind", strings.TrimSpace(*req.Kind))) } if taskID != nil { fields = append(fields, zap.String("task_id", taskID.String())) } s.logWarn("desktop task lease query failed", err, fields...) return nil, response.ErrInternal(50088, "desktop_task_lease_failed", "failed to lease desktop task") } if _, attemptErr := s.repo.CreateAttempt(ctx, repository.CreateDesktopTaskAttemptParams{ DesktopID: attemptID, TaskID: task.DesktopID, ClientID: client.ID, LeaseTokenHash: tokenHash, }); attemptErr != nil { s.logWarn("desktop task attempt insert failed", attemptErr, zap.String("task_id", task.DesktopID.String())) } s.publishTaskEvent(ctx, task, "task_leased") if s.logger != nil { s.logger.Info("desktop task leased", zap.String("client_id", client.ID.String()), zap.String("task_id", task.DesktopID.String()), zap.String("kind", task.Kind), zap.String("status", task.Status), ) } view := buildDesktopTaskView(task) attemptIDText := attemptID.String() return &LeaseDesktopTaskResponse{ Task: &view, AttemptID: &attemptIDText, LeaseToken: &rawToken, LeaseExpiresAt: task.LeaseExpiresAt, }, nil } func (s *DesktopTaskService) recheckQueuedPublishTasksForClient(ctx context.Context, client *repository.DesktopClient) error { if s == nil || s.pool == nil || s.compliance == nil || client == nil { return nil } rows, err := s.pool.Query(ctx, ` SELECT j.desktop_id, j.tenant_id, j.workspace_id, j.created_by_user_id, j.article_id, j.article_version_id, COALESCE(array_agg(DISTINCT dt.platform_id) FILTER (WHERE dt.platform_id <> ''), '{}') AS platforms, BOOL_OR(a.deleted_at IS NOT NULL) AS article_deleted FROM desktop_tasks dt JOIN desktop_publish_jobs j ON j.desktop_id = dt.job_id LEFT JOIN articles a ON a.id = j.article_id AND a.tenant_id = j.tenant_id WHERE dt.target_client_id = $1 AND dt.kind = 'publish' AND dt.status = 'queued' AND j.status = 'queued' AND j.article_id IS NOT NULL AND j.article_version_id IS NOT NULL GROUP BY j.desktop_id, j.tenant_id, j.workspace_id, j.created_by_user_id, j.article_id, j.article_version_id ORDER BY MIN(dt.created_at) LIMIT 5 `, client.ID) if err != nil { s.logWarn("desktop publish compliance recheck query failed", err, zap.String("client_id", client.ID.String())) return response.ErrInternal(50118, "desktop_publish_compliance_recheck_failed", "failed to load publish jobs for compliance recheck") } defer rows.Close() type candidate struct { JobID uuid.UUID TenantID int64 WorkspaceID int64 CreatedByUserID int64 ArticleID int64 ArticleVersionID int64 Platforms []string ArticleDeleted bool } candidates := make([]candidate, 0) for rows.Next() { var item candidate if scanErr := rows.Scan(&item.JobID, &item.TenantID, &item.WorkspaceID, &item.CreatedByUserID, &item.ArticleID, &item.ArticleVersionID, &item.Platforms, &item.ArticleDeleted); scanErr != nil { s.logWarn("desktop publish compliance recheck scan failed", scanErr, zap.String("client_id", client.ID.String())) return response.ErrInternal(50118, "desktop_publish_compliance_recheck_failed", "failed to scan publish job for compliance recheck") } if item.WorkspaceID == client.WorkspaceID && item.TenantID == client.TenantID { candidates = append(candidates, item) } } if err := rows.Err(); err != nil { s.logWarn("desktop publish compliance recheck rows failed", err, zap.String("client_id", client.ID.String())) return response.ErrInternal(50118, "desktop_publish_compliance_recheck_failed", "failed to iterate publish jobs for compliance recheck") } for _, item := range candidates { if item.ArticleDeleted { if cancelErr := s.cancelUnavailableArticleQueuedPublishTasks(ctx, item.TenantID, item.ArticleID); cancelErr != nil { return cancelErr } continue } gate, gateErr := s.compliance.GateForPublish(ctx, tenantcompliance.GateForPublishRequest{ TenantID: item.TenantID, ArticleID: item.ArticleID, ArticleVersionID: item.ArticleVersionID, ActorID: item.CreatedByUserID, TargetPlatforms: item.Platforms, TriggerSource: "scheduler_recheck", }) if gateErr != nil && gate != nil && gate.Decision == sharedcompliance.GateDecisionBlock { if markErr := s.compliance.MarkPublishJobBlockedByCompliance(ctx, item.JobID, gate.Result); markErr != nil { return markErr } continue } if gateErr != nil { if isComplianceInvalidArticleVersionError(gateErr) { if cancelErr := s.cancelUnavailableArticleQueuedPublishTasks(ctx, item.TenantID, item.ArticleID); cancelErr != nil { return cancelErr } continue } return response.ErrServiceUnavailable(41004, "compliance_engine_unavailable", "scheduled publish compliance recheck failed") } if gate != nil && gate.Decision == sharedcompliance.GateDecisionBlock { if markErr := s.compliance.MarkPublishJobBlockedByCompliance(ctx, item.JobID, gate.Result); markErr != nil { return markErr } } } return nil } func (s *DesktopTaskService) cancelUnavailableArticleQueuedPublishTasks(ctx context.Context, tenantID, articleID int64) error { if s == nil || s.pool == nil || tenantID <= 0 || articleID <= 0 { return nil } tx, err := s.pool.Begin(ctx) if err != nil { return response.ErrInternal(50122, "publish_task_cancel_unavailable_article_begin_failed", "failed to start unavailable article publish task cleanup") } defer tx.Rollback(ctx) taskRefs, err := queuedPublishTaskRefsForArticle(ctx, tx, tenantID, articleID) if err != nil { return err } if err := cancelQueuedPublishTasksForUnavailableArticle(ctx, tx, tenantID, articleID); err != nil { return err } if err := tx.Commit(ctx); err != nil { return response.ErrInternal(50123, "publish_task_cancel_unavailable_article_commit_failed", "failed to commit unavailable article publish task cleanup") } for _, ref := range taskRefs { task, getErr := s.repo.GetByDesktopID(ctx, ref.TaskID, ref.WorkspaceID) if getErr == nil && task != nil { s.publishTaskEvent(ctx, task, "task_canceled") } } return nil } type desktopTaskRef struct { TaskID uuid.UUID WorkspaceID int64 } func queuedPublishTaskRefsForArticle(ctx context.Context, tx pgx.Tx, tenantID, articleID int64) ([]desktopTaskRef, error) { if tx == nil || tenantID <= 0 || articleID <= 0 { return nil, nil } rows, err := tx.Query(ctx, ` SELECT dt.desktop_id, dt.workspace_id FROM desktop_tasks dt JOIN desktop_publish_jobs j ON j.desktop_id = dt.job_id AND j.tenant_id = dt.tenant_id WHERE dt.tenant_id = $1 AND dt.kind = 'publish' AND dt.status = 'queued' AND j.article_id = $2 AND j.status = 'queued' ORDER BY dt.created_at ASC, dt.desktop_id ASC `, tenantID, articleID) if err != nil { return nil, response.ErrInternal(50124, "publish_task_cancel_unavailable_article_lookup_failed", "failed to inspect unavailable article publish tasks") } defer rows.Close() taskRefs := make([]desktopTaskRef, 0) for rows.Next() { var ref desktopTaskRef if scanErr := rows.Scan(&ref.TaskID, &ref.WorkspaceID); scanErr != nil { return nil, response.ErrInternal(50124, "publish_task_cancel_unavailable_article_lookup_failed", "failed to scan unavailable article publish tasks") } taskRefs = append(taskRefs, ref) } if err := rows.Err(); err != nil { return nil, response.ErrInternal(50124, "publish_task_cancel_unavailable_article_lookup_failed", "failed to iterate unavailable article publish tasks") } return taskRefs, nil } func isComplianceInvalidArticleVersionError(err error) bool { appErr, ok := err.(*response.AppError) return ok && appErr.Code == 41005 && appErr.Message == "invalid_article_version" } const desktopTaskRepositoryReturningColumns = ` t.desktop_id, t.job_id, t.tenant_id, t.workspace_id, t.target_account_id, t.target_client_id, t.platform_id, t.kind, t.payload, t.status, t.priority, t.lane, t.lane_weight, t.source, t.scheduler_group_key, t.monitor_task_id, t.supersedes_task_id, t.control_flags, t.interrupt_generation, t.dedup_key, t.active_attempt_id, t.lease_expires_at, t.attempts, t.result, t.error, t.started_at, t.interrupted_at, t.interrupt_reason, t.enqueued_at, t.created_at, t.updated_at` type desktopTaskRepositoryScanner interface { Scan(dest ...any) error } func (s *DesktopTaskService) leaseNextQueuedMonitorTask( ctx context.Context, client *repository.DesktopClient, params repository.DesktopTaskLeaseParams, ) (*repository.DesktopTask, error) { accountIDs, err := s.loadMonitorLeaseAccountIDs(ctx, client) if err != nil { return nil, err } hasAccountIDs := len(accountIDs) > 0 row := s.pool.QueryRow(ctx, ` WITH candidate AS ( SELECT dt.desktop_id FROM desktop_tasks AS dt WHERE dt.tenant_id = $1 AND dt.workspace_id = $2 AND dt.kind = 'monitor' AND dt.status = 'queued' AND ( dt.target_client_id = $3 OR ($4::boolean AND dt.target_account_id = ANY($5::uuid[])) ) ORDER BY dt.lane_weight DESC, dt.priority DESC, COALESCE(dt.enqueued_at, dt.created_at) ASC, dt.created_at ASC, dt.desktop_id ASC LIMIT 1 FOR UPDATE SKIP LOCKED ) UPDATE desktop_tasks AS t SET target_client_id = $3, active_attempt_id = $6, lease_token_hash = $7, lease_expires_at = now() + interval '10 minutes', status = 'in_progress', attempts = t.attempts + 1, updated_at = now() FROM candidate WHERE t.desktop_id = candidate.desktop_id RETURNING `+desktopTaskRepositoryReturningColumns, client.TenantID, client.WorkspaceID, client.ID, hasAccountIDs, accountIDs, params.AttemptID, params.LeaseTokenHash, ) return scanRepositoryDesktopTask(row) } func (s *DesktopTaskService) leaseNextQueuedPublishTask( ctx context.Context, client *repository.DesktopClient, params repository.DesktopTaskLeaseParams, ) (*repository.DesktopTask, error) { row := s.pool.QueryRow(ctx, ` WITH candidate AS ( SELECT dt.desktop_id FROM desktop_tasks AS dt WHERE dt.target_client_id = $1 AND dt.kind = 'publish' AND dt.status = 'queued' AND dt.attempts < $4 AND ( NOT EXISTS ( SELECT 1 FROM desktop_publish_jobs AS j WHERE j.desktop_id = dt.job_id AND j.status <> 'queued' ) ) ORDER BY dt.lane_weight DESC, dt.priority DESC, COALESCE(dt.enqueued_at, dt.created_at) ASC, dt.created_at ASC, dt.desktop_id ASC LIMIT 1 FOR UPDATE OF dt SKIP LOCKED ) UPDATE desktop_tasks AS t SET active_attempt_id = $2, lease_token_hash = $3, lease_expires_at = now() + interval '3 minutes', status = 'in_progress', attempts = t.attempts + 1, started_at = COALESCE(t.started_at, now()), updated_at = now() FROM candidate WHERE t.desktop_id = candidate.desktop_id RETURNING `+desktopTaskRepositoryReturningColumns, client.ID, params.AttemptID, params.LeaseTokenHash, desktopPublishMaxAttempts, ) return scanRepositoryDesktopTask(row) } func (s *DesktopTaskService) leaseQueuedDesktopTaskByID( ctx context.Context, client *repository.DesktopClient, desktopID uuid.UUID, params repository.DesktopTaskLeaseParams, ) (*repository.DesktopTask, error) { accountIDs, err := s.loadMonitorLeaseAccountIDs(ctx, client) if err != nil { return nil, err } hasAccountIDs := len(accountIDs) > 0 row := s.pool.QueryRow(ctx, ` WITH candidate AS ( SELECT dt.desktop_id FROM desktop_tasks AS dt WHERE dt.desktop_id = $1 AND dt.workspace_id = $2 AND dt.status = 'queued' AND (dt.kind <> 'publish' OR dt.attempts < $9) AND ( ( dt.kind = 'monitor' AND dt.tenant_id = $3 AND ( dt.target_client_id = $4 OR ($5::boolean AND dt.target_account_id = ANY($6::uuid[])) ) ) OR ( dt.kind <> 'monitor' AND dt.target_client_id = $4 ) ) LIMIT 1 FOR UPDATE SKIP LOCKED ) UPDATE desktop_tasks AS t SET target_client_id = CASE WHEN t.kind = 'monitor' THEN $4 ELSE t.target_client_id END, active_attempt_id = $7, lease_token_hash = $8, lease_expires_at = now() + CASE WHEN t.kind = 'publish' THEN interval '3 minutes' ELSE interval '10 minutes' END, status = 'in_progress', attempts = t.attempts + 1, started_at = COALESCE(t.started_at, now()), updated_at = now() FROM candidate WHERE t.desktop_id = candidate.desktop_id RETURNING `+desktopTaskRepositoryReturningColumns, desktopID, client.WorkspaceID, client.TenantID, client.ID, hasAccountIDs, accountIDs, params.AttemptID, params.LeaseTokenHash, desktopPublishMaxAttempts, ) return scanRepositoryDesktopTask(row) } func (s *DesktopTaskService) loadMonitorLeaseAccountIDs(ctx context.Context, client *repository.DesktopClient) ([]uuid.UUID, error) { if s == nil || client == nil || s.pool == nil { return nil, nil } accountIDs := loadTrackedDesktopClientAccountIDs(ctx, s.redis, client.ID) rows, err := s.pool.Query(ctx, ` SELECT desktop_id FROM platform_accounts WHERE tenant_id = $1 AND workspace_id = $2 AND client_id = $3 AND deleted_at IS NULL AND platform_id = ANY($4::text[]) `, client.TenantID, client.WorkspaceID, client.ID, monitoringPlatformIDs(defaultMonitoringPlatforms)) if err != nil { return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to resolve monitor lease accounts") } defer rows.Close() for rows.Next() { var accountID uuid.UUID if scanErr := rows.Scan(&accountID); scanErr != nil { return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to parse monitor lease accounts") } accountIDs = append(accountIDs, accountID) } if err := rows.Err(); err != nil { return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to iterate monitor lease accounts") } return uniqueUUIDs(accountIDs), nil } func scanRepositoryDesktopTask(row desktopTaskRepositoryScanner) (*repository.DesktopTask, error) { var ( task repository.DesktopTask schedulerGroupKey pgtype.Text monitorTaskID pgtype.Int8 supersedesTaskID pgtype.UUID dedupKey pgtype.Text activeAttemptID pgtype.UUID leaseExpiresAt pgtype.Timestamptz startedAt pgtype.Timestamptz interruptedAt pgtype.Timestamptz interruptReason pgtype.Text enqueuedAt pgtype.Timestamptz ) if err := row.Scan( &task.DesktopID, &task.JobID, &task.TenantID, &task.WorkspaceID, &task.TargetAccountID, &task.TargetClientID, &task.Platform, &task.Kind, &task.Payload, &task.Status, &task.Priority, &task.Lane, &task.LaneWeight, &task.Source, &schedulerGroupKey, &monitorTaskID, &supersedesTaskID, &task.ControlFlags, &task.InterruptGeneration, &dedupKey, &activeAttemptID, &leaseExpiresAt, &task.Attempts, &task.Result, &task.Error, &startedAt, &interruptedAt, &interruptReason, &enqueuedAt, &task.CreatedAt, &task.UpdatedAt, ); err != nil { return nil, err } task.SchedulerGroupKey = desktopTaskNullableText(schedulerGroupKey) task.MonitorTaskID = desktopTaskNullableInt64(monitorTaskID) task.SupersedesTaskID = desktopTaskNullableUUID(supersedesTaskID) task.DedupKey = desktopTaskNullableText(dedupKey) task.ActiveAttemptID = desktopTaskNullableUUID(activeAttemptID) task.LeaseExpiresAt = desktopTaskNullableTime(leaseExpiresAt) task.StartedAt = desktopTaskNullableTime(startedAt) task.InterruptedAt = desktopTaskNullableTime(interruptedAt) task.InterruptReason = desktopTaskNullableText(interruptReason) if enqueuedAt.Valid { task.EnqueuedAt = enqueuedAt.Time } return &task, nil } func desktopTaskNullableText(value pgtype.Text) *string { if !value.Valid { return nil } text := value.String return &text } func desktopTaskNullableInt64(value pgtype.Int8) *int64 { if !value.Valid { return nil } number := value.Int64 return &number } func desktopTaskNullableUUID(value pgtype.UUID) *uuid.UUID { if !value.Valid { return nil } resolved, err := uuid.FromBytes(value.Bytes[:]) if err != nil { return nil } return &resolved } func desktopTaskNullableTime(value pgtype.Timestamptz) *time.Time { if !value.Valid { return nil } timestamp := value.Time return ×tamp } func (s *DesktopTaskService) Extend(ctx context.Context, client *repository.DesktopClient, desktopID uuid.UUID, req ExtendDesktopTaskRequest) (*DesktopTaskView, error) { if client == nil { return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required") } if err := s.dropStaleMonitorTasksForClient(ctx, client); err != nil { return nil, err } if s.redis != nil { presence := loadDesktopClientPresence(ctx, s.redis, []uuid.UUID{client.ID}) if presence != nil && !presence[client.ID] { return nil, response.ErrConflict(40986, "desktop_client_offline", "desktop client presence has expired") } } task, err := s.repo.ExtendLease(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(req.LeaseToken))) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return nil, response.ErrConflict(40983, "desktop_task_lease_invalid", "desktop task lease token is invalid or expired") } return nil, response.ErrInternal(50089, "desktop_task_extend_failed", "failed to extend desktop task lease") } s.publishTaskEvent(ctx, task, "task_extended") view := buildDesktopTaskView(task) return &view, nil } // MarkPublishSubmitStarted records a durable intent marker right before the client performs the // irreversible platform submit POST. Lease-recovery uses it to avoid silently re-posting a // non-idempotent article: once submit may have started, recovery routes the task to 'unknown' // (manual reconcile) instead of auto-requeueing it. The client must receive a // positive marker write before it performs the platform submit; a stale/lost // lease is rejected so the client can stop before the irreversible request. func (s *DesktopTaskService) MarkPublishSubmitStarted(ctx context.Context, client *repository.DesktopClient, taskID uuid.UUID, leaseToken string) error { if s == nil || s.pool == nil || client == nil { return nil } leaseToken = strings.TrimSpace(leaseToken) if leaseToken == "" { return response.ErrBadRequest(40001, "invalid_params", "lease_token is required") } tag, err := s.pool.Exec(ctx, ` UPDATE desktop_tasks SET publish_submit_started_at = COALESCE(publish_submit_started_at, NOW()), updated_at = NOW() WHERE desktop_id = $1 AND target_client_id = $2 AND kind = 'publish' AND status = 'in_progress' AND lease_token_hash = $3 `, taskID, client.ID, HashDesktopClientToken(leaseToken)) if err != nil { return response.ErrInternal(50200, "desktop_task_submit_marker_failed", "failed to mark publish submit started") } if tag.RowsAffected() == 0 { return response.ErrConflict(40983, "desktop_task_lease_invalid", "desktop task lease token is invalid or expired") } return nil } func (s *DesktopTaskService) Complete(ctx context.Context, client *repository.DesktopClient, desktopID uuid.UUID, req CompleteDesktopTaskRequest) (*DesktopTaskView, error) { if client == nil { return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required") } finalStatus := normalizeDesktopTaskTerminalStatus(req.Status) resultJSON, err := marshalOptionalJSON(req.Payload) if err != nil { return nil, response.ErrBadRequest(40086, "invalid_desktop_task_payload", "payload must be serializable") } errorJSON, err := marshalOptionalJSON(req.Error) if err != nil { return nil, response.ErrBadRequest(40087, "invalid_desktop_task_error", "error must be serializable") } tx, err := s.pool.Begin(ctx) if err != nil { return nil, response.ErrInternal(50103, "desktop_task_complete_begin_failed", "failed to start desktop task completion") } defer tx.Rollback(ctx) taskRepo := repository.NewDesktopTaskRepository(tx) previousTask, _ := taskRepo.GetByDesktopID(ctx, desktopID, client.WorkspaceID) previousAttemptID := activeAttemptID(previousTask) task, err := taskRepo.Complete(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(req.LeaseToken)), finalStatus, resultJSON, errorJSON) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return nil, response.ErrConflict(40983, "desktop_task_lease_invalid", "desktop task lease token is invalid or expired") } return nil, response.ErrInternal(50091, "desktop_task_complete_failed", "failed to complete desktop task") } if previousAttemptID != nil { if finishErr := taskRepo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{ TaskID: task.DesktopID, AttemptID: *previousAttemptID, FinalStatus: &finalStatus, Error: errorJSON, }); finishErr != nil { s.logWarn("desktop task attempt finish failed after complete", finishErr, zap.String("task_id", task.DesktopID.String())) } } publishOutcome, err := syncDesktopPublishTaskState(ctx, tx, task) if err != nil { return nil, err } bulkFailure, err := bulkFailQueuedDesktopTasksForAuthorizationFailure(ctx, tx, task, errorJSON) if err != nil { return nil, err } if err := tx.Commit(ctx); err != nil { return nil, response.ErrInternal(50104, "desktop_task_complete_commit_failed", "failed to commit desktop task completion") } if publishOutcome != nil { invalidateArticleCaches(ctx, s.cache, publishOutcome.TenantID, &publishOutcome.ArticleID) if syncErr := s.syncMonitoringArticleAlias(ctx, publishOutcome.ArticleAlias); syncErr != nil { s.logWarn("monitoring article alias sync failed after publish complete", syncErr, zap.Int64("article_id", publishOutcome.ArticleID)) } } for _, outcome := range bulkFailure.PublishOutcomes { invalidateArticleCaches(ctx, s.cache, outcome.TenantID, &outcome.ArticleID) if outcome.ArticleAlias != nil { if syncErr := s.syncMonitoringArticleAlias(ctx, outcome.ArticleAlias); syncErr != nil { s.logWarn("monitoring article alias sync failed after bulk authorization failure", syncErr, zap.Int64("article_id", outcome.ArticleID)) } } } s.publishTaskEvent(ctx, task, "task_completed") for _, failedTask := range bulkFailure.Tasks { s.publishTaskEvent(ctx, failedTask, "task_completed") } view := buildDesktopTaskView(task) return &view, nil } func (s *DesktopTaskService) Cancel(ctx context.Context, client *repository.DesktopClient, desktopID uuid.UUID, req CancelDesktopTaskRequest) (*DesktopTaskView, error) { if client == nil { return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required") } errorJSON, err := marshalOptionalJSON(map[string]any{ "reason": optionalStringValue(req.Reason), "source": "desktop_client", }) if err != nil { return nil, response.ErrBadRequest(40088, "invalid_desktop_task_cancel", "cancel payload must be serializable") } tx, err := s.pool.Begin(ctx) if err != nil { return nil, response.ErrInternal(50107, "desktop_task_cancel_begin_failed", "failed to start desktop task cancel") } defer tx.Rollback(ctx) taskRepo := repository.NewDesktopTaskRepository(tx) previousTask, _ := taskRepo.GetByDesktopID(ctx, desktopID, client.WorkspaceID) previousAttemptID := activeAttemptID(previousTask) var task *repository.DesktopTask if req.LeaseToken != nil && strings.TrimSpace(*req.LeaseToken) != "" { task, err = taskRepo.CancelByLease(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(*req.LeaseToken)), errorJSON) } else { task, err = taskRepo.CancelByClient(ctx, desktopID, client.ID, errorJSON) } if err != nil { if errors.Is(err, pgx.ErrNoRows) { return nil, response.ErrConflict(40984, "desktop_task_cancel_conflict", "desktop task cannot be canceled in its current state") } return nil, response.ErrInternal(50092, "desktop_task_cancel_failed", "failed to cancel desktop task") } if previousAttemptID != nil && req.LeaseToken != nil && strings.TrimSpace(*req.LeaseToken) != "" { if finishErr := taskRepo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{ TaskID: task.DesktopID, AttemptID: *previousAttemptID, FinalStatus: literalStringPtr("aborted"), Error: errorJSON, }); finishErr != nil { s.logWarn("desktop task attempt finish failed after cancel", finishErr, zap.String("task_id", task.DesktopID.String())) } } var replacementTask *repository.DesktopTask if shouldRequeuePreemptedMonitorTask(task, req.Reason) { replacementTask, err = requeuePreemptedMonitorTask(ctx, tx, taskRepo, task) if err != nil { return nil, err } } if err := tx.Commit(ctx); err != nil { return nil, response.ErrInternal(50108, "desktop_task_cancel_commit_failed", "failed to commit desktop task cancel") } s.publishTaskEvent(ctx, task, "task_canceled") if replacementTask != nil { s.publishTaskEvent(ctx, replacementTask, "task_available") } view := buildDesktopTaskView(task) return &view, nil } func (s *DesktopTaskService) TenantCancel(ctx context.Context, actor auth.Actor, desktopID uuid.UUID, req CancelDesktopTaskRequest) (*DesktopTaskView, error) { if actor.PrimaryWorkspaceID == 0 { return nil, response.ErrUnauthorized(40132, "workspace_scope_missing", "workspace context is required") } errorJSON, err := marshalOptionalJSON(map[string]any{ "reason": optionalStringValue(req.Reason), "source": "tenant_web", }) if err != nil { return nil, response.ErrBadRequest(40088, "invalid_desktop_task_cancel", "cancel payload must be serializable") } task, err := s.repo.TenantCancel(ctx, desktopID, actor.PrimaryWorkspaceID, errorJSON) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return nil, response.ErrConflict(40984, "desktop_task_cancel_conflict", "desktop task cannot be canceled in its current state") } return nil, response.ErrInternal(50093, "desktop_task_tenant_cancel_failed", "failed to cancel desktop task") } s.publishTaskEvent(ctx, task, "task_canceled") view := buildDesktopTaskView(task) return &view, nil } func (s *DesktopTaskService) Reconcile(ctx context.Context, actor auth.Actor, desktopID uuid.UUID, req ReconcileDesktopTaskRequest) (*DesktopTaskView, error) { if actor.PrimaryWorkspaceID == 0 { return nil, response.ErrUnauthorized(40132, "workspace_scope_missing", "workspace context is required") } resultJSON, err := marshalOptionalJSON(req.Result) if err != nil { return nil, response.ErrBadRequest(40089, "invalid_desktop_task_reconcile_result", "result must be serializable") } errorJSON, err := marshalOptionalJSON(req.Error) if err != nil { return nil, response.ErrBadRequest(40090, "invalid_desktop_task_reconcile_error", "error must be serializable") } tx, err := s.pool.Begin(ctx) if err != nil { return nil, response.ErrInternal(50105, "desktop_task_reconcile_begin_failed", "failed to start desktop task reconcile") } defer tx.Rollback(ctx) taskRepo := repository.NewDesktopTaskRepository(tx) task, err := taskRepo.Reconcile(ctx, desktopID, actor.PrimaryWorkspaceID, req.Status, resultJSON, errorJSON) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return nil, response.ErrConflict(40985, "desktop_task_reconcile_conflict", "desktop task is not waiting for reconcile") } return nil, response.ErrInternal(50094, "desktop_task_reconcile_failed", "failed to reconcile desktop task") } publishOutcome, err := syncDesktopPublishTaskState(ctx, tx, task) if err != nil { return nil, err } if err := tx.Commit(ctx); err != nil { return nil, response.ErrInternal(50106, "desktop_task_reconcile_commit_failed", "failed to commit desktop task reconcile") } if publishOutcome != nil { invalidateArticleCaches(ctx, s.cache, publishOutcome.TenantID, &publishOutcome.ArticleID) if syncErr := s.syncMonitoringArticleAlias(ctx, publishOutcome.ArticleAlias); syncErr != nil { s.logWarn("monitoring article alias sync failed after publish reconcile", syncErr, zap.Int64("article_id", publishOutcome.ArticleID)) } } s.publishTaskEvent(ctx, task, reconcileDesktopTaskEventType(task.Kind, req.Status)) view := buildDesktopTaskView(task) return &view, nil } func (s *DesktopTaskService) ListPublishTasks(ctx context.Context, client *repository.DesktopClient, req ListPublishTasksRequest) (*DesktopPublishTaskList, error) { if client == nil { return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required") } return s.listPublishTasksForOwner(ctx, publishTaskListOwner{ TenantID: client.TenantID, WorkspaceID: client.WorkspaceID, UserID: client.UserID, }, req) } func (s *DesktopTaskService) ListTenantPublishTasks(ctx context.Context, actor auth.Actor, req ListPublishTasksRequest) (*DesktopPublishTaskList, error) { workspaceID := auth.CurrentWorkspaceID(ctx) if actor.TenantID == 0 || actor.UserID == 0 || workspaceID == 0 { return nil, response.ErrUnauthorized(40132, "workspace_scope_missing", "workspace context is required") } return s.listPublishTasksForOwner(ctx, publishTaskListOwner{ TenantID: actor.TenantID, WorkspaceID: workspaceID, UserID: actor.UserID, }, req) } func (s *DesktopTaskService) listPublishTasksForOwner(ctx context.Context, owner publishTaskListOwner, req ListPublishTasksRequest) (*DesktopPublishTaskList, error) { if s.pool == nil { return nil, response.ErrServiceUnavailable(50342, "desktop_publish_job_store_unavailable", "desktop publish job store is unavailable") } page := req.Page if page <= 0 { page = 1 } pageSize := req.PageSize if pageSize <= 0 || pageSize > 50 { pageSize = 10 } title := strings.TrimSpace(req.Title) pendingStatuses := []string{"queued", "in_progress"} historyStatuses := []string{"succeeded", "failed", "unknown", "aborted"} pendingItems, err := s.listPublishTasksByStatuses( ctx, owner, pendingStatuses, title, 0, 0, `CASE WHEN t.status = 'in_progress' THEN 0 ELSE 1 END ASC, t.created_at ASC, t.desktop_id DESC`, ) if err != nil { return nil, err } historyTotal, err := s.countPublishTasksByStatuses(ctx, owner, historyStatuses, title) if err != nil { return nil, err } offset := (page - 1) * pageSize items := make([]DesktopTaskView, 0, pageSize) total := len(pendingItems) + historyTotal if offset < len(pendingItems) { end := offset + pageSize if end > len(pendingItems) { end = len(pendingItems) } items = append(items, pendingItems[offset:end]...) } remaining := pageSize - len(items) historyOffset := offset - len(pendingItems) if historyOffset < 0 { historyOffset = 0 } if remaining > 0 && historyOffset < historyTotal { historyItems, listErr := s.listPublishTasksByStatuses( ctx, owner, historyStatuses, title, remaining, historyOffset, `t.updated_at DESC, t.desktop_id DESC`, ) if listErr != nil { return nil, listErr } items = append(items, historyItems...) } return &DesktopPublishTaskList{ Items: items, Page: page, PageSize: pageSize, Total: total, PendingCount: len(pendingItems), HistoryTotal: historyTotal, }, nil } func (s *DesktopTaskService) listPublishTasksByStatuses( ctx context.Context, owner publishTaskListOwner, statuses []string, title string, limit int, offset int, orderBy string, ) ([]DesktopTaskView, error) { query := ` SELECT t.desktop_id, t.job_id, t.tenant_id, t.workspace_id, t.target_account_id, t.target_client_id, t.platform_id, t.kind, t.payload, t.status, t.dedup_key, t.active_attempt_id, t.lease_expires_at, t.attempts, t.result, t.error, t.created_at, t.updated_at, j.status, j.compliance_blocked_record_id, j.compliance_blocked_at, j.compliance_blocked_reason FROM desktop_tasks AS t JOIN desktop_publish_jobs AS j ON j.desktop_id = t.job_id AND j.workspace_id = t.workspace_id WHERE t.tenant_id = $1 AND t.workspace_id = $2 AND j.tenant_id = $1 AND j.created_by_user_id = $3 AND t.kind = 'publish' AND t.status = ANY($4) AND ( $5 = '' OR COALESCE(t.payload->>'title', j.title, '') ILIKE '%' || $5 || '%' ) ` args := []any{owner.TenantID, owner.WorkspaceID, owner.UserID, statuses, title} if strings.TrimSpace(orderBy) != "" { query += "\nORDER BY " + orderBy } if limit > 0 { query += "\nLIMIT $6 OFFSET $7" args = append(args, limit, offset) } rows, err := s.pool.Query(ctx, query, args...) if err != nil { return nil, response.ErrInternal(50109, "desktop_publish_tasks_query_failed", "failed to list desktop publish tasks") } defer rows.Close() return scanDesktopTaskRows(rows) } func (s *DesktopTaskService) countPublishTasksByStatuses( ctx context.Context, owner publishTaskListOwner, statuses []string, title string, ) (int, error) { var count int err := s.pool.QueryRow(ctx, ` SELECT COUNT(1) FROM desktop_tasks AS t JOIN desktop_publish_jobs AS j ON j.desktop_id = t.job_id AND j.workspace_id = t.workspace_id WHERE t.tenant_id = $1 AND t.workspace_id = $2 AND j.tenant_id = $1 AND j.created_by_user_id = $3 AND t.kind = 'publish' AND t.status = ANY($4) AND ( $5 = '' OR COALESCE(t.payload->>'title', j.title, '') ILIKE '%' || $5 || '%' ) `, owner.TenantID, owner.WorkspaceID, owner.UserID, statuses, title).Scan(&count) if err != nil { return 0, response.ErrInternal(50109, "desktop_publish_tasks_query_failed", "failed to list desktop publish tasks") } return count, nil } func scanDesktopTaskRows(rows pgx.Rows) ([]DesktopTaskView, error) { items := make([]DesktopTaskView, 0) for rows.Next() { var ( desktopID uuid.UUID jobID uuid.UUID tenantID int64 workspaceID int64 targetAccountID uuid.UUID targetClientID uuid.UUID platform string kind string payload []byte status string dedupKey pgtype.Text activeAttemptID pgtype.UUID leaseExpiresAt pgtype.Timestamptz attempts int32 resultJSON []byte errorJSON []byte createdAt time.Time updatedAt time.Time jobStatus string blockedRecordID pgtype.Int8 blockedAt pgtype.Timestamptz blockedReason pgtype.Text ) if err := rows.Scan( &desktopID, &jobID, &tenantID, &workspaceID, &targetAccountID, &targetClientID, &platform, &kind, &payload, &status, &dedupKey, &activeAttemptID, &leaseExpiresAt, &attempts, &resultJSON, &errorJSON, &createdAt, &updatedAt, &jobStatus, &blockedRecordID, &blockedAt, &blockedReason, ); err != nil { return nil, response.ErrInternal(50110, "desktop_publish_tasks_scan_failed", "failed to scan desktop publish tasks") } var dedupKeyText *string if dedupKey.Valid { value := dedupKey.String dedupKeyText = &value } var activeAttemptIDText *string if activeAttemptID.Valid { value, convErr := uuid.FromBytes(activeAttemptID.Bytes[:]) if convErr == nil { text := value.String() activeAttemptIDText = &text } } var leaseExpiresAtValue *time.Time if leaseExpiresAt.Valid { value := leaseExpiresAt.Time leaseExpiresAtValue = &value } publishJobStatusText := strings.TrimSpace(jobStatus) var publishJobStatus *string if publishJobStatusText != "" { publishJobStatus = &publishJobStatusText } var complianceBlockedRecordID *int64 if blockedRecordID.Valid { value := blockedRecordID.Int64 complianceBlockedRecordID = &value } var complianceBlockedAt *time.Time if blockedAt.Valid { value := blockedAt.Time complianceBlockedAt = &value } var complianceBlockedReason *string if blockedReason.Valid { value := blockedReason.String complianceBlockedReason = &value } items = append(items, DesktopTaskView{ ID: desktopID.String(), JobID: jobID.String(), TenantID: tenantID, WorkspaceID: workspaceID, TargetAccountID: targetAccountID.String(), TargetClientID: targetClientID.String(), Platform: platform, Kind: kind, Payload: json.RawMessage(payload), Status: normalizeDesktopTaskTerminalStatus(status), PublishJobStatus: publishJobStatus, ComplianceBlockedRecordID: complianceBlockedRecordID, ComplianceBlockedAt: complianceBlockedAt, ComplianceBlockedReason: complianceBlockedReason, DedupKey: dedupKeyText, ActiveAttemptID: activeAttemptIDText, LeaseExpiresAt: leaseExpiresAtValue, Attempts: int(attempts), Result: json.RawMessage(resultJSON), Error: json.RawMessage(errorJSON), CreatedAt: createdAt, UpdatedAt: updatedAt, }) } if err := rows.Err(); err != nil { return nil, response.ErrInternal(50109, "desktop_publish_tasks_query_failed", "failed to list desktop publish tasks") } return items, nil } func (s *DesktopTaskService) classifyLeaseError(ctx context.Context, client *repository.DesktopClient, taskID *uuid.UUID) error { if taskID == nil { return response.ErrConflict(40986, "desktop_task_unavailable", "no desktop task is currently available") } task, err := s.repo.GetByDesktopID(ctx, *taskID, client.WorkspaceID) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return response.ErrNotFound(40482, "desktop_task_not_found", "desktop task not found") } return response.ErrInternal(50095, "desktop_task_lookup_failed", "failed to inspect desktop task state") } if task.TargetClientID != client.ID && !s.clientCanLeaseMonitorTask(ctx, client, task) { return response.ErrForbidden(40381, "desktop_task_not_target_client", "desktop task is assigned to another desktop client") } if task.Status == "in_progress" { return response.ErrConflict(40989, "desktop_task_already_in_progress", "desktop task is already in progress") } return response.ErrConflict(40991, "desktop_task_not_queued", "desktop task is not queued") } func (s *DesktopTaskService) clientCanLeaseMonitorTask(ctx context.Context, client *repository.DesktopClient, task *repository.DesktopTask) bool { if s == nil || client == nil || task == nil || task.Kind != "monitor" || task.TargetAccountID == uuid.Nil { return false } accountIDs, err := s.loadMonitorLeaseAccountIDs(ctx, client) if err != nil { return false } for _, accountID := range accountIDs { if accountID == task.TargetAccountID { return true } } return false } func buildDesktopTaskView(task *repository.DesktopTask) DesktopTaskView { var activeAttemptID *string if task.ActiveAttemptID != nil { value := task.ActiveAttemptID.String() activeAttemptID = &value } return DesktopTaskView{ ID: task.DesktopID.String(), JobID: task.JobID.String(), TenantID: task.TenantID, WorkspaceID: task.WorkspaceID, TargetAccountID: task.TargetAccountID.String(), TargetClientID: task.TargetClientID.String(), Platform: task.Platform, Kind: task.Kind, Payload: json.RawMessage(task.Payload), Status: normalizeDesktopTaskTerminalStatus(task.Status), DedupKey: task.DedupKey, ActiveAttemptID: activeAttemptID, LeaseExpiresAt: task.LeaseExpiresAt, Attempts: task.Attempts, Result: json.RawMessage(task.Result), Error: json.RawMessage(task.Error), CreatedAt: task.CreatedAt, UpdatedAt: task.UpdatedAt, } } func marshalOptionalJSON(value any) ([]byte, error) { if value == nil { return nil, nil } return json.Marshal(value) } func optionalStringValue(value *string) string { if value == nil { return "" } return strings.TrimSpace(*value) } func literalStringPtr(value string) *string { return &value } func normalizeDesktopTaskTerminalStatus(status string) string { switch strings.TrimSpace(status) { case "unknown": return "failed" default: return strings.TrimSpace(status) } } type desktopTaskRecoveryMode string const ( desktopTaskRecoveryModeStartup desktopTaskRecoveryMode = "startup" desktopTaskRecoveryModeDisconnect desktopTaskRecoveryMode = "disconnect" desktopTaskRecoveryModeLeaseExpiry desktopTaskRecoveryMode = "lease_expired" ) type recoveredDesktopTaskLease struct { TaskID uuid.UUID WorkspaceID int64 Kind string Status string ActiveAttempt pgtype.UUID Attempts int SubmitStarted bool ErrorJSON []byte } type PublishLeaseRecoveryResult struct { Requeued int `json:"requeued"` Failed int `json:"failed"` Uncertain int `json:"uncertain"` } // resolvePublishRecoveryOutcome decides what to do with a publish task whose lease was lost. // // If the client had already entered the irreversible submit phase (publish_submit_started_at // is set), the article may already exist on the platform. Auto-requeueing would silently // re-post a non-idempotent article, so we route the task to 'unknown' (kept in the dedup // active set, surfaced for manual reconcile) instead. Tasks that never reached submit are // safe to auto-requeue while attempts remain, and give up to 'failed' once exhausted. func resolvePublishRecoveryOutcome(submitStarted bool, attempts int) (status string, payloadKind string) { switch { case submitStarted: return "unknown", "publish_uncertain" case attempts >= desktopPublishMaxAttempts: return "failed", "publish_final" default: return "queued", "publish" } } func (s *DesktopTaskService) recoverClientTasksOnStartup(ctx context.Context, client *repository.DesktopClient) error { return s.recoverClientTasks(ctx, client, desktopTaskRecoveryModeStartup) } func (s *DesktopTaskService) recoverClientTasksOnDisconnect(ctx context.Context, client *repository.DesktopClient) error { return s.recoverClientTasks(ctx, client, desktopTaskRecoveryModeDisconnect) } func (s *DesktopTaskService) recoverExpiredClientTasks(ctx context.Context, client *repository.DesktopClient) error { return s.recoverClientTasks(ctx, client, desktopTaskRecoveryModeLeaseExpiry) } func (s *DesktopTaskService) recoverClientTasks( ctx context.Context, client *repository.DesktopClient, mode desktopTaskRecoveryMode, ) error { if s == nil || client == nil || s.pool == nil { return nil } publishErrorJSON, publishReason, publishMessage, err := desktopTaskRecoveryPayload(mode, "publish") if err != nil { return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload") } publishFinalErrorJSON, _, _, err := desktopTaskRecoveryPayload(mode, "publish_final") if err != nil { return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload") } publishUncertainErrorJSON, _, _, err := desktopTaskRecoveryPayload(mode, "publish_uncertain") if err != nil { return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload") } monitorErrorJSON, monitorReason, _, err := desktopTaskRecoveryPayload(mode, "monitor") if err != nil { return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode monitor desktop task recovery payload") } tx, err := s.pool.Begin(ctx) if err != nil { return response.ErrInternal(50196, "desktop_task_recovery_begin_failed", "failed to start desktop task recovery") } defer tx.Rollback(ctx) repo := repository.NewDesktopTaskRepository(tx) query := desktopTaskRecoverySelectQuery(mode) queryArgs := []any{client.ID, client.WorkspaceID} rows, err := tx.Query(ctx, query, queryArgs...) if err != nil { s.logWarn("desktop task recovery query failed", err, zap.String("client_id", client.ID.String()), zap.String("mode", string(mode))) return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks") } defer rows.Close() recovered := make([]recoveredDesktopTaskLease, 0) for rows.Next() { var item recoveredDesktopTaskLease if scanErr := rows.Scan( &item.TaskID, &item.WorkspaceID, &item.Kind, &item.Status, &item.ActiveAttempt, &item.Attempts, &item.SubmitStarted, ); scanErr != nil { s.logWarn("desktop task recovery scan failed", scanErr, zap.String("client_id", client.ID.String()), zap.String("mode", string(mode))) return response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to parse recovered desktop tasks") } recovered = append(recovered, item) } if err := rows.Err(); err != nil { s.logWarn("desktop task recovery rows failed", err, zap.String("client_id", client.ID.String()), zap.String("mode", string(mode))) return response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to iterate recovered desktop tasks") } publishOutcomes := make([]*desktopPublishSyncOutcome, 0) for index := range recovered { item := &recovered[index] if item.Kind == "monitor" { if _, execErr := tx.Exec(ctx, ` UPDATE desktop_tasks SET status = 'queued', error = $2, active_attempt_id = NULL, lease_token_hash = NULL, lease_expires_at = NULL, interrupted_at = COALESCE(interrupted_at, NOW()), interrupt_reason = $3, enqueued_at = NOW(), updated_at = NOW() WHERE desktop_id = $1 `, item.TaskID, monitorErrorJSON, monitorReason); execErr != nil { s.logWarn("desktop monitor task recovery update failed", execErr, zap.String("task_id", item.TaskID.String()), zap.String("mode", string(mode))) return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks") } item.Status = "queued" continue } finalStatus, payloadKind := resolvePublishRecoveryOutcome(item.SubmitStarted, item.Attempts) nextErrorJSON := publishErrorJSON switch payloadKind { case "publish_final": nextErrorJSON = publishFinalErrorJSON case "publish_uncertain": nextErrorJSON = publishUncertainErrorJSON } if _, execErr := tx.Exec(ctx, ` UPDATE desktop_tasks SET status = $2, error = $3, result = CASE WHEN $2 = 'queued' THEN NULL ELSE result END, active_attempt_id = NULL, lease_token_hash = NULL, lease_expires_at = NULL, interrupted_at = COALESCE(interrupted_at, NOW()), interrupt_reason = COALESCE(interrupt_reason, $4), enqueued_at = CASE WHEN $2 = 'queued' THEN NOW() ELSE enqueued_at END, updated_at = NOW() WHERE desktop_id = $1 `, item.TaskID, finalStatus, nextErrorJSON, publishReason); execErr != nil { s.logWarn("desktop publish task recovery update failed", execErr, zap.String("task_id", item.TaskID.String()), zap.String("mode", string(mode))) return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks") } item.Status = finalStatus item.ErrorJSON = nextErrorJSON // Only terminal 'failed' tasks finalize the publish_record; 'unknown' (may already be // published) is left in the dedup active set for manual reconcile, 'queued' is requeued. if finalStatus == "failed" { task, getErr := repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID) if getErr != nil { s.logWarn("desktop publish task recovery reload failed", getErr, zap.String("task_id", item.TaskID.String()), zap.String("mode", string(mode))) return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks") } publishOutcome, syncErr := syncDesktopPublishTaskState(ctx, tx, task) if syncErr != nil { return syncErr } if publishOutcome != nil { publishOutcomes = append(publishOutcomes, publishOutcome) } } } for _, item := range recovered { if !item.ActiveAttempt.Valid { continue } attemptID, convErr := uuid.FromBytes(item.ActiveAttempt.Bytes[:]) if convErr != nil { s.logWarn("desktop recovery attempt id decode failed", convErr, zap.String("task_id", item.TaskID.String())) continue } finalStatus := literalStringPtr(item.Status) errorJSON := item.ErrorJSON if len(errorJSON) == 0 { errorJSON = publishErrorJSON } if item.Kind == "monitor" { finalStatus = literalStringPtr("aborted") errorJSON = monitorErrorJSON } if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{ TaskID: item.TaskID, AttemptID: attemptID, FinalStatus: finalStatus, Error: errorJSON, }); finishErr != nil { s.logWarn("desktop recovery attempt finish failed", finishErr, zap.String("task_id", item.TaskID.String())) } } if err := tx.Commit(ctx); err != nil { return response.ErrInternal(50199, "desktop_task_recovery_commit_failed", "failed to commit desktop task recovery") } s.afterRecoveredPublishOutcomes(ctx, publishOutcomes) for _, item := range recovered { task, getErr := s.repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID) if getErr != nil { s.logWarn("desktop recovery reload failed", getErr, zap.String("task_id", item.TaskID.String())) continue } eventType := "task_completed" if item.Status == "queued" { eventType = "task_available" } s.publishTaskEvent(ctx, task, eventType) } if len(recovered) > 0 && s.logger != nil { s.logger.Info("desktop tasks recovered", zap.String("client_id", client.ID.String()), zap.String("mode", string(mode)), zap.Int("count", len(recovered)), zap.String("publish_message", publishMessage), ) } return nil } func (s *DesktopTaskService) RecoverExpiredPublishTasks(ctx context.Context, limit int) (PublishLeaseRecoveryResult, error) { var result PublishLeaseRecoveryResult if s == nil || s.pool == nil { return result, nil } if limit <= 0 { limit = 1000 } publishErrorJSON, publishReason, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "publish") if err != nil { return result, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload") } publishFinalErrorJSON, _, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "publish_final") if err != nil { return result, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload") } publishUncertainErrorJSON, _, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "publish_uncertain") if err != nil { return result, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload") } tx, err := s.pool.Begin(ctx) if err != nil { return result, response.ErrInternal(50196, "desktop_task_recovery_begin_failed", "failed to start desktop task recovery") } defer tx.Rollback(ctx) rows, err := tx.Query(ctx, ` SELECT t.desktop_id, t.workspace_id, t.kind, t.status, t.active_attempt_id, t.attempts, (t.publish_submit_started_at IS NOT NULL) AS submit_started FROM desktop_tasks AS t WHERE t.kind = 'publish' AND t.status = 'in_progress' AND t.lease_expires_at IS NOT NULL AND t.lease_expires_at < NOW() ORDER BY t.lease_expires_at ASC, t.updated_at ASC, t.desktop_id ASC LIMIT $1 FOR UPDATE OF t SKIP LOCKED `, limit) if err != nil { s.logWarn("expired desktop publish recovery query failed", err) return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks") } defer rows.Close() recovered := make([]recoveredDesktopTaskLease, 0) for rows.Next() { var item recoveredDesktopTaskLease if scanErr := rows.Scan( &item.TaskID, &item.WorkspaceID, &item.Kind, &item.Status, &item.ActiveAttempt, &item.Attempts, &item.SubmitStarted, ); scanErr != nil { s.logWarn("expired desktop publish recovery scan failed", scanErr) return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to parse recovered desktop tasks") } recovered = append(recovered, item) } if err := rows.Err(); err != nil { s.logWarn("expired desktop publish recovery rows failed", err) return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to iterate recovered desktop tasks") } repo := repository.NewDesktopTaskRepository(tx) publishOutcomes := make([]*desktopPublishSyncOutcome, 0) for index := range recovered { item := &recovered[index] finalStatus, payloadKind := resolvePublishRecoveryOutcome(item.SubmitStarted, item.Attempts) nextErrorJSON := publishErrorJSON switch payloadKind { case "publish_final": nextErrorJSON = publishFinalErrorJSON case "publish_uncertain": nextErrorJSON = publishUncertainErrorJSON } if _, execErr := tx.Exec(ctx, ` UPDATE desktop_tasks SET status = $2, error = $3, result = CASE WHEN $2 = 'queued' THEN NULL ELSE result END, active_attempt_id = NULL, lease_token_hash = NULL, lease_expires_at = NULL, interrupted_at = COALESCE(interrupted_at, NOW()), interrupt_reason = COALESCE(interrupt_reason, $4), enqueued_at = CASE WHEN $2 = 'queued' THEN NOW() ELSE enqueued_at END, updated_at = NOW() WHERE desktop_id = $1 AND status = 'in_progress' `, item.TaskID, finalStatus, nextErrorJSON, publishReason); execErr != nil { s.logWarn("expired desktop publish recovery update failed", execErr, zap.String("task_id", item.TaskID.String())) return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks") } item.Status = finalStatus item.ErrorJSON = nextErrorJSON switch finalStatus { case "failed": result.Failed++ task, getErr := repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID) if getErr != nil { s.logWarn("expired desktop publish recovery reload failed", getErr, zap.String("task_id", item.TaskID.String())) return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks") } publishOutcome, syncErr := syncDesktopPublishTaskState(ctx, tx, task) if syncErr != nil { return result, syncErr } if publishOutcome != nil { publishOutcomes = append(publishOutcomes, publishOutcome) } case "unknown": // May already be published — keep as 'unknown' (still in the dedup active set) for // manual reconcile. Do NOT sync to a terminal publish_record state and do NOT requeue. result.Uncertain++ default: result.Requeued++ } } for _, item := range recovered { if !item.ActiveAttempt.Valid { continue } attemptID, convErr := uuid.FromBytes(item.ActiveAttempt.Bytes[:]) if convErr != nil { s.logWarn("expired desktop publish recovery attempt id decode failed", convErr, zap.String("task_id", item.TaskID.String())) continue } if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{ TaskID: item.TaskID, AttemptID: attemptID, FinalStatus: literalStringPtr(item.Status), Error: item.ErrorJSON, }); finishErr != nil { s.logWarn("expired desktop publish recovery attempt finish failed", finishErr, zap.String("task_id", item.TaskID.String())) } } if err := tx.Commit(ctx); err != nil { return result, response.ErrInternal(50199, "desktop_task_recovery_commit_failed", "failed to commit desktop task recovery") } s.afterRecoveredPublishOutcomes(ctx, publishOutcomes) for _, item := range recovered { task, getErr := s.repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID) if getErr != nil { s.logWarn("expired desktop publish recovery reload after commit failed", getErr, zap.String("task_id", item.TaskID.String())) continue } eventType := "task_completed" if item.Status == "queued" { eventType = "task_available" } s.publishTaskEvent(ctx, task, eventType) } if len(recovered) > 0 && s.logger != nil { s.logger.Info("expired desktop publish tasks recovered", zap.Int("requeued", result.Requeued), zap.Int("failed", result.Failed), zap.Int("uncertain", result.Uncertain), ) } return result, nil } func desktopTaskRecoverySelectQuery(mode desktopTaskRecoveryMode) string { query := ` SELECT desktop_id, workspace_id, kind, status, active_attempt_id, attempts, publish_submit_started_at IS NOT NULL AS submit_started FROM desktop_tasks WHERE target_client_id = $1 AND workspace_id = $2 AND status = 'in_progress' ` if mode == desktopTaskRecoveryModeLeaseExpiry { query += ` AND lease_expires_at IS NOT NULL AND lease_expires_at < NOW() ` } query += ` FOR UPDATE ` return query } func desktopTaskRecoveryPayload(mode desktopTaskRecoveryMode, kind string) ([]byte, string, string, error) { reason := strings.TrimSpace(string(mode)) message := "desktop task was recovered after the client lost the active lease" source := "desktop_task_recovery" isPublishFinal := kind == "publish_final" isPublishUncertain := kind == "publish_uncertain" if isPublishFinal { message = fmt.Sprintf("desktop publish task exceeded %d attempts during recovery; task has been marked failed", desktopPublishMaxAttempts) } else if isPublishUncertain { message = "desktop publish task may have already been submitted to the platform before the lease was lost; kept as unknown for manual reconcile to avoid duplicate publishing" } switch mode { case desktopTaskRecoveryModeStartup: source = "desktop_client_startup" if isPublishFinal { message = fmt.Sprintf("desktop client restarted while the publish task was in progress; task exceeded %d attempts and has been marked failed", desktopPublishMaxAttempts) } else if isPublishUncertain { message = "desktop client restarted while the publish task was mid-submit; kept as unknown for manual reconcile to avoid duplicate publishing" } else if kind == "monitor" { message = "desktop client restarted while the monitor task was in progress; task has been re-queued" } else { message = "desktop client restarted while the publish task was in progress; task has been re-queued for retry" } case desktopTaskRecoveryModeDisconnect: source = "desktop_client_offline" if isPublishFinal { message = fmt.Sprintf("desktop client went offline while the publish task was in progress; task exceeded %d attempts and has been marked failed", desktopPublishMaxAttempts) } else if isPublishUncertain { message = "desktop client went offline while the publish task was mid-submit; kept as unknown for manual reconcile to avoid duplicate publishing" } else if kind == "monitor" { message = "desktop client went offline while the monitor task was in progress; task has been re-queued" } else { message = "desktop client went offline while the publish task was in progress; task has been re-queued for retry" } case desktopTaskRecoveryModeLeaseExpiry: source = "desktop_task_lease_expiry" if isPublishFinal { message = fmt.Sprintf("desktop task lease expired before publish completion; task exceeded %d attempts and has been marked failed", desktopPublishMaxAttempts) } else if isPublishUncertain { message = "desktop task lease expired while the publish task was mid-submit; kept as unknown for manual reconcile to avoid duplicate publishing" } else if kind == "monitor" { message = "desktop task lease expired before monitor completion; task has been re-queued" } else { message = "desktop task lease expired before publish completion; task has been re-queued for retry" } } fields := map[string]any{ "reason": reason, "message": message, "source": source, } if isPublishUncertain { // Surfaced to the desktop UI so a manual retry of a maybe-already-published task asks for // explicit confirmation instead of silently re-posting. fields["publish_submit_uncertain"] = true } payload, err := marshalOptionalJSON(fields) if err != nil { return nil, "", "", err } return payload, reason, message, nil } func (s *DesktopTaskService) dropStaleMonitorTasksForClient(ctx context.Context, client *repository.DesktopClient) error { if s == nil || client == nil || s.pool == nil || s.monitoringPool == nil { return nil } _, currentBusinessDate := monitoringBusinessDayAndDateAt(time.Now()) errorJSON, err := marshalOptionalJSON(map[string]any{ "reason": monitoringStaleTaskDropReason, "message": monitoringStaleTaskDropError, "current_business_date": currentBusinessDate, "source": "business_day_rollover", }) if err != nil { return response.ErrInternal(50190, "desktop_task_stale_payload_failed", "failed to encode stale monitor desktop task payload") } tx, err := s.pool.Begin(ctx) if err != nil { return response.ErrInternal(50191, "desktop_task_stale_begin_failed", "failed to start stale monitor desktop cleanup") } defer tx.Rollback(ctx) rows, err := tx.Query(ctx, ` WITH stale AS ( SELECT desktop_id, active_attempt_id FROM desktop_tasks WHERE target_client_id = $1 AND kind = 'monitor' AND status IN ('queued', 'in_progress', 'unknown') AND COALESCE( NULLIF(payload ->> 'business_date', ''), NULLIF(payload ->> 'businessDate', ''), NULLIF(payload ->> 'metric_date', ''), NULLIF(payload ->> 'date', '') ) < $2 FOR UPDATE ), updated AS ( UPDATE desktop_tasks AS t SET status = 'aborted', error = $3, active_attempt_id = NULL, lease_token_hash = NULL, lease_expires_at = NULL, interrupted_at = COALESCE(interrupted_at, NOW()), interrupt_reason = $4, updated_at = NOW() FROM stale WHERE t.desktop_id = stale.desktop_id RETURNING t.desktop_id ) SELECT stale.desktop_id, stale.active_attempt_id FROM stale INNER JOIN updated ON updated.desktop_id = stale.desktop_id `, client.ID, currentBusinessDate, errorJSON, monitoringStaleTaskDropReason) if err != nil { return response.ErrInternal(50192, "desktop_task_stale_query_failed", "failed to select stale monitor desktop tasks") } defer rows.Close() repo := repository.NewDesktopTaskRepository(tx) droppedTaskIDs := make([]uuid.UUID, 0) for rows.Next() { var ( taskID uuid.UUID attemptID pgtype.UUID ) if scanErr := rows.Scan(&taskID, &attemptID); scanErr != nil { return response.ErrInternal(50193, "desktop_task_stale_scan_failed", "failed to parse stale monitor desktop tasks") } droppedTaskIDs = append(droppedTaskIDs, taskID) if !attemptID.Valid { continue } resolvedAttemptID, convErr := uuid.FromBytes(attemptID.Bytes[:]) if convErr != nil { s.logWarn("desktop stale monitor attempt id decode failed", convErr, zap.String("task_id", taskID.String())) continue } if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{ TaskID: taskID, AttemptID: resolvedAttemptID, FinalStatus: literalStringPtr("aborted"), Error: errorJSON, }); finishErr != nil { s.logWarn("desktop stale monitor attempt finish failed", finishErr, zap.String("task_id", taskID.String())) } } if err := rows.Err(); err != nil { return response.ErrInternal(50193, "desktop_task_stale_scan_failed", "failed to iterate stale monitor desktop tasks") } if err := tx.Commit(ctx); err != nil { return response.ErrInternal(50194, "desktop_task_stale_commit_failed", "failed to commit stale monitor desktop cleanup") } for _, taskID := range droppedTaskIDs { task, getErr := s.repo.GetByDesktopID(ctx, taskID, client.WorkspaceID) if getErr != nil { s.logWarn("desktop stale monitor reload failed", getErr, zap.String("task_id", taskID.String())) continue } s.publishTaskEvent(ctx, task, "task_canceled") } if _, err := s.monitoringPool.Exec(ctx, ` UPDATE monitoring_collect_tasks SET status = 'skipped', lease_token_hash = NULL, leased_to_executor = NULL, leased_at = NULL, lease_expires_at = NULL, callback_received_at = COALESCE(callback_received_at, NOW()), completed_at = COALESCE(completed_at, NOW()), skip_reason = $4, error_message = COALESCE(NULLIF(error_message, ''), $5), updated_at = NOW() WHERE tenant_id = $1 AND collector_type = $2 AND COALESCE(execution_owner, 'legacy') = 'desktop_tasks' AND target_client_id = $3 AND business_date < $6::date AND callback_received_at IS NULL AND status IN ('pending', 'leased', 'expired') `, client.TenantID, monitoringCollectorType, client.ID, monitoringStaleTaskDropReason, monitoringStaleTaskDropError, currentBusinessDate); err != nil { return response.ErrInternal(50041, "task_skip_failed", "failed to drop stale phase2 monitoring tasks") } return nil } func activeAttemptID(task *repository.DesktopTask) *uuid.UUID { if task == nil || task.ActiveAttemptID == nil { return nil } value := *task.ActiveAttemptID return &value } func shouldRequeuePreemptedMonitorTask(task *repository.DesktopTask, reason *string) bool { if task == nil || task.Kind != "monitor" { return false } return strings.EqualFold(strings.TrimSpace(optionalStringValue(reason)), "collect_now_preempt") } func requeuePreemptedMonitorTask( ctx context.Context, tx pgx.Tx, repo repository.DesktopTaskRepository, task *repository.DesktopTask, ) (*repository.DesktopTask, error) { if task == nil { return nil, nil } replacementID := uuid.New() replacementJobID := uuid.New() if _, err := tx.Exec(ctx, ` INSERT INTO desktop_tasks ( desktop_id, job_id, tenant_id, workspace_id, target_account_id, target_client_id, platform_id, kind, payload, status, priority, lane, lane_weight, source, scheduler_group_key, monitor_task_id, supersedes_task_id, control_flags, interrupt_generation, enqueued_at ) VALUES ( $1, $2, $3, $4, $5, $6, $7, 'monitor', $8, 'queued', 50, 'retry', 20, 'retry_recovery', $9, $10, $11, jsonb_build_object( 'interrupt_requested', false, 'requeued_at', NOW()::text, 'requeued_from_task_id', $11::text, 'requeued_reason', 'collect_now_preempt' ), $12, NOW() ) `, replacementID, replacementJobID, task.TenantID, task.WorkspaceID, task.TargetAccountID, task.TargetClientID, task.Platform, task.Payload, nullableString(task.SchedulerGroupKey), nullableInt64(task.MonitorTaskID), task.DesktopID, task.InterruptGeneration); err != nil { return nil, response.ErrInternal(50111, "desktop_task_requeue_failed", "failed to create replacement retry monitor desktop task") } next, err := repo.GetByDesktopID(ctx, replacementID, task.WorkspaceID) if err != nil { return nil, response.ErrInternal(50112, "desktop_task_lookup_failed", "failed to reload replacement retry monitor desktop task") } return next, nil } func (s *DesktopTaskService) publishTaskEvent(ctx context.Context, task *repository.DesktopTask, eventType string) { publishDesktopTaskLifecycleEvent(ctx, s.messaging, s.logger, task, eventType) } func (s *DesktopTaskService) afterRecoveredPublishOutcomes(ctx context.Context, outcomes []*desktopPublishSyncOutcome) { for _, outcome := range outcomes { if outcome == nil { continue } invalidateArticleCaches(ctx, s.cache, outcome.TenantID, &outcome.ArticleID) if outcome.ArticleAlias != nil { if syncErr := s.syncMonitoringArticleAlias(ctx, outcome.ArticleAlias); syncErr != nil { s.logWarn("monitoring article alias sync failed after publish recovery", syncErr, zap.Int64("article_id", outcome.ArticleID)) } } } } func reconcileDesktopTaskEventType(_ string, status string) string { if status == "retry" { return "task_available" } return "task_reconciled" } func (s *DesktopTaskService) logWarn(message string, err error, fields ...zap.Field) { if s.logger == nil || err == nil { return } fields = append(fields, zap.Error(err)) s.logger.Warn(message, fields...) }