Files
geo/server/internal/tenant/app/desktop_task_service.go
T
root 28633cf570
Desktop Client Build / Resolve Build Metadata (push) Successful in 44s
Backend CI / Backend (push) Successful in 17m5s
Desktop Client Build / Build Desktop Client (push) Successful in 25m13s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 42s
feat(monitoring): fail same-client queued tasks on desktop authorization failure
When a desktop monitoring task fails with a non-retryable authorization failure
(login expired, challenge required, risk control), all pending same-client/same-platform
tasks for the same business day are immediately bulk-failed as non-retryable, avoiding
wasted execution attempts. Adds preflight authorization checks before task execution,
enriches failure payloads with disposition metadata (code, category, retryable flags),
and triggers account health reports on auth state degradation.

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
2026-05-15 23:50:50 +08:00

1787 lines
56 KiB
Go

package app
import (
"context"
"encoding/json"
"errors"
"strings"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
goredis "github.com/redis/go-redis/v9"
"go.uber.org/zap"
"github.com/geo-platform/tenant-api/internal/shared/auth"
sharedcache "github.com/geo-platform/tenant-api/internal/shared/cache"
sharedcompliance "github.com/geo-platform/tenant-api/internal/shared/compliance"
"github.com/geo-platform/tenant-api/internal/shared/config"
"github.com/geo-platform/tenant-api/internal/shared/messaging/rabbitmq"
"github.com/geo-platform/tenant-api/internal/shared/response"
tenantcompliance "github.com/geo-platform/tenant-api/internal/tenant/compliance"
"github.com/geo-platform/tenant-api/internal/tenant/repository"
)
type DesktopTaskService struct {
pool *pgxpool.Pool
monitoringPool *pgxpool.Pool
repo repository.DesktopTaskRepository
cache sharedcache.Cache
redis *goredis.Client
messaging *rabbitmq.Client
logger *zap.Logger
compliance *tenantcompliance.Service
}
func NewDesktopTaskService(pool *pgxpool.Pool, monitoringPool *pgxpool.Pool, messaging *rabbitmq.Client, logger *zap.Logger) *DesktopTaskService {
return &DesktopTaskService{
pool: pool,
monitoringPool: monitoringPool,
repo: repository.NewDesktopTaskRepository(pool),
messaging: messaging,
logger: logger,
compliance: tenantcompliance.NewService(pool, config.NewStaticProvider(&config.Config{
Compliance: config.ComplianceConfig{Enabled: true},
}), logger).WithRabbitMQ(messaging),
}
}
func NewDesktopTaskServiceWithConfig(pool *pgxpool.Pool, monitoringPool *pgxpool.Pool, messaging *rabbitmq.Client, logger *zap.Logger, cfg config.Provider) *DesktopTaskService {
svc := NewDesktopTaskService(pool, monitoringPool, messaging, logger)
svc.compliance = tenantcompliance.NewService(pool, cfg, logger).WithRabbitMQ(messaging)
return svc
}
func (s *DesktopTaskService) WithCache(c sharedcache.Cache) *DesktopTaskService {
s.cache = c
return s
}
func (s *DesktopTaskService) WithRedis(redis *goredis.Client) *DesktopTaskService {
if s != nil {
s.redis = redis
}
return s
}
type DesktopTaskView struct {
ID string `json:"id"`
JobID string `json:"job_id"`
TenantID int64 `json:"tenant_id"`
WorkspaceID int64 `json:"workspace_id"`
TargetAccountID string `json:"target_account_id"`
TargetClientID string `json:"target_client_id"`
Platform string `json:"platform"`
Kind string `json:"kind"`
Payload json.RawMessage `json:"payload"`
Status string `json:"status"`
PublishJobStatus *string `json:"publish_job_status,omitempty"`
ComplianceBlockedRecordID *int64 `json:"compliance_blocked_record_id,omitempty"`
ComplianceBlockedAt *time.Time `json:"compliance_blocked_at,omitempty"`
ComplianceBlockedReason *string `json:"compliance_blocked_reason,omitempty"`
DedupKey *string `json:"dedup_key"`
ActiveAttemptID *string `json:"active_attempt_id"`
LeaseExpiresAt *time.Time `json:"lease_expires_at"`
Attempts int `json:"attempts"`
Result json.RawMessage `json:"result"`
Error json.RawMessage `json:"error"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
type LeaseDesktopTaskRequest struct {
TaskID *string `json:"task_id"`
Kind *string `json:"kind" binding:"omitempty,oneof=publish monitor"`
}
type LeaseDesktopTaskResponse struct {
Task *DesktopTaskView `json:"task"`
AttemptID *string `json:"attempt_id,omitempty"`
LeaseToken *string `json:"lease_token,omitempty"`
LeaseExpiresAt *time.Time `json:"lease_expires_at,omitempty"`
}
type ExtendDesktopTaskRequest struct {
LeaseToken string `json:"lease_token" binding:"required"`
}
type CompleteDesktopTaskRequest struct {
LeaseToken string `json:"lease_token" binding:"required"`
Status string `json:"status" binding:"required,oneof=succeeded failed unknown"`
Payload map[string]any `json:"payload"`
Error map[string]any `json:"error"`
}
type CancelDesktopTaskRequest struct {
LeaseToken *string `json:"lease_token"`
Reason *string `json:"reason"`
}
type ReconcileDesktopTaskRequest struct {
Status string `json:"status" binding:"required,oneof=succeeded failed aborted retry"`
Result map[string]any `json:"result"`
Error map[string]any `json:"error"`
}
type ListPublishTasksRequest struct {
Page int `form:"page"`
PageSize int `form:"page_size"`
Title string `form:"title"`
}
type DesktopPublishTaskList struct {
Items []DesktopTaskView `json:"items"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Total int `json:"total"`
PendingCount int `json:"pending_count"`
HistoryTotal int `json:"history_total"`
}
type publishTaskListOwner struct {
TenantID int64
WorkspaceID int64
UserID int64
}
func (s *DesktopTaskService) Lease(ctx context.Context, client *repository.DesktopClient, req LeaseDesktopTaskRequest, routeTaskID *uuid.UUID) (*LeaseDesktopTaskResponse, error) {
if client == nil {
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
}
taskID := routeTaskID
if taskID == nil && req.TaskID != nil && strings.TrimSpace(*req.TaskID) != "" {
parsed, err := uuid.Parse(strings.TrimSpace(*req.TaskID))
if err != nil {
return nil, response.ErrBadRequest(40084, "invalid_desktop_task_id", "task id must be a uuid")
}
taskID = &parsed
}
if err := s.dropStaleMonitorTasksForClient(ctx, client); err != nil {
return nil, err
}
if err := s.recoverExpiredClientTasks(ctx, client); err != nil {
return nil, err
}
if err := s.recheckQueuedPublishTasksForClient(ctx, client); err != nil {
return nil, err
}
rawToken, tokenHash, err := newDesktopClientToken()
if err != nil {
return nil, response.ErrInternal(50087, "desktop_task_lease_token_failed", "failed to generate desktop task lease token")
}
attemptID := uuid.New()
leaseParams := repository.DesktopTaskLeaseParams{
AttemptID: attemptID,
LeaseTokenHash: tokenHash,
}
if s.logger != nil {
fields := []zap.Field{
zap.String("client_id", client.ID.String()),
}
if req.Kind != nil {
fields = append(fields, zap.String("requested_kind", strings.TrimSpace(*req.Kind)))
}
if taskID != nil {
fields = append(fields, zap.String("task_id", taskID.String()))
}
s.logger.Debug("desktop task lease requested", fields...)
}
var task *repository.DesktopTask
switch {
case taskID != nil:
task, err = s.leaseQueuedDesktopTaskByID(ctx, client, *taskID, leaseParams)
case req.Kind != nil && strings.TrimSpace(*req.Kind) == "monitor":
task, err = s.leaseNextQueuedMonitorTask(ctx, client, leaseParams)
default:
task, err = s.repo.LeaseNextQueued(ctx, client.ID, req.Kind, leaseParams)
}
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
if s.logger != nil {
fields := []zap.Field{
zap.String("client_id", client.ID.String()),
}
if req.Kind != nil {
fields = append(fields, zap.String("requested_kind", strings.TrimSpace(*req.Kind)))
}
if taskID != nil {
fields = append(fields, zap.String("task_id", taskID.String()))
}
s.logger.Debug("desktop task lease found no matching queued task", fields...)
}
if taskID == nil {
return &LeaseDesktopTaskResponse{}, nil
}
return nil, s.classifyLeaseError(ctx, client, taskID)
}
fields := []zap.Field{
zap.String("client_id", client.ID.String()),
}
if req.Kind != nil {
fields = append(fields, zap.String("requested_kind", strings.TrimSpace(*req.Kind)))
}
if taskID != nil {
fields = append(fields, zap.String("task_id", taskID.String()))
}
s.logWarn("desktop task lease query failed", err, fields...)
return nil, response.ErrInternal(50088, "desktop_task_lease_failed", "failed to lease desktop task")
}
if _, attemptErr := s.repo.CreateAttempt(ctx, repository.CreateDesktopTaskAttemptParams{
DesktopID: attemptID,
TaskID: task.DesktopID,
ClientID: client.ID,
LeaseTokenHash: tokenHash,
}); attemptErr != nil {
s.logWarn("desktop task attempt insert failed", attemptErr, zap.String("task_id", task.DesktopID.String()))
}
s.publishTaskEvent(ctx, task, "task_leased")
if s.logger != nil {
s.logger.Info("desktop task leased",
zap.String("client_id", client.ID.String()),
zap.String("task_id", task.DesktopID.String()),
zap.String("kind", task.Kind),
zap.String("status", task.Status),
)
}
view := buildDesktopTaskView(task)
attemptIDText := attemptID.String()
return &LeaseDesktopTaskResponse{
Task: &view,
AttemptID: &attemptIDText,
LeaseToken: &rawToken,
LeaseExpiresAt: task.LeaseExpiresAt,
}, nil
}
func (s *DesktopTaskService) recheckQueuedPublishTasksForClient(ctx context.Context, client *repository.DesktopClient) error {
if s == nil || s.pool == nil || s.compliance == nil || client == nil {
return nil
}
rows, err := s.pool.Query(ctx, `
SELECT
j.desktop_id,
j.tenant_id,
j.workspace_id,
j.created_by_user_id,
j.article_id,
j.article_version_id,
COALESCE(array_agg(DISTINCT dt.platform_id) FILTER (WHERE dt.platform_id <> ''), '{}') AS platforms
FROM desktop_tasks dt
JOIN desktop_publish_jobs j ON j.desktop_id = dt.job_id
WHERE dt.target_client_id = $1
AND dt.kind = 'publish'
AND dt.status = 'queued'
AND j.status = 'queued'
AND j.article_id IS NOT NULL
AND j.article_version_id IS NOT NULL
GROUP BY j.desktop_id, j.tenant_id, j.workspace_id, j.created_by_user_id, j.article_id, j.article_version_id
ORDER BY MIN(dt.created_at)
LIMIT 5
`, client.ID)
if err != nil {
s.logWarn("desktop publish compliance recheck query failed", err, zap.String("client_id", client.ID.String()))
return response.ErrInternal(50118, "desktop_publish_compliance_recheck_failed", "failed to load publish jobs for compliance recheck")
}
defer rows.Close()
type candidate struct {
JobID uuid.UUID
TenantID int64
WorkspaceID int64
CreatedByUserID int64
ArticleID int64
ArticleVersionID int64
Platforms []string
}
candidates := make([]candidate, 0)
for rows.Next() {
var item candidate
if scanErr := rows.Scan(&item.JobID, &item.TenantID, &item.WorkspaceID, &item.CreatedByUserID, &item.ArticleID, &item.ArticleVersionID, &item.Platforms); scanErr != nil {
s.logWarn("desktop publish compliance recheck scan failed", scanErr, zap.String("client_id", client.ID.String()))
return response.ErrInternal(50118, "desktop_publish_compliance_recheck_failed", "failed to scan publish job for compliance recheck")
}
if item.WorkspaceID == client.WorkspaceID && item.TenantID == client.TenantID {
candidates = append(candidates, item)
}
}
if err := rows.Err(); err != nil {
s.logWarn("desktop publish compliance recheck rows failed", err, zap.String("client_id", client.ID.String()))
return response.ErrInternal(50118, "desktop_publish_compliance_recheck_failed", "failed to iterate publish jobs for compliance recheck")
}
for _, item := range candidates {
gate, gateErr := s.compliance.GateForPublish(ctx, tenantcompliance.GateForPublishRequest{
TenantID: item.TenantID,
ArticleID: item.ArticleID,
ArticleVersionID: item.ArticleVersionID,
ActorID: item.CreatedByUserID,
TargetPlatforms: item.Platforms,
TriggerSource: "scheduler_recheck",
})
if gateErr != nil && gate != nil && gate.Decision == sharedcompliance.GateDecisionBlock {
if markErr := s.compliance.MarkPublishJobBlockedByCompliance(ctx, item.JobID, gate.Result); markErr != nil {
return markErr
}
continue
}
if gateErr != nil {
return response.ErrServiceUnavailable(41004, "compliance_engine_unavailable", "scheduled publish compliance recheck failed")
}
if gate != nil && gate.Decision == sharedcompliance.GateDecisionBlock {
if markErr := s.compliance.MarkPublishJobBlockedByCompliance(ctx, item.JobID, gate.Result); markErr != nil {
return markErr
}
}
}
return nil
}
const desktopTaskRepositoryReturningColumns = `
t.desktop_id,
t.job_id,
t.tenant_id,
t.workspace_id,
t.target_account_id,
t.target_client_id,
t.platform_id,
t.kind,
t.payload,
t.status,
t.priority,
t.lane,
t.lane_weight,
t.source,
t.scheduler_group_key,
t.monitor_task_id,
t.supersedes_task_id,
t.control_flags,
t.interrupt_generation,
t.dedup_key,
t.active_attempt_id,
t.lease_expires_at,
t.attempts,
t.result,
t.error,
t.started_at,
t.interrupted_at,
t.interrupt_reason,
t.enqueued_at,
t.created_at,
t.updated_at`
type desktopTaskRepositoryScanner interface {
Scan(dest ...any) error
}
func (s *DesktopTaskService) leaseNextQueuedMonitorTask(
ctx context.Context,
client *repository.DesktopClient,
params repository.DesktopTaskLeaseParams,
) (*repository.DesktopTask, error) {
accountIDs, err := s.loadMonitorLeaseAccountIDs(ctx, client)
if err != nil {
return nil, err
}
hasAccountIDs := len(accountIDs) > 0
row := s.pool.QueryRow(ctx, `
WITH candidate AS (
SELECT dt.desktop_id
FROM desktop_tasks AS dt
WHERE dt.tenant_id = $1
AND dt.workspace_id = $2
AND dt.kind = 'monitor'
AND dt.status = 'queued'
AND (
dt.target_client_id = $3
OR ($4::boolean AND dt.target_account_id = ANY($5::uuid[]))
)
ORDER BY dt.lane_weight DESC,
dt.priority DESC,
COALESCE(dt.enqueued_at, dt.created_at) ASC,
dt.created_at ASC,
dt.desktop_id ASC
LIMIT 1
FOR UPDATE SKIP LOCKED
)
UPDATE desktop_tasks AS t
SET target_client_id = $3,
active_attempt_id = $6,
lease_token_hash = $7,
lease_expires_at = now() + interval '10 minutes',
status = 'in_progress',
attempts = t.attempts + 1,
updated_at = now()
FROM candidate
WHERE t.desktop_id = candidate.desktop_id
RETURNING `+desktopTaskRepositoryReturningColumns,
client.TenantID,
client.WorkspaceID,
client.ID,
hasAccountIDs,
accountIDs,
params.AttemptID,
params.LeaseTokenHash,
)
return scanRepositoryDesktopTask(row)
}
func (s *DesktopTaskService) leaseQueuedDesktopTaskByID(
ctx context.Context,
client *repository.DesktopClient,
desktopID uuid.UUID,
params repository.DesktopTaskLeaseParams,
) (*repository.DesktopTask, error) {
accountIDs, err := s.loadMonitorLeaseAccountIDs(ctx, client)
if err != nil {
return nil, err
}
hasAccountIDs := len(accountIDs) > 0
row := s.pool.QueryRow(ctx, `
WITH candidate AS (
SELECT dt.desktop_id
FROM desktop_tasks AS dt
WHERE dt.desktop_id = $1
AND dt.workspace_id = $2
AND dt.status = 'queued'
AND (
(
dt.kind = 'monitor'
AND dt.tenant_id = $3
AND (
dt.target_client_id = $4
OR ($5::boolean AND dt.target_account_id = ANY($6::uuid[]))
)
)
OR (
dt.kind <> 'monitor'
AND dt.target_client_id = $4
)
)
LIMIT 1
FOR UPDATE SKIP LOCKED
)
UPDATE desktop_tasks AS t
SET target_client_id = CASE WHEN t.kind = 'monitor' THEN $4 ELSE t.target_client_id END,
active_attempt_id = $7,
lease_token_hash = $8,
lease_expires_at = now() + interval '10 minutes',
status = 'in_progress',
attempts = t.attempts + 1,
updated_at = now()
FROM candidate
WHERE t.desktop_id = candidate.desktop_id
RETURNING `+desktopTaskRepositoryReturningColumns,
desktopID,
client.WorkspaceID,
client.TenantID,
client.ID,
hasAccountIDs,
accountIDs,
params.AttemptID,
params.LeaseTokenHash,
)
return scanRepositoryDesktopTask(row)
}
func (s *DesktopTaskService) loadMonitorLeaseAccountIDs(ctx context.Context, client *repository.DesktopClient) ([]uuid.UUID, error) {
if s == nil || client == nil || s.pool == nil {
return nil, nil
}
accountIDs := loadTrackedDesktopClientAccountIDs(ctx, s.redis, client.ID)
rows, err := s.pool.Query(ctx, `
SELECT desktop_id
FROM platform_accounts
WHERE tenant_id = $1
AND workspace_id = $2
AND client_id = $3
AND deleted_at IS NULL
AND platform_id = ANY($4::text[])
`, client.TenantID, client.WorkspaceID, client.ID, monitoringPlatformIDs(defaultMonitoringPlatforms))
if err != nil {
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to resolve monitor lease accounts")
}
defer rows.Close()
for rows.Next() {
var accountID uuid.UUID
if scanErr := rows.Scan(&accountID); scanErr != nil {
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to parse monitor lease accounts")
}
accountIDs = append(accountIDs, accountID)
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to iterate monitor lease accounts")
}
return uniqueUUIDs(accountIDs), nil
}
func scanRepositoryDesktopTask(row desktopTaskRepositoryScanner) (*repository.DesktopTask, error) {
var (
task repository.DesktopTask
schedulerGroupKey pgtype.Text
monitorTaskID pgtype.Int8
supersedesTaskID pgtype.UUID
dedupKey pgtype.Text
activeAttemptID pgtype.UUID
leaseExpiresAt pgtype.Timestamptz
startedAt pgtype.Timestamptz
interruptedAt pgtype.Timestamptz
interruptReason pgtype.Text
enqueuedAt pgtype.Timestamptz
)
if err := row.Scan(
&task.DesktopID,
&task.JobID,
&task.TenantID,
&task.WorkspaceID,
&task.TargetAccountID,
&task.TargetClientID,
&task.Platform,
&task.Kind,
&task.Payload,
&task.Status,
&task.Priority,
&task.Lane,
&task.LaneWeight,
&task.Source,
&schedulerGroupKey,
&monitorTaskID,
&supersedesTaskID,
&task.ControlFlags,
&task.InterruptGeneration,
&dedupKey,
&activeAttemptID,
&leaseExpiresAt,
&task.Attempts,
&task.Result,
&task.Error,
&startedAt,
&interruptedAt,
&interruptReason,
&enqueuedAt,
&task.CreatedAt,
&task.UpdatedAt,
); err != nil {
return nil, err
}
task.SchedulerGroupKey = desktopTaskNullableText(schedulerGroupKey)
task.MonitorTaskID = desktopTaskNullableInt64(monitorTaskID)
task.SupersedesTaskID = desktopTaskNullableUUID(supersedesTaskID)
task.DedupKey = desktopTaskNullableText(dedupKey)
task.ActiveAttemptID = desktopTaskNullableUUID(activeAttemptID)
task.LeaseExpiresAt = desktopTaskNullableTime(leaseExpiresAt)
task.StartedAt = desktopTaskNullableTime(startedAt)
task.InterruptedAt = desktopTaskNullableTime(interruptedAt)
task.InterruptReason = desktopTaskNullableText(interruptReason)
if enqueuedAt.Valid {
task.EnqueuedAt = enqueuedAt.Time
}
return &task, nil
}
func desktopTaskNullableText(value pgtype.Text) *string {
if !value.Valid {
return nil
}
text := value.String
return &text
}
func desktopTaskNullableInt64(value pgtype.Int8) *int64 {
if !value.Valid {
return nil
}
number := value.Int64
return &number
}
func desktopTaskNullableUUID(value pgtype.UUID) *uuid.UUID {
if !value.Valid {
return nil
}
resolved, err := uuid.FromBytes(value.Bytes[:])
if err != nil {
return nil
}
return &resolved
}
func desktopTaskNullableTime(value pgtype.Timestamptz) *time.Time {
if !value.Valid {
return nil
}
timestamp := value.Time
return &timestamp
}
func (s *DesktopTaskService) Extend(ctx context.Context, client *repository.DesktopClient, desktopID uuid.UUID, req ExtendDesktopTaskRequest) (*DesktopTaskView, error) {
if client == nil {
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
}
if err := s.dropStaleMonitorTasksForClient(ctx, client); err != nil {
return nil, err
}
task, err := s.repo.ExtendLease(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(req.LeaseToken)))
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, response.ErrConflict(40983, "desktop_task_lease_invalid", "desktop task lease token is invalid or expired")
}
return nil, response.ErrInternal(50089, "desktop_task_extend_failed", "failed to extend desktop task lease")
}
s.publishTaskEvent(ctx, task, "task_extended")
view := buildDesktopTaskView(task)
return &view, nil
}
func (s *DesktopTaskService) Complete(ctx context.Context, client *repository.DesktopClient, desktopID uuid.UUID, req CompleteDesktopTaskRequest) (*DesktopTaskView, error) {
if client == nil {
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
}
finalStatus := normalizeDesktopTaskTerminalStatus(req.Status)
resultJSON, err := marshalOptionalJSON(req.Payload)
if err != nil {
return nil, response.ErrBadRequest(40086, "invalid_desktop_task_payload", "payload must be serializable")
}
errorJSON, err := marshalOptionalJSON(req.Error)
if err != nil {
return nil, response.ErrBadRequest(40087, "invalid_desktop_task_error", "error must be serializable")
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return nil, response.ErrInternal(50103, "desktop_task_complete_begin_failed", "failed to start desktop task completion")
}
defer tx.Rollback(ctx)
taskRepo := repository.NewDesktopTaskRepository(tx)
previousTask, _ := taskRepo.GetByDesktopID(ctx, desktopID, client.WorkspaceID)
previousAttemptID := activeAttemptID(previousTask)
task, err := taskRepo.Complete(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(req.LeaseToken)), finalStatus, resultJSON, errorJSON)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, response.ErrConflict(40983, "desktop_task_lease_invalid", "desktop task lease token is invalid or expired")
}
return nil, response.ErrInternal(50091, "desktop_task_complete_failed", "failed to complete desktop task")
}
if previousAttemptID != nil {
if finishErr := taskRepo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
TaskID: task.DesktopID,
AttemptID: *previousAttemptID,
FinalStatus: &finalStatus,
Error: errorJSON,
}); finishErr != nil {
s.logWarn("desktop task attempt finish failed after complete", finishErr, zap.String("task_id", task.DesktopID.String()))
}
}
publishOutcome, err := syncDesktopPublishTaskState(ctx, tx, task)
if err != nil {
return nil, err
}
bulkFailure, err := bulkFailQueuedDesktopTasksForAuthorizationFailure(ctx, tx, task, errorJSON)
if err != nil {
return nil, err
}
if err := tx.Commit(ctx); err != nil {
return nil, response.ErrInternal(50104, "desktop_task_complete_commit_failed", "failed to commit desktop task completion")
}
if publishOutcome != nil {
invalidateArticleCaches(ctx, s.cache, publishOutcome.TenantID, &publishOutcome.ArticleID)
if syncErr := s.syncMonitoringArticleAlias(ctx, publishOutcome.ArticleAlias); syncErr != nil {
s.logWarn("monitoring article alias sync failed after publish complete", syncErr, zap.Int64("article_id", publishOutcome.ArticleID))
}
}
for _, outcome := range bulkFailure.PublishOutcomes {
invalidateArticleCaches(ctx, s.cache, outcome.TenantID, &outcome.ArticleID)
if outcome.ArticleAlias != nil {
if syncErr := s.syncMonitoringArticleAlias(ctx, outcome.ArticleAlias); syncErr != nil {
s.logWarn("monitoring article alias sync failed after bulk authorization failure", syncErr, zap.Int64("article_id", outcome.ArticleID))
}
}
}
s.publishTaskEvent(ctx, task, "task_completed")
for _, failedTask := range bulkFailure.Tasks {
s.publishTaskEvent(ctx, failedTask, "task_completed")
}
view := buildDesktopTaskView(task)
return &view, nil
}
func (s *DesktopTaskService) Cancel(ctx context.Context, client *repository.DesktopClient, desktopID uuid.UUID, req CancelDesktopTaskRequest) (*DesktopTaskView, error) {
if client == nil {
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
}
errorJSON, err := marshalOptionalJSON(map[string]any{
"reason": optionalStringValue(req.Reason),
"source": "desktop_client",
})
if err != nil {
return nil, response.ErrBadRequest(40088, "invalid_desktop_task_cancel", "cancel payload must be serializable")
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return nil, response.ErrInternal(50107, "desktop_task_cancel_begin_failed", "failed to start desktop task cancel")
}
defer tx.Rollback(ctx)
taskRepo := repository.NewDesktopTaskRepository(tx)
previousTask, _ := taskRepo.GetByDesktopID(ctx, desktopID, client.WorkspaceID)
previousAttemptID := activeAttemptID(previousTask)
var task *repository.DesktopTask
if req.LeaseToken != nil && strings.TrimSpace(*req.LeaseToken) != "" {
task, err = taskRepo.CancelByLease(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(*req.LeaseToken)), errorJSON)
} else {
task, err = taskRepo.CancelByClient(ctx, desktopID, client.ID, errorJSON)
}
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, response.ErrConflict(40984, "desktop_task_cancel_conflict", "desktop task cannot be canceled in its current state")
}
return nil, response.ErrInternal(50092, "desktop_task_cancel_failed", "failed to cancel desktop task")
}
if previousAttemptID != nil && req.LeaseToken != nil && strings.TrimSpace(*req.LeaseToken) != "" {
if finishErr := taskRepo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
TaskID: task.DesktopID,
AttemptID: *previousAttemptID,
FinalStatus: literalStringPtr("aborted"),
Error: errorJSON,
}); finishErr != nil {
s.logWarn("desktop task attempt finish failed after cancel", finishErr, zap.String("task_id", task.DesktopID.String()))
}
}
var replacementTask *repository.DesktopTask
if shouldRequeuePreemptedMonitorTask(task, req.Reason) {
replacementTask, err = requeuePreemptedMonitorTask(ctx, tx, taskRepo, task)
if err != nil {
return nil, err
}
}
if err := tx.Commit(ctx); err != nil {
return nil, response.ErrInternal(50108, "desktop_task_cancel_commit_failed", "failed to commit desktop task cancel")
}
s.publishTaskEvent(ctx, task, "task_canceled")
if replacementTask != nil {
s.publishTaskEvent(ctx, replacementTask, "task_available")
}
view := buildDesktopTaskView(task)
return &view, nil
}
func (s *DesktopTaskService) TenantCancel(ctx context.Context, actor auth.Actor, desktopID uuid.UUID, req CancelDesktopTaskRequest) (*DesktopTaskView, error) {
if actor.PrimaryWorkspaceID == 0 {
return nil, response.ErrUnauthorized(40132, "workspace_scope_missing", "workspace context is required")
}
errorJSON, err := marshalOptionalJSON(map[string]any{
"reason": optionalStringValue(req.Reason),
"source": "tenant_web",
})
if err != nil {
return nil, response.ErrBadRequest(40088, "invalid_desktop_task_cancel", "cancel payload must be serializable")
}
task, err := s.repo.TenantCancel(ctx, desktopID, actor.PrimaryWorkspaceID, errorJSON)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, response.ErrConflict(40984, "desktop_task_cancel_conflict", "desktop task cannot be canceled in its current state")
}
return nil, response.ErrInternal(50093, "desktop_task_tenant_cancel_failed", "failed to cancel desktop task")
}
s.publishTaskEvent(ctx, task, "task_canceled")
view := buildDesktopTaskView(task)
return &view, nil
}
func (s *DesktopTaskService) Reconcile(ctx context.Context, actor auth.Actor, desktopID uuid.UUID, req ReconcileDesktopTaskRequest) (*DesktopTaskView, error) {
if actor.PrimaryWorkspaceID == 0 {
return nil, response.ErrUnauthorized(40132, "workspace_scope_missing", "workspace context is required")
}
resultJSON, err := marshalOptionalJSON(req.Result)
if err != nil {
return nil, response.ErrBadRequest(40089, "invalid_desktop_task_reconcile_result", "result must be serializable")
}
errorJSON, err := marshalOptionalJSON(req.Error)
if err != nil {
return nil, response.ErrBadRequest(40090, "invalid_desktop_task_reconcile_error", "error must be serializable")
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return nil, response.ErrInternal(50105, "desktop_task_reconcile_begin_failed", "failed to start desktop task reconcile")
}
defer tx.Rollback(ctx)
taskRepo := repository.NewDesktopTaskRepository(tx)
task, err := taskRepo.Reconcile(ctx, desktopID, actor.PrimaryWorkspaceID, req.Status, resultJSON, errorJSON)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, response.ErrConflict(40985, "desktop_task_reconcile_conflict", "desktop task is not waiting for reconcile")
}
return nil, response.ErrInternal(50094, "desktop_task_reconcile_failed", "failed to reconcile desktop task")
}
publishOutcome, err := syncDesktopPublishTaskState(ctx, tx, task)
if err != nil {
return nil, err
}
if err := tx.Commit(ctx); err != nil {
return nil, response.ErrInternal(50106, "desktop_task_reconcile_commit_failed", "failed to commit desktop task reconcile")
}
if publishOutcome != nil {
invalidateArticleCaches(ctx, s.cache, publishOutcome.TenantID, &publishOutcome.ArticleID)
if syncErr := s.syncMonitoringArticleAlias(ctx, publishOutcome.ArticleAlias); syncErr != nil {
s.logWarn("monitoring article alias sync failed after publish reconcile", syncErr, zap.Int64("article_id", publishOutcome.ArticleID))
}
}
eventType := "task_reconciled"
if task.Kind == "monitor" && req.Status == "retry" {
eventType = "task_available"
}
s.publishTaskEvent(ctx, task, eventType)
view := buildDesktopTaskView(task)
return &view, nil
}
func (s *DesktopTaskService) ListPublishTasks(ctx context.Context, client *repository.DesktopClient, req ListPublishTasksRequest) (*DesktopPublishTaskList, error) {
if client == nil {
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
}
return s.listPublishTasksForOwner(ctx, publishTaskListOwner{
TenantID: client.TenantID,
WorkspaceID: client.WorkspaceID,
UserID: client.UserID,
}, req)
}
func (s *DesktopTaskService) ListTenantPublishTasks(ctx context.Context, actor auth.Actor, req ListPublishTasksRequest) (*DesktopPublishTaskList, error) {
workspaceID := auth.CurrentWorkspaceID(ctx)
if actor.TenantID == 0 || actor.UserID == 0 || workspaceID == 0 {
return nil, response.ErrUnauthorized(40132, "workspace_scope_missing", "workspace context is required")
}
return s.listPublishTasksForOwner(ctx, publishTaskListOwner{
TenantID: actor.TenantID,
WorkspaceID: workspaceID,
UserID: actor.UserID,
}, req)
}
func (s *DesktopTaskService) listPublishTasksForOwner(ctx context.Context, owner publishTaskListOwner, req ListPublishTasksRequest) (*DesktopPublishTaskList, error) {
if s.pool == nil {
return nil, response.ErrServiceUnavailable(50342, "desktop_publish_job_store_unavailable", "desktop publish job store is unavailable")
}
page := req.Page
if page <= 0 {
page = 1
}
pageSize := req.PageSize
if pageSize <= 0 || pageSize > 50 {
pageSize = 10
}
title := strings.TrimSpace(req.Title)
pendingStatuses := []string{"queued", "in_progress"}
historyStatuses := []string{"succeeded", "failed", "unknown", "aborted"}
pendingItems, err := s.listPublishTasksByStatuses(
ctx,
owner,
pendingStatuses,
title,
0,
0,
`CASE WHEN t.status = 'in_progress' THEN 0 ELSE 1 END ASC, t.created_at ASC, t.desktop_id DESC`,
)
if err != nil {
return nil, err
}
historyTotal, err := s.countPublishTasksByStatuses(ctx, owner, historyStatuses, title)
if err != nil {
return nil, err
}
offset := (page - 1) * pageSize
items := make([]DesktopTaskView, 0, pageSize)
total := len(pendingItems) + historyTotal
if offset < len(pendingItems) {
end := offset + pageSize
if end > len(pendingItems) {
end = len(pendingItems)
}
items = append(items, pendingItems[offset:end]...)
}
remaining := pageSize - len(items)
historyOffset := offset - len(pendingItems)
if historyOffset < 0 {
historyOffset = 0
}
if remaining > 0 && historyOffset < historyTotal {
historyItems, listErr := s.listPublishTasksByStatuses(
ctx,
owner,
historyStatuses,
title,
remaining,
historyOffset,
`t.updated_at DESC, t.desktop_id DESC`,
)
if listErr != nil {
return nil, listErr
}
items = append(items, historyItems...)
}
return &DesktopPublishTaskList{
Items: items,
Page: page,
PageSize: pageSize,
Total: total,
PendingCount: len(pendingItems),
HistoryTotal: historyTotal,
}, nil
}
func (s *DesktopTaskService) listPublishTasksByStatuses(
ctx context.Context,
owner publishTaskListOwner,
statuses []string,
title string,
limit int,
offset int,
orderBy string,
) ([]DesktopTaskView, error) {
query := `
SELECT
t.desktop_id,
t.job_id,
t.tenant_id,
t.workspace_id,
t.target_account_id,
t.target_client_id,
t.platform_id,
t.kind,
t.payload,
t.status,
t.dedup_key,
t.active_attempt_id,
t.lease_expires_at,
t.attempts,
t.result,
t.error,
t.created_at,
t.updated_at,
j.status,
j.compliance_blocked_record_id,
j.compliance_blocked_at,
j.compliance_blocked_reason
FROM desktop_tasks AS t
JOIN desktop_publish_jobs AS j
ON j.desktop_id = t.job_id
AND j.workspace_id = t.workspace_id
WHERE t.tenant_id = $1
AND t.workspace_id = $2
AND j.tenant_id = $1
AND j.created_by_user_id = $3
AND t.kind = 'publish'
AND t.status = ANY($4)
AND (
$5 = ''
OR COALESCE(t.payload->>'title', j.title, '') ILIKE '%' || $5 || '%'
)
`
args := []any{owner.TenantID, owner.WorkspaceID, owner.UserID, statuses, title}
if strings.TrimSpace(orderBy) != "" {
query += "\nORDER BY " + orderBy
}
if limit > 0 {
query += "\nLIMIT $6 OFFSET $7"
args = append(args, limit, offset)
}
rows, err := s.pool.Query(ctx, query, args...)
if err != nil {
return nil, response.ErrInternal(50109, "desktop_publish_tasks_query_failed", "failed to list desktop publish tasks")
}
defer rows.Close()
return scanDesktopTaskRows(rows)
}
func (s *DesktopTaskService) countPublishTasksByStatuses(
ctx context.Context,
owner publishTaskListOwner,
statuses []string,
title string,
) (int, error) {
var count int
err := s.pool.QueryRow(ctx, `
SELECT COUNT(1)
FROM desktop_tasks AS t
JOIN desktop_publish_jobs AS j
ON j.desktop_id = t.job_id
AND j.workspace_id = t.workspace_id
WHERE t.tenant_id = $1
AND t.workspace_id = $2
AND j.tenant_id = $1
AND j.created_by_user_id = $3
AND t.kind = 'publish'
AND t.status = ANY($4)
AND (
$5 = ''
OR COALESCE(t.payload->>'title', j.title, '') ILIKE '%' || $5 || '%'
)
`, owner.TenantID, owner.WorkspaceID, owner.UserID, statuses, title).Scan(&count)
if err != nil {
return 0, response.ErrInternal(50109, "desktop_publish_tasks_query_failed", "failed to list desktop publish tasks")
}
return count, nil
}
func scanDesktopTaskRows(rows pgx.Rows) ([]DesktopTaskView, error) {
items := make([]DesktopTaskView, 0)
for rows.Next() {
var (
desktopID uuid.UUID
jobID uuid.UUID
tenantID int64
workspaceID int64
targetAccountID uuid.UUID
targetClientID uuid.UUID
platform string
kind string
payload []byte
status string
dedupKey pgtype.Text
activeAttemptID pgtype.UUID
leaseExpiresAt pgtype.Timestamptz
attempts int32
resultJSON []byte
errorJSON []byte
createdAt time.Time
updatedAt time.Time
jobStatus string
blockedRecordID pgtype.Int8
blockedAt pgtype.Timestamptz
blockedReason pgtype.Text
)
if err := rows.Scan(
&desktopID,
&jobID,
&tenantID,
&workspaceID,
&targetAccountID,
&targetClientID,
&platform,
&kind,
&payload,
&status,
&dedupKey,
&activeAttemptID,
&leaseExpiresAt,
&attempts,
&resultJSON,
&errorJSON,
&createdAt,
&updatedAt,
&jobStatus,
&blockedRecordID,
&blockedAt,
&blockedReason,
); err != nil {
return nil, response.ErrInternal(50110, "desktop_publish_tasks_scan_failed", "failed to scan desktop publish tasks")
}
var dedupKeyText *string
if dedupKey.Valid {
value := dedupKey.String
dedupKeyText = &value
}
var activeAttemptIDText *string
if activeAttemptID.Valid {
value, convErr := uuid.FromBytes(activeAttemptID.Bytes[:])
if convErr == nil {
text := value.String()
activeAttemptIDText = &text
}
}
var leaseExpiresAtValue *time.Time
if leaseExpiresAt.Valid {
value := leaseExpiresAt.Time
leaseExpiresAtValue = &value
}
publishJobStatusText := strings.TrimSpace(jobStatus)
var publishJobStatus *string
if publishJobStatusText != "" {
publishJobStatus = &publishJobStatusText
}
var complianceBlockedRecordID *int64
if blockedRecordID.Valid {
value := blockedRecordID.Int64
complianceBlockedRecordID = &value
}
var complianceBlockedAt *time.Time
if blockedAt.Valid {
value := blockedAt.Time
complianceBlockedAt = &value
}
var complianceBlockedReason *string
if blockedReason.Valid {
value := blockedReason.String
complianceBlockedReason = &value
}
items = append(items, DesktopTaskView{
ID: desktopID.String(),
JobID: jobID.String(),
TenantID: tenantID,
WorkspaceID: workspaceID,
TargetAccountID: targetAccountID.String(),
TargetClientID: targetClientID.String(),
Platform: platform,
Kind: kind,
Payload: json.RawMessage(payload),
Status: normalizeDesktopTaskTerminalStatus(status),
PublishJobStatus: publishJobStatus,
ComplianceBlockedRecordID: complianceBlockedRecordID,
ComplianceBlockedAt: complianceBlockedAt,
ComplianceBlockedReason: complianceBlockedReason,
DedupKey: dedupKeyText,
ActiveAttemptID: activeAttemptIDText,
LeaseExpiresAt: leaseExpiresAtValue,
Attempts: int(attempts),
Result: json.RawMessage(resultJSON),
Error: json.RawMessage(errorJSON),
CreatedAt: createdAt,
UpdatedAt: updatedAt,
})
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50109, "desktop_publish_tasks_query_failed", "failed to list desktop publish tasks")
}
return items, nil
}
func (s *DesktopTaskService) classifyLeaseError(ctx context.Context, client *repository.DesktopClient, taskID *uuid.UUID) error {
if taskID == nil {
return response.ErrConflict(40986, "desktop_task_unavailable", "no desktop task is currently available")
}
task, err := s.repo.GetByDesktopID(ctx, *taskID, client.WorkspaceID)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return response.ErrNotFound(40482, "desktop_task_not_found", "desktop task not found")
}
return response.ErrInternal(50095, "desktop_task_lookup_failed", "failed to inspect desktop task state")
}
if task.TargetClientID != client.ID && !s.clientCanLeaseMonitorTask(ctx, client, task) {
return response.ErrForbidden(40381, "desktop_task_not_target_client", "desktop task is assigned to another desktop client")
}
if task.Status == "in_progress" {
return response.ErrConflict(40989, "desktop_task_already_in_progress", "desktop task is already in progress")
}
return response.ErrConflict(40991, "desktop_task_not_queued", "desktop task is not queued")
}
func (s *DesktopTaskService) clientCanLeaseMonitorTask(ctx context.Context, client *repository.DesktopClient, task *repository.DesktopTask) bool {
if s == nil || client == nil || task == nil || task.Kind != "monitor" || task.TargetAccountID == uuid.Nil {
return false
}
accountIDs, err := s.loadMonitorLeaseAccountIDs(ctx, client)
if err != nil {
return false
}
for _, accountID := range accountIDs {
if accountID == task.TargetAccountID {
return true
}
}
return false
}
func buildDesktopTaskView(task *repository.DesktopTask) DesktopTaskView {
var activeAttemptID *string
if task.ActiveAttemptID != nil {
value := task.ActiveAttemptID.String()
activeAttemptID = &value
}
return DesktopTaskView{
ID: task.DesktopID.String(),
JobID: task.JobID.String(),
TenantID: task.TenantID,
WorkspaceID: task.WorkspaceID,
TargetAccountID: task.TargetAccountID.String(),
TargetClientID: task.TargetClientID.String(),
Platform: task.Platform,
Kind: task.Kind,
Payload: json.RawMessage(task.Payload),
Status: normalizeDesktopTaskTerminalStatus(task.Status),
DedupKey: task.DedupKey,
ActiveAttemptID: activeAttemptID,
LeaseExpiresAt: task.LeaseExpiresAt,
Attempts: task.Attempts,
Result: json.RawMessage(task.Result),
Error: json.RawMessage(task.Error),
CreatedAt: task.CreatedAt,
UpdatedAt: task.UpdatedAt,
}
}
func marshalOptionalJSON(value any) ([]byte, error) {
if value == nil {
return nil, nil
}
return json.Marshal(value)
}
func optionalStringValue(value *string) string {
if value == nil {
return ""
}
return strings.TrimSpace(*value)
}
func literalStringPtr(value string) *string {
return &value
}
func normalizeDesktopTaskTerminalStatus(status string) string {
switch strings.TrimSpace(status) {
case "unknown":
return "failed"
default:
return strings.TrimSpace(status)
}
}
type desktopTaskRecoveryMode string
const (
desktopTaskRecoveryModeStartup desktopTaskRecoveryMode = "startup"
desktopTaskRecoveryModeDisconnect desktopTaskRecoveryMode = "disconnect"
desktopTaskRecoveryModeLeaseExpiry desktopTaskRecoveryMode = "lease_expired"
)
type recoveredDesktopTaskLease struct {
TaskID uuid.UUID
WorkspaceID int64
Kind string
Status string
ActiveAttempt pgtype.UUID
}
func (s *DesktopTaskService) recoverClientTasksOnStartup(ctx context.Context, client *repository.DesktopClient) error {
return s.recoverClientTasks(ctx, client, desktopTaskRecoveryModeStartup)
}
func (s *DesktopTaskService) recoverClientTasksOnDisconnect(ctx context.Context, client *repository.DesktopClient) error {
return s.recoverClientTasks(ctx, client, desktopTaskRecoveryModeDisconnect)
}
func (s *DesktopTaskService) recoverExpiredClientTasks(ctx context.Context, client *repository.DesktopClient) error {
return s.recoverClientTasks(ctx, client, desktopTaskRecoveryModeLeaseExpiry)
}
func (s *DesktopTaskService) recoverClientTasks(
ctx context.Context,
client *repository.DesktopClient,
mode desktopTaskRecoveryMode,
) error {
if s == nil || client == nil || s.pool == nil {
return nil
}
publishErrorJSON, publishReason, publishMessage, err := desktopTaskRecoveryPayload(mode, "publish")
if err != nil {
return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
}
monitorErrorJSON, monitorReason, _, err := desktopTaskRecoveryPayload(mode, "monitor")
if err != nil {
return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode monitor desktop task recovery payload")
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return response.ErrInternal(50196, "desktop_task_recovery_begin_failed", "failed to start desktop task recovery")
}
defer tx.Rollback(ctx)
repo := repository.NewDesktopTaskRepository(tx)
query := `
SELECT
desktop_id,
workspace_id,
kind,
active_attempt_id
FROM desktop_tasks
WHERE target_client_id = $1
AND workspace_id = $2
AND status = 'in_progress'
`
queryArgs := []any{client.ID, client.WorkspaceID}
if mode == desktopTaskRecoveryModeLeaseExpiry {
query += `
AND lease_expires_at IS NOT NULL
AND lease_expires_at < NOW()
`
}
query += `
FOR UPDATE
`
rows, err := tx.Query(ctx, query, queryArgs...)
if err != nil {
s.logWarn("desktop task recovery query failed", err, zap.String("client_id", client.ID.String()), zap.String("mode", string(mode)))
return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
}
defer rows.Close()
recovered := make([]recoveredDesktopTaskLease, 0)
for rows.Next() {
var item recoveredDesktopTaskLease
if scanErr := rows.Scan(
&item.TaskID,
&item.WorkspaceID,
&item.Kind,
&item.Status,
&item.ActiveAttempt,
); scanErr != nil {
return response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to parse recovered desktop tasks")
}
recovered = append(recovered, item)
}
if err := rows.Err(); err != nil {
return response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to iterate recovered desktop tasks")
}
for index := range recovered {
item := &recovered[index]
if item.Kind == "monitor" {
if _, execErr := tx.Exec(ctx, `
UPDATE desktop_tasks
SET status = 'queued',
error = $2,
active_attempt_id = NULL,
lease_token_hash = NULL,
lease_expires_at = NULL,
interrupted_at = COALESCE(interrupted_at, NOW()),
interrupt_reason = $3,
enqueued_at = NOW(),
updated_at = NOW()
WHERE desktop_id = $1
`, item.TaskID, monitorErrorJSON, monitorReason); execErr != nil {
s.logWarn("desktop monitor task recovery update failed", execErr, zap.String("task_id", item.TaskID.String()), zap.String("mode", string(mode)))
return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
}
item.Status = "queued"
continue
}
if _, execErr := tx.Exec(ctx, `
UPDATE desktop_tasks
SET status = 'unknown',
error = $2,
active_attempt_id = NULL,
lease_token_hash = NULL,
lease_expires_at = NULL,
interrupted_at = COALESCE(interrupted_at, NOW()),
interrupt_reason = COALESCE(interrupt_reason, $3),
updated_at = NOW()
WHERE desktop_id = $1
`, item.TaskID, publishErrorJSON, publishReason); execErr != nil {
s.logWarn("desktop publish task recovery update failed", execErr, zap.String("task_id", item.TaskID.String()), zap.String("mode", string(mode)))
return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
}
item.Status = "unknown"
}
for _, item := range recovered {
if !item.ActiveAttempt.Valid {
continue
}
attemptID, convErr := uuid.FromBytes(item.ActiveAttempt.Bytes[:])
if convErr != nil {
s.logWarn("desktop recovery attempt id decode failed", convErr, zap.String("task_id", item.TaskID.String()))
continue
}
finalStatus := literalStringPtr(item.Status)
errorJSON := publishErrorJSON
if item.Kind == "monitor" {
finalStatus = literalStringPtr("aborted")
errorJSON = monitorErrorJSON
}
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
TaskID: item.TaskID,
AttemptID: attemptID,
FinalStatus: finalStatus,
Error: errorJSON,
}); finishErr != nil {
s.logWarn("desktop recovery attempt finish failed", finishErr, zap.String("task_id", item.TaskID.String()))
}
}
if err := tx.Commit(ctx); err != nil {
return response.ErrInternal(50199, "desktop_task_recovery_commit_failed", "failed to commit desktop task recovery")
}
for _, item := range recovered {
task, getErr := s.repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID)
if getErr != nil {
s.logWarn("desktop recovery reload failed", getErr, zap.String("task_id", item.TaskID.String()))
continue
}
eventType := "task_completed"
if item.Kind == "monitor" {
eventType = "task_available"
}
s.publishTaskEvent(ctx, task, eventType)
}
if len(recovered) > 0 && s.logger != nil {
s.logger.Info("desktop tasks recovered",
zap.String("client_id", client.ID.String()),
zap.String("mode", string(mode)),
zap.Int("count", len(recovered)),
zap.String("publish_message", publishMessage),
)
}
return nil
}
func desktopTaskRecoveryPayload(mode desktopTaskRecoveryMode, kind string) ([]byte, string, string, error) {
reason := strings.TrimSpace(string(mode))
message := "desktop task was recovered after the client lost the active lease"
source := "desktop_task_recovery"
switch mode {
case desktopTaskRecoveryModeStartup:
source = "desktop_client_startup"
if kind == "monitor" {
message = "desktop client restarted while the monitor task was in progress; task has been re-queued"
} else {
message = "desktop client restarted while the publish task was in progress; task has been moved to unknown for manual reconcile"
}
case desktopTaskRecoveryModeDisconnect:
source = "desktop_client_offline"
if kind == "monitor" {
message = "desktop client went offline while the monitor task was in progress; task has been re-queued"
} else {
message = "desktop client went offline while the publish task was in progress; task has been moved to unknown for manual reconcile"
}
case desktopTaskRecoveryModeLeaseExpiry:
source = "desktop_task_lease_expiry"
if kind == "monitor" {
message = "desktop task lease expired before monitor completion; task has been re-queued"
} else {
message = "desktop task lease expired before publish completion; task has been moved to unknown for manual reconcile"
}
}
payload, err := marshalOptionalJSON(map[string]any{
"reason": reason,
"message": message,
"source": source,
})
if err != nil {
return nil, "", "", err
}
return payload, reason, message, nil
}
func (s *DesktopTaskService) dropStaleMonitorTasksForClient(ctx context.Context, client *repository.DesktopClient) error {
if s == nil || client == nil || s.pool == nil || s.monitoringPool == nil {
return nil
}
_, currentBusinessDate := monitoringBusinessDayAndDateAt(time.Now())
errorJSON, err := marshalOptionalJSON(map[string]any{
"reason": monitoringStaleTaskDropReason,
"message": monitoringStaleTaskDropError,
"current_business_date": currentBusinessDate,
"source": "business_day_rollover",
})
if err != nil {
return response.ErrInternal(50190, "desktop_task_stale_payload_failed", "failed to encode stale monitor desktop task payload")
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return response.ErrInternal(50191, "desktop_task_stale_begin_failed", "failed to start stale monitor desktop cleanup")
}
defer tx.Rollback(ctx)
rows, err := tx.Query(ctx, `
WITH stale AS (
SELECT
desktop_id,
active_attempt_id
FROM desktop_tasks
WHERE target_client_id = $1
AND kind = 'monitor'
AND status IN ('queued', 'in_progress', 'unknown')
AND COALESCE(
NULLIF(payload ->> 'business_date', ''),
NULLIF(payload ->> 'businessDate', ''),
NULLIF(payload ->> 'metric_date', ''),
NULLIF(payload ->> 'date', '')
) < $2
FOR UPDATE
),
updated AS (
UPDATE desktop_tasks AS t
SET status = 'aborted',
error = $3,
active_attempt_id = NULL,
lease_token_hash = NULL,
lease_expires_at = NULL,
interrupted_at = COALESCE(interrupted_at, NOW()),
interrupt_reason = $4,
updated_at = NOW()
FROM stale
WHERE t.desktop_id = stale.desktop_id
RETURNING t.desktop_id
)
SELECT stale.desktop_id, stale.active_attempt_id
FROM stale
INNER JOIN updated ON updated.desktop_id = stale.desktop_id
`, client.ID, currentBusinessDate, errorJSON, monitoringStaleTaskDropReason)
if err != nil {
return response.ErrInternal(50192, "desktop_task_stale_query_failed", "failed to select stale monitor desktop tasks")
}
defer rows.Close()
repo := repository.NewDesktopTaskRepository(tx)
droppedTaskIDs := make([]uuid.UUID, 0)
for rows.Next() {
var (
taskID uuid.UUID
attemptID pgtype.UUID
)
if scanErr := rows.Scan(&taskID, &attemptID); scanErr != nil {
return response.ErrInternal(50193, "desktop_task_stale_scan_failed", "failed to parse stale monitor desktop tasks")
}
droppedTaskIDs = append(droppedTaskIDs, taskID)
if !attemptID.Valid {
continue
}
resolvedAttemptID, convErr := uuid.FromBytes(attemptID.Bytes[:])
if convErr != nil {
s.logWarn("desktop stale monitor attempt id decode failed", convErr, zap.String("task_id", taskID.String()))
continue
}
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
TaskID: taskID,
AttemptID: resolvedAttemptID,
FinalStatus: literalStringPtr("aborted"),
Error: errorJSON,
}); finishErr != nil {
s.logWarn("desktop stale monitor attempt finish failed", finishErr, zap.String("task_id", taskID.String()))
}
}
if err := rows.Err(); err != nil {
return response.ErrInternal(50193, "desktop_task_stale_scan_failed", "failed to iterate stale monitor desktop tasks")
}
if err := tx.Commit(ctx); err != nil {
return response.ErrInternal(50194, "desktop_task_stale_commit_failed", "failed to commit stale monitor desktop cleanup")
}
for _, taskID := range droppedTaskIDs {
task, getErr := s.repo.GetByDesktopID(ctx, taskID, client.WorkspaceID)
if getErr != nil {
s.logWarn("desktop stale monitor reload failed", getErr, zap.String("task_id", taskID.String()))
continue
}
s.publishTaskEvent(ctx, task, "task_canceled")
}
if _, err := s.monitoringPool.Exec(ctx, `
UPDATE monitoring_collect_tasks
SET status = 'skipped',
lease_token_hash = NULL,
leased_to_executor = NULL,
leased_at = NULL,
lease_expires_at = NULL,
callback_received_at = COALESCE(callback_received_at, NOW()),
completed_at = COALESCE(completed_at, NOW()),
skip_reason = $4,
error_message = COALESCE(NULLIF(error_message, ''), $5),
updated_at = NOW()
WHERE tenant_id = $1
AND collector_type = $2
AND COALESCE(execution_owner, 'legacy') = 'desktop_tasks'
AND target_client_id = $3
AND business_date < $6::date
AND callback_received_at IS NULL
AND status IN ('pending', 'leased', 'expired')
`, client.TenantID, monitoringCollectorType, client.ID, monitoringStaleTaskDropReason, monitoringStaleTaskDropError, currentBusinessDate); err != nil {
return response.ErrInternal(50041, "task_skip_failed", "failed to drop stale phase2 monitoring tasks")
}
return nil
}
func activeAttemptID(task *repository.DesktopTask) *uuid.UUID {
if task == nil || task.ActiveAttemptID == nil {
return nil
}
value := *task.ActiveAttemptID
return &value
}
func shouldRequeuePreemptedMonitorTask(task *repository.DesktopTask, reason *string) bool {
if task == nil || task.Kind != "monitor" {
return false
}
return strings.EqualFold(strings.TrimSpace(optionalStringValue(reason)), "collect_now_preempt")
}
func requeuePreemptedMonitorTask(
ctx context.Context,
tx pgx.Tx,
repo repository.DesktopTaskRepository,
task *repository.DesktopTask,
) (*repository.DesktopTask, error) {
if task == nil {
return nil, nil
}
replacementID := uuid.New()
replacementJobID := uuid.New()
if _, err := tx.Exec(ctx, `
INSERT INTO desktop_tasks (
desktop_id,
job_id,
tenant_id,
workspace_id,
target_account_id,
target_client_id,
platform_id,
kind,
payload,
status,
priority,
lane,
lane_weight,
source,
scheduler_group_key,
monitor_task_id,
supersedes_task_id,
control_flags,
interrupt_generation,
enqueued_at
)
VALUES (
$1,
$2,
$3,
$4,
$5,
$6,
$7,
'monitor',
$8,
'queued',
50,
'retry',
20,
'retry_recovery',
$9,
$10,
$11,
jsonb_build_object(
'interrupt_requested', false,
'requeued_at', NOW()::text,
'requeued_from_task_id', $11::text,
'requeued_reason', 'collect_now_preempt'
),
$12,
NOW()
)
`, replacementID, replacementJobID, task.TenantID, task.WorkspaceID, task.TargetAccountID, task.TargetClientID, task.Platform, task.Payload, nullableString(task.SchedulerGroupKey), nullableInt64(task.MonitorTaskID), task.DesktopID, task.InterruptGeneration); err != nil {
return nil, response.ErrInternal(50111, "desktop_task_requeue_failed", "failed to create replacement retry monitor desktop task")
}
next, err := repo.GetByDesktopID(ctx, replacementID, task.WorkspaceID)
if err != nil {
return nil, response.ErrInternal(50112, "desktop_task_lookup_failed", "failed to reload replacement retry monitor desktop task")
}
return next, nil
}
func (s *DesktopTaskService) publishTaskEvent(ctx context.Context, task *repository.DesktopTask, eventType string) {
publishDesktopTaskLifecycleEvent(ctx, s.messaging, s.logger, task, eventType)
}
func (s *DesktopTaskService) logWarn(message string, err error, fields ...zap.Field) {
if s.logger == nil || err == nil {
return
}
fields = append(fields, zap.Error(err))
s.logger.Warn(message, fields...)
}