Files
geo/server/internal/tenant/app/desktop_task_service.go
T
root a5d89f0c48 fix(publish): cancel queued publish tasks when the article is unavailable
Stale publish jobs for deleted or invalid articles previously lingered in the
queue and kept failing compliance rechecks. Now they are cancelled cleanly and
batch/article publish status is recalculated.

- cancel queued desktop tasks, jobs and publish records on article delete
- detect deleted articles and invalid-article-version compliance errors during
  the client publish recheck and cancel the affected tasks
- add cancelQueuedPublishTasksForUnavailableArticle helper and a unit test for
  the compliance-error classifier

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-14 21:03:00 +08:00

2283 lines
75 KiB
Go

package app
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
goredis "github.com/redis/go-redis/v9"
"go.uber.org/zap"
"github.com/geo-platform/tenant-api/internal/shared/auth"
sharedcache "github.com/geo-platform/tenant-api/internal/shared/cache"
sharedcompliance "github.com/geo-platform/tenant-api/internal/shared/compliance"
"github.com/geo-platform/tenant-api/internal/shared/config"
"github.com/geo-platform/tenant-api/internal/shared/messaging/rabbitmq"
"github.com/geo-platform/tenant-api/internal/shared/response"
tenantcompliance "github.com/geo-platform/tenant-api/internal/tenant/compliance"
"github.com/geo-platform/tenant-api/internal/tenant/repository"
)
const desktopPublishMaxAttempts = 3
type DesktopTaskService struct {
pool *pgxpool.Pool
monitoringPool *pgxpool.Pool
repo repository.DesktopTaskRepository
cache sharedcache.Cache
redis *goredis.Client
messaging *rabbitmq.Client
logger *zap.Logger
compliance *tenantcompliance.Service
}
func NewDesktopTaskService(pool *pgxpool.Pool, monitoringPool *pgxpool.Pool, messaging *rabbitmq.Client, logger *zap.Logger) *DesktopTaskService {
return &DesktopTaskService{
pool: pool,
monitoringPool: monitoringPool,
repo: repository.NewDesktopTaskRepository(pool),
messaging: messaging,
logger: logger,
compliance: tenantcompliance.NewService(pool, config.NewStaticProvider(&config.Config{
Compliance: config.ComplianceConfig{Enabled: true},
}), logger).WithRabbitMQ(messaging),
}
}
func NewDesktopTaskServiceWithConfig(pool *pgxpool.Pool, monitoringPool *pgxpool.Pool, messaging *rabbitmq.Client, logger *zap.Logger, cfg config.Provider) *DesktopTaskService {
svc := NewDesktopTaskService(pool, monitoringPool, messaging, logger)
svc.compliance = tenantcompliance.NewService(pool, cfg, logger).WithRabbitMQ(messaging)
return svc
}
func (s *DesktopTaskService) WithCache(c sharedcache.Cache) *DesktopTaskService {
s.cache = c
return s
}
func (s *DesktopTaskService) WithRedis(redis *goredis.Client) *DesktopTaskService {
if s != nil {
s.redis = redis
}
return s
}
type DesktopTaskView struct {
ID string `json:"id"`
JobID string `json:"job_id"`
TenantID int64 `json:"tenant_id"`
WorkspaceID int64 `json:"workspace_id"`
TargetAccountID string `json:"target_account_id"`
TargetClientID string `json:"target_client_id"`
Platform string `json:"platform"`
Kind string `json:"kind"`
Payload json.RawMessage `json:"payload"`
Status string `json:"status"`
PublishJobStatus *string `json:"publish_job_status,omitempty"`
ComplianceBlockedRecordID *int64 `json:"compliance_blocked_record_id,omitempty"`
ComplianceBlockedAt *time.Time `json:"compliance_blocked_at,omitempty"`
ComplianceBlockedReason *string `json:"compliance_blocked_reason,omitempty"`
DedupKey *string `json:"dedup_key"`
ActiveAttemptID *string `json:"active_attempt_id"`
LeaseExpiresAt *time.Time `json:"lease_expires_at"`
Attempts int `json:"attempts"`
Result json.RawMessage `json:"result"`
Error json.RawMessage `json:"error"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
type LeaseDesktopTaskRequest struct {
TaskID *string `json:"task_id"`
Kind *string `json:"kind" binding:"omitempty,oneof=publish monitor"`
}
type LeaseDesktopTaskResponse struct {
Task *DesktopTaskView `json:"task"`
AttemptID *string `json:"attempt_id,omitempty"`
LeaseToken *string `json:"lease_token,omitempty"`
LeaseExpiresAt *time.Time `json:"lease_expires_at,omitempty"`
}
type ExtendDesktopTaskRequest struct {
LeaseToken string `json:"lease_token" binding:"required"`
}
type MarkPublishSubmitStartedRequest struct {
LeaseToken string `json:"lease_token" binding:"required"`
}
type CompleteDesktopTaskRequest struct {
LeaseToken string `json:"lease_token" binding:"required"`
Status string `json:"status" binding:"required,oneof=succeeded failed unknown"`
Payload map[string]any `json:"payload"`
Error map[string]any `json:"error"`
}
type CancelDesktopTaskRequest struct {
LeaseToken *string `json:"lease_token"`
Reason *string `json:"reason"`
}
type ReconcileDesktopTaskRequest struct {
Status string `json:"status" binding:"required,oneof=succeeded failed aborted retry"`
Result map[string]any `json:"result"`
Error map[string]any `json:"error"`
}
type ListPublishTasksRequest struct {
Page int `form:"page"`
PageSize int `form:"page_size"`
Title string `form:"title"`
}
type DesktopPublishTaskList struct {
Items []DesktopTaskView `json:"items"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Total int `json:"total"`
PendingCount int `json:"pending_count"`
HistoryTotal int `json:"history_total"`
}
type publishTaskListOwner struct {
TenantID int64
WorkspaceID int64
UserID int64
}
func (s *DesktopTaskService) Lease(ctx context.Context, client *repository.DesktopClient, req LeaseDesktopTaskRequest, routeTaskID *uuid.UUID) (*LeaseDesktopTaskResponse, error) {
if client == nil {
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
}
markDesktopTaskConsumerPresent(ctx, s.redis, client.ID)
taskID := routeTaskID
if taskID == nil && req.TaskID != nil && strings.TrimSpace(*req.TaskID) != "" {
parsed, err := uuid.Parse(strings.TrimSpace(*req.TaskID))
if err != nil {
return nil, response.ErrBadRequest(40084, "invalid_desktop_task_id", "task id must be a uuid")
}
taskID = &parsed
}
if err := s.dropStaleMonitorTasksForClient(ctx, client); err != nil {
return nil, err
}
if err := s.recoverExpiredClientTasks(ctx, client); err != nil {
return nil, err
}
if err := s.recheckQueuedPublishTasksForClient(ctx, client); err != nil {
return nil, err
}
rawToken, tokenHash, err := newDesktopClientToken()
if err != nil {
return nil, response.ErrInternal(50087, "desktop_task_lease_token_failed", "failed to generate desktop task lease token")
}
attemptID := uuid.New()
leaseParams := repository.DesktopTaskLeaseParams{
AttemptID: attemptID,
LeaseTokenHash: tokenHash,
}
if s.logger != nil {
fields := []zap.Field{
zap.String("client_id", client.ID.String()),
}
if req.Kind != nil {
fields = append(fields, zap.String("requested_kind", strings.TrimSpace(*req.Kind)))
}
if taskID != nil {
fields = append(fields, zap.String("task_id", taskID.String()))
}
s.logger.Debug("desktop task lease requested", fields...)
}
var task *repository.DesktopTask
switch {
case taskID != nil:
task, err = s.leaseQueuedDesktopTaskByID(ctx, client, *taskID, leaseParams)
case req.Kind != nil && strings.TrimSpace(*req.Kind) == "monitor":
task, err = s.leaseNextQueuedMonitorTask(ctx, client, leaseParams)
case req.Kind != nil && strings.TrimSpace(*req.Kind) == "publish":
task, err = s.leaseNextQueuedPublishTask(ctx, client, leaseParams)
default:
task, err = s.leaseNextQueuedPublishTask(ctx, client, leaseParams)
}
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
if s.logger != nil {
fields := []zap.Field{
zap.String("client_id", client.ID.String()),
}
if req.Kind != nil {
fields = append(fields, zap.String("requested_kind", strings.TrimSpace(*req.Kind)))
}
if taskID != nil {
fields = append(fields, zap.String("task_id", taskID.String()))
}
s.logger.Debug("desktop task lease found no matching queued task", fields...)
}
if taskID == nil {
return &LeaseDesktopTaskResponse{}, nil
}
return nil, s.classifyLeaseError(ctx, client, taskID)
}
fields := []zap.Field{
zap.String("client_id", client.ID.String()),
}
if req.Kind != nil {
fields = append(fields, zap.String("requested_kind", strings.TrimSpace(*req.Kind)))
}
if taskID != nil {
fields = append(fields, zap.String("task_id", taskID.String()))
}
s.logWarn("desktop task lease query failed", err, fields...)
return nil, response.ErrInternal(50088, "desktop_task_lease_failed", "failed to lease desktop task")
}
if _, attemptErr := s.repo.CreateAttempt(ctx, repository.CreateDesktopTaskAttemptParams{
DesktopID: attemptID,
TaskID: task.DesktopID,
ClientID: client.ID,
LeaseTokenHash: tokenHash,
}); attemptErr != nil {
s.logWarn("desktop task attempt insert failed", attemptErr, zap.String("task_id", task.DesktopID.String()))
}
s.publishTaskEvent(ctx, task, "task_leased")
if s.logger != nil {
s.logger.Info("desktop task leased",
zap.String("client_id", client.ID.String()),
zap.String("task_id", task.DesktopID.String()),
zap.String("kind", task.Kind),
zap.String("status", task.Status),
)
}
view := buildDesktopTaskView(task)
attemptIDText := attemptID.String()
return &LeaseDesktopTaskResponse{
Task: &view,
AttemptID: &attemptIDText,
LeaseToken: &rawToken,
LeaseExpiresAt: task.LeaseExpiresAt,
}, nil
}
func (s *DesktopTaskService) recheckQueuedPublishTasksForClient(ctx context.Context, client *repository.DesktopClient) error {
if s == nil || s.pool == nil || s.compliance == nil || client == nil {
return nil
}
rows, err := s.pool.Query(ctx, `
SELECT
j.desktop_id,
j.tenant_id,
j.workspace_id,
j.created_by_user_id,
j.article_id,
j.article_version_id,
COALESCE(array_agg(DISTINCT dt.platform_id) FILTER (WHERE dt.platform_id <> ''), '{}') AS platforms,
BOOL_OR(a.deleted_at IS NOT NULL) AS article_deleted
FROM desktop_tasks dt
JOIN desktop_publish_jobs j ON j.desktop_id = dt.job_id
LEFT JOIN articles a ON a.id = j.article_id AND a.tenant_id = j.tenant_id
WHERE dt.target_client_id = $1
AND dt.kind = 'publish'
AND dt.status = 'queued'
AND j.status = 'queued'
AND j.article_id IS NOT NULL
AND j.article_version_id IS NOT NULL
GROUP BY j.desktop_id, j.tenant_id, j.workspace_id, j.created_by_user_id, j.article_id, j.article_version_id
ORDER BY MIN(dt.created_at)
LIMIT 5
`, client.ID)
if err != nil {
s.logWarn("desktop publish compliance recheck query failed", err, zap.String("client_id", client.ID.String()))
return response.ErrInternal(50118, "desktop_publish_compliance_recheck_failed", "failed to load publish jobs for compliance recheck")
}
defer rows.Close()
type candidate struct {
JobID uuid.UUID
TenantID int64
WorkspaceID int64
CreatedByUserID int64
ArticleID int64
ArticleVersionID int64
Platforms []string
ArticleDeleted bool
}
candidates := make([]candidate, 0)
for rows.Next() {
var item candidate
if scanErr := rows.Scan(&item.JobID, &item.TenantID, &item.WorkspaceID, &item.CreatedByUserID, &item.ArticleID, &item.ArticleVersionID, &item.Platforms, &item.ArticleDeleted); scanErr != nil {
s.logWarn("desktop publish compliance recheck scan failed", scanErr, zap.String("client_id", client.ID.String()))
return response.ErrInternal(50118, "desktop_publish_compliance_recheck_failed", "failed to scan publish job for compliance recheck")
}
if item.WorkspaceID == client.WorkspaceID && item.TenantID == client.TenantID {
candidates = append(candidates, item)
}
}
if err := rows.Err(); err != nil {
s.logWarn("desktop publish compliance recheck rows failed", err, zap.String("client_id", client.ID.String()))
return response.ErrInternal(50118, "desktop_publish_compliance_recheck_failed", "failed to iterate publish jobs for compliance recheck")
}
for _, item := range candidates {
if item.ArticleDeleted {
if cancelErr := s.cancelUnavailableArticleQueuedPublishTasks(ctx, item.TenantID, item.ArticleID); cancelErr != nil {
return cancelErr
}
continue
}
gate, gateErr := s.compliance.GateForPublish(ctx, tenantcompliance.GateForPublishRequest{
TenantID: item.TenantID,
ArticleID: item.ArticleID,
ArticleVersionID: item.ArticleVersionID,
ActorID: item.CreatedByUserID,
TargetPlatforms: item.Platforms,
TriggerSource: "scheduler_recheck",
})
if gateErr != nil && gate != nil && gate.Decision == sharedcompliance.GateDecisionBlock {
if markErr := s.compliance.MarkPublishJobBlockedByCompliance(ctx, item.JobID, gate.Result); markErr != nil {
return markErr
}
continue
}
if gateErr != nil {
if isComplianceInvalidArticleVersionError(gateErr) {
if cancelErr := s.cancelUnavailableArticleQueuedPublishTasks(ctx, item.TenantID, item.ArticleID); cancelErr != nil {
return cancelErr
}
continue
}
return response.ErrServiceUnavailable(41004, "compliance_engine_unavailable", "scheduled publish compliance recheck failed")
}
if gate != nil && gate.Decision == sharedcompliance.GateDecisionBlock {
if markErr := s.compliance.MarkPublishJobBlockedByCompliance(ctx, item.JobID, gate.Result); markErr != nil {
return markErr
}
}
}
return nil
}
func (s *DesktopTaskService) cancelUnavailableArticleQueuedPublishTasks(ctx context.Context, tenantID, articleID int64) error {
if s == nil || s.pool == nil || tenantID <= 0 || articleID <= 0 {
return nil
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return response.ErrInternal(50122, "publish_task_cancel_unavailable_article_begin_failed", "failed to start unavailable article publish task cleanup")
}
defer tx.Rollback(ctx)
taskRefs, err := queuedPublishTaskRefsForArticle(ctx, tx, tenantID, articleID)
if err != nil {
return err
}
if err := cancelQueuedPublishTasksForUnavailableArticle(ctx, tx, tenantID, articleID); err != nil {
return err
}
if err := tx.Commit(ctx); err != nil {
return response.ErrInternal(50123, "publish_task_cancel_unavailable_article_commit_failed", "failed to commit unavailable article publish task cleanup")
}
for _, ref := range taskRefs {
task, getErr := s.repo.GetByDesktopID(ctx, ref.TaskID, ref.WorkspaceID)
if getErr == nil && task != nil {
s.publishTaskEvent(ctx, task, "task_canceled")
}
}
return nil
}
type desktopTaskRef struct {
TaskID uuid.UUID
WorkspaceID int64
}
func queuedPublishTaskRefsForArticle(ctx context.Context, tx pgx.Tx, tenantID, articleID int64) ([]desktopTaskRef, error) {
if tx == nil || tenantID <= 0 || articleID <= 0 {
return nil, nil
}
rows, err := tx.Query(ctx, `
SELECT dt.desktop_id, dt.workspace_id
FROM desktop_tasks dt
JOIN desktop_publish_jobs j
ON j.desktop_id = dt.job_id
AND j.tenant_id = dt.tenant_id
WHERE dt.tenant_id = $1
AND dt.kind = 'publish'
AND dt.status = 'queued'
AND j.article_id = $2
AND j.status = 'queued'
ORDER BY dt.created_at ASC, dt.desktop_id ASC
`, tenantID, articleID)
if err != nil {
return nil, response.ErrInternal(50124, "publish_task_cancel_unavailable_article_lookup_failed", "failed to inspect unavailable article publish tasks")
}
defer rows.Close()
taskRefs := make([]desktopTaskRef, 0)
for rows.Next() {
var ref desktopTaskRef
if scanErr := rows.Scan(&ref.TaskID, &ref.WorkspaceID); scanErr != nil {
return nil, response.ErrInternal(50124, "publish_task_cancel_unavailable_article_lookup_failed", "failed to scan unavailable article publish tasks")
}
taskRefs = append(taskRefs, ref)
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50124, "publish_task_cancel_unavailable_article_lookup_failed", "failed to iterate unavailable article publish tasks")
}
return taskRefs, nil
}
func isComplianceInvalidArticleVersionError(err error) bool {
appErr, ok := err.(*response.AppError)
return ok && appErr.Code == 41005 && appErr.Message == "invalid_article_version"
}
const desktopTaskRepositoryReturningColumns = `
t.desktop_id,
t.job_id,
t.tenant_id,
t.workspace_id,
t.target_account_id,
t.target_client_id,
t.platform_id,
t.kind,
t.payload,
t.status,
t.priority,
t.lane,
t.lane_weight,
t.source,
t.scheduler_group_key,
t.monitor_task_id,
t.supersedes_task_id,
t.control_flags,
t.interrupt_generation,
t.dedup_key,
t.active_attempt_id,
t.lease_expires_at,
t.attempts,
t.result,
t.error,
t.started_at,
t.interrupted_at,
t.interrupt_reason,
t.enqueued_at,
t.created_at,
t.updated_at`
type desktopTaskRepositoryScanner interface {
Scan(dest ...any) error
}
func (s *DesktopTaskService) leaseNextQueuedMonitorTask(
ctx context.Context,
client *repository.DesktopClient,
params repository.DesktopTaskLeaseParams,
) (*repository.DesktopTask, error) {
accountIDs, err := s.loadMonitorLeaseAccountIDs(ctx, client)
if err != nil {
return nil, err
}
hasAccountIDs := len(accountIDs) > 0
row := s.pool.QueryRow(ctx, `
WITH candidate AS (
SELECT dt.desktop_id
FROM desktop_tasks AS dt
WHERE dt.tenant_id = $1
AND dt.workspace_id = $2
AND dt.kind = 'monitor'
AND dt.status = 'queued'
AND (
dt.target_client_id = $3
OR ($4::boolean AND dt.target_account_id = ANY($5::uuid[]))
)
ORDER BY dt.lane_weight DESC,
dt.priority DESC,
COALESCE(dt.enqueued_at, dt.created_at) ASC,
dt.created_at ASC,
dt.desktop_id ASC
LIMIT 1
FOR UPDATE SKIP LOCKED
)
UPDATE desktop_tasks AS t
SET target_client_id = $3,
active_attempt_id = $6,
lease_token_hash = $7,
lease_expires_at = now() + interval '10 minutes',
status = 'in_progress',
attempts = t.attempts + 1,
updated_at = now()
FROM candidate
WHERE t.desktop_id = candidate.desktop_id
RETURNING `+desktopTaskRepositoryReturningColumns,
client.TenantID,
client.WorkspaceID,
client.ID,
hasAccountIDs,
accountIDs,
params.AttemptID,
params.LeaseTokenHash,
)
return scanRepositoryDesktopTask(row)
}
func (s *DesktopTaskService) leaseNextQueuedPublishTask(
ctx context.Context,
client *repository.DesktopClient,
params repository.DesktopTaskLeaseParams,
) (*repository.DesktopTask, error) {
row := s.pool.QueryRow(ctx, `
WITH candidate AS (
SELECT dt.desktop_id
FROM desktop_tasks AS dt
WHERE dt.target_client_id = $1
AND dt.kind = 'publish'
AND dt.status = 'queued'
AND dt.attempts < $4
AND (
NOT EXISTS (
SELECT 1
FROM desktop_publish_jobs AS j
WHERE j.desktop_id = dt.job_id
AND j.status <> 'queued'
)
)
ORDER BY dt.lane_weight DESC,
dt.priority DESC,
COALESCE(dt.enqueued_at, dt.created_at) ASC,
dt.created_at ASC,
dt.desktop_id ASC
LIMIT 1
FOR UPDATE OF dt SKIP LOCKED
)
UPDATE desktop_tasks AS t
SET active_attempt_id = $2,
lease_token_hash = $3,
lease_expires_at = now() + interval '3 minutes',
status = 'in_progress',
attempts = t.attempts + 1,
started_at = COALESCE(t.started_at, now()),
updated_at = now()
FROM candidate
WHERE t.desktop_id = candidate.desktop_id
RETURNING `+desktopTaskRepositoryReturningColumns,
client.ID,
params.AttemptID,
params.LeaseTokenHash,
desktopPublishMaxAttempts,
)
return scanRepositoryDesktopTask(row)
}
func (s *DesktopTaskService) leaseQueuedDesktopTaskByID(
ctx context.Context,
client *repository.DesktopClient,
desktopID uuid.UUID,
params repository.DesktopTaskLeaseParams,
) (*repository.DesktopTask, error) {
accountIDs, err := s.loadMonitorLeaseAccountIDs(ctx, client)
if err != nil {
return nil, err
}
hasAccountIDs := len(accountIDs) > 0
row := s.pool.QueryRow(ctx, `
WITH candidate AS (
SELECT dt.desktop_id
FROM desktop_tasks AS dt
WHERE dt.desktop_id = $1
AND dt.workspace_id = $2
AND dt.status = 'queued'
AND (dt.kind <> 'publish' OR dt.attempts < $9)
AND (
(
dt.kind = 'monitor'
AND dt.tenant_id = $3
AND (
dt.target_client_id = $4
OR ($5::boolean AND dt.target_account_id = ANY($6::uuid[]))
)
)
OR (
dt.kind <> 'monitor'
AND dt.target_client_id = $4
)
)
LIMIT 1
FOR UPDATE SKIP LOCKED
)
UPDATE desktop_tasks AS t
SET target_client_id = CASE WHEN t.kind = 'monitor' THEN $4 ELSE t.target_client_id END,
active_attempt_id = $7,
lease_token_hash = $8,
lease_expires_at = now() + CASE WHEN t.kind = 'publish' THEN interval '3 minutes' ELSE interval '10 minutes' END,
status = 'in_progress',
attempts = t.attempts + 1,
started_at = COALESCE(t.started_at, now()),
updated_at = now()
FROM candidate
WHERE t.desktop_id = candidate.desktop_id
RETURNING `+desktopTaskRepositoryReturningColumns,
desktopID,
client.WorkspaceID,
client.TenantID,
client.ID,
hasAccountIDs,
accountIDs,
params.AttemptID,
params.LeaseTokenHash,
desktopPublishMaxAttempts,
)
return scanRepositoryDesktopTask(row)
}
func (s *DesktopTaskService) loadMonitorLeaseAccountIDs(ctx context.Context, client *repository.DesktopClient) ([]uuid.UUID, error) {
if s == nil || client == nil || s.pool == nil {
return nil, nil
}
accountIDs := loadTrackedDesktopClientAccountIDs(ctx, s.redis, client.ID)
rows, err := s.pool.Query(ctx, `
SELECT desktop_id
FROM platform_accounts
WHERE tenant_id = $1
AND workspace_id = $2
AND client_id = $3
AND deleted_at IS NULL
AND platform_id = ANY($4::text[])
`, client.TenantID, client.WorkspaceID, client.ID, monitoringPlatformIDs(defaultMonitoringPlatforms))
if err != nil {
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to resolve monitor lease accounts")
}
defer rows.Close()
for rows.Next() {
var accountID uuid.UUID
if scanErr := rows.Scan(&accountID); scanErr != nil {
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to parse monitor lease accounts")
}
accountIDs = append(accountIDs, accountID)
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to iterate monitor lease accounts")
}
return uniqueUUIDs(accountIDs), nil
}
func scanRepositoryDesktopTask(row desktopTaskRepositoryScanner) (*repository.DesktopTask, error) {
var (
task repository.DesktopTask
schedulerGroupKey pgtype.Text
monitorTaskID pgtype.Int8
supersedesTaskID pgtype.UUID
dedupKey pgtype.Text
activeAttemptID pgtype.UUID
leaseExpiresAt pgtype.Timestamptz
startedAt pgtype.Timestamptz
interruptedAt pgtype.Timestamptz
interruptReason pgtype.Text
enqueuedAt pgtype.Timestamptz
)
if err := row.Scan(
&task.DesktopID,
&task.JobID,
&task.TenantID,
&task.WorkspaceID,
&task.TargetAccountID,
&task.TargetClientID,
&task.Platform,
&task.Kind,
&task.Payload,
&task.Status,
&task.Priority,
&task.Lane,
&task.LaneWeight,
&task.Source,
&schedulerGroupKey,
&monitorTaskID,
&supersedesTaskID,
&task.ControlFlags,
&task.InterruptGeneration,
&dedupKey,
&activeAttemptID,
&leaseExpiresAt,
&task.Attempts,
&task.Result,
&task.Error,
&startedAt,
&interruptedAt,
&interruptReason,
&enqueuedAt,
&task.CreatedAt,
&task.UpdatedAt,
); err != nil {
return nil, err
}
task.SchedulerGroupKey = desktopTaskNullableText(schedulerGroupKey)
task.MonitorTaskID = desktopTaskNullableInt64(monitorTaskID)
task.SupersedesTaskID = desktopTaskNullableUUID(supersedesTaskID)
task.DedupKey = desktopTaskNullableText(dedupKey)
task.ActiveAttemptID = desktopTaskNullableUUID(activeAttemptID)
task.LeaseExpiresAt = desktopTaskNullableTime(leaseExpiresAt)
task.StartedAt = desktopTaskNullableTime(startedAt)
task.InterruptedAt = desktopTaskNullableTime(interruptedAt)
task.InterruptReason = desktopTaskNullableText(interruptReason)
if enqueuedAt.Valid {
task.EnqueuedAt = enqueuedAt.Time
}
return &task, nil
}
func desktopTaskNullableText(value pgtype.Text) *string {
if !value.Valid {
return nil
}
text := value.String
return &text
}
func desktopTaskNullableInt64(value pgtype.Int8) *int64 {
if !value.Valid {
return nil
}
number := value.Int64
return &number
}
func desktopTaskNullableUUID(value pgtype.UUID) *uuid.UUID {
if !value.Valid {
return nil
}
resolved, err := uuid.FromBytes(value.Bytes[:])
if err != nil {
return nil
}
return &resolved
}
func desktopTaskNullableTime(value pgtype.Timestamptz) *time.Time {
if !value.Valid {
return nil
}
timestamp := value.Time
return &timestamp
}
func (s *DesktopTaskService) Extend(ctx context.Context, client *repository.DesktopClient, desktopID uuid.UUID, req ExtendDesktopTaskRequest) (*DesktopTaskView, error) {
if client == nil {
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
}
if err := s.dropStaleMonitorTasksForClient(ctx, client); err != nil {
return nil, err
}
if s.redis != nil {
presence := loadDesktopClientPresence(ctx, s.redis, []uuid.UUID{client.ID})
if presence != nil && !presence[client.ID] {
return nil, response.ErrConflict(40986, "desktop_client_offline", "desktop client presence has expired")
}
}
task, err := s.repo.ExtendLease(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(req.LeaseToken)))
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, response.ErrConflict(40983, "desktop_task_lease_invalid", "desktop task lease token is invalid or expired")
}
return nil, response.ErrInternal(50089, "desktop_task_extend_failed", "failed to extend desktop task lease")
}
s.publishTaskEvent(ctx, task, "task_extended")
view := buildDesktopTaskView(task)
return &view, nil
}
// MarkPublishSubmitStarted records a durable intent marker right before the client performs the
// irreversible platform submit POST. Lease-recovery uses it to avoid silently re-posting a
// non-idempotent article: once submit may have started, recovery routes the task to 'unknown'
// (manual reconcile) instead of auto-requeueing it. The client must receive a
// positive marker write before it performs the platform submit; a stale/lost
// lease is rejected so the client can stop before the irreversible request.
func (s *DesktopTaskService) MarkPublishSubmitStarted(ctx context.Context, client *repository.DesktopClient, taskID uuid.UUID, leaseToken string) error {
if s == nil || s.pool == nil || client == nil {
return nil
}
leaseToken = strings.TrimSpace(leaseToken)
if leaseToken == "" {
return response.ErrBadRequest(40001, "invalid_params", "lease_token is required")
}
tag, err := s.pool.Exec(ctx, `
UPDATE desktop_tasks
SET publish_submit_started_at = COALESCE(publish_submit_started_at, NOW()),
updated_at = NOW()
WHERE desktop_id = $1
AND target_client_id = $2
AND kind = 'publish'
AND status = 'in_progress'
AND lease_token_hash = $3
`, taskID, client.ID, HashDesktopClientToken(leaseToken))
if err != nil {
return response.ErrInternal(50200, "desktop_task_submit_marker_failed", "failed to mark publish submit started")
}
if tag.RowsAffected() == 0 {
return response.ErrConflict(40983, "desktop_task_lease_invalid", "desktop task lease token is invalid or expired")
}
return nil
}
func (s *DesktopTaskService) Complete(ctx context.Context, client *repository.DesktopClient, desktopID uuid.UUID, req CompleteDesktopTaskRequest) (*DesktopTaskView, error) {
if client == nil {
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
}
finalStatus := normalizeDesktopTaskTerminalStatus(req.Status)
resultJSON, err := marshalOptionalJSON(req.Payload)
if err != nil {
return nil, response.ErrBadRequest(40086, "invalid_desktop_task_payload", "payload must be serializable")
}
errorJSON, err := marshalOptionalJSON(req.Error)
if err != nil {
return nil, response.ErrBadRequest(40087, "invalid_desktop_task_error", "error must be serializable")
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return nil, response.ErrInternal(50103, "desktop_task_complete_begin_failed", "failed to start desktop task completion")
}
defer tx.Rollback(ctx)
taskRepo := repository.NewDesktopTaskRepository(tx)
previousTask, _ := taskRepo.GetByDesktopID(ctx, desktopID, client.WorkspaceID)
previousAttemptID := activeAttemptID(previousTask)
task, err := taskRepo.Complete(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(req.LeaseToken)), finalStatus, resultJSON, errorJSON)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, response.ErrConflict(40983, "desktop_task_lease_invalid", "desktop task lease token is invalid or expired")
}
return nil, response.ErrInternal(50091, "desktop_task_complete_failed", "failed to complete desktop task")
}
if previousAttemptID != nil {
if finishErr := taskRepo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
TaskID: task.DesktopID,
AttemptID: *previousAttemptID,
FinalStatus: &finalStatus,
Error: errorJSON,
}); finishErr != nil {
s.logWarn("desktop task attempt finish failed after complete", finishErr, zap.String("task_id", task.DesktopID.String()))
}
}
publishOutcome, err := syncDesktopPublishTaskState(ctx, tx, task)
if err != nil {
return nil, err
}
bulkFailure, err := bulkFailQueuedDesktopTasksForAuthorizationFailure(ctx, tx, task, errorJSON)
if err != nil {
return nil, err
}
if err := tx.Commit(ctx); err != nil {
return nil, response.ErrInternal(50104, "desktop_task_complete_commit_failed", "failed to commit desktop task completion")
}
if publishOutcome != nil {
invalidateArticleCaches(ctx, s.cache, publishOutcome.TenantID, &publishOutcome.ArticleID)
if syncErr := s.syncMonitoringArticleAlias(ctx, publishOutcome.ArticleAlias); syncErr != nil {
s.logWarn("monitoring article alias sync failed after publish complete", syncErr, zap.Int64("article_id", publishOutcome.ArticleID))
}
}
for _, outcome := range bulkFailure.PublishOutcomes {
invalidateArticleCaches(ctx, s.cache, outcome.TenantID, &outcome.ArticleID)
if outcome.ArticleAlias != nil {
if syncErr := s.syncMonitoringArticleAlias(ctx, outcome.ArticleAlias); syncErr != nil {
s.logWarn("monitoring article alias sync failed after bulk authorization failure", syncErr, zap.Int64("article_id", outcome.ArticleID))
}
}
}
s.publishTaskEvent(ctx, task, "task_completed")
for _, failedTask := range bulkFailure.Tasks {
s.publishTaskEvent(ctx, failedTask, "task_completed")
}
view := buildDesktopTaskView(task)
return &view, nil
}
func (s *DesktopTaskService) Cancel(ctx context.Context, client *repository.DesktopClient, desktopID uuid.UUID, req CancelDesktopTaskRequest) (*DesktopTaskView, error) {
if client == nil {
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
}
errorJSON, err := marshalOptionalJSON(map[string]any{
"reason": optionalStringValue(req.Reason),
"source": "desktop_client",
})
if err != nil {
return nil, response.ErrBadRequest(40088, "invalid_desktop_task_cancel", "cancel payload must be serializable")
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return nil, response.ErrInternal(50107, "desktop_task_cancel_begin_failed", "failed to start desktop task cancel")
}
defer tx.Rollback(ctx)
taskRepo := repository.NewDesktopTaskRepository(tx)
previousTask, _ := taskRepo.GetByDesktopID(ctx, desktopID, client.WorkspaceID)
previousAttemptID := activeAttemptID(previousTask)
var task *repository.DesktopTask
if req.LeaseToken != nil && strings.TrimSpace(*req.LeaseToken) != "" {
task, err = taskRepo.CancelByLease(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(*req.LeaseToken)), errorJSON)
} else {
task, err = taskRepo.CancelByClient(ctx, desktopID, client.ID, errorJSON)
}
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, response.ErrConflict(40984, "desktop_task_cancel_conflict", "desktop task cannot be canceled in its current state")
}
return nil, response.ErrInternal(50092, "desktop_task_cancel_failed", "failed to cancel desktop task")
}
if previousAttemptID != nil && req.LeaseToken != nil && strings.TrimSpace(*req.LeaseToken) != "" {
if finishErr := taskRepo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
TaskID: task.DesktopID,
AttemptID: *previousAttemptID,
FinalStatus: literalStringPtr("aborted"),
Error: errorJSON,
}); finishErr != nil {
s.logWarn("desktop task attempt finish failed after cancel", finishErr, zap.String("task_id", task.DesktopID.String()))
}
}
var replacementTask *repository.DesktopTask
if shouldRequeuePreemptedMonitorTask(task, req.Reason) {
replacementTask, err = requeuePreemptedMonitorTask(ctx, tx, taskRepo, task)
if err != nil {
return nil, err
}
}
if err := tx.Commit(ctx); err != nil {
return nil, response.ErrInternal(50108, "desktop_task_cancel_commit_failed", "failed to commit desktop task cancel")
}
s.publishTaskEvent(ctx, task, "task_canceled")
if replacementTask != nil {
s.publishTaskEvent(ctx, replacementTask, "task_available")
}
view := buildDesktopTaskView(task)
return &view, nil
}
func (s *DesktopTaskService) TenantCancel(ctx context.Context, actor auth.Actor, desktopID uuid.UUID, req CancelDesktopTaskRequest) (*DesktopTaskView, error) {
if actor.PrimaryWorkspaceID == 0 {
return nil, response.ErrUnauthorized(40132, "workspace_scope_missing", "workspace context is required")
}
errorJSON, err := marshalOptionalJSON(map[string]any{
"reason": optionalStringValue(req.Reason),
"source": "tenant_web",
})
if err != nil {
return nil, response.ErrBadRequest(40088, "invalid_desktop_task_cancel", "cancel payload must be serializable")
}
task, err := s.repo.TenantCancel(ctx, desktopID, actor.PrimaryWorkspaceID, errorJSON)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, response.ErrConflict(40984, "desktop_task_cancel_conflict", "desktop task cannot be canceled in its current state")
}
return nil, response.ErrInternal(50093, "desktop_task_tenant_cancel_failed", "failed to cancel desktop task")
}
s.publishTaskEvent(ctx, task, "task_canceled")
view := buildDesktopTaskView(task)
return &view, nil
}
func (s *DesktopTaskService) Reconcile(ctx context.Context, actor auth.Actor, desktopID uuid.UUID, req ReconcileDesktopTaskRequest) (*DesktopTaskView, error) {
if actor.PrimaryWorkspaceID == 0 {
return nil, response.ErrUnauthorized(40132, "workspace_scope_missing", "workspace context is required")
}
resultJSON, err := marshalOptionalJSON(req.Result)
if err != nil {
return nil, response.ErrBadRequest(40089, "invalid_desktop_task_reconcile_result", "result must be serializable")
}
errorJSON, err := marshalOptionalJSON(req.Error)
if err != nil {
return nil, response.ErrBadRequest(40090, "invalid_desktop_task_reconcile_error", "error must be serializable")
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return nil, response.ErrInternal(50105, "desktop_task_reconcile_begin_failed", "failed to start desktop task reconcile")
}
defer tx.Rollback(ctx)
taskRepo := repository.NewDesktopTaskRepository(tx)
task, err := taskRepo.Reconcile(ctx, desktopID, actor.PrimaryWorkspaceID, req.Status, resultJSON, errorJSON)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, response.ErrConflict(40985, "desktop_task_reconcile_conflict", "desktop task is not waiting for reconcile")
}
return nil, response.ErrInternal(50094, "desktop_task_reconcile_failed", "failed to reconcile desktop task")
}
publishOutcome, err := syncDesktopPublishTaskState(ctx, tx, task)
if err != nil {
return nil, err
}
if err := tx.Commit(ctx); err != nil {
return nil, response.ErrInternal(50106, "desktop_task_reconcile_commit_failed", "failed to commit desktop task reconcile")
}
if publishOutcome != nil {
invalidateArticleCaches(ctx, s.cache, publishOutcome.TenantID, &publishOutcome.ArticleID)
if syncErr := s.syncMonitoringArticleAlias(ctx, publishOutcome.ArticleAlias); syncErr != nil {
s.logWarn("monitoring article alias sync failed after publish reconcile", syncErr, zap.Int64("article_id", publishOutcome.ArticleID))
}
}
s.publishTaskEvent(ctx, task, reconcileDesktopTaskEventType(task.Kind, req.Status))
view := buildDesktopTaskView(task)
return &view, nil
}
func (s *DesktopTaskService) ListPublishTasks(ctx context.Context, client *repository.DesktopClient, req ListPublishTasksRequest) (*DesktopPublishTaskList, error) {
if client == nil {
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
}
return s.listPublishTasksForOwner(ctx, publishTaskListOwner{
TenantID: client.TenantID,
WorkspaceID: client.WorkspaceID,
UserID: client.UserID,
}, req)
}
func (s *DesktopTaskService) ListTenantPublishTasks(ctx context.Context, actor auth.Actor, req ListPublishTasksRequest) (*DesktopPublishTaskList, error) {
workspaceID := auth.CurrentWorkspaceID(ctx)
if actor.TenantID == 0 || actor.UserID == 0 || workspaceID == 0 {
return nil, response.ErrUnauthorized(40132, "workspace_scope_missing", "workspace context is required")
}
return s.listPublishTasksForOwner(ctx, publishTaskListOwner{
TenantID: actor.TenantID,
WorkspaceID: workspaceID,
UserID: actor.UserID,
}, req)
}
func (s *DesktopTaskService) listPublishTasksForOwner(ctx context.Context, owner publishTaskListOwner, req ListPublishTasksRequest) (*DesktopPublishTaskList, error) {
if s.pool == nil {
return nil, response.ErrServiceUnavailable(50342, "desktop_publish_job_store_unavailable", "desktop publish job store is unavailable")
}
page := req.Page
if page <= 0 {
page = 1
}
pageSize := req.PageSize
if pageSize <= 0 || pageSize > 50 {
pageSize = 10
}
title := strings.TrimSpace(req.Title)
pendingStatuses := []string{"queued", "in_progress"}
historyStatuses := []string{"succeeded", "failed", "unknown", "aborted"}
pendingItems, err := s.listPublishTasksByStatuses(
ctx,
owner,
pendingStatuses,
title,
0,
0,
`CASE WHEN t.status = 'in_progress' THEN 0 ELSE 1 END ASC, t.created_at ASC, t.desktop_id DESC`,
)
if err != nil {
return nil, err
}
historyTotal, err := s.countPublishTasksByStatuses(ctx, owner, historyStatuses, title)
if err != nil {
return nil, err
}
offset := (page - 1) * pageSize
items := make([]DesktopTaskView, 0, pageSize)
total := len(pendingItems) + historyTotal
if offset < len(pendingItems) {
end := offset + pageSize
if end > len(pendingItems) {
end = len(pendingItems)
}
items = append(items, pendingItems[offset:end]...)
}
remaining := pageSize - len(items)
historyOffset := offset - len(pendingItems)
if historyOffset < 0 {
historyOffset = 0
}
if remaining > 0 && historyOffset < historyTotal {
historyItems, listErr := s.listPublishTasksByStatuses(
ctx,
owner,
historyStatuses,
title,
remaining,
historyOffset,
`t.updated_at DESC, t.desktop_id DESC`,
)
if listErr != nil {
return nil, listErr
}
items = append(items, historyItems...)
}
return &DesktopPublishTaskList{
Items: items,
Page: page,
PageSize: pageSize,
Total: total,
PendingCount: len(pendingItems),
HistoryTotal: historyTotal,
}, nil
}
func (s *DesktopTaskService) listPublishTasksByStatuses(
ctx context.Context,
owner publishTaskListOwner,
statuses []string,
title string,
limit int,
offset int,
orderBy string,
) ([]DesktopTaskView, error) {
query := `
SELECT
t.desktop_id,
t.job_id,
t.tenant_id,
t.workspace_id,
t.target_account_id,
t.target_client_id,
t.platform_id,
t.kind,
t.payload,
t.status,
t.dedup_key,
t.active_attempt_id,
t.lease_expires_at,
t.attempts,
t.result,
t.error,
t.created_at,
t.updated_at,
j.status,
j.compliance_blocked_record_id,
j.compliance_blocked_at,
j.compliance_blocked_reason
FROM desktop_tasks AS t
JOIN desktop_publish_jobs AS j
ON j.desktop_id = t.job_id
AND j.workspace_id = t.workspace_id
WHERE t.tenant_id = $1
AND t.workspace_id = $2
AND j.tenant_id = $1
AND j.created_by_user_id = $3
AND t.kind = 'publish'
AND t.status = ANY($4)
AND (
$5 = ''
OR COALESCE(t.payload->>'title', j.title, '') ILIKE '%' || $5 || '%'
)
`
args := []any{owner.TenantID, owner.WorkspaceID, owner.UserID, statuses, title}
if strings.TrimSpace(orderBy) != "" {
query += "\nORDER BY " + orderBy
}
if limit > 0 {
query += "\nLIMIT $6 OFFSET $7"
args = append(args, limit, offset)
}
rows, err := s.pool.Query(ctx, query, args...)
if err != nil {
return nil, response.ErrInternal(50109, "desktop_publish_tasks_query_failed", "failed to list desktop publish tasks")
}
defer rows.Close()
return scanDesktopTaskRows(rows)
}
func (s *DesktopTaskService) countPublishTasksByStatuses(
ctx context.Context,
owner publishTaskListOwner,
statuses []string,
title string,
) (int, error) {
var count int
err := s.pool.QueryRow(ctx, `
SELECT COUNT(1)
FROM desktop_tasks AS t
JOIN desktop_publish_jobs AS j
ON j.desktop_id = t.job_id
AND j.workspace_id = t.workspace_id
WHERE t.tenant_id = $1
AND t.workspace_id = $2
AND j.tenant_id = $1
AND j.created_by_user_id = $3
AND t.kind = 'publish'
AND t.status = ANY($4)
AND (
$5 = ''
OR COALESCE(t.payload->>'title', j.title, '') ILIKE '%' || $5 || '%'
)
`, owner.TenantID, owner.WorkspaceID, owner.UserID, statuses, title).Scan(&count)
if err != nil {
return 0, response.ErrInternal(50109, "desktop_publish_tasks_query_failed", "failed to list desktop publish tasks")
}
return count, nil
}
func scanDesktopTaskRows(rows pgx.Rows) ([]DesktopTaskView, error) {
items := make([]DesktopTaskView, 0)
for rows.Next() {
var (
desktopID uuid.UUID
jobID uuid.UUID
tenantID int64
workspaceID int64
targetAccountID uuid.UUID
targetClientID uuid.UUID
platform string
kind string
payload []byte
status string
dedupKey pgtype.Text
activeAttemptID pgtype.UUID
leaseExpiresAt pgtype.Timestamptz
attempts int32
resultJSON []byte
errorJSON []byte
createdAt time.Time
updatedAt time.Time
jobStatus string
blockedRecordID pgtype.Int8
blockedAt pgtype.Timestamptz
blockedReason pgtype.Text
)
if err := rows.Scan(
&desktopID,
&jobID,
&tenantID,
&workspaceID,
&targetAccountID,
&targetClientID,
&platform,
&kind,
&payload,
&status,
&dedupKey,
&activeAttemptID,
&leaseExpiresAt,
&attempts,
&resultJSON,
&errorJSON,
&createdAt,
&updatedAt,
&jobStatus,
&blockedRecordID,
&blockedAt,
&blockedReason,
); err != nil {
return nil, response.ErrInternal(50110, "desktop_publish_tasks_scan_failed", "failed to scan desktop publish tasks")
}
var dedupKeyText *string
if dedupKey.Valid {
value := dedupKey.String
dedupKeyText = &value
}
var activeAttemptIDText *string
if activeAttemptID.Valid {
value, convErr := uuid.FromBytes(activeAttemptID.Bytes[:])
if convErr == nil {
text := value.String()
activeAttemptIDText = &text
}
}
var leaseExpiresAtValue *time.Time
if leaseExpiresAt.Valid {
value := leaseExpiresAt.Time
leaseExpiresAtValue = &value
}
publishJobStatusText := strings.TrimSpace(jobStatus)
var publishJobStatus *string
if publishJobStatusText != "" {
publishJobStatus = &publishJobStatusText
}
var complianceBlockedRecordID *int64
if blockedRecordID.Valid {
value := blockedRecordID.Int64
complianceBlockedRecordID = &value
}
var complianceBlockedAt *time.Time
if blockedAt.Valid {
value := blockedAt.Time
complianceBlockedAt = &value
}
var complianceBlockedReason *string
if blockedReason.Valid {
value := blockedReason.String
complianceBlockedReason = &value
}
items = append(items, DesktopTaskView{
ID: desktopID.String(),
JobID: jobID.String(),
TenantID: tenantID,
WorkspaceID: workspaceID,
TargetAccountID: targetAccountID.String(),
TargetClientID: targetClientID.String(),
Platform: platform,
Kind: kind,
Payload: json.RawMessage(payload),
Status: normalizeDesktopTaskTerminalStatus(status),
PublishJobStatus: publishJobStatus,
ComplianceBlockedRecordID: complianceBlockedRecordID,
ComplianceBlockedAt: complianceBlockedAt,
ComplianceBlockedReason: complianceBlockedReason,
DedupKey: dedupKeyText,
ActiveAttemptID: activeAttemptIDText,
LeaseExpiresAt: leaseExpiresAtValue,
Attempts: int(attempts),
Result: json.RawMessage(resultJSON),
Error: json.RawMessage(errorJSON),
CreatedAt: createdAt,
UpdatedAt: updatedAt,
})
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50109, "desktop_publish_tasks_query_failed", "failed to list desktop publish tasks")
}
return items, nil
}
func (s *DesktopTaskService) classifyLeaseError(ctx context.Context, client *repository.DesktopClient, taskID *uuid.UUID) error {
if taskID == nil {
return response.ErrConflict(40986, "desktop_task_unavailable", "no desktop task is currently available")
}
task, err := s.repo.GetByDesktopID(ctx, *taskID, client.WorkspaceID)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return response.ErrNotFound(40482, "desktop_task_not_found", "desktop task not found")
}
return response.ErrInternal(50095, "desktop_task_lookup_failed", "failed to inspect desktop task state")
}
if task.TargetClientID != client.ID && !s.clientCanLeaseMonitorTask(ctx, client, task) {
return response.ErrForbidden(40381, "desktop_task_not_target_client", "desktop task is assigned to another desktop client")
}
if task.Status == "in_progress" {
return response.ErrConflict(40989, "desktop_task_already_in_progress", "desktop task is already in progress")
}
return response.ErrConflict(40991, "desktop_task_not_queued", "desktop task is not queued")
}
func (s *DesktopTaskService) clientCanLeaseMonitorTask(ctx context.Context, client *repository.DesktopClient, task *repository.DesktopTask) bool {
if s == nil || client == nil || task == nil || task.Kind != "monitor" || task.TargetAccountID == uuid.Nil {
return false
}
accountIDs, err := s.loadMonitorLeaseAccountIDs(ctx, client)
if err != nil {
return false
}
for _, accountID := range accountIDs {
if accountID == task.TargetAccountID {
return true
}
}
return false
}
func buildDesktopTaskView(task *repository.DesktopTask) DesktopTaskView {
var activeAttemptID *string
if task.ActiveAttemptID != nil {
value := task.ActiveAttemptID.String()
activeAttemptID = &value
}
return DesktopTaskView{
ID: task.DesktopID.String(),
JobID: task.JobID.String(),
TenantID: task.TenantID,
WorkspaceID: task.WorkspaceID,
TargetAccountID: task.TargetAccountID.String(),
TargetClientID: task.TargetClientID.String(),
Platform: task.Platform,
Kind: task.Kind,
Payload: json.RawMessage(task.Payload),
Status: normalizeDesktopTaskTerminalStatus(task.Status),
DedupKey: task.DedupKey,
ActiveAttemptID: activeAttemptID,
LeaseExpiresAt: task.LeaseExpiresAt,
Attempts: task.Attempts,
Result: json.RawMessage(task.Result),
Error: json.RawMessage(task.Error),
CreatedAt: task.CreatedAt,
UpdatedAt: task.UpdatedAt,
}
}
func marshalOptionalJSON(value any) ([]byte, error) {
if value == nil {
return nil, nil
}
return json.Marshal(value)
}
func optionalStringValue(value *string) string {
if value == nil {
return ""
}
return strings.TrimSpace(*value)
}
func literalStringPtr(value string) *string {
return &value
}
func normalizeDesktopTaskTerminalStatus(status string) string {
switch strings.TrimSpace(status) {
case "unknown":
return "failed"
default:
return strings.TrimSpace(status)
}
}
type desktopTaskRecoveryMode string
const (
desktopTaskRecoveryModeStartup desktopTaskRecoveryMode = "startup"
desktopTaskRecoveryModeDisconnect desktopTaskRecoveryMode = "disconnect"
desktopTaskRecoveryModeLeaseExpiry desktopTaskRecoveryMode = "lease_expired"
)
type recoveredDesktopTaskLease struct {
TaskID uuid.UUID
WorkspaceID int64
Kind string
Status string
ActiveAttempt pgtype.UUID
Attempts int
SubmitStarted bool
ErrorJSON []byte
}
type PublishLeaseRecoveryResult struct {
Requeued int `json:"requeued"`
Failed int `json:"failed"`
Uncertain int `json:"uncertain"`
}
// resolvePublishRecoveryOutcome decides what to do with a publish task whose lease was lost.
//
// If the client had already entered the irreversible submit phase (publish_submit_started_at
// is set), the article may already exist on the platform. Auto-requeueing would silently
// re-post a non-idempotent article, so we route the task to 'unknown' (kept in the dedup
// active set, surfaced for manual reconcile) instead. Tasks that never reached submit are
// safe to auto-requeue while attempts remain, and give up to 'failed' once exhausted.
func resolvePublishRecoveryOutcome(submitStarted bool, attempts int) (status string, payloadKind string) {
switch {
case submitStarted:
return "unknown", "publish_uncertain"
case attempts >= desktopPublishMaxAttempts:
return "failed", "publish_final"
default:
return "queued", "publish"
}
}
func (s *DesktopTaskService) recoverClientTasksOnStartup(ctx context.Context, client *repository.DesktopClient) error {
return s.recoverClientTasks(ctx, client, desktopTaskRecoveryModeStartup)
}
func (s *DesktopTaskService) recoverClientTasksOnDisconnect(ctx context.Context, client *repository.DesktopClient) error {
return s.recoverClientTasks(ctx, client, desktopTaskRecoveryModeDisconnect)
}
func (s *DesktopTaskService) recoverExpiredClientTasks(ctx context.Context, client *repository.DesktopClient) error {
return s.recoverClientTasks(ctx, client, desktopTaskRecoveryModeLeaseExpiry)
}
func (s *DesktopTaskService) recoverClientTasks(
ctx context.Context,
client *repository.DesktopClient,
mode desktopTaskRecoveryMode,
) error {
if s == nil || client == nil || s.pool == nil {
return nil
}
publishErrorJSON, publishReason, publishMessage, err := desktopTaskRecoveryPayload(mode, "publish")
if err != nil {
return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
}
publishFinalErrorJSON, _, _, err := desktopTaskRecoveryPayload(mode, "publish_final")
if err != nil {
return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
}
publishUncertainErrorJSON, _, _, err := desktopTaskRecoveryPayload(mode, "publish_uncertain")
if err != nil {
return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
}
monitorErrorJSON, monitorReason, _, err := desktopTaskRecoveryPayload(mode, "monitor")
if err != nil {
return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode monitor desktop task recovery payload")
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return response.ErrInternal(50196, "desktop_task_recovery_begin_failed", "failed to start desktop task recovery")
}
defer tx.Rollback(ctx)
repo := repository.NewDesktopTaskRepository(tx)
query := desktopTaskRecoverySelectQuery(mode)
queryArgs := []any{client.ID, client.WorkspaceID}
rows, err := tx.Query(ctx, query, queryArgs...)
if err != nil {
s.logWarn("desktop task recovery query failed", err, zap.String("client_id", client.ID.String()), zap.String("mode", string(mode)))
return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
}
defer rows.Close()
recovered := make([]recoveredDesktopTaskLease, 0)
for rows.Next() {
var item recoveredDesktopTaskLease
if scanErr := rows.Scan(
&item.TaskID,
&item.WorkspaceID,
&item.Kind,
&item.Status,
&item.ActiveAttempt,
&item.Attempts,
&item.SubmitStarted,
); scanErr != nil {
s.logWarn("desktop task recovery scan failed", scanErr, zap.String("client_id", client.ID.String()), zap.String("mode", string(mode)))
return response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to parse recovered desktop tasks")
}
recovered = append(recovered, item)
}
if err := rows.Err(); err != nil {
s.logWarn("desktop task recovery rows failed", err, zap.String("client_id", client.ID.String()), zap.String("mode", string(mode)))
return response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to iterate recovered desktop tasks")
}
publishOutcomes := make([]*desktopPublishSyncOutcome, 0)
for index := range recovered {
item := &recovered[index]
if item.Kind == "monitor" {
if _, execErr := tx.Exec(ctx, `
UPDATE desktop_tasks
SET status = 'queued',
error = $2,
active_attempt_id = NULL,
lease_token_hash = NULL,
lease_expires_at = NULL,
interrupted_at = COALESCE(interrupted_at, NOW()),
interrupt_reason = $3,
enqueued_at = NOW(),
updated_at = NOW()
WHERE desktop_id = $1
`, item.TaskID, monitorErrorJSON, monitorReason); execErr != nil {
s.logWarn("desktop monitor task recovery update failed", execErr, zap.String("task_id", item.TaskID.String()), zap.String("mode", string(mode)))
return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
}
item.Status = "queued"
continue
}
finalStatus, payloadKind := resolvePublishRecoveryOutcome(item.SubmitStarted, item.Attempts)
nextErrorJSON := publishErrorJSON
switch payloadKind {
case "publish_final":
nextErrorJSON = publishFinalErrorJSON
case "publish_uncertain":
nextErrorJSON = publishUncertainErrorJSON
}
if _, execErr := tx.Exec(ctx, `
UPDATE desktop_tasks
SET status = $2,
error = $3,
result = CASE WHEN $2 = 'queued' THEN NULL ELSE result END,
active_attempt_id = NULL,
lease_token_hash = NULL,
lease_expires_at = NULL,
interrupted_at = COALESCE(interrupted_at, NOW()),
interrupt_reason = COALESCE(interrupt_reason, $4),
enqueued_at = CASE WHEN $2 = 'queued' THEN NOW() ELSE enqueued_at END,
updated_at = NOW()
WHERE desktop_id = $1
`, item.TaskID, finalStatus, nextErrorJSON, publishReason); execErr != nil {
s.logWarn("desktop publish task recovery update failed", execErr, zap.String("task_id", item.TaskID.String()), zap.String("mode", string(mode)))
return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
}
item.Status = finalStatus
item.ErrorJSON = nextErrorJSON
// Only terminal 'failed' tasks finalize the publish_record; 'unknown' (may already be
// published) is left in the dedup active set for manual reconcile, 'queued' is requeued.
if finalStatus == "failed" {
task, getErr := repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID)
if getErr != nil {
s.logWarn("desktop publish task recovery reload failed", getErr, zap.String("task_id", item.TaskID.String()), zap.String("mode", string(mode)))
return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
}
publishOutcome, syncErr := syncDesktopPublishTaskState(ctx, tx, task)
if syncErr != nil {
return syncErr
}
if publishOutcome != nil {
publishOutcomes = append(publishOutcomes, publishOutcome)
}
}
}
for _, item := range recovered {
if !item.ActiveAttempt.Valid {
continue
}
attemptID, convErr := uuid.FromBytes(item.ActiveAttempt.Bytes[:])
if convErr != nil {
s.logWarn("desktop recovery attempt id decode failed", convErr, zap.String("task_id", item.TaskID.String()))
continue
}
finalStatus := literalStringPtr(item.Status)
errorJSON := item.ErrorJSON
if len(errorJSON) == 0 {
errorJSON = publishErrorJSON
}
if item.Kind == "monitor" {
finalStatus = literalStringPtr("aborted")
errorJSON = monitorErrorJSON
}
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
TaskID: item.TaskID,
AttemptID: attemptID,
FinalStatus: finalStatus,
Error: errorJSON,
}); finishErr != nil {
s.logWarn("desktop recovery attempt finish failed", finishErr, zap.String("task_id", item.TaskID.String()))
}
}
if err := tx.Commit(ctx); err != nil {
return response.ErrInternal(50199, "desktop_task_recovery_commit_failed", "failed to commit desktop task recovery")
}
s.afterRecoveredPublishOutcomes(ctx, publishOutcomes)
for _, item := range recovered {
task, getErr := s.repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID)
if getErr != nil {
s.logWarn("desktop recovery reload failed", getErr, zap.String("task_id", item.TaskID.String()))
continue
}
eventType := "task_completed"
if item.Status == "queued" {
eventType = "task_available"
}
s.publishTaskEvent(ctx, task, eventType)
}
if len(recovered) > 0 && s.logger != nil {
s.logger.Info("desktop tasks recovered",
zap.String("client_id", client.ID.String()),
zap.String("mode", string(mode)),
zap.Int("count", len(recovered)),
zap.String("publish_message", publishMessage),
)
}
return nil
}
func (s *DesktopTaskService) RecoverExpiredPublishTasks(ctx context.Context, limit int) (PublishLeaseRecoveryResult, error) {
var result PublishLeaseRecoveryResult
if s == nil || s.pool == nil {
return result, nil
}
if limit <= 0 {
limit = 1000
}
publishErrorJSON, publishReason, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "publish")
if err != nil {
return result, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
}
publishFinalErrorJSON, _, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "publish_final")
if err != nil {
return result, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
}
publishUncertainErrorJSON, _, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "publish_uncertain")
if err != nil {
return result, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return result, response.ErrInternal(50196, "desktop_task_recovery_begin_failed", "failed to start desktop task recovery")
}
defer tx.Rollback(ctx)
rows, err := tx.Query(ctx, `
SELECT
t.desktop_id,
t.workspace_id,
t.kind,
t.status,
t.active_attempt_id,
t.attempts,
(t.publish_submit_started_at IS NOT NULL) AS submit_started
FROM desktop_tasks AS t
WHERE t.kind = 'publish'
AND t.status = 'in_progress'
AND t.lease_expires_at IS NOT NULL
AND t.lease_expires_at < NOW()
ORDER BY t.lease_expires_at ASC, t.updated_at ASC, t.desktop_id ASC
LIMIT $1
FOR UPDATE OF t SKIP LOCKED
`, limit)
if err != nil {
s.logWarn("expired desktop publish recovery query failed", err)
return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
}
defer rows.Close()
recovered := make([]recoveredDesktopTaskLease, 0)
for rows.Next() {
var item recoveredDesktopTaskLease
if scanErr := rows.Scan(
&item.TaskID,
&item.WorkspaceID,
&item.Kind,
&item.Status,
&item.ActiveAttempt,
&item.Attempts,
&item.SubmitStarted,
); scanErr != nil {
s.logWarn("expired desktop publish recovery scan failed", scanErr)
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to parse recovered desktop tasks")
}
recovered = append(recovered, item)
}
if err := rows.Err(); err != nil {
s.logWarn("expired desktop publish recovery rows failed", err)
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to iterate recovered desktop tasks")
}
repo := repository.NewDesktopTaskRepository(tx)
publishOutcomes := make([]*desktopPublishSyncOutcome, 0)
for index := range recovered {
item := &recovered[index]
finalStatus, payloadKind := resolvePublishRecoveryOutcome(item.SubmitStarted, item.Attempts)
nextErrorJSON := publishErrorJSON
switch payloadKind {
case "publish_final":
nextErrorJSON = publishFinalErrorJSON
case "publish_uncertain":
nextErrorJSON = publishUncertainErrorJSON
}
if _, execErr := tx.Exec(ctx, `
UPDATE desktop_tasks
SET status = $2,
error = $3,
result = CASE WHEN $2 = 'queued' THEN NULL ELSE result END,
active_attempt_id = NULL,
lease_token_hash = NULL,
lease_expires_at = NULL,
interrupted_at = COALESCE(interrupted_at, NOW()),
interrupt_reason = COALESCE(interrupt_reason, $4),
enqueued_at = CASE WHEN $2 = 'queued' THEN NOW() ELSE enqueued_at END,
updated_at = NOW()
WHERE desktop_id = $1
AND status = 'in_progress'
`, item.TaskID, finalStatus, nextErrorJSON, publishReason); execErr != nil {
s.logWarn("expired desktop publish recovery update failed", execErr, zap.String("task_id", item.TaskID.String()))
return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
}
item.Status = finalStatus
item.ErrorJSON = nextErrorJSON
switch finalStatus {
case "failed":
result.Failed++
task, getErr := repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID)
if getErr != nil {
s.logWarn("expired desktop publish recovery reload failed", getErr, zap.String("task_id", item.TaskID.String()))
return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
}
publishOutcome, syncErr := syncDesktopPublishTaskState(ctx, tx, task)
if syncErr != nil {
return result, syncErr
}
if publishOutcome != nil {
publishOutcomes = append(publishOutcomes, publishOutcome)
}
case "unknown":
// May already be published — keep as 'unknown' (still in the dedup active set) for
// manual reconcile. Do NOT sync to a terminal publish_record state and do NOT requeue.
result.Uncertain++
default:
result.Requeued++
}
}
for _, item := range recovered {
if !item.ActiveAttempt.Valid {
continue
}
attemptID, convErr := uuid.FromBytes(item.ActiveAttempt.Bytes[:])
if convErr != nil {
s.logWarn("expired desktop publish recovery attempt id decode failed", convErr, zap.String("task_id", item.TaskID.String()))
continue
}
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
TaskID: item.TaskID,
AttemptID: attemptID,
FinalStatus: literalStringPtr(item.Status),
Error: item.ErrorJSON,
}); finishErr != nil {
s.logWarn("expired desktop publish recovery attempt finish failed", finishErr, zap.String("task_id", item.TaskID.String()))
}
}
if err := tx.Commit(ctx); err != nil {
return result, response.ErrInternal(50199, "desktop_task_recovery_commit_failed", "failed to commit desktop task recovery")
}
s.afterRecoveredPublishOutcomes(ctx, publishOutcomes)
for _, item := range recovered {
task, getErr := s.repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID)
if getErr != nil {
s.logWarn("expired desktop publish recovery reload after commit failed", getErr, zap.String("task_id", item.TaskID.String()))
continue
}
eventType := "task_completed"
if item.Status == "queued" {
eventType = "task_available"
}
s.publishTaskEvent(ctx, task, eventType)
}
if len(recovered) > 0 && s.logger != nil {
s.logger.Info("expired desktop publish tasks recovered",
zap.Int("requeued", result.Requeued),
zap.Int("failed", result.Failed),
zap.Int("uncertain", result.Uncertain),
)
}
return result, nil
}
func desktopTaskRecoverySelectQuery(mode desktopTaskRecoveryMode) string {
query := `
SELECT
desktop_id,
workspace_id,
kind,
status,
active_attempt_id,
attempts,
publish_submit_started_at IS NOT NULL AS submit_started
FROM desktop_tasks
WHERE target_client_id = $1
AND workspace_id = $2
AND status = 'in_progress'
`
if mode == desktopTaskRecoveryModeLeaseExpiry {
query += `
AND lease_expires_at IS NOT NULL
AND lease_expires_at < NOW()
`
}
query += `
FOR UPDATE
`
return query
}
func desktopTaskRecoveryPayload(mode desktopTaskRecoveryMode, kind string) ([]byte, string, string, error) {
reason := strings.TrimSpace(string(mode))
message := "desktop task was recovered after the client lost the active lease"
source := "desktop_task_recovery"
isPublishFinal := kind == "publish_final"
isPublishUncertain := kind == "publish_uncertain"
if isPublishFinal {
message = fmt.Sprintf("desktop publish task exceeded %d attempts during recovery; task has been marked failed", desktopPublishMaxAttempts)
} else if isPublishUncertain {
message = "desktop publish task may have already been submitted to the platform before the lease was lost; kept as unknown for manual reconcile to avoid duplicate publishing"
}
switch mode {
case desktopTaskRecoveryModeStartup:
source = "desktop_client_startup"
if isPublishFinal {
message = fmt.Sprintf("desktop client restarted while the publish task was in progress; task exceeded %d attempts and has been marked failed", desktopPublishMaxAttempts)
} else if isPublishUncertain {
message = "desktop client restarted while the publish task was mid-submit; kept as unknown for manual reconcile to avoid duplicate publishing"
} else if kind == "monitor" {
message = "desktop client restarted while the monitor task was in progress; task has been re-queued"
} else {
message = "desktop client restarted while the publish task was in progress; task has been re-queued for retry"
}
case desktopTaskRecoveryModeDisconnect:
source = "desktop_client_offline"
if isPublishFinal {
message = fmt.Sprintf("desktop client went offline while the publish task was in progress; task exceeded %d attempts and has been marked failed", desktopPublishMaxAttempts)
} else if isPublishUncertain {
message = "desktop client went offline while the publish task was mid-submit; kept as unknown for manual reconcile to avoid duplicate publishing"
} else if kind == "monitor" {
message = "desktop client went offline while the monitor task was in progress; task has been re-queued"
} else {
message = "desktop client went offline while the publish task was in progress; task has been re-queued for retry"
}
case desktopTaskRecoveryModeLeaseExpiry:
source = "desktop_task_lease_expiry"
if isPublishFinal {
message = fmt.Sprintf("desktop task lease expired before publish completion; task exceeded %d attempts and has been marked failed", desktopPublishMaxAttempts)
} else if isPublishUncertain {
message = "desktop task lease expired while the publish task was mid-submit; kept as unknown for manual reconcile to avoid duplicate publishing"
} else if kind == "monitor" {
message = "desktop task lease expired before monitor completion; task has been re-queued"
} else {
message = "desktop task lease expired before publish completion; task has been re-queued for retry"
}
}
fields := map[string]any{
"reason": reason,
"message": message,
"source": source,
}
if isPublishUncertain {
// Surfaced to the desktop UI so a manual retry of a maybe-already-published task asks for
// explicit confirmation instead of silently re-posting.
fields["publish_submit_uncertain"] = true
}
payload, err := marshalOptionalJSON(fields)
if err != nil {
return nil, "", "", err
}
return payload, reason, message, nil
}
func (s *DesktopTaskService) dropStaleMonitorTasksForClient(ctx context.Context, client *repository.DesktopClient) error {
if s == nil || client == nil || s.pool == nil || s.monitoringPool == nil {
return nil
}
_, currentBusinessDate := monitoringBusinessDayAndDateAt(time.Now())
errorJSON, err := marshalOptionalJSON(map[string]any{
"reason": monitoringStaleTaskDropReason,
"message": monitoringStaleTaskDropError,
"current_business_date": currentBusinessDate,
"source": "business_day_rollover",
})
if err != nil {
return response.ErrInternal(50190, "desktop_task_stale_payload_failed", "failed to encode stale monitor desktop task payload")
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return response.ErrInternal(50191, "desktop_task_stale_begin_failed", "failed to start stale monitor desktop cleanup")
}
defer tx.Rollback(ctx)
rows, err := tx.Query(ctx, `
WITH stale AS (
SELECT
desktop_id,
active_attempt_id
FROM desktop_tasks
WHERE target_client_id = $1
AND kind = 'monitor'
AND status IN ('queued', 'in_progress', 'unknown')
AND COALESCE(
NULLIF(payload ->> 'business_date', ''),
NULLIF(payload ->> 'businessDate', ''),
NULLIF(payload ->> 'metric_date', ''),
NULLIF(payload ->> 'date', '')
) < $2
FOR UPDATE
),
updated AS (
UPDATE desktop_tasks AS t
SET status = 'aborted',
error = $3,
active_attempt_id = NULL,
lease_token_hash = NULL,
lease_expires_at = NULL,
interrupted_at = COALESCE(interrupted_at, NOW()),
interrupt_reason = $4,
updated_at = NOW()
FROM stale
WHERE t.desktop_id = stale.desktop_id
RETURNING t.desktop_id
)
SELECT stale.desktop_id, stale.active_attempt_id
FROM stale
INNER JOIN updated ON updated.desktop_id = stale.desktop_id
`, client.ID, currentBusinessDate, errorJSON, monitoringStaleTaskDropReason)
if err != nil {
return response.ErrInternal(50192, "desktop_task_stale_query_failed", "failed to select stale monitor desktop tasks")
}
defer rows.Close()
repo := repository.NewDesktopTaskRepository(tx)
droppedTaskIDs := make([]uuid.UUID, 0)
for rows.Next() {
var (
taskID uuid.UUID
attemptID pgtype.UUID
)
if scanErr := rows.Scan(&taskID, &attemptID); scanErr != nil {
return response.ErrInternal(50193, "desktop_task_stale_scan_failed", "failed to parse stale monitor desktop tasks")
}
droppedTaskIDs = append(droppedTaskIDs, taskID)
if !attemptID.Valid {
continue
}
resolvedAttemptID, convErr := uuid.FromBytes(attemptID.Bytes[:])
if convErr != nil {
s.logWarn("desktop stale monitor attempt id decode failed", convErr, zap.String("task_id", taskID.String()))
continue
}
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
TaskID: taskID,
AttemptID: resolvedAttemptID,
FinalStatus: literalStringPtr("aborted"),
Error: errorJSON,
}); finishErr != nil {
s.logWarn("desktop stale monitor attempt finish failed", finishErr, zap.String("task_id", taskID.String()))
}
}
if err := rows.Err(); err != nil {
return response.ErrInternal(50193, "desktop_task_stale_scan_failed", "failed to iterate stale monitor desktop tasks")
}
if err := tx.Commit(ctx); err != nil {
return response.ErrInternal(50194, "desktop_task_stale_commit_failed", "failed to commit stale monitor desktop cleanup")
}
for _, taskID := range droppedTaskIDs {
task, getErr := s.repo.GetByDesktopID(ctx, taskID, client.WorkspaceID)
if getErr != nil {
s.logWarn("desktop stale monitor reload failed", getErr, zap.String("task_id", taskID.String()))
continue
}
s.publishTaskEvent(ctx, task, "task_canceled")
}
if _, err := s.monitoringPool.Exec(ctx, `
UPDATE monitoring_collect_tasks
SET status = 'skipped',
lease_token_hash = NULL,
leased_to_executor = NULL,
leased_at = NULL,
lease_expires_at = NULL,
callback_received_at = COALESCE(callback_received_at, NOW()),
completed_at = COALESCE(completed_at, NOW()),
skip_reason = $4,
error_message = COALESCE(NULLIF(error_message, ''), $5),
updated_at = NOW()
WHERE tenant_id = $1
AND collector_type = $2
AND COALESCE(execution_owner, 'legacy') = 'desktop_tasks'
AND target_client_id = $3
AND business_date < $6::date
AND callback_received_at IS NULL
AND status IN ('pending', 'leased', 'expired')
`, client.TenantID, monitoringCollectorType, client.ID, monitoringStaleTaskDropReason, monitoringStaleTaskDropError, currentBusinessDate); err != nil {
return response.ErrInternal(50041, "task_skip_failed", "failed to drop stale phase2 monitoring tasks")
}
return nil
}
func activeAttemptID(task *repository.DesktopTask) *uuid.UUID {
if task == nil || task.ActiveAttemptID == nil {
return nil
}
value := *task.ActiveAttemptID
return &value
}
func shouldRequeuePreemptedMonitorTask(task *repository.DesktopTask, reason *string) bool {
if task == nil || task.Kind != "monitor" {
return false
}
return strings.EqualFold(strings.TrimSpace(optionalStringValue(reason)), "collect_now_preempt")
}
func requeuePreemptedMonitorTask(
ctx context.Context,
tx pgx.Tx,
repo repository.DesktopTaskRepository,
task *repository.DesktopTask,
) (*repository.DesktopTask, error) {
if task == nil {
return nil, nil
}
replacementID := uuid.New()
replacementJobID := uuid.New()
if _, err := tx.Exec(ctx, `
INSERT INTO desktop_tasks (
desktop_id,
job_id,
tenant_id,
workspace_id,
target_account_id,
target_client_id,
platform_id,
kind,
payload,
status,
priority,
lane,
lane_weight,
source,
scheduler_group_key,
monitor_task_id,
supersedes_task_id,
control_flags,
interrupt_generation,
enqueued_at
)
VALUES (
$1,
$2,
$3,
$4,
$5,
$6,
$7,
'monitor',
$8,
'queued',
50,
'retry',
20,
'retry_recovery',
$9,
$10,
$11,
jsonb_build_object(
'interrupt_requested', false,
'requeued_at', NOW()::text,
'requeued_from_task_id', $11::text,
'requeued_reason', 'collect_now_preempt'
),
$12,
NOW()
)
`, replacementID, replacementJobID, task.TenantID, task.WorkspaceID, task.TargetAccountID, task.TargetClientID, task.Platform, task.Payload, nullableString(task.SchedulerGroupKey), nullableInt64(task.MonitorTaskID), task.DesktopID, task.InterruptGeneration); err != nil {
return nil, response.ErrInternal(50111, "desktop_task_requeue_failed", "failed to create replacement retry monitor desktop task")
}
next, err := repo.GetByDesktopID(ctx, replacementID, task.WorkspaceID)
if err != nil {
return nil, response.ErrInternal(50112, "desktop_task_lookup_failed", "failed to reload replacement retry monitor desktop task")
}
return next, nil
}
func (s *DesktopTaskService) publishTaskEvent(ctx context.Context, task *repository.DesktopTask, eventType string) {
publishDesktopTaskLifecycleEvent(ctx, s.messaging, s.logger, task, eventType)
}
func (s *DesktopTaskService) afterRecoveredPublishOutcomes(ctx context.Context, outcomes []*desktopPublishSyncOutcome) {
for _, outcome := range outcomes {
if outcome == nil {
continue
}
invalidateArticleCaches(ctx, s.cache, outcome.TenantID, &outcome.ArticleID)
if outcome.ArticleAlias != nil {
if syncErr := s.syncMonitoringArticleAlias(ctx, outcome.ArticleAlias); syncErr != nil {
s.logWarn("monitoring article alias sync failed after publish recovery", syncErr, zap.Int64("article_id", outcome.ArticleID))
}
}
}
}
func reconcileDesktopTaskEventType(_ string, status string) string {
if status == "retry" {
return "task_available"
}
return "task_reconciled"
}
func (s *DesktopTaskService) logWarn(message string, err error, fields ...zap.Field) {
if s.logger == nil || err == nil {
return
}
fields = append(fields, zap.Error(err))
s.logger.Warn(message, fields...)
}