28633cf570
Desktop Client Build / Resolve Build Metadata (push) Successful in 44s
Backend CI / Backend (push) Successful in 17m5s
Desktop Client Build / Build Desktop Client (push) Successful in 25m13s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 42s
When a desktop monitoring task fails with a non-retryable authorization failure (login expired, challenge required, risk control), all pending same-client/same-platform tasks for the same business day are immediately bulk-failed as non-retryable, avoiding wasted execution attempts. Adds preflight authorization checks before task execution, enriches failure payloads with disposition metadata (code, category, retryable flags), and triggers account health reports on auth state degradation. Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
1787 lines
56 KiB
Go
1787 lines
56 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
goredis "github.com/redis/go-redis/v9"
|
|
"go.uber.org/zap"
|
|
|
|
"github.com/geo-platform/tenant-api/internal/shared/auth"
|
|
sharedcache "github.com/geo-platform/tenant-api/internal/shared/cache"
|
|
sharedcompliance "github.com/geo-platform/tenant-api/internal/shared/compliance"
|
|
"github.com/geo-platform/tenant-api/internal/shared/config"
|
|
"github.com/geo-platform/tenant-api/internal/shared/messaging/rabbitmq"
|
|
"github.com/geo-platform/tenant-api/internal/shared/response"
|
|
tenantcompliance "github.com/geo-platform/tenant-api/internal/tenant/compliance"
|
|
"github.com/geo-platform/tenant-api/internal/tenant/repository"
|
|
)
|
|
|
|
type DesktopTaskService struct {
|
|
pool *pgxpool.Pool
|
|
monitoringPool *pgxpool.Pool
|
|
repo repository.DesktopTaskRepository
|
|
cache sharedcache.Cache
|
|
redis *goredis.Client
|
|
messaging *rabbitmq.Client
|
|
logger *zap.Logger
|
|
compliance *tenantcompliance.Service
|
|
}
|
|
|
|
func NewDesktopTaskService(pool *pgxpool.Pool, monitoringPool *pgxpool.Pool, messaging *rabbitmq.Client, logger *zap.Logger) *DesktopTaskService {
|
|
return &DesktopTaskService{
|
|
pool: pool,
|
|
monitoringPool: monitoringPool,
|
|
repo: repository.NewDesktopTaskRepository(pool),
|
|
messaging: messaging,
|
|
logger: logger,
|
|
compliance: tenantcompliance.NewService(pool, config.NewStaticProvider(&config.Config{
|
|
Compliance: config.ComplianceConfig{Enabled: true},
|
|
}), logger).WithRabbitMQ(messaging),
|
|
}
|
|
}
|
|
|
|
func NewDesktopTaskServiceWithConfig(pool *pgxpool.Pool, monitoringPool *pgxpool.Pool, messaging *rabbitmq.Client, logger *zap.Logger, cfg config.Provider) *DesktopTaskService {
|
|
svc := NewDesktopTaskService(pool, monitoringPool, messaging, logger)
|
|
svc.compliance = tenantcompliance.NewService(pool, cfg, logger).WithRabbitMQ(messaging)
|
|
return svc
|
|
}
|
|
|
|
func (s *DesktopTaskService) WithCache(c sharedcache.Cache) *DesktopTaskService {
|
|
s.cache = c
|
|
return s
|
|
}
|
|
|
|
func (s *DesktopTaskService) WithRedis(redis *goredis.Client) *DesktopTaskService {
|
|
if s != nil {
|
|
s.redis = redis
|
|
}
|
|
return s
|
|
}
|
|
|
|
type DesktopTaskView struct {
|
|
ID string `json:"id"`
|
|
JobID string `json:"job_id"`
|
|
TenantID int64 `json:"tenant_id"`
|
|
WorkspaceID int64 `json:"workspace_id"`
|
|
TargetAccountID string `json:"target_account_id"`
|
|
TargetClientID string `json:"target_client_id"`
|
|
Platform string `json:"platform"`
|
|
Kind string `json:"kind"`
|
|
Payload json.RawMessage `json:"payload"`
|
|
Status string `json:"status"`
|
|
PublishJobStatus *string `json:"publish_job_status,omitempty"`
|
|
ComplianceBlockedRecordID *int64 `json:"compliance_blocked_record_id,omitempty"`
|
|
ComplianceBlockedAt *time.Time `json:"compliance_blocked_at,omitempty"`
|
|
ComplianceBlockedReason *string `json:"compliance_blocked_reason,omitempty"`
|
|
DedupKey *string `json:"dedup_key"`
|
|
ActiveAttemptID *string `json:"active_attempt_id"`
|
|
LeaseExpiresAt *time.Time `json:"lease_expires_at"`
|
|
Attempts int `json:"attempts"`
|
|
Result json.RawMessage `json:"result"`
|
|
Error json.RawMessage `json:"error"`
|
|
CreatedAt time.Time `json:"created_at"`
|
|
UpdatedAt time.Time `json:"updated_at"`
|
|
}
|
|
|
|
type LeaseDesktopTaskRequest struct {
|
|
TaskID *string `json:"task_id"`
|
|
Kind *string `json:"kind" binding:"omitempty,oneof=publish monitor"`
|
|
}
|
|
|
|
type LeaseDesktopTaskResponse struct {
|
|
Task *DesktopTaskView `json:"task"`
|
|
AttemptID *string `json:"attempt_id,omitempty"`
|
|
LeaseToken *string `json:"lease_token,omitempty"`
|
|
LeaseExpiresAt *time.Time `json:"lease_expires_at,omitempty"`
|
|
}
|
|
|
|
type ExtendDesktopTaskRequest struct {
|
|
LeaseToken string `json:"lease_token" binding:"required"`
|
|
}
|
|
|
|
type CompleteDesktopTaskRequest struct {
|
|
LeaseToken string `json:"lease_token" binding:"required"`
|
|
Status string `json:"status" binding:"required,oneof=succeeded failed unknown"`
|
|
Payload map[string]any `json:"payload"`
|
|
Error map[string]any `json:"error"`
|
|
}
|
|
|
|
type CancelDesktopTaskRequest struct {
|
|
LeaseToken *string `json:"lease_token"`
|
|
Reason *string `json:"reason"`
|
|
}
|
|
|
|
type ReconcileDesktopTaskRequest struct {
|
|
Status string `json:"status" binding:"required,oneof=succeeded failed aborted retry"`
|
|
Result map[string]any `json:"result"`
|
|
Error map[string]any `json:"error"`
|
|
}
|
|
|
|
type ListPublishTasksRequest struct {
|
|
Page int `form:"page"`
|
|
PageSize int `form:"page_size"`
|
|
Title string `form:"title"`
|
|
}
|
|
|
|
type DesktopPublishTaskList struct {
|
|
Items []DesktopTaskView `json:"items"`
|
|
Page int `json:"page"`
|
|
PageSize int `json:"page_size"`
|
|
Total int `json:"total"`
|
|
PendingCount int `json:"pending_count"`
|
|
HistoryTotal int `json:"history_total"`
|
|
}
|
|
|
|
type publishTaskListOwner struct {
|
|
TenantID int64
|
|
WorkspaceID int64
|
|
UserID int64
|
|
}
|
|
|
|
func (s *DesktopTaskService) Lease(ctx context.Context, client *repository.DesktopClient, req LeaseDesktopTaskRequest, routeTaskID *uuid.UUID) (*LeaseDesktopTaskResponse, error) {
|
|
if client == nil {
|
|
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
|
|
}
|
|
|
|
taskID := routeTaskID
|
|
if taskID == nil && req.TaskID != nil && strings.TrimSpace(*req.TaskID) != "" {
|
|
parsed, err := uuid.Parse(strings.TrimSpace(*req.TaskID))
|
|
if err != nil {
|
|
return nil, response.ErrBadRequest(40084, "invalid_desktop_task_id", "task id must be a uuid")
|
|
}
|
|
taskID = &parsed
|
|
}
|
|
|
|
if err := s.dropStaleMonitorTasksForClient(ctx, client); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := s.recoverExpiredClientTasks(ctx, client); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := s.recheckQueuedPublishTasksForClient(ctx, client); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
rawToken, tokenHash, err := newDesktopClientToken()
|
|
if err != nil {
|
|
return nil, response.ErrInternal(50087, "desktop_task_lease_token_failed", "failed to generate desktop task lease token")
|
|
}
|
|
|
|
attemptID := uuid.New()
|
|
leaseParams := repository.DesktopTaskLeaseParams{
|
|
AttemptID: attemptID,
|
|
LeaseTokenHash: tokenHash,
|
|
}
|
|
|
|
if s.logger != nil {
|
|
fields := []zap.Field{
|
|
zap.String("client_id", client.ID.String()),
|
|
}
|
|
if req.Kind != nil {
|
|
fields = append(fields, zap.String("requested_kind", strings.TrimSpace(*req.Kind)))
|
|
}
|
|
if taskID != nil {
|
|
fields = append(fields, zap.String("task_id", taskID.String()))
|
|
}
|
|
s.logger.Debug("desktop task lease requested", fields...)
|
|
}
|
|
|
|
var task *repository.DesktopTask
|
|
switch {
|
|
case taskID != nil:
|
|
task, err = s.leaseQueuedDesktopTaskByID(ctx, client, *taskID, leaseParams)
|
|
case req.Kind != nil && strings.TrimSpace(*req.Kind) == "monitor":
|
|
task, err = s.leaseNextQueuedMonitorTask(ctx, client, leaseParams)
|
|
default:
|
|
task, err = s.repo.LeaseNextQueued(ctx, client.ID, req.Kind, leaseParams)
|
|
}
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
if s.logger != nil {
|
|
fields := []zap.Field{
|
|
zap.String("client_id", client.ID.String()),
|
|
}
|
|
if req.Kind != nil {
|
|
fields = append(fields, zap.String("requested_kind", strings.TrimSpace(*req.Kind)))
|
|
}
|
|
if taskID != nil {
|
|
fields = append(fields, zap.String("task_id", taskID.String()))
|
|
}
|
|
s.logger.Debug("desktop task lease found no matching queued task", fields...)
|
|
}
|
|
if taskID == nil {
|
|
return &LeaseDesktopTaskResponse{}, nil
|
|
}
|
|
return nil, s.classifyLeaseError(ctx, client, taskID)
|
|
}
|
|
fields := []zap.Field{
|
|
zap.String("client_id", client.ID.String()),
|
|
}
|
|
if req.Kind != nil {
|
|
fields = append(fields, zap.String("requested_kind", strings.TrimSpace(*req.Kind)))
|
|
}
|
|
if taskID != nil {
|
|
fields = append(fields, zap.String("task_id", taskID.String()))
|
|
}
|
|
s.logWarn("desktop task lease query failed", err, fields...)
|
|
return nil, response.ErrInternal(50088, "desktop_task_lease_failed", "failed to lease desktop task")
|
|
}
|
|
|
|
if _, attemptErr := s.repo.CreateAttempt(ctx, repository.CreateDesktopTaskAttemptParams{
|
|
DesktopID: attemptID,
|
|
TaskID: task.DesktopID,
|
|
ClientID: client.ID,
|
|
LeaseTokenHash: tokenHash,
|
|
}); attemptErr != nil {
|
|
s.logWarn("desktop task attempt insert failed", attemptErr, zap.String("task_id", task.DesktopID.String()))
|
|
}
|
|
|
|
s.publishTaskEvent(ctx, task, "task_leased")
|
|
|
|
if s.logger != nil {
|
|
s.logger.Info("desktop task leased",
|
|
zap.String("client_id", client.ID.String()),
|
|
zap.String("task_id", task.DesktopID.String()),
|
|
zap.String("kind", task.Kind),
|
|
zap.String("status", task.Status),
|
|
)
|
|
}
|
|
|
|
view := buildDesktopTaskView(task)
|
|
attemptIDText := attemptID.String()
|
|
return &LeaseDesktopTaskResponse{
|
|
Task: &view,
|
|
AttemptID: &attemptIDText,
|
|
LeaseToken: &rawToken,
|
|
LeaseExpiresAt: task.LeaseExpiresAt,
|
|
}, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) recheckQueuedPublishTasksForClient(ctx context.Context, client *repository.DesktopClient) error {
|
|
if s == nil || s.pool == nil || s.compliance == nil || client == nil {
|
|
return nil
|
|
}
|
|
rows, err := s.pool.Query(ctx, `
|
|
SELECT
|
|
j.desktop_id,
|
|
j.tenant_id,
|
|
j.workspace_id,
|
|
j.created_by_user_id,
|
|
j.article_id,
|
|
j.article_version_id,
|
|
COALESCE(array_agg(DISTINCT dt.platform_id) FILTER (WHERE dt.platform_id <> ''), '{}') AS platforms
|
|
FROM desktop_tasks dt
|
|
JOIN desktop_publish_jobs j ON j.desktop_id = dt.job_id
|
|
WHERE dt.target_client_id = $1
|
|
AND dt.kind = 'publish'
|
|
AND dt.status = 'queued'
|
|
AND j.status = 'queued'
|
|
AND j.article_id IS NOT NULL
|
|
AND j.article_version_id IS NOT NULL
|
|
GROUP BY j.desktop_id, j.tenant_id, j.workspace_id, j.created_by_user_id, j.article_id, j.article_version_id
|
|
ORDER BY MIN(dt.created_at)
|
|
LIMIT 5
|
|
`, client.ID)
|
|
if err != nil {
|
|
s.logWarn("desktop publish compliance recheck query failed", err, zap.String("client_id", client.ID.String()))
|
|
return response.ErrInternal(50118, "desktop_publish_compliance_recheck_failed", "failed to load publish jobs for compliance recheck")
|
|
}
|
|
defer rows.Close()
|
|
|
|
type candidate struct {
|
|
JobID uuid.UUID
|
|
TenantID int64
|
|
WorkspaceID int64
|
|
CreatedByUserID int64
|
|
ArticleID int64
|
|
ArticleVersionID int64
|
|
Platforms []string
|
|
}
|
|
candidates := make([]candidate, 0)
|
|
for rows.Next() {
|
|
var item candidate
|
|
if scanErr := rows.Scan(&item.JobID, &item.TenantID, &item.WorkspaceID, &item.CreatedByUserID, &item.ArticleID, &item.ArticleVersionID, &item.Platforms); scanErr != nil {
|
|
s.logWarn("desktop publish compliance recheck scan failed", scanErr, zap.String("client_id", client.ID.String()))
|
|
return response.ErrInternal(50118, "desktop_publish_compliance_recheck_failed", "failed to scan publish job for compliance recheck")
|
|
}
|
|
if item.WorkspaceID == client.WorkspaceID && item.TenantID == client.TenantID {
|
|
candidates = append(candidates, item)
|
|
}
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
s.logWarn("desktop publish compliance recheck rows failed", err, zap.String("client_id", client.ID.String()))
|
|
return response.ErrInternal(50118, "desktop_publish_compliance_recheck_failed", "failed to iterate publish jobs for compliance recheck")
|
|
}
|
|
|
|
for _, item := range candidates {
|
|
gate, gateErr := s.compliance.GateForPublish(ctx, tenantcompliance.GateForPublishRequest{
|
|
TenantID: item.TenantID,
|
|
ArticleID: item.ArticleID,
|
|
ArticleVersionID: item.ArticleVersionID,
|
|
ActorID: item.CreatedByUserID,
|
|
TargetPlatforms: item.Platforms,
|
|
TriggerSource: "scheduler_recheck",
|
|
})
|
|
if gateErr != nil && gate != nil && gate.Decision == sharedcompliance.GateDecisionBlock {
|
|
if markErr := s.compliance.MarkPublishJobBlockedByCompliance(ctx, item.JobID, gate.Result); markErr != nil {
|
|
return markErr
|
|
}
|
|
continue
|
|
}
|
|
if gateErr != nil {
|
|
return response.ErrServiceUnavailable(41004, "compliance_engine_unavailable", "scheduled publish compliance recheck failed")
|
|
}
|
|
if gate != nil && gate.Decision == sharedcompliance.GateDecisionBlock {
|
|
if markErr := s.compliance.MarkPublishJobBlockedByCompliance(ctx, item.JobID, gate.Result); markErr != nil {
|
|
return markErr
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
const desktopTaskRepositoryReturningColumns = `
|
|
t.desktop_id,
|
|
t.job_id,
|
|
t.tenant_id,
|
|
t.workspace_id,
|
|
t.target_account_id,
|
|
t.target_client_id,
|
|
t.platform_id,
|
|
t.kind,
|
|
t.payload,
|
|
t.status,
|
|
t.priority,
|
|
t.lane,
|
|
t.lane_weight,
|
|
t.source,
|
|
t.scheduler_group_key,
|
|
t.monitor_task_id,
|
|
t.supersedes_task_id,
|
|
t.control_flags,
|
|
t.interrupt_generation,
|
|
t.dedup_key,
|
|
t.active_attempt_id,
|
|
t.lease_expires_at,
|
|
t.attempts,
|
|
t.result,
|
|
t.error,
|
|
t.started_at,
|
|
t.interrupted_at,
|
|
t.interrupt_reason,
|
|
t.enqueued_at,
|
|
t.created_at,
|
|
t.updated_at`
|
|
|
|
type desktopTaskRepositoryScanner interface {
|
|
Scan(dest ...any) error
|
|
}
|
|
|
|
func (s *DesktopTaskService) leaseNextQueuedMonitorTask(
|
|
ctx context.Context,
|
|
client *repository.DesktopClient,
|
|
params repository.DesktopTaskLeaseParams,
|
|
) (*repository.DesktopTask, error) {
|
|
accountIDs, err := s.loadMonitorLeaseAccountIDs(ctx, client)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
hasAccountIDs := len(accountIDs) > 0
|
|
|
|
row := s.pool.QueryRow(ctx, `
|
|
WITH candidate AS (
|
|
SELECT dt.desktop_id
|
|
FROM desktop_tasks AS dt
|
|
WHERE dt.tenant_id = $1
|
|
AND dt.workspace_id = $2
|
|
AND dt.kind = 'monitor'
|
|
AND dt.status = 'queued'
|
|
AND (
|
|
dt.target_client_id = $3
|
|
OR ($4::boolean AND dt.target_account_id = ANY($5::uuid[]))
|
|
)
|
|
ORDER BY dt.lane_weight DESC,
|
|
dt.priority DESC,
|
|
COALESCE(dt.enqueued_at, dt.created_at) ASC,
|
|
dt.created_at ASC,
|
|
dt.desktop_id ASC
|
|
LIMIT 1
|
|
FOR UPDATE SKIP LOCKED
|
|
)
|
|
UPDATE desktop_tasks AS t
|
|
SET target_client_id = $3,
|
|
active_attempt_id = $6,
|
|
lease_token_hash = $7,
|
|
lease_expires_at = now() + interval '10 minutes',
|
|
status = 'in_progress',
|
|
attempts = t.attempts + 1,
|
|
updated_at = now()
|
|
FROM candidate
|
|
WHERE t.desktop_id = candidate.desktop_id
|
|
RETURNING `+desktopTaskRepositoryReturningColumns,
|
|
client.TenantID,
|
|
client.WorkspaceID,
|
|
client.ID,
|
|
hasAccountIDs,
|
|
accountIDs,
|
|
params.AttemptID,
|
|
params.LeaseTokenHash,
|
|
)
|
|
return scanRepositoryDesktopTask(row)
|
|
}
|
|
|
|
func (s *DesktopTaskService) leaseQueuedDesktopTaskByID(
|
|
ctx context.Context,
|
|
client *repository.DesktopClient,
|
|
desktopID uuid.UUID,
|
|
params repository.DesktopTaskLeaseParams,
|
|
) (*repository.DesktopTask, error) {
|
|
accountIDs, err := s.loadMonitorLeaseAccountIDs(ctx, client)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
hasAccountIDs := len(accountIDs) > 0
|
|
|
|
row := s.pool.QueryRow(ctx, `
|
|
WITH candidate AS (
|
|
SELECT dt.desktop_id
|
|
FROM desktop_tasks AS dt
|
|
WHERE dt.desktop_id = $1
|
|
AND dt.workspace_id = $2
|
|
AND dt.status = 'queued'
|
|
AND (
|
|
(
|
|
dt.kind = 'monitor'
|
|
AND dt.tenant_id = $3
|
|
AND (
|
|
dt.target_client_id = $4
|
|
OR ($5::boolean AND dt.target_account_id = ANY($6::uuid[]))
|
|
)
|
|
)
|
|
OR (
|
|
dt.kind <> 'monitor'
|
|
AND dt.target_client_id = $4
|
|
)
|
|
)
|
|
LIMIT 1
|
|
FOR UPDATE SKIP LOCKED
|
|
)
|
|
UPDATE desktop_tasks AS t
|
|
SET target_client_id = CASE WHEN t.kind = 'monitor' THEN $4 ELSE t.target_client_id END,
|
|
active_attempt_id = $7,
|
|
lease_token_hash = $8,
|
|
lease_expires_at = now() + interval '10 minutes',
|
|
status = 'in_progress',
|
|
attempts = t.attempts + 1,
|
|
updated_at = now()
|
|
FROM candidate
|
|
WHERE t.desktop_id = candidate.desktop_id
|
|
RETURNING `+desktopTaskRepositoryReturningColumns,
|
|
desktopID,
|
|
client.WorkspaceID,
|
|
client.TenantID,
|
|
client.ID,
|
|
hasAccountIDs,
|
|
accountIDs,
|
|
params.AttemptID,
|
|
params.LeaseTokenHash,
|
|
)
|
|
return scanRepositoryDesktopTask(row)
|
|
}
|
|
|
|
func (s *DesktopTaskService) loadMonitorLeaseAccountIDs(ctx context.Context, client *repository.DesktopClient) ([]uuid.UUID, error) {
|
|
if s == nil || client == nil || s.pool == nil {
|
|
return nil, nil
|
|
}
|
|
|
|
accountIDs := loadTrackedDesktopClientAccountIDs(ctx, s.redis, client.ID)
|
|
rows, err := s.pool.Query(ctx, `
|
|
SELECT desktop_id
|
|
FROM platform_accounts
|
|
WHERE tenant_id = $1
|
|
AND workspace_id = $2
|
|
AND client_id = $3
|
|
AND deleted_at IS NULL
|
|
AND platform_id = ANY($4::text[])
|
|
`, client.TenantID, client.WorkspaceID, client.ID, monitoringPlatformIDs(defaultMonitoringPlatforms))
|
|
if err != nil {
|
|
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to resolve monitor lease accounts")
|
|
}
|
|
defer rows.Close()
|
|
|
|
for rows.Next() {
|
|
var accountID uuid.UUID
|
|
if scanErr := rows.Scan(&accountID); scanErr != nil {
|
|
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to parse monitor lease accounts")
|
|
}
|
|
accountIDs = append(accountIDs, accountID)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to iterate monitor lease accounts")
|
|
}
|
|
return uniqueUUIDs(accountIDs), nil
|
|
}
|
|
|
|
func scanRepositoryDesktopTask(row desktopTaskRepositoryScanner) (*repository.DesktopTask, error) {
|
|
var (
|
|
task repository.DesktopTask
|
|
schedulerGroupKey pgtype.Text
|
|
monitorTaskID pgtype.Int8
|
|
supersedesTaskID pgtype.UUID
|
|
dedupKey pgtype.Text
|
|
activeAttemptID pgtype.UUID
|
|
leaseExpiresAt pgtype.Timestamptz
|
|
startedAt pgtype.Timestamptz
|
|
interruptedAt pgtype.Timestamptz
|
|
interruptReason pgtype.Text
|
|
enqueuedAt pgtype.Timestamptz
|
|
)
|
|
if err := row.Scan(
|
|
&task.DesktopID,
|
|
&task.JobID,
|
|
&task.TenantID,
|
|
&task.WorkspaceID,
|
|
&task.TargetAccountID,
|
|
&task.TargetClientID,
|
|
&task.Platform,
|
|
&task.Kind,
|
|
&task.Payload,
|
|
&task.Status,
|
|
&task.Priority,
|
|
&task.Lane,
|
|
&task.LaneWeight,
|
|
&task.Source,
|
|
&schedulerGroupKey,
|
|
&monitorTaskID,
|
|
&supersedesTaskID,
|
|
&task.ControlFlags,
|
|
&task.InterruptGeneration,
|
|
&dedupKey,
|
|
&activeAttemptID,
|
|
&leaseExpiresAt,
|
|
&task.Attempts,
|
|
&task.Result,
|
|
&task.Error,
|
|
&startedAt,
|
|
&interruptedAt,
|
|
&interruptReason,
|
|
&enqueuedAt,
|
|
&task.CreatedAt,
|
|
&task.UpdatedAt,
|
|
); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
task.SchedulerGroupKey = desktopTaskNullableText(schedulerGroupKey)
|
|
task.MonitorTaskID = desktopTaskNullableInt64(monitorTaskID)
|
|
task.SupersedesTaskID = desktopTaskNullableUUID(supersedesTaskID)
|
|
task.DedupKey = desktopTaskNullableText(dedupKey)
|
|
task.ActiveAttemptID = desktopTaskNullableUUID(activeAttemptID)
|
|
task.LeaseExpiresAt = desktopTaskNullableTime(leaseExpiresAt)
|
|
task.StartedAt = desktopTaskNullableTime(startedAt)
|
|
task.InterruptedAt = desktopTaskNullableTime(interruptedAt)
|
|
task.InterruptReason = desktopTaskNullableText(interruptReason)
|
|
if enqueuedAt.Valid {
|
|
task.EnqueuedAt = enqueuedAt.Time
|
|
}
|
|
return &task, nil
|
|
}
|
|
|
|
func desktopTaskNullableText(value pgtype.Text) *string {
|
|
if !value.Valid {
|
|
return nil
|
|
}
|
|
text := value.String
|
|
return &text
|
|
}
|
|
|
|
func desktopTaskNullableInt64(value pgtype.Int8) *int64 {
|
|
if !value.Valid {
|
|
return nil
|
|
}
|
|
number := value.Int64
|
|
return &number
|
|
}
|
|
|
|
func desktopTaskNullableUUID(value pgtype.UUID) *uuid.UUID {
|
|
if !value.Valid {
|
|
return nil
|
|
}
|
|
resolved, err := uuid.FromBytes(value.Bytes[:])
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
return &resolved
|
|
}
|
|
|
|
func desktopTaskNullableTime(value pgtype.Timestamptz) *time.Time {
|
|
if !value.Valid {
|
|
return nil
|
|
}
|
|
timestamp := value.Time
|
|
return ×tamp
|
|
}
|
|
|
|
func (s *DesktopTaskService) Extend(ctx context.Context, client *repository.DesktopClient, desktopID uuid.UUID, req ExtendDesktopTaskRequest) (*DesktopTaskView, error) {
|
|
if client == nil {
|
|
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
|
|
}
|
|
|
|
if err := s.dropStaleMonitorTasksForClient(ctx, client); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
task, err := s.repo.ExtendLease(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(req.LeaseToken)))
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil, response.ErrConflict(40983, "desktop_task_lease_invalid", "desktop task lease token is invalid or expired")
|
|
}
|
|
return nil, response.ErrInternal(50089, "desktop_task_extend_failed", "failed to extend desktop task lease")
|
|
}
|
|
|
|
s.publishTaskEvent(ctx, task, "task_extended")
|
|
|
|
view := buildDesktopTaskView(task)
|
|
return &view, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) Complete(ctx context.Context, client *repository.DesktopClient, desktopID uuid.UUID, req CompleteDesktopTaskRequest) (*DesktopTaskView, error) {
|
|
if client == nil {
|
|
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
|
|
}
|
|
|
|
finalStatus := normalizeDesktopTaskTerminalStatus(req.Status)
|
|
resultJSON, err := marshalOptionalJSON(req.Payload)
|
|
if err != nil {
|
|
return nil, response.ErrBadRequest(40086, "invalid_desktop_task_payload", "payload must be serializable")
|
|
}
|
|
errorJSON, err := marshalOptionalJSON(req.Error)
|
|
if err != nil {
|
|
return nil, response.ErrBadRequest(40087, "invalid_desktop_task_error", "error must be serializable")
|
|
}
|
|
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return nil, response.ErrInternal(50103, "desktop_task_complete_begin_failed", "failed to start desktop task completion")
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
|
|
taskRepo := repository.NewDesktopTaskRepository(tx)
|
|
previousTask, _ := taskRepo.GetByDesktopID(ctx, desktopID, client.WorkspaceID)
|
|
previousAttemptID := activeAttemptID(previousTask)
|
|
|
|
task, err := taskRepo.Complete(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(req.LeaseToken)), finalStatus, resultJSON, errorJSON)
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil, response.ErrConflict(40983, "desktop_task_lease_invalid", "desktop task lease token is invalid or expired")
|
|
}
|
|
return nil, response.ErrInternal(50091, "desktop_task_complete_failed", "failed to complete desktop task")
|
|
}
|
|
|
|
if previousAttemptID != nil {
|
|
if finishErr := taskRepo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
|
|
TaskID: task.DesktopID,
|
|
AttemptID: *previousAttemptID,
|
|
FinalStatus: &finalStatus,
|
|
Error: errorJSON,
|
|
}); finishErr != nil {
|
|
s.logWarn("desktop task attempt finish failed after complete", finishErr, zap.String("task_id", task.DesktopID.String()))
|
|
}
|
|
}
|
|
|
|
publishOutcome, err := syncDesktopPublishTaskState(ctx, tx, task)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
bulkFailure, err := bulkFailQueuedDesktopTasksForAuthorizationFailure(ctx, tx, task, errorJSON)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return nil, response.ErrInternal(50104, "desktop_task_complete_commit_failed", "failed to commit desktop task completion")
|
|
}
|
|
|
|
if publishOutcome != nil {
|
|
invalidateArticleCaches(ctx, s.cache, publishOutcome.TenantID, &publishOutcome.ArticleID)
|
|
if syncErr := s.syncMonitoringArticleAlias(ctx, publishOutcome.ArticleAlias); syncErr != nil {
|
|
s.logWarn("monitoring article alias sync failed after publish complete", syncErr, zap.Int64("article_id", publishOutcome.ArticleID))
|
|
}
|
|
}
|
|
for _, outcome := range bulkFailure.PublishOutcomes {
|
|
invalidateArticleCaches(ctx, s.cache, outcome.TenantID, &outcome.ArticleID)
|
|
if outcome.ArticleAlias != nil {
|
|
if syncErr := s.syncMonitoringArticleAlias(ctx, outcome.ArticleAlias); syncErr != nil {
|
|
s.logWarn("monitoring article alias sync failed after bulk authorization failure", syncErr, zap.Int64("article_id", outcome.ArticleID))
|
|
}
|
|
}
|
|
}
|
|
|
|
s.publishTaskEvent(ctx, task, "task_completed")
|
|
for _, failedTask := range bulkFailure.Tasks {
|
|
s.publishTaskEvent(ctx, failedTask, "task_completed")
|
|
}
|
|
|
|
view := buildDesktopTaskView(task)
|
|
return &view, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) Cancel(ctx context.Context, client *repository.DesktopClient, desktopID uuid.UUID, req CancelDesktopTaskRequest) (*DesktopTaskView, error) {
|
|
if client == nil {
|
|
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
|
|
}
|
|
|
|
errorJSON, err := marshalOptionalJSON(map[string]any{
|
|
"reason": optionalStringValue(req.Reason),
|
|
"source": "desktop_client",
|
|
})
|
|
if err != nil {
|
|
return nil, response.ErrBadRequest(40088, "invalid_desktop_task_cancel", "cancel payload must be serializable")
|
|
}
|
|
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return nil, response.ErrInternal(50107, "desktop_task_cancel_begin_failed", "failed to start desktop task cancel")
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
|
|
taskRepo := repository.NewDesktopTaskRepository(tx)
|
|
previousTask, _ := taskRepo.GetByDesktopID(ctx, desktopID, client.WorkspaceID)
|
|
previousAttemptID := activeAttemptID(previousTask)
|
|
|
|
var task *repository.DesktopTask
|
|
if req.LeaseToken != nil && strings.TrimSpace(*req.LeaseToken) != "" {
|
|
task, err = taskRepo.CancelByLease(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(*req.LeaseToken)), errorJSON)
|
|
} else {
|
|
task, err = taskRepo.CancelByClient(ctx, desktopID, client.ID, errorJSON)
|
|
}
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil, response.ErrConflict(40984, "desktop_task_cancel_conflict", "desktop task cannot be canceled in its current state")
|
|
}
|
|
return nil, response.ErrInternal(50092, "desktop_task_cancel_failed", "failed to cancel desktop task")
|
|
}
|
|
|
|
if previousAttemptID != nil && req.LeaseToken != nil && strings.TrimSpace(*req.LeaseToken) != "" {
|
|
if finishErr := taskRepo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
|
|
TaskID: task.DesktopID,
|
|
AttemptID: *previousAttemptID,
|
|
FinalStatus: literalStringPtr("aborted"),
|
|
Error: errorJSON,
|
|
}); finishErr != nil {
|
|
s.logWarn("desktop task attempt finish failed after cancel", finishErr, zap.String("task_id", task.DesktopID.String()))
|
|
}
|
|
}
|
|
|
|
var replacementTask *repository.DesktopTask
|
|
if shouldRequeuePreemptedMonitorTask(task, req.Reason) {
|
|
replacementTask, err = requeuePreemptedMonitorTask(ctx, tx, taskRepo, task)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return nil, response.ErrInternal(50108, "desktop_task_cancel_commit_failed", "failed to commit desktop task cancel")
|
|
}
|
|
|
|
s.publishTaskEvent(ctx, task, "task_canceled")
|
|
if replacementTask != nil {
|
|
s.publishTaskEvent(ctx, replacementTask, "task_available")
|
|
}
|
|
|
|
view := buildDesktopTaskView(task)
|
|
return &view, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) TenantCancel(ctx context.Context, actor auth.Actor, desktopID uuid.UUID, req CancelDesktopTaskRequest) (*DesktopTaskView, error) {
|
|
if actor.PrimaryWorkspaceID == 0 {
|
|
return nil, response.ErrUnauthorized(40132, "workspace_scope_missing", "workspace context is required")
|
|
}
|
|
|
|
errorJSON, err := marshalOptionalJSON(map[string]any{
|
|
"reason": optionalStringValue(req.Reason),
|
|
"source": "tenant_web",
|
|
})
|
|
if err != nil {
|
|
return nil, response.ErrBadRequest(40088, "invalid_desktop_task_cancel", "cancel payload must be serializable")
|
|
}
|
|
|
|
task, err := s.repo.TenantCancel(ctx, desktopID, actor.PrimaryWorkspaceID, errorJSON)
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil, response.ErrConflict(40984, "desktop_task_cancel_conflict", "desktop task cannot be canceled in its current state")
|
|
}
|
|
return nil, response.ErrInternal(50093, "desktop_task_tenant_cancel_failed", "failed to cancel desktop task")
|
|
}
|
|
|
|
s.publishTaskEvent(ctx, task, "task_canceled")
|
|
|
|
view := buildDesktopTaskView(task)
|
|
return &view, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) Reconcile(ctx context.Context, actor auth.Actor, desktopID uuid.UUID, req ReconcileDesktopTaskRequest) (*DesktopTaskView, error) {
|
|
if actor.PrimaryWorkspaceID == 0 {
|
|
return nil, response.ErrUnauthorized(40132, "workspace_scope_missing", "workspace context is required")
|
|
}
|
|
|
|
resultJSON, err := marshalOptionalJSON(req.Result)
|
|
if err != nil {
|
|
return nil, response.ErrBadRequest(40089, "invalid_desktop_task_reconcile_result", "result must be serializable")
|
|
}
|
|
errorJSON, err := marshalOptionalJSON(req.Error)
|
|
if err != nil {
|
|
return nil, response.ErrBadRequest(40090, "invalid_desktop_task_reconcile_error", "error must be serializable")
|
|
}
|
|
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return nil, response.ErrInternal(50105, "desktop_task_reconcile_begin_failed", "failed to start desktop task reconcile")
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
|
|
taskRepo := repository.NewDesktopTaskRepository(tx)
|
|
task, err := taskRepo.Reconcile(ctx, desktopID, actor.PrimaryWorkspaceID, req.Status, resultJSON, errorJSON)
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil, response.ErrConflict(40985, "desktop_task_reconcile_conflict", "desktop task is not waiting for reconcile")
|
|
}
|
|
return nil, response.ErrInternal(50094, "desktop_task_reconcile_failed", "failed to reconcile desktop task")
|
|
}
|
|
|
|
publishOutcome, err := syncDesktopPublishTaskState(ctx, tx, task)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return nil, response.ErrInternal(50106, "desktop_task_reconcile_commit_failed", "failed to commit desktop task reconcile")
|
|
}
|
|
|
|
if publishOutcome != nil {
|
|
invalidateArticleCaches(ctx, s.cache, publishOutcome.TenantID, &publishOutcome.ArticleID)
|
|
if syncErr := s.syncMonitoringArticleAlias(ctx, publishOutcome.ArticleAlias); syncErr != nil {
|
|
s.logWarn("monitoring article alias sync failed after publish reconcile", syncErr, zap.Int64("article_id", publishOutcome.ArticleID))
|
|
}
|
|
}
|
|
|
|
eventType := "task_reconciled"
|
|
if task.Kind == "monitor" && req.Status == "retry" {
|
|
eventType = "task_available"
|
|
}
|
|
s.publishTaskEvent(ctx, task, eventType)
|
|
|
|
view := buildDesktopTaskView(task)
|
|
return &view, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) ListPublishTasks(ctx context.Context, client *repository.DesktopClient, req ListPublishTasksRequest) (*DesktopPublishTaskList, error) {
|
|
if client == nil {
|
|
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
|
|
}
|
|
return s.listPublishTasksForOwner(ctx, publishTaskListOwner{
|
|
TenantID: client.TenantID,
|
|
WorkspaceID: client.WorkspaceID,
|
|
UserID: client.UserID,
|
|
}, req)
|
|
}
|
|
|
|
func (s *DesktopTaskService) ListTenantPublishTasks(ctx context.Context, actor auth.Actor, req ListPublishTasksRequest) (*DesktopPublishTaskList, error) {
|
|
workspaceID := auth.CurrentWorkspaceID(ctx)
|
|
if actor.TenantID == 0 || actor.UserID == 0 || workspaceID == 0 {
|
|
return nil, response.ErrUnauthorized(40132, "workspace_scope_missing", "workspace context is required")
|
|
}
|
|
return s.listPublishTasksForOwner(ctx, publishTaskListOwner{
|
|
TenantID: actor.TenantID,
|
|
WorkspaceID: workspaceID,
|
|
UserID: actor.UserID,
|
|
}, req)
|
|
}
|
|
|
|
func (s *DesktopTaskService) listPublishTasksForOwner(ctx context.Context, owner publishTaskListOwner, req ListPublishTasksRequest) (*DesktopPublishTaskList, error) {
|
|
if s.pool == nil {
|
|
return nil, response.ErrServiceUnavailable(50342, "desktop_publish_job_store_unavailable", "desktop publish job store is unavailable")
|
|
}
|
|
|
|
page := req.Page
|
|
if page <= 0 {
|
|
page = 1
|
|
}
|
|
pageSize := req.PageSize
|
|
if pageSize <= 0 || pageSize > 50 {
|
|
pageSize = 10
|
|
}
|
|
|
|
title := strings.TrimSpace(req.Title)
|
|
pendingStatuses := []string{"queued", "in_progress"}
|
|
historyStatuses := []string{"succeeded", "failed", "unknown", "aborted"}
|
|
|
|
pendingItems, err := s.listPublishTasksByStatuses(
|
|
ctx,
|
|
owner,
|
|
pendingStatuses,
|
|
title,
|
|
0,
|
|
0,
|
|
`CASE WHEN t.status = 'in_progress' THEN 0 ELSE 1 END ASC, t.created_at ASC, t.desktop_id DESC`,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
historyTotal, err := s.countPublishTasksByStatuses(ctx, owner, historyStatuses, title)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
offset := (page - 1) * pageSize
|
|
items := make([]DesktopTaskView, 0, pageSize)
|
|
total := len(pendingItems) + historyTotal
|
|
|
|
if offset < len(pendingItems) {
|
|
end := offset + pageSize
|
|
if end > len(pendingItems) {
|
|
end = len(pendingItems)
|
|
}
|
|
items = append(items, pendingItems[offset:end]...)
|
|
}
|
|
|
|
remaining := pageSize - len(items)
|
|
historyOffset := offset - len(pendingItems)
|
|
if historyOffset < 0 {
|
|
historyOffset = 0
|
|
}
|
|
|
|
if remaining > 0 && historyOffset < historyTotal {
|
|
historyItems, listErr := s.listPublishTasksByStatuses(
|
|
ctx,
|
|
owner,
|
|
historyStatuses,
|
|
title,
|
|
remaining,
|
|
historyOffset,
|
|
`t.updated_at DESC, t.desktop_id DESC`,
|
|
)
|
|
if listErr != nil {
|
|
return nil, listErr
|
|
}
|
|
items = append(items, historyItems...)
|
|
}
|
|
|
|
return &DesktopPublishTaskList{
|
|
Items: items,
|
|
Page: page,
|
|
PageSize: pageSize,
|
|
Total: total,
|
|
PendingCount: len(pendingItems),
|
|
HistoryTotal: historyTotal,
|
|
}, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) listPublishTasksByStatuses(
|
|
ctx context.Context,
|
|
owner publishTaskListOwner,
|
|
statuses []string,
|
|
title string,
|
|
limit int,
|
|
offset int,
|
|
orderBy string,
|
|
) ([]DesktopTaskView, error) {
|
|
query := `
|
|
SELECT
|
|
t.desktop_id,
|
|
t.job_id,
|
|
t.tenant_id,
|
|
t.workspace_id,
|
|
t.target_account_id,
|
|
t.target_client_id,
|
|
t.platform_id,
|
|
t.kind,
|
|
t.payload,
|
|
t.status,
|
|
t.dedup_key,
|
|
t.active_attempt_id,
|
|
t.lease_expires_at,
|
|
t.attempts,
|
|
t.result,
|
|
t.error,
|
|
t.created_at,
|
|
t.updated_at,
|
|
j.status,
|
|
j.compliance_blocked_record_id,
|
|
j.compliance_blocked_at,
|
|
j.compliance_blocked_reason
|
|
FROM desktop_tasks AS t
|
|
JOIN desktop_publish_jobs AS j
|
|
ON j.desktop_id = t.job_id
|
|
AND j.workspace_id = t.workspace_id
|
|
WHERE t.tenant_id = $1
|
|
AND t.workspace_id = $2
|
|
AND j.tenant_id = $1
|
|
AND j.created_by_user_id = $3
|
|
AND t.kind = 'publish'
|
|
AND t.status = ANY($4)
|
|
AND (
|
|
$5 = ''
|
|
OR COALESCE(t.payload->>'title', j.title, '') ILIKE '%' || $5 || '%'
|
|
)
|
|
`
|
|
args := []any{owner.TenantID, owner.WorkspaceID, owner.UserID, statuses, title}
|
|
|
|
if strings.TrimSpace(orderBy) != "" {
|
|
query += "\nORDER BY " + orderBy
|
|
}
|
|
if limit > 0 {
|
|
query += "\nLIMIT $6 OFFSET $7"
|
|
args = append(args, limit, offset)
|
|
}
|
|
|
|
rows, err := s.pool.Query(ctx, query, args...)
|
|
if err != nil {
|
|
return nil, response.ErrInternal(50109, "desktop_publish_tasks_query_failed", "failed to list desktop publish tasks")
|
|
}
|
|
defer rows.Close()
|
|
|
|
return scanDesktopTaskRows(rows)
|
|
}
|
|
|
|
func (s *DesktopTaskService) countPublishTasksByStatuses(
|
|
ctx context.Context,
|
|
owner publishTaskListOwner,
|
|
statuses []string,
|
|
title string,
|
|
) (int, error) {
|
|
var count int
|
|
err := s.pool.QueryRow(ctx, `
|
|
SELECT COUNT(1)
|
|
FROM desktop_tasks AS t
|
|
JOIN desktop_publish_jobs AS j
|
|
ON j.desktop_id = t.job_id
|
|
AND j.workspace_id = t.workspace_id
|
|
WHERE t.tenant_id = $1
|
|
AND t.workspace_id = $2
|
|
AND j.tenant_id = $1
|
|
AND j.created_by_user_id = $3
|
|
AND t.kind = 'publish'
|
|
AND t.status = ANY($4)
|
|
AND (
|
|
$5 = ''
|
|
OR COALESCE(t.payload->>'title', j.title, '') ILIKE '%' || $5 || '%'
|
|
)
|
|
`, owner.TenantID, owner.WorkspaceID, owner.UserID, statuses, title).Scan(&count)
|
|
if err != nil {
|
|
return 0, response.ErrInternal(50109, "desktop_publish_tasks_query_failed", "failed to list desktop publish tasks")
|
|
}
|
|
return count, nil
|
|
}
|
|
|
|
func scanDesktopTaskRows(rows pgx.Rows) ([]DesktopTaskView, error) {
|
|
items := make([]DesktopTaskView, 0)
|
|
for rows.Next() {
|
|
var (
|
|
desktopID uuid.UUID
|
|
jobID uuid.UUID
|
|
tenantID int64
|
|
workspaceID int64
|
|
targetAccountID uuid.UUID
|
|
targetClientID uuid.UUID
|
|
platform string
|
|
kind string
|
|
payload []byte
|
|
status string
|
|
dedupKey pgtype.Text
|
|
activeAttemptID pgtype.UUID
|
|
leaseExpiresAt pgtype.Timestamptz
|
|
attempts int32
|
|
resultJSON []byte
|
|
errorJSON []byte
|
|
createdAt time.Time
|
|
updatedAt time.Time
|
|
jobStatus string
|
|
blockedRecordID pgtype.Int8
|
|
blockedAt pgtype.Timestamptz
|
|
blockedReason pgtype.Text
|
|
)
|
|
if err := rows.Scan(
|
|
&desktopID,
|
|
&jobID,
|
|
&tenantID,
|
|
&workspaceID,
|
|
&targetAccountID,
|
|
&targetClientID,
|
|
&platform,
|
|
&kind,
|
|
&payload,
|
|
&status,
|
|
&dedupKey,
|
|
&activeAttemptID,
|
|
&leaseExpiresAt,
|
|
&attempts,
|
|
&resultJSON,
|
|
&errorJSON,
|
|
&createdAt,
|
|
&updatedAt,
|
|
&jobStatus,
|
|
&blockedRecordID,
|
|
&blockedAt,
|
|
&blockedReason,
|
|
); err != nil {
|
|
return nil, response.ErrInternal(50110, "desktop_publish_tasks_scan_failed", "failed to scan desktop publish tasks")
|
|
}
|
|
|
|
var dedupKeyText *string
|
|
if dedupKey.Valid {
|
|
value := dedupKey.String
|
|
dedupKeyText = &value
|
|
}
|
|
|
|
var activeAttemptIDText *string
|
|
if activeAttemptID.Valid {
|
|
value, convErr := uuid.FromBytes(activeAttemptID.Bytes[:])
|
|
if convErr == nil {
|
|
text := value.String()
|
|
activeAttemptIDText = &text
|
|
}
|
|
}
|
|
|
|
var leaseExpiresAtValue *time.Time
|
|
if leaseExpiresAt.Valid {
|
|
value := leaseExpiresAt.Time
|
|
leaseExpiresAtValue = &value
|
|
}
|
|
|
|
publishJobStatusText := strings.TrimSpace(jobStatus)
|
|
var publishJobStatus *string
|
|
if publishJobStatusText != "" {
|
|
publishJobStatus = &publishJobStatusText
|
|
}
|
|
|
|
var complianceBlockedRecordID *int64
|
|
if blockedRecordID.Valid {
|
|
value := blockedRecordID.Int64
|
|
complianceBlockedRecordID = &value
|
|
}
|
|
|
|
var complianceBlockedAt *time.Time
|
|
if blockedAt.Valid {
|
|
value := blockedAt.Time
|
|
complianceBlockedAt = &value
|
|
}
|
|
|
|
var complianceBlockedReason *string
|
|
if blockedReason.Valid {
|
|
value := blockedReason.String
|
|
complianceBlockedReason = &value
|
|
}
|
|
|
|
items = append(items, DesktopTaskView{
|
|
ID: desktopID.String(),
|
|
JobID: jobID.String(),
|
|
TenantID: tenantID,
|
|
WorkspaceID: workspaceID,
|
|
TargetAccountID: targetAccountID.String(),
|
|
TargetClientID: targetClientID.String(),
|
|
Platform: platform,
|
|
Kind: kind,
|
|
Payload: json.RawMessage(payload),
|
|
Status: normalizeDesktopTaskTerminalStatus(status),
|
|
PublishJobStatus: publishJobStatus,
|
|
ComplianceBlockedRecordID: complianceBlockedRecordID,
|
|
ComplianceBlockedAt: complianceBlockedAt,
|
|
ComplianceBlockedReason: complianceBlockedReason,
|
|
DedupKey: dedupKeyText,
|
|
ActiveAttemptID: activeAttemptIDText,
|
|
LeaseExpiresAt: leaseExpiresAtValue,
|
|
Attempts: int(attempts),
|
|
Result: json.RawMessage(resultJSON),
|
|
Error: json.RawMessage(errorJSON),
|
|
CreatedAt: createdAt,
|
|
UpdatedAt: updatedAt,
|
|
})
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, response.ErrInternal(50109, "desktop_publish_tasks_query_failed", "failed to list desktop publish tasks")
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) classifyLeaseError(ctx context.Context, client *repository.DesktopClient, taskID *uuid.UUID) error {
|
|
if taskID == nil {
|
|
return response.ErrConflict(40986, "desktop_task_unavailable", "no desktop task is currently available")
|
|
}
|
|
|
|
task, err := s.repo.GetByDesktopID(ctx, *taskID, client.WorkspaceID)
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return response.ErrNotFound(40482, "desktop_task_not_found", "desktop task not found")
|
|
}
|
|
return response.ErrInternal(50095, "desktop_task_lookup_failed", "failed to inspect desktop task state")
|
|
}
|
|
|
|
if task.TargetClientID != client.ID && !s.clientCanLeaseMonitorTask(ctx, client, task) {
|
|
return response.ErrForbidden(40381, "desktop_task_not_target_client", "desktop task is assigned to another desktop client")
|
|
}
|
|
|
|
if task.Status == "in_progress" {
|
|
return response.ErrConflict(40989, "desktop_task_already_in_progress", "desktop task is already in progress")
|
|
}
|
|
|
|
return response.ErrConflict(40991, "desktop_task_not_queued", "desktop task is not queued")
|
|
}
|
|
|
|
func (s *DesktopTaskService) clientCanLeaseMonitorTask(ctx context.Context, client *repository.DesktopClient, task *repository.DesktopTask) bool {
|
|
if s == nil || client == nil || task == nil || task.Kind != "monitor" || task.TargetAccountID == uuid.Nil {
|
|
return false
|
|
}
|
|
accountIDs, err := s.loadMonitorLeaseAccountIDs(ctx, client)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
for _, accountID := range accountIDs {
|
|
if accountID == task.TargetAccountID {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func buildDesktopTaskView(task *repository.DesktopTask) DesktopTaskView {
|
|
var activeAttemptID *string
|
|
if task.ActiveAttemptID != nil {
|
|
value := task.ActiveAttemptID.String()
|
|
activeAttemptID = &value
|
|
}
|
|
|
|
return DesktopTaskView{
|
|
ID: task.DesktopID.String(),
|
|
JobID: task.JobID.String(),
|
|
TenantID: task.TenantID,
|
|
WorkspaceID: task.WorkspaceID,
|
|
TargetAccountID: task.TargetAccountID.String(),
|
|
TargetClientID: task.TargetClientID.String(),
|
|
Platform: task.Platform,
|
|
Kind: task.Kind,
|
|
Payload: json.RawMessage(task.Payload),
|
|
Status: normalizeDesktopTaskTerminalStatus(task.Status),
|
|
DedupKey: task.DedupKey,
|
|
ActiveAttemptID: activeAttemptID,
|
|
LeaseExpiresAt: task.LeaseExpiresAt,
|
|
Attempts: task.Attempts,
|
|
Result: json.RawMessage(task.Result),
|
|
Error: json.RawMessage(task.Error),
|
|
CreatedAt: task.CreatedAt,
|
|
UpdatedAt: task.UpdatedAt,
|
|
}
|
|
}
|
|
|
|
func marshalOptionalJSON(value any) ([]byte, error) {
|
|
if value == nil {
|
|
return nil, nil
|
|
}
|
|
return json.Marshal(value)
|
|
}
|
|
|
|
func optionalStringValue(value *string) string {
|
|
if value == nil {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(*value)
|
|
}
|
|
|
|
func literalStringPtr(value string) *string {
|
|
return &value
|
|
}
|
|
|
|
func normalizeDesktopTaskTerminalStatus(status string) string {
|
|
switch strings.TrimSpace(status) {
|
|
case "unknown":
|
|
return "failed"
|
|
default:
|
|
return strings.TrimSpace(status)
|
|
}
|
|
}
|
|
|
|
type desktopTaskRecoveryMode string
|
|
|
|
const (
|
|
desktopTaskRecoveryModeStartup desktopTaskRecoveryMode = "startup"
|
|
desktopTaskRecoveryModeDisconnect desktopTaskRecoveryMode = "disconnect"
|
|
desktopTaskRecoveryModeLeaseExpiry desktopTaskRecoveryMode = "lease_expired"
|
|
)
|
|
|
|
type recoveredDesktopTaskLease struct {
|
|
TaskID uuid.UUID
|
|
WorkspaceID int64
|
|
Kind string
|
|
Status string
|
|
ActiveAttempt pgtype.UUID
|
|
}
|
|
|
|
func (s *DesktopTaskService) recoverClientTasksOnStartup(ctx context.Context, client *repository.DesktopClient) error {
|
|
return s.recoverClientTasks(ctx, client, desktopTaskRecoveryModeStartup)
|
|
}
|
|
|
|
func (s *DesktopTaskService) recoverClientTasksOnDisconnect(ctx context.Context, client *repository.DesktopClient) error {
|
|
return s.recoverClientTasks(ctx, client, desktopTaskRecoveryModeDisconnect)
|
|
}
|
|
|
|
func (s *DesktopTaskService) recoverExpiredClientTasks(ctx context.Context, client *repository.DesktopClient) error {
|
|
return s.recoverClientTasks(ctx, client, desktopTaskRecoveryModeLeaseExpiry)
|
|
}
|
|
|
|
func (s *DesktopTaskService) recoverClientTasks(
|
|
ctx context.Context,
|
|
client *repository.DesktopClient,
|
|
mode desktopTaskRecoveryMode,
|
|
) error {
|
|
if s == nil || client == nil || s.pool == nil {
|
|
return nil
|
|
}
|
|
|
|
publishErrorJSON, publishReason, publishMessage, err := desktopTaskRecoveryPayload(mode, "publish")
|
|
if err != nil {
|
|
return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
|
|
}
|
|
monitorErrorJSON, monitorReason, _, err := desktopTaskRecoveryPayload(mode, "monitor")
|
|
if err != nil {
|
|
return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode monitor desktop task recovery payload")
|
|
}
|
|
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return response.ErrInternal(50196, "desktop_task_recovery_begin_failed", "failed to start desktop task recovery")
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
|
|
repo := repository.NewDesktopTaskRepository(tx)
|
|
query := `
|
|
SELECT
|
|
desktop_id,
|
|
workspace_id,
|
|
kind,
|
|
active_attempt_id
|
|
FROM desktop_tasks
|
|
WHERE target_client_id = $1
|
|
AND workspace_id = $2
|
|
AND status = 'in_progress'
|
|
`
|
|
queryArgs := []any{client.ID, client.WorkspaceID}
|
|
if mode == desktopTaskRecoveryModeLeaseExpiry {
|
|
query += `
|
|
AND lease_expires_at IS NOT NULL
|
|
AND lease_expires_at < NOW()
|
|
`
|
|
}
|
|
query += `
|
|
FOR UPDATE
|
|
`
|
|
|
|
rows, err := tx.Query(ctx, query, queryArgs...)
|
|
if err != nil {
|
|
s.logWarn("desktop task recovery query failed", err, zap.String("client_id", client.ID.String()), zap.String("mode", string(mode)))
|
|
return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
|
|
}
|
|
defer rows.Close()
|
|
|
|
recovered := make([]recoveredDesktopTaskLease, 0)
|
|
for rows.Next() {
|
|
var item recoveredDesktopTaskLease
|
|
if scanErr := rows.Scan(
|
|
&item.TaskID,
|
|
&item.WorkspaceID,
|
|
&item.Kind,
|
|
&item.Status,
|
|
&item.ActiveAttempt,
|
|
); scanErr != nil {
|
|
return response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to parse recovered desktop tasks")
|
|
}
|
|
recovered = append(recovered, item)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to iterate recovered desktop tasks")
|
|
}
|
|
|
|
for index := range recovered {
|
|
item := &recovered[index]
|
|
if item.Kind == "monitor" {
|
|
if _, execErr := tx.Exec(ctx, `
|
|
UPDATE desktop_tasks
|
|
SET status = 'queued',
|
|
error = $2,
|
|
active_attempt_id = NULL,
|
|
lease_token_hash = NULL,
|
|
lease_expires_at = NULL,
|
|
interrupted_at = COALESCE(interrupted_at, NOW()),
|
|
interrupt_reason = $3,
|
|
enqueued_at = NOW(),
|
|
updated_at = NOW()
|
|
WHERE desktop_id = $1
|
|
`, item.TaskID, monitorErrorJSON, monitorReason); execErr != nil {
|
|
s.logWarn("desktop monitor task recovery update failed", execErr, zap.String("task_id", item.TaskID.String()), zap.String("mode", string(mode)))
|
|
return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
|
|
}
|
|
item.Status = "queued"
|
|
continue
|
|
}
|
|
|
|
if _, execErr := tx.Exec(ctx, `
|
|
UPDATE desktop_tasks
|
|
SET status = 'unknown',
|
|
error = $2,
|
|
active_attempt_id = NULL,
|
|
lease_token_hash = NULL,
|
|
lease_expires_at = NULL,
|
|
interrupted_at = COALESCE(interrupted_at, NOW()),
|
|
interrupt_reason = COALESCE(interrupt_reason, $3),
|
|
updated_at = NOW()
|
|
WHERE desktop_id = $1
|
|
`, item.TaskID, publishErrorJSON, publishReason); execErr != nil {
|
|
s.logWarn("desktop publish task recovery update failed", execErr, zap.String("task_id", item.TaskID.String()), zap.String("mode", string(mode)))
|
|
return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
|
|
}
|
|
item.Status = "unknown"
|
|
}
|
|
|
|
for _, item := range recovered {
|
|
if !item.ActiveAttempt.Valid {
|
|
continue
|
|
}
|
|
|
|
attemptID, convErr := uuid.FromBytes(item.ActiveAttempt.Bytes[:])
|
|
if convErr != nil {
|
|
s.logWarn("desktop recovery attempt id decode failed", convErr, zap.String("task_id", item.TaskID.String()))
|
|
continue
|
|
}
|
|
|
|
finalStatus := literalStringPtr(item.Status)
|
|
errorJSON := publishErrorJSON
|
|
if item.Kind == "monitor" {
|
|
finalStatus = literalStringPtr("aborted")
|
|
errorJSON = monitorErrorJSON
|
|
}
|
|
|
|
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
|
|
TaskID: item.TaskID,
|
|
AttemptID: attemptID,
|
|
FinalStatus: finalStatus,
|
|
Error: errorJSON,
|
|
}); finishErr != nil {
|
|
s.logWarn("desktop recovery attempt finish failed", finishErr, zap.String("task_id", item.TaskID.String()))
|
|
}
|
|
}
|
|
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return response.ErrInternal(50199, "desktop_task_recovery_commit_failed", "failed to commit desktop task recovery")
|
|
}
|
|
|
|
for _, item := range recovered {
|
|
task, getErr := s.repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID)
|
|
if getErr != nil {
|
|
s.logWarn("desktop recovery reload failed", getErr, zap.String("task_id", item.TaskID.String()))
|
|
continue
|
|
}
|
|
|
|
eventType := "task_completed"
|
|
if item.Kind == "monitor" {
|
|
eventType = "task_available"
|
|
}
|
|
s.publishTaskEvent(ctx, task, eventType)
|
|
}
|
|
|
|
if len(recovered) > 0 && s.logger != nil {
|
|
s.logger.Info("desktop tasks recovered",
|
|
zap.String("client_id", client.ID.String()),
|
|
zap.String("mode", string(mode)),
|
|
zap.Int("count", len(recovered)),
|
|
zap.String("publish_message", publishMessage),
|
|
)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func desktopTaskRecoveryPayload(mode desktopTaskRecoveryMode, kind string) ([]byte, string, string, error) {
|
|
reason := strings.TrimSpace(string(mode))
|
|
message := "desktop task was recovered after the client lost the active lease"
|
|
source := "desktop_task_recovery"
|
|
|
|
switch mode {
|
|
case desktopTaskRecoveryModeStartup:
|
|
source = "desktop_client_startup"
|
|
if kind == "monitor" {
|
|
message = "desktop client restarted while the monitor task was in progress; task has been re-queued"
|
|
} else {
|
|
message = "desktop client restarted while the publish task was in progress; task has been moved to unknown for manual reconcile"
|
|
}
|
|
case desktopTaskRecoveryModeDisconnect:
|
|
source = "desktop_client_offline"
|
|
if kind == "monitor" {
|
|
message = "desktop client went offline while the monitor task was in progress; task has been re-queued"
|
|
} else {
|
|
message = "desktop client went offline while the publish task was in progress; task has been moved to unknown for manual reconcile"
|
|
}
|
|
case desktopTaskRecoveryModeLeaseExpiry:
|
|
source = "desktop_task_lease_expiry"
|
|
if kind == "monitor" {
|
|
message = "desktop task lease expired before monitor completion; task has been re-queued"
|
|
} else {
|
|
message = "desktop task lease expired before publish completion; task has been moved to unknown for manual reconcile"
|
|
}
|
|
}
|
|
|
|
payload, err := marshalOptionalJSON(map[string]any{
|
|
"reason": reason,
|
|
"message": message,
|
|
"source": source,
|
|
})
|
|
if err != nil {
|
|
return nil, "", "", err
|
|
}
|
|
return payload, reason, message, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) dropStaleMonitorTasksForClient(ctx context.Context, client *repository.DesktopClient) error {
|
|
if s == nil || client == nil || s.pool == nil || s.monitoringPool == nil {
|
|
return nil
|
|
}
|
|
|
|
_, currentBusinessDate := monitoringBusinessDayAndDateAt(time.Now())
|
|
errorJSON, err := marshalOptionalJSON(map[string]any{
|
|
"reason": monitoringStaleTaskDropReason,
|
|
"message": monitoringStaleTaskDropError,
|
|
"current_business_date": currentBusinessDate,
|
|
"source": "business_day_rollover",
|
|
})
|
|
if err != nil {
|
|
return response.ErrInternal(50190, "desktop_task_stale_payload_failed", "failed to encode stale monitor desktop task payload")
|
|
}
|
|
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return response.ErrInternal(50191, "desktop_task_stale_begin_failed", "failed to start stale monitor desktop cleanup")
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
|
|
rows, err := tx.Query(ctx, `
|
|
WITH stale AS (
|
|
SELECT
|
|
desktop_id,
|
|
active_attempt_id
|
|
FROM desktop_tasks
|
|
WHERE target_client_id = $1
|
|
AND kind = 'monitor'
|
|
AND status IN ('queued', 'in_progress', 'unknown')
|
|
AND COALESCE(
|
|
NULLIF(payload ->> 'business_date', ''),
|
|
NULLIF(payload ->> 'businessDate', ''),
|
|
NULLIF(payload ->> 'metric_date', ''),
|
|
NULLIF(payload ->> 'date', '')
|
|
) < $2
|
|
FOR UPDATE
|
|
),
|
|
updated AS (
|
|
UPDATE desktop_tasks AS t
|
|
SET status = 'aborted',
|
|
error = $3,
|
|
active_attempt_id = NULL,
|
|
lease_token_hash = NULL,
|
|
lease_expires_at = NULL,
|
|
interrupted_at = COALESCE(interrupted_at, NOW()),
|
|
interrupt_reason = $4,
|
|
updated_at = NOW()
|
|
FROM stale
|
|
WHERE t.desktop_id = stale.desktop_id
|
|
RETURNING t.desktop_id
|
|
)
|
|
SELECT stale.desktop_id, stale.active_attempt_id
|
|
FROM stale
|
|
INNER JOIN updated ON updated.desktop_id = stale.desktop_id
|
|
`, client.ID, currentBusinessDate, errorJSON, monitoringStaleTaskDropReason)
|
|
if err != nil {
|
|
return response.ErrInternal(50192, "desktop_task_stale_query_failed", "failed to select stale monitor desktop tasks")
|
|
}
|
|
defer rows.Close()
|
|
|
|
repo := repository.NewDesktopTaskRepository(tx)
|
|
droppedTaskIDs := make([]uuid.UUID, 0)
|
|
for rows.Next() {
|
|
var (
|
|
taskID uuid.UUID
|
|
attemptID pgtype.UUID
|
|
)
|
|
if scanErr := rows.Scan(&taskID, &attemptID); scanErr != nil {
|
|
return response.ErrInternal(50193, "desktop_task_stale_scan_failed", "failed to parse stale monitor desktop tasks")
|
|
}
|
|
|
|
droppedTaskIDs = append(droppedTaskIDs, taskID)
|
|
|
|
if !attemptID.Valid {
|
|
continue
|
|
}
|
|
|
|
resolvedAttemptID, convErr := uuid.FromBytes(attemptID.Bytes[:])
|
|
if convErr != nil {
|
|
s.logWarn("desktop stale monitor attempt id decode failed", convErr, zap.String("task_id", taskID.String()))
|
|
continue
|
|
}
|
|
|
|
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
|
|
TaskID: taskID,
|
|
AttemptID: resolvedAttemptID,
|
|
FinalStatus: literalStringPtr("aborted"),
|
|
Error: errorJSON,
|
|
}); finishErr != nil {
|
|
s.logWarn("desktop stale monitor attempt finish failed", finishErr, zap.String("task_id", taskID.String()))
|
|
}
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return response.ErrInternal(50193, "desktop_task_stale_scan_failed", "failed to iterate stale monitor desktop tasks")
|
|
}
|
|
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return response.ErrInternal(50194, "desktop_task_stale_commit_failed", "failed to commit stale monitor desktop cleanup")
|
|
}
|
|
|
|
for _, taskID := range droppedTaskIDs {
|
|
task, getErr := s.repo.GetByDesktopID(ctx, taskID, client.WorkspaceID)
|
|
if getErr != nil {
|
|
s.logWarn("desktop stale monitor reload failed", getErr, zap.String("task_id", taskID.String()))
|
|
continue
|
|
}
|
|
s.publishTaskEvent(ctx, task, "task_canceled")
|
|
}
|
|
|
|
if _, err := s.monitoringPool.Exec(ctx, `
|
|
UPDATE monitoring_collect_tasks
|
|
SET status = 'skipped',
|
|
lease_token_hash = NULL,
|
|
leased_to_executor = NULL,
|
|
leased_at = NULL,
|
|
lease_expires_at = NULL,
|
|
callback_received_at = COALESCE(callback_received_at, NOW()),
|
|
completed_at = COALESCE(completed_at, NOW()),
|
|
skip_reason = $4,
|
|
error_message = COALESCE(NULLIF(error_message, ''), $5),
|
|
updated_at = NOW()
|
|
WHERE tenant_id = $1
|
|
AND collector_type = $2
|
|
AND COALESCE(execution_owner, 'legacy') = 'desktop_tasks'
|
|
AND target_client_id = $3
|
|
AND business_date < $6::date
|
|
AND callback_received_at IS NULL
|
|
AND status IN ('pending', 'leased', 'expired')
|
|
`, client.TenantID, monitoringCollectorType, client.ID, monitoringStaleTaskDropReason, monitoringStaleTaskDropError, currentBusinessDate); err != nil {
|
|
return response.ErrInternal(50041, "task_skip_failed", "failed to drop stale phase2 monitoring tasks")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func activeAttemptID(task *repository.DesktopTask) *uuid.UUID {
|
|
if task == nil || task.ActiveAttemptID == nil {
|
|
return nil
|
|
}
|
|
value := *task.ActiveAttemptID
|
|
return &value
|
|
}
|
|
|
|
func shouldRequeuePreemptedMonitorTask(task *repository.DesktopTask, reason *string) bool {
|
|
if task == nil || task.Kind != "monitor" {
|
|
return false
|
|
}
|
|
return strings.EqualFold(strings.TrimSpace(optionalStringValue(reason)), "collect_now_preempt")
|
|
}
|
|
|
|
func requeuePreemptedMonitorTask(
|
|
ctx context.Context,
|
|
tx pgx.Tx,
|
|
repo repository.DesktopTaskRepository,
|
|
task *repository.DesktopTask,
|
|
) (*repository.DesktopTask, error) {
|
|
if task == nil {
|
|
return nil, nil
|
|
}
|
|
replacementID := uuid.New()
|
|
replacementJobID := uuid.New()
|
|
if _, err := tx.Exec(ctx, `
|
|
INSERT INTO desktop_tasks (
|
|
desktop_id,
|
|
job_id,
|
|
tenant_id,
|
|
workspace_id,
|
|
target_account_id,
|
|
target_client_id,
|
|
platform_id,
|
|
kind,
|
|
payload,
|
|
status,
|
|
priority,
|
|
lane,
|
|
lane_weight,
|
|
source,
|
|
scheduler_group_key,
|
|
monitor_task_id,
|
|
supersedes_task_id,
|
|
control_flags,
|
|
interrupt_generation,
|
|
enqueued_at
|
|
)
|
|
VALUES (
|
|
$1,
|
|
$2,
|
|
$3,
|
|
$4,
|
|
$5,
|
|
$6,
|
|
$7,
|
|
'monitor',
|
|
$8,
|
|
'queued',
|
|
50,
|
|
'retry',
|
|
20,
|
|
'retry_recovery',
|
|
$9,
|
|
$10,
|
|
$11,
|
|
jsonb_build_object(
|
|
'interrupt_requested', false,
|
|
'requeued_at', NOW()::text,
|
|
'requeued_from_task_id', $11::text,
|
|
'requeued_reason', 'collect_now_preempt'
|
|
),
|
|
$12,
|
|
NOW()
|
|
)
|
|
`, replacementID, replacementJobID, task.TenantID, task.WorkspaceID, task.TargetAccountID, task.TargetClientID, task.Platform, task.Payload, nullableString(task.SchedulerGroupKey), nullableInt64(task.MonitorTaskID), task.DesktopID, task.InterruptGeneration); err != nil {
|
|
return nil, response.ErrInternal(50111, "desktop_task_requeue_failed", "failed to create replacement retry monitor desktop task")
|
|
}
|
|
|
|
next, err := repo.GetByDesktopID(ctx, replacementID, task.WorkspaceID)
|
|
if err != nil {
|
|
return nil, response.ErrInternal(50112, "desktop_task_lookup_failed", "failed to reload replacement retry monitor desktop task")
|
|
}
|
|
return next, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) publishTaskEvent(ctx context.Context, task *repository.DesktopTask, eventType string) {
|
|
publishDesktopTaskLifecycleEvent(ctx, s.messaging, s.logger, task, eventType)
|
|
}
|
|
|
|
func (s *DesktopTaskService) logWarn(message string, err error, fields ...zap.Field) {
|
|
if s.logger == nil || err == nil {
|
|
return
|
|
}
|
|
fields = append(fields, zap.Error(err))
|
|
s.logger.Warn(message, fields...)
|
|
}
|