root
6e0519a232
- Implemented a new BrandAssetCleanupWorker to handle the cleanup of brand-related assets after a brand is deleted. - Added SQL queries for cleaning up articles, keywords, questions, competitors, and monitoring data associated with a brand. - Introduced a new API endpoint to delete publish records. - Updated the router to include the new delete publish record endpoint. - Added tests for the BrandAssetCleanupWorker to ensure proper functionality. - Created migration scripts to support soft deletion of publish records and to add the brand asset cleanup scheduler job.
2325 lines
77 KiB
Go
2325 lines
77 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
goredis "github.com/redis/go-redis/v9"
|
|
"go.uber.org/zap"
|
|
|
|
"github.com/geo-platform/tenant-api/internal/shared/auth"
|
|
sharedcache "github.com/geo-platform/tenant-api/internal/shared/cache"
|
|
sharedcompliance "github.com/geo-platform/tenant-api/internal/shared/compliance"
|
|
"github.com/geo-platform/tenant-api/internal/shared/config"
|
|
"github.com/geo-platform/tenant-api/internal/shared/messaging/rabbitmq"
|
|
"github.com/geo-platform/tenant-api/internal/shared/response"
|
|
tenantcompliance "github.com/geo-platform/tenant-api/internal/tenant/compliance"
|
|
"github.com/geo-platform/tenant-api/internal/tenant/repository"
|
|
)
|
|
|
|
const desktopPublishMaxAttempts = 3
|
|
|
|
type DesktopTaskService struct {
|
|
pool *pgxpool.Pool
|
|
monitoringPool *pgxpool.Pool
|
|
repo repository.DesktopTaskRepository
|
|
cache sharedcache.Cache
|
|
redis *goredis.Client
|
|
messaging *rabbitmq.Client
|
|
logger *zap.Logger
|
|
compliance *tenantcompliance.Service
|
|
}
|
|
|
|
func NewDesktopTaskService(pool *pgxpool.Pool, monitoringPool *pgxpool.Pool, messaging *rabbitmq.Client, logger *zap.Logger) *DesktopTaskService {
|
|
return &DesktopTaskService{
|
|
pool: pool,
|
|
monitoringPool: monitoringPool,
|
|
repo: repository.NewDesktopTaskRepository(pool),
|
|
messaging: messaging,
|
|
logger: logger,
|
|
compliance: tenantcompliance.NewService(pool, config.NewStaticProvider(&config.Config{
|
|
Compliance: config.ComplianceConfig{Enabled: true},
|
|
}), logger).WithRabbitMQ(messaging),
|
|
}
|
|
}
|
|
|
|
func NewDesktopTaskServiceWithConfig(pool *pgxpool.Pool, monitoringPool *pgxpool.Pool, messaging *rabbitmq.Client, logger *zap.Logger, cfg config.Provider) *DesktopTaskService {
|
|
svc := NewDesktopTaskService(pool, monitoringPool, messaging, logger)
|
|
svc.compliance = tenantcompliance.NewService(pool, cfg, logger).WithRabbitMQ(messaging)
|
|
return svc
|
|
}
|
|
|
|
func (s *DesktopTaskService) WithCache(c sharedcache.Cache) *DesktopTaskService {
|
|
s.cache = c
|
|
return s
|
|
}
|
|
|
|
func (s *DesktopTaskService) WithRedis(redis *goredis.Client) *DesktopTaskService {
|
|
if s != nil {
|
|
s.redis = redis
|
|
}
|
|
return s
|
|
}
|
|
|
|
type DesktopTaskView struct {
|
|
ID string `json:"id"`
|
|
JobID string `json:"job_id"`
|
|
TenantID int64 `json:"tenant_id"`
|
|
WorkspaceID int64 `json:"workspace_id"`
|
|
TargetAccountID string `json:"target_account_id"`
|
|
TargetClientID string `json:"target_client_id"`
|
|
Platform string `json:"platform"`
|
|
Kind string `json:"kind"`
|
|
Payload json.RawMessage `json:"payload"`
|
|
Status string `json:"status"`
|
|
PublishJobStatus *string `json:"publish_job_status,omitempty"`
|
|
ComplianceBlockedRecordID *int64 `json:"compliance_blocked_record_id,omitempty"`
|
|
ComplianceBlockedAt *time.Time `json:"compliance_blocked_at,omitempty"`
|
|
ComplianceBlockedReason *string `json:"compliance_blocked_reason,omitempty"`
|
|
DedupKey *string `json:"dedup_key"`
|
|
ActiveAttemptID *string `json:"active_attempt_id"`
|
|
LeaseExpiresAt *time.Time `json:"lease_expires_at"`
|
|
Attempts int `json:"attempts"`
|
|
Result json.RawMessage `json:"result"`
|
|
Error json.RawMessage `json:"error"`
|
|
CreatedAt time.Time `json:"created_at"`
|
|
UpdatedAt time.Time `json:"updated_at"`
|
|
}
|
|
|
|
type LeaseDesktopTaskRequest struct {
|
|
TaskID *string `json:"task_id"`
|
|
Kind *string `json:"kind" binding:"omitempty,oneof=publish monitor"`
|
|
}
|
|
|
|
type LeaseDesktopTaskResponse struct {
|
|
Task *DesktopTaskView `json:"task"`
|
|
AttemptID *string `json:"attempt_id,omitempty"`
|
|
LeaseToken *string `json:"lease_token,omitempty"`
|
|
LeaseExpiresAt *time.Time `json:"lease_expires_at,omitempty"`
|
|
}
|
|
|
|
type ExtendDesktopTaskRequest struct {
|
|
LeaseToken string `json:"lease_token" binding:"required"`
|
|
}
|
|
|
|
type MarkPublishSubmitStartedRequest struct {
|
|
LeaseToken string `json:"lease_token" binding:"required"`
|
|
}
|
|
|
|
type CompleteDesktopTaskRequest struct {
|
|
LeaseToken string `json:"lease_token" binding:"required"`
|
|
Status string `json:"status" binding:"required,oneof=succeeded failed unknown"`
|
|
Payload map[string]any `json:"payload"`
|
|
Error map[string]any `json:"error"`
|
|
}
|
|
|
|
type CancelDesktopTaskRequest struct {
|
|
LeaseToken *string `json:"lease_token"`
|
|
Reason *string `json:"reason"`
|
|
}
|
|
|
|
type ReconcileDesktopTaskRequest struct {
|
|
Status string `json:"status" binding:"required,oneof=succeeded failed aborted retry"`
|
|
Result map[string]any `json:"result"`
|
|
Error map[string]any `json:"error"`
|
|
}
|
|
|
|
type ListPublishTasksRequest struct {
|
|
Page int `form:"page"`
|
|
PageSize int `form:"page_size"`
|
|
Title string `form:"title"`
|
|
}
|
|
|
|
type DesktopPublishTaskList struct {
|
|
Items []DesktopTaskView `json:"items"`
|
|
Page int `json:"page"`
|
|
PageSize int `json:"page_size"`
|
|
Total int `json:"total"`
|
|
PendingCount int `json:"pending_count"`
|
|
HistoryTotal int `json:"history_total"`
|
|
}
|
|
|
|
type publishTaskListOwner struct {
|
|
TenantID int64
|
|
WorkspaceID int64
|
|
UserID int64
|
|
}
|
|
|
|
func (s *DesktopTaskService) Lease(ctx context.Context, client *repository.DesktopClient, req LeaseDesktopTaskRequest, routeTaskID *uuid.UUID) (*LeaseDesktopTaskResponse, error) {
|
|
if client == nil {
|
|
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
|
|
}
|
|
markDesktopTaskConsumerPresent(ctx, s.redis, client.ID)
|
|
|
|
taskID := routeTaskID
|
|
if taskID == nil && req.TaskID != nil && strings.TrimSpace(*req.TaskID) != "" {
|
|
parsed, err := uuid.Parse(strings.TrimSpace(*req.TaskID))
|
|
if err != nil {
|
|
return nil, response.ErrBadRequest(40084, "invalid_desktop_task_id", "task id must be a uuid")
|
|
}
|
|
taskID = &parsed
|
|
}
|
|
|
|
if err := s.dropStaleMonitorTasksForClient(ctx, client); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := s.recoverExpiredClientTasks(ctx, client); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := s.recheckQueuedPublishTasksForClient(ctx, client); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
rawToken, tokenHash, err := newDesktopClientToken()
|
|
if err != nil {
|
|
return nil, response.ErrInternal(50087, "desktop_task_lease_token_failed", "failed to generate desktop task lease token")
|
|
}
|
|
|
|
attemptID := uuid.New()
|
|
leaseParams := repository.DesktopTaskLeaseParams{
|
|
AttemptID: attemptID,
|
|
LeaseTokenHash: tokenHash,
|
|
}
|
|
|
|
if s.logger != nil {
|
|
fields := []zap.Field{
|
|
zap.String("client_id", client.ID.String()),
|
|
}
|
|
if req.Kind != nil {
|
|
fields = append(fields, zap.String("requested_kind", strings.TrimSpace(*req.Kind)))
|
|
}
|
|
if taskID != nil {
|
|
fields = append(fields, zap.String("task_id", taskID.String()))
|
|
}
|
|
s.logger.Debug("desktop task lease requested", fields...)
|
|
}
|
|
|
|
var task *repository.DesktopTask
|
|
switch {
|
|
case taskID != nil:
|
|
task, err = s.leaseQueuedDesktopTaskByID(ctx, client, *taskID, leaseParams)
|
|
case req.Kind != nil && strings.TrimSpace(*req.Kind) == "monitor":
|
|
task, err = s.leaseNextQueuedMonitorTask(ctx, client, leaseParams)
|
|
case req.Kind != nil && strings.TrimSpace(*req.Kind) == "publish":
|
|
task, err = s.leaseNextQueuedPublishTask(ctx, client, leaseParams)
|
|
default:
|
|
task, err = s.leaseNextQueuedPublishTask(ctx, client, leaseParams)
|
|
}
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
if s.logger != nil {
|
|
fields := []zap.Field{
|
|
zap.String("client_id", client.ID.String()),
|
|
}
|
|
if req.Kind != nil {
|
|
fields = append(fields, zap.String("requested_kind", strings.TrimSpace(*req.Kind)))
|
|
}
|
|
if taskID != nil {
|
|
fields = append(fields, zap.String("task_id", taskID.String()))
|
|
}
|
|
s.logger.Debug("desktop task lease found no matching queued task", fields...)
|
|
}
|
|
if taskID == nil {
|
|
return &LeaseDesktopTaskResponse{}, nil
|
|
}
|
|
return nil, s.classifyLeaseError(ctx, client, taskID)
|
|
}
|
|
fields := []zap.Field{
|
|
zap.String("client_id", client.ID.String()),
|
|
}
|
|
if req.Kind != nil {
|
|
fields = append(fields, zap.String("requested_kind", strings.TrimSpace(*req.Kind)))
|
|
}
|
|
if taskID != nil {
|
|
fields = append(fields, zap.String("task_id", taskID.String()))
|
|
}
|
|
s.logWarn("desktop task lease query failed", err, fields...)
|
|
return nil, response.ErrInternal(50088, "desktop_task_lease_failed", "failed to lease desktop task")
|
|
}
|
|
|
|
if _, attemptErr := s.repo.CreateAttempt(ctx, repository.CreateDesktopTaskAttemptParams{
|
|
DesktopID: attemptID,
|
|
TaskID: task.DesktopID,
|
|
ClientID: client.ID,
|
|
LeaseTokenHash: tokenHash,
|
|
}); attemptErr != nil {
|
|
s.logWarn("desktop task attempt insert failed", attemptErr, zap.String("task_id", task.DesktopID.String()))
|
|
}
|
|
|
|
s.publishTaskEvent(ctx, task, "task_leased")
|
|
|
|
if s.logger != nil {
|
|
s.logger.Info("desktop task leased",
|
|
zap.String("client_id", client.ID.String()),
|
|
zap.String("task_id", task.DesktopID.String()),
|
|
zap.String("kind", task.Kind),
|
|
zap.String("status", task.Status),
|
|
)
|
|
}
|
|
|
|
view := buildDesktopTaskView(task)
|
|
attemptIDText := attemptID.String()
|
|
return &LeaseDesktopTaskResponse{
|
|
Task: &view,
|
|
AttemptID: &attemptIDText,
|
|
LeaseToken: &rawToken,
|
|
LeaseExpiresAt: task.LeaseExpiresAt,
|
|
}, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) recheckQueuedPublishTasksForClient(ctx context.Context, client *repository.DesktopClient) error {
|
|
if s == nil || s.pool == nil || s.compliance == nil || client == nil {
|
|
return nil
|
|
}
|
|
rows, err := s.pool.Query(ctx, `
|
|
SELECT
|
|
j.desktop_id,
|
|
j.tenant_id,
|
|
j.workspace_id,
|
|
j.created_by_user_id,
|
|
j.article_id,
|
|
j.article_version_id,
|
|
COALESCE(array_agg(DISTINCT dt.platform_id) FILTER (WHERE dt.platform_id <> ''), '{}') AS platforms,
|
|
BOOL_OR(a.deleted_at IS NOT NULL) AS article_deleted
|
|
FROM desktop_tasks dt
|
|
JOIN desktop_publish_jobs j ON j.desktop_id = dt.job_id
|
|
LEFT JOIN articles a ON a.id = j.article_id AND a.tenant_id = j.tenant_id
|
|
WHERE dt.target_client_id = $1
|
|
AND dt.kind = 'publish'
|
|
AND dt.status = 'queued'
|
|
AND j.status = 'queued'
|
|
AND j.article_id IS NOT NULL
|
|
AND j.article_version_id IS NOT NULL
|
|
GROUP BY j.desktop_id, j.tenant_id, j.workspace_id, j.created_by_user_id, j.article_id, j.article_version_id
|
|
ORDER BY MIN(dt.created_at)
|
|
LIMIT 5
|
|
`, client.ID)
|
|
if err != nil {
|
|
s.logWarn("desktop publish compliance recheck query failed", err, zap.String("client_id", client.ID.String()))
|
|
return response.ErrInternal(50118, "desktop_publish_compliance_recheck_failed", "failed to load publish jobs for compliance recheck")
|
|
}
|
|
defer rows.Close()
|
|
|
|
type candidate struct {
|
|
JobID uuid.UUID
|
|
TenantID int64
|
|
WorkspaceID int64
|
|
CreatedByUserID int64
|
|
ArticleID int64
|
|
ArticleVersionID int64
|
|
Platforms []string
|
|
ArticleDeleted bool
|
|
}
|
|
candidates := make([]candidate, 0)
|
|
for rows.Next() {
|
|
var item candidate
|
|
if scanErr := rows.Scan(&item.JobID, &item.TenantID, &item.WorkspaceID, &item.CreatedByUserID, &item.ArticleID, &item.ArticleVersionID, &item.Platforms, &item.ArticleDeleted); scanErr != nil {
|
|
s.logWarn("desktop publish compliance recheck scan failed", scanErr, zap.String("client_id", client.ID.String()))
|
|
return response.ErrInternal(50118, "desktop_publish_compliance_recheck_failed", "failed to scan publish job for compliance recheck")
|
|
}
|
|
if item.WorkspaceID == client.WorkspaceID && item.TenantID == client.TenantID {
|
|
candidates = append(candidates, item)
|
|
}
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
s.logWarn("desktop publish compliance recheck rows failed", err, zap.String("client_id", client.ID.String()))
|
|
return response.ErrInternal(50118, "desktop_publish_compliance_recheck_failed", "failed to iterate publish jobs for compliance recheck")
|
|
}
|
|
|
|
for _, item := range candidates {
|
|
if item.ArticleDeleted {
|
|
if cancelErr := s.cancelUnavailableArticleQueuedPublishTasks(ctx, item.TenantID, item.ArticleID); cancelErr != nil {
|
|
return cancelErr
|
|
}
|
|
continue
|
|
}
|
|
gate, gateErr := s.compliance.GateForPublish(ctx, tenantcompliance.GateForPublishRequest{
|
|
TenantID: item.TenantID,
|
|
ArticleID: item.ArticleID,
|
|
ArticleVersionID: item.ArticleVersionID,
|
|
ActorID: item.CreatedByUserID,
|
|
TargetPlatforms: item.Platforms,
|
|
TriggerSource: "scheduler_recheck",
|
|
})
|
|
if gateErr != nil && gate != nil && gate.Decision == sharedcompliance.GateDecisionBlock {
|
|
if markErr := s.compliance.MarkPublishJobBlockedByCompliance(ctx, item.JobID, gate.Result); markErr != nil {
|
|
return markErr
|
|
}
|
|
continue
|
|
}
|
|
if gateErr != nil {
|
|
if isComplianceInvalidArticleVersionError(gateErr) {
|
|
if cancelErr := s.cancelUnavailableArticleQueuedPublishTasks(ctx, item.TenantID, item.ArticleID); cancelErr != nil {
|
|
return cancelErr
|
|
}
|
|
continue
|
|
}
|
|
return response.ErrServiceUnavailable(41004, "compliance_engine_unavailable", "scheduled publish compliance recheck failed")
|
|
}
|
|
if gate != nil && gate.Decision == sharedcompliance.GateDecisionBlock {
|
|
if markErr := s.compliance.MarkPublishJobBlockedByCompliance(ctx, item.JobID, gate.Result); markErr != nil {
|
|
return markErr
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) cancelUnavailableArticleQueuedPublishTasks(ctx context.Context, tenantID, articleID int64) error {
|
|
if s == nil || s.pool == nil || tenantID <= 0 || articleID <= 0 {
|
|
return nil
|
|
}
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return response.ErrInternal(50122, "publish_task_cancel_unavailable_article_begin_failed", "failed to start unavailable article publish task cleanup")
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
|
|
taskRefs, err := queuedPublishTaskRefsForArticle(ctx, tx, tenantID, articleID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := cancelQueuedPublishTasksForUnavailableArticle(ctx, tx, tenantID, articleID); err != nil {
|
|
return err
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return response.ErrInternal(50123, "publish_task_cancel_unavailable_article_commit_failed", "failed to commit unavailable article publish task cleanup")
|
|
}
|
|
|
|
for _, ref := range taskRefs {
|
|
task, getErr := s.repo.GetByDesktopID(ctx, ref.TaskID, ref.WorkspaceID)
|
|
if getErr == nil && task != nil {
|
|
s.publishTaskEvent(ctx, task, "task_canceled")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type desktopTaskRef struct {
|
|
TaskID uuid.UUID
|
|
WorkspaceID int64
|
|
}
|
|
|
|
func queuedPublishTaskRefsForArticle(ctx context.Context, tx pgx.Tx, tenantID, articleID int64) ([]desktopTaskRef, error) {
|
|
if tx == nil || tenantID <= 0 || articleID <= 0 {
|
|
return nil, nil
|
|
}
|
|
rows, err := tx.Query(ctx, `
|
|
SELECT dt.desktop_id, dt.workspace_id
|
|
FROM desktop_tasks dt
|
|
JOIN desktop_publish_jobs j
|
|
ON j.desktop_id = dt.job_id
|
|
AND j.tenant_id = dt.tenant_id
|
|
WHERE dt.tenant_id = $1
|
|
AND dt.kind = 'publish'
|
|
AND dt.status = 'queued'
|
|
AND j.article_id = $2
|
|
AND j.status = 'queued'
|
|
ORDER BY dt.created_at ASC, dt.desktop_id ASC
|
|
`, tenantID, articleID)
|
|
if err != nil {
|
|
return nil, response.ErrInternal(50124, "publish_task_cancel_unavailable_article_lookup_failed", "failed to inspect unavailable article publish tasks")
|
|
}
|
|
defer rows.Close()
|
|
|
|
taskRefs := make([]desktopTaskRef, 0)
|
|
for rows.Next() {
|
|
var ref desktopTaskRef
|
|
if scanErr := rows.Scan(&ref.TaskID, &ref.WorkspaceID); scanErr != nil {
|
|
return nil, response.ErrInternal(50124, "publish_task_cancel_unavailable_article_lookup_failed", "failed to scan unavailable article publish tasks")
|
|
}
|
|
taskRefs = append(taskRefs, ref)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, response.ErrInternal(50124, "publish_task_cancel_unavailable_article_lookup_failed", "failed to iterate unavailable article publish tasks")
|
|
}
|
|
return taskRefs, nil
|
|
}
|
|
|
|
func isComplianceInvalidArticleVersionError(err error) bool {
|
|
appErr, ok := err.(*response.AppError)
|
|
return ok && appErr.Code == 41005 && appErr.Message == "invalid_article_version"
|
|
}
|
|
|
|
const desktopTaskRepositoryReturningColumns = `
|
|
t.desktop_id,
|
|
t.job_id,
|
|
t.tenant_id,
|
|
t.workspace_id,
|
|
t.target_account_id,
|
|
t.target_client_id,
|
|
t.platform_id,
|
|
t.kind,
|
|
t.payload,
|
|
t.status,
|
|
t.priority,
|
|
t.lane,
|
|
t.lane_weight,
|
|
t.source,
|
|
t.scheduler_group_key,
|
|
t.monitor_task_id,
|
|
t.supersedes_task_id,
|
|
t.control_flags,
|
|
t.interrupt_generation,
|
|
t.dedup_key,
|
|
t.active_attempt_id,
|
|
t.lease_expires_at,
|
|
t.attempts,
|
|
t.result,
|
|
t.error,
|
|
t.started_at,
|
|
t.interrupted_at,
|
|
t.interrupt_reason,
|
|
t.enqueued_at,
|
|
t.created_at,
|
|
t.updated_at`
|
|
|
|
type desktopTaskRepositoryScanner interface {
|
|
Scan(dest ...any) error
|
|
}
|
|
|
|
func (s *DesktopTaskService) leaseNextQueuedMonitorTask(
|
|
ctx context.Context,
|
|
client *repository.DesktopClient,
|
|
params repository.DesktopTaskLeaseParams,
|
|
) (*repository.DesktopTask, error) {
|
|
accountIDs, err := s.loadMonitorLeaseAccountIDs(ctx, client)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
hasAccountIDs := len(accountIDs) > 0
|
|
|
|
row := s.pool.QueryRow(ctx, `
|
|
WITH candidate AS (
|
|
SELECT dt.desktop_id
|
|
FROM desktop_tasks AS dt
|
|
WHERE dt.tenant_id = $1
|
|
AND dt.workspace_id = $2
|
|
AND dt.kind = 'monitor'
|
|
AND dt.status = 'queued'
|
|
AND (
|
|
dt.target_client_id = $3
|
|
OR ($4::boolean AND dt.target_account_id = ANY($5::uuid[]))
|
|
)
|
|
ORDER BY dt.lane_weight DESC,
|
|
dt.priority DESC,
|
|
COALESCE(dt.enqueued_at, dt.created_at) ASC,
|
|
dt.created_at ASC,
|
|
dt.desktop_id ASC
|
|
LIMIT 1
|
|
FOR UPDATE SKIP LOCKED
|
|
)
|
|
UPDATE desktop_tasks AS t
|
|
SET target_client_id = $3,
|
|
active_attempt_id = $6,
|
|
lease_token_hash = $7,
|
|
lease_expires_at = now() + interval '10 minutes',
|
|
status = 'in_progress',
|
|
attempts = t.attempts + 1,
|
|
updated_at = now()
|
|
FROM candidate
|
|
WHERE t.desktop_id = candidate.desktop_id
|
|
RETURNING `+desktopTaskRepositoryReturningColumns,
|
|
client.TenantID,
|
|
client.WorkspaceID,
|
|
client.ID,
|
|
hasAccountIDs,
|
|
accountIDs,
|
|
params.AttemptID,
|
|
params.LeaseTokenHash,
|
|
)
|
|
return scanRepositoryDesktopTask(row)
|
|
}
|
|
|
|
func (s *DesktopTaskService) leaseNextQueuedPublishTask(
|
|
ctx context.Context,
|
|
client *repository.DesktopClient,
|
|
params repository.DesktopTaskLeaseParams,
|
|
) (*repository.DesktopTask, error) {
|
|
row := s.pool.QueryRow(ctx, `
|
|
WITH candidate AS (
|
|
SELECT dt.desktop_id
|
|
FROM desktop_tasks AS dt
|
|
WHERE dt.target_client_id = $1
|
|
AND dt.kind = 'publish'
|
|
AND dt.status = 'queued'
|
|
AND dt.attempts < $4
|
|
AND EXISTS (
|
|
SELECT 1
|
|
FROM platform_accounts AS pa
|
|
WHERE pa.desktop_id = dt.target_account_id
|
|
AND pa.workspace_id = dt.workspace_id
|
|
AND pa.client_id = dt.target_client_id
|
|
AND pa.deleted_at IS NULL
|
|
AND pa.delete_requested_at IS NULL
|
|
)
|
|
AND (
|
|
NOT EXISTS (
|
|
SELECT 1
|
|
FROM desktop_publish_jobs AS j
|
|
WHERE j.desktop_id = dt.job_id
|
|
AND j.status <> 'queued'
|
|
)
|
|
)
|
|
ORDER BY dt.lane_weight DESC,
|
|
dt.priority DESC,
|
|
COALESCE(dt.enqueued_at, dt.created_at) ASC,
|
|
dt.created_at ASC,
|
|
dt.desktop_id ASC
|
|
LIMIT 1
|
|
FOR UPDATE OF dt SKIP LOCKED
|
|
)
|
|
UPDATE desktop_tasks AS t
|
|
SET active_attempt_id = $2,
|
|
lease_token_hash = $3,
|
|
lease_expires_at = now() + interval '3 minutes',
|
|
status = 'in_progress',
|
|
attempts = t.attempts + 1,
|
|
started_at = COALESCE(t.started_at, now()),
|
|
updated_at = now()
|
|
FROM candidate
|
|
WHERE t.desktop_id = candidate.desktop_id
|
|
RETURNING `+desktopTaskRepositoryReturningColumns,
|
|
client.ID,
|
|
params.AttemptID,
|
|
params.LeaseTokenHash,
|
|
desktopPublishMaxAttempts,
|
|
)
|
|
return scanRepositoryDesktopTask(row)
|
|
}
|
|
|
|
func (s *DesktopTaskService) leaseQueuedDesktopTaskByID(
|
|
ctx context.Context,
|
|
client *repository.DesktopClient,
|
|
desktopID uuid.UUID,
|
|
params repository.DesktopTaskLeaseParams,
|
|
) (*repository.DesktopTask, error) {
|
|
accountIDs, err := s.loadMonitorLeaseAccountIDs(ctx, client)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
hasAccountIDs := len(accountIDs) > 0
|
|
|
|
row := s.pool.QueryRow(ctx, `
|
|
WITH candidate AS (
|
|
SELECT dt.desktop_id
|
|
FROM desktop_tasks AS dt
|
|
WHERE dt.desktop_id = $1
|
|
AND dt.workspace_id = $2
|
|
AND dt.status = 'queued'
|
|
AND (dt.kind <> 'publish' OR dt.attempts < $9)
|
|
AND (
|
|
dt.kind <> 'publish'
|
|
OR EXISTS (
|
|
SELECT 1
|
|
FROM platform_accounts AS pa
|
|
WHERE pa.desktop_id = dt.target_account_id
|
|
AND pa.workspace_id = dt.workspace_id
|
|
AND pa.client_id = dt.target_client_id
|
|
AND pa.deleted_at IS NULL
|
|
AND pa.delete_requested_at IS NULL
|
|
)
|
|
)
|
|
AND (
|
|
(
|
|
dt.kind = 'monitor'
|
|
AND dt.tenant_id = $3
|
|
AND (
|
|
dt.target_client_id = $4
|
|
OR ($5::boolean AND dt.target_account_id = ANY($6::uuid[]))
|
|
)
|
|
)
|
|
OR (
|
|
dt.kind <> 'monitor'
|
|
AND dt.target_client_id = $4
|
|
)
|
|
)
|
|
LIMIT 1
|
|
FOR UPDATE SKIP LOCKED
|
|
)
|
|
UPDATE desktop_tasks AS t
|
|
SET target_client_id = CASE WHEN t.kind = 'monitor' THEN $4 ELSE t.target_client_id END,
|
|
active_attempt_id = $7,
|
|
lease_token_hash = $8,
|
|
lease_expires_at = now() + CASE WHEN t.kind = 'publish' THEN interval '3 minutes' ELSE interval '10 minutes' END,
|
|
status = 'in_progress',
|
|
attempts = t.attempts + 1,
|
|
started_at = COALESCE(t.started_at, now()),
|
|
updated_at = now()
|
|
FROM candidate
|
|
WHERE t.desktop_id = candidate.desktop_id
|
|
RETURNING `+desktopTaskRepositoryReturningColumns,
|
|
desktopID,
|
|
client.WorkspaceID,
|
|
client.TenantID,
|
|
client.ID,
|
|
hasAccountIDs,
|
|
accountIDs,
|
|
params.AttemptID,
|
|
params.LeaseTokenHash,
|
|
desktopPublishMaxAttempts,
|
|
)
|
|
return scanRepositoryDesktopTask(row)
|
|
}
|
|
|
|
func (s *DesktopTaskService) loadMonitorLeaseAccountIDs(ctx context.Context, client *repository.DesktopClient) ([]uuid.UUID, error) {
|
|
if s == nil || client == nil || s.pool == nil {
|
|
return nil, nil
|
|
}
|
|
|
|
accountIDs := loadTrackedDesktopClientAccountIDs(ctx, s.redis, client.ID)
|
|
rows, err := s.pool.Query(ctx, `
|
|
SELECT desktop_id
|
|
FROM platform_accounts
|
|
WHERE tenant_id = $1
|
|
AND workspace_id = $2
|
|
AND client_id = $3
|
|
AND deleted_at IS NULL
|
|
AND platform_id = ANY($4::text[])
|
|
`, client.TenantID, client.WorkspaceID, client.ID, monitoringPlatformIDs(defaultMonitoringPlatforms))
|
|
if err != nil {
|
|
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to resolve monitor lease accounts")
|
|
}
|
|
defer rows.Close()
|
|
|
|
for rows.Next() {
|
|
var accountID uuid.UUID
|
|
if scanErr := rows.Scan(&accountID); scanErr != nil {
|
|
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to parse monitor lease accounts")
|
|
}
|
|
accountIDs = append(accountIDs, accountID)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to iterate monitor lease accounts")
|
|
}
|
|
return uniqueUUIDs(accountIDs), nil
|
|
}
|
|
|
|
func scanRepositoryDesktopTask(row desktopTaskRepositoryScanner) (*repository.DesktopTask, error) {
|
|
var (
|
|
task repository.DesktopTask
|
|
schedulerGroupKey pgtype.Text
|
|
monitorTaskID pgtype.Int8
|
|
supersedesTaskID pgtype.UUID
|
|
dedupKey pgtype.Text
|
|
activeAttemptID pgtype.UUID
|
|
leaseExpiresAt pgtype.Timestamptz
|
|
startedAt pgtype.Timestamptz
|
|
interruptedAt pgtype.Timestamptz
|
|
interruptReason pgtype.Text
|
|
enqueuedAt pgtype.Timestamptz
|
|
)
|
|
if err := row.Scan(
|
|
&task.DesktopID,
|
|
&task.JobID,
|
|
&task.TenantID,
|
|
&task.WorkspaceID,
|
|
&task.TargetAccountID,
|
|
&task.TargetClientID,
|
|
&task.Platform,
|
|
&task.Kind,
|
|
&task.Payload,
|
|
&task.Status,
|
|
&task.Priority,
|
|
&task.Lane,
|
|
&task.LaneWeight,
|
|
&task.Source,
|
|
&schedulerGroupKey,
|
|
&monitorTaskID,
|
|
&supersedesTaskID,
|
|
&task.ControlFlags,
|
|
&task.InterruptGeneration,
|
|
&dedupKey,
|
|
&activeAttemptID,
|
|
&leaseExpiresAt,
|
|
&task.Attempts,
|
|
&task.Result,
|
|
&task.Error,
|
|
&startedAt,
|
|
&interruptedAt,
|
|
&interruptReason,
|
|
&enqueuedAt,
|
|
&task.CreatedAt,
|
|
&task.UpdatedAt,
|
|
); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
task.SchedulerGroupKey = desktopTaskNullableText(schedulerGroupKey)
|
|
task.MonitorTaskID = desktopTaskNullableInt64(monitorTaskID)
|
|
task.SupersedesTaskID = desktopTaskNullableUUID(supersedesTaskID)
|
|
task.DedupKey = desktopTaskNullableText(dedupKey)
|
|
task.ActiveAttemptID = desktopTaskNullableUUID(activeAttemptID)
|
|
task.LeaseExpiresAt = desktopTaskNullableTime(leaseExpiresAt)
|
|
task.StartedAt = desktopTaskNullableTime(startedAt)
|
|
task.InterruptedAt = desktopTaskNullableTime(interruptedAt)
|
|
task.InterruptReason = desktopTaskNullableText(interruptReason)
|
|
if enqueuedAt.Valid {
|
|
task.EnqueuedAt = enqueuedAt.Time
|
|
}
|
|
return &task, nil
|
|
}
|
|
|
|
func desktopTaskNullableText(value pgtype.Text) *string {
|
|
if !value.Valid {
|
|
return nil
|
|
}
|
|
text := value.String
|
|
return &text
|
|
}
|
|
|
|
func desktopTaskNullableInt64(value pgtype.Int8) *int64 {
|
|
if !value.Valid {
|
|
return nil
|
|
}
|
|
number := value.Int64
|
|
return &number
|
|
}
|
|
|
|
func desktopTaskNullableUUID(value pgtype.UUID) *uuid.UUID {
|
|
if !value.Valid {
|
|
return nil
|
|
}
|
|
resolved, err := uuid.FromBytes(value.Bytes[:])
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
return &resolved
|
|
}
|
|
|
|
func desktopTaskNullableTime(value pgtype.Timestamptz) *time.Time {
|
|
if !value.Valid {
|
|
return nil
|
|
}
|
|
timestamp := value.Time
|
|
return ×tamp
|
|
}
|
|
|
|
func (s *DesktopTaskService) Extend(ctx context.Context, client *repository.DesktopClient, desktopID uuid.UUID, req ExtendDesktopTaskRequest) (*DesktopTaskView, error) {
|
|
if client == nil {
|
|
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
|
|
}
|
|
|
|
if err := s.dropStaleMonitorTasksForClient(ctx, client); err != nil {
|
|
return nil, err
|
|
}
|
|
if s.redis != nil {
|
|
presence := loadDesktopClientPresence(ctx, s.redis, []uuid.UUID{client.ID})
|
|
if presence != nil && !presence[client.ID] {
|
|
return nil, response.ErrConflict(40986, "desktop_client_offline", "desktop client presence has expired")
|
|
}
|
|
}
|
|
|
|
task, err := s.repo.ExtendLease(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(req.LeaseToken)))
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil, response.ErrConflict(40983, "desktop_task_lease_invalid", "desktop task lease token is invalid or expired")
|
|
}
|
|
return nil, response.ErrInternal(50089, "desktop_task_extend_failed", "failed to extend desktop task lease")
|
|
}
|
|
|
|
s.publishTaskEvent(ctx, task, "task_extended")
|
|
|
|
view := buildDesktopTaskView(task)
|
|
return &view, nil
|
|
}
|
|
|
|
// MarkPublishSubmitStarted records a durable intent marker right before the client performs the
|
|
// irreversible platform submit POST. Lease-recovery uses it to avoid silently re-posting a
|
|
// non-idempotent article: once submit may have started, recovery routes the task to 'unknown'
|
|
// (manual reconcile) instead of auto-requeueing it. The client must receive a
|
|
// positive marker write before it performs the platform submit; a stale/lost
|
|
// lease is rejected so the client can stop before the irreversible request.
|
|
func (s *DesktopTaskService) MarkPublishSubmitStarted(ctx context.Context, client *repository.DesktopClient, taskID uuid.UUID, leaseToken string) error {
|
|
if s == nil || s.pool == nil || client == nil {
|
|
return nil
|
|
}
|
|
leaseToken = strings.TrimSpace(leaseToken)
|
|
if leaseToken == "" {
|
|
return response.ErrBadRequest(40001, "invalid_params", "lease_token is required")
|
|
}
|
|
tag, err := s.pool.Exec(ctx, `
|
|
UPDATE desktop_tasks
|
|
SET publish_submit_started_at = COALESCE(publish_submit_started_at, NOW()),
|
|
updated_at = NOW()
|
|
WHERE desktop_id = $1
|
|
AND target_client_id = $2
|
|
AND kind = 'publish'
|
|
AND status = 'in_progress'
|
|
AND lease_token_hash = $3
|
|
`, taskID, client.ID, HashDesktopClientToken(leaseToken))
|
|
if err != nil {
|
|
return response.ErrInternal(50200, "desktop_task_submit_marker_failed", "failed to mark publish submit started")
|
|
}
|
|
if tag.RowsAffected() == 0 {
|
|
return response.ErrConflict(40983, "desktop_task_lease_invalid", "desktop task lease token is invalid or expired")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) Complete(ctx context.Context, client *repository.DesktopClient, desktopID uuid.UUID, req CompleteDesktopTaskRequest) (*DesktopTaskView, error) {
|
|
if client == nil {
|
|
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
|
|
}
|
|
|
|
finalStatus := normalizeDesktopTaskTerminalStatus(req.Status)
|
|
resultJSON, err := marshalOptionalJSON(req.Payload)
|
|
if err != nil {
|
|
return nil, response.ErrBadRequest(40086, "invalid_desktop_task_payload", "payload must be serializable")
|
|
}
|
|
errorJSON, err := marshalOptionalJSON(req.Error)
|
|
if err != nil {
|
|
return nil, response.ErrBadRequest(40087, "invalid_desktop_task_error", "error must be serializable")
|
|
}
|
|
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return nil, response.ErrInternal(50103, "desktop_task_complete_begin_failed", "failed to start desktop task completion")
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
|
|
taskRepo := repository.NewDesktopTaskRepository(tx)
|
|
previousTask, _ := taskRepo.GetByDesktopID(ctx, desktopID, client.WorkspaceID)
|
|
previousAttemptID := activeAttemptID(previousTask)
|
|
|
|
task, err := taskRepo.Complete(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(req.LeaseToken)), finalStatus, resultJSON, errorJSON)
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil, response.ErrConflict(40983, "desktop_task_lease_invalid", "desktop task lease token is invalid or expired")
|
|
}
|
|
return nil, response.ErrInternal(50091, "desktop_task_complete_failed", "failed to complete desktop task")
|
|
}
|
|
|
|
if previousAttemptID != nil {
|
|
if finishErr := taskRepo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
|
|
TaskID: task.DesktopID,
|
|
AttemptID: *previousAttemptID,
|
|
FinalStatus: &finalStatus,
|
|
Error: errorJSON,
|
|
}); finishErr != nil {
|
|
s.logWarn("desktop task attempt finish failed after complete", finishErr, zap.String("task_id", task.DesktopID.String()))
|
|
}
|
|
}
|
|
|
|
publishOutcome, err := syncDesktopPublishTaskState(ctx, tx, task)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
bulkFailure, err := bulkFailQueuedDesktopTasksForAuthorizationFailure(ctx, tx, task, errorJSON)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return nil, response.ErrInternal(50104, "desktop_task_complete_commit_failed", "failed to commit desktop task completion")
|
|
}
|
|
|
|
if publishOutcome != nil {
|
|
invalidateArticleCaches(ctx, s.cache, publishOutcome.TenantID, &publishOutcome.ArticleID)
|
|
if syncErr := s.syncMonitoringArticleAlias(ctx, publishOutcome.ArticleAlias); syncErr != nil {
|
|
s.logWarn("monitoring article alias sync failed after publish complete", syncErr, zap.Int64("article_id", publishOutcome.ArticleID))
|
|
}
|
|
}
|
|
for _, outcome := range bulkFailure.PublishOutcomes {
|
|
invalidateArticleCaches(ctx, s.cache, outcome.TenantID, &outcome.ArticleID)
|
|
if outcome.ArticleAlias != nil {
|
|
if syncErr := s.syncMonitoringArticleAlias(ctx, outcome.ArticleAlias); syncErr != nil {
|
|
s.logWarn("monitoring article alias sync failed after bulk authorization failure", syncErr, zap.Int64("article_id", outcome.ArticleID))
|
|
}
|
|
}
|
|
}
|
|
|
|
s.publishTaskEvent(ctx, task, "task_completed")
|
|
for _, failedTask := range bulkFailure.Tasks {
|
|
s.publishTaskEvent(ctx, failedTask, "task_completed")
|
|
}
|
|
|
|
view := buildDesktopTaskView(task)
|
|
return &view, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) Cancel(ctx context.Context, client *repository.DesktopClient, desktopID uuid.UUID, req CancelDesktopTaskRequest) (*DesktopTaskView, error) {
|
|
if client == nil {
|
|
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
|
|
}
|
|
|
|
errorJSON, err := marshalOptionalJSON(map[string]any{
|
|
"reason": optionalStringValue(req.Reason),
|
|
"source": "desktop_client",
|
|
})
|
|
if err != nil {
|
|
return nil, response.ErrBadRequest(40088, "invalid_desktop_task_cancel", "cancel payload must be serializable")
|
|
}
|
|
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return nil, response.ErrInternal(50107, "desktop_task_cancel_begin_failed", "failed to start desktop task cancel")
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
|
|
taskRepo := repository.NewDesktopTaskRepository(tx)
|
|
previousTask, _ := taskRepo.GetByDesktopID(ctx, desktopID, client.WorkspaceID)
|
|
previousAttemptID := activeAttemptID(previousTask)
|
|
|
|
var task *repository.DesktopTask
|
|
if req.LeaseToken != nil && strings.TrimSpace(*req.LeaseToken) != "" {
|
|
task, err = taskRepo.CancelByLease(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(*req.LeaseToken)), errorJSON)
|
|
} else {
|
|
task, err = taskRepo.CancelQueuedPublishByOwner(ctx, desktopID, client.TenantID, client.WorkspaceID, client.UserID, errorJSON)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
task, err = taskRepo.CancelByClient(ctx, desktopID, client.ID, errorJSON)
|
|
}
|
|
}
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil, response.ErrConflict(40984, "desktop_task_cancel_conflict", "desktop task cannot be canceled in its current state")
|
|
}
|
|
return nil, response.ErrInternal(50092, "desktop_task_cancel_failed", "failed to cancel desktop task")
|
|
}
|
|
|
|
if previousAttemptID != nil && req.LeaseToken != nil && strings.TrimSpace(*req.LeaseToken) != "" {
|
|
if finishErr := taskRepo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
|
|
TaskID: task.DesktopID,
|
|
AttemptID: *previousAttemptID,
|
|
FinalStatus: literalStringPtr("aborted"),
|
|
Error: errorJSON,
|
|
}); finishErr != nil {
|
|
s.logWarn("desktop task attempt finish failed after cancel", finishErr, zap.String("task_id", task.DesktopID.String()))
|
|
}
|
|
}
|
|
|
|
var replacementTask *repository.DesktopTask
|
|
if shouldRequeuePreemptedMonitorTask(task, req.Reason) {
|
|
replacementTask, err = requeuePreemptedMonitorTask(ctx, tx, taskRepo, task)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return nil, response.ErrInternal(50108, "desktop_task_cancel_commit_failed", "failed to commit desktop task cancel")
|
|
}
|
|
|
|
s.publishTaskEvent(ctx, task, "task_canceled")
|
|
if replacementTask != nil {
|
|
s.publishTaskEvent(ctx, replacementTask, "task_available")
|
|
}
|
|
|
|
view := buildDesktopTaskView(task)
|
|
return &view, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) TenantCancel(ctx context.Context, actor auth.Actor, desktopID uuid.UUID, req CancelDesktopTaskRequest) (*DesktopTaskView, error) {
|
|
workspaceID := auth.CurrentWorkspaceID(ctx)
|
|
if actor.TenantID == 0 || actor.UserID == 0 || workspaceID == 0 {
|
|
return nil, response.ErrUnauthorized(40132, "workspace_scope_missing", "workspace context is required")
|
|
}
|
|
|
|
errorJSON, err := marshalOptionalJSON(map[string]any{
|
|
"reason": optionalStringValue(req.Reason),
|
|
"source": "tenant_web",
|
|
})
|
|
if err != nil {
|
|
return nil, response.ErrBadRequest(40088, "invalid_desktop_task_cancel", "cancel payload must be serializable")
|
|
}
|
|
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return nil, response.ErrInternal(50107, "desktop_task_cancel_begin_failed", "failed to start desktop task cancel")
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
|
|
taskRepo := repository.NewDesktopTaskRepository(tx)
|
|
task, err := taskRepo.TenantCancel(ctx, desktopID, actor.TenantID, workspaceID, actor.UserID, errorJSON)
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil, response.ErrConflict(40984, "desktop_task_cancel_conflict", "desktop task cannot be canceled in its current state")
|
|
}
|
|
return nil, response.ErrInternal(50093, "desktop_task_tenant_cancel_failed", "failed to cancel desktop task")
|
|
}
|
|
|
|
publishOutcome, err := syncDesktopPublishTaskState(ctx, tx, task)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return nil, response.ErrInternal(50108, "desktop_task_cancel_commit_failed", "failed to commit desktop task cancel")
|
|
}
|
|
|
|
s.publishTaskEvent(ctx, task, "task_canceled")
|
|
s.afterRecoveredPublishOutcomes(ctx, []*desktopPublishSyncOutcome{publishOutcome})
|
|
|
|
view := buildDesktopTaskView(task)
|
|
return &view, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) Reconcile(ctx context.Context, actor auth.Actor, desktopID uuid.UUID, req ReconcileDesktopTaskRequest) (*DesktopTaskView, error) {
|
|
if actor.PrimaryWorkspaceID == 0 {
|
|
return nil, response.ErrUnauthorized(40132, "workspace_scope_missing", "workspace context is required")
|
|
}
|
|
|
|
resultJSON, err := marshalOptionalJSON(req.Result)
|
|
if err != nil {
|
|
return nil, response.ErrBadRequest(40089, "invalid_desktop_task_reconcile_result", "result must be serializable")
|
|
}
|
|
errorJSON, err := marshalOptionalJSON(req.Error)
|
|
if err != nil {
|
|
return nil, response.ErrBadRequest(40090, "invalid_desktop_task_reconcile_error", "error must be serializable")
|
|
}
|
|
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return nil, response.ErrInternal(50105, "desktop_task_reconcile_begin_failed", "failed to start desktop task reconcile")
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
|
|
taskRepo := repository.NewDesktopTaskRepository(tx)
|
|
task, err := taskRepo.Reconcile(ctx, desktopID, actor.PrimaryWorkspaceID, req.Status, resultJSON, errorJSON)
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil, response.ErrConflict(40985, "desktop_task_reconcile_conflict", "desktop task is not waiting for reconcile")
|
|
}
|
|
return nil, response.ErrInternal(50094, "desktop_task_reconcile_failed", "failed to reconcile desktop task")
|
|
}
|
|
|
|
publishOutcome, err := syncDesktopPublishTaskState(ctx, tx, task)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return nil, response.ErrInternal(50106, "desktop_task_reconcile_commit_failed", "failed to commit desktop task reconcile")
|
|
}
|
|
|
|
if publishOutcome != nil {
|
|
invalidateArticleCaches(ctx, s.cache, publishOutcome.TenantID, &publishOutcome.ArticleID)
|
|
if syncErr := s.syncMonitoringArticleAlias(ctx, publishOutcome.ArticleAlias); syncErr != nil {
|
|
s.logWarn("monitoring article alias sync failed after publish reconcile", syncErr, zap.Int64("article_id", publishOutcome.ArticleID))
|
|
}
|
|
}
|
|
|
|
s.publishTaskEvent(ctx, task, reconcileDesktopTaskEventType(task.Kind, req.Status))
|
|
|
|
view := buildDesktopTaskView(task)
|
|
return &view, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) ListPublishTasks(ctx context.Context, client *repository.DesktopClient, req ListPublishTasksRequest) (*DesktopPublishTaskList, error) {
|
|
if client == nil {
|
|
return nil, response.ErrUnauthorized(40108, "desktop_client_missing", "desktop client context is required")
|
|
}
|
|
return s.listPublishTasksForOwner(ctx, publishTaskListOwner{
|
|
TenantID: client.TenantID,
|
|
WorkspaceID: client.WorkspaceID,
|
|
UserID: client.UserID,
|
|
}, req)
|
|
}
|
|
|
|
func (s *DesktopTaskService) ListTenantPublishTasks(ctx context.Context, actor auth.Actor, req ListPublishTasksRequest) (*DesktopPublishTaskList, error) {
|
|
workspaceID := auth.CurrentWorkspaceID(ctx)
|
|
if actor.TenantID == 0 || actor.UserID == 0 || workspaceID == 0 {
|
|
return nil, response.ErrUnauthorized(40132, "workspace_scope_missing", "workspace context is required")
|
|
}
|
|
return s.listPublishTasksForOwner(ctx, publishTaskListOwner{
|
|
TenantID: actor.TenantID,
|
|
WorkspaceID: workspaceID,
|
|
UserID: actor.UserID,
|
|
}, req)
|
|
}
|
|
|
|
func (s *DesktopTaskService) listPublishTasksForOwner(ctx context.Context, owner publishTaskListOwner, req ListPublishTasksRequest) (*DesktopPublishTaskList, error) {
|
|
if s.pool == nil {
|
|
return nil, response.ErrServiceUnavailable(50342, "desktop_publish_job_store_unavailable", "desktop publish job store is unavailable")
|
|
}
|
|
|
|
page := req.Page
|
|
if page <= 0 {
|
|
page = 1
|
|
}
|
|
pageSize := req.PageSize
|
|
if pageSize <= 0 || pageSize > 50 {
|
|
pageSize = 10
|
|
}
|
|
|
|
title := strings.TrimSpace(req.Title)
|
|
pendingStatuses := []string{"queued", "in_progress"}
|
|
historyStatuses := []string{"succeeded", "failed", "unknown", "aborted"}
|
|
|
|
pendingItems, err := s.listPublishTasksByStatuses(
|
|
ctx,
|
|
owner,
|
|
pendingStatuses,
|
|
title,
|
|
0,
|
|
0,
|
|
`CASE WHEN t.status = 'in_progress' THEN 0 ELSE 1 END ASC, t.created_at ASC, t.desktop_id DESC`,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
historyTotal, err := s.countPublishTasksByStatuses(ctx, owner, historyStatuses, title)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
offset := (page - 1) * pageSize
|
|
items := make([]DesktopTaskView, 0, pageSize)
|
|
total := len(pendingItems) + historyTotal
|
|
|
|
if offset < len(pendingItems) {
|
|
end := offset + pageSize
|
|
if end > len(pendingItems) {
|
|
end = len(pendingItems)
|
|
}
|
|
items = append(items, pendingItems[offset:end]...)
|
|
}
|
|
|
|
remaining := pageSize - len(items)
|
|
historyOffset := offset - len(pendingItems)
|
|
if historyOffset < 0 {
|
|
historyOffset = 0
|
|
}
|
|
|
|
if remaining > 0 && historyOffset < historyTotal {
|
|
historyItems, listErr := s.listPublishTasksByStatuses(
|
|
ctx,
|
|
owner,
|
|
historyStatuses,
|
|
title,
|
|
remaining,
|
|
historyOffset,
|
|
`t.updated_at DESC, t.desktop_id DESC`,
|
|
)
|
|
if listErr != nil {
|
|
return nil, listErr
|
|
}
|
|
items = append(items, historyItems...)
|
|
}
|
|
|
|
return &DesktopPublishTaskList{
|
|
Items: items,
|
|
Page: page,
|
|
PageSize: pageSize,
|
|
Total: total,
|
|
PendingCount: len(pendingItems),
|
|
HistoryTotal: historyTotal,
|
|
}, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) listPublishTasksByStatuses(
|
|
ctx context.Context,
|
|
owner publishTaskListOwner,
|
|
statuses []string,
|
|
title string,
|
|
limit int,
|
|
offset int,
|
|
orderBy string,
|
|
) ([]DesktopTaskView, error) {
|
|
query := `
|
|
SELECT
|
|
t.desktop_id,
|
|
t.job_id,
|
|
t.tenant_id,
|
|
t.workspace_id,
|
|
t.target_account_id,
|
|
t.target_client_id,
|
|
t.platform_id,
|
|
t.kind,
|
|
t.payload,
|
|
t.status,
|
|
t.dedup_key,
|
|
t.active_attempt_id,
|
|
t.lease_expires_at,
|
|
t.attempts,
|
|
t.result,
|
|
t.error,
|
|
t.created_at,
|
|
t.updated_at,
|
|
j.status,
|
|
j.compliance_blocked_record_id,
|
|
j.compliance_blocked_at,
|
|
j.compliance_blocked_reason
|
|
FROM desktop_tasks AS t
|
|
JOIN desktop_publish_jobs AS j
|
|
ON j.desktop_id = t.job_id
|
|
AND j.workspace_id = t.workspace_id
|
|
WHERE t.tenant_id = $1
|
|
AND t.workspace_id = $2
|
|
AND j.tenant_id = $1
|
|
AND j.created_by_user_id = $3
|
|
AND t.kind = 'publish'
|
|
AND t.status = ANY($4)
|
|
AND (
|
|
$5 = ''
|
|
OR COALESCE(t.payload->>'title', j.title, '') ILIKE '%' || $5 || '%'
|
|
)
|
|
`
|
|
args := []any{owner.TenantID, owner.WorkspaceID, owner.UserID, statuses, title}
|
|
|
|
if strings.TrimSpace(orderBy) != "" {
|
|
query += "\nORDER BY " + orderBy
|
|
}
|
|
if limit > 0 {
|
|
query += "\nLIMIT $6 OFFSET $7"
|
|
args = append(args, limit, offset)
|
|
}
|
|
|
|
rows, err := s.pool.Query(ctx, query, args...)
|
|
if err != nil {
|
|
return nil, response.ErrInternal(50109, "desktop_publish_tasks_query_failed", "failed to list desktop publish tasks")
|
|
}
|
|
defer rows.Close()
|
|
|
|
return scanDesktopTaskRows(rows)
|
|
}
|
|
|
|
func (s *DesktopTaskService) countPublishTasksByStatuses(
|
|
ctx context.Context,
|
|
owner publishTaskListOwner,
|
|
statuses []string,
|
|
title string,
|
|
) (int, error) {
|
|
var count int
|
|
err := s.pool.QueryRow(ctx, `
|
|
SELECT COUNT(1)
|
|
FROM desktop_tasks AS t
|
|
JOIN desktop_publish_jobs AS j
|
|
ON j.desktop_id = t.job_id
|
|
AND j.workspace_id = t.workspace_id
|
|
WHERE t.tenant_id = $1
|
|
AND t.workspace_id = $2
|
|
AND j.tenant_id = $1
|
|
AND j.created_by_user_id = $3
|
|
AND t.kind = 'publish'
|
|
AND t.status = ANY($4)
|
|
AND (
|
|
$5 = ''
|
|
OR COALESCE(t.payload->>'title', j.title, '') ILIKE '%' || $5 || '%'
|
|
)
|
|
`, owner.TenantID, owner.WorkspaceID, owner.UserID, statuses, title).Scan(&count)
|
|
if err != nil {
|
|
return 0, response.ErrInternal(50109, "desktop_publish_tasks_query_failed", "failed to list desktop publish tasks")
|
|
}
|
|
return count, nil
|
|
}
|
|
|
|
func scanDesktopTaskRows(rows pgx.Rows) ([]DesktopTaskView, error) {
|
|
items := make([]DesktopTaskView, 0)
|
|
for rows.Next() {
|
|
var (
|
|
desktopID uuid.UUID
|
|
jobID uuid.UUID
|
|
tenantID int64
|
|
workspaceID int64
|
|
targetAccountID uuid.UUID
|
|
targetClientID uuid.UUID
|
|
platform string
|
|
kind string
|
|
payload []byte
|
|
status string
|
|
dedupKey pgtype.Text
|
|
activeAttemptID pgtype.UUID
|
|
leaseExpiresAt pgtype.Timestamptz
|
|
attempts int32
|
|
resultJSON []byte
|
|
errorJSON []byte
|
|
createdAt time.Time
|
|
updatedAt time.Time
|
|
jobStatus string
|
|
blockedRecordID pgtype.Int8
|
|
blockedAt pgtype.Timestamptz
|
|
blockedReason pgtype.Text
|
|
)
|
|
if err := rows.Scan(
|
|
&desktopID,
|
|
&jobID,
|
|
&tenantID,
|
|
&workspaceID,
|
|
&targetAccountID,
|
|
&targetClientID,
|
|
&platform,
|
|
&kind,
|
|
&payload,
|
|
&status,
|
|
&dedupKey,
|
|
&activeAttemptID,
|
|
&leaseExpiresAt,
|
|
&attempts,
|
|
&resultJSON,
|
|
&errorJSON,
|
|
&createdAt,
|
|
&updatedAt,
|
|
&jobStatus,
|
|
&blockedRecordID,
|
|
&blockedAt,
|
|
&blockedReason,
|
|
); err != nil {
|
|
return nil, response.ErrInternal(50110, "desktop_publish_tasks_scan_failed", "failed to scan desktop publish tasks")
|
|
}
|
|
|
|
var dedupKeyText *string
|
|
if dedupKey.Valid {
|
|
value := dedupKey.String
|
|
dedupKeyText = &value
|
|
}
|
|
|
|
var activeAttemptIDText *string
|
|
if activeAttemptID.Valid {
|
|
value, convErr := uuid.FromBytes(activeAttemptID.Bytes[:])
|
|
if convErr == nil {
|
|
text := value.String()
|
|
activeAttemptIDText = &text
|
|
}
|
|
}
|
|
|
|
var leaseExpiresAtValue *time.Time
|
|
if leaseExpiresAt.Valid {
|
|
value := leaseExpiresAt.Time
|
|
leaseExpiresAtValue = &value
|
|
}
|
|
|
|
publishJobStatusText := strings.TrimSpace(jobStatus)
|
|
var publishJobStatus *string
|
|
if publishJobStatusText != "" {
|
|
publishJobStatus = &publishJobStatusText
|
|
}
|
|
|
|
var complianceBlockedRecordID *int64
|
|
if blockedRecordID.Valid {
|
|
value := blockedRecordID.Int64
|
|
complianceBlockedRecordID = &value
|
|
}
|
|
|
|
var complianceBlockedAt *time.Time
|
|
if blockedAt.Valid {
|
|
value := blockedAt.Time
|
|
complianceBlockedAt = &value
|
|
}
|
|
|
|
var complianceBlockedReason *string
|
|
if blockedReason.Valid {
|
|
value := blockedReason.String
|
|
complianceBlockedReason = &value
|
|
}
|
|
|
|
items = append(items, DesktopTaskView{
|
|
ID: desktopID.String(),
|
|
JobID: jobID.String(),
|
|
TenantID: tenantID,
|
|
WorkspaceID: workspaceID,
|
|
TargetAccountID: targetAccountID.String(),
|
|
TargetClientID: targetClientID.String(),
|
|
Platform: platform,
|
|
Kind: kind,
|
|
Payload: json.RawMessage(payload),
|
|
Status: normalizeDesktopTaskTerminalStatus(status),
|
|
PublishJobStatus: publishJobStatus,
|
|
ComplianceBlockedRecordID: complianceBlockedRecordID,
|
|
ComplianceBlockedAt: complianceBlockedAt,
|
|
ComplianceBlockedReason: complianceBlockedReason,
|
|
DedupKey: dedupKeyText,
|
|
ActiveAttemptID: activeAttemptIDText,
|
|
LeaseExpiresAt: leaseExpiresAtValue,
|
|
Attempts: int(attempts),
|
|
Result: json.RawMessage(resultJSON),
|
|
Error: json.RawMessage(errorJSON),
|
|
CreatedAt: createdAt,
|
|
UpdatedAt: updatedAt,
|
|
})
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, response.ErrInternal(50109, "desktop_publish_tasks_query_failed", "failed to list desktop publish tasks")
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) classifyLeaseError(ctx context.Context, client *repository.DesktopClient, taskID *uuid.UUID) error {
|
|
if taskID == nil {
|
|
return response.ErrConflict(40986, "desktop_task_unavailable", "no desktop task is currently available")
|
|
}
|
|
|
|
task, err := s.repo.GetByDesktopID(ctx, *taskID, client.WorkspaceID)
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return response.ErrNotFound(40482, "desktop_task_not_found", "desktop task not found")
|
|
}
|
|
return response.ErrInternal(50095, "desktop_task_lookup_failed", "failed to inspect desktop task state")
|
|
}
|
|
|
|
if task.TargetClientID != client.ID && !s.clientCanLeaseMonitorTask(ctx, client, task) {
|
|
return response.ErrForbidden(40381, "desktop_task_not_target_client", "desktop task is assigned to another desktop client")
|
|
}
|
|
|
|
if task.Status == "in_progress" {
|
|
return response.ErrConflict(40989, "desktop_task_already_in_progress", "desktop task is already in progress")
|
|
}
|
|
|
|
return response.ErrConflict(40991, "desktop_task_not_queued", "desktop task is not queued")
|
|
}
|
|
|
|
func (s *DesktopTaskService) clientCanLeaseMonitorTask(ctx context.Context, client *repository.DesktopClient, task *repository.DesktopTask) bool {
|
|
if s == nil || client == nil || task == nil || task.Kind != "monitor" || task.TargetAccountID == uuid.Nil {
|
|
return false
|
|
}
|
|
accountIDs, err := s.loadMonitorLeaseAccountIDs(ctx, client)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
for _, accountID := range accountIDs {
|
|
if accountID == task.TargetAccountID {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func buildDesktopTaskView(task *repository.DesktopTask) DesktopTaskView {
|
|
var activeAttemptID *string
|
|
if task.ActiveAttemptID != nil {
|
|
value := task.ActiveAttemptID.String()
|
|
activeAttemptID = &value
|
|
}
|
|
|
|
return DesktopTaskView{
|
|
ID: task.DesktopID.String(),
|
|
JobID: task.JobID.String(),
|
|
TenantID: task.TenantID,
|
|
WorkspaceID: task.WorkspaceID,
|
|
TargetAccountID: task.TargetAccountID.String(),
|
|
TargetClientID: task.TargetClientID.String(),
|
|
Platform: task.Platform,
|
|
Kind: task.Kind,
|
|
Payload: json.RawMessage(task.Payload),
|
|
Status: normalizeDesktopTaskTerminalStatus(task.Status),
|
|
DedupKey: task.DedupKey,
|
|
ActiveAttemptID: activeAttemptID,
|
|
LeaseExpiresAt: task.LeaseExpiresAt,
|
|
Attempts: task.Attempts,
|
|
Result: json.RawMessage(task.Result),
|
|
Error: json.RawMessage(task.Error),
|
|
CreatedAt: task.CreatedAt,
|
|
UpdatedAt: task.UpdatedAt,
|
|
}
|
|
}
|
|
|
|
func marshalOptionalJSON(value any) ([]byte, error) {
|
|
if value == nil {
|
|
return nil, nil
|
|
}
|
|
return json.Marshal(value)
|
|
}
|
|
|
|
func optionalStringValue(value *string) string {
|
|
if value == nil {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(*value)
|
|
}
|
|
|
|
func literalStringPtr(value string) *string {
|
|
return &value
|
|
}
|
|
|
|
func normalizeDesktopTaskTerminalStatus(status string) string {
|
|
switch strings.TrimSpace(status) {
|
|
case "unknown":
|
|
return "failed"
|
|
default:
|
|
return strings.TrimSpace(status)
|
|
}
|
|
}
|
|
|
|
type desktopTaskRecoveryMode string
|
|
|
|
const (
|
|
desktopTaskRecoveryModeStartup desktopTaskRecoveryMode = "startup"
|
|
desktopTaskRecoveryModeDisconnect desktopTaskRecoveryMode = "disconnect"
|
|
desktopTaskRecoveryModeLeaseExpiry desktopTaskRecoveryMode = "lease_expired"
|
|
)
|
|
|
|
type recoveredDesktopTaskLease struct {
|
|
TaskID uuid.UUID
|
|
WorkspaceID int64
|
|
Kind string
|
|
Status string
|
|
ActiveAttempt pgtype.UUID
|
|
Attempts int
|
|
SubmitStarted bool
|
|
ErrorJSON []byte
|
|
}
|
|
|
|
type PublishLeaseRecoveryResult struct {
|
|
Requeued int `json:"requeued"`
|
|
Failed int `json:"failed"`
|
|
Uncertain int `json:"uncertain"`
|
|
}
|
|
|
|
// resolvePublishRecoveryOutcome decides what to do with a publish task whose lease was lost.
|
|
//
|
|
// If the client had already entered the irreversible submit phase (publish_submit_started_at
|
|
// is set), the article may already exist on the platform. Auto-requeueing would silently
|
|
// re-post a non-idempotent article, so we route the task to 'unknown' (kept in the dedup
|
|
// active set, surfaced for manual reconcile) instead. Tasks that never reached submit are
|
|
// safe to auto-requeue while attempts remain, and give up to 'failed' once exhausted.
|
|
func resolvePublishRecoveryOutcome(submitStarted bool, attempts int) (status string, payloadKind string) {
|
|
switch {
|
|
case submitStarted:
|
|
return "unknown", "publish_uncertain"
|
|
case attempts >= desktopPublishMaxAttempts:
|
|
return "failed", "publish_final"
|
|
default:
|
|
return "queued", "publish"
|
|
}
|
|
}
|
|
|
|
func (s *DesktopTaskService) recoverClientTasksOnStartup(ctx context.Context, client *repository.DesktopClient) error {
|
|
return s.recoverClientTasks(ctx, client, desktopTaskRecoveryModeStartup)
|
|
}
|
|
|
|
func (s *DesktopTaskService) recoverClientTasksOnDisconnect(ctx context.Context, client *repository.DesktopClient) error {
|
|
return s.recoverClientTasks(ctx, client, desktopTaskRecoveryModeDisconnect)
|
|
}
|
|
|
|
func (s *DesktopTaskService) recoverExpiredClientTasks(ctx context.Context, client *repository.DesktopClient) error {
|
|
return s.recoverClientTasks(ctx, client, desktopTaskRecoveryModeLeaseExpiry)
|
|
}
|
|
|
|
func (s *DesktopTaskService) recoverClientTasks(
|
|
ctx context.Context,
|
|
client *repository.DesktopClient,
|
|
mode desktopTaskRecoveryMode,
|
|
) error {
|
|
if s == nil || client == nil || s.pool == nil {
|
|
return nil
|
|
}
|
|
|
|
publishErrorJSON, publishReason, publishMessage, err := desktopTaskRecoveryPayload(mode, "publish")
|
|
if err != nil {
|
|
return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
|
|
}
|
|
publishFinalErrorJSON, _, _, err := desktopTaskRecoveryPayload(mode, "publish_final")
|
|
if err != nil {
|
|
return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
|
|
}
|
|
publishUncertainErrorJSON, _, _, err := desktopTaskRecoveryPayload(mode, "publish_uncertain")
|
|
if err != nil {
|
|
return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
|
|
}
|
|
monitorErrorJSON, monitorReason, _, err := desktopTaskRecoveryPayload(mode, "monitor")
|
|
if err != nil {
|
|
return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode monitor desktop task recovery payload")
|
|
}
|
|
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return response.ErrInternal(50196, "desktop_task_recovery_begin_failed", "failed to start desktop task recovery")
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
|
|
repo := repository.NewDesktopTaskRepository(tx)
|
|
query := desktopTaskRecoverySelectQuery(mode)
|
|
queryArgs := []any{client.ID, client.WorkspaceID}
|
|
|
|
rows, err := tx.Query(ctx, query, queryArgs...)
|
|
if err != nil {
|
|
s.logWarn("desktop task recovery query failed", err, zap.String("client_id", client.ID.String()), zap.String("mode", string(mode)))
|
|
return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
|
|
}
|
|
defer rows.Close()
|
|
|
|
recovered := make([]recoveredDesktopTaskLease, 0)
|
|
for rows.Next() {
|
|
var item recoveredDesktopTaskLease
|
|
if scanErr := rows.Scan(
|
|
&item.TaskID,
|
|
&item.WorkspaceID,
|
|
&item.Kind,
|
|
&item.Status,
|
|
&item.ActiveAttempt,
|
|
&item.Attempts,
|
|
&item.SubmitStarted,
|
|
); scanErr != nil {
|
|
s.logWarn("desktop task recovery scan failed", scanErr, zap.String("client_id", client.ID.String()), zap.String("mode", string(mode)))
|
|
return response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to parse recovered desktop tasks")
|
|
}
|
|
recovered = append(recovered, item)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
s.logWarn("desktop task recovery rows failed", err, zap.String("client_id", client.ID.String()), zap.String("mode", string(mode)))
|
|
return response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to iterate recovered desktop tasks")
|
|
}
|
|
|
|
publishOutcomes := make([]*desktopPublishSyncOutcome, 0)
|
|
for index := range recovered {
|
|
item := &recovered[index]
|
|
if item.Kind == "monitor" {
|
|
if _, execErr := tx.Exec(ctx, `
|
|
UPDATE desktop_tasks
|
|
SET status = 'queued',
|
|
error = $2,
|
|
active_attempt_id = NULL,
|
|
lease_token_hash = NULL,
|
|
lease_expires_at = NULL,
|
|
interrupted_at = COALESCE(interrupted_at, NOW()),
|
|
interrupt_reason = $3,
|
|
enqueued_at = NOW(),
|
|
updated_at = NOW()
|
|
WHERE desktop_id = $1
|
|
`, item.TaskID, monitorErrorJSON, monitorReason); execErr != nil {
|
|
s.logWarn("desktop monitor task recovery update failed", execErr, zap.String("task_id", item.TaskID.String()), zap.String("mode", string(mode)))
|
|
return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
|
|
}
|
|
item.Status = "queued"
|
|
continue
|
|
}
|
|
|
|
finalStatus, payloadKind := resolvePublishRecoveryOutcome(item.SubmitStarted, item.Attempts)
|
|
nextErrorJSON := publishErrorJSON
|
|
switch payloadKind {
|
|
case "publish_final":
|
|
nextErrorJSON = publishFinalErrorJSON
|
|
case "publish_uncertain":
|
|
nextErrorJSON = publishUncertainErrorJSON
|
|
}
|
|
if _, execErr := tx.Exec(ctx, `
|
|
UPDATE desktop_tasks
|
|
SET status = $2,
|
|
error = $3,
|
|
result = CASE WHEN $2 = 'queued' THEN NULL ELSE result END,
|
|
active_attempt_id = NULL,
|
|
lease_token_hash = NULL,
|
|
lease_expires_at = NULL,
|
|
interrupted_at = COALESCE(interrupted_at, NOW()),
|
|
interrupt_reason = COALESCE(interrupt_reason, $4),
|
|
enqueued_at = CASE WHEN $2 = 'queued' THEN NOW() ELSE enqueued_at END,
|
|
updated_at = NOW()
|
|
WHERE desktop_id = $1
|
|
`, item.TaskID, finalStatus, nextErrorJSON, publishReason); execErr != nil {
|
|
s.logWarn("desktop publish task recovery update failed", execErr, zap.String("task_id", item.TaskID.String()), zap.String("mode", string(mode)))
|
|
return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
|
|
}
|
|
item.Status = finalStatus
|
|
item.ErrorJSON = nextErrorJSON
|
|
// Only terminal 'failed' tasks finalize the publish_record; 'unknown' (may already be
|
|
// published) is left in the dedup active set for manual reconcile, 'queued' is requeued.
|
|
if finalStatus == "failed" {
|
|
task, getErr := repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID)
|
|
if getErr != nil {
|
|
s.logWarn("desktop publish task recovery reload failed", getErr, zap.String("task_id", item.TaskID.String()), zap.String("mode", string(mode)))
|
|
return response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
|
|
}
|
|
publishOutcome, syncErr := syncDesktopPublishTaskState(ctx, tx, task)
|
|
if syncErr != nil {
|
|
return syncErr
|
|
}
|
|
if publishOutcome != nil {
|
|
publishOutcomes = append(publishOutcomes, publishOutcome)
|
|
}
|
|
}
|
|
}
|
|
|
|
for _, item := range recovered {
|
|
if !item.ActiveAttempt.Valid {
|
|
continue
|
|
}
|
|
|
|
attemptID, convErr := uuid.FromBytes(item.ActiveAttempt.Bytes[:])
|
|
if convErr != nil {
|
|
s.logWarn("desktop recovery attempt id decode failed", convErr, zap.String("task_id", item.TaskID.String()))
|
|
continue
|
|
}
|
|
|
|
finalStatus := literalStringPtr(item.Status)
|
|
errorJSON := item.ErrorJSON
|
|
if len(errorJSON) == 0 {
|
|
errorJSON = publishErrorJSON
|
|
}
|
|
if item.Kind == "monitor" {
|
|
finalStatus = literalStringPtr("aborted")
|
|
errorJSON = monitorErrorJSON
|
|
}
|
|
|
|
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
|
|
TaskID: item.TaskID,
|
|
AttemptID: attemptID,
|
|
FinalStatus: finalStatus,
|
|
Error: errorJSON,
|
|
}); finishErr != nil {
|
|
s.logWarn("desktop recovery attempt finish failed", finishErr, zap.String("task_id", item.TaskID.String()))
|
|
}
|
|
}
|
|
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return response.ErrInternal(50199, "desktop_task_recovery_commit_failed", "failed to commit desktop task recovery")
|
|
}
|
|
|
|
s.afterRecoveredPublishOutcomes(ctx, publishOutcomes)
|
|
|
|
for _, item := range recovered {
|
|
task, getErr := s.repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID)
|
|
if getErr != nil {
|
|
s.logWarn("desktop recovery reload failed", getErr, zap.String("task_id", item.TaskID.String()))
|
|
continue
|
|
}
|
|
|
|
eventType := "task_completed"
|
|
if item.Status == "queued" {
|
|
eventType = "task_available"
|
|
}
|
|
s.publishTaskEvent(ctx, task, eventType)
|
|
}
|
|
|
|
if len(recovered) > 0 && s.logger != nil {
|
|
s.logger.Info("desktop tasks recovered",
|
|
zap.String("client_id", client.ID.String()),
|
|
zap.String("mode", string(mode)),
|
|
zap.Int("count", len(recovered)),
|
|
zap.String("publish_message", publishMessage),
|
|
)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) RecoverExpiredPublishTasks(ctx context.Context, limit int) (PublishLeaseRecoveryResult, error) {
|
|
var result PublishLeaseRecoveryResult
|
|
if s == nil || s.pool == nil {
|
|
return result, nil
|
|
}
|
|
if limit <= 0 {
|
|
limit = 1000
|
|
}
|
|
|
|
publishErrorJSON, publishReason, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "publish")
|
|
if err != nil {
|
|
return result, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
|
|
}
|
|
publishFinalErrorJSON, _, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "publish_final")
|
|
if err != nil {
|
|
return result, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
|
|
}
|
|
publishUncertainErrorJSON, _, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "publish_uncertain")
|
|
if err != nil {
|
|
return result, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode publish desktop task recovery payload")
|
|
}
|
|
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return result, response.ErrInternal(50196, "desktop_task_recovery_begin_failed", "failed to start desktop task recovery")
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
|
|
rows, err := tx.Query(ctx, `
|
|
SELECT
|
|
t.desktop_id,
|
|
t.workspace_id,
|
|
t.kind,
|
|
t.status,
|
|
t.active_attempt_id,
|
|
t.attempts,
|
|
(t.publish_submit_started_at IS NOT NULL) AS submit_started
|
|
FROM desktop_tasks AS t
|
|
WHERE t.kind = 'publish'
|
|
AND t.status = 'in_progress'
|
|
AND t.lease_expires_at IS NOT NULL
|
|
AND t.lease_expires_at < NOW()
|
|
ORDER BY t.lease_expires_at ASC, t.updated_at ASC, t.desktop_id ASC
|
|
LIMIT $1
|
|
FOR UPDATE OF t SKIP LOCKED
|
|
`, limit)
|
|
if err != nil {
|
|
s.logWarn("expired desktop publish recovery query failed", err)
|
|
return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
|
|
}
|
|
defer rows.Close()
|
|
|
|
recovered := make([]recoveredDesktopTaskLease, 0)
|
|
for rows.Next() {
|
|
var item recoveredDesktopTaskLease
|
|
if scanErr := rows.Scan(
|
|
&item.TaskID,
|
|
&item.WorkspaceID,
|
|
&item.Kind,
|
|
&item.Status,
|
|
&item.ActiveAttempt,
|
|
&item.Attempts,
|
|
&item.SubmitStarted,
|
|
); scanErr != nil {
|
|
s.logWarn("expired desktop publish recovery scan failed", scanErr)
|
|
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to parse recovered desktop tasks")
|
|
}
|
|
recovered = append(recovered, item)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
s.logWarn("expired desktop publish recovery rows failed", err)
|
|
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to iterate recovered desktop tasks")
|
|
}
|
|
|
|
repo := repository.NewDesktopTaskRepository(tx)
|
|
publishOutcomes := make([]*desktopPublishSyncOutcome, 0)
|
|
for index := range recovered {
|
|
item := &recovered[index]
|
|
finalStatus, payloadKind := resolvePublishRecoveryOutcome(item.SubmitStarted, item.Attempts)
|
|
nextErrorJSON := publishErrorJSON
|
|
switch payloadKind {
|
|
case "publish_final":
|
|
nextErrorJSON = publishFinalErrorJSON
|
|
case "publish_uncertain":
|
|
nextErrorJSON = publishUncertainErrorJSON
|
|
}
|
|
if _, execErr := tx.Exec(ctx, `
|
|
UPDATE desktop_tasks
|
|
SET status = $2,
|
|
error = $3,
|
|
result = CASE WHEN $2 = 'queued' THEN NULL ELSE result END,
|
|
active_attempt_id = NULL,
|
|
lease_token_hash = NULL,
|
|
lease_expires_at = NULL,
|
|
interrupted_at = COALESCE(interrupted_at, NOW()),
|
|
interrupt_reason = COALESCE(interrupt_reason, $4),
|
|
enqueued_at = CASE WHEN $2 = 'queued' THEN NOW() ELSE enqueued_at END,
|
|
updated_at = NOW()
|
|
WHERE desktop_id = $1
|
|
AND status = 'in_progress'
|
|
`, item.TaskID, finalStatus, nextErrorJSON, publishReason); execErr != nil {
|
|
s.logWarn("expired desktop publish recovery update failed", execErr, zap.String("task_id", item.TaskID.String()))
|
|
return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
|
|
}
|
|
|
|
item.Status = finalStatus
|
|
item.ErrorJSON = nextErrorJSON
|
|
switch finalStatus {
|
|
case "failed":
|
|
result.Failed++
|
|
task, getErr := repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID)
|
|
if getErr != nil {
|
|
s.logWarn("expired desktop publish recovery reload failed", getErr, zap.String("task_id", item.TaskID.String()))
|
|
return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
|
|
}
|
|
publishOutcome, syncErr := syncDesktopPublishTaskState(ctx, tx, task)
|
|
if syncErr != nil {
|
|
return result, syncErr
|
|
}
|
|
if publishOutcome != nil {
|
|
publishOutcomes = append(publishOutcomes, publishOutcome)
|
|
}
|
|
case "unknown":
|
|
// May already be published — keep as 'unknown' (still in the dedup active set) for
|
|
// manual reconcile. Do NOT sync to a terminal publish_record state and do NOT requeue.
|
|
result.Uncertain++
|
|
default:
|
|
result.Requeued++
|
|
}
|
|
}
|
|
|
|
for _, item := range recovered {
|
|
if !item.ActiveAttempt.Valid {
|
|
continue
|
|
}
|
|
|
|
attemptID, convErr := uuid.FromBytes(item.ActiveAttempt.Bytes[:])
|
|
if convErr != nil {
|
|
s.logWarn("expired desktop publish recovery attempt id decode failed", convErr, zap.String("task_id", item.TaskID.String()))
|
|
continue
|
|
}
|
|
|
|
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
|
|
TaskID: item.TaskID,
|
|
AttemptID: attemptID,
|
|
FinalStatus: literalStringPtr(item.Status),
|
|
Error: item.ErrorJSON,
|
|
}); finishErr != nil {
|
|
s.logWarn("expired desktop publish recovery attempt finish failed", finishErr, zap.String("task_id", item.TaskID.String()))
|
|
}
|
|
}
|
|
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return result, response.ErrInternal(50199, "desktop_task_recovery_commit_failed", "failed to commit desktop task recovery")
|
|
}
|
|
|
|
s.afterRecoveredPublishOutcomes(ctx, publishOutcomes)
|
|
|
|
for _, item := range recovered {
|
|
task, getErr := s.repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID)
|
|
if getErr != nil {
|
|
s.logWarn("expired desktop publish recovery reload after commit failed", getErr, zap.String("task_id", item.TaskID.String()))
|
|
continue
|
|
}
|
|
eventType := "task_completed"
|
|
if item.Status == "queued" {
|
|
eventType = "task_available"
|
|
}
|
|
s.publishTaskEvent(ctx, task, eventType)
|
|
}
|
|
|
|
if len(recovered) > 0 && s.logger != nil {
|
|
s.logger.Info("expired desktop publish tasks recovered",
|
|
zap.Int("requeued", result.Requeued),
|
|
zap.Int("failed", result.Failed),
|
|
zap.Int("uncertain", result.Uncertain),
|
|
)
|
|
}
|
|
|
|
return result, nil
|
|
}
|
|
|
|
func desktopTaskRecoverySelectQuery(mode desktopTaskRecoveryMode) string {
|
|
query := `
|
|
SELECT
|
|
desktop_id,
|
|
workspace_id,
|
|
kind,
|
|
status,
|
|
active_attempt_id,
|
|
attempts,
|
|
publish_submit_started_at IS NOT NULL AS submit_started
|
|
FROM desktop_tasks
|
|
WHERE target_client_id = $1
|
|
AND workspace_id = $2
|
|
AND status = 'in_progress'
|
|
`
|
|
if mode == desktopTaskRecoveryModeLeaseExpiry {
|
|
query += `
|
|
AND lease_expires_at IS NOT NULL
|
|
AND lease_expires_at < NOW()
|
|
`
|
|
}
|
|
query += `
|
|
FOR UPDATE
|
|
`
|
|
return query
|
|
}
|
|
|
|
func desktopTaskRecoveryPayload(mode desktopTaskRecoveryMode, kind string) ([]byte, string, string, error) {
|
|
reason := strings.TrimSpace(string(mode))
|
|
message := "desktop task was recovered after the client lost the active lease"
|
|
source := "desktop_task_recovery"
|
|
isPublishFinal := kind == "publish_final"
|
|
isPublishUncertain := kind == "publish_uncertain"
|
|
|
|
if isPublishFinal {
|
|
message = fmt.Sprintf("desktop publish task exceeded %d attempts during recovery; task has been marked failed", desktopPublishMaxAttempts)
|
|
} else if isPublishUncertain {
|
|
message = "desktop publish task may have already been submitted to the platform before the lease was lost; kept as unknown for manual reconcile to avoid duplicate publishing"
|
|
}
|
|
|
|
switch mode {
|
|
case desktopTaskRecoveryModeStartup:
|
|
source = "desktop_client_startup"
|
|
if isPublishFinal {
|
|
message = fmt.Sprintf("desktop client restarted while the publish task was in progress; task exceeded %d attempts and has been marked failed", desktopPublishMaxAttempts)
|
|
} else if isPublishUncertain {
|
|
message = "desktop client restarted while the publish task was mid-submit; kept as unknown for manual reconcile to avoid duplicate publishing"
|
|
} else if kind == "monitor" {
|
|
message = "desktop client restarted while the monitor task was in progress; task has been re-queued"
|
|
} else {
|
|
message = "desktop client restarted while the publish task was in progress; task has been re-queued for retry"
|
|
}
|
|
case desktopTaskRecoveryModeDisconnect:
|
|
source = "desktop_client_offline"
|
|
if isPublishFinal {
|
|
message = fmt.Sprintf("desktop client went offline while the publish task was in progress; task exceeded %d attempts and has been marked failed", desktopPublishMaxAttempts)
|
|
} else if isPublishUncertain {
|
|
message = "desktop client went offline while the publish task was mid-submit; kept as unknown for manual reconcile to avoid duplicate publishing"
|
|
} else if kind == "monitor" {
|
|
message = "desktop client went offline while the monitor task was in progress; task has been re-queued"
|
|
} else {
|
|
message = "desktop client went offline while the publish task was in progress; task has been re-queued for retry"
|
|
}
|
|
case desktopTaskRecoveryModeLeaseExpiry:
|
|
source = "desktop_task_lease_expiry"
|
|
if isPublishFinal {
|
|
message = fmt.Sprintf("desktop task lease expired before publish completion; task exceeded %d attempts and has been marked failed", desktopPublishMaxAttempts)
|
|
} else if isPublishUncertain {
|
|
message = "desktop task lease expired while the publish task was mid-submit; kept as unknown for manual reconcile to avoid duplicate publishing"
|
|
} else if kind == "monitor" {
|
|
message = "desktop task lease expired before monitor completion; task has been re-queued"
|
|
} else {
|
|
message = "desktop task lease expired before publish completion; task has been re-queued for retry"
|
|
}
|
|
}
|
|
|
|
fields := map[string]any{
|
|
"reason": reason,
|
|
"message": message,
|
|
"source": source,
|
|
}
|
|
if isPublishUncertain {
|
|
// Surfaced to the desktop UI so a manual retry of a maybe-already-published task asks for
|
|
// explicit confirmation instead of silently re-posting.
|
|
fields["publish_submit_uncertain"] = true
|
|
}
|
|
payload, err := marshalOptionalJSON(fields)
|
|
if err != nil {
|
|
return nil, "", "", err
|
|
}
|
|
return payload, reason, message, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) dropStaleMonitorTasksForClient(ctx context.Context, client *repository.DesktopClient) error {
|
|
if s == nil || client == nil || s.pool == nil || s.monitoringPool == nil {
|
|
return nil
|
|
}
|
|
|
|
_, currentBusinessDate := monitoringBusinessDayAndDateAt(time.Now())
|
|
errorJSON, err := marshalOptionalJSON(map[string]any{
|
|
"reason": monitoringStaleTaskDropReason,
|
|
"message": monitoringStaleTaskDropError,
|
|
"current_business_date": currentBusinessDate,
|
|
"source": "business_day_rollover",
|
|
})
|
|
if err != nil {
|
|
return response.ErrInternal(50190, "desktop_task_stale_payload_failed", "failed to encode stale monitor desktop task payload")
|
|
}
|
|
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return response.ErrInternal(50191, "desktop_task_stale_begin_failed", "failed to start stale monitor desktop cleanup")
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
|
|
rows, err := tx.Query(ctx, `
|
|
WITH stale AS (
|
|
SELECT
|
|
desktop_id,
|
|
active_attempt_id
|
|
FROM desktop_tasks
|
|
WHERE target_client_id = $1
|
|
AND kind = 'monitor'
|
|
AND status IN ('queued', 'in_progress', 'unknown')
|
|
AND COALESCE(
|
|
NULLIF(payload ->> 'business_date', ''),
|
|
NULLIF(payload ->> 'businessDate', ''),
|
|
NULLIF(payload ->> 'metric_date', ''),
|
|
NULLIF(payload ->> 'date', '')
|
|
) < $2
|
|
FOR UPDATE
|
|
),
|
|
updated AS (
|
|
UPDATE desktop_tasks AS t
|
|
SET status = 'aborted',
|
|
error = $3,
|
|
active_attempt_id = NULL,
|
|
lease_token_hash = NULL,
|
|
lease_expires_at = NULL,
|
|
interrupted_at = COALESCE(interrupted_at, NOW()),
|
|
interrupt_reason = $4,
|
|
updated_at = NOW()
|
|
FROM stale
|
|
WHERE t.desktop_id = stale.desktop_id
|
|
RETURNING t.desktop_id
|
|
)
|
|
SELECT stale.desktop_id, stale.active_attempt_id
|
|
FROM stale
|
|
INNER JOIN updated ON updated.desktop_id = stale.desktop_id
|
|
`, client.ID, currentBusinessDate, errorJSON, monitoringStaleTaskDropReason)
|
|
if err != nil {
|
|
return response.ErrInternal(50192, "desktop_task_stale_query_failed", "failed to select stale monitor desktop tasks")
|
|
}
|
|
defer rows.Close()
|
|
|
|
repo := repository.NewDesktopTaskRepository(tx)
|
|
droppedTaskIDs := make([]uuid.UUID, 0)
|
|
for rows.Next() {
|
|
var (
|
|
taskID uuid.UUID
|
|
attemptID pgtype.UUID
|
|
)
|
|
if scanErr := rows.Scan(&taskID, &attemptID); scanErr != nil {
|
|
return response.ErrInternal(50193, "desktop_task_stale_scan_failed", "failed to parse stale monitor desktop tasks")
|
|
}
|
|
|
|
droppedTaskIDs = append(droppedTaskIDs, taskID)
|
|
|
|
if !attemptID.Valid {
|
|
continue
|
|
}
|
|
|
|
resolvedAttemptID, convErr := uuid.FromBytes(attemptID.Bytes[:])
|
|
if convErr != nil {
|
|
s.logWarn("desktop stale monitor attempt id decode failed", convErr, zap.String("task_id", taskID.String()))
|
|
continue
|
|
}
|
|
|
|
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
|
|
TaskID: taskID,
|
|
AttemptID: resolvedAttemptID,
|
|
FinalStatus: literalStringPtr("aborted"),
|
|
Error: errorJSON,
|
|
}); finishErr != nil {
|
|
s.logWarn("desktop stale monitor attempt finish failed", finishErr, zap.String("task_id", taskID.String()))
|
|
}
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return response.ErrInternal(50193, "desktop_task_stale_scan_failed", "failed to iterate stale monitor desktop tasks")
|
|
}
|
|
|
|
if err := tx.Commit(ctx); err != nil {
|
|
return response.ErrInternal(50194, "desktop_task_stale_commit_failed", "failed to commit stale monitor desktop cleanup")
|
|
}
|
|
|
|
for _, taskID := range droppedTaskIDs {
|
|
task, getErr := s.repo.GetByDesktopID(ctx, taskID, client.WorkspaceID)
|
|
if getErr != nil {
|
|
s.logWarn("desktop stale monitor reload failed", getErr, zap.String("task_id", taskID.String()))
|
|
continue
|
|
}
|
|
s.publishTaskEvent(ctx, task, "task_canceled")
|
|
}
|
|
|
|
if _, err := s.monitoringPool.Exec(ctx, `
|
|
UPDATE monitoring_collect_tasks
|
|
SET status = 'skipped',
|
|
lease_token_hash = NULL,
|
|
leased_to_executor = NULL,
|
|
leased_at = NULL,
|
|
lease_expires_at = NULL,
|
|
callback_received_at = COALESCE(callback_received_at, NOW()),
|
|
completed_at = COALESCE(completed_at, NOW()),
|
|
skip_reason = $4,
|
|
error_message = COALESCE(NULLIF(error_message, ''), $5),
|
|
updated_at = NOW()
|
|
WHERE tenant_id = $1
|
|
AND collector_type = $2
|
|
AND COALESCE(execution_owner, 'legacy') = 'desktop_tasks'
|
|
AND target_client_id = $3
|
|
AND business_date < $6::date
|
|
AND callback_received_at IS NULL
|
|
AND status IN ('pending', 'leased', 'expired')
|
|
`, client.TenantID, monitoringCollectorType, client.ID, monitoringStaleTaskDropReason, monitoringStaleTaskDropError, currentBusinessDate); err != nil {
|
|
return response.ErrInternal(50041, "task_skip_failed", "failed to drop stale phase2 monitoring tasks")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func activeAttemptID(task *repository.DesktopTask) *uuid.UUID {
|
|
if task == nil || task.ActiveAttemptID == nil {
|
|
return nil
|
|
}
|
|
value := *task.ActiveAttemptID
|
|
return &value
|
|
}
|
|
|
|
func shouldRequeuePreemptedMonitorTask(task *repository.DesktopTask, reason *string) bool {
|
|
if task == nil || task.Kind != "monitor" {
|
|
return false
|
|
}
|
|
return strings.EqualFold(strings.TrimSpace(optionalStringValue(reason)), "collect_now_preempt")
|
|
}
|
|
|
|
func requeuePreemptedMonitorTask(
|
|
ctx context.Context,
|
|
tx pgx.Tx,
|
|
repo repository.DesktopTaskRepository,
|
|
task *repository.DesktopTask,
|
|
) (*repository.DesktopTask, error) {
|
|
if task == nil {
|
|
return nil, nil
|
|
}
|
|
replacementID := uuid.New()
|
|
replacementJobID := uuid.New()
|
|
if _, err := tx.Exec(ctx, `
|
|
INSERT INTO desktop_tasks (
|
|
desktop_id,
|
|
job_id,
|
|
tenant_id,
|
|
workspace_id,
|
|
target_account_id,
|
|
target_client_id,
|
|
platform_id,
|
|
kind,
|
|
payload,
|
|
status,
|
|
priority,
|
|
lane,
|
|
lane_weight,
|
|
source,
|
|
scheduler_group_key,
|
|
monitor_task_id,
|
|
supersedes_task_id,
|
|
control_flags,
|
|
interrupt_generation,
|
|
enqueued_at
|
|
)
|
|
VALUES (
|
|
$1,
|
|
$2,
|
|
$3,
|
|
$4,
|
|
$5,
|
|
$6,
|
|
$7,
|
|
'monitor',
|
|
$8,
|
|
'queued',
|
|
50,
|
|
'retry',
|
|
20,
|
|
'retry_recovery',
|
|
$9,
|
|
$10,
|
|
$11,
|
|
jsonb_build_object(
|
|
'interrupt_requested', false,
|
|
'requeued_at', NOW()::text,
|
|
'requeued_from_task_id', $11::text,
|
|
'requeued_reason', 'collect_now_preempt'
|
|
),
|
|
$12,
|
|
NOW()
|
|
)
|
|
`, replacementID, replacementJobID, task.TenantID, task.WorkspaceID, task.TargetAccountID, task.TargetClientID, task.Platform, task.Payload, nullableString(task.SchedulerGroupKey), nullableInt64(task.MonitorTaskID), task.DesktopID, task.InterruptGeneration); err != nil {
|
|
return nil, response.ErrInternal(50111, "desktop_task_requeue_failed", "failed to create replacement retry monitor desktop task")
|
|
}
|
|
|
|
next, err := repo.GetByDesktopID(ctx, replacementID, task.WorkspaceID)
|
|
if err != nil {
|
|
return nil, response.ErrInternal(50112, "desktop_task_lookup_failed", "failed to reload replacement retry monitor desktop task")
|
|
}
|
|
return next, nil
|
|
}
|
|
|
|
func (s *DesktopTaskService) publishTaskEvent(ctx context.Context, task *repository.DesktopTask, eventType string) {
|
|
publishDesktopTaskLifecycleEvent(ctx, s.messaging, s.logger, task, eventType)
|
|
}
|
|
|
|
func (s *DesktopTaskService) afterRecoveredPublishOutcomes(ctx context.Context, outcomes []*desktopPublishSyncOutcome) {
|
|
for _, outcome := range outcomes {
|
|
if outcome == nil {
|
|
continue
|
|
}
|
|
invalidateArticleCaches(ctx, s.cache, outcome.TenantID, &outcome.ArticleID)
|
|
if outcome.ArticleAlias != nil {
|
|
if syncErr := s.syncMonitoringArticleAlias(ctx, outcome.ArticleAlias); syncErr != nil {
|
|
s.logWarn("monitoring article alias sync failed after publish recovery", syncErr, zap.Int64("article_id", outcome.ArticleID))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func reconcileDesktopTaskEventType(_ string, status string) string {
|
|
if status == "retry" {
|
|
return "task_available"
|
|
}
|
|
return "task_reconciled"
|
|
}
|
|
|
|
func (s *DesktopTaskService) logWarn(message string, err error, fields ...zap.Field) {
|
|
if s.logger == nil || err == nil {
|
|
return
|
|
}
|
|
fields = append(fields, zap.Error(err))
|
|
s.logger.Warn(message, fields...)
|
|
}
|