fix: lease monitor tasks by account identity
This commit is contained in:
@@ -28,6 +28,7 @@ import (
|
|||||||
const (
|
const (
|
||||||
desktopPublishMaxAttempts = 3
|
desktopPublishMaxAttempts = 3
|
||||||
maxConcurrentMonitorPlatformsPerDesktopClient = 2
|
maxConcurrentMonitorPlatformsPerDesktopClient = 2
|
||||||
|
monitorLeaseCandidateScanLimitSQL = "16"
|
||||||
monitorSucceededTaskCooldownSQL = "2 seconds"
|
monitorSucceededTaskCooldownSQL = "2 seconds"
|
||||||
monitorUnhealthyTaskCooldownSQL = "30 seconds"
|
monitorUnhealthyTaskCooldownSQL = "30 seconds"
|
||||||
)
|
)
|
||||||
@@ -519,12 +520,6 @@ func (s *DesktopTaskService) leaseNextQueuedMonitorTask(
|
|||||||
client *repository.DesktopClient,
|
client *repository.DesktopClient,
|
||||||
params repository.DesktopTaskLeaseParams,
|
params repository.DesktopTaskLeaseParams,
|
||||||
) (*repository.DesktopTask, error) {
|
) (*repository.DesktopTask, error) {
|
||||||
accountIDs, err := s.loadMonitorLeaseAccountIDs(ctx, client)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
hasAccountIDs := len(accountIDs) > 0
|
|
||||||
|
|
||||||
tx, err := beginDesktopMonitorLeaseTx(ctx, s.pool, client.ID)
|
tx, err := beginDesktopMonitorLeaseTx(ctx, s.pool, client.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -535,8 +530,6 @@ func (s *DesktopTaskService) leaseNextQueuedMonitorTask(
|
|||||||
client.TenantID,
|
client.TenantID,
|
||||||
client.WorkspaceID,
|
client.WorkspaceID,
|
||||||
client.ID,
|
client.ID,
|
||||||
hasAccountIDs,
|
|
||||||
accountIDs,
|
|
||||||
params.AttemptID,
|
params.AttemptID,
|
||||||
params.LeaseTokenHash,
|
params.LeaseTokenHash,
|
||||||
maxConcurrentMonitorPlatformsPerDesktopClient,
|
maxConcurrentMonitorPlatformsPerDesktopClient,
|
||||||
@@ -553,37 +546,71 @@ func (s *DesktopTaskService) leaseNextQueuedMonitorTask(
|
|||||||
|
|
||||||
func leaseNextQueuedMonitorTaskSQL() string {
|
func leaseNextQueuedMonitorTaskSQL() string {
|
||||||
return `
|
return `
|
||||||
WITH candidate AS (
|
WITH current_accounts AS (
|
||||||
SELECT dt.desktop_id
|
SELECT desktop_id, user_id, platform_id, platform_uid, account_fingerprint
|
||||||
FROM desktop_tasks AS dt
|
FROM platform_accounts
|
||||||
WHERE dt.tenant_id = $1
|
WHERE tenant_id = $1
|
||||||
AND dt.workspace_id = $2
|
AND workspace_id = $2
|
||||||
AND dt.kind = 'monitor'
|
AND client_id = $3
|
||||||
AND dt.status = 'queued'
|
AND deleted_at IS NULL
|
||||||
AND NOT EXISTS (
|
AND delete_requested_at IS NULL
|
||||||
|
),
|
||||||
|
candidate_pool AS (
|
||||||
|
SELECT dt.desktop_id,
|
||||||
|
ca.desktop_id AS lease_account_id,
|
||||||
|
dt.lane_weight,
|
||||||
|
dt.priority,
|
||||||
|
dt.enqueued_at,
|
||||||
|
dt.created_at,
|
||||||
|
` + monitorAccountLeaseLockSQL("target_account") + ` AS account_slot_lock_key
|
||||||
|
FROM current_accounts AS ca
|
||||||
|
JOIN platform_accounts AS target_account
|
||||||
|
ON target_account.tenant_id = $1
|
||||||
|
AND target_account.workspace_id = $2
|
||||||
|
AND target_account.user_id = ca.user_id
|
||||||
|
AND target_account.platform_id = ca.platform_id
|
||||||
|
AND ` + monitorAccountIdentityMatchSQL("ca", "target_account") + `
|
||||||
|
JOIN desktop_tasks AS dt
|
||||||
|
ON dt.target_account_id = target_account.desktop_id
|
||||||
|
AND dt.tenant_id = $1
|
||||||
|
AND dt.workspace_id = $2
|
||||||
|
AND dt.platform_id = ca.platform_id
|
||||||
|
AND dt.kind = 'monitor'
|
||||||
|
AND dt.status = 'queued'
|
||||||
|
WHERE NOT EXISTS (
|
||||||
SELECT 1
|
SELECT 1
|
||||||
FROM desktop_tasks AS active
|
FROM desktop_tasks AS active
|
||||||
|
JOIN platform_accounts AS active_account
|
||||||
|
ON active_account.desktop_id = active.target_account_id
|
||||||
|
AND active_account.tenant_id = active.tenant_id
|
||||||
|
AND active_account.workspace_id = active.workspace_id
|
||||||
WHERE active.tenant_id = dt.tenant_id
|
WHERE active.tenant_id = dt.tenant_id
|
||||||
AND active.workspace_id = dt.workspace_id
|
AND active.workspace_id = dt.workspace_id
|
||||||
AND active.kind = 'monitor'
|
AND active.kind = 'monitor'
|
||||||
AND active.target_client_id = $3
|
|
||||||
AND active.platform_id = dt.platform_id
|
AND active.platform_id = dt.platform_id
|
||||||
AND active.status = 'in_progress'
|
AND active.status = 'in_progress'
|
||||||
AND active.desktop_id <> dt.desktop_id
|
AND active.desktop_id <> dt.desktop_id
|
||||||
|
AND active_account.user_id = target_account.user_id
|
||||||
|
AND ` + monitorAccountIdentityMatchSQL("active_account", "target_account") + `
|
||||||
)
|
)
|
||||||
AND NOT EXISTS (
|
AND NOT EXISTS (
|
||||||
SELECT 1
|
SELECT 1
|
||||||
FROM desktop_tasks AS recent
|
FROM desktop_tasks AS recent
|
||||||
|
JOIN platform_accounts AS recent_account
|
||||||
|
ON recent_account.desktop_id = recent.target_account_id
|
||||||
|
AND recent_account.tenant_id = recent.tenant_id
|
||||||
|
AND recent_account.workspace_id = recent.workspace_id
|
||||||
WHERE recent.tenant_id = dt.tenant_id
|
WHERE recent.tenant_id = dt.tenant_id
|
||||||
AND recent.workspace_id = dt.workspace_id
|
AND recent.workspace_id = dt.workspace_id
|
||||||
AND recent.kind = 'monitor'
|
AND recent.kind = 'monitor'
|
||||||
AND recent.target_client_id = $3
|
|
||||||
AND recent.platform_id = dt.platform_id
|
AND recent.platform_id = dt.platform_id
|
||||||
AND (
|
AND (
|
||||||
(recent.status = 'succeeded' AND recent.updated_at >= now() - interval '` + monitorSucceededTaskCooldownSQL + `')
|
(recent.status = 'succeeded' AND recent.updated_at >= now() - interval '` + monitorSucceededTaskCooldownSQL + `')
|
||||||
OR (recent.status IN ('failed', 'unknown', 'aborted') AND recent.updated_at >= now() - interval '` + monitorUnhealthyTaskCooldownSQL + `')
|
OR (recent.status IN ('failed', 'unknown', 'aborted') AND recent.updated_at >= now() - interval '` + monitorUnhealthyTaskCooldownSQL + `')
|
||||||
)
|
)
|
||||||
AND recent.desktop_id <> dt.desktop_id
|
AND recent.desktop_id <> dt.desktop_id
|
||||||
|
AND recent_account.user_id = target_account.user_id
|
||||||
|
AND ` + monitorAccountIdentityMatchSQL("recent_account", "target_account") + `
|
||||||
)
|
)
|
||||||
AND (
|
AND (
|
||||||
SELECT COUNT(DISTINCT active_platform.platform_id)
|
SELECT COUNT(DISTINCT active_platform.platform_id)
|
||||||
@@ -594,23 +621,44 @@ func leaseNextQueuedMonitorTaskSQL() string {
|
|||||||
AND active_platform.target_client_id = $3
|
AND active_platform.target_client_id = $3
|
||||||
AND active_platform.status = 'in_progress'
|
AND active_platform.status = 'in_progress'
|
||||||
AND active_platform.desktop_id <> dt.desktop_id
|
AND active_platform.desktop_id <> dt.desktop_id
|
||||||
) < $8::integer
|
) < $6::integer
|
||||||
AND (
|
AND NOT EXISTS (
|
||||||
dt.target_client_id = $3
|
SELECT 1
|
||||||
OR ($4::boolean AND dt.target_account_id = ANY($5::uuid[]))
|
FROM desktop_tasks AS active_client_platform
|
||||||
|
WHERE active_client_platform.tenant_id = dt.tenant_id
|
||||||
|
AND active_client_platform.workspace_id = dt.workspace_id
|
||||||
|
AND active_client_platform.kind = 'monitor'
|
||||||
|
AND active_client_platform.target_client_id = $3
|
||||||
|
AND active_client_platform.platform_id = dt.platform_id
|
||||||
|
AND active_client_platform.status = 'in_progress'
|
||||||
|
AND active_client_platform.desktop_id <> dt.desktop_id
|
||||||
)
|
)
|
||||||
ORDER BY dt.lane_weight DESC,
|
ORDER BY dt.lane_weight DESC,
|
||||||
dt.priority DESC,
|
dt.priority DESC,
|
||||||
COALESCE(dt.enqueued_at, dt.created_at) ASC,
|
COALESCE(dt.enqueued_at, dt.created_at) ASC,
|
||||||
dt.created_at ASC,
|
dt.created_at ASC,
|
||||||
dt.desktop_id ASC
|
dt.desktop_id ASC,
|
||||||
|
ca.desktop_id ASC
|
||||||
|
LIMIT ` + monitorLeaseCandidateScanLimitSQL + `
|
||||||
|
FOR UPDATE OF dt SKIP LOCKED
|
||||||
|
),
|
||||||
|
candidate AS (
|
||||||
|
SELECT desktop_id, lease_account_id
|
||||||
|
FROM candidate_pool
|
||||||
|
WHERE pg_try_advisory_xact_lock(account_slot_lock_key)
|
||||||
|
ORDER BY lane_weight DESC,
|
||||||
|
priority DESC,
|
||||||
|
COALESCE(enqueued_at, created_at) ASC,
|
||||||
|
created_at ASC,
|
||||||
|
desktop_id ASC,
|
||||||
|
lease_account_id ASC
|
||||||
LIMIT 1
|
LIMIT 1
|
||||||
FOR UPDATE SKIP LOCKED
|
|
||||||
)
|
)
|
||||||
UPDATE desktop_tasks AS t
|
UPDATE desktop_tasks AS t
|
||||||
SET target_client_id = $3,
|
SET target_client_id = $3,
|
||||||
active_attempt_id = $6,
|
target_account_id = candidate.lease_account_id,
|
||||||
lease_token_hash = $7,
|
active_attempt_id = $4,
|
||||||
|
lease_token_hash = $5,
|
||||||
lease_expires_at = now() + interval '10 minutes',
|
lease_expires_at = now() + interval '10 minutes',
|
||||||
status = 'in_progress',
|
status = 'in_progress',
|
||||||
attempts = t.attempts + 1,
|
attempts = t.attempts + 1,
|
||||||
@@ -620,6 +668,51 @@ func leaseNextQueuedMonitorTaskSQL() string {
|
|||||||
RETURNING ` + desktopTaskRepositoryReturningColumns
|
RETURNING ` + desktopTaskRepositoryReturningColumns
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func monitorAccountLeaseLockSQL(alias string) string {
|
||||||
|
return fmt.Sprintf(`hashtextextended(
|
||||||
|
concat_ws(
|
||||||
|
':',
|
||||||
|
'monitor_desktop_account_slot',
|
||||||
|
%s.tenant_id::text,
|
||||||
|
%s.workspace_id::text,
|
||||||
|
%s.user_id::text,
|
||||||
|
%s.platform_id,
|
||||||
|
CASE
|
||||||
|
WHEN btrim(COALESCE(%s.account_fingerprint, '')) <> '' THEN 'fp:' || %s.account_fingerprint
|
||||||
|
WHEN btrim(COALESCE(%s.platform_uid, '')) <> '' THEN 'uid:' || %s.platform_uid
|
||||||
|
ELSE 'id:' || %s.desktop_id::text
|
||||||
|
END
|
||||||
|
),
|
||||||
|
0
|
||||||
|
)`,
|
||||||
|
alias,
|
||||||
|
alias,
|
||||||
|
alias,
|
||||||
|
alias,
|
||||||
|
alias, alias,
|
||||||
|
alias, alias,
|
||||||
|
alias,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitorAccountIdentityMatchSQL(leftAlias, rightAlias string) string {
|
||||||
|
return fmt.Sprintf(`(
|
||||||
|
%s.desktop_id = %s.desktop_id
|
||||||
|
OR (
|
||||||
|
btrim(COALESCE(%s.account_fingerprint, '')) <> ''
|
||||||
|
AND %s.account_fingerprint = %s.account_fingerprint
|
||||||
|
)
|
||||||
|
OR (
|
||||||
|
btrim(COALESCE(%s.platform_uid, '')) <> ''
|
||||||
|
AND %s.platform_uid = %s.platform_uid
|
||||||
|
)
|
||||||
|
)`,
|
||||||
|
leftAlias, rightAlias,
|
||||||
|
leftAlias, leftAlias, rightAlias,
|
||||||
|
leftAlias, leftAlias, rightAlias,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
func (s *DesktopTaskService) leaseNextQueuedPublishTask(
|
func (s *DesktopTaskService) leaseNextQueuedPublishTask(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client *repository.DesktopClient,
|
client *repository.DesktopClient,
|
||||||
@@ -683,12 +776,6 @@ func (s *DesktopTaskService) leaseQueuedDesktopTaskByID(
|
|||||||
desktopID uuid.UUID,
|
desktopID uuid.UUID,
|
||||||
params repository.DesktopTaskLeaseParams,
|
params repository.DesktopTaskLeaseParams,
|
||||||
) (*repository.DesktopTask, error) {
|
) (*repository.DesktopTask, error) {
|
||||||
accountIDs, err := s.loadMonitorLeaseAccountIDs(ctx, client)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
hasAccountIDs := len(accountIDs) > 0
|
|
||||||
|
|
||||||
tx, err := beginDesktopMonitorLeaseTx(ctx, s.pool, client.ID)
|
tx, err := beginDesktopMonitorLeaseTx(ctx, s.pool, client.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -700,8 +787,6 @@ func (s *DesktopTaskService) leaseQueuedDesktopTaskByID(
|
|||||||
client.WorkspaceID,
|
client.WorkspaceID,
|
||||||
client.TenantID,
|
client.TenantID,
|
||||||
client.ID,
|
client.ID,
|
||||||
hasAccountIDs,
|
|
||||||
accountIDs,
|
|
||||||
params.AttemptID,
|
params.AttemptID,
|
||||||
params.LeaseTokenHash,
|
params.LeaseTokenHash,
|
||||||
desktopPublishMaxAttempts,
|
desktopPublishMaxAttempts,
|
||||||
@@ -719,13 +804,35 @@ func (s *DesktopTaskService) leaseQueuedDesktopTaskByID(
|
|||||||
|
|
||||||
func leaseQueuedDesktopTaskByIDSQL() string {
|
func leaseQueuedDesktopTaskByIDSQL() string {
|
||||||
return `
|
return `
|
||||||
WITH candidate AS (
|
WITH current_accounts AS (
|
||||||
SELECT dt.desktop_id
|
SELECT desktop_id, user_id, platform_id, platform_uid, account_fingerprint
|
||||||
|
FROM platform_accounts
|
||||||
|
WHERE tenant_id = $3
|
||||||
|
AND workspace_id = $2
|
||||||
|
AND client_id = $4
|
||||||
|
AND deleted_at IS NULL
|
||||||
|
AND delete_requested_at IS NULL
|
||||||
|
),
|
||||||
|
candidate AS (
|
||||||
|
SELECT dt.desktop_id,
|
||||||
|
dt.kind,
|
||||||
|
ca.desktop_id AS lease_account_id,
|
||||||
|
` + monitorAccountLeaseLockSQL("target_account") + ` AS account_slot_lock_key
|
||||||
FROM desktop_tasks AS dt
|
FROM desktop_tasks AS dt
|
||||||
|
LEFT JOIN platform_accounts AS target_account
|
||||||
|
ON dt.kind = 'monitor'
|
||||||
|
AND target_account.desktop_id = dt.target_account_id
|
||||||
|
AND target_account.tenant_id = dt.tenant_id
|
||||||
|
AND target_account.workspace_id = dt.workspace_id
|
||||||
|
LEFT JOIN current_accounts AS ca
|
||||||
|
ON dt.kind = 'monitor'
|
||||||
|
AND ca.user_id = target_account.user_id
|
||||||
|
AND ca.platform_id = dt.platform_id
|
||||||
|
AND ` + monitorAccountIdentityMatchSQL("ca", "target_account") + `
|
||||||
WHERE dt.desktop_id = $1
|
WHERE dt.desktop_id = $1
|
||||||
AND dt.workspace_id = $2
|
AND dt.workspace_id = $2
|
||||||
AND dt.status = 'queued'
|
AND dt.status = 'queued'
|
||||||
AND (dt.kind <> 'publish' OR dt.attempts < $9)
|
AND (dt.kind <> 'publish' OR dt.attempts < $7)
|
||||||
AND (
|
AND (
|
||||||
dt.kind <> 'publish'
|
dt.kind <> 'publish'
|
||||||
OR EXISTS (
|
OR EXISTS (
|
||||||
@@ -738,18 +845,27 @@ func leaseQueuedDesktopTaskByIDSQL() string {
|
|||||||
AND pa.delete_requested_at IS NULL
|
AND pa.delete_requested_at IS NULL
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
AND (
|
||||||
|
dt.kind <> 'monitor'
|
||||||
|
OR ca.desktop_id IS NOT NULL
|
||||||
|
)
|
||||||
AND (
|
AND (
|
||||||
dt.kind <> 'monitor'
|
dt.kind <> 'monitor'
|
||||||
OR NOT EXISTS (
|
OR NOT EXISTS (
|
||||||
SELECT 1
|
SELECT 1
|
||||||
FROM desktop_tasks AS active
|
FROM desktop_tasks AS active
|
||||||
|
JOIN platform_accounts AS active_account
|
||||||
|
ON active_account.desktop_id = active.target_account_id
|
||||||
|
AND active_account.tenant_id = active.tenant_id
|
||||||
|
AND active_account.workspace_id = active.workspace_id
|
||||||
WHERE active.tenant_id = dt.tenant_id
|
WHERE active.tenant_id = dt.tenant_id
|
||||||
AND active.workspace_id = dt.workspace_id
|
AND active.workspace_id = dt.workspace_id
|
||||||
AND active.kind = 'monitor'
|
AND active.kind = 'monitor'
|
||||||
AND active.target_client_id = $4
|
|
||||||
AND active.platform_id = dt.platform_id
|
AND active.platform_id = dt.platform_id
|
||||||
AND active.status = 'in_progress'
|
AND active.status = 'in_progress'
|
||||||
AND active.desktop_id <> dt.desktop_id
|
AND active.desktop_id <> dt.desktop_id
|
||||||
|
AND active_account.user_id = target_account.user_id
|
||||||
|
AND ` + monitorAccountIdentityMatchSQL("active_account", "target_account") + `
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
AND (
|
AND (
|
||||||
@@ -757,16 +873,21 @@ func leaseQueuedDesktopTaskByIDSQL() string {
|
|||||||
OR NOT EXISTS (
|
OR NOT EXISTS (
|
||||||
SELECT 1
|
SELECT 1
|
||||||
FROM desktop_tasks AS recent
|
FROM desktop_tasks AS recent
|
||||||
|
JOIN platform_accounts AS recent_account
|
||||||
|
ON recent_account.desktop_id = recent.target_account_id
|
||||||
|
AND recent_account.tenant_id = recent.tenant_id
|
||||||
|
AND recent_account.workspace_id = recent.workspace_id
|
||||||
WHERE recent.tenant_id = dt.tenant_id
|
WHERE recent.tenant_id = dt.tenant_id
|
||||||
AND recent.workspace_id = dt.workspace_id
|
AND recent.workspace_id = dt.workspace_id
|
||||||
AND recent.kind = 'monitor'
|
AND recent.kind = 'monitor'
|
||||||
AND recent.target_client_id = $4
|
|
||||||
AND recent.platform_id = dt.platform_id
|
AND recent.platform_id = dt.platform_id
|
||||||
AND (
|
AND (
|
||||||
(recent.status = 'succeeded' AND recent.updated_at >= now() - interval '` + monitorSucceededTaskCooldownSQL + `')
|
(recent.status = 'succeeded' AND recent.updated_at >= now() - interval '` + monitorSucceededTaskCooldownSQL + `')
|
||||||
OR (recent.status IN ('failed', 'unknown', 'aborted') AND recent.updated_at >= now() - interval '` + monitorUnhealthyTaskCooldownSQL + `')
|
OR (recent.status IN ('failed', 'unknown', 'aborted') AND recent.updated_at >= now() - interval '` + monitorUnhealthyTaskCooldownSQL + `')
|
||||||
)
|
)
|
||||||
AND recent.desktop_id <> dt.desktop_id
|
AND recent.desktop_id <> dt.desktop_id
|
||||||
|
AND recent_account.user_id = target_account.user_id
|
||||||
|
AND ` + monitorAccountIdentityMatchSQL("recent_account", "target_account") + `
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
AND (
|
AND (
|
||||||
@@ -780,36 +901,55 @@ func leaseQueuedDesktopTaskByIDSQL() string {
|
|||||||
AND active_platform.target_client_id = $4
|
AND active_platform.target_client_id = $4
|
||||||
AND active_platform.status = 'in_progress'
|
AND active_platform.status = 'in_progress'
|
||||||
AND active_platform.desktop_id <> dt.desktop_id
|
AND active_platform.desktop_id <> dt.desktop_id
|
||||||
) < $10::integer
|
) < $8::integer
|
||||||
|
)
|
||||||
|
AND (
|
||||||
|
dt.kind <> 'monitor'
|
||||||
|
OR NOT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM desktop_tasks AS active_client_platform
|
||||||
|
WHERE active_client_platform.tenant_id = dt.tenant_id
|
||||||
|
AND active_client_platform.workspace_id = dt.workspace_id
|
||||||
|
AND active_client_platform.kind = 'monitor'
|
||||||
|
AND active_client_platform.target_client_id = $4
|
||||||
|
AND active_client_platform.platform_id = dt.platform_id
|
||||||
|
AND active_client_platform.status = 'in_progress'
|
||||||
|
AND active_client_platform.desktop_id <> dt.desktop_id
|
||||||
|
)
|
||||||
)
|
)
|
||||||
AND (
|
AND (
|
||||||
(
|
(
|
||||||
dt.kind = 'monitor'
|
dt.kind = 'monitor'
|
||||||
AND dt.tenant_id = $3
|
AND dt.tenant_id = $3
|
||||||
AND (
|
|
||||||
dt.target_client_id = $4
|
|
||||||
OR ($5::boolean AND dt.target_account_id = ANY($6::uuid[]))
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
OR (
|
OR (
|
||||||
dt.kind <> 'monitor'
|
dt.kind <> 'monitor'
|
||||||
AND dt.target_client_id = $4
|
AND dt.target_client_id = $4
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
ORDER BY ca.desktop_id ASC NULLS LAST
|
||||||
|
LIMIT 1
|
||||||
|
FOR UPDATE OF dt SKIP LOCKED
|
||||||
|
),
|
||||||
|
locked_candidate AS (
|
||||||
|
SELECT desktop_id, lease_account_id
|
||||||
|
FROM candidate
|
||||||
|
WHERE kind <> 'monitor'
|
||||||
|
OR pg_try_advisory_xact_lock(account_slot_lock_key)
|
||||||
LIMIT 1
|
LIMIT 1
|
||||||
FOR UPDATE SKIP LOCKED
|
|
||||||
)
|
)
|
||||||
UPDATE desktop_tasks AS t
|
UPDATE desktop_tasks AS t
|
||||||
SET target_client_id = CASE WHEN t.kind = 'monitor' THEN $4 ELSE t.target_client_id END,
|
SET target_client_id = CASE WHEN t.kind = 'monitor' THEN $4 ELSE t.target_client_id END,
|
||||||
active_attempt_id = $7,
|
target_account_id = CASE WHEN t.kind = 'monitor' THEN locked_candidate.lease_account_id ELSE t.target_account_id END,
|
||||||
lease_token_hash = $8,
|
active_attempt_id = $5,
|
||||||
|
lease_token_hash = $6,
|
||||||
lease_expires_at = now() + CASE WHEN t.kind = 'publish' THEN interval '3 minutes' ELSE interval '10 minutes' END,
|
lease_expires_at = now() + CASE WHEN t.kind = 'publish' THEN interval '3 minutes' ELSE interval '10 minutes' END,
|
||||||
status = 'in_progress',
|
status = 'in_progress',
|
||||||
attempts = t.attempts + 1,
|
attempts = t.attempts + 1,
|
||||||
started_at = COALESCE(t.started_at, now()),
|
started_at = COALESCE(t.started_at, now()),
|
||||||
updated_at = now()
|
updated_at = now()
|
||||||
FROM candidate
|
FROM locked_candidate
|
||||||
WHERE t.desktop_id = candidate.desktop_id
|
WHERE t.desktop_id = locked_candidate.desktop_id
|
||||||
RETURNING ` + desktopTaskRepositoryReturningColumns
|
RETURNING ` + desktopTaskRepositoryReturningColumns
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -828,39 +968,6 @@ func beginDesktopMonitorLeaseTx(ctx context.Context, pool *pgxpool.Pool, clientI
|
|||||||
return tx, nil
|
return tx, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *DesktopTaskService) loadMonitorLeaseAccountIDs(ctx context.Context, client *repository.DesktopClient) ([]uuid.UUID, error) {
|
|
||||||
if s == nil || client == nil || s.pool == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
accountIDs := loadTrackedDesktopClientAccountIDs(ctx, s.redis, client.ID)
|
|
||||||
rows, err := s.pool.Query(ctx, `
|
|
||||||
SELECT desktop_id
|
|
||||||
FROM platform_accounts
|
|
||||||
WHERE tenant_id = $1
|
|
||||||
AND workspace_id = $2
|
|
||||||
AND client_id = $3
|
|
||||||
AND deleted_at IS NULL
|
|
||||||
AND platform_id = ANY($4::text[])
|
|
||||||
`, client.TenantID, client.WorkspaceID, client.ID, monitoringPlatformIDs(defaultMonitoringPlatforms))
|
|
||||||
if err != nil {
|
|
||||||
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to resolve monitor lease accounts")
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
|
|
||||||
for rows.Next() {
|
|
||||||
var accountID uuid.UUID
|
|
||||||
if scanErr := rows.Scan(&accountID); scanErr != nil {
|
|
||||||
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to parse monitor lease accounts")
|
|
||||||
}
|
|
||||||
accountIDs = append(accountIDs, accountID)
|
|
||||||
}
|
|
||||||
if err := rows.Err(); err != nil {
|
|
||||||
return nil, response.ErrInternal(50117, "desktop_account_lookup_failed", "failed to iterate monitor lease accounts")
|
|
||||||
}
|
|
||||||
return uniqueUUIDs(accountIDs), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func scanRepositoryDesktopTask(row desktopTaskRepositoryScanner) (*repository.DesktopTask, error) {
|
func scanRepositoryDesktopTask(row desktopTaskRepositoryScanner) (*repository.DesktopTask, error) {
|
||||||
var (
|
var (
|
||||||
task repository.DesktopTask
|
task repository.DesktopTask
|
||||||
@@ -1662,16 +1769,28 @@ func (s *DesktopTaskService) clientCanLeaseMonitorTask(ctx context.Context, clie
|
|||||||
if s == nil || client == nil || task == nil || task.Kind != "monitor" || task.TargetAccountID == uuid.Nil {
|
if s == nil || client == nil || task == nil || task.Kind != "monitor" || task.TargetAccountID == uuid.Nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
accountIDs, err := s.loadMonitorLeaseAccountIDs(ctx, client)
|
var canLease bool
|
||||||
if err != nil {
|
if err := s.pool.QueryRow(ctx, `
|
||||||
|
SELECT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM platform_accounts AS target_account
|
||||||
|
JOIN platform_accounts AS client_account
|
||||||
|
ON client_account.tenant_id = target_account.tenant_id
|
||||||
|
AND client_account.workspace_id = target_account.workspace_id
|
||||||
|
AND client_account.user_id = target_account.user_id
|
||||||
|
AND client_account.platform_id = target_account.platform_id
|
||||||
|
AND client_account.client_id = $4
|
||||||
|
AND client_account.deleted_at IS NULL
|
||||||
|
AND client_account.delete_requested_at IS NULL
|
||||||
|
AND `+monitorAccountIdentityMatchSQL("client_account", "target_account")+`
|
||||||
|
WHERE target_account.desktop_id = $1
|
||||||
|
AND target_account.tenant_id = $2
|
||||||
|
AND target_account.workspace_id = $3
|
||||||
|
)
|
||||||
|
`, task.TargetAccountID, task.TenantID, task.WorkspaceID, client.ID).Scan(&canLease); err != nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
for _, accountID := range accountIDs {
|
return canLease
|
||||||
if accountID == task.TargetAccountID {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildDesktopTaskView(task *repository.DesktopTask) DesktopTaskView {
|
func buildDesktopTaskView(task *repository.DesktopTask) DesktopTaskView {
|
||||||
|
|||||||
@@ -239,19 +239,28 @@ func TestBuildCountPublishTasksByStatusesQueryHidesDeletedPublishRecords(t *test
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLeaseNextQueuedMonitorTaskSQLSerializesPerClientPlatform(t *testing.T) {
|
func TestLeaseNextQueuedMonitorTaskSQLUsesAccountIdentityAndClientSlots(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
normalized := normalizeSQLForDesktopTaskTest(leaseNextQueuedMonitorTaskSQL())
|
normalized := normalizeSQLForDesktopTaskTest(leaseNextQueuedMonitorTaskSQL())
|
||||||
for _, fragment := range []string{
|
for _, fragment := range []string{
|
||||||
|
"WITH current_accounts AS",
|
||||||
|
"client_id = $3",
|
||||||
|
"dt.target_account_id = target_account.desktop_id",
|
||||||
|
"target_account.account_fingerprint",
|
||||||
|
"target_account.platform_uid",
|
||||||
|
"hashtextextended",
|
||||||
|
"pg_try_advisory_xact_lock(account_slot_lock_key)",
|
||||||
|
"SET target_client_id = $3, target_account_id = candidate.lease_account_id",
|
||||||
"NOT EXISTS ( SELECT 1 FROM desktop_tasks AS active",
|
"NOT EXISTS ( SELECT 1 FROM desktop_tasks AS active",
|
||||||
"active.target_client_id = $3",
|
|
||||||
"active.platform_id = dt.platform_id",
|
"active.platform_id = dt.platform_id",
|
||||||
"active.status = 'in_progress'",
|
"active.status = 'in_progress'",
|
||||||
"active.desktop_id <> dt.desktop_id",
|
"active.desktop_id <> dt.desktop_id",
|
||||||
"COUNT(DISTINCT active_platform.platform_id)",
|
"COUNT(DISTINCT active_platform.platform_id)",
|
||||||
"active_platform.target_client_id = $3",
|
"active_platform.target_client_id = $3",
|
||||||
"< $8::integer",
|
"< $6::integer",
|
||||||
|
"active_client_platform.target_client_id = $3",
|
||||||
|
"active_client_platform.platform_id = dt.platform_id",
|
||||||
"recent.status = 'succeeded'",
|
"recent.status = 'succeeded'",
|
||||||
"recent.updated_at >= now() - interval '2 seconds'",
|
"recent.updated_at >= now() - interval '2 seconds'",
|
||||||
"recent.status IN ('failed', 'unknown', 'aborted')",
|
"recent.status IN ('failed', 'unknown', 'aborted')",
|
||||||
@@ -264,19 +273,30 @@ func TestLeaseNextQueuedMonitorTaskSQLSerializesPerClientPlatform(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLeaseQueuedDesktopTaskByIDSQLSerializesMonitorPerClientPlatform(t *testing.T) {
|
func TestLeaseQueuedDesktopTaskByIDSQLUsesAccountIdentityAndClientSlots(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
normalized := normalizeSQLForDesktopTaskTest(leaseQueuedDesktopTaskByIDSQL())
|
normalized := normalizeSQLForDesktopTaskTest(leaseQueuedDesktopTaskByIDSQL())
|
||||||
for _, fragment := range []string{
|
for _, fragment := range []string{
|
||||||
|
"WITH current_accounts AS",
|
||||||
|
"client_id = $4",
|
||||||
|
"dt.kind <> 'monitor' OR ca.desktop_id IS NOT NULL",
|
||||||
|
"target_account.account_fingerprint",
|
||||||
|
"target_account.platform_uid",
|
||||||
|
"SELECT dt.desktop_id, dt.kind, ca.desktop_id AS lease_account_id",
|
||||||
|
"locked_candidate AS",
|
||||||
|
"pg_try_advisory_xact_lock(account_slot_lock_key)",
|
||||||
|
"FROM locked_candidate WHERE t.desktop_id = locked_candidate.desktop_id",
|
||||||
|
"target_account_id = CASE WHEN t.kind = 'monitor' THEN locked_candidate.lease_account_id ELSE t.target_account_id END",
|
||||||
"dt.kind <> 'monitor' OR NOT EXISTS",
|
"dt.kind <> 'monitor' OR NOT EXISTS",
|
||||||
"active.target_client_id = $4",
|
|
||||||
"active.platform_id = dt.platform_id",
|
"active.platform_id = dt.platform_id",
|
||||||
"active.status = 'in_progress'",
|
"active.status = 'in_progress'",
|
||||||
"active.desktop_id <> dt.desktop_id",
|
"active.desktop_id <> dt.desktop_id",
|
||||||
"COUNT(DISTINCT active_platform.platform_id)",
|
"COUNT(DISTINCT active_platform.platform_id)",
|
||||||
"active_platform.target_client_id = $4",
|
"active_platform.target_client_id = $4",
|
||||||
"< $10::integer",
|
"< $8::integer",
|
||||||
|
"active_client_platform.target_client_id = $4",
|
||||||
|
"active_client_platform.platform_id = dt.platform_id",
|
||||||
"recent.status = 'succeeded'",
|
"recent.status = 'succeeded'",
|
||||||
"recent.updated_at >= now() - interval '2 seconds'",
|
"recent.updated_at >= now() - interval '2 seconds'",
|
||||||
"recent.status IN ('failed', 'unknown', 'aborted')",
|
"recent.status IN ('failed', 'unknown', 'aborted')",
|
||||||
|
|||||||
@@ -0,0 +1,5 @@
|
|||||||
|
DROP INDEX CONCURRENTLY IF EXISTS idx_platform_accounts_monitor_identity_uid;
|
||||||
|
DROP INDEX CONCURRENTLY IF EXISTS idx_platform_accounts_monitor_identity_fingerprint;
|
||||||
|
DROP INDEX CONCURRENTLY IF EXISTS idx_desktop_tasks_monitor_active_client_platform;
|
||||||
|
DROP INDEX CONCURRENTLY IF EXISTS idx_desktop_tasks_monitor_active_platform_account;
|
||||||
|
DROP INDEX CONCURRENTLY IF EXISTS idx_desktop_tasks_monitor_queue_account_order;
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_desktop_tasks_monitor_queue_account_order
|
||||||
|
ON desktop_tasks (
|
||||||
|
tenant_id,
|
||||||
|
workspace_id,
|
||||||
|
target_account_id,
|
||||||
|
platform_id,
|
||||||
|
lane_weight DESC,
|
||||||
|
priority DESC,
|
||||||
|
enqueued_at ASC,
|
||||||
|
created_at ASC,
|
||||||
|
desktop_id ASC
|
||||||
|
)
|
||||||
|
WHERE kind = 'monitor'
|
||||||
|
AND status = 'queued';
|
||||||
|
|
||||||
|
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_desktop_tasks_monitor_active_platform_account
|
||||||
|
ON desktop_tasks (
|
||||||
|
tenant_id,
|
||||||
|
workspace_id,
|
||||||
|
platform_id,
|
||||||
|
status,
|
||||||
|
target_account_id,
|
||||||
|
updated_at DESC
|
||||||
|
)
|
||||||
|
WHERE kind = 'monitor'
|
||||||
|
AND status IN ('in_progress', 'succeeded', 'failed', 'unknown', 'aborted');
|
||||||
|
|
||||||
|
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_desktop_tasks_monitor_active_client_platform
|
||||||
|
ON desktop_tasks (
|
||||||
|
tenant_id,
|
||||||
|
workspace_id,
|
||||||
|
target_client_id,
|
||||||
|
platform_id,
|
||||||
|
status,
|
||||||
|
desktop_id
|
||||||
|
)
|
||||||
|
WHERE kind = 'monitor'
|
||||||
|
AND status = 'in_progress';
|
||||||
|
|
||||||
|
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_platform_accounts_monitor_identity_fingerprint
|
||||||
|
ON platform_accounts (
|
||||||
|
tenant_id,
|
||||||
|
workspace_id,
|
||||||
|
user_id,
|
||||||
|
platform_id,
|
||||||
|
account_fingerprint
|
||||||
|
)
|
||||||
|
WHERE deleted_at IS NULL
|
||||||
|
AND delete_requested_at IS NULL
|
||||||
|
AND btrim(COALESCE(account_fingerprint, '')) <> '';
|
||||||
|
|
||||||
|
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_platform_accounts_monitor_identity_uid
|
||||||
|
ON platform_accounts (
|
||||||
|
tenant_id,
|
||||||
|
workspace_id,
|
||||||
|
user_id,
|
||||||
|
platform_id,
|
||||||
|
platform_uid
|
||||||
|
)
|
||||||
|
WHERE deleted_at IS NULL
|
||||||
|
AND delete_requested_at IS NULL
|
||||||
|
AND btrim(COALESCE(platform_uid, '')) <> '';
|
||||||
Reference in New Issue
Block a user