fix: queue monitor tasks and recover doubao dom answers
Desktop Client Build / Resolve Build Metadata (push) Successful in 26s
Backend CI / Backend (push) Successful in 15m54s
Desktop Client Build / Build Desktop Client (push) Successful in 22m56s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 28s

This commit is contained in:
2026-06-22 16:37:41 +08:00
parent 12681105f2
commit 8f7a83bba9
6 changed files with 154 additions and 73 deletions
@@ -25,7 +25,10 @@ import (
"github.com/geo-platform/tenant-api/internal/tenant/repository"
)
const desktopPublishMaxAttempts = 3
const (
desktopPublishMaxAttempts = 3
maxConcurrentMonitorPlatformsPerDesktopClient = 2
)
var errDesktopTaskLeaseDeferred = errors.New("desktop task lease deferred")
@@ -520,7 +523,13 @@ func (s *DesktopTaskService) leaseNextQueuedMonitorTask(
}
hasAccountIDs := len(accountIDs) > 0
row := s.pool.QueryRow(ctx, leaseNextQueuedMonitorTaskSQL(),
tx, err := beginDesktopMonitorLeaseTx(ctx, s.pool, client.ID)
if err != nil {
return nil, err
}
defer tx.Rollback(ctx)
row := tx.QueryRow(ctx, leaseNextQueuedMonitorTaskSQL(),
client.TenantID,
client.WorkspaceID,
client.ID,
@@ -528,8 +537,16 @@ func (s *DesktopTaskService) leaseNextQueuedMonitorTask(
accountIDs,
params.AttemptID,
params.LeaseTokenHash,
maxConcurrentMonitorPlatformsPerDesktopClient,
)
return scanRepositoryDesktopTask(row)
task, err := scanRepositoryDesktopTask(row)
if err != nil {
return nil, err
}
if err := tx.Commit(ctx); err != nil {
return nil, response.ErrInternal(50088, "desktop_task_lease_failed", "failed to commit desktop monitor task lease")
}
return task, nil
}
func leaseNextQueuedMonitorTaskSQL() string {
@@ -564,6 +581,16 @@ func leaseNextQueuedMonitorTaskSQL() string {
AND recent.updated_at >= now() - interval '30 seconds'
AND recent.desktop_id <> dt.desktop_id
)
AND (
SELECT COUNT(DISTINCT active_platform.platform_id)
FROM desktop_tasks AS active_platform
WHERE active_platform.tenant_id = dt.tenant_id
AND active_platform.workspace_id = dt.workspace_id
AND active_platform.kind = 'monitor'
AND active_platform.target_client_id = $3
AND active_platform.status = 'in_progress'
AND active_platform.desktop_id <> dt.desktop_id
) < $8::integer
AND (
dt.target_client_id = $3
OR ($4::boolean AND dt.target_account_id = ANY($5::uuid[]))
@@ -658,7 +685,13 @@ func (s *DesktopTaskService) leaseQueuedDesktopTaskByID(
}
hasAccountIDs := len(accountIDs) > 0
row := s.pool.QueryRow(ctx, leaseQueuedDesktopTaskByIDSQL(),
tx, err := beginDesktopMonitorLeaseTx(ctx, s.pool, client.ID)
if err != nil {
return nil, err
}
defer tx.Rollback(ctx)
row := tx.QueryRow(ctx, leaseQueuedDesktopTaskByIDSQL(),
desktopID,
client.WorkspaceID,
client.TenantID,
@@ -668,8 +701,16 @@ func (s *DesktopTaskService) leaseQueuedDesktopTaskByID(
params.AttemptID,
params.LeaseTokenHash,
desktopPublishMaxAttempts,
maxConcurrentMonitorPlatformsPerDesktopClient,
)
return scanRepositoryDesktopTask(row)
task, err := scanRepositoryDesktopTask(row)
if err != nil {
return nil, err
}
if err := tx.Commit(ctx); err != nil {
return nil, response.ErrInternal(50088, "desktop_task_lease_failed", "failed to commit desktop task lease")
}
return task, nil
}
func leaseQueuedDesktopTaskByIDSQL() string {
@@ -722,6 +763,19 @@ func leaseQueuedDesktopTaskByIDSQL() string {
AND recent.desktop_id <> dt.desktop_id
)
)
AND (
dt.kind <> 'monitor'
OR (
SELECT COUNT(DISTINCT active_platform.platform_id)
FROM desktop_tasks AS active_platform
WHERE active_platform.tenant_id = dt.tenant_id
AND active_platform.workspace_id = dt.workspace_id
AND active_platform.kind = 'monitor'
AND active_platform.target_client_id = $4
AND active_platform.status = 'in_progress'
AND active_platform.desktop_id <> dt.desktop_id
) < $10::integer
)
AND (
(
dt.kind = 'monitor'
@@ -753,6 +807,21 @@ func leaseQueuedDesktopTaskByIDSQL() string {
RETURNING ` + desktopTaskRepositoryReturningColumns
}
func beginDesktopMonitorLeaseTx(ctx context.Context, pool *pgxpool.Pool, clientID uuid.UUID) (pgx.Tx, error) {
if pool == nil {
return nil, response.ErrInternal(50088, "desktop_task_lease_failed", "desktop task pool is not available")
}
tx, err := pool.Begin(ctx)
if err != nil {
return nil, response.ErrInternal(50088, "desktop_task_lease_failed", "failed to start desktop monitor task lease")
}
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, monitorDesktopTaskClientSlotLockKey(clientID)); err != nil {
_ = tx.Rollback(ctx)
return nil, response.ErrInternal(50088, "desktop_task_lease_failed", "failed to lock desktop monitor task lease")
}
return tx, nil
}
func (s *DesktopTaskService) loadMonitorLeaseAccountIDs(ctx context.Context, client *repository.DesktopClient) ([]uuid.UUID, error) {
if s == nil || client == nil || s.pool == nil {
return nil, nil
@@ -249,6 +249,9 @@ func TestLeaseNextQueuedMonitorTaskSQLSerializesPerClientPlatform(t *testing.T)
"active.platform_id = dt.platform_id",
"active.status = 'in_progress'",
"active.desktop_id <> dt.desktop_id",
"COUNT(DISTINCT active_platform.platform_id)",
"active_platform.target_client_id = $3",
"< $8::integer",
"recent.status IN ('succeeded', 'failed', 'unknown', 'aborted')",
"recent.updated_at >= now() - interval '30 seconds'",
} {
@@ -268,6 +271,9 @@ func TestLeaseQueuedDesktopTaskByIDSQLSerializesMonitorPerClientPlatform(t *test
"active.platform_id = dt.platform_id",
"active.status = 'in_progress'",
"active.desktop_id <> dt.desktop_id",
"COUNT(DISTINCT active_platform.platform_id)",
"active_platform.target_client_id = $4",
"< $10::integer",
"recent.status IN ('succeeded', 'failed', 'unknown', 'aborted')",
"recent.updated_at >= now() - interval '30 seconds'",
} {
@@ -246,14 +246,6 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
spec.TargetClientID = target.ClientID
}
available, throttleErr := acquireMonitorDesktopTaskPlatformSlot(ctx, tx, spec)
if throttleErr != nil {
return nil, nil, throttleErr
}
if !available {
continue
}
task, shouldPublish, fallbackLegacy, upsertErr := s.upsertMonitorDesktopTask(ctx, tx, repo, spec)
if upsertErr != nil {
return nil, nil, upsertErr
@@ -282,43 +274,8 @@ func monitorDesktopTaskSpecsNeedTargetLookup(specs []monitorDesktopTaskSpec) boo
return false
}
func acquireMonitorDesktopTaskPlatformSlot(ctx context.Context, tx pgx.Tx, spec monitorDesktopTaskSpec) (bool, error) {
if tx == nil || spec.TargetClientID == uuid.Nil || strings.TrimSpace(spec.PlatformID) == "" {
return false, nil
}
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, monitorDesktopTaskPlatformSlotLockKey(spec.TargetClientID, spec.PlatformID)); err != nil {
return false, response.ErrInternal(50123, "desktop_task_throttle_lock_failed", "failed to acquire phase2 monitor desktop platform slot")
}
var busy bool
if err := tx.QueryRow(ctx, monitorDesktopTaskPlatformSlotBusySQL(), spec.TenantID, spec.WorkspaceID, spec.TargetClientID, normalizeMonitoringPlatformID(spec.PlatformID), spec.MonitorTaskID).Scan(&busy); err != nil {
return false, response.ErrInternal(50124, "desktop_task_throttle_lookup_failed", "failed to inspect phase2 monitor desktop platform slot")
}
return !busy, nil
}
func monitorDesktopTaskPlatformSlotBusySQL() string {
return `
SELECT EXISTS (
SELECT 1
FROM desktop_tasks
WHERE tenant_id = $1
AND workspace_id = $2
AND kind = 'monitor'
AND target_client_id = $3
AND platform_id = $4
AND status = 'in_progress'
AND (
$5::bigint <= 0
OR monitor_task_id IS NULL
OR monitor_task_id <> $5
)
)
`
}
func monitorDesktopTaskPlatformSlotLockKey(clientID uuid.UUID, platformID string) int64 {
return int64(monitoringDailyStableHash("monitor_desktop_platform_slot", clientID.String(), normalizeMonitoringPlatformID(platformID)))
func monitorDesktopTaskClientSlotLockKey(clientID uuid.UUID) int64 {
return int64(monitoringDailyStableHash("monitor_desktop_client_slot", clientID.String()))
}
func (s *MonitoringService) loadMonitorDesktopTaskTargets(
@@ -207,21 +207,6 @@ func TestMonitoringSkipOutcomeKeepsOrdinarySkip(t *testing.T) {
assert.Equal(t, "stale_business_date", outcome.StoredSkipReason)
}
func TestMonitorDesktopTaskPlatformSlotBusySQLThrottlesPerClientPlatformInProgress(t *testing.T) {
query := strings.Join(strings.Fields(monitorDesktopTaskPlatformSlotBusySQL()), " ")
for _, fragment := range []string{
"FROM desktop_tasks",
"kind = 'monitor'",
"target_client_id = $3",
"platform_id = $4",
"status = 'in_progress'",
"monitor_task_id <> $5",
} {
assert.Contains(t, query, fragment)
}
}
func TestRequestPhase2MonitorInterruptsAcceptsExcludedPlatforms(t *testing.T) {
clientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000701")
targets, err := (&MonitoringService{}).requestPhase2MonitorInterrupts(