fix monitor desktop unknown recovery
This commit is contained in:
@@ -27,6 +27,8 @@ import (
|
|||||||
|
|
||||||
const desktopPublishMaxAttempts = 3
|
const desktopPublishMaxAttempts = 3
|
||||||
|
|
||||||
|
var errDesktopTaskLeaseDeferred = errors.New("desktop task lease deferred")
|
||||||
|
|
||||||
type DesktopTaskService struct {
|
type DesktopTaskService struct {
|
||||||
pool *pgxpool.Pool
|
pool *pgxpool.Pool
|
||||||
monitoringPool *pgxpool.Pool
|
monitoringPool *pgxpool.Pool
|
||||||
@@ -247,7 +249,11 @@ func (s *DesktopTaskService) Lease(ctx context.Context, client *repository.Deskt
|
|||||||
if taskID == nil {
|
if taskID == nil {
|
||||||
return &LeaseDesktopTaskResponse{}, nil
|
return &LeaseDesktopTaskResponse{}, nil
|
||||||
}
|
}
|
||||||
return nil, s.classifyLeaseError(ctx, client, taskID)
|
classifiedErr := s.classifyLeaseError(ctx, client, taskID)
|
||||||
|
if errors.Is(classifiedErr, errDesktopTaskLeaseDeferred) {
|
||||||
|
return &LeaseDesktopTaskResponse{}, nil
|
||||||
|
}
|
||||||
|
return nil, classifiedErr
|
||||||
}
|
}
|
||||||
fields := []zap.Field{
|
fields := []zap.Field{
|
||||||
zap.String("client_id", client.ID.String()),
|
zap.String("client_id", client.ID.String()),
|
||||||
@@ -966,6 +972,9 @@ func (s *DesktopTaskService) Complete(ctx context.Context, client *repository.De
|
|||||||
taskRepo := repository.NewDesktopTaskRepository(tx)
|
taskRepo := repository.NewDesktopTaskRepository(tx)
|
||||||
previousTask, _ := taskRepo.GetByDesktopID(ctx, desktopID, client.WorkspaceID)
|
previousTask, _ := taskRepo.GetByDesktopID(ctx, desktopID, client.WorkspaceID)
|
||||||
previousAttemptID := activeAttemptID(previousTask)
|
previousAttemptID := activeAttemptID(previousTask)
|
||||||
|
if previousTask != nil {
|
||||||
|
finalStatus = normalizeDesktopTaskCompletionStatusForKind(previousTask.Kind, finalStatus)
|
||||||
|
}
|
||||||
|
|
||||||
task, err := taskRepo.Complete(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(req.LeaseToken)), finalStatus, resultJSON, errorJSON)
|
task, err := taskRepo.Complete(ctx, desktopID, HashDesktopClientToken(strings.TrimSpace(req.LeaseToken)), finalStatus, resultJSON, errorJSON)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1558,10 +1567,19 @@ func (s *DesktopTaskService) classifyLeaseError(ctx context.Context, client *rep
|
|||||||
return response.ErrForbidden(40381, "desktop_task_not_target_client", "desktop task is assigned to another desktop client")
|
return response.ErrForbidden(40381, "desktop_task_not_target_client", "desktop task is assigned to another desktop client")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return classifyDesktopTaskLeaseState(task)
|
||||||
|
}
|
||||||
|
|
||||||
|
func classifyDesktopTaskLeaseState(task *repository.DesktopTask) error {
|
||||||
|
if task == nil {
|
||||||
|
return response.ErrNotFound(40482, "desktop_task_not_found", "desktop task not found")
|
||||||
|
}
|
||||||
|
if task.Kind == "monitor" && task.Status == "queued" {
|
||||||
|
return errDesktopTaskLeaseDeferred
|
||||||
|
}
|
||||||
if task.Status == "in_progress" {
|
if task.Status == "in_progress" {
|
||||||
return response.ErrConflict(40989, "desktop_task_already_in_progress", "desktop task is already in progress")
|
return response.ErrConflict(40989, "desktop_task_already_in_progress", "desktop task is already in progress")
|
||||||
}
|
}
|
||||||
|
|
||||||
return response.ErrConflict(40991, "desktop_task_not_queued", "desktop task is not queued")
|
return response.ErrConflict(40991, "desktop_task_not_queued", "desktop task is not queued")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1646,13 +1664,22 @@ func normalizeDesktopTaskCompletionStatus(status string) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeDesktopTaskViewStatus(kind string, status string) string {
|
func normalizeDesktopTaskCompletionStatusForKind(kind string, status string) string {
|
||||||
if strings.TrimSpace(kind) == "publish" {
|
if strings.TrimSpace(kind) == "monitor" && strings.TrimSpace(status) == "unknown" {
|
||||||
return normalizeDesktopTaskTerminalStatus(status)
|
return "failed"
|
||||||
}
|
}
|
||||||
return strings.TrimSpace(status)
|
return strings.TrimSpace(status)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func normalizeDesktopTaskViewStatus(kind string, status string) string {
|
||||||
|
switch strings.TrimSpace(kind) {
|
||||||
|
case "monitor", "publish":
|
||||||
|
return normalizeDesktopTaskTerminalStatus(status)
|
||||||
|
default:
|
||||||
|
return strings.TrimSpace(status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type desktopTaskRecoveryMode string
|
type desktopTaskRecoveryMode string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|||||||
@@ -1,11 +1,13 @@
|
|||||||
package app
|
package app
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/geo-platform/tenant-api/internal/shared/response"
|
"github.com/geo-platform/tenant-api/internal/shared/response"
|
||||||
|
"github.com/geo-platform/tenant-api/internal/tenant/repository"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDesktopTaskRecoverySelectQueryMatchesRecoveredLeaseScanOrder(t *testing.T) {
|
func TestDesktopTaskRecoverySelectQueryMatchesRecoveredLeaseScanOrder(t *testing.T) {
|
||||||
@@ -138,19 +140,34 @@ func TestResolvePublishRecoveryOutcome(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNormalizeDesktopTaskCompletionStatusPreservesUnknown(t *testing.T) {
|
func TestNormalizeDesktopTaskCompletionStatusForKindFailsMonitorUnknown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
if got := normalizeDesktopTaskCompletionStatus("unknown"); got != "unknown" {
|
if got := normalizeDesktopTaskCompletionStatusForKind("monitor", "unknown"); got != "failed" {
|
||||||
t.Fatalf("normalizeDesktopTaskCompletionStatus(unknown) = %q, want unknown", got)
|
t.Fatalf("monitor completion status = %q, want failed", got)
|
||||||
|
}
|
||||||
|
if got := normalizeDesktopTaskCompletionStatusForKind("publish", "unknown"); got != "unknown" {
|
||||||
|
t.Fatalf("publish completion status = %q, want unknown", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNormalizeDesktopTaskViewStatusOnlyMapsPublishUnknownToFailed(t *testing.T) {
|
func TestClassifyLeaseErrorDefersQueuedMonitorTask(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
if got := normalizeDesktopTaskViewStatus("monitor", "unknown"); got != "unknown" {
|
task := &repository.DesktopTask{
|
||||||
t.Fatalf("monitor unknown view status = %q, want unknown", got)
|
Kind: "monitor",
|
||||||
|
Status: "queued",
|
||||||
|
}
|
||||||
|
if got := classifyDesktopTaskLeaseState(task); !errors.Is(got, errDesktopTaskLeaseDeferred) {
|
||||||
|
t.Fatalf("classifyDesktopTaskLeaseState() = %v, want deferred", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeDesktopTaskViewStatusMapsMonitorAndPublishUnknownToFailed(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
if got := normalizeDesktopTaskViewStatus("monitor", "unknown"); got != "failed" {
|
||||||
|
t.Fatalf("monitor unknown view status = %q, want failed", got)
|
||||||
}
|
}
|
||||||
if got := normalizeDesktopTaskViewStatus("publish", "unknown"); got != "failed" {
|
if got := normalizeDesktopTaskViewStatus("publish", "unknown"); got != "failed" {
|
||||||
t.Fatalf("publish unknown view status = %q, want failed", got)
|
t.Fatalf("publish unknown view status = %q, want failed", got)
|
||||||
|
|||||||
@@ -945,6 +945,7 @@ func (s *MonitoringCallbackService) skipTaskForInstallation(
|
|||||||
}
|
}
|
||||||
|
|
||||||
skipReason := normalizeSkipReason(req.SkipReason)
|
skipReason := normalizeSkipReason(req.SkipReason)
|
||||||
|
skipOutcome := monitoringSkipOutcomeForReason(skipReason)
|
||||||
|
|
||||||
tx, err := s.monitoringPool.Begin(ctx)
|
tx, err := s.monitoringPool.Begin(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -954,14 +955,14 @@ func (s *MonitoringCallbackService) skipTaskForInstallation(
|
|||||||
|
|
||||||
if _, err := tx.Exec(ctx, `
|
if _, err := tx.Exec(ctx, `
|
||||||
UPDATE monitoring_collect_tasks
|
UPDATE monitoring_collect_tasks
|
||||||
SET status = 'skipped',
|
SET status = $2,
|
||||||
callback_received_at = NOW(),
|
callback_received_at = NOW(),
|
||||||
completed_at = $2,
|
completed_at = $3,
|
||||||
skip_reason = $3,
|
skip_reason = $4,
|
||||||
error_message = $4,
|
error_message = $5,
|
||||||
updated_at = NOW()
|
updated_at = NOW()
|
||||||
WHERE id = $1
|
WHERE id = $1
|
||||||
`, task.ID, completedAt, skipReason, nullableString(req.ErrorMessage)); err != nil {
|
`, task.ID, skipOutcome.TaskStatus, completedAt, nullableStringPtr(skipOutcome.StoredSkipReason), nullableString(req.ErrorMessage)); err != nil {
|
||||||
return nil, response.ErrInternal(50041, "task_update_failed", "failed to update monitoring task skip status")
|
return nil, response.ErrInternal(50041, "task_update_failed", "failed to update monitoring task skip status")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -978,12 +979,34 @@ func (s *MonitoringCallbackService) skipTaskForInstallation(
|
|||||||
completedAtText := completedAt.Format(time.RFC3339)
|
completedAtText := completedAt.Format(time.RFC3339)
|
||||||
return &MonitoringSkipTaskResponse{
|
return &MonitoringSkipTaskResponse{
|
||||||
TaskID: task.ID,
|
TaskID: task.ID,
|
||||||
TaskStatus: "skipped",
|
TaskStatus: skipOutcome.TaskStatus,
|
||||||
SkipReason: skipReason,
|
SkipReason: skipReason,
|
||||||
CompletedAt: &completedAtText,
|
CompletedAt: &completedAtText,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type monitoringSkipOutcome struct {
|
||||||
|
TaskStatus string
|
||||||
|
StoredSkipReason string
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitoringSkipOutcomeForReason(skipReason string) monitoringSkipOutcome {
|
||||||
|
switch strings.TrimSpace(skipReason) {
|
||||||
|
case "runtime_unknown":
|
||||||
|
return monitoringSkipOutcome{TaskStatus: "failed"}
|
||||||
|
default:
|
||||||
|
return monitoringSkipOutcome{TaskStatus: "skipped", StoredSkipReason: strings.TrimSpace(skipReason)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func nullableStringPtr(value string) *string {
|
||||||
|
trimmed := strings.TrimSpace(value)
|
||||||
|
if trimmed == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &trimmed
|
||||||
|
}
|
||||||
|
|
||||||
func (s *MonitoringCallbackService) CancelTaskForDesktopClient(
|
func (s *MonitoringCallbackService) CancelTaskForDesktopClient(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client *repository.DesktopClient,
|
client *repository.DesktopClient,
|
||||||
@@ -1504,7 +1527,7 @@ func (s *MonitoringCallbackService) loadActiveDesktopMonitorExecutionLease(
|
|||||||
FROM desktop_tasks
|
FROM desktop_tasks
|
||||||
WHERE kind = 'monitor'
|
WHERE kind = 'monitor'
|
||||||
AND monitor_task_id = $1
|
AND monitor_task_id = $1
|
||||||
AND status IN ('queued', 'in_progress', 'unknown')
|
AND status IN ('queued', 'in_progress')
|
||||||
ORDER BY created_at DESC, id DESC
|
ORDER BY created_at DESC, id DESC
|
||||||
LIMIT 1
|
LIMIT 1
|
||||||
`, monitorTaskID).Scan(
|
`, monitorTaskID).Scan(
|
||||||
@@ -2349,16 +2372,7 @@ func (s *MonitoringCallbackService) completeLinkedDesktopMonitorTask(
|
|||||||
workspaceID int64
|
workspaceID int64
|
||||||
activeAttemptID pgtype.UUID
|
activeAttemptID pgtype.UUID
|
||||||
)
|
)
|
||||||
if err := tx.QueryRow(ctx, `
|
if err := tx.QueryRow(ctx, completeLinkedDesktopMonitorTaskLookupSQL(), task.ID).Scan(&desktopID, &workspaceID, &activeAttemptID); err != nil {
|
||||||
SELECT desktop_id, workspace_id, active_attempt_id
|
|
||||||
FROM desktop_tasks
|
|
||||||
WHERE kind = 'monitor'
|
|
||||||
AND monitor_task_id = $1
|
|
||||||
AND status IN ('in_progress', 'unknown')
|
|
||||||
ORDER BY created_at DESC, id DESC
|
|
||||||
LIMIT 1
|
|
||||||
FOR UPDATE
|
|
||||||
`, task.ID).Scan(&desktopID, &workspaceID, &activeAttemptID); err != nil {
|
|
||||||
if err == pgx.ErrNoRows {
|
if err == pgx.ErrNoRows {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -2434,6 +2448,19 @@ func (s *MonitoringCallbackService) completeLinkedDesktopMonitorTask(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func completeLinkedDesktopMonitorTaskLookupSQL() string {
|
||||||
|
return `
|
||||||
|
SELECT desktop_id, workspace_id, active_attempt_id
|
||||||
|
FROM desktop_tasks
|
||||||
|
WHERE kind = 'monitor'
|
||||||
|
AND monitor_task_id = $1
|
||||||
|
AND status = 'in_progress'
|
||||||
|
ORDER BY created_at DESC, id DESC
|
||||||
|
LIMIT 1
|
||||||
|
FOR UPDATE
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
func (s *MonitoringCallbackService) replaceCitationFacts(ctx context.Context, tx pgx.Tx, tenantID, runID int64) error {
|
func (s *MonitoringCallbackService) replaceCitationFacts(ctx context.Context, tx pgx.Tx, tenantID, runID int64) error {
|
||||||
if _, err := tx.Exec(ctx, `
|
if _, err := tx.Exec(ctx, `
|
||||||
DELETE FROM monitoring_citation_facts
|
DELETE FROM monitoring_citation_facts
|
||||||
|
|||||||
@@ -612,16 +612,7 @@ func (s *MonitoringService) upsertMonitorDesktopTask(
|
|||||||
existingLeaseExpiresAt pgtype.Timestamptz
|
existingLeaseExpiresAt pgtype.Timestamptz
|
||||||
existingActiveAttempt pgtype.UUID
|
existingActiveAttempt pgtype.UUID
|
||||||
)
|
)
|
||||||
err := tx.QueryRow(ctx, `
|
err := tx.QueryRow(ctx, upsertMonitorDesktopTaskActiveLookupSQL(), spec.MonitorTaskID).Scan(&existingDesktopID, &existingStatus, &existingLeaseExpiresAt, &existingActiveAttempt)
|
||||||
SELECT desktop_id, status, lease_expires_at, active_attempt_id
|
|
||||||
FROM desktop_tasks
|
|
||||||
WHERE kind = 'monitor'
|
|
||||||
AND monitor_task_id = $1
|
|
||||||
AND status IN ('queued', 'in_progress', 'unknown')
|
|
||||||
ORDER BY created_at DESC, id DESC
|
|
||||||
LIMIT 1
|
|
||||||
FOR UPDATE
|
|
||||||
`, spec.MonitorTaskID).Scan(&existingDesktopID, &existingStatus, &existingLeaseExpiresAt, &existingActiveAttempt)
|
|
||||||
switch err {
|
switch err {
|
||||||
case nil:
|
case nil:
|
||||||
task, getErr := repo.GetByDesktopID(ctx, existingDesktopID, spec.WorkspaceID)
|
task, getErr := repo.GetByDesktopID(ctx, existingDesktopID, spec.WorkspaceID)
|
||||||
@@ -771,6 +762,19 @@ func (s *MonitoringService) upsertMonitorDesktopTask(
|
|||||||
return task, true, false, nil
|
return task, true, false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func upsertMonitorDesktopTaskActiveLookupSQL() string {
|
||||||
|
return `
|
||||||
|
SELECT desktop_id, status, lease_expires_at, active_attempt_id
|
||||||
|
FROM desktop_tasks
|
||||||
|
WHERE kind = 'monitor'
|
||||||
|
AND monitor_task_id = $1
|
||||||
|
AND status IN ('queued', 'in_progress')
|
||||||
|
ORDER BY created_at DESC, id DESC
|
||||||
|
LIMIT 1
|
||||||
|
FOR UPDATE
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
func (s *MonitoringService) fallbackMonitorDesktopTasksToLegacy(ctx context.Context, taskIDs []int64) error {
|
func (s *MonitoringService) fallbackMonitorDesktopTasksToLegacy(ctx context.Context, taskIDs []int64) error {
|
||||||
if s == nil || s.monitoringPool == nil || len(taskIDs) == 0 {
|
if s == nil || s.monitoringPool == nil || len(taskIDs) == 0 {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -179,6 +179,34 @@ func TestMarshalMonitorDesktopTaskPayloadUsesQuestionTextNotFallbackTitle(t *tes
|
|||||||
assert.Equal(t, "通义千问 · 幽灵门轨道长度定制", decoded["title"])
|
assert.Equal(t, "通义千问 · 幽灵门轨道长度定制", decoded["title"])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestUpsertMonitorDesktopTaskIgnoresUnknownAsActive(t *testing.T) {
|
||||||
|
query := strings.Join(strings.Fields(upsertMonitorDesktopTaskActiveLookupSQL()), " ")
|
||||||
|
|
||||||
|
assert.Contains(t, query, "status IN ('queued', 'in_progress')")
|
||||||
|
assert.NotContains(t, query, "'unknown'")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompleteLinkedDesktopMonitorTaskOnlyFinalizesInProgress(t *testing.T) {
|
||||||
|
query := strings.Join(strings.Fields(completeLinkedDesktopMonitorTaskLookupSQL()), " ")
|
||||||
|
|
||||||
|
assert.Contains(t, query, "status = 'in_progress'")
|
||||||
|
assert.NotContains(t, query, "'unknown'")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitoringSkipOutcomeForRuntimeUnknownFailsTask(t *testing.T) {
|
||||||
|
outcome := monitoringSkipOutcomeForReason("runtime_unknown")
|
||||||
|
|
||||||
|
assert.Equal(t, "failed", outcome.TaskStatus)
|
||||||
|
assert.Empty(t, outcome.StoredSkipReason)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitoringSkipOutcomeKeepsOrdinarySkip(t *testing.T) {
|
||||||
|
outcome := monitoringSkipOutcomeForReason("stale_business_date")
|
||||||
|
|
||||||
|
assert.Equal(t, "skipped", outcome.TaskStatus)
|
||||||
|
assert.Equal(t, "stale_business_date", outcome.StoredSkipReason)
|
||||||
|
}
|
||||||
|
|
||||||
func TestMonitorDesktopTaskPlatformSlotBusySQLThrottlesPerClientPlatformInProgress(t *testing.T) {
|
func TestMonitorDesktopTaskPlatformSlotBusySQLThrottlesPerClientPlatformInProgress(t *testing.T) {
|
||||||
query := strings.Join(strings.Fields(monitorDesktopTaskPlatformSlotBusySQL()), " ")
|
query := strings.Join(strings.Fields(monitorDesktopTaskPlatformSlotBusySQL()), " ")
|
||||||
|
|
||||||
|
|||||||
@@ -54,4 +54,4 @@ CREATE UNIQUE INDEX IF NOT EXISTS uq_desktop_tasks_monitor_active
|
|||||||
ON desktop_tasks (monitor_task_id)
|
ON desktop_tasks (monitor_task_id)
|
||||||
WHERE kind = 'monitor'
|
WHERE kind = 'monitor'
|
||||||
AND monitor_task_id IS NOT NULL
|
AND monitor_task_id IS NOT NULL
|
||||||
AND status IN ('queued', 'in_progress', 'unknown');
|
AND status IN ('queued', 'in_progress');
|
||||||
|
|||||||
@@ -0,0 +1,7 @@
|
|||||||
|
DROP INDEX IF EXISTS uq_desktop_tasks_monitor_active;
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_desktop_tasks_monitor_active
|
||||||
|
ON desktop_tasks (monitor_task_id)
|
||||||
|
WHERE kind = 'monitor'
|
||||||
|
AND monitor_task_id IS NOT NULL
|
||||||
|
AND status IN ('queued', 'in_progress', 'unknown');
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
UPDATE desktop_tasks
|
||||||
|
SET status = 'failed',
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE kind = 'monitor'
|
||||||
|
AND status = 'unknown';
|
||||||
|
|
||||||
|
DROP INDEX IF EXISTS uq_desktop_tasks_monitor_active;
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_desktop_tasks_monitor_active
|
||||||
|
ON desktop_tasks (monitor_task_id)
|
||||||
|
WHERE kind = 'monitor'
|
||||||
|
AND monitor_task_id IS NOT NULL
|
||||||
|
AND status IN ('queued', 'in_progress');
|
||||||
Reference in New Issue
Block a user