root/geo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix monitor desktop task recovery

Browse Source
This commit is contained in:
liangxu
2026-06-25 18:34:58 +08:00
parent cbbec8da85
commit 4d07f40fdf
3 changed files with 275 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/tenant/app/desktop_task_service.go
+118 -20
View File
@@ -2012,6 +2012,10 @@ func (s *DesktopTaskService) recoverClientTasks(
if err != nil {
return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode monitor desktop task recovery payload")
}
monitorUnrecoverableErrorJSON, monitorUnrecoverableReason, _, err := desktopTaskRecoveryPayload(mode, "monitor_unrecoverable")
if err != nil {
return response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode monitor desktop task recovery payload")
}
tx, err := s.pool.Begin(ctx)
if err != nil {
@@ -2069,7 +2073,9 @@ func (s *DesktopTaskService) recoverClientTasks(
return failErr
}
if len(failedMonitorTaskIDs) == 0 {
continue
finalStatus = "aborted"
monitorErrorForTask = monitorUnrecoverableErrorJSON
monitorReasonForTask = monitorUnrecoverableReason
}
}
} else if item.MonitorTaskID.Valid {
@@ -2078,7 +2084,9 @@ func (s *DesktopTaskService) recoverClientTasks(
return requeueErr
}
if len(requeuedMonitorTaskIDs) == 0 {
continue
finalStatus = "aborted"
monitorErrorForTask = monitorUnrecoverableErrorJSON
monitorReasonForTask = monitorUnrecoverableReason
}
}
if _, execErr := tx.Exec(ctx, `
@@ -2414,6 +2422,10 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
if err != nil {
return result, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode monitor desktop task recovery payload")
}
unrecoverableErrorJSON, unrecoverableReason, _, err := desktopTaskRecoveryPayload(desktopTaskRecoveryModeLeaseExpiry, "monitor_unrecoverable")
if err != nil {
return result, response.ErrInternal(50195, "desktop_task_recovery_payload_failed", "failed to encode monitor desktop task recovery payload")
}
tx, err := s.pool.Begin(ctx)
if err != nil {
@@ -2464,39 +2476,70 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
requeuedMonitorTaskSet := int64Set(requeuedMonitorTaskIDs)
recoverableDesktopIDs := make([]uuid.UUID, 0, len(recovered))
recoverable := make([]recoveredDesktopTaskLease, 0, len(recovered))
unrecoverableDesktopIDs := make([]uuid.UUID, 0, len(recovered))
unrecoverable := make([]recoveredDesktopTaskLease, 0, len(recovered))
for _, item := range recovered {
if !item.MonitorTaskID.Valid {
continue
}
if _, ok := requeuedMonitorTaskSet[item.MonitorTaskID.Int64]; !ok {
item.Status = "aborted"
item.ErrorJSON = unrecoverableErrorJSON
unrecoverableDesktopIDs = append(unrecoverableDesktopIDs, item.TaskID)
unrecoverable = append(unrecoverable, item)
continue
}
recoverableDesktopIDs = append(recoverableDesktopIDs, item.TaskID)
recoverable = append(recoverable, item)
}
if len(recoverableDesktopIDs) == 0 {
if len(recoverableDesktopIDs) == 0 && len(unrecoverableDesktopIDs) == 0 {
return result, nil
}
updateRows, err := tx.Query(ctx, requeueExpiredMonitorDesktopTasksSQL(), recoverableDesktopIDs, errorJSON, reason)
if err != nil {
s.logWarn("expired desktop monitor recovery update failed", err)
return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
}
defer updateRows.Close()
updatedDesktopIDs := make(map[uuid.UUID]struct{}, len(recoverableDesktopIDs))
for updateRows.Next() {
var desktopID uuid.UUID
if scanErr := updateRows.Scan(&desktopID); scanErr != nil {
s.logWarn("expired desktop monitor recovery update scan failed", scanErr)
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to parse recovered desktop tasks")
if len(recoverableDesktopIDs) > 0 {
updateRows, err := tx.Query(ctx, requeueExpiredMonitorDesktopTasksSQL(), recoverableDesktopIDs, errorJSON, reason)
if err != nil {
s.logWarn("expired desktop monitor recovery update failed", err)
return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
}
defer updateRows.Close()
for updateRows.Next() {
var desktopID uuid.UUID
if scanErr := updateRows.Scan(&desktopID); scanErr != nil {
s.logWarn("expired desktop monitor recovery update scan failed", scanErr)
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to parse recovered desktop tasks")
}
updatedDesktopIDs[desktopID] = struct{}{}
}
if err := updateRows.Err(); err != nil {
s.logWarn("expired desktop monitor recovery update rows failed", err)
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to iterate recovered desktop tasks")
}
updatedDesktopIDs[desktopID] = struct{}{}
}
if err := updateRows.Err(); err != nil {
s.logWarn("expired desktop monitor recovery update rows failed", err)
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to iterate recovered desktop tasks")
abortedDesktopIDs := make(map[uuid.UUID]struct{}, len(unrecoverableDesktopIDs))
if len(unrecoverableDesktopIDs) > 0 {
abortRows, err := tx.Query(ctx, abortExpiredMonitorDesktopTasksSQL(), unrecoverableDesktopIDs, unrecoverableErrorJSON, unrecoverableReason)
if err != nil {
s.logWarn("expired desktop monitor unrecoverable update failed", err)
return result, response.ErrInternal(50197, "desktop_task_recovery_query_failed", "failed to select recoverable desktop tasks")
}
defer abortRows.Close()
for abortRows.Next() {
var desktopID uuid.UUID
if scanErr := abortRows.Scan(&desktopID); scanErr != nil {
s.logWarn("expired desktop monitor unrecoverable update scan failed", scanErr)
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to parse recovered desktop tasks")
}
abortedDesktopIDs[desktopID] = struct{}{}
}
if err := abortRows.Err(); err != nil {
s.logWarn("expired desktop monitor unrecoverable update rows failed", err)
return result, response.ErrInternal(50198, "desktop_task_recovery_scan_failed", "failed to iterate recovered desktop tasks")
}
}
repo := repository.NewDesktopTaskRepository(tx)
@@ -2523,12 +2566,36 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
s.logWarn("expired desktop monitor recovery attempt finish failed", finishErr, zap.String("task_id", item.TaskID.String()))
}
}
for _, item := range unrecoverable {
if _, ok := abortedDesktopIDs[item.TaskID]; !ok {
continue
}
if !item.ActiveAttempt.Valid {
continue
}
attemptID, convErr := uuid.FromBytes(item.ActiveAttempt.Bytes[:])
if convErr != nil {
s.logWarn("expired desktop monitor unrecoverable attempt id decode failed", convErr, zap.String("task_id", item.TaskID.String()))
continue
}
if finishErr := repo.FinishAttempt(ctx, repository.FinishDesktopTaskAttemptParams{
TaskID: item.TaskID,
AttemptID: attemptID,
FinalStatus: literalStringPtr("aborted"),
Error: unrecoverableErrorJSON,
}); finishErr != nil {
s.logWarn("expired desktop monitor unrecoverable attempt finish failed", finishErr, zap.String("task_id", item.TaskID.String()))
}
}
if err := tx.Commit(ctx); err != nil {
return result, response.ErrInternal(50199, "desktop_task_recovery_commit_failed", "failed to commit desktop task recovery")
}
result.Requeued = len(updatedDesktopIDs)
result.Failed = len(abortedDesktopIDs)
for _, item := range recoverable {
if _, ok := updatedDesktopIDs[item.TaskID]; !ok {
@@ -2541,10 +2608,22 @@ func (s *DesktopTaskService) RecoverExpiredMonitorTasks(ctx context.Context, lim
}
s.publishTaskEvent(ctx, task, "task_available")
}
for _, item := range unrecoverable {
if _, ok := abortedDesktopIDs[item.TaskID]; !ok {
continue
}
task, getErr := s.repo.GetByDesktopID(ctx, item.TaskID, item.WorkspaceID)
if getErr != nil {
s.logWarn("expired desktop monitor unrecoverable reload after commit failed", getErr, zap.String("task_id", item.TaskID.String()))
continue
}
s.publishTaskEvent(ctx, task, "task_completed")
}
if result.Requeued > 0 && s.logger != nil {
if (result.Requeued > 0 || result.Failed > 0) && s.logger != nil {
s.logger.Info("expired desktop monitor tasks recovered",
zap.Int("requeued", result.Requeued),
zap.Int("failed", result.Failed),
)
}
@@ -2594,6 +2673,25 @@ func requeueExpiredMonitorDesktopTasksSQL() string {
`
}
func abortExpiredMonitorDesktopTasksSQL() string {
return `
UPDATE desktop_tasks
SET status = 'aborted',
error = $2,
result = NULL,
active_attempt_id = NULL,
lease_token_hash = NULL,
lease_expires_at = NULL,
interrupted_at = COALESCE(interrupted_at, NOW()),
interrupt_reason = COALESCE(interrupt_reason, $3),
updated_at = NOW()
WHERE desktop_id = ANY($1::uuid[])
AND kind = 'monitor'
AND status = 'in_progress'
RETURNING desktop_id
`
}
func (s *DesktopTaskService) requeueMonitorCollectTasksForDesktopRecovery(ctx context.Context, monitorTaskIDs []int64, errorJSON []byte) ([]int64, error) {
if s == nil || s.monitoringPool == nil || len(monitorTaskIDs) == 0 {
return nil, nil