fix: fail expired monitor desktop leases
Frontend CI / Frontend (push) Successful in 2m26s
Backend CI / Backend (push) Failing after 8m42s

This commit is contained in:
2026-06-23 01:03:35 +08:00
parent 8e92737bc2
commit d458dd7477
8 changed files with 514 additions and 21 deletions
@@ -11,22 +11,24 @@ import (
)
const (
defaultMonitoringLeaseRecoveryInterval = 30 * time.Minute
defaultMonitoringLeaseRecoveryInterval = 1 * time.Minute
defaultMonitoringLeaseRecoveryTimeout = 30 * time.Second
defaultMonitoringLeaseRecoveryLimit = 1000
)
type MonitoringLeaseRecoveryWorker struct {
monitoringPool *pgxpool.Pool
desktopTasks *tenantapp.DesktopTaskService
logger *zap.Logger
interval time.Duration
timeout time.Duration
limit int
}
func NewMonitoringLeaseRecoveryWorker(monitoringPool *pgxpool.Pool, logger *zap.Logger) *MonitoringLeaseRecoveryWorker {
func NewMonitoringLeaseRecoveryWorker(monitoringPool *pgxpool.Pool, desktopTasks *tenantapp.DesktopTaskService, logger *zap.Logger) *MonitoringLeaseRecoveryWorker {
return &MonitoringLeaseRecoveryWorker{
monitoringPool: monitoringPool,
desktopTasks: desktopTasks,
logger: logger,
interval: defaultMonitoringLeaseRecoveryInterval,
timeout: defaultMonitoringLeaseRecoveryTimeout,
@@ -89,8 +91,23 @@ func (w *MonitoringLeaseRecoveryWorker) runOnce(parent context.Context) {
return
}
if expiredCount > 0 && w.logger != nil {
w.logger.Info("monitoring lease recovery completed", zap.Int64("expired_task_count", expiredCount))
var monitorResult tenantapp.MonitorLeaseRecoveryResult
if w.desktopTasks != nil {
var recoverErr error
monitorResult, recoverErr = w.desktopTasks.RecoverExpiredMonitorTasks(ctx, w.limit)
if recoverErr != nil {
if w.logger != nil {
w.logger.Warn("monitor desktop lease recovery failed", zap.Error(recoverErr))
}
return
}
}
if (expiredCount > 0 || monitorResult.Failed > 0) && w.logger != nil {
w.logger.Info("monitoring lease recovery completed",
zap.Int64("expired_task_count", expiredCount),
zap.Int("failed_desktop_task_count", monitorResult.Failed),
)
}
}
@@ -126,9 +143,18 @@ func (w *MonitoringLeaseRecoveryWorker) RunOnce(parent context.Context, run JobR
if err := tx.Commit(ctx); err != nil {
return map[string]any{"stage": "commit"}, err
}
var monitorResult tenantapp.MonitorLeaseRecoveryResult
if w.desktopTasks != nil {
var recoverErr error
monitorResult, recoverErr = w.desktopTasks.RecoverExpiredMonitorTasks(ctx, limit)
if recoverErr != nil {
return map[string]any{"stage": "recover_desktop_tasks"}, recoverErr
}
}
return map[string]any{
"expired_task_count": expiredCount,
"duration_ms": time.Since(startedAt).Milliseconds(),
"batch_size": limit,
"expired_task_count": expiredCount,
"failed_desktop_monitor_count": monitorResult.Failed,
"duration_ms": time.Since(startedAt).Milliseconds(),
"batch_size": limit,
}, nil
}