geo/server/internal/scheduler/monitoring_lease_recovery_worker.go

package scheduler

import (
	"context"
	"time"

	"github.com/jackc/pgx/v5/pgxpool"
	"go.uber.org/zap"

	tenantapp "github.com/geo-platform/tenant-api/internal/tenant/app"
)

const (
	defaultMonitoringLeaseRecoveryInterval = 10 * time.Minute
	defaultMonitoringLeaseRecoveryTimeout  = 30 * time.Second
	defaultMonitoringLeaseRecoveryLimit    = 1000
)

type MonitoringLeaseRecoveryWorker struct {
	monitoringPool *pgxpool.Pool
	desktopTasks   *tenantapp.DesktopTaskService
	logger         *zap.Logger
	interval       time.Duration
	timeout        time.Duration
	limit          int
}

func NewMonitoringLeaseRecoveryWorker(monitoringPool *pgxpool.Pool, desktopTasks *tenantapp.DesktopTaskService, logger *zap.Logger) *MonitoringLeaseRecoveryWorker {
	return &MonitoringLeaseRecoveryWorker{
		monitoringPool: monitoringPool,
		desktopTasks:   desktopTasks,
		logger:         logger,
		interval:       defaultMonitoringLeaseRecoveryInterval,
		timeout:        defaultMonitoringLeaseRecoveryTimeout,
		limit:          defaultMonitoringLeaseRecoveryLimit,
	}
}

func (w *MonitoringLeaseRecoveryWorker) Start(ctx context.Context) {
	go w.Run(ctx)
}

func (w *MonitoringLeaseRecoveryWorker) Run(ctx context.Context) {
	if w == nil || w.monitoringPool == nil {
		return
	}
	w.run(ctx)
}

func (w *MonitoringLeaseRecoveryWorker) run(ctx context.Context) {
	w.runOnce(context.Background())

	ticker := time.NewTicker(w.interval)
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			w.runOnce(context.Background())
		}
	}
}

func (w *MonitoringLeaseRecoveryWorker) runOnce(parent context.Context) {
	ctx, cancel := context.WithTimeout(parent, w.timeout)
	defer cancel()

	tx, err := w.monitoringPool.Begin(ctx)
	if err != nil {
		if w.logger != nil {
			w.logger.Warn("monitoring lease recovery begin failed", zap.Error(err))
		}
		return
	}
	defer tx.Rollback(ctx)

	expiredCount, err := tenantapp.ExpireMonitoringLeasedTasks(ctx, tx, nil, time.Now().UTC(), w.limit)
	if err != nil {
		if w.logger != nil {
			w.logger.Warn("monitoring lease recovery failed", tenantapp.MonitoringWorkerErrorFields(err)...)
		}
		return
	}

	if err := tx.Commit(ctx); err != nil {
		if w.logger != nil {
			w.logger.Warn("monitoring lease recovery commit failed", zap.Error(err))
		}
		return
	}

	var monitorResult tenantapp.MonitorLeaseRecoveryResult
	if w.desktopTasks != nil {
		var recoverErr error
		monitorResult, recoverErr = w.desktopTasks.RecoverExpiredMonitorTasks(ctx, w.limit)
		if recoverErr != nil {
			if w.logger != nil {
				w.logger.Warn("monitor desktop lease recovery failed", zap.Error(recoverErr))
			}
			return
		}
	}

	if (expiredCount > 0 || monitorResult.Failed > 0) && w.logger != nil {
		w.logger.Info("monitoring lease recovery completed",
			zap.Int64("expired_task_count", expiredCount),
			zap.Int("failed_desktop_task_count", monitorResult.Failed),
		)
	}
}

func (w *MonitoringLeaseRecoveryWorker) RunOnce(parent context.Context, run JobRunContext) (map[string]any, error) {
	if w == nil || w.monitoringPool == nil {
		return map[string]any{"skipped": true, "reason": "worker_not_ready"}, nil
	}
	startedAt := time.Now()
	timeout := w.timeout
	if run.Job != nil && run.Job.TimeoutSeconds > 0 {
		timeout = time.Duration(run.Job.TimeoutSeconds) * time.Second
	}
	limit := w.limit
	if run.Job != nil && run.Job.BatchSize != nil && *run.Job.BatchSize > 0 {
		limit = *run.Job.BatchSize
	}
	if value := AsInt(run.Config, "batch_size", 0); value > 0 && (run.Job == nil || run.Job.BatchSize == nil) {
		limit = value
	}
	ctx, cancel := context.WithTimeout(parent, timeout)
	defer cancel()

	tx, err := w.monitoringPool.Begin(ctx)
	if err != nil {
		return map[string]any{"stage": "begin"}, err
	}
	defer tx.Rollback(ctx)

	expiredCount, err := tenantapp.ExpireMonitoringLeasedTasks(ctx, tx, nil, time.Now().UTC(), limit)
	if err != nil {
		return map[string]any{"stage": "expire"}, err
	}
	if err := tx.Commit(ctx); err != nil {
		return map[string]any{"stage": "commit"}, err
	}
	var monitorResult tenantapp.MonitorLeaseRecoveryResult
	if w.desktopTasks != nil {
		var recoverErr error
		monitorResult, recoverErr = w.desktopTasks.RecoverExpiredMonitorTasks(ctx, limit)
		if recoverErr != nil {
			return map[string]any{"stage": "recover_desktop_tasks"}, recoverErr
		}
	}
	return map[string]any{
		"expired_task_count":           expiredCount,
		"failed_desktop_monitor_count": monitorResult.Failed,
		"duration_ms":                  time.Since(startedAt).Milliseconds(),
		"batch_size":                   limit,
	}, nil
}