package scheduler import ( "context" "time" "github.com/jackc/pgx/v5/pgxpool" "go.uber.org/zap" tenantapp "github.com/geo-platform/tenant-api/internal/tenant/app" ) const ( defaultMonitoringLeaseRecoveryInterval = 30 * time.Minute defaultMonitoringLeaseRecoveryTimeout = 30 * time.Second defaultMonitoringLeaseRecoveryLimit = 1000 ) type MonitoringLeaseRecoveryWorker struct { monitoringPool *pgxpool.Pool logger *zap.Logger interval time.Duration timeout time.Duration limit int } func NewMonitoringLeaseRecoveryWorker(monitoringPool *pgxpool.Pool, logger *zap.Logger) *MonitoringLeaseRecoveryWorker { return &MonitoringLeaseRecoveryWorker{ monitoringPool: monitoringPool, logger: logger, interval: defaultMonitoringLeaseRecoveryInterval, timeout: defaultMonitoringLeaseRecoveryTimeout, limit: defaultMonitoringLeaseRecoveryLimit, } } func (w *MonitoringLeaseRecoveryWorker) Start(ctx context.Context) { go w.Run(ctx) } func (w *MonitoringLeaseRecoveryWorker) Run(ctx context.Context) { if w == nil || w.monitoringPool == nil { return } w.run(ctx) } func (w *MonitoringLeaseRecoveryWorker) run(ctx context.Context) { w.runOnce(context.Background()) ticker := time.NewTicker(w.interval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: w.runOnce(context.Background()) } } } func (w *MonitoringLeaseRecoveryWorker) runOnce(parent context.Context) { ctx, cancel := context.WithTimeout(parent, w.timeout) defer cancel() tx, err := w.monitoringPool.Begin(ctx) if err != nil { if w.logger != nil { w.logger.Warn("monitoring lease recovery begin failed", zap.Error(err)) } return } defer tx.Rollback(ctx) expiredCount, err := tenantapp.ExpireMonitoringLeasedTasks(ctx, tx, nil, time.Now().UTC(), w.limit) if err != nil { if w.logger != nil { w.logger.Warn("monitoring lease recovery failed", tenantapp.MonitoringWorkerErrorFields(err)...) } return } if err := tx.Commit(ctx); err != nil { if w.logger != nil { w.logger.Warn("monitoring lease recovery commit failed", zap.Error(err)) } return } if expiredCount > 0 && w.logger != nil { w.logger.Info("monitoring lease recovery completed", zap.Int64("expired_task_count", expiredCount)) } } func (w *MonitoringLeaseRecoveryWorker) RunOnce(parent context.Context, run JobRunContext) (map[string]any, error) { if w == nil || w.monitoringPool == nil { return map[string]any{"skipped": true, "reason": "worker_not_ready"}, nil } startedAt := time.Now() timeout := w.timeout if run.Job != nil && run.Job.TimeoutSeconds > 0 { timeout = time.Duration(run.Job.TimeoutSeconds) * time.Second } limit := w.limit if run.Job != nil && run.Job.BatchSize != nil && *run.Job.BatchSize > 0 { limit = *run.Job.BatchSize } if value := AsInt(run.Config, "batch_size", 0); value > 0 && (run.Job == nil || run.Job.BatchSize == nil) { limit = value } ctx, cancel := context.WithTimeout(parent, timeout) defer cancel() tx, err := w.monitoringPool.Begin(ctx) if err != nil { return map[string]any{"stage": "begin"}, err } defer tx.Rollback(ctx) expiredCount, err := tenantapp.ExpireMonitoringLeasedTasks(ctx, tx, nil, time.Now().UTC(), limit) if err != nil { return map[string]any{"stage": "expire"}, err } if err := tx.Commit(ctx); err != nil { return map[string]any{"stage": "commit"}, err } return map[string]any{ "expired_task_count": expiredCount, "duration_ms": time.Since(startedAt).Milliseconds(), "batch_size": limit, }, nil }