package scheduler import ( "context" "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "go.uber.org/zap" tenantapp "github.com/geo-platform/tenant-api/internal/tenant/app" ) const ( defaultMonitoringResultRecoveryInterval = 1 * time.Minute defaultMonitoringResultRecoveryTimeout = 30 * time.Second defaultMonitoringResultRecoveryLimit = 100 ) type MonitoringResultRecoveryWorker struct { monitoringPool *pgxpool.Pool service *tenantapp.MonitoringCallbackService logger *zap.Logger interval time.Duration timeout time.Duration limit int } type monitoringRecoverableTaskResult struct { ID int64 CallbackReceivedAt time.Time RequestPayloadJSON []byte } func NewMonitoringResultRecoveryWorker( monitoringPool *pgxpool.Pool, service *tenantapp.MonitoringCallbackService, logger *zap.Logger, ) *MonitoringResultRecoveryWorker { return &MonitoringResultRecoveryWorker{ monitoringPool: monitoringPool, service: service, logger: logger, interval: defaultMonitoringResultRecoveryInterval, timeout: defaultMonitoringResultRecoveryTimeout, limit: defaultMonitoringResultRecoveryLimit, } } func (w *MonitoringResultRecoveryWorker) Start(ctx context.Context) { go w.Run(ctx) } func (w *MonitoringResultRecoveryWorker) Run(ctx context.Context) { if w == nil || w.monitoringPool == nil || w.service == nil { return } w.run(ctx) } func (w *MonitoringResultRecoveryWorker) run(ctx context.Context) { w.runOnce(context.Background()) ticker := time.NewTicker(w.interval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: w.runOnce(context.Background()) } } } func (w *MonitoringResultRecoveryWorker) runOnce(parent context.Context) { ctx, cancel := context.WithTimeout(parent, w.timeout) defer cancel() tx, err := w.monitoringPool.Begin(ctx) if err != nil { if w.logger != nil { w.logger.Warn("monitoring result recovery begin failed", zap.Error(err)) } return } defer tx.Rollback(ctx) items, err := loadMonitoringRecoverableTaskResults(ctx, tx, w.limit) if err != nil { if w.logger != nil { w.logger.Warn("monitoring result recovery failed", tenantapp.MonitoringWorkerErrorFields(err)...) } return } if err := tx.Commit(ctx); err != nil { if w.logger != nil { w.logger.Warn("monitoring result recovery commit failed", zap.Error(err)) } return } if len(items) == 0 { return } republishedCount := 0 for _, item := range items { if w.tryRepublish(parent, item) { republishedCount++ } } if republishedCount > 0 && w.logger != nil { w.logger.Info("monitoring result recovery republished tasks", zap.Int("task_count", republishedCount)) } } func (w *MonitoringResultRecoveryWorker) tryRepublish(parent context.Context, item monitoringRecoverableTaskResult) bool { if err := tenantapp.ValidateMonitoringTaskResultEnvelope(item.RequestPayloadJSON); err != nil { w.service.PatchTaskResultQueueMeta(item.ID, item.CallbackReceivedAt, tenantapp.MonitoringTaskResultQueueMetaPatch{ QueueStatus: "invalid_payload", QueueClaimedAt: nil, QueueEnqueuedAt: nil, LastQueuePublishError: stringPtr(err.Error()), }) if w.logger != nil { w.logger.Warn("monitoring result recovery skipped invalid payload", zap.Int64("task_id", item.ID), zap.Error(err)) } return false } ctx, cancel := context.WithTimeout(parent, 5*time.Second) defer cancel() if err := w.service.PublishTaskResultEnvelope(ctx, item.RequestPayloadJSON); err != nil { w.service.PatchTaskResultQueueMeta(item.ID, item.CallbackReceivedAt, tenantapp.MonitoringTaskResultQueueMetaPatch{ QueueStatus: "publish_failed", QueueClaimedAt: nil, QueueEnqueuedAt: nil, LastQueuePublishError: stringPtr(err.Error()), }) if w.logger != nil { w.logger.Warn("monitoring result recovery republish failed", zap.Int64("task_id", item.ID), zap.Error(err)) } return false } enqueuedAt := time.Now().UTC().Round(0) w.service.PatchTaskResultQueueMeta(item.ID, item.CallbackReceivedAt, tenantapp.MonitoringTaskResultQueueMetaPatch{ QueueStatus: "queued", QueueClaimedAt: nil, QueueEnqueuedAt: &enqueuedAt, LastQueuePublishError: nil, }) return true } func loadMonitoringRecoverableTaskResults(ctx context.Context, tx pgx.Tx, limit int) ([]monitoringRecoverableTaskResult, error) { if limit <= 0 { limit = defaultMonitoringResultRecoveryLimit } rows, err := tx.Query(ctx, ` SELECT id, callback_received_at, request_payload_json FROM monitoring_collect_tasks WHERE collector_type = $1 AND ( status = 'received' OR ( COALESCE(execution_owner, 'legacy') = 'desktop_tasks' AND status = 'pending' AND callback_received_at IS NOT NULL ) ) AND callback_received_at IS NOT NULL AND request_payload_json IS NOT NULL AND jsonb_typeof(request_payload_json) = 'object' AND COALESCE(NULLIF(request_payload_json ->> 'queue_status', ''), 'pending') IN ('pending', 'publish_failed') ORDER BY callback_received_at ASC, id ASC FOR UPDATE SKIP LOCKED LIMIT $2 `, "plugin", limit) if err != nil { return nil, tenantapp.InternalMonitoringError(50041, "result_recovery_query_failed", "failed to load recoverable monitoring results", err) } defer rows.Close() result := make([]monitoringRecoverableTaskResult, 0, limit) for rows.Next() { var item monitoringRecoverableTaskResult if scanErr := rows.Scan(&item.ID, &item.CallbackReceivedAt, &item.RequestPayloadJSON); scanErr != nil { return nil, tenantapp.InternalMonitoringError(50041, "result_recovery_scan_failed", "failed to parse recoverable monitoring results", scanErr) } result = append(result, item) } if err := rows.Err(); err != nil { return nil, tenantapp.InternalMonitoringError(50041, "result_recovery_scan_failed", "failed to iterate recoverable monitoring results", err) } return result, nil } func stringPtr(value string) *string { if value == "" { return nil } result := value return &result }