geo/server/internal/scheduler/monitoring_result_recovery_worker.go

package scheduler

import (
	"context"
	"time"

	"github.com/jackc/pgx/v5"
	"github.com/jackc/pgx/v5/pgxpool"
	"go.uber.org/zap"

	tenantapp "github.com/geo-platform/tenant-api/internal/tenant/app"
)

const (
	defaultMonitoringResultRecoveryInterval = 1 * time.Minute
	defaultMonitoringResultRecoveryTimeout  = 30 * time.Second
	defaultMonitoringResultRecoveryLimit    = 100
)

type MonitoringResultRecoveryWorker struct {
	monitoringPool *pgxpool.Pool
	service        *tenantapp.MonitoringCallbackService
	logger         *zap.Logger
	interval       time.Duration
	timeout        time.Duration
	limit          int
}

type monitoringRecoverableTaskResult struct {
	ID                 int64
	CallbackReceivedAt time.Time
	RequestPayloadJSON []byte
}

func NewMonitoringResultRecoveryWorker(
	monitoringPool *pgxpool.Pool,
	service *tenantapp.MonitoringCallbackService,
	logger *zap.Logger,
) *MonitoringResultRecoveryWorker {
	return &MonitoringResultRecoveryWorker{
		monitoringPool: monitoringPool,
		service:        service,
		logger:         logger,
		interval:       defaultMonitoringResultRecoveryInterval,
		timeout:        defaultMonitoringResultRecoveryTimeout,
		limit:          defaultMonitoringResultRecoveryLimit,
	}
}

func (w *MonitoringResultRecoveryWorker) Start(ctx context.Context) {
	go w.Run(ctx)
}

func (w *MonitoringResultRecoveryWorker) Run(ctx context.Context) {
	if w == nil || w.monitoringPool == nil || w.service == nil {
		return
	}
	w.run(ctx)
}

func (w *MonitoringResultRecoveryWorker) run(ctx context.Context) {
	w.runOnce(context.Background())

	ticker := time.NewTicker(w.interval)
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			w.runOnce(context.Background())
		}
	}
}

func (w *MonitoringResultRecoveryWorker) runOnce(parent context.Context) {
	ctx, cancel := context.WithTimeout(parent, w.timeout)
	defer cancel()

	tx, err := w.monitoringPool.Begin(ctx)
	if err != nil {
		if w.logger != nil {
			w.logger.Warn("monitoring result recovery begin failed", zap.Error(err))
		}
		return
	}
	defer tx.Rollback(ctx)

	items, err := loadMonitoringRecoverableTaskResults(ctx, tx, w.limit)
	if err != nil {
		if w.logger != nil {
			w.logger.Warn("monitoring result recovery failed", tenantapp.MonitoringWorkerErrorFields(err)...)
		}
		return
	}

	if err := tx.Commit(ctx); err != nil {
		if w.logger != nil {
			w.logger.Warn("monitoring result recovery commit failed", zap.Error(err))
		}
		return
	}

	if len(items) == 0 {
		return
	}

	republishedCount := 0
	for _, item := range items {
		if w.tryRepublish(parent, item) {
			republishedCount++
		}
	}

	if republishedCount > 0 && w.logger != nil {
		w.logger.Info("monitoring result recovery republished tasks", zap.Int("task_count", republishedCount))
	}
}

func (w *MonitoringResultRecoveryWorker) tryRepublish(parent context.Context, item monitoringRecoverableTaskResult) bool {
	if err := tenantapp.ValidateMonitoringTaskResultEnvelope(item.RequestPayloadJSON); err != nil {
		w.service.PatchTaskResultQueueMeta(item.ID, item.CallbackReceivedAt, tenantapp.MonitoringTaskResultQueueMetaPatch{
			QueueStatus:           "invalid_payload",
			QueueClaimedAt:        nil,
			QueueEnqueuedAt:       nil,
			LastQueuePublishError: stringPtr(err.Error()),
		})
		if w.logger != nil {
			w.logger.Warn("monitoring result recovery skipped invalid payload", zap.Int64("task_id", item.ID), zap.Error(err))
		}
		return false
	}

	ctx, cancel := context.WithTimeout(parent, 5*time.Second)
	defer cancel()

	if err := w.service.PublishTaskResultEnvelope(ctx, item.RequestPayloadJSON); err != nil {
		w.service.PatchTaskResultQueueMeta(item.ID, item.CallbackReceivedAt, tenantapp.MonitoringTaskResultQueueMetaPatch{
			QueueStatus:           "publish_failed",
			QueueClaimedAt:        nil,
			QueueEnqueuedAt:       nil,
			LastQueuePublishError: stringPtr(err.Error()),
		})
		if w.logger != nil {
			w.logger.Warn("monitoring result recovery republish failed", zap.Int64("task_id", item.ID), zap.Error(err))
		}
		return false
	}

	enqueuedAt := time.Now().UTC().Round(0)
	w.service.PatchTaskResultQueueMeta(item.ID, item.CallbackReceivedAt, tenantapp.MonitoringTaskResultQueueMetaPatch{
		QueueStatus:           "queued",
		QueueClaimedAt:        nil,
		QueueEnqueuedAt:       &enqueuedAt,
		LastQueuePublishError: nil,
	})

	return true
}

func loadMonitoringRecoverableTaskResults(ctx context.Context, tx pgx.Tx, limit int) ([]monitoringRecoverableTaskResult, error) {
	if limit <= 0 {
		limit = defaultMonitoringResultRecoveryLimit
	}

	rows, err := tx.Query(ctx, `
		SELECT
			id,
			callback_received_at,
			request_payload_json
		FROM monitoring_collect_tasks
		WHERE collector_type = $1
		  AND (
			status = 'received'
			OR (
				COALESCE(execution_owner, 'legacy') = 'desktop_tasks'
				AND status = 'pending'
				AND callback_received_at IS NOT NULL
			)
		  )
		  AND callback_received_at IS NOT NULL
		  AND request_payload_json IS NOT NULL
		  AND jsonb_typeof(request_payload_json) = 'object'
		  AND COALESCE(NULLIF(request_payload_json ->> 'queue_status', ''), 'pending') IN ('pending', 'publish_failed')
		ORDER BY callback_received_at ASC, id ASC
		FOR UPDATE SKIP LOCKED
		LIMIT $2
	`, "plugin", limit)
	if err != nil {
		return nil, tenantapp.InternalMonitoringError(50041, "result_recovery_query_failed", "failed to load recoverable monitoring results", err)
	}
	defer rows.Close()

	result := make([]monitoringRecoverableTaskResult, 0, limit)
	for rows.Next() {
		var item monitoringRecoverableTaskResult
		if scanErr := rows.Scan(&item.ID, &item.CallbackReceivedAt, &item.RequestPayloadJSON); scanErr != nil {
			return nil, tenantapp.InternalMonitoringError(50041, "result_recovery_scan_failed", "failed to parse recoverable monitoring results", scanErr)
		}
		result = append(result, item)
	}
	if err := rows.Err(); err != nil {
		return nil, tenantapp.InternalMonitoringError(50041, "result_recovery_scan_failed", "failed to iterate recoverable monitoring results", err)
	}

	return result, nil
}

func stringPtr(value string) *string {
	if value == "" {
		return nil
	}
	result := value
	return &result
}