feat(infra): add scheduler and worker-generate standalone processes

Extract monitoring recovery/inspection workers and schedule dispatch into a dedicated scheduler process. Add worker-generate process for article generation and template-assist queue consumption. Introduce shared/schedule runtime for cron-based worker lifecycle management. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-15 14:19:21 +08:00
parent b46cfaaa61
commit 3f5b5c1e01
11 changed files with 1588 additions and 1 deletions
@@ -0,0 +1,205 @@
+package scheduler
+
+import (
+	"context"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+	"go.uber.org/zap"
+
+	tenantapp "github.com/geo-platform/tenant-api/internal/tenant/app"
+)
+
+const (
+	defaultMonitoringResultRecoveryInterval = 1 * time.Minute
+	defaultMonitoringResultRecoveryTimeout  = 30 * time.Second
+	defaultMonitoringResultRecoveryLimit    = 100
+)
+
+type MonitoringResultRecoveryWorker struct {
+	monitoringPool *pgxpool.Pool
+	service        *tenantapp.MonitoringCallbackService
+	logger         *zap.Logger
+	interval       time.Duration
+	timeout        time.Duration
+	limit          int
+}
+
+type monitoringRecoverableTaskResult struct {
+	ID                 int64
+	CallbackReceivedAt time.Time
+	RequestPayloadJSON []byte
+}
+
+func NewMonitoringResultRecoveryWorker(
+	monitoringPool *pgxpool.Pool,
+	service *tenantapp.MonitoringCallbackService,
+	logger *zap.Logger,
+) *MonitoringResultRecoveryWorker {
+	return &MonitoringResultRecoveryWorker{
+		monitoringPool: monitoringPool,
+		service:        service,
+		logger:         logger,
+		interval:       defaultMonitoringResultRecoveryInterval,
+		timeout:        defaultMonitoringResultRecoveryTimeout,
+		limit:          defaultMonitoringResultRecoveryLimit,
+	}
+}
+
+func (w *MonitoringResultRecoveryWorker) Start(ctx context.Context) {
+	if w == nil || w.monitoringPool == nil || w.service == nil {
+		return
+	}
+	go w.run(ctx)
+}
+
+func (w *MonitoringResultRecoveryWorker) run(ctx context.Context) {
+	w.runOnce(ctx)
+
+	ticker := time.NewTicker(w.interval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			w.runOnce(ctx)
+		}
+	}
+}
+
+func (w *MonitoringResultRecoveryWorker) runOnce(parent context.Context) {
+	ctx, cancel := context.WithTimeout(parent, w.timeout)
+	defer cancel()
+
+	tx, err := w.monitoringPool.Begin(ctx)
+	if err != nil {
+		if w.logger != nil {
+			w.logger.Warn("monitoring result recovery begin failed", zap.Error(err))
+		}
+		return
+	}
+	defer tx.Rollback(ctx)
+
+	items, err := loadMonitoringRecoverableTaskResults(ctx, tx, w.limit)
+	if err != nil {
+		if w.logger != nil {
+			w.logger.Warn("monitoring result recovery failed", tenantapp.MonitoringWorkerErrorFields(err)...)
+		}
+		return
+	}
+
+	if err := tx.Commit(ctx); err != nil {
+		if w.logger != nil {
+			w.logger.Warn("monitoring result recovery commit failed", zap.Error(err))
+		}
+		return
+	}
+
+	if len(items) == 0 {
+		return
+	}
+
+	republishedCount := 0
+	for _, item := range items {
+		if w.tryRepublish(parent, item) {
+			republishedCount++
+		}
+	}
+
+	if republishedCount > 0 && w.logger != nil {
+		w.logger.Info("monitoring result recovery republished tasks", zap.Int("task_count", republishedCount))
+	}
+}
+
+func (w *MonitoringResultRecoveryWorker) tryRepublish(parent context.Context, item monitoringRecoverableTaskResult) bool {
+	if err := tenantapp.ValidateMonitoringTaskResultEnvelope(item.RequestPayloadJSON); err != nil {
+		w.service.PatchTaskResultQueueMeta(item.ID, item.CallbackReceivedAt, tenantapp.MonitoringTaskResultQueueMetaPatch{
+			QueueStatus:           "invalid_payload",
+			QueueClaimedAt:        nil,
+			QueueEnqueuedAt:       nil,
+			LastQueuePublishError: stringPtr(err.Error()),
+		})
+		if w.logger != nil {
+			w.logger.Warn("monitoring result recovery skipped invalid payload", zap.Int64("task_id", item.ID), zap.Error(err))
+		}
+		return false
+	}
+
+	ctx, cancel := context.WithTimeout(parent, 5*time.Second)
+	defer cancel()
+
+	if err := w.service.PublishTaskResultEnvelope(ctx, item.RequestPayloadJSON); err != nil {
+		w.service.PatchTaskResultQueueMeta(item.ID, item.CallbackReceivedAt, tenantapp.MonitoringTaskResultQueueMetaPatch{
+			QueueStatus:           "publish_failed",
+			QueueClaimedAt:        nil,
+			QueueEnqueuedAt:       nil,
+			LastQueuePublishError: stringPtr(err.Error()),
+		})
+		if w.logger != nil {
+			w.logger.Warn("monitoring result recovery republish failed", zap.Int64("task_id", item.ID), zap.Error(err))
+		}
+		return false
+	}
+
+	enqueuedAt := time.Now().UTC().Round(0)
+	w.service.PatchTaskResultQueueMeta(item.ID, item.CallbackReceivedAt, tenantapp.MonitoringTaskResultQueueMetaPatch{
+		QueueStatus:           "queued",
+		QueueClaimedAt:        nil,
+		QueueEnqueuedAt:       &enqueuedAt,
+		LastQueuePublishError: nil,
+	})
+
+	return true
+}
+
+func loadMonitoringRecoverableTaskResults(ctx context.Context, tx pgx.Tx, limit int) ([]monitoringRecoverableTaskResult, error) {
+	if limit <= 0 {
+		limit = defaultMonitoringResultRecoveryLimit
+	}
+
+	rows, err := tx.Query(ctx, `
+		SELECT
+			id,
+			callback_received_at,
+			request_payload_json
+		FROM monitoring_collect_tasks
+		WHERE collector_type = $1
+		  AND status = 'received'
+		  AND callback_received_at IS NOT NULL
+		  AND request_payload_json IS NOT NULL
+		  AND jsonb_typeof(request_payload_json) = 'object'
+		  AND COALESCE(NULLIF(request_payload_json ->> 'queue_status', ''), 'pending') IN ('pending', 'publish_failed')
+		ORDER BY callback_received_at ASC, id ASC
+		FOR UPDATE SKIP LOCKED
+		LIMIT $2
+	`, "plugin", limit)
+	if err != nil {
+		return nil, tenantapp.InternalMonitoringError(50041, "result_recovery_query_failed", "failed to load recoverable monitoring results", err)
+	}
+	defer rows.Close()
+
+	result := make([]monitoringRecoverableTaskResult, 0, limit)
+	for rows.Next() {
+		var item monitoringRecoverableTaskResult
+		if scanErr := rows.Scan(&item.ID, &item.CallbackReceivedAt, &item.RequestPayloadJSON); scanErr != nil {
+			return nil, tenantapp.InternalMonitoringError(50041, "result_recovery_scan_failed", "failed to parse recoverable monitoring results", scanErr)
+		}
+		result = append(result, item)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, tenantapp.InternalMonitoringError(50041, "result_recovery_scan_failed", "failed to iterate recoverable monitoring results", err)
+	}
+
+	return result, nil
+}
+
+func stringPtr(value string) *string {
+	if value == "" {
+		return nil
+	}
+	result := value
+	return &result
+}