root
c0253c98f9
Scheduler now runs a token-protected HTTP server on scheduler.http_port (default 8081, -1 disables) that exposes Prometheus /metrics and the daily-task JSON snapshot endpoint. Workers gain a blocking Run(ctx) and are supervised by a WaitGroup: on SIGINT/SIGTERM the metrics server shuts down first, worker ticks stop scheduling but ongoing runOnce calls keep their own context and finish, and the process waits up to 60s before exiting. Adds scheduler.http_host/http_port/internal_metrics_token to config with SCHEDULER_* env overrides and exposes port 8081 in the Docker image. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
119 lines
3.1 KiB
Go
119 lines
3.1 KiB
Go
package scheduler
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
"go.uber.org/zap"
|
|
|
|
tenantapp "github.com/geo-platform/tenant-api/internal/tenant/app"
|
|
)
|
|
|
|
const (
|
|
defaultMonitoringReceivedInspectionInterval = 5 * time.Minute
|
|
defaultMonitoringReceivedInspectionTimeout = 30 * time.Second
|
|
defaultMonitoringReceivedAlertThreshold = 15 * time.Minute
|
|
defaultMonitoringReceivedInspectLimit = 100
|
|
)
|
|
|
|
type MonitoringReceivedInspectionWorker struct {
|
|
monitoringPool *pgxpool.Pool
|
|
logger *zap.Logger
|
|
interval time.Duration
|
|
timeout time.Duration
|
|
threshold time.Duration
|
|
limit int
|
|
}
|
|
|
|
func NewMonitoringReceivedInspectionWorker(monitoringPool *pgxpool.Pool, logger *zap.Logger) *MonitoringReceivedInspectionWorker {
|
|
return &MonitoringReceivedInspectionWorker{
|
|
monitoringPool: monitoringPool,
|
|
logger: logger,
|
|
interval: defaultMonitoringReceivedInspectionInterval,
|
|
timeout: defaultMonitoringReceivedInspectionTimeout,
|
|
threshold: defaultMonitoringReceivedAlertThreshold,
|
|
limit: defaultMonitoringReceivedInspectLimit,
|
|
}
|
|
}
|
|
|
|
func (w *MonitoringReceivedInspectionWorker) Start(ctx context.Context) {
|
|
go w.Run(ctx)
|
|
}
|
|
|
|
func (w *MonitoringReceivedInspectionWorker) Run(ctx context.Context) {
|
|
if w == nil || w.monitoringPool == nil {
|
|
return
|
|
}
|
|
w.run(ctx)
|
|
}
|
|
|
|
func (w *MonitoringReceivedInspectionWorker) run(ctx context.Context) {
|
|
w.runOnce(context.Background())
|
|
|
|
ticker := time.NewTicker(w.interval)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
w.runOnce(context.Background())
|
|
}
|
|
}
|
|
}
|
|
|
|
func (w *MonitoringReceivedInspectionWorker) runOnce(parent context.Context) {
|
|
ctx, cancel := context.WithTimeout(parent, w.timeout)
|
|
defer cancel()
|
|
|
|
tx, err := w.monitoringPool.Begin(ctx)
|
|
if err != nil {
|
|
if w.logger != nil {
|
|
w.logger.Warn("monitoring received watchdog begin failed", zap.Error(err))
|
|
}
|
|
return
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
|
|
now := time.Now().UTC()
|
|
items, err := tenantapp.MarkMonitoringStaleReceivedTasks(ctx, tx, now, w.threshold, w.limit)
|
|
if err != nil {
|
|
if w.logger != nil {
|
|
w.logger.Warn("monitoring received watchdog failed", tenantapp.MonitoringWorkerErrorFields(err)...)
|
|
}
|
|
return
|
|
}
|
|
|
|
if err := tx.Commit(ctx); err != nil {
|
|
if w.logger != nil {
|
|
w.logger.Warn("monitoring received watchdog commit failed", zap.Error(err))
|
|
}
|
|
return
|
|
}
|
|
|
|
if len(items) == 0 || w.logger == nil {
|
|
return
|
|
}
|
|
|
|
w.logger.Warn("monitoring received watchdog found overdue tasks",
|
|
zap.Int("task_count", len(items)),
|
|
zap.Duration("threshold", w.threshold),
|
|
)
|
|
|
|
for _, item := range items {
|
|
w.logger.Warn("monitoring task stuck in received state",
|
|
zap.Int64("task_id", item.ID),
|
|
zap.Int64("tenant_id", item.TenantID),
|
|
zap.Int64("brand_id", item.BrandID),
|
|
zap.Int64("question_id", item.QuestionID),
|
|
zap.String("collector_type", item.CollectorType),
|
|
zap.String("ai_platform_id", item.AIPlatformID),
|
|
zap.String("business_date", item.BusinessDate.Format("2006-01-02")),
|
|
zap.Time("callback_received_at", item.CallbackReceivedAt.UTC()),
|
|
zap.Duration("received_lag", now.Sub(item.CallbackReceivedAt.UTC())),
|
|
)
|
|
}
|
|
}
|