feat(scheduler): expose metrics HTTP and add graceful shutdown

Scheduler now runs a token-protected HTTP server on scheduler.http_port
(default 8081, -1 disables) that exposes Prometheus /metrics and the
daily-task JSON snapshot endpoint. Workers gain a blocking Run(ctx) and
are supervised by a WaitGroup: on SIGINT/SIGTERM the metrics server
shuts down first, worker ticks stop scheduling but ongoing runOnce
calls keep their own context and finish, and the process waits up to
60s before exiting. Adds scheduler.http_host/http_port/internal_metrics_token
to config with SCHEDULER_* env overrides and exposes port 8081 in the
Docker image.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-04-24 22:20:43 +08:00
parent ff86933c24
commit c0253c98f9
10 changed files with 328 additions and 19 deletions
@@ -20,10 +20,14 @@ func NewKnowledgeDeletedCleanupWorker(service *tenantapp.KnowledgeService) *Know
}
func (w *KnowledgeDeletedCleanupWorker) Start(ctx context.Context) {
go w.Run(ctx)
}
func (w *KnowledgeDeletedCleanupWorker) Run(ctx context.Context) {
if w == nil || w.service == nil {
return
}
go w.run(ctx)
w.run(ctx)
}
func (w *KnowledgeDeletedCleanupWorker) run(ctx context.Context) {
@@ -32,14 +32,18 @@ func NewMonitoringLeaseRecoveryWorker(monitoringPool *pgxpool.Pool, logger *zap.
}
func (w *MonitoringLeaseRecoveryWorker) Start(ctx context.Context) {
go w.Run(ctx)
}
func (w *MonitoringLeaseRecoveryWorker) Run(ctx context.Context) {
if w == nil || w.monitoringPool == nil {
return
}
go w.run(ctx)
w.run(ctx)
}
func (w *MonitoringLeaseRecoveryWorker) run(ctx context.Context) {
w.runOnce(ctx)
w.runOnce(context.Background())
ticker := time.NewTicker(w.interval)
defer ticker.Stop()
@@ -49,7 +53,7 @@ func (w *MonitoringLeaseRecoveryWorker) run(ctx context.Context) {
case <-ctx.Done():
return
case <-ticker.C:
w.runOnce(ctx)
w.runOnce(context.Background())
}
}
}
@@ -38,14 +38,18 @@ func NewMonitoringReceivedInspectionWorker(monitoringPool *pgxpool.Pool, logger
}
func (w *MonitoringReceivedInspectionWorker) Start(ctx context.Context) {
go w.Run(ctx)
}
func (w *MonitoringReceivedInspectionWorker) Run(ctx context.Context) {
if w == nil || w.monitoringPool == nil {
return
}
go w.run(ctx)
w.run(ctx)
}
func (w *MonitoringReceivedInspectionWorker) run(ctx context.Context) {
w.runOnce(ctx)
w.runOnce(context.Background())
ticker := time.NewTicker(w.interval)
defer ticker.Stop()
@@ -55,7 +59,7 @@ func (w *MonitoringReceivedInspectionWorker) run(ctx context.Context) {
case <-ctx.Done():
return
case <-ticker.C:
w.runOnce(ctx)
w.runOnce(context.Background())
}
}
}
@@ -48,14 +48,18 @@ func NewMonitoringResultRecoveryWorker(
}
func (w *MonitoringResultRecoveryWorker) Start(ctx context.Context) {
go w.Run(ctx)
}
func (w *MonitoringResultRecoveryWorker) Run(ctx context.Context) {
if w == nil || w.monitoringPool == nil || w.service == nil {
return
}
go w.run(ctx)
w.run(ctx)
}
func (w *MonitoringResultRecoveryWorker) run(ctx context.Context) {
w.runOnce(ctx)
w.runOnce(context.Background())
ticker := time.NewTicker(w.interval)
defer ticker.Stop()
@@ -65,7 +69,7 @@ func (w *MonitoringResultRecoveryWorker) run(ctx context.Context) {
case <-ctx.Done():
return
case <-ticker.C:
w.runOnce(ctx)
w.runOnce(context.Background())
}
}
}
@@ -80,14 +80,18 @@ func NewScheduleDispatchWorker(
}
func (w *ScheduleDispatchWorker) Start(ctx context.Context) {
go w.Run(ctx)
}
func (w *ScheduleDispatchWorker) Run(ctx context.Context) {
if w == nil || w.pool == nil || w.promptRuleService == nil {
return
}
go w.run(ctx)
w.run(ctx)
}
func (w *ScheduleDispatchWorker) run(ctx context.Context) {
w.runOnce(ctx)
w.runOnce(context.Background())
ticker := time.NewTicker(w.interval)
defer ticker.Stop()
@@ -97,7 +101,7 @@ func (w *ScheduleDispatchWorker) run(ctx context.Context) {
case <-ctx.Done():
return
case <-ticker.C:
w.runOnce(ctx)
w.runOnce(context.Background())
}
}
}