feat(scheduler): expose metrics HTTP and add graceful shutdown
Scheduler now runs a token-protected HTTP server on scheduler.http_port (default 8081, -1 disables) that exposes Prometheus /metrics and the daily-task JSON snapshot endpoint. Workers gain a blocking Run(ctx) and are supervised by a WaitGroup: on SIGINT/SIGTERM the metrics server shuts down first, worker ticks stop scheduling but ongoing runOnce calls keep their own context and finish, and the process waits up to 60s before exiting. Adds scheduler.http_host/http_port/internal_metrics_token to config with SCHEDULER_* env overrides and exposes port 8081 in the Docker image. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func TestSchedulerHTTPAddrDefaultsToLoopback(t *testing.T) {
|
||||
if got := schedulerHTTPAddr("", 8081); got != "127.0.0.1:8081" {
|
||||
t.Fatalf("schedulerHTTPAddr() = %q, want 127.0.0.1:8081", got)
|
||||
}
|
||||
if got := schedulerHTTPAddr("0.0.0.0", 8081); got != "0.0.0.0:8081" {
|
||||
t.Fatalf("schedulerHTTPAddr() = %q, want 0.0.0.0:8081", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchedulerMetricsAuthMiddleware(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
headerName string
|
||||
header string
|
||||
wantStatus int
|
||||
}{
|
||||
{name: "missing token", wantStatus: http.StatusUnauthorized},
|
||||
{name: "bad bearer token", headerName: "Authorization", header: "Bearer bad", wantStatus: http.StatusUnauthorized},
|
||||
{name: "valid bearer token", headerName: "Authorization", header: "Bearer secret-token", wantStatus: http.StatusNoContent},
|
||||
{name: "valid internal token header", headerName: "X-Internal-Metrics-Token", header: "secret-token", wantStatus: http.StatusNoContent},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
router := gin.New()
|
||||
router.GET("/metrics", schedulerMetricsAuthMiddleware("secret-token"), func(c *gin.Context) {
|
||||
c.Status(http.StatusNoContent)
|
||||
})
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
if tt.headerName != "" {
|
||||
req.Header.Set(tt.headerName, tt.header)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != tt.wantStatus {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, tt.wantStatus)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartSchedulerWorkerWaitsForRunToReturn(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
started := make(chan struct{})
|
||||
startSchedulerWorker(ctx, &wg, "test_worker", testSchedulerLogger{}, func(ctx context.Context) {
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
})
|
||||
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("worker did not start")
|
||||
}
|
||||
|
||||
cancel()
|
||||
if ok := waitSchedulerWorkers(&wg, time.Second, testSchedulerLogger{}); !ok {
|
||||
t.Fatal("expected worker shutdown wait to complete")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitSchedulerWorkersTimesOut(t *testing.T) {
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
defer wg.Done()
|
||||
|
||||
if ok := waitSchedulerWorkers(&wg, time.Nanosecond, testSchedulerLogger{}); ok {
|
||||
t.Fatal("expected worker shutdown wait to time out")
|
||||
}
|
||||
}
|
||||
|
||||
type testSchedulerLogger struct{}
|
||||
|
||||
func (testSchedulerLogger) Infof(string, ...interface{}) {}
|
||||
func (testSchedulerLogger) Errorf(string, ...interface{}) {}
|
||||
Reference in New Issue
Block a user