feat(server): hot-reload config store and reloadable infra clients

- Replace single Load() with a watching Store that re-reads config files
  (and config.local.yaml) and fans out a ReloadEvent with a per-field diff
  so consumers can decide whether the change is hot-applicable or requires
  a process restart.
- Wrap llm, retrieval, vector store, and object storage clients in
  Reloadable* shells so the bootstrap can swap their underlying impls when
  config changes without re-instantiating handlers.
- Make jwt.Manager and ops TokenIssuer mutable under a lock so secrets and
  TTLs can be rotated live; thread default plan code through a setter on
  the ops AdminUserService.
- Wire ConfigStore through bootstrap and every cmd/main.go, scheduler /
  worker / tenant-api / ops-api start the watcher; services and handlers
  take a config.Provider so they always read current values for things
  like generation.stream_enabled, scheduler dispatch, retrieval, etc.
- Switch shared/config decoding off viper to a Kratos-derived runtime
  package so env placeholders (\${VAR:default}) resolve consistently and
  the same source machinery powers both the loader and the watcher.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-01 16:01:23 +08:00
parent ce2d8a2907
commit 618399f86d
61 changed files with 3186 additions and 496 deletions
+5 -3
View File
@@ -26,6 +26,8 @@ func main() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
app.StartConfigWatcher(ctx)
assistworker.NewKolAssistWorker(
app.DB,
app.RabbitMQ,
@@ -33,9 +35,9 @@ func main() {
app.LLM,
app.Logger,
app.Cache,
app.Config.Generation.WorkerConcurrency,
app.Config.Generation.ArticleTimeout,
).Start(ctx)
app.Config().Generation.WorkerConcurrency,
app.Config().Generation.ArticleTimeout,
).WithConfigProvider(app.ConfigStore).Start(ctx)
app.Logger.Info("kol-assist-worker started")
+21 -1
View File
@@ -30,10 +30,12 @@ func main() {
configPath = p
}
cfg, err := opsconfig.Load(configPath)
configStore, err := opsconfig.NewStore(configPath)
if err != nil {
log.Fatalf("ops-api load config: %v", err)
}
defer configStore.Close()
cfg := configStore.Current()
logger, err := observability.NewLogger(cfg.Log.Level, cfg.Log.Format)
if err != nil {
@@ -92,6 +94,23 @@ func main() {
logger.Sugar().Fatalf("ops-api seed default admin: %v", err)
}
watcherCtx, watcherCancel := context.WithCancel(context.Background())
defer watcherCancel()
go func() {
if err := configStore.Watch(watcherCtx, logger, func(event opsconfig.ReloadEvent) {
if event.Current == nil {
return
}
issuer.Update(event.Current.JWT.Secret, event.Current.JWT.AccessTTL)
adminUserSvc.UpdateDefaultPlanCode(event.Current.AdminUsers.DefaultPlanCode)
if opsconfig.RestartRequired(event.Changes) {
logger.Warn("ops config hot reload changed fields that require process restart")
}
}); err != nil {
logger.Warn("ops config hot reload watcher stopped", zap.Error(err))
}
}()
if cfg.Server.Mode == "release" {
gin.SetMode(gin.ReleaseMode)
}
@@ -130,6 +149,7 @@ func main() {
<-quit
logger.Info("ops-api shutting down...")
watcherCancel()
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := srv.Shutdown(shutdownCtx); err != nil {
+37 -22
View File
@@ -36,7 +36,8 @@ func main() {
}
defer app.Close()
generationCfg := app.Config.Generation
generationProvider := tenantapp.NewGenerationStreamDisabledConfigProvider(app.ConfigStore)
generationCfg := app.Config().Generation
generationCfg.StreamEnabled = false
knowledgeSvc := tenantapp.NewKnowledgeService(
@@ -46,11 +47,11 @@ func main() {
app.ObjectStorage,
app.LLM,
app.Logger,
app.Config.Retrieval,
app.Config.LLM,
app.Config().Retrieval,
app.Config().LLM,
0,
0,
)
).WithConfigProvider(app.ConfigStore)
promptRuleSvc := tenantapp.NewPromptRuleGenerationService(
app.DB,
@@ -59,8 +60,8 @@ func main() {
knowledgeSvc,
app.GenerationStreams,
generationCfg,
app.Config.LLM.MaxOutputTokens,
).WithCache(app.Cache).WithLogger(app.Logger)
app.Config().LLM.MaxOutputTokens,
).WithCache(app.Cache).WithLogger(app.Logger).WithConfigProvider(generationProvider)
monitoringCallbackService := tenantapp.NewMonitoringCallbackService(
app.DB,
@@ -69,20 +70,22 @@ func main() {
app.LLM,
app.Logger,
)
monitoringService := tenantapp.NewMonitoringService(
app.DB,
app.MonitoringDB,
app.RabbitMQ,
app.Config.MonitoringDispatch,
app.Config.BrandLibrary,
app.Logger,
).WithRedis(app.Redis)
monitoringService := app.MonitoringService
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
app.StartConfigWatcher(ctx)
var workerWG sync.WaitGroup
scheduleDispatchWorker := internalscheduler.NewScheduleDispatchWorker(app.DB, app.RabbitMQ, promptRuleSvc, app.Cache, app.Logger, app.Config.Scheduler)
scheduleDispatchWorker := internalscheduler.NewScheduleDispatchWorker(
app.DB,
app.RabbitMQ,
promptRuleSvc,
app.Cache,
app.Logger,
app.Config().Scheduler,
).WithConfigProvider(app.ConfigStore)
monitoringDailyTaskWorker := tenantapp.NewMonitoringDailyTaskWorker(monitoringService, monitoringCallbackService, app.Logger)
monitoringResultRecoveryWorker := internalscheduler.NewMonitoringResultRecoveryWorker(app.MonitoringDB, monitoringCallbackService, app.Logger)
monitoringLeaseRecoveryWorker := internalscheduler.NewMonitoringLeaseRecoveryWorker(app.MonitoringDB, app.Logger)
@@ -97,10 +100,10 @@ func main() {
startSchedulerWorker(ctx, &workerWG, "knowledge_deleted_cleanup", app.Logger.Sugar(), knowledgeDeletedCleanupWorker.Run)
var metricsServer *http.Server
if app.Config.Scheduler.HTTPPort > 0 {
if app.Config().Scheduler.HTTPPort > 0 {
metricsServer = startSchedulerMetricsServer(app)
} else {
app.Logger.Sugar().Infof("scheduler metrics http server disabled by http_port=%d", app.Config.Scheduler.HTTPPort)
app.Logger.Sugar().Infof("scheduler metrics http server disabled by http_port=%d", app.Config().Scheduler.HTTPPort)
}
app.Logger.Info("scheduler started")
@@ -163,13 +166,13 @@ func waitSchedulerWorkers(wg *sync.WaitGroup, timeout time.Duration, logger inte
}
func startSchedulerMetricsServer(app *bootstrap.App) *http.Server {
token := strings.TrimSpace(app.Config.Scheduler.InternalMetricsToken)
token := strings.TrimSpace(app.Config().Scheduler.InternalMetricsToken)
if token == "" {
app.Logger.Warn("scheduler metrics http server not started: scheduler.internal_metrics_token is required")
return nil
}
authMiddleware := schedulerMetricsAuthMiddleware(token)
authMiddleware := schedulerMetricsAuthMiddleware(app)
app.Engine.GET("/api/internal/metrics/monitoring/daily-tasks", authMiddleware, func(c *gin.Context) {
response.Success(c, tenantapp.MonitoringDailyTaskMetricsSnapshotValue())
})
@@ -178,7 +181,7 @@ func startSchedulerMetricsServer(app *bootstrap.App) *http.Server {
prometheusHandler.ServeHTTP(c.Writer, c.Request)
})
addr := schedulerHTTPAddr(app.Config.Scheduler.HTTPHost, app.Config.Scheduler.HTTPPort)
addr := schedulerHTTPAddr(app.Config().Scheduler.HTTPHost, app.Config().Scheduler.HTTPPort)
server := &http.Server{
Addr: addr,
Handler: app.Engine,
@@ -219,9 +222,21 @@ func schedulerHTTPAddr(host string, port int) string {
return net.JoinHostPort(host, strconv.Itoa(port))
}
func schedulerMetricsAuthMiddleware(token string) gin.HandlerFunc {
expected := strings.TrimSpace(token)
func schedulerMetricsAuthMiddleware(app *bootstrap.App) gin.HandlerFunc {
return schedulerMetricsAuthMiddlewareWithToken(func() string {
if cfg := app.Config(); cfg != nil {
return cfg.Scheduler.InternalMetricsToken
}
return ""
})
}
func schedulerMetricsAuthMiddlewareWithToken(token func() string) gin.HandlerFunc {
return func(c *gin.Context) {
expected := ""
if token != nil {
expected = strings.TrimSpace(token())
}
if expected == "" || !schedulerMetricsTokenMatches(c, expected) {
response.Error(c, response.ErrUnauthorized(40181, "internal_metrics_unauthorized", "valid internal metrics token is required"))
c.Abort()
+1 -1
View File
@@ -38,7 +38,7 @@ func TestSchedulerMetricsAuthMiddleware(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
router := gin.New()
router.GET("/metrics", schedulerMetricsAuthMiddleware("secret-token"), func(c *gin.Context) {
router.GET("/metrics", schedulerMetricsAuthMiddlewareWithToken(func() string { return "secret-token" }), func(c *gin.Context) {
c.Status(http.StatusNoContent)
})
+6 -4
View File
@@ -28,22 +28,24 @@ func main() {
workerCtx, workerCancel := context.WithCancel(context.Background())
defer workerCancel()
app.StartConfigWatcher(workerCtx)
app.GenerationStreams.Run(workerCtx)
app.DesktopTaskStreams.Run(workerCtx)
app.DesktopDispatch.Run(workerCtx)
monitoringService := tenantapp.NewMonitoringService(app.DB, app.MonitoringDB, app.RabbitMQ, app.Config.MonitoringDispatch, app.Config.BrandLibrary, app.Logger).WithRedis(app.Redis)
monitoringService := app.MonitoringService
monitoringCallbackService := tenantapp.NewMonitoringCallbackService(app.DB, app.MonitoringDB, app.RabbitMQ, app.LLM, app.Logger)
tenantapp.NewMonitoringCollectOutboxWorker(monitoringService, app.Logger).Start(workerCtx)
tenantapp.NewMonitoringResultIngestWorker(monitoringCallbackService, app.RabbitMQ, app.Logger, app.Config.MonitoringWorkers).Start(workerCtx)
tenantapp.NewMonitoringProjectionRebuildWorker(monitoringCallbackService, app.RabbitMQ, app.Logger, app.Config.MonitoringWorkers).Start(workerCtx)
tenantapp.NewMonitoringResultIngestWorker(monitoringCallbackService, app.RabbitMQ, app.Logger, app.Config().MonitoringWorkers).Start(workerCtx)
tenantapp.NewMonitoringProjectionRebuildWorker(monitoringCallbackService, app.RabbitMQ, app.Logger, app.Config().MonitoringWorkers).Start(workerCtx)
tenantapp.NewDesktopAccountHealthSinkWorker(app.DB, app.RabbitMQ, app.Logger).Start(workerCtx)
imageService := tenantapp.NewImageService(app.DB, app.ObjectStorage, app.RabbitMQ, app.Logger)
tenantapp.NewImageAssetWorker(imageService, app.RabbitMQ, app.Logger).Start(workerCtx)
transport.RegisterRoutes(app)
addr := fmt.Sprintf(":%d", app.Config.Server.Port)
addr := fmt.Sprintf(":%d", app.Config().Server.Port)
app.Logger.Sugar().Infof("tenant-api starting on %s", addr)
go func() {
+20 -17
View File
@@ -25,7 +25,8 @@ func main() {
}
defer app.Close()
generationCfg := app.Config.Generation
generationProvider := tenantapp.NewGenerationStreamDisabledConfigProvider(app.ConfigStore)
generationCfg := app.Config().Generation
generationCfg.StreamEnabled = false
knowledgeSvc := tenantapp.NewKnowledgeService(
@@ -35,11 +36,11 @@ func main() {
app.ObjectStorage,
app.LLM,
app.Logger,
app.Config.Retrieval,
app.Config.LLM,
app.Config().Retrieval,
app.Config().LLM,
0,
0,
)
).WithConfigProvider(app.ConfigStore)
templateSvc := tenantapp.NewTemplateService(
app.DB,
@@ -52,8 +53,8 @@ func main() {
knowledgeSvc,
app.GenerationStreams,
generationCfg,
app.Config.LLM.MaxOutputTokens,
).WithCache(app.Cache)
app.Config().LLM.MaxOutputTokens,
).WithCache(app.Cache).WithConfigProvider(generationProvider)
promptRuleSvc := tenantapp.NewPromptRuleGenerationService(
app.DB,
@@ -62,8 +63,8 @@ func main() {
knowledgeSvc,
app.GenerationStreams,
generationCfg,
app.Config.LLM.MaxOutputTokens,
).WithCache(app.Cache).WithLogger(app.Logger)
app.Config().LLM.MaxOutputTokens,
).WithCache(app.Cache).WithLogger(app.Logger).WithConfigProvider(generationProvider)
publishJobSvc := tenantapp.NewPublishJobService(
app.DB,
@@ -80,8 +81,8 @@ func main() {
knowledgeSvc,
app.GenerationStreams,
generationCfg,
app.Config.LLM.MaxOutputTokens,
).WithCache(app.Cache)
app.Config().LLM.MaxOutputTokens,
).WithCache(app.Cache).WithConfigProvider(generationProvider)
kolWorker := generateworker.NewKolGenerationWorker(
app.DB,
@@ -89,12 +90,14 @@ func main() {
knowledgeSvc,
app.Logger,
generationCfg,
app.Config.LLM.MaxOutputTokens,
).WithCache(app.Cache)
app.Config().LLM.MaxOutputTokens,
).WithCache(app.Cache).WithConfigProvider(app.ConfigStore)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
app.StartConfigWatcher(ctx)
generateworker.NewArticleGenerationWorker(
app.DB,
app.RabbitMQ,
@@ -103,16 +106,16 @@ func main() {
imitationSvc,
kolWorker,
app.Logger,
app.Config.Generation,
).Start(ctx)
app.Config().Generation,
).WithConfigProvider(app.ConfigStore).Start(ctx)
generateworker.NewTemplateAssistWorker(
app.RabbitMQ,
templateSvc,
app.Logger,
app.Config.Generation.WorkerConcurrency,
app.Config.Generation.ArticleTimeout,
).Start(ctx)
app.Config().Generation.WorkerConcurrency,
app.Config().Generation.ArticleTimeout,
).WithConfigProvider(app.ConfigStore).Start(ctx)
app.Logger.Info("worker-generate started")