feat(infra): add RabbitMQ generation/template-assist queue configs and scheduler settings

Introduce generation task and template-assist queue declarations with
DLX/DLQ support in RabbitMQ client. Add scheduler dispatch and
monitoring worker concurrency configs. Include publish channel pool
for high-throughput message publishing.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-04-15 14:18:55 +08:00
parent 1b01caac0f
commit b46cfaaa61
6 changed files with 481 additions and 11 deletions
+106
View File
@@ -14,6 +14,8 @@ type Config struct {
Database DatabaseConfig `mapstructure:"database"`
MonitoringDatabase DatabaseConfig `mapstructure:"monitoring_database"`
RabbitMQ RabbitMQConfig `mapstructure:"rabbitmq"`
Scheduler SchedulerConfig `mapstructure:"scheduler"`
MonitoringWorkers MonitoringConfig `mapstructure:"monitoring_workers"`
Redis RedisConfig `mapstructure:"redis"`
Qdrant QdrantConfig `mapstructure:"qdrant"`
ObjectStorage ObjectStorageConfig `mapstructure:"object_storage"`
@@ -65,7 +67,34 @@ type RabbitMQConfig struct {
MonitorProjectionDLX string `mapstructure:"monitor_projection_dlx"`
MonitorProjectionDLQ string `mapstructure:"monitor_projection_dlq"`
MonitorProjectionDLQRouteKey string `mapstructure:"monitor_projection_dlq_routing_key"`
GenerationExchange string `mapstructure:"generation_exchange"`
GenerationRoutingKey string `mapstructure:"generation_routing_key"`
GenerationQueue string `mapstructure:"generation_queue"`
GenerationDLX string `mapstructure:"generation_dlx"`
GenerationDLQ string `mapstructure:"generation_dlq"`
GenerationDLQRouteKey string `mapstructure:"generation_dlq_routing_key"`
GenerationEventExchange string `mapstructure:"generation_event_exchange"`
TemplateAssistExchange string `mapstructure:"template_assist_exchange"`
TemplateAssistRoutingKey string `mapstructure:"template_assist_routing_key"`
TemplateAssistQueue string `mapstructure:"template_assist_queue"`
TemplateAssistDLX string `mapstructure:"template_assist_dlx"`
TemplateAssistDLQ string `mapstructure:"template_assist_dlq"`
TemplateAssistDLQRouteKey string `mapstructure:"template_assist_dlq_routing_key"`
ConsumerPrefetch int `mapstructure:"consumer_prefetch"`
PublishChannelPoolSize int `mapstructure:"publish_channel_pool_size"`
}
type SchedulerConfig struct {
DispatchInterval time.Duration `mapstructure:"dispatch_interval"`
DispatchTimeout time.Duration `mapstructure:"dispatch_timeout"`
DispatchBatchSize int `mapstructure:"dispatch_batch_size"`
DispatchConcurrency int `mapstructure:"dispatch_concurrency"`
GenerationQueueBackpressureLimit int `mapstructure:"generation_queue_backpressure_limit"`
}
type MonitoringConfig struct {
ResultIngestConcurrency int `mapstructure:"result_ingest_concurrency"`
ProjectionRebuildConcurrency int `mapstructure:"projection_rebuild_concurrency"`
}
type QdrantConfig struct {
@@ -159,6 +188,8 @@ func Load(configPath string) (*Config, error) {
applyEnvOverrides(&cfg)
normalizeRabbitMQConfig(&cfg.RabbitMQ)
normalizeSchedulerConfig(&cfg.Scheduler)
normalizeMonitoringConfig(&cfg.MonitoringWorkers)
return &cfg, nil
}
@@ -266,6 +297,81 @@ func normalizeRabbitMQConfig(cfg *RabbitMQConfig) {
if strings.TrimSpace(cfg.MonitorProjectionDLQRouteKey) == "" {
cfg.MonitorProjectionDLQRouteKey = "monitor.projection.rebuild.dlq"
}
if strings.TrimSpace(cfg.GenerationExchange) == "" {
cfg.GenerationExchange = "generation.task"
}
if strings.TrimSpace(cfg.GenerationRoutingKey) == "" {
cfg.GenerationRoutingKey = "generation.task.run"
}
if strings.TrimSpace(cfg.GenerationQueue) == "" {
cfg.GenerationQueue = "generation.task.run"
}
if strings.TrimSpace(cfg.GenerationDLX) == "" {
cfg.GenerationDLX = "generation.task.dlx"
}
if strings.TrimSpace(cfg.GenerationDLQ) == "" {
cfg.GenerationDLQ = "generation.task.run.dlq"
}
if strings.TrimSpace(cfg.GenerationDLQRouteKey) == "" {
cfg.GenerationDLQRouteKey = "generation.task.run.dlq"
}
if strings.TrimSpace(cfg.GenerationEventExchange) == "" {
cfg.GenerationEventExchange = "generation.event"
}
if strings.TrimSpace(cfg.TemplateAssistExchange) == "" {
cfg.TemplateAssistExchange = "template.assist"
}
if strings.TrimSpace(cfg.TemplateAssistRoutingKey) == "" {
cfg.TemplateAssistRoutingKey = "template.assist.run"
}
if strings.TrimSpace(cfg.TemplateAssistQueue) == "" {
cfg.TemplateAssistQueue = "template.assist.run"
}
if strings.TrimSpace(cfg.TemplateAssistDLX) == "" {
cfg.TemplateAssistDLX = "template.assist.dlx"
}
if strings.TrimSpace(cfg.TemplateAssistDLQ) == "" {
cfg.TemplateAssistDLQ = "template.assist.run.dlq"
}
if strings.TrimSpace(cfg.TemplateAssistDLQRouteKey) == "" {
cfg.TemplateAssistDLQRouteKey = "template.assist.run.dlq"
}
if cfg.PublishChannelPoolSize <= 0 {
cfg.PublishChannelPoolSize = 16
}
}
func normalizeSchedulerConfig(cfg *SchedulerConfig) {
if cfg == nil {
return
}
if cfg.DispatchInterval <= 0 {
cfg.DispatchInterval = 15 * time.Second
}
if cfg.DispatchTimeout <= 0 {
cfg.DispatchTimeout = 30 * time.Second
}
if cfg.DispatchBatchSize <= 0 {
cfg.DispatchBatchSize = 20
}
if cfg.DispatchConcurrency <= 0 {
cfg.DispatchConcurrency = 1
}
}
func normalizeMonitoringConfig(cfg *MonitoringConfig) {
if cfg == nil {
return
}
if cfg.ResultIngestConcurrency <= 0 {
cfg.ResultIngestConcurrency = 1
}
if cfg.ProjectionRebuildConcurrency <= 0 {
cfg.ProjectionRebuildConcurrency = 1
}
}
func lookupFirstNonEmptyEnv(keys ...string) (string, bool) {