feat(infra): add RabbitMQ generation/template-assist queue configs and scheduler settings
Introduce generation task and template-assist queue declarations with DLX/DLQ support in RabbitMQ client. Add scheduler dispatch and monitoring worker concurrency configs. Include publish channel pool for high-throughput message publishing. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -14,6 +14,8 @@ type Config struct {
|
||||
Database DatabaseConfig `mapstructure:"database"`
|
||||
MonitoringDatabase DatabaseConfig `mapstructure:"monitoring_database"`
|
||||
RabbitMQ RabbitMQConfig `mapstructure:"rabbitmq"`
|
||||
Scheduler SchedulerConfig `mapstructure:"scheduler"`
|
||||
MonitoringWorkers MonitoringConfig `mapstructure:"monitoring_workers"`
|
||||
Redis RedisConfig `mapstructure:"redis"`
|
||||
Qdrant QdrantConfig `mapstructure:"qdrant"`
|
||||
ObjectStorage ObjectStorageConfig `mapstructure:"object_storage"`
|
||||
@@ -65,7 +67,34 @@ type RabbitMQConfig struct {
|
||||
MonitorProjectionDLX string `mapstructure:"monitor_projection_dlx"`
|
||||
MonitorProjectionDLQ string `mapstructure:"monitor_projection_dlq"`
|
||||
MonitorProjectionDLQRouteKey string `mapstructure:"monitor_projection_dlq_routing_key"`
|
||||
GenerationExchange string `mapstructure:"generation_exchange"`
|
||||
GenerationRoutingKey string `mapstructure:"generation_routing_key"`
|
||||
GenerationQueue string `mapstructure:"generation_queue"`
|
||||
GenerationDLX string `mapstructure:"generation_dlx"`
|
||||
GenerationDLQ string `mapstructure:"generation_dlq"`
|
||||
GenerationDLQRouteKey string `mapstructure:"generation_dlq_routing_key"`
|
||||
GenerationEventExchange string `mapstructure:"generation_event_exchange"`
|
||||
TemplateAssistExchange string `mapstructure:"template_assist_exchange"`
|
||||
TemplateAssistRoutingKey string `mapstructure:"template_assist_routing_key"`
|
||||
TemplateAssistQueue string `mapstructure:"template_assist_queue"`
|
||||
TemplateAssistDLX string `mapstructure:"template_assist_dlx"`
|
||||
TemplateAssistDLQ string `mapstructure:"template_assist_dlq"`
|
||||
TemplateAssistDLQRouteKey string `mapstructure:"template_assist_dlq_routing_key"`
|
||||
ConsumerPrefetch int `mapstructure:"consumer_prefetch"`
|
||||
PublishChannelPoolSize int `mapstructure:"publish_channel_pool_size"`
|
||||
}
|
||||
|
||||
type SchedulerConfig struct {
|
||||
DispatchInterval time.Duration `mapstructure:"dispatch_interval"`
|
||||
DispatchTimeout time.Duration `mapstructure:"dispatch_timeout"`
|
||||
DispatchBatchSize int `mapstructure:"dispatch_batch_size"`
|
||||
DispatchConcurrency int `mapstructure:"dispatch_concurrency"`
|
||||
GenerationQueueBackpressureLimit int `mapstructure:"generation_queue_backpressure_limit"`
|
||||
}
|
||||
|
||||
type MonitoringConfig struct {
|
||||
ResultIngestConcurrency int `mapstructure:"result_ingest_concurrency"`
|
||||
ProjectionRebuildConcurrency int `mapstructure:"projection_rebuild_concurrency"`
|
||||
}
|
||||
|
||||
type QdrantConfig struct {
|
||||
@@ -159,6 +188,8 @@ func Load(configPath string) (*Config, error) {
|
||||
|
||||
applyEnvOverrides(&cfg)
|
||||
normalizeRabbitMQConfig(&cfg.RabbitMQ)
|
||||
normalizeSchedulerConfig(&cfg.Scheduler)
|
||||
normalizeMonitoringConfig(&cfg.MonitoringWorkers)
|
||||
|
||||
return &cfg, nil
|
||||
}
|
||||
@@ -266,6 +297,81 @@ func normalizeRabbitMQConfig(cfg *RabbitMQConfig) {
|
||||
if strings.TrimSpace(cfg.MonitorProjectionDLQRouteKey) == "" {
|
||||
cfg.MonitorProjectionDLQRouteKey = "monitor.projection.rebuild.dlq"
|
||||
}
|
||||
|
||||
if strings.TrimSpace(cfg.GenerationExchange) == "" {
|
||||
cfg.GenerationExchange = "generation.task"
|
||||
}
|
||||
if strings.TrimSpace(cfg.GenerationRoutingKey) == "" {
|
||||
cfg.GenerationRoutingKey = "generation.task.run"
|
||||
}
|
||||
if strings.TrimSpace(cfg.GenerationQueue) == "" {
|
||||
cfg.GenerationQueue = "generation.task.run"
|
||||
}
|
||||
if strings.TrimSpace(cfg.GenerationDLX) == "" {
|
||||
cfg.GenerationDLX = "generation.task.dlx"
|
||||
}
|
||||
if strings.TrimSpace(cfg.GenerationDLQ) == "" {
|
||||
cfg.GenerationDLQ = "generation.task.run.dlq"
|
||||
}
|
||||
if strings.TrimSpace(cfg.GenerationDLQRouteKey) == "" {
|
||||
cfg.GenerationDLQRouteKey = "generation.task.run.dlq"
|
||||
}
|
||||
if strings.TrimSpace(cfg.GenerationEventExchange) == "" {
|
||||
cfg.GenerationEventExchange = "generation.event"
|
||||
}
|
||||
if strings.TrimSpace(cfg.TemplateAssistExchange) == "" {
|
||||
cfg.TemplateAssistExchange = "template.assist"
|
||||
}
|
||||
if strings.TrimSpace(cfg.TemplateAssistRoutingKey) == "" {
|
||||
cfg.TemplateAssistRoutingKey = "template.assist.run"
|
||||
}
|
||||
if strings.TrimSpace(cfg.TemplateAssistQueue) == "" {
|
||||
cfg.TemplateAssistQueue = "template.assist.run"
|
||||
}
|
||||
if strings.TrimSpace(cfg.TemplateAssistDLX) == "" {
|
||||
cfg.TemplateAssistDLX = "template.assist.dlx"
|
||||
}
|
||||
if strings.TrimSpace(cfg.TemplateAssistDLQ) == "" {
|
||||
cfg.TemplateAssistDLQ = "template.assist.run.dlq"
|
||||
}
|
||||
if strings.TrimSpace(cfg.TemplateAssistDLQRouteKey) == "" {
|
||||
cfg.TemplateAssistDLQRouteKey = "template.assist.run.dlq"
|
||||
}
|
||||
if cfg.PublishChannelPoolSize <= 0 {
|
||||
cfg.PublishChannelPoolSize = 16
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeSchedulerConfig(cfg *SchedulerConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if cfg.DispatchInterval <= 0 {
|
||||
cfg.DispatchInterval = 15 * time.Second
|
||||
}
|
||||
if cfg.DispatchTimeout <= 0 {
|
||||
cfg.DispatchTimeout = 30 * time.Second
|
||||
}
|
||||
if cfg.DispatchBatchSize <= 0 {
|
||||
cfg.DispatchBatchSize = 20
|
||||
}
|
||||
if cfg.DispatchConcurrency <= 0 {
|
||||
cfg.DispatchConcurrency = 1
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeMonitoringConfig(cfg *MonitoringConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if cfg.ResultIngestConcurrency <= 0 {
|
||||
cfg.ResultIngestConcurrency = 1
|
||||
}
|
||||
if cfg.ProjectionRebuildConcurrency <= 0 {
|
||||
cfg.ProjectionRebuildConcurrency = 1
|
||||
}
|
||||
}
|
||||
|
||||
func lookupFirstNonEmptyEnv(keys ...string) (string, bool) {
|
||||
|
||||
Reference in New Issue
Block a user