Files
geo/server/internal/shared/config/config.go
T

498 lines
17 KiB
Go

package config
import (
"fmt"
"os"
"strings"
"time"
"github.com/spf13/viper"
)
type Config struct {
Server ServerConfig `mapstructure:"server"`
Database DatabaseConfig `mapstructure:"database"`
MonitoringDatabase DatabaseConfig `mapstructure:"monitoring_database"`
RabbitMQ RabbitMQConfig `mapstructure:"rabbitmq"`
Scheduler SchedulerConfig `mapstructure:"scheduler"`
MonitoringWorkers MonitoringConfig `mapstructure:"monitoring_workers"`
BrandLibrary BrandLibraryConfig `mapstructure:"brand_library"`
Redis RedisConfig `mapstructure:"redis"`
Qdrant QdrantConfig `mapstructure:"qdrant"`
ObjectStorage ObjectStorageConfig `mapstructure:"object_storage"`
Cache CacheConfig `mapstructure:"cache"`
JWT JWTConfig `mapstructure:"jwt"`
Log LogConfig `mapstructure:"log"`
LLM LLMConfig `mapstructure:"llm"`
Retrieval RetrievalConfig `mapstructure:"retrieval"`
Generation GenerationConfig `mapstructure:"generation"`
}
type ServerConfig struct {
Port int `mapstructure:"port"`
Mode string `mapstructure:"mode"`
}
type DatabaseConfig struct {
Host string `mapstructure:"host"`
Port int `mapstructure:"port"`
User string `mapstructure:"user"`
Password string `mapstructure:"password"`
DBName string `mapstructure:"dbname"`
SSLMode string `mapstructure:"sslmode"`
MaxOpenConns int `mapstructure:"max_open_conns"`
MaxIdleConns int `mapstructure:"max_idle_conns"`
}
func (d DatabaseConfig) DSN() string {
return fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=%s",
d.User, d.Password, d.Host, d.Port, d.DBName, d.SSLMode)
}
type RedisConfig struct {
Addr string `mapstructure:"addr"`
DB int `mapstructure:"db"`
}
type RabbitMQConfig struct {
URL string `mapstructure:"url"`
MonitorResultExchange string `mapstructure:"monitor_result_exchange"`
MonitorResultRoutingKey string `mapstructure:"monitor_result_routing_key"`
MonitorResultQueue string `mapstructure:"monitor_result_queue"`
MonitorResultDLX string `mapstructure:"monitor_result_dlx"`
MonitorResultDLQ string `mapstructure:"monitor_result_dlq"`
MonitorResultDLQRouteKey string `mapstructure:"monitor_result_dlq_routing_key"`
MonitorProjectionExchange string `mapstructure:"monitor_projection_exchange"`
MonitorProjectionRoutingKey string `mapstructure:"monitor_projection_routing_key"`
MonitorProjectionQueue string `mapstructure:"monitor_projection_queue"`
MonitorProjectionDLX string `mapstructure:"monitor_projection_dlx"`
MonitorProjectionDLQ string `mapstructure:"monitor_projection_dlq"`
MonitorProjectionDLQRouteKey string `mapstructure:"monitor_projection_dlq_routing_key"`
GenerationExchange string `mapstructure:"generation_exchange"`
GenerationRoutingKey string `mapstructure:"generation_routing_key"`
GenerationQueue string `mapstructure:"generation_queue"`
GenerationDLX string `mapstructure:"generation_dlx"`
GenerationDLQ string `mapstructure:"generation_dlq"`
GenerationDLQRouteKey string `mapstructure:"generation_dlq_routing_key"`
GenerationEventExchange string `mapstructure:"generation_event_exchange"`
TemplateAssistExchange string `mapstructure:"template_assist_exchange"`
TemplateAssistRoutingKey string `mapstructure:"template_assist_routing_key"`
TemplateAssistQueue string `mapstructure:"template_assist_queue"`
TemplateAssistDLX string `mapstructure:"template_assist_dlx"`
TemplateAssistDLQ string `mapstructure:"template_assist_dlq"`
TemplateAssistDLQRouteKey string `mapstructure:"template_assist_dlq_routing_key"`
KolAssistExchange string `mapstructure:"kol_assist_exchange"`
KolAssistRoutingKey string `mapstructure:"kol_assist_routing_key"`
KolAssistQueue string `mapstructure:"kol_assist_queue"`
KolAssistDLX string `mapstructure:"kol_assist_dlx"`
KolAssistDLQ string `mapstructure:"kol_assist_dlq"`
KolAssistDLQRouteKey string `mapstructure:"kol_assist_dlq_routing_key"`
ImageAssetExchange string `mapstructure:"image_asset_exchange"`
ImageAssetRoutingKey string `mapstructure:"image_asset_routing_key"`
ImageAssetQueue string `mapstructure:"image_asset_queue"`
ImageAssetDLX string `mapstructure:"image_asset_dlx"`
ImageAssetDLQ string `mapstructure:"image_asset_dlq"`
ImageAssetDLQRouteKey string `mapstructure:"image_asset_dlq_routing_key"`
ConsumerPrefetch int `mapstructure:"consumer_prefetch"`
PublishChannelPoolSize int `mapstructure:"publish_channel_pool_size"`
}
type SchedulerConfig struct {
DispatchInterval time.Duration `mapstructure:"dispatch_interval"`
DispatchTimeout time.Duration `mapstructure:"dispatch_timeout"`
DispatchBatchSize int `mapstructure:"dispatch_batch_size"`
DispatchConcurrency int `mapstructure:"dispatch_concurrency"`
GenerationQueueBackpressureLimit int `mapstructure:"generation_queue_backpressure_limit"`
}
type MonitoringConfig struct {
ResultIngestConcurrency int `mapstructure:"result_ingest_concurrency"`
ProjectionRebuildConcurrency int `mapstructure:"projection_rebuild_concurrency"`
}
type BrandLibraryConfig struct {
FreeBrandLimit int `mapstructure:"free_brand_limit"`
PaidBrandLimit int `mapstructure:"paid_brand_limit"`
MaxKeywords int `mapstructure:"max_keywords"`
MaxQuestionsPerKeyword int `mapstructure:"max_questions_per_keyword"`
}
func (c BrandLibraryConfig) BrandLimitForPlan(planCode string) int {
if strings.EqualFold(strings.TrimSpace(planCode), "free") {
return c.FreeBrandLimit
}
return c.PaidBrandLimit
}
type QdrantConfig struct {
URL string `mapstructure:"url"`
APIKey string `mapstructure:"api_key"`
Collection string `mapstructure:"collection"`
Timeout time.Duration `mapstructure:"timeout"`
}
type ObjectStorageConfig struct {
Provider string `mapstructure:"provider"`
Endpoint string `mapstructure:"endpoint"`
AccessKey string `mapstructure:"access_key"`
SecretKey string `mapstructure:"secret_key"`
Bucket string `mapstructure:"bucket"`
UseSSL bool `mapstructure:"use_ssl"`
PublicBaseURL string `mapstructure:"public_base_url"`
Region string `mapstructure:"region"`
}
type JWTConfig struct {
Secret string `mapstructure:"secret"`
AccessTTL time.Duration `mapstructure:"access_ttl"`
RefreshTTL time.Duration `mapstructure:"refresh_ttl"`
}
type LogConfig struct {
Level string `mapstructure:"level"`
Format string `mapstructure:"format"`
}
type LLMConfig struct {
Provider string `mapstructure:"provider"`
APIKey string `mapstructure:"api_key"`
BaseURL string `mapstructure:"base_url"`
Model string `mapstructure:"model"`
KnowledgeURLModel string `mapstructure:"knowledge_url_model"`
Timeout time.Duration `mapstructure:"timeout"`
MaxOutputTokens int64 `mapstructure:"max_output_tokens"`
Temperature float64 `mapstructure:"temperature"`
TopP float64 `mapstructure:"top_p"`
ReasoningEffort string `mapstructure:"reasoning_effort"`
WebSearchLimit int32 `mapstructure:"web_search_limit"`
}
type RetrievalConfig struct {
Provider string `mapstructure:"provider"`
BaseURL string `mapstructure:"base_url"`
APIKey string `mapstructure:"api_key"`
EmbeddingModel string `mapstructure:"embedding_model"`
RerankerModel string `mapstructure:"reranker_model"`
Timeout time.Duration `mapstructure:"timeout"`
ChunkSize int `mapstructure:"chunk_size"`
ChunkOverlap int `mapstructure:"chunk_overlap"`
EmbeddingBatchSize int `mapstructure:"embedding_batch_size"`
RecallLimit int `mapstructure:"recall_limit"`
RerankTopN int `mapstructure:"rerank_top_n"`
MaxChunksPerDoc int `mapstructure:"max_chunks_per_doc"`
OverlapTokens int `mapstructure:"overlap_tokens"`
}
type CacheConfig struct {
Driver string `mapstructure:"driver"` // "redis" or "memory"
}
type GenerationConfig struct {
QueueSize int `mapstructure:"queue_size"`
WorkerConcurrency int `mapstructure:"worker_concurrency"`
StreamEnabled bool `mapstructure:"stream_enabled"`
ArticleTimeout time.Duration `mapstructure:"article_timeout"`
}
func Load(configPath string) (*Config, error) {
v := viper.New()
v.SetConfigFile(configPath)
v.AutomaticEnv()
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
if err := v.ReadInConfig(); err != nil {
return nil, fmt.Errorf("read config: %w", err)
}
// Allow local overrides
v.SetConfigFile(strings.Replace(configPath, ".yaml", ".local.yaml", 1))
_ = v.MergeInConfig()
var cfg Config
if err := v.Unmarshal(&cfg); err != nil {
return nil, fmt.Errorf("unmarshal config: %w", err)
}
applyEnvOverrides(&cfg)
normalizeRabbitMQConfig(&cfg.RabbitMQ)
normalizeSchedulerConfig(&cfg.Scheduler)
normalizeMonitoringConfig(&cfg.MonitoringWorkers)
normalizeBrandLibraryConfig(&cfg.BrandLibrary)
return &cfg, nil
}
func applyEnvOverrides(cfg *Config) {
if cfg == nil {
return
}
if apiKey, ok := lookupNonEmptyEnv("LLM_API_KEY"); ok {
cfg.LLM.APIKey = apiKey
}
if model, ok := lookupNonEmptyEnv("LLM_KNOWLEDGE_URL_MODEL"); ok {
cfg.LLM.KnowledgeURLModel = model
}
if apiKey, ok := lookupNonEmptyEnv("QDRANT_API_KEY"); ok {
cfg.Qdrant.APIKey = apiKey
}
if rabbitmqURL, ok := lookupNonEmptyEnv("RABBITMQ_URL"); ok {
cfg.RabbitMQ.URL = rabbitmqURL
}
if url, ok := lookupNonEmptyEnv("QDRANT_URL"); ok {
cfg.Qdrant.URL = url
}
if collection, ok := lookupNonEmptyEnv("QDRANT_COLLECTION"); ok {
cfg.Qdrant.Collection = collection
}
if endpoint, ok := lookupNonEmptyEnv("OBJECT_STORAGE_ENDPOINT"); ok {
cfg.ObjectStorage.Endpoint = endpoint
}
if accessKey, ok := lookupNonEmptyEnv("OBJECT_STORAGE_ACCESS_KEY"); ok {
cfg.ObjectStorage.AccessKey = accessKey
}
if secretKey, ok := lookupNonEmptyEnv("OBJECT_STORAGE_SECRET_KEY"); ok {
cfg.ObjectStorage.SecretKey = secretKey
}
if bucket, ok := lookupNonEmptyEnv("OBJECT_STORAGE_BUCKET"); ok {
cfg.ObjectStorage.Bucket = bucket
}
if provider, ok := lookupNonEmptyEnv("OBJECT_STORAGE_PROVIDER"); ok {
cfg.ObjectStorage.Provider = provider
}
if publicBaseURL, ok := lookupNonEmptyEnv("OBJECT_STORAGE_PUBLIC_BASE_URL"); ok {
cfg.ObjectStorage.PublicBaseURL = publicBaseURL
}
if region, ok := lookupNonEmptyEnv("OBJECT_STORAGE_REGION"); ok {
cfg.ObjectStorage.Region = region
}
if useSSL, ok := lookupBoolEnv("OBJECT_STORAGE_USE_SSL"); ok {
cfg.ObjectStorage.UseSSL = useSSL
}
if apiKey, ok := lookupFirstNonEmptyEnv("SILICONFLOW_API_KEY", "RETRIEVAL_API_KEY", "EMBEDDING_API_KEY", "RERANKER_API_KEY"); ok {
cfg.Retrieval.APIKey = apiKey
}
if baseURL, ok := lookupFirstNonEmptyEnv("SILICONFLOW_BASE_URL", "RETRIEVAL_BASE_URL"); ok {
cfg.Retrieval.BaseURL = baseURL
}
if model, ok := lookupFirstNonEmptyEnv("SILICONFLOW_EMBEDDING_MODEL", "RETRIEVAL_EMBEDDING_MODEL"); ok {
cfg.Retrieval.EmbeddingModel = model
}
if model, ok := lookupFirstNonEmptyEnv("SILICONFLOW_RERANKER_MODEL", "RETRIEVAL_RERANKER_MODEL"); ok {
cfg.Retrieval.RerankerModel = model
}
}
func normalizeRabbitMQConfig(cfg *RabbitMQConfig) {
if cfg == nil {
return
}
if strings.TrimSpace(cfg.MonitorResultExchange) == "" {
cfg.MonitorResultExchange = "monitor.result"
}
if strings.TrimSpace(cfg.MonitorResultRoutingKey) == "" {
cfg.MonitorResultRoutingKey = "monitor.result.ingest"
}
if strings.TrimSpace(cfg.MonitorResultQueue) == "" {
cfg.MonitorResultQueue = "monitor.result.ingest"
}
if strings.TrimSpace(cfg.MonitorResultDLX) == "" {
cfg.MonitorResultDLX = "monitor.result.dlx"
}
if strings.TrimSpace(cfg.MonitorResultDLQ) == "" {
cfg.MonitorResultDLQ = "monitor.result.ingest.dlq"
}
if strings.TrimSpace(cfg.MonitorResultDLQRouteKey) == "" {
cfg.MonitorResultDLQRouteKey = "monitor.result.ingest.dlq"
}
if strings.TrimSpace(cfg.MonitorProjectionExchange) == "" {
cfg.MonitorProjectionExchange = "monitor.projection"
}
if strings.TrimSpace(cfg.MonitorProjectionRoutingKey) == "" {
cfg.MonitorProjectionRoutingKey = "monitor.projection.rebuild"
}
if strings.TrimSpace(cfg.MonitorProjectionQueue) == "" {
cfg.MonitorProjectionQueue = "monitor.projection.rebuild"
}
if strings.TrimSpace(cfg.MonitorProjectionDLX) == "" {
cfg.MonitorProjectionDLX = "monitor.projection.dlx"
}
if strings.TrimSpace(cfg.MonitorProjectionDLQ) == "" {
cfg.MonitorProjectionDLQ = "monitor.projection.rebuild.dlq"
}
if strings.TrimSpace(cfg.MonitorProjectionDLQRouteKey) == "" {
cfg.MonitorProjectionDLQRouteKey = "monitor.projection.rebuild.dlq"
}
if strings.TrimSpace(cfg.GenerationExchange) == "" {
cfg.GenerationExchange = "generation.task"
}
if strings.TrimSpace(cfg.GenerationRoutingKey) == "" {
cfg.GenerationRoutingKey = "generation.task.run"
}
if strings.TrimSpace(cfg.GenerationQueue) == "" {
cfg.GenerationQueue = "generation.task.run"
}
if strings.TrimSpace(cfg.GenerationDLX) == "" {
cfg.GenerationDLX = "generation.task.dlx"
}
if strings.TrimSpace(cfg.GenerationDLQ) == "" {
cfg.GenerationDLQ = "generation.task.run.dlq"
}
if strings.TrimSpace(cfg.GenerationDLQRouteKey) == "" {
cfg.GenerationDLQRouteKey = "generation.task.run.dlq"
}
if strings.TrimSpace(cfg.GenerationEventExchange) == "" {
cfg.GenerationEventExchange = "generation.event"
}
if strings.TrimSpace(cfg.TemplateAssistExchange) == "" {
cfg.TemplateAssistExchange = "template.assist"
}
if strings.TrimSpace(cfg.TemplateAssistRoutingKey) == "" {
cfg.TemplateAssistRoutingKey = "template.assist.run"
}
if strings.TrimSpace(cfg.TemplateAssistQueue) == "" {
cfg.TemplateAssistQueue = "template.assist.run"
}
if strings.TrimSpace(cfg.TemplateAssistDLX) == "" {
cfg.TemplateAssistDLX = "template.assist.dlx"
}
if strings.TrimSpace(cfg.TemplateAssistDLQ) == "" {
cfg.TemplateAssistDLQ = "template.assist.run.dlq"
}
if strings.TrimSpace(cfg.TemplateAssistDLQRouteKey) == "" {
cfg.TemplateAssistDLQRouteKey = "template.assist.run.dlq"
}
if strings.TrimSpace(cfg.KolAssistExchange) == "" {
cfg.KolAssistExchange = "kol.assist"
}
if strings.TrimSpace(cfg.KolAssistRoutingKey) == "" {
cfg.KolAssistRoutingKey = "kol.assist.run"
}
if strings.TrimSpace(cfg.KolAssistQueue) == "" {
cfg.KolAssistQueue = "kol.assist.run"
}
if strings.TrimSpace(cfg.KolAssistDLX) == "" {
cfg.KolAssistDLX = "kol.assist.dlx"
}
if strings.TrimSpace(cfg.KolAssistDLQ) == "" {
cfg.KolAssistDLQ = "kol.assist.run.dlq"
}
if strings.TrimSpace(cfg.KolAssistDLQRouteKey) == "" {
cfg.KolAssistDLQRouteKey = "kol.assist.run.dlq"
}
if strings.TrimSpace(cfg.ImageAssetExchange) == "" {
cfg.ImageAssetExchange = "image.asset"
}
if strings.TrimSpace(cfg.ImageAssetRoutingKey) == "" {
cfg.ImageAssetRoutingKey = "image.asset.finalize"
}
if strings.TrimSpace(cfg.ImageAssetQueue) == "" {
cfg.ImageAssetQueue = "image.asset.finalize"
}
if strings.TrimSpace(cfg.ImageAssetDLX) == "" {
cfg.ImageAssetDLX = "image.asset.dlx"
}
if strings.TrimSpace(cfg.ImageAssetDLQ) == "" {
cfg.ImageAssetDLQ = "image.asset.finalize.dlq"
}
if strings.TrimSpace(cfg.ImageAssetDLQRouteKey) == "" {
cfg.ImageAssetDLQRouteKey = "image.asset.finalize.dlq"
}
if cfg.PublishChannelPoolSize <= 0 {
cfg.PublishChannelPoolSize = 16
}
}
func normalizeSchedulerConfig(cfg *SchedulerConfig) {
if cfg == nil {
return
}
if cfg.DispatchInterval <= 0 {
cfg.DispatchInterval = 15 * time.Second
}
if cfg.DispatchTimeout <= 0 {
cfg.DispatchTimeout = 30 * time.Second
}
if cfg.DispatchBatchSize <= 0 {
cfg.DispatchBatchSize = 20
}
if cfg.DispatchConcurrency <= 0 {
cfg.DispatchConcurrency = 1
}
}
func normalizeMonitoringConfig(cfg *MonitoringConfig) {
if cfg == nil {
return
}
if cfg.ResultIngestConcurrency <= 0 {
cfg.ResultIngestConcurrency = 1
}
if cfg.ProjectionRebuildConcurrency <= 0 {
cfg.ProjectionRebuildConcurrency = 1
}
}
func normalizeBrandLibraryConfig(cfg *BrandLibraryConfig) {
if cfg == nil {
return
}
if cfg.FreeBrandLimit <= 0 {
cfg.FreeBrandLimit = 1
}
if cfg.PaidBrandLimit <= 0 {
cfg.PaidBrandLimit = 2
}
if cfg.MaxKeywords <= 0 {
cfg.MaxKeywords = 5
}
if cfg.MaxQuestionsPerKeyword <= 0 {
cfg.MaxQuestionsPerKeyword = 5
}
}
func lookupFirstNonEmptyEnv(keys ...string) (string, bool) {
for _, key := range keys {
if value, ok := lookupNonEmptyEnv(key); ok {
return value, true
}
}
return "", false
}
func lookupNonEmptyEnv(key string) (string, bool) {
value, ok := os.LookupEnv(key)
if !ok {
return "", false
}
value = strings.TrimSpace(value)
if value == "" {
return "", false
}
return value, true
}
func lookupBoolEnv(key string) (bool, bool) {
value, ok := os.LookupEnv(key)
if !ok {
return false, false
}
switch strings.ToLower(strings.TrimSpace(value)) {
case "1", "true", "yes", "on":
return true, true
case "0", "false", "no", "off":
return false, true
default:
return false, false
}
}