root
6066f43a7d
- Implement monitoring service with heartbeat, lease tasks, resume tasks, and task result handling. - Create monitoring time utilities for business date calculations. - Add unit tests for date window resolution and business day handling. - Define database schema for monitoring-related tables including quotas, daily reports, and task management. - Establish migration scripts for creating and dropping monitoring tables.
309 lines
10 KiB
Go
309 lines
10 KiB
Go
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/spf13/viper"
|
|
)
|
|
|
|
type Config struct {
|
|
Server ServerConfig `mapstructure:"server"`
|
|
Database DatabaseConfig `mapstructure:"database"`
|
|
MonitoringDatabase DatabaseConfig `mapstructure:"monitoring_database"`
|
|
RabbitMQ RabbitMQConfig `mapstructure:"rabbitmq"`
|
|
Redis RedisConfig `mapstructure:"redis"`
|
|
Qdrant QdrantConfig `mapstructure:"qdrant"`
|
|
ObjectStorage ObjectStorageConfig `mapstructure:"object_storage"`
|
|
Cache CacheConfig `mapstructure:"cache"`
|
|
JWT JWTConfig `mapstructure:"jwt"`
|
|
Log LogConfig `mapstructure:"log"`
|
|
LLM LLMConfig `mapstructure:"llm"`
|
|
Retrieval RetrievalConfig `mapstructure:"retrieval"`
|
|
Generation GenerationConfig `mapstructure:"generation"`
|
|
}
|
|
|
|
type ServerConfig struct {
|
|
Port int `mapstructure:"port"`
|
|
Mode string `mapstructure:"mode"`
|
|
}
|
|
|
|
type DatabaseConfig struct {
|
|
Host string `mapstructure:"host"`
|
|
Port int `mapstructure:"port"`
|
|
User string `mapstructure:"user"`
|
|
Password string `mapstructure:"password"`
|
|
DBName string `mapstructure:"dbname"`
|
|
SSLMode string `mapstructure:"sslmode"`
|
|
MaxOpenConns int `mapstructure:"max_open_conns"`
|
|
MaxIdleConns int `mapstructure:"max_idle_conns"`
|
|
}
|
|
|
|
func (d DatabaseConfig) DSN() string {
|
|
return fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=%s",
|
|
d.User, d.Password, d.Host, d.Port, d.DBName, d.SSLMode)
|
|
}
|
|
|
|
type RedisConfig struct {
|
|
Addr string `mapstructure:"addr"`
|
|
DB int `mapstructure:"db"`
|
|
}
|
|
|
|
type RabbitMQConfig struct {
|
|
URL string `mapstructure:"url"`
|
|
MonitorResultExchange string `mapstructure:"monitor_result_exchange"`
|
|
MonitorResultRoutingKey string `mapstructure:"monitor_result_routing_key"`
|
|
MonitorResultQueue string `mapstructure:"monitor_result_queue"`
|
|
MonitorResultDLX string `mapstructure:"monitor_result_dlx"`
|
|
MonitorResultDLQ string `mapstructure:"monitor_result_dlq"`
|
|
MonitorResultDLQRouteKey string `mapstructure:"monitor_result_dlq_routing_key"`
|
|
MonitorProjectionExchange string `mapstructure:"monitor_projection_exchange"`
|
|
MonitorProjectionRoutingKey string `mapstructure:"monitor_projection_routing_key"`
|
|
MonitorProjectionQueue string `mapstructure:"monitor_projection_queue"`
|
|
MonitorProjectionDLX string `mapstructure:"monitor_projection_dlx"`
|
|
MonitorProjectionDLQ string `mapstructure:"monitor_projection_dlq"`
|
|
MonitorProjectionDLQRouteKey string `mapstructure:"monitor_projection_dlq_routing_key"`
|
|
ConsumerPrefetch int `mapstructure:"consumer_prefetch"`
|
|
}
|
|
|
|
type QdrantConfig struct {
|
|
URL string `mapstructure:"url"`
|
|
APIKey string `mapstructure:"api_key"`
|
|
Collection string `mapstructure:"collection"`
|
|
Timeout time.Duration `mapstructure:"timeout"`
|
|
}
|
|
|
|
type ObjectStorageConfig struct {
|
|
Provider string `mapstructure:"provider"`
|
|
Endpoint string `mapstructure:"endpoint"`
|
|
AccessKey string `mapstructure:"access_key"`
|
|
SecretKey string `mapstructure:"secret_key"`
|
|
Bucket string `mapstructure:"bucket"`
|
|
UseSSL bool `mapstructure:"use_ssl"`
|
|
PublicBaseURL string `mapstructure:"public_base_url"`
|
|
Region string `mapstructure:"region"`
|
|
}
|
|
|
|
type JWTConfig struct {
|
|
Secret string `mapstructure:"secret"`
|
|
AccessTTL time.Duration `mapstructure:"access_ttl"`
|
|
RefreshTTL time.Duration `mapstructure:"refresh_ttl"`
|
|
}
|
|
|
|
type LogConfig struct {
|
|
Level string `mapstructure:"level"`
|
|
Format string `mapstructure:"format"`
|
|
}
|
|
|
|
type LLMConfig struct {
|
|
Provider string `mapstructure:"provider"`
|
|
APIKey string `mapstructure:"api_key"`
|
|
BaseURL string `mapstructure:"base_url"`
|
|
Model string `mapstructure:"model"`
|
|
KnowledgeURLModel string `mapstructure:"knowledge_url_model"`
|
|
Timeout time.Duration `mapstructure:"timeout"`
|
|
MaxOutputTokens int64 `mapstructure:"max_output_tokens"`
|
|
Temperature float64 `mapstructure:"temperature"`
|
|
TopP float64 `mapstructure:"top_p"`
|
|
ReasoningEffort string `mapstructure:"reasoning_effort"`
|
|
WebSearchLimit int32 `mapstructure:"web_search_limit"`
|
|
}
|
|
|
|
type RetrievalConfig struct {
|
|
Provider string `mapstructure:"provider"`
|
|
BaseURL string `mapstructure:"base_url"`
|
|
APIKey string `mapstructure:"api_key"`
|
|
EmbeddingModel string `mapstructure:"embedding_model"`
|
|
RerankerModel string `mapstructure:"reranker_model"`
|
|
Timeout time.Duration `mapstructure:"timeout"`
|
|
ChunkSize int `mapstructure:"chunk_size"`
|
|
ChunkOverlap int `mapstructure:"chunk_overlap"`
|
|
EmbeddingBatchSize int `mapstructure:"embedding_batch_size"`
|
|
RecallLimit int `mapstructure:"recall_limit"`
|
|
RerankTopN int `mapstructure:"rerank_top_n"`
|
|
MaxChunksPerDoc int `mapstructure:"max_chunks_per_doc"`
|
|
OverlapTokens int `mapstructure:"overlap_tokens"`
|
|
}
|
|
|
|
type CacheConfig struct {
|
|
Driver string `mapstructure:"driver"` // "redis" or "memory"
|
|
}
|
|
|
|
type GenerationConfig struct {
|
|
QueueSize int `mapstructure:"queue_size"`
|
|
WorkerConcurrency int `mapstructure:"worker_concurrency"`
|
|
StreamEnabled bool `mapstructure:"stream_enabled"`
|
|
ArticleTimeout time.Duration `mapstructure:"article_timeout"`
|
|
}
|
|
|
|
func Load(configPath string) (*Config, error) {
|
|
v := viper.New()
|
|
v.SetConfigFile(configPath)
|
|
v.AutomaticEnv()
|
|
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
|
|
|
|
if err := v.ReadInConfig(); err != nil {
|
|
return nil, fmt.Errorf("read config: %w", err)
|
|
}
|
|
|
|
// Allow local overrides
|
|
v.SetConfigFile(strings.Replace(configPath, ".yaml", ".local.yaml", 1))
|
|
_ = v.MergeInConfig()
|
|
|
|
var cfg Config
|
|
if err := v.Unmarshal(&cfg); err != nil {
|
|
return nil, fmt.Errorf("unmarshal config: %w", err)
|
|
}
|
|
|
|
applyEnvOverrides(&cfg)
|
|
normalizeRabbitMQConfig(&cfg.RabbitMQ)
|
|
|
|
return &cfg, nil
|
|
}
|
|
|
|
func applyEnvOverrides(cfg *Config) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
|
|
if apiKey, ok := lookupNonEmptyEnv("LLM_API_KEY"); ok {
|
|
cfg.LLM.APIKey = apiKey
|
|
}
|
|
if model, ok := lookupNonEmptyEnv("LLM_KNOWLEDGE_URL_MODEL"); ok {
|
|
cfg.LLM.KnowledgeURLModel = model
|
|
}
|
|
if apiKey, ok := lookupNonEmptyEnv("QDRANT_API_KEY"); ok {
|
|
cfg.Qdrant.APIKey = apiKey
|
|
}
|
|
if rabbitmqURL, ok := lookupNonEmptyEnv("RABBITMQ_URL"); ok {
|
|
cfg.RabbitMQ.URL = rabbitmqURL
|
|
}
|
|
if url, ok := lookupNonEmptyEnv("QDRANT_URL"); ok {
|
|
cfg.Qdrant.URL = url
|
|
}
|
|
if collection, ok := lookupNonEmptyEnv("QDRANT_COLLECTION"); ok {
|
|
cfg.Qdrant.Collection = collection
|
|
}
|
|
if endpoint, ok := lookupNonEmptyEnv("OBJECT_STORAGE_ENDPOINT"); ok {
|
|
cfg.ObjectStorage.Endpoint = endpoint
|
|
}
|
|
if accessKey, ok := lookupNonEmptyEnv("OBJECT_STORAGE_ACCESS_KEY"); ok {
|
|
cfg.ObjectStorage.AccessKey = accessKey
|
|
}
|
|
if secretKey, ok := lookupNonEmptyEnv("OBJECT_STORAGE_SECRET_KEY"); ok {
|
|
cfg.ObjectStorage.SecretKey = secretKey
|
|
}
|
|
if bucket, ok := lookupNonEmptyEnv("OBJECT_STORAGE_BUCKET"); ok {
|
|
cfg.ObjectStorage.Bucket = bucket
|
|
}
|
|
if provider, ok := lookupNonEmptyEnv("OBJECT_STORAGE_PROVIDER"); ok {
|
|
cfg.ObjectStorage.Provider = provider
|
|
}
|
|
if publicBaseURL, ok := lookupNonEmptyEnv("OBJECT_STORAGE_PUBLIC_BASE_URL"); ok {
|
|
cfg.ObjectStorage.PublicBaseURL = publicBaseURL
|
|
}
|
|
if region, ok := lookupNonEmptyEnv("OBJECT_STORAGE_REGION"); ok {
|
|
cfg.ObjectStorage.Region = region
|
|
}
|
|
if useSSL, ok := lookupBoolEnv("OBJECT_STORAGE_USE_SSL"); ok {
|
|
cfg.ObjectStorage.UseSSL = useSSL
|
|
}
|
|
if apiKey, ok := lookupFirstNonEmptyEnv("SILICONFLOW_API_KEY", "RETRIEVAL_API_KEY", "EMBEDDING_API_KEY", "RERANKER_API_KEY"); ok {
|
|
cfg.Retrieval.APIKey = apiKey
|
|
}
|
|
if baseURL, ok := lookupFirstNonEmptyEnv("SILICONFLOW_BASE_URL", "RETRIEVAL_BASE_URL"); ok {
|
|
cfg.Retrieval.BaseURL = baseURL
|
|
}
|
|
if model, ok := lookupFirstNonEmptyEnv("SILICONFLOW_EMBEDDING_MODEL", "RETRIEVAL_EMBEDDING_MODEL"); ok {
|
|
cfg.Retrieval.EmbeddingModel = model
|
|
}
|
|
if model, ok := lookupFirstNonEmptyEnv("SILICONFLOW_RERANKER_MODEL", "RETRIEVAL_RERANKER_MODEL"); ok {
|
|
cfg.Retrieval.RerankerModel = model
|
|
}
|
|
}
|
|
|
|
func normalizeRabbitMQConfig(cfg *RabbitMQConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
|
|
if strings.TrimSpace(cfg.MonitorResultExchange) == "" {
|
|
cfg.MonitorResultExchange = "monitor.result"
|
|
}
|
|
if strings.TrimSpace(cfg.MonitorResultRoutingKey) == "" {
|
|
cfg.MonitorResultRoutingKey = "monitor.result.ingest"
|
|
}
|
|
if strings.TrimSpace(cfg.MonitorResultQueue) == "" {
|
|
cfg.MonitorResultQueue = "monitor.result.ingest"
|
|
}
|
|
if strings.TrimSpace(cfg.MonitorResultDLX) == "" {
|
|
cfg.MonitorResultDLX = "monitor.result.dlx"
|
|
}
|
|
if strings.TrimSpace(cfg.MonitorResultDLQ) == "" {
|
|
cfg.MonitorResultDLQ = "monitor.result.ingest.dlq"
|
|
}
|
|
if strings.TrimSpace(cfg.MonitorResultDLQRouteKey) == "" {
|
|
cfg.MonitorResultDLQRouteKey = "monitor.result.ingest.dlq"
|
|
}
|
|
|
|
if strings.TrimSpace(cfg.MonitorProjectionExchange) == "" {
|
|
cfg.MonitorProjectionExchange = "monitor.projection"
|
|
}
|
|
if strings.TrimSpace(cfg.MonitorProjectionRoutingKey) == "" {
|
|
cfg.MonitorProjectionRoutingKey = "monitor.projection.rebuild"
|
|
}
|
|
if strings.TrimSpace(cfg.MonitorProjectionQueue) == "" {
|
|
cfg.MonitorProjectionQueue = "monitor.projection.rebuild"
|
|
}
|
|
if strings.TrimSpace(cfg.MonitorProjectionDLX) == "" {
|
|
cfg.MonitorProjectionDLX = "monitor.projection.dlx"
|
|
}
|
|
if strings.TrimSpace(cfg.MonitorProjectionDLQ) == "" {
|
|
cfg.MonitorProjectionDLQ = "monitor.projection.rebuild.dlq"
|
|
}
|
|
if strings.TrimSpace(cfg.MonitorProjectionDLQRouteKey) == "" {
|
|
cfg.MonitorProjectionDLQRouteKey = "monitor.projection.rebuild.dlq"
|
|
}
|
|
}
|
|
|
|
func lookupFirstNonEmptyEnv(keys ...string) (string, bool) {
|
|
for _, key := range keys {
|
|
if value, ok := lookupNonEmptyEnv(key); ok {
|
|
return value, true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func lookupNonEmptyEnv(key string) (string, bool) {
|
|
value, ok := os.LookupEnv(key)
|
|
if !ok {
|
|
return "", false
|
|
}
|
|
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
return "", false
|
|
}
|
|
|
|
return value, true
|
|
}
|
|
|
|
func lookupBoolEnv(key string) (bool, bool) {
|
|
value, ok := os.LookupEnv(key)
|
|
if !ok {
|
|
return false, false
|
|
}
|
|
|
|
switch strings.ToLower(strings.TrimSpace(value)) {
|
|
case "1", "true", "yes", "on":
|
|
return true, true
|
|
case "0", "false", "no", "off":
|
|
return false, true
|
|
default:
|
|
return false, false
|
|
}
|
|
}
|