feat: add knowledge management functionality with CRUD operations and database schema
- Implemented KnowledgeHandler for managing knowledge groups and items, including listing, creating, updating, and deleting operations. - Added database migration scripts to create necessary tables for knowledge management, including knowledge_groups, knowledge_items, knowledge_parse_tasks, and knowledge_chunks_meta. - Introduced prompt_rule_knowledge_groups table to associate prompt rules with knowledge groups.
This commit is contained in:
@@ -10,14 +10,17 @@ import (
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Server ServerConfig `mapstructure:"server"`
|
||||
Database DatabaseConfig `mapstructure:"database"`
|
||||
Redis RedisConfig `mapstructure:"redis"`
|
||||
Cache CacheConfig `mapstructure:"cache"`
|
||||
JWT JWTConfig `mapstructure:"jwt"`
|
||||
Log LogConfig `mapstructure:"log"`
|
||||
LLM LLMConfig `mapstructure:"llm"`
|
||||
Generation GenerationConfig `mapstructure:"generation"`
|
||||
Server ServerConfig `mapstructure:"server"`
|
||||
Database DatabaseConfig `mapstructure:"database"`
|
||||
Redis RedisConfig `mapstructure:"redis"`
|
||||
Qdrant QdrantConfig `mapstructure:"qdrant"`
|
||||
ObjectStorage ObjectStorageConfig `mapstructure:"object_storage"`
|
||||
Cache CacheConfig `mapstructure:"cache"`
|
||||
JWT JWTConfig `mapstructure:"jwt"`
|
||||
Log LogConfig `mapstructure:"log"`
|
||||
LLM LLMConfig `mapstructure:"llm"`
|
||||
Retrieval RetrievalConfig `mapstructure:"retrieval"`
|
||||
Generation GenerationConfig `mapstructure:"generation"`
|
||||
}
|
||||
|
||||
type ServerConfig struct {
|
||||
@@ -46,6 +49,24 @@ type RedisConfig struct {
|
||||
DB int `mapstructure:"db"`
|
||||
}
|
||||
|
||||
type QdrantConfig struct {
|
||||
URL string `mapstructure:"url"`
|
||||
APIKey string `mapstructure:"api_key"`
|
||||
Collection string `mapstructure:"collection"`
|
||||
Timeout time.Duration `mapstructure:"timeout"`
|
||||
}
|
||||
|
||||
type ObjectStorageConfig struct {
|
||||
Provider string `mapstructure:"provider"`
|
||||
Endpoint string `mapstructure:"endpoint"`
|
||||
AccessKey string `mapstructure:"access_key"`
|
||||
SecretKey string `mapstructure:"secret_key"`
|
||||
Bucket string `mapstructure:"bucket"`
|
||||
UseSSL bool `mapstructure:"use_ssl"`
|
||||
PublicBaseURL string `mapstructure:"public_base_url"`
|
||||
Region string `mapstructure:"region"`
|
||||
}
|
||||
|
||||
type JWTConfig struct {
|
||||
Secret string `mapstructure:"secret"`
|
||||
AccessTTL time.Duration `mapstructure:"access_ttl"`
|
||||
@@ -70,6 +91,22 @@ type LLMConfig struct {
|
||||
WebSearchLimit int32 `mapstructure:"web_search_limit"`
|
||||
}
|
||||
|
||||
type RetrievalConfig struct {
|
||||
Provider string `mapstructure:"provider"`
|
||||
BaseURL string `mapstructure:"base_url"`
|
||||
APIKey string `mapstructure:"api_key"`
|
||||
EmbeddingModel string `mapstructure:"embedding_model"`
|
||||
RerankerModel string `mapstructure:"reranker_model"`
|
||||
Timeout time.Duration `mapstructure:"timeout"`
|
||||
ChunkSize int `mapstructure:"chunk_size"`
|
||||
ChunkOverlap int `mapstructure:"chunk_overlap"`
|
||||
EmbeddingBatchSize int `mapstructure:"embedding_batch_size"`
|
||||
RecallLimit int `mapstructure:"recall_limit"`
|
||||
RerankTopN int `mapstructure:"rerank_top_n"`
|
||||
MaxChunksPerDoc int `mapstructure:"max_chunks_per_doc"`
|
||||
OverlapTokens int `mapstructure:"overlap_tokens"`
|
||||
}
|
||||
|
||||
type CacheConfig struct {
|
||||
Driver string `mapstructure:"driver"` // "redis" or "memory"
|
||||
}
|
||||
@@ -113,6 +150,54 @@ func applyEnvOverrides(cfg *Config) {
|
||||
if apiKey, ok := lookupNonEmptyEnv("LLM_API_KEY"); ok {
|
||||
cfg.LLM.APIKey = apiKey
|
||||
}
|
||||
if apiKey, ok := lookupNonEmptyEnv("QDRANT_API_KEY"); ok {
|
||||
cfg.Qdrant.APIKey = apiKey
|
||||
}
|
||||
if url, ok := lookupNonEmptyEnv("QDRANT_URL"); ok {
|
||||
cfg.Qdrant.URL = url
|
||||
}
|
||||
if collection, ok := lookupNonEmptyEnv("QDRANT_COLLECTION"); ok {
|
||||
cfg.Qdrant.Collection = collection
|
||||
}
|
||||
if endpoint, ok := lookupNonEmptyEnv("OBJECT_STORAGE_ENDPOINT"); ok {
|
||||
cfg.ObjectStorage.Endpoint = endpoint
|
||||
}
|
||||
if accessKey, ok := lookupNonEmptyEnv("OBJECT_STORAGE_ACCESS_KEY"); ok {
|
||||
cfg.ObjectStorage.AccessKey = accessKey
|
||||
}
|
||||
if secretKey, ok := lookupNonEmptyEnv("OBJECT_STORAGE_SECRET_KEY"); ok {
|
||||
cfg.ObjectStorage.SecretKey = secretKey
|
||||
}
|
||||
if bucket, ok := lookupNonEmptyEnv("OBJECT_STORAGE_BUCKET"); ok {
|
||||
cfg.ObjectStorage.Bucket = bucket
|
||||
}
|
||||
if provider, ok := lookupNonEmptyEnv("OBJECT_STORAGE_PROVIDER"); ok {
|
||||
cfg.ObjectStorage.Provider = provider
|
||||
}
|
||||
if publicBaseURL, ok := lookupNonEmptyEnv("OBJECT_STORAGE_PUBLIC_BASE_URL"); ok {
|
||||
cfg.ObjectStorage.PublicBaseURL = publicBaseURL
|
||||
}
|
||||
if apiKey, ok := lookupFirstNonEmptyEnv("SILICONFLOW_API_KEY", "RETRIEVAL_API_KEY", "EMBEDDING_API_KEY", "RERANKER_API_KEY"); ok {
|
||||
cfg.Retrieval.APIKey = apiKey
|
||||
}
|
||||
if baseURL, ok := lookupFirstNonEmptyEnv("SILICONFLOW_BASE_URL", "RETRIEVAL_BASE_URL"); ok {
|
||||
cfg.Retrieval.BaseURL = baseURL
|
||||
}
|
||||
if model, ok := lookupFirstNonEmptyEnv("SILICONFLOW_EMBEDDING_MODEL", "RETRIEVAL_EMBEDDING_MODEL"); ok {
|
||||
cfg.Retrieval.EmbeddingModel = model
|
||||
}
|
||||
if model, ok := lookupFirstNonEmptyEnv("SILICONFLOW_RERANKER_MODEL", "RETRIEVAL_RERANKER_MODEL"); ok {
|
||||
cfg.Retrieval.RerankerModel = model
|
||||
}
|
||||
}
|
||||
|
||||
func lookupFirstNonEmptyEnv(keys ...string) (string, bool) {
|
||||
for _, key := range keys {
|
||||
if value, ok := lookupNonEmptyEnv(key); ok {
|
||||
return value, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func lookupNonEmptyEnv(key string) (string, bool) {
|
||||
|
||||
@@ -44,6 +44,51 @@ llm:
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPrefersEnvRetrievalAndQdrantSettings(t *testing.T) {
|
||||
t.Setenv("SILICONFLOW_API_KEY", "siliconflow-env-key")
|
||||
t.Setenv("SILICONFLOW_EMBEDDING_MODEL", "env-embedding")
|
||||
t.Setenv("SILICONFLOW_RERANKER_MODEL", "env-reranker")
|
||||
t.Setenv("QDRANT_API_KEY", "qdrant-env-key")
|
||||
t.Setenv("QDRANT_URL", "qdrant-env:6334")
|
||||
t.Setenv("QDRANT_COLLECTION", "env_collection")
|
||||
|
||||
configPath := writeTestConfig(t, `
|
||||
qdrant:
|
||||
url: qdrant-config:6334
|
||||
api_key: "qdrant-config-key"
|
||||
collection: "config_collection"
|
||||
retrieval:
|
||||
provider: siliconflow
|
||||
api_key: "retrieval-config-key"
|
||||
embedding_model: "config-embedding"
|
||||
reranker_model: "config-reranker"
|
||||
`)
|
||||
|
||||
cfg, err := Load(configPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
|
||||
if cfg.Retrieval.APIKey != "siliconflow-env-key" {
|
||||
t.Fatalf("expected env retrieval api key, got %q", cfg.Retrieval.APIKey)
|
||||
}
|
||||
if cfg.Retrieval.EmbeddingModel != "env-embedding" {
|
||||
t.Fatalf("expected env embedding model, got %q", cfg.Retrieval.EmbeddingModel)
|
||||
}
|
||||
if cfg.Retrieval.RerankerModel != "env-reranker" {
|
||||
t.Fatalf("expected env reranker model, got %q", cfg.Retrieval.RerankerModel)
|
||||
}
|
||||
if cfg.Qdrant.APIKey != "qdrant-env-key" {
|
||||
t.Fatalf("expected env qdrant api key, got %q", cfg.Qdrant.APIKey)
|
||||
}
|
||||
if cfg.Qdrant.URL != "qdrant-env:6334" {
|
||||
t.Fatalf("expected env qdrant url, got %q", cfg.Qdrant.URL)
|
||||
}
|
||||
if cfg.Qdrant.Collection != "env_collection" {
|
||||
t.Fatalf("expected env qdrant collection, got %q", cfg.Qdrant.Collection)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestConfig(t *testing.T, body string) string {
|
||||
t.Helper()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user