feat: add knowledge management functionality with CRUD operations and database schema
- Implemented KnowledgeHandler for managing knowledge groups and items, including listing, creating, updating, and deleting operations. - Added database migration scripts to create necessary tables for knowledge management, including knowledge_groups, knowledge_items, knowledge_parse_tasks, and knowledge_chunks_meta. - Introduced prompt_rule_knowledge_groups table to associate prompt rules with knowledge groups.
This commit is contained in:
@@ -10,14 +10,17 @@ import (
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Server ServerConfig `mapstructure:"server"`
|
||||
Database DatabaseConfig `mapstructure:"database"`
|
||||
Redis RedisConfig `mapstructure:"redis"`
|
||||
Cache CacheConfig `mapstructure:"cache"`
|
||||
JWT JWTConfig `mapstructure:"jwt"`
|
||||
Log LogConfig `mapstructure:"log"`
|
||||
LLM LLMConfig `mapstructure:"llm"`
|
||||
Generation GenerationConfig `mapstructure:"generation"`
|
||||
Server ServerConfig `mapstructure:"server"`
|
||||
Database DatabaseConfig `mapstructure:"database"`
|
||||
Redis RedisConfig `mapstructure:"redis"`
|
||||
Qdrant QdrantConfig `mapstructure:"qdrant"`
|
||||
ObjectStorage ObjectStorageConfig `mapstructure:"object_storage"`
|
||||
Cache CacheConfig `mapstructure:"cache"`
|
||||
JWT JWTConfig `mapstructure:"jwt"`
|
||||
Log LogConfig `mapstructure:"log"`
|
||||
LLM LLMConfig `mapstructure:"llm"`
|
||||
Retrieval RetrievalConfig `mapstructure:"retrieval"`
|
||||
Generation GenerationConfig `mapstructure:"generation"`
|
||||
}
|
||||
|
||||
type ServerConfig struct {
|
||||
@@ -46,6 +49,24 @@ type RedisConfig struct {
|
||||
DB int `mapstructure:"db"`
|
||||
}
|
||||
|
||||
type QdrantConfig struct {
|
||||
URL string `mapstructure:"url"`
|
||||
APIKey string `mapstructure:"api_key"`
|
||||
Collection string `mapstructure:"collection"`
|
||||
Timeout time.Duration `mapstructure:"timeout"`
|
||||
}
|
||||
|
||||
type ObjectStorageConfig struct {
|
||||
Provider string `mapstructure:"provider"`
|
||||
Endpoint string `mapstructure:"endpoint"`
|
||||
AccessKey string `mapstructure:"access_key"`
|
||||
SecretKey string `mapstructure:"secret_key"`
|
||||
Bucket string `mapstructure:"bucket"`
|
||||
UseSSL bool `mapstructure:"use_ssl"`
|
||||
PublicBaseURL string `mapstructure:"public_base_url"`
|
||||
Region string `mapstructure:"region"`
|
||||
}
|
||||
|
||||
type JWTConfig struct {
|
||||
Secret string `mapstructure:"secret"`
|
||||
AccessTTL time.Duration `mapstructure:"access_ttl"`
|
||||
@@ -70,6 +91,22 @@ type LLMConfig struct {
|
||||
WebSearchLimit int32 `mapstructure:"web_search_limit"`
|
||||
}
|
||||
|
||||
type RetrievalConfig struct {
|
||||
Provider string `mapstructure:"provider"`
|
||||
BaseURL string `mapstructure:"base_url"`
|
||||
APIKey string `mapstructure:"api_key"`
|
||||
EmbeddingModel string `mapstructure:"embedding_model"`
|
||||
RerankerModel string `mapstructure:"reranker_model"`
|
||||
Timeout time.Duration `mapstructure:"timeout"`
|
||||
ChunkSize int `mapstructure:"chunk_size"`
|
||||
ChunkOverlap int `mapstructure:"chunk_overlap"`
|
||||
EmbeddingBatchSize int `mapstructure:"embedding_batch_size"`
|
||||
RecallLimit int `mapstructure:"recall_limit"`
|
||||
RerankTopN int `mapstructure:"rerank_top_n"`
|
||||
MaxChunksPerDoc int `mapstructure:"max_chunks_per_doc"`
|
||||
OverlapTokens int `mapstructure:"overlap_tokens"`
|
||||
}
|
||||
|
||||
type CacheConfig struct {
|
||||
Driver string `mapstructure:"driver"` // "redis" or "memory"
|
||||
}
|
||||
@@ -113,6 +150,54 @@ func applyEnvOverrides(cfg *Config) {
|
||||
if apiKey, ok := lookupNonEmptyEnv("LLM_API_KEY"); ok {
|
||||
cfg.LLM.APIKey = apiKey
|
||||
}
|
||||
if apiKey, ok := lookupNonEmptyEnv("QDRANT_API_KEY"); ok {
|
||||
cfg.Qdrant.APIKey = apiKey
|
||||
}
|
||||
if url, ok := lookupNonEmptyEnv("QDRANT_URL"); ok {
|
||||
cfg.Qdrant.URL = url
|
||||
}
|
||||
if collection, ok := lookupNonEmptyEnv("QDRANT_COLLECTION"); ok {
|
||||
cfg.Qdrant.Collection = collection
|
||||
}
|
||||
if endpoint, ok := lookupNonEmptyEnv("OBJECT_STORAGE_ENDPOINT"); ok {
|
||||
cfg.ObjectStorage.Endpoint = endpoint
|
||||
}
|
||||
if accessKey, ok := lookupNonEmptyEnv("OBJECT_STORAGE_ACCESS_KEY"); ok {
|
||||
cfg.ObjectStorage.AccessKey = accessKey
|
||||
}
|
||||
if secretKey, ok := lookupNonEmptyEnv("OBJECT_STORAGE_SECRET_KEY"); ok {
|
||||
cfg.ObjectStorage.SecretKey = secretKey
|
||||
}
|
||||
if bucket, ok := lookupNonEmptyEnv("OBJECT_STORAGE_BUCKET"); ok {
|
||||
cfg.ObjectStorage.Bucket = bucket
|
||||
}
|
||||
if provider, ok := lookupNonEmptyEnv("OBJECT_STORAGE_PROVIDER"); ok {
|
||||
cfg.ObjectStorage.Provider = provider
|
||||
}
|
||||
if publicBaseURL, ok := lookupNonEmptyEnv("OBJECT_STORAGE_PUBLIC_BASE_URL"); ok {
|
||||
cfg.ObjectStorage.PublicBaseURL = publicBaseURL
|
||||
}
|
||||
if apiKey, ok := lookupFirstNonEmptyEnv("SILICONFLOW_API_KEY", "RETRIEVAL_API_KEY", "EMBEDDING_API_KEY", "RERANKER_API_KEY"); ok {
|
||||
cfg.Retrieval.APIKey = apiKey
|
||||
}
|
||||
if baseURL, ok := lookupFirstNonEmptyEnv("SILICONFLOW_BASE_URL", "RETRIEVAL_BASE_URL"); ok {
|
||||
cfg.Retrieval.BaseURL = baseURL
|
||||
}
|
||||
if model, ok := lookupFirstNonEmptyEnv("SILICONFLOW_EMBEDDING_MODEL", "RETRIEVAL_EMBEDDING_MODEL"); ok {
|
||||
cfg.Retrieval.EmbeddingModel = model
|
||||
}
|
||||
if model, ok := lookupFirstNonEmptyEnv("SILICONFLOW_RERANKER_MODEL", "RETRIEVAL_RERANKER_MODEL"); ok {
|
||||
cfg.Retrieval.RerankerModel = model
|
||||
}
|
||||
}
|
||||
|
||||
func lookupFirstNonEmptyEnv(keys ...string) (string, bool) {
|
||||
for _, key := range keys {
|
||||
if value, ok := lookupNonEmptyEnv(key); ok {
|
||||
return value, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func lookupNonEmptyEnv(key string) (string, bool) {
|
||||
|
||||
@@ -44,6 +44,51 @@ llm:
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPrefersEnvRetrievalAndQdrantSettings(t *testing.T) {
|
||||
t.Setenv("SILICONFLOW_API_KEY", "siliconflow-env-key")
|
||||
t.Setenv("SILICONFLOW_EMBEDDING_MODEL", "env-embedding")
|
||||
t.Setenv("SILICONFLOW_RERANKER_MODEL", "env-reranker")
|
||||
t.Setenv("QDRANT_API_KEY", "qdrant-env-key")
|
||||
t.Setenv("QDRANT_URL", "qdrant-env:6334")
|
||||
t.Setenv("QDRANT_COLLECTION", "env_collection")
|
||||
|
||||
configPath := writeTestConfig(t, `
|
||||
qdrant:
|
||||
url: qdrant-config:6334
|
||||
api_key: "qdrant-config-key"
|
||||
collection: "config_collection"
|
||||
retrieval:
|
||||
provider: siliconflow
|
||||
api_key: "retrieval-config-key"
|
||||
embedding_model: "config-embedding"
|
||||
reranker_model: "config-reranker"
|
||||
`)
|
||||
|
||||
cfg, err := Load(configPath)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error = %v", err)
|
||||
}
|
||||
|
||||
if cfg.Retrieval.APIKey != "siliconflow-env-key" {
|
||||
t.Fatalf("expected env retrieval api key, got %q", cfg.Retrieval.APIKey)
|
||||
}
|
||||
if cfg.Retrieval.EmbeddingModel != "env-embedding" {
|
||||
t.Fatalf("expected env embedding model, got %q", cfg.Retrieval.EmbeddingModel)
|
||||
}
|
||||
if cfg.Retrieval.RerankerModel != "env-reranker" {
|
||||
t.Fatalf("expected env reranker model, got %q", cfg.Retrieval.RerankerModel)
|
||||
}
|
||||
if cfg.Qdrant.APIKey != "qdrant-env-key" {
|
||||
t.Fatalf("expected env qdrant api key, got %q", cfg.Qdrant.APIKey)
|
||||
}
|
||||
if cfg.Qdrant.URL != "qdrant-env:6334" {
|
||||
t.Fatalf("expected env qdrant url, got %q", cfg.Qdrant.URL)
|
||||
}
|
||||
if cfg.Qdrant.Collection != "env_collection" {
|
||||
t.Fatalf("expected env qdrant collection, got %q", cfg.Qdrant.Collection)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestConfig(t *testing.T, body string) string {
|
||||
t.Helper()
|
||||
|
||||
|
||||
@@ -5,19 +5,43 @@ import (
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/response"
|
||||
)
|
||||
|
||||
func Logger(logger *zap.Logger) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
start := time.Now()
|
||||
c.Next()
|
||||
logger.Info("request",
|
||||
|
||||
fields := []zap.Field{
|
||||
zap.String("method", c.Request.Method),
|
||||
zap.String("path", c.Request.URL.Path),
|
||||
zap.Int("status", c.Writer.Status()),
|
||||
zap.Duration("latency", time.Since(start)),
|
||||
zap.String("request_id", RequestIDFromGin(c)),
|
||||
zap.String("client_ip", c.ClientIP()),
|
||||
)
|
||||
}
|
||||
|
||||
if appErr, ok := response.AppErrorFromGin(c); ok {
|
||||
fields = append(fields,
|
||||
zap.Int("app_code", appErr.Code),
|
||||
zap.String("app_message", appErr.Message),
|
||||
)
|
||||
if appErr.Detail != "" {
|
||||
fields = append(fields, zap.String("app_detail", appErr.Detail))
|
||||
}
|
||||
} else if len(c.Errors) > 0 {
|
||||
fields = append(fields, zap.String("error", c.Errors.String()))
|
||||
}
|
||||
|
||||
switch status := c.Writer.Status(); {
|
||||
case status >= 500:
|
||||
logger.Error("request", fields...)
|
||||
case status >= 400:
|
||||
logger.Warn("request", fields...)
|
||||
default:
|
||||
logger.Info("request", fields...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package objectstorage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/config"
|
||||
)
|
||||
|
||||
var ErrNotConfigured = errors.New("object storage is not configured")
|
||||
|
||||
type Client interface {
|
||||
Validate() error
|
||||
PutBytes(ctx context.Context, objectKey string, content []byte, contentType string) error
|
||||
GetBytes(ctx context.Context, objectKey string) ([]byte, error)
|
||||
Delete(ctx context.Context, objectKey string) error
|
||||
}
|
||||
|
||||
func New(cfg config.ObjectStorageConfig, logger *zap.Logger) Client {
|
||||
provider := strings.ToLower(strings.TrimSpace(cfg.Provider))
|
||||
switch provider {
|
||||
case "", "disabled":
|
||||
return disabledClient{reason: "object storage is disabled"}
|
||||
case "minio":
|
||||
return NewMinIOClient(cfg, logger)
|
||||
default:
|
||||
return disabledClient{reason: fmt.Sprintf("unsupported object storage provider %q", cfg.Provider)}
|
||||
}
|
||||
}
|
||||
|
||||
type disabledClient struct {
|
||||
reason string
|
||||
}
|
||||
|
||||
func (c disabledClient) Validate() error {
|
||||
if c.reason == "" {
|
||||
c.reason = "missing object storage configuration"
|
||||
}
|
||||
return fmt.Errorf("%w: %s", ErrNotConfigured, c.reason)
|
||||
}
|
||||
|
||||
func (c disabledClient) PutBytes(context.Context, string, []byte, string) error {
|
||||
return c.Validate()
|
||||
}
|
||||
|
||||
func (c disabledClient) GetBytes(context.Context, string) ([]byte, error) {
|
||||
return nil, c.Validate()
|
||||
}
|
||||
|
||||
func (c disabledClient) Delete(context.Context, string) error {
|
||||
return c.Validate()
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package objectstorage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/config"
|
||||
"github.com/geo-platform/tenant-api/internal/shared/middleware"
|
||||
)
|
||||
|
||||
type minioClient struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
region string
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
func NewMinIOClient(cfg config.ObjectStorageConfig, logger *zap.Logger) Client {
|
||||
endpoint := strings.TrimSpace(cfg.Endpoint)
|
||||
accessKey := strings.TrimSpace(cfg.AccessKey)
|
||||
secretKey := strings.TrimSpace(cfg.SecretKey)
|
||||
bucket := strings.TrimSpace(cfg.Bucket)
|
||||
if endpoint == "" || accessKey == "" || secretKey == "" || bucket == "" {
|
||||
return disabledClient{reason: "missing object_storage minio endpoint/access_key/secret_key/bucket"}
|
||||
}
|
||||
|
||||
client, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: cfg.UseSSL,
|
||||
Region: strings.TrimSpace(cfg.Region),
|
||||
})
|
||||
if err != nil {
|
||||
return disabledClient{reason: fmt.Sprintf("init minio client failed: %v", err)}
|
||||
}
|
||||
|
||||
return &minioClient{
|
||||
client: client,
|
||||
bucket: bucket,
|
||||
region: strings.TrimSpace(cfg.Region),
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *minioClient) Validate() error {
|
||||
if c == nil || c.client == nil {
|
||||
return fmt.Errorf("%w: minio client is not initialized", ErrNotConfigured)
|
||||
}
|
||||
if strings.TrimSpace(c.bucket) == "" {
|
||||
return fmt.Errorf("%w: minio bucket is empty", ErrNotConfigured)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *minioClient) PutBytes(
|
||||
ctx context.Context,
|
||||
objectKey string,
|
||||
content []byte,
|
||||
contentType string,
|
||||
) error {
|
||||
if err := c.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(objectKey) == "" {
|
||||
return fmt.Errorf("object key is required")
|
||||
}
|
||||
|
||||
exists, err := c.client.BucketExists(ctx, c.bucket)
|
||||
if err != nil {
|
||||
c.logError(ctx, "minio bucket exists check failed",
|
||||
zap.String("bucket", c.bucket),
|
||||
zap.String("object_key", objectKey),
|
||||
zap.Error(err),
|
||||
)
|
||||
return fmt.Errorf("check minio bucket: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
if err := c.client.MakeBucket(ctx, c.bucket, minio.MakeBucketOptions{Region: c.region}); err != nil {
|
||||
c.logError(ctx, "minio create bucket failed",
|
||||
zap.String("bucket", c.bucket),
|
||||
zap.String("object_key", objectKey),
|
||||
zap.Error(err),
|
||||
)
|
||||
return fmt.Errorf("create minio bucket: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
_, err = c.client.PutObject(ctx, c.bucket, objectKey, bytes.NewReader(content), int64(len(content)), minio.PutObjectOptions{
|
||||
ContentType: contentType,
|
||||
})
|
||||
if err != nil {
|
||||
c.logError(ctx, "minio put object failed",
|
||||
zap.String("bucket", c.bucket),
|
||||
zap.String("object_key", objectKey),
|
||||
zap.Int("content_size", len(content)),
|
||||
zap.Error(err),
|
||||
)
|
||||
return fmt.Errorf("put minio object: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *minioClient) GetBytes(ctx context.Context, objectKey string) ([]byte, error) {
|
||||
if err := c.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reader, err := c.client.GetObject(ctx, c.bucket, objectKey, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
c.logError(ctx, "minio get object failed",
|
||||
zap.String("bucket", c.bucket),
|
||||
zap.String("object_key", objectKey),
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil, fmt.Errorf("get minio object: %w", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
c.logError(ctx, "minio read object failed",
|
||||
zap.String("bucket", c.bucket),
|
||||
zap.String("object_key", objectKey),
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil, fmt.Errorf("read minio object: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (c *minioClient) Delete(ctx context.Context, objectKey string) error {
|
||||
if err := c.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(objectKey) == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := c.client.RemoveObject(ctx, c.bucket, objectKey, minio.RemoveObjectOptions{})
|
||||
if err != nil {
|
||||
c.logError(ctx, "minio delete object failed",
|
||||
zap.String("bucket", c.bucket),
|
||||
zap.String("object_key", objectKey),
|
||||
zap.Error(err),
|
||||
)
|
||||
return fmt.Errorf("delete minio object: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *minioClient) logError(ctx context.Context, msg string, fields ...zap.Field) {
|
||||
if c == nil || c.logger == nil {
|
||||
return
|
||||
}
|
||||
if requestID := middleware.RequestIDFromContext(ctx); requestID != "" {
|
||||
fields = append(fields, zap.String("request_id", requestID))
|
||||
}
|
||||
c.logger.Error(msg, fields...)
|
||||
}
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
const ginAppErrorKey = "app_error"
|
||||
|
||||
type AppError struct {
|
||||
HTTPStatus int `json:"-"`
|
||||
Code int `json:"code"`
|
||||
@@ -41,6 +43,8 @@ func SuccessWithStatus(c *gin.Context, status int, data interface{}) {
|
||||
|
||||
func Error(c *gin.Context, err error) {
|
||||
appErr := Normalize(err)
|
||||
c.Set(ginAppErrorKey, appErr)
|
||||
_ = c.Error(appErr)
|
||||
c.JSON(appErr.HTTPStatus, gin.H{
|
||||
"code": appErr.Code,
|
||||
"message": appErr.Message,
|
||||
@@ -62,6 +66,18 @@ func Normalize(err error) *AppError {
|
||||
}
|
||||
}
|
||||
|
||||
func AppErrorFromGin(c *gin.Context) (*AppError, bool) {
|
||||
if c == nil {
|
||||
return nil, false
|
||||
}
|
||||
value, ok := c.Get(ginAppErrorKey)
|
||||
if !ok || value == nil {
|
||||
return nil, false
|
||||
}
|
||||
appErr, ok := value.(*AppError)
|
||||
return appErr, ok && appErr != nil
|
||||
}
|
||||
|
||||
func getRequestID(c *gin.Context) string {
|
||||
if rid, ok := c.Get("request_id"); ok {
|
||||
return rid.(string)
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package retrieval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/config"
|
||||
)
|
||||
|
||||
var ErrNotConfigured = errors.New("retrieval provider is not configured")
|
||||
|
||||
type Provider interface {
|
||||
Validate() error
|
||||
Embed(ctx context.Context, inputs []string) ([][]float64, error)
|
||||
Rerank(ctx context.Context, query string, documents []string, topN int) ([]RerankResult, error)
|
||||
}
|
||||
|
||||
type RerankResult struct {
|
||||
Index int
|
||||
RelevanceScore float64
|
||||
Text string
|
||||
}
|
||||
|
||||
func NewProvider(cfg config.RetrievalConfig, logger *zap.Logger) Provider {
|
||||
provider := strings.ToLower(strings.TrimSpace(cfg.Provider))
|
||||
switch provider {
|
||||
case "", "disabled":
|
||||
return disabledProvider{reason: "retrieval provider is disabled"}
|
||||
case "siliconflow":
|
||||
return NewSiliconFlowProvider(cfg, logger)
|
||||
default:
|
||||
return disabledProvider{reason: fmt.Sprintf("unsupported retrieval provider %q", cfg.Provider)}
|
||||
}
|
||||
}
|
||||
|
||||
type disabledProvider struct {
|
||||
reason string
|
||||
}
|
||||
|
||||
func (p disabledProvider) Validate() error {
|
||||
if p.reason == "" {
|
||||
p.reason = "missing retrieval configuration"
|
||||
}
|
||||
return fmt.Errorf("%w: %s", ErrNotConfigured, p.reason)
|
||||
}
|
||||
|
||||
func (p disabledProvider) Embed(context.Context, []string) ([][]float64, error) {
|
||||
return nil, p.Validate()
|
||||
}
|
||||
|
||||
func (p disabledProvider) Rerank(context.Context, string, []string, int) ([]RerankResult, error) {
|
||||
return nil, p.Validate()
|
||||
}
|
||||
@@ -0,0 +1,426 @@
|
||||
package retrieval
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
sdk "github.com/qdrant/go-client/qdrant"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/config"
|
||||
"github.com/geo-platform/tenant-api/internal/shared/middleware"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultQdrantHost = "localhost"
|
||||
defaultQdrantGRPCPort = 6334
|
||||
defaultQdrantCollection = "geo_knowledge_chunks"
|
||||
)
|
||||
|
||||
type VectorStore interface {
|
||||
Validate() error
|
||||
EnsureCollection(ctx context.Context, vectorSize int) error
|
||||
Upsert(ctx context.Context, points []VectorPoint) error
|
||||
Search(ctx context.Context, req SearchRequest) ([]SearchPoint, error)
|
||||
DeletePoints(ctx context.Context, pointIDs []string) error
|
||||
}
|
||||
|
||||
type VectorPoint struct {
|
||||
ID string
|
||||
Vector []float64
|
||||
Payload map[string]any
|
||||
}
|
||||
|
||||
type SearchRequest struct {
|
||||
Vector []float64
|
||||
TenantID int64
|
||||
GroupIDs []int64
|
||||
Limit int
|
||||
}
|
||||
|
||||
type SearchPoint struct {
|
||||
ID string
|
||||
Score float64
|
||||
Payload map[string]any
|
||||
}
|
||||
|
||||
type qdrantStore struct {
|
||||
client *sdk.Client
|
||||
collection string
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
type disabledVectorStore struct {
|
||||
reason string
|
||||
}
|
||||
|
||||
func NewQdrantStore(cfg config.QdrantConfig, logger *zap.Logger) VectorStore {
|
||||
host, port, useTLS, err := parseQdrantEndpoint(cfg.URL)
|
||||
if err != nil {
|
||||
return disabledVectorStore{reason: fmt.Sprintf("invalid qdrant.url: %v", err)}
|
||||
}
|
||||
|
||||
collection := strings.TrimSpace(cfg.Collection)
|
||||
if collection == "" {
|
||||
collection = defaultQdrantCollection
|
||||
}
|
||||
|
||||
client, err := sdk.NewClient(&sdk.Config{
|
||||
Host: host,
|
||||
Port: port,
|
||||
APIKey: strings.TrimSpace(cfg.APIKey),
|
||||
UseTLS: useTLS,
|
||||
SkipCompatibilityCheck: true,
|
||||
})
|
||||
if err != nil {
|
||||
return disabledVectorStore{reason: fmt.Sprintf("init qdrant client failed: %v", err)}
|
||||
}
|
||||
|
||||
return &qdrantStore{
|
||||
client: client,
|
||||
collection: collection,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *qdrantStore) Validate() error {
|
||||
if s == nil || s.client == nil {
|
||||
return fmt.Errorf("%w: qdrant client is not initialized", ErrNotConfigured)
|
||||
}
|
||||
if strings.TrimSpace(s.collection) == "" {
|
||||
return fmt.Errorf("%w: qdrant collection is empty", ErrNotConfigured)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *qdrantStore) EnsureCollection(ctx context.Context, vectorSize int) error {
|
||||
if err := s.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if vectorSize <= 0 {
|
||||
return fmt.Errorf("invalid qdrant vector size %d", vectorSize)
|
||||
}
|
||||
|
||||
exists, err := s.client.CollectionExists(ctx, s.collection)
|
||||
if err != nil {
|
||||
s.logError(ctx, "qdrant collection exists check failed",
|
||||
zap.String("collection", s.collection),
|
||||
zap.Error(err),
|
||||
)
|
||||
return fmt.Errorf("query qdrant collection: %w", err)
|
||||
}
|
||||
if exists {
|
||||
return nil
|
||||
}
|
||||
|
||||
err = s.client.CreateCollection(ctx, &sdk.CreateCollection{
|
||||
CollectionName: s.collection,
|
||||
VectorsConfig: sdk.NewVectorsConfig(&sdk.VectorParams{
|
||||
Size: uint64(vectorSize),
|
||||
Distance: sdk.Distance_Cosine,
|
||||
}),
|
||||
})
|
||||
if err != nil {
|
||||
s.logError(ctx, "qdrant create collection failed",
|
||||
zap.String("collection", s.collection),
|
||||
zap.Int("vector_size", vectorSize),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *qdrantStore) Upsert(ctx context.Context, points []VectorPoint) error {
|
||||
if err := s.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(points) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
converted := make([]*sdk.PointStruct, 0, len(points))
|
||||
for _, point := range points {
|
||||
if strings.TrimSpace(point.ID) == "" || len(point.Vector) == 0 {
|
||||
continue
|
||||
}
|
||||
converted = append(converted, &sdk.PointStruct{
|
||||
Id: sdk.NewID(point.ID),
|
||||
Vectors: sdk.NewVectors(float64SliceToFloat32(point.Vector)...),
|
||||
Payload: sdk.NewValueMap(point.Payload),
|
||||
})
|
||||
}
|
||||
if len(converted) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
wait := true
|
||||
_, err := s.client.Upsert(ctx, &sdk.UpsertPoints{
|
||||
CollectionName: s.collection,
|
||||
Wait: &wait,
|
||||
Points: converted,
|
||||
})
|
||||
if err != nil {
|
||||
s.logError(ctx, "qdrant upsert failed",
|
||||
zap.String("collection", s.collection),
|
||||
zap.Int("point_count", len(converted)),
|
||||
zap.Error(err),
|
||||
)
|
||||
return fmt.Errorf("upsert qdrant points: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *qdrantStore) Search(ctx context.Context, req SearchRequest) ([]SearchPoint, error) {
|
||||
if err := s.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(req.Vector) == 0 || req.TenantID <= 0 {
|
||||
return []SearchPoint{}, nil
|
||||
}
|
||||
limit := req.Limit
|
||||
if limit <= 0 {
|
||||
limit = 10
|
||||
}
|
||||
limitValue := uint64(limit)
|
||||
|
||||
filter := &sdk.Filter{
|
||||
Must: []*sdk.Condition{
|
||||
sdk.NewMatchInt("tenant_id", req.TenantID),
|
||||
sdk.NewMatchKeyword("status", "active"),
|
||||
},
|
||||
}
|
||||
groupIDs := normalizeGroupIDs(req.GroupIDs)
|
||||
if len(groupIDs) > 0 {
|
||||
filter.Should = []*sdk.Condition{
|
||||
sdk.NewMatchInts("group_id", groupIDs...),
|
||||
}
|
||||
}
|
||||
|
||||
points, err := s.client.Query(ctx, &sdk.QueryPoints{
|
||||
CollectionName: s.collection,
|
||||
Query: sdk.NewQueryDense(float64SliceToFloat32(req.Vector)),
|
||||
Filter: filter,
|
||||
Limit: &limitValue,
|
||||
WithPayload: sdk.NewWithPayload(true),
|
||||
})
|
||||
if err != nil {
|
||||
s.logError(ctx, "qdrant search failed",
|
||||
zap.String("collection", s.collection),
|
||||
zap.Int64("tenant_id", req.TenantID),
|
||||
zap.Int("group_count", len(groupIDs)),
|
||||
zap.Int("limit", limit),
|
||||
zap.Error(err),
|
||||
)
|
||||
return nil, fmt.Errorf("query qdrant points: %w", err)
|
||||
}
|
||||
|
||||
results := make([]SearchPoint, 0, len(points))
|
||||
for _, point := range points {
|
||||
if point == nil {
|
||||
continue
|
||||
}
|
||||
results = append(results, SearchPoint{
|
||||
ID: pointIDToString(point.GetId()),
|
||||
Score: float64(point.GetScore()),
|
||||
Payload: valueMapToAny(point.GetPayload()),
|
||||
})
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s *qdrantStore) DeletePoints(ctx context.Context, pointIDs []string) error {
|
||||
if err := s.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(pointIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
ids := make([]*sdk.PointId, 0, len(pointIDs))
|
||||
for _, pointID := range pointIDs {
|
||||
pointID = strings.TrimSpace(pointID)
|
||||
if pointID == "" {
|
||||
continue
|
||||
}
|
||||
ids = append(ids, sdk.NewID(pointID))
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
wait := true
|
||||
_, err := s.client.Delete(ctx, &sdk.DeletePoints{
|
||||
CollectionName: s.collection,
|
||||
Wait: &wait,
|
||||
Points: sdk.NewPointsSelector(ids...),
|
||||
})
|
||||
if err != nil {
|
||||
s.logError(ctx, "qdrant delete points failed",
|
||||
zap.String("collection", s.collection),
|
||||
zap.Int("point_count", len(ids)),
|
||||
zap.Error(err),
|
||||
)
|
||||
return fmt.Errorf("delete qdrant points: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *qdrantStore) logError(ctx context.Context, msg string, fields ...zap.Field) {
|
||||
if s == nil || s.logger == nil {
|
||||
return
|
||||
}
|
||||
if requestID := middleware.RequestIDFromContext(ctx); requestID != "" {
|
||||
fields = append(fields, zap.String("request_id", requestID))
|
||||
}
|
||||
s.logger.Error(msg, fields...)
|
||||
}
|
||||
|
||||
func parseQdrantEndpoint(raw string) (string, int, bool, error) {
|
||||
value := strings.TrimSpace(raw)
|
||||
if value == "" {
|
||||
return defaultQdrantHost, defaultQdrantGRPCPort, false, nil
|
||||
}
|
||||
|
||||
if strings.Contains(value, "://") {
|
||||
parsed, err := url.Parse(value)
|
||||
if err != nil {
|
||||
return "", 0, false, err
|
||||
}
|
||||
port := defaultQdrantGRPCPort
|
||||
if parsed.Port() != "" {
|
||||
port, err = strconv.Atoi(parsed.Port())
|
||||
if err != nil {
|
||||
return "", 0, false, err
|
||||
}
|
||||
}
|
||||
host := parsed.Hostname()
|
||||
if host == "" {
|
||||
host = defaultQdrantHost
|
||||
}
|
||||
return host, port, parsed.Scheme == "https", nil
|
||||
}
|
||||
|
||||
host, portText, err := net.SplitHostPort(value)
|
||||
if err == nil {
|
||||
port, convErr := strconv.Atoi(portText)
|
||||
if convErr != nil {
|
||||
return "", 0, false, convErr
|
||||
}
|
||||
return host, port, false, nil
|
||||
}
|
||||
|
||||
if strings.Contains(err.Error(), "missing port in address") {
|
||||
return value, defaultQdrantGRPCPort, false, nil
|
||||
}
|
||||
|
||||
return "", 0, false, err
|
||||
}
|
||||
|
||||
func float64SliceToFloat32(values []float64) []float32 {
|
||||
result := make([]float32, 0, len(values))
|
||||
for _, value := range values {
|
||||
result = append(result, float32(value))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func pointIDToString(id *sdk.PointId) string {
|
||||
if id == nil {
|
||||
return ""
|
||||
}
|
||||
if uuid := strings.TrimSpace(id.GetUuid()); uuid != "" {
|
||||
return uuid
|
||||
}
|
||||
if num := id.GetNum(); num > 0 {
|
||||
return strconv.FormatUint(num, 10)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func valueMapToAny(values map[string]*sdk.Value) map[string]any {
|
||||
if len(values) == 0 {
|
||||
return map[string]any{}
|
||||
}
|
||||
result := make(map[string]any, len(values))
|
||||
for key, value := range values {
|
||||
result[key] = valueToAny(value)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func valueToAny(value *sdk.Value) any {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
switch value.GetKind().(type) {
|
||||
case *sdk.Value_StringValue:
|
||||
return value.GetStringValue()
|
||||
case *sdk.Value_IntegerValue:
|
||||
return value.GetIntegerValue()
|
||||
case *sdk.Value_DoubleValue:
|
||||
return value.GetDoubleValue()
|
||||
case *sdk.Value_BoolValue:
|
||||
return value.GetBoolValue()
|
||||
case *sdk.Value_ListValue:
|
||||
list := value.GetListValue()
|
||||
if list == nil {
|
||||
return []any{}
|
||||
}
|
||||
items := make([]any, 0, len(list.GetValues()))
|
||||
for _, item := range list.GetValues() {
|
||||
items = append(items, valueToAny(item))
|
||||
}
|
||||
return items
|
||||
case *sdk.Value_StructValue:
|
||||
structValue := value.GetStructValue()
|
||||
if structValue == nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
return valueMapToAny(structValue.GetFields())
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeGroupIDs(ids []int64) []int64 {
|
||||
result := make([]int64, 0, len(ids))
|
||||
seen := make(map[int64]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
if id <= 0 {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
result = append(result, id)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (s disabledVectorStore) Validate() error {
|
||||
if s.reason == "" {
|
||||
s.reason = "missing qdrant configuration"
|
||||
}
|
||||
return fmt.Errorf("%w: %s", ErrNotConfigured, s.reason)
|
||||
}
|
||||
|
||||
func (s disabledVectorStore) EnsureCollection(context.Context, int) error {
|
||||
return s.Validate()
|
||||
}
|
||||
|
||||
func (s disabledVectorStore) Upsert(context.Context, []VectorPoint) error {
|
||||
return s.Validate()
|
||||
}
|
||||
|
||||
func (s disabledVectorStore) Search(context.Context, SearchRequest) ([]SearchPoint, error) {
|
||||
return nil, s.Validate()
|
||||
}
|
||||
|
||||
func (s disabledVectorStore) DeletePoints(context.Context, []string) error {
|
||||
return s.Validate()
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
package retrieval
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/config"
|
||||
"github.com/geo-platform/tenant-api/internal/shared/middleware"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultSiliconFlowBaseURL = "https://api.siliconflow.cn/v1"
|
||||
defaultSiliconFlowEmbeddingModel = "BAAI/bge-m3"
|
||||
defaultSiliconFlowRerankerModel = "BAAI/bge-reranker-v2-m3"
|
||||
defaultSiliconFlowTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
type siliconFlowProvider struct {
|
||||
client *http.Client
|
||||
logger *zap.Logger
|
||||
baseURL string
|
||||
apiKey string
|
||||
embeddingModel string
|
||||
rerankerModel string
|
||||
maxChunksPerDoc int
|
||||
overlapTokens int
|
||||
}
|
||||
|
||||
func NewSiliconFlowProvider(cfg config.RetrievalConfig, logger *zap.Logger) Provider {
|
||||
if strings.TrimSpace(cfg.APIKey) == "" {
|
||||
return disabledProvider{reason: "missing retrieval.api_key or SiliconFlow api key env (SILICONFLOW_API_KEY / RETRIEVAL_API_KEY)"}
|
||||
}
|
||||
|
||||
baseURL := strings.TrimSpace(cfg.BaseURL)
|
||||
if baseURL == "" {
|
||||
baseURL = defaultSiliconFlowBaseURL
|
||||
}
|
||||
|
||||
timeout := cfg.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = defaultSiliconFlowTimeout
|
||||
}
|
||||
|
||||
embeddingModel := strings.TrimSpace(cfg.EmbeddingModel)
|
||||
if embeddingModel == "" {
|
||||
embeddingModel = defaultSiliconFlowEmbeddingModel
|
||||
}
|
||||
|
||||
rerankerModel := strings.TrimSpace(cfg.RerankerModel)
|
||||
if rerankerModel == "" {
|
||||
rerankerModel = defaultSiliconFlowRerankerModel
|
||||
}
|
||||
|
||||
maxChunksPerDoc := cfg.MaxChunksPerDoc
|
||||
if maxChunksPerDoc <= 0 {
|
||||
maxChunksPerDoc = 6
|
||||
}
|
||||
|
||||
overlapTokens := cfg.OverlapTokens
|
||||
if overlapTokens <= 0 {
|
||||
overlapTokens = 60
|
||||
}
|
||||
|
||||
return &siliconFlowProvider{
|
||||
client: &http.Client{Timeout: timeout},
|
||||
logger: logger,
|
||||
baseURL: strings.TrimRight(baseURL, "/"),
|
||||
apiKey: strings.TrimSpace(cfg.APIKey),
|
||||
embeddingModel: embeddingModel,
|
||||
rerankerModel: rerankerModel,
|
||||
maxChunksPerDoc: maxChunksPerDoc,
|
||||
overlapTokens: overlapTokens,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *siliconFlowProvider) Validate() error {
|
||||
if p == nil || p.client == nil {
|
||||
return fmt.Errorf("%w: siliconflow client is not initialized", ErrNotConfigured)
|
||||
}
|
||||
if strings.TrimSpace(p.apiKey) == "" {
|
||||
return fmt.Errorf("%w: siliconflow api key is empty", ErrNotConfigured)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *siliconFlowProvider) Embed(ctx context.Context, inputs []string) ([][]float64, error) {
|
||||
if err := p.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(inputs) == 0 {
|
||||
return [][]float64{}, nil
|
||||
}
|
||||
|
||||
type embeddingRequest struct {
|
||||
Model string `json:"model"`
|
||||
Input []string `json:"input"`
|
||||
}
|
||||
type embeddingItem struct {
|
||||
Embedding []float64 `json:"embedding"`
|
||||
Index int `json:"index"`
|
||||
}
|
||||
type embeddingResponse struct {
|
||||
Data []embeddingItem `json:"data"`
|
||||
Error *struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
payload := embeddingRequest{
|
||||
Model: p.embeddingModel,
|
||||
Input: inputs,
|
||||
}
|
||||
|
||||
var resp embeddingResponse
|
||||
if err := p.doJSON(ctx, http.MethodPost, "/embeddings", payload, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.Error != nil && strings.TrimSpace(resp.Error.Message) != "" {
|
||||
return nil, errors.New(resp.Error.Message)
|
||||
}
|
||||
|
||||
vectors := make([][]float64, len(inputs))
|
||||
for _, item := range resp.Data {
|
||||
if item.Index < 0 || item.Index >= len(vectors) {
|
||||
continue
|
||||
}
|
||||
vectors[item.Index] = item.Embedding
|
||||
}
|
||||
for index, vector := range vectors {
|
||||
if len(vector) == 0 {
|
||||
return nil, fmt.Errorf("embedding vector missing at index %d", index)
|
||||
}
|
||||
}
|
||||
|
||||
return vectors, nil
|
||||
}
|
||||
|
||||
func (p *siliconFlowProvider) Rerank(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
documents []string,
|
||||
topN int,
|
||||
) ([]RerankResult, error) {
|
||||
if err := p.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(query) == "" || len(documents) == 0 {
|
||||
return []RerankResult{}, nil
|
||||
}
|
||||
if topN <= 0 || topN > len(documents) {
|
||||
topN = len(documents)
|
||||
}
|
||||
|
||||
type rerankRequest struct {
|
||||
Model string `json:"model"`
|
||||
Query string `json:"query"`
|
||||
Documents []string `json:"documents"`
|
||||
TopN int `json:"top_n,omitempty"`
|
||||
ReturnDocuments bool `json:"return_documents"`
|
||||
MaxChunksPerDoc int `json:"max_chunks_per_doc,omitempty"`
|
||||
OverlapTokens int `json:"overlap_tokens,omitempty"`
|
||||
}
|
||||
type rerankResponse struct {
|
||||
Results []struct {
|
||||
Index int `json:"index"`
|
||||
RelevanceScore float64 `json:"relevance_score"`
|
||||
Document struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"document"`
|
||||
} `json:"results"`
|
||||
Error *struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
payload := rerankRequest{
|
||||
Model: p.rerankerModel,
|
||||
Query: strings.TrimSpace(query),
|
||||
Documents: documents,
|
||||
TopN: topN,
|
||||
ReturnDocuments: true,
|
||||
MaxChunksPerDoc: p.maxChunksPerDoc,
|
||||
OverlapTokens: p.overlapTokens,
|
||||
}
|
||||
|
||||
var resp rerankResponse
|
||||
if err := p.doJSON(ctx, http.MethodPost, "/rerank", payload, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.Error != nil && strings.TrimSpace(resp.Error.Message) != "" {
|
||||
return nil, errors.New(resp.Error.Message)
|
||||
}
|
||||
|
||||
results := make([]RerankResult, 0, len(resp.Results))
|
||||
for _, item := range resp.Results {
|
||||
text := item.Document.Text
|
||||
if text == "" && item.Index >= 0 && item.Index < len(documents) {
|
||||
text = documents[item.Index]
|
||||
}
|
||||
results = append(results, RerankResult{
|
||||
Index: item.Index,
|
||||
RelevanceScore: item.RelevanceScore,
|
||||
Text: text,
|
||||
})
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (p *siliconFlowProvider) doJSON(
|
||||
ctx context.Context,
|
||||
method, path string,
|
||||
payload any,
|
||||
target any,
|
||||
) error {
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
p.logError(ctx, "siliconflow marshal request failed",
|
||||
zap.String("method", method),
|
||||
zap.String("path", path),
|
||||
zap.Error(err),
|
||||
)
|
||||
return fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, p.baseURL+path, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
p.logError(ctx, "siliconflow create request failed",
|
||||
zap.String("method", method),
|
||||
zap.String("url", p.baseURL+path),
|
||||
zap.Error(err),
|
||||
)
|
||||
return fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+p.apiKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
p.logError(ctx, "siliconflow request failed",
|
||||
zap.String("method", method),
|
||||
zap.String("url", req.URL.String()),
|
||||
zap.Error(err),
|
||||
)
|
||||
return fmt.Errorf("request siliconflow: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
p.logError(ctx, "siliconflow read response failed",
|
||||
zap.String("method", method),
|
||||
zap.String("url", req.URL.String()),
|
||||
zap.Int("status", resp.StatusCode),
|
||||
zap.Error(err),
|
||||
)
|
||||
return fmt.Errorf("read response body: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode >= http.StatusBadRequest {
|
||||
p.logError(ctx, "siliconflow non-success response",
|
||||
zap.String("method", method),
|
||||
zap.String("url", req.URL.String()),
|
||||
zap.Int("status", resp.StatusCode),
|
||||
zap.String("response_body", truncateForLog(string(data), 512)),
|
||||
)
|
||||
return fmt.Errorf("siliconflow request failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(data)))
|
||||
}
|
||||
|
||||
if target == nil || len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(data, target); err != nil {
|
||||
p.logError(ctx, "siliconflow decode response failed",
|
||||
zap.String("method", method),
|
||||
zap.String("url", req.URL.String()),
|
||||
zap.Int("status", resp.StatusCode),
|
||||
zap.String("response_body", truncateForLog(string(data), 512)),
|
||||
zap.Error(err),
|
||||
)
|
||||
return fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *siliconFlowProvider) logError(ctx context.Context, msg string, fields ...zap.Field) {
|
||||
if p == nil || p.logger == nil {
|
||||
return
|
||||
}
|
||||
if requestID := middleware.RequestIDFromContext(ctx); requestID != "" {
|
||||
fields = append(fields, zap.String("request_id", requestID))
|
||||
}
|
||||
p.logger.Error(msg, fields...)
|
||||
}
|
||||
|
||||
func truncateForLog(value string, limit int) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if limit <= 0 || len(value) <= limit {
|
||||
return value
|
||||
}
|
||||
return value[:limit] + "...(truncated)"
|
||||
}
|
||||
Reference in New Issue
Block a user