feat(tenant): update retrieval configuration with increased recall limit and rerank top N values
This commit is contained in:
@@ -145,6 +145,7 @@ type KnowledgeSnippet struct {
|
||||
SourceType string
|
||||
SourceURI *string
|
||||
Text string
|
||||
SearchText string
|
||||
Score float64
|
||||
PreciseFacts []string
|
||||
}
|
||||
@@ -900,7 +901,7 @@ func (s *KnowledgeService) ResolveContext(
|
||||
|
||||
documents := make([]string, 0, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
documents = append(documents, candidate.Text)
|
||||
documents = append(documents, candidate.RerankText())
|
||||
}
|
||||
|
||||
reranked, err := s.provider.Rerank(ctx, query, documents, minInt(runtimeCfg.RerankTopN, len(documents)))
|
||||
@@ -1018,19 +1019,40 @@ func knowledgeSnippetsFromSearchPoints(points []retrieval.SearchPoint) []Knowled
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
sourceURI := optionalKnowledgeString(point.Payload["source_uri"])
|
||||
snippets = append(snippets, KnowledgeSnippet{
|
||||
PointID: strings.TrimSpace(point.ID),
|
||||
GroupID: anyInt64(point.Payload["group_id"]),
|
||||
GroupName: strings.TrimSpace(anyString(point.Payload["group_name"])),
|
||||
KnowledgeItemID: anyInt64(point.Payload["item_id"]),
|
||||
ItemName: strings.TrimSpace(anyString(point.Payload["item_name"])),
|
||||
SourceType: strings.TrimSpace(anyString(point.Payload["source_type"])),
|
||||
SourceURI: sourceURI,
|
||||
Text: text,
|
||||
SearchText: strings.TrimSpace(anyString(point.Payload["search_text"])),
|
||||
Score: point.Score,
|
||||
})
|
||||
}
|
||||
return snippets
|
||||
}
|
||||
|
||||
func (snippet KnowledgeSnippet) RerankText() string {
|
||||
if text := strings.TrimSpace(snippet.SearchText); text != "" {
|
||||
return text
|
||||
}
|
||||
return buildKnowledgeSearchText(knowledgeSearchTextInput{
|
||||
GroupName: snippet.GroupName,
|
||||
ItemName: snippet.ItemName,
|
||||
SourceType: snippet.SourceType,
|
||||
SourceURI: derefKnowledgeString(snippet.SourceURI),
|
||||
Text: snippet.Text,
|
||||
})
|
||||
}
|
||||
|
||||
func (snippet KnowledgeSnippet) dedupeKey() string {
|
||||
return normalizeKnowledgeSnippetTextKey(snippet.RerankText())
|
||||
}
|
||||
|
||||
func dedupeKnowledgeSearchCandidates(candidates []KnowledgeSnippet) []KnowledgeSnippet {
|
||||
if len(candidates) == 0 {
|
||||
return []KnowledgeSnippet{}
|
||||
@@ -1046,7 +1068,7 @@ func dedupeKnowledgeSearchCandidates(candidates []KnowledgeSnippet) []KnowledgeS
|
||||
}
|
||||
seenPoints[pointID] = struct{}{}
|
||||
}
|
||||
textKey := normalizeKnowledgeSnippetTextKey(candidate.Text)
|
||||
textKey := candidate.dedupeKey()
|
||||
if textKey == "" {
|
||||
continue
|
||||
}
|
||||
@@ -1090,7 +1112,7 @@ func selectKnowledgeRerankedSnippets(
|
||||
return
|
||||
}
|
||||
}
|
||||
textKey := normalizeKnowledgeSnippetTextKey(snippet.Text)
|
||||
textKey := snippet.dedupeKey()
|
||||
if textKey == "" {
|
||||
return
|
||||
}
|
||||
@@ -1118,9 +1140,6 @@ func selectKnowledgeRerankedSnippets(
|
||||
}
|
||||
snippet := candidates[item.Index]
|
||||
snippet.Score = item.RelevanceScore
|
||||
if text := strings.TrimSpace(item.Text); text != "" {
|
||||
snippet.Text = text
|
||||
}
|
||||
appendSnippet(snippet, true)
|
||||
}
|
||||
|
||||
@@ -1870,7 +1889,18 @@ func (s *KnowledgeService) executeParseJob(ctx context.Context, job knowledgePar
|
||||
}
|
||||
}
|
||||
|
||||
embeddings, err := s.embedChunks(ctx, chunks)
|
||||
searchTexts := make([]string, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
searchTexts = append(searchTexts, buildKnowledgeSearchText(knowledgeSearchTextInput{
|
||||
GroupName: item.GroupName,
|
||||
ItemName: item.Name,
|
||||
SourceType: item.SourceType,
|
||||
SourceURI: derefKnowledgeString(item.SourceURI),
|
||||
Text: chunk,
|
||||
}))
|
||||
}
|
||||
|
||||
embeddings, err := s.embedChunks(ctx, searchTexts)
|
||||
if err != nil {
|
||||
_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
|
||||
return
|
||||
@@ -1910,7 +1940,9 @@ func (s *KnowledgeService) executeParseJob(ctx context.Context, job knowledgePar
|
||||
"item_version": item.ItemVersion,
|
||||
"status": "active",
|
||||
"source_type": item.SourceType,
|
||||
"source_uri": derefKnowledgeString(item.SourceURI),
|
||||
"text": chunk,
|
||||
"search_text": searchTexts[index],
|
||||
},
|
||||
})
|
||||
metas = append(metas, knowledgeChunkMetaInput{
|
||||
@@ -2227,6 +2259,49 @@ func derefKnowledgeString(value *string) string {
|
||||
return strings.TrimSpace(*value)
|
||||
}
|
||||
|
||||
func optionalKnowledgeString(value any) *string {
|
||||
text := strings.TrimSpace(anyString(value))
|
||||
if text == "" {
|
||||
return nil
|
||||
}
|
||||
return &text
|
||||
}
|
||||
|
||||
type knowledgeSearchTextInput struct {
|
||||
GroupName string
|
||||
ItemName string
|
||||
SourceType string
|
||||
SourceURI string
|
||||
Text string
|
||||
}
|
||||
|
||||
func buildKnowledgeSearchText(input knowledgeSearchTextInput) string {
|
||||
text := strings.TrimSpace(input.Text)
|
||||
if text == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
metadata := make([]string, 0, 4)
|
||||
if groupName := strings.TrimSpace(input.GroupName); groupName != "" {
|
||||
metadata = append(metadata, "目录: "+groupName)
|
||||
}
|
||||
if itemName := strings.TrimSpace(input.ItemName); itemName != "" {
|
||||
metadata = append(metadata, "标题: "+itemName)
|
||||
}
|
||||
if sourceType := strings.TrimSpace(input.SourceType); sourceType != "" {
|
||||
metadata = append(metadata, "来源类型: "+sourceType)
|
||||
}
|
||||
if sourceURI := strings.TrimSpace(input.SourceURI); sourceURI != "" {
|
||||
metadata = append(metadata, "来源链接: "+sourceURI)
|
||||
}
|
||||
if len(metadata) == 0 {
|
||||
return text
|
||||
}
|
||||
|
||||
metadata = append(metadata, "内容:", text)
|
||||
return strings.Join(metadata, "\n")
|
||||
}
|
||||
|
||||
func (s *KnowledgeService) ensureGroupAccessible(ctx context.Context, tenantID, groupID int64) error {
|
||||
var exists bool
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
|
||||
Reference in New Issue
Block a user