From 85bb373a8b3dd87b128b4fe5697939f051847f31 Mon Sep 17 00:00:00 2001 From: liangxu Date: Mon, 1 Jun 2026 15:49:14 +0800 Subject: [PATCH] feat(tenant): update retrieval configuration with increased recall limit and rerank top N values --- deploy/config.yaml | 4 +- deploy/k3s/config/config.yaml | 6 +- server/configs/config.yaml | 4 +- .../tenant/app/knowledge_prompt_test.go | 114 +++++++++++++++++- .../internal/tenant/app/knowledge_service.go | 89 ++++++++++++-- server/internal/tenant/app/runtime_config.go | 4 +- 6 files changed, 204 insertions(+), 17 deletions(-) diff --git a/deploy/config.yaml b/deploy/config.yaml index d986497..97abf7d 100644 --- a/deploy/config.yaml +++ b/deploy/config.yaml @@ -237,8 +237,8 @@ retrieval: chunk_size: 900 chunk_overlap: 120 embedding_batch_size: 16 - recall_limit: 12 - rerank_top_n: 6 + recall_limit: 30 + rerank_top_n: 8 max_chunks_per_doc: 6 overlap_tokens: 60 diff --git a/deploy/k3s/config/config.yaml b/deploy/k3s/config/config.yaml index 350465f..d19cd7d 100644 --- a/deploy/k3s/config/config.yaml +++ b/deploy/k3s/config/config.yaml @@ -219,8 +219,8 @@ retrieval: chunk_size: 900 chunk_overlap: 120 embedding_batch_size: 16 - recall_limit: 12 - rerank_top_n: 6 + recall_limit: 30 + rerank_top_n: 8 max_chunks_per_doc: 6 overlap_tokens: 60 @@ -284,4 +284,4 @@ media_supply: published_max_pages: 3 search_options_ttl: 12h upstream_lock_ttl: 2m - upstream_min_interval: 800ms \ No newline at end of file + upstream_min_interval: 800ms diff --git a/server/configs/config.yaml b/server/configs/config.yaml index 6acc28b..92d0a76 100644 --- a/server/configs/config.yaml +++ b/server/configs/config.yaml @@ -223,8 +223,8 @@ retrieval: chunk_size: 900 chunk_overlap: 120 embedding_batch_size: 16 - recall_limit: 12 - rerank_top_n: 6 + recall_limit: 30 + rerank_top_n: 8 max_chunks_per_doc: 6 overlap_tokens: 60 diff --git a/server/internal/tenant/app/knowledge_prompt_test.go b/server/internal/tenant/app/knowledge_prompt_test.go index b3702b2..0e4d7cc 100644 --- a/server/internal/tenant/app/knowledge_prompt_test.go +++ b/server/internal/tenant/app/knowledge_prompt_test.go @@ -112,7 +112,11 @@ func TestKnowledgeSnippetsFromSearchPointsSkipsMissingTextAndDedupes(t *testing. { ID: "point-2", Payload: map[string]any{ - "text": "支持全屋定制方案", + "group_id": int64(10), + "group_name": "产品资料", + "item_id": int64(100), + "item_name": "旗舰款", + "text": "支持全屋定制方案", }, }, { @@ -129,6 +133,84 @@ func TestKnowledgeSnippetsFromSearchPointsSkipsMissingTextAndDedupes(t *testing. } } +func TestKnowledgeSearchTextIncludesMetadataForRetrieval(t *testing.T) { + got := buildKnowledgeSearchText(knowledgeSearchTextInput{ + GroupName: "产品资料", + ItemName: "旗舰款", + SourceType: "website", + SourceURI: "https://example.com/product", + Text: "支持全屋定制方案", + }) + + for _, expected := range []string{ + "目录: 产品资料", + "标题: 旗舰款", + "来源类型: website", + "来源链接: https://example.com/product", + "内容:\n支持全屋定制方案", + } { + if !strings.Contains(got, expected) { + t.Fatalf("buildKnowledgeSearchText() = %q, want %q", got, expected) + } + } +} + +func TestKnowledgeSnippetsFromSearchPointsKeepsSearchTextForRerank(t *testing.T) { + snippets := knowledgeSnippetsFromSearchPoints([]retrieval.SearchPoint{ + { + ID: "point-1", + Score: 0.91, + Payload: map[string]any{ + "group_id": int64(10), + "group_name": "产品资料", + "item_id": int64(100), + "item_name": "旗舰款", + "source_type": "website", + "source_uri": "https://example.com/product", + "text": "支持全屋定制方案", + "search_text": "目录: 产品资料\n标题: 旗舰款\n内容:\n支持全屋定制方案", + }, + }, + }) + + if len(snippets) != 1 { + t.Fatalf("snippets length = %d, want 1", len(snippets)) + } + if snippets[0].Text != "支持全屋定制方案" { + t.Fatalf("Text = %q, want original chunk text", snippets[0].Text) + } + if snippets[0].RerankText() != "目录: 产品资料\n标题: 旗舰款\n内容:\n支持全屋定制方案" { + t.Fatalf("RerankText() = %q, want search text", snippets[0].RerankText()) + } + if snippets[0].SourceURI == nil || *snippets[0].SourceURI != "https://example.com/product" { + t.Fatalf("SourceURI = %#v, want payload source uri", snippets[0].SourceURI) + } +} + +func TestDedupeKnowledgeSearchCandidatesUsesMetadataAwareSearchText(t *testing.T) { + candidates := []KnowledgeSnippet{ + { + PointID: "p1", + KnowledgeItemID: 1, + ItemName: "A 款", + Text: "支持全屋定制方案", + SearchText: "标题: A 款\n内容:\n支持全屋定制方案", + }, + { + PointID: "p2", + KnowledgeItemID: 2, + ItemName: "B 款", + Text: "支持全屋定制方案", + SearchText: "标题: B 款\n内容:\n支持全屋定制方案", + }, + } + + deduped := dedupeKnowledgeSearchCandidates(candidates) + if len(deduped) != 2 { + t.Fatalf("deduped length = %d, want 2: %#v", len(deduped), deduped) + } +} + func TestSelectKnowledgeRerankedSnippetsFillsAndDiversifiesResults(t *testing.T) { candidates := []KnowledgeSnippet{ {PointID: "p1", KnowledgeItemID: 1, Text: "同一文档片段 1", Score: 0.90}, @@ -154,6 +236,36 @@ func TestSelectKnowledgeRerankedSnippetsFillsAndDiversifiesResults(t *testing.T) } } +func TestSelectKnowledgeRerankedSnippetsKeepsOriginalChunkText(t *testing.T) { + candidates := []KnowledgeSnippet{ + { + PointID: "p1", + KnowledgeItemID: 1, + Text: "原始正文片段", + SearchText: "目录: 产品资料\n标题: 旗舰款\n内容:\n原始正文片段", + Score: 0.90, + }, + } + + selected := selectKnowledgeRerankedSnippets(candidates, []retrieval.RerankResult{ + { + Index: 0, + RelevanceScore: 0.99, + Text: "目录: 产品资料\n标题: 旗舰款\n内容:\n原始正文片段", + }, + }, 1) + + if len(selected) != 1 { + t.Fatalf("selected length = %d, want 1", len(selected)) + } + if selected[0].Text != "原始正文片段" { + t.Fatalf("selected text = %q, want original chunk text", selected[0].Text) + } + if selected[0].Score != 0.99 { + t.Fatalf("selected score = %v, want rerank score", selected[0].Score) + } +} + func containsExactString(items []string, target string) bool { for _, item := range items { if item == target { diff --git a/server/internal/tenant/app/knowledge_service.go b/server/internal/tenant/app/knowledge_service.go index 43db001..dd48cb3 100644 --- a/server/internal/tenant/app/knowledge_service.go +++ b/server/internal/tenant/app/knowledge_service.go @@ -145,6 +145,7 @@ type KnowledgeSnippet struct { SourceType string SourceURI *string Text string + SearchText string Score float64 PreciseFacts []string } @@ -900,7 +901,7 @@ func (s *KnowledgeService) ResolveContext( documents := make([]string, 0, len(candidates)) for _, candidate := range candidates { - documents = append(documents, candidate.Text) + documents = append(documents, candidate.RerankText()) } reranked, err := s.provider.Rerank(ctx, query, documents, minInt(runtimeCfg.RerankTopN, len(documents))) @@ -1018,19 +1019,40 @@ func knowledgeSnippetsFromSearchPoints(points []retrieval.SearchPoint) []Knowled if text == "" { continue } + sourceURI := optionalKnowledgeString(point.Payload["source_uri"]) snippets = append(snippets, KnowledgeSnippet{ PointID: strings.TrimSpace(point.ID), GroupID: anyInt64(point.Payload["group_id"]), GroupName: strings.TrimSpace(anyString(point.Payload["group_name"])), KnowledgeItemID: anyInt64(point.Payload["item_id"]), ItemName: strings.TrimSpace(anyString(point.Payload["item_name"])), + SourceType: strings.TrimSpace(anyString(point.Payload["source_type"])), + SourceURI: sourceURI, Text: text, + SearchText: strings.TrimSpace(anyString(point.Payload["search_text"])), Score: point.Score, }) } return snippets } +func (snippet KnowledgeSnippet) RerankText() string { + if text := strings.TrimSpace(snippet.SearchText); text != "" { + return text + } + return buildKnowledgeSearchText(knowledgeSearchTextInput{ + GroupName: snippet.GroupName, + ItemName: snippet.ItemName, + SourceType: snippet.SourceType, + SourceURI: derefKnowledgeString(snippet.SourceURI), + Text: snippet.Text, + }) +} + +func (snippet KnowledgeSnippet) dedupeKey() string { + return normalizeKnowledgeSnippetTextKey(snippet.RerankText()) +} + func dedupeKnowledgeSearchCandidates(candidates []KnowledgeSnippet) []KnowledgeSnippet { if len(candidates) == 0 { return []KnowledgeSnippet{} @@ -1046,7 +1068,7 @@ func dedupeKnowledgeSearchCandidates(candidates []KnowledgeSnippet) []KnowledgeS } seenPoints[pointID] = struct{}{} } - textKey := normalizeKnowledgeSnippetTextKey(candidate.Text) + textKey := candidate.dedupeKey() if textKey == "" { continue } @@ -1090,7 +1112,7 @@ func selectKnowledgeRerankedSnippets( return } } - textKey := normalizeKnowledgeSnippetTextKey(snippet.Text) + textKey := snippet.dedupeKey() if textKey == "" { return } @@ -1118,9 +1140,6 @@ func selectKnowledgeRerankedSnippets( } snippet := candidates[item.Index] snippet.Score = item.RelevanceScore - if text := strings.TrimSpace(item.Text); text != "" { - snippet.Text = text - } appendSnippet(snippet, true) } @@ -1870,7 +1889,18 @@ func (s *KnowledgeService) executeParseJob(ctx context.Context, job knowledgePar } } - embeddings, err := s.embedChunks(ctx, chunks) + searchTexts := make([]string, 0, len(chunks)) + for _, chunk := range chunks { + searchTexts = append(searchTexts, buildKnowledgeSearchText(knowledgeSearchTextInput{ + GroupName: item.GroupName, + ItemName: item.Name, + SourceType: item.SourceType, + SourceURI: derefKnowledgeString(item.SourceURI), + Text: chunk, + })) + } + + embeddings, err := s.embedChunks(ctx, searchTexts) if err != nil { _ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err) return @@ -1910,7 +1940,9 @@ func (s *KnowledgeService) executeParseJob(ctx context.Context, job knowledgePar "item_version": item.ItemVersion, "status": "active", "source_type": item.SourceType, + "source_uri": derefKnowledgeString(item.SourceURI), "text": chunk, + "search_text": searchTexts[index], }, }) metas = append(metas, knowledgeChunkMetaInput{ @@ -2227,6 +2259,49 @@ func derefKnowledgeString(value *string) string { return strings.TrimSpace(*value) } +func optionalKnowledgeString(value any) *string { + text := strings.TrimSpace(anyString(value)) + if text == "" { + return nil + } + return &text +} + +type knowledgeSearchTextInput struct { + GroupName string + ItemName string + SourceType string + SourceURI string + Text string +} + +func buildKnowledgeSearchText(input knowledgeSearchTextInput) string { + text := strings.TrimSpace(input.Text) + if text == "" { + return "" + } + + metadata := make([]string, 0, 4) + if groupName := strings.TrimSpace(input.GroupName); groupName != "" { + metadata = append(metadata, "目录: "+groupName) + } + if itemName := strings.TrimSpace(input.ItemName); itemName != "" { + metadata = append(metadata, "标题: "+itemName) + } + if sourceType := strings.TrimSpace(input.SourceType); sourceType != "" { + metadata = append(metadata, "来源类型: "+sourceType) + } + if sourceURI := strings.TrimSpace(input.SourceURI); sourceURI != "" { + metadata = append(metadata, "来源链接: "+sourceURI) + } + if len(metadata) == 0 { + return text + } + + metadata = append(metadata, "内容:", text) + return strings.Join(metadata, "\n") +} + func (s *KnowledgeService) ensureGroupAccessible(ctx context.Context, tenantID, groupID int64) error { var exists bool err := s.pool.QueryRow(ctx, ` diff --git a/server/internal/tenant/app/runtime_config.go b/server/internal/tenant/app/runtime_config.go index 13f28a8..2394fd8 100644 --- a/server/internal/tenant/app/runtime_config.go +++ b/server/internal/tenant/app/runtime_config.go @@ -77,11 +77,11 @@ func knowledgeRuntimeFromConfig(retrievalCfg config.RetrievalConfig, llmCfg conf } recallLimit := retrievalCfg.RecallLimit if recallLimit <= 0 { - recallLimit = 12 + recallLimit = 30 } rerankTopN := retrievalCfg.RerankTopN if rerankTopN <= 0 { - rerankTopN = 6 + rerankTopN = 8 } model := strings.TrimSpace(llmCfg.KnowledgeURLModel)