feat(tenant): update retrieval configuration with increased recall limit and rerank top N values

2026-06-01 15:49:14 +08:00
parent 76df672133
commit 85bb373a8b
6 changed files with 204 additions and 17 deletions
@@ -112,7 +112,11 @@ func TestKnowledgeSnippetsFromSearchPointsSkipsMissingTextAndDedupes(t *testing.
 		{
 			ID: "point-2",
 			Payload: map[string]any{
-				"text": "支持全屋定制方案",
+				"group_id":   int64(10),
+				"group_name": "产品资料",
+				"item_id":    int64(100),
+				"item_name":  "旗舰款",
+				"text":       "支持全屋定制方案",
 			},
 		},
 		{
@@ -129,6 +133,84 @@ func TestKnowledgeSnippetsFromSearchPointsSkipsMissingTextAndDedupes(t *testing.
 	}
 }

+func TestKnowledgeSearchTextIncludesMetadataForRetrieval(t *testing.T) {
+	got := buildKnowledgeSearchText(knowledgeSearchTextInput{
+		GroupName:  "产品资料",
+		ItemName:   "旗舰款",
+		SourceType: "website",
+		SourceURI:  "https://example.com/product",
+		Text:       "支持全屋定制方案",
+	})
+
+	for _, expected := range []string{
+		"目录: 产品资料",
+		"标题: 旗舰款",
+		"来源类型: website",
+		"来源链接: https://example.com/product",
+		"内容:\n支持全屋定制方案",
+	} {
+		if !strings.Contains(got, expected) {
+			t.Fatalf("buildKnowledgeSearchText() = %q, want %q", got, expected)
+		}
+	}
+}
+
+func TestKnowledgeSnippetsFromSearchPointsKeepsSearchTextForRerank(t *testing.T) {
+	snippets := knowledgeSnippetsFromSearchPoints([]retrieval.SearchPoint{
+		{
+			ID:    "point-1",
+			Score: 0.91,
+			Payload: map[string]any{
+				"group_id":    int64(10),
+				"group_name":  "产品资料",
+				"item_id":     int64(100),
+				"item_name":   "旗舰款",
+				"source_type": "website",
+				"source_uri":  "https://example.com/product",
+				"text":        "支持全屋定制方案",
+				"search_text": "目录: 产品资料\n标题: 旗舰款\n内容:\n支持全屋定制方案",
+			},
+		},
+	})
+
+	if len(snippets) != 1 {
+		t.Fatalf("snippets length = %d, want 1", len(snippets))
+	}
+	if snippets[0].Text != "支持全屋定制方案" {
+		t.Fatalf("Text = %q, want original chunk text", snippets[0].Text)
+	}
+	if snippets[0].RerankText() != "目录: 产品资料\n标题: 旗舰款\n内容:\n支持全屋定制方案" {
+		t.Fatalf("RerankText() = %q, want search text", snippets[0].RerankText())
+	}
+	if snippets[0].SourceURI == nil || *snippets[0].SourceURI != "https://example.com/product" {
+		t.Fatalf("SourceURI = %#v, want payload source uri", snippets[0].SourceURI)
+	}
+}
+
+func TestDedupeKnowledgeSearchCandidatesUsesMetadataAwareSearchText(t *testing.T) {
+	candidates := []KnowledgeSnippet{
+		{
+			PointID:         "p1",
+			KnowledgeItemID: 1,
+			ItemName:        "A 款",
+			Text:            "支持全屋定制方案",
+			SearchText:      "标题: A 款\n内容:\n支持全屋定制方案",
+		},
+		{
+			PointID:         "p2",
+			KnowledgeItemID: 2,
+			ItemName:        "B 款",
+			Text:            "支持全屋定制方案",
+			SearchText:      "标题: B 款\n内容:\n支持全屋定制方案",
+		},
+	}
+
+	deduped := dedupeKnowledgeSearchCandidates(candidates)
+	if len(deduped) != 2 {
+		t.Fatalf("deduped length = %d, want 2: %#v", len(deduped), deduped)
+	}
+}
+
 func TestSelectKnowledgeRerankedSnippetsFillsAndDiversifiesResults(t *testing.T) {
 	candidates := []KnowledgeSnippet{
 		{PointID: "p1", KnowledgeItemID: 1, Text: "同一文档片段 1", Score: 0.90},
@@ -154,6 +236,36 @@ func TestSelectKnowledgeRerankedSnippetsFillsAndDiversifiesResults(t *testing.T)
 	}
 }

+func TestSelectKnowledgeRerankedSnippetsKeepsOriginalChunkText(t *testing.T) {
+	candidates := []KnowledgeSnippet{
+		{
+			PointID:         "p1",
+			KnowledgeItemID: 1,
+			Text:            "原始正文片段",
+			SearchText:      "目录: 产品资料\n标题: 旗舰款\n内容:\n原始正文片段",
+			Score:           0.90,
+		},
+	}
+
+	selected := selectKnowledgeRerankedSnippets(candidates, []retrieval.RerankResult{
+		{
+			Index:          0,
+			RelevanceScore: 0.99,
+			Text:           "目录: 产品资料\n标题: 旗舰款\n内容:\n原始正文片段",
+		},
+	}, 1)
+
+	if len(selected) != 1 {
+		t.Fatalf("selected length = %d, want 1", len(selected))
+	}
+	if selected[0].Text != "原始正文片段" {
+		t.Fatalf("selected text = %q, want original chunk text", selected[0].Text)
+	}
+	if selected[0].Score != 0.99 {
+		t.Fatalf("selected score = %v, want rerank score", selected[0].Score)
+	}
+}
+
 func containsExactString(items []string, target string) bool {
 	for _, item := range items {
 		if item == target {