From 63abe403c0ebbe52a488a89caa0b9b1d9d01ed06 Mon Sep 17 00:00:00 2001 From: liangxu Date: Mon, 27 Apr 2026 11:02:37 +0800 Subject: [PATCH] feat(knowledge): broaden context resolution and prompt query - Walk knowledge group children recursively so selecting a parent group pulls snippets from its descendants too. - Filter knowledge items to status='completed' across the prompt and active-snippet queries to avoid leaking in-progress documents. - Dedupe vector candidates by point id and normalized text, then pick rerank+vector results with a per-item cap so prompts stay diverse. - Tag each rendered snippet with a [Kn] reference id for prompt citing. - Build the rule knowledge query from task params (title/keywords/etc.) before falling back to rule prompt content so retrieval matches the user's actual ask. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../tenant/app/knowledge_prompt_test.go | 61 +++++ .../internal/tenant/app/knowledge_service.go | 250 ++++++++++++++---- .../tenant/app/prompt_generate_service.go | 25 +- 3 files changed, 281 insertions(+), 55 deletions(-) diff --git a/server/internal/tenant/app/knowledge_prompt_test.go b/server/internal/tenant/app/knowledge_prompt_test.go index acc8ebe..b3702b2 100644 --- a/server/internal/tenant/app/knowledge_prompt_test.go +++ b/server/internal/tenant/app/knowledge_prompt_test.go @@ -3,6 +3,8 @@ package app import ( "strings" "testing" + + "github.com/geo-platform/tenant-api/internal/shared/retrieval" ) func TestExtractKnowledgePreciseFactsKeepsAddressAndContactLines(t *testing.T) { @@ -43,6 +45,7 @@ func TestRenderPromptSectionIncludesPreciseFactsAndSourceURI(t *testing.T) { for _, expected := range []string{ "以下为必须严格按原文保留的高精度事实", "- 门店信息 / 合肥门店：地址：合肥市蜀山区望江西路 123 号", + "1. [K1] 门店信息 / 合肥门店", "来源链接: https://example.com/store", "这里是一段门店介绍。", } { @@ -93,6 +96,64 @@ func TestExtractKnowledgePreciseFactsKeepsBusinessScopeSection(t *testing.T) { } } +func TestKnowledgeSnippetsFromSearchPointsSkipsMissingTextAndDedupes(t *testing.T) { + snippets := dedupeKnowledgeSearchCandidates(knowledgeSnippetsFromSearchPoints([]retrieval.SearchPoint{ + { + ID: "point-1", + Score: 0.91, + Payload: map[string]any{ + "group_id": int64(10), + "group_name": "产品资料", + "item_id": int64(100), + "item_name": "旗舰款", + "text": " 支持全屋定制方案 ", + }, + }, + { + ID: "point-2", + Payload: map[string]any{ + "text": "支持全屋定制方案", + }, + }, + { + ID: "point-3", + Payload: map[string]any{}, + }, + })) + + if len(snippets) != 1 { + t.Fatalf("deduped snippets length = %d, want 1: %#v", len(snippets), snippets) + } + if snippets[0].Text != "支持全屋定制方案" { + t.Fatalf("snippet text = %q, want trimmed text", snippets[0].Text) + } +} + +func TestSelectKnowledgeRerankedSnippetsFillsAndDiversifiesResults(t *testing.T) { + candidates := []KnowledgeSnippet{ + {PointID: "p1", KnowledgeItemID: 1, Text: "同一文档片段 1", Score: 0.90}, + {PointID: "p2", KnowledgeItemID: 1, Text: "同一文档片段 2", Score: 0.89}, + {PointID: "p3", KnowledgeItemID: 1, Text: "同一文档片段 3", Score: 0.88}, + {PointID: "p4", KnowledgeItemID: 2, Text: "第二文档片段", Score: 0.80}, + {PointID: "p5", KnowledgeItemID: 3, Text: "第三文档片段", Score: 0.70}, + } + + selected := selectKnowledgeRerankedSnippets(candidates, []retrieval.RerankResult{ + {Index: 2, RelevanceScore: 0.99}, + {Index: 99, RelevanceScore: 1}, + }, 4) + + if len(selected) != 4 { + t.Fatalf("selected length = %d, want 4: %#v", len(selected), selected) + } + if selected[0].PointID != "p3" || selected[0].Score != 0.99 { + t.Fatalf("selected[0] = %#v, want reranked p3 with rerank score", selected[0]) + } + if selected[1].PointID != "p1" || selected[2].PointID != "p4" || selected[3].PointID != "p5" { + t.Fatalf("selected order = %#v, want reranked result filled with diverse vector candidates", selected) + } +} + func containsExactString(items []string, target string) bool { for _, item := range items { if item == target { diff --git a/server/internal/tenant/app/knowledge_service.go b/server/internal/tenant/app/knowledge_service.go index b6449fb..dc17766 100644 --- a/server/internal/tenant/app/knowledge_service.go +++ b/server/internal/tenant/app/knowledge_service.go @@ -31,6 +31,7 @@ const ( defaultKnowledgeQueueSize = 128 defaultKnowledgeCleanupPoll = 15 * time.Second defaultKnowledgeChildGroupName = "默认目录" + maxKnowledgeSnippetsPerItem = 2 ) type KnowledgeService struct { @@ -743,8 +744,8 @@ func (s *KnowledgeService) ResolveContext( groupIDs []int64, query string, ) (*KnowledgeContext, error) { - groupIDs = normalizeInt64IDs(groupIDs) - if len(groupIDs) == 0 { + requestedGroupIDs := normalizeInt64IDs(groupIDs) + if len(requestedGroupIDs) == 0 { return nil, nil } if strings.TrimSpace(query) == "" { @@ -757,19 +758,19 @@ func (s *KnowledgeService) ResolveContext( return nil, response.ErrServiceUnavailable(50352, "qdrant_unavailable", err.Error()) } - groupNames, err := s.listGroupNames(ctx, tenantID, groupIDs) + searchGroupIDs, groupNames, err := s.resolveKnowledgeSearchScope(ctx, tenantID, requestedGroupIDs) if err != nil { return nil, err } - if len(groupNames) == 0 { + if len(searchGroupIDs) == 0 || len(groupNames) == 0 { return nil, nil } baseContext := &KnowledgeContext{ - GroupIDs: groupIDs, + GroupIDs: searchGroupIDs, GroupNames: groupNames, Snippets: []KnowledgeSnippet{}, - PreciseFacts: s.collectKnowledgeContextPreciseFacts(ctx, tenantID, groupIDs, nil), + PreciseFacts: s.collectKnowledgeContextPreciseFacts(ctx, tenantID, searchGroupIDs, nil), } vectors, err := s.provider.Embed(ctx, []string{query}) @@ -783,7 +784,7 @@ func (s *KnowledgeService) ResolveContext( searchPoints, err := s.store.Search(ctx, retrieval.SearchRequest{ Vector: vectors[0], TenantID: tenantID, - GroupIDs: groupIDs, + GroupIDs: searchGroupIDs, Limit: s.recallLimit, }) if err != nil { @@ -794,24 +795,7 @@ func (s *KnowledgeService) ResolveContext( return baseContext, nil } - candidates := make([]KnowledgeSnippet, 0, len(searchPoints)) - documents := make([]string, 0, len(searchPoints)) - for _, point := range searchPoints { - text := strings.TrimSpace(anyString(point.Payload["text"])) - if text == "" { - continue - } - candidates = append(candidates, KnowledgeSnippet{ - PointID: point.ID, - GroupID: anyInt64(point.Payload["group_id"]), - GroupName: strings.TrimSpace(anyString(point.Payload["group_name"])), - KnowledgeItemID: anyInt64(point.Payload["item_id"]), - ItemName: strings.TrimSpace(anyString(point.Payload["item_name"])), - Text: text, - Score: point.Score, - }) - documents = append(documents, text) - } + candidates := dedupeKnowledgeSearchCandidates(knowledgeSnippetsFromSearchPoints(searchPoints)) if len(candidates) == 0 { s.logKnowledgeResolve(baseContext, tenantID, query, "no_candidate_text", nil, nil) return baseContext, nil @@ -826,37 +810,25 @@ func (s *KnowledgeService) ResolveContext( return baseContext, nil } - documents = documents[:0] + documents := make([]string, 0, len(candidates)) for _, candidate := range candidates { documents = append(documents, candidate.Text) } reranked, err := s.provider.Rerank(ctx, query, documents, minInt(s.rerankTopN, len(documents))) if err != nil { - selected := candidates[:minInt(len(candidates), s.rerankTopN)] + selected := selectKnowledgeCandidateSnippets(candidates, s.rerankTopN) selected = s.enrichKnowledgePromptSnippets(ctx, tenantID, selected) baseContext.Snippets = selected - baseContext.PreciseFacts = s.collectKnowledgeContextPreciseFacts(ctx, tenantID, groupIDs, selected) + baseContext.PreciseFacts = s.collectKnowledgeContextPreciseFacts(ctx, tenantID, searchGroupIDs, selected) s.logKnowledgeResolve(baseContext, tenantID, query, "rerank_failed_fallback", candidates, selected) return baseContext, nil } - selected := make([]KnowledgeSnippet, 0, len(reranked)) - for _, item := range reranked { - if item.Index < 0 || item.Index >= len(candidates) { - continue - } - snippet := candidates[item.Index] - snippet.Score = item.RelevanceScore - if strings.TrimSpace(item.Text) != "" { - snippet.Text = strings.TrimSpace(item.Text) - } - selected = append(selected, snippet) - } - + selected := selectKnowledgeRerankedSnippets(candidates, reranked, s.rerankTopN) selected = s.enrichKnowledgePromptSnippets(ctx, tenantID, selected) baseContext.Snippets = selected - baseContext.PreciseFacts = s.collectKnowledgeContextPreciseFacts(ctx, tenantID, groupIDs, selected) + baseContext.PreciseFacts = s.collectKnowledgeContextPreciseFacts(ctx, tenantID, searchGroupIDs, selected) s.logKnowledgeResolve(baseContext, tenantID, query, "resolved", candidates, selected) return baseContext, nil } @@ -948,6 +920,143 @@ func knowledgeLogPreview(value string, maxRunes int) string { return strings.TrimSpace(string(runes[:maxRunes])) + "..." } +func knowledgeSnippetsFromSearchPoints(points []retrieval.SearchPoint) []KnowledgeSnippet { + if len(points) == 0 { + return []KnowledgeSnippet{} + } + + snippets := make([]KnowledgeSnippet, 0, len(points)) + for _, point := range points { + text := strings.TrimSpace(anyString(point.Payload["text"])) + if text == "" { + continue + } + snippets = append(snippets, KnowledgeSnippet{ + PointID: strings.TrimSpace(point.ID), + GroupID: anyInt64(point.Payload["group_id"]), + GroupName: strings.TrimSpace(anyString(point.Payload["group_name"])), + KnowledgeItemID: anyInt64(point.Payload["item_id"]), + ItemName: strings.TrimSpace(anyString(point.Payload["item_name"])), + Text: text, + Score: point.Score, + }) + } + return snippets +} + +func dedupeKnowledgeSearchCandidates(candidates []KnowledgeSnippet) []KnowledgeSnippet { + if len(candidates) == 0 { + return []KnowledgeSnippet{} + } + + seenPoints := make(map[string]struct{}, len(candidates)) + seenTexts := make(map[string]struct{}, len(candidates)) + deduped := make([]KnowledgeSnippet, 0, len(candidates)) + for _, candidate := range candidates { + if pointID := strings.TrimSpace(candidate.PointID); pointID != "" { + if _, exists := seenPoints[pointID]; exists { + continue + } + seenPoints[pointID] = struct{}{} + } + textKey := normalizeKnowledgeSnippetTextKey(candidate.Text) + if textKey == "" { + continue + } + if _, exists := seenTexts[textKey]; exists { + continue + } + seenTexts[textKey] = struct{}{} + candidate.Text = strings.TrimSpace(candidate.Text) + deduped = append(deduped, candidate) + } + return deduped +} + +func selectKnowledgeCandidateSnippets(candidates []KnowledgeSnippet, limit int) []KnowledgeSnippet { + return selectKnowledgeRerankedSnippets(candidates, nil, limit) +} + +func selectKnowledgeRerankedSnippets( + candidates []KnowledgeSnippet, + reranked []retrieval.RerankResult, + limit int, +) []KnowledgeSnippet { + if len(candidates) == 0 { + return []KnowledgeSnippet{} + } + if limit <= 0 || limit > len(candidates) { + limit = len(candidates) + } + + selected := make([]KnowledgeSnippet, 0, limit) + seenPoints := make(map[string]struct{}, limit) + seenTexts := make(map[string]struct{}, limit) + itemCounts := make(map[int64]int, limit) + + appendSnippet := func(snippet KnowledgeSnippet, enforceItemLimit bool) { + if len(selected) >= limit { + return + } + if pointID := strings.TrimSpace(snippet.PointID); pointID != "" { + if _, exists := seenPoints[pointID]; exists { + return + } + } + textKey := normalizeKnowledgeSnippetTextKey(snippet.Text) + if textKey == "" { + return + } + if _, exists := seenTexts[textKey]; exists { + return + } + if enforceItemLimit && snippet.KnowledgeItemID > 0 && itemCounts[snippet.KnowledgeItemID] >= maxKnowledgeSnippetsPerItem { + return + } + + snippet.Text = strings.TrimSpace(snippet.Text) + selected = append(selected, snippet) + if pointID := strings.TrimSpace(snippet.PointID); pointID != "" { + seenPoints[pointID] = struct{}{} + } + seenTexts[textKey] = struct{}{} + if snippet.KnowledgeItemID > 0 { + itemCounts[snippet.KnowledgeItemID]++ + } + } + + for _, item := range reranked { + if item.Index < 0 || item.Index >= len(candidates) { + continue + } + snippet := candidates[item.Index] + snippet.Score = item.RelevanceScore + if text := strings.TrimSpace(item.Text); text != "" { + snippet.Text = text + } + appendSnippet(snippet, true) + } + + for _, candidate := range candidates { + appendSnippet(candidate, true) + } + if len(selected) < limit { + for _, candidate := range candidates { + appendSnippet(candidate, false) + } + } + + return selected +} + +func normalizeKnowledgeSnippetTextKey(text string) string { + text = strings.TrimSpace(text) + if text == "" { + return "" + } + return strings.ToLower(strings.Join(strings.Fields(text), " ")) +} + func (s *KnowledgeService) RenderPromptSection(ctx *KnowledgeContext) string { if ctx == nil || (len(ctx.Snippets) == 0 && len(ctx.PreciseFacts) == 0) { return "" @@ -970,7 +1079,8 @@ func (s *KnowledgeService) RenderPromptSection(ctx *KnowledgeContext) string { for index, snippet := range ctx.Snippets { label := knowledgeSnippetPromptLabel(snippet, true, index) - block := fmt.Sprintf("%d. %s\n%s", index+1, label, snippet.Text) + referenceID := knowledgeSnippetReferenceID(index) + block := fmt.Sprintf("%d. [%s] %s\n%s", index+1, referenceID, label, snippet.Text) if sourceURI := strings.TrimSpace(derefKnowledgeString(snippet.SourceURI)); sourceURI != "" { block = fmt.Sprintf("%s\n来源链接: %s", block, sourceURI) } @@ -1059,6 +1169,7 @@ func (s *KnowledgeService) loadKnowledgePromptItems( JOIN knowledge_groups kg ON kg.id = ki.group_id AND kg.deleted_at IS NULL WHERE ki.tenant_id = $1 AND ki.id = ANY($2::bigint[]) + AND ki.status = 'completed' AND ki.deleted_at IS NULL `, tenantID, itemIDs) if err != nil { @@ -1188,6 +1299,7 @@ func (s *KnowledgeService) loadKnowledgePromptItemsByGroupIDs( FROM knowledge_items ki JOIN knowledge_groups kg ON kg.id = ki.group_id AND kg.deleted_at IS NULL WHERE ki.tenant_id = $1 + AND ki.status = 'completed' AND ki.deleted_at IS NULL AND ki.group_id IN (SELECT id FROM selected_groups) ORDER BY @@ -1264,6 +1376,10 @@ func knowledgeSnippetPromptLabel(snippet KnowledgeSnippet, fallbackToDefault boo return label } +func knowledgeSnippetReferenceID(index int) string { + return fmt.Sprintf("K%d", index+1) +} + func extractKnowledgePreciseFacts(text string) []string { normalized := normalizeKnowledgeMarkdown(text) if normalized == "" { @@ -2323,6 +2439,7 @@ func (s *KnowledgeService) filterActiveKnowledgeSnippets( FROM knowledge_items WHERE tenant_id = $1 AND id = ANY($2::bigint[]) + AND status = 'completed' AND deleted_at IS NULL `, tenantID, itemIDs) if err != nil { @@ -2622,37 +2739,62 @@ func (s *KnowledgeService) listKnowledgeItemsByIDs(ctx context.Context, tenantID return items, nil } -func (s *KnowledgeService) listGroupNames(ctx context.Context, tenantID int64, groupIDs []int64) ([]string, error) { +func (s *KnowledgeService) resolveKnowledgeSearchScope(ctx context.Context, tenantID int64, groupIDs []int64) ([]int64, []string, error) { groupIDs = normalizeInt64IDs(groupIDs) if len(groupIDs) == 0 { - return []string{}, nil + return []int64{}, []string{}, nil } rows, err := s.pool.Query(ctx, ` + WITH RECURSIVE selected_groups AS ( + SELECT id, name, parent_id, sort_order, created_at + FROM knowledge_groups + WHERE tenant_id = $1 + AND id = ANY($2::bigint[]) + AND deleted_at IS NULL + UNION + SELECT child.id, child.name, child.parent_id, child.sort_order, child.created_at + FROM knowledge_groups child + JOIN selected_groups parent ON child.parent_id = parent.id + WHERE child.tenant_id = $1 + AND child.deleted_at IS NULL + ) SELECT id, name - FROM knowledge_groups - WHERE tenant_id = $1 AND id = ANY($2::bigint[]) AND deleted_at IS NULL + FROM selected_groups ORDER BY sort_order, created_at, id `, tenantID, groupIDs) if err != nil { - return nil, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge groups") + return nil, nil, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge groups") } defer rows.Close() + resolvedIDs := make([]int64, 0, len(groupIDs)) names := make([]string, 0, len(groupIDs)) + seenIDs := make(map[int64]struct{}, len(groupIDs)) + seenNames := make(map[string]struct{}, len(groupIDs)) for rows.Next() { var ( groupID int64 name string ) if err := rows.Scan(&groupID, &name); err != nil { - return nil, response.ErrInternal(50050, "knowledge_group_query_failed", err.Error()) + return nil, nil, response.ErrInternal(50050, "knowledge_group_query_failed", err.Error()) } - if strings.TrimSpace(name) != "" { - names = append(names, strings.TrimSpace(name)) + if groupID > 0 { + if _, exists := seenIDs[groupID]; !exists { + seenIDs[groupID] = struct{}{} + resolvedIDs = append(resolvedIDs, groupID) + } + } + name = strings.TrimSpace(name) + if name != "" { + if _, exists := seenNames[name]; !exists { + seenNames[name] = struct{}{} + names = append(names, name) + } } } - return names, nil + return resolvedIDs, names, nil } func normalizeInt64IDs(ids []int64) []int64 { @@ -2677,8 +2819,12 @@ func normalizeInt64IDs(ids []int64) []int64 { func anyString(value any) string { switch typed := value.(type) { + case nil: + return "" case string: return typed + case []byte: + return string(typed) default: return fmt.Sprintf("%v", typed) } diff --git a/server/internal/tenant/app/prompt_generate_service.go b/server/internal/tenant/app/prompt_generate_service.go index 35dbae6..064ed16 100644 --- a/server/internal/tenant/app/prompt_generate_service.go +++ b/server/internal/tenant/app/prompt_generate_service.go @@ -657,16 +657,35 @@ func buildPromptRuleKnowledgeQuery(rule *promptRuleGenerationRecord, params map[ if rule.Scene != nil && strings.TrimSpace(*rule.Scene) != "" { parts = append(parts, strings.TrimSpace(*rule.Scene)) } - if text := strings.TrimSpace(rule.PromptContent); text != "" { - parts = append(parts, text) - } } if taskName := strings.TrimSpace(extractString(params, "task_name")); taskName != "" { parts = append(parts, taskName) } + for _, key := range []string{ + "title", + "topic", + "subject", + "brand_name", + "product_name", + "primary_keyword", + "key_points", + "extra_requirements", + } { + if text := strings.TrimSpace(extractString(params, key)); text != "" { + parts = append(parts, text) + } + } + if keywords := extractStringList(params["keywords"], 16); len(keywords) > 0 { + parts = append(parts, strings.Join(keywords, "\n")) + } if target := strings.TrimSpace(extractString(params, "target_platform")); target != "" { parts = append(parts, target) } + if rule != nil { + if text := strings.TrimSpace(rule.PromptContent); text != "" { + parts = append(parts, text) + } + } return strings.Join(parts, "\n") }