feat(knowledge): broaden context resolution and prompt query

- Walk knowledge group children recursively so selecting a parent group
  pulls snippets from its descendants too.
- Filter knowledge items to status='completed' across the prompt and
  active-snippet queries to avoid leaking in-progress documents.
- Dedupe vector candidates by point id and normalized text, then pick
  rerank+vector results with a per-item cap so prompts stay diverse.
- Tag each rendered snippet with a [Kn] reference id for prompt citing.
- Build the rule knowledge query from task params (title/keywords/etc.)
  before falling back to rule prompt content so retrieval matches the
  user's actual ask.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-04-27 11:02:37 +08:00
parent 72adc43e63
commit 63abe403c0
3 changed files with 281 additions and 55 deletions
@@ -3,6 +3,8 @@ package app
import (
"strings"
"testing"
"github.com/geo-platform/tenant-api/internal/shared/retrieval"
)
func TestExtractKnowledgePreciseFactsKeepsAddressAndContactLines(t *testing.T) {
@@ -43,6 +45,7 @@ func TestRenderPromptSectionIncludesPreciseFactsAndSourceURI(t *testing.T) {
for _, expected := range []string{
"以下为必须严格按原文保留的高精度事实",
"- 门店信息 / 合肥门店:地址:合肥市蜀山区望江西路 123 号",
"1. [K1] 门店信息 / 合肥门店",
"来源链接: https://example.com/store",
"这里是一段门店介绍。",
} {
@@ -93,6 +96,64 @@ func TestExtractKnowledgePreciseFactsKeepsBusinessScopeSection(t *testing.T) {
}
}
func TestKnowledgeSnippetsFromSearchPointsSkipsMissingTextAndDedupes(t *testing.T) {
snippets := dedupeKnowledgeSearchCandidates(knowledgeSnippetsFromSearchPoints([]retrieval.SearchPoint{
{
ID: "point-1",
Score: 0.91,
Payload: map[string]any{
"group_id": int64(10),
"group_name": "产品资料",
"item_id": int64(100),
"item_name": "旗舰款",
"text": " 支持全屋定制方案 ",
},
},
{
ID: "point-2",
Payload: map[string]any{
"text": "支持全屋定制方案",
},
},
{
ID: "point-3",
Payload: map[string]any{},
},
}))
if len(snippets) != 1 {
t.Fatalf("deduped snippets length = %d, want 1: %#v", len(snippets), snippets)
}
if snippets[0].Text != "支持全屋定制方案" {
t.Fatalf("snippet text = %q, want trimmed text", snippets[0].Text)
}
}
func TestSelectKnowledgeRerankedSnippetsFillsAndDiversifiesResults(t *testing.T) {
candidates := []KnowledgeSnippet{
{PointID: "p1", KnowledgeItemID: 1, Text: "同一文档片段 1", Score: 0.90},
{PointID: "p2", KnowledgeItemID: 1, Text: "同一文档片段 2", Score: 0.89},
{PointID: "p3", KnowledgeItemID: 1, Text: "同一文档片段 3", Score: 0.88},
{PointID: "p4", KnowledgeItemID: 2, Text: "第二文档片段", Score: 0.80},
{PointID: "p5", KnowledgeItemID: 3, Text: "第三文档片段", Score: 0.70},
}
selected := selectKnowledgeRerankedSnippets(candidates, []retrieval.RerankResult{
{Index: 2, RelevanceScore: 0.99},
{Index: 99, RelevanceScore: 1},
}, 4)
if len(selected) != 4 {
t.Fatalf("selected length = %d, want 4: %#v", len(selected), selected)
}
if selected[0].PointID != "p3" || selected[0].Score != 0.99 {
t.Fatalf("selected[0] = %#v, want reranked p3 with rerank score", selected[0])
}
if selected[1].PointID != "p1" || selected[2].PointID != "p4" || selected[3].PointID != "p5" {
t.Fatalf("selected order = %#v, want reranked result filled with diverse vector candidates", selected)
}
}
func containsExactString(items []string, target string) bool {
for _, item := range items {
if item == target {
+198 -52
View File
@@ -31,6 +31,7 @@ const (
defaultKnowledgeQueueSize = 128
defaultKnowledgeCleanupPoll = 15 * time.Second
defaultKnowledgeChildGroupName = "默认目录"
maxKnowledgeSnippetsPerItem = 2
)
type KnowledgeService struct {
@@ -743,8 +744,8 @@ func (s *KnowledgeService) ResolveContext(
groupIDs []int64,
query string,
) (*KnowledgeContext, error) {
groupIDs = normalizeInt64IDs(groupIDs)
if len(groupIDs) == 0 {
requestedGroupIDs := normalizeInt64IDs(groupIDs)
if len(requestedGroupIDs) == 0 {
return nil, nil
}
if strings.TrimSpace(query) == "" {
@@ -757,19 +758,19 @@ func (s *KnowledgeService) ResolveContext(
return nil, response.ErrServiceUnavailable(50352, "qdrant_unavailable", err.Error())
}
groupNames, err := s.listGroupNames(ctx, tenantID, groupIDs)
searchGroupIDs, groupNames, err := s.resolveKnowledgeSearchScope(ctx, tenantID, requestedGroupIDs)
if err != nil {
return nil, err
}
if len(groupNames) == 0 {
if len(searchGroupIDs) == 0 || len(groupNames) == 0 {
return nil, nil
}
baseContext := &KnowledgeContext{
GroupIDs: groupIDs,
GroupIDs: searchGroupIDs,
GroupNames: groupNames,
Snippets: []KnowledgeSnippet{},
PreciseFacts: s.collectKnowledgeContextPreciseFacts(ctx, tenantID, groupIDs, nil),
PreciseFacts: s.collectKnowledgeContextPreciseFacts(ctx, tenantID, searchGroupIDs, nil),
}
vectors, err := s.provider.Embed(ctx, []string{query})
@@ -783,7 +784,7 @@ func (s *KnowledgeService) ResolveContext(
searchPoints, err := s.store.Search(ctx, retrieval.SearchRequest{
Vector: vectors[0],
TenantID: tenantID,
GroupIDs: groupIDs,
GroupIDs: searchGroupIDs,
Limit: s.recallLimit,
})
if err != nil {
@@ -794,24 +795,7 @@ func (s *KnowledgeService) ResolveContext(
return baseContext, nil
}
candidates := make([]KnowledgeSnippet, 0, len(searchPoints))
documents := make([]string, 0, len(searchPoints))
for _, point := range searchPoints {
text := strings.TrimSpace(anyString(point.Payload["text"]))
if text == "" {
continue
}
candidates = append(candidates, KnowledgeSnippet{
PointID: point.ID,
GroupID: anyInt64(point.Payload["group_id"]),
GroupName: strings.TrimSpace(anyString(point.Payload["group_name"])),
KnowledgeItemID: anyInt64(point.Payload["item_id"]),
ItemName: strings.TrimSpace(anyString(point.Payload["item_name"])),
Text: text,
Score: point.Score,
})
documents = append(documents, text)
}
candidates := dedupeKnowledgeSearchCandidates(knowledgeSnippetsFromSearchPoints(searchPoints))
if len(candidates) == 0 {
s.logKnowledgeResolve(baseContext, tenantID, query, "no_candidate_text", nil, nil)
return baseContext, nil
@@ -826,37 +810,25 @@ func (s *KnowledgeService) ResolveContext(
return baseContext, nil
}
documents = documents[:0]
documents := make([]string, 0, len(candidates))
for _, candidate := range candidates {
documents = append(documents, candidate.Text)
}
reranked, err := s.provider.Rerank(ctx, query, documents, minInt(s.rerankTopN, len(documents)))
if err != nil {
selected := candidates[:minInt(len(candidates), s.rerankTopN)]
selected := selectKnowledgeCandidateSnippets(candidates, s.rerankTopN)
selected = s.enrichKnowledgePromptSnippets(ctx, tenantID, selected)
baseContext.Snippets = selected
baseContext.PreciseFacts = s.collectKnowledgeContextPreciseFacts(ctx, tenantID, groupIDs, selected)
baseContext.PreciseFacts = s.collectKnowledgeContextPreciseFacts(ctx, tenantID, searchGroupIDs, selected)
s.logKnowledgeResolve(baseContext, tenantID, query, "rerank_failed_fallback", candidates, selected)
return baseContext, nil
}
selected := make([]KnowledgeSnippet, 0, len(reranked))
for _, item := range reranked {
if item.Index < 0 || item.Index >= len(candidates) {
continue
}
snippet := candidates[item.Index]
snippet.Score = item.RelevanceScore
if strings.TrimSpace(item.Text) != "" {
snippet.Text = strings.TrimSpace(item.Text)
}
selected = append(selected, snippet)
}
selected := selectKnowledgeRerankedSnippets(candidates, reranked, s.rerankTopN)
selected = s.enrichKnowledgePromptSnippets(ctx, tenantID, selected)
baseContext.Snippets = selected
baseContext.PreciseFacts = s.collectKnowledgeContextPreciseFacts(ctx, tenantID, groupIDs, selected)
baseContext.PreciseFacts = s.collectKnowledgeContextPreciseFacts(ctx, tenantID, searchGroupIDs, selected)
s.logKnowledgeResolve(baseContext, tenantID, query, "resolved", candidates, selected)
return baseContext, nil
}
@@ -948,6 +920,143 @@ func knowledgeLogPreview(value string, maxRunes int) string {
return strings.TrimSpace(string(runes[:maxRunes])) + "..."
}
func knowledgeSnippetsFromSearchPoints(points []retrieval.SearchPoint) []KnowledgeSnippet {
if len(points) == 0 {
return []KnowledgeSnippet{}
}
snippets := make([]KnowledgeSnippet, 0, len(points))
for _, point := range points {
text := strings.TrimSpace(anyString(point.Payload["text"]))
if text == "" {
continue
}
snippets = append(snippets, KnowledgeSnippet{
PointID: strings.TrimSpace(point.ID),
GroupID: anyInt64(point.Payload["group_id"]),
GroupName: strings.TrimSpace(anyString(point.Payload["group_name"])),
KnowledgeItemID: anyInt64(point.Payload["item_id"]),
ItemName: strings.TrimSpace(anyString(point.Payload["item_name"])),
Text: text,
Score: point.Score,
})
}
return snippets
}
func dedupeKnowledgeSearchCandidates(candidates []KnowledgeSnippet) []KnowledgeSnippet {
if len(candidates) == 0 {
return []KnowledgeSnippet{}
}
seenPoints := make(map[string]struct{}, len(candidates))
seenTexts := make(map[string]struct{}, len(candidates))
deduped := make([]KnowledgeSnippet, 0, len(candidates))
for _, candidate := range candidates {
if pointID := strings.TrimSpace(candidate.PointID); pointID != "" {
if _, exists := seenPoints[pointID]; exists {
continue
}
seenPoints[pointID] = struct{}{}
}
textKey := normalizeKnowledgeSnippetTextKey(candidate.Text)
if textKey == "" {
continue
}
if _, exists := seenTexts[textKey]; exists {
continue
}
seenTexts[textKey] = struct{}{}
candidate.Text = strings.TrimSpace(candidate.Text)
deduped = append(deduped, candidate)
}
return deduped
}
func selectKnowledgeCandidateSnippets(candidates []KnowledgeSnippet, limit int) []KnowledgeSnippet {
return selectKnowledgeRerankedSnippets(candidates, nil, limit)
}
func selectKnowledgeRerankedSnippets(
candidates []KnowledgeSnippet,
reranked []retrieval.RerankResult,
limit int,
) []KnowledgeSnippet {
if len(candidates) == 0 {
return []KnowledgeSnippet{}
}
if limit <= 0 || limit > len(candidates) {
limit = len(candidates)
}
selected := make([]KnowledgeSnippet, 0, limit)
seenPoints := make(map[string]struct{}, limit)
seenTexts := make(map[string]struct{}, limit)
itemCounts := make(map[int64]int, limit)
appendSnippet := func(snippet KnowledgeSnippet, enforceItemLimit bool) {
if len(selected) >= limit {
return
}
if pointID := strings.TrimSpace(snippet.PointID); pointID != "" {
if _, exists := seenPoints[pointID]; exists {
return
}
}
textKey := normalizeKnowledgeSnippetTextKey(snippet.Text)
if textKey == "" {
return
}
if _, exists := seenTexts[textKey]; exists {
return
}
if enforceItemLimit && snippet.KnowledgeItemID > 0 && itemCounts[snippet.KnowledgeItemID] >= maxKnowledgeSnippetsPerItem {
return
}
snippet.Text = strings.TrimSpace(snippet.Text)
selected = append(selected, snippet)
if pointID := strings.TrimSpace(snippet.PointID); pointID != "" {
seenPoints[pointID] = struct{}{}
}
seenTexts[textKey] = struct{}{}
if snippet.KnowledgeItemID > 0 {
itemCounts[snippet.KnowledgeItemID]++
}
}
for _, item := range reranked {
if item.Index < 0 || item.Index >= len(candidates) {
continue
}
snippet := candidates[item.Index]
snippet.Score = item.RelevanceScore
if text := strings.TrimSpace(item.Text); text != "" {
snippet.Text = text
}
appendSnippet(snippet, true)
}
for _, candidate := range candidates {
appendSnippet(candidate, true)
}
if len(selected) < limit {
for _, candidate := range candidates {
appendSnippet(candidate, false)
}
}
return selected
}
func normalizeKnowledgeSnippetTextKey(text string) string {
text = strings.TrimSpace(text)
if text == "" {
return ""
}
return strings.ToLower(strings.Join(strings.Fields(text), " "))
}
func (s *KnowledgeService) RenderPromptSection(ctx *KnowledgeContext) string {
if ctx == nil || (len(ctx.Snippets) == 0 && len(ctx.PreciseFacts) == 0) {
return ""
@@ -970,7 +1079,8 @@ func (s *KnowledgeService) RenderPromptSection(ctx *KnowledgeContext) string {
for index, snippet := range ctx.Snippets {
label := knowledgeSnippetPromptLabel(snippet, true, index)
block := fmt.Sprintf("%d. %s\n%s", index+1, label, snippet.Text)
referenceID := knowledgeSnippetReferenceID(index)
block := fmt.Sprintf("%d. [%s] %s\n%s", index+1, referenceID, label, snippet.Text)
if sourceURI := strings.TrimSpace(derefKnowledgeString(snippet.SourceURI)); sourceURI != "" {
block = fmt.Sprintf("%s\n来源链接: %s", block, sourceURI)
}
@@ -1059,6 +1169,7 @@ func (s *KnowledgeService) loadKnowledgePromptItems(
JOIN knowledge_groups kg ON kg.id = ki.group_id AND kg.deleted_at IS NULL
WHERE ki.tenant_id = $1
AND ki.id = ANY($2::bigint[])
AND ki.status = 'completed'
AND ki.deleted_at IS NULL
`, tenantID, itemIDs)
if err != nil {
@@ -1188,6 +1299,7 @@ func (s *KnowledgeService) loadKnowledgePromptItemsByGroupIDs(
FROM knowledge_items ki
JOIN knowledge_groups kg ON kg.id = ki.group_id AND kg.deleted_at IS NULL
WHERE ki.tenant_id = $1
AND ki.status = 'completed'
AND ki.deleted_at IS NULL
AND ki.group_id IN (SELECT id FROM selected_groups)
ORDER BY
@@ -1264,6 +1376,10 @@ func knowledgeSnippetPromptLabel(snippet KnowledgeSnippet, fallbackToDefault boo
return label
}
func knowledgeSnippetReferenceID(index int) string {
return fmt.Sprintf("K%d", index+1)
}
func extractKnowledgePreciseFacts(text string) []string {
normalized := normalizeKnowledgeMarkdown(text)
if normalized == "" {
@@ -2323,6 +2439,7 @@ func (s *KnowledgeService) filterActiveKnowledgeSnippets(
FROM knowledge_items
WHERE tenant_id = $1
AND id = ANY($2::bigint[])
AND status = 'completed'
AND deleted_at IS NULL
`, tenantID, itemIDs)
if err != nil {
@@ -2622,37 +2739,62 @@ func (s *KnowledgeService) listKnowledgeItemsByIDs(ctx context.Context, tenantID
return items, nil
}
func (s *KnowledgeService) listGroupNames(ctx context.Context, tenantID int64, groupIDs []int64) ([]string, error) {
func (s *KnowledgeService) resolveKnowledgeSearchScope(ctx context.Context, tenantID int64, groupIDs []int64) ([]int64, []string, error) {
groupIDs = normalizeInt64IDs(groupIDs)
if len(groupIDs) == 0 {
return []string{}, nil
return []int64{}, []string{}, nil
}
rows, err := s.pool.Query(ctx, `
WITH RECURSIVE selected_groups AS (
SELECT id, name, parent_id, sort_order, created_at
FROM knowledge_groups
WHERE tenant_id = $1
AND id = ANY($2::bigint[])
AND deleted_at IS NULL
UNION
SELECT child.id, child.name, child.parent_id, child.sort_order, child.created_at
FROM knowledge_groups child
JOIN selected_groups parent ON child.parent_id = parent.id
WHERE child.tenant_id = $1
AND child.deleted_at IS NULL
)
SELECT id, name
FROM knowledge_groups
WHERE tenant_id = $1 AND id = ANY($2::bigint[]) AND deleted_at IS NULL
FROM selected_groups
ORDER BY sort_order, created_at, id
`, tenantID, groupIDs)
if err != nil {
return nil, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge groups")
return nil, nil, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge groups")
}
defer rows.Close()
resolvedIDs := make([]int64, 0, len(groupIDs))
names := make([]string, 0, len(groupIDs))
seenIDs := make(map[int64]struct{}, len(groupIDs))
seenNames := make(map[string]struct{}, len(groupIDs))
for rows.Next() {
var (
groupID int64
name string
)
if err := rows.Scan(&groupID, &name); err != nil {
return nil, response.ErrInternal(50050, "knowledge_group_query_failed", err.Error())
return nil, nil, response.ErrInternal(50050, "knowledge_group_query_failed", err.Error())
}
if strings.TrimSpace(name) != "" {
names = append(names, strings.TrimSpace(name))
if groupID > 0 {
if _, exists := seenIDs[groupID]; !exists {
seenIDs[groupID] = struct{}{}
resolvedIDs = append(resolvedIDs, groupID)
}
}
name = strings.TrimSpace(name)
if name != "" {
if _, exists := seenNames[name]; !exists {
seenNames[name] = struct{}{}
names = append(names, name)
}
}
}
return names, nil
return resolvedIDs, names, nil
}
func normalizeInt64IDs(ids []int64) []int64 {
@@ -2677,8 +2819,12 @@ func normalizeInt64IDs(ids []int64) []int64 {
func anyString(value any) string {
switch typed := value.(type) {
case nil:
return ""
case string:
return typed
case []byte:
return string(typed)
default:
return fmt.Sprintf("%v", typed)
}
@@ -657,16 +657,35 @@ func buildPromptRuleKnowledgeQuery(rule *promptRuleGenerationRecord, params map[
if rule.Scene != nil && strings.TrimSpace(*rule.Scene) != "" {
parts = append(parts, strings.TrimSpace(*rule.Scene))
}
if text := strings.TrimSpace(rule.PromptContent); text != "" {
parts = append(parts, text)
}
}
if taskName := strings.TrimSpace(extractString(params, "task_name")); taskName != "" {
parts = append(parts, taskName)
}
for _, key := range []string{
"title",
"topic",
"subject",
"brand_name",
"product_name",
"primary_keyword",
"key_points",
"extra_requirements",
} {
if text := strings.TrimSpace(extractString(params, key)); text != "" {
parts = append(parts, text)
}
}
if keywords := extractStringList(params["keywords"], 16); len(keywords) > 0 {
parts = append(parts, strings.Join(keywords, "\n"))
}
if target := strings.TrimSpace(extractString(params, "target_platform")); target != "" {
parts = append(parts, target)
}
if rule != nil {
if text := strings.TrimSpace(rule.PromptContent); text != "" {
parts = append(parts, text)
}
}
return strings.Join(parts, "\n")
}