feat: Enhance Kol Generation Service with web search and knowledge group support
- Added `EnableWebSearch` and `KnowledgeGroupIDs` fields to `KolGenerationSubmitRequest`. - Updated `Submit` method in `KolGenerationService` to handle new request fields. - Integrated web search and knowledge group validation based on card configuration. - Introduced caching mechanisms in `KolPackageService`, `KolPromptService`, and `KolSubscriptionAdminService` to improve performance. - Implemented knowledge context resolution in `KolGenerationWorker` to enrich prompts with relevant knowledge snippets. - Added utility functions for handling card configuration in both backend and frontend. - Created tests for knowledge extraction and rendering to ensure accuracy and reliability.
This commit is contained in:
@@ -137,9 +137,10 @@ type KnowledgeURLItemRequest struct {
|
||||
}
|
||||
|
||||
type KnowledgeContext struct {
|
||||
GroupIDs []int64
|
||||
GroupNames []string
|
||||
Snippets []KnowledgeSnippet
|
||||
GroupIDs []int64
|
||||
GroupNames []string
|
||||
Snippets []KnowledgeSnippet
|
||||
PreciseFacts []string
|
||||
}
|
||||
|
||||
type KnowledgeSnippet struct {
|
||||
@@ -148,8 +149,11 @@ type KnowledgeSnippet struct {
|
||||
GroupName string
|
||||
KnowledgeItemID int64
|
||||
ItemName string
|
||||
SourceType string
|
||||
SourceURI *string
|
||||
Text string
|
||||
Score float64
|
||||
PreciseFacts []string
|
||||
}
|
||||
|
||||
func NewKnowledgeService(
|
||||
@@ -761,6 +765,13 @@ func (s *KnowledgeService) ResolveContext(
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
baseContext := &KnowledgeContext{
|
||||
GroupIDs: groupIDs,
|
||||
GroupNames: groupNames,
|
||||
Snippets: []KnowledgeSnippet{},
|
||||
PreciseFacts: s.collectKnowledgeContextPreciseFacts(ctx, tenantID, groupIDs, nil),
|
||||
}
|
||||
|
||||
vectors, err := s.provider.Embed(ctx, []string{query})
|
||||
if err != nil {
|
||||
return nil, response.ErrServiceUnavailable(50353, "embedding_failed", err.Error())
|
||||
@@ -779,7 +790,7 @@ func (s *KnowledgeService) ResolveContext(
|
||||
return nil, response.ErrServiceUnavailable(50354, "qdrant_search_failed", err.Error())
|
||||
}
|
||||
if len(searchPoints) == 0 {
|
||||
return &KnowledgeContext{GroupIDs: groupIDs, GroupNames: groupNames, Snippets: []KnowledgeSnippet{}}, nil
|
||||
return baseContext, nil
|
||||
}
|
||||
|
||||
candidates := make([]KnowledgeSnippet, 0, len(searchPoints))
|
||||
@@ -801,7 +812,7 @@ func (s *KnowledgeService) ResolveContext(
|
||||
documents = append(documents, text)
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
return &KnowledgeContext{GroupIDs: groupIDs, GroupNames: groupNames, Snippets: []KnowledgeSnippet{}}, nil
|
||||
return baseContext, nil
|
||||
}
|
||||
|
||||
candidates, err = s.filterActiveKnowledgeSnippets(ctx, tenantID, candidates)
|
||||
@@ -809,7 +820,7 @@ func (s *KnowledgeService) ResolveContext(
|
||||
return nil, err
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
return &KnowledgeContext{GroupIDs: groupIDs, GroupNames: groupNames, Snippets: []KnowledgeSnippet{}}, nil
|
||||
return baseContext, nil
|
||||
}
|
||||
|
||||
documents = documents[:0]
|
||||
@@ -820,7 +831,10 @@ func (s *KnowledgeService) ResolveContext(
|
||||
reranked, err := s.provider.Rerank(ctx, query, documents, minInt(s.rerankTopN, len(documents)))
|
||||
if err != nil {
|
||||
selected := candidates[:minInt(len(candidates), s.rerankTopN)]
|
||||
return &KnowledgeContext{GroupIDs: groupIDs, GroupNames: groupNames, Snippets: selected}, nil
|
||||
selected = s.enrichKnowledgePromptSnippets(ctx, tenantID, selected)
|
||||
baseContext.Snippets = selected
|
||||
baseContext.PreciseFacts = s.collectKnowledgeContextPreciseFacts(ctx, tenantID, groupIDs, selected)
|
||||
return baseContext, nil
|
||||
}
|
||||
|
||||
selected := make([]KnowledgeSnippet, 0, len(reranked))
|
||||
@@ -836,33 +850,532 @@ func (s *KnowledgeService) ResolveContext(
|
||||
selected = append(selected, snippet)
|
||||
}
|
||||
|
||||
return &KnowledgeContext{
|
||||
GroupIDs: groupIDs,
|
||||
GroupNames: groupNames,
|
||||
Snippets: selected,
|
||||
}, nil
|
||||
selected = s.enrichKnowledgePromptSnippets(ctx, tenantID, selected)
|
||||
baseContext.Snippets = selected
|
||||
baseContext.PreciseFacts = s.collectKnowledgeContextPreciseFacts(ctx, tenantID, groupIDs, selected)
|
||||
return baseContext, nil
|
||||
}
|
||||
|
||||
func (s *KnowledgeService) RenderPromptSection(ctx *KnowledgeContext) string {
|
||||
if ctx == nil || len(ctx.Snippets) == 0 {
|
||||
if ctx == nil || (len(ctx.Snippets) == 0 && len(ctx.PreciseFacts) == 0) {
|
||||
return ""
|
||||
}
|
||||
|
||||
lines := prompts.KnowledgePromptIntroLines()
|
||||
preciseFacts := ctx.PreciseFacts
|
||||
if len(preciseFacts) == 0 {
|
||||
preciseFacts = collectKnowledgePromptPreciseFacts(ctx.Snippets)
|
||||
}
|
||||
if len(preciseFacts) > 0 {
|
||||
lines = append(lines,
|
||||
"以下为必须严格按原文保留的高精度事实；如果正文需要引用，必须逐字一致，不得改写、翻译、推测或补全：",
|
||||
)
|
||||
for _, fact := range preciseFacts {
|
||||
lines = append(lines, "- "+fact)
|
||||
}
|
||||
lines = append(lines, "")
|
||||
}
|
||||
|
||||
for index, snippet := range ctx.Snippets {
|
||||
label := strings.TrimSpace(snippet.ItemName)
|
||||
if label == "" {
|
||||
label = prompts.KnowledgeSnippetLabel(index)
|
||||
label := knowledgeSnippetPromptLabel(snippet, true, index)
|
||||
block := fmt.Sprintf("%d. %s\n%s", index+1, label, snippet.Text)
|
||||
if sourceURI := strings.TrimSpace(derefKnowledgeString(snippet.SourceURI)); sourceURI != "" {
|
||||
block = fmt.Sprintf("%s\n来源链接: %s", block, sourceURI)
|
||||
}
|
||||
groupName := strings.TrimSpace(snippet.GroupName)
|
||||
if groupName != "" {
|
||||
label = fmt.Sprintf("%s / %s", groupName, label)
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("%d. %s\n%s", index+1, label, snippet.Text))
|
||||
lines = append(lines, block)
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
type knowledgePromptItemRecord struct {
|
||||
ID int64
|
||||
GroupID int64
|
||||
GroupName string
|
||||
Name string
|
||||
SourceType string
|
||||
SourceURI *string
|
||||
ContentText *string
|
||||
MarkdownContent *string
|
||||
}
|
||||
|
||||
func (s *KnowledgeService) enrichKnowledgePromptSnippets(
|
||||
ctx context.Context,
|
||||
tenantID int64,
|
||||
snippets []KnowledgeSnippet,
|
||||
) []KnowledgeSnippet {
|
||||
if len(snippets) == 0 {
|
||||
return []KnowledgeSnippet{}
|
||||
}
|
||||
|
||||
itemIDs := make([]int64, 0, len(snippets))
|
||||
for _, snippet := range snippets {
|
||||
if snippet.KnowledgeItemID > 0 {
|
||||
itemIDs = append(itemIDs, snippet.KnowledgeItemID)
|
||||
}
|
||||
}
|
||||
itemIDs = normalizeInt64IDs(itemIDs)
|
||||
if len(itemIDs) == 0 {
|
||||
return snippets
|
||||
}
|
||||
|
||||
items, err := s.loadKnowledgePromptItems(ctx, tenantID, itemIDs)
|
||||
if err != nil {
|
||||
if s.logger != nil {
|
||||
s.logger.Warn("load knowledge prompt items failed", zap.Error(err))
|
||||
}
|
||||
return snippets
|
||||
}
|
||||
|
||||
itemMap := make(map[int64]knowledgePromptItemRecord, len(items))
|
||||
for _, item := range items {
|
||||
itemMap[item.ID] = item
|
||||
}
|
||||
|
||||
enriched := make([]KnowledgeSnippet, 0, len(snippets))
|
||||
for _, snippet := range snippets {
|
||||
item, ok := itemMap[snippet.KnowledgeItemID]
|
||||
if ok {
|
||||
snippet.GroupID = item.GroupID
|
||||
snippet.GroupName = item.GroupName
|
||||
snippet.ItemName = item.Name
|
||||
snippet.SourceType = item.SourceType
|
||||
snippet.SourceURI = item.SourceURI
|
||||
sourceText := derefKnowledgeString(item.MarkdownContent)
|
||||
if strings.TrimSpace(sourceText) == "" {
|
||||
sourceText = derefKnowledgeString(item.ContentText)
|
||||
}
|
||||
snippet.PreciseFacts = extractKnowledgePreciseFacts(sourceText)
|
||||
}
|
||||
enriched = append(enriched, snippet)
|
||||
}
|
||||
|
||||
return enriched
|
||||
}
|
||||
|
||||
func (s *KnowledgeService) loadKnowledgePromptItems(
|
||||
ctx context.Context,
|
||||
tenantID int64,
|
||||
itemIDs []int64,
|
||||
) ([]knowledgePromptItemRecord, error) {
|
||||
if len(itemIDs) == 0 {
|
||||
return []knowledgePromptItemRecord{}, nil
|
||||
}
|
||||
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT ki.id, ki.group_id, kg.name, ki.name, ki.source_type, ki.source_uri, ki.content_text, ki.markdown_content
|
||||
FROM knowledge_items ki
|
||||
JOIN knowledge_groups kg ON kg.id = ki.group_id AND kg.deleted_at IS NULL
|
||||
WHERE ki.tenant_id = $1
|
||||
AND ki.id = ANY($2::bigint[])
|
||||
AND ki.deleted_at IS NULL
|
||||
`, tenantID, itemIDs)
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(50051, "knowledge_items_query_failed", "failed to load knowledge prompt items")
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
items := make([]knowledgePromptItemRecord, 0, len(itemIDs))
|
||||
for rows.Next() {
|
||||
var item knowledgePromptItemRecord
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.GroupID,
|
||||
&item.GroupName,
|
||||
&item.Name,
|
||||
&item.SourceType,
|
||||
&item.SourceURI,
|
||||
&item.ContentText,
|
||||
&item.MarkdownContent,
|
||||
); err != nil {
|
||||
return nil, response.ErrInternal(50051, "knowledge_items_scan_failed", err.Error())
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (s *KnowledgeService) collectKnowledgeContextPreciseFacts(
|
||||
ctx context.Context,
|
||||
tenantID int64,
|
||||
groupIDs []int64,
|
||||
snippets []KnowledgeSnippet,
|
||||
) []string {
|
||||
groupIDs = normalizeInt64IDs(groupIDs)
|
||||
if len(groupIDs) == 0 {
|
||||
return collectKnowledgePromptPreciseFacts(snippets)
|
||||
}
|
||||
|
||||
matchedItemIDs := make([]int64, 0, len(snippets))
|
||||
for _, snippet := range snippets {
|
||||
if snippet.KnowledgeItemID > 0 {
|
||||
matchedItemIDs = append(matchedItemIDs, snippet.KnowledgeItemID)
|
||||
}
|
||||
}
|
||||
matchedItemIDs = normalizeInt64IDs(matchedItemIDs)
|
||||
|
||||
items, err := s.loadKnowledgePromptItemsByGroupIDs(ctx, tenantID, groupIDs, matchedItemIDs, 24)
|
||||
if err != nil {
|
||||
if s.logger != nil {
|
||||
s.logger.Warn("collect knowledge context precise facts failed", zap.Error(err))
|
||||
}
|
||||
return collectKnowledgePromptPreciseFacts(snippets)
|
||||
}
|
||||
|
||||
facts := make([]string, 0, 24)
|
||||
seen := make(map[string]struct{}, 24)
|
||||
for _, item := range items {
|
||||
sourceText := derefKnowledgeString(item.MarkdownContent)
|
||||
if strings.TrimSpace(sourceText) == "" {
|
||||
sourceText = derefKnowledgeString(item.ContentText)
|
||||
}
|
||||
|
||||
label := strings.TrimSpace(item.Name)
|
||||
if strings.TrimSpace(item.GroupName) != "" && label != "" {
|
||||
label = fmt.Sprintf("%s / %s", strings.TrimSpace(item.GroupName), label)
|
||||
} else if strings.TrimSpace(item.GroupName) != "" {
|
||||
label = strings.TrimSpace(item.GroupName)
|
||||
}
|
||||
|
||||
for _, fact := range extractKnowledgePreciseFacts(sourceText) {
|
||||
value := strings.TrimSpace(fact)
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
entry := value
|
||||
if label != "" {
|
||||
entry = fmt.Sprintf("%s：%s", label, value)
|
||||
}
|
||||
if _, exists := seen[entry]; exists {
|
||||
continue
|
||||
}
|
||||
seen[entry] = struct{}{}
|
||||
facts = append(facts, entry)
|
||||
if len(facts) >= 24 {
|
||||
return facts
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(facts) == 0 {
|
||||
return collectKnowledgePromptPreciseFacts(snippets)
|
||||
}
|
||||
return facts
|
||||
}
|
||||
|
||||
func (s *KnowledgeService) loadKnowledgePromptItemsByGroupIDs(
|
||||
ctx context.Context,
|
||||
tenantID int64,
|
||||
groupIDs []int64,
|
||||
prioritizedItemIDs []int64,
|
||||
limit int,
|
||||
) ([]knowledgePromptItemRecord, error) {
|
||||
groupIDs = normalizeInt64IDs(groupIDs)
|
||||
prioritizedItemIDs = normalizeInt64IDs(prioritizedItemIDs)
|
||||
if len(groupIDs) == 0 {
|
||||
return []knowledgePromptItemRecord{}, nil
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 24
|
||||
}
|
||||
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
WITH RECURSIVE selected_groups AS (
|
||||
SELECT id
|
||||
FROM knowledge_groups
|
||||
WHERE tenant_id = $1
|
||||
AND id = ANY($2::bigint[])
|
||||
AND deleted_at IS NULL
|
||||
UNION ALL
|
||||
SELECT child.id
|
||||
FROM knowledge_groups child
|
||||
JOIN selected_groups parent ON child.parent_id = parent.id
|
||||
WHERE child.tenant_id = $1
|
||||
AND child.deleted_at IS NULL
|
||||
)
|
||||
SELECT ki.id, ki.group_id, kg.name, ki.name, ki.source_type, ki.source_uri, ki.content_text, ki.markdown_content
|
||||
FROM knowledge_items ki
|
||||
JOIN knowledge_groups kg ON kg.id = ki.group_id AND kg.deleted_at IS NULL
|
||||
WHERE ki.tenant_id = $1
|
||||
AND ki.deleted_at IS NULL
|
||||
AND ki.group_id IN (SELECT id FROM selected_groups)
|
||||
ORDER BY
|
||||
CASE
|
||||
WHEN ki.id = ANY($3::bigint[]) THEN 0
|
||||
ELSE 1
|
||||
END,
|
||||
ki.updated_at DESC,
|
||||
ki.id DESC
|
||||
LIMIT $4
|
||||
`, tenantID, groupIDs, prioritizedItemIDs, limit)
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(50051, "knowledge_items_query_failed", "failed to load knowledge prompt items by groups")
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
items := make([]knowledgePromptItemRecord, 0, limit)
|
||||
for rows.Next() {
|
||||
var item knowledgePromptItemRecord
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.GroupID,
|
||||
&item.GroupName,
|
||||
&item.Name,
|
||||
&item.SourceType,
|
||||
&item.SourceURI,
|
||||
&item.ContentText,
|
||||
&item.MarkdownContent,
|
||||
); err != nil {
|
||||
return nil, response.ErrInternal(50051, "knowledge_items_scan_failed", err.Error())
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func collectKnowledgePromptPreciseFacts(snippets []KnowledgeSnippet) []string {
|
||||
if len(snippets) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
facts := make([]string, 0, len(snippets)*2)
|
||||
seen := make(map[string]struct{}, len(snippets)*2)
|
||||
for index, snippet := range snippets {
|
||||
label := knowledgeSnippetPromptLabel(snippet, false, index)
|
||||
for _, fact := range snippet.PreciseFacts {
|
||||
value := strings.TrimSpace(fact)
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
entry := fmt.Sprintf("%s：%s", label, value)
|
||||
if _, exists := seen[entry]; exists {
|
||||
continue
|
||||
}
|
||||
seen[entry] = struct{}{}
|
||||
facts = append(facts, entry)
|
||||
}
|
||||
}
|
||||
return facts
|
||||
}
|
||||
|
||||
func knowledgeSnippetPromptLabel(snippet KnowledgeSnippet, fallbackToDefault bool, index int) string {
|
||||
label := strings.TrimSpace(snippet.ItemName)
|
||||
if label == "" && fallbackToDefault {
|
||||
label = prompts.KnowledgeSnippetLabel(index)
|
||||
}
|
||||
groupName := strings.TrimSpace(snippet.GroupName)
|
||||
if groupName != "" && label != "" {
|
||||
return fmt.Sprintf("%s / %s", groupName, label)
|
||||
}
|
||||
if groupName != "" {
|
||||
return groupName
|
||||
}
|
||||
return label
|
||||
}
|
||||
|
||||
func extractKnowledgePreciseFacts(text string) []string {
|
||||
normalized := normalizeKnowledgeMarkdown(text)
|
||||
if normalized == "" {
|
||||
normalized = normalizeKnowledgeText(text)
|
||||
}
|
||||
if normalized == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
lines := strings.Split(normalized, "\n")
|
||||
facts := make([]string, 0, 12)
|
||||
seen := make(map[string]struct{}, 12)
|
||||
|
||||
appendFact := func(value string) {
|
||||
value = normalizeKnowledgeInlineText(trimKnowledgeFactLine(value))
|
||||
if value == "" {
|
||||
return
|
||||
}
|
||||
if _, exists := seen[value]; exists {
|
||||
return
|
||||
}
|
||||
seen[value] = struct{}{}
|
||||
facts = append(facts, value)
|
||||
}
|
||||
|
||||
for index := 0; index < len(lines); index++ {
|
||||
line := trimKnowledgeFactLine(lines[index])
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
|
||||
if knowledgePreciseFactLabelOnly(line) {
|
||||
sectionContent := collectKnowledgePreciseFactSection(lines, index, line)
|
||||
if sectionContent != "" {
|
||||
appendFact(strings.TrimRight(strings.TrimSpace(line), ":：") + "：" + sectionContent)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if knowledgeLineContainsPreciseFact(line) {
|
||||
appendFact(line)
|
||||
}
|
||||
}
|
||||
|
||||
if len(facts) > 12 {
|
||||
return facts[:12]
|
||||
}
|
||||
return facts
|
||||
}
|
||||
|
||||
func collectKnowledgePreciseFactSection(lines []string, startIndex int, label string) string {
|
||||
maxLines := knowledgePreciseFactSectionLineLimit(label)
|
||||
if maxLines <= 0 {
|
||||
maxLines = 1
|
||||
}
|
||||
|
||||
items := make([]string, 0, maxLines)
|
||||
for index := startIndex + 1; index < len(lines) && len(items) < maxLines; index++ {
|
||||
line := trimKnowledgeFactLine(lines[index])
|
||||
if line == "" {
|
||||
if len(items) > 0 {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "#") {
|
||||
break
|
||||
}
|
||||
if knowledgePreciseFactLabelOnly(line) {
|
||||
break
|
||||
}
|
||||
if len(items) > 0 && knowledgeLineStartsAnotherPreciseFact(line) && !knowledgeLineContainsBusinessFact(line) {
|
||||
break
|
||||
}
|
||||
items = append(items, line)
|
||||
}
|
||||
|
||||
return strings.Join(items, "；")
|
||||
}
|
||||
|
||||
func knowledgePreciseFactSectionLineLimit(line string) int {
|
||||
if knowledgeLineContainsBusinessFact(line) {
|
||||
return 4
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
func trimKnowledgeFactLine(value string) string {
|
||||
line := strings.TrimSpace(value)
|
||||
line = strings.TrimLeft(line, "-*• \t")
|
||||
line = strings.TrimLeft(line, "0123456789.、)） \t")
|
||||
return strings.TrimSpace(line)
|
||||
}
|
||||
|
||||
func knowledgePreciseFactLabelOnly(line string) bool {
|
||||
normalized := strings.ToLower(strings.Trim(strings.TrimSpace(line), ":："))
|
||||
switch normalized {
|
||||
case "地址", "门店地址", "公司地址", "办公地址", "总部地址", "联系地址", "location", "address",
|
||||
"电话", "联系电话", "客服热线", "phone", "tel", "telephone",
|
||||
"官网", "网址", "网站", "website", "url",
|
||||
"邮箱", "电子邮箱", "email", "e-mail",
|
||||
"营业时间", "开放时间", "服务时间", "办公时间", "hours", "opening hours",
|
||||
"主营业务", "主要业务", "核心业务", "业务范围", "经营范围", "服务范围",
|
||||
"主营产品", "主要产品", "核心产品", "产品范围",
|
||||
"main business", "core business", "business scope", "service scope",
|
||||
"main products", "core products", "product scope":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func knowledgeLineContainsPreciseFact(line string) bool {
|
||||
lower := strings.ToLower(strings.TrimSpace(line))
|
||||
if lower == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
keywords := []string{
|
||||
"地址", "门店地址", "公司地址", "办公地址", "总部地址", "联系地址", "邮编", "位置", "坐标",
|
||||
"电话", "联系电话", "客服热线", "热线", "传真",
|
||||
"官网", "网址", "网站", "链接",
|
||||
"邮箱", "电子邮箱", "邮件",
|
||||
"营业时间", "开放时间", "服务时间", "办公时间",
|
||||
"主营业务", "主要业务", "核心业务", "业务范围", "经营范围", "服务范围",
|
||||
"主营产品", "主要产品", "核心产品", "产品范围",
|
||||
"address", "location", "phone", "tel", "telephone", "website", "email", "e-mail",
|
||||
"opening hours", "business hours", "postcode", "zip code",
|
||||
"main business", "core business", "business scope", "service scope",
|
||||
"main products", "core products", "product scope",
|
||||
}
|
||||
for _, keyword := range keywords {
|
||||
if strings.Contains(lower, keyword) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if strings.Contains(lower, "http://") || strings.Contains(lower, "https://") {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(lower, "@") {
|
||||
return true
|
||||
}
|
||||
|
||||
digits := 0
|
||||
for _, r := range lower {
|
||||
if r >= '0' && r <= '9' {
|
||||
digits++
|
||||
}
|
||||
}
|
||||
return digits >= 7 && (strings.Contains(lower, "-") || strings.Contains(lower, " ") || strings.Contains(lower, "("))
|
||||
}
|
||||
|
||||
func knowledgeLineStartsAnotherPreciseFact(line string) bool {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "" {
|
||||
return false
|
||||
}
|
||||
if knowledgePreciseFactLabelOnly(trimmed) {
|
||||
return true
|
||||
}
|
||||
|
||||
lower := strings.ToLower(trimmed)
|
||||
labels := []string{
|
||||
"地址：", "门店地址：", "公司地址：", "办公地址：", "总部地址：", "联系地址：",
|
||||
"电话：", "联系电话：", "客服热线：", "传真：",
|
||||
"官网：", "网址：", "网站：", "邮箱：", "电子邮箱：",
|
||||
"营业时间：", "开放时间：", "服务时间：", "办公时间：",
|
||||
"主营业务：", "主要业务：", "核心业务：", "业务范围：", "经营范围：", "服务范围：",
|
||||
"主营产品：", "主要产品：", "核心产品：", "产品范围：",
|
||||
"address:", "location:", "phone:", "tel:", "telephone:",
|
||||
"website:", "email:", "e-mail:",
|
||||
"opening hours:", "business hours:",
|
||||
"main business:", "core business:", "business scope:", "service scope:",
|
||||
"main products:", "core products:", "product scope:",
|
||||
}
|
||||
for _, label := range labels {
|
||||
if strings.HasPrefix(lower, label) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func knowledgeLineContainsBusinessFact(line string) bool {
|
||||
lower := strings.ToLower(strings.TrimSpace(line))
|
||||
if lower == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
keywords := []string{
|
||||
"主营业务", "主要业务", "核心业务", "业务范围", "经营范围", "服务范围",
|
||||
"主营产品", "主要产品", "核心产品", "产品范围",
|
||||
"main business", "core business", "business scope", "service scope",
|
||||
"main products", "core products", "product scope",
|
||||
}
|
||||
for _, keyword := range keywords {
|
||||
if strings.Contains(lower, keyword) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type knowledgeIngestInput struct {
|
||||
GroupID int64
|
||||
Name string
|
||||
@@ -1036,7 +1549,12 @@ func (s *KnowledgeService) executeParseJob(ctx context.Context, job knowledgePar
|
||||
}
|
||||
}
|
||||
|
||||
chunks := splitKnowledgeTextIntoChunks(parsedContent.Text, s.chunkSize, s.chunkOverlap)
|
||||
chunkSource := strings.TrimSpace(parsedContent.Markdown)
|
||||
if chunkSource == "" {
|
||||
chunkSource = strings.TrimSpace(parsedContent.Text)
|
||||
}
|
||||
|
||||
chunks := splitKnowledgeTextIntoChunks(chunkSource, s.chunkSize, s.chunkOverlap)
|
||||
if len(chunks) == 0 {
|
||||
err = errors.New("knowledge content is empty after parsing")
|
||||
_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
|
||||
|
||||
Reference in New Issue
Block a user