feat(knowledge): multi-query retrieval with LLM query rewrite

Add a shared structured query builder (knowledge_query.go) and switch
all generation paths to it, so retrieval queries carry task intent
(type, brand, region, keywords, key points) instead of a raw prompt.

ResolveContext now rewrites the query into multiple sub-questions via
the URL-markdown model, embeds and searches each, and merges/dedupes
candidates before rerank; falls back to parsed fallback questions when
rewrite is unavailable or returns too few. Resolve logs gain query
list, count, and rewrite stage/model for observability.

- knowledge_query.go/_test.go: BuildKnowledgeQuery + intent normalization
- knowledge_service.go: per-query embed/search loop, query rewrite, logs
- template_prompt/template_service/prompt_generate/article_imitation/
  kol_generation_worker: adopt the shared query builder
- generation_observability: richer task error logging
This commit is contained in:
2026-06-02 14:50:12 +08:00
parent 85bb373a8b
commit ba2f117265
11 changed files with 734 additions and 102 deletions
@@ -469,7 +469,7 @@ func (s *ArticleImitationService) failGeneration(ctx context.Context, job articl
now := time.Now()
logCtx := articleImitationJobLogContext(job, stage, "", "failed", GenerationTaskResultFailure)
RecordGenerationTaskEvent(GenerationTaskEventFailure, logCtx)
LogGenerationTaskWarn(s.logger, "imitation generation failed", logCtx, zap.Error(genErr))
LogGenerationTaskWarn(s.logger, "imitation generation failed", logCtx, GenerationTaskErrorFields(genErr)...)
title := strings.TrimSpace(job.InitialTitle)
if title == "" {
title = articleImitationFallbackTitle
@@ -659,22 +659,18 @@ func buildImitationKnowledgeQuery(params map[string]interface{}) string {
return ""
}
parts := make([]string, 0, 8)
appendValue := func(value string) {
value = strings.TrimSpace(value)
if value == "" {
return
}
parts = append(parts, value)
intent := knowledgeQueryIntent{
TaskType: "仿写文章",
Title: extractString(params, "source_title"),
Topic: firstNonEmptyText(extractString(params, "source_title"), extractString(params, "extra_requirements")),
BrandName: extractString(params, "brand_name"),
Region: extractString(params, "region"),
PrimaryKeyword: firstNonEmptyText(extractString(params, "source_title"), knowledgeQueryKeywordText(extractStringList(params["keywords"], 16))),
Keywords: mergeKnowledgeQueryKeywords(params["keywords"]),
KeyPoints: extractString(params, "preserve_points"),
Extra: strings.Join([]string{extractString(params, "avoid_points"), extractString(params, "extra_requirements")}, "\n"),
}
appendValue(extractString(params, "source_title"))
appendValue(extractString(params, "brand_name"))
appendValue(extractString(params, "region"))
appendValue(strings.Join(extractStringList(params["keywords"], 16), "\n"))
appendValue(extractString(params, "extra_requirements"))
return strings.Join(parts, "\n")
return buildKnowledgeQueryInputText(intent)
}
func appendPromptLine(b *strings.Builder, label, value string) {
@@ -1,11 +1,14 @@
package app
import (
"errors"
"strings"
"sync/atomic"
"time"
"go.uber.org/zap"
"github.com/geo-platform/tenant-api/internal/shared/response"
)
const GenerationTaskStatusRunning = "running"
@@ -103,6 +106,30 @@ func LogGenerationTaskError(logger *zap.Logger, message string, ctx GenerationTa
logger.Error(message, append(GenerationTaskLogFields(ctx), fields...)...)
}
func GenerationTaskErrorFields(err error) []zap.Field {
if err == nil {
return []zap.Field{}
}
fields := []zap.Field{zap.Error(err)}
var appErr *response.AppError
if errors.As(err, &appErr) {
if appErr.Code > 0 {
fields = append(fields, zap.Int("error_code", appErr.Code))
}
if appErr.HTTPStatus > 0 {
fields = append(fields, zap.Int("http_status", appErr.HTTPStatus))
}
if message := strings.TrimSpace(appErr.Message); message != "" {
fields = append(fields, zap.String("error_message", message))
}
if detail := strings.TrimSpace(appErr.Detail); detail != "" {
fields = append(fields, zap.String("error_detail", detail))
}
}
return fields
}
type GenerationTaskMetricsSnapshot struct {
TransitionTotal int64 `json:"transition_total"`
FailureTotal int64 `json:"failure_total"`
@@ -4,6 +4,7 @@ import (
"strings"
"testing"
"github.com/geo-platform/tenant-api/internal/shared/response"
"github.com/stretchr/testify/assert"
)
@@ -91,6 +92,34 @@ func TestGenerationTaskMetricsRecordsEventsAndPrometheus(t *testing.T) {
assert.True(t, strings.HasSuffix(lastEventLine, " 1"), lastEventLine)
}
func TestFormatGenerationFailureKeepsAppErrorBriefForUI(t *testing.T) {
err := response.ErrServiceUnavailable(50353, "embedding_failed", "request siliconflow: timeout")
got := formatGenerationFailure("knowledge_resolve", err)
assert.Equal(t, "知识库检索失败 [knowledge_resolve]: embedding_failed", got)
}
func TestGenerationTaskErrorFieldsIncludesAppErrorDetailForLogs(t *testing.T) {
err := response.ErrServiceUnavailable(50353, "embedding_failed", "request siliconflow: timeout")
fields := GenerationTaskErrorFields(err)
values := map[string]any{}
for _, field := range fields {
switch {
case field.String != "":
values[field.Key] = field.String
case field.Integer != 0:
values[field.Key] = int(field.Integer)
}
}
assert.Equal(t, "embedding_failed", values["error_message"])
assert.Equal(t, "request siliconflow: timeout", values["error_detail"])
assert.Equal(t, 50353, values["error_code"])
assert.Equal(t, 503, values["http_status"])
}
func findPrometheusMetricLine(output, metricName string) string {
for _, line := range strings.Split(output, "\n") {
if strings.HasPrefix(line, metricName+"{") || strings.HasPrefix(line, metricName+" ") {
@@ -0,0 +1,293 @@
package app
import (
"encoding/json"
"fmt"
"strings"
)
const maxKnowledgeQueryQuestions = 3
type knowledgeQueryIntent struct {
TaskType string
TemplateName string
Title string
Topic string
BrandName string
Region string
PrimaryKeyword string
Keywords []string
KeyPoints string
Extra string
}
type KnowledgeQueryInput struct {
TaskType string
TemplateName string
Title string
Topic string
BrandName string
Region string
PrimaryKeyword string
Keywords []string
KeyPoints string
Extra string
}
func BuildKnowledgeQuery(input KnowledgeQueryInput) string {
intent := knowledgeQueryIntent{
TaskType: input.TaskType,
TemplateName: input.TemplateName,
Title: input.Title,
Topic: input.Topic,
BrandName: input.BrandName,
Region: input.Region,
PrimaryKeyword: input.PrimaryKeyword,
Keywords: input.Keywords,
KeyPoints: input.KeyPoints,
Extra: input.Extra,
}
return buildKnowledgeQueryInputText(intent)
}
func buildKnowledgeQueryQuestions(intent knowledgeQueryIntent) string {
questions := buildKnowledgeQueryQuestionList(intent)
return strings.Join(questions, "\n")
}
func buildKnowledgeQueryInputText(intent knowledgeQueryIntent) string {
intent = normalizeKnowledgeQueryIntent(intent)
lines := make([]string, 0, 12)
appendLine := func(label, value string) {
value = strings.TrimSpace(value)
if value != "" {
lines = append(lines, fmt.Sprintf("%s%s", label, value))
}
}
appendLine("文章类型", intent.TaskType)
appendLine("模板名称", intent.TemplateName)
appendLine("标题", intent.Title)
appendLine("主题", intent.Topic)
appendLine("品牌", intent.BrandName)
appendLine("地域", intent.Region)
appendLine("主关键词", intent.PrimaryKeyword)
appendLine("关键词", knowledgeQueryKeywordText(intent.Keywords))
appendLine("重点要求", intent.KeyPoints)
appendLine("其他要求", intent.Extra)
fallback := buildKnowledgeQueryQuestionList(intent)
if len(fallback) > 0 {
lines = append(lines, "兜底检索问题:")
for _, question := range fallback {
lines = append(lines, "- "+question)
}
}
return strings.Join(lines, "\n")
}
func buildKnowledgeQueryRewritePrompt(rawQuery string) string {
rawQuery = strings.TrimSpace(rawQuery)
if rawQuery == "" {
return ""
}
return strings.TrimSpace(fmt.Sprintf(`你是知识库检索 Query Rewrite 助手。请根据用户的文章生成输入,改写出 3 个最适合去企业/产品知识库中检索的自然语言问题。
目标:
1. 让检索问题覆盖“文章类型/写作任务”真正需要的知识,而不是只重复品牌名或地域词。
2. 优先围绕用户输入的标题、主题、品牌、地域、关键词、重点要求生成问题。
3. 问题应适合在知识库中召回:业务范围、产品服务、优势案例、地址联系方式、资质事实、用户痛点、对比维度、避坑点、适用场景。
4. 如果输入中出现“兜底检索问题”,只能作为参考;你需要根据上方结构化字段重新生成更贴近本次文章的 3 个问题。
规则:
- 必须输出 3 个问题。
- 每个问题 18-60 个中文字符,尽量具体。
- 不要生成泛泛的问题,如“这个品牌怎么样”。
- 不要编造输入中没有的品牌、地域、产品或事实。
- 只输出 JSON,不要 Markdown,不要解释。
用户输入:
%s`, rawQuery))
}
func parseKnowledgeQueryRewriteOutput(content string) []string {
content = strings.TrimSpace(content)
if content == "" {
return []string{}
}
content = strings.TrimPrefix(content, "```json")
content = strings.TrimPrefix(content, "```")
content = strings.TrimSuffix(content, "```")
content = strings.TrimSpace(content)
var payload struct {
Questions []string `json:"questions"`
}
if err := json.Unmarshal([]byte(content), &payload); err == nil && len(payload.Questions) > 0 {
return normalizeKnowledgeQueryQuestions(payload.Questions)
}
var list []string
if err := json.Unmarshal([]byte(content), &list); err == nil && len(list) > 0 {
return normalizeKnowledgeQueryQuestions(list)
}
return []string{}
}
func normalizeKnowledgeQueryQuestions(values []string) []string {
questions := make([]string, 0, maxKnowledgeQueryQuestions)
for _, value := range values {
appendKnowledgeQuestion(&questions, value)
if len(questions) >= maxKnowledgeQueryQuestions {
break
}
}
return questions
}
func buildKnowledgeQueryQuestionList(intent knowledgeQueryIntent) []string {
intent = normalizeKnowledgeQueryIntent(intent)
questions := make([]string, 0, maxKnowledgeQueryQuestions)
subject := firstNonEmptyText(intent.Topic, intent.Title, intent.PrimaryKeyword, intent.BrandName, "当前主题")
brandScope := firstNonEmptyText(joinKnowledgeQueryParts("", intent.BrandName, intent.Region), intent.BrandName, subject)
keywordScope := knowledgeQueryKeywordText(intent.Keywords)
switch {
case strings.Contains(intent.TaskType, "推荐榜") || strings.Contains(strings.ToLower(intent.TaskType), "top"):
appendKnowledgeQuestion(&questions, fmt.Sprintf("%s 中有哪些与 %s 相关的品牌、产品、服务能力和选型依据?", subject, brandScope))
appendKnowledgeQuestion(&questions, fmt.Sprintf("%s 写作时需要引用哪些关于 %s 的准确事实、优势、案例、地址或联系方式?", intent.TaskType, firstNonEmptyText(intent.BrandName, subject)))
appendKnowledgeQuestion(&questions, fmt.Sprintf("%s 相关关键词 %s 对应的用户关注点、避坑点和对比维度是什么?", subject, firstNonEmptyText(keywordScope, intent.PrimaryKeyword, subject)))
case strings.Contains(intent.TaskType, "评测") || strings.Contains(strings.ToLower(intent.TaskType), "review"):
appendKnowledgeQuestion(&questions, fmt.Sprintf("%s 的产品特性、参数、卖点、适用人群和使用场景是什么?", subject))
appendKnowledgeQuestion(&questions, fmt.Sprintf("%s 有哪些真实体验、优缺点、案例、价格或售后信息可用于评测?", firstNonEmptyText(intent.BrandName, subject)))
appendKnowledgeQuestion(&questions, fmt.Sprintf("围绕 %s 和 %s,用户最关心哪些购买决策问题?", subject, firstNonEmptyText(keywordScope, intent.PrimaryKeyword, subject)))
case strings.Contains(intent.TaskType, "仿写"):
appendKnowledgeQuestion(&questions, fmt.Sprintf("仿写 %s 时,%s 有哪些必须保留或补充的品牌事实和业务信息?", subject, firstNonEmptyText(intent.BrandName, subject)))
appendKnowledgeQuestion(&questions, fmt.Sprintf("%s 相关关键词 %s 对应的产品、服务、案例和用户需求是什么?", subject, firstNonEmptyText(keywordScope, intent.PrimaryKeyword, subject)))
appendKnowledgeQuestion(&questions, fmt.Sprintf("生成 %s 时需要避免遗漏哪些地址、联系方式、优势、资质或落地案例?", firstNonEmptyText(intent.TaskType, "仿写文章")))
default:
appendKnowledgeQuestion(&questions, fmt.Sprintf("%s 这篇%s需要引用哪些核心事实、产品服务、案例和用户关注点?", subject, firstNonEmptyText(intent.TaskType, "文章")))
appendKnowledgeQuestion(&questions, fmt.Sprintf("%s 与 %s 相关的准确资料、品牌信息、地址、联系方式和业务范围是什么?", firstNonEmptyText(intent.BrandName, subject), firstNonEmptyText(keywordScope, intent.PrimaryKeyword, subject)))
appendKnowledgeQuestion(&questions, fmt.Sprintf("围绕 %s,用户最可能关心哪些问题、痛点、对比维度和解决方案?", firstNonEmptyText(keywordScope, subject)))
}
appendKnowledgeQuestion(&questions, intent.KeyPoints)
appendKnowledgeQuestion(&questions, intent.Extra)
return questions
}
func normalizeKnowledgeQueryIntent(intent knowledgeQueryIntent) knowledgeQueryIntent {
intent.TaskType = strings.TrimSpace(intent.TaskType)
intent.TemplateName = strings.TrimSpace(intent.TemplateName)
intent.Title = strings.TrimSpace(intent.Title)
intent.Topic = strings.TrimSpace(intent.Topic)
intent.BrandName = strings.TrimSpace(intent.BrandName)
intent.Region = strings.TrimSpace(intent.Region)
intent.PrimaryKeyword = strings.TrimSpace(intent.PrimaryKeyword)
intent.KeyPoints = strings.TrimSpace(intent.KeyPoints)
intent.Extra = strings.TrimSpace(intent.Extra)
intent.Keywords = normalizeKnowledgeQueryKeywords(intent.Keywords, 6)
if intent.TaskType == "" {
intent.TaskType = strings.TrimSpace(intent.TemplateName)
}
return intent
}
func appendKnowledgeQuestion(questions *[]string, question string) {
if questions == nil || len(*questions) >= maxKnowledgeQueryQuestions {
return
}
question = normalizeKnowledgeQueryQuestion(question)
if question == "" {
return
}
for _, existing := range *questions {
if normalizeKnowledgeQueryQuestion(existing) == question {
return
}
}
*questions = append(*questions, question)
}
func normalizeKnowledgeQueryQuestion(question string) string {
question = strings.TrimSpace(question)
if question == "" {
return ""
}
question = strings.Join(strings.Fields(question), " ")
for _, bad := range []string{" ", " ", " ", " ?"} {
question = strings.ReplaceAll(question, bad, strings.TrimSpace(bad))
}
question = strings.Trim(question, " ,;")
if question == "" {
return ""
}
if !strings.HasSuffix(question, "") && !strings.HasSuffix(question, "?") {
question += ""
}
return question
}
func mergeKnowledgeQueryKeywords(values ...interface{}) []string {
keywords := make([]string, 0)
for _, value := range values {
keywords = append(keywords, extractStringList(value, 16)...)
if text := strings.TrimSpace(formatPromptValue(value)); text != "" && len(extractStringList(value, 1)) == 0 {
keywords = append(keywords, text)
}
}
return normalizeKnowledgeQueryKeywords(keywords, 6)
}
func normalizeKnowledgeQueryKeywords(keywords []string, limit int) []string {
result := make([]string, 0, len(keywords))
seen := make(map[string]struct{}, len(keywords))
for _, keyword := range keywords {
keyword = strings.TrimSpace(keyword)
if keyword == "" {
continue
}
key := strings.ToLower(strings.Join(strings.Fields(keyword), " "))
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
result = append(result, keyword)
if limit > 0 && len(result) >= limit {
break
}
}
return result
}
func knowledgeQueryKeywordText(keywords []string) string {
keywords = normalizeKnowledgeQueryKeywords(keywords, 6)
return strings.Join(keywords, "、")
}
func firstNonEmptyPromptString(params map[string]interface{}, keys ...string) string {
for _, key := range keys {
if text := strings.TrimSpace(stringValue(params[key])); text != "" {
return text
}
}
return ""
}
func firstNonEmptyText(values ...string) string {
for _, value := range values {
if text := strings.TrimSpace(value); text != "" {
return text
}
}
return ""
}
func joinKnowledgeQueryParts(sep string, values ...string) string {
parts := make([]string, 0, len(values))
for _, value := range values {
if text := strings.TrimSpace(value); text != "" {
parts = append(parts, text)
}
}
return strings.Join(parts, sep)
}
@@ -0,0 +1,141 @@
package app
import (
"context"
"errors"
"strings"
"testing"
"github.com/geo-platform/tenant-api/internal/shared/llm"
)
func TestBuildGenerationKnowledgeQueryIncludesContextAndFallbackQuestions(t *testing.T) {
query := buildGenerationKnowledgeQuery("top_x_article", "Top X 推荐文章", map[string]interface{}{
"topic": "合肥全屋定制",
"brand_name": "安徽海翔家居",
"region": "合肥",
"primary_keyword": "合肥定制家具",
"supplemental_questions": []string{"合肥家具品牌怎么选", "全屋定制避坑"},
"key_points": "突出门店与服务能力",
})
for _, expected := range []string{
"文章类型:推荐榜文章",
"模板名称:Top X 推荐文章",
"主题:合肥全屋定制",
"品牌:安徽海翔家居",
"地域:合肥",
"主关键词:合肥定制家具",
"兜底检索问题:",
} {
if !strings.Contains(query, expected) {
t.Fatalf("buildGenerationKnowledgeQuery() = %q, want %q", query, expected)
}
}
questions := normalizeKnowledgeResolveQueries(query)
if len(questions) != maxKnowledgeQueryQuestions {
t.Fatalf("normalizeKnowledgeResolveQueries() len = %d, want %d: %#v", len(questions), maxKnowledgeQueryQuestions, questions)
}
for _, question := range questions {
if !strings.HasSuffix(question, "") {
t.Fatalf("question = %q, want Chinese question mark", question)
}
}
}
func TestParseKnowledgeQueryRewriteOutputAcceptsObjectAndArray(t *testing.T) {
objectQuestions := parseKnowledgeQueryRewriteOutput(`{"questions":["安徽海翔家居在合肥有哪些门店和联系方式?","安徽海翔家居主营哪些全屋定制产品和服务?","合肥全屋定制用户选择时关注哪些案例和售后?"]}`)
if len(objectQuestions) != maxKnowledgeQueryQuestions {
t.Fatalf("object questions len = %d, want %d: %#v", len(objectQuestions), maxKnowledgeQueryQuestions, objectQuestions)
}
if objectQuestions[0] != "安徽海翔家居在合肥有哪些门店和联系方式?" {
t.Fatalf("first object question = %q", objectQuestions[0])
}
arrayQuestions := parseKnowledgeQueryRewriteOutput(`["安徽海翔家居有什么品牌优势?","合肥全屋定制有哪些案例可引用?","全屋定制选型需要哪些避坑信息?"]`)
if len(arrayQuestions) != maxKnowledgeQueryQuestions {
t.Fatalf("array questions len = %d, want %d: %#v", len(arrayQuestions), maxKnowledgeQueryQuestions, arrayQuestions)
}
}
func TestNormalizeKnowledgeResolveQueriesPrefersFallbackQuestionBlock(t *testing.T) {
query := strings.Join([]string{
"文章类型:推荐榜文章",
"主题:合肥全屋定制",
"品牌:安徽海翔家居",
"重点要求:",
"请突出本地服务",
"不要写成硬广",
"兜底检索问题:",
"- 安徽海翔家居在合肥有哪些门店和联系方式?",
"- 安徽海翔家居主营哪些全屋定制产品和服务?",
"- 合肥全屋定制用户选择时关注哪些案例和售后?",
}, "\n")
got := normalizeKnowledgeResolveQueries(query)
want := []string{
"安徽海翔家居在合肥有哪些门店和联系方式?",
"安徽海翔家居主营哪些全屋定制产品和服务?",
"合肥全屋定制用户选择时关注哪些案例和售后?",
}
if len(got) != len(want) {
t.Fatalf("normalizeKnowledgeResolveQueries() = %#v, want %#v", got, want)
}
for index := range want {
if got[index] != want[index] {
t.Fatalf("normalizeKnowledgeResolveQueries()[%d] = %q, want %q", index, got[index], want[index])
}
}
}
func TestRewriteKnowledgeResolveQueriesUsesKnowledgeURLModelAndJSONObject(t *testing.T) {
client := &fakeKnowledgeQueryLLM{
content: `{"questions":["安徽海翔家居在合肥有哪些门店和联系方式?","安徽海翔家居主营哪些全屋定制产品和服务?","合肥全屋定制用户选择时关注哪些案例和售后?"]}`,
}
svc := &KnowledgeService{
llmClient: client,
cfg: knowledgeRuntimeConfig{URLMarkdownModel: "knowledge-url-fast"},
}
questions, err := svc.rewriteKnowledgeResolveQueries(context.Background(), "品牌:安徽海翔家居\n地域:合肥")
if err != nil {
t.Fatalf("rewriteKnowledgeResolveQueries() error = %v", err)
}
if len(questions) != maxKnowledgeQueryQuestions {
t.Fatalf("questions len = %d, want %d: %#v", len(questions), maxKnowledgeQueryQuestions, questions)
}
if client.request.Model != "knowledge-url-fast" {
t.Fatalf("Generate model = %q, want knowledge-url-fast", client.request.Model)
}
if client.request.ResponseFormat == nil || client.request.ResponseFormat.Type != llm.ResponseFormatTypeJSONObject {
t.Fatalf("ResponseFormat = %#v, want json_object", client.request.ResponseFormat)
}
if !strings.Contains(client.request.Prompt, "品牌:安徽海翔家居") || !strings.Contains(client.request.Prompt, "必须输出 3 个问题") {
t.Fatalf("Prompt = %q, want raw user input and three-question instruction", client.request.Prompt)
}
}
type fakeKnowledgeQueryLLM struct {
request llm.GenerateRequest
content string
err error
}
func (f *fakeKnowledgeQueryLLM) Validate() error {
if f.err != nil {
return f.err
}
return nil
}
func (f *fakeKnowledgeQueryLLM) Generate(_ context.Context, req llm.GenerateRequest, _ func(string)) (*llm.GenerateResult, error) {
f.request = req
if f.err != nil {
return nil, f.err
}
if strings.TrimSpace(f.content) == "" {
return nil, errors.New("empty fake content")
}
return &llm.GenerateResult{Content: f.content, Model: req.Model}, nil
}
+150 -18
View File
@@ -836,9 +836,34 @@ func (s *KnowledgeService) ResolveContext(
if len(requestedGroupIDs) == 0 {
return nil, nil
}
if strings.TrimSpace(query) == "" {
queries := normalizeKnowledgeResolveQueries(query)
if len(queries) == 0 {
return nil, nil
}
queryRewriteStage := "fallback"
queryRewriteModel := strings.TrimSpace(s.runtimeConfig().URLMarkdownModel)
if rewritten, err := s.rewriteKnowledgeResolveQueries(ctx, query); err != nil {
if s.logger != nil {
s.logger.Warn("knowledge query rewrite failed",
zap.String("query", knowledgeLogPreview(query, 600)),
zap.String("query_rewrite_model", queryRewriteModel),
zap.Strings("fallback_queries", queries),
zap.Error(err),
)
}
} else if len(rewritten) >= maxKnowledgeQueryQuestions {
queries = rewritten
queryRewriteStage = "model"
} else if len(rewritten) > 0 && s.logger != nil {
s.logger.Warn("knowledge query rewrite returned insufficient questions",
zap.String("query", knowledgeLogPreview(query, 600)),
zap.String("query_rewrite_model", queryRewriteModel),
zap.Strings("model_queries", rewritten),
zap.Int("model_query_count", len(rewritten)),
zap.Strings("fallback_queries", queries),
)
}
rerankQuery := strings.Join(queries, "\n")
if err := s.provider.Validate(); err != nil {
return nil, response.ErrServiceUnavailable(50351, "retrieval_unavailable", err.Error())
}
@@ -861,32 +886,39 @@ func (s *KnowledgeService) ResolveContext(
PreciseFacts: s.collectKnowledgeContextPreciseFacts(ctx, tenantID, searchGroupIDs, nil),
}
vectors, err := s.provider.Embed(ctx, []string{query})
vectors, err := s.provider.Embed(ctx, queries)
if err != nil {
return nil, response.ErrServiceUnavailable(50353, "embedding_failed", err.Error())
}
if len(vectors) == 0 || len(vectors[0]) == 0 {
return nil, response.ErrServiceUnavailable(50353, "embedding_failed", "embedding vector is empty")
if len(vectors) != len(queries) {
return nil, response.ErrServiceUnavailable(50353, "embedding_failed", "embedding result count mismatch")
}
runtimeCfg := s.runtimeConfig()
searchPoints, err := s.store.Search(ctx, retrieval.SearchRequest{
Vector: vectors[0],
TenantID: tenantID,
GroupIDs: searchGroupIDs,
Limit: runtimeCfg.RecallLimit,
})
if err != nil {
return nil, response.ErrServiceUnavailable(50354, "qdrant_search_failed", err.Error())
searchPoints := make([]retrieval.SearchPoint, 0, len(queries)*runtimeCfg.RecallLimit)
for index, vector := range vectors {
if len(vector) == 0 {
return nil, response.ErrServiceUnavailable(50353, "embedding_failed", fmt.Sprintf("embedding vector is empty at query index %d", index))
}
points, err := s.store.Search(ctx, retrieval.SearchRequest{
Vector: vector,
TenantID: tenantID,
GroupIDs: searchGroupIDs,
Limit: runtimeCfg.RecallLimit,
})
if err != nil {
return nil, response.ErrServiceUnavailable(50354, "qdrant_search_failed", err.Error())
}
searchPoints = append(searchPoints, points...)
}
if len(searchPoints) == 0 {
s.logKnowledgeResolve(baseContext, tenantID, query, "no_search_points", nil, nil)
s.logKnowledgeResolve(baseContext, tenantID, rerankQuery, "no_search_points", queryRewriteStage, queryRewriteModel, nil, nil)
return baseContext, nil
}
candidates := dedupeKnowledgeSearchCandidates(knowledgeSnippetsFromSearchPoints(searchPoints))
if len(candidates) == 0 {
s.logKnowledgeResolve(baseContext, tenantID, query, "no_candidate_text", nil, nil)
s.logKnowledgeResolve(baseContext, tenantID, rerankQuery, "no_candidate_text", queryRewriteStage, queryRewriteModel, nil, nil)
return baseContext, nil
}
@@ -895,7 +927,7 @@ func (s *KnowledgeService) ResolveContext(
return nil, err
}
if len(candidates) == 0 {
s.logKnowledgeResolve(baseContext, tenantID, query, "no_active_candidates", nil, nil)
s.logKnowledgeResolve(baseContext, tenantID, rerankQuery, "no_active_candidates", queryRewriteStage, queryRewriteModel, nil, nil)
return baseContext, nil
}
@@ -904,13 +936,13 @@ func (s *KnowledgeService) ResolveContext(
documents = append(documents, candidate.RerankText())
}
reranked, err := s.provider.Rerank(ctx, query, documents, minInt(runtimeCfg.RerankTopN, len(documents)))
reranked, err := s.provider.Rerank(ctx, rerankQuery, documents, minInt(runtimeCfg.RerankTopN, len(documents)))
if err != nil {
selected := selectKnowledgeCandidateSnippets(candidates, runtimeCfg.RerankTopN)
selected = s.enrichKnowledgePromptSnippets(ctx, tenantID, selected)
baseContext.Snippets = selected
baseContext.PreciseFacts = s.collectKnowledgeContextPreciseFacts(ctx, tenantID, searchGroupIDs, selected)
s.logKnowledgeResolve(baseContext, tenantID, query, "rerank_failed_fallback", candidates, selected)
s.logKnowledgeResolve(baseContext, tenantID, rerankQuery, "rerank_failed_fallback", queryRewriteStage, queryRewriteModel, candidates, selected)
return baseContext, nil
}
@@ -918,21 +950,48 @@ func (s *KnowledgeService) ResolveContext(
selected = s.enrichKnowledgePromptSnippets(ctx, tenantID, selected)
baseContext.Snippets = selected
baseContext.PreciseFacts = s.collectKnowledgeContextPreciseFacts(ctx, tenantID, searchGroupIDs, selected)
s.logKnowledgeResolve(baseContext, tenantID, query, "resolved", candidates, selected)
s.logKnowledgeResolve(baseContext, tenantID, rerankQuery, "resolved", queryRewriteStage, queryRewriteModel, candidates, selected)
return baseContext, nil
}
func (s *KnowledgeService) rewriteKnowledgeResolveQueries(ctx context.Context, query string) ([]string, error) {
if s == nil || s.llmClient == nil {
return []string{}, nil
}
prompt := buildKnowledgeQueryRewritePrompt(query)
if prompt == "" {
return []string{}, nil
}
runtimeCfg := s.runtimeConfig()
result, err := s.llmClient.Generate(ctx, llm.GenerateRequest{
Model: runtimeCfg.URLMarkdownModel,
Prompt: prompt,
Timeout: 15 * time.Second,
MaxOutputTokens: 600,
ResponseFormat: &llm.ResponseFormat{
Type: llm.ResponseFormatTypeJSONObject,
},
}, nil)
if err != nil {
return nil, err
}
return parseKnowledgeQueryRewriteOutput(result.Content), nil
}
func (s *KnowledgeService) logKnowledgeResolve(
knowledgeCtx *KnowledgeContext,
tenantID int64,
query string,
stage string,
queryRewriteStage string,
queryRewriteModel string,
candidates []KnowledgeSnippet,
selected []KnowledgeSnippet,
) {
if s == nil || s.logger == nil || knowledgeCtx == nil {
return
}
queries := normalizeKnowledgeResolveQueries(query)
s.logger.Info("knowledge resolve result",
zap.String("stage", strings.TrimSpace(stage)),
@@ -940,6 +999,10 @@ func (s *KnowledgeService) logKnowledgeResolve(
zap.Int64s("group_ids", knowledgeCtx.GroupIDs),
zap.Strings("group_names", knowledgeCtx.GroupNames),
zap.String("query", knowledgeLogPreview(query, 600)),
zap.Strings("queries", queries),
zap.Int("query_count", len(queries)),
zap.String("query_rewrite_stage", strings.TrimSpace(queryRewriteStage)),
zap.String("query_rewrite_model", strings.TrimSpace(queryRewriteModel)),
zap.Int("candidate_count", len(candidates)),
zap.Int("selected_count", len(selected)),
zap.Int("precise_fact_count", len(knowledgeCtx.PreciseFacts)),
@@ -947,6 +1010,75 @@ func (s *KnowledgeService) logKnowledgeResolve(
)
}
func normalizeKnowledgeResolveQueries(query string) []string {
lines := strings.Split(strings.TrimSpace(query), "\n")
queries := make([]string, 0, minInt(maxKnowledgeQueryQuestions, len(lines)))
seen := make(map[string]struct{}, len(lines))
for index, line := range lines {
if strings.TrimSuffix(strings.TrimSpace(line), "") != "兜底检索问题" {
continue
}
for _, fallbackLine := range lines[index+1:] {
text := strings.TrimSpace(fallbackLine)
if text == "" {
continue
}
if !strings.HasPrefix(text, "-") {
break
}
appendNormalizedKnowledgeResolveQuery(&queries, seen, strings.TrimSpace(strings.TrimPrefix(text, "-")))
if len(queries) >= maxKnowledgeQueryQuestions {
return queries
}
}
if len(queries) > 0 {
return queries
}
}
inFallbackQuestions := false
for _, line := range lines {
text := strings.TrimSpace(line)
if text == "" {
continue
}
if strings.TrimSuffix(text, "") == "兜底检索问题" {
inFallbackQuestions = true
continue
}
if strings.Contains(text, "") && !strings.HasPrefix(text, "-") && !inFallbackQuestions {
continue
}
if inFallbackQuestions && !strings.HasPrefix(text, "-") {
continue
}
text = strings.TrimPrefix(text, "-")
text = strings.TrimSpace(text)
appendNormalizedKnowledgeResolveQuery(&queries, seen, text)
if len(queries) >= maxKnowledgeQueryQuestions {
break
}
}
return queries
}
func appendNormalizedKnowledgeResolveQuery(queries *[]string, seen map[string]struct{}, text string) {
if queries == nil || len(*queries) >= maxKnowledgeQueryQuestions {
return
}
text = strings.TrimSpace(text)
if text == "" {
return
}
key := strings.ToLower(strings.Join(strings.Fields(text), " "))
if _, exists := seen[key]; exists {
return
}
seen[key] = struct{}{}
*queries = append(*queries, text)
}
// for debug
func summarizeKnowledgeSnippetsForLog(snippets []KnowledgeSnippet) []map[string]any {
if len(snippets) == 0 {
@@ -718,7 +718,7 @@ func (s *PromptRuleGenerationService) failGeneration(ctx context.Context, job pr
now := time.Now()
logCtx := promptRuleGenerationJobLogContext(job, stage, "", "failed", GenerationTaskResultFailure)
RecordGenerationTaskEvent(GenerationTaskEventFailure, logCtx)
LogGenerationTaskWarn(s.logger, "prompt rule generation failed", logCtx, zap.Error(genErr))
LogGenerationTaskWarn(s.logger, "prompt rule generation failed", logCtx, GenerationTaskErrorFields(genErr)...)
failureTitle := strings.TrimSpace(extractString(job.InputParams, "task_name"))
if failureTitle == "" {
failureTitle = title
@@ -857,41 +857,28 @@ func buildPromptRuleGenerationPrompt(rule *promptRuleGenerationRecord, params ma
}
func buildPromptRuleKnowledgeQuery(rule *promptRuleGenerationRecord, params map[string]interface{}) string {
parts := make([]string, 0, 6)
intent := knowledgeQueryIntent{
TaskType: "自定义生成",
Title: firstNonEmptyPromptString(params, "task_name", "title", "topic", "subject"),
Topic: firstNonEmptyPromptString(params, "topic", "subject", "title"),
BrandName: extractString(params, "brand_name"),
PrimaryKeyword: extractString(params, "primary_keyword"),
Keywords: mergeKnowledgeQueryKeywords(params["keywords"], params["primary_keyword"]),
KeyPoints: extractString(params, "key_points"),
Extra: extractString(params, "extra_requirements"),
}
if rule != nil {
if text := strings.TrimSpace(rule.Name); text != "" {
parts = append(parts, text)
if name := strings.TrimSpace(rule.Name); name != "" {
intent.TemplateName = name
if intent.Title == "" {
intent.Title = name
}
}
if rule.Scene != nil && strings.TrimSpace(*rule.Scene) != "" {
parts = append(parts, strings.TrimSpace(*rule.Scene))
intent.TaskType = strings.TrimSpace(*rule.Scene)
}
}
if taskName := strings.TrimSpace(extractString(params, "task_name")); taskName != "" {
parts = append(parts, taskName)
}
for _, key := range []string{
"title",
"topic",
"subject",
"brand_name",
"product_name",
"primary_keyword",
"key_points",
"extra_requirements",
} {
if text := strings.TrimSpace(extractString(params, key)); text != "" {
parts = append(parts, text)
}
}
if keywords := extractStringList(params["keywords"], 16); len(keywords) > 0 {
parts = append(parts, strings.Join(keywords, "\n"))
}
if rule != nil {
if text := strings.TrimSpace(rule.PromptContent); text != "" {
parts = append(parts, text)
}
}
return strings.Join(parts, "\n")
return buildKnowledgeQueryInputText(intent)
}
func extractGenerateCount(extra map[string]interface{}) int {
+27 -20
View File
@@ -48,32 +48,39 @@ func buildGenerationPrompt(
return strings.Join(sections, "\n\n")
}
func buildGenerationKnowledgeQuery(params map[string]interface{}) string {
func buildGenerationKnowledgeQuery(templateKey, templateName string, params map[string]interface{}) string {
if len(params) == 0 {
return ""
}
parts := make([]string, 0, 8)
appendValue := func(value string) {
value = strings.TrimSpace(value)
if value == "" {
return
}
parts = append(parts, value)
intent := knowledgeQueryIntent{
TaskType: templateKnowledgeTaskType(templateKey, templateName),
TemplateName: templateName,
Title: firstNonEmptyPromptString(params, "title", "topic", "product_name", "subject"),
Topic: firstNonEmptyPromptString(params, "topic", "subject", "product_name", "title"),
BrandName: stringValue(params["brand_name"]),
Region: firstNonEmptyPromptString(params, "region", "city", "location", "area", "target_region", "service_area"),
PrimaryKeyword: firstNonEmptyPromptString(params, "brand_question", "primary_keyword"),
Keywords: mergeKnowledgeQueryKeywords(params["keywords"], params["supplemental_questions"], params["primary_keyword"]),
KeyPoints: firstNonEmptyPromptString(params, "key_points", "review_intro_hook"),
}
return buildKnowledgeQueryInputText(intent)
}
appendValue(stringValue(params["title"]))
appendValue(stringValue(params["topic"]))
appendValue(stringValue(params["product_name"]))
appendValue(stringValue(params["subject"]))
appendValue(stringValue(params["brand_name"]))
appendValue(stringValue(params["brand_question"]))
appendValue(stringValue(params["primary_keyword"]))
appendValue(formatPromptValue(params["supplemental_questions"]))
appendValue(stringValue(params["key_points"]))
appendValue(stringValue(params["review_intro_hook"]))
return strings.Join(parts, "\n")
func templateKnowledgeTaskType(templateKey, templateName string) string {
switch strings.TrimSpace(templateKey) {
case "top_x_article":
return "推荐榜文章"
case "product_review":
return "产品评测文章"
case "research_report":
return "研究报告"
default:
if name := strings.TrimSpace(templateName); name != "" {
return name
}
return "文章生成"
}
}
func buildTemplateSpecificWritingRules(templateKey string, params map[string]interface{}) string {
@@ -491,7 +491,7 @@ func (s *TemplateService) executeGeneration(ctx context.Context, job generationJ
ctx,
job.TenantID,
knowledgeGroupIDs,
buildGenerationKnowledgeQuery(job.Params),
buildGenerationKnowledgeQuery(job.TemplateKey, job.TemplateName, job.Params),
)
if resolveErr != nil {
s.failGeneration(ctx, job, title, "knowledge_resolve", resolveErr)
@@ -604,7 +604,7 @@ func (s *TemplateService) failGeneration(ctx context.Context, job generationJob,
now := time.Now()
logCtx := generationJobLogContext(job, stage, "", "failed", GenerationTaskResultFailure)
RecordGenerationTaskEvent(GenerationTaskEventFailure, logCtx)
LogGenerationTaskWarn(s.logger, "template generation failed", logCtx, zap.Error(genErr))
LogGenerationTaskWarn(s.logger, "template generation failed", logCtx, GenerationTaskErrorFields(genErr)...)
auditRepo := repository.NewAuditRepository(s.pool)
articleRepo := repository.NewArticleRepository(s.pool)
@@ -251,7 +251,7 @@ func (w *ArticleGenerationWorker) handleTaskFailure(ctx context.Context, task te
task.TaskType = strings.TrimSpace(task.TaskType)
logCtx := generationWorkerTaskLogContext(task, stage, "", "failed", tenantapp.GenerationTaskResultFailure, 0, "")
tenantapp.RecordGenerationTaskEvent(tenantapp.GenerationTaskEventFailure, logCtx)
tenantapp.LogGenerationTaskWarn(w.logger, "article generation task failed", logCtx, zap.Error(taskErr))
tenantapp.LogGenerationTaskWarn(w.logger, "article generation task failed", logCtx, tenantapp.GenerationTaskErrorFields(taskErr)...)
var err error
switch {
case task.TaskType == kolGenerationTaskType && w.kolWorker != nil:
@@ -333,15 +333,11 @@ func dedupeKolIDs(ids []int64) []int64 {
}
func buildKolKnowledgeQuery(input map[string]interface{}) string {
parts := make([]string, 0, 8)
if promptName := strings.TrimSpace(kolStringInput(input, "prompt_name")); promptName != "" {
parts = append(parts, promptName)
}
if platformHint := strings.TrimSpace(kolStringInput(input, "platform_hint")); platformHint != "" {
parts = append(parts, platformHint)
}
promptName := strings.TrimSpace(kolStringInput(input, "prompt_name"))
platformHint := strings.TrimSpace(kolStringInput(input, "platform_hint"))
variables, _ := input["variables"].(map[string]interface{})
keywords := make([]string, 0)
keyPoints := make([]string, 0)
if len(variables) > 0 {
keys := make([]string, 0, len(variables))
for key := range variables {
@@ -353,17 +349,41 @@ func buildKolKnowledgeQuery(input map[string]interface{}) string {
if valueText == "" {
continue
}
parts = append(parts, fmt.Sprintf("%s: %s", key, valueText))
switch {
case strings.Contains(strings.ToLower(key), "keyword") || strings.Contains(key, "关键词"):
keywords = append(keywords, valueText)
case strings.Contains(strings.ToLower(key), "brand") || strings.Contains(key, "品牌"):
keyPoints = append(keyPoints, fmt.Sprintf("%s: %s", key, valueText))
default:
keyPoints = append(keyPoints, fmt.Sprintf("%s: %s", key, valueText))
}
}
}
if len(parts) == 0 {
if renderedPrompt := strings.TrimSpace(kolStringInput(input, "rendered_prompt")); renderedPrompt != "" {
parts = append(parts, renderedPrompt)
renderedPrompt := strings.TrimSpace(kolStringInput(input, "rendered_prompt"))
query := tenantapp.BuildKnowledgeQuery(tenantapp.KnowledgeQueryInput{
TaskType: firstNonEmptyKolText(platformHint, "KOL 内容生成"),
TemplateName: promptName,
Title: firstNonEmptyKolText(promptName, renderedPrompt),
Topic: firstNonEmptyKolText(promptName, renderedPrompt),
PrimaryKeyword: firstNonEmptyKolText(strings.Join(keywords, "、"), promptName),
Keywords: keywords,
KeyPoints: strings.Join(keyPoints, "\n"),
Extra: renderedPrompt,
})
if strings.TrimSpace(query) != "" {
return kolLimitText(query, 2000)
}
return kolLimitText(renderedPrompt, 2000)
}
func firstNonEmptyKolText(values ...string) string {
for _, value := range values {
if text := strings.TrimSpace(value); text != "" {
return text
}
}
return kolLimitText(strings.Join(parts, "\n"), 2000)
return ""
}
func kolValueString(value interface{}) string {