From dbd7747742df93d9b302a7b0e0456704ef831c12 Mon Sep 17 00:00:00 2001 From: liangxu Date: Sun, 5 Apr 2026 20:41:42 +0800 Subject: [PATCH] feat: add knowledge markdown formatting and detail retrieval - Implemented KnowledgeWebsiteMarkdownPrompt for formatting webpage content into Markdown. - Added GetItemDetail method in KnowledgeHandler to retrieve knowledge item details. - Introduced KnowledgeDetailDrawer component for displaying knowledge item details in the admin web interface. - Created MarkdownPreview component for rendering Markdown content. - Added knowledge chunking and URL parsing logic to handle webpage content extraction and formatting. - Updated database schema to include markdown_content in knowledge_items table. --- apps/admin-web/package.json | 2 + .../src/components/KnowledgeDetailDrawer.vue | 180 +++++++ .../src/components/MarkdownPreview.vue | 114 ++++ apps/admin-web/src/lib/api.ts | 4 + apps/admin-web/src/main.ts | 2 + apps/admin-web/src/views/KnowledgeView.vue | 146 ++++-- packages/shared-types/src/index.ts | 4 + pnpm-lock.yaml | 65 +++ server/configs/config.local.yaml.example | 1 + server/configs/config.yaml | 1 + server/go.mod | 2 + server/go.sum | 4 + server/internal/shared/config/config.go | 24 +- server/internal/shared/llm/ark.go | 9 +- server/internal/shared/llm/client.go | 1 + .../internal/tenant/app/knowledge_chunking.go | 122 +++++ .../internal/tenant/app/knowledge_parser.go | 495 ++++++++++++------ .../internal/tenant/app/knowledge_service.go | 305 +++++++++-- .../tenant/app/knowledge_url_parser.go | 266 ++++++++++ server/internal/tenant/prompts/runtime.go | 30 ++ .../tenant/transport/article_handler.go | 2 + .../tenant/transport/knowledge_handler.go | 17 + server/internal/tenant/transport/router.go | 1 + .../tenant/transport/template_handler.go | 2 + ...rkdown_content_to_knowledge_items.down.sql | 2 + ...markdown_content_to_knowledge_items.up.sql | 7 + 26 files changed, 1556 insertions(+), 252 deletions(-) create mode 100644 apps/admin-web/src/components/KnowledgeDetailDrawer.vue create mode 100644 apps/admin-web/src/components/MarkdownPreview.vue create mode 100644 server/internal/tenant/app/knowledge_chunking.go create mode 100644 server/internal/tenant/app/knowledge_url_parser.go create mode 100644 server/migrations/20260405173000_add_markdown_content_to_knowledge_items.down.sql create mode 100644 server/migrations/20260405173000_add_markdown_content_to_knowledge_items.up.sql diff --git a/apps/admin-web/package.json b/apps/admin-web/package.json index f0f48bf..a02e8f5 100644 --- a/apps/admin-web/package.json +++ b/apps/admin-web/package.json @@ -21,6 +21,7 @@ "@vueuse/core": "^14.2.1", "ant-design-vue": "^4.2.6", "dayjs": "^1.11.20", + "markdown-it": "^14.1.1", "pinia": "^3.0.4", "vue": "^3.5.31", "vue-i18n": "^10.0.5", @@ -28,6 +29,7 @@ "zod": "^4.3.6" }, "devDependencies": { + "@types/markdown-it": "^14.1.2", "@types/node": "^24.0.0", "@vitejs/plugin-vue": "^6.0.5", "typescript": "^5.9.3", diff --git a/apps/admin-web/src/components/KnowledgeDetailDrawer.vue b/apps/admin-web/src/components/KnowledgeDetailDrawer.vue new file mode 100644 index 0000000..4cf36ea --- /dev/null +++ b/apps/admin-web/src/components/KnowledgeDetailDrawer.vue @@ -0,0 +1,180 @@ + + + + + diff --git a/apps/admin-web/src/components/MarkdownPreview.vue b/apps/admin-web/src/components/MarkdownPreview.vue new file mode 100644 index 0000000..3507736 --- /dev/null +++ b/apps/admin-web/src/components/MarkdownPreview.vue @@ -0,0 +1,114 @@ + + + + + diff --git a/apps/admin-web/src/lib/api.ts b/apps/admin-web/src/lib/api.ts index 600e9e3..4cb774a 100644 --- a/apps/admin-web/src/lib/api.ts +++ b/apps/admin-web/src/lib/api.ts @@ -21,6 +21,7 @@ import type { KnowledgeGroup, KnowledgeGroupRequest, KnowledgeItem, + KnowledgeItemDetail, KnowledgeTextItemRequest, KnowledgeURLItemRequest, Keyword, @@ -248,6 +249,9 @@ export const knowledgeApi = { params: groupId ? { group_id: groupId } : undefined, }); }, + detail(id: number) { + return apiClient.get(`/api/tenant/knowledge/items/${id}`); + }, createTextItem(payload: KnowledgeTextItemRequest) { return apiClient.post( "/api/tenant/knowledge/items/text", diff --git a/apps/admin-web/src/main.ts b/apps/admin-web/src/main.ts index e1251bb..2f45264 100644 --- a/apps/admin-web/src/main.ts +++ b/apps/admin-web/src/main.ts @@ -26,6 +26,7 @@ import { Popconfirm, Progress, Radio, + Result, Row, Select, Skeleton, @@ -92,6 +93,7 @@ app.component("ATextarea", Input.TextArea); Popconfirm, Progress, Radio, + Result, Row, Select, Skeleton, diff --git a/apps/admin-web/src/views/KnowledgeView.vue b/apps/admin-web/src/views/KnowledgeView.vue index b1fc638..3f8a847 100644 --- a/apps/admin-web/src/views/KnowledgeView.vue +++ b/apps/admin-web/src/views/KnowledgeView.vue @@ -1,17 +1,18 @@ @@ -570,11 +587,16 @@ function editItem(): void { {{ formatDateTime(record.created_at) }} @@ -615,7 +637,7 @@ function editItem(): void { " ok-text="确 定" cancel-text="取 消" - :ok-button-props="{ style: { width: '80px', borderRadius: '4px' } }" + :ok-button-props="itemModalOkButtonProps" :cancel-button-props="{ style: { width: '80px', borderRadius: '4px' } }" >
@@ -644,7 +666,7 @@ function editItem(): void {

点击或拖拽文件至此区域即可上传

- 请上传doc、pdf、txt、xls、xlsx格式，不超过30M的文件。严禁上传敏感数据或其他非法文件。 + 请上传 docx、pdf、txt、md、xls、xlsx 格式，不超过30M的文件。严禁上传敏感数据或其他非法文件。

已选择文件：{{ uploadFiles.length }}个

@@ -655,9 +677,17 @@ function editItem(): void { +
+ 当前 {{ websiteUrls.length }} / {{ MAX_WEBSITE_URLS }} 个网址 + ，请删除多余网址后再提交 +
@@ -690,6 +720,12 @@ function editItem(): void {
+ + @@ -776,7 +812,7 @@ function editItem(): void { } .knowledge-sidebar__add { - color: #9ca3af; + color: #666; font-size: 13px; cursor: pointer; display: flex; @@ -823,12 +859,21 @@ function editItem(): void { color: #6b7280; } -.knowledge-actions { +/* Actions */ +.table-actions-row { display: flex; align-items: center; justify-content: flex-end; + gap: 4px; } +.action-btn { + color: #8c8c8c; + font-size: 14px; +} +.action-eye:hover { color: #13c2c2; background: #e6fffb; } +.action-delete:hover { color: #ff4d4f; background: #fff2f0; } + .knowledge-form { display: flex; flex-direction: column; @@ -871,6 +916,17 @@ function editItem(): void { margin-top: 4px; } +.knowledge-url-hint { + margin-top: 8px; + color: #6b7280; + font-size: 12px; + line-height: 1.5; +} + +.knowledge-url-hint--error { + color: #ff4d4f; +} + :deep(.custom-dragger.ant-upload-wrapper .ant-upload-drag) { border: 1px dashed #7eb0f7; background-color: #fff; diff --git a/packages/shared-types/src/index.ts b/packages/shared-types/src/index.ts index ed0d4f2..d2260ba 100644 --- a/packages/shared-types/src/index.ts +++ b/packages/shared-types/src/index.ts @@ -556,6 +556,10 @@ export interface KnowledgeItem { updated_at: string; } +export interface KnowledgeItemDetail extends KnowledgeItem { + markdown_content: string | null; +} + export interface KnowledgeTextItemRequest { group_id: number; name: string; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6327156..7bf4091 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -43,6 +43,9 @@ importers: dayjs: specifier: ^1.11.20 version: 1.11.20 + markdown-it: + specifier: ^14.1.1 + version: 14.1.1 pinia: specifier: ^3.0.4 version: 3.0.4(typescript@5.9.3)(vue@3.5.31(typescript@5.9.3)) @@ -59,6 +62,9 @@ importers: specifier: ^4.3.6 version: 4.3.6 devDependencies: + '@types/markdown-it': + specifier: ^14.1.2 + version: 14.1.2 '@types/node': specifier: ^24.0.0 version: 24.12.0 @@ -790,15 +796,24 @@ packages: '@types/katex@0.16.8': resolution: {integrity: sha512-trgaNyfU+Xh2Tc+ABIb44a5AYUpicB3uwirOioeOkNPPbmgRNtcWyDeeFRzjPZENO9Vq8gvVqfhaaXWLlevVwg==} + '@types/linkify-it@5.0.0': + resolution: {integrity: sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==} + '@types/lodash-es@4.17.12': resolution: {integrity: sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==} '@types/lodash@4.17.24': resolution: {integrity: sha512-gIW7lQLZbue7lRSWEFql49QJJWThrTFFeIMJdp3eH4tKoxm1OvEPg02rm4wCCSHS0cL3/Fizimb35b7k8atwsQ==} + '@types/markdown-it@14.1.2': + resolution: {integrity: sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog==} + '@types/mdast@4.0.4': resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==} + '@types/mdurl@2.0.0': + resolution: {integrity: sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==} + '@types/minimatch@3.0.5': resolution: {integrity: sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==} @@ -951,6 +966,9 @@ packages: peerDependencies: vue: '>=3.2.0' + argparse@2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + array-differ@4.0.0: resolution: {integrity: sha512-Q6VPTLMsmXZ47ENG3V+wQyZS1ZxXMxFyYzA+Z/GMrJ6yIutAIEf9wTyroTzmGjNfox9/h3GdGBCVh43GVFx4Uw==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -1760,6 +1778,9 @@ packages: canvas: optional: true + linkify-it@5.0.0: + resolution: {integrity: sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==} + listr2@10.2.1: resolution: {integrity: sha512-7I5knELsJKTUjXG+A6BkKAiGkW1i25fNa/xlUl9hFtk15WbE9jndA89xu5FzQKrY5llajE1hfZZFMILXkDHk/Q==} engines: {node: '>=22.13.0'} @@ -1821,6 +1842,10 @@ packages: many-keys-map@2.0.1: resolution: {integrity: sha512-DHnZAD4phTbZ+qnJdjoNEVU1NecYoSdbOOoVmTDH46AuxDkEVh3MxTVpXq10GtcTC6mndN9dkv1rNfpjRcLnOw==} + markdown-it@14.1.1: + resolution: {integrity: sha512-BuU2qnTti9YKgK5N+IeMubp14ZUKUUw7yeJbkjtosvHiP0AZ5c8IAgEMk79D0eC8F23r4Ac/q8cAIFdm2FtyoA==} + hasBin: true + markdown-table@3.0.4: resolution: {integrity: sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==} @@ -1875,6 +1900,9 @@ packages: mdast-util-to-string@4.0.0: resolution: {integrity: sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==} + mdurl@2.0.0: + resolution: {integrity: sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==} + micromark-core-commonmark@2.0.3: resolution: {integrity: sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==} @@ -2226,6 +2254,10 @@ packages: engines: {node: '>=18.0.0'} hasBin: true + punycode.js@2.3.1: + resolution: {integrity: sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==} + engines: {node: '>=6'} + pupa@3.3.0: resolution: {integrity: sha512-LjgDO2zPtoXP2wJpDjZrGdojii1uqO0cnwKoIoUzkfS98HDmbeiGmYiXo3lXeFlq2xvne1QFQhwYXSUCLKtEuA==} engines: {node: '>=12.20'} @@ -2495,6 +2527,9 @@ packages: engines: {node: '>=14.17'} hasBin: true + uc.micro@2.1.0: + resolution: {integrity: sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==} + ufo@1.6.3: resolution: {integrity: sha512-yDJTmhydvl5lJzBmy/hyOAA0d+aqCBuwl818haVdYCRrWV84o7YyeVm4QlVHStqNrrJSTb6jKuFAVqAFsr+K3Q==} @@ -3711,16 +3746,25 @@ snapshots: '@types/katex@0.16.8': {} + '@types/linkify-it@5.0.0': {} + '@types/lodash-es@4.17.12': dependencies: '@types/lodash': 4.17.24 '@types/lodash@4.17.24': {} + '@types/markdown-it@14.1.2': + dependencies: + '@types/linkify-it': 5.0.0 + '@types/mdurl': 2.0.0 + '@types/mdast@4.0.4': dependencies: '@types/unist': 3.0.3 + '@types/mdurl@2.0.0': {} + '@types/minimatch@3.0.5': {} '@types/ms@2.1.0': {} @@ -3930,6 +3974,8 @@ snapshots: vue-types: 3.0.2(vue@3.5.31(typescript@5.9.3)) warning: 4.0.3 + argparse@2.0.1: {} + array-differ@4.0.0: {} array-tree-filter@2.1.0: {} @@ -4648,6 +4694,10 @@ snapshots: htmlparser2: 10.1.0 uhyphen: 0.2.0 + linkify-it@5.0.0: + dependencies: + uc.micro: 2.1.0 + listr2@10.2.1: dependencies: cli-truncate: 5.2.0 @@ -4710,6 +4760,15 @@ snapshots: many-keys-map@2.0.1: {} + markdown-it@14.1.1: + dependencies: + argparse: 2.0.1 + entities: 4.5.0 + linkify-it: 5.0.0 + mdurl: 2.0.0 + punycode.js: 2.3.1 + uc.micro: 2.1.0 + markdown-table@3.0.4: {} marked@17.0.5: {} @@ -4838,6 +4897,8 @@ snapshots: dependencies: '@types/mdast': 4.0.4 + mdurl@2.0.0: {} + micromark-core-commonmark@2.0.3: dependencies: decode-named-character-reference: 1.3.0 @@ -5348,6 +5409,8 @@ snapshots: ofetch: 1.5.1 zod: 4.3.6 + punycode.js@2.3.1: {} + pupa@3.3.0: dependencies: escape-goat: 4.0.0 @@ -5634,6 +5697,8 @@ snapshots: typescript@5.9.3: {} + uc.micro@2.1.0: {} + ufo@1.6.3: {} uhyphen@0.2.0: {} diff --git a/server/configs/config.local.yaml.example b/server/configs/config.local.yaml.example index 9316f0f..fdca851 100644 --- a/server/configs/config.local.yaml.example +++ b/server/configs/config.local.yaml.example @@ -36,6 +36,7 @@ llm: provider: ark base_url: https://ark.cn-beijing.volces.com/api/v3 model: doubao-seed-2-0-lite-260215 + knowledge_url_model: doubao-seed-2-0-mini-260215 timeout: 2m max_output_tokens: 4000 temperature: 0.7 diff --git a/server/configs/config.yaml b/server/configs/config.yaml index 4472d75..b14df68 100644 --- a/server/configs/config.yaml +++ b/server/configs/config.yaml @@ -49,6 +49,7 @@ llm: base_url: https://ark.cn-beijing.volces.com/api/v3 api_key: "7fb6c66b-129c-4935-9617-709236c4205a" model: doubao-seed-2-0-lite-260215 + knowledge_url_model: doubao-seed-2-0-mini-260215 timeout: 2m max_output_tokens: 16000 temperature: 0.7 diff --git a/server/go.mod b/server/go.mod index 42fa287..a5baccb 100644 --- a/server/go.mod +++ b/server/go.mod @@ -28,6 +28,8 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7 // indirect + github.com/extrame/xls v0.0.1 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect diff --git a/server/go.sum b/server/go.sum index 1c7c495..5d2907f 100644 --- a/server/go.sum +++ b/server/go.sum @@ -27,6 +27,10 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7 h1:n+nk0bNe2+gVbRI8WRbLFVwwcBQ0rr5p+gzkKb6ol8c= +github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7/go.mod h1:GPpMrAfHdb8IdQ1/R2uIRBsNfnPnwsYE9YYI5WyY1zw= +github.com/extrame/xls v0.0.1 h1:jI7L/o3z73TyyENPopsLS/Jlekm3nF1a/kF5hKBvy/k= +github.com/extrame/xls v0.0.1/go.mod h1:iACcgahst7BboCpIMSpnFs4SKyU9ZjsvZBfNbUxZOJI= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= diff --git a/server/internal/shared/config/config.go b/server/internal/shared/config/config.go index 422bea0..8668115 100644 --- a/server/internal/shared/config/config.go +++ b/server/internal/shared/config/config.go @@ -79,16 +79,17 @@ type LogConfig struct { } type LLMConfig struct { - Provider string `mapstructure:"provider"` - APIKey string `mapstructure:"api_key"` - BaseURL string `mapstructure:"base_url"` - Model string `mapstructure:"model"` - Timeout time.Duration `mapstructure:"timeout"` - MaxOutputTokens int64 `mapstructure:"max_output_tokens"` - Temperature float64 `mapstructure:"temperature"` - TopP float64 `mapstructure:"top_p"` - ReasoningEffort string `mapstructure:"reasoning_effort"` - WebSearchLimit int32 `mapstructure:"web_search_limit"` + Provider string `mapstructure:"provider"` + APIKey string `mapstructure:"api_key"` + BaseURL string `mapstructure:"base_url"` + Model string `mapstructure:"model"` + KnowledgeURLModel string `mapstructure:"knowledge_url_model"` + Timeout time.Duration `mapstructure:"timeout"` + MaxOutputTokens int64 `mapstructure:"max_output_tokens"` + Temperature float64 `mapstructure:"temperature"` + TopP float64 `mapstructure:"top_p"` + ReasoningEffort string `mapstructure:"reasoning_effort"` + WebSearchLimit int32 `mapstructure:"web_search_limit"` } type RetrievalConfig struct { @@ -150,6 +151,9 @@ func applyEnvOverrides(cfg *Config) { if apiKey, ok := lookupNonEmptyEnv("LLM_API_KEY"); ok { cfg.LLM.APIKey = apiKey } + if model, ok := lookupNonEmptyEnv("LLM_KNOWLEDGE_URL_MODEL"); ok { + cfg.LLM.KnowledgeURLModel = model + } if apiKey, ok := lookupNonEmptyEnv("QDRANT_API_KEY"); ok { cfg.Qdrant.APIKey = apiKey } diff --git a/server/internal/shared/llm/ark.go b/server/internal/shared/llm/ark.go index c04b7d8..dcaf0fc 100644 --- a/server/internal/shared/llm/ark.go +++ b/server/internal/shared/llm/ark.go @@ -113,6 +113,11 @@ func (c *arkClient) Generate(ctx context.Context, req GenerateRequest, onDelta f maxOutputTokens = req.MaxOutputTokens } + model := c.model + if strings.TrimSpace(req.Model) != "" { + model = strings.TrimSpace(req.Model) + } + callCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() @@ -154,7 +159,7 @@ func (c *arkClient) Generate(ctx context.Context, req GenerateRequest, onDelta f } stream, err := c.client.CreateResponsesStream(callCtx, &responses.ResponsesRequest{ - Model: c.model, + Model: model, MaxOutputTokens: &maxOutputTokens, Store: &store, Stream: &streamValue, @@ -232,7 +237,7 @@ func (c *arkClient) Generate(ctx context.Context, req GenerateRequest, onDelta f return &GenerateResult{ Content: result, - Model: c.model, + Model: model, }, nil } diff --git a/server/internal/shared/llm/client.go b/server/internal/shared/llm/client.go index 688b0ba..7c4ca03 100644 --- a/server/internal/shared/llm/client.go +++ b/server/internal/shared/llm/client.go @@ -13,6 +13,7 @@ import ( var ErrNotConfigured = errors.New("llm provider is not configured") type GenerateRequest struct { + Model string Prompt string Timeout time.Duration MaxOutputTokens int64 diff --git a/server/internal/tenant/app/knowledge_chunking.go b/server/internal/tenant/app/knowledge_chunking.go new file mode 100644 index 0000000..012f106 --- /dev/null +++ b/server/internal/tenant/app/knowledge_chunking.go @@ -0,0 +1,122 @@ +package app + +import "strings" + +func splitKnowledgeTextIntoChunks(text string, chunkSize, chunkOverlap int) []string { + text = strings.TrimSpace(text) + if text == "" { + return nil + } + + if chunkSize <= 0 { + chunkSize = 900 + } + if chunkOverlap < 0 { + chunkOverlap = 0 + } + if chunkOverlap >= chunkSize { + chunkOverlap = chunkSize / 4 + } + + lines := strings.Split(text, "\n") + chunks := make([]string, 0, len(lines)) + current := make([]string, 0, 16) + currentLen := 0 + + resetWithOverlap := func(chunk string) { + current = current[:0] + currentLen = 0 + if chunkOverlap <= 0 { + return + } + overlap := takeLastKnowledgeRunes(chunk, chunkOverlap) + if overlap == "" { + return + } + current = append(current, overlap) + currentLen = len([]rune(overlap)) + } + + flush := func() { + if len(current) == 0 { + return + } + chunk := strings.TrimSpace(strings.Join(current, "\n")) + if chunk == "" { + current = current[:0] + currentLen = 0 + return + } + chunks = append(chunks, chunk) + resetWithOverlap(chunk) + } + + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + lineRunes := len([]rune(line)) + if lineRunes > chunkSize { + flush() + for _, segment := range splitKnowledgeLongLine(line, chunkSize) { + if strings.TrimSpace(segment) == "" { + continue + } + chunks = append(chunks, segment) + resetWithOverlap(segment) + } + continue + } + + nextLen := currentLen + lineRunes + if len(current) > 0 { + nextLen++ + } + if nextLen > chunkSize { + flush() + } + + current = append(current, line) + currentLen += lineRunes + if len(current) > 1 { + currentLen++ + } + } + + if len(current) > 0 { + chunk := strings.TrimSpace(strings.Join(current, "\n")) + if chunk != "" { + chunks = append(chunks, chunk) + } + } + + return chunks +} + +func takeLastKnowledgeRunes(text string, size int) string { + if size <= 0 { + return "" + } + + runes := []rune(strings.TrimSpace(text)) + if len(runes) <= size { + return string(runes) + } + return strings.TrimSpace(string(runes[len(runes)-size:])) +} + +func estimateKnowledgeTokenCount(text string) int { + text = strings.TrimSpace(text) + if text == "" { + return 0 + } + + runeCount := len([]rune(text)) + tokenCount := runeCount / 2 + if tokenCount == 0 { + return 1 + } + return tokenCount +} diff --git a/server/internal/tenant/app/knowledge_parser.go b/server/internal/tenant/app/knowledge_parser.go index 73181ec..d51c9ba 100644 --- a/server/internal/tenant/app/knowledge_parser.go +++ b/server/internal/tenant/app/knowledge_parser.go @@ -3,76 +3,113 @@ package app import ( "archive/zip" "bytes" - "context" "encoding/xml" "fmt" "io" - "net/http" "path/filepath" "regexp" "strings" - "time" "unicode/utf8" + "github.com/extrame/xls" pdf "github.com/ledongthuc/pdf" "github.com/xuri/excelize/v2" "golang.org/x/net/html" ) -const ( - maxKnowledgeFetchBytes = 5 << 20 +var ( + whitespacePattern = regexp.MustCompile(`[ \t]+`) + genericKnowledgeMarkdownTitleRegex = regexp.MustCompile(`^第\s*\d+\s*部分$`) ) -var whitespacePattern = regexp.MustCompile(`[ \t]+`) +const maxKnowledgeSpreadsheetRows = 100000 -func extractKnowledgeTextFromURL(ctx context.Context, rawURL string) (string, error) { - data, contentType, err := fetchKnowledgeURLSnapshot(ctx, rawURL) - if err != nil { - return "", err - } - return extractKnowledgeTextFromSnapshot(resolveSnapshotFileName(rawURL, contentType), data) +type knowledgeParsedContent struct { + Text string + Markdown string } -func fetchKnowledgeURLSnapshot(ctx context.Context, rawURL string) ([]byte, string, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimSpace(rawURL), nil) - if err != nil { - return nil, "", fmt.Errorf("create request: %w", err) +func parseKnowledgeTextInput(name, content string) *knowledgeParsedContent { + text := normalizeKnowledgeText(content) + return &knowledgeParsedContent{ + Text: text, + Markdown: buildKnowledgeMarkdownFromText(name, text), } - req.Header.Set("User-Agent", "geo-knowledge-fetcher/1.0") - - client := &http.Client{Timeout: 20 * time.Second} - resp, err := client.Do(req) - if err != nil { - return nil, "", fmt.Errorf("fetch url: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode >= http.StatusBadRequest { - return nil, "", fmt.Errorf("fetch url failed: status=%d", resp.StatusCode) - } - - data, err := io.ReadAll(io.LimitReader(resp.Body, maxKnowledgeFetchBytes)) - if err != nil { - return nil, "", fmt.Errorf("read url body: %w", err) - } - - contentType := strings.ToLower(resp.Header.Get("Content-Type")) - return data, contentType, nil } -func resolveSnapshotFileName(rawURL string, contentType string) string { - if strings.Contains(strings.ToLower(contentType), "text/plain") { - return rawURL + ".txt" - } - return rawURL + ".html" -} - -func extractKnowledgeTextFromSnapshot(fileName string, content []byte) (string, error) { +func extractKnowledgeContentFromFile(fileName string, content []byte) (*knowledgeParsedContent, error) { ext := strings.ToLower(filepath.Ext(fileName)) - if ext != ".html" && ext != ".htm" { - return extractKnowledgeTextFromFile(fileName, content) + + switch ext { + case ".txt", ".json", ".csv": + return extractUTF8KnowledgeText(fileName, content, false) + case ".md", ".markdown": + return extractUTF8KnowledgeText(fileName, content, true) + case ".html", ".htm": + text, err := extractHTMLText(content) + if err != nil { + return nil, err + } + return &knowledgeParsedContent{ + Text: text, + Markdown: buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text), + }, nil + case ".docx": + text, err := extractDOCXText(content) + if err != nil { + return nil, err + } + return &knowledgeParsedContent{ + Text: text, + Markdown: buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text), + }, nil + case ".xlsx": + return extractXLSXContent(content) + case ".xls": + return extractXLSContent(content) + case ".pdf": + text, err := extractPDFText(content) + if err != nil { + return nil, err + } + return &knowledgeParsedContent{ + Text: text, + Markdown: buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text), + }, nil + default: + if utf8.Valid(content) { + return extractUTF8KnowledgeText(fileName, content, false) + } + return nil, fmt.Errorf("unsupported file type %s", ext) + } +} + +func extractUTF8KnowledgeText(fileName string, content []byte, preserveMarkdown bool) (*knowledgeParsedContent, error) { + if !utf8.Valid(content) { + return nil, fmt.Errorf("file %s is not valid UTF-8 text", fileName) } + raw := string(content) + text := normalizeKnowledgeText(raw) + if text == "" { + return nil, fmt.Errorf("file %s contains no readable text", fileName) + } + + markdown := buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text) + if preserveMarkdown { + normalizedMarkdown := normalizeKnowledgeMarkdown(raw) + if normalizedMarkdown != "" { + markdown = normalizedMarkdown + } + } + + return &knowledgeParsedContent{ + Text: text, + Markdown: markdown, + }, nil +} + +func extractHTMLText(content []byte) (string, error) { doc, err := html.Parse(bytes.NewReader(content)) if err != nil { return "", fmt.Errorf("parse html: %w", err) @@ -106,32 +143,11 @@ func extractKnowledgeTextFromSnapshot(fileName string, content []byte) (string, } walk(doc, false) - return normalizeKnowledgeText(strings.Join(parts, "\n")), nil -} - -func extractKnowledgeTextFromFile(fileName string, content []byte) (string, error) { - ext := strings.ToLower(filepath.Ext(fileName)) - - switch ext { - case ".txt", ".md", ".markdown", ".json", ".csv", ".html", ".htm": - if !utf8.Valid(content) { - return "", fmt.Errorf("file %s is not valid UTF-8 text", fileName) - } - return normalizeKnowledgeText(string(content)), nil - case ".docx": - return extractDOCXText(content) - case ".xlsx": - return extractXLSXText(content) - case ".pdf": - return extractPDFText(content) - case ".doc", ".xls": - return "", fmt.Errorf("file type %s is not supported yet", ext) - default: - if utf8.Valid(content) { - return normalizeKnowledgeText(string(content)), nil - } - return "", fmt.Errorf("unsupported file type %s", ext) + result := normalizeKnowledgeText(strings.Join(parts, "\n")) + if result == "" { + return "", fmt.Errorf("html contains no readable text") } + return result, nil } func extractDOCXText(content []byte) (string, error) { @@ -173,46 +189,177 @@ func extractDOCXText(content []byte) (string, error) { } } } + if len(parts) == 0 { return "", fmt.Errorf("docx contains no readable text") } return normalizeKnowledgeText(strings.Join(parts, "\n")), nil } -func extractXLSXText(content []byte) (string, error) { +func extractXLSXContent(content []byte) (*knowledgeParsedContent, error) { book, err := excelize.OpenReader(bytes.NewReader(content)) if err != nil { - return "", fmt.Errorf("open xlsx: %w", err) + return nil, fmt.Errorf("open xlsx: %w", err) } defer func() { _ = book.Close() }() - lines := make([]string, 0) + markdownSections := make([]string, 0) + textSections := make([]string, 0) for _, sheet := range book.GetSheetList() { rows, err := book.GetRows(sheet) if err != nil { - return "", fmt.Errorf("read xlsx rows: %w", err) + return nil, fmt.Errorf("read xlsx rows: %w", err) } - if len(rows) == 0 { + + cleaned := normalizeSpreadsheetRows(rows) + if len(cleaned) == 0 { continue } - lines = append(lines, sheet) - for _, row := range rows { - cells := make([]string, 0, len(row)) - for _, cell := range row { - cell = strings.TrimSpace(cell) - if cell != "" { - cells = append(cells, cell) - } - } - if len(cells) > 0 { - lines = append(lines, strings.Join(cells, " | ")) - } + + markdownSections = append(markdownSections, buildKnowledgeSheetMarkdown(sheet, cleaned)) + textSections = append(textSections, buildKnowledgeSheetPlainText(sheet, cleaned)) + } + + if len(markdownSections) == 0 { + return nil, fmt.Errorf("xlsx contains no readable text") + } + + return &knowledgeParsedContent{ + Text: normalizeKnowledgeText(strings.Join(textSections, "\n\n")), + Markdown: strings.TrimSpace(strings.Join(markdownSections, "\n\n")), + }, nil +} + +func extractXLSContent(content []byte) (*knowledgeParsedContent, error) { + book, err := xls.OpenReader(bytes.NewReader(content), "utf-8") + if err != nil { + return nil, fmt.Errorf("open xls: %w", err) + } + + // The upstream xls library panics for sparse rows when calling sheet.Row(i), + // so we rely on its safer bulk reader here instead of random row access. + rows := book.ReadAllCells(maxKnowledgeSpreadsheetRows) + cleaned := normalizeSpreadsheetRows(rows) + if len(cleaned) == 0 { + return nil, fmt.Errorf("xls contains no readable text") + } + + title := "XLS 工作簿" + if book.NumSheets() == 1 { + if sheet := book.GetSheet(0); sheet != nil && strings.TrimSpace(sheet.Name) != "" { + title = strings.TrimSpace(sheet.Name) } } - if len(lines) == 0 { - return "", fmt.Errorf("xlsx contains no readable text") + + return &knowledgeParsedContent{ + Text: normalizeKnowledgeText(buildKnowledgeSheetPlainText(title, cleaned)), + Markdown: buildKnowledgeSheetMarkdown(title, cleaned), + }, nil +} + +func normalizeSpreadsheetRows(rows [][]string) [][]string { + cleaned := make([][]string, 0, len(rows)) + for _, row := range rows { + cells := make([]string, len(row)) + copy(cells, row) + + for index := range cells { + cells[index] = normalizeKnowledgeInlineText(cells[index]) + } + + lastNonEmpty := -1 + for index, cell := range cells { + if cell != "" { + lastNonEmpty = index + } + } + if lastNonEmpty < 0 { + continue + } + + cleaned = append(cleaned, cells[:lastNonEmpty+1]) } - return normalizeKnowledgeText(strings.Join(lines, "\n")), nil + return cleaned +} + +func buildKnowledgeSheetPlainText(sheetName string, rows [][]string) string { + lines := make([]string, 0, len(rows)+1) + if strings.TrimSpace(sheetName) != "" { + lines = append(lines, strings.TrimSpace(sheetName)) + } + for _, row := range rows { + lines = append(lines, strings.Join(row, " | ")) + } + return strings.Join(lines, "\n") +} + +func buildKnowledgeSheetMarkdown(sheetName string, rows [][]string) string { + if len(rows) == 0 { + return "" + } + + width := 0 + for _, row := range rows { + if len(row) > width { + width = len(row) + } + } + if width == 0 { + return "" + } + + normalizedRows := make([][]string, 0, len(rows)) + for _, row := range rows { + normalizedRows = append(normalizedRows, padKnowledgeRow(row, width)) + } + + var builder strings.Builder + if title := strings.TrimSpace(sheetName); title != "" { + builder.WriteString("## ") + builder.WriteString(title) + builder.WriteString("\n\n") + } + + writeKnowledgeMarkdownTableRow(&builder, normalizedRows[0]) + writeKnowledgeMarkdownTableSeparator(&builder, width) + for _, row := range normalizedRows[1:] { + writeKnowledgeMarkdownTableRow(&builder, row) + } + + return strings.TrimSpace(builder.String()) +} + +func padKnowledgeRow(row []string, width int) []string { + result := make([]string, width) + copy(result, row) + return result +} + +func writeKnowledgeMarkdownTableRow(builder *strings.Builder, row []string) { + builder.WriteString("|") + for _, cell := range row { + builder.WriteString(" ") + builder.WriteString(escapeKnowledgeMarkdownTableCell(cell)) + builder.WriteString(" |") + } + builder.WriteString("\n") +} + +func writeKnowledgeMarkdownTableSeparator(builder *strings.Builder, width int) { + builder.WriteString("|") + for index := 0; index < width; index++ { + builder.WriteString(" --- |") + } + builder.WriteString("\n") +} + +func escapeKnowledgeMarkdownTableCell(cell string) string { + cell = strings.TrimSpace(cell) + cell = strings.ReplaceAll(cell, "\\", "\\\\") + cell = strings.ReplaceAll(cell, "|", "\\|") + cell = strings.ReplaceAll(cell, "\r", " ") + cell = strings.ReplaceAll(cell, "\n", " / ") + return cell } func extractPDFText(content []byte) (string, error) { @@ -223,7 +370,7 @@ func extractPDFText(content []byte) (string, error) { var builder strings.Builder totalPage := reader.NumPage() - for pageIndex := 1; pageIndex <= totalPage; pageIndex += 1 { + for pageIndex := 1; pageIndex <= totalPage; pageIndex++ { page := reader.Page(pageIndex) if page.V.IsNull() { continue @@ -245,12 +392,116 @@ func extractPDFText(content []byte) (string, error) { return result, nil } -func normalizeKnowledgeText(input string) string { +func defaultKnowledgeTitle(fileName string) string { + base := filepath.Base(strings.TrimSpace(fileName)) + if base == "" { + return "" + } + return strings.TrimSuffix(base, filepath.Ext(base)) +} + +func buildKnowledgeMarkdownFromText(title, text string) string { + text = normalizeKnowledgeText(text) + title = strings.TrimSpace(title) + if text == "" { + if title == "" { + return "" + } + return "# " + title + } + + lines := strings.Split(text, "\n") + if title != "" && len(lines) > 0 && strings.EqualFold(strings.TrimSpace(lines[0]), title) { + lines = lines[1:] + } + + paragraphs := make([]string, 0, len(lines)) + for _, line := range lines { + line = strings.TrimSpace(line) + if line != "" { + paragraphs = append(paragraphs, line) + } + } + + sections := make([]string, 0, 2) + if title != "" { + sections = append(sections, "# "+title) + } + if len(paragraphs) > 0 { + sections = append(sections, strings.Join(paragraphs, "\n\n")) + } + return strings.TrimSpace(strings.Join(sections, "\n\n")) +} + +func extractKnowledgeMarkdownTitle(markdown string) string { + markdown = normalizeKnowledgeMarkdown(markdown) + if markdown == "" { + return "" + } + + for _, line := range strings.Split(markdown, "\n") { + line = strings.TrimSpace(line) + if line == "" || !strings.HasPrefix(line, "#") { + continue + } + + level := 0 + for level < len(line) && line[level] == '#' { + level++ + } + if level == len(line) || line[level] != ' ' { + continue + } + + title := strings.Join(strings.Fields(strings.TrimSpace(line[level+1:])), " ") + title = strings.Trim(title, "# \t") + if title == "" || genericKnowledgeMarkdownTitleRegex.MatchString(title) { + continue + } + switch title { + case "摘要", "概述", "概要", "目录", "正文": + continue + } + + runes := []rune(title) + if len(runes) > 200 { + title = string(runes[:200]) + } + return title + } + + return "" +} + +func normalizeKnowledgeMarkdown(input string) string { + input = strings.ReplaceAll(input, "\r\n", "\n") input = strings.ReplaceAll(input, "\u00a0", " ") + lines := strings.Split(input, "\n") cleaned := make([]string, 0, len(lines)) + lastBlank := false for _, line := range lines { - line = whitespacePattern.ReplaceAllString(strings.TrimSpace(line), " ") + line = strings.TrimRight(line, " \t") + if strings.TrimSpace(line) == "" { + if lastBlank { + continue + } + lastBlank = true + cleaned = append(cleaned, "") + continue + } + lastBlank = false + cleaned = append(cleaned, line) + } + return strings.TrimSpace(strings.Join(cleaned, "\n")) +} + +func normalizeKnowledgeText(input string) string { + input = strings.ReplaceAll(input, "\u00a0", " ") + lines := strings.Split(strings.ReplaceAll(input, "\r\n", "\n"), "\n") + cleaned := make([]string, 0, len(lines)) + for _, line := range lines { + line = normalizeKnowledgeInlineText(line) if line != "" { cleaned = append(cleaned, line) } @@ -258,58 +509,6 @@ func normalizeKnowledgeText(input string) string { return strings.TrimSpace(strings.Join(cleaned, "\n")) } -func splitKnowledgeTextIntoChunks(text string, chunkSize, chunkOverlap int) []string { - text = normalizeKnowledgeText(text) - if text == "" { - return nil - } - - runes := []rune(text) - if chunkSize <= 0 { - chunkSize = 900 - } - if chunkOverlap < 0 { - chunkOverlap = 0 - } - if chunkOverlap >= chunkSize { - chunkOverlap = chunkSize / 4 - } - - chunks := make([]string, 0) - for start := 0; start < len(runes); { - end := start + chunkSize - if end > len(runes) { - end = len(runes) - } - - if end < len(runes) { - for offset := end; offset > start+chunkSize/2; offset -= 1 { - if runes[offset-1] == '\n' || runes[offset-1] == '。' || runes[offset-1] == '.' { - end = offset - break - } - } - } - - chunk := strings.TrimSpace(string(runes[start:end])) - if chunk != "" { - chunks = append(chunks, chunk) - } - if end >= len(runes) { - break - } - start = end - chunkOverlap - if start < 0 { - start = 0 - } - } - - return chunks -} - -func estimateKnowledgeTokenCount(text string) int { - if text == "" { - return 0 - } - return utf8.RuneCountInString(text) +func normalizeKnowledgeInlineText(input string) string { + return whitespacePattern.ReplaceAllString(strings.TrimSpace(input), " ") } diff --git a/server/internal/tenant/app/knowledge_service.go b/server/internal/tenant/app/knowledge_service.go index f90ab69..25a2e28 100644 --- a/server/internal/tenant/app/knowledge_service.go +++ b/server/internal/tenant/app/knowledge_service.go @@ -4,8 +4,10 @@ import ( "context" "errors" "fmt" + "net/http" "net/url" "path/filepath" + "runtime/debug" "slices" "strings" "sync" @@ -18,6 +20,7 @@ import ( "github.com/geo-platform/tenant-api/internal/shared/auth" "github.com/geo-platform/tenant-api/internal/shared/config" + "github.com/geo-platform/tenant-api/internal/shared/llm" "github.com/geo-platform/tenant-api/internal/shared/objectstorage" "github.com/geo-platform/tenant-api/internal/shared/response" "github.com/geo-platform/tenant-api/internal/shared/retrieval" @@ -25,8 +28,9 @@ import ( ) const ( - defaultKnowledgeQueueSize = 128 - defaultKnowledgeCleanupPoll = 15 * time.Second + defaultKnowledgeQueueSize = 128 + defaultKnowledgeCleanupPoll = 15 * time.Second + defaultKnowledgeChildGroupName = "默认目录" ) type KnowledgeService struct { @@ -34,12 +38,17 @@ type KnowledgeService struct { provider retrieval.Provider store retrieval.VectorStore objectStorage objectstorage.Client + llmClient llm.Client logger *zap.Logger chunkSize int chunkOverlap int embeddingBatchSize int recallLimit int rerankTopN int + arkBaseURL string + arkAPIKey string + urlMarkdownModel string + httpClient *http.Client jobs chan knowledgeParseJob cleanupMu sync.Mutex cleanupInFlight map[string]struct{} @@ -66,6 +75,7 @@ type knowledgeItemRecord struct { SourceURI *string StorageKey string ContentText *string + Markdown *string ItemVersion int } @@ -99,6 +109,21 @@ type KnowledgeItemResponse struct { UpdatedAt string `json:"updated_at"` } +type KnowledgeItemDetailResponse struct { + ID int64 `json:"id"` + GroupID int64 `json:"group_id"` + GroupName string `json:"group_name"` + Name string `json:"name"` + SourceType string `json:"source_type"` + SourceURI *string `json:"source_uri"` + Status string `json:"status"` + SizeBytes int64 `json:"size_bytes"` + ErrorMessage *string `json:"error_message"` + MarkdownContent *string `json:"markdown_content"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + type KnowledgeTextItemRequest struct { GroupID int64 `json:"group_id" binding:"required"` Name string `json:"name" binding:"required"` @@ -132,8 +157,10 @@ func NewKnowledgeService( provider retrieval.Provider, store retrieval.VectorStore, objectStorage objectstorage.Client, + llmClient llm.Client, logger *zap.Logger, cfg config.RetrievalConfig, + llmCfg config.LLMConfig, queueSize int, workerCount int, ) *KnowledgeService { @@ -163,14 +190,25 @@ func NewKnowledgeService( provider: provider, store: store, objectStorage: objectStorage, + llmClient: llmClient, logger: logger, chunkSize: chunkSize, chunkOverlap: chunkOverlap, embeddingBatchSize: batchSize, recallLimit: recallLimit, rerankTopN: rerankTopN, + arkBaseURL: strings.TrimSpace(llmCfg.BaseURL), + arkAPIKey: strings.TrimSpace(llmCfg.APIKey), + urlMarkdownModel: strings.TrimSpace(llmCfg.KnowledgeURLModel), + httpClient: &http.Client{Timeout: defaultKnowledgeURLParseTimeout}, cleanupInFlight: make(map[string]struct{}), } + if svc.urlMarkdownModel == "" { + svc.urlMarkdownModel = strings.TrimSpace(llmCfg.Model) + } + if svc.arkBaseURL == "" { + svc.arkBaseURL = defaultKnowledgeArkBaseURL + } if workerCount > 0 { if queueSize <= 0 { @@ -521,15 +559,58 @@ func (s *KnowledgeService) ListItems(ctx context.Context, groupID *int64) ([]Kno return items, nil } +func (s *KnowledgeService) GetItemDetail(ctx context.Context, itemID int64) (*KnowledgeItemDetailResponse, error) { + actor := auth.MustActor(ctx) + + var ( + item KnowledgeItemDetailResponse + sourceURI *string + errorMessage *string + markdownContent *string + createdAt, updatedAt time.Time + ) + + err := s.pool.QueryRow(ctx, ` + SELECT ki.id, ki.group_id, kg.name, ki.name, ki.source_type, ki.source_uri, ki.status, + ki.size_bytes, ki.error_message, ki.markdown_content, ki.created_at, ki.updated_at + FROM knowledge_items ki + JOIN knowledge_groups kg ON kg.id = ki.group_id AND kg.deleted_at IS NULL + WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NULL + `, itemID, actor.TenantID).Scan( + &item.ID, + &item.GroupID, + &item.GroupName, + &item.Name, + &item.SourceType, + &sourceURI, + &item.Status, + &item.SizeBytes, + &errorMessage, + &markdownContent, + &createdAt, + &updatedAt, + ) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, response.ErrNotFound(40450, "knowledge_item_not_found", "knowledge item not found") + } + return nil, response.ErrInternal(50051, "knowledge_items_query_failed", "failed to load knowledge item") + } + + item.SourceURI = sourceURI + item.ErrorMessage = errorMessage + item.MarkdownContent = markdownContent + item.CreatedAt = createdAt.Format(time.RFC3339) + item.UpdatedAt = updatedAt.Format(time.RFC3339) + return &item, nil +} + func (s *KnowledgeService) CreateTextItem(ctx context.Context, req KnowledgeTextItemRequest) (*KnowledgeItemResponse, error) { actor := auth.MustActor(ctx) if err := s.validateIngestionDependencies("text"); err != nil { return nil, err } - if err := s.ensureStorableGroup(ctx, actor.TenantID, req.GroupID); err != nil { - return nil, err - } content := normalizeKnowledgeText(req.Content) if content == "" { @@ -551,33 +632,17 @@ func (s *KnowledgeService) CreateURLItem(ctx context.Context, req KnowledgeURLIt if err := s.validateIngestionDependencies("website"); err != nil { return nil, err } - if err := s.ensureStorableGroup(ctx, actor.TenantID, req.GroupID); err != nil { - return nil, err - } rawURL := strings.TrimSpace(req.URL) - parsed, err := url.ParseRequestURI(rawURL) - if err != nil { + if _, err := url.ParseRequestURI(rawURL); err != nil { return nil, response.ErrBadRequest(40052, "knowledge_url_invalid", "knowledge url is invalid") } - snapshot, contentType, err := fetchKnowledgeURLSnapshot(ctx, parsed.String()) - if err != nil { - return nil, response.ErrBadRequest(40052, "knowledge_url_fetch_failed", err.Error()) - } - - objectKey := buildKnowledgeObjectKey(actor.TenantID, "website", resolveSnapshotFileName(parsed.String(), contentType)) - if err := s.objectStorage.PutBytes(ctx, objectKey, snapshot, contentType); err != nil { - return nil, response.ErrServiceUnavailable(50357, "knowledge_storage_unavailable", err.Error()) - } - return s.queueKnowledgeItem(ctx, actor.TenantID, knowledgeIngestInput{ GroupID: req.GroupID, Name: strings.TrimSpace(req.Name), SourceType: "website", SourceURI: &rawURL, - StorageKey: objectKey, - SizeBytes: int64(len(snapshot)), }) } @@ -593,9 +658,6 @@ func (s *KnowledgeService) CreateFileItem( if err := s.validateIngestionDependencies("document"); err != nil { return nil, err } - if err := s.ensureStorableGroup(ctx, actor.TenantID, groupID); err != nil { - return nil, err - } displayName := strings.TrimSpace(name) if displayName == "" { @@ -819,11 +881,19 @@ func (s *KnowledgeService) validateIngestionDependencies(sourceType string) erro if err := s.store.Validate(); err != nil { return response.ErrServiceUnavailable(50352, "qdrant_unavailable", err.Error()) } - if sourceType == "document" || sourceType == "website" { + if sourceType == "document" { if err := s.objectStorage.Validate(); err != nil { return response.ErrServiceUnavailable(50357, "knowledge_storage_unavailable", err.Error()) } } + if sourceType == "website" { + if strings.TrimSpace(s.arkAPIKey) == "" { + return response.ErrServiceUnavailable(50358, "knowledge_webpage_parser_unavailable", "ark api key is not configured") + } + if err := s.llmClient.Validate(); err != nil { + return response.ErrServiceUnavailable(50359, "knowledge_markdown_formatter_unavailable", err.Error()) + } + } return nil } @@ -832,9 +902,11 @@ func (s *KnowledgeService) queueKnowledgeItem( tenantID int64, input knowledgeIngestInput, ) (*KnowledgeItemResponse, error) { - if err := s.ensureStorableGroup(ctx, tenantID, input.GroupID); err != nil { + resolvedGroupID, err := s.resolveKnowledgeStorageGroupID(ctx, tenantID, input.GroupID) + if err != nil { return nil, err } + input.GroupID = resolvedGroupID input.Name = strings.TrimSpace(input.Name) if input.Name == "" { @@ -888,7 +960,25 @@ func (s *KnowledgeService) enqueueParseJob(job knowledgeParseJob) error { func (s *KnowledgeService) runParseWorker() { for job := range s.jobs { - s.executeParseJob(context.Background(), job) + func(job knowledgeParseJob) { + defer func() { + if recovered := recover(); recovered != nil { + err := fmt.Errorf("knowledge parse panic: %v", recovered) + if s.logger != nil { + s.logger.Error("knowledge parse worker panicked", + zap.Int64("tenant_id", job.TenantID), + zap.Int64("knowledge_item_id", job.ItemID), + zap.Int64("parse_task_id", job.ParseTaskID), + zap.Any("panic", recovered), + zap.ByteString("stack", debug.Stack()), + ) + } + _ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err) + } + }() + + s.executeParseJob(context.Background(), job) + }(job) } } @@ -936,13 +1026,18 @@ func (s *KnowledgeService) executeParseJob(ctx context.Context, job knowledgePar return } - content, err := s.readKnowledgeItemContent(ctx, item) + parsedContent, err := s.readKnowledgeItemParsedContent(ctx, item) if err != nil { _ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err) return } + if item.SourceType == "website" { + if resolvedName := extractKnowledgeMarkdownTitle(parsedContent.Markdown); resolvedName != "" { + item.Name = resolvedName + } + } - chunks := splitKnowledgeTextIntoChunks(content, s.chunkSize, s.chunkOverlap) + chunks := splitKnowledgeTextIntoChunks(parsedContent.Text, s.chunkSize, s.chunkOverlap) if len(chunks) == 0 { err = errors.New("knowledge content is empty after parsing") _ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err) @@ -1018,7 +1113,7 @@ func (s *KnowledgeService) executeParseJob(ctx context.Context, job knowledgePar return } - if err = s.markKnowledgeIngestionCompleted(ctx, job.TenantID, job.ItemID, job.ParseTaskID, metas); err != nil { + if err = s.markKnowledgeIngestionCompleted(ctx, job.TenantID, job.ItemID, job.ParseTaskID, item.Name, parsedContent, metas); err != nil { _ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err) return } @@ -1030,9 +1125,10 @@ func (s *KnowledgeService) loadKnowledgeItemForParse( ) (*knowledgeItemRecord, error) { record := &knowledgeItemRecord{} var contentText *string + var markdownContent *string err := s.pool.QueryRow(ctx, ` SELECT ki.id, ki.tenant_id, ki.group_id, kg.name, ki.name, ki.source_type, - ki.source_uri, ki.storage_key, ki.content_text, ki.item_version + ki.source_uri, ki.storage_key, ki.content_text, ki.markdown_content, ki.item_version FROM knowledge_items ki JOIN knowledge_groups kg ON kg.id = ki.group_id WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NULL @@ -1046,37 +1142,38 @@ func (s *KnowledgeService) loadKnowledgeItemForParse( &record.SourceURI, &record.StorageKey, &contentText, + &markdownContent, &record.ItemVersion, ) if err != nil { return nil, nil } record.ContentText = contentText + record.Markdown = markdownContent return record, nil } -func (s *KnowledgeService) readKnowledgeItemContent(ctx context.Context, item *knowledgeItemRecord) (string, error) { +func (s *KnowledgeService) readKnowledgeItemParsedContent(ctx context.Context, item *knowledgeItemRecord) (*knowledgeParsedContent, error) { if item == nil { - return "", fmt.Errorf("knowledge item is empty") + return nil, fmt.Errorf("knowledge item is empty") } switch item.SourceType { case "text": - return normalizeKnowledgeText(derefKnowledgeString(item.ContentText)), nil + return parseKnowledgeTextInput(item.Name, derefKnowledgeString(item.ContentText)), nil case "document": data, err := s.objectStorage.GetBytes(ctx, item.StorageKey) if err != nil { - return "", err + return nil, err } - return extractKnowledgeTextFromFile(item.StorageKey, data) + return extractKnowledgeContentFromFile(item.StorageKey, data) case "website": - data, err := s.objectStorage.GetBytes(ctx, item.StorageKey) - if err != nil { - return "", err + if item.SourceURI == nil || strings.TrimSpace(*item.SourceURI) == "" { + return nil, fmt.Errorf("knowledge website url is empty") } - return extractKnowledgeTextFromSnapshot(item.StorageKey, data) + return s.parseWebsiteKnowledge(ctx, strings.TrimSpace(*item.SourceURI)) default: - return "", fmt.Errorf("unsupported knowledge source type %q", item.SourceType) + return nil, fmt.Errorf("unsupported knowledge source type %q", item.SourceType) } } @@ -1103,9 +1200,9 @@ func (s *KnowledgeService) createPendingKnowledgeItem( err = tx.QueryRow(ctx, ` INSERT INTO knowledge_items ( tenant_id, group_id, source_type, name, source_uri, storage_key, - content_text, status, size_bytes, item_version + content_text, markdown_content, status, size_bytes, item_version ) - VALUES ($1, $2, $3, $4, $5, $6, $7, 'pending', $8, 1) + VALUES ($1, $2, $3, $4, $5, $6, $7, NULL, 'pending', $8, 1) RETURNING id `, tenantID, input.GroupID, input.SourceType, input.Name, input.SourceURI, input.StorageKey, input.ContentText, input.SizeBytes).Scan(&itemID) if err != nil { @@ -1159,8 +1256,21 @@ func (s *KnowledgeService) markKnowledgeIngestionRunning( func (s *KnowledgeService) markKnowledgeIngestionCompleted( ctx context.Context, tenantID, itemID, parseTaskID int64, + itemName string, + content *knowledgeParsedContent, metas []knowledgeChunkMetaInput, ) error { + if content == nil { + return errors.New("knowledge content is empty") + } + if strings.TrimSpace(content.Text) == "" { + return errors.New("knowledge text content is empty") + } + if strings.TrimSpace(content.Markdown) == "" { + content.Markdown = buildKnowledgeMarkdownFromText("", content.Text) + } + itemName = strings.TrimSpace(itemName) + tx, err := s.pool.Begin(ctx) if err != nil { return err @@ -1188,9 +1298,14 @@ func (s *KnowledgeService) markKnowledgeIngestionCompleted( if _, err := tx.Exec(ctx, ` UPDATE knowledge_items - SET status = 'completed', error_message = NULL, updated_at = NOW() + SET status = 'completed', + error_message = NULL, + name = CASE WHEN $3 <> '' THEN $3 ELSE name END, + content_text = $4, + markdown_content = $5, + updated_at = NOW() WHERE id = $1 AND tenant_id = $2 - `, itemID, tenantID); err != nil { + `, itemID, tenantID, itemName, content.Text, content.Markdown); err != nil { return err } @@ -1370,6 +1485,99 @@ func (s *KnowledgeService) ensureStorableGroup(ctx context.Context, tenantID, gr return nil } +func (s *KnowledgeService) resolveKnowledgeStorageGroupID(ctx context.Context, tenantID, groupID int64) (int64, error) { + tx, err := s.pool.Begin(ctx) + if err != nil { + return 0, response.ErrInternal(50055, "knowledge_group_resolve_failed", "failed to begin knowledge group transaction") + } + defer func() { _ = tx.Rollback(ctx) }() + + resolvedGroupID, err := s.resolveKnowledgeStorageGroupIDTx(ctx, tx, tenantID, groupID) + if err != nil { + return 0, err + } + + if err := tx.Commit(ctx); err != nil { + return 0, response.ErrInternal(50055, "knowledge_group_resolve_failed", "failed to commit knowledge group transaction") + } + return resolvedGroupID, nil +} + +func (s *KnowledgeService) resolveKnowledgeStorageGroupIDTx( + ctx context.Context, + tx pgx.Tx, + tenantID, groupID int64, +) (int64, error) { + var parentID *int64 + if err := tx.QueryRow(ctx, ` + SELECT parent_id + FROM knowledge_groups + WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL + FOR UPDATE + `, groupID, tenantID).Scan(&parentID); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return 0, response.ErrNotFound(40451, "knowledge_group_not_found", "knowledge group not found") + } + return 0, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge group") + } + + if parentID != nil { + var grandParentID *int64 + if err := tx.QueryRow(ctx, ` + SELECT parent_id + FROM knowledge_groups + WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL + `, *parentID, tenantID).Scan(&grandParentID); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return 0, response.ErrNotFound(40451, "knowledge_group_not_found", "knowledge group not found") + } + return 0, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge group") + } + if grandParentID != nil { + return 0, response.ErrBadRequest(40053, "knowledge_group_storage_invalid", "knowledge items can only be stored in second-level groups") + } + return groupID, nil + } + + var childGroupID int64 + if err := tx.QueryRow(ctx, ` + SELECT id + FROM knowledge_groups + WHERE tenant_id = $1 + AND parent_id = $2 + AND name = $3 + AND deleted_at IS NULL + ORDER BY id + LIMIT 1 + `, tenantID, groupID, defaultKnowledgeChildGroupName).Scan(&childGroupID); err == nil { + return childGroupID, nil + } else if !errors.Is(err, pgx.ErrNoRows) { + return 0, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge group") + } + + if err := tx.QueryRow(ctx, ` + INSERT INTO knowledge_groups (tenant_id, name, parent_id, sort_order) + VALUES ($1, $2, $3, 0) + RETURNING id + `, tenantID, defaultKnowledgeChildGroupName, groupID).Scan(&childGroupID); err == nil { + return childGroupID, nil + } + + if err := tx.QueryRow(ctx, ` + SELECT id + FROM knowledge_groups + WHERE tenant_id = $1 + AND parent_id = $2 + AND name = $3 + AND deleted_at IS NULL + ORDER BY id + LIMIT 1 + `, tenantID, groupID, defaultKnowledgeChildGroupName).Scan(&childGroupID); err != nil { + return 0, response.ErrConflict(40950, "knowledge_group_create_failed", "failed to create default knowledge group") + } + return childGroupID, nil +} + func (s *KnowledgeService) listGroupSubtreeIDs( ctx context.Context, tx pgx.Tx, @@ -1688,9 +1896,10 @@ func (s *KnowledgeService) loadKnowledgeItemForCleanup( ) (*knowledgeItemRecord, error) { record := &knowledgeItemRecord{} var contentText *string + var markdownContent *string err := s.pool.QueryRow(ctx, ` SELECT ki.id, ki.tenant_id, ki.group_id, kg.name, ki.name, ki.source_type, - ki.source_uri, ki.storage_key, ki.content_text, ki.item_version + ki.source_uri, ki.storage_key, ki.content_text, ki.markdown_content, ki.item_version FROM knowledge_items ki JOIN knowledge_groups kg ON kg.id = ki.group_id WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NOT NULL @@ -1704,12 +1913,14 @@ func (s *KnowledgeService) loadKnowledgeItemForCleanup( &record.SourceURI, &record.StorageKey, &contentText, + &markdownContent, &record.ItemVersion, ) if err != nil { return nil, nil } record.ContentText = contentText + record.Markdown = markdownContent return record, nil } diff --git a/server/internal/tenant/app/knowledge_url_parser.go b/server/internal/tenant/app/knowledge_url_parser.go new file mode 100644 index 0000000..958c536 --- /dev/null +++ b/server/internal/tenant/app/knowledge_url_parser.go @@ -0,0 +1,266 @@ +package app + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/geo-platform/tenant-api/internal/shared/llm" + "github.com/geo-platform/tenant-api/internal/tenant/prompts" + "go.uber.org/zap" +) + +const ( + defaultKnowledgeArkBaseURL = "https://ark.cn-beijing.volces.com/api/v3" + defaultKnowledgeURLParseTimeout = 60 * time.Second + defaultKnowledgeURLMarkdownTimeout = 90 * time.Second + defaultKnowledgeURLMarkdownChunkChars = 12000 + defaultKnowledgeURLMarkdownOutputToken = 12000 +) + +type arkToolExecuteRequest struct { + ActionName string `json:"action_name"` + ToolName string `json:"tool_name"` + Timeout int `json:"timeout,omitempty"` + Parameters map[string]any `json:"parameters,omitempty"` +} + +type arkToolExecuteResponse struct { + StatusCode int `json:"status_code"` + Data arkToolExecuteData `json:"data"` +} + +type arkToolExecuteData struct { + ArkWebDataList []arkWebDataItem `json:"ark_web_data_list"` +} + +type arkWebDataItem struct { + URL string `json:"url"` + Title string `json:"title"` + Content string `json:"content"` + ErrorMessage string `json:"error_message"` + ErrorCode string `json:"error_code"` +} + +func (s *KnowledgeService) parseWebsiteKnowledge(ctx context.Context, rawURL string) (*knowledgeParsedContent, error) { + item, err := s.executeLinkReader(ctx, rawURL) + if err != nil { + return nil, err + } + + text := normalizeKnowledgeText(item.Content) + if text == "" { + return nil, fmt.Errorf("webpage parser returned empty content") + } + + title := strings.TrimSpace(item.Title) + markdown, err := s.formatWebsiteMarkdown(ctx, title, text, rawURL) + if err != nil { + if s.logger != nil { + s.logger.Warn("knowledge website markdown formatting failed, fallback to local markdown", + zap.String("url", rawURL), + zap.Error(err), + ) + } + markdown = buildKnowledgeMarkdownFromText(title, text) + } + + return &knowledgeParsedContent{ + Text: text, + Markdown: markdown, + }, nil +} + +func (s *KnowledgeService) executeLinkReader(ctx context.Context, rawURL string) (*arkWebDataItem, error) { + baseURL := strings.TrimRight(strings.TrimSpace(s.arkBaseURL), "/") + if baseURL == "" { + baseURL = defaultKnowledgeArkBaseURL + } + + timeout := defaultKnowledgeURLParseTimeout + callCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + body, err := json.Marshal(arkToolExecuteRequest{ + ActionName: "LinkReader", + ToolName: "LinkReader", + Timeout: int(timeout / time.Second), + Parameters: map[string]any{ + "url_list": []string{rawURL}, + }, + }) + if err != nil { + return nil, fmt.Errorf("marshal webpage parser request: %w", err) + } + + req, err := http.NewRequestWithContext(callCtx, http.MethodPost, baseURL+"/tools/execute", bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("create webpage parser request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+s.arkAPIKey) + req.Header.Set("Content-Type", "application/json") + + resp, err := s.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("execute webpage parser: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode >= http.StatusBadRequest { + data, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return nil, fmt.Errorf("webpage parser failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(data))) + } + + var parsed arkToolExecuteResponse + if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { + return nil, fmt.Errorf("decode webpage parser response: %w", err) + } + if parsed.StatusCode != 0 && parsed.StatusCode != http.StatusOK { + return nil, fmt.Errorf("webpage parser returned status_code=%d", parsed.StatusCode) + } + if len(parsed.Data.ArkWebDataList) == 0 { + return nil, fmt.Errorf("webpage parser returned no content") + } + + item := parsed.Data.ArkWebDataList[0] + if strings.TrimSpace(item.ErrorCode) != "" || strings.TrimSpace(item.ErrorMessage) != "" { + return nil, fmt.Errorf("webpage parser returned error: %s %s", strings.TrimSpace(item.ErrorCode), strings.TrimSpace(item.ErrorMessage)) + } + return &item, nil +} + +func (s *KnowledgeService) formatWebsiteMarkdown(ctx context.Context, title, content, rawURL string) (string, error) { + content = strings.TrimSpace(content) + if content == "" { + return "", fmt.Errorf("website content is empty") + } + + chunks := splitKnowledgeMarkdownChunks(content, defaultKnowledgeURLMarkdownChunkChars) + if len(chunks) == 0 { + return "", fmt.Errorf("website content is empty after chunking") + } + + sections := make([]string, 0, len(chunks)+1) + for index, chunk := range chunks { + result, err := s.llmClient.Generate(ctx, llm.GenerateRequest{ + Model: s.urlMarkdownModel, + Prompt: prompts.KnowledgeWebsiteMarkdownPrompt(title, rawURL, chunk, index, len(chunks)), + Timeout: defaultKnowledgeURLMarkdownTimeout, + MaxOutputTokens: defaultKnowledgeURLMarkdownOutputToken, + }, nil) + if err != nil { + return "", err + } + + section := sanitizeLLMMarkdown(strings.TrimSpace(result.Content)) + if section == "" { + continue + } + if index > 0 && !strings.HasPrefix(section, "#") { + section = fmt.Sprintf("## 第 %d 部分\n\n%s", index+1, section) + } + sections = append(sections, section) + } + + if len(sections) == 0 { + return "", fmt.Errorf("website formatter returned empty markdown") + } + + markdown := strings.TrimSpace(strings.Join(sections, "\n\n")) + if title != "" && !strings.HasPrefix(markdown, "# ") { + markdown = "# " + title + "\n\n" + markdown + } + return markdown, nil +} + +func splitKnowledgeMarkdownChunks(content string, maxChars int) []string { + if maxChars <= 0 { + maxChars = defaultKnowledgeURLMarkdownChunkChars + } + + lines := strings.Split(content, "\n") + chunks := make([]string, 0, len(lines)) + current := make([]string, 0, 16) + currentLen := 0 + + flush := func() { + if len(current) == 0 { + return + } + chunks = append(chunks, strings.TrimSpace(strings.Join(current, "\n"))) + current = current[:0] + currentLen = 0 + } + + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + if len([]rune(line)) > maxChars { + flush() + for _, segment := range splitKnowledgeLongLine(line, maxChars) { + if strings.TrimSpace(segment) != "" { + chunks = append(chunks, segment) + } + } + continue + } + + nextLen := currentLen + len([]rune(line)) + if len(current) > 0 { + nextLen++ + } + if nextLen > maxChars { + flush() + } + + current = append(current, line) + currentLen += len([]rune(line)) + if len(current) > 1 { + currentLen++ + } + } + + flush() + return chunks +} + +func splitKnowledgeLongLine(line string, maxChars int) []string { + runes := []rune(line) + if len(runes) <= maxChars { + return []string{line} + } + + segments := make([]string, 0, (len(runes)/maxChars)+1) + for start := 0; start < len(runes); start += maxChars { + end := start + maxChars + if end > len(runes) { + end = len(runes) + } + segments = append(segments, strings.TrimSpace(string(runes[start:end]))) + } + return segments +} + +func sanitizeLLMMarkdown(content string) string { + content = strings.TrimSpace(content) + if !strings.HasPrefix(content, "```") { + return content + } + + lines := strings.Split(content, "\n") + if len(lines) < 3 { + return strings.Trim(content, "`") + } + if !strings.HasPrefix(lines[0], "```") || strings.TrimSpace(lines[len(lines)-1]) != "```" { + return content + } + return strings.TrimSpace(strings.Join(lines[1:len(lines)-1], "\n")) +} diff --git a/server/internal/tenant/prompts/runtime.go b/server/internal/tenant/prompts/runtime.go index f0f590e..2de0dfc 100644 --- a/server/internal/tenant/prompts/runtime.go +++ b/server/internal/tenant/prompts/runtime.go @@ -317,3 +317,33 @@ func KnowledgePromptIntroLines() []string { func KnowledgeSnippetLabel(index int) string { return fmt.Sprintf("知识片段 %d", index+1) } + +func KnowledgeWebsiteMarkdownPrompt(title, rawURL, content string, chunkIndex, chunkCount int) string { + scope := "以下是网页正文。" + if chunkCount > 1 { + scope = fmt.Sprintf("以下是网页正文的第 %d/%d 段。", chunkIndex+1, chunkCount) + } + + var builder strings.Builder + builder.WriteString("你是知识库 Markdown 整理助手。\n") + builder.WriteString("任务：把给定网页正文整理为适合知识库只读预览的 Markdown。\n") + builder.WriteString("要求：\n") + builder.WriteString("1. 只根据原文整理，不要补充事实，不要总结扩写。\n") + builder.WriteString("2. 保留标题、列表、表格、编号、联系方式等关键信息。\n") + builder.WriteString("3. 如果原文明显是表格，请输出 Markdown 表格。\n") + builder.WriteString("4. 不要输出解释，不要输出代码块包裹，只输出 Markdown 正文。\n") + if strings.TrimSpace(title) != "" { + builder.WriteString("网页标题：") + builder.WriteString(title) + builder.WriteString("\n") + } + if strings.TrimSpace(rawURL) != "" { + builder.WriteString("网页地址：") + builder.WriteString(rawURL) + builder.WriteString("\n") + } + builder.WriteString(scope) + builder.WriteString("\n\n") + builder.WriteString(content) + return builder.String() +} diff --git a/server/internal/tenant/transport/article_handler.go b/server/internal/tenant/transport/article_handler.go index 2bc875e..7b97d7d 100644 --- a/server/internal/tenant/transport/article_handler.go +++ b/server/internal/tenant/transport/article_handler.go @@ -27,8 +27,10 @@ func NewArticleHandler(a *bootstrap.App) *ArticleHandler { a.RetrievalProvider, a.VectorStore, a.ObjectStorage, + a.LLM, a.Logger, a.Config.Retrieval, + a.Config.LLM, 0, 0, ) diff --git a/server/internal/tenant/transport/knowledge_handler.go b/server/internal/tenant/transport/knowledge_handler.go index 31984c4..41ffeb6 100644 --- a/server/internal/tenant/transport/knowledge_handler.go +++ b/server/internal/tenant/transport/knowledge_handler.go @@ -23,8 +23,10 @@ func NewKnowledgeHandler(a *bootstrap.App) *KnowledgeHandler { a.RetrievalProvider, a.VectorStore, a.ObjectStorage, + a.LLM, a.Logger, a.Config.Retrieval, + a.Config.LLM, a.Config.Generation.QueueSize, a.Config.Generation.WorkerConcurrency, ), @@ -108,6 +110,21 @@ func (h *KnowledgeHandler) ListItems(c *gin.Context) { response.Success(c, data) } +func (h *KnowledgeHandler) GetItemDetail(c *gin.Context) { + itemID, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil { + response.Error(c, response.ErrBadRequest(40001, "invalid_id", "knowledge item id must be a number")) + return + } + + data, err := h.svc.GetItemDetail(c.Request.Context(), itemID) + if err != nil { + response.Error(c, err) + return + } + response.Success(c, data) +} + func (h *KnowledgeHandler) CreateTextItem(c *gin.Context) { var req app.KnowledgeTextItemRequest if err := c.ShouldBindJSON(&req); err != nil { diff --git a/server/internal/tenant/transport/router.go b/server/internal/tenant/transport/router.go index aeaef80..a483030 100644 --- a/server/internal/tenant/transport/router.go +++ b/server/internal/tenant/transport/router.go @@ -90,6 +90,7 @@ func RegisterRoutes(app *bootstrap.App) { knowledge.PUT("/groups/:gid", knowledgeHandler.UpdateGroup) knowledge.DELETE("/groups/:gid", knowledgeHandler.DeleteGroup) knowledge.GET("/items", knowledgeHandler.ListItems) + knowledge.GET("/items/:id", knowledgeHandler.GetItemDetail) knowledge.POST("/items/text", knowledgeHandler.CreateTextItem) knowledge.POST("/items/url", knowledgeHandler.CreateURLItem) knowledge.POST("/items/file", knowledgeHandler.CreateFileItem) diff --git a/server/internal/tenant/transport/template_handler.go b/server/internal/tenant/transport/template_handler.go index aa02415..9b48d54 100644 --- a/server/internal/tenant/transport/template_handler.go +++ b/server/internal/tenant/transport/template_handler.go @@ -21,8 +21,10 @@ func NewTemplateHandler(a *bootstrap.App) *TemplateHandler { a.RetrievalProvider, a.VectorStore, a.ObjectStorage, + a.LLM, a.Logger, a.Config.Retrieval, + a.Config.LLM, 0, 0, ) diff --git a/server/migrations/20260405173000_add_markdown_content_to_knowledge_items.down.sql b/server/migrations/20260405173000_add_markdown_content_to_knowledge_items.down.sql new file mode 100644 index 0000000..a88e279 --- /dev/null +++ b/server/migrations/20260405173000_add_markdown_content_to_knowledge_items.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE knowledge_items + DROP COLUMN IF EXISTS markdown_content; diff --git a/server/migrations/20260405173000_add_markdown_content_to_knowledge_items.up.sql b/server/migrations/20260405173000_add_markdown_content_to_knowledge_items.up.sql new file mode 100644 index 0000000..6581412 --- /dev/null +++ b/server/migrations/20260405173000_add_markdown_content_to_knowledge_items.up.sql @@ -0,0 +1,7 @@ +ALTER TABLE knowledge_items + ADD COLUMN markdown_content TEXT; + +UPDATE knowledge_items +SET markdown_content = content_text +WHERE markdown_content IS NULL + AND content_text IS NOT NULL;