feat: add knowledge markdown formatting and detail retrieval

- Implemented KnowledgeWebsiteMarkdownPrompt for formatting webpage content into Markdown.
- Added GetItemDetail method in KnowledgeHandler to retrieve knowledge item details.
- Introduced KnowledgeDetailDrawer component for displaying knowledge item details in the admin web interface.
- Created MarkdownPreview component for rendering Markdown content.
- Added knowledge chunking and URL parsing logic to handle webpage content extraction and formatting.
- Updated database schema to include markdown_content in knowledge_items table.
This commit is contained in:
2026-04-05 20:41:42 +08:00
parent 446f865cdf
commit dbd7747742
26 changed files with 1556 additions and 252 deletions
+2
View File
@@ -21,6 +21,7 @@
"@vueuse/core": "^14.2.1", "@vueuse/core": "^14.2.1",
"ant-design-vue": "^4.2.6", "ant-design-vue": "^4.2.6",
"dayjs": "^1.11.20", "dayjs": "^1.11.20",
"markdown-it": "^14.1.1",
"pinia": "^3.0.4", "pinia": "^3.0.4",
"vue": "^3.5.31", "vue": "^3.5.31",
"vue-i18n": "^10.0.5", "vue-i18n": "^10.0.5",
@@ -28,6 +29,7 @@
"zod": "^4.3.6" "zod": "^4.3.6"
}, },
"devDependencies": { "devDependencies": {
"@types/markdown-it": "^14.1.2",
"@types/node": "^24.0.0", "@types/node": "^24.0.0",
"@vitejs/plugin-vue": "^6.0.5", "@vitejs/plugin-vue": "^6.0.5",
"typescript": "^5.9.3", "typescript": "^5.9.3",
@@ -0,0 +1,180 @@
<script setup lang="ts">
import { useQuery, useQueryClient } from "@tanstack/vue-query";
import { computed, watch } from "vue";
import MarkdownPreview from "@/components/MarkdownPreview.vue";
import { knowledgeApi } from "@/lib/api";
import { formatDateTime } from "@/lib/display";
const props = defineProps<{
open: boolean;
itemId: number | null;
}>();
const emit = defineEmits<{
close: [];
}>();
const queryClient = useQueryClient();
const enabled = computed(() => props.open && Boolean(props.itemId));
const detailQuery = useQuery({
queryKey: computed(() => ["knowledge", "detail", props.itemId]),
enabled,
queryFn: () => knowledgeApi.detail(props.itemId as number),
});
const detail = computed(() => detailQuery.data.value);
const hasMarkdown = computed(() => Boolean(detail.value?.markdown_content?.trim()));
const isProcessing = computed(() =>
detail.value?.status === "pending" || detail.value?.status === "processing",
);
const isFailed = computed(() => detail.value?.status === "failed");
const statusLabel = computed(() => formatKnowledgeStatus(detail.value?.status));
const statusColor = computed(() => {
if (detail.value?.status === "completed") return "success";
if (detail.value?.status === "failed") return "error";
if (isProcessing.value) return "processing";
return "default";
});
watch(
[() => props.open, () => props.itemId, () => detail.value?.status],
([open, itemId, status], _prev, onCleanup) => {
if (!open || !itemId || (status !== "pending" && status !== "processing")) {
return;
}
const timer = window.setInterval(() => {
void Promise.all([
detailQuery.refetch(),
queryClient.invalidateQueries({ queryKey: ["knowledge", "items"] }),
]);
}, 2500);
onCleanup(() => {
window.clearInterval(timer);
});
},
{ immediate: true },
);
function handleClose(): void {
emit("close");
}
function formatKnowledgeStatus(status?: string): string {
switch (status) {
case "completed":
return "学习完成";
case "processing":
return "处理中";
case "failed":
return "解析失败";
case "pending":
return "待处理";
case "deleted":
return "已删除";
default:
return status || "未知状态";
}
}
</script>
<template>
<a-drawer
:open="open"
width="760"
title="知识详情"
class="knowledge-detail-drawer"
@close="handleClose"
>
<div v-if="detailQuery.isPending.value" class="knowledge-detail-drawer__loading">
<a-skeleton active :paragraph="{ rows: 10 }" />
</div>
<template v-else-if="detail">
<section class="knowledge-detail-drawer__hero">
<div>
<h2>{{ detail.name }}</h2>
<div class="knowledge-detail-drawer__meta">
<span>{{ detail.group_name }}</span>
<span>{{ formatDateTime(detail.created_at) }}</span>
<span v-if="detail.source_uri">{{ detail.source_uri }}</span>
</div>
</div>
<a-tag :color="statusColor">{{ statusLabel }}</a-tag>
</section>
<a-alert
v-if="isProcessing"
type="info"
show-icon
message="处理中"
description="内容正在解析并整理为 Markdown,抽屉会自动刷新。"
class="knowledge-detail-drawer__alert"
/>
<a-alert
v-else-if="isFailed"
type="error"
show-icon
message="解析失败"
:description="detail.error_message || '当前内容解析失败,请稍后重试。'"
class="knowledge-detail-drawer__alert"
/>
<div v-if="hasMarkdown" class="knowledge-detail-drawer__content">
<MarkdownPreview :content="detail.markdown_content || ''" />
</div>
<a-empty v-else description="无可预览文本" />
</template>
<a-result
v-else
status="404"
title="未找到知识内容"
sub-title="该内容可能已删除或当前租户无权访问"
/>
</a-drawer>
</template>
<style scoped>
.knowledge-detail-drawer__loading {
padding-top: 8px;
}
.knowledge-detail-drawer__hero {
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: 16px;
margin-bottom: 16px;
}
.knowledge-detail-drawer__hero h2 {
margin: 0 0 8px;
color: #111827;
font-size: 22px;
font-weight: 700;
}
.knowledge-detail-drawer__meta {
display: flex;
flex-wrap: wrap;
gap: 8px 16px;
color: #6b7280;
font-size: 13px;
}
.knowledge-detail-drawer__alert {
margin-bottom: 16px;
}
.knowledge-detail-drawer__content {
padding: 20px 24px;
border: 1px solid #f0f0f0;
border-radius: 16px;
background: #fff;
}
</style>
@@ -0,0 +1,114 @@
<script setup lang="ts">
import MarkdownIt from "markdown-it";
import { computed } from "vue";
const props = defineProps<{
content: string;
}>();
const renderer = new MarkdownIt({
html: false,
linkify: true,
breaks: true,
});
const html = computed(() => renderer.render(props.content || ""));
</script>
<template>
<div class="markdown-preview" v-html="html" />
</template>
<style scoped>
.markdown-preview {
color: #111827;
font-size: 14px;
line-height: 1.75;
}
.markdown-preview :deep(h1),
.markdown-preview :deep(h2),
.markdown-preview :deep(h3),
.markdown-preview :deep(h4) {
margin: 1.2em 0 0.6em;
color: #111827;
font-weight: 700;
}
.markdown-preview :deep(h1) {
font-size: 28px;
}
.markdown-preview :deep(h2) {
font-size: 22px;
}
.markdown-preview :deep(h3) {
font-size: 18px;
}
.markdown-preview :deep(p),
.markdown-preview :deep(ul),
.markdown-preview :deep(ol),
.markdown-preview :deep(blockquote) {
margin: 0 0 1em;
}
.markdown-preview :deep(ul),
.markdown-preview :deep(ol) {
padding-left: 1.5em;
}
.markdown-preview :deep(a) {
color: #1677ff;
}
.markdown-preview :deep(code) {
padding: 2px 6px;
border-radius: 6px;
background: #f3f4f6;
font-size: 13px;
}
.markdown-preview :deep(pre) {
overflow-x: auto;
padding: 16px;
border-radius: 12px;
background: #111827;
color: #f9fafb;
}
.markdown-preview :deep(pre code) {
padding: 0;
background: transparent;
color: inherit;
}
.markdown-preview :deep(table) {
width: 100%;
margin: 0 0 1em;
border-collapse: collapse;
overflow: hidden;
border: 1px solid #e5e7eb;
border-radius: 12px;
}
.markdown-preview :deep(th),
.markdown-preview :deep(td) {
padding: 10px 12px;
border: 1px solid #e5e7eb;
text-align: left;
vertical-align: top;
}
.markdown-preview :deep(th) {
background: #f9fafb;
font-weight: 600;
}
.markdown-preview :deep(hr) {
margin: 24px 0;
border: 0;
border-top: 1px solid #e5e7eb;
}
</style>
+4
View File
@@ -21,6 +21,7 @@ import type {
KnowledgeGroup, KnowledgeGroup,
KnowledgeGroupRequest, KnowledgeGroupRequest,
KnowledgeItem, KnowledgeItem,
KnowledgeItemDetail,
KnowledgeTextItemRequest, KnowledgeTextItemRequest,
KnowledgeURLItemRequest, KnowledgeURLItemRequest,
Keyword, Keyword,
@@ -248,6 +249,9 @@ export const knowledgeApi = {
params: groupId ? { group_id: groupId } : undefined, params: groupId ? { group_id: groupId } : undefined,
}); });
}, },
detail(id: number) {
return apiClient.get<KnowledgeItemDetail>(`/api/tenant/knowledge/items/${id}`);
},
createTextItem(payload: KnowledgeTextItemRequest) { createTextItem(payload: KnowledgeTextItemRequest) {
return apiClient.post<KnowledgeItem, KnowledgeTextItemRequest>( return apiClient.post<KnowledgeItem, KnowledgeTextItemRequest>(
"/api/tenant/knowledge/items/text", "/api/tenant/knowledge/items/text",
+2
View File
@@ -26,6 +26,7 @@ import {
Popconfirm, Popconfirm,
Progress, Progress,
Radio, Radio,
Result,
Row, Row,
Select, Select,
Skeleton, Skeleton,
@@ -92,6 +93,7 @@ app.component("ATextarea", Input.TextArea);
Popconfirm, Popconfirm,
Progress, Progress,
Radio, Radio,
Result,
Row, Row,
Select, Select,
Skeleton, Skeleton,
+101 -45
View File
@@ -1,17 +1,18 @@
<script setup lang="ts"> <script setup lang="ts">
import { MoreOutlined, PlusOutlined, UploadOutlined } from "@ant-design/icons-vue"; import { DeleteOutlined, EyeOutlined, MoreOutlined, PlusOutlined, UploadOutlined } from "@ant-design/icons-vue";
import { useMutation, useQuery, useQueryClient } from "@tanstack/vue-query"; import { useMutation, useQuery, useQueryClient } from "@tanstack/vue-query";
import { message, Modal, type TableColumnsType, type UploadProps } from "ant-design-vue"; import { message, Modal, type TableColumnsType, type UploadProps } from "ant-design-vue";
import type { KnowledgeGroup, KnowledgeItem } from "@geo/shared-types"; import type { KnowledgeGroup, KnowledgeItem } from "@geo/shared-types";
import { computed, reactive, ref, watch } from "vue"; import { computed, reactive, ref, watch } from "vue";
import KnowledgeDetailDrawer from "@/components/KnowledgeDetailDrawer.vue";
import { knowledgeApi } from "@/lib/api"; import { knowledgeApi } from "@/lib/api";
import { formatDateTime } from "@/lib/display"; import { formatDateTime } from "@/lib/display";
import { formatError } from "@/lib/errors"; import { formatError } from "@/lib/errors";
type KnowledgeSourceType = "document" | "website" | "text"; type KnowledgeSourceType = "document" | "website" | "text";
type KnowledgeGroupLevel = "root" | "child"; type KnowledgeGroupLevel = "root" | "child";
const DEFAULT_CHILD_GROUP_NAME = "默认目录"; const MAX_WEBSITE_URLS = 3;
interface TreeNode { interface TreeNode {
title: string; title: string;
@@ -29,6 +30,8 @@ const editingGroup = ref<KnowledgeGroup | null>(null);
const itemModalOpen = ref(false); const itemModalOpen = ref(false);
const itemSourceType = ref<KnowledgeSourceType>("document"); const itemSourceType = ref<KnowledgeSourceType>("document");
const uploadFiles = ref<File[]>([]); const uploadFiles = ref<File[]>([]);
const previewItemId = ref<number | null>(null);
const previewDrawerOpen = ref(false);
const groupForm = reactive({ const groupForm = reactive({
name: "", name: "",
@@ -67,6 +70,17 @@ const groupMap = computed(() => buildGroupMap(rootGroups.value));
const treeData = computed(() => rootGroups.value.map((item) => toSidebarTreeNode(item))); const treeData = computed(() => rootGroups.value.map((item) => toSidebarTreeNode(item)));
const storableTreeData = computed(() => rootGroups.value.map((item) => toStorageTreeNode(item))); const storableTreeData = computed(() => rootGroups.value.map((item) => toStorageTreeNode(item)));
const storableGroupCount = computed(() => countStorableGroups(rootGroups.value)); const storableGroupCount = computed(() => countStorableGroups(rootGroups.value));
const websiteUrls = computed(() =>
urlsText.value
.split("\n")
.map((item) => item.trim())
.filter(Boolean),
);
const websiteUrlLimitExceeded = computed(() => websiteUrls.value.length > MAX_WEBSITE_URLS);
const itemModalOkButtonProps = computed(() => ({
style: { width: "80px", borderRadius: "4px" },
disabled: itemSourceType.value === "website" && websiteUrlLimitExceeded.value,
}));
const groupModalTitle = computed(() => { const groupModalTitle = computed(() => {
if (editingGroup.value) { if (editingGroup.value) {
return isRootGroup(editingGroup.value) ? "编辑分组" : "编辑子目录"; return isRootGroup(editingGroup.value) ? "编辑分组" : "编辑子目录";
@@ -119,13 +133,11 @@ const groupMutations = {
const itemMutation = useMutation({ const itemMutation = useMutation({
mutationFn: async () => { mutationFn: async () => {
let targetGroupId = itemForm.group_id; const targetGroupId = itemForm.group_id;
if (!targetGroupId) { if (!targetGroupId) {
throw new Error("请选择分组目录"); throw new Error("请选择分组目录");
} }
targetGroupId = await resolveItemTargetGroupId(targetGroupId);
const promises: Promise<any>[] = []; const promises: Promise<any>[] = [];
if (itemSourceType.value === "document") { if (itemSourceType.value === "document") {
@@ -142,10 +154,13 @@ const itemMutation = useMutation({
); );
} }
} else if (itemSourceType.value === "website") { } else if (itemSourceType.value === "website") {
const urls = urlsText.value.split('\n').map(u => u.trim()).filter(Boolean); const urls = websiteUrls.value;
if (urls.length === 0) { if (urls.length === 0) {
throw new Error("请输入包含URL的文本"); throw new Error("请输入包含URL的文本");
} }
if (urls.length > MAX_WEBSITE_URLS) {
throw new Error(`一次最多支持 ${MAX_WEBSITE_URLS} 个网址`);
}
for (const url of urls) { for (const url of urls) {
let name = url; let name = url;
try { name = new URL(url).hostname + new URL(url).pathname } catch (e) {} try { name = new URL(url).hostname + new URL(url).pathname } catch (e) {}
@@ -173,12 +188,31 @@ const itemMutation = useMutation({
} }
} }
await Promise.all(promises); const results = await Promise.allSettled(promises);
const successCount = results.filter((result) => result.status === "fulfilled").length;
const failedResults = results.filter((result): result is PromiseRejectedResult => result.status === "rejected");
if (failedResults.length > 0 && successCount === 0) {
throw failedResults[0].reason;
}
return {
totalCount: promises.length,
successCount,
failedCount: failedResults.length,
firstError: failedResults[0]?.reason,
};
}, },
onSuccess: async () => { onSuccess: async (result) => {
message.success("内容已提交"); if (result.failedCount === 0) {
itemModalOpen.value = false; message.success(result.successCount > 1 ? `已提交 ${result.successCount} 条内容` : "内容已提交");
resetItemForm(); itemModalOpen.value = false;
resetItemForm();
} else {
message.warning(
`已成功提交 ${result.successCount} 条,${result.failedCount} 条失败${result.firstError ? `${formatError(result.firstError)}` : ""}`,
);
}
await queryClient.invalidateQueries({ queryKey: ["knowledge"] }); await queryClient.invalidateQueries({ queryKey: ["knowledge"] });
}, },
onError: (error) => message.error(formatError(error)), onError: (error) => message.error(formatError(error)),
@@ -310,29 +344,6 @@ function isStorableGroupId(id: number | null | undefined): boolean {
return Boolean(group && isStorableGroup(group)); return Boolean(group && isStorableGroup(group));
} }
function findChildGroupByName(parentId: number, name: string): KnowledgeGroup | undefined {
const parentGroup = getGroupById(parentId);
return parentGroup?.children.find((child) => child.name === name);
}
async function resolveItemTargetGroupId(groupId: number): Promise<number> {
if (!isNodeRootGroup(groupId)) {
return groupId;
}
const defaultChild = findChildGroupByName(groupId, DEFAULT_CHILD_GROUP_NAME);
if (defaultChild) {
return defaultChild.id;
}
const newGroup = await knowledgeApi.createGroup({
name: DEFAULT_CHILD_GROUP_NAME,
parent_id: groupId,
});
return newGroup.id;
}
function resolveDefaultItemGroupId(): number | undefined { function resolveDefaultItemGroupId(): number | undefined {
if (selectedGroupId.value) { if (selectedGroupId.value) {
return selectedGroupId.value; return selectedGroupId.value;
@@ -481,8 +492,14 @@ function formatSize(value: number): string {
return `${(value / 1024 / 1024).toFixed(1)} MB`; return `${(value / 1024 / 1024).toFixed(1)} MB`;
} }
function editItem(): void { function openItemDetail(itemId: number): void {
message.info("内容编辑功能开发中"); previewItemId.value = itemId;
previewDrawerOpen.value = true;
}
function closeItemDetail(): void {
previewDrawerOpen.value = false;
previewItemId.value = null;
} }
</script> </script>
@@ -570,11 +587,16 @@ function editItem(): void {
{{ formatDateTime(record.created_at) }} {{ formatDateTime(record.created_at) }}
</template> </template>
<template v-else-if="column.key === 'actions'"> <template v-else-if="column.key === 'actions'">
<div class="knowledge-actions"> <div class="table-actions-row">
<a-button type="link" style="padding: 0 4px" @click="editItem()">编辑</a-button> <a-tooltip title="查看">
<a-divider type="vertical" /> <a-button type="text" shape="circle" size="small" class="action-btn action-eye" @click="openItemDetail(record.id)">
<EyeOutlined />
</a-button>
</a-tooltip>
<a-popconfirm title="确认删除该内容?" @confirm="deleteItemMutation.mutate(record.id)"> <a-popconfirm title="确认删除该内容?" @confirm="deleteItemMutation.mutate(record.id)">
<a-button type="link" style="padding: 0 4px">删除</a-button> <a-button type="text" shape="circle" size="small" class="action-btn action-delete">
<DeleteOutlined />
</a-button>
</a-popconfirm> </a-popconfirm>
</div> </div>
</template> </template>
@@ -615,7 +637,7 @@ function editItem(): void {
" "
ok-text=" " ok-text=" "
cancel-text=" " cancel-text=" "
:ok-button-props="{ style: { width: '80px', borderRadius: '4px' } }" :ok-button-props="itemModalOkButtonProps"
:cancel-button-props="{ style: { width: '80px', borderRadius: '4px' } }" :cancel-button-props="{ style: { width: '80px', borderRadius: '4px' } }"
> >
<div class="knowledge-form-add"> <div class="knowledge-form-add">
@@ -644,7 +666,7 @@ function editItem(): void {
</div> </div>
<p class="dragger-text">点击或拖拽文件至此区域即可上传</p> <p class="dragger-text">点击或拖拽文件至此区域即可上传</p>
<p class="dragger-hint"> <p class="dragger-hint">
请上传docpdftxtxlsxlsx格式不超过30M的文件严禁上传敏感数据或其他非法文件 请上传 docxpdftxtmdxlsxlsx 格式不超过30M的文件严禁上传敏感数据或其他非法文件
</p> </p>
<p class="dragger-count">已选择文件{{ uploadFiles.length }}</p> <p class="dragger-count">已选择文件{{ uploadFiles.length }}</p>
</div> </div>
@@ -655,9 +677,17 @@ function editItem(): void {
<a-textarea <a-textarea
v-model:value="urlsText" v-model:value="urlsText"
:rows="11" :rows="11"
placeholder="请输入产品相关网页URL,每行一个" :status="websiteUrlLimitExceeded ? 'error' : undefined"
placeholder="请输入产品相关网页URL,每行一个,最多支持3个"
class="custom-textarea" class="custom-textarea"
/> />
<div
class="knowledge-url-hint"
:class="{ 'knowledge-url-hint--error': websiteUrlLimitExceeded }"
>
当前 {{ websiteUrls.length }} / {{ MAX_WEBSITE_URLS }} 个网址
<span v-if="websiteUrlLimitExceeded">请删除多余网址后再提交</span>
</div>
</div> </div>
<div v-else class="knowledge-content-wrapper"> <div v-else class="knowledge-content-wrapper">
@@ -690,6 +720,12 @@ function editItem(): void {
</div> </div>
</div> </div>
</a-modal> </a-modal>
<KnowledgeDetailDrawer
:open="previewDrawerOpen"
:item-id="previewItemId"
@close="closeItemDetail"
/>
</div> </div>
</template> </template>
@@ -776,7 +812,7 @@ function editItem(): void {
} }
.knowledge-sidebar__add { .knowledge-sidebar__add {
color: #9ca3af; color: #666;
font-size: 13px; font-size: 13px;
cursor: pointer; cursor: pointer;
display: flex; display: flex;
@@ -823,12 +859,21 @@ function editItem(): void {
color: #6b7280; color: #6b7280;
} }
.knowledge-actions { /* Actions */
.table-actions-row {
display: flex; display: flex;
align-items: center; align-items: center;
justify-content: flex-end; justify-content: flex-end;
gap: 4px;
} }
.action-btn {
color: #8c8c8c;
font-size: 14px;
}
.action-eye:hover { color: #13c2c2; background: #e6fffb; }
.action-delete:hover { color: #ff4d4f; background: #fff2f0; }
.knowledge-form { .knowledge-form {
display: flex; display: flex;
flex-direction: column; flex-direction: column;
@@ -871,6 +916,17 @@ function editItem(): void {
margin-top: 4px; margin-top: 4px;
} }
.knowledge-url-hint {
margin-top: 8px;
color: #6b7280;
font-size: 12px;
line-height: 1.5;
}
.knowledge-url-hint--error {
color: #ff4d4f;
}
:deep(.custom-dragger.ant-upload-wrapper .ant-upload-drag) { :deep(.custom-dragger.ant-upload-wrapper .ant-upload-drag) {
border: 1px dashed #7eb0f7; border: 1px dashed #7eb0f7;
background-color: #fff; background-color: #fff;
+4
View File
@@ -556,6 +556,10 @@ export interface KnowledgeItem {
updated_at: string; updated_at: string;
} }
export interface KnowledgeItemDetail extends KnowledgeItem {
markdown_content: string | null;
}
export interface KnowledgeTextItemRequest { export interface KnowledgeTextItemRequest {
group_id: number; group_id: number;
name: string; name: string;
+65
View File
@@ -43,6 +43,9 @@ importers:
dayjs: dayjs:
specifier: ^1.11.20 specifier: ^1.11.20
version: 1.11.20 version: 1.11.20
markdown-it:
specifier: ^14.1.1
version: 14.1.1
pinia: pinia:
specifier: ^3.0.4 specifier: ^3.0.4
version: 3.0.4(typescript@5.9.3)(vue@3.5.31(typescript@5.9.3)) version: 3.0.4(typescript@5.9.3)(vue@3.5.31(typescript@5.9.3))
@@ -59,6 +62,9 @@ importers:
specifier: ^4.3.6 specifier: ^4.3.6
version: 4.3.6 version: 4.3.6
devDependencies: devDependencies:
'@types/markdown-it':
specifier: ^14.1.2
version: 14.1.2
'@types/node': '@types/node':
specifier: ^24.0.0 specifier: ^24.0.0
version: 24.12.0 version: 24.12.0
@@ -790,15 +796,24 @@ packages:
'@types/katex@0.16.8': '@types/katex@0.16.8':
resolution: {integrity: sha512-trgaNyfU+Xh2Tc+ABIb44a5AYUpicB3uwirOioeOkNPPbmgRNtcWyDeeFRzjPZENO9Vq8gvVqfhaaXWLlevVwg==} resolution: {integrity: sha512-trgaNyfU+Xh2Tc+ABIb44a5AYUpicB3uwirOioeOkNPPbmgRNtcWyDeeFRzjPZENO9Vq8gvVqfhaaXWLlevVwg==}
'@types/linkify-it@5.0.0':
resolution: {integrity: sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==}
'@types/lodash-es@4.17.12': '@types/lodash-es@4.17.12':
resolution: {integrity: sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==} resolution: {integrity: sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==}
'@types/lodash@4.17.24': '@types/lodash@4.17.24':
resolution: {integrity: sha512-gIW7lQLZbue7lRSWEFql49QJJWThrTFFeIMJdp3eH4tKoxm1OvEPg02rm4wCCSHS0cL3/Fizimb35b7k8atwsQ==} resolution: {integrity: sha512-gIW7lQLZbue7lRSWEFql49QJJWThrTFFeIMJdp3eH4tKoxm1OvEPg02rm4wCCSHS0cL3/Fizimb35b7k8atwsQ==}
'@types/markdown-it@14.1.2':
resolution: {integrity: sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog==}
'@types/mdast@4.0.4': '@types/mdast@4.0.4':
resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==} resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==}
'@types/mdurl@2.0.0':
resolution: {integrity: sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==}
'@types/minimatch@3.0.5': '@types/minimatch@3.0.5':
resolution: {integrity: sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==} resolution: {integrity: sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==}
@@ -951,6 +966,9 @@ packages:
peerDependencies: peerDependencies:
vue: '>=3.2.0' vue: '>=3.2.0'
argparse@2.0.1:
resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==}
array-differ@4.0.0: array-differ@4.0.0:
resolution: {integrity: sha512-Q6VPTLMsmXZ47ENG3V+wQyZS1ZxXMxFyYzA+Z/GMrJ6yIutAIEf9wTyroTzmGjNfox9/h3GdGBCVh43GVFx4Uw==} resolution: {integrity: sha512-Q6VPTLMsmXZ47ENG3V+wQyZS1ZxXMxFyYzA+Z/GMrJ6yIutAIEf9wTyroTzmGjNfox9/h3GdGBCVh43GVFx4Uw==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
@@ -1760,6 +1778,9 @@ packages:
canvas: canvas:
optional: true optional: true
linkify-it@5.0.0:
resolution: {integrity: sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==}
listr2@10.2.1: listr2@10.2.1:
resolution: {integrity: sha512-7I5knELsJKTUjXG+A6BkKAiGkW1i25fNa/xlUl9hFtk15WbE9jndA89xu5FzQKrY5llajE1hfZZFMILXkDHk/Q==} resolution: {integrity: sha512-7I5knELsJKTUjXG+A6BkKAiGkW1i25fNa/xlUl9hFtk15WbE9jndA89xu5FzQKrY5llajE1hfZZFMILXkDHk/Q==}
engines: {node: '>=22.13.0'} engines: {node: '>=22.13.0'}
@@ -1821,6 +1842,10 @@ packages:
many-keys-map@2.0.1: many-keys-map@2.0.1:
resolution: {integrity: sha512-DHnZAD4phTbZ+qnJdjoNEVU1NecYoSdbOOoVmTDH46AuxDkEVh3MxTVpXq10GtcTC6mndN9dkv1rNfpjRcLnOw==} resolution: {integrity: sha512-DHnZAD4phTbZ+qnJdjoNEVU1NecYoSdbOOoVmTDH46AuxDkEVh3MxTVpXq10GtcTC6mndN9dkv1rNfpjRcLnOw==}
markdown-it@14.1.1:
resolution: {integrity: sha512-BuU2qnTti9YKgK5N+IeMubp14ZUKUUw7yeJbkjtosvHiP0AZ5c8IAgEMk79D0eC8F23r4Ac/q8cAIFdm2FtyoA==}
hasBin: true
markdown-table@3.0.4: markdown-table@3.0.4:
resolution: {integrity: sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==} resolution: {integrity: sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==}
@@ -1875,6 +1900,9 @@ packages:
mdast-util-to-string@4.0.0: mdast-util-to-string@4.0.0:
resolution: {integrity: sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==} resolution: {integrity: sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==}
mdurl@2.0.0:
resolution: {integrity: sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==}
micromark-core-commonmark@2.0.3: micromark-core-commonmark@2.0.3:
resolution: {integrity: sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==} resolution: {integrity: sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==}
@@ -2226,6 +2254,10 @@ packages:
engines: {node: '>=18.0.0'} engines: {node: '>=18.0.0'}
hasBin: true hasBin: true
punycode.js@2.3.1:
resolution: {integrity: sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==}
engines: {node: '>=6'}
pupa@3.3.0: pupa@3.3.0:
resolution: {integrity: sha512-LjgDO2zPtoXP2wJpDjZrGdojii1uqO0cnwKoIoUzkfS98HDmbeiGmYiXo3lXeFlq2xvne1QFQhwYXSUCLKtEuA==} resolution: {integrity: sha512-LjgDO2zPtoXP2wJpDjZrGdojii1uqO0cnwKoIoUzkfS98HDmbeiGmYiXo3lXeFlq2xvne1QFQhwYXSUCLKtEuA==}
engines: {node: '>=12.20'} engines: {node: '>=12.20'}
@@ -2495,6 +2527,9 @@ packages:
engines: {node: '>=14.17'} engines: {node: '>=14.17'}
hasBin: true hasBin: true
uc.micro@2.1.0:
resolution: {integrity: sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==}
ufo@1.6.3: ufo@1.6.3:
resolution: {integrity: sha512-yDJTmhydvl5lJzBmy/hyOAA0d+aqCBuwl818haVdYCRrWV84o7YyeVm4QlVHStqNrrJSTb6jKuFAVqAFsr+K3Q==} resolution: {integrity: sha512-yDJTmhydvl5lJzBmy/hyOAA0d+aqCBuwl818haVdYCRrWV84o7YyeVm4QlVHStqNrrJSTb6jKuFAVqAFsr+K3Q==}
@@ -3711,16 +3746,25 @@ snapshots:
'@types/katex@0.16.8': {} '@types/katex@0.16.8': {}
'@types/linkify-it@5.0.0': {}
'@types/lodash-es@4.17.12': '@types/lodash-es@4.17.12':
dependencies: dependencies:
'@types/lodash': 4.17.24 '@types/lodash': 4.17.24
'@types/lodash@4.17.24': {} '@types/lodash@4.17.24': {}
'@types/markdown-it@14.1.2':
dependencies:
'@types/linkify-it': 5.0.0
'@types/mdurl': 2.0.0
'@types/mdast@4.0.4': '@types/mdast@4.0.4':
dependencies: dependencies:
'@types/unist': 3.0.3 '@types/unist': 3.0.3
'@types/mdurl@2.0.0': {}
'@types/minimatch@3.0.5': {} '@types/minimatch@3.0.5': {}
'@types/ms@2.1.0': {} '@types/ms@2.1.0': {}
@@ -3930,6 +3974,8 @@ snapshots:
vue-types: 3.0.2(vue@3.5.31(typescript@5.9.3)) vue-types: 3.0.2(vue@3.5.31(typescript@5.9.3))
warning: 4.0.3 warning: 4.0.3
argparse@2.0.1: {}
array-differ@4.0.0: {} array-differ@4.0.0: {}
array-tree-filter@2.1.0: {} array-tree-filter@2.1.0: {}
@@ -4648,6 +4694,10 @@ snapshots:
htmlparser2: 10.1.0 htmlparser2: 10.1.0
uhyphen: 0.2.0 uhyphen: 0.2.0
linkify-it@5.0.0:
dependencies:
uc.micro: 2.1.0
listr2@10.2.1: listr2@10.2.1:
dependencies: dependencies:
cli-truncate: 5.2.0 cli-truncate: 5.2.0
@@ -4710,6 +4760,15 @@ snapshots:
many-keys-map@2.0.1: {} many-keys-map@2.0.1: {}
markdown-it@14.1.1:
dependencies:
argparse: 2.0.1
entities: 4.5.0
linkify-it: 5.0.0
mdurl: 2.0.0
punycode.js: 2.3.1
uc.micro: 2.1.0
markdown-table@3.0.4: {} markdown-table@3.0.4: {}
marked@17.0.5: {} marked@17.0.5: {}
@@ -4838,6 +4897,8 @@ snapshots:
dependencies: dependencies:
'@types/mdast': 4.0.4 '@types/mdast': 4.0.4
mdurl@2.0.0: {}
micromark-core-commonmark@2.0.3: micromark-core-commonmark@2.0.3:
dependencies: dependencies:
decode-named-character-reference: 1.3.0 decode-named-character-reference: 1.3.0
@@ -5348,6 +5409,8 @@ snapshots:
ofetch: 1.5.1 ofetch: 1.5.1
zod: 4.3.6 zod: 4.3.6
punycode.js@2.3.1: {}
pupa@3.3.0: pupa@3.3.0:
dependencies: dependencies:
escape-goat: 4.0.0 escape-goat: 4.0.0
@@ -5634,6 +5697,8 @@ snapshots:
typescript@5.9.3: {} typescript@5.9.3: {}
uc.micro@2.1.0: {}
ufo@1.6.3: {} ufo@1.6.3: {}
uhyphen@0.2.0: {} uhyphen@0.2.0: {}
+1
View File
@@ -36,6 +36,7 @@ llm:
provider: ark provider: ark
base_url: https://ark.cn-beijing.volces.com/api/v3 base_url: https://ark.cn-beijing.volces.com/api/v3
model: doubao-seed-2-0-lite-260215 model: doubao-seed-2-0-lite-260215
knowledge_url_model: doubao-seed-2-0-mini-260215
timeout: 2m timeout: 2m
max_output_tokens: 4000 max_output_tokens: 4000
temperature: 0.7 temperature: 0.7
+1
View File
@@ -49,6 +49,7 @@ llm:
base_url: https://ark.cn-beijing.volces.com/api/v3 base_url: https://ark.cn-beijing.volces.com/api/v3
api_key: "7fb6c66b-129c-4935-9617-709236c4205a" api_key: "7fb6c66b-129c-4935-9617-709236c4205a"
model: doubao-seed-2-0-lite-260215 model: doubao-seed-2-0-lite-260215
knowledge_url_model: doubao-seed-2-0-mini-260215
timeout: 2m timeout: 2m
max_output_tokens: 16000 max_output_tokens: 16000
temperature: 0.7 temperature: 0.7
+2
View File
@@ -28,6 +28,8 @@ require (
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect
github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7 // indirect
github.com/extrame/xls v0.0.1 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect
+4
View File
@@ -27,6 +27,10 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7 h1:n+nk0bNe2+gVbRI8WRbLFVwwcBQ0rr5p+gzkKb6ol8c=
github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7/go.mod h1:GPpMrAfHdb8IdQ1/R2uIRBsNfnPnwsYE9YYI5WyY1zw=
github.com/extrame/xls v0.0.1 h1:jI7L/o3z73TyyENPopsLS/Jlekm3nF1a/kF5hKBvy/k=
github.com/extrame/xls v0.0.1/go.mod h1:iACcgahst7BboCpIMSpnFs4SKyU9ZjsvZBfNbUxZOJI=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+14 -10
View File
@@ -79,16 +79,17 @@ type LogConfig struct {
} }
type LLMConfig struct { type LLMConfig struct {
Provider string `mapstructure:"provider"` Provider string `mapstructure:"provider"`
APIKey string `mapstructure:"api_key"` APIKey string `mapstructure:"api_key"`
BaseURL string `mapstructure:"base_url"` BaseURL string `mapstructure:"base_url"`
Model string `mapstructure:"model"` Model string `mapstructure:"model"`
Timeout time.Duration `mapstructure:"timeout"` KnowledgeURLModel string `mapstructure:"knowledge_url_model"`
MaxOutputTokens int64 `mapstructure:"max_output_tokens"` Timeout time.Duration `mapstructure:"timeout"`
Temperature float64 `mapstructure:"temperature"` MaxOutputTokens int64 `mapstructure:"max_output_tokens"`
TopP float64 `mapstructure:"top_p"` Temperature float64 `mapstructure:"temperature"`
ReasoningEffort string `mapstructure:"reasoning_effort"` TopP float64 `mapstructure:"top_p"`
WebSearchLimit int32 `mapstructure:"web_search_limit"` ReasoningEffort string `mapstructure:"reasoning_effort"`
WebSearchLimit int32 `mapstructure:"web_search_limit"`
} }
type RetrievalConfig struct { type RetrievalConfig struct {
@@ -150,6 +151,9 @@ func applyEnvOverrides(cfg *Config) {
if apiKey, ok := lookupNonEmptyEnv("LLM_API_KEY"); ok { if apiKey, ok := lookupNonEmptyEnv("LLM_API_KEY"); ok {
cfg.LLM.APIKey = apiKey cfg.LLM.APIKey = apiKey
} }
if model, ok := lookupNonEmptyEnv("LLM_KNOWLEDGE_URL_MODEL"); ok {
cfg.LLM.KnowledgeURLModel = model
}
if apiKey, ok := lookupNonEmptyEnv("QDRANT_API_KEY"); ok { if apiKey, ok := lookupNonEmptyEnv("QDRANT_API_KEY"); ok {
cfg.Qdrant.APIKey = apiKey cfg.Qdrant.APIKey = apiKey
} }
+7 -2
View File
@@ -113,6 +113,11 @@ func (c *arkClient) Generate(ctx context.Context, req GenerateRequest, onDelta f
maxOutputTokens = req.MaxOutputTokens maxOutputTokens = req.MaxOutputTokens
} }
model := c.model
if strings.TrimSpace(req.Model) != "" {
model = strings.TrimSpace(req.Model)
}
callCtx, cancel := context.WithTimeout(ctx, timeout) callCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel() defer cancel()
@@ -154,7 +159,7 @@ func (c *arkClient) Generate(ctx context.Context, req GenerateRequest, onDelta f
} }
stream, err := c.client.CreateResponsesStream(callCtx, &responses.ResponsesRequest{ stream, err := c.client.CreateResponsesStream(callCtx, &responses.ResponsesRequest{
Model: c.model, Model: model,
MaxOutputTokens: &maxOutputTokens, MaxOutputTokens: &maxOutputTokens,
Store: &store, Store: &store,
Stream: &streamValue, Stream: &streamValue,
@@ -232,7 +237,7 @@ func (c *arkClient) Generate(ctx context.Context, req GenerateRequest, onDelta f
return &GenerateResult{ return &GenerateResult{
Content: result, Content: result,
Model: c.model, Model: model,
}, nil }, nil
} }
+1
View File
@@ -13,6 +13,7 @@ import (
var ErrNotConfigured = errors.New("llm provider is not configured") var ErrNotConfigured = errors.New("llm provider is not configured")
type GenerateRequest struct { type GenerateRequest struct {
Model string
Prompt string Prompt string
Timeout time.Duration Timeout time.Duration
MaxOutputTokens int64 MaxOutputTokens int64
@@ -0,0 +1,122 @@
package app
import "strings"
func splitKnowledgeTextIntoChunks(text string, chunkSize, chunkOverlap int) []string {
text = strings.TrimSpace(text)
if text == "" {
return nil
}
if chunkSize <= 0 {
chunkSize = 900
}
if chunkOverlap < 0 {
chunkOverlap = 0
}
if chunkOverlap >= chunkSize {
chunkOverlap = chunkSize / 4
}
lines := strings.Split(text, "\n")
chunks := make([]string, 0, len(lines))
current := make([]string, 0, 16)
currentLen := 0
resetWithOverlap := func(chunk string) {
current = current[:0]
currentLen = 0
if chunkOverlap <= 0 {
return
}
overlap := takeLastKnowledgeRunes(chunk, chunkOverlap)
if overlap == "" {
return
}
current = append(current, overlap)
currentLen = len([]rune(overlap))
}
flush := func() {
if len(current) == 0 {
return
}
chunk := strings.TrimSpace(strings.Join(current, "\n"))
if chunk == "" {
current = current[:0]
currentLen = 0
return
}
chunks = append(chunks, chunk)
resetWithOverlap(chunk)
}
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
continue
}
lineRunes := len([]rune(line))
if lineRunes > chunkSize {
flush()
for _, segment := range splitKnowledgeLongLine(line, chunkSize) {
if strings.TrimSpace(segment) == "" {
continue
}
chunks = append(chunks, segment)
resetWithOverlap(segment)
}
continue
}
nextLen := currentLen + lineRunes
if len(current) > 0 {
nextLen++
}
if nextLen > chunkSize {
flush()
}
current = append(current, line)
currentLen += lineRunes
if len(current) > 1 {
currentLen++
}
}
if len(current) > 0 {
chunk := strings.TrimSpace(strings.Join(current, "\n"))
if chunk != "" {
chunks = append(chunks, chunk)
}
}
return chunks
}
func takeLastKnowledgeRunes(text string, size int) string {
if size <= 0 {
return ""
}
runes := []rune(strings.TrimSpace(text))
if len(runes) <= size {
return string(runes)
}
return strings.TrimSpace(string(runes[len(runes)-size:]))
}
func estimateKnowledgeTokenCount(text string) int {
text = strings.TrimSpace(text)
if text == "" {
return 0
}
runeCount := len([]rune(text))
tokenCount := runeCount / 2
if tokenCount == 0 {
return 1
}
return tokenCount
}
+347 -148
View File
@@ -3,76 +3,113 @@ package app
import ( import (
"archive/zip" "archive/zip"
"bytes" "bytes"
"context"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
"io" "io"
"net/http"
"path/filepath" "path/filepath"
"regexp" "regexp"
"strings" "strings"
"time"
"unicode/utf8" "unicode/utf8"
"github.com/extrame/xls"
pdf "github.com/ledongthuc/pdf" pdf "github.com/ledongthuc/pdf"
"github.com/xuri/excelize/v2" "github.com/xuri/excelize/v2"
"golang.org/x/net/html" "golang.org/x/net/html"
) )
const ( var (
maxKnowledgeFetchBytes = 5 << 20 whitespacePattern = regexp.MustCompile(`[ \t]+`)
genericKnowledgeMarkdownTitleRegex = regexp.MustCompile(`^第\s*\d+\s*部分$`)
) )
var whitespacePattern = regexp.MustCompile(`[ \t]+`) const maxKnowledgeSpreadsheetRows = 100000
func extractKnowledgeTextFromURL(ctx context.Context, rawURL string) (string, error) { type knowledgeParsedContent struct {
data, contentType, err := fetchKnowledgeURLSnapshot(ctx, rawURL) Text string
if err != nil { Markdown string
return "", err
}
return extractKnowledgeTextFromSnapshot(resolveSnapshotFileName(rawURL, contentType), data)
} }
func fetchKnowledgeURLSnapshot(ctx context.Context, rawURL string) ([]byte, string, error) { func parseKnowledgeTextInput(name, content string) *knowledgeParsedContent {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimSpace(rawURL), nil) text := normalizeKnowledgeText(content)
if err != nil { return &knowledgeParsedContent{
return nil, "", fmt.Errorf("create request: %w", err) Text: text,
Markdown: buildKnowledgeMarkdownFromText(name, text),
} }
req.Header.Set("User-Agent", "geo-knowledge-fetcher/1.0")
client := &http.Client{Timeout: 20 * time.Second}
resp, err := client.Do(req)
if err != nil {
return nil, "", fmt.Errorf("fetch url: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode >= http.StatusBadRequest {
return nil, "", fmt.Errorf("fetch url failed: status=%d", resp.StatusCode)
}
data, err := io.ReadAll(io.LimitReader(resp.Body, maxKnowledgeFetchBytes))
if err != nil {
return nil, "", fmt.Errorf("read url body: %w", err)
}
contentType := strings.ToLower(resp.Header.Get("Content-Type"))
return data, contentType, nil
} }
func resolveSnapshotFileName(rawURL string, contentType string) string { func extractKnowledgeContentFromFile(fileName string, content []byte) (*knowledgeParsedContent, error) {
if strings.Contains(strings.ToLower(contentType), "text/plain") {
return rawURL + ".txt"
}
return rawURL + ".html"
}
func extractKnowledgeTextFromSnapshot(fileName string, content []byte) (string, error) {
ext := strings.ToLower(filepath.Ext(fileName)) ext := strings.ToLower(filepath.Ext(fileName))
if ext != ".html" && ext != ".htm" {
return extractKnowledgeTextFromFile(fileName, content) switch ext {
case ".txt", ".json", ".csv":
return extractUTF8KnowledgeText(fileName, content, false)
case ".md", ".markdown":
return extractUTF8KnowledgeText(fileName, content, true)
case ".html", ".htm":
text, err := extractHTMLText(content)
if err != nil {
return nil, err
}
return &knowledgeParsedContent{
Text: text,
Markdown: buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text),
}, nil
case ".docx":
text, err := extractDOCXText(content)
if err != nil {
return nil, err
}
return &knowledgeParsedContent{
Text: text,
Markdown: buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text),
}, nil
case ".xlsx":
return extractXLSXContent(content)
case ".xls":
return extractXLSContent(content)
case ".pdf":
text, err := extractPDFText(content)
if err != nil {
return nil, err
}
return &knowledgeParsedContent{
Text: text,
Markdown: buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text),
}, nil
default:
if utf8.Valid(content) {
return extractUTF8KnowledgeText(fileName, content, false)
}
return nil, fmt.Errorf("unsupported file type %s", ext)
}
}
func extractUTF8KnowledgeText(fileName string, content []byte, preserveMarkdown bool) (*knowledgeParsedContent, error) {
if !utf8.Valid(content) {
return nil, fmt.Errorf("file %s is not valid UTF-8 text", fileName)
} }
raw := string(content)
text := normalizeKnowledgeText(raw)
if text == "" {
return nil, fmt.Errorf("file %s contains no readable text", fileName)
}
markdown := buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text)
if preserveMarkdown {
normalizedMarkdown := normalizeKnowledgeMarkdown(raw)
if normalizedMarkdown != "" {
markdown = normalizedMarkdown
}
}
return &knowledgeParsedContent{
Text: text,
Markdown: markdown,
}, nil
}
func extractHTMLText(content []byte) (string, error) {
doc, err := html.Parse(bytes.NewReader(content)) doc, err := html.Parse(bytes.NewReader(content))
if err != nil { if err != nil {
return "", fmt.Errorf("parse html: %w", err) return "", fmt.Errorf("parse html: %w", err)
@@ -106,32 +143,11 @@ func extractKnowledgeTextFromSnapshot(fileName string, content []byte) (string,
} }
walk(doc, false) walk(doc, false)
return normalizeKnowledgeText(strings.Join(parts, "\n")), nil result := normalizeKnowledgeText(strings.Join(parts, "\n"))
} if result == "" {
return "", fmt.Errorf("html contains no readable text")
func extractKnowledgeTextFromFile(fileName string, content []byte) (string, error) {
ext := strings.ToLower(filepath.Ext(fileName))
switch ext {
case ".txt", ".md", ".markdown", ".json", ".csv", ".html", ".htm":
if !utf8.Valid(content) {
return "", fmt.Errorf("file %s is not valid UTF-8 text", fileName)
}
return normalizeKnowledgeText(string(content)), nil
case ".docx":
return extractDOCXText(content)
case ".xlsx":
return extractXLSXText(content)
case ".pdf":
return extractPDFText(content)
case ".doc", ".xls":
return "", fmt.Errorf("file type %s is not supported yet", ext)
default:
if utf8.Valid(content) {
return normalizeKnowledgeText(string(content)), nil
}
return "", fmt.Errorf("unsupported file type %s", ext)
} }
return result, nil
} }
func extractDOCXText(content []byte) (string, error) { func extractDOCXText(content []byte) (string, error) {
@@ -173,46 +189,177 @@ func extractDOCXText(content []byte) (string, error) {
} }
} }
} }
if len(parts) == 0 { if len(parts) == 0 {
return "", fmt.Errorf("docx contains no readable text") return "", fmt.Errorf("docx contains no readable text")
} }
return normalizeKnowledgeText(strings.Join(parts, "\n")), nil return normalizeKnowledgeText(strings.Join(parts, "\n")), nil
} }
func extractXLSXText(content []byte) (string, error) { func extractXLSXContent(content []byte) (*knowledgeParsedContent, error) {
book, err := excelize.OpenReader(bytes.NewReader(content)) book, err := excelize.OpenReader(bytes.NewReader(content))
if err != nil { if err != nil {
return "", fmt.Errorf("open xlsx: %w", err) return nil, fmt.Errorf("open xlsx: %w", err)
} }
defer func() { _ = book.Close() }() defer func() { _ = book.Close() }()
lines := make([]string, 0) markdownSections := make([]string, 0)
textSections := make([]string, 0)
for _, sheet := range book.GetSheetList() { for _, sheet := range book.GetSheetList() {
rows, err := book.GetRows(sheet) rows, err := book.GetRows(sheet)
if err != nil { if err != nil {
return "", fmt.Errorf("read xlsx rows: %w", err) return nil, fmt.Errorf("read xlsx rows: %w", err)
} }
if len(rows) == 0 {
cleaned := normalizeSpreadsheetRows(rows)
if len(cleaned) == 0 {
continue continue
} }
lines = append(lines, sheet)
for _, row := range rows { markdownSections = append(markdownSections, buildKnowledgeSheetMarkdown(sheet, cleaned))
cells := make([]string, 0, len(row)) textSections = append(textSections, buildKnowledgeSheetPlainText(sheet, cleaned))
for _, cell := range row { }
cell = strings.TrimSpace(cell)
if cell != "" { if len(markdownSections) == 0 {
cells = append(cells, cell) return nil, fmt.Errorf("xlsx contains no readable text")
} }
}
if len(cells) > 0 { return &knowledgeParsedContent{
lines = append(lines, strings.Join(cells, " | ")) Text: normalizeKnowledgeText(strings.Join(textSections, "\n\n")),
} Markdown: strings.TrimSpace(strings.Join(markdownSections, "\n\n")),
}, nil
}
func extractXLSContent(content []byte) (*knowledgeParsedContent, error) {
book, err := xls.OpenReader(bytes.NewReader(content), "utf-8")
if err != nil {
return nil, fmt.Errorf("open xls: %w", err)
}
// The upstream xls library panics for sparse rows when calling sheet.Row(i),
// so we rely on its safer bulk reader here instead of random row access.
rows := book.ReadAllCells(maxKnowledgeSpreadsheetRows)
cleaned := normalizeSpreadsheetRows(rows)
if len(cleaned) == 0 {
return nil, fmt.Errorf("xls contains no readable text")
}
title := "XLS 工作簿"
if book.NumSheets() == 1 {
if sheet := book.GetSheet(0); sheet != nil && strings.TrimSpace(sheet.Name) != "" {
title = strings.TrimSpace(sheet.Name)
} }
} }
if len(lines) == 0 {
return "", fmt.Errorf("xlsx contains no readable text") return &knowledgeParsedContent{
Text: normalizeKnowledgeText(buildKnowledgeSheetPlainText(title, cleaned)),
Markdown: buildKnowledgeSheetMarkdown(title, cleaned),
}, nil
}
func normalizeSpreadsheetRows(rows [][]string) [][]string {
cleaned := make([][]string, 0, len(rows))
for _, row := range rows {
cells := make([]string, len(row))
copy(cells, row)
for index := range cells {
cells[index] = normalizeKnowledgeInlineText(cells[index])
}
lastNonEmpty := -1
for index, cell := range cells {
if cell != "" {
lastNonEmpty = index
}
}
if lastNonEmpty < 0 {
continue
}
cleaned = append(cleaned, cells[:lastNonEmpty+1])
} }
return normalizeKnowledgeText(strings.Join(lines, "\n")), nil return cleaned
}
func buildKnowledgeSheetPlainText(sheetName string, rows [][]string) string {
lines := make([]string, 0, len(rows)+1)
if strings.TrimSpace(sheetName) != "" {
lines = append(lines, strings.TrimSpace(sheetName))
}
for _, row := range rows {
lines = append(lines, strings.Join(row, " | "))
}
return strings.Join(lines, "\n")
}
func buildKnowledgeSheetMarkdown(sheetName string, rows [][]string) string {
if len(rows) == 0 {
return ""
}
width := 0
for _, row := range rows {
if len(row) > width {
width = len(row)
}
}
if width == 0 {
return ""
}
normalizedRows := make([][]string, 0, len(rows))
for _, row := range rows {
normalizedRows = append(normalizedRows, padKnowledgeRow(row, width))
}
var builder strings.Builder
if title := strings.TrimSpace(sheetName); title != "" {
builder.WriteString("## ")
builder.WriteString(title)
builder.WriteString("\n\n")
}
writeKnowledgeMarkdownTableRow(&builder, normalizedRows[0])
writeKnowledgeMarkdownTableSeparator(&builder, width)
for _, row := range normalizedRows[1:] {
writeKnowledgeMarkdownTableRow(&builder, row)
}
return strings.TrimSpace(builder.String())
}
func padKnowledgeRow(row []string, width int) []string {
result := make([]string, width)
copy(result, row)
return result
}
func writeKnowledgeMarkdownTableRow(builder *strings.Builder, row []string) {
builder.WriteString("|")
for _, cell := range row {
builder.WriteString(" ")
builder.WriteString(escapeKnowledgeMarkdownTableCell(cell))
builder.WriteString(" |")
}
builder.WriteString("\n")
}
func writeKnowledgeMarkdownTableSeparator(builder *strings.Builder, width int) {
builder.WriteString("|")
for index := 0; index < width; index++ {
builder.WriteString(" --- |")
}
builder.WriteString("\n")
}
func escapeKnowledgeMarkdownTableCell(cell string) string {
cell = strings.TrimSpace(cell)
cell = strings.ReplaceAll(cell, "\\", "\\\\")
cell = strings.ReplaceAll(cell, "|", "\\|")
cell = strings.ReplaceAll(cell, "\r", " ")
cell = strings.ReplaceAll(cell, "\n", " / ")
return cell
} }
func extractPDFText(content []byte) (string, error) { func extractPDFText(content []byte) (string, error) {
@@ -223,7 +370,7 @@ func extractPDFText(content []byte) (string, error) {
var builder strings.Builder var builder strings.Builder
totalPage := reader.NumPage() totalPage := reader.NumPage()
for pageIndex := 1; pageIndex <= totalPage; pageIndex += 1 { for pageIndex := 1; pageIndex <= totalPage; pageIndex++ {
page := reader.Page(pageIndex) page := reader.Page(pageIndex)
if page.V.IsNull() { if page.V.IsNull() {
continue continue
@@ -245,12 +392,116 @@ func extractPDFText(content []byte) (string, error) {
return result, nil return result, nil
} }
func normalizeKnowledgeText(input string) string { func defaultKnowledgeTitle(fileName string) string {
base := filepath.Base(strings.TrimSpace(fileName))
if base == "" {
return ""
}
return strings.TrimSuffix(base, filepath.Ext(base))
}
func buildKnowledgeMarkdownFromText(title, text string) string {
text = normalizeKnowledgeText(text)
title = strings.TrimSpace(title)
if text == "" {
if title == "" {
return ""
}
return "# " + title
}
lines := strings.Split(text, "\n")
if title != "" && len(lines) > 0 && strings.EqualFold(strings.TrimSpace(lines[0]), title) {
lines = lines[1:]
}
paragraphs := make([]string, 0, len(lines))
for _, line := range lines {
line = strings.TrimSpace(line)
if line != "" {
paragraphs = append(paragraphs, line)
}
}
sections := make([]string, 0, 2)
if title != "" {
sections = append(sections, "# "+title)
}
if len(paragraphs) > 0 {
sections = append(sections, strings.Join(paragraphs, "\n\n"))
}
return strings.TrimSpace(strings.Join(sections, "\n\n"))
}
func extractKnowledgeMarkdownTitle(markdown string) string {
markdown = normalizeKnowledgeMarkdown(markdown)
if markdown == "" {
return ""
}
for _, line := range strings.Split(markdown, "\n") {
line = strings.TrimSpace(line)
if line == "" || !strings.HasPrefix(line, "#") {
continue
}
level := 0
for level < len(line) && line[level] == '#' {
level++
}
if level == len(line) || line[level] != ' ' {
continue
}
title := strings.Join(strings.Fields(strings.TrimSpace(line[level+1:])), " ")
title = strings.Trim(title, "# \t")
if title == "" || genericKnowledgeMarkdownTitleRegex.MatchString(title) {
continue
}
switch title {
case "摘要", "概述", "概要", "目录", "正文":
continue
}
runes := []rune(title)
if len(runes) > 200 {
title = string(runes[:200])
}
return title
}
return ""
}
func normalizeKnowledgeMarkdown(input string) string {
input = strings.ReplaceAll(input, "\r\n", "\n")
input = strings.ReplaceAll(input, "\u00a0", " ") input = strings.ReplaceAll(input, "\u00a0", " ")
lines := strings.Split(input, "\n") lines := strings.Split(input, "\n")
cleaned := make([]string, 0, len(lines)) cleaned := make([]string, 0, len(lines))
lastBlank := false
for _, line := range lines { for _, line := range lines {
line = whitespacePattern.ReplaceAllString(strings.TrimSpace(line), " ") line = strings.TrimRight(line, " \t")
if strings.TrimSpace(line) == "" {
if lastBlank {
continue
}
lastBlank = true
cleaned = append(cleaned, "")
continue
}
lastBlank = false
cleaned = append(cleaned, line)
}
return strings.TrimSpace(strings.Join(cleaned, "\n"))
}
func normalizeKnowledgeText(input string) string {
input = strings.ReplaceAll(input, "\u00a0", " ")
lines := strings.Split(strings.ReplaceAll(input, "\r\n", "\n"), "\n")
cleaned := make([]string, 0, len(lines))
for _, line := range lines {
line = normalizeKnowledgeInlineText(line)
if line != "" { if line != "" {
cleaned = append(cleaned, line) cleaned = append(cleaned, line)
} }
@@ -258,58 +509,6 @@ func normalizeKnowledgeText(input string) string {
return strings.TrimSpace(strings.Join(cleaned, "\n")) return strings.TrimSpace(strings.Join(cleaned, "\n"))
} }
func splitKnowledgeTextIntoChunks(text string, chunkSize, chunkOverlap int) []string { func normalizeKnowledgeInlineText(input string) string {
text = normalizeKnowledgeText(text) return whitespacePattern.ReplaceAllString(strings.TrimSpace(input), " ")
if text == "" {
return nil
}
runes := []rune(text)
if chunkSize <= 0 {
chunkSize = 900
}
if chunkOverlap < 0 {
chunkOverlap = 0
}
if chunkOverlap >= chunkSize {
chunkOverlap = chunkSize / 4
}
chunks := make([]string, 0)
for start := 0; start < len(runes); {
end := start + chunkSize
if end > len(runes) {
end = len(runes)
}
if end < len(runes) {
for offset := end; offset > start+chunkSize/2; offset -= 1 {
if runes[offset-1] == '\n' || runes[offset-1] == '。' || runes[offset-1] == '.' {
end = offset
break
}
}
}
chunk := strings.TrimSpace(string(runes[start:end]))
if chunk != "" {
chunks = append(chunks, chunk)
}
if end >= len(runes) {
break
}
start = end - chunkOverlap
if start < 0 {
start = 0
}
}
return chunks
}
func estimateKnowledgeTokenCount(text string) int {
if text == "" {
return 0
}
return utf8.RuneCountInString(text)
} }
+258 -47
View File
@@ -4,8 +4,10 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"net/http"
"net/url" "net/url"
"path/filepath" "path/filepath"
"runtime/debug"
"slices" "slices"
"strings" "strings"
"sync" "sync"
@@ -18,6 +20,7 @@ import (
"github.com/geo-platform/tenant-api/internal/shared/auth" "github.com/geo-platform/tenant-api/internal/shared/auth"
"github.com/geo-platform/tenant-api/internal/shared/config" "github.com/geo-platform/tenant-api/internal/shared/config"
"github.com/geo-platform/tenant-api/internal/shared/llm"
"github.com/geo-platform/tenant-api/internal/shared/objectstorage" "github.com/geo-platform/tenant-api/internal/shared/objectstorage"
"github.com/geo-platform/tenant-api/internal/shared/response" "github.com/geo-platform/tenant-api/internal/shared/response"
"github.com/geo-platform/tenant-api/internal/shared/retrieval" "github.com/geo-platform/tenant-api/internal/shared/retrieval"
@@ -25,8 +28,9 @@ import (
) )
const ( const (
defaultKnowledgeQueueSize = 128 defaultKnowledgeQueueSize = 128
defaultKnowledgeCleanupPoll = 15 * time.Second defaultKnowledgeCleanupPoll = 15 * time.Second
defaultKnowledgeChildGroupName = "默认目录"
) )
type KnowledgeService struct { type KnowledgeService struct {
@@ -34,12 +38,17 @@ type KnowledgeService struct {
provider retrieval.Provider provider retrieval.Provider
store retrieval.VectorStore store retrieval.VectorStore
objectStorage objectstorage.Client objectStorage objectstorage.Client
llmClient llm.Client
logger *zap.Logger logger *zap.Logger
chunkSize int chunkSize int
chunkOverlap int chunkOverlap int
embeddingBatchSize int embeddingBatchSize int
recallLimit int recallLimit int
rerankTopN int rerankTopN int
arkBaseURL string
arkAPIKey string
urlMarkdownModel string
httpClient *http.Client
jobs chan knowledgeParseJob jobs chan knowledgeParseJob
cleanupMu sync.Mutex cleanupMu sync.Mutex
cleanupInFlight map[string]struct{} cleanupInFlight map[string]struct{}
@@ -66,6 +75,7 @@ type knowledgeItemRecord struct {
SourceURI *string SourceURI *string
StorageKey string StorageKey string
ContentText *string ContentText *string
Markdown *string
ItemVersion int ItemVersion int
} }
@@ -99,6 +109,21 @@ type KnowledgeItemResponse struct {
UpdatedAt string `json:"updated_at"` UpdatedAt string `json:"updated_at"`
} }
type KnowledgeItemDetailResponse struct {
ID int64 `json:"id"`
GroupID int64 `json:"group_id"`
GroupName string `json:"group_name"`
Name string `json:"name"`
SourceType string `json:"source_type"`
SourceURI *string `json:"source_uri"`
Status string `json:"status"`
SizeBytes int64 `json:"size_bytes"`
ErrorMessage *string `json:"error_message"`
MarkdownContent *string `json:"markdown_content"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
type KnowledgeTextItemRequest struct { type KnowledgeTextItemRequest struct {
GroupID int64 `json:"group_id" binding:"required"` GroupID int64 `json:"group_id" binding:"required"`
Name string `json:"name" binding:"required"` Name string `json:"name" binding:"required"`
@@ -132,8 +157,10 @@ func NewKnowledgeService(
provider retrieval.Provider, provider retrieval.Provider,
store retrieval.VectorStore, store retrieval.VectorStore,
objectStorage objectstorage.Client, objectStorage objectstorage.Client,
llmClient llm.Client,
logger *zap.Logger, logger *zap.Logger,
cfg config.RetrievalConfig, cfg config.RetrievalConfig,
llmCfg config.LLMConfig,
queueSize int, queueSize int,
workerCount int, workerCount int,
) *KnowledgeService { ) *KnowledgeService {
@@ -163,14 +190,25 @@ func NewKnowledgeService(
provider: provider, provider: provider,
store: store, store: store,
objectStorage: objectStorage, objectStorage: objectStorage,
llmClient: llmClient,
logger: logger, logger: logger,
chunkSize: chunkSize, chunkSize: chunkSize,
chunkOverlap: chunkOverlap, chunkOverlap: chunkOverlap,
embeddingBatchSize: batchSize, embeddingBatchSize: batchSize,
recallLimit: recallLimit, recallLimit: recallLimit,
rerankTopN: rerankTopN, rerankTopN: rerankTopN,
arkBaseURL: strings.TrimSpace(llmCfg.BaseURL),
arkAPIKey: strings.TrimSpace(llmCfg.APIKey),
urlMarkdownModel: strings.TrimSpace(llmCfg.KnowledgeURLModel),
httpClient: &http.Client{Timeout: defaultKnowledgeURLParseTimeout},
cleanupInFlight: make(map[string]struct{}), cleanupInFlight: make(map[string]struct{}),
} }
if svc.urlMarkdownModel == "" {
svc.urlMarkdownModel = strings.TrimSpace(llmCfg.Model)
}
if svc.arkBaseURL == "" {
svc.arkBaseURL = defaultKnowledgeArkBaseURL
}
if workerCount > 0 { if workerCount > 0 {
if queueSize <= 0 { if queueSize <= 0 {
@@ -521,15 +559,58 @@ func (s *KnowledgeService) ListItems(ctx context.Context, groupID *int64) ([]Kno
return items, nil return items, nil
} }
func (s *KnowledgeService) GetItemDetail(ctx context.Context, itemID int64) (*KnowledgeItemDetailResponse, error) {
actor := auth.MustActor(ctx)
var (
item KnowledgeItemDetailResponse
sourceURI *string
errorMessage *string
markdownContent *string
createdAt, updatedAt time.Time
)
err := s.pool.QueryRow(ctx, `
SELECT ki.id, ki.group_id, kg.name, ki.name, ki.source_type, ki.source_uri, ki.status,
ki.size_bytes, ki.error_message, ki.markdown_content, ki.created_at, ki.updated_at
FROM knowledge_items ki
JOIN knowledge_groups kg ON kg.id = ki.group_id AND kg.deleted_at IS NULL
WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NULL
`, itemID, actor.TenantID).Scan(
&item.ID,
&item.GroupID,
&item.GroupName,
&item.Name,
&item.SourceType,
&sourceURI,
&item.Status,
&item.SizeBytes,
&errorMessage,
&markdownContent,
&createdAt,
&updatedAt,
)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, response.ErrNotFound(40450, "knowledge_item_not_found", "knowledge item not found")
}
return nil, response.ErrInternal(50051, "knowledge_items_query_failed", "failed to load knowledge item")
}
item.SourceURI = sourceURI
item.ErrorMessage = errorMessage
item.MarkdownContent = markdownContent
item.CreatedAt = createdAt.Format(time.RFC3339)
item.UpdatedAt = updatedAt.Format(time.RFC3339)
return &item, nil
}
func (s *KnowledgeService) CreateTextItem(ctx context.Context, req KnowledgeTextItemRequest) (*KnowledgeItemResponse, error) { func (s *KnowledgeService) CreateTextItem(ctx context.Context, req KnowledgeTextItemRequest) (*KnowledgeItemResponse, error) {
actor := auth.MustActor(ctx) actor := auth.MustActor(ctx)
if err := s.validateIngestionDependencies("text"); err != nil { if err := s.validateIngestionDependencies("text"); err != nil {
return nil, err return nil, err
} }
if err := s.ensureStorableGroup(ctx, actor.TenantID, req.GroupID); err != nil {
return nil, err
}
content := normalizeKnowledgeText(req.Content) content := normalizeKnowledgeText(req.Content)
if content == "" { if content == "" {
@@ -551,33 +632,17 @@ func (s *KnowledgeService) CreateURLItem(ctx context.Context, req KnowledgeURLIt
if err := s.validateIngestionDependencies("website"); err != nil { if err := s.validateIngestionDependencies("website"); err != nil {
return nil, err return nil, err
} }
if err := s.ensureStorableGroup(ctx, actor.TenantID, req.GroupID); err != nil {
return nil, err
}
rawURL := strings.TrimSpace(req.URL) rawURL := strings.TrimSpace(req.URL)
parsed, err := url.ParseRequestURI(rawURL) if _, err := url.ParseRequestURI(rawURL); err != nil {
if err != nil {
return nil, response.ErrBadRequest(40052, "knowledge_url_invalid", "knowledge url is invalid") return nil, response.ErrBadRequest(40052, "knowledge_url_invalid", "knowledge url is invalid")
} }
snapshot, contentType, err := fetchKnowledgeURLSnapshot(ctx, parsed.String())
if err != nil {
return nil, response.ErrBadRequest(40052, "knowledge_url_fetch_failed", err.Error())
}
objectKey := buildKnowledgeObjectKey(actor.TenantID, "website", resolveSnapshotFileName(parsed.String(), contentType))
if err := s.objectStorage.PutBytes(ctx, objectKey, snapshot, contentType); err != nil {
return nil, response.ErrServiceUnavailable(50357, "knowledge_storage_unavailable", err.Error())
}
return s.queueKnowledgeItem(ctx, actor.TenantID, knowledgeIngestInput{ return s.queueKnowledgeItem(ctx, actor.TenantID, knowledgeIngestInput{
GroupID: req.GroupID, GroupID: req.GroupID,
Name: strings.TrimSpace(req.Name), Name: strings.TrimSpace(req.Name),
SourceType: "website", SourceType: "website",
SourceURI: &rawURL, SourceURI: &rawURL,
StorageKey: objectKey,
SizeBytes: int64(len(snapshot)),
}) })
} }
@@ -593,9 +658,6 @@ func (s *KnowledgeService) CreateFileItem(
if err := s.validateIngestionDependencies("document"); err != nil { if err := s.validateIngestionDependencies("document"); err != nil {
return nil, err return nil, err
} }
if err := s.ensureStorableGroup(ctx, actor.TenantID, groupID); err != nil {
return nil, err
}
displayName := strings.TrimSpace(name) displayName := strings.TrimSpace(name)
if displayName == "" { if displayName == "" {
@@ -819,11 +881,19 @@ func (s *KnowledgeService) validateIngestionDependencies(sourceType string) erro
if err := s.store.Validate(); err != nil { if err := s.store.Validate(); err != nil {
return response.ErrServiceUnavailable(50352, "qdrant_unavailable", err.Error()) return response.ErrServiceUnavailable(50352, "qdrant_unavailable", err.Error())
} }
if sourceType == "document" || sourceType == "website" { if sourceType == "document" {
if err := s.objectStorage.Validate(); err != nil { if err := s.objectStorage.Validate(); err != nil {
return response.ErrServiceUnavailable(50357, "knowledge_storage_unavailable", err.Error()) return response.ErrServiceUnavailable(50357, "knowledge_storage_unavailable", err.Error())
} }
} }
if sourceType == "website" {
if strings.TrimSpace(s.arkAPIKey) == "" {
return response.ErrServiceUnavailable(50358, "knowledge_webpage_parser_unavailable", "ark api key is not configured")
}
if err := s.llmClient.Validate(); err != nil {
return response.ErrServiceUnavailable(50359, "knowledge_markdown_formatter_unavailable", err.Error())
}
}
return nil return nil
} }
@@ -832,9 +902,11 @@ func (s *KnowledgeService) queueKnowledgeItem(
tenantID int64, tenantID int64,
input knowledgeIngestInput, input knowledgeIngestInput,
) (*KnowledgeItemResponse, error) { ) (*KnowledgeItemResponse, error) {
if err := s.ensureStorableGroup(ctx, tenantID, input.GroupID); err != nil { resolvedGroupID, err := s.resolveKnowledgeStorageGroupID(ctx, tenantID, input.GroupID)
if err != nil {
return nil, err return nil, err
} }
input.GroupID = resolvedGroupID
input.Name = strings.TrimSpace(input.Name) input.Name = strings.TrimSpace(input.Name)
if input.Name == "" { if input.Name == "" {
@@ -888,7 +960,25 @@ func (s *KnowledgeService) enqueueParseJob(job knowledgeParseJob) error {
func (s *KnowledgeService) runParseWorker() { func (s *KnowledgeService) runParseWorker() {
for job := range s.jobs { for job := range s.jobs {
s.executeParseJob(context.Background(), job) func(job knowledgeParseJob) {
defer func() {
if recovered := recover(); recovered != nil {
err := fmt.Errorf("knowledge parse panic: %v", recovered)
if s.logger != nil {
s.logger.Error("knowledge parse worker panicked",
zap.Int64("tenant_id", job.TenantID),
zap.Int64("knowledge_item_id", job.ItemID),
zap.Int64("parse_task_id", job.ParseTaskID),
zap.Any("panic", recovered),
zap.ByteString("stack", debug.Stack()),
)
}
_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
}
}()
s.executeParseJob(context.Background(), job)
}(job)
} }
} }
@@ -936,13 +1026,18 @@ func (s *KnowledgeService) executeParseJob(ctx context.Context, job knowledgePar
return return
} }
content, err := s.readKnowledgeItemContent(ctx, item) parsedContent, err := s.readKnowledgeItemParsedContent(ctx, item)
if err != nil { if err != nil {
_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err) _ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
return return
} }
if item.SourceType == "website" {
if resolvedName := extractKnowledgeMarkdownTitle(parsedContent.Markdown); resolvedName != "" {
item.Name = resolvedName
}
}
chunks := splitKnowledgeTextIntoChunks(content, s.chunkSize, s.chunkOverlap) chunks := splitKnowledgeTextIntoChunks(parsedContent.Text, s.chunkSize, s.chunkOverlap)
if len(chunks) == 0 { if len(chunks) == 0 {
err = errors.New("knowledge content is empty after parsing") err = errors.New("knowledge content is empty after parsing")
_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err) _ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
@@ -1018,7 +1113,7 @@ func (s *KnowledgeService) executeParseJob(ctx context.Context, job knowledgePar
return return
} }
if err = s.markKnowledgeIngestionCompleted(ctx, job.TenantID, job.ItemID, job.ParseTaskID, metas); err != nil { if err = s.markKnowledgeIngestionCompleted(ctx, job.TenantID, job.ItemID, job.ParseTaskID, item.Name, parsedContent, metas); err != nil {
_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err) _ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
return return
} }
@@ -1030,9 +1125,10 @@ func (s *KnowledgeService) loadKnowledgeItemForParse(
) (*knowledgeItemRecord, error) { ) (*knowledgeItemRecord, error) {
record := &knowledgeItemRecord{} record := &knowledgeItemRecord{}
var contentText *string var contentText *string
var markdownContent *string
err := s.pool.QueryRow(ctx, ` err := s.pool.QueryRow(ctx, `
SELECT ki.id, ki.tenant_id, ki.group_id, kg.name, ki.name, ki.source_type, SELECT ki.id, ki.tenant_id, ki.group_id, kg.name, ki.name, ki.source_type,
ki.source_uri, ki.storage_key, ki.content_text, ki.item_version ki.source_uri, ki.storage_key, ki.content_text, ki.markdown_content, ki.item_version
FROM knowledge_items ki FROM knowledge_items ki
JOIN knowledge_groups kg ON kg.id = ki.group_id JOIN knowledge_groups kg ON kg.id = ki.group_id
WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NULL WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NULL
@@ -1046,37 +1142,38 @@ func (s *KnowledgeService) loadKnowledgeItemForParse(
&record.SourceURI, &record.SourceURI,
&record.StorageKey, &record.StorageKey,
&contentText, &contentText,
&markdownContent,
&record.ItemVersion, &record.ItemVersion,
) )
if err != nil { if err != nil {
return nil, nil return nil, nil
} }
record.ContentText = contentText record.ContentText = contentText
record.Markdown = markdownContent
return record, nil return record, nil
} }
func (s *KnowledgeService) readKnowledgeItemContent(ctx context.Context, item *knowledgeItemRecord) (string, error) { func (s *KnowledgeService) readKnowledgeItemParsedContent(ctx context.Context, item *knowledgeItemRecord) (*knowledgeParsedContent, error) {
if item == nil { if item == nil {
return "", fmt.Errorf("knowledge item is empty") return nil, fmt.Errorf("knowledge item is empty")
} }
switch item.SourceType { switch item.SourceType {
case "text": case "text":
return normalizeKnowledgeText(derefKnowledgeString(item.ContentText)), nil return parseKnowledgeTextInput(item.Name, derefKnowledgeString(item.ContentText)), nil
case "document": case "document":
data, err := s.objectStorage.GetBytes(ctx, item.StorageKey) data, err := s.objectStorage.GetBytes(ctx, item.StorageKey)
if err != nil { if err != nil {
return "", err return nil, err
} }
return extractKnowledgeTextFromFile(item.StorageKey, data) return extractKnowledgeContentFromFile(item.StorageKey, data)
case "website": case "website":
data, err := s.objectStorage.GetBytes(ctx, item.StorageKey) if item.SourceURI == nil || strings.TrimSpace(*item.SourceURI) == "" {
if err != nil { return nil, fmt.Errorf("knowledge website url is empty")
return "", err
} }
return extractKnowledgeTextFromSnapshot(item.StorageKey, data) return s.parseWebsiteKnowledge(ctx, strings.TrimSpace(*item.SourceURI))
default: default:
return "", fmt.Errorf("unsupported knowledge source type %q", item.SourceType) return nil, fmt.Errorf("unsupported knowledge source type %q", item.SourceType)
} }
} }
@@ -1103,9 +1200,9 @@ func (s *KnowledgeService) createPendingKnowledgeItem(
err = tx.QueryRow(ctx, ` err = tx.QueryRow(ctx, `
INSERT INTO knowledge_items ( INSERT INTO knowledge_items (
tenant_id, group_id, source_type, name, source_uri, storage_key, tenant_id, group_id, source_type, name, source_uri, storage_key,
content_text, status, size_bytes, item_version content_text, markdown_content, status, size_bytes, item_version
) )
VALUES ($1, $2, $3, $4, $5, $6, $7, 'pending', $8, 1) VALUES ($1, $2, $3, $4, $5, $6, $7, NULL, 'pending', $8, 1)
RETURNING id RETURNING id
`, tenantID, input.GroupID, input.SourceType, input.Name, input.SourceURI, input.StorageKey, input.ContentText, input.SizeBytes).Scan(&itemID) `, tenantID, input.GroupID, input.SourceType, input.Name, input.SourceURI, input.StorageKey, input.ContentText, input.SizeBytes).Scan(&itemID)
if err != nil { if err != nil {
@@ -1159,8 +1256,21 @@ func (s *KnowledgeService) markKnowledgeIngestionRunning(
func (s *KnowledgeService) markKnowledgeIngestionCompleted( func (s *KnowledgeService) markKnowledgeIngestionCompleted(
ctx context.Context, ctx context.Context,
tenantID, itemID, parseTaskID int64, tenantID, itemID, parseTaskID int64,
itemName string,
content *knowledgeParsedContent,
metas []knowledgeChunkMetaInput, metas []knowledgeChunkMetaInput,
) error { ) error {
if content == nil {
return errors.New("knowledge content is empty")
}
if strings.TrimSpace(content.Text) == "" {
return errors.New("knowledge text content is empty")
}
if strings.TrimSpace(content.Markdown) == "" {
content.Markdown = buildKnowledgeMarkdownFromText("", content.Text)
}
itemName = strings.TrimSpace(itemName)
tx, err := s.pool.Begin(ctx) tx, err := s.pool.Begin(ctx)
if err != nil { if err != nil {
return err return err
@@ -1188,9 +1298,14 @@ func (s *KnowledgeService) markKnowledgeIngestionCompleted(
if _, err := tx.Exec(ctx, ` if _, err := tx.Exec(ctx, `
UPDATE knowledge_items UPDATE knowledge_items
SET status = 'completed', error_message = NULL, updated_at = NOW() SET status = 'completed',
error_message = NULL,
name = CASE WHEN $3 <> '' THEN $3 ELSE name END,
content_text = $4,
markdown_content = $5,
updated_at = NOW()
WHERE id = $1 AND tenant_id = $2 WHERE id = $1 AND tenant_id = $2
`, itemID, tenantID); err != nil { `, itemID, tenantID, itemName, content.Text, content.Markdown); err != nil {
return err return err
} }
@@ -1370,6 +1485,99 @@ func (s *KnowledgeService) ensureStorableGroup(ctx context.Context, tenantID, gr
return nil return nil
} }
func (s *KnowledgeService) resolveKnowledgeStorageGroupID(ctx context.Context, tenantID, groupID int64) (int64, error) {
tx, err := s.pool.Begin(ctx)
if err != nil {
return 0, response.ErrInternal(50055, "knowledge_group_resolve_failed", "failed to begin knowledge group transaction")
}
defer func() { _ = tx.Rollback(ctx) }()
resolvedGroupID, err := s.resolveKnowledgeStorageGroupIDTx(ctx, tx, tenantID, groupID)
if err != nil {
return 0, err
}
if err := tx.Commit(ctx); err != nil {
return 0, response.ErrInternal(50055, "knowledge_group_resolve_failed", "failed to commit knowledge group transaction")
}
return resolvedGroupID, nil
}
func (s *KnowledgeService) resolveKnowledgeStorageGroupIDTx(
ctx context.Context,
tx pgx.Tx,
tenantID, groupID int64,
) (int64, error) {
var parentID *int64
if err := tx.QueryRow(ctx, `
SELECT parent_id
FROM knowledge_groups
WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL
FOR UPDATE
`, groupID, tenantID).Scan(&parentID); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return 0, response.ErrNotFound(40451, "knowledge_group_not_found", "knowledge group not found")
}
return 0, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge group")
}
if parentID != nil {
var grandParentID *int64
if err := tx.QueryRow(ctx, `
SELECT parent_id
FROM knowledge_groups
WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL
`, *parentID, tenantID).Scan(&grandParentID); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return 0, response.ErrNotFound(40451, "knowledge_group_not_found", "knowledge group not found")
}
return 0, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge group")
}
if grandParentID != nil {
return 0, response.ErrBadRequest(40053, "knowledge_group_storage_invalid", "knowledge items can only be stored in second-level groups")
}
return groupID, nil
}
var childGroupID int64
if err := tx.QueryRow(ctx, `
SELECT id
FROM knowledge_groups
WHERE tenant_id = $1
AND parent_id = $2
AND name = $3
AND deleted_at IS NULL
ORDER BY id
LIMIT 1
`, tenantID, groupID, defaultKnowledgeChildGroupName).Scan(&childGroupID); err == nil {
return childGroupID, nil
} else if !errors.Is(err, pgx.ErrNoRows) {
return 0, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge group")
}
if err := tx.QueryRow(ctx, `
INSERT INTO knowledge_groups (tenant_id, name, parent_id, sort_order)
VALUES ($1, $2, $3, 0)
RETURNING id
`, tenantID, defaultKnowledgeChildGroupName, groupID).Scan(&childGroupID); err == nil {
return childGroupID, nil
}
if err := tx.QueryRow(ctx, `
SELECT id
FROM knowledge_groups
WHERE tenant_id = $1
AND parent_id = $2
AND name = $3
AND deleted_at IS NULL
ORDER BY id
LIMIT 1
`, tenantID, groupID, defaultKnowledgeChildGroupName).Scan(&childGroupID); err != nil {
return 0, response.ErrConflict(40950, "knowledge_group_create_failed", "failed to create default knowledge group")
}
return childGroupID, nil
}
func (s *KnowledgeService) listGroupSubtreeIDs( func (s *KnowledgeService) listGroupSubtreeIDs(
ctx context.Context, ctx context.Context,
tx pgx.Tx, tx pgx.Tx,
@@ -1688,9 +1896,10 @@ func (s *KnowledgeService) loadKnowledgeItemForCleanup(
) (*knowledgeItemRecord, error) { ) (*knowledgeItemRecord, error) {
record := &knowledgeItemRecord{} record := &knowledgeItemRecord{}
var contentText *string var contentText *string
var markdownContent *string
err := s.pool.QueryRow(ctx, ` err := s.pool.QueryRow(ctx, `
SELECT ki.id, ki.tenant_id, ki.group_id, kg.name, ki.name, ki.source_type, SELECT ki.id, ki.tenant_id, ki.group_id, kg.name, ki.name, ki.source_type,
ki.source_uri, ki.storage_key, ki.content_text, ki.item_version ki.source_uri, ki.storage_key, ki.content_text, ki.markdown_content, ki.item_version
FROM knowledge_items ki FROM knowledge_items ki
JOIN knowledge_groups kg ON kg.id = ki.group_id JOIN knowledge_groups kg ON kg.id = ki.group_id
WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NOT NULL WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NOT NULL
@@ -1704,12 +1913,14 @@ func (s *KnowledgeService) loadKnowledgeItemForCleanup(
&record.SourceURI, &record.SourceURI,
&record.StorageKey, &record.StorageKey,
&contentText, &contentText,
&markdownContent,
&record.ItemVersion, &record.ItemVersion,
) )
if err != nil { if err != nil {
return nil, nil return nil, nil
} }
record.ContentText = contentText record.ContentText = contentText
record.Markdown = markdownContent
return record, nil return record, nil
} }
@@ -0,0 +1,266 @@
package app
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/geo-platform/tenant-api/internal/shared/llm"
"github.com/geo-platform/tenant-api/internal/tenant/prompts"
"go.uber.org/zap"
)
const (
defaultKnowledgeArkBaseURL = "https://ark.cn-beijing.volces.com/api/v3"
defaultKnowledgeURLParseTimeout = 60 * time.Second
defaultKnowledgeURLMarkdownTimeout = 90 * time.Second
defaultKnowledgeURLMarkdownChunkChars = 12000
defaultKnowledgeURLMarkdownOutputToken = 12000
)
type arkToolExecuteRequest struct {
ActionName string `json:"action_name"`
ToolName string `json:"tool_name"`
Timeout int `json:"timeout,omitempty"`
Parameters map[string]any `json:"parameters,omitempty"`
}
type arkToolExecuteResponse struct {
StatusCode int `json:"status_code"`
Data arkToolExecuteData `json:"data"`
}
type arkToolExecuteData struct {
ArkWebDataList []arkWebDataItem `json:"ark_web_data_list"`
}
type arkWebDataItem struct {
URL string `json:"url"`
Title string `json:"title"`
Content string `json:"content"`
ErrorMessage string `json:"error_message"`
ErrorCode string `json:"error_code"`
}
func (s *KnowledgeService) parseWebsiteKnowledge(ctx context.Context, rawURL string) (*knowledgeParsedContent, error) {
item, err := s.executeLinkReader(ctx, rawURL)
if err != nil {
return nil, err
}
text := normalizeKnowledgeText(item.Content)
if text == "" {
return nil, fmt.Errorf("webpage parser returned empty content")
}
title := strings.TrimSpace(item.Title)
markdown, err := s.formatWebsiteMarkdown(ctx, title, text, rawURL)
if err != nil {
if s.logger != nil {
s.logger.Warn("knowledge website markdown formatting failed, fallback to local markdown",
zap.String("url", rawURL),
zap.Error(err),
)
}
markdown = buildKnowledgeMarkdownFromText(title, text)
}
return &knowledgeParsedContent{
Text: text,
Markdown: markdown,
}, nil
}
func (s *KnowledgeService) executeLinkReader(ctx context.Context, rawURL string) (*arkWebDataItem, error) {
baseURL := strings.TrimRight(strings.TrimSpace(s.arkBaseURL), "/")
if baseURL == "" {
baseURL = defaultKnowledgeArkBaseURL
}
timeout := defaultKnowledgeURLParseTimeout
callCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
body, err := json.Marshal(arkToolExecuteRequest{
ActionName: "LinkReader",
ToolName: "LinkReader",
Timeout: int(timeout / time.Second),
Parameters: map[string]any{
"url_list": []string{rawURL},
},
})
if err != nil {
return nil, fmt.Errorf("marshal webpage parser request: %w", err)
}
req, err := http.NewRequestWithContext(callCtx, http.MethodPost, baseURL+"/tools/execute", bytes.NewReader(body))
if err != nil {
return nil, fmt.Errorf("create webpage parser request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+s.arkAPIKey)
req.Header.Set("Content-Type", "application/json")
resp, err := s.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("execute webpage parser: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode >= http.StatusBadRequest {
data, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
return nil, fmt.Errorf("webpage parser failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(data)))
}
var parsed arkToolExecuteResponse
if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
return nil, fmt.Errorf("decode webpage parser response: %w", err)
}
if parsed.StatusCode != 0 && parsed.StatusCode != http.StatusOK {
return nil, fmt.Errorf("webpage parser returned status_code=%d", parsed.StatusCode)
}
if len(parsed.Data.ArkWebDataList) == 0 {
return nil, fmt.Errorf("webpage parser returned no content")
}
item := parsed.Data.ArkWebDataList[0]
if strings.TrimSpace(item.ErrorCode) != "" || strings.TrimSpace(item.ErrorMessage) != "" {
return nil, fmt.Errorf("webpage parser returned error: %s %s", strings.TrimSpace(item.ErrorCode), strings.TrimSpace(item.ErrorMessage))
}
return &item, nil
}
func (s *KnowledgeService) formatWebsiteMarkdown(ctx context.Context, title, content, rawURL string) (string, error) {
content = strings.TrimSpace(content)
if content == "" {
return "", fmt.Errorf("website content is empty")
}
chunks := splitKnowledgeMarkdownChunks(content, defaultKnowledgeURLMarkdownChunkChars)
if len(chunks) == 0 {
return "", fmt.Errorf("website content is empty after chunking")
}
sections := make([]string, 0, len(chunks)+1)
for index, chunk := range chunks {
result, err := s.llmClient.Generate(ctx, llm.GenerateRequest{
Model: s.urlMarkdownModel,
Prompt: prompts.KnowledgeWebsiteMarkdownPrompt(title, rawURL, chunk, index, len(chunks)),
Timeout: defaultKnowledgeURLMarkdownTimeout,
MaxOutputTokens: defaultKnowledgeURLMarkdownOutputToken,
}, nil)
if err != nil {
return "", err
}
section := sanitizeLLMMarkdown(strings.TrimSpace(result.Content))
if section == "" {
continue
}
if index > 0 && !strings.HasPrefix(section, "#") {
section = fmt.Sprintf("## 第 %d 部分\n\n%s", index+1, section)
}
sections = append(sections, section)
}
if len(sections) == 0 {
return "", fmt.Errorf("website formatter returned empty markdown")
}
markdown := strings.TrimSpace(strings.Join(sections, "\n\n"))
if title != "" && !strings.HasPrefix(markdown, "# ") {
markdown = "# " + title + "\n\n" + markdown
}
return markdown, nil
}
func splitKnowledgeMarkdownChunks(content string, maxChars int) []string {
if maxChars <= 0 {
maxChars = defaultKnowledgeURLMarkdownChunkChars
}
lines := strings.Split(content, "\n")
chunks := make([]string, 0, len(lines))
current := make([]string, 0, 16)
currentLen := 0
flush := func() {
if len(current) == 0 {
return
}
chunks = append(chunks, strings.TrimSpace(strings.Join(current, "\n")))
current = current[:0]
currentLen = 0
}
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
continue
}
if len([]rune(line)) > maxChars {
flush()
for _, segment := range splitKnowledgeLongLine(line, maxChars) {
if strings.TrimSpace(segment) != "" {
chunks = append(chunks, segment)
}
}
continue
}
nextLen := currentLen + len([]rune(line))
if len(current) > 0 {
nextLen++
}
if nextLen > maxChars {
flush()
}
current = append(current, line)
currentLen += len([]rune(line))
if len(current) > 1 {
currentLen++
}
}
flush()
return chunks
}
func splitKnowledgeLongLine(line string, maxChars int) []string {
runes := []rune(line)
if len(runes) <= maxChars {
return []string{line}
}
segments := make([]string, 0, (len(runes)/maxChars)+1)
for start := 0; start < len(runes); start += maxChars {
end := start + maxChars
if end > len(runes) {
end = len(runes)
}
segments = append(segments, strings.TrimSpace(string(runes[start:end])))
}
return segments
}
func sanitizeLLMMarkdown(content string) string {
content = strings.TrimSpace(content)
if !strings.HasPrefix(content, "```") {
return content
}
lines := strings.Split(content, "\n")
if len(lines) < 3 {
return strings.Trim(content, "`")
}
if !strings.HasPrefix(lines[0], "```") || strings.TrimSpace(lines[len(lines)-1]) != "```" {
return content
}
return strings.TrimSpace(strings.Join(lines[1:len(lines)-1], "\n"))
}
+30
View File
@@ -317,3 +317,33 @@ func KnowledgePromptIntroLines() []string {
func KnowledgeSnippetLabel(index int) string { func KnowledgeSnippetLabel(index int) string {
return fmt.Sprintf("知识片段 %d", index+1) return fmt.Sprintf("知识片段 %d", index+1)
} }
func KnowledgeWebsiteMarkdownPrompt(title, rawURL, content string, chunkIndex, chunkCount int) string {
scope := "以下是网页正文。"
if chunkCount > 1 {
scope = fmt.Sprintf("以下是网页正文的第 %d/%d 段。", chunkIndex+1, chunkCount)
}
var builder strings.Builder
builder.WriteString("你是知识库 Markdown 整理助手。\n")
builder.WriteString("任务:把给定网页正文整理为适合知识库只读预览的 Markdown。\n")
builder.WriteString("要求:\n")
builder.WriteString("1. 只根据原文整理,不要补充事实,不要总结扩写。\n")
builder.WriteString("2. 保留标题、列表、表格、编号、联系方式等关键信息。\n")
builder.WriteString("3. 如果原文明显是表格,请输出 Markdown 表格。\n")
builder.WriteString("4. 不要输出解释,不要输出代码块包裹,只输出 Markdown 正文。\n")
if strings.TrimSpace(title) != "" {
builder.WriteString("网页标题:")
builder.WriteString(title)
builder.WriteString("\n")
}
if strings.TrimSpace(rawURL) != "" {
builder.WriteString("网页地址:")
builder.WriteString(rawURL)
builder.WriteString("\n")
}
builder.WriteString(scope)
builder.WriteString("\n\n")
builder.WriteString(content)
return builder.String()
}
@@ -27,8 +27,10 @@ func NewArticleHandler(a *bootstrap.App) *ArticleHandler {
a.RetrievalProvider, a.RetrievalProvider,
a.VectorStore, a.VectorStore,
a.ObjectStorage, a.ObjectStorage,
a.LLM,
a.Logger, a.Logger,
a.Config.Retrieval, a.Config.Retrieval,
a.Config.LLM,
0, 0,
0, 0,
) )
@@ -23,8 +23,10 @@ func NewKnowledgeHandler(a *bootstrap.App) *KnowledgeHandler {
a.RetrievalProvider, a.RetrievalProvider,
a.VectorStore, a.VectorStore,
a.ObjectStorage, a.ObjectStorage,
a.LLM,
a.Logger, a.Logger,
a.Config.Retrieval, a.Config.Retrieval,
a.Config.LLM,
a.Config.Generation.QueueSize, a.Config.Generation.QueueSize,
a.Config.Generation.WorkerConcurrency, a.Config.Generation.WorkerConcurrency,
), ),
@@ -108,6 +110,21 @@ func (h *KnowledgeHandler) ListItems(c *gin.Context) {
response.Success(c, data) response.Success(c, data)
} }
func (h *KnowledgeHandler) GetItemDetail(c *gin.Context) {
itemID, err := strconv.ParseInt(c.Param("id"), 10, 64)
if err != nil {
response.Error(c, response.ErrBadRequest(40001, "invalid_id", "knowledge item id must be a number"))
return
}
data, err := h.svc.GetItemDetail(c.Request.Context(), itemID)
if err != nil {
response.Error(c, err)
return
}
response.Success(c, data)
}
func (h *KnowledgeHandler) CreateTextItem(c *gin.Context) { func (h *KnowledgeHandler) CreateTextItem(c *gin.Context) {
var req app.KnowledgeTextItemRequest var req app.KnowledgeTextItemRequest
if err := c.ShouldBindJSON(&req); err != nil { if err := c.ShouldBindJSON(&req); err != nil {
@@ -90,6 +90,7 @@ func RegisterRoutes(app *bootstrap.App) {
knowledge.PUT("/groups/:gid", knowledgeHandler.UpdateGroup) knowledge.PUT("/groups/:gid", knowledgeHandler.UpdateGroup)
knowledge.DELETE("/groups/:gid", knowledgeHandler.DeleteGroup) knowledge.DELETE("/groups/:gid", knowledgeHandler.DeleteGroup)
knowledge.GET("/items", knowledgeHandler.ListItems) knowledge.GET("/items", knowledgeHandler.ListItems)
knowledge.GET("/items/:id", knowledgeHandler.GetItemDetail)
knowledge.POST("/items/text", knowledgeHandler.CreateTextItem) knowledge.POST("/items/text", knowledgeHandler.CreateTextItem)
knowledge.POST("/items/url", knowledgeHandler.CreateURLItem) knowledge.POST("/items/url", knowledgeHandler.CreateURLItem)
knowledge.POST("/items/file", knowledgeHandler.CreateFileItem) knowledge.POST("/items/file", knowledgeHandler.CreateFileItem)
@@ -21,8 +21,10 @@ func NewTemplateHandler(a *bootstrap.App) *TemplateHandler {
a.RetrievalProvider, a.RetrievalProvider,
a.VectorStore, a.VectorStore,
a.ObjectStorage, a.ObjectStorage,
a.LLM,
a.Logger, a.Logger,
a.Config.Retrieval, a.Config.Retrieval,
a.Config.LLM,
0, 0,
0, 0,
) )
@@ -0,0 +1,2 @@
ALTER TABLE knowledge_items
DROP COLUMN IF EXISTS markdown_content;
@@ -0,0 +1,7 @@
ALTER TABLE knowledge_items
ADD COLUMN markdown_content TEXT;
UPDATE knowledge_items
SET markdown_content = content_text
WHERE markdown_content IS NULL
AND content_text IS NOT NULL;