feat: add knowledge markdown formatting and detail retrieval
- Implemented KnowledgeWebsiteMarkdownPrompt for formatting webpage content into Markdown. - Added GetItemDetail method in KnowledgeHandler to retrieve knowledge item details. - Introduced KnowledgeDetailDrawer component for displaying knowledge item details in the admin web interface. - Created MarkdownPreview component for rendering Markdown content. - Added knowledge chunking and URL parsing logic to handle webpage content extraction and formatting. - Updated database schema to include markdown_content in knowledge_items table.
This commit is contained in:
@@ -21,6 +21,7 @@
|
|||||||
"@vueuse/core": "^14.2.1",
|
"@vueuse/core": "^14.2.1",
|
||||||
"ant-design-vue": "^4.2.6",
|
"ant-design-vue": "^4.2.6",
|
||||||
"dayjs": "^1.11.20",
|
"dayjs": "^1.11.20",
|
||||||
|
"markdown-it": "^14.1.1",
|
||||||
"pinia": "^3.0.4",
|
"pinia": "^3.0.4",
|
||||||
"vue": "^3.5.31",
|
"vue": "^3.5.31",
|
||||||
"vue-i18n": "^10.0.5",
|
"vue-i18n": "^10.0.5",
|
||||||
@@ -28,6 +29,7 @@
|
|||||||
"zod": "^4.3.6"
|
"zod": "^4.3.6"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
"@types/markdown-it": "^14.1.2",
|
||||||
"@types/node": "^24.0.0",
|
"@types/node": "^24.0.0",
|
||||||
"@vitejs/plugin-vue": "^6.0.5",
|
"@vitejs/plugin-vue": "^6.0.5",
|
||||||
"typescript": "^5.9.3",
|
"typescript": "^5.9.3",
|
||||||
|
|||||||
@@ -0,0 +1,180 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
import { useQuery, useQueryClient } from "@tanstack/vue-query";
|
||||||
|
import { computed, watch } from "vue";
|
||||||
|
|
||||||
|
import MarkdownPreview from "@/components/MarkdownPreview.vue";
|
||||||
|
import { knowledgeApi } from "@/lib/api";
|
||||||
|
import { formatDateTime } from "@/lib/display";
|
||||||
|
|
||||||
|
const props = defineProps<{
|
||||||
|
open: boolean;
|
||||||
|
itemId: number | null;
|
||||||
|
}>();
|
||||||
|
|
||||||
|
const emit = defineEmits<{
|
||||||
|
close: [];
|
||||||
|
}>();
|
||||||
|
|
||||||
|
const queryClient = useQueryClient();
|
||||||
|
const enabled = computed(() => props.open && Boolean(props.itemId));
|
||||||
|
|
||||||
|
const detailQuery = useQuery({
|
||||||
|
queryKey: computed(() => ["knowledge", "detail", props.itemId]),
|
||||||
|
enabled,
|
||||||
|
queryFn: () => knowledgeApi.detail(props.itemId as number),
|
||||||
|
});
|
||||||
|
|
||||||
|
const detail = computed(() => detailQuery.data.value);
|
||||||
|
const hasMarkdown = computed(() => Boolean(detail.value?.markdown_content?.trim()));
|
||||||
|
const isProcessing = computed(() =>
|
||||||
|
detail.value?.status === "pending" || detail.value?.status === "processing",
|
||||||
|
);
|
||||||
|
const isFailed = computed(() => detail.value?.status === "failed");
|
||||||
|
const statusLabel = computed(() => formatKnowledgeStatus(detail.value?.status));
|
||||||
|
const statusColor = computed(() => {
|
||||||
|
if (detail.value?.status === "completed") return "success";
|
||||||
|
if (detail.value?.status === "failed") return "error";
|
||||||
|
if (isProcessing.value) return "processing";
|
||||||
|
return "default";
|
||||||
|
});
|
||||||
|
|
||||||
|
watch(
|
||||||
|
[() => props.open, () => props.itemId, () => detail.value?.status],
|
||||||
|
([open, itemId, status], _prev, onCleanup) => {
|
||||||
|
if (!open || !itemId || (status !== "pending" && status !== "processing")) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const timer = window.setInterval(() => {
|
||||||
|
void Promise.all([
|
||||||
|
detailQuery.refetch(),
|
||||||
|
queryClient.invalidateQueries({ queryKey: ["knowledge", "items"] }),
|
||||||
|
]);
|
||||||
|
}, 2500);
|
||||||
|
|
||||||
|
onCleanup(() => {
|
||||||
|
window.clearInterval(timer);
|
||||||
|
});
|
||||||
|
},
|
||||||
|
{ immediate: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
function handleClose(): void {
|
||||||
|
emit("close");
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatKnowledgeStatus(status?: string): string {
|
||||||
|
switch (status) {
|
||||||
|
case "completed":
|
||||||
|
return "学习完成";
|
||||||
|
case "processing":
|
||||||
|
return "处理中";
|
||||||
|
case "failed":
|
||||||
|
return "解析失败";
|
||||||
|
case "pending":
|
||||||
|
return "待处理";
|
||||||
|
case "deleted":
|
||||||
|
return "已删除";
|
||||||
|
default:
|
||||||
|
return status || "未知状态";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<a-drawer
|
||||||
|
:open="open"
|
||||||
|
width="760"
|
||||||
|
title="知识详情"
|
||||||
|
class="knowledge-detail-drawer"
|
||||||
|
@close="handleClose"
|
||||||
|
>
|
||||||
|
<div v-if="detailQuery.isPending.value" class="knowledge-detail-drawer__loading">
|
||||||
|
<a-skeleton active :paragraph="{ rows: 10 }" />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<template v-else-if="detail">
|
||||||
|
<section class="knowledge-detail-drawer__hero">
|
||||||
|
<div>
|
||||||
|
<h2>{{ detail.name }}</h2>
|
||||||
|
<div class="knowledge-detail-drawer__meta">
|
||||||
|
<span>{{ detail.group_name }}</span>
|
||||||
|
<span>{{ formatDateTime(detail.created_at) }}</span>
|
||||||
|
<span v-if="detail.source_uri">{{ detail.source_uri }}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<a-tag :color="statusColor">{{ statusLabel }}</a-tag>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<a-alert
|
||||||
|
v-if="isProcessing"
|
||||||
|
type="info"
|
||||||
|
show-icon
|
||||||
|
message="处理中"
|
||||||
|
description="内容正在解析并整理为 Markdown,抽屉会自动刷新。"
|
||||||
|
class="knowledge-detail-drawer__alert"
|
||||||
|
/>
|
||||||
|
|
||||||
|
<a-alert
|
||||||
|
v-else-if="isFailed"
|
||||||
|
type="error"
|
||||||
|
show-icon
|
||||||
|
message="解析失败"
|
||||||
|
:description="detail.error_message || '当前内容解析失败,请稍后重试。'"
|
||||||
|
class="knowledge-detail-drawer__alert"
|
||||||
|
/>
|
||||||
|
|
||||||
|
<div v-if="hasMarkdown" class="knowledge-detail-drawer__content">
|
||||||
|
<MarkdownPreview :content="detail.markdown_content || ''" />
|
||||||
|
</div>
|
||||||
|
<a-empty v-else description="无可预览文本" />
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<a-result
|
||||||
|
v-else
|
||||||
|
status="404"
|
||||||
|
title="未找到知识内容"
|
||||||
|
sub-title="该内容可能已删除,或当前租户无权访问。"
|
||||||
|
/>
|
||||||
|
</a-drawer>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.knowledge-detail-drawer__loading {
|
||||||
|
padding-top: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.knowledge-detail-drawer__hero {
|
||||||
|
display: flex;
|
||||||
|
align-items: flex-start;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 16px;
|
||||||
|
margin-bottom: 16px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.knowledge-detail-drawer__hero h2 {
|
||||||
|
margin: 0 0 8px;
|
||||||
|
color: #111827;
|
||||||
|
font-size: 22px;
|
||||||
|
font-weight: 700;
|
||||||
|
}
|
||||||
|
|
||||||
|
.knowledge-detail-drawer__meta {
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
gap: 8px 16px;
|
||||||
|
color: #6b7280;
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.knowledge-detail-drawer__alert {
|
||||||
|
margin-bottom: 16px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.knowledge-detail-drawer__content {
|
||||||
|
padding: 20px 24px;
|
||||||
|
border: 1px solid #f0f0f0;
|
||||||
|
border-radius: 16px;
|
||||||
|
background: #fff;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
import MarkdownIt from "markdown-it";
|
||||||
|
import { computed } from "vue";
|
||||||
|
|
||||||
|
const props = defineProps<{
|
||||||
|
content: string;
|
||||||
|
}>();
|
||||||
|
|
||||||
|
const renderer = new MarkdownIt({
|
||||||
|
html: false,
|
||||||
|
linkify: true,
|
||||||
|
breaks: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const html = computed(() => renderer.render(props.content || ""));
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<div class="markdown-preview" v-html="html" />
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.markdown-preview {
|
||||||
|
color: #111827;
|
||||||
|
font-size: 14px;
|
||||||
|
line-height: 1.75;
|
||||||
|
}
|
||||||
|
|
||||||
|
.markdown-preview :deep(h1),
|
||||||
|
.markdown-preview :deep(h2),
|
||||||
|
.markdown-preview :deep(h3),
|
||||||
|
.markdown-preview :deep(h4) {
|
||||||
|
margin: 1.2em 0 0.6em;
|
||||||
|
color: #111827;
|
||||||
|
font-weight: 700;
|
||||||
|
}
|
||||||
|
|
||||||
|
.markdown-preview :deep(h1) {
|
||||||
|
font-size: 28px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.markdown-preview :deep(h2) {
|
||||||
|
font-size: 22px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.markdown-preview :deep(h3) {
|
||||||
|
font-size: 18px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.markdown-preview :deep(p),
|
||||||
|
.markdown-preview :deep(ul),
|
||||||
|
.markdown-preview :deep(ol),
|
||||||
|
.markdown-preview :deep(blockquote) {
|
||||||
|
margin: 0 0 1em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.markdown-preview :deep(ul),
|
||||||
|
.markdown-preview :deep(ol) {
|
||||||
|
padding-left: 1.5em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.markdown-preview :deep(a) {
|
||||||
|
color: #1677ff;
|
||||||
|
}
|
||||||
|
|
||||||
|
.markdown-preview :deep(code) {
|
||||||
|
padding: 2px 6px;
|
||||||
|
border-radius: 6px;
|
||||||
|
background: #f3f4f6;
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.markdown-preview :deep(pre) {
|
||||||
|
overflow-x: auto;
|
||||||
|
padding: 16px;
|
||||||
|
border-radius: 12px;
|
||||||
|
background: #111827;
|
||||||
|
color: #f9fafb;
|
||||||
|
}
|
||||||
|
|
||||||
|
.markdown-preview :deep(pre code) {
|
||||||
|
padding: 0;
|
||||||
|
background: transparent;
|
||||||
|
color: inherit;
|
||||||
|
}
|
||||||
|
|
||||||
|
.markdown-preview :deep(table) {
|
||||||
|
width: 100%;
|
||||||
|
margin: 0 0 1em;
|
||||||
|
border-collapse: collapse;
|
||||||
|
overflow: hidden;
|
||||||
|
border: 1px solid #e5e7eb;
|
||||||
|
border-radius: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.markdown-preview :deep(th),
|
||||||
|
.markdown-preview :deep(td) {
|
||||||
|
padding: 10px 12px;
|
||||||
|
border: 1px solid #e5e7eb;
|
||||||
|
text-align: left;
|
||||||
|
vertical-align: top;
|
||||||
|
}
|
||||||
|
|
||||||
|
.markdown-preview :deep(th) {
|
||||||
|
background: #f9fafb;
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
|
||||||
|
.markdown-preview :deep(hr) {
|
||||||
|
margin: 24px 0;
|
||||||
|
border: 0;
|
||||||
|
border-top: 1px solid #e5e7eb;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -21,6 +21,7 @@ import type {
|
|||||||
KnowledgeGroup,
|
KnowledgeGroup,
|
||||||
KnowledgeGroupRequest,
|
KnowledgeGroupRequest,
|
||||||
KnowledgeItem,
|
KnowledgeItem,
|
||||||
|
KnowledgeItemDetail,
|
||||||
KnowledgeTextItemRequest,
|
KnowledgeTextItemRequest,
|
||||||
KnowledgeURLItemRequest,
|
KnowledgeURLItemRequest,
|
||||||
Keyword,
|
Keyword,
|
||||||
@@ -248,6 +249,9 @@ export const knowledgeApi = {
|
|||||||
params: groupId ? { group_id: groupId } : undefined,
|
params: groupId ? { group_id: groupId } : undefined,
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
|
detail(id: number) {
|
||||||
|
return apiClient.get<KnowledgeItemDetail>(`/api/tenant/knowledge/items/${id}`);
|
||||||
|
},
|
||||||
createTextItem(payload: KnowledgeTextItemRequest) {
|
createTextItem(payload: KnowledgeTextItemRequest) {
|
||||||
return apiClient.post<KnowledgeItem, KnowledgeTextItemRequest>(
|
return apiClient.post<KnowledgeItem, KnowledgeTextItemRequest>(
|
||||||
"/api/tenant/knowledge/items/text",
|
"/api/tenant/knowledge/items/text",
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ import {
|
|||||||
Popconfirm,
|
Popconfirm,
|
||||||
Progress,
|
Progress,
|
||||||
Radio,
|
Radio,
|
||||||
|
Result,
|
||||||
Row,
|
Row,
|
||||||
Select,
|
Select,
|
||||||
Skeleton,
|
Skeleton,
|
||||||
@@ -92,6 +93,7 @@ app.component("ATextarea", Input.TextArea);
|
|||||||
Popconfirm,
|
Popconfirm,
|
||||||
Progress,
|
Progress,
|
||||||
Radio,
|
Radio,
|
||||||
|
Result,
|
||||||
Row,
|
Row,
|
||||||
Select,
|
Select,
|
||||||
Skeleton,
|
Skeleton,
|
||||||
|
|||||||
@@ -1,17 +1,18 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { MoreOutlined, PlusOutlined, UploadOutlined } from "@ant-design/icons-vue";
|
import { DeleteOutlined, EyeOutlined, MoreOutlined, PlusOutlined, UploadOutlined } from "@ant-design/icons-vue";
|
||||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/vue-query";
|
import { useMutation, useQuery, useQueryClient } from "@tanstack/vue-query";
|
||||||
import { message, Modal, type TableColumnsType, type UploadProps } from "ant-design-vue";
|
import { message, Modal, type TableColumnsType, type UploadProps } from "ant-design-vue";
|
||||||
import type { KnowledgeGroup, KnowledgeItem } from "@geo/shared-types";
|
import type { KnowledgeGroup, KnowledgeItem } from "@geo/shared-types";
|
||||||
import { computed, reactive, ref, watch } from "vue";
|
import { computed, reactive, ref, watch } from "vue";
|
||||||
|
|
||||||
|
import KnowledgeDetailDrawer from "@/components/KnowledgeDetailDrawer.vue";
|
||||||
import { knowledgeApi } from "@/lib/api";
|
import { knowledgeApi } from "@/lib/api";
|
||||||
import { formatDateTime } from "@/lib/display";
|
import { formatDateTime } from "@/lib/display";
|
||||||
import { formatError } from "@/lib/errors";
|
import { formatError } from "@/lib/errors";
|
||||||
|
|
||||||
type KnowledgeSourceType = "document" | "website" | "text";
|
type KnowledgeSourceType = "document" | "website" | "text";
|
||||||
type KnowledgeGroupLevel = "root" | "child";
|
type KnowledgeGroupLevel = "root" | "child";
|
||||||
const DEFAULT_CHILD_GROUP_NAME = "默认目录";
|
const MAX_WEBSITE_URLS = 3;
|
||||||
|
|
||||||
interface TreeNode {
|
interface TreeNode {
|
||||||
title: string;
|
title: string;
|
||||||
@@ -29,6 +30,8 @@ const editingGroup = ref<KnowledgeGroup | null>(null);
|
|||||||
const itemModalOpen = ref(false);
|
const itemModalOpen = ref(false);
|
||||||
const itemSourceType = ref<KnowledgeSourceType>("document");
|
const itemSourceType = ref<KnowledgeSourceType>("document");
|
||||||
const uploadFiles = ref<File[]>([]);
|
const uploadFiles = ref<File[]>([]);
|
||||||
|
const previewItemId = ref<number | null>(null);
|
||||||
|
const previewDrawerOpen = ref(false);
|
||||||
|
|
||||||
const groupForm = reactive({
|
const groupForm = reactive({
|
||||||
name: "",
|
name: "",
|
||||||
@@ -67,6 +70,17 @@ const groupMap = computed(() => buildGroupMap(rootGroups.value));
|
|||||||
const treeData = computed(() => rootGroups.value.map((item) => toSidebarTreeNode(item)));
|
const treeData = computed(() => rootGroups.value.map((item) => toSidebarTreeNode(item)));
|
||||||
const storableTreeData = computed(() => rootGroups.value.map((item) => toStorageTreeNode(item)));
|
const storableTreeData = computed(() => rootGroups.value.map((item) => toStorageTreeNode(item)));
|
||||||
const storableGroupCount = computed(() => countStorableGroups(rootGroups.value));
|
const storableGroupCount = computed(() => countStorableGroups(rootGroups.value));
|
||||||
|
const websiteUrls = computed(() =>
|
||||||
|
urlsText.value
|
||||||
|
.split("\n")
|
||||||
|
.map((item) => item.trim())
|
||||||
|
.filter(Boolean),
|
||||||
|
);
|
||||||
|
const websiteUrlLimitExceeded = computed(() => websiteUrls.value.length > MAX_WEBSITE_URLS);
|
||||||
|
const itemModalOkButtonProps = computed(() => ({
|
||||||
|
style: { width: "80px", borderRadius: "4px" },
|
||||||
|
disabled: itemSourceType.value === "website" && websiteUrlLimitExceeded.value,
|
||||||
|
}));
|
||||||
const groupModalTitle = computed(() => {
|
const groupModalTitle = computed(() => {
|
||||||
if (editingGroup.value) {
|
if (editingGroup.value) {
|
||||||
return isRootGroup(editingGroup.value) ? "编辑分组" : "编辑子目录";
|
return isRootGroup(editingGroup.value) ? "编辑分组" : "编辑子目录";
|
||||||
@@ -119,13 +133,11 @@ const groupMutations = {
|
|||||||
|
|
||||||
const itemMutation = useMutation({
|
const itemMutation = useMutation({
|
||||||
mutationFn: async () => {
|
mutationFn: async () => {
|
||||||
let targetGroupId = itemForm.group_id;
|
const targetGroupId = itemForm.group_id;
|
||||||
if (!targetGroupId) {
|
if (!targetGroupId) {
|
||||||
throw new Error("请选择分组目录");
|
throw new Error("请选择分组目录");
|
||||||
}
|
}
|
||||||
|
|
||||||
targetGroupId = await resolveItemTargetGroupId(targetGroupId);
|
|
||||||
|
|
||||||
const promises: Promise<any>[] = [];
|
const promises: Promise<any>[] = [];
|
||||||
|
|
||||||
if (itemSourceType.value === "document") {
|
if (itemSourceType.value === "document") {
|
||||||
@@ -142,10 +154,13 @@ const itemMutation = useMutation({
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
} else if (itemSourceType.value === "website") {
|
} else if (itemSourceType.value === "website") {
|
||||||
const urls = urlsText.value.split('\n').map(u => u.trim()).filter(Boolean);
|
const urls = websiteUrls.value;
|
||||||
if (urls.length === 0) {
|
if (urls.length === 0) {
|
||||||
throw new Error("请输入包含URL的文本");
|
throw new Error("请输入包含URL的文本");
|
||||||
}
|
}
|
||||||
|
if (urls.length > MAX_WEBSITE_URLS) {
|
||||||
|
throw new Error(`一次最多支持 ${MAX_WEBSITE_URLS} 个网址`);
|
||||||
|
}
|
||||||
for (const url of urls) {
|
for (const url of urls) {
|
||||||
let name = url;
|
let name = url;
|
||||||
try { name = new URL(url).hostname + new URL(url).pathname } catch (e) {}
|
try { name = new URL(url).hostname + new URL(url).pathname } catch (e) {}
|
||||||
@@ -173,12 +188,31 @@ const itemMutation = useMutation({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
await Promise.all(promises);
|
const results = await Promise.allSettled(promises);
|
||||||
|
const successCount = results.filter((result) => result.status === "fulfilled").length;
|
||||||
|
const failedResults = results.filter((result): result is PromiseRejectedResult => result.status === "rejected");
|
||||||
|
|
||||||
|
if (failedResults.length > 0 && successCount === 0) {
|
||||||
|
throw failedResults[0].reason;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
totalCount: promises.length,
|
||||||
|
successCount,
|
||||||
|
failedCount: failedResults.length,
|
||||||
|
firstError: failedResults[0]?.reason,
|
||||||
|
};
|
||||||
},
|
},
|
||||||
onSuccess: async () => {
|
onSuccess: async (result) => {
|
||||||
message.success("内容已提交");
|
if (result.failedCount === 0) {
|
||||||
itemModalOpen.value = false;
|
message.success(result.successCount > 1 ? `已提交 ${result.successCount} 条内容` : "内容已提交");
|
||||||
resetItemForm();
|
itemModalOpen.value = false;
|
||||||
|
resetItemForm();
|
||||||
|
} else {
|
||||||
|
message.warning(
|
||||||
|
`已成功提交 ${result.successCount} 条,${result.failedCount} 条失败${result.firstError ? `:${formatError(result.firstError)}` : ""}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
await queryClient.invalidateQueries({ queryKey: ["knowledge"] });
|
await queryClient.invalidateQueries({ queryKey: ["knowledge"] });
|
||||||
},
|
},
|
||||||
onError: (error) => message.error(formatError(error)),
|
onError: (error) => message.error(formatError(error)),
|
||||||
@@ -310,29 +344,6 @@ function isStorableGroupId(id: number | null | undefined): boolean {
|
|||||||
return Boolean(group && isStorableGroup(group));
|
return Boolean(group && isStorableGroup(group));
|
||||||
}
|
}
|
||||||
|
|
||||||
function findChildGroupByName(parentId: number, name: string): KnowledgeGroup | undefined {
|
|
||||||
const parentGroup = getGroupById(parentId);
|
|
||||||
return parentGroup?.children.find((child) => child.name === name);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function resolveItemTargetGroupId(groupId: number): Promise<number> {
|
|
||||||
if (!isNodeRootGroup(groupId)) {
|
|
||||||
return groupId;
|
|
||||||
}
|
|
||||||
|
|
||||||
const defaultChild = findChildGroupByName(groupId, DEFAULT_CHILD_GROUP_NAME);
|
|
||||||
if (defaultChild) {
|
|
||||||
return defaultChild.id;
|
|
||||||
}
|
|
||||||
|
|
||||||
const newGroup = await knowledgeApi.createGroup({
|
|
||||||
name: DEFAULT_CHILD_GROUP_NAME,
|
|
||||||
parent_id: groupId,
|
|
||||||
});
|
|
||||||
return newGroup.id;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
function resolveDefaultItemGroupId(): number | undefined {
|
function resolveDefaultItemGroupId(): number | undefined {
|
||||||
if (selectedGroupId.value) {
|
if (selectedGroupId.value) {
|
||||||
return selectedGroupId.value;
|
return selectedGroupId.value;
|
||||||
@@ -481,8 +492,14 @@ function formatSize(value: number): string {
|
|||||||
return `${(value / 1024 / 1024).toFixed(1)} MB`;
|
return `${(value / 1024 / 1024).toFixed(1)} MB`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function editItem(): void {
|
function openItemDetail(itemId: number): void {
|
||||||
message.info("内容编辑功能开发中");
|
previewItemId.value = itemId;
|
||||||
|
previewDrawerOpen.value = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function closeItemDetail(): void {
|
||||||
|
previewDrawerOpen.value = false;
|
||||||
|
previewItemId.value = null;
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
@@ -570,11 +587,16 @@ function editItem(): void {
|
|||||||
{{ formatDateTime(record.created_at) }}
|
{{ formatDateTime(record.created_at) }}
|
||||||
</template>
|
</template>
|
||||||
<template v-else-if="column.key === 'actions'">
|
<template v-else-if="column.key === 'actions'">
|
||||||
<div class="knowledge-actions">
|
<div class="table-actions-row">
|
||||||
<a-button type="link" style="padding: 0 4px" @click="editItem()">编辑</a-button>
|
<a-tooltip title="查看">
|
||||||
<a-divider type="vertical" />
|
<a-button type="text" shape="circle" size="small" class="action-btn action-eye" @click="openItemDetail(record.id)">
|
||||||
|
<EyeOutlined />
|
||||||
|
</a-button>
|
||||||
|
</a-tooltip>
|
||||||
<a-popconfirm title="确认删除该内容?" @confirm="deleteItemMutation.mutate(record.id)">
|
<a-popconfirm title="确认删除该内容?" @confirm="deleteItemMutation.mutate(record.id)">
|
||||||
<a-button type="link" style="padding: 0 4px">删除</a-button>
|
<a-button type="text" shape="circle" size="small" class="action-btn action-delete">
|
||||||
|
<DeleteOutlined />
|
||||||
|
</a-button>
|
||||||
</a-popconfirm>
|
</a-popconfirm>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
@@ -615,7 +637,7 @@ function editItem(): void {
|
|||||||
"
|
"
|
||||||
ok-text="确 定"
|
ok-text="确 定"
|
||||||
cancel-text="取 消"
|
cancel-text="取 消"
|
||||||
:ok-button-props="{ style: { width: '80px', borderRadius: '4px' } }"
|
:ok-button-props="itemModalOkButtonProps"
|
||||||
:cancel-button-props="{ style: { width: '80px', borderRadius: '4px' } }"
|
:cancel-button-props="{ style: { width: '80px', borderRadius: '4px' } }"
|
||||||
>
|
>
|
||||||
<div class="knowledge-form-add">
|
<div class="knowledge-form-add">
|
||||||
@@ -644,7 +666,7 @@ function editItem(): void {
|
|||||||
</div>
|
</div>
|
||||||
<p class="dragger-text">点击或拖拽文件至此区域即可上传</p>
|
<p class="dragger-text">点击或拖拽文件至此区域即可上传</p>
|
||||||
<p class="dragger-hint">
|
<p class="dragger-hint">
|
||||||
请上传doc、pdf、txt、xls、xlsx格式,不超过30M的文件。严禁上传敏感数据或其他非法文件。
|
请上传 docx、pdf、txt、md、xls、xlsx 格式,不超过30M的文件。严禁上传敏感数据或其他非法文件。
|
||||||
</p>
|
</p>
|
||||||
<p class="dragger-count">已选择文件:{{ uploadFiles.length }}个</p>
|
<p class="dragger-count">已选择文件:{{ uploadFiles.length }}个</p>
|
||||||
</div>
|
</div>
|
||||||
@@ -655,9 +677,17 @@ function editItem(): void {
|
|||||||
<a-textarea
|
<a-textarea
|
||||||
v-model:value="urlsText"
|
v-model:value="urlsText"
|
||||||
:rows="11"
|
:rows="11"
|
||||||
placeholder="请输入产品相关网页URL,每行一个"
|
:status="websiteUrlLimitExceeded ? 'error' : undefined"
|
||||||
|
placeholder="请输入产品相关网页URL,每行一个,最多支持3个"
|
||||||
class="custom-textarea"
|
class="custom-textarea"
|
||||||
/>
|
/>
|
||||||
|
<div
|
||||||
|
class="knowledge-url-hint"
|
||||||
|
:class="{ 'knowledge-url-hint--error': websiteUrlLimitExceeded }"
|
||||||
|
>
|
||||||
|
当前 {{ websiteUrls.length }} / {{ MAX_WEBSITE_URLS }} 个网址
|
||||||
|
<span v-if="websiteUrlLimitExceeded">,请删除多余网址后再提交</span>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div v-else class="knowledge-content-wrapper">
|
<div v-else class="knowledge-content-wrapper">
|
||||||
@@ -690,6 +720,12 @@ function editItem(): void {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</a-modal>
|
</a-modal>
|
||||||
|
|
||||||
|
<KnowledgeDetailDrawer
|
||||||
|
:open="previewDrawerOpen"
|
||||||
|
:item-id="previewItemId"
|
||||||
|
@close="closeItemDetail"
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
@@ -776,7 +812,7 @@ function editItem(): void {
|
|||||||
}
|
}
|
||||||
|
|
||||||
.knowledge-sidebar__add {
|
.knowledge-sidebar__add {
|
||||||
color: #9ca3af;
|
color: #666;
|
||||||
font-size: 13px;
|
font-size: 13px;
|
||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
display: flex;
|
display: flex;
|
||||||
@@ -823,12 +859,21 @@ function editItem(): void {
|
|||||||
color: #6b7280;
|
color: #6b7280;
|
||||||
}
|
}
|
||||||
|
|
||||||
.knowledge-actions {
|
/* Actions */
|
||||||
|
.table-actions-row {
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
justify-content: flex-end;
|
justify-content: flex-end;
|
||||||
|
gap: 4px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.action-btn {
|
||||||
|
color: #8c8c8c;
|
||||||
|
font-size: 14px;
|
||||||
|
}
|
||||||
|
.action-eye:hover { color: #13c2c2; background: #e6fffb; }
|
||||||
|
.action-delete:hover { color: #ff4d4f; background: #fff2f0; }
|
||||||
|
|
||||||
.knowledge-form {
|
.knowledge-form {
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
@@ -871,6 +916,17 @@ function editItem(): void {
|
|||||||
margin-top: 4px;
|
margin-top: 4px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.knowledge-url-hint {
|
||||||
|
margin-top: 8px;
|
||||||
|
color: #6b7280;
|
||||||
|
font-size: 12px;
|
||||||
|
line-height: 1.5;
|
||||||
|
}
|
||||||
|
|
||||||
|
.knowledge-url-hint--error {
|
||||||
|
color: #ff4d4f;
|
||||||
|
}
|
||||||
|
|
||||||
:deep(.custom-dragger.ant-upload-wrapper .ant-upload-drag) {
|
:deep(.custom-dragger.ant-upload-wrapper .ant-upload-drag) {
|
||||||
border: 1px dashed #7eb0f7;
|
border: 1px dashed #7eb0f7;
|
||||||
background-color: #fff;
|
background-color: #fff;
|
||||||
|
|||||||
@@ -556,6 +556,10 @@ export interface KnowledgeItem {
|
|||||||
updated_at: string;
|
updated_at: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface KnowledgeItemDetail extends KnowledgeItem {
|
||||||
|
markdown_content: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
export interface KnowledgeTextItemRequest {
|
export interface KnowledgeTextItemRequest {
|
||||||
group_id: number;
|
group_id: number;
|
||||||
name: string;
|
name: string;
|
||||||
|
|||||||
Generated
+65
@@ -43,6 +43,9 @@ importers:
|
|||||||
dayjs:
|
dayjs:
|
||||||
specifier: ^1.11.20
|
specifier: ^1.11.20
|
||||||
version: 1.11.20
|
version: 1.11.20
|
||||||
|
markdown-it:
|
||||||
|
specifier: ^14.1.1
|
||||||
|
version: 14.1.1
|
||||||
pinia:
|
pinia:
|
||||||
specifier: ^3.0.4
|
specifier: ^3.0.4
|
||||||
version: 3.0.4(typescript@5.9.3)(vue@3.5.31(typescript@5.9.3))
|
version: 3.0.4(typescript@5.9.3)(vue@3.5.31(typescript@5.9.3))
|
||||||
@@ -59,6 +62,9 @@ importers:
|
|||||||
specifier: ^4.3.6
|
specifier: ^4.3.6
|
||||||
version: 4.3.6
|
version: 4.3.6
|
||||||
devDependencies:
|
devDependencies:
|
||||||
|
'@types/markdown-it':
|
||||||
|
specifier: ^14.1.2
|
||||||
|
version: 14.1.2
|
||||||
'@types/node':
|
'@types/node':
|
||||||
specifier: ^24.0.0
|
specifier: ^24.0.0
|
||||||
version: 24.12.0
|
version: 24.12.0
|
||||||
@@ -790,15 +796,24 @@ packages:
|
|||||||
'@types/katex@0.16.8':
|
'@types/katex@0.16.8':
|
||||||
resolution: {integrity: sha512-trgaNyfU+Xh2Tc+ABIb44a5AYUpicB3uwirOioeOkNPPbmgRNtcWyDeeFRzjPZENO9Vq8gvVqfhaaXWLlevVwg==}
|
resolution: {integrity: sha512-trgaNyfU+Xh2Tc+ABIb44a5AYUpicB3uwirOioeOkNPPbmgRNtcWyDeeFRzjPZENO9Vq8gvVqfhaaXWLlevVwg==}
|
||||||
|
|
||||||
|
'@types/linkify-it@5.0.0':
|
||||||
|
resolution: {integrity: sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==}
|
||||||
|
|
||||||
'@types/lodash-es@4.17.12':
|
'@types/lodash-es@4.17.12':
|
||||||
resolution: {integrity: sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==}
|
resolution: {integrity: sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==}
|
||||||
|
|
||||||
'@types/lodash@4.17.24':
|
'@types/lodash@4.17.24':
|
||||||
resolution: {integrity: sha512-gIW7lQLZbue7lRSWEFql49QJJWThrTFFeIMJdp3eH4tKoxm1OvEPg02rm4wCCSHS0cL3/Fizimb35b7k8atwsQ==}
|
resolution: {integrity: sha512-gIW7lQLZbue7lRSWEFql49QJJWThrTFFeIMJdp3eH4tKoxm1OvEPg02rm4wCCSHS0cL3/Fizimb35b7k8atwsQ==}
|
||||||
|
|
||||||
|
'@types/markdown-it@14.1.2':
|
||||||
|
resolution: {integrity: sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog==}
|
||||||
|
|
||||||
'@types/mdast@4.0.4':
|
'@types/mdast@4.0.4':
|
||||||
resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==}
|
resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==}
|
||||||
|
|
||||||
|
'@types/mdurl@2.0.0':
|
||||||
|
resolution: {integrity: sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==}
|
||||||
|
|
||||||
'@types/minimatch@3.0.5':
|
'@types/minimatch@3.0.5':
|
||||||
resolution: {integrity: sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==}
|
resolution: {integrity: sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==}
|
||||||
|
|
||||||
@@ -951,6 +966,9 @@ packages:
|
|||||||
peerDependencies:
|
peerDependencies:
|
||||||
vue: '>=3.2.0'
|
vue: '>=3.2.0'
|
||||||
|
|
||||||
|
argparse@2.0.1:
|
||||||
|
resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==}
|
||||||
|
|
||||||
array-differ@4.0.0:
|
array-differ@4.0.0:
|
||||||
resolution: {integrity: sha512-Q6VPTLMsmXZ47ENG3V+wQyZS1ZxXMxFyYzA+Z/GMrJ6yIutAIEf9wTyroTzmGjNfox9/h3GdGBCVh43GVFx4Uw==}
|
resolution: {integrity: sha512-Q6VPTLMsmXZ47ENG3V+wQyZS1ZxXMxFyYzA+Z/GMrJ6yIutAIEf9wTyroTzmGjNfox9/h3GdGBCVh43GVFx4Uw==}
|
||||||
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
|
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
|
||||||
@@ -1760,6 +1778,9 @@ packages:
|
|||||||
canvas:
|
canvas:
|
||||||
optional: true
|
optional: true
|
||||||
|
|
||||||
|
linkify-it@5.0.0:
|
||||||
|
resolution: {integrity: sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==}
|
||||||
|
|
||||||
listr2@10.2.1:
|
listr2@10.2.1:
|
||||||
resolution: {integrity: sha512-7I5knELsJKTUjXG+A6BkKAiGkW1i25fNa/xlUl9hFtk15WbE9jndA89xu5FzQKrY5llajE1hfZZFMILXkDHk/Q==}
|
resolution: {integrity: sha512-7I5knELsJKTUjXG+A6BkKAiGkW1i25fNa/xlUl9hFtk15WbE9jndA89xu5FzQKrY5llajE1hfZZFMILXkDHk/Q==}
|
||||||
engines: {node: '>=22.13.0'}
|
engines: {node: '>=22.13.0'}
|
||||||
@@ -1821,6 +1842,10 @@ packages:
|
|||||||
many-keys-map@2.0.1:
|
many-keys-map@2.0.1:
|
||||||
resolution: {integrity: sha512-DHnZAD4phTbZ+qnJdjoNEVU1NecYoSdbOOoVmTDH46AuxDkEVh3MxTVpXq10GtcTC6mndN9dkv1rNfpjRcLnOw==}
|
resolution: {integrity: sha512-DHnZAD4phTbZ+qnJdjoNEVU1NecYoSdbOOoVmTDH46AuxDkEVh3MxTVpXq10GtcTC6mndN9dkv1rNfpjRcLnOw==}
|
||||||
|
|
||||||
|
markdown-it@14.1.1:
|
||||||
|
resolution: {integrity: sha512-BuU2qnTti9YKgK5N+IeMubp14ZUKUUw7yeJbkjtosvHiP0AZ5c8IAgEMk79D0eC8F23r4Ac/q8cAIFdm2FtyoA==}
|
||||||
|
hasBin: true
|
||||||
|
|
||||||
markdown-table@3.0.4:
|
markdown-table@3.0.4:
|
||||||
resolution: {integrity: sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==}
|
resolution: {integrity: sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==}
|
||||||
|
|
||||||
@@ -1875,6 +1900,9 @@ packages:
|
|||||||
mdast-util-to-string@4.0.0:
|
mdast-util-to-string@4.0.0:
|
||||||
resolution: {integrity: sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==}
|
resolution: {integrity: sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==}
|
||||||
|
|
||||||
|
mdurl@2.0.0:
|
||||||
|
resolution: {integrity: sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==}
|
||||||
|
|
||||||
micromark-core-commonmark@2.0.3:
|
micromark-core-commonmark@2.0.3:
|
||||||
resolution: {integrity: sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==}
|
resolution: {integrity: sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==}
|
||||||
|
|
||||||
@@ -2226,6 +2254,10 @@ packages:
|
|||||||
engines: {node: '>=18.0.0'}
|
engines: {node: '>=18.0.0'}
|
||||||
hasBin: true
|
hasBin: true
|
||||||
|
|
||||||
|
punycode.js@2.3.1:
|
||||||
|
resolution: {integrity: sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==}
|
||||||
|
engines: {node: '>=6'}
|
||||||
|
|
||||||
pupa@3.3.0:
|
pupa@3.3.0:
|
||||||
resolution: {integrity: sha512-LjgDO2zPtoXP2wJpDjZrGdojii1uqO0cnwKoIoUzkfS98HDmbeiGmYiXo3lXeFlq2xvne1QFQhwYXSUCLKtEuA==}
|
resolution: {integrity: sha512-LjgDO2zPtoXP2wJpDjZrGdojii1uqO0cnwKoIoUzkfS98HDmbeiGmYiXo3lXeFlq2xvne1QFQhwYXSUCLKtEuA==}
|
||||||
engines: {node: '>=12.20'}
|
engines: {node: '>=12.20'}
|
||||||
@@ -2495,6 +2527,9 @@ packages:
|
|||||||
engines: {node: '>=14.17'}
|
engines: {node: '>=14.17'}
|
||||||
hasBin: true
|
hasBin: true
|
||||||
|
|
||||||
|
uc.micro@2.1.0:
|
||||||
|
resolution: {integrity: sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==}
|
||||||
|
|
||||||
ufo@1.6.3:
|
ufo@1.6.3:
|
||||||
resolution: {integrity: sha512-yDJTmhydvl5lJzBmy/hyOAA0d+aqCBuwl818haVdYCRrWV84o7YyeVm4QlVHStqNrrJSTb6jKuFAVqAFsr+K3Q==}
|
resolution: {integrity: sha512-yDJTmhydvl5lJzBmy/hyOAA0d+aqCBuwl818haVdYCRrWV84o7YyeVm4QlVHStqNrrJSTb6jKuFAVqAFsr+K3Q==}
|
||||||
|
|
||||||
@@ -3711,16 +3746,25 @@ snapshots:
|
|||||||
|
|
||||||
'@types/katex@0.16.8': {}
|
'@types/katex@0.16.8': {}
|
||||||
|
|
||||||
|
'@types/linkify-it@5.0.0': {}
|
||||||
|
|
||||||
'@types/lodash-es@4.17.12':
|
'@types/lodash-es@4.17.12':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/lodash': 4.17.24
|
'@types/lodash': 4.17.24
|
||||||
|
|
||||||
'@types/lodash@4.17.24': {}
|
'@types/lodash@4.17.24': {}
|
||||||
|
|
||||||
|
'@types/markdown-it@14.1.2':
|
||||||
|
dependencies:
|
||||||
|
'@types/linkify-it': 5.0.0
|
||||||
|
'@types/mdurl': 2.0.0
|
||||||
|
|
||||||
'@types/mdast@4.0.4':
|
'@types/mdast@4.0.4':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/unist': 3.0.3
|
'@types/unist': 3.0.3
|
||||||
|
|
||||||
|
'@types/mdurl@2.0.0': {}
|
||||||
|
|
||||||
'@types/minimatch@3.0.5': {}
|
'@types/minimatch@3.0.5': {}
|
||||||
|
|
||||||
'@types/ms@2.1.0': {}
|
'@types/ms@2.1.0': {}
|
||||||
@@ -3930,6 +3974,8 @@ snapshots:
|
|||||||
vue-types: 3.0.2(vue@3.5.31(typescript@5.9.3))
|
vue-types: 3.0.2(vue@3.5.31(typescript@5.9.3))
|
||||||
warning: 4.0.3
|
warning: 4.0.3
|
||||||
|
|
||||||
|
argparse@2.0.1: {}
|
||||||
|
|
||||||
array-differ@4.0.0: {}
|
array-differ@4.0.0: {}
|
||||||
|
|
||||||
array-tree-filter@2.1.0: {}
|
array-tree-filter@2.1.0: {}
|
||||||
@@ -4648,6 +4694,10 @@ snapshots:
|
|||||||
htmlparser2: 10.1.0
|
htmlparser2: 10.1.0
|
||||||
uhyphen: 0.2.0
|
uhyphen: 0.2.0
|
||||||
|
|
||||||
|
linkify-it@5.0.0:
|
||||||
|
dependencies:
|
||||||
|
uc.micro: 2.1.0
|
||||||
|
|
||||||
listr2@10.2.1:
|
listr2@10.2.1:
|
||||||
dependencies:
|
dependencies:
|
||||||
cli-truncate: 5.2.0
|
cli-truncate: 5.2.0
|
||||||
@@ -4710,6 +4760,15 @@ snapshots:
|
|||||||
|
|
||||||
many-keys-map@2.0.1: {}
|
many-keys-map@2.0.1: {}
|
||||||
|
|
||||||
|
markdown-it@14.1.1:
|
||||||
|
dependencies:
|
||||||
|
argparse: 2.0.1
|
||||||
|
entities: 4.5.0
|
||||||
|
linkify-it: 5.0.0
|
||||||
|
mdurl: 2.0.0
|
||||||
|
punycode.js: 2.3.1
|
||||||
|
uc.micro: 2.1.0
|
||||||
|
|
||||||
markdown-table@3.0.4: {}
|
markdown-table@3.0.4: {}
|
||||||
|
|
||||||
marked@17.0.5: {}
|
marked@17.0.5: {}
|
||||||
@@ -4838,6 +4897,8 @@ snapshots:
|
|||||||
dependencies:
|
dependencies:
|
||||||
'@types/mdast': 4.0.4
|
'@types/mdast': 4.0.4
|
||||||
|
|
||||||
|
mdurl@2.0.0: {}
|
||||||
|
|
||||||
micromark-core-commonmark@2.0.3:
|
micromark-core-commonmark@2.0.3:
|
||||||
dependencies:
|
dependencies:
|
||||||
decode-named-character-reference: 1.3.0
|
decode-named-character-reference: 1.3.0
|
||||||
@@ -5348,6 +5409,8 @@ snapshots:
|
|||||||
ofetch: 1.5.1
|
ofetch: 1.5.1
|
||||||
zod: 4.3.6
|
zod: 4.3.6
|
||||||
|
|
||||||
|
punycode.js@2.3.1: {}
|
||||||
|
|
||||||
pupa@3.3.0:
|
pupa@3.3.0:
|
||||||
dependencies:
|
dependencies:
|
||||||
escape-goat: 4.0.0
|
escape-goat: 4.0.0
|
||||||
@@ -5634,6 +5697,8 @@ snapshots:
|
|||||||
|
|
||||||
typescript@5.9.3: {}
|
typescript@5.9.3: {}
|
||||||
|
|
||||||
|
uc.micro@2.1.0: {}
|
||||||
|
|
||||||
ufo@1.6.3: {}
|
ufo@1.6.3: {}
|
||||||
|
|
||||||
uhyphen@0.2.0: {}
|
uhyphen@0.2.0: {}
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ llm:
|
|||||||
provider: ark
|
provider: ark
|
||||||
base_url: https://ark.cn-beijing.volces.com/api/v3
|
base_url: https://ark.cn-beijing.volces.com/api/v3
|
||||||
model: doubao-seed-2-0-lite-260215
|
model: doubao-seed-2-0-lite-260215
|
||||||
|
knowledge_url_model: doubao-seed-2-0-mini-260215
|
||||||
timeout: 2m
|
timeout: 2m
|
||||||
max_output_tokens: 4000
|
max_output_tokens: 4000
|
||||||
temperature: 0.7
|
temperature: 0.7
|
||||||
|
|||||||
@@ -49,6 +49,7 @@ llm:
|
|||||||
base_url: https://ark.cn-beijing.volces.com/api/v3
|
base_url: https://ark.cn-beijing.volces.com/api/v3
|
||||||
api_key: "7fb6c66b-129c-4935-9617-709236c4205a"
|
api_key: "7fb6c66b-129c-4935-9617-709236c4205a"
|
||||||
model: doubao-seed-2-0-lite-260215
|
model: doubao-seed-2-0-lite-260215
|
||||||
|
knowledge_url_model: doubao-seed-2-0-mini-260215
|
||||||
timeout: 2m
|
timeout: 2m
|
||||||
max_output_tokens: 16000
|
max_output_tokens: 16000
|
||||||
temperature: 0.7
|
temperature: 0.7
|
||||||
|
|||||||
@@ -28,6 +28,8 @@ require (
|
|||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||||
|
github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7 // indirect
|
||||||
|
github.com/extrame/xls v0.0.1 // indirect
|
||||||
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
||||||
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
|
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
|
||||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||||
|
|||||||
@@ -27,6 +27,10 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
|
|||||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||||
|
github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7 h1:n+nk0bNe2+gVbRI8WRbLFVwwcBQ0rr5p+gzkKb6ol8c=
|
||||||
|
github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7/go.mod h1:GPpMrAfHdb8IdQ1/R2uIRBsNfnPnwsYE9YYI5WyY1zw=
|
||||||
|
github.com/extrame/xls v0.0.1 h1:jI7L/o3z73TyyENPopsLS/Jlekm3nF1a/kF5hKBvy/k=
|
||||||
|
github.com/extrame/xls v0.0.1/go.mod h1:iACcgahst7BboCpIMSpnFs4SKyU9ZjsvZBfNbUxZOJI=
|
||||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||||
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
|
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
|
||||||
|
|||||||
@@ -79,16 +79,17 @@ type LogConfig struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type LLMConfig struct {
|
type LLMConfig struct {
|
||||||
Provider string `mapstructure:"provider"`
|
Provider string `mapstructure:"provider"`
|
||||||
APIKey string `mapstructure:"api_key"`
|
APIKey string `mapstructure:"api_key"`
|
||||||
BaseURL string `mapstructure:"base_url"`
|
BaseURL string `mapstructure:"base_url"`
|
||||||
Model string `mapstructure:"model"`
|
Model string `mapstructure:"model"`
|
||||||
Timeout time.Duration `mapstructure:"timeout"`
|
KnowledgeURLModel string `mapstructure:"knowledge_url_model"`
|
||||||
MaxOutputTokens int64 `mapstructure:"max_output_tokens"`
|
Timeout time.Duration `mapstructure:"timeout"`
|
||||||
Temperature float64 `mapstructure:"temperature"`
|
MaxOutputTokens int64 `mapstructure:"max_output_tokens"`
|
||||||
TopP float64 `mapstructure:"top_p"`
|
Temperature float64 `mapstructure:"temperature"`
|
||||||
ReasoningEffort string `mapstructure:"reasoning_effort"`
|
TopP float64 `mapstructure:"top_p"`
|
||||||
WebSearchLimit int32 `mapstructure:"web_search_limit"`
|
ReasoningEffort string `mapstructure:"reasoning_effort"`
|
||||||
|
WebSearchLimit int32 `mapstructure:"web_search_limit"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type RetrievalConfig struct {
|
type RetrievalConfig struct {
|
||||||
@@ -150,6 +151,9 @@ func applyEnvOverrides(cfg *Config) {
|
|||||||
if apiKey, ok := lookupNonEmptyEnv("LLM_API_KEY"); ok {
|
if apiKey, ok := lookupNonEmptyEnv("LLM_API_KEY"); ok {
|
||||||
cfg.LLM.APIKey = apiKey
|
cfg.LLM.APIKey = apiKey
|
||||||
}
|
}
|
||||||
|
if model, ok := lookupNonEmptyEnv("LLM_KNOWLEDGE_URL_MODEL"); ok {
|
||||||
|
cfg.LLM.KnowledgeURLModel = model
|
||||||
|
}
|
||||||
if apiKey, ok := lookupNonEmptyEnv("QDRANT_API_KEY"); ok {
|
if apiKey, ok := lookupNonEmptyEnv("QDRANT_API_KEY"); ok {
|
||||||
cfg.Qdrant.APIKey = apiKey
|
cfg.Qdrant.APIKey = apiKey
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -113,6 +113,11 @@ func (c *arkClient) Generate(ctx context.Context, req GenerateRequest, onDelta f
|
|||||||
maxOutputTokens = req.MaxOutputTokens
|
maxOutputTokens = req.MaxOutputTokens
|
||||||
}
|
}
|
||||||
|
|
||||||
|
model := c.model
|
||||||
|
if strings.TrimSpace(req.Model) != "" {
|
||||||
|
model = strings.TrimSpace(req.Model)
|
||||||
|
}
|
||||||
|
|
||||||
callCtx, cancel := context.WithTimeout(ctx, timeout)
|
callCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
@@ -154,7 +159,7 @@ func (c *arkClient) Generate(ctx context.Context, req GenerateRequest, onDelta f
|
|||||||
}
|
}
|
||||||
|
|
||||||
stream, err := c.client.CreateResponsesStream(callCtx, &responses.ResponsesRequest{
|
stream, err := c.client.CreateResponsesStream(callCtx, &responses.ResponsesRequest{
|
||||||
Model: c.model,
|
Model: model,
|
||||||
MaxOutputTokens: &maxOutputTokens,
|
MaxOutputTokens: &maxOutputTokens,
|
||||||
Store: &store,
|
Store: &store,
|
||||||
Stream: &streamValue,
|
Stream: &streamValue,
|
||||||
@@ -232,7 +237,7 @@ func (c *arkClient) Generate(ctx context.Context, req GenerateRequest, onDelta f
|
|||||||
|
|
||||||
return &GenerateResult{
|
return &GenerateResult{
|
||||||
Content: result,
|
Content: result,
|
||||||
Model: c.model,
|
Model: model,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
var ErrNotConfigured = errors.New("llm provider is not configured")
|
var ErrNotConfigured = errors.New("llm provider is not configured")
|
||||||
|
|
||||||
type GenerateRequest struct {
|
type GenerateRequest struct {
|
||||||
|
Model string
|
||||||
Prompt string
|
Prompt string
|
||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
MaxOutputTokens int64
|
MaxOutputTokens int64
|
||||||
|
|||||||
@@ -0,0 +1,122 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
func splitKnowledgeTextIntoChunks(text string, chunkSize, chunkOverlap int) []string {
|
||||||
|
text = strings.TrimSpace(text)
|
||||||
|
if text == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if chunkSize <= 0 {
|
||||||
|
chunkSize = 900
|
||||||
|
}
|
||||||
|
if chunkOverlap < 0 {
|
||||||
|
chunkOverlap = 0
|
||||||
|
}
|
||||||
|
if chunkOverlap >= chunkSize {
|
||||||
|
chunkOverlap = chunkSize / 4
|
||||||
|
}
|
||||||
|
|
||||||
|
lines := strings.Split(text, "\n")
|
||||||
|
chunks := make([]string, 0, len(lines))
|
||||||
|
current := make([]string, 0, 16)
|
||||||
|
currentLen := 0
|
||||||
|
|
||||||
|
resetWithOverlap := func(chunk string) {
|
||||||
|
current = current[:0]
|
||||||
|
currentLen = 0
|
||||||
|
if chunkOverlap <= 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
overlap := takeLastKnowledgeRunes(chunk, chunkOverlap)
|
||||||
|
if overlap == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
current = append(current, overlap)
|
||||||
|
currentLen = len([]rune(overlap))
|
||||||
|
}
|
||||||
|
|
||||||
|
flush := func() {
|
||||||
|
if len(current) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
chunk := strings.TrimSpace(strings.Join(current, "\n"))
|
||||||
|
if chunk == "" {
|
||||||
|
current = current[:0]
|
||||||
|
currentLen = 0
|
||||||
|
return
|
||||||
|
}
|
||||||
|
chunks = append(chunks, chunk)
|
||||||
|
resetWithOverlap(chunk)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, line := range lines {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
lineRunes := len([]rune(line))
|
||||||
|
if lineRunes > chunkSize {
|
||||||
|
flush()
|
||||||
|
for _, segment := range splitKnowledgeLongLine(line, chunkSize) {
|
||||||
|
if strings.TrimSpace(segment) == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
chunks = append(chunks, segment)
|
||||||
|
resetWithOverlap(segment)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
nextLen := currentLen + lineRunes
|
||||||
|
if len(current) > 0 {
|
||||||
|
nextLen++
|
||||||
|
}
|
||||||
|
if nextLen > chunkSize {
|
||||||
|
flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
current = append(current, line)
|
||||||
|
currentLen += lineRunes
|
||||||
|
if len(current) > 1 {
|
||||||
|
currentLen++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(current) > 0 {
|
||||||
|
chunk := strings.TrimSpace(strings.Join(current, "\n"))
|
||||||
|
if chunk != "" {
|
||||||
|
chunks = append(chunks, chunk)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return chunks
|
||||||
|
}
|
||||||
|
|
||||||
|
func takeLastKnowledgeRunes(text string, size int) string {
|
||||||
|
if size <= 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
runes := []rune(strings.TrimSpace(text))
|
||||||
|
if len(runes) <= size {
|
||||||
|
return string(runes)
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(string(runes[len(runes)-size:]))
|
||||||
|
}
|
||||||
|
|
||||||
|
func estimateKnowledgeTokenCount(text string) int {
|
||||||
|
text = strings.TrimSpace(text)
|
||||||
|
if text == "" {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
runeCount := len([]rune(text))
|
||||||
|
tokenCount := runeCount / 2
|
||||||
|
if tokenCount == 0 {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return tokenCount
|
||||||
|
}
|
||||||
@@ -3,76 +3,113 @@ package app
|
|||||||
import (
|
import (
|
||||||
"archive/zip"
|
"archive/zip"
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"github.com/extrame/xls"
|
||||||
pdf "github.com/ledongthuc/pdf"
|
pdf "github.com/ledongthuc/pdf"
|
||||||
"github.com/xuri/excelize/v2"
|
"github.com/xuri/excelize/v2"
|
||||||
"golang.org/x/net/html"
|
"golang.org/x/net/html"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
var (
|
||||||
maxKnowledgeFetchBytes = 5 << 20
|
whitespacePattern = regexp.MustCompile(`[ \t]+`)
|
||||||
|
genericKnowledgeMarkdownTitleRegex = regexp.MustCompile(`^第\s*\d+\s*部分$`)
|
||||||
)
|
)
|
||||||
|
|
||||||
var whitespacePattern = regexp.MustCompile(`[ \t]+`)
|
const maxKnowledgeSpreadsheetRows = 100000
|
||||||
|
|
||||||
func extractKnowledgeTextFromURL(ctx context.Context, rawURL string) (string, error) {
|
type knowledgeParsedContent struct {
|
||||||
data, contentType, err := fetchKnowledgeURLSnapshot(ctx, rawURL)
|
Text string
|
||||||
if err != nil {
|
Markdown string
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return extractKnowledgeTextFromSnapshot(resolveSnapshotFileName(rawURL, contentType), data)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func fetchKnowledgeURLSnapshot(ctx context.Context, rawURL string) ([]byte, string, error) {
|
func parseKnowledgeTextInput(name, content string) *knowledgeParsedContent {
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimSpace(rawURL), nil)
|
text := normalizeKnowledgeText(content)
|
||||||
if err != nil {
|
return &knowledgeParsedContent{
|
||||||
return nil, "", fmt.Errorf("create request: %w", err)
|
Text: text,
|
||||||
|
Markdown: buildKnowledgeMarkdownFromText(name, text),
|
||||||
}
|
}
|
||||||
req.Header.Set("User-Agent", "geo-knowledge-fetcher/1.0")
|
|
||||||
|
|
||||||
client := &http.Client{Timeout: 20 * time.Second}
|
|
||||||
resp, err := client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", fmt.Errorf("fetch url: %w", err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode >= http.StatusBadRequest {
|
|
||||||
return nil, "", fmt.Errorf("fetch url failed: status=%d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := io.ReadAll(io.LimitReader(resp.Body, maxKnowledgeFetchBytes))
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", fmt.Errorf("read url body: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
contentType := strings.ToLower(resp.Header.Get("Content-Type"))
|
|
||||||
return data, contentType, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveSnapshotFileName(rawURL string, contentType string) string {
|
func extractKnowledgeContentFromFile(fileName string, content []byte) (*knowledgeParsedContent, error) {
|
||||||
if strings.Contains(strings.ToLower(contentType), "text/plain") {
|
|
||||||
return rawURL + ".txt"
|
|
||||||
}
|
|
||||||
return rawURL + ".html"
|
|
||||||
}
|
|
||||||
|
|
||||||
func extractKnowledgeTextFromSnapshot(fileName string, content []byte) (string, error) {
|
|
||||||
ext := strings.ToLower(filepath.Ext(fileName))
|
ext := strings.ToLower(filepath.Ext(fileName))
|
||||||
if ext != ".html" && ext != ".htm" {
|
|
||||||
return extractKnowledgeTextFromFile(fileName, content)
|
switch ext {
|
||||||
|
case ".txt", ".json", ".csv":
|
||||||
|
return extractUTF8KnowledgeText(fileName, content, false)
|
||||||
|
case ".md", ".markdown":
|
||||||
|
return extractUTF8KnowledgeText(fileName, content, true)
|
||||||
|
case ".html", ".htm":
|
||||||
|
text, err := extractHTMLText(content)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &knowledgeParsedContent{
|
||||||
|
Text: text,
|
||||||
|
Markdown: buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text),
|
||||||
|
}, nil
|
||||||
|
case ".docx":
|
||||||
|
text, err := extractDOCXText(content)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &knowledgeParsedContent{
|
||||||
|
Text: text,
|
||||||
|
Markdown: buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text),
|
||||||
|
}, nil
|
||||||
|
case ".xlsx":
|
||||||
|
return extractXLSXContent(content)
|
||||||
|
case ".xls":
|
||||||
|
return extractXLSContent(content)
|
||||||
|
case ".pdf":
|
||||||
|
text, err := extractPDFText(content)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &knowledgeParsedContent{
|
||||||
|
Text: text,
|
||||||
|
Markdown: buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text),
|
||||||
|
}, nil
|
||||||
|
default:
|
||||||
|
if utf8.Valid(content) {
|
||||||
|
return extractUTF8KnowledgeText(fileName, content, false)
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("unsupported file type %s", ext)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractUTF8KnowledgeText(fileName string, content []byte, preserveMarkdown bool) (*knowledgeParsedContent, error) {
|
||||||
|
if !utf8.Valid(content) {
|
||||||
|
return nil, fmt.Errorf("file %s is not valid UTF-8 text", fileName)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
raw := string(content)
|
||||||
|
text := normalizeKnowledgeText(raw)
|
||||||
|
if text == "" {
|
||||||
|
return nil, fmt.Errorf("file %s contains no readable text", fileName)
|
||||||
|
}
|
||||||
|
|
||||||
|
markdown := buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text)
|
||||||
|
if preserveMarkdown {
|
||||||
|
normalizedMarkdown := normalizeKnowledgeMarkdown(raw)
|
||||||
|
if normalizedMarkdown != "" {
|
||||||
|
markdown = normalizedMarkdown
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return &knowledgeParsedContent{
|
||||||
|
Text: text,
|
||||||
|
Markdown: markdown,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractHTMLText(content []byte) (string, error) {
|
||||||
doc, err := html.Parse(bytes.NewReader(content))
|
doc, err := html.Parse(bytes.NewReader(content))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("parse html: %w", err)
|
return "", fmt.Errorf("parse html: %w", err)
|
||||||
@@ -106,32 +143,11 @@ func extractKnowledgeTextFromSnapshot(fileName string, content []byte) (string,
|
|||||||
}
|
}
|
||||||
walk(doc, false)
|
walk(doc, false)
|
||||||
|
|
||||||
return normalizeKnowledgeText(strings.Join(parts, "\n")), nil
|
result := normalizeKnowledgeText(strings.Join(parts, "\n"))
|
||||||
}
|
if result == "" {
|
||||||
|
return "", fmt.Errorf("html contains no readable text")
|
||||||
func extractKnowledgeTextFromFile(fileName string, content []byte) (string, error) {
|
|
||||||
ext := strings.ToLower(filepath.Ext(fileName))
|
|
||||||
|
|
||||||
switch ext {
|
|
||||||
case ".txt", ".md", ".markdown", ".json", ".csv", ".html", ".htm":
|
|
||||||
if !utf8.Valid(content) {
|
|
||||||
return "", fmt.Errorf("file %s is not valid UTF-8 text", fileName)
|
|
||||||
}
|
|
||||||
return normalizeKnowledgeText(string(content)), nil
|
|
||||||
case ".docx":
|
|
||||||
return extractDOCXText(content)
|
|
||||||
case ".xlsx":
|
|
||||||
return extractXLSXText(content)
|
|
||||||
case ".pdf":
|
|
||||||
return extractPDFText(content)
|
|
||||||
case ".doc", ".xls":
|
|
||||||
return "", fmt.Errorf("file type %s is not supported yet", ext)
|
|
||||||
default:
|
|
||||||
if utf8.Valid(content) {
|
|
||||||
return normalizeKnowledgeText(string(content)), nil
|
|
||||||
}
|
|
||||||
return "", fmt.Errorf("unsupported file type %s", ext)
|
|
||||||
}
|
}
|
||||||
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func extractDOCXText(content []byte) (string, error) {
|
func extractDOCXText(content []byte) (string, error) {
|
||||||
@@ -173,46 +189,177 @@ func extractDOCXText(content []byte) (string, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(parts) == 0 {
|
if len(parts) == 0 {
|
||||||
return "", fmt.Errorf("docx contains no readable text")
|
return "", fmt.Errorf("docx contains no readable text")
|
||||||
}
|
}
|
||||||
return normalizeKnowledgeText(strings.Join(parts, "\n")), nil
|
return normalizeKnowledgeText(strings.Join(parts, "\n")), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func extractXLSXText(content []byte) (string, error) {
|
func extractXLSXContent(content []byte) (*knowledgeParsedContent, error) {
|
||||||
book, err := excelize.OpenReader(bytes.NewReader(content))
|
book, err := excelize.OpenReader(bytes.NewReader(content))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("open xlsx: %w", err)
|
return nil, fmt.Errorf("open xlsx: %w", err)
|
||||||
}
|
}
|
||||||
defer func() { _ = book.Close() }()
|
defer func() { _ = book.Close() }()
|
||||||
|
|
||||||
lines := make([]string, 0)
|
markdownSections := make([]string, 0)
|
||||||
|
textSections := make([]string, 0)
|
||||||
for _, sheet := range book.GetSheetList() {
|
for _, sheet := range book.GetSheetList() {
|
||||||
rows, err := book.GetRows(sheet)
|
rows, err := book.GetRows(sheet)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("read xlsx rows: %w", err)
|
return nil, fmt.Errorf("read xlsx rows: %w", err)
|
||||||
}
|
}
|
||||||
if len(rows) == 0 {
|
|
||||||
|
cleaned := normalizeSpreadsheetRows(rows)
|
||||||
|
if len(cleaned) == 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
lines = append(lines, sheet)
|
|
||||||
for _, row := range rows {
|
markdownSections = append(markdownSections, buildKnowledgeSheetMarkdown(sheet, cleaned))
|
||||||
cells := make([]string, 0, len(row))
|
textSections = append(textSections, buildKnowledgeSheetPlainText(sheet, cleaned))
|
||||||
for _, cell := range row {
|
}
|
||||||
cell = strings.TrimSpace(cell)
|
|
||||||
if cell != "" {
|
if len(markdownSections) == 0 {
|
||||||
cells = append(cells, cell)
|
return nil, fmt.Errorf("xlsx contains no readable text")
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if len(cells) > 0 {
|
return &knowledgeParsedContent{
|
||||||
lines = append(lines, strings.Join(cells, " | "))
|
Text: normalizeKnowledgeText(strings.Join(textSections, "\n\n")),
|
||||||
}
|
Markdown: strings.TrimSpace(strings.Join(markdownSections, "\n\n")),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractXLSContent(content []byte) (*knowledgeParsedContent, error) {
|
||||||
|
book, err := xls.OpenReader(bytes.NewReader(content), "utf-8")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("open xls: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The upstream xls library panics for sparse rows when calling sheet.Row(i),
|
||||||
|
// so we rely on its safer bulk reader here instead of random row access.
|
||||||
|
rows := book.ReadAllCells(maxKnowledgeSpreadsheetRows)
|
||||||
|
cleaned := normalizeSpreadsheetRows(rows)
|
||||||
|
if len(cleaned) == 0 {
|
||||||
|
return nil, fmt.Errorf("xls contains no readable text")
|
||||||
|
}
|
||||||
|
|
||||||
|
title := "XLS 工作簿"
|
||||||
|
if book.NumSheets() == 1 {
|
||||||
|
if sheet := book.GetSheet(0); sheet != nil && strings.TrimSpace(sheet.Name) != "" {
|
||||||
|
title = strings.TrimSpace(sheet.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(lines) == 0 {
|
|
||||||
return "", fmt.Errorf("xlsx contains no readable text")
|
return &knowledgeParsedContent{
|
||||||
|
Text: normalizeKnowledgeText(buildKnowledgeSheetPlainText(title, cleaned)),
|
||||||
|
Markdown: buildKnowledgeSheetMarkdown(title, cleaned),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeSpreadsheetRows(rows [][]string) [][]string {
|
||||||
|
cleaned := make([][]string, 0, len(rows))
|
||||||
|
for _, row := range rows {
|
||||||
|
cells := make([]string, len(row))
|
||||||
|
copy(cells, row)
|
||||||
|
|
||||||
|
for index := range cells {
|
||||||
|
cells[index] = normalizeKnowledgeInlineText(cells[index])
|
||||||
|
}
|
||||||
|
|
||||||
|
lastNonEmpty := -1
|
||||||
|
for index, cell := range cells {
|
||||||
|
if cell != "" {
|
||||||
|
lastNonEmpty = index
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if lastNonEmpty < 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
cleaned = append(cleaned, cells[:lastNonEmpty+1])
|
||||||
}
|
}
|
||||||
return normalizeKnowledgeText(strings.Join(lines, "\n")), nil
|
return cleaned
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildKnowledgeSheetPlainText(sheetName string, rows [][]string) string {
|
||||||
|
lines := make([]string, 0, len(rows)+1)
|
||||||
|
if strings.TrimSpace(sheetName) != "" {
|
||||||
|
lines = append(lines, strings.TrimSpace(sheetName))
|
||||||
|
}
|
||||||
|
for _, row := range rows {
|
||||||
|
lines = append(lines, strings.Join(row, " | "))
|
||||||
|
}
|
||||||
|
return strings.Join(lines, "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildKnowledgeSheetMarkdown(sheetName string, rows [][]string) string {
|
||||||
|
if len(rows) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
width := 0
|
||||||
|
for _, row := range rows {
|
||||||
|
if len(row) > width {
|
||||||
|
width = len(row)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if width == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
normalizedRows := make([][]string, 0, len(rows))
|
||||||
|
for _, row := range rows {
|
||||||
|
normalizedRows = append(normalizedRows, padKnowledgeRow(row, width))
|
||||||
|
}
|
||||||
|
|
||||||
|
var builder strings.Builder
|
||||||
|
if title := strings.TrimSpace(sheetName); title != "" {
|
||||||
|
builder.WriteString("## ")
|
||||||
|
builder.WriteString(title)
|
||||||
|
builder.WriteString("\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
writeKnowledgeMarkdownTableRow(&builder, normalizedRows[0])
|
||||||
|
writeKnowledgeMarkdownTableSeparator(&builder, width)
|
||||||
|
for _, row := range normalizedRows[1:] {
|
||||||
|
writeKnowledgeMarkdownTableRow(&builder, row)
|
||||||
|
}
|
||||||
|
|
||||||
|
return strings.TrimSpace(builder.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
func padKnowledgeRow(row []string, width int) []string {
|
||||||
|
result := make([]string, width)
|
||||||
|
copy(result, row)
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeKnowledgeMarkdownTableRow(builder *strings.Builder, row []string) {
|
||||||
|
builder.WriteString("|")
|
||||||
|
for _, cell := range row {
|
||||||
|
builder.WriteString(" ")
|
||||||
|
builder.WriteString(escapeKnowledgeMarkdownTableCell(cell))
|
||||||
|
builder.WriteString(" |")
|
||||||
|
}
|
||||||
|
builder.WriteString("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeKnowledgeMarkdownTableSeparator(builder *strings.Builder, width int) {
|
||||||
|
builder.WriteString("|")
|
||||||
|
for index := 0; index < width; index++ {
|
||||||
|
builder.WriteString(" --- |")
|
||||||
|
}
|
||||||
|
builder.WriteString("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
func escapeKnowledgeMarkdownTableCell(cell string) string {
|
||||||
|
cell = strings.TrimSpace(cell)
|
||||||
|
cell = strings.ReplaceAll(cell, "\\", "\\\\")
|
||||||
|
cell = strings.ReplaceAll(cell, "|", "\\|")
|
||||||
|
cell = strings.ReplaceAll(cell, "\r", " ")
|
||||||
|
cell = strings.ReplaceAll(cell, "\n", " / ")
|
||||||
|
return cell
|
||||||
}
|
}
|
||||||
|
|
||||||
func extractPDFText(content []byte) (string, error) {
|
func extractPDFText(content []byte) (string, error) {
|
||||||
@@ -223,7 +370,7 @@ func extractPDFText(content []byte) (string, error) {
|
|||||||
|
|
||||||
var builder strings.Builder
|
var builder strings.Builder
|
||||||
totalPage := reader.NumPage()
|
totalPage := reader.NumPage()
|
||||||
for pageIndex := 1; pageIndex <= totalPage; pageIndex += 1 {
|
for pageIndex := 1; pageIndex <= totalPage; pageIndex++ {
|
||||||
page := reader.Page(pageIndex)
|
page := reader.Page(pageIndex)
|
||||||
if page.V.IsNull() {
|
if page.V.IsNull() {
|
||||||
continue
|
continue
|
||||||
@@ -245,12 +392,116 @@ func extractPDFText(content []byte) (string, error) {
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeKnowledgeText(input string) string {
|
func defaultKnowledgeTitle(fileName string) string {
|
||||||
|
base := filepath.Base(strings.TrimSpace(fileName))
|
||||||
|
if base == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimSuffix(base, filepath.Ext(base))
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildKnowledgeMarkdownFromText(title, text string) string {
|
||||||
|
text = normalizeKnowledgeText(text)
|
||||||
|
title = strings.TrimSpace(title)
|
||||||
|
if text == "" {
|
||||||
|
if title == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return "# " + title
|
||||||
|
}
|
||||||
|
|
||||||
|
lines := strings.Split(text, "\n")
|
||||||
|
if title != "" && len(lines) > 0 && strings.EqualFold(strings.TrimSpace(lines[0]), title) {
|
||||||
|
lines = lines[1:]
|
||||||
|
}
|
||||||
|
|
||||||
|
paragraphs := make([]string, 0, len(lines))
|
||||||
|
for _, line := range lines {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line != "" {
|
||||||
|
paragraphs = append(paragraphs, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sections := make([]string, 0, 2)
|
||||||
|
if title != "" {
|
||||||
|
sections = append(sections, "# "+title)
|
||||||
|
}
|
||||||
|
if len(paragraphs) > 0 {
|
||||||
|
sections = append(sections, strings.Join(paragraphs, "\n\n"))
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(strings.Join(sections, "\n\n"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractKnowledgeMarkdownTitle(markdown string) string {
|
||||||
|
markdown = normalizeKnowledgeMarkdown(markdown)
|
||||||
|
if markdown == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, line := range strings.Split(markdown, "\n") {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" || !strings.HasPrefix(line, "#") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
level := 0
|
||||||
|
for level < len(line) && line[level] == '#' {
|
||||||
|
level++
|
||||||
|
}
|
||||||
|
if level == len(line) || line[level] != ' ' {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
title := strings.Join(strings.Fields(strings.TrimSpace(line[level+1:])), " ")
|
||||||
|
title = strings.Trim(title, "# \t")
|
||||||
|
if title == "" || genericKnowledgeMarkdownTitleRegex.MatchString(title) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch title {
|
||||||
|
case "摘要", "概述", "概要", "目录", "正文":
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
runes := []rune(title)
|
||||||
|
if len(runes) > 200 {
|
||||||
|
title = string(runes[:200])
|
||||||
|
}
|
||||||
|
return title
|
||||||
|
}
|
||||||
|
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeKnowledgeMarkdown(input string) string {
|
||||||
|
input = strings.ReplaceAll(input, "\r\n", "\n")
|
||||||
input = strings.ReplaceAll(input, "\u00a0", " ")
|
input = strings.ReplaceAll(input, "\u00a0", " ")
|
||||||
|
|
||||||
lines := strings.Split(input, "\n")
|
lines := strings.Split(input, "\n")
|
||||||
cleaned := make([]string, 0, len(lines))
|
cleaned := make([]string, 0, len(lines))
|
||||||
|
lastBlank := false
|
||||||
for _, line := range lines {
|
for _, line := range lines {
|
||||||
line = whitespacePattern.ReplaceAllString(strings.TrimSpace(line), " ")
|
line = strings.TrimRight(line, " \t")
|
||||||
|
if strings.TrimSpace(line) == "" {
|
||||||
|
if lastBlank {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
lastBlank = true
|
||||||
|
cleaned = append(cleaned, "")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
lastBlank = false
|
||||||
|
cleaned = append(cleaned, line)
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(strings.Join(cleaned, "\n"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeKnowledgeText(input string) string {
|
||||||
|
input = strings.ReplaceAll(input, "\u00a0", " ")
|
||||||
|
lines := strings.Split(strings.ReplaceAll(input, "\r\n", "\n"), "\n")
|
||||||
|
cleaned := make([]string, 0, len(lines))
|
||||||
|
for _, line := range lines {
|
||||||
|
line = normalizeKnowledgeInlineText(line)
|
||||||
if line != "" {
|
if line != "" {
|
||||||
cleaned = append(cleaned, line)
|
cleaned = append(cleaned, line)
|
||||||
}
|
}
|
||||||
@@ -258,58 +509,6 @@ func normalizeKnowledgeText(input string) string {
|
|||||||
return strings.TrimSpace(strings.Join(cleaned, "\n"))
|
return strings.TrimSpace(strings.Join(cleaned, "\n"))
|
||||||
}
|
}
|
||||||
|
|
||||||
func splitKnowledgeTextIntoChunks(text string, chunkSize, chunkOverlap int) []string {
|
func normalizeKnowledgeInlineText(input string) string {
|
||||||
text = normalizeKnowledgeText(text)
|
return whitespacePattern.ReplaceAllString(strings.TrimSpace(input), " ")
|
||||||
if text == "" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
runes := []rune(text)
|
|
||||||
if chunkSize <= 0 {
|
|
||||||
chunkSize = 900
|
|
||||||
}
|
|
||||||
if chunkOverlap < 0 {
|
|
||||||
chunkOverlap = 0
|
|
||||||
}
|
|
||||||
if chunkOverlap >= chunkSize {
|
|
||||||
chunkOverlap = chunkSize / 4
|
|
||||||
}
|
|
||||||
|
|
||||||
chunks := make([]string, 0)
|
|
||||||
for start := 0; start < len(runes); {
|
|
||||||
end := start + chunkSize
|
|
||||||
if end > len(runes) {
|
|
||||||
end = len(runes)
|
|
||||||
}
|
|
||||||
|
|
||||||
if end < len(runes) {
|
|
||||||
for offset := end; offset > start+chunkSize/2; offset -= 1 {
|
|
||||||
if runes[offset-1] == '\n' || runes[offset-1] == '。' || runes[offset-1] == '.' {
|
|
||||||
end = offset
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
chunk := strings.TrimSpace(string(runes[start:end]))
|
|
||||||
if chunk != "" {
|
|
||||||
chunks = append(chunks, chunk)
|
|
||||||
}
|
|
||||||
if end >= len(runes) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
start = end - chunkOverlap
|
|
||||||
if start < 0 {
|
|
||||||
start = 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return chunks
|
|
||||||
}
|
|
||||||
|
|
||||||
func estimateKnowledgeTokenCount(text string) int {
|
|
||||||
if text == "" {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
return utf8.RuneCountInString(text)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,8 +4,10 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime/debug"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -18,6 +20,7 @@ import (
|
|||||||
|
|
||||||
"github.com/geo-platform/tenant-api/internal/shared/auth"
|
"github.com/geo-platform/tenant-api/internal/shared/auth"
|
||||||
"github.com/geo-platform/tenant-api/internal/shared/config"
|
"github.com/geo-platform/tenant-api/internal/shared/config"
|
||||||
|
"github.com/geo-platform/tenant-api/internal/shared/llm"
|
||||||
"github.com/geo-platform/tenant-api/internal/shared/objectstorage"
|
"github.com/geo-platform/tenant-api/internal/shared/objectstorage"
|
||||||
"github.com/geo-platform/tenant-api/internal/shared/response"
|
"github.com/geo-platform/tenant-api/internal/shared/response"
|
||||||
"github.com/geo-platform/tenant-api/internal/shared/retrieval"
|
"github.com/geo-platform/tenant-api/internal/shared/retrieval"
|
||||||
@@ -25,8 +28,9 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
defaultKnowledgeQueueSize = 128
|
defaultKnowledgeQueueSize = 128
|
||||||
defaultKnowledgeCleanupPoll = 15 * time.Second
|
defaultKnowledgeCleanupPoll = 15 * time.Second
|
||||||
|
defaultKnowledgeChildGroupName = "默认目录"
|
||||||
)
|
)
|
||||||
|
|
||||||
type KnowledgeService struct {
|
type KnowledgeService struct {
|
||||||
@@ -34,12 +38,17 @@ type KnowledgeService struct {
|
|||||||
provider retrieval.Provider
|
provider retrieval.Provider
|
||||||
store retrieval.VectorStore
|
store retrieval.VectorStore
|
||||||
objectStorage objectstorage.Client
|
objectStorage objectstorage.Client
|
||||||
|
llmClient llm.Client
|
||||||
logger *zap.Logger
|
logger *zap.Logger
|
||||||
chunkSize int
|
chunkSize int
|
||||||
chunkOverlap int
|
chunkOverlap int
|
||||||
embeddingBatchSize int
|
embeddingBatchSize int
|
||||||
recallLimit int
|
recallLimit int
|
||||||
rerankTopN int
|
rerankTopN int
|
||||||
|
arkBaseURL string
|
||||||
|
arkAPIKey string
|
||||||
|
urlMarkdownModel string
|
||||||
|
httpClient *http.Client
|
||||||
jobs chan knowledgeParseJob
|
jobs chan knowledgeParseJob
|
||||||
cleanupMu sync.Mutex
|
cleanupMu sync.Mutex
|
||||||
cleanupInFlight map[string]struct{}
|
cleanupInFlight map[string]struct{}
|
||||||
@@ -66,6 +75,7 @@ type knowledgeItemRecord struct {
|
|||||||
SourceURI *string
|
SourceURI *string
|
||||||
StorageKey string
|
StorageKey string
|
||||||
ContentText *string
|
ContentText *string
|
||||||
|
Markdown *string
|
||||||
ItemVersion int
|
ItemVersion int
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -99,6 +109,21 @@ type KnowledgeItemResponse struct {
|
|||||||
UpdatedAt string `json:"updated_at"`
|
UpdatedAt string `json:"updated_at"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type KnowledgeItemDetailResponse struct {
|
||||||
|
ID int64 `json:"id"`
|
||||||
|
GroupID int64 `json:"group_id"`
|
||||||
|
GroupName string `json:"group_name"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
SourceType string `json:"source_type"`
|
||||||
|
SourceURI *string `json:"source_uri"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
SizeBytes int64 `json:"size_bytes"`
|
||||||
|
ErrorMessage *string `json:"error_message"`
|
||||||
|
MarkdownContent *string `json:"markdown_content"`
|
||||||
|
CreatedAt string `json:"created_at"`
|
||||||
|
UpdatedAt string `json:"updated_at"`
|
||||||
|
}
|
||||||
|
|
||||||
type KnowledgeTextItemRequest struct {
|
type KnowledgeTextItemRequest struct {
|
||||||
GroupID int64 `json:"group_id" binding:"required"`
|
GroupID int64 `json:"group_id" binding:"required"`
|
||||||
Name string `json:"name" binding:"required"`
|
Name string `json:"name" binding:"required"`
|
||||||
@@ -132,8 +157,10 @@ func NewKnowledgeService(
|
|||||||
provider retrieval.Provider,
|
provider retrieval.Provider,
|
||||||
store retrieval.VectorStore,
|
store retrieval.VectorStore,
|
||||||
objectStorage objectstorage.Client,
|
objectStorage objectstorage.Client,
|
||||||
|
llmClient llm.Client,
|
||||||
logger *zap.Logger,
|
logger *zap.Logger,
|
||||||
cfg config.RetrievalConfig,
|
cfg config.RetrievalConfig,
|
||||||
|
llmCfg config.LLMConfig,
|
||||||
queueSize int,
|
queueSize int,
|
||||||
workerCount int,
|
workerCount int,
|
||||||
) *KnowledgeService {
|
) *KnowledgeService {
|
||||||
@@ -163,14 +190,25 @@ func NewKnowledgeService(
|
|||||||
provider: provider,
|
provider: provider,
|
||||||
store: store,
|
store: store,
|
||||||
objectStorage: objectStorage,
|
objectStorage: objectStorage,
|
||||||
|
llmClient: llmClient,
|
||||||
logger: logger,
|
logger: logger,
|
||||||
chunkSize: chunkSize,
|
chunkSize: chunkSize,
|
||||||
chunkOverlap: chunkOverlap,
|
chunkOverlap: chunkOverlap,
|
||||||
embeddingBatchSize: batchSize,
|
embeddingBatchSize: batchSize,
|
||||||
recallLimit: recallLimit,
|
recallLimit: recallLimit,
|
||||||
rerankTopN: rerankTopN,
|
rerankTopN: rerankTopN,
|
||||||
|
arkBaseURL: strings.TrimSpace(llmCfg.BaseURL),
|
||||||
|
arkAPIKey: strings.TrimSpace(llmCfg.APIKey),
|
||||||
|
urlMarkdownModel: strings.TrimSpace(llmCfg.KnowledgeURLModel),
|
||||||
|
httpClient: &http.Client{Timeout: defaultKnowledgeURLParseTimeout},
|
||||||
cleanupInFlight: make(map[string]struct{}),
|
cleanupInFlight: make(map[string]struct{}),
|
||||||
}
|
}
|
||||||
|
if svc.urlMarkdownModel == "" {
|
||||||
|
svc.urlMarkdownModel = strings.TrimSpace(llmCfg.Model)
|
||||||
|
}
|
||||||
|
if svc.arkBaseURL == "" {
|
||||||
|
svc.arkBaseURL = defaultKnowledgeArkBaseURL
|
||||||
|
}
|
||||||
|
|
||||||
if workerCount > 0 {
|
if workerCount > 0 {
|
||||||
if queueSize <= 0 {
|
if queueSize <= 0 {
|
||||||
@@ -521,15 +559,58 @@ func (s *KnowledgeService) ListItems(ctx context.Context, groupID *int64) ([]Kno
|
|||||||
return items, nil
|
return items, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *KnowledgeService) GetItemDetail(ctx context.Context, itemID int64) (*KnowledgeItemDetailResponse, error) {
|
||||||
|
actor := auth.MustActor(ctx)
|
||||||
|
|
||||||
|
var (
|
||||||
|
item KnowledgeItemDetailResponse
|
||||||
|
sourceURI *string
|
||||||
|
errorMessage *string
|
||||||
|
markdownContent *string
|
||||||
|
createdAt, updatedAt time.Time
|
||||||
|
)
|
||||||
|
|
||||||
|
err := s.pool.QueryRow(ctx, `
|
||||||
|
SELECT ki.id, ki.group_id, kg.name, ki.name, ki.source_type, ki.source_uri, ki.status,
|
||||||
|
ki.size_bytes, ki.error_message, ki.markdown_content, ki.created_at, ki.updated_at
|
||||||
|
FROM knowledge_items ki
|
||||||
|
JOIN knowledge_groups kg ON kg.id = ki.group_id AND kg.deleted_at IS NULL
|
||||||
|
WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NULL
|
||||||
|
`, itemID, actor.TenantID).Scan(
|
||||||
|
&item.ID,
|
||||||
|
&item.GroupID,
|
||||||
|
&item.GroupName,
|
||||||
|
&item.Name,
|
||||||
|
&item.SourceType,
|
||||||
|
&sourceURI,
|
||||||
|
&item.Status,
|
||||||
|
&item.SizeBytes,
|
||||||
|
&errorMessage,
|
||||||
|
&markdownContent,
|
||||||
|
&createdAt,
|
||||||
|
&updatedAt,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return nil, response.ErrNotFound(40450, "knowledge_item_not_found", "knowledge item not found")
|
||||||
|
}
|
||||||
|
return nil, response.ErrInternal(50051, "knowledge_items_query_failed", "failed to load knowledge item")
|
||||||
|
}
|
||||||
|
|
||||||
|
item.SourceURI = sourceURI
|
||||||
|
item.ErrorMessage = errorMessage
|
||||||
|
item.MarkdownContent = markdownContent
|
||||||
|
item.CreatedAt = createdAt.Format(time.RFC3339)
|
||||||
|
item.UpdatedAt = updatedAt.Format(time.RFC3339)
|
||||||
|
return &item, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *KnowledgeService) CreateTextItem(ctx context.Context, req KnowledgeTextItemRequest) (*KnowledgeItemResponse, error) {
|
func (s *KnowledgeService) CreateTextItem(ctx context.Context, req KnowledgeTextItemRequest) (*KnowledgeItemResponse, error) {
|
||||||
actor := auth.MustActor(ctx)
|
actor := auth.MustActor(ctx)
|
||||||
|
|
||||||
if err := s.validateIngestionDependencies("text"); err != nil {
|
if err := s.validateIngestionDependencies("text"); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := s.ensureStorableGroup(ctx, actor.TenantID, req.GroupID); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
content := normalizeKnowledgeText(req.Content)
|
content := normalizeKnowledgeText(req.Content)
|
||||||
if content == "" {
|
if content == "" {
|
||||||
@@ -551,33 +632,17 @@ func (s *KnowledgeService) CreateURLItem(ctx context.Context, req KnowledgeURLIt
|
|||||||
if err := s.validateIngestionDependencies("website"); err != nil {
|
if err := s.validateIngestionDependencies("website"); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := s.ensureStorableGroup(ctx, actor.TenantID, req.GroupID); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
rawURL := strings.TrimSpace(req.URL)
|
rawURL := strings.TrimSpace(req.URL)
|
||||||
parsed, err := url.ParseRequestURI(rawURL)
|
if _, err := url.ParseRequestURI(rawURL); err != nil {
|
||||||
if err != nil {
|
|
||||||
return nil, response.ErrBadRequest(40052, "knowledge_url_invalid", "knowledge url is invalid")
|
return nil, response.ErrBadRequest(40052, "knowledge_url_invalid", "knowledge url is invalid")
|
||||||
}
|
}
|
||||||
|
|
||||||
snapshot, contentType, err := fetchKnowledgeURLSnapshot(ctx, parsed.String())
|
|
||||||
if err != nil {
|
|
||||||
return nil, response.ErrBadRequest(40052, "knowledge_url_fetch_failed", err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
objectKey := buildKnowledgeObjectKey(actor.TenantID, "website", resolveSnapshotFileName(parsed.String(), contentType))
|
|
||||||
if err := s.objectStorage.PutBytes(ctx, objectKey, snapshot, contentType); err != nil {
|
|
||||||
return nil, response.ErrServiceUnavailable(50357, "knowledge_storage_unavailable", err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
return s.queueKnowledgeItem(ctx, actor.TenantID, knowledgeIngestInput{
|
return s.queueKnowledgeItem(ctx, actor.TenantID, knowledgeIngestInput{
|
||||||
GroupID: req.GroupID,
|
GroupID: req.GroupID,
|
||||||
Name: strings.TrimSpace(req.Name),
|
Name: strings.TrimSpace(req.Name),
|
||||||
SourceType: "website",
|
SourceType: "website",
|
||||||
SourceURI: &rawURL,
|
SourceURI: &rawURL,
|
||||||
StorageKey: objectKey,
|
|
||||||
SizeBytes: int64(len(snapshot)),
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -593,9 +658,6 @@ func (s *KnowledgeService) CreateFileItem(
|
|||||||
if err := s.validateIngestionDependencies("document"); err != nil {
|
if err := s.validateIngestionDependencies("document"); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := s.ensureStorableGroup(ctx, actor.TenantID, groupID); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
displayName := strings.TrimSpace(name)
|
displayName := strings.TrimSpace(name)
|
||||||
if displayName == "" {
|
if displayName == "" {
|
||||||
@@ -819,11 +881,19 @@ func (s *KnowledgeService) validateIngestionDependencies(sourceType string) erro
|
|||||||
if err := s.store.Validate(); err != nil {
|
if err := s.store.Validate(); err != nil {
|
||||||
return response.ErrServiceUnavailable(50352, "qdrant_unavailable", err.Error())
|
return response.ErrServiceUnavailable(50352, "qdrant_unavailable", err.Error())
|
||||||
}
|
}
|
||||||
if sourceType == "document" || sourceType == "website" {
|
if sourceType == "document" {
|
||||||
if err := s.objectStorage.Validate(); err != nil {
|
if err := s.objectStorage.Validate(); err != nil {
|
||||||
return response.ErrServiceUnavailable(50357, "knowledge_storage_unavailable", err.Error())
|
return response.ErrServiceUnavailable(50357, "knowledge_storage_unavailable", err.Error())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if sourceType == "website" {
|
||||||
|
if strings.TrimSpace(s.arkAPIKey) == "" {
|
||||||
|
return response.ErrServiceUnavailable(50358, "knowledge_webpage_parser_unavailable", "ark api key is not configured")
|
||||||
|
}
|
||||||
|
if err := s.llmClient.Validate(); err != nil {
|
||||||
|
return response.ErrServiceUnavailable(50359, "knowledge_markdown_formatter_unavailable", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -832,9 +902,11 @@ func (s *KnowledgeService) queueKnowledgeItem(
|
|||||||
tenantID int64,
|
tenantID int64,
|
||||||
input knowledgeIngestInput,
|
input knowledgeIngestInput,
|
||||||
) (*KnowledgeItemResponse, error) {
|
) (*KnowledgeItemResponse, error) {
|
||||||
if err := s.ensureStorableGroup(ctx, tenantID, input.GroupID); err != nil {
|
resolvedGroupID, err := s.resolveKnowledgeStorageGroupID(ctx, tenantID, input.GroupID)
|
||||||
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
input.GroupID = resolvedGroupID
|
||||||
|
|
||||||
input.Name = strings.TrimSpace(input.Name)
|
input.Name = strings.TrimSpace(input.Name)
|
||||||
if input.Name == "" {
|
if input.Name == "" {
|
||||||
@@ -888,7 +960,25 @@ func (s *KnowledgeService) enqueueParseJob(job knowledgeParseJob) error {
|
|||||||
|
|
||||||
func (s *KnowledgeService) runParseWorker() {
|
func (s *KnowledgeService) runParseWorker() {
|
||||||
for job := range s.jobs {
|
for job := range s.jobs {
|
||||||
s.executeParseJob(context.Background(), job)
|
func(job knowledgeParseJob) {
|
||||||
|
defer func() {
|
||||||
|
if recovered := recover(); recovered != nil {
|
||||||
|
err := fmt.Errorf("knowledge parse panic: %v", recovered)
|
||||||
|
if s.logger != nil {
|
||||||
|
s.logger.Error("knowledge parse worker panicked",
|
||||||
|
zap.Int64("tenant_id", job.TenantID),
|
||||||
|
zap.Int64("knowledge_item_id", job.ItemID),
|
||||||
|
zap.Int64("parse_task_id", job.ParseTaskID),
|
||||||
|
zap.Any("panic", recovered),
|
||||||
|
zap.ByteString("stack", debug.Stack()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
s.executeParseJob(context.Background(), job)
|
||||||
|
}(job)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -936,13 +1026,18 @@ func (s *KnowledgeService) executeParseJob(ctx context.Context, job knowledgePar
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
content, err := s.readKnowledgeItemContent(ctx, item)
|
parsedContent, err := s.readKnowledgeItemParsedContent(ctx, item)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
|
_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if item.SourceType == "website" {
|
||||||
|
if resolvedName := extractKnowledgeMarkdownTitle(parsedContent.Markdown); resolvedName != "" {
|
||||||
|
item.Name = resolvedName
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
chunks := splitKnowledgeTextIntoChunks(content, s.chunkSize, s.chunkOverlap)
|
chunks := splitKnowledgeTextIntoChunks(parsedContent.Text, s.chunkSize, s.chunkOverlap)
|
||||||
if len(chunks) == 0 {
|
if len(chunks) == 0 {
|
||||||
err = errors.New("knowledge content is empty after parsing")
|
err = errors.New("knowledge content is empty after parsing")
|
||||||
_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
|
_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
|
||||||
@@ -1018,7 +1113,7 @@ func (s *KnowledgeService) executeParseJob(ctx context.Context, job knowledgePar
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = s.markKnowledgeIngestionCompleted(ctx, job.TenantID, job.ItemID, job.ParseTaskID, metas); err != nil {
|
if err = s.markKnowledgeIngestionCompleted(ctx, job.TenantID, job.ItemID, job.ParseTaskID, item.Name, parsedContent, metas); err != nil {
|
||||||
_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
|
_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -1030,9 +1125,10 @@ func (s *KnowledgeService) loadKnowledgeItemForParse(
|
|||||||
) (*knowledgeItemRecord, error) {
|
) (*knowledgeItemRecord, error) {
|
||||||
record := &knowledgeItemRecord{}
|
record := &knowledgeItemRecord{}
|
||||||
var contentText *string
|
var contentText *string
|
||||||
|
var markdownContent *string
|
||||||
err := s.pool.QueryRow(ctx, `
|
err := s.pool.QueryRow(ctx, `
|
||||||
SELECT ki.id, ki.tenant_id, ki.group_id, kg.name, ki.name, ki.source_type,
|
SELECT ki.id, ki.tenant_id, ki.group_id, kg.name, ki.name, ki.source_type,
|
||||||
ki.source_uri, ki.storage_key, ki.content_text, ki.item_version
|
ki.source_uri, ki.storage_key, ki.content_text, ki.markdown_content, ki.item_version
|
||||||
FROM knowledge_items ki
|
FROM knowledge_items ki
|
||||||
JOIN knowledge_groups kg ON kg.id = ki.group_id
|
JOIN knowledge_groups kg ON kg.id = ki.group_id
|
||||||
WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NULL
|
WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NULL
|
||||||
@@ -1046,37 +1142,38 @@ func (s *KnowledgeService) loadKnowledgeItemForParse(
|
|||||||
&record.SourceURI,
|
&record.SourceURI,
|
||||||
&record.StorageKey,
|
&record.StorageKey,
|
||||||
&contentText,
|
&contentText,
|
||||||
|
&markdownContent,
|
||||||
&record.ItemVersion,
|
&record.ItemVersion,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
record.ContentText = contentText
|
record.ContentText = contentText
|
||||||
|
record.Markdown = markdownContent
|
||||||
return record, nil
|
return record, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *KnowledgeService) readKnowledgeItemContent(ctx context.Context, item *knowledgeItemRecord) (string, error) {
|
func (s *KnowledgeService) readKnowledgeItemParsedContent(ctx context.Context, item *knowledgeItemRecord) (*knowledgeParsedContent, error) {
|
||||||
if item == nil {
|
if item == nil {
|
||||||
return "", fmt.Errorf("knowledge item is empty")
|
return nil, fmt.Errorf("knowledge item is empty")
|
||||||
}
|
}
|
||||||
|
|
||||||
switch item.SourceType {
|
switch item.SourceType {
|
||||||
case "text":
|
case "text":
|
||||||
return normalizeKnowledgeText(derefKnowledgeString(item.ContentText)), nil
|
return parseKnowledgeTextInput(item.Name, derefKnowledgeString(item.ContentText)), nil
|
||||||
case "document":
|
case "document":
|
||||||
data, err := s.objectStorage.GetBytes(ctx, item.StorageKey)
|
data, err := s.objectStorage.GetBytes(ctx, item.StorageKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return nil, err
|
||||||
}
|
}
|
||||||
return extractKnowledgeTextFromFile(item.StorageKey, data)
|
return extractKnowledgeContentFromFile(item.StorageKey, data)
|
||||||
case "website":
|
case "website":
|
||||||
data, err := s.objectStorage.GetBytes(ctx, item.StorageKey)
|
if item.SourceURI == nil || strings.TrimSpace(*item.SourceURI) == "" {
|
||||||
if err != nil {
|
return nil, fmt.Errorf("knowledge website url is empty")
|
||||||
return "", err
|
|
||||||
}
|
}
|
||||||
return extractKnowledgeTextFromSnapshot(item.StorageKey, data)
|
return s.parseWebsiteKnowledge(ctx, strings.TrimSpace(*item.SourceURI))
|
||||||
default:
|
default:
|
||||||
return "", fmt.Errorf("unsupported knowledge source type %q", item.SourceType)
|
return nil, fmt.Errorf("unsupported knowledge source type %q", item.SourceType)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1103,9 +1200,9 @@ func (s *KnowledgeService) createPendingKnowledgeItem(
|
|||||||
err = tx.QueryRow(ctx, `
|
err = tx.QueryRow(ctx, `
|
||||||
INSERT INTO knowledge_items (
|
INSERT INTO knowledge_items (
|
||||||
tenant_id, group_id, source_type, name, source_uri, storage_key,
|
tenant_id, group_id, source_type, name, source_uri, storage_key,
|
||||||
content_text, status, size_bytes, item_version
|
content_text, markdown_content, status, size_bytes, item_version
|
||||||
)
|
)
|
||||||
VALUES ($1, $2, $3, $4, $5, $6, $7, 'pending', $8, 1)
|
VALUES ($1, $2, $3, $4, $5, $6, $7, NULL, 'pending', $8, 1)
|
||||||
RETURNING id
|
RETURNING id
|
||||||
`, tenantID, input.GroupID, input.SourceType, input.Name, input.SourceURI, input.StorageKey, input.ContentText, input.SizeBytes).Scan(&itemID)
|
`, tenantID, input.GroupID, input.SourceType, input.Name, input.SourceURI, input.StorageKey, input.ContentText, input.SizeBytes).Scan(&itemID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1159,8 +1256,21 @@ func (s *KnowledgeService) markKnowledgeIngestionRunning(
|
|||||||
func (s *KnowledgeService) markKnowledgeIngestionCompleted(
|
func (s *KnowledgeService) markKnowledgeIngestionCompleted(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
tenantID, itemID, parseTaskID int64,
|
tenantID, itemID, parseTaskID int64,
|
||||||
|
itemName string,
|
||||||
|
content *knowledgeParsedContent,
|
||||||
metas []knowledgeChunkMetaInput,
|
metas []knowledgeChunkMetaInput,
|
||||||
) error {
|
) error {
|
||||||
|
if content == nil {
|
||||||
|
return errors.New("knowledge content is empty")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(content.Text) == "" {
|
||||||
|
return errors.New("knowledge text content is empty")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(content.Markdown) == "" {
|
||||||
|
content.Markdown = buildKnowledgeMarkdownFromText("", content.Text)
|
||||||
|
}
|
||||||
|
itemName = strings.TrimSpace(itemName)
|
||||||
|
|
||||||
tx, err := s.pool.Begin(ctx)
|
tx, err := s.pool.Begin(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -1188,9 +1298,14 @@ func (s *KnowledgeService) markKnowledgeIngestionCompleted(
|
|||||||
|
|
||||||
if _, err := tx.Exec(ctx, `
|
if _, err := tx.Exec(ctx, `
|
||||||
UPDATE knowledge_items
|
UPDATE knowledge_items
|
||||||
SET status = 'completed', error_message = NULL, updated_at = NOW()
|
SET status = 'completed',
|
||||||
|
error_message = NULL,
|
||||||
|
name = CASE WHEN $3 <> '' THEN $3 ELSE name END,
|
||||||
|
content_text = $4,
|
||||||
|
markdown_content = $5,
|
||||||
|
updated_at = NOW()
|
||||||
WHERE id = $1 AND tenant_id = $2
|
WHERE id = $1 AND tenant_id = $2
|
||||||
`, itemID, tenantID); err != nil {
|
`, itemID, tenantID, itemName, content.Text, content.Markdown); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1370,6 +1485,99 @@ func (s *KnowledgeService) ensureStorableGroup(ctx context.Context, tenantID, gr
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *KnowledgeService) resolveKnowledgeStorageGroupID(ctx context.Context, tenantID, groupID int64) (int64, error) {
|
||||||
|
tx, err := s.pool.Begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return 0, response.ErrInternal(50055, "knowledge_group_resolve_failed", "failed to begin knowledge group transaction")
|
||||||
|
}
|
||||||
|
defer func() { _ = tx.Rollback(ctx) }()
|
||||||
|
|
||||||
|
resolvedGroupID, err := s.resolveKnowledgeStorageGroupIDTx(ctx, tx, tenantID, groupID)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tx.Commit(ctx); err != nil {
|
||||||
|
return 0, response.ErrInternal(50055, "knowledge_group_resolve_failed", "failed to commit knowledge group transaction")
|
||||||
|
}
|
||||||
|
return resolvedGroupID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *KnowledgeService) resolveKnowledgeStorageGroupIDTx(
|
||||||
|
ctx context.Context,
|
||||||
|
tx pgx.Tx,
|
||||||
|
tenantID, groupID int64,
|
||||||
|
) (int64, error) {
|
||||||
|
var parentID *int64
|
||||||
|
if err := tx.QueryRow(ctx, `
|
||||||
|
SELECT parent_id
|
||||||
|
FROM knowledge_groups
|
||||||
|
WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL
|
||||||
|
FOR UPDATE
|
||||||
|
`, groupID, tenantID).Scan(&parentID); err != nil {
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return 0, response.ErrNotFound(40451, "knowledge_group_not_found", "knowledge group not found")
|
||||||
|
}
|
||||||
|
return 0, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge group")
|
||||||
|
}
|
||||||
|
|
||||||
|
if parentID != nil {
|
||||||
|
var grandParentID *int64
|
||||||
|
if err := tx.QueryRow(ctx, `
|
||||||
|
SELECT parent_id
|
||||||
|
FROM knowledge_groups
|
||||||
|
WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL
|
||||||
|
`, *parentID, tenantID).Scan(&grandParentID); err != nil {
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return 0, response.ErrNotFound(40451, "knowledge_group_not_found", "knowledge group not found")
|
||||||
|
}
|
||||||
|
return 0, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge group")
|
||||||
|
}
|
||||||
|
if grandParentID != nil {
|
||||||
|
return 0, response.ErrBadRequest(40053, "knowledge_group_storage_invalid", "knowledge items can only be stored in second-level groups")
|
||||||
|
}
|
||||||
|
return groupID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var childGroupID int64
|
||||||
|
if err := tx.QueryRow(ctx, `
|
||||||
|
SELECT id
|
||||||
|
FROM knowledge_groups
|
||||||
|
WHERE tenant_id = $1
|
||||||
|
AND parent_id = $2
|
||||||
|
AND name = $3
|
||||||
|
AND deleted_at IS NULL
|
||||||
|
ORDER BY id
|
||||||
|
LIMIT 1
|
||||||
|
`, tenantID, groupID, defaultKnowledgeChildGroupName).Scan(&childGroupID); err == nil {
|
||||||
|
return childGroupID, nil
|
||||||
|
} else if !errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return 0, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge group")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tx.QueryRow(ctx, `
|
||||||
|
INSERT INTO knowledge_groups (tenant_id, name, parent_id, sort_order)
|
||||||
|
VALUES ($1, $2, $3, 0)
|
||||||
|
RETURNING id
|
||||||
|
`, tenantID, defaultKnowledgeChildGroupName, groupID).Scan(&childGroupID); err == nil {
|
||||||
|
return childGroupID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tx.QueryRow(ctx, `
|
||||||
|
SELECT id
|
||||||
|
FROM knowledge_groups
|
||||||
|
WHERE tenant_id = $1
|
||||||
|
AND parent_id = $2
|
||||||
|
AND name = $3
|
||||||
|
AND deleted_at IS NULL
|
||||||
|
ORDER BY id
|
||||||
|
LIMIT 1
|
||||||
|
`, tenantID, groupID, defaultKnowledgeChildGroupName).Scan(&childGroupID); err != nil {
|
||||||
|
return 0, response.ErrConflict(40950, "knowledge_group_create_failed", "failed to create default knowledge group")
|
||||||
|
}
|
||||||
|
return childGroupID, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *KnowledgeService) listGroupSubtreeIDs(
|
func (s *KnowledgeService) listGroupSubtreeIDs(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
tx pgx.Tx,
|
tx pgx.Tx,
|
||||||
@@ -1688,9 +1896,10 @@ func (s *KnowledgeService) loadKnowledgeItemForCleanup(
|
|||||||
) (*knowledgeItemRecord, error) {
|
) (*knowledgeItemRecord, error) {
|
||||||
record := &knowledgeItemRecord{}
|
record := &knowledgeItemRecord{}
|
||||||
var contentText *string
|
var contentText *string
|
||||||
|
var markdownContent *string
|
||||||
err := s.pool.QueryRow(ctx, `
|
err := s.pool.QueryRow(ctx, `
|
||||||
SELECT ki.id, ki.tenant_id, ki.group_id, kg.name, ki.name, ki.source_type,
|
SELECT ki.id, ki.tenant_id, ki.group_id, kg.name, ki.name, ki.source_type,
|
||||||
ki.source_uri, ki.storage_key, ki.content_text, ki.item_version
|
ki.source_uri, ki.storage_key, ki.content_text, ki.markdown_content, ki.item_version
|
||||||
FROM knowledge_items ki
|
FROM knowledge_items ki
|
||||||
JOIN knowledge_groups kg ON kg.id = ki.group_id
|
JOIN knowledge_groups kg ON kg.id = ki.group_id
|
||||||
WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NOT NULL
|
WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NOT NULL
|
||||||
@@ -1704,12 +1913,14 @@ func (s *KnowledgeService) loadKnowledgeItemForCleanup(
|
|||||||
&record.SourceURI,
|
&record.SourceURI,
|
||||||
&record.StorageKey,
|
&record.StorageKey,
|
||||||
&contentText,
|
&contentText,
|
||||||
|
&markdownContent,
|
||||||
&record.ItemVersion,
|
&record.ItemVersion,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
record.ContentText = contentText
|
record.ContentText = contentText
|
||||||
|
record.Markdown = markdownContent
|
||||||
return record, nil
|
return record, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,266 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/geo-platform/tenant-api/internal/shared/llm"
|
||||||
|
"github.com/geo-platform/tenant-api/internal/tenant/prompts"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
defaultKnowledgeArkBaseURL = "https://ark.cn-beijing.volces.com/api/v3"
|
||||||
|
defaultKnowledgeURLParseTimeout = 60 * time.Second
|
||||||
|
defaultKnowledgeURLMarkdownTimeout = 90 * time.Second
|
||||||
|
defaultKnowledgeURLMarkdownChunkChars = 12000
|
||||||
|
defaultKnowledgeURLMarkdownOutputToken = 12000
|
||||||
|
)
|
||||||
|
|
||||||
|
type arkToolExecuteRequest struct {
|
||||||
|
ActionName string `json:"action_name"`
|
||||||
|
ToolName string `json:"tool_name"`
|
||||||
|
Timeout int `json:"timeout,omitempty"`
|
||||||
|
Parameters map[string]any `json:"parameters,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type arkToolExecuteResponse struct {
|
||||||
|
StatusCode int `json:"status_code"`
|
||||||
|
Data arkToolExecuteData `json:"data"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type arkToolExecuteData struct {
|
||||||
|
ArkWebDataList []arkWebDataItem `json:"ark_web_data_list"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type arkWebDataItem struct {
|
||||||
|
URL string `json:"url"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
Content string `json:"content"`
|
||||||
|
ErrorMessage string `json:"error_message"`
|
||||||
|
ErrorCode string `json:"error_code"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *KnowledgeService) parseWebsiteKnowledge(ctx context.Context, rawURL string) (*knowledgeParsedContent, error) {
|
||||||
|
item, err := s.executeLinkReader(ctx, rawURL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
text := normalizeKnowledgeText(item.Content)
|
||||||
|
if text == "" {
|
||||||
|
return nil, fmt.Errorf("webpage parser returned empty content")
|
||||||
|
}
|
||||||
|
|
||||||
|
title := strings.TrimSpace(item.Title)
|
||||||
|
markdown, err := s.formatWebsiteMarkdown(ctx, title, text, rawURL)
|
||||||
|
if err != nil {
|
||||||
|
if s.logger != nil {
|
||||||
|
s.logger.Warn("knowledge website markdown formatting failed, fallback to local markdown",
|
||||||
|
zap.String("url", rawURL),
|
||||||
|
zap.Error(err),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
markdown = buildKnowledgeMarkdownFromText(title, text)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &knowledgeParsedContent{
|
||||||
|
Text: text,
|
||||||
|
Markdown: markdown,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *KnowledgeService) executeLinkReader(ctx context.Context, rawURL string) (*arkWebDataItem, error) {
|
||||||
|
baseURL := strings.TrimRight(strings.TrimSpace(s.arkBaseURL), "/")
|
||||||
|
if baseURL == "" {
|
||||||
|
baseURL = defaultKnowledgeArkBaseURL
|
||||||
|
}
|
||||||
|
|
||||||
|
timeout := defaultKnowledgeURLParseTimeout
|
||||||
|
callCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
body, err := json.Marshal(arkToolExecuteRequest{
|
||||||
|
ActionName: "LinkReader",
|
||||||
|
ToolName: "LinkReader",
|
||||||
|
Timeout: int(timeout / time.Second),
|
||||||
|
Parameters: map[string]any{
|
||||||
|
"url_list": []string{rawURL},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("marshal webpage parser request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(callCtx, http.MethodPost, baseURL+"/tools/execute", bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("create webpage parser request: %w", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("Authorization", "Bearer "+s.arkAPIKey)
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
resp, err := s.httpClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("execute webpage parser: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode >= http.StatusBadRequest {
|
||||||
|
data, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||||
|
return nil, fmt.Errorf("webpage parser failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(data)))
|
||||||
|
}
|
||||||
|
|
||||||
|
var parsed arkToolExecuteResponse
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
|
||||||
|
return nil, fmt.Errorf("decode webpage parser response: %w", err)
|
||||||
|
}
|
||||||
|
if parsed.StatusCode != 0 && parsed.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("webpage parser returned status_code=%d", parsed.StatusCode)
|
||||||
|
}
|
||||||
|
if len(parsed.Data.ArkWebDataList) == 0 {
|
||||||
|
return nil, fmt.Errorf("webpage parser returned no content")
|
||||||
|
}
|
||||||
|
|
||||||
|
item := parsed.Data.ArkWebDataList[0]
|
||||||
|
if strings.TrimSpace(item.ErrorCode) != "" || strings.TrimSpace(item.ErrorMessage) != "" {
|
||||||
|
return nil, fmt.Errorf("webpage parser returned error: %s %s", strings.TrimSpace(item.ErrorCode), strings.TrimSpace(item.ErrorMessage))
|
||||||
|
}
|
||||||
|
return &item, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *KnowledgeService) formatWebsiteMarkdown(ctx context.Context, title, content, rawURL string) (string, error) {
|
||||||
|
content = strings.TrimSpace(content)
|
||||||
|
if content == "" {
|
||||||
|
return "", fmt.Errorf("website content is empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
chunks := splitKnowledgeMarkdownChunks(content, defaultKnowledgeURLMarkdownChunkChars)
|
||||||
|
if len(chunks) == 0 {
|
||||||
|
return "", fmt.Errorf("website content is empty after chunking")
|
||||||
|
}
|
||||||
|
|
||||||
|
sections := make([]string, 0, len(chunks)+1)
|
||||||
|
for index, chunk := range chunks {
|
||||||
|
result, err := s.llmClient.Generate(ctx, llm.GenerateRequest{
|
||||||
|
Model: s.urlMarkdownModel,
|
||||||
|
Prompt: prompts.KnowledgeWebsiteMarkdownPrompt(title, rawURL, chunk, index, len(chunks)),
|
||||||
|
Timeout: defaultKnowledgeURLMarkdownTimeout,
|
||||||
|
MaxOutputTokens: defaultKnowledgeURLMarkdownOutputToken,
|
||||||
|
}, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
section := sanitizeLLMMarkdown(strings.TrimSpace(result.Content))
|
||||||
|
if section == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if index > 0 && !strings.HasPrefix(section, "#") {
|
||||||
|
section = fmt.Sprintf("## 第 %d 部分\n\n%s", index+1, section)
|
||||||
|
}
|
||||||
|
sections = append(sections, section)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(sections) == 0 {
|
||||||
|
return "", fmt.Errorf("website formatter returned empty markdown")
|
||||||
|
}
|
||||||
|
|
||||||
|
markdown := strings.TrimSpace(strings.Join(sections, "\n\n"))
|
||||||
|
if title != "" && !strings.HasPrefix(markdown, "# ") {
|
||||||
|
markdown = "# " + title + "\n\n" + markdown
|
||||||
|
}
|
||||||
|
return markdown, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func splitKnowledgeMarkdownChunks(content string, maxChars int) []string {
|
||||||
|
if maxChars <= 0 {
|
||||||
|
maxChars = defaultKnowledgeURLMarkdownChunkChars
|
||||||
|
}
|
||||||
|
|
||||||
|
lines := strings.Split(content, "\n")
|
||||||
|
chunks := make([]string, 0, len(lines))
|
||||||
|
current := make([]string, 0, 16)
|
||||||
|
currentLen := 0
|
||||||
|
|
||||||
|
flush := func() {
|
||||||
|
if len(current) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
chunks = append(chunks, strings.TrimSpace(strings.Join(current, "\n")))
|
||||||
|
current = current[:0]
|
||||||
|
currentLen = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, line := range lines {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if len([]rune(line)) > maxChars {
|
||||||
|
flush()
|
||||||
|
for _, segment := range splitKnowledgeLongLine(line, maxChars) {
|
||||||
|
if strings.TrimSpace(segment) != "" {
|
||||||
|
chunks = append(chunks, segment)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
nextLen := currentLen + len([]rune(line))
|
||||||
|
if len(current) > 0 {
|
||||||
|
nextLen++
|
||||||
|
}
|
||||||
|
if nextLen > maxChars {
|
||||||
|
flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
current = append(current, line)
|
||||||
|
currentLen += len([]rune(line))
|
||||||
|
if len(current) > 1 {
|
||||||
|
currentLen++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
flush()
|
||||||
|
return chunks
|
||||||
|
}
|
||||||
|
|
||||||
|
func splitKnowledgeLongLine(line string, maxChars int) []string {
|
||||||
|
runes := []rune(line)
|
||||||
|
if len(runes) <= maxChars {
|
||||||
|
return []string{line}
|
||||||
|
}
|
||||||
|
|
||||||
|
segments := make([]string, 0, (len(runes)/maxChars)+1)
|
||||||
|
for start := 0; start < len(runes); start += maxChars {
|
||||||
|
end := start + maxChars
|
||||||
|
if end > len(runes) {
|
||||||
|
end = len(runes)
|
||||||
|
}
|
||||||
|
segments = append(segments, strings.TrimSpace(string(runes[start:end])))
|
||||||
|
}
|
||||||
|
return segments
|
||||||
|
}
|
||||||
|
|
||||||
|
func sanitizeLLMMarkdown(content string) string {
|
||||||
|
content = strings.TrimSpace(content)
|
||||||
|
if !strings.HasPrefix(content, "```") {
|
||||||
|
return content
|
||||||
|
}
|
||||||
|
|
||||||
|
lines := strings.Split(content, "\n")
|
||||||
|
if len(lines) < 3 {
|
||||||
|
return strings.Trim(content, "`")
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(lines[0], "```") || strings.TrimSpace(lines[len(lines)-1]) != "```" {
|
||||||
|
return content
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(strings.Join(lines[1:len(lines)-1], "\n"))
|
||||||
|
}
|
||||||
@@ -317,3 +317,33 @@ func KnowledgePromptIntroLines() []string {
|
|||||||
func KnowledgeSnippetLabel(index int) string {
|
func KnowledgeSnippetLabel(index int) string {
|
||||||
return fmt.Sprintf("知识片段 %d", index+1)
|
return fmt.Sprintf("知识片段 %d", index+1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func KnowledgeWebsiteMarkdownPrompt(title, rawURL, content string, chunkIndex, chunkCount int) string {
|
||||||
|
scope := "以下是网页正文。"
|
||||||
|
if chunkCount > 1 {
|
||||||
|
scope = fmt.Sprintf("以下是网页正文的第 %d/%d 段。", chunkIndex+1, chunkCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
var builder strings.Builder
|
||||||
|
builder.WriteString("你是知识库 Markdown 整理助手。\n")
|
||||||
|
builder.WriteString("任务:把给定网页正文整理为适合知识库只读预览的 Markdown。\n")
|
||||||
|
builder.WriteString("要求:\n")
|
||||||
|
builder.WriteString("1. 只根据原文整理,不要补充事实,不要总结扩写。\n")
|
||||||
|
builder.WriteString("2. 保留标题、列表、表格、编号、联系方式等关键信息。\n")
|
||||||
|
builder.WriteString("3. 如果原文明显是表格,请输出 Markdown 表格。\n")
|
||||||
|
builder.WriteString("4. 不要输出解释,不要输出代码块包裹,只输出 Markdown 正文。\n")
|
||||||
|
if strings.TrimSpace(title) != "" {
|
||||||
|
builder.WriteString("网页标题:")
|
||||||
|
builder.WriteString(title)
|
||||||
|
builder.WriteString("\n")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(rawURL) != "" {
|
||||||
|
builder.WriteString("网页地址:")
|
||||||
|
builder.WriteString(rawURL)
|
||||||
|
builder.WriteString("\n")
|
||||||
|
}
|
||||||
|
builder.WriteString(scope)
|
||||||
|
builder.WriteString("\n\n")
|
||||||
|
builder.WriteString(content)
|
||||||
|
return builder.String()
|
||||||
|
}
|
||||||
|
|||||||
@@ -27,8 +27,10 @@ func NewArticleHandler(a *bootstrap.App) *ArticleHandler {
|
|||||||
a.RetrievalProvider,
|
a.RetrievalProvider,
|
||||||
a.VectorStore,
|
a.VectorStore,
|
||||||
a.ObjectStorage,
|
a.ObjectStorage,
|
||||||
|
a.LLM,
|
||||||
a.Logger,
|
a.Logger,
|
||||||
a.Config.Retrieval,
|
a.Config.Retrieval,
|
||||||
|
a.Config.LLM,
|
||||||
0,
|
0,
|
||||||
0,
|
0,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -23,8 +23,10 @@ func NewKnowledgeHandler(a *bootstrap.App) *KnowledgeHandler {
|
|||||||
a.RetrievalProvider,
|
a.RetrievalProvider,
|
||||||
a.VectorStore,
|
a.VectorStore,
|
||||||
a.ObjectStorage,
|
a.ObjectStorage,
|
||||||
|
a.LLM,
|
||||||
a.Logger,
|
a.Logger,
|
||||||
a.Config.Retrieval,
|
a.Config.Retrieval,
|
||||||
|
a.Config.LLM,
|
||||||
a.Config.Generation.QueueSize,
|
a.Config.Generation.QueueSize,
|
||||||
a.Config.Generation.WorkerConcurrency,
|
a.Config.Generation.WorkerConcurrency,
|
||||||
),
|
),
|
||||||
@@ -108,6 +110,21 @@ func (h *KnowledgeHandler) ListItems(c *gin.Context) {
|
|||||||
response.Success(c, data)
|
response.Success(c, data)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *KnowledgeHandler) GetItemDetail(c *gin.Context) {
|
||||||
|
itemID, err := strconv.ParseInt(c.Param("id"), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
response.Error(c, response.ErrBadRequest(40001, "invalid_id", "knowledge item id must be a number"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := h.svc.GetItemDetail(c.Request.Context(), itemID)
|
||||||
|
if err != nil {
|
||||||
|
response.Error(c, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
response.Success(c, data)
|
||||||
|
}
|
||||||
|
|
||||||
func (h *KnowledgeHandler) CreateTextItem(c *gin.Context) {
|
func (h *KnowledgeHandler) CreateTextItem(c *gin.Context) {
|
||||||
var req app.KnowledgeTextItemRequest
|
var req app.KnowledgeTextItemRequest
|
||||||
if err := c.ShouldBindJSON(&req); err != nil {
|
if err := c.ShouldBindJSON(&req); err != nil {
|
||||||
|
|||||||
@@ -90,6 +90,7 @@ func RegisterRoutes(app *bootstrap.App) {
|
|||||||
knowledge.PUT("/groups/:gid", knowledgeHandler.UpdateGroup)
|
knowledge.PUT("/groups/:gid", knowledgeHandler.UpdateGroup)
|
||||||
knowledge.DELETE("/groups/:gid", knowledgeHandler.DeleteGroup)
|
knowledge.DELETE("/groups/:gid", knowledgeHandler.DeleteGroup)
|
||||||
knowledge.GET("/items", knowledgeHandler.ListItems)
|
knowledge.GET("/items", knowledgeHandler.ListItems)
|
||||||
|
knowledge.GET("/items/:id", knowledgeHandler.GetItemDetail)
|
||||||
knowledge.POST("/items/text", knowledgeHandler.CreateTextItem)
|
knowledge.POST("/items/text", knowledgeHandler.CreateTextItem)
|
||||||
knowledge.POST("/items/url", knowledgeHandler.CreateURLItem)
|
knowledge.POST("/items/url", knowledgeHandler.CreateURLItem)
|
||||||
knowledge.POST("/items/file", knowledgeHandler.CreateFileItem)
|
knowledge.POST("/items/file", knowledgeHandler.CreateFileItem)
|
||||||
|
|||||||
@@ -21,8 +21,10 @@ func NewTemplateHandler(a *bootstrap.App) *TemplateHandler {
|
|||||||
a.RetrievalProvider,
|
a.RetrievalProvider,
|
||||||
a.VectorStore,
|
a.VectorStore,
|
||||||
a.ObjectStorage,
|
a.ObjectStorage,
|
||||||
|
a.LLM,
|
||||||
a.Logger,
|
a.Logger,
|
||||||
a.Config.Retrieval,
|
a.Config.Retrieval,
|
||||||
|
a.Config.LLM,
|
||||||
0,
|
0,
|
||||||
0,
|
0,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,2 @@
|
|||||||
|
ALTER TABLE knowledge_items
|
||||||
|
DROP COLUMN IF EXISTS markdown_content;
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
ALTER TABLE knowledge_items
|
||||||
|
ADD COLUMN markdown_content TEXT;
|
||||||
|
|
||||||
|
UPDATE knowledge_items
|
||||||
|
SET markdown_content = content_text
|
||||||
|
WHERE markdown_content IS NULL
|
||||||
|
AND content_text IS NOT NULL;
|
||||||
Reference in New Issue
Block a user