root
dbd7747742
- Implemented KnowledgeWebsiteMarkdownPrompt for formatting webpage content into Markdown. - Added GetItemDetail method in KnowledgeHandler to retrieve knowledge item details. - Introduced KnowledgeDetailDrawer component for displaying knowledge item details in the admin web interface. - Created MarkdownPreview component for rendering Markdown content. - Added knowledge chunking and URL parsing logic to handle webpage content extraction and formatting. - Updated database schema to include markdown_content in knowledge_items table.
515 lines
13 KiB
Go
515 lines
13 KiB
Go
package app
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bytes"
|
|
"encoding/xml"
|
|
"fmt"
|
|
"io"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"unicode/utf8"
|
|
|
|
"github.com/extrame/xls"
|
|
pdf "github.com/ledongthuc/pdf"
|
|
"github.com/xuri/excelize/v2"
|
|
"golang.org/x/net/html"
|
|
)
|
|
|
|
var (
|
|
whitespacePattern = regexp.MustCompile(`[ \t]+`)
|
|
genericKnowledgeMarkdownTitleRegex = regexp.MustCompile(`^第\s*\d+\s*部分$`)
|
|
)
|
|
|
|
const maxKnowledgeSpreadsheetRows = 100000
|
|
|
|
type knowledgeParsedContent struct {
|
|
Text string
|
|
Markdown string
|
|
}
|
|
|
|
func parseKnowledgeTextInput(name, content string) *knowledgeParsedContent {
|
|
text := normalizeKnowledgeText(content)
|
|
return &knowledgeParsedContent{
|
|
Text: text,
|
|
Markdown: buildKnowledgeMarkdownFromText(name, text),
|
|
}
|
|
}
|
|
|
|
func extractKnowledgeContentFromFile(fileName string, content []byte) (*knowledgeParsedContent, error) {
|
|
ext := strings.ToLower(filepath.Ext(fileName))
|
|
|
|
switch ext {
|
|
case ".txt", ".json", ".csv":
|
|
return extractUTF8KnowledgeText(fileName, content, false)
|
|
case ".md", ".markdown":
|
|
return extractUTF8KnowledgeText(fileName, content, true)
|
|
case ".html", ".htm":
|
|
text, err := extractHTMLText(content)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &knowledgeParsedContent{
|
|
Text: text,
|
|
Markdown: buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text),
|
|
}, nil
|
|
case ".docx":
|
|
text, err := extractDOCXText(content)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &knowledgeParsedContent{
|
|
Text: text,
|
|
Markdown: buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text),
|
|
}, nil
|
|
case ".xlsx":
|
|
return extractXLSXContent(content)
|
|
case ".xls":
|
|
return extractXLSContent(content)
|
|
case ".pdf":
|
|
text, err := extractPDFText(content)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &knowledgeParsedContent{
|
|
Text: text,
|
|
Markdown: buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text),
|
|
}, nil
|
|
default:
|
|
if utf8.Valid(content) {
|
|
return extractUTF8KnowledgeText(fileName, content, false)
|
|
}
|
|
return nil, fmt.Errorf("unsupported file type %s", ext)
|
|
}
|
|
}
|
|
|
|
func extractUTF8KnowledgeText(fileName string, content []byte, preserveMarkdown bool) (*knowledgeParsedContent, error) {
|
|
if !utf8.Valid(content) {
|
|
return nil, fmt.Errorf("file %s is not valid UTF-8 text", fileName)
|
|
}
|
|
|
|
raw := string(content)
|
|
text := normalizeKnowledgeText(raw)
|
|
if text == "" {
|
|
return nil, fmt.Errorf("file %s contains no readable text", fileName)
|
|
}
|
|
|
|
markdown := buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text)
|
|
if preserveMarkdown {
|
|
normalizedMarkdown := normalizeKnowledgeMarkdown(raw)
|
|
if normalizedMarkdown != "" {
|
|
markdown = normalizedMarkdown
|
|
}
|
|
}
|
|
|
|
return &knowledgeParsedContent{
|
|
Text: text,
|
|
Markdown: markdown,
|
|
}, nil
|
|
}
|
|
|
|
func extractHTMLText(content []byte) (string, error) {
|
|
doc, err := html.Parse(bytes.NewReader(content))
|
|
if err != nil {
|
|
return "", fmt.Errorf("parse html: %w", err)
|
|
}
|
|
|
|
var parts []string
|
|
var walk func(*html.Node, bool)
|
|
walk = func(node *html.Node, skip bool) {
|
|
if node == nil {
|
|
return
|
|
}
|
|
|
|
nextSkip := skip
|
|
if node.Type == html.ElementNode {
|
|
switch strings.ToLower(node.Data) {
|
|
case "script", "style", "noscript", "svg":
|
|
nextSkip = true
|
|
}
|
|
}
|
|
|
|
if node.Type == html.TextNode && !nextSkip {
|
|
text := strings.TrimSpace(node.Data)
|
|
if text != "" {
|
|
parts = append(parts, text)
|
|
}
|
|
}
|
|
|
|
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
|
walk(child, nextSkip)
|
|
}
|
|
}
|
|
walk(doc, false)
|
|
|
|
result := normalizeKnowledgeText(strings.Join(parts, "\n"))
|
|
if result == "" {
|
|
return "", fmt.Errorf("html contains no readable text")
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func extractDOCXText(content []byte) (string, error) {
|
|
reader, err := zip.NewReader(bytes.NewReader(content), int64(len(content)))
|
|
if err != nil {
|
|
return "", fmt.Errorf("open docx: %w", err)
|
|
}
|
|
|
|
var parts []string
|
|
for _, file := range reader.File {
|
|
if file.Name != "word/document.xml" {
|
|
continue
|
|
}
|
|
rc, err := file.Open()
|
|
if err != nil {
|
|
return "", fmt.Errorf("open docx entry: %w", err)
|
|
}
|
|
data, err := io.ReadAll(rc)
|
|
rc.Close()
|
|
if err != nil {
|
|
return "", fmt.Errorf("read docx entry: %w", err)
|
|
}
|
|
|
|
decoder := xml.NewDecoder(bytes.NewReader(data))
|
|
for {
|
|
token, tokenErr := decoder.Token()
|
|
if tokenErr == io.EOF {
|
|
break
|
|
}
|
|
if tokenErr != nil {
|
|
return "", fmt.Errorf("decode docx xml: %w", tokenErr)
|
|
}
|
|
switch typed := token.(type) {
|
|
case xml.CharData:
|
|
text := strings.TrimSpace(string(typed))
|
|
if text != "" {
|
|
parts = append(parts, text)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(parts) == 0 {
|
|
return "", fmt.Errorf("docx contains no readable text")
|
|
}
|
|
return normalizeKnowledgeText(strings.Join(parts, "\n")), nil
|
|
}
|
|
|
|
func extractXLSXContent(content []byte) (*knowledgeParsedContent, error) {
|
|
book, err := excelize.OpenReader(bytes.NewReader(content))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open xlsx: %w", err)
|
|
}
|
|
defer func() { _ = book.Close() }()
|
|
|
|
markdownSections := make([]string, 0)
|
|
textSections := make([]string, 0)
|
|
for _, sheet := range book.GetSheetList() {
|
|
rows, err := book.GetRows(sheet)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read xlsx rows: %w", err)
|
|
}
|
|
|
|
cleaned := normalizeSpreadsheetRows(rows)
|
|
if len(cleaned) == 0 {
|
|
continue
|
|
}
|
|
|
|
markdownSections = append(markdownSections, buildKnowledgeSheetMarkdown(sheet, cleaned))
|
|
textSections = append(textSections, buildKnowledgeSheetPlainText(sheet, cleaned))
|
|
}
|
|
|
|
if len(markdownSections) == 0 {
|
|
return nil, fmt.Errorf("xlsx contains no readable text")
|
|
}
|
|
|
|
return &knowledgeParsedContent{
|
|
Text: normalizeKnowledgeText(strings.Join(textSections, "\n\n")),
|
|
Markdown: strings.TrimSpace(strings.Join(markdownSections, "\n\n")),
|
|
}, nil
|
|
}
|
|
|
|
func extractXLSContent(content []byte) (*knowledgeParsedContent, error) {
|
|
book, err := xls.OpenReader(bytes.NewReader(content), "utf-8")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open xls: %w", err)
|
|
}
|
|
|
|
// The upstream xls library panics for sparse rows when calling sheet.Row(i),
|
|
// so we rely on its safer bulk reader here instead of random row access.
|
|
rows := book.ReadAllCells(maxKnowledgeSpreadsheetRows)
|
|
cleaned := normalizeSpreadsheetRows(rows)
|
|
if len(cleaned) == 0 {
|
|
return nil, fmt.Errorf("xls contains no readable text")
|
|
}
|
|
|
|
title := "XLS 工作簿"
|
|
if book.NumSheets() == 1 {
|
|
if sheet := book.GetSheet(0); sheet != nil && strings.TrimSpace(sheet.Name) != "" {
|
|
title = strings.TrimSpace(sheet.Name)
|
|
}
|
|
}
|
|
|
|
return &knowledgeParsedContent{
|
|
Text: normalizeKnowledgeText(buildKnowledgeSheetPlainText(title, cleaned)),
|
|
Markdown: buildKnowledgeSheetMarkdown(title, cleaned),
|
|
}, nil
|
|
}
|
|
|
|
func normalizeSpreadsheetRows(rows [][]string) [][]string {
|
|
cleaned := make([][]string, 0, len(rows))
|
|
for _, row := range rows {
|
|
cells := make([]string, len(row))
|
|
copy(cells, row)
|
|
|
|
for index := range cells {
|
|
cells[index] = normalizeKnowledgeInlineText(cells[index])
|
|
}
|
|
|
|
lastNonEmpty := -1
|
|
for index, cell := range cells {
|
|
if cell != "" {
|
|
lastNonEmpty = index
|
|
}
|
|
}
|
|
if lastNonEmpty < 0 {
|
|
continue
|
|
}
|
|
|
|
cleaned = append(cleaned, cells[:lastNonEmpty+1])
|
|
}
|
|
return cleaned
|
|
}
|
|
|
|
func buildKnowledgeSheetPlainText(sheetName string, rows [][]string) string {
|
|
lines := make([]string, 0, len(rows)+1)
|
|
if strings.TrimSpace(sheetName) != "" {
|
|
lines = append(lines, strings.TrimSpace(sheetName))
|
|
}
|
|
for _, row := range rows {
|
|
lines = append(lines, strings.Join(row, " | "))
|
|
}
|
|
return strings.Join(lines, "\n")
|
|
}
|
|
|
|
func buildKnowledgeSheetMarkdown(sheetName string, rows [][]string) string {
|
|
if len(rows) == 0 {
|
|
return ""
|
|
}
|
|
|
|
width := 0
|
|
for _, row := range rows {
|
|
if len(row) > width {
|
|
width = len(row)
|
|
}
|
|
}
|
|
if width == 0 {
|
|
return ""
|
|
}
|
|
|
|
normalizedRows := make([][]string, 0, len(rows))
|
|
for _, row := range rows {
|
|
normalizedRows = append(normalizedRows, padKnowledgeRow(row, width))
|
|
}
|
|
|
|
var builder strings.Builder
|
|
if title := strings.TrimSpace(sheetName); title != "" {
|
|
builder.WriteString("## ")
|
|
builder.WriteString(title)
|
|
builder.WriteString("\n\n")
|
|
}
|
|
|
|
writeKnowledgeMarkdownTableRow(&builder, normalizedRows[0])
|
|
writeKnowledgeMarkdownTableSeparator(&builder, width)
|
|
for _, row := range normalizedRows[1:] {
|
|
writeKnowledgeMarkdownTableRow(&builder, row)
|
|
}
|
|
|
|
return strings.TrimSpace(builder.String())
|
|
}
|
|
|
|
func padKnowledgeRow(row []string, width int) []string {
|
|
result := make([]string, width)
|
|
copy(result, row)
|
|
return result
|
|
}
|
|
|
|
func writeKnowledgeMarkdownTableRow(builder *strings.Builder, row []string) {
|
|
builder.WriteString("|")
|
|
for _, cell := range row {
|
|
builder.WriteString(" ")
|
|
builder.WriteString(escapeKnowledgeMarkdownTableCell(cell))
|
|
builder.WriteString(" |")
|
|
}
|
|
builder.WriteString("\n")
|
|
}
|
|
|
|
func writeKnowledgeMarkdownTableSeparator(builder *strings.Builder, width int) {
|
|
builder.WriteString("|")
|
|
for index := 0; index < width; index++ {
|
|
builder.WriteString(" --- |")
|
|
}
|
|
builder.WriteString("\n")
|
|
}
|
|
|
|
func escapeKnowledgeMarkdownTableCell(cell string) string {
|
|
cell = strings.TrimSpace(cell)
|
|
cell = strings.ReplaceAll(cell, "\\", "\\\\")
|
|
cell = strings.ReplaceAll(cell, "|", "\\|")
|
|
cell = strings.ReplaceAll(cell, "\r", " ")
|
|
cell = strings.ReplaceAll(cell, "\n", " / ")
|
|
return cell
|
|
}
|
|
|
|
func extractPDFText(content []byte) (string, error) {
|
|
reader, err := pdf.NewReader(bytes.NewReader(content), int64(len(content)))
|
|
if err != nil {
|
|
return "", fmt.Errorf("open pdf: %w", err)
|
|
}
|
|
|
|
var builder strings.Builder
|
|
totalPage := reader.NumPage()
|
|
for pageIndex := 1; pageIndex <= totalPage; pageIndex++ {
|
|
page := reader.Page(pageIndex)
|
|
if page.V.IsNull() {
|
|
continue
|
|
}
|
|
text, err := page.GetPlainText(nil)
|
|
if err != nil {
|
|
return "", fmt.Errorf("extract pdf text: %w", err)
|
|
}
|
|
if text != "" {
|
|
builder.WriteString(text)
|
|
builder.WriteString("\n")
|
|
}
|
|
}
|
|
|
|
result := normalizeKnowledgeText(builder.String())
|
|
if result == "" {
|
|
return "", fmt.Errorf("pdf contains no readable text")
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func defaultKnowledgeTitle(fileName string) string {
|
|
base := filepath.Base(strings.TrimSpace(fileName))
|
|
if base == "" {
|
|
return ""
|
|
}
|
|
return strings.TrimSuffix(base, filepath.Ext(base))
|
|
}
|
|
|
|
func buildKnowledgeMarkdownFromText(title, text string) string {
|
|
text = normalizeKnowledgeText(text)
|
|
title = strings.TrimSpace(title)
|
|
if text == "" {
|
|
if title == "" {
|
|
return ""
|
|
}
|
|
return "# " + title
|
|
}
|
|
|
|
lines := strings.Split(text, "\n")
|
|
if title != "" && len(lines) > 0 && strings.EqualFold(strings.TrimSpace(lines[0]), title) {
|
|
lines = lines[1:]
|
|
}
|
|
|
|
paragraphs := make([]string, 0, len(lines))
|
|
for _, line := range lines {
|
|
line = strings.TrimSpace(line)
|
|
if line != "" {
|
|
paragraphs = append(paragraphs, line)
|
|
}
|
|
}
|
|
|
|
sections := make([]string, 0, 2)
|
|
if title != "" {
|
|
sections = append(sections, "# "+title)
|
|
}
|
|
if len(paragraphs) > 0 {
|
|
sections = append(sections, strings.Join(paragraphs, "\n\n"))
|
|
}
|
|
return strings.TrimSpace(strings.Join(sections, "\n\n"))
|
|
}
|
|
|
|
func extractKnowledgeMarkdownTitle(markdown string) string {
|
|
markdown = normalizeKnowledgeMarkdown(markdown)
|
|
if markdown == "" {
|
|
return ""
|
|
}
|
|
|
|
for _, line := range strings.Split(markdown, "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" || !strings.HasPrefix(line, "#") {
|
|
continue
|
|
}
|
|
|
|
level := 0
|
|
for level < len(line) && line[level] == '#' {
|
|
level++
|
|
}
|
|
if level == len(line) || line[level] != ' ' {
|
|
continue
|
|
}
|
|
|
|
title := strings.Join(strings.Fields(strings.TrimSpace(line[level+1:])), " ")
|
|
title = strings.Trim(title, "# \t")
|
|
if title == "" || genericKnowledgeMarkdownTitleRegex.MatchString(title) {
|
|
continue
|
|
}
|
|
switch title {
|
|
case "摘要", "概述", "概要", "目录", "正文":
|
|
continue
|
|
}
|
|
|
|
runes := []rune(title)
|
|
if len(runes) > 200 {
|
|
title = string(runes[:200])
|
|
}
|
|
return title
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
func normalizeKnowledgeMarkdown(input string) string {
|
|
input = strings.ReplaceAll(input, "\r\n", "\n")
|
|
input = strings.ReplaceAll(input, "\u00a0", " ")
|
|
|
|
lines := strings.Split(input, "\n")
|
|
cleaned := make([]string, 0, len(lines))
|
|
lastBlank := false
|
|
for _, line := range lines {
|
|
line = strings.TrimRight(line, " \t")
|
|
if strings.TrimSpace(line) == "" {
|
|
if lastBlank {
|
|
continue
|
|
}
|
|
lastBlank = true
|
|
cleaned = append(cleaned, "")
|
|
continue
|
|
}
|
|
lastBlank = false
|
|
cleaned = append(cleaned, line)
|
|
}
|
|
return strings.TrimSpace(strings.Join(cleaned, "\n"))
|
|
}
|
|
|
|
func normalizeKnowledgeText(input string) string {
|
|
input = strings.ReplaceAll(input, "\u00a0", " ")
|
|
lines := strings.Split(strings.ReplaceAll(input, "\r\n", "\n"), "\n")
|
|
cleaned := make([]string, 0, len(lines))
|
|
for _, line := range lines {
|
|
line = normalizeKnowledgeInlineText(line)
|
|
if line != "" {
|
|
cleaned = append(cleaned, line)
|
|
}
|
|
}
|
|
return strings.TrimSpace(strings.Join(cleaned, "\n"))
|
|
}
|
|
|
|
func normalizeKnowledgeInlineText(input string) string {
|
|
return whitespacePattern.ReplaceAllString(strings.TrimSpace(input), " ")
|
|
}
|