feat: add knowledge markdown formatting and detail retrieval

- Implemented KnowledgeWebsiteMarkdownPrompt for formatting webpage content into Markdown.
- Added GetItemDetail method in KnowledgeHandler to retrieve knowledge item details.
- Introduced KnowledgeDetailDrawer component for displaying knowledge item details in the admin web interface.
- Created MarkdownPreview component for rendering Markdown content.
- Added knowledge chunking and URL parsing logic to handle webpage content extraction and formatting.
- Updated database schema to include markdown_content in knowledge_items table.
This commit is contained in:
2026-04-05 20:41:42 +08:00
parent 446f865cdf
commit dbd7747742
26 changed files with 1556 additions and 252 deletions
+347 -148
View File
@@ -3,76 +3,113 @@ package app
import (
"archive/zip"
"bytes"
"context"
"encoding/xml"
"fmt"
"io"
"net/http"
"path/filepath"
"regexp"
"strings"
"time"
"unicode/utf8"
"github.com/extrame/xls"
pdf "github.com/ledongthuc/pdf"
"github.com/xuri/excelize/v2"
"golang.org/x/net/html"
)
const (
maxKnowledgeFetchBytes = 5 << 20
var (
whitespacePattern = regexp.MustCompile(`[ \t]+`)
genericKnowledgeMarkdownTitleRegex = regexp.MustCompile(`^第\s*\d+\s*部分$`)
)
var whitespacePattern = regexp.MustCompile(`[ \t]+`)
const maxKnowledgeSpreadsheetRows = 100000
func extractKnowledgeTextFromURL(ctx context.Context, rawURL string) (string, error) {
data, contentType, err := fetchKnowledgeURLSnapshot(ctx, rawURL)
if err != nil {
return "", err
}
return extractKnowledgeTextFromSnapshot(resolveSnapshotFileName(rawURL, contentType), data)
type knowledgeParsedContent struct {
Text string
Markdown string
}
func fetchKnowledgeURLSnapshot(ctx context.Context, rawURL string) ([]byte, string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimSpace(rawURL), nil)
if err != nil {
return nil, "", fmt.Errorf("create request: %w", err)
func parseKnowledgeTextInput(name, content string) *knowledgeParsedContent {
text := normalizeKnowledgeText(content)
return &knowledgeParsedContent{
Text: text,
Markdown: buildKnowledgeMarkdownFromText(name, text),
}
req.Header.Set("User-Agent", "geo-knowledge-fetcher/1.0")
client := &http.Client{Timeout: 20 * time.Second}
resp, err := client.Do(req)
if err != nil {
return nil, "", fmt.Errorf("fetch url: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode >= http.StatusBadRequest {
return nil, "", fmt.Errorf("fetch url failed: status=%d", resp.StatusCode)
}
data, err := io.ReadAll(io.LimitReader(resp.Body, maxKnowledgeFetchBytes))
if err != nil {
return nil, "", fmt.Errorf("read url body: %w", err)
}
contentType := strings.ToLower(resp.Header.Get("Content-Type"))
return data, contentType, nil
}
func resolveSnapshotFileName(rawURL string, contentType string) string {
if strings.Contains(strings.ToLower(contentType), "text/plain") {
return rawURL + ".txt"
}
return rawURL + ".html"
}
func extractKnowledgeTextFromSnapshot(fileName string, content []byte) (string, error) {
func extractKnowledgeContentFromFile(fileName string, content []byte) (*knowledgeParsedContent, error) {
ext := strings.ToLower(filepath.Ext(fileName))
if ext != ".html" && ext != ".htm" {
return extractKnowledgeTextFromFile(fileName, content)
switch ext {
case ".txt", ".json", ".csv":
return extractUTF8KnowledgeText(fileName, content, false)
case ".md", ".markdown":
return extractUTF8KnowledgeText(fileName, content, true)
case ".html", ".htm":
text, err := extractHTMLText(content)
if err != nil {
return nil, err
}
return &knowledgeParsedContent{
Text: text,
Markdown: buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text),
}, nil
case ".docx":
text, err := extractDOCXText(content)
if err != nil {
return nil, err
}
return &knowledgeParsedContent{
Text: text,
Markdown: buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text),
}, nil
case ".xlsx":
return extractXLSXContent(content)
case ".xls":
return extractXLSContent(content)
case ".pdf":
text, err := extractPDFText(content)
if err != nil {
return nil, err
}
return &knowledgeParsedContent{
Text: text,
Markdown: buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text),
}, nil
default:
if utf8.Valid(content) {
return extractUTF8KnowledgeText(fileName, content, false)
}
return nil, fmt.Errorf("unsupported file type %s", ext)
}
}
func extractUTF8KnowledgeText(fileName string, content []byte, preserveMarkdown bool) (*knowledgeParsedContent, error) {
if !utf8.Valid(content) {
return nil, fmt.Errorf("file %s is not valid UTF-8 text", fileName)
}
raw := string(content)
text := normalizeKnowledgeText(raw)
if text == "" {
return nil, fmt.Errorf("file %s contains no readable text", fileName)
}
markdown := buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text)
if preserveMarkdown {
normalizedMarkdown := normalizeKnowledgeMarkdown(raw)
if normalizedMarkdown != "" {
markdown = normalizedMarkdown
}
}
return &knowledgeParsedContent{
Text: text,
Markdown: markdown,
}, nil
}
func extractHTMLText(content []byte) (string, error) {
doc, err := html.Parse(bytes.NewReader(content))
if err != nil {
return "", fmt.Errorf("parse html: %w", err)
@@ -106,32 +143,11 @@ func extractKnowledgeTextFromSnapshot(fileName string, content []byte) (string,
}
walk(doc, false)
return normalizeKnowledgeText(strings.Join(parts, "\n")), nil
}
func extractKnowledgeTextFromFile(fileName string, content []byte) (string, error) {
ext := strings.ToLower(filepath.Ext(fileName))
switch ext {
case ".txt", ".md", ".markdown", ".json", ".csv", ".html", ".htm":
if !utf8.Valid(content) {
return "", fmt.Errorf("file %s is not valid UTF-8 text", fileName)
}
return normalizeKnowledgeText(string(content)), nil
case ".docx":
return extractDOCXText(content)
case ".xlsx":
return extractXLSXText(content)
case ".pdf":
return extractPDFText(content)
case ".doc", ".xls":
return "", fmt.Errorf("file type %s is not supported yet", ext)
default:
if utf8.Valid(content) {
return normalizeKnowledgeText(string(content)), nil
}
return "", fmt.Errorf("unsupported file type %s", ext)
result := normalizeKnowledgeText(strings.Join(parts, "\n"))
if result == "" {
return "", fmt.Errorf("html contains no readable text")
}
return result, nil
}
func extractDOCXText(content []byte) (string, error) {
@@ -173,46 +189,177 @@ func extractDOCXText(content []byte) (string, error) {
}
}
}
if len(parts) == 0 {
return "", fmt.Errorf("docx contains no readable text")
}
return normalizeKnowledgeText(strings.Join(parts, "\n")), nil
}
func extractXLSXText(content []byte) (string, error) {
func extractXLSXContent(content []byte) (*knowledgeParsedContent, error) {
book, err := excelize.OpenReader(bytes.NewReader(content))
if err != nil {
return "", fmt.Errorf("open xlsx: %w", err)
return nil, fmt.Errorf("open xlsx: %w", err)
}
defer func() { _ = book.Close() }()
lines := make([]string, 0)
markdownSections := make([]string, 0)
textSections := make([]string, 0)
for _, sheet := range book.GetSheetList() {
rows, err := book.GetRows(sheet)
if err != nil {
return "", fmt.Errorf("read xlsx rows: %w", err)
return nil, fmt.Errorf("read xlsx rows: %w", err)
}
if len(rows) == 0 {
cleaned := normalizeSpreadsheetRows(rows)
if len(cleaned) == 0 {
continue
}
lines = append(lines, sheet)
for _, row := range rows {
cells := make([]string, 0, len(row))
for _, cell := range row {
cell = strings.TrimSpace(cell)
if cell != "" {
cells = append(cells, cell)
}
}
if len(cells) > 0 {
lines = append(lines, strings.Join(cells, " | "))
}
markdownSections = append(markdownSections, buildKnowledgeSheetMarkdown(sheet, cleaned))
textSections = append(textSections, buildKnowledgeSheetPlainText(sheet, cleaned))
}
if len(markdownSections) == 0 {
return nil, fmt.Errorf("xlsx contains no readable text")
}
return &knowledgeParsedContent{
Text: normalizeKnowledgeText(strings.Join(textSections, "\n\n")),
Markdown: strings.TrimSpace(strings.Join(markdownSections, "\n\n")),
}, nil
}
func extractXLSContent(content []byte) (*knowledgeParsedContent, error) {
book, err := xls.OpenReader(bytes.NewReader(content), "utf-8")
if err != nil {
return nil, fmt.Errorf("open xls: %w", err)
}
// The upstream xls library panics for sparse rows when calling sheet.Row(i),
// so we rely on its safer bulk reader here instead of random row access.
rows := book.ReadAllCells(maxKnowledgeSpreadsheetRows)
cleaned := normalizeSpreadsheetRows(rows)
if len(cleaned) == 0 {
return nil, fmt.Errorf("xls contains no readable text")
}
title := "XLS 工作簿"
if book.NumSheets() == 1 {
if sheet := book.GetSheet(0); sheet != nil && strings.TrimSpace(sheet.Name) != "" {
title = strings.TrimSpace(sheet.Name)
}
}
if len(lines) == 0 {
return "", fmt.Errorf("xlsx contains no readable text")
return &knowledgeParsedContent{
Text: normalizeKnowledgeText(buildKnowledgeSheetPlainText(title, cleaned)),
Markdown: buildKnowledgeSheetMarkdown(title, cleaned),
}, nil
}
func normalizeSpreadsheetRows(rows [][]string) [][]string {
cleaned := make([][]string, 0, len(rows))
for _, row := range rows {
cells := make([]string, len(row))
copy(cells, row)
for index := range cells {
cells[index] = normalizeKnowledgeInlineText(cells[index])
}
lastNonEmpty := -1
for index, cell := range cells {
if cell != "" {
lastNonEmpty = index
}
}
if lastNonEmpty < 0 {
continue
}
cleaned = append(cleaned, cells[:lastNonEmpty+1])
}
return normalizeKnowledgeText(strings.Join(lines, "\n")), nil
return cleaned
}
func buildKnowledgeSheetPlainText(sheetName string, rows [][]string) string {
lines := make([]string, 0, len(rows)+1)
if strings.TrimSpace(sheetName) != "" {
lines = append(lines, strings.TrimSpace(sheetName))
}
for _, row := range rows {
lines = append(lines, strings.Join(row, " | "))
}
return strings.Join(lines, "\n")
}
func buildKnowledgeSheetMarkdown(sheetName string, rows [][]string) string {
if len(rows) == 0 {
return ""
}
width := 0
for _, row := range rows {
if len(row) > width {
width = len(row)
}
}
if width == 0 {
return ""
}
normalizedRows := make([][]string, 0, len(rows))
for _, row := range rows {
normalizedRows = append(normalizedRows, padKnowledgeRow(row, width))
}
var builder strings.Builder
if title := strings.TrimSpace(sheetName); title != "" {
builder.WriteString("## ")
builder.WriteString(title)
builder.WriteString("\n\n")
}
writeKnowledgeMarkdownTableRow(&builder, normalizedRows[0])
writeKnowledgeMarkdownTableSeparator(&builder, width)
for _, row := range normalizedRows[1:] {
writeKnowledgeMarkdownTableRow(&builder, row)
}
return strings.TrimSpace(builder.String())
}
func padKnowledgeRow(row []string, width int) []string {
result := make([]string, width)
copy(result, row)
return result
}
func writeKnowledgeMarkdownTableRow(builder *strings.Builder, row []string) {
builder.WriteString("|")
for _, cell := range row {
builder.WriteString(" ")
builder.WriteString(escapeKnowledgeMarkdownTableCell(cell))
builder.WriteString(" |")
}
builder.WriteString("\n")
}
func writeKnowledgeMarkdownTableSeparator(builder *strings.Builder, width int) {
builder.WriteString("|")
for index := 0; index < width; index++ {
builder.WriteString(" --- |")
}
builder.WriteString("\n")
}
func escapeKnowledgeMarkdownTableCell(cell string) string {
cell = strings.TrimSpace(cell)
cell = strings.ReplaceAll(cell, "\\", "\\\\")
cell = strings.ReplaceAll(cell, "|", "\\|")
cell = strings.ReplaceAll(cell, "\r", " ")
cell = strings.ReplaceAll(cell, "\n", " / ")
return cell
}
func extractPDFText(content []byte) (string, error) {
@@ -223,7 +370,7 @@ func extractPDFText(content []byte) (string, error) {
var builder strings.Builder
totalPage := reader.NumPage()
for pageIndex := 1; pageIndex <= totalPage; pageIndex += 1 {
for pageIndex := 1; pageIndex <= totalPage; pageIndex++ {
page := reader.Page(pageIndex)
if page.V.IsNull() {
continue
@@ -245,12 +392,116 @@ func extractPDFText(content []byte) (string, error) {
return result, nil
}
func normalizeKnowledgeText(input string) string {
func defaultKnowledgeTitle(fileName string) string {
base := filepath.Base(strings.TrimSpace(fileName))
if base == "" {
return ""
}
return strings.TrimSuffix(base, filepath.Ext(base))
}
func buildKnowledgeMarkdownFromText(title, text string) string {
text = normalizeKnowledgeText(text)
title = strings.TrimSpace(title)
if text == "" {
if title == "" {
return ""
}
return "# " + title
}
lines := strings.Split(text, "\n")
if title != "" && len(lines) > 0 && strings.EqualFold(strings.TrimSpace(lines[0]), title) {
lines = lines[1:]
}
paragraphs := make([]string, 0, len(lines))
for _, line := range lines {
line = strings.TrimSpace(line)
if line != "" {
paragraphs = append(paragraphs, line)
}
}
sections := make([]string, 0, 2)
if title != "" {
sections = append(sections, "# "+title)
}
if len(paragraphs) > 0 {
sections = append(sections, strings.Join(paragraphs, "\n\n"))
}
return strings.TrimSpace(strings.Join(sections, "\n\n"))
}
func extractKnowledgeMarkdownTitle(markdown string) string {
markdown = normalizeKnowledgeMarkdown(markdown)
if markdown == "" {
return ""
}
for _, line := range strings.Split(markdown, "\n") {
line = strings.TrimSpace(line)
if line == "" || !strings.HasPrefix(line, "#") {
continue
}
level := 0
for level < len(line) && line[level] == '#' {
level++
}
if level == len(line) || line[level] != ' ' {
continue
}
title := strings.Join(strings.Fields(strings.TrimSpace(line[level+1:])), " ")
title = strings.Trim(title, "# \t")
if title == "" || genericKnowledgeMarkdownTitleRegex.MatchString(title) {
continue
}
switch title {
case "摘要", "概述", "概要", "目录", "正文":
continue
}
runes := []rune(title)
if len(runes) > 200 {
title = string(runes[:200])
}
return title
}
return ""
}
func normalizeKnowledgeMarkdown(input string) string {
input = strings.ReplaceAll(input, "\r\n", "\n")
input = strings.ReplaceAll(input, "\u00a0", " ")
lines := strings.Split(input, "\n")
cleaned := make([]string, 0, len(lines))
lastBlank := false
for _, line := range lines {
line = whitespacePattern.ReplaceAllString(strings.TrimSpace(line), " ")
line = strings.TrimRight(line, " \t")
if strings.TrimSpace(line) == "" {
if lastBlank {
continue
}
lastBlank = true
cleaned = append(cleaned, "")
continue
}
lastBlank = false
cleaned = append(cleaned, line)
}
return strings.TrimSpace(strings.Join(cleaned, "\n"))
}
func normalizeKnowledgeText(input string) string {
input = strings.ReplaceAll(input, "\u00a0", " ")
lines := strings.Split(strings.ReplaceAll(input, "\r\n", "\n"), "\n")
cleaned := make([]string, 0, len(lines))
for _, line := range lines {
line = normalizeKnowledgeInlineText(line)
if line != "" {
cleaned = append(cleaned, line)
}
@@ -258,58 +509,6 @@ func normalizeKnowledgeText(input string) string {
return strings.TrimSpace(strings.Join(cleaned, "\n"))
}
func splitKnowledgeTextIntoChunks(text string, chunkSize, chunkOverlap int) []string {
text = normalizeKnowledgeText(text)
if text == "" {
return nil
}
runes := []rune(text)
if chunkSize <= 0 {
chunkSize = 900
}
if chunkOverlap < 0 {
chunkOverlap = 0
}
if chunkOverlap >= chunkSize {
chunkOverlap = chunkSize / 4
}
chunks := make([]string, 0)
for start := 0; start < len(runes); {
end := start + chunkSize
if end > len(runes) {
end = len(runes)
}
if end < len(runes) {
for offset := end; offset > start+chunkSize/2; offset -= 1 {
if runes[offset-1] == '\n' || runes[offset-1] == '。' || runes[offset-1] == '.' {
end = offset
break
}
}
}
chunk := strings.TrimSpace(string(runes[start:end]))
if chunk != "" {
chunks = append(chunks, chunk)
}
if end >= len(runes) {
break
}
start = end - chunkOverlap
if start < 0 {
start = 0
}
}
return chunks
}
func estimateKnowledgeTokenCount(text string) int {
if text == "" {
return 0
}
return utf8.RuneCountInString(text)
func normalizeKnowledgeInlineText(input string) string {
return whitespacePattern.ReplaceAllString(strings.TrimSpace(input), " ")
}