feat: add knowledge markdown formatting and detail retrieval

- Implemented KnowledgeWebsiteMarkdownPrompt for formatting webpage content into Markdown.
- Added GetItemDetail method in KnowledgeHandler to retrieve knowledge item details.
- Introduced KnowledgeDetailDrawer component for displaying knowledge item details in the admin web interface.
- Created MarkdownPreview component for rendering Markdown content.
- Added knowledge chunking and URL parsing logic to handle webpage content extraction and formatting.
- Updated database schema to include markdown_content in knowledge_items table.

server/configs/config.local.yaml.example

@@ -36,6 +36,7 @@ llm:
   provider: ark
   base_url: https://ark.cn-beijing.volces.com/api/v3
   model: doubao-seed-2-0-lite-260215
+  knowledge_url_model: doubao-seed-2-0-mini-260215
   timeout: 2m
   max_output_tokens: 4000
   temperature: 0.7

server/configs/config.yaml

@@ -49,6 +49,7 @@ llm:
   base_url: https://ark.cn-beijing.volces.com/api/v3
   api_key: "7fb6c66b-129c-4935-9617-709236c4205a"
   model: doubao-seed-2-0-lite-260215
+  knowledge_url_model: doubao-seed-2-0-mini-260215
   timeout: 2m
   max_output_tokens: 16000
   temperature: 0.7

server/go.mod

@@ -28,6 +28,8 @@ require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7 // indirect
+	github.com/extrame/xls v0.0.1 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect

server/go.sum

@@ -27,6 +27,10 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7 h1:n+nk0bNe2+gVbRI8WRbLFVwwcBQ0rr5p+gzkKb6ol8c=
+github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7/go.mod h1:GPpMrAfHdb8IdQ1/R2uIRBsNfnPnwsYE9YYI5WyY1zw=
+github.com/extrame/xls v0.0.1 h1:jI7L/o3z73TyyENPopsLS/Jlekm3nF1a/kF5hKBvy/k=
+github.com/extrame/xls v0.0.1/go.mod h1:iACcgahst7BboCpIMSpnFs4SKyU9ZjsvZBfNbUxZOJI=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=

server/internal/shared/config/config.go

@@ -79,16 +79,17 @@ type LogConfig struct {
 }
 
 type LLMConfig struct {
-	Provider        string        `mapstructure:"provider"`
-	APIKey          string        `mapstructure:"api_key"`
-	BaseURL         string        `mapstructure:"base_url"`
-	Model           string        `mapstructure:"model"`
-	Timeout         time.Duration `mapstructure:"timeout"`
-	MaxOutputTokens int64         `mapstructure:"max_output_tokens"`
-	Temperature     float64       `mapstructure:"temperature"`
-	TopP            float64       `mapstructure:"top_p"`
-	ReasoningEffort string        `mapstructure:"reasoning_effort"`
-	WebSearchLimit  int32         `mapstructure:"web_search_limit"`
+	Provider          string        `mapstructure:"provider"`
+	APIKey            string        `mapstructure:"api_key"`
+	BaseURL           string        `mapstructure:"base_url"`
+	Model             string        `mapstructure:"model"`
+	KnowledgeURLModel string        `mapstructure:"knowledge_url_model"`
+	Timeout           time.Duration `mapstructure:"timeout"`
+	MaxOutputTokens   int64         `mapstructure:"max_output_tokens"`
+	Temperature       float64       `mapstructure:"temperature"`
+	TopP              float64       `mapstructure:"top_p"`
+	ReasoningEffort   string        `mapstructure:"reasoning_effort"`
+	WebSearchLimit    int32         `mapstructure:"web_search_limit"`
 }
 
 type RetrievalConfig struct {
@@ -150,6 +151,9 @@ func applyEnvOverrides(cfg *Config) {
 	if apiKey, ok := lookupNonEmptyEnv("LLM_API_KEY"); ok {
 		cfg.LLM.APIKey = apiKey
 	}
+	if model, ok := lookupNonEmptyEnv("LLM_KNOWLEDGE_URL_MODEL"); ok {
+		cfg.LLM.KnowledgeURLModel = model
+	}
 	if apiKey, ok := lookupNonEmptyEnv("QDRANT_API_KEY"); ok {
 		cfg.Qdrant.APIKey = apiKey
 	}

server/internal/shared/llm/ark.go

@@ -113,6 +113,11 @@ func (c *arkClient) Generate(ctx context.Context, req GenerateRequest, onDelta f
 		maxOutputTokens = req.MaxOutputTokens
 	}
 
+	model := c.model
+	if strings.TrimSpace(req.Model) != "" {
+		model = strings.TrimSpace(req.Model)
+	}
+
 	callCtx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
 
@@ -154,7 +159,7 @@ func (c *arkClient) Generate(ctx context.Context, req GenerateRequest, onDelta f
 	}
 
 	stream, err := c.client.CreateResponsesStream(callCtx, &responses.ResponsesRequest{
-		Model:           c.model,
+		Model:           model,
 		MaxOutputTokens: &maxOutputTokens,
 		Store:           &store,
 		Stream:          &streamValue,
@@ -232,7 +237,7 @@ func (c *arkClient) Generate(ctx context.Context, req GenerateRequest, onDelta f
 
 	return &GenerateResult{
 		Content: result,
-		Model:   c.model,
+		Model:   model,
 	}, nil
 }
 

server/internal/shared/llm/client.go

@@ -13,6 +13,7 @@ import (
 var ErrNotConfigured = errors.New("llm provider is not configured")
 
 type GenerateRequest struct {
+	Model           string
 	Prompt          string
 	Timeout         time.Duration
 	MaxOutputTokens int64

server/internal/tenant/app/knowledge_chunking.go

@@ -0,0 +1,122 @@
+package app
+
+import "strings"
+
+func splitKnowledgeTextIntoChunks(text string, chunkSize, chunkOverlap int) []string {
+	text = strings.TrimSpace(text)
+	if text == "" {
+		return nil
+	}
+
+	if chunkSize <= 0 {
+		chunkSize = 900
+	}
+	if chunkOverlap < 0 {
+		chunkOverlap = 0
+	}
+	if chunkOverlap >= chunkSize {
+		chunkOverlap = chunkSize / 4
+	}
+
+	lines := strings.Split(text, "\n")
+	chunks := make([]string, 0, len(lines))
+	current := make([]string, 0, 16)
+	currentLen := 0
+
+	resetWithOverlap := func(chunk string) {
+		current = current[:0]
+		currentLen = 0
+		if chunkOverlap <= 0 {
+			return
+		}
+		overlap := takeLastKnowledgeRunes(chunk, chunkOverlap)
+		if overlap == "" {
+			return
+		}
+		current = append(current, overlap)
+		currentLen = len([]rune(overlap))
+	}
+
+	flush := func() {
+		if len(current) == 0 {
+			return
+		}
+		chunk := strings.TrimSpace(strings.Join(current, "\n"))
+		if chunk == "" {
+			current = current[:0]
+			currentLen = 0
+			return
+		}
+		chunks = append(chunks, chunk)
+		resetWithOverlap(chunk)
+	}
+
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+
+		lineRunes := len([]rune(line))
+		if lineRunes > chunkSize {
+			flush()
+			for _, segment := range splitKnowledgeLongLine(line, chunkSize) {
+				if strings.TrimSpace(segment) == "" {
+					continue
+				}
+				chunks = append(chunks, segment)
+				resetWithOverlap(segment)
+			}
+			continue
+		}
+
+		nextLen := currentLen + lineRunes
+		if len(current) > 0 {
+			nextLen++
+		}
+		if nextLen > chunkSize {
+			flush()
+		}
+
+		current = append(current, line)
+		currentLen += lineRunes
+		if len(current) > 1 {
+			currentLen++
+		}
+	}
+
+	if len(current) > 0 {
+		chunk := strings.TrimSpace(strings.Join(current, "\n"))
+		if chunk != "" {
+			chunks = append(chunks, chunk)
+		}
+	}
+
+	return chunks
+}
+
+func takeLastKnowledgeRunes(text string, size int) string {
+	if size <= 0 {
+		return ""
+	}
+
+	runes := []rune(strings.TrimSpace(text))
+	if len(runes) <= size {
+		return string(runes)
+	}
+	return strings.TrimSpace(string(runes[len(runes)-size:]))
+}
+
+func estimateKnowledgeTokenCount(text string) int {
+	text = strings.TrimSpace(text)
+	if text == "" {
+		return 0
+	}
+
+	runeCount := len([]rune(text))
+	tokenCount := runeCount / 2
+	if tokenCount == 0 {
+		return 1
+	}
+	return tokenCount
+}

server/internal/tenant/app/knowledge_parser.go

@@ -3,76 +3,113 @@ package app
 import (
 	"archive/zip"
 	"bytes"
-	"context"
 	"encoding/xml"
 	"fmt"
 	"io"
-	"net/http"
 	"path/filepath"
 	"regexp"
 	"strings"
-	"time"
 	"unicode/utf8"
 
+	"github.com/extrame/xls"
 	pdf "github.com/ledongthuc/pdf"
 	"github.com/xuri/excelize/v2"
 	"golang.org/x/net/html"
 )
 
-const (
-	maxKnowledgeFetchBytes = 5 << 20
+var (
+	whitespacePattern                  = regexp.MustCompile(`[ \t]+`)
+	genericKnowledgeMarkdownTitleRegex = regexp.MustCompile(`^第\s*\d+\s*部分$`)
 )
 
-var whitespacePattern = regexp.MustCompile(`[ \t]+`)
+const maxKnowledgeSpreadsheetRows = 100000
 
-func extractKnowledgeTextFromURL(ctx context.Context, rawURL string) (string, error) {
-	data, contentType, err := fetchKnowledgeURLSnapshot(ctx, rawURL)
-	if err != nil {
-		return "", err
-	}
-	return extractKnowledgeTextFromSnapshot(resolveSnapshotFileName(rawURL, contentType), data)
+type knowledgeParsedContent struct {
+	Text     string
+	Markdown string
 }
 
-func fetchKnowledgeURLSnapshot(ctx context.Context, rawURL string) ([]byte, string, error) {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimSpace(rawURL), nil)
-	if err != nil {
-		return nil, "", fmt.Errorf("create request: %w", err)
+func parseKnowledgeTextInput(name, content string) *knowledgeParsedContent {
+	text := normalizeKnowledgeText(content)
+	return &knowledgeParsedContent{
+		Text:     text,
+		Markdown: buildKnowledgeMarkdownFromText(name, text),
 	}
-	req.Header.Set("User-Agent", "geo-knowledge-fetcher/1.0")
-
-	client := &http.Client{Timeout: 20 * time.Second}
-	resp, err := client.Do(req)
-	if err != nil {
-		return nil, "", fmt.Errorf("fetch url: %w", err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode >= http.StatusBadRequest {
-		return nil, "", fmt.Errorf("fetch url failed: status=%d", resp.StatusCode)
-	}
-
-	data, err := io.ReadAll(io.LimitReader(resp.Body, maxKnowledgeFetchBytes))
-	if err != nil {
-		return nil, "", fmt.Errorf("read url body: %w", err)
-	}
-
-	contentType := strings.ToLower(resp.Header.Get("Content-Type"))
-	return data, contentType, nil
 }
 
-func resolveSnapshotFileName(rawURL string, contentType string) string {
-	if strings.Contains(strings.ToLower(contentType), "text/plain") {
-		return rawURL + ".txt"
-	}
-	return rawURL + ".html"
-}
-
-func extractKnowledgeTextFromSnapshot(fileName string, content []byte) (string, error) {
+func extractKnowledgeContentFromFile(fileName string, content []byte) (*knowledgeParsedContent, error) {
 	ext := strings.ToLower(filepath.Ext(fileName))
-	if ext != ".html" && ext != ".htm" {
-		return extractKnowledgeTextFromFile(fileName, content)
+
+	switch ext {
+	case ".txt", ".json", ".csv":
+		return extractUTF8KnowledgeText(fileName, content, false)
+	case ".md", ".markdown":
+		return extractUTF8KnowledgeText(fileName, content, true)
+	case ".html", ".htm":
+		text, err := extractHTMLText(content)
+		if err != nil {
+			return nil, err
+		}
+		return &knowledgeParsedContent{
+			Text:     text,
+			Markdown: buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text),
+		}, nil
+	case ".docx":
+		text, err := extractDOCXText(content)
+		if err != nil {
+			return nil, err
+		}
+		return &knowledgeParsedContent{
+			Text:     text,
+			Markdown: buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text),
+		}, nil
+	case ".xlsx":
+		return extractXLSXContent(content)
+	case ".xls":
+		return extractXLSContent(content)
+	case ".pdf":
+		text, err := extractPDFText(content)
+		if err != nil {
+			return nil, err
+		}
+		return &knowledgeParsedContent{
+			Text:     text,
+			Markdown: buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text),
+		}, nil
+	default:
+		if utf8.Valid(content) {
+			return extractUTF8KnowledgeText(fileName, content, false)
+		}
+		return nil, fmt.Errorf("unsupported file type %s", ext)
+	}
+}
+
+func extractUTF8KnowledgeText(fileName string, content []byte, preserveMarkdown bool) (*knowledgeParsedContent, error) {
+	if !utf8.Valid(content) {
+		return nil, fmt.Errorf("file %s is not valid UTF-8 text", fileName)
 	}
 
+	raw := string(content)
+	text := normalizeKnowledgeText(raw)
+	if text == "" {
+		return nil, fmt.Errorf("file %s contains no readable text", fileName)
+	}
+
+	markdown := buildKnowledgeMarkdownFromText(defaultKnowledgeTitle(fileName), text)
+	if preserveMarkdown {
+		normalizedMarkdown := normalizeKnowledgeMarkdown(raw)
+		if normalizedMarkdown != "" {
+			markdown = normalizedMarkdown
+		}
+	}
+
+	return &knowledgeParsedContent{
+		Text:     text,
+		Markdown: markdown,
+	}, nil
+}
+
+func extractHTMLText(content []byte) (string, error) {
 	doc, err := html.Parse(bytes.NewReader(content))
 	if err != nil {
 		return "", fmt.Errorf("parse html: %w", err)
@@ -106,32 +143,11 @@ func extractKnowledgeTextFromSnapshot(fileName string, content []byte) (string,
 	}
 	walk(doc, false)
 
-	return normalizeKnowledgeText(strings.Join(parts, "\n")), nil
-}
-
-func extractKnowledgeTextFromFile(fileName string, content []byte) (string, error) {
-	ext := strings.ToLower(filepath.Ext(fileName))
-
-	switch ext {
-	case ".txt", ".md", ".markdown", ".json", ".csv", ".html", ".htm":
-		if !utf8.Valid(content) {
-			return "", fmt.Errorf("file %s is not valid UTF-8 text", fileName)
-		}
-		return normalizeKnowledgeText(string(content)), nil
-	case ".docx":
-		return extractDOCXText(content)
-	case ".xlsx":
-		return extractXLSXText(content)
-	case ".pdf":
-		return extractPDFText(content)
-	case ".doc", ".xls":
-		return "", fmt.Errorf("file type %s is not supported yet", ext)
-	default:
-		if utf8.Valid(content) {
-			return normalizeKnowledgeText(string(content)), nil
-		}
-		return "", fmt.Errorf("unsupported file type %s", ext)
+	result := normalizeKnowledgeText(strings.Join(parts, "\n"))
+	if result == "" {
+		return "", fmt.Errorf("html contains no readable text")
 	}
+	return result, nil
 }
 
 func extractDOCXText(content []byte) (string, error) {
@@ -173,46 +189,177 @@ func extractDOCXText(content []byte) (string, error) {
 			}
 		}
 	}
+
 	if len(parts) == 0 {
 		return "", fmt.Errorf("docx contains no readable text")
 	}
 	return normalizeKnowledgeText(strings.Join(parts, "\n")), nil
 }
 
-func extractXLSXText(content []byte) (string, error) {
+func extractXLSXContent(content []byte) (*knowledgeParsedContent, error) {
 	book, err := excelize.OpenReader(bytes.NewReader(content))
 	if err != nil {
-		return "", fmt.Errorf("open xlsx: %w", err)
+		return nil, fmt.Errorf("open xlsx: %w", err)
 	}
 	defer func() { _ = book.Close() }()
 
-	lines := make([]string, 0)
+	markdownSections := make([]string, 0)
+	textSections := make([]string, 0)
 	for _, sheet := range book.GetSheetList() {
 		rows, err := book.GetRows(sheet)
 		if err != nil {
-			return "", fmt.Errorf("read xlsx rows: %w", err)
+			return nil, fmt.Errorf("read xlsx rows: %w", err)
 		}
-		if len(rows) == 0 {
+
+		cleaned := normalizeSpreadsheetRows(rows)
+		if len(cleaned) == 0 {
 			continue
 		}
-		lines = append(lines, sheet)
-		for _, row := range rows {
-			cells := make([]string, 0, len(row))
-			for _, cell := range row {
-				cell = strings.TrimSpace(cell)
-				if cell != "" {
-					cells = append(cells, cell)
-				}
-			}
-			if len(cells) > 0 {
-				lines = append(lines, strings.Join(cells, " | "))
-			}
+
+		markdownSections = append(markdownSections, buildKnowledgeSheetMarkdown(sheet, cleaned))
+		textSections = append(textSections, buildKnowledgeSheetPlainText(sheet, cleaned))
+	}
+
+	if len(markdownSections) == 0 {
+		return nil, fmt.Errorf("xlsx contains no readable text")
+	}
+
+	return &knowledgeParsedContent{
+		Text:     normalizeKnowledgeText(strings.Join(textSections, "\n\n")),
+		Markdown: strings.TrimSpace(strings.Join(markdownSections, "\n\n")),
+	}, nil
+}
+
+func extractXLSContent(content []byte) (*knowledgeParsedContent, error) {
+	book, err := xls.OpenReader(bytes.NewReader(content), "utf-8")
+	if err != nil {
+		return nil, fmt.Errorf("open xls: %w", err)
+	}
+
+	// The upstream xls library panics for sparse rows when calling sheet.Row(i),
+	// so we rely on its safer bulk reader here instead of random row access.
+	rows := book.ReadAllCells(maxKnowledgeSpreadsheetRows)
+	cleaned := normalizeSpreadsheetRows(rows)
+	if len(cleaned) == 0 {
+		return nil, fmt.Errorf("xls contains no readable text")
+	}
+
+	title := "XLS 工作簿"
+	if book.NumSheets() == 1 {
+		if sheet := book.GetSheet(0); sheet != nil && strings.TrimSpace(sheet.Name) != "" {
+			title = strings.TrimSpace(sheet.Name)
 		}
 	}
-	if len(lines) == 0 {
-		return "", fmt.Errorf("xlsx contains no readable text")
+
+	return &knowledgeParsedContent{
+		Text:     normalizeKnowledgeText(buildKnowledgeSheetPlainText(title, cleaned)),
+		Markdown: buildKnowledgeSheetMarkdown(title, cleaned),
+	}, nil
+}
+
+func normalizeSpreadsheetRows(rows [][]string) [][]string {
+	cleaned := make([][]string, 0, len(rows))
+	for _, row := range rows {
+		cells := make([]string, len(row))
+		copy(cells, row)
+
+		for index := range cells {
+			cells[index] = normalizeKnowledgeInlineText(cells[index])
+		}
+
+		lastNonEmpty := -1
+		for index, cell := range cells {
+			if cell != "" {
+				lastNonEmpty = index
+			}
+		}
+		if lastNonEmpty < 0 {
+			continue
+		}
+
+		cleaned = append(cleaned, cells[:lastNonEmpty+1])
 	}
-	return normalizeKnowledgeText(strings.Join(lines, "\n")), nil
+	return cleaned
+}
+
+func buildKnowledgeSheetPlainText(sheetName string, rows [][]string) string {
+	lines := make([]string, 0, len(rows)+1)
+	if strings.TrimSpace(sheetName) != "" {
+		lines = append(lines, strings.TrimSpace(sheetName))
+	}
+	for _, row := range rows {
+		lines = append(lines, strings.Join(row, " | "))
+	}
+	return strings.Join(lines, "\n")
+}
+
+func buildKnowledgeSheetMarkdown(sheetName string, rows [][]string) string {
+	if len(rows) == 0 {
+		return ""
+	}
+
+	width := 0
+	for _, row := range rows {
+		if len(row) > width {
+			width = len(row)
+		}
+	}
+	if width == 0 {
+		return ""
+	}
+
+	normalizedRows := make([][]string, 0, len(rows))
+	for _, row := range rows {
+		normalizedRows = append(normalizedRows, padKnowledgeRow(row, width))
+	}
+
+	var builder strings.Builder
+	if title := strings.TrimSpace(sheetName); title != "" {
+		builder.WriteString("## ")
+		builder.WriteString(title)
+		builder.WriteString("\n\n")
+	}
+
+	writeKnowledgeMarkdownTableRow(&builder, normalizedRows[0])
+	writeKnowledgeMarkdownTableSeparator(&builder, width)
+	for _, row := range normalizedRows[1:] {
+		writeKnowledgeMarkdownTableRow(&builder, row)
+	}
+
+	return strings.TrimSpace(builder.String())
+}
+
+func padKnowledgeRow(row []string, width int) []string {
+	result := make([]string, width)
+	copy(result, row)
+	return result
+}
+
+func writeKnowledgeMarkdownTableRow(builder *strings.Builder, row []string) {
+	builder.WriteString("|")
+	for _, cell := range row {
+		builder.WriteString(" ")
+		builder.WriteString(escapeKnowledgeMarkdownTableCell(cell))
+		builder.WriteString(" |")
+	}
+	builder.WriteString("\n")
+}
+
+func writeKnowledgeMarkdownTableSeparator(builder *strings.Builder, width int) {
+	builder.WriteString("|")
+	for index := 0; index < width; index++ {
+		builder.WriteString(" --- |")
+	}
+	builder.WriteString("\n")
+}
+
+func escapeKnowledgeMarkdownTableCell(cell string) string {
+	cell = strings.TrimSpace(cell)
+	cell = strings.ReplaceAll(cell, "\\", "\\\\")
+	cell = strings.ReplaceAll(cell, "|", "\\|")
+	cell = strings.ReplaceAll(cell, "\r", " ")
+	cell = strings.ReplaceAll(cell, "\n", " / ")
+	return cell
 }
 
 func extractPDFText(content []byte) (string, error) {
@@ -223,7 +370,7 @@ func extractPDFText(content []byte) (string, error) {
 
 	var builder strings.Builder
 	totalPage := reader.NumPage()
-	for pageIndex := 1; pageIndex <= totalPage; pageIndex += 1 {
+	for pageIndex := 1; pageIndex <= totalPage; pageIndex++ {
 		page := reader.Page(pageIndex)
 		if page.V.IsNull() {
 			continue
@@ -245,12 +392,116 @@ func extractPDFText(content []byte) (string, error) {
 	return result, nil
 }
 
-func normalizeKnowledgeText(input string) string {
+func defaultKnowledgeTitle(fileName string) string {
+	base := filepath.Base(strings.TrimSpace(fileName))
+	if base == "" {
+		return ""
+	}
+	return strings.TrimSuffix(base, filepath.Ext(base))
+}
+
+func buildKnowledgeMarkdownFromText(title, text string) string {
+	text = normalizeKnowledgeText(text)
+	title = strings.TrimSpace(title)
+	if text == "" {
+		if title == "" {
+			return ""
+		}
+		return "# " + title
+	}
+
+	lines := strings.Split(text, "\n")
+	if title != "" && len(lines) > 0 && strings.EqualFold(strings.TrimSpace(lines[0]), title) {
+		lines = lines[1:]
+	}
+
+	paragraphs := make([]string, 0, len(lines))
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line != "" {
+			paragraphs = append(paragraphs, line)
+		}
+	}
+
+	sections := make([]string, 0, 2)
+	if title != "" {
+		sections = append(sections, "# "+title)
+	}
+	if len(paragraphs) > 0 {
+		sections = append(sections, strings.Join(paragraphs, "\n\n"))
+	}
+	return strings.TrimSpace(strings.Join(sections, "\n\n"))
+}
+
+func extractKnowledgeMarkdownTitle(markdown string) string {
+	markdown = normalizeKnowledgeMarkdown(markdown)
+	if markdown == "" {
+		return ""
+	}
+
+	for _, line := range strings.Split(markdown, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || !strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		level := 0
+		for level < len(line) && line[level] == '#' {
+			level++
+		}
+		if level == len(line) || line[level] != ' ' {
+			continue
+		}
+
+		title := strings.Join(strings.Fields(strings.TrimSpace(line[level+1:])), " ")
+		title = strings.Trim(title, "# \t")
+		if title == "" || genericKnowledgeMarkdownTitleRegex.MatchString(title) {
+			continue
+		}
+		switch title {
+		case "摘要", "概述", "概要", "目录", "正文":
+			continue
+		}
+
+		runes := []rune(title)
+		if len(runes) > 200 {
+			title = string(runes[:200])
+		}
+		return title
+	}
+
+	return ""
+}
+
+func normalizeKnowledgeMarkdown(input string) string {
+	input = strings.ReplaceAll(input, "\r\n", "\n")
 	input = strings.ReplaceAll(input, "\u00a0", " ")
+
 	lines := strings.Split(input, "\n")
 	cleaned := make([]string, 0, len(lines))
+	lastBlank := false
 	for _, line := range lines {
-		line = whitespacePattern.ReplaceAllString(strings.TrimSpace(line), " ")
+		line = strings.TrimRight(line, " \t")
+		if strings.TrimSpace(line) == "" {
+			if lastBlank {
+				continue
+			}
+			lastBlank = true
+			cleaned = append(cleaned, "")
+			continue
+		}
+		lastBlank = false
+		cleaned = append(cleaned, line)
+	}
+	return strings.TrimSpace(strings.Join(cleaned, "\n"))
+}
+
+func normalizeKnowledgeText(input string) string {
+	input = strings.ReplaceAll(input, "\u00a0", " ")
+	lines := strings.Split(strings.ReplaceAll(input, "\r\n", "\n"), "\n")
+	cleaned := make([]string, 0, len(lines))
+	for _, line := range lines {
+		line = normalizeKnowledgeInlineText(line)
 		if line != "" {
 			cleaned = append(cleaned, line)
 		}
@@ -258,58 +509,6 @@ func normalizeKnowledgeText(input string) string {
 	return strings.TrimSpace(strings.Join(cleaned, "\n"))
 }
 
-func splitKnowledgeTextIntoChunks(text string, chunkSize, chunkOverlap int) []string {
-	text = normalizeKnowledgeText(text)
-	if text == "" {
-		return nil
-	}
-
-	runes := []rune(text)
-	if chunkSize <= 0 {
-		chunkSize = 900
-	}
-	if chunkOverlap < 0 {
-		chunkOverlap = 0
-	}
-	if chunkOverlap >= chunkSize {
-		chunkOverlap = chunkSize / 4
-	}
-
-	chunks := make([]string, 0)
-	for start := 0; start < len(runes); {
-		end := start + chunkSize
-		if end > len(runes) {
-			end = len(runes)
-		}
-
-		if end < len(runes) {
-			for offset := end; offset > start+chunkSize/2; offset -= 1 {
-				if runes[offset-1] == '\n' || runes[offset-1] == '。' || runes[offset-1] == '.' {
-					end = offset
-					break
-				}
-			}
-		}
-
-		chunk := strings.TrimSpace(string(runes[start:end]))
-		if chunk != "" {
-			chunks = append(chunks, chunk)
-		}
-		if end >= len(runes) {
-			break
-		}
-		start = end - chunkOverlap
-		if start < 0 {
-			start = 0
-		}
-	}
-
-	return chunks
-}
-
-func estimateKnowledgeTokenCount(text string) int {
-	if text == "" {
-		return 0
-	}
-	return utf8.RuneCountInString(text)
+func normalizeKnowledgeInlineText(input string) string {
+	return whitespacePattern.ReplaceAllString(strings.TrimSpace(input), " ")
 }

server/internal/tenant/app/knowledge_service.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/http"
 	"net/url"
 	"path/filepath"
+	"runtime/debug"
 	"slices"
 	"strings"
 	"sync"
@@ -18,6 +20,7 @@ import (
 
 	"github.com/geo-platform/tenant-api/internal/shared/auth"
 	"github.com/geo-platform/tenant-api/internal/shared/config"
+	"github.com/geo-platform/tenant-api/internal/shared/llm"
 	"github.com/geo-platform/tenant-api/internal/shared/objectstorage"
 	"github.com/geo-platform/tenant-api/internal/shared/response"
 	"github.com/geo-platform/tenant-api/internal/shared/retrieval"
@@ -25,8 +28,9 @@ import (
 )
 
 const (
-	defaultKnowledgeQueueSize   = 128
-	defaultKnowledgeCleanupPoll = 15 * time.Second
+	defaultKnowledgeQueueSize      = 128
+	defaultKnowledgeCleanupPoll    = 15 * time.Second
+	defaultKnowledgeChildGroupName = "默认目录"
 )
 
 type KnowledgeService struct {
@@ -34,12 +38,17 @@ type KnowledgeService struct {
 	provider           retrieval.Provider
 	store              retrieval.VectorStore
 	objectStorage      objectstorage.Client
+	llmClient          llm.Client
 	logger             *zap.Logger
 	chunkSize          int
 	chunkOverlap       int
 	embeddingBatchSize int
 	recallLimit        int
 	rerankTopN         int
+	arkBaseURL         string
+	arkAPIKey          string
+	urlMarkdownModel   string
+	httpClient         *http.Client
 	jobs               chan knowledgeParseJob
 	cleanupMu          sync.Mutex
 	cleanupInFlight    map[string]struct{}
@@ -66,6 +75,7 @@ type knowledgeItemRecord struct {
 	SourceURI   *string
 	StorageKey  string
 	ContentText *string
+	Markdown    *string
 	ItemVersion int
 }
 
@@ -99,6 +109,21 @@ type KnowledgeItemResponse struct {
 	UpdatedAt    string  `json:"updated_at"`
 }
 
+type KnowledgeItemDetailResponse struct {
+	ID              int64   `json:"id"`
+	GroupID         int64   `json:"group_id"`
+	GroupName       string  `json:"group_name"`
+	Name            string  `json:"name"`
+	SourceType      string  `json:"source_type"`
+	SourceURI       *string `json:"source_uri"`
+	Status          string  `json:"status"`
+	SizeBytes       int64   `json:"size_bytes"`
+	ErrorMessage    *string `json:"error_message"`
+	MarkdownContent *string `json:"markdown_content"`
+	CreatedAt       string  `json:"created_at"`
+	UpdatedAt       string  `json:"updated_at"`
+}
+
 type KnowledgeTextItemRequest struct {
 	GroupID int64  `json:"group_id" binding:"required"`
 	Name    string `json:"name" binding:"required"`
@@ -132,8 +157,10 @@ func NewKnowledgeService(
 	provider retrieval.Provider,
 	store retrieval.VectorStore,
 	objectStorage objectstorage.Client,
+	llmClient llm.Client,
 	logger *zap.Logger,
 	cfg config.RetrievalConfig,
+	llmCfg config.LLMConfig,
 	queueSize int,
 	workerCount int,
 ) *KnowledgeService {
@@ -163,14 +190,25 @@ func NewKnowledgeService(
 		provider:           provider,
 		store:              store,
 		objectStorage:      objectStorage,
+		llmClient:          llmClient,
 		logger:             logger,
 		chunkSize:          chunkSize,
 		chunkOverlap:       chunkOverlap,
 		embeddingBatchSize: batchSize,
 		recallLimit:        recallLimit,
 		rerankTopN:         rerankTopN,
+		arkBaseURL:         strings.TrimSpace(llmCfg.BaseURL),
+		arkAPIKey:          strings.TrimSpace(llmCfg.APIKey),
+		urlMarkdownModel:   strings.TrimSpace(llmCfg.KnowledgeURLModel),
+		httpClient:         &http.Client{Timeout: defaultKnowledgeURLParseTimeout},
 		cleanupInFlight:    make(map[string]struct{}),
 	}
+	if svc.urlMarkdownModel == "" {
+		svc.urlMarkdownModel = strings.TrimSpace(llmCfg.Model)
+	}
+	if svc.arkBaseURL == "" {
+		svc.arkBaseURL = defaultKnowledgeArkBaseURL
+	}
 
 	if workerCount > 0 {
 		if queueSize <= 0 {
@@ -521,15 +559,58 @@ func (s *KnowledgeService) ListItems(ctx context.Context, groupID *int64) ([]Kno
 	return items, nil
 }
 
+func (s *KnowledgeService) GetItemDetail(ctx context.Context, itemID int64) (*KnowledgeItemDetailResponse, error) {
+	actor := auth.MustActor(ctx)
+
+	var (
+		item                 KnowledgeItemDetailResponse
+		sourceURI            *string
+		errorMessage         *string
+		markdownContent      *string
+		createdAt, updatedAt time.Time
+	)
+
+	err := s.pool.QueryRow(ctx, `
+		SELECT ki.id, ki.group_id, kg.name, ki.name, ki.source_type, ki.source_uri, ki.status,
+		       ki.size_bytes, ki.error_message, ki.markdown_content, ki.created_at, ki.updated_at
+		FROM knowledge_items ki
+		JOIN knowledge_groups kg ON kg.id = ki.group_id AND kg.deleted_at IS NULL
+		WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NULL
+	`, itemID, actor.TenantID).Scan(
+		&item.ID,
+		&item.GroupID,
+		&item.GroupName,
+		&item.Name,
+		&item.SourceType,
+		&sourceURI,
+		&item.Status,
+		&item.SizeBytes,
+		&errorMessage,
+		&markdownContent,
+		&createdAt,
+		&updatedAt,
+	)
+	if err != nil {
+		if errors.Is(err, pgx.ErrNoRows) {
+			return nil, response.ErrNotFound(40450, "knowledge_item_not_found", "knowledge item not found")
+		}
+		return nil, response.ErrInternal(50051, "knowledge_items_query_failed", "failed to load knowledge item")
+	}
+
+	item.SourceURI = sourceURI
+	item.ErrorMessage = errorMessage
+	item.MarkdownContent = markdownContent
+	item.CreatedAt = createdAt.Format(time.RFC3339)
+	item.UpdatedAt = updatedAt.Format(time.RFC3339)
+	return &item, nil
+}
+
 func (s *KnowledgeService) CreateTextItem(ctx context.Context, req KnowledgeTextItemRequest) (*KnowledgeItemResponse, error) {
 	actor := auth.MustActor(ctx)
 
 	if err := s.validateIngestionDependencies("text"); err != nil {
 		return nil, err
 	}
-	if err := s.ensureStorableGroup(ctx, actor.TenantID, req.GroupID); err != nil {
-		return nil, err
-	}
 
 	content := normalizeKnowledgeText(req.Content)
 	if content == "" {
@@ -551,33 +632,17 @@ func (s *KnowledgeService) CreateURLItem(ctx context.Context, req KnowledgeURLIt
 	if err := s.validateIngestionDependencies("website"); err != nil {
 		return nil, err
 	}
-	if err := s.ensureStorableGroup(ctx, actor.TenantID, req.GroupID); err != nil {
-		return nil, err
-	}
 
 	rawURL := strings.TrimSpace(req.URL)
-	parsed, err := url.ParseRequestURI(rawURL)
-	if err != nil {
+	if _, err := url.ParseRequestURI(rawURL); err != nil {
 		return nil, response.ErrBadRequest(40052, "knowledge_url_invalid", "knowledge url is invalid")
 	}
 
-	snapshot, contentType, err := fetchKnowledgeURLSnapshot(ctx, parsed.String())
-	if err != nil {
-		return nil, response.ErrBadRequest(40052, "knowledge_url_fetch_failed", err.Error())
-	}
-
-	objectKey := buildKnowledgeObjectKey(actor.TenantID, "website", resolveSnapshotFileName(parsed.String(), contentType))
-	if err := s.objectStorage.PutBytes(ctx, objectKey, snapshot, contentType); err != nil {
-		return nil, response.ErrServiceUnavailable(50357, "knowledge_storage_unavailable", err.Error())
-	}
-
 	return s.queueKnowledgeItem(ctx, actor.TenantID, knowledgeIngestInput{
 		GroupID:    req.GroupID,
 		Name:       strings.TrimSpace(req.Name),
 		SourceType: "website",
 		SourceURI:  &rawURL,
-		StorageKey: objectKey,
-		SizeBytes:  int64(len(snapshot)),
 	})
 }
 
@@ -593,9 +658,6 @@ func (s *KnowledgeService) CreateFileItem(
 	if err := s.validateIngestionDependencies("document"); err != nil {
 		return nil, err
 	}
-	if err := s.ensureStorableGroup(ctx, actor.TenantID, groupID); err != nil {
-		return nil, err
-	}
 
 	displayName := strings.TrimSpace(name)
 	if displayName == "" {
@@ -819,11 +881,19 @@ func (s *KnowledgeService) validateIngestionDependencies(sourceType string) erro
 	if err := s.store.Validate(); err != nil {
 		return response.ErrServiceUnavailable(50352, "qdrant_unavailable", err.Error())
 	}
-	if sourceType == "document" || sourceType == "website" {
+	if sourceType == "document" {
 		if err := s.objectStorage.Validate(); err != nil {
 			return response.ErrServiceUnavailable(50357, "knowledge_storage_unavailable", err.Error())
 		}
 	}
+	if sourceType == "website" {
+		if strings.TrimSpace(s.arkAPIKey) == "" {
+			return response.ErrServiceUnavailable(50358, "knowledge_webpage_parser_unavailable", "ark api key is not configured")
+		}
+		if err := s.llmClient.Validate(); err != nil {
+			return response.ErrServiceUnavailable(50359, "knowledge_markdown_formatter_unavailable", err.Error())
+		}
+	}
 	return nil
 }
 
@@ -832,9 +902,11 @@ func (s *KnowledgeService) queueKnowledgeItem(
 	tenantID int64,
 	input knowledgeIngestInput,
 ) (*KnowledgeItemResponse, error) {
-	if err := s.ensureStorableGroup(ctx, tenantID, input.GroupID); err != nil {
+	resolvedGroupID, err := s.resolveKnowledgeStorageGroupID(ctx, tenantID, input.GroupID)
+	if err != nil {
 		return nil, err
 	}
+	input.GroupID = resolvedGroupID
 
 	input.Name = strings.TrimSpace(input.Name)
 	if input.Name == "" {
@@ -888,7 +960,25 @@ func (s *KnowledgeService) enqueueParseJob(job knowledgeParseJob) error {
 
 func (s *KnowledgeService) runParseWorker() {
 	for job := range s.jobs {
-		s.executeParseJob(context.Background(), job)
+		func(job knowledgeParseJob) {
+			defer func() {
+				if recovered := recover(); recovered != nil {
+					err := fmt.Errorf("knowledge parse panic: %v", recovered)
+					if s.logger != nil {
+						s.logger.Error("knowledge parse worker panicked",
+							zap.Int64("tenant_id", job.TenantID),
+							zap.Int64("knowledge_item_id", job.ItemID),
+							zap.Int64("parse_task_id", job.ParseTaskID),
+							zap.Any("panic", recovered),
+							zap.ByteString("stack", debug.Stack()),
+						)
+					}
+					_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
+				}
+			}()
+
+			s.executeParseJob(context.Background(), job)
+		}(job)
 	}
 }
 
@@ -936,13 +1026,18 @@ func (s *KnowledgeService) executeParseJob(ctx context.Context, job knowledgePar
 		return
 	}
 
-	content, err := s.readKnowledgeItemContent(ctx, item)
+	parsedContent, err := s.readKnowledgeItemParsedContent(ctx, item)
 	if err != nil {
 		_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
 		return
 	}
+	if item.SourceType == "website" {
+		if resolvedName := extractKnowledgeMarkdownTitle(parsedContent.Markdown); resolvedName != "" {
+			item.Name = resolvedName
+		}
+	}
 
-	chunks := splitKnowledgeTextIntoChunks(content, s.chunkSize, s.chunkOverlap)
+	chunks := splitKnowledgeTextIntoChunks(parsedContent.Text, s.chunkSize, s.chunkOverlap)
 	if len(chunks) == 0 {
 		err = errors.New("knowledge content is empty after parsing")
 		_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
@@ -1018,7 +1113,7 @@ func (s *KnowledgeService) executeParseJob(ctx context.Context, job knowledgePar
 		return
 	}
 
-	if err = s.markKnowledgeIngestionCompleted(ctx, job.TenantID, job.ItemID, job.ParseTaskID, metas); err != nil {
+	if err = s.markKnowledgeIngestionCompleted(ctx, job.TenantID, job.ItemID, job.ParseTaskID, item.Name, parsedContent, metas); err != nil {
 		_ = s.markKnowledgeIngestionFailed(context.Background(), job.TenantID, job.ItemID, job.ParseTaskID, err)
 		return
 	}
@@ -1030,9 +1125,10 @@ func (s *KnowledgeService) loadKnowledgeItemForParse(
 ) (*knowledgeItemRecord, error) {
 	record := &knowledgeItemRecord{}
 	var contentText *string
+	var markdownContent *string
 	err := s.pool.QueryRow(ctx, `
 		SELECT ki.id, ki.tenant_id, ki.group_id, kg.name, ki.name, ki.source_type,
-		       ki.source_uri, ki.storage_key, ki.content_text, ki.item_version
+		       ki.source_uri, ki.storage_key, ki.content_text, ki.markdown_content, ki.item_version
 		FROM knowledge_items ki
 		JOIN knowledge_groups kg ON kg.id = ki.group_id
 		WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NULL
@@ -1046,37 +1142,38 @@ func (s *KnowledgeService) loadKnowledgeItemForParse(
 		&record.SourceURI,
 		&record.StorageKey,
 		&contentText,
+		&markdownContent,
 		&record.ItemVersion,
 	)
 	if err != nil {
 		return nil, nil
 	}
 	record.ContentText = contentText
+	record.Markdown = markdownContent
 	return record, nil
 }
 
-func (s *KnowledgeService) readKnowledgeItemContent(ctx context.Context, item *knowledgeItemRecord) (string, error) {
+func (s *KnowledgeService) readKnowledgeItemParsedContent(ctx context.Context, item *knowledgeItemRecord) (*knowledgeParsedContent, error) {
 	if item == nil {
-		return "", fmt.Errorf("knowledge item is empty")
+		return nil, fmt.Errorf("knowledge item is empty")
 	}
 
 	switch item.SourceType {
 	case "text":
-		return normalizeKnowledgeText(derefKnowledgeString(item.ContentText)), nil
+		return parseKnowledgeTextInput(item.Name, derefKnowledgeString(item.ContentText)), nil
 	case "document":
 		data, err := s.objectStorage.GetBytes(ctx, item.StorageKey)
 		if err != nil {
-			return "", err
+			return nil, err
 		}
-		return extractKnowledgeTextFromFile(item.StorageKey, data)
+		return extractKnowledgeContentFromFile(item.StorageKey, data)
 	case "website":
-		data, err := s.objectStorage.GetBytes(ctx, item.StorageKey)
-		if err != nil {
-			return "", err
+		if item.SourceURI == nil || strings.TrimSpace(*item.SourceURI) == "" {
+			return nil, fmt.Errorf("knowledge website url is empty")
 		}
-		return extractKnowledgeTextFromSnapshot(item.StorageKey, data)
+		return s.parseWebsiteKnowledge(ctx, strings.TrimSpace(*item.SourceURI))
 	default:
-		return "", fmt.Errorf("unsupported knowledge source type %q", item.SourceType)
+		return nil, fmt.Errorf("unsupported knowledge source type %q", item.SourceType)
 	}
 }
 
@@ -1103,9 +1200,9 @@ func (s *KnowledgeService) createPendingKnowledgeItem(
 	err = tx.QueryRow(ctx, `
 		INSERT INTO knowledge_items (
 			tenant_id, group_id, source_type, name, source_uri, storage_key,
-			content_text, status, size_bytes, item_version
+			content_text, markdown_content, status, size_bytes, item_version
 		)
-		VALUES ($1, $2, $3, $4, $5, $6, $7, 'pending', $8, 1)
+		VALUES ($1, $2, $3, $4, $5, $6, $7, NULL, 'pending', $8, 1)
 		RETURNING id
 	`, tenantID, input.GroupID, input.SourceType, input.Name, input.SourceURI, input.StorageKey, input.ContentText, input.SizeBytes).Scan(&itemID)
 	if err != nil {
@@ -1159,8 +1256,21 @@ func (s *KnowledgeService) markKnowledgeIngestionRunning(
 func (s *KnowledgeService) markKnowledgeIngestionCompleted(
 	ctx context.Context,
 	tenantID, itemID, parseTaskID int64,
+	itemName string,
+	content *knowledgeParsedContent,
 	metas []knowledgeChunkMetaInput,
 ) error {
+	if content == nil {
+		return errors.New("knowledge content is empty")
+	}
+	if strings.TrimSpace(content.Text) == "" {
+		return errors.New("knowledge text content is empty")
+	}
+	if strings.TrimSpace(content.Markdown) == "" {
+		content.Markdown = buildKnowledgeMarkdownFromText("", content.Text)
+	}
+	itemName = strings.TrimSpace(itemName)
+
 	tx, err := s.pool.Begin(ctx)
 	if err != nil {
 		return err
@@ -1188,9 +1298,14 @@ func (s *KnowledgeService) markKnowledgeIngestionCompleted(
 
 	if _, err := tx.Exec(ctx, `
 		UPDATE knowledge_items
-		SET status = 'completed', error_message = NULL, updated_at = NOW()
+		SET status = 'completed',
+		    error_message = NULL,
+		    name = CASE WHEN $3 <> '' THEN $3 ELSE name END,
+		    content_text = $4,
+		    markdown_content = $5,
+		    updated_at = NOW()
 		WHERE id = $1 AND tenant_id = $2
-	`, itemID, tenantID); err != nil {
+	`, itemID, tenantID, itemName, content.Text, content.Markdown); err != nil {
 		return err
 	}
 
@@ -1370,6 +1485,99 @@ func (s *KnowledgeService) ensureStorableGroup(ctx context.Context, tenantID, gr
 	return nil
 }
 
+func (s *KnowledgeService) resolveKnowledgeStorageGroupID(ctx context.Context, tenantID, groupID int64) (int64, error) {
+	tx, err := s.pool.Begin(ctx)
+	if err != nil {
+		return 0, response.ErrInternal(50055, "knowledge_group_resolve_failed", "failed to begin knowledge group transaction")
+	}
+	defer func() { _ = tx.Rollback(ctx) }()
+
+	resolvedGroupID, err := s.resolveKnowledgeStorageGroupIDTx(ctx, tx, tenantID, groupID)
+	if err != nil {
+		return 0, err
+	}
+
+	if err := tx.Commit(ctx); err != nil {
+		return 0, response.ErrInternal(50055, "knowledge_group_resolve_failed", "failed to commit knowledge group transaction")
+	}
+	return resolvedGroupID, nil
+}
+
+func (s *KnowledgeService) resolveKnowledgeStorageGroupIDTx(
+	ctx context.Context,
+	tx pgx.Tx,
+	tenantID, groupID int64,
+) (int64, error) {
+	var parentID *int64
+	if err := tx.QueryRow(ctx, `
+		SELECT parent_id
+		FROM knowledge_groups
+		WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL
+		FOR UPDATE
+	`, groupID, tenantID).Scan(&parentID); err != nil {
+		if errors.Is(err, pgx.ErrNoRows) {
+			return 0, response.ErrNotFound(40451, "knowledge_group_not_found", "knowledge group not found")
+		}
+		return 0, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge group")
+	}
+
+	if parentID != nil {
+		var grandParentID *int64
+		if err := tx.QueryRow(ctx, `
+			SELECT parent_id
+			FROM knowledge_groups
+			WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL
+		`, *parentID, tenantID).Scan(&grandParentID); err != nil {
+			if errors.Is(err, pgx.ErrNoRows) {
+				return 0, response.ErrNotFound(40451, "knowledge_group_not_found", "knowledge group not found")
+			}
+			return 0, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge group")
+		}
+		if grandParentID != nil {
+			return 0, response.ErrBadRequest(40053, "knowledge_group_storage_invalid", "knowledge items can only be stored in second-level groups")
+		}
+		return groupID, nil
+	}
+
+	var childGroupID int64
+	if err := tx.QueryRow(ctx, `
+		SELECT id
+		FROM knowledge_groups
+		WHERE tenant_id = $1
+		  AND parent_id = $2
+		  AND name = $3
+		  AND deleted_at IS NULL
+		ORDER BY id
+		LIMIT 1
+	`, tenantID, groupID, defaultKnowledgeChildGroupName).Scan(&childGroupID); err == nil {
+		return childGroupID, nil
+	} else if !errors.Is(err, pgx.ErrNoRows) {
+		return 0, response.ErrInternal(50050, "knowledge_group_query_failed", "failed to query knowledge group")
+	}
+
+	if err := tx.QueryRow(ctx, `
+		INSERT INTO knowledge_groups (tenant_id, name, parent_id, sort_order)
+		VALUES ($1, $2, $3, 0)
+		RETURNING id
+	`, tenantID, defaultKnowledgeChildGroupName, groupID).Scan(&childGroupID); err == nil {
+		return childGroupID, nil
+	}
+
+	if err := tx.QueryRow(ctx, `
+		SELECT id
+		FROM knowledge_groups
+		WHERE tenant_id = $1
+		  AND parent_id = $2
+		  AND name = $3
+		  AND deleted_at IS NULL
+		ORDER BY id
+		LIMIT 1
+	`, tenantID, groupID, defaultKnowledgeChildGroupName).Scan(&childGroupID); err != nil {
+		return 0, response.ErrConflict(40950, "knowledge_group_create_failed", "failed to create default knowledge group")
+	}
+	return childGroupID, nil
+}
+
 func (s *KnowledgeService) listGroupSubtreeIDs(
 	ctx context.Context,
 	tx pgx.Tx,
@@ -1688,9 +1896,10 @@ func (s *KnowledgeService) loadKnowledgeItemForCleanup(
 ) (*knowledgeItemRecord, error) {
 	record := &knowledgeItemRecord{}
 	var contentText *string
+	var markdownContent *string
 	err := s.pool.QueryRow(ctx, `
 		SELECT ki.id, ki.tenant_id, ki.group_id, kg.name, ki.name, ki.source_type,
-		       ki.source_uri, ki.storage_key, ki.content_text, ki.item_version
+		       ki.source_uri, ki.storage_key, ki.content_text, ki.markdown_content, ki.item_version
 		FROM knowledge_items ki
 		JOIN knowledge_groups kg ON kg.id = ki.group_id
 		WHERE ki.id = $1 AND ki.tenant_id = $2 AND ki.deleted_at IS NOT NULL
@@ -1704,12 +1913,14 @@ func (s *KnowledgeService) loadKnowledgeItemForCleanup(
 		&record.SourceURI,
 		&record.StorageKey,
 		&contentText,
+		&markdownContent,
 		&record.ItemVersion,
 	)
 	if err != nil {
 		return nil, nil
 	}
 	record.ContentText = contentText
+	record.Markdown = markdownContent
 	return record, nil
 }
 

server/internal/tenant/app/knowledge_url_parser.go

@@ -0,0 +1,266 @@
+package app
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/geo-platform/tenant-api/internal/shared/llm"
+	"github.com/geo-platform/tenant-api/internal/tenant/prompts"
+	"go.uber.org/zap"
+)
+
+const (
+	defaultKnowledgeArkBaseURL             = "https://ark.cn-beijing.volces.com/api/v3"
+	defaultKnowledgeURLParseTimeout        = 60 * time.Second
+	defaultKnowledgeURLMarkdownTimeout     = 90 * time.Second
+	defaultKnowledgeURLMarkdownChunkChars  = 12000
+	defaultKnowledgeURLMarkdownOutputToken = 12000
+)
+
+type arkToolExecuteRequest struct {
+	ActionName string         `json:"action_name"`
+	ToolName   string         `json:"tool_name"`
+	Timeout    int            `json:"timeout,omitempty"`
+	Parameters map[string]any `json:"parameters,omitempty"`
+}
+
+type arkToolExecuteResponse struct {
+	StatusCode int                `json:"status_code"`
+	Data       arkToolExecuteData `json:"data"`
+}
+
+type arkToolExecuteData struct {
+	ArkWebDataList []arkWebDataItem `json:"ark_web_data_list"`
+}
+
+type arkWebDataItem struct {
+	URL          string `json:"url"`
+	Title        string `json:"title"`
+	Content      string `json:"content"`
+	ErrorMessage string `json:"error_message"`
+	ErrorCode    string `json:"error_code"`
+}
+
+func (s *KnowledgeService) parseWebsiteKnowledge(ctx context.Context, rawURL string) (*knowledgeParsedContent, error) {
+	item, err := s.executeLinkReader(ctx, rawURL)
+	if err != nil {
+		return nil, err
+	}
+
+	text := normalizeKnowledgeText(item.Content)
+	if text == "" {
+		return nil, fmt.Errorf("webpage parser returned empty content")
+	}
+
+	title := strings.TrimSpace(item.Title)
+	markdown, err := s.formatWebsiteMarkdown(ctx, title, text, rawURL)
+	if err != nil {
+		if s.logger != nil {
+			s.logger.Warn("knowledge website markdown formatting failed, fallback to local markdown",
+				zap.String("url", rawURL),
+				zap.Error(err),
+			)
+		}
+		markdown = buildKnowledgeMarkdownFromText(title, text)
+	}
+
+	return &knowledgeParsedContent{
+		Text:     text,
+		Markdown: markdown,
+	}, nil
+}
+
+func (s *KnowledgeService) executeLinkReader(ctx context.Context, rawURL string) (*arkWebDataItem, error) {
+	baseURL := strings.TrimRight(strings.TrimSpace(s.arkBaseURL), "/")
+	if baseURL == "" {
+		baseURL = defaultKnowledgeArkBaseURL
+	}
+
+	timeout := defaultKnowledgeURLParseTimeout
+	callCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	body, err := json.Marshal(arkToolExecuteRequest{
+		ActionName: "LinkReader",
+		ToolName:   "LinkReader",
+		Timeout:    int(timeout / time.Second),
+		Parameters: map[string]any{
+			"url_list": []string{rawURL},
+		},
+	})
+	if err != nil {
+		return nil, fmt.Errorf("marshal webpage parser request: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(callCtx, http.MethodPost, baseURL+"/tools/execute", bytes.NewReader(body))
+	if err != nil {
+		return nil, fmt.Errorf("create webpage parser request: %w", err)
+	}
+	req.Header.Set("Authorization", "Bearer "+s.arkAPIKey)
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := s.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("execute webpage parser: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= http.StatusBadRequest {
+		data, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+		return nil, fmt.Errorf("webpage parser failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(data)))
+	}
+
+	var parsed arkToolExecuteResponse
+	if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
+		return nil, fmt.Errorf("decode webpage parser response: %w", err)
+	}
+	if parsed.StatusCode != 0 && parsed.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("webpage parser returned status_code=%d", parsed.StatusCode)
+	}
+	if len(parsed.Data.ArkWebDataList) == 0 {
+		return nil, fmt.Errorf("webpage parser returned no content")
+	}
+
+	item := parsed.Data.ArkWebDataList[0]
+	if strings.TrimSpace(item.ErrorCode) != "" || strings.TrimSpace(item.ErrorMessage) != "" {
+		return nil, fmt.Errorf("webpage parser returned error: %s %s", strings.TrimSpace(item.ErrorCode), strings.TrimSpace(item.ErrorMessage))
+	}
+	return &item, nil
+}
+
+func (s *KnowledgeService) formatWebsiteMarkdown(ctx context.Context, title, content, rawURL string) (string, error) {
+	content = strings.TrimSpace(content)
+	if content == "" {
+		return "", fmt.Errorf("website content is empty")
+	}
+
+	chunks := splitKnowledgeMarkdownChunks(content, defaultKnowledgeURLMarkdownChunkChars)
+	if len(chunks) == 0 {
+		return "", fmt.Errorf("website content is empty after chunking")
+	}
+
+	sections := make([]string, 0, len(chunks)+1)
+	for index, chunk := range chunks {
+		result, err := s.llmClient.Generate(ctx, llm.GenerateRequest{
+			Model:           s.urlMarkdownModel,
+			Prompt:          prompts.KnowledgeWebsiteMarkdownPrompt(title, rawURL, chunk, index, len(chunks)),
+			Timeout:         defaultKnowledgeURLMarkdownTimeout,
+			MaxOutputTokens: defaultKnowledgeURLMarkdownOutputToken,
+		}, nil)
+		if err != nil {
+			return "", err
+		}
+
+		section := sanitizeLLMMarkdown(strings.TrimSpace(result.Content))
+		if section == "" {
+			continue
+		}
+		if index > 0 && !strings.HasPrefix(section, "#") {
+			section = fmt.Sprintf("## 第 %d 部分\n\n%s", index+1, section)
+		}
+		sections = append(sections, section)
+	}
+
+	if len(sections) == 0 {
+		return "", fmt.Errorf("website formatter returned empty markdown")
+	}
+
+	markdown := strings.TrimSpace(strings.Join(sections, "\n\n"))
+	if title != "" && !strings.HasPrefix(markdown, "# ") {
+		markdown = "# " + title + "\n\n" + markdown
+	}
+	return markdown, nil
+}
+
+func splitKnowledgeMarkdownChunks(content string, maxChars int) []string {
+	if maxChars <= 0 {
+		maxChars = defaultKnowledgeURLMarkdownChunkChars
+	}
+
+	lines := strings.Split(content, "\n")
+	chunks := make([]string, 0, len(lines))
+	current := make([]string, 0, 16)
+	currentLen := 0
+
+	flush := func() {
+		if len(current) == 0 {
+			return
+		}
+		chunks = append(chunks, strings.TrimSpace(strings.Join(current, "\n")))
+		current = current[:0]
+		currentLen = 0
+	}
+
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+
+		if len([]rune(line)) > maxChars {
+			flush()
+			for _, segment := range splitKnowledgeLongLine(line, maxChars) {
+				if strings.TrimSpace(segment) != "" {
+					chunks = append(chunks, segment)
+				}
+			}
+			continue
+		}
+
+		nextLen := currentLen + len([]rune(line))
+		if len(current) > 0 {
+			nextLen++
+		}
+		if nextLen > maxChars {
+			flush()
+		}
+
+		current = append(current, line)
+		currentLen += len([]rune(line))
+		if len(current) > 1 {
+			currentLen++
+		}
+	}
+
+	flush()
+	return chunks
+}
+
+func splitKnowledgeLongLine(line string, maxChars int) []string {
+	runes := []rune(line)
+	if len(runes) <= maxChars {
+		return []string{line}
+	}
+
+	segments := make([]string, 0, (len(runes)/maxChars)+1)
+	for start := 0; start < len(runes); start += maxChars {
+		end := start + maxChars
+		if end > len(runes) {
+			end = len(runes)
+		}
+		segments = append(segments, strings.TrimSpace(string(runes[start:end])))
+	}
+	return segments
+}
+
+func sanitizeLLMMarkdown(content string) string {
+	content = strings.TrimSpace(content)
+	if !strings.HasPrefix(content, "```") {
+		return content
+	}
+
+	lines := strings.Split(content, "\n")
+	if len(lines) < 3 {
+		return strings.Trim(content, "`")
+	}
+	if !strings.HasPrefix(lines[0], "```") || strings.TrimSpace(lines[len(lines)-1]) != "```" {
+		return content
+	}
+	return strings.TrimSpace(strings.Join(lines[1:len(lines)-1], "\n"))
+}

server/internal/tenant/prompts/runtime.go

@@ -317,3 +317,33 @@ func KnowledgePromptIntroLines() []string {
 func KnowledgeSnippetLabel(index int) string {
 	return fmt.Sprintf("知识片段 %d", index+1)
 }
+
+func KnowledgeWebsiteMarkdownPrompt(title, rawURL, content string, chunkIndex, chunkCount int) string {
+	scope := "以下是网页正文。"
+	if chunkCount > 1 {
+		scope = fmt.Sprintf("以下是网页正文的第 %d/%d 段。", chunkIndex+1, chunkCount)
+	}
+
+	var builder strings.Builder
+	builder.WriteString("你是知识库 Markdown 整理助手。\n")
+	builder.WriteString("任务：把给定网页正文整理为适合知识库只读预览的 Markdown。\n")
+	builder.WriteString("要求：\n")
+	builder.WriteString("1. 只根据原文整理，不要补充事实，不要总结扩写。\n")
+	builder.WriteString("2. 保留标题、列表、表格、编号、联系方式等关键信息。\n")
+	builder.WriteString("3. 如果原文明显是表格，请输出 Markdown 表格。\n")
+	builder.WriteString("4. 不要输出解释，不要输出代码块包裹，只输出 Markdown 正文。\n")
+	if strings.TrimSpace(title) != "" {
+		builder.WriteString("网页标题：")
+		builder.WriteString(title)
+		builder.WriteString("\n")
+	}
+	if strings.TrimSpace(rawURL) != "" {
+		builder.WriteString("网页地址：")
+		builder.WriteString(rawURL)
+		builder.WriteString("\n")
+	}
+	builder.WriteString(scope)
+	builder.WriteString("\n\n")
+	builder.WriteString(content)
+	return builder.String()
+}

server/internal/tenant/transport/article_handler.go

@@ -27,8 +27,10 @@ func NewArticleHandler(a *bootstrap.App) *ArticleHandler {
 		a.RetrievalProvider,
 		a.VectorStore,
 		a.ObjectStorage,
+		a.LLM,
 		a.Logger,
 		a.Config.Retrieval,
+		a.Config.LLM,
 		0,
 		0,
 	)

server/internal/tenant/transport/knowledge_handler.go

@@ -23,8 +23,10 @@ func NewKnowledgeHandler(a *bootstrap.App) *KnowledgeHandler {
 			a.RetrievalProvider,
 			a.VectorStore,
 			a.ObjectStorage,
+			a.LLM,
 			a.Logger,
 			a.Config.Retrieval,
+			a.Config.LLM,
 			a.Config.Generation.QueueSize,
 			a.Config.Generation.WorkerConcurrency,
 		),
@@ -108,6 +110,21 @@ func (h *KnowledgeHandler) ListItems(c *gin.Context) {
 	response.Success(c, data)
 }
 
+func (h *KnowledgeHandler) GetItemDetail(c *gin.Context) {
+	itemID, err := strconv.ParseInt(c.Param("id"), 10, 64)
+	if err != nil {
+		response.Error(c, response.ErrBadRequest(40001, "invalid_id", "knowledge item id must be a number"))
+		return
+	}
+
+	data, err := h.svc.GetItemDetail(c.Request.Context(), itemID)
+	if err != nil {
+		response.Error(c, err)
+		return
+	}
+	response.Success(c, data)
+}
+
 func (h *KnowledgeHandler) CreateTextItem(c *gin.Context) {
 	var req app.KnowledgeTextItemRequest
 	if err := c.ShouldBindJSON(&req); err != nil {

server/internal/tenant/transport/router.go

@@ -90,6 +90,7 @@ func RegisterRoutes(app *bootstrap.App) {
 	knowledge.PUT("/groups/:gid", knowledgeHandler.UpdateGroup)
 	knowledge.DELETE("/groups/:gid", knowledgeHandler.DeleteGroup)
 	knowledge.GET("/items", knowledgeHandler.ListItems)
+	knowledge.GET("/items/:id", knowledgeHandler.GetItemDetail)
 	knowledge.POST("/items/text", knowledgeHandler.CreateTextItem)
 	knowledge.POST("/items/url", knowledgeHandler.CreateURLItem)
 	knowledge.POST("/items/file", knowledgeHandler.CreateFileItem)

server/internal/tenant/transport/template_handler.go

@@ -21,8 +21,10 @@ func NewTemplateHandler(a *bootstrap.App) *TemplateHandler {
 		a.RetrievalProvider,
 		a.VectorStore,
 		a.ObjectStorage,
+		a.LLM,
 		a.Logger,
 		a.Config.Retrieval,
+		a.Config.LLM,
 		0,
 		0,
 	)

server/migrations/20260405173000_add_markdown_content_to_knowledge_items.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE knowledge_items
+    DROP COLUMN IF EXISTS markdown_content;

server/migrations/20260405173000_add_markdown_content_to_knowledge_items.up.sql

@@ -0,0 +1,7 @@
+ALTER TABLE knowledge_items
+    ADD COLUMN markdown_content TEXT;
+
+UPDATE knowledge_items
+SET markdown_content = content_text
+WHERE markdown_content IS NULL
+  AND content_text IS NOT NULL;