feat: add knowledge markdown formatting and detail retrieval

- Implemented KnowledgeWebsiteMarkdownPrompt for formatting webpage content into Markdown.
- Added GetItemDetail method in KnowledgeHandler to retrieve knowledge item details.
- Introduced KnowledgeDetailDrawer component for displaying knowledge item details in the admin web interface.
- Created MarkdownPreview component for rendering Markdown content.
- Added knowledge chunking and URL parsing logic to handle webpage content extraction and formatting.
- Updated database schema to include markdown_content in knowledge_items table.

server/internal/tenant/app/knowledge_chunking.go

@@ -0,0 +1,122 @@
+package app
+
+import "strings"
+
+func splitKnowledgeTextIntoChunks(text string, chunkSize, chunkOverlap int) []string {
+	text = strings.TrimSpace(text)
+	if text == "" {
+		return nil
+	}
+
+	if chunkSize <= 0 {
+		chunkSize = 900
+	}
+	if chunkOverlap < 0 {
+		chunkOverlap = 0
+	}
+	if chunkOverlap >= chunkSize {
+		chunkOverlap = chunkSize / 4
+	}
+
+	lines := strings.Split(text, "\n")
+	chunks := make([]string, 0, len(lines))
+	current := make([]string, 0, 16)
+	currentLen := 0
+
+	resetWithOverlap := func(chunk string) {
+		current = current[:0]
+		currentLen = 0
+		if chunkOverlap <= 0 {
+			return
+		}
+		overlap := takeLastKnowledgeRunes(chunk, chunkOverlap)
+		if overlap == "" {
+			return
+		}
+		current = append(current, overlap)
+		currentLen = len([]rune(overlap))
+	}
+
+	flush := func() {
+		if len(current) == 0 {
+			return
+		}
+		chunk := strings.TrimSpace(strings.Join(current, "\n"))
+		if chunk == "" {
+			current = current[:0]
+			currentLen = 0
+			return
+		}
+		chunks = append(chunks, chunk)
+		resetWithOverlap(chunk)
+	}
+
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+
+		lineRunes := len([]rune(line))
+		if lineRunes > chunkSize {
+			flush()
+			for _, segment := range splitKnowledgeLongLine(line, chunkSize) {
+				if strings.TrimSpace(segment) == "" {
+					continue
+				}
+				chunks = append(chunks, segment)
+				resetWithOverlap(segment)
+			}
+			continue
+		}
+
+		nextLen := currentLen + lineRunes
+		if len(current) > 0 {
+			nextLen++
+		}
+		if nextLen > chunkSize {
+			flush()
+		}
+
+		current = append(current, line)
+		currentLen += lineRunes
+		if len(current) > 1 {
+			currentLen++
+		}
+	}
+
+	if len(current) > 0 {
+		chunk := strings.TrimSpace(strings.Join(current, "\n"))
+		if chunk != "" {
+			chunks = append(chunks, chunk)
+		}
+	}
+
+	return chunks
+}
+
+func takeLastKnowledgeRunes(text string, size int) string {
+	if size <= 0 {
+		return ""
+	}
+
+	runes := []rune(strings.TrimSpace(text))
+	if len(runes) <= size {
+		return string(runes)
+	}
+	return strings.TrimSpace(string(runes[len(runes)-size:]))
+}
+
+func estimateKnowledgeTokenCount(text string) int {
+	text = strings.TrimSpace(text)
+	if text == "" {
+		return 0
+	}
+
+	runeCount := len([]rune(text))
+	tokenCount := runeCount / 2
+	if tokenCount == 0 {
+		return 1
+	}
+	return tokenCount
+}