root
dbd7747742
- Implemented KnowledgeWebsiteMarkdownPrompt for formatting webpage content into Markdown. - Added GetItemDetail method in KnowledgeHandler to retrieve knowledge item details. - Introduced KnowledgeDetailDrawer component for displaying knowledge item details in the admin web interface. - Created MarkdownPreview component for rendering Markdown content. - Added knowledge chunking and URL parsing logic to handle webpage content extraction and formatting. - Updated database schema to include markdown_content in knowledge_items table.
123 lines
2.2 KiB
Go
123 lines
2.2 KiB
Go
package app
|
|
|
|
import "strings"
|
|
|
|
func splitKnowledgeTextIntoChunks(text string, chunkSize, chunkOverlap int) []string {
|
|
text = strings.TrimSpace(text)
|
|
if text == "" {
|
|
return nil
|
|
}
|
|
|
|
if chunkSize <= 0 {
|
|
chunkSize = 900
|
|
}
|
|
if chunkOverlap < 0 {
|
|
chunkOverlap = 0
|
|
}
|
|
if chunkOverlap >= chunkSize {
|
|
chunkOverlap = chunkSize / 4
|
|
}
|
|
|
|
lines := strings.Split(text, "\n")
|
|
chunks := make([]string, 0, len(lines))
|
|
current := make([]string, 0, 16)
|
|
currentLen := 0
|
|
|
|
resetWithOverlap := func(chunk string) {
|
|
current = current[:0]
|
|
currentLen = 0
|
|
if chunkOverlap <= 0 {
|
|
return
|
|
}
|
|
overlap := takeLastKnowledgeRunes(chunk, chunkOverlap)
|
|
if overlap == "" {
|
|
return
|
|
}
|
|
current = append(current, overlap)
|
|
currentLen = len([]rune(overlap))
|
|
}
|
|
|
|
flush := func() {
|
|
if len(current) == 0 {
|
|
return
|
|
}
|
|
chunk := strings.TrimSpace(strings.Join(current, "\n"))
|
|
if chunk == "" {
|
|
current = current[:0]
|
|
currentLen = 0
|
|
return
|
|
}
|
|
chunks = append(chunks, chunk)
|
|
resetWithOverlap(chunk)
|
|
}
|
|
|
|
for _, line := range lines {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
continue
|
|
}
|
|
|
|
lineRunes := len([]rune(line))
|
|
if lineRunes > chunkSize {
|
|
flush()
|
|
for _, segment := range splitKnowledgeLongLine(line, chunkSize) {
|
|
if strings.TrimSpace(segment) == "" {
|
|
continue
|
|
}
|
|
chunks = append(chunks, segment)
|
|
resetWithOverlap(segment)
|
|
}
|
|
continue
|
|
}
|
|
|
|
nextLen := currentLen + lineRunes
|
|
if len(current) > 0 {
|
|
nextLen++
|
|
}
|
|
if nextLen > chunkSize {
|
|
flush()
|
|
}
|
|
|
|
current = append(current, line)
|
|
currentLen += lineRunes
|
|
if len(current) > 1 {
|
|
currentLen++
|
|
}
|
|
}
|
|
|
|
if len(current) > 0 {
|
|
chunk := strings.TrimSpace(strings.Join(current, "\n"))
|
|
if chunk != "" {
|
|
chunks = append(chunks, chunk)
|
|
}
|
|
}
|
|
|
|
return chunks
|
|
}
|
|
|
|
func takeLastKnowledgeRunes(text string, size int) string {
|
|
if size <= 0 {
|
|
return ""
|
|
}
|
|
|
|
runes := []rune(strings.TrimSpace(text))
|
|
if len(runes) <= size {
|
|
return string(runes)
|
|
}
|
|
return strings.TrimSpace(string(runes[len(runes)-size:]))
|
|
}
|
|
|
|
func estimateKnowledgeTokenCount(text string) int {
|
|
text = strings.TrimSpace(text)
|
|
if text == "" {
|
|
return 0
|
|
}
|
|
|
|
runeCount := len([]rune(text))
|
|
tokenCount := runeCount / 2
|
|
if tokenCount == 0 {
|
|
return 1
|
|
}
|
|
return tokenCount
|
|
}
|