package app
import (
"regexp"
"strings"
)
const (
generatedArticleSchemeURLPattern = `https?://[^\s<>"',。;、?!\]))}]+`
generatedArticleDomainPattern = `(?:www\.)?[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*\.(?:com|cn|net|org|io|ai|co|cc|top|xyz|shop|site|online|tech|app|gov|edu|info|biz|me|tv)(?:/[^\s<>"',。;、?!\]))}]*)?`
generatedArticleURLPattern = `(?:` + generatedArticleSchemeURLPattern + `|` + generatedArticleDomainPattern + `)`
)
var (
generatedArticleMarkdownLinkPattern = regexp.MustCompile(`!?\[([^\]\n]*)\]\(([^)\s]+)\)`)
generatedArticleHTMLImageParagraph = regexp.MustCompile(`(?is)
]*>\s*
]*\bsrc\s*=\s*["']?` + generatedArticleURLPattern + `["']?[^>]*>\s*
`)
generatedArticleHTMLImagePattern = regexp.MustCompile(`(?is)
]*\bsrc\s*=\s*["']?` + generatedArticleURLPattern + `["']?[^>]*>`)
generatedArticleURLLabelPattern = regexp.MustCompile(`(?i)(?:官网|官方网站|网站|网址|链接|URL|website|official website)\s*[::]?\s*` + generatedArticleURLPattern)
generatedArticleBareURLPattern = regexp.MustCompile(`(?i)\b` + generatedArticleURLPattern)
generatedArticleDanglingURLLabelPattern = regexp.MustCompile(`(?i)(?:官网|官方网站|网站|网址|链接|URL|website|official website)\s*[::]?\s*$`)
generatedArticleBlankLinePattern = regexp.MustCompile(`\n{3,}`)
generatedArticlePunctuationGapPattern = regexp.MustCompile(`\s+([,。;、?!,.;!?])`)
generatedArticleEmptyPunctuationPattern = regexp.MustCompile(`[::]\s*([,。;、?!,.;!?])`)
generatedArticleRepeatedSpacePattern = regexp.MustCompile(`[ \t]{2,}`)
)
func sanitizeGeneratedArticleMarkdown(markdown string) string {
text := strings.TrimSpace(markdown)
if text == "" {
return ""
}
text = generatedArticleMarkdownLinkPattern.ReplaceAllStringFunc(text, func(match string) string {
parts := generatedArticleMarkdownLinkPattern.FindStringSubmatch(match)
if len(parts) != 3 || !looksLikeGeneratedArticleURL(parts[2]) {
return match
}
if strings.HasPrefix(match, "![") {
return ""
}
label := strings.TrimSpace(parts[1])
if label == "" || looksLikeGeneratedArticleURL(label) {
return ""
}
return label
})
text = generatedArticleHTMLImageParagraph.ReplaceAllString(text, "")
text = generatedArticleHTMLImagePattern.ReplaceAllString(text, "")
text = generatedArticleURLLabelPattern.ReplaceAllString(text, "")
text = generatedArticleBareURLPattern.ReplaceAllString(text, "")
text = cleanGeneratedArticleURLArtifacts(text)
return strings.TrimSpace(text)
}
func sanitizeGeneratedOutlineText(text string) string {
text = sanitizeGeneratedArticleMarkdown(text)
replacer := strings.NewReplacer(
"网站列表", "品牌信息整理",
"官网列表", "品牌信息整理",
"网址列表", "品牌信息整理",
"官方网站", "公开信息",
"官方网址", "公开信息",
)
return strings.TrimSpace(replacer.Replace(text))
}
func looksLikeGeneratedArticleURL(text string) bool {
text = strings.TrimSpace(text)
if text == "" {
return false
}
return generatedArticleBareURLPattern.MatchString(text)
}
func cleanGeneratedArticleURLArtifacts(text string) string {
lines := strings.Split(text, "\n")
cleaned := make([]string, 0, len(lines))
for _, line := range lines {
originalLine := line
line = strings.TrimRight(line, " \t")
line = generatedArticleDanglingURLLabelPattern.ReplaceAllString(line, "")
line = strings.ReplaceAll(line, "网站列表", "品牌信息整理")
line = strings.ReplaceAll(line, "官网列表", "品牌信息整理")
line = strings.ReplaceAll(line, "网址列表", "品牌信息整理")
line = generatedArticleEmptyPunctuationPattern.ReplaceAllString(line, "$1")
line = generatedArticlePunctuationGapPattern.ReplaceAllString(line, "$1")
line = generatedArticleRepeatedSpacePattern.ReplaceAllString(line, " ")
if line != originalLine {
line = strings.TrimRight(line, " \t::,,;;")
}
if strings.TrimSpace(line) == "" {
if strings.TrimSpace(originalLine) == "" {
cleaned = append(cleaned, line)
}
continue
}
if isGeneratedArticleURLOnlyLine(line) {
continue
}
cleaned = append(cleaned, line)
}
return generatedArticleBlankLinePattern.ReplaceAllString(strings.Join(cleaned, "\n"), "\n\n")
}
func isGeneratedArticleURLOnlyLine(line string) bool {
trimmed := strings.TrimSpace(line)
trimmed = strings.TrimLeft(trimmed, "#-*+> \t")
trimmed = strings.Trim(trimmed, " \t::,,;;。")
if trimmed == "" {
return true
}
switch strings.ToLower(trimmed) {
case "官网", "官方网站", "官方网址", "网站", "网址", "链接", "url", "website", "official website":
return true
default:
return false
}
}