geo/server/internal/tenant/app/article_url_sanitizer.go

package app

import (
	"regexp"
	"strings"
)

const (
	generatedArticleSchemeURLPattern = `https?://[^\s<>"'，。；、？！\])）}]+`
	generatedArticleDomainPattern    = `(?:www\.)?[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*\.(?:com|cn|net|org|io|ai|co|cc|top|xyz|shop|site|online|tech|app|gov|edu|info|biz|me|tv)(?:/[^\s<>"'，。；、？！\])）}]*)?`
	generatedArticleURLPattern       = `(?:` + generatedArticleSchemeURLPattern + `|` + generatedArticleDomainPattern + `)`
)

var (
	generatedArticleMarkdownLinkPattern     = regexp.MustCompile(`!?\[([^\]\n]*)\]\(([^)\s]+)\)`)
	generatedArticleHTMLImageParagraph      = regexp.MustCompile(`(?is)<p\b[^>]*>\s*<img\b[^>]*\bsrc\s*=\s*["']?` + generatedArticleURLPattern + `["']?[^>]*>\s*</p>`)
	generatedArticleHTMLImagePattern        = regexp.MustCompile(`(?is)<img\b[^>]*\bsrc\s*=\s*["']?` + generatedArticleURLPattern + `["']?[^>]*>`)
	generatedArticleURLLabelPattern         = regexp.MustCompile(`(?i)(?:官网|官方网站|网站|网址|链接|URL|website|official website)\s*[:：]?\s*` + generatedArticleURLPattern)
	generatedArticleBareURLPattern          = regexp.MustCompile(`(?i)\b` + generatedArticleURLPattern)
	generatedArticleDanglingURLLabelPattern = regexp.MustCompile(`(?i)(?:官网|官方网站|网站|网址|链接|URL|website|official website)\s*[:：]?\s*$`)
	generatedArticleBlankLinePattern        = regexp.MustCompile(`\n{3,}`)
	generatedArticlePunctuationGapPattern   = regexp.MustCompile(`\s+([，。；、？！,.;!?])`)
	generatedArticleEmptyPunctuationPattern = regexp.MustCompile(`[：:]\s*([，。；、？！,.;!?])`)
	generatedArticleRepeatedSpacePattern    = regexp.MustCompile(`[ \t]{2,}`)
)

func sanitizeGeneratedArticleMarkdown(markdown string) string {
	text := strings.TrimSpace(markdown)
	if text == "" {
		return ""
	}

	text = generatedArticleMarkdownLinkPattern.ReplaceAllStringFunc(text, func(match string) string {
		parts := generatedArticleMarkdownLinkPattern.FindStringSubmatch(match)
		if len(parts) != 3 || !looksLikeGeneratedArticleURL(parts[2]) {
			return match
		}
		if strings.HasPrefix(match, "![") {
			return ""
		}
		label := strings.TrimSpace(parts[1])
		if label == "" || looksLikeGeneratedArticleURL(label) {
			return ""
		}
		return label
	})

	text = generatedArticleHTMLImageParagraph.ReplaceAllString(text, "")
	text = generatedArticleHTMLImagePattern.ReplaceAllString(text, "")
	text = generatedArticleURLLabelPattern.ReplaceAllString(text, "")
	text = generatedArticleBareURLPattern.ReplaceAllString(text, "")
	text = cleanGeneratedArticleURLArtifacts(text)
	return strings.TrimSpace(text)
}

func sanitizeGeneratedOutlineText(text string) string {
	text = sanitizeGeneratedArticleMarkdown(text)
	replacer := strings.NewReplacer(
		"网站列表", "品牌信息整理",
		"官网列表", "品牌信息整理",
		"网址列表", "品牌信息整理",
		"官方网站", "公开信息",
		"官方网址", "公开信息",
	)
	return strings.TrimSpace(replacer.Replace(text))
}

func looksLikeGeneratedArticleURL(text string) bool {
	text = strings.TrimSpace(text)
	if text == "" {
		return false
	}
	return generatedArticleBareURLPattern.MatchString(text)
}

func cleanGeneratedArticleURLArtifacts(text string) string {
	lines := strings.Split(text, "\n")
	cleaned := make([]string, 0, len(lines))
	for _, line := range lines {
		originalLine := line
		line = strings.TrimRight(line, " \t")
		line = generatedArticleDanglingURLLabelPattern.ReplaceAllString(line, "")
		line = strings.ReplaceAll(line, "网站列表", "品牌信息整理")
		line = strings.ReplaceAll(line, "官网列表", "品牌信息整理")
		line = strings.ReplaceAll(line, "网址列表", "品牌信息整理")
		line = generatedArticleEmptyPunctuationPattern.ReplaceAllString(line, "$1")
		line = generatedArticlePunctuationGapPattern.ReplaceAllString(line, "$1")
		line = generatedArticleRepeatedSpacePattern.ReplaceAllString(line, " ")
		if line != originalLine {
			line = strings.TrimRight(line, " \t:：,，;；")
		}
		if strings.TrimSpace(line) == "" {
			if strings.TrimSpace(originalLine) == "" {
				cleaned = append(cleaned, line)
			}
			continue
		}
		if isGeneratedArticleURLOnlyLine(line) {
			continue
		}
		cleaned = append(cleaned, line)
	}
	return generatedArticleBlankLinePattern.ReplaceAllString(strings.Join(cleaned, "\n"), "\n\n")
}

func isGeneratedArticleURLOnlyLine(line string) bool {
	trimmed := strings.TrimSpace(line)
	trimmed = strings.TrimLeft(trimmed, "#-*+> \t")
	trimmed = strings.Trim(trimmed, " \t:：,，;；。")
	if trimmed == "" {
		return true
	}
	switch strings.ToLower(trimmed) {
	case "官网", "官方网站", "官方网址", "网站", "网址", "链接", "url", "website", "official website":
		return true
	default:
		return false
	}
}