314 lines
7.7 KiB
Go
314 lines
7.7 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"net"
|
|
"net/url"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
|
|
"github.com/geo-platform/tenant-api/internal/shared/response"
|
|
)
|
|
|
|
type monitoringPublishedAlias struct {
|
|
ArticleID int64
|
|
PublishRecordID *int64
|
|
ArticleTitle string
|
|
PublishPlatform string
|
|
OriginalURL string
|
|
NormalizedURL string
|
|
SourceType string
|
|
SourceID int64
|
|
SourcePriority int
|
|
Ambiguous bool
|
|
}
|
|
|
|
type monitoringPublishedAliasIndex map[string]monitoringPublishedAlias
|
|
|
|
type monitoringAliasQuerier interface {
|
|
Query(ctx context.Context, sql string, args ...interface{}) (pgx.Rows, error)
|
|
}
|
|
|
|
func loadMonitoringPublishedAliasIndex(ctx context.Context, q monitoringAliasQuerier, tenantID int64) (monitoringPublishedAliasIndex, error) {
|
|
rows, err := q.Query(ctx, `
|
|
WITH alias_sources AS (
|
|
SELECT
|
|
article_id,
|
|
publish_record_id,
|
|
COALESCE(article_title_snapshot, '未命名文章') AS article_title,
|
|
COALESCE(publish_platform_name_snapshot, '已发布内容') AS publish_platform,
|
|
original_url,
|
|
normalized_url,
|
|
'published_article' AS source_type,
|
|
article_id AS source_id,
|
|
1 AS source_priority,
|
|
updated_at AS sort_updated_at,
|
|
id AS sort_id
|
|
FROM monitoring_article_url_aliases
|
|
WHERE tenant_id = $1
|
|
AND confidence = 'high'
|
|
UNION ALL
|
|
SELECT
|
|
NULL AS article_id,
|
|
NULL AS publish_record_id,
|
|
COALESCE(NULLIF(article_title, ''), '未命名文章') AS article_title,
|
|
COALESCE(NULLIF(publish_platform, ''), '外部文章') AS publish_platform,
|
|
original_url,
|
|
normalized_url,
|
|
'manual_mark' AS source_type,
|
|
id AS source_id,
|
|
2 AS source_priority,
|
|
updated_at AS sort_updated_at,
|
|
id AS sort_id
|
|
FROM monitoring_user_marked_articles
|
|
WHERE tenant_id = $1
|
|
AND expires_at > NOW()
|
|
)
|
|
SELECT
|
|
article_id,
|
|
publish_record_id,
|
|
article_title,
|
|
publish_platform,
|
|
original_url,
|
|
normalized_url,
|
|
source_type,
|
|
source_id,
|
|
source_priority
|
|
FROM alias_sources
|
|
ORDER BY source_priority ASC, sort_updated_at DESC, sort_id DESC
|
|
`, tenantID)
|
|
if err != nil {
|
|
return nil, responseInternalAliasLookupError()
|
|
}
|
|
defer rows.Close()
|
|
|
|
index := make(monitoringPublishedAliasIndex)
|
|
for rows.Next() {
|
|
var item monitoringPublishedAlias
|
|
var originalURL string
|
|
var normalizedURL string
|
|
var publishRecordID sql.NullInt64
|
|
var articleID sql.NullInt64
|
|
if scanErr := rows.Scan(
|
|
&articleID,
|
|
&publishRecordID,
|
|
&item.ArticleTitle,
|
|
&item.PublishPlatform,
|
|
&originalURL,
|
|
&normalizedURL,
|
|
&item.SourceType,
|
|
&item.SourceID,
|
|
&item.SourcePriority,
|
|
); scanErr != nil {
|
|
return nil, responseInternalAliasScanError()
|
|
}
|
|
if articleID.Valid {
|
|
item.ArticleID = articleID.Int64
|
|
}
|
|
item.PublishRecordID = nullableInt64Value(publishRecordID)
|
|
item.OriginalURL = originalURL
|
|
item.NormalizedURL = normalizedURL
|
|
for _, key := range monitoringCitationCandidateKeys(originalURL, normalizedURL) {
|
|
existing, exists := index[key]
|
|
if exists && existing.SourcePriority < item.SourcePriority {
|
|
continue
|
|
}
|
|
if exists && item.SourcePriority < existing.SourcePriority {
|
|
index[key] = item
|
|
continue
|
|
}
|
|
if exists && (existing.SourceType != item.SourceType || existing.SourceID != item.SourceID) {
|
|
existing.Ambiguous = true
|
|
index[key] = existing
|
|
continue
|
|
}
|
|
if !exists {
|
|
index[key] = item
|
|
}
|
|
}
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, responseInternalAliasScanError()
|
|
}
|
|
|
|
return index, nil
|
|
}
|
|
|
|
func (index monitoringPublishedAliasIndex) Match(rawURL, normalizedURL string) (monitoringPublishedAlias, bool) {
|
|
for _, key := range monitoringCitationCandidateKeys(rawURL, normalizedURL) {
|
|
item, ok := index[key]
|
|
if ok && !item.Ambiguous {
|
|
return item, true
|
|
}
|
|
}
|
|
return monitoringPublishedAlias{}, false
|
|
}
|
|
|
|
func monitoringCitationCandidateKeys(rawURL, normalizedURL string) []string {
|
|
keys := []string{
|
|
monitoringCitationURLMatchKey(rawURL),
|
|
monitoringCitationURLMatchKey(normalizedURL),
|
|
}
|
|
result := make([]string, 0, len(keys))
|
|
seen := make(map[string]struct{}, len(keys))
|
|
for _, key := range keys {
|
|
if key == "" {
|
|
continue
|
|
}
|
|
if _, ok := seen[key]; ok {
|
|
continue
|
|
}
|
|
seen[key] = struct{}{}
|
|
result = append(result, key)
|
|
}
|
|
return result
|
|
}
|
|
|
|
func monitoringCitationURLMatchKey(raw string) string {
|
|
trimmed := strings.TrimSpace(raw)
|
|
if trimmed == "" {
|
|
return ""
|
|
}
|
|
|
|
parsed, err := url.Parse(trimmed)
|
|
if err != nil || parsed.Hostname() == "" {
|
|
return ""
|
|
}
|
|
|
|
host := monitoringCanonicalCitationHost(parsed.Hostname())
|
|
if host == "" {
|
|
return ""
|
|
}
|
|
|
|
if strings.HasSuffix(host, "163.com") {
|
|
if id := monitoringNeteaseArticleID(trimmed); id != "" {
|
|
return "163.com/dy/article/" + strings.ToLower(id)
|
|
}
|
|
}
|
|
|
|
if host == "baijiahao.baidu.com" {
|
|
if id := extractBaijiahaoArticleIDFromURL(&trimmed); id != "" {
|
|
return host + "/s?id=" + strings.ToLower(id)
|
|
}
|
|
}
|
|
|
|
path := strings.TrimRight(parsed.EscapedPath(), "/")
|
|
if path == "" {
|
|
path = "/"
|
|
}
|
|
|
|
if host == "mp.weixin.qq.com" {
|
|
if strings.HasPrefix(path, "/s/") && len(path) > len("/s/") {
|
|
return host + path
|
|
}
|
|
if query := monitoringCanonicalQuery(parsed, []string{"__biz", "mid", "idx", "sn"}); query != "" {
|
|
return host + path + "?" + query
|
|
}
|
|
return ""
|
|
}
|
|
|
|
if path == "/" {
|
|
if query := monitoringCanonicalQuery(parsed, monitoringGenericContentQueryKeys()); query != "" {
|
|
return host + path + "?" + query
|
|
}
|
|
return ""
|
|
}
|
|
|
|
if query := monitoringCanonicalQuery(parsed, monitoringGenericContentQueryKeys()); query != "" {
|
|
return host + path + "?" + query
|
|
}
|
|
return host + path
|
|
}
|
|
|
|
func monitoringCanonicalCitationHost(value string) string {
|
|
host := strings.ToLower(strings.TrimSpace(value))
|
|
if host == "" {
|
|
return ""
|
|
}
|
|
if withoutPort, _, err := net.SplitHostPort(host); err == nil {
|
|
host = withoutPort
|
|
}
|
|
host = strings.Trim(host, ".")
|
|
parts := strings.Split(host, ".")
|
|
if len(parts) <= 2 {
|
|
return host
|
|
}
|
|
switch parts[0] {
|
|
case "www", "m", "wap", "mobile", "3g", "amp":
|
|
return strings.Join(parts[1:], ".")
|
|
default:
|
|
return host
|
|
}
|
|
}
|
|
|
|
func monitoringCanonicalQuery(parsed *url.URL, allowedKeys []string) string {
|
|
if parsed == nil || len(allowedKeys) == 0 {
|
|
return ""
|
|
}
|
|
values := parsed.Query()
|
|
pairs := make([]string, 0, len(allowedKeys))
|
|
for _, key := range allowedKeys {
|
|
value := strings.TrimSpace(values.Get(key))
|
|
if value == "" {
|
|
continue
|
|
}
|
|
pairs = append(pairs, url.QueryEscape(key)+"="+url.QueryEscape(value))
|
|
}
|
|
sort.Strings(pairs)
|
|
return strings.Join(pairs, "&")
|
|
}
|
|
|
|
func monitoringGenericContentQueryKeys() []string {
|
|
return []string{"id", "article_id", "articleId", "docid", "docId", "postId"}
|
|
}
|
|
|
|
func monitoringNeteaseArticleID(rawURL string) string {
|
|
if id := extractWangyihaoArticleIDFromURL(&rawURL); id != "" {
|
|
return id
|
|
}
|
|
parsed, err := url.Parse(strings.TrimSpace(rawURL))
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
segments := strings.Split(strings.Trim(parsed.Path, "/"), "/")
|
|
for index := len(segments) - 1; index >= 0; index-- {
|
|
id := strings.TrimSuffix(strings.TrimSpace(segments[index]), ".html")
|
|
if monitoringLooksLikeNeteaseArticleID(id) {
|
|
return id
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func monitoringLooksLikeNeteaseArticleID(value string) bool {
|
|
if len(value) < 8 || len(value) > 32 {
|
|
return false
|
|
}
|
|
hasDigit := false
|
|
hasLetter := false
|
|
for _, r := range value {
|
|
switch {
|
|
case r >= '0' && r <= '9':
|
|
hasDigit = true
|
|
case r >= 'A' && r <= 'Z':
|
|
hasLetter = true
|
|
case r >= 'a' && r <= 'z':
|
|
hasLetter = true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
return hasDigit && hasLetter
|
|
}
|
|
|
|
func responseInternalAliasLookupError() error {
|
|
return response.ErrInternal(50041, "alias_lookup_failed", "failed to resolve article aliases")
|
|
}
|
|
|
|
func responseInternalAliasScanError() error {
|
|
return response.ErrInternal(50041, "alias_scan_failed", "failed to parse article alias resolution")
|
|
}
|