feat(monitoring): match citations by canonical link and scope summary by brand/keyword/platform/date
Replace the domain+title fuzzy alias scoring with a canonical candidate-key index built from published-link aliases — only exact matches now resolve to a SaaS source, eliminating false positives across articles sharing a host (e.g. m.163.com vs www.163.com). Surface the summary scoping that already existed on loadCitationRanking / loadCitedArticles by accepting brand_id, keyword_id, ai_platform_id and business_date on CitationSummary, plumbing them through the handler and admin tracking view. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,251 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"net"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/response"
|
||||
)
|
||||
|
||||
type monitoringPublishedAlias struct {
|
||||
ArticleID int64
|
||||
PublishRecordID *int64
|
||||
ArticleTitle string
|
||||
PublishPlatform string
|
||||
Ambiguous bool
|
||||
}
|
||||
|
||||
type monitoringPublishedAliasIndex map[string]monitoringPublishedAlias
|
||||
|
||||
type monitoringAliasQuerier interface {
|
||||
Query(ctx context.Context, sql string, args ...interface{}) (pgx.Rows, error)
|
||||
}
|
||||
|
||||
func loadMonitoringPublishedAliasIndex(ctx context.Context, q monitoringAliasQuerier, tenantID int64) (monitoringPublishedAliasIndex, error) {
|
||||
rows, err := q.Query(ctx, `
|
||||
SELECT
|
||||
article_id,
|
||||
publish_record_id,
|
||||
COALESCE(article_title_snapshot, '未命名文章') AS article_title,
|
||||
COALESCE(publish_platform_name_snapshot, '已发布内容') AS publish_platform,
|
||||
original_url,
|
||||
normalized_url
|
||||
FROM monitoring_article_url_aliases
|
||||
WHERE tenant_id = $1
|
||||
AND confidence = 'high'
|
||||
ORDER BY updated_at DESC, id DESC
|
||||
`, tenantID)
|
||||
if err != nil {
|
||||
return nil, responseInternalAliasLookupError()
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
index := make(monitoringPublishedAliasIndex)
|
||||
for rows.Next() {
|
||||
var item monitoringPublishedAlias
|
||||
var originalURL string
|
||||
var normalizedURL string
|
||||
var publishRecordID sql.NullInt64
|
||||
if scanErr := rows.Scan(&item.ArticleID, &publishRecordID, &item.ArticleTitle, &item.PublishPlatform, &originalURL, &normalizedURL); scanErr != nil {
|
||||
return nil, responseInternalAliasScanError()
|
||||
}
|
||||
item.PublishRecordID = nullableInt64Value(publishRecordID)
|
||||
for _, key := range monitoringCitationCandidateKeys(originalURL, normalizedURL) {
|
||||
existing, exists := index[key]
|
||||
if exists && existing.ArticleID != item.ArticleID {
|
||||
existing.Ambiguous = true
|
||||
index[key] = existing
|
||||
continue
|
||||
}
|
||||
if !exists {
|
||||
index[key] = item
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, responseInternalAliasScanError()
|
||||
}
|
||||
|
||||
return index, nil
|
||||
}
|
||||
|
||||
func (index monitoringPublishedAliasIndex) Match(rawURL, normalizedURL string) (monitoringPublishedAlias, bool) {
|
||||
for _, key := range monitoringCitationCandidateKeys(rawURL, normalizedURL) {
|
||||
item, ok := index[key]
|
||||
if ok && !item.Ambiguous {
|
||||
return item, true
|
||||
}
|
||||
}
|
||||
return monitoringPublishedAlias{}, false
|
||||
}
|
||||
|
||||
func monitoringCitationCandidateKeys(rawURL, normalizedURL string) []string {
|
||||
keys := []string{
|
||||
monitoringCitationURLMatchKey(rawURL),
|
||||
monitoringCitationURLMatchKey(normalizedURL),
|
||||
}
|
||||
result := make([]string, 0, len(keys))
|
||||
seen := make(map[string]struct{}, len(keys))
|
||||
for _, key := range keys {
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
result = append(result, key)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func monitoringCitationURLMatchKey(raw string) string {
|
||||
trimmed := strings.TrimSpace(raw)
|
||||
if trimmed == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
parsed, err := url.Parse(trimmed)
|
||||
if err != nil || parsed.Hostname() == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
host := monitoringCanonicalCitationHost(parsed.Hostname())
|
||||
if host == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
if strings.HasSuffix(host, "163.com") {
|
||||
if id := monitoringNeteaseArticleID(trimmed); id != "" {
|
||||
return "163.com/dy/article/" + strings.ToLower(id)
|
||||
}
|
||||
}
|
||||
|
||||
if host == "baijiahao.baidu.com" {
|
||||
if id := extractBaijiahaoArticleIDFromURL(&trimmed); id != "" {
|
||||
return host + "/s?id=" + strings.ToLower(id)
|
||||
}
|
||||
}
|
||||
|
||||
path := strings.TrimRight(parsed.EscapedPath(), "/")
|
||||
if path == "" {
|
||||
path = "/"
|
||||
}
|
||||
|
||||
if host == "mp.weixin.qq.com" {
|
||||
if strings.HasPrefix(path, "/s/") && len(path) > len("/s/") {
|
||||
return host + path
|
||||
}
|
||||
if query := monitoringCanonicalQuery(parsed, []string{"__biz", "mid", "idx", "sn"}); query != "" {
|
||||
return host + path + "?" + query
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
if path == "/" {
|
||||
if query := monitoringCanonicalQuery(parsed, monitoringGenericContentQueryKeys()); query != "" {
|
||||
return host + path + "?" + query
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
if query := monitoringCanonicalQuery(parsed, monitoringGenericContentQueryKeys()); query != "" {
|
||||
return host + path + "?" + query
|
||||
}
|
||||
return host + path
|
||||
}
|
||||
|
||||
func monitoringCanonicalCitationHost(value string) string {
|
||||
host := strings.ToLower(strings.TrimSpace(value))
|
||||
if host == "" {
|
||||
return ""
|
||||
}
|
||||
if withoutPort, _, err := net.SplitHostPort(host); err == nil {
|
||||
host = withoutPort
|
||||
}
|
||||
host = strings.Trim(host, ".")
|
||||
parts := strings.Split(host, ".")
|
||||
if len(parts) <= 2 {
|
||||
return host
|
||||
}
|
||||
switch parts[0] {
|
||||
case "www", "m", "wap", "mobile", "3g", "amp":
|
||||
return strings.Join(parts[1:], ".")
|
||||
default:
|
||||
return host
|
||||
}
|
||||
}
|
||||
|
||||
func monitoringCanonicalQuery(parsed *url.URL, allowedKeys []string) string {
|
||||
if parsed == nil || len(allowedKeys) == 0 {
|
||||
return ""
|
||||
}
|
||||
values := parsed.Query()
|
||||
pairs := make([]string, 0, len(allowedKeys))
|
||||
for _, key := range allowedKeys {
|
||||
value := strings.TrimSpace(values.Get(key))
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
pairs = append(pairs, url.QueryEscape(key)+"="+url.QueryEscape(value))
|
||||
}
|
||||
sort.Strings(pairs)
|
||||
return strings.Join(pairs, "&")
|
||||
}
|
||||
|
||||
func monitoringGenericContentQueryKeys() []string {
|
||||
return []string{"id", "article_id", "articleId", "docid", "docId", "postId"}
|
||||
}
|
||||
|
||||
func monitoringNeteaseArticleID(rawURL string) string {
|
||||
if id := extractWangyihaoArticleIDFromURL(&rawURL); id != "" {
|
||||
return id
|
||||
}
|
||||
parsed, err := url.Parse(strings.TrimSpace(rawURL))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
segments := strings.Split(strings.Trim(parsed.Path, "/"), "/")
|
||||
for index := len(segments) - 1; index >= 0; index-- {
|
||||
id := strings.TrimSuffix(strings.TrimSpace(segments[index]), ".html")
|
||||
if monitoringLooksLikeNeteaseArticleID(id) {
|
||||
return id
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func monitoringLooksLikeNeteaseArticleID(value string) bool {
|
||||
if len(value) < 8 || len(value) > 32 {
|
||||
return false
|
||||
}
|
||||
hasDigit := false
|
||||
hasLetter := false
|
||||
for _, r := range value {
|
||||
switch {
|
||||
case r >= '0' && r <= '9':
|
||||
hasDigit = true
|
||||
case r >= 'A' && r <= 'Z':
|
||||
hasLetter = true
|
||||
case r >= 'a' && r <= 'z':
|
||||
hasLetter = true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return hasDigit && hasLetter
|
||||
}
|
||||
|
||||
func responseInternalAliasLookupError() error {
|
||||
return response.ErrInternal(50041, "alias_lookup_failed", "failed to resolve article aliases")
|
||||
}
|
||||
|
||||
func responseInternalAliasScanError() error {
|
||||
return response.ErrInternal(50041, "alias_scan_failed", "failed to parse article alias resolution")
|
||||
}
|
||||
Reference in New Issue
Block a user