root/geo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(monitoring): attribute SaaS citations by registrable domain

Browse Source 
Persist host/registrable_domain/site_key on monitoring article URL
aliases, populate them on desktop-publish completion, and use them in
the citation pipeline to count how often SaaS-published content
appears as a citation source. Expose a /citation-summary endpoint with
a 7/30-day window switch and surface the new source-share metrics in
the tracking dashboard.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
liangxu
2026-04-30 17:12:09 +08:00
parent 6e6e19cccb
commit fb166fa9fe
15 changed files with 836 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/tenant/app/monitoring_callback_service.go
+255 -20
View File
@@ -11,6 +11,7 @@ import (
"strconv"
"strings"
"time"
"unicode"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
@@ -274,9 +275,31 @@ type monitoringLeasedTaskRow struct {
}
type monitoringAliasResolution struct {
ArticleID *int64
PublishRecordID *int64
NormalizedURLKey string
ArticleID *int64
PublishRecordID *int64
NormalizedURLKey string
ResolutionStatus string
ResolutionConfidence string
}
type monitoringAliasLookupInput struct {
NormalizedURL string
RegistrableDomain string
SiteKey string
Host string
LastPathSegment *string
TitleKey string
}
type monitoringAliasCandidate struct {
NormalizedURL string
ArticleID *int64
PublishRecordID *int64
RegistrableDomain string
SiteKey string
Host string
LastPathSegment string
TitleKey string
}
type monitoringCitationFact struct {
@@ -2400,7 +2423,7 @@ func (s *MonitoringCallbackService) buildCitationFacts(ctx context.Context, tx p
resolveContentCitations := monitoringPlatformResolvesContentCitations(platformID)
aliasMap := map[string]monitoringAliasResolution{}
if resolveContentCitations {
normalizedURLs := make([]string, 0, len(inputs))
lookupInputs := make([]monitoringAliasLookupInput, 0, len(inputs))
seenURLs := make(map[string]struct{}, len(inputs))
for _, item := range inputs {
key := strings.TrimSpace(monitoringStringValue(item.NormalizedURL))
@@ -2414,11 +2437,23 @@ func (s *MonitoringCallbackService) buildCitationFacts(ctx context.Context, tx p
continue
}
seenURLs[key] = struct{}{}
normalizedURLs = append(normalizedURLs, key)
host, registrableDomain, _, _ := deriveCitationURLParts(item.URL, key)
siteKey := strings.TrimSpace(monitoringStringValue(item.SiteKey))
if siteKey == "" {
siteKey = registrableDomain
}
lookupInputs = append(lookupInputs, monitoringAliasLookupInput{
NormalizedURL: key,
RegistrableDomain: strings.ToLower(strings.TrimSpace(registrableDomain)),
SiteKey: strings.ToLower(strings.TrimSpace(siteKey)),
Host: strings.ToLower(strings.TrimSpace(host)),
LastPathSegment: monitoringAliasLastPathSegment(key),
TitleKey: normalizeMonitoringAliasTitle(monitoringStringValue(item.Title)),
})
}
var err error
aliasMap, err = s.loadAliasResolutions(ctx, tx, tenantID, normalizedURLs)
aliasMap, err = s.loadAliasResolutions(ctx, tx, tenantID, lookupInputs)
if err != nil {
return nil, err
}
@@ -2443,40 +2478,205 @@ func (s *MonitoringCallbackService) buildCitationFacts(ctx context.Context, tx p
return result, nil
}
func (s *MonitoringCallbackService) loadAliasResolutions(ctx context.Context, tx pgx.Tx, tenantID int64, normalizedURLs []string) (map[string]monitoringAliasResolution, error) {
func (s *MonitoringCallbackService) loadAliasResolutions(ctx context.Context, tx pgx.Tx, tenantID int64, lookupInputs []monitoringAliasLookupInput) (map[string]monitoringAliasResolution, error) {
result := make(map[string]monitoringAliasResolution)
if len(normalizedURLs) == 0 {
if len(lookupInputs) == 0 {
return result, nil
}
normalizedURLs := make([]string, 0, len(lookupInputs))
domainKeys := make([]string, 0, len(lookupInputs)*3)
seenDomainKeys := map[string]struct{}{}
for _, input := range lookupInputs {
if input.NormalizedURL != "" {
normalizedURLs = append(normalizedURLs, input.NormalizedURL)
}
for _, key := range []string{input.RegistrableDomain, input.SiteKey, input.Host} {
key = strings.ToLower(strings.TrimSpace(key))
if key == "" {
continue
}
if _, ok := seenDomainKeys[key]; ok {
continue
}
seenDomainKeys[key] = struct{}{}
domainKeys = append(domainKeys, key)
}
}
rows, err := tx.Query(ctx, `
SELECT normalized_url, article_id, publish_record_id
SELECT
normalized_url,
article_id,
publish_record_id,
registrable_domain,
site_key,
host,
last_path_segment,
article_title_snapshot
FROM monitoring_article_url_aliases
WHERE tenant_id = $1
AND normalized_url = ANY($2)
`, tenantID, normalizedURLs)
AND (
normalized_url = ANY($2::text[])
OR NULLIF(LOWER(TRIM(registrable_domain)), '') = ANY($3::text[])
OR NULLIF(LOWER(TRIM(site_key)), '') = ANY($3::text[])
OR NULLIF(LOWER(TRIM(host)), '') = ANY($3::text[])
)
`, tenantID, normalizedURLs, domainKeys)
if err != nil {
return nil, response.ErrInternal(50041, "alias_lookup_failed", "failed to resolve article aliases")
}
defer rows.Close()
candidates := make([]monitoringAliasCandidate, 0)
for rows.Next() {
var normalizedURL string
var candidate monitoringAliasCandidate
var articleID sql.NullInt64
var publishRecordID sql.NullInt64
if scanErr := rows.Scan(&normalizedURL, &articleID, &publishRecordID); scanErr != nil {
var registrableDomain sql.NullString
var siteKey sql.NullString
var host sql.NullString
var lastPathSegment sql.NullString
var articleTitle sql.NullString
if scanErr := rows.Scan(
&candidate.NormalizedURL,
&articleID,
&publishRecordID,
&registrableDomain,
&siteKey,
&host,
&lastPathSegment,
&articleTitle,
); scanErr != nil {
return nil, response.ErrInternal(50041, "alias_scan_failed", "failed to parse article alias resolution")
}
result[normalizedURL] = monitoringAliasResolution{
ArticleID: nullableInt64Value(articleID),
PublishRecordID: nullableInt64Value(publishRecordID),
NormalizedURLKey: normalizedURL,
candidate.ArticleID = nullableInt64Value(articleID)
candidate.PublishRecordID = nullableInt64Value(publishRecordID)
candidate.RegistrableDomain = strings.ToLower(strings.TrimSpace(nullableStringText(registrableDomain)))
candidate.SiteKey = strings.ToLower(strings.TrimSpace(nullableStringText(siteKey)))
candidate.Host = strings.ToLower(strings.TrimSpace(nullableStringText(host)))
candidate.LastPathSegment = strings.TrimSpace(nullableStringText(lastPathSegment))
candidate.TitleKey = normalizeMonitoringAliasTitle(nullableStringText(articleTitle))
candidates = append(candidates, candidate)
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "alias_scan_failed", "failed to iterate article aliases")
}
for _, input := range lookupInputs {
bestScore := -1
bestAmbiguous := false
var best monitoringAliasCandidate
domainMatchedArticleIDs := map[int64]struct{}{}
for _, candidate := range candidates {
if monitoringAliasDomainMatches(input, candidate) && candidate.ArticleID != nil {
domainMatchedArticleIDs[*candidate.ArticleID] = struct{}{}
}
score := scoreMonitoringAliasCandidate(input, candidate)
if score < 0 {
continue
}
if score > bestScore {
bestScore = score
bestAmbiguous = false
best = candidate
continue
}
if score == bestScore && !sameInt64Pointer(candidate.ArticleID, best.ArticleID) {
bestAmbiguous = true
}
}
if bestScore == 40 && len(domainMatchedArticleIDs) == 1 {
bestScore = 60
}
if bestScore < 60 || bestAmbiguous {
continue
}
confidence := "medium"
status := "domain_matched"
if bestScore >= 100 {
confidence = "high"
status = "resolved"
} else if bestScore < 80 {
confidence = "low"
}
result[input.NormalizedURL] = monitoringAliasResolution{
ArticleID: cloneInt64(best.ArticleID),
PublishRecordID: cloneInt64(best.PublishRecordID),
NormalizedURLKey: best.NormalizedURL,
ResolutionStatus: status,
ResolutionConfidence: confidence,
}
}
return result, nil
}
func scoreMonitoringAliasCandidate(input monitoringAliasLookupInput, candidate monitoringAliasCandidate) int {
if input.NormalizedURL != "" && candidate.NormalizedURL == input.NormalizedURL {
return 100
}
if !monitoringAliasDomainMatches(input, candidate) {
return -1
}
score := 40
if input.LastPathSegment != nil && strings.TrimSpace(*input.LastPathSegment) != "" && candidate.LastPathSegment == strings.TrimSpace(*input.LastPathSegment) {
score += 30
}
if input.TitleKey != "" && candidate.TitleKey != "" {
switch {
case input.TitleKey == candidate.TitleKey:
score += 30
case strings.Contains(input.TitleKey, candidate.TitleKey) || strings.Contains(candidate.TitleKey, input.TitleKey):
score += 15
}
}
return score
}
func monitoringAliasDomainMatches(input monitoringAliasLookupInput, candidate monitoringAliasCandidate) bool {
inputKeys := []string{input.RegistrableDomain, input.SiteKey, input.Host}
candidateKeys := []string{candidate.RegistrableDomain, candidate.SiteKey, candidate.Host}
for _, inputKey := range inputKeys {
inputKey = strings.ToLower(strings.TrimSpace(inputKey))
if inputKey == "" {
continue
}
for _, candidateKey := range candidateKeys {
if inputKey == strings.ToLower(strings.TrimSpace(candidateKey)) {
return true
}
}
}
return false
}
func normalizeMonitoringAliasTitle(value string) string {
return strings.Map(func(r rune) rune {
if unicode.IsLetter(r) || unicode.IsNumber(r) {
return unicode.ToLower(r)
}
return -1
}, value)
}
func nullableStringText(value sql.NullString) string {
if !value.Valid {
return ""
}
return strings.TrimSpace(value.String)
}
func sameInt64Pointer(left, right *int64) bool {
if left == nil || right == nil {
return left == right
}
return *left == *right
}
func (s *MonitoringCallbackService) insertCitationFact(ctx context.Context, tx pgx.Tx, task *monitoringCollectTask, runID int64, fact monitoringCitationFact) error {
if _, err := tx.Exec(ctx, `
INSERT INTO monitoring_citation_facts (
@@ -2809,10 +3009,13 @@ func buildMonitoringCitationFact(input MonitoringSourceItem, aliasMap map[string
var articleID *int64
var publishRecordID *int64
var aliasResolution *monitoringAliasResolution
if resolveContentCitations {
articleID = cloneInt64(input.ArticleID)
publishRecordID = cloneInt64(input.PublishRecordID)
if alias, ok := aliasMap[normalizedURL]; ok {
aliasCopy := alias
aliasResolution = &aliasCopy
if articleID == nil {
articleID = cloneInt64(alias.ArticleID)
}
@@ -2824,11 +3027,19 @@ func buildMonitoringCitationFact(input MonitoringSourceItem, aliasMap map[string
resolutionStatus := strings.TrimSpace(monitoringStringValue(input.ResolutionStatus))
if resolutionStatus == "" {
resolutionStatus = "resolved"
if aliasResolution != nil && strings.TrimSpace(aliasResolution.ResolutionStatus) != "" {
resolutionStatus = strings.TrimSpace(aliasResolution.ResolutionStatus)
} else {
resolutionStatus = "resolved"
}
}
resolutionConfidence := strings.TrimSpace(monitoringStringValue(input.ResolutionConfidence))
if resolutionConfidence == "" {
resolutionConfidence = "high"
if aliasResolution != nil && strings.TrimSpace(aliasResolution.ResolutionConfidence) != "" {
resolutionConfidence = strings.TrimSpace(aliasResolution.ResolutionConfidence)
} else {
resolutionConfidence = "high"
}
}
return monitoringCitationFact{
@@ -3113,7 +3324,7 @@ func normalizeCitationURL(raw string) string {
parsed.Host = strings.ToLower(parsed.Host)
parsed.Fragment = ""
parsed.RawFragment = ""
parsed.RawQuery = ""
parsed.RawQuery = normalizeCitationQuery(parsed)
if parsed.Path != "/" {
parsed.Path = strings.TrimRight(parsed.Path, "/")
@@ -3122,6 +3333,30 @@ func normalizeCitationURL(raw string) string {
return parsed.String()
}
func normalizeCitationQuery(parsed *url.URL) string {
if parsed == nil {
return ""
}
host := strings.ToLower(strings.TrimSpace(parsed.Hostname()))
allowedParams := []string{}
switch {
case host == "baijiahao.baidu.com":
allowedParams = []string{"id"}
}
if len(allowedParams) == 0 {
return ""
}
query := parsed.Query()
normalized := url.Values{}
for _, key := range allowedParams {
if value := strings.TrimSpace(query.Get(key)); value != "" {
normalized.Set(key, value)
}
}
return normalized.Encode()
}
func deriveCitationURLParts(rawURL, normalizedURL string) (string, string, *string, *string) {
candidate := strings.TrimSpace(normalizedURL)
if candidate == "" {