feat: add monitoring marked articles and retention updates
Frontend CI / Frontend (push) Successful in 7m47s
Backend CI / Backend (push) Successful in 19m26s

This commit is contained in:
2026-06-17 12:48:41 +08:00
parent 9ed857e159
commit 31c4dd9358
40 changed files with 2373 additions and 488 deletions
+372 -298
View File
@@ -10,6 +10,7 @@ import (
"fmt"
"math/big"
"sort"
"strconv"
"strings"
"sync"
"time"
@@ -192,9 +193,12 @@ type MonitoringCitationRanking struct {
}
type MonitoringCitedArticle struct {
ArticleID int64 `json:"article_id"`
ArticleID *int64 `json:"article_id"`
ArticleTitle string `json:"article_title"`
PublishPlatform string `json:"publish_platform"`
SourceType string `json:"source_type"`
SourceID int64 `json:"source_id"`
SourceURL string `json:"source_url"`
CitationCount int64 `json:"citation_count"`
CitationRate *float64 `json:"citation_rate"`
SourceShare *float64 `json:"source_share"`
@@ -237,6 +241,9 @@ type MonitoringQuestionDetailCitation struct {
SiteKey string `json:"site_key"`
FaviconURL string `json:"favicon_url"`
ArticleID *int64 `json:"article_id"`
SourceType *string `json:"source_type,omitempty"`
SourceID *int64 `json:"source_id,omitempty"`
SourceURL *string `json:"source_url,omitempty"`
ArticleTitle *string `json:"article_title,omitempty"`
ResolutionStatus string `json:"resolution_status"`
ResolutionConfidence string `json:"resolution_confidence"`
@@ -254,9 +261,12 @@ type MonitoringQuestionCitationStats struct {
}
type MonitoringQuestionContentCitation struct {
ArticleID int64 `json:"article_id"`
ArticleID *int64 `json:"article_id"`
ArticleTitle string `json:"article_title"`
PublishPlatform string `json:"publish_platform"`
SourceType string `json:"source_type"`
SourceID int64 `json:"source_id"`
SourceURL string `json:"source_url"`
AIPlatformID string `json:"ai_platform_id"`
PlatformName string `json:"platform_name"`
CitationCount int64 `json:"citation_count"`
@@ -3458,11 +3468,6 @@ func (s *MonitoringService) loadCitationRanking(
}
platformQueryIDs := monitoringPlatformQueryIDs(aiPlatformID)
aliasIndex, err := loadMonitoringPublishedAliasIndex(ctx, s.monitoringPool, tenantID)
if err != nil {
return nil, err
}
type citationRankingAggregate struct {
CitedAnswerCount int64
CitedArticleCount int64
@@ -3473,18 +3478,57 @@ func (s *MonitoringService) loadCitationRanking(
aggregates := make(map[string]citationRankingAggregate)
rows, err := s.monitoringPool.Query(ctx, `
WITH filtered_runs AS (
SELECT id, ai_platform_id
FROM question_monitor_runs
WHERE tenant_id = $1
AND ($2::bigint = 0 OR brand_id = $2)
AND collector_type = $3
AND status = 'succeeded'
AND business_date BETWEEN $4::date AND $5::date
AND ($6::bigint[] IS NULL OR question_id = ANY($6))
AND ($7::text[] IS NULL OR ai_platform_id = ANY($7))
),
sample_counts AS (
SELECT ai_platform_id, COUNT(*)::bigint AS sample_count
FROM filtered_runs
GROUP BY ai_platform_id
),
citation_counts AS (
SELECT
fr.ai_platform_id,
COUNT(cf.id)::bigint AS citation_source_count,
COUNT(cf.id) FILTER (
WHERE fr.ai_platform_id <> 'qwen'
AND cf.content_source_type IS NOT NULL
AND cf.content_source_id IS NOT NULL
)::bigint AS saas_source_count,
COUNT(DISTINCT cf.run_id) FILTER (
WHERE fr.ai_platform_id <> 'qwen'
AND cf.content_source_type IS NOT NULL
AND cf.content_source_id IS NOT NULL
)::bigint AS cited_answer_count,
COUNT(DISTINCT cf.content_source_type || ':' || cf.content_source_id::text) FILTER (
WHERE fr.ai_platform_id <> 'qwen'
AND cf.content_source_type IS NOT NULL
AND cf.content_source_id IS NOT NULL
)::bigint AS cited_article_count
FROM filtered_runs fr
JOIN monitoring_citation_facts cf
ON cf.tenant_id = $1
AND cf.run_id = fr.id
AND cf.business_date BETWEEN $4::date AND $5::date
GROUP BY fr.ai_platform_id
)
SELECT
r.ai_platform_id,
COUNT(*) AS sample_count
FROM question_monitor_runs r
WHERE r.tenant_id = $1
AND ($2::bigint = 0 OR r.brand_id = $2)
AND r.collector_type = $3
AND r.status = 'succeeded'
AND r.business_date BETWEEN $4::date AND $5::date
AND ($6::bigint[] IS NULL OR r.question_id = ANY($6))
AND ($7::text[] IS NULL OR r.ai_platform_id = ANY($7))
GROUP BY r.ai_platform_id
sc.ai_platform_id,
sc.sample_count,
COALESCE(cc.citation_source_count, 0)::bigint,
COALESCE(cc.saas_source_count, 0)::bigint,
COALESCE(cc.cited_answer_count, 0)::bigint,
COALESCE(cc.cited_article_count, 0)::bigint
FROM sample_counts sc
LEFT JOIN citation_counts cc ON cc.ai_platform_id = sc.ai_platform_id
`, tenantID, brandID, monitoringCollectorType, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), nullableInt64Array(questionIDs), nullableStringArray(platformQueryIDs))
if err != nil {
return nil, response.ErrInternal(50041, "query_failed", "failed to load citation ranking")
@@ -3494,7 +3538,18 @@ func (s *MonitoringService) loadCitationRanking(
for rows.Next() {
var platformID string
var sampleCount int64
if scanErr := rows.Scan(&platformID, &sampleCount); scanErr != nil {
var citationSourceCount int64
var saasSourceCount int64
var citedAnswerCount int64
var citedArticleCount int64
if scanErr := rows.Scan(
&platformID,
&sampleCount,
&citationSourceCount,
&saasSourceCount,
&citedAnswerCount,
&citedArticleCount,
); scanErr != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse citation ranking")
}
platformID = normalizeMonitoringPlatformID(platformID)
@@ -3503,74 +3558,16 @@ func (s *MonitoringService) loadCitationRanking(
}
item := aggregates[platformID]
item.SampleCount += sampleCount
item.CitationSourceCount += citationSourceCount
item.SaaSSourceCount += saasSourceCount
item.CitedAnswerCount += citedAnswerCount
item.CitedArticleCount += citedArticleCount
aggregates[platformID] = item
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate citation ranking")
}
citedRuns := make(map[string]map[int64]struct{})
citedArticles := make(map[string]map[int64]struct{})
rows, err = s.monitoringPool.Query(ctx, `
SELECT
r.ai_platform_id,
cf.run_id,
cf.cited_url,
cf.normalized_url
FROM question_monitor_runs r
JOIN monitoring_citation_facts cf
ON cf.tenant_id = r.tenant_id AND cf.run_id = r.id
WHERE r.tenant_id = $1
AND ($2::bigint = 0 OR r.brand_id = $2)
AND r.collector_type = $3
AND r.status = 'succeeded'
AND r.business_date BETWEEN $4::date AND $5::date
AND ($6::bigint[] IS NULL OR r.question_id = ANY($6))
AND ($7::text[] IS NULL OR r.ai_platform_id = ANY($7))
`, tenantID, brandID, monitoringCollectorType, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), nullableInt64Array(questionIDs), nullableStringArray(platformQueryIDs))
if err != nil {
return nil, response.ErrInternal(50041, "query_failed", "failed to load citation ranking")
}
defer rows.Close()
for rows.Next() {
var platformID string
var runID int64
var citedURL string
var normalizedURL string
if scanErr := rows.Scan(&platformID, &runID, &citedURL, &normalizedURL); scanErr != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse citation ranking")
}
platformID = normalizeMonitoringPlatformID(platformID)
if platformID == "" {
continue
}
item := aggregates[platformID]
item.CitationSourceCount++
if platformID != "qwen" {
if alias, ok := aliasIndex.Match(citedURL, normalizedURL); ok {
item.SaaSSourceCount++
if _, exists := citedRuns[platformID]; !exists {
citedRuns[platformID] = make(map[int64]struct{})
}
citedRuns[platformID][runID] = struct{}{}
if _, exists := citedArticles[platformID]; !exists {
citedArticles[platformID] = make(map[int64]struct{})
}
citedArticles[platformID][alias.ArticleID] = struct{}{}
}
}
aggregates[platformID] = item
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate citation ranking")
}
for platformID, item := range aggregates {
item.CitedAnswerCount = int64(len(citedRuns[platformID]))
item.CitedArticleCount = int64(len(citedArticles[platformID]))
aggregates[platformID] = item
}
items := make([]MonitoringCitationRanking, 0, len(aggregates))
for platformID, aggregate := range aggregates {
if aggregate.SaaSSourceCount == 0 {
@@ -3611,123 +3608,9 @@ func (s *MonitoringService) loadCitedArticles(
startDate, endDate time.Time,
aiPlatformID *string,
) ([]MonitoringCitedArticle, error) {
if questionIDs != nil && len(questionIDs) == 0 {
return []MonitoringCitedArticle{}, nil
}
platformQueryIDs := monitoringPlatformQueryIDs(aiPlatformID)
aliasIndex, err := loadMonitoringPublishedAliasIndex(ctx, s.monitoringPool, tenantID)
if err != nil {
return nil, err
}
var totalSampleCount int64
if err := s.monitoringPool.QueryRow(ctx, `
SELECT COUNT(*)
FROM question_monitor_runs r
WHERE r.tenant_id = $1
AND ($2::bigint = 0 OR r.brand_id = $2)
AND r.collector_type = $3
AND r.status = 'succeeded'
AND r.business_date BETWEEN $4::date AND $5::date
AND ($6::bigint[] IS NULL OR r.question_id = ANY($6))
AND ($7::text[] IS NULL OR r.ai_platform_id = ANY($7))
`, tenantID, brandID, monitoringCollectorType, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), nullableInt64Array(questionIDs), nullableStringArray(platformQueryIDs)).Scan(&totalSampleCount); err != nil {
return nil, response.ErrInternal(50041, "query_failed", "failed to count cited article denominator")
}
rows, err := s.monitoringPool.Query(ctx, `
SELECT
cf.cited_url,
cf.normalized_url,
NULLIF(cf.article_id, 0),
NULLIF(cf.cited_title, '')
FROM monitoring_citation_facts cf
JOIN question_monitor_runs r
ON r.tenant_id = cf.tenant_id AND r.id = cf.run_id
WHERE cf.tenant_id = $1
AND ($2::bigint = 0 OR cf.brand_id = $2)
AND r.ai_platform_id <> 'qwen'
AND r.collector_type = $3
AND r.status = 'succeeded'
AND r.business_date BETWEEN $4::date AND $5::date
AND ($6::bigint[] IS NULL OR r.question_id = ANY($6))
AND ($7::text[] IS NULL OR r.ai_platform_id = ANY($7))
`, tenantID, brandID, monitoringCollectorType, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), nullableInt64Array(questionIDs), nullableStringArray(platformQueryIDs))
if err != nil {
return nil, response.ErrInternal(50041, "query_failed", "failed to load cited articles")
}
defer rows.Close()
type citedArticleAggregate struct {
ArticleID int64
ArticleTitle string
PublishPlatform string
CitationCount int64
}
aggregates := make(map[int64]citedArticleAggregate)
var totalArticleCitationCount int64
for rows.Next() {
var citedURL string
var normalizedURL string
var factArticleID sql.NullInt64
var factCitedTitle sql.NullString
if scanErr := rows.Scan(&citedURL, &normalizedURL, &factArticleID, &factCitedTitle); scanErr != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse cited articles")
}
alias, ok := aliasIndex.Match(citedURL, normalizedURL)
articleID := factArticleID.Int64
if !factArticleID.Valid && ok {
articleID = alias.ArticleID
}
if articleID <= 0 {
continue
}
item := aggregates[articleID]
if item.ArticleID == 0 {
item.ArticleID = articleID
if ok && strings.TrimSpace(alias.ArticleTitle) != "" {
item.ArticleTitle = alias.ArticleTitle
} else if factCitedTitle.Valid && strings.TrimSpace(factCitedTitle.String) != "" {
item.ArticleTitle = factCitedTitle.String
} else {
item.ArticleTitle = "未命名文章"
}
if ok && strings.TrimSpace(alias.PublishPlatform) != "" {
item.PublishPlatform = alias.PublishPlatform
} else {
item.PublishPlatform = "已发布内容"
}
}
item.CitationCount++
totalArticleCitationCount++
aggregates[articleID] = item
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate cited articles")
}
items := make([]MonitoringCitedArticle, 0, len(aggregates))
for _, item := range aggregates {
items = append(items, MonitoringCitedArticle{
ArticleID: item.ArticleID,
ArticleTitle: item.ArticleTitle,
PublishPlatform: item.PublishPlatform,
CitationCount: item.CitationCount,
CitationRate: divideAsPointer(item.CitationCount, totalSampleCount),
SourceShare: divideAsPointer(item.CitationCount, totalArticleCitationCount),
})
}
sort.Slice(items, func(left, right int) bool {
if items[left].CitationCount != items[right].CitationCount {
return items[left].CitationCount > items[right].CitationCount
}
return items[left].ArticleID < items[right].ArticleID
})
return items, nil
pageReq := MonitoringPageRequest{Page: 1, PageSize: maxMonitoringListPageSize}
items, _, err := s.loadCitedArticlesPage(ctx, tenantID, brandID, questionIDs, startDate, endDate, aiPlatformID, pageReq)
return items, err
}
func (s *MonitoringService) loadCitedArticlesPage(
@@ -3761,94 +3644,238 @@ func (s *MonitoringService) loadCitedArticlesPage(
return nil, MonitoringPage{}, response.ErrInternal(50041, "query_failed", "failed to count cited article denominator")
}
type citedArticleStats struct {
Total int64
TotalCitationCount int64
sourceRows, totalArticleCitationCount, totalSources, err := s.loadCitedArticleSourceRows(ctx, tenantID, brandID, questionIDs, startDateText, endDateText, platformQueryIDs, pageReq)
if err != nil {
return nil, MonitoringPage{}, err
}
var stats citedArticleStats
if err := s.monitoringPool.QueryRow(ctx, `
WITH article_counts AS (
SELECT COALESCE(NULLIF(cf.article_id, 0), alias.article_id) AS article_id,
COUNT(*)::bigint AS citation_count
FROM monitoring_citation_facts cf
JOIN question_monitor_runs r
ON r.tenant_id = cf.tenant_id AND r.id = cf.run_id
LEFT JOIN monitoring_article_url_aliases alias
ON alias.tenant_id = cf.tenant_id
AND alias.normalized_url = cf.normalized_url
AND alias.confidence = 'high'
WHERE cf.tenant_id = $1
AND ($2::bigint = 0 OR cf.brand_id = $2)
AND r.ai_platform_id <> 'qwen'
AND r.collector_type = $3
AND r.status = 'succeeded'
AND r.business_date BETWEEN $4::date AND $5::date
AND ($6::bigint[] IS NULL OR r.question_id = ANY($6))
AND ($7::text[] IS NULL OR r.ai_platform_id = ANY($7))
AND COALESCE(NULLIF(cf.article_id, 0), alias.article_id) IS NOT NULL
GROUP BY COALESCE(NULLIF(cf.article_id, 0), alias.article_id)
)
SELECT COUNT(*)::bigint, COALESCE(SUM(citation_count), 0)::bigint
FROM article_counts
`, tenantID, brandID, monitoringCollectorType, startDateText, endDateText, nullableInt64Array(questionIDs), nullableStringArray(platformQueryIDs)).Scan(&stats.Total, &stats.TotalCitationCount); err != nil {
return nil, MonitoringPage{}, response.ErrInternal(50041, "query_failed", "failed to count cited articles")
}
page := monitoringPageFromRequest(pageReq, stats.Total)
if stats.Total == 0 {
page := monitoringPageFromRequest(pageReq, totalSources)
if len(sourceRows) == 0 {
return []MonitoringCitedArticle{}, page, nil
}
metaByKey, err := s.loadCitedArticleSourceMeta(ctx, tenantID, sourceRows)
if err != nil {
return nil, MonitoringPage{}, err
}
items := make([]MonitoringCitedArticle, 0, len(sourceRows))
for _, source := range sourceRows {
item := metaByKey[monitoringContentSourceKey(source.SourceType, source.SourceID)]
if item.SourceID == 0 {
item = MonitoringCitedArticle{
ArticleTitle: "未命名文章",
PublishPlatform: "外部文章",
SourceType: source.SourceType,
SourceID: source.SourceID,
}
}
item.CitationCount = source.CitationCount
item.CitationRate = divideAsPointer(source.CitationCount, totalSampleCount)
item.SourceShare = divideAsPointer(source.CitationCount, totalArticleCitationCount)
items = append(items, item)
}
return items, page, nil
}
type monitoringCitedArticleSourceRow struct {
SourceType string
SourceID int64
CitationCount int64
}
func (s *MonitoringService) loadCitedArticleSourceRows(
ctx context.Context,
tenantID, brandID int64,
questionIDs []int64,
startDateText, endDateText string,
platformQueryIDs []string,
pageReq MonitoringPageRequest,
) ([]monitoringCitedArticleSourceRow, int64, int64, error) {
rows, err := s.monitoringPool.Query(ctx, `
WITH article_counts AS (
WITH filtered_sources AS (
SELECT
COALESCE(NULLIF(cf.article_id, 0), alias.article_id) AS article_id,
COALESCE(MAX(alias.article_title_snapshot), MAX(NULLIF(cf.cited_title, '')), '未命名文章') AS article_title,
COALESCE(MAX(alias.publish_platform_name_snapshot), '已发布内容') AS publish_platform,
cf.content_source_type,
cf.content_source_id,
COUNT(*)::bigint AS citation_count
FROM monitoring_citation_facts cf
JOIN question_monitor_runs r
ON r.tenant_id = cf.tenant_id AND r.id = cf.run_id
LEFT JOIN monitoring_article_url_aliases alias
ON alias.tenant_id = cf.tenant_id
AND alias.normalized_url = cf.normalized_url
AND alias.confidence = 'high'
ON r.tenant_id = cf.tenant_id
AND r.id = cf.run_id
WHERE cf.tenant_id = $1
AND ($2::bigint = 0 OR cf.brand_id = $2)
AND r.ai_platform_id <> 'qwen'
AND cf.ai_platform_id <> 'qwen'
AND cf.business_date BETWEEN $4::date AND $5::date
AND cf.content_source_type IS NOT NULL
AND cf.content_source_id IS NOT NULL
AND r.collector_type = $3
AND r.status = 'succeeded'
AND r.business_date BETWEEN $4::date AND $5::date
AND ($6::bigint[] IS NULL OR r.question_id = ANY($6))
AND ($7::text[] IS NULL OR r.ai_platform_id = ANY($7))
AND COALESCE(NULLIF(cf.article_id, 0), alias.article_id) IS NOT NULL
GROUP BY COALESCE(NULLIF(cf.article_id, 0), alias.article_id)
GROUP BY cf.content_source_type, cf.content_source_id
),
counts AS (
SELECT
COALESCE(SUM(citation_count), 0)::bigint AS total_citation_count,
COUNT(*)::bigint AS total_source_count
FROM filtered_sources
),
paged_sources AS (
SELECT content_source_type, content_source_id, citation_count
FROM filtered_sources
ORDER BY citation_count DESC, content_source_type ASC, content_source_id ASC
LIMIT $8 OFFSET $9
)
SELECT article_id, article_title, publish_platform, citation_count
FROM article_counts
ORDER BY citation_count DESC, article_id ASC
LIMIT $8 OFFSET $9
SELECT
ps.content_source_type,
ps.content_source_id,
ps.citation_count,
counts.total_citation_count,
counts.total_source_count
FROM counts
LEFT JOIN paged_sources ps ON TRUE
`, tenantID, brandID, monitoringCollectorType, startDateText, endDateText, nullableInt64Array(questionIDs), nullableStringArray(platformQueryIDs), pageReq.PageSize, monitoringPageOffset(pageReq))
if err != nil {
return nil, MonitoringPage{}, response.ErrInternal(50041, "query_failed", "failed to load cited articles")
return nil, 0, 0, response.ErrInternal(50041, "query_failed", "failed to load cited articles")
}
defer rows.Close()
items := make([]MonitoringCitedArticle, 0, pageReq.PageSize)
items := make([]monitoringCitedArticleSourceRow, 0, pageReq.PageSize)
var totalCitationCount int64
var totalSourceCount int64
for rows.Next() {
var item MonitoringCitedArticle
if scanErr := rows.Scan(&item.ArticleID, &item.ArticleTitle, &item.PublishPlatform, &item.CitationCount); scanErr != nil {
return nil, MonitoringPage{}, response.ErrInternal(50041, "scan_failed", "failed to parse cited articles")
var sourceType sql.NullString
var sourceID sql.NullInt64
var citationCount sql.NullInt64
if scanErr := rows.Scan(
&sourceType,
&sourceID,
&citationCount,
&totalCitationCount,
&totalSourceCount,
); scanErr != nil {
return nil, 0, 0, response.ErrInternal(50041, "scan_failed", "failed to parse cited articles")
}
if !sourceType.Valid || !sourceID.Valid || !citationCount.Valid {
continue
}
item := monitoringCitedArticleSourceRow{
SourceType: strings.TrimSpace(sourceType.String),
SourceID: sourceID.Int64,
CitationCount: citationCount.Int64,
}
if item.SourceType == "" || item.SourceID <= 0 {
continue
}
item.CitationRate = divideAsPointer(item.CitationCount, totalSampleCount)
item.SourceShare = divideAsPointer(item.CitationCount, stats.TotalCitationCount)
items = append(items, item)
}
if err := rows.Err(); err != nil {
return nil, MonitoringPage{}, response.ErrInternal(50041, "scan_failed", "failed to iterate cited articles")
return nil, 0, 0, response.ErrInternal(50041, "scan_failed", "failed to iterate cited articles")
}
return items, page, nil
return items, totalCitationCount, totalSourceCount, nil
}
func (s *MonitoringService) loadCitedArticleSourceMeta(ctx context.Context, tenantID int64, sources []monitoringCitedArticleSourceRow) (map[string]MonitoringCitedArticle, error) {
result := make(map[string]MonitoringCitedArticle, len(sources))
publishedIDs := make([]int64, 0, len(sources))
manualIDs := make([]int64, 0, len(sources))
for _, source := range sources {
switch source.SourceType {
case "published_article":
publishedIDs = append(publishedIDs, source.SourceID)
case "manual_mark":
manualIDs = append(manualIDs, source.SourceID)
}
}
if len(publishedIDs) > 0 {
rows, err := s.monitoringPool.Query(ctx, `
SELECT DISTINCT ON (article_id)
article_id,
COALESCE(NULLIF(article_title_snapshot, ''), '未命名文章') AS article_title,
COALESCE(NULLIF(publish_platform_name_snapshot, ''), '已发布内容') AS publish_platform,
COALESCE(NULLIF(original_url, ''), normalized_url) AS source_url
FROM monitoring_article_url_aliases
WHERE tenant_id = $1
AND confidence = 'high'
AND article_id = ANY($2)
ORDER BY article_id, updated_at DESC, id DESC
`, tenantID, publishedIDs)
if err != nil {
return nil, response.ErrInternal(50041, "query_failed", "failed to load cited article metadata")
}
defer rows.Close()
for rows.Next() {
var articleID int64
var item MonitoringCitedArticle
if scanErr := rows.Scan(&articleID, &item.ArticleTitle, &item.PublishPlatform, &item.SourceURL); scanErr != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse cited article metadata")
}
value := articleID
item.ArticleID = &value
item.ArticleTitle = repairMonitoringMojibake(item.ArticleTitle)
item.PublishPlatform = repairMonitoringMojibake(item.PublishPlatform)
item.SourceType = "published_article"
item.SourceID = articleID
result[monitoringContentSourceKey(item.SourceType, item.SourceID)] = item
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate cited article metadata")
}
}
if len(manualIDs) > 0 {
rows, err := s.monitoringPool.Query(ctx, `
SELECT
id,
COALESCE(NULLIF(article_title, ''), '未命名文章') AS article_title,
COALESCE(NULLIF(publish_platform, ''), '外部文章') AS publish_platform,
COALESCE(NULLIF(original_url, ''), normalized_url) AS source_url
FROM monitoring_user_marked_articles
WHERE tenant_id = $1
AND id = ANY($2)
`, tenantID, manualIDs)
if err != nil {
return nil, response.ErrInternal(50041, "query_failed", "failed to load marked article metadata")
}
defer rows.Close()
for rows.Next() {
var item MonitoringCitedArticle
if scanErr := rows.Scan(&item.SourceID, &item.ArticleTitle, &item.PublishPlatform, &item.SourceURL); scanErr != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse marked article metadata")
}
item.ArticleTitle = repairMonitoringMojibake(item.ArticleTitle)
item.PublishPlatform = repairMonitoringMojibake(item.PublishPlatform)
item.SourceType = "manual_mark"
result[monitoringContentSourceKey(item.SourceType, item.SourceID)] = item
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate marked article metadata")
}
}
return result, nil
}
func monitoringContentSourceKey(sourceType string, sourceID int64) string {
return strings.TrimSpace(sourceType) + ":" + strconv.FormatInt(sourceID, 10)
}
func monitoringAliasContentKey(alias monitoringPublishedAlias) string {
sourceType := strings.TrimSpace(alias.SourceType)
sourceID := alias.SourceID
if sourceType == "" {
sourceType = "published_article"
}
if sourceID == 0 && alias.ArticleID > 0 {
sourceID = alias.ArticleID
}
if sourceID > 0 {
return sourceType + ":" + strconv.FormatInt(sourceID, 10)
}
key := strings.TrimSpace(alias.NormalizedURL)
if key == "" {
key = strings.TrimSpace(alias.OriginalURL)
}
return sourceType + ":url:" + key
}
func (s *MonitoringService) loadQuestionText(ctx context.Context, tenantID, brandID, questionID int64) (string, error) {
@@ -4130,10 +4157,35 @@ func (s *MonitoringService) loadCitationsForRuns(ctx context.Context, tenantID i
}
var articleID *int64
var articleTitle *string
var sourceType *string
var sourceID *int64
var sourceURL *string
if alias, ok := aliasIndex.Match(citedURL, normalizedURL); ok {
articleID = &alias.ArticleID
if alias.ArticleID > 0 {
value := alias.ArticleID
articleID = &value
}
title := repairMonitoringMojibake(alias.ArticleTitle)
articleTitle = &title
aliasSourceType := strings.TrimSpace(alias.SourceType)
if aliasSourceType == "" {
aliasSourceType = "published_article"
}
sourceType = &aliasSourceType
aliasSourceID := alias.SourceID
if aliasSourceID == 0 && alias.ArticleID > 0 {
aliasSourceID = alias.ArticleID
}
if aliasSourceID > 0 {
sourceID = &aliasSourceID
}
aliasSourceURL := strings.TrimSpace(alias.OriginalURL)
if aliasSourceURL == "" {
aliasSourceURL = strings.TrimSpace(alias.NormalizedURL)
}
if aliasSourceURL != "" {
sourceURL = &aliasSourceURL
}
resolutionStatus = "resolved"
resolutionConfidence = "high"
}
@@ -4144,6 +4196,9 @@ func (s *MonitoringService) loadCitationsForRuns(ctx context.Context, tenantID i
SiteKey: siteKey,
FaviconURL: faviconURL(siteKey),
ArticleID: articleID,
SourceType: sourceType,
SourceID: sourceID,
SourceURL: sourceURL,
ArticleTitle: articleTitle,
ResolutionStatus: resolutionStatus,
ResolutionConfidence: resolutionConfidence,
@@ -4161,21 +4216,16 @@ func (s *MonitoringService) loadQuestionCitationAnalysis(ctx context.Context, te
return []MonitoringQuestionCitationStats{}, nil
}
aliasIndex, err := loadMonitoringPublishedAliasIndex(ctx, s.monitoringPool, tenantID)
if err != nil {
return nil, err
}
rows, err := s.monitoringPool.Query(ctx, `
WITH filtered_facts AS (
SELECT
r.ai_platform_id,
cf.id,
cf.cited_url,
cf.normalized_url,
cf.host,
cf.registrable_domain,
cf.site_key
cf.site_key,
cf.content_source_type,
cf.content_source_id
FROM question_monitor_runs r
JOIN monitoring_citation_facts cf
ON cf.tenant_id = r.tenant_id AND cf.run_id = r.id
@@ -4220,11 +4270,11 @@ func (s *MonitoringService) loadQuestionCitationAnalysis(ctx context.Context, te
)
SELECT
ff.ai_platform_id,
ff.cited_url,
ff.normalized_url,
COALESCE(dm.mapped_site_name, ff.site_key, ff.registrable_domain) AS site_name,
COALESCE(dm.mapped_site_key, ff.site_key) AS site_key,
COALESCE(dm.mapped_domain, ff.site_key, ff.registrable_domain) AS site_domain
COALESCE(dm.mapped_domain, ff.site_key, ff.registrable_domain) AS site_domain,
ff.content_source_type,
ff.content_source_id
FROM filtered_facts ff
LEFT JOIN domain_mappings dm
ON dm.host = ff.host
@@ -4246,12 +4296,12 @@ func (s *MonitoringService) loadQuestionCitationAnalysis(ctx context.Context, te
totals := make(map[string]int64)
for rows.Next() {
var platformID string
var citedURL string
var normalizedURL string
var siteName string
var siteKey string
var siteDomain string
if scanErr := rows.Scan(&platformID, &citedURL, &normalizedURL, &siteName, &siteKey, &siteDomain); scanErr != nil {
var sourceType sql.NullString
var sourceID sql.NullInt64
if scanErr := rows.Scan(&platformID, &siteName, &siteKey, &siteDomain, &sourceType, &sourceID); scanErr != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse question citation analysis")
}
platformID = normalizeMonitoringPlatformID(platformID)
@@ -4273,10 +4323,8 @@ func (s *MonitoringService) loadQuestionCitationAnalysis(ctx context.Context, te
}
}
item.CitationCount++
if platformID != "qwen" {
if _, ok := aliasIndex.Match(citedURL, normalizedURL); ok {
item.ContentCitationCount++
}
if platformID != "qwen" && sourceType.Valid && strings.TrimSpace(sourceType.String) != "" && sourceID.Valid {
item.ContentCitationCount++
}
aggregates[key] = item
totals[platformID]++
@@ -4311,16 +4359,11 @@ func (s *MonitoringService) loadQuestionContentCitations(ctx context.Context, te
return []MonitoringQuestionContentCitation{}, nil
}
aliasIndex, err := loadMonitoringPublishedAliasIndex(ctx, s.monitoringPool, tenantID)
if err != nil {
return nil, err
}
rows, err := s.monitoringPool.Query(ctx, `
SELECT
r.ai_platform_id,
cf.cited_url,
cf.normalized_url
cf.content_source_type,
cf.content_source_id
FROM question_monitor_runs r
JOIN monitoring_citation_facts cf
ON cf.tenant_id = r.tenant_id AND cf.run_id = r.id
@@ -4336,16 +4379,17 @@ func (s *MonitoringService) loadQuestionContentCitations(ctx context.Context, te
type contentCitationKey struct {
PlatformID string
ArticleID int64
SourceType string
SourceID int64
}
aggregates := make(map[contentCitationKey]MonitoringQuestionContentCitation)
totals := make(map[string]int64)
for rows.Next() {
var platformID string
var citedURL string
var normalizedURL string
if scanErr := rows.Scan(&platformID, &citedURL, &normalizedURL); scanErr != nil {
var sourceType sql.NullString
var sourceID sql.NullInt64
if scanErr := rows.Scan(&platformID, &sourceType, &sourceID); scanErr != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse content citations")
}
platformID = normalizeMonitoringPlatformID(platformID)
@@ -4353,22 +4397,22 @@ func (s *MonitoringService) loadQuestionContentCitations(ctx context.Context, te
continue
}
totals[platformID]++
alias, ok := aliasIndex.Match(citedURL, normalizedURL)
if !ok {
normalizedSourceType := strings.TrimSpace(sourceType.String)
if !sourceType.Valid || normalizedSourceType == "" || !sourceID.Valid || sourceID.Int64 <= 0 {
continue
}
key := contentCitationKey{
PlatformID: platformID,
ArticleID: alias.ArticleID,
SourceType: normalizedSourceType,
SourceID: sourceID.Int64,
}
item := aggregates[key]
if item.ArticleID == 0 {
if item.SourceID == 0 {
item = MonitoringQuestionContentCitation{
ArticleID: alias.ArticleID,
ArticleTitle: alias.ArticleTitle,
PublishPlatform: alias.PublishPlatform,
AIPlatformID: platformID,
PlatformName: platformDisplayName(platformID),
SourceType: normalizedSourceType,
SourceID: sourceID.Int64,
AIPlatformID: platformID,
PlatformName: platformDisplayName(platformID),
}
}
item.CitationCount++
@@ -4377,9 +4421,36 @@ func (s *MonitoringService) loadQuestionContentCitations(ctx context.Context, te
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate content citations")
}
if len(aggregates) == 0 {
return []MonitoringQuestionContentCitation{}, nil
}
sourceRows := make([]monitoringCitedArticleSourceRow, 0, len(aggregates))
for key := range aggregates {
sourceRows = append(sourceRows, monitoringCitedArticleSourceRow{
SourceType: key.SourceType,
SourceID: key.SourceID,
})
}
metaByKey, err := s.loadCitedArticleSourceMeta(ctx, tenantID, sourceRows)
if err != nil {
return nil, err
}
items := make([]MonitoringQuestionContentCitation, 0, len(aggregates))
for _, item := range aggregates {
for key, item := range aggregates {
if meta := metaByKey[monitoringContentSourceKey(key.SourceType, key.SourceID)]; meta.SourceID > 0 {
item.ArticleID = meta.ArticleID
item.ArticleTitle = meta.ArticleTitle
item.PublishPlatform = meta.PublishPlatform
item.SourceURL = meta.SourceURL
}
if strings.TrimSpace(item.ArticleTitle) == "" {
item.ArticleTitle = "未命名文章"
}
if strings.TrimSpace(item.PublishPlatform) == "" {
item.PublishPlatform = "外部文章"
}
item.CitationRate = divideAsPointer(item.CitationCount, totals[item.AIPlatformID])
items = append(items, item)
}
@@ -4390,7 +4461,10 @@ func (s *MonitoringService) loadQuestionContentCitations(ctx context.Context, te
if items[left].CitationCount != items[right].CitationCount {
return items[left].CitationCount > items[right].CitationCount
}
return items[left].ArticleID < items[right].ArticleID
if items[left].SourceType != items[right].SourceType {
return items[left].SourceType < items[right].SourceType
}
return items[left].SourceID < items[right].SourceID
})
return items, nil