feat: add monitoring marked articles and retention updates
This commit is contained in:
@@ -10,6 +10,7 @@ import (
|
||||
"fmt"
|
||||
"math/big"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -192,9 +193,12 @@ type MonitoringCitationRanking struct {
|
||||
}
|
||||
|
||||
type MonitoringCitedArticle struct {
|
||||
ArticleID int64 `json:"article_id"`
|
||||
ArticleID *int64 `json:"article_id"`
|
||||
ArticleTitle string `json:"article_title"`
|
||||
PublishPlatform string `json:"publish_platform"`
|
||||
SourceType string `json:"source_type"`
|
||||
SourceID int64 `json:"source_id"`
|
||||
SourceURL string `json:"source_url"`
|
||||
CitationCount int64 `json:"citation_count"`
|
||||
CitationRate *float64 `json:"citation_rate"`
|
||||
SourceShare *float64 `json:"source_share"`
|
||||
@@ -237,6 +241,9 @@ type MonitoringQuestionDetailCitation struct {
|
||||
SiteKey string `json:"site_key"`
|
||||
FaviconURL string `json:"favicon_url"`
|
||||
ArticleID *int64 `json:"article_id"`
|
||||
SourceType *string `json:"source_type,omitempty"`
|
||||
SourceID *int64 `json:"source_id,omitempty"`
|
||||
SourceURL *string `json:"source_url,omitempty"`
|
||||
ArticleTitle *string `json:"article_title,omitempty"`
|
||||
ResolutionStatus string `json:"resolution_status"`
|
||||
ResolutionConfidence string `json:"resolution_confidence"`
|
||||
@@ -254,9 +261,12 @@ type MonitoringQuestionCitationStats struct {
|
||||
}
|
||||
|
||||
type MonitoringQuestionContentCitation struct {
|
||||
ArticleID int64 `json:"article_id"`
|
||||
ArticleID *int64 `json:"article_id"`
|
||||
ArticleTitle string `json:"article_title"`
|
||||
PublishPlatform string `json:"publish_platform"`
|
||||
SourceType string `json:"source_type"`
|
||||
SourceID int64 `json:"source_id"`
|
||||
SourceURL string `json:"source_url"`
|
||||
AIPlatformID string `json:"ai_platform_id"`
|
||||
PlatformName string `json:"platform_name"`
|
||||
CitationCount int64 `json:"citation_count"`
|
||||
@@ -3458,11 +3468,6 @@ func (s *MonitoringService) loadCitationRanking(
|
||||
}
|
||||
platformQueryIDs := monitoringPlatformQueryIDs(aiPlatformID)
|
||||
|
||||
aliasIndex, err := loadMonitoringPublishedAliasIndex(ctx, s.monitoringPool, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
type citationRankingAggregate struct {
|
||||
CitedAnswerCount int64
|
||||
CitedArticleCount int64
|
||||
@@ -3473,18 +3478,57 @@ func (s *MonitoringService) loadCitationRanking(
|
||||
|
||||
aggregates := make(map[string]citationRankingAggregate)
|
||||
rows, err := s.monitoringPool.Query(ctx, `
|
||||
WITH filtered_runs AS (
|
||||
SELECT id, ai_platform_id
|
||||
FROM question_monitor_runs
|
||||
WHERE tenant_id = $1
|
||||
AND ($2::bigint = 0 OR brand_id = $2)
|
||||
AND collector_type = $3
|
||||
AND status = 'succeeded'
|
||||
AND business_date BETWEEN $4::date AND $5::date
|
||||
AND ($6::bigint[] IS NULL OR question_id = ANY($6))
|
||||
AND ($7::text[] IS NULL OR ai_platform_id = ANY($7))
|
||||
),
|
||||
sample_counts AS (
|
||||
SELECT ai_platform_id, COUNT(*)::bigint AS sample_count
|
||||
FROM filtered_runs
|
||||
GROUP BY ai_platform_id
|
||||
),
|
||||
citation_counts AS (
|
||||
SELECT
|
||||
fr.ai_platform_id,
|
||||
COUNT(cf.id)::bigint AS citation_source_count,
|
||||
COUNT(cf.id) FILTER (
|
||||
WHERE fr.ai_platform_id <> 'qwen'
|
||||
AND cf.content_source_type IS NOT NULL
|
||||
AND cf.content_source_id IS NOT NULL
|
||||
)::bigint AS saas_source_count,
|
||||
COUNT(DISTINCT cf.run_id) FILTER (
|
||||
WHERE fr.ai_platform_id <> 'qwen'
|
||||
AND cf.content_source_type IS NOT NULL
|
||||
AND cf.content_source_id IS NOT NULL
|
||||
)::bigint AS cited_answer_count,
|
||||
COUNT(DISTINCT cf.content_source_type || ':' || cf.content_source_id::text) FILTER (
|
||||
WHERE fr.ai_platform_id <> 'qwen'
|
||||
AND cf.content_source_type IS NOT NULL
|
||||
AND cf.content_source_id IS NOT NULL
|
||||
)::bigint AS cited_article_count
|
||||
FROM filtered_runs fr
|
||||
JOIN monitoring_citation_facts cf
|
||||
ON cf.tenant_id = $1
|
||||
AND cf.run_id = fr.id
|
||||
AND cf.business_date BETWEEN $4::date AND $5::date
|
||||
GROUP BY fr.ai_platform_id
|
||||
)
|
||||
SELECT
|
||||
r.ai_platform_id,
|
||||
COUNT(*) AS sample_count
|
||||
FROM question_monitor_runs r
|
||||
WHERE r.tenant_id = $1
|
||||
AND ($2::bigint = 0 OR r.brand_id = $2)
|
||||
AND r.collector_type = $3
|
||||
AND r.status = 'succeeded'
|
||||
AND r.business_date BETWEEN $4::date AND $5::date
|
||||
AND ($6::bigint[] IS NULL OR r.question_id = ANY($6))
|
||||
AND ($7::text[] IS NULL OR r.ai_platform_id = ANY($7))
|
||||
GROUP BY r.ai_platform_id
|
||||
sc.ai_platform_id,
|
||||
sc.sample_count,
|
||||
COALESCE(cc.citation_source_count, 0)::bigint,
|
||||
COALESCE(cc.saas_source_count, 0)::bigint,
|
||||
COALESCE(cc.cited_answer_count, 0)::bigint,
|
||||
COALESCE(cc.cited_article_count, 0)::bigint
|
||||
FROM sample_counts sc
|
||||
LEFT JOIN citation_counts cc ON cc.ai_platform_id = sc.ai_platform_id
|
||||
`, tenantID, brandID, monitoringCollectorType, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), nullableInt64Array(questionIDs), nullableStringArray(platformQueryIDs))
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(50041, "query_failed", "failed to load citation ranking")
|
||||
@@ -3494,7 +3538,18 @@ func (s *MonitoringService) loadCitationRanking(
|
||||
for rows.Next() {
|
||||
var platformID string
|
||||
var sampleCount int64
|
||||
if scanErr := rows.Scan(&platformID, &sampleCount); scanErr != nil {
|
||||
var citationSourceCount int64
|
||||
var saasSourceCount int64
|
||||
var citedAnswerCount int64
|
||||
var citedArticleCount int64
|
||||
if scanErr := rows.Scan(
|
||||
&platformID,
|
||||
&sampleCount,
|
||||
&citationSourceCount,
|
||||
&saasSourceCount,
|
||||
&citedAnswerCount,
|
||||
&citedArticleCount,
|
||||
); scanErr != nil {
|
||||
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse citation ranking")
|
||||
}
|
||||
platformID = normalizeMonitoringPlatformID(platformID)
|
||||
@@ -3503,74 +3558,16 @@ func (s *MonitoringService) loadCitationRanking(
|
||||
}
|
||||
item := aggregates[platformID]
|
||||
item.SampleCount += sampleCount
|
||||
item.CitationSourceCount += citationSourceCount
|
||||
item.SaaSSourceCount += saasSourceCount
|
||||
item.CitedAnswerCount += citedAnswerCount
|
||||
item.CitedArticleCount += citedArticleCount
|
||||
aggregates[platformID] = item
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate citation ranking")
|
||||
}
|
||||
|
||||
citedRuns := make(map[string]map[int64]struct{})
|
||||
citedArticles := make(map[string]map[int64]struct{})
|
||||
rows, err = s.monitoringPool.Query(ctx, `
|
||||
SELECT
|
||||
r.ai_platform_id,
|
||||
cf.run_id,
|
||||
cf.cited_url,
|
||||
cf.normalized_url
|
||||
FROM question_monitor_runs r
|
||||
JOIN monitoring_citation_facts cf
|
||||
ON cf.tenant_id = r.tenant_id AND cf.run_id = r.id
|
||||
WHERE r.tenant_id = $1
|
||||
AND ($2::bigint = 0 OR r.brand_id = $2)
|
||||
AND r.collector_type = $3
|
||||
AND r.status = 'succeeded'
|
||||
AND r.business_date BETWEEN $4::date AND $5::date
|
||||
AND ($6::bigint[] IS NULL OR r.question_id = ANY($6))
|
||||
AND ($7::text[] IS NULL OR r.ai_platform_id = ANY($7))
|
||||
`, tenantID, brandID, monitoringCollectorType, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), nullableInt64Array(questionIDs), nullableStringArray(platformQueryIDs))
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(50041, "query_failed", "failed to load citation ranking")
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var platformID string
|
||||
var runID int64
|
||||
var citedURL string
|
||||
var normalizedURL string
|
||||
if scanErr := rows.Scan(&platformID, &runID, &citedURL, &normalizedURL); scanErr != nil {
|
||||
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse citation ranking")
|
||||
}
|
||||
platformID = normalizeMonitoringPlatformID(platformID)
|
||||
if platformID == "" {
|
||||
continue
|
||||
}
|
||||
item := aggregates[platformID]
|
||||
item.CitationSourceCount++
|
||||
if platformID != "qwen" {
|
||||
if alias, ok := aliasIndex.Match(citedURL, normalizedURL); ok {
|
||||
item.SaaSSourceCount++
|
||||
if _, exists := citedRuns[platformID]; !exists {
|
||||
citedRuns[platformID] = make(map[int64]struct{})
|
||||
}
|
||||
citedRuns[platformID][runID] = struct{}{}
|
||||
if _, exists := citedArticles[platformID]; !exists {
|
||||
citedArticles[platformID] = make(map[int64]struct{})
|
||||
}
|
||||
citedArticles[platformID][alias.ArticleID] = struct{}{}
|
||||
}
|
||||
}
|
||||
aggregates[platformID] = item
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate citation ranking")
|
||||
}
|
||||
for platformID, item := range aggregates {
|
||||
item.CitedAnswerCount = int64(len(citedRuns[platformID]))
|
||||
item.CitedArticleCount = int64(len(citedArticles[platformID]))
|
||||
aggregates[platformID] = item
|
||||
}
|
||||
|
||||
items := make([]MonitoringCitationRanking, 0, len(aggregates))
|
||||
for platformID, aggregate := range aggregates {
|
||||
if aggregate.SaaSSourceCount == 0 {
|
||||
@@ -3611,123 +3608,9 @@ func (s *MonitoringService) loadCitedArticles(
|
||||
startDate, endDate time.Time,
|
||||
aiPlatformID *string,
|
||||
) ([]MonitoringCitedArticle, error) {
|
||||
if questionIDs != nil && len(questionIDs) == 0 {
|
||||
return []MonitoringCitedArticle{}, nil
|
||||
}
|
||||
platformQueryIDs := monitoringPlatformQueryIDs(aiPlatformID)
|
||||
|
||||
aliasIndex, err := loadMonitoringPublishedAliasIndex(ctx, s.monitoringPool, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var totalSampleCount int64
|
||||
if err := s.monitoringPool.QueryRow(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM question_monitor_runs r
|
||||
WHERE r.tenant_id = $1
|
||||
AND ($2::bigint = 0 OR r.brand_id = $2)
|
||||
AND r.collector_type = $3
|
||||
AND r.status = 'succeeded'
|
||||
AND r.business_date BETWEEN $4::date AND $5::date
|
||||
AND ($6::bigint[] IS NULL OR r.question_id = ANY($6))
|
||||
AND ($7::text[] IS NULL OR r.ai_platform_id = ANY($7))
|
||||
`, tenantID, brandID, monitoringCollectorType, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), nullableInt64Array(questionIDs), nullableStringArray(platformQueryIDs)).Scan(&totalSampleCount); err != nil {
|
||||
return nil, response.ErrInternal(50041, "query_failed", "failed to count cited article denominator")
|
||||
}
|
||||
|
||||
rows, err := s.monitoringPool.Query(ctx, `
|
||||
SELECT
|
||||
cf.cited_url,
|
||||
cf.normalized_url,
|
||||
NULLIF(cf.article_id, 0),
|
||||
NULLIF(cf.cited_title, '')
|
||||
FROM monitoring_citation_facts cf
|
||||
JOIN question_monitor_runs r
|
||||
ON r.tenant_id = cf.tenant_id AND r.id = cf.run_id
|
||||
WHERE cf.tenant_id = $1
|
||||
AND ($2::bigint = 0 OR cf.brand_id = $2)
|
||||
AND r.ai_platform_id <> 'qwen'
|
||||
AND r.collector_type = $3
|
||||
AND r.status = 'succeeded'
|
||||
AND r.business_date BETWEEN $4::date AND $5::date
|
||||
AND ($6::bigint[] IS NULL OR r.question_id = ANY($6))
|
||||
AND ($7::text[] IS NULL OR r.ai_platform_id = ANY($7))
|
||||
`, tenantID, brandID, monitoringCollectorType, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), nullableInt64Array(questionIDs), nullableStringArray(platformQueryIDs))
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(50041, "query_failed", "failed to load cited articles")
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
type citedArticleAggregate struct {
|
||||
ArticleID int64
|
||||
ArticleTitle string
|
||||
PublishPlatform string
|
||||
CitationCount int64
|
||||
}
|
||||
aggregates := make(map[int64]citedArticleAggregate)
|
||||
var totalArticleCitationCount int64
|
||||
for rows.Next() {
|
||||
var citedURL string
|
||||
var normalizedURL string
|
||||
var factArticleID sql.NullInt64
|
||||
var factCitedTitle sql.NullString
|
||||
if scanErr := rows.Scan(&citedURL, &normalizedURL, &factArticleID, &factCitedTitle); scanErr != nil {
|
||||
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse cited articles")
|
||||
}
|
||||
alias, ok := aliasIndex.Match(citedURL, normalizedURL)
|
||||
|
||||
articleID := factArticleID.Int64
|
||||
if !factArticleID.Valid && ok {
|
||||
articleID = alias.ArticleID
|
||||
}
|
||||
if articleID <= 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
item := aggregates[articleID]
|
||||
if item.ArticleID == 0 {
|
||||
item.ArticleID = articleID
|
||||
if ok && strings.TrimSpace(alias.ArticleTitle) != "" {
|
||||
item.ArticleTitle = alias.ArticleTitle
|
||||
} else if factCitedTitle.Valid && strings.TrimSpace(factCitedTitle.String) != "" {
|
||||
item.ArticleTitle = factCitedTitle.String
|
||||
} else {
|
||||
item.ArticleTitle = "未命名文章"
|
||||
}
|
||||
if ok && strings.TrimSpace(alias.PublishPlatform) != "" {
|
||||
item.PublishPlatform = alias.PublishPlatform
|
||||
} else {
|
||||
item.PublishPlatform = "已发布内容"
|
||||
}
|
||||
}
|
||||
item.CitationCount++
|
||||
totalArticleCitationCount++
|
||||
aggregates[articleID] = item
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate cited articles")
|
||||
}
|
||||
|
||||
items := make([]MonitoringCitedArticle, 0, len(aggregates))
|
||||
for _, item := range aggregates {
|
||||
items = append(items, MonitoringCitedArticle{
|
||||
ArticleID: item.ArticleID,
|
||||
ArticleTitle: item.ArticleTitle,
|
||||
PublishPlatform: item.PublishPlatform,
|
||||
CitationCount: item.CitationCount,
|
||||
CitationRate: divideAsPointer(item.CitationCount, totalSampleCount),
|
||||
SourceShare: divideAsPointer(item.CitationCount, totalArticleCitationCount),
|
||||
})
|
||||
}
|
||||
sort.Slice(items, func(left, right int) bool {
|
||||
if items[left].CitationCount != items[right].CitationCount {
|
||||
return items[left].CitationCount > items[right].CitationCount
|
||||
}
|
||||
return items[left].ArticleID < items[right].ArticleID
|
||||
})
|
||||
|
||||
return items, nil
|
||||
pageReq := MonitoringPageRequest{Page: 1, PageSize: maxMonitoringListPageSize}
|
||||
items, _, err := s.loadCitedArticlesPage(ctx, tenantID, brandID, questionIDs, startDate, endDate, aiPlatformID, pageReq)
|
||||
return items, err
|
||||
}
|
||||
|
||||
func (s *MonitoringService) loadCitedArticlesPage(
|
||||
@@ -3761,94 +3644,238 @@ func (s *MonitoringService) loadCitedArticlesPage(
|
||||
return nil, MonitoringPage{}, response.ErrInternal(50041, "query_failed", "failed to count cited article denominator")
|
||||
}
|
||||
|
||||
type citedArticleStats struct {
|
||||
Total int64
|
||||
TotalCitationCount int64
|
||||
sourceRows, totalArticleCitationCount, totalSources, err := s.loadCitedArticleSourceRows(ctx, tenantID, brandID, questionIDs, startDateText, endDateText, platformQueryIDs, pageReq)
|
||||
if err != nil {
|
||||
return nil, MonitoringPage{}, err
|
||||
}
|
||||
var stats citedArticleStats
|
||||
if err := s.monitoringPool.QueryRow(ctx, `
|
||||
WITH article_counts AS (
|
||||
SELECT COALESCE(NULLIF(cf.article_id, 0), alias.article_id) AS article_id,
|
||||
COUNT(*)::bigint AS citation_count
|
||||
FROM monitoring_citation_facts cf
|
||||
JOIN question_monitor_runs r
|
||||
ON r.tenant_id = cf.tenant_id AND r.id = cf.run_id
|
||||
LEFT JOIN monitoring_article_url_aliases alias
|
||||
ON alias.tenant_id = cf.tenant_id
|
||||
AND alias.normalized_url = cf.normalized_url
|
||||
AND alias.confidence = 'high'
|
||||
WHERE cf.tenant_id = $1
|
||||
AND ($2::bigint = 0 OR cf.brand_id = $2)
|
||||
AND r.ai_platform_id <> 'qwen'
|
||||
AND r.collector_type = $3
|
||||
AND r.status = 'succeeded'
|
||||
AND r.business_date BETWEEN $4::date AND $5::date
|
||||
AND ($6::bigint[] IS NULL OR r.question_id = ANY($6))
|
||||
AND ($7::text[] IS NULL OR r.ai_platform_id = ANY($7))
|
||||
AND COALESCE(NULLIF(cf.article_id, 0), alias.article_id) IS NOT NULL
|
||||
GROUP BY COALESCE(NULLIF(cf.article_id, 0), alias.article_id)
|
||||
)
|
||||
SELECT COUNT(*)::bigint, COALESCE(SUM(citation_count), 0)::bigint
|
||||
FROM article_counts
|
||||
`, tenantID, brandID, monitoringCollectorType, startDateText, endDateText, nullableInt64Array(questionIDs), nullableStringArray(platformQueryIDs)).Scan(&stats.Total, &stats.TotalCitationCount); err != nil {
|
||||
return nil, MonitoringPage{}, response.ErrInternal(50041, "query_failed", "failed to count cited articles")
|
||||
}
|
||||
|
||||
page := monitoringPageFromRequest(pageReq, stats.Total)
|
||||
if stats.Total == 0 {
|
||||
page := monitoringPageFromRequest(pageReq, totalSources)
|
||||
if len(sourceRows) == 0 {
|
||||
return []MonitoringCitedArticle{}, page, nil
|
||||
}
|
||||
|
||||
metaByKey, err := s.loadCitedArticleSourceMeta(ctx, tenantID, sourceRows)
|
||||
if err != nil {
|
||||
return nil, MonitoringPage{}, err
|
||||
}
|
||||
items := make([]MonitoringCitedArticle, 0, len(sourceRows))
|
||||
for _, source := range sourceRows {
|
||||
item := metaByKey[monitoringContentSourceKey(source.SourceType, source.SourceID)]
|
||||
if item.SourceID == 0 {
|
||||
item = MonitoringCitedArticle{
|
||||
ArticleTitle: "未命名文章",
|
||||
PublishPlatform: "外部文章",
|
||||
SourceType: source.SourceType,
|
||||
SourceID: source.SourceID,
|
||||
}
|
||||
}
|
||||
item.CitationCount = source.CitationCount
|
||||
item.CitationRate = divideAsPointer(source.CitationCount, totalSampleCount)
|
||||
item.SourceShare = divideAsPointer(source.CitationCount, totalArticleCitationCount)
|
||||
items = append(items, item)
|
||||
}
|
||||
|
||||
return items, page, nil
|
||||
}
|
||||
|
||||
type monitoringCitedArticleSourceRow struct {
|
||||
SourceType string
|
||||
SourceID int64
|
||||
CitationCount int64
|
||||
}
|
||||
|
||||
func (s *MonitoringService) loadCitedArticleSourceRows(
|
||||
ctx context.Context,
|
||||
tenantID, brandID int64,
|
||||
questionIDs []int64,
|
||||
startDateText, endDateText string,
|
||||
platformQueryIDs []string,
|
||||
pageReq MonitoringPageRequest,
|
||||
) ([]monitoringCitedArticleSourceRow, int64, int64, error) {
|
||||
rows, err := s.monitoringPool.Query(ctx, `
|
||||
WITH article_counts AS (
|
||||
WITH filtered_sources AS (
|
||||
SELECT
|
||||
COALESCE(NULLIF(cf.article_id, 0), alias.article_id) AS article_id,
|
||||
COALESCE(MAX(alias.article_title_snapshot), MAX(NULLIF(cf.cited_title, '')), '未命名文章') AS article_title,
|
||||
COALESCE(MAX(alias.publish_platform_name_snapshot), '已发布内容') AS publish_platform,
|
||||
cf.content_source_type,
|
||||
cf.content_source_id,
|
||||
COUNT(*)::bigint AS citation_count
|
||||
FROM monitoring_citation_facts cf
|
||||
JOIN question_monitor_runs r
|
||||
ON r.tenant_id = cf.tenant_id AND r.id = cf.run_id
|
||||
LEFT JOIN monitoring_article_url_aliases alias
|
||||
ON alias.tenant_id = cf.tenant_id
|
||||
AND alias.normalized_url = cf.normalized_url
|
||||
AND alias.confidence = 'high'
|
||||
ON r.tenant_id = cf.tenant_id
|
||||
AND r.id = cf.run_id
|
||||
WHERE cf.tenant_id = $1
|
||||
AND ($2::bigint = 0 OR cf.brand_id = $2)
|
||||
AND r.ai_platform_id <> 'qwen'
|
||||
AND cf.ai_platform_id <> 'qwen'
|
||||
AND cf.business_date BETWEEN $4::date AND $5::date
|
||||
AND cf.content_source_type IS NOT NULL
|
||||
AND cf.content_source_id IS NOT NULL
|
||||
AND r.collector_type = $3
|
||||
AND r.status = 'succeeded'
|
||||
AND r.business_date BETWEEN $4::date AND $5::date
|
||||
AND ($6::bigint[] IS NULL OR r.question_id = ANY($6))
|
||||
AND ($7::text[] IS NULL OR r.ai_platform_id = ANY($7))
|
||||
AND COALESCE(NULLIF(cf.article_id, 0), alias.article_id) IS NOT NULL
|
||||
GROUP BY COALESCE(NULLIF(cf.article_id, 0), alias.article_id)
|
||||
GROUP BY cf.content_source_type, cf.content_source_id
|
||||
),
|
||||
counts AS (
|
||||
SELECT
|
||||
COALESCE(SUM(citation_count), 0)::bigint AS total_citation_count,
|
||||
COUNT(*)::bigint AS total_source_count
|
||||
FROM filtered_sources
|
||||
),
|
||||
paged_sources AS (
|
||||
SELECT content_source_type, content_source_id, citation_count
|
||||
FROM filtered_sources
|
||||
ORDER BY citation_count DESC, content_source_type ASC, content_source_id ASC
|
||||
LIMIT $8 OFFSET $9
|
||||
)
|
||||
SELECT article_id, article_title, publish_platform, citation_count
|
||||
FROM article_counts
|
||||
ORDER BY citation_count DESC, article_id ASC
|
||||
LIMIT $8 OFFSET $9
|
||||
SELECT
|
||||
ps.content_source_type,
|
||||
ps.content_source_id,
|
||||
ps.citation_count,
|
||||
counts.total_citation_count,
|
||||
counts.total_source_count
|
||||
FROM counts
|
||||
LEFT JOIN paged_sources ps ON TRUE
|
||||
`, tenantID, brandID, monitoringCollectorType, startDateText, endDateText, nullableInt64Array(questionIDs), nullableStringArray(platformQueryIDs), pageReq.PageSize, monitoringPageOffset(pageReq))
|
||||
if err != nil {
|
||||
return nil, MonitoringPage{}, response.ErrInternal(50041, "query_failed", "failed to load cited articles")
|
||||
return nil, 0, 0, response.ErrInternal(50041, "query_failed", "failed to load cited articles")
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
items := make([]MonitoringCitedArticle, 0, pageReq.PageSize)
|
||||
items := make([]monitoringCitedArticleSourceRow, 0, pageReq.PageSize)
|
||||
var totalCitationCount int64
|
||||
var totalSourceCount int64
|
||||
for rows.Next() {
|
||||
var item MonitoringCitedArticle
|
||||
if scanErr := rows.Scan(&item.ArticleID, &item.ArticleTitle, &item.PublishPlatform, &item.CitationCount); scanErr != nil {
|
||||
return nil, MonitoringPage{}, response.ErrInternal(50041, "scan_failed", "failed to parse cited articles")
|
||||
var sourceType sql.NullString
|
||||
var sourceID sql.NullInt64
|
||||
var citationCount sql.NullInt64
|
||||
if scanErr := rows.Scan(
|
||||
&sourceType,
|
||||
&sourceID,
|
||||
&citationCount,
|
||||
&totalCitationCount,
|
||||
&totalSourceCount,
|
||||
); scanErr != nil {
|
||||
return nil, 0, 0, response.ErrInternal(50041, "scan_failed", "failed to parse cited articles")
|
||||
}
|
||||
if !sourceType.Valid || !sourceID.Valid || !citationCount.Valid {
|
||||
continue
|
||||
}
|
||||
item := monitoringCitedArticleSourceRow{
|
||||
SourceType: strings.TrimSpace(sourceType.String),
|
||||
SourceID: sourceID.Int64,
|
||||
CitationCount: citationCount.Int64,
|
||||
}
|
||||
if item.SourceType == "" || item.SourceID <= 0 {
|
||||
continue
|
||||
}
|
||||
item.CitationRate = divideAsPointer(item.CitationCount, totalSampleCount)
|
||||
item.SourceShare = divideAsPointer(item.CitationCount, stats.TotalCitationCount)
|
||||
items = append(items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, MonitoringPage{}, response.ErrInternal(50041, "scan_failed", "failed to iterate cited articles")
|
||||
return nil, 0, 0, response.ErrInternal(50041, "scan_failed", "failed to iterate cited articles")
|
||||
}
|
||||
|
||||
return items, page, nil
|
||||
return items, totalCitationCount, totalSourceCount, nil
|
||||
}
|
||||
|
||||
func (s *MonitoringService) loadCitedArticleSourceMeta(ctx context.Context, tenantID int64, sources []monitoringCitedArticleSourceRow) (map[string]MonitoringCitedArticle, error) {
|
||||
result := make(map[string]MonitoringCitedArticle, len(sources))
|
||||
publishedIDs := make([]int64, 0, len(sources))
|
||||
manualIDs := make([]int64, 0, len(sources))
|
||||
for _, source := range sources {
|
||||
switch source.SourceType {
|
||||
case "published_article":
|
||||
publishedIDs = append(publishedIDs, source.SourceID)
|
||||
case "manual_mark":
|
||||
manualIDs = append(manualIDs, source.SourceID)
|
||||
}
|
||||
}
|
||||
if len(publishedIDs) > 0 {
|
||||
rows, err := s.monitoringPool.Query(ctx, `
|
||||
SELECT DISTINCT ON (article_id)
|
||||
article_id,
|
||||
COALESCE(NULLIF(article_title_snapshot, ''), '未命名文章') AS article_title,
|
||||
COALESCE(NULLIF(publish_platform_name_snapshot, ''), '已发布内容') AS publish_platform,
|
||||
COALESCE(NULLIF(original_url, ''), normalized_url) AS source_url
|
||||
FROM monitoring_article_url_aliases
|
||||
WHERE tenant_id = $1
|
||||
AND confidence = 'high'
|
||||
AND article_id = ANY($2)
|
||||
ORDER BY article_id, updated_at DESC, id DESC
|
||||
`, tenantID, publishedIDs)
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(50041, "query_failed", "failed to load cited article metadata")
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var articleID int64
|
||||
var item MonitoringCitedArticle
|
||||
if scanErr := rows.Scan(&articleID, &item.ArticleTitle, &item.PublishPlatform, &item.SourceURL); scanErr != nil {
|
||||
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse cited article metadata")
|
||||
}
|
||||
value := articleID
|
||||
item.ArticleID = &value
|
||||
item.ArticleTitle = repairMonitoringMojibake(item.ArticleTitle)
|
||||
item.PublishPlatform = repairMonitoringMojibake(item.PublishPlatform)
|
||||
item.SourceType = "published_article"
|
||||
item.SourceID = articleID
|
||||
result[monitoringContentSourceKey(item.SourceType, item.SourceID)] = item
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate cited article metadata")
|
||||
}
|
||||
}
|
||||
if len(manualIDs) > 0 {
|
||||
rows, err := s.monitoringPool.Query(ctx, `
|
||||
SELECT
|
||||
id,
|
||||
COALESCE(NULLIF(article_title, ''), '未命名文章') AS article_title,
|
||||
COALESCE(NULLIF(publish_platform, ''), '外部文章') AS publish_platform,
|
||||
COALESCE(NULLIF(original_url, ''), normalized_url) AS source_url
|
||||
FROM monitoring_user_marked_articles
|
||||
WHERE tenant_id = $1
|
||||
AND id = ANY($2)
|
||||
`, tenantID, manualIDs)
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(50041, "query_failed", "failed to load marked article metadata")
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var item MonitoringCitedArticle
|
||||
if scanErr := rows.Scan(&item.SourceID, &item.ArticleTitle, &item.PublishPlatform, &item.SourceURL); scanErr != nil {
|
||||
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse marked article metadata")
|
||||
}
|
||||
item.ArticleTitle = repairMonitoringMojibake(item.ArticleTitle)
|
||||
item.PublishPlatform = repairMonitoringMojibake(item.PublishPlatform)
|
||||
item.SourceType = "manual_mark"
|
||||
result[monitoringContentSourceKey(item.SourceType, item.SourceID)] = item
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate marked article metadata")
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func monitoringContentSourceKey(sourceType string, sourceID int64) string {
|
||||
return strings.TrimSpace(sourceType) + ":" + strconv.FormatInt(sourceID, 10)
|
||||
}
|
||||
|
||||
func monitoringAliasContentKey(alias monitoringPublishedAlias) string {
|
||||
sourceType := strings.TrimSpace(alias.SourceType)
|
||||
sourceID := alias.SourceID
|
||||
if sourceType == "" {
|
||||
sourceType = "published_article"
|
||||
}
|
||||
if sourceID == 0 && alias.ArticleID > 0 {
|
||||
sourceID = alias.ArticleID
|
||||
}
|
||||
if sourceID > 0 {
|
||||
return sourceType + ":" + strconv.FormatInt(sourceID, 10)
|
||||
}
|
||||
key := strings.TrimSpace(alias.NormalizedURL)
|
||||
if key == "" {
|
||||
key = strings.TrimSpace(alias.OriginalURL)
|
||||
}
|
||||
return sourceType + ":url:" + key
|
||||
}
|
||||
|
||||
func (s *MonitoringService) loadQuestionText(ctx context.Context, tenantID, brandID, questionID int64) (string, error) {
|
||||
@@ -4130,10 +4157,35 @@ func (s *MonitoringService) loadCitationsForRuns(ctx context.Context, tenantID i
|
||||
}
|
||||
var articleID *int64
|
||||
var articleTitle *string
|
||||
var sourceType *string
|
||||
var sourceID *int64
|
||||
var sourceURL *string
|
||||
if alias, ok := aliasIndex.Match(citedURL, normalizedURL); ok {
|
||||
articleID = &alias.ArticleID
|
||||
if alias.ArticleID > 0 {
|
||||
value := alias.ArticleID
|
||||
articleID = &value
|
||||
}
|
||||
title := repairMonitoringMojibake(alias.ArticleTitle)
|
||||
articleTitle = &title
|
||||
aliasSourceType := strings.TrimSpace(alias.SourceType)
|
||||
if aliasSourceType == "" {
|
||||
aliasSourceType = "published_article"
|
||||
}
|
||||
sourceType = &aliasSourceType
|
||||
aliasSourceID := alias.SourceID
|
||||
if aliasSourceID == 0 && alias.ArticleID > 0 {
|
||||
aliasSourceID = alias.ArticleID
|
||||
}
|
||||
if aliasSourceID > 0 {
|
||||
sourceID = &aliasSourceID
|
||||
}
|
||||
aliasSourceURL := strings.TrimSpace(alias.OriginalURL)
|
||||
if aliasSourceURL == "" {
|
||||
aliasSourceURL = strings.TrimSpace(alias.NormalizedURL)
|
||||
}
|
||||
if aliasSourceURL != "" {
|
||||
sourceURL = &aliasSourceURL
|
||||
}
|
||||
resolutionStatus = "resolved"
|
||||
resolutionConfidence = "high"
|
||||
}
|
||||
@@ -4144,6 +4196,9 @@ func (s *MonitoringService) loadCitationsForRuns(ctx context.Context, tenantID i
|
||||
SiteKey: siteKey,
|
||||
FaviconURL: faviconURL(siteKey),
|
||||
ArticleID: articleID,
|
||||
SourceType: sourceType,
|
||||
SourceID: sourceID,
|
||||
SourceURL: sourceURL,
|
||||
ArticleTitle: articleTitle,
|
||||
ResolutionStatus: resolutionStatus,
|
||||
ResolutionConfidence: resolutionConfidence,
|
||||
@@ -4161,21 +4216,16 @@ func (s *MonitoringService) loadQuestionCitationAnalysis(ctx context.Context, te
|
||||
return []MonitoringQuestionCitationStats{}, nil
|
||||
}
|
||||
|
||||
aliasIndex, err := loadMonitoringPublishedAliasIndex(ctx, s.monitoringPool, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rows, err := s.monitoringPool.Query(ctx, `
|
||||
WITH filtered_facts AS (
|
||||
SELECT
|
||||
r.ai_platform_id,
|
||||
cf.id,
|
||||
cf.cited_url,
|
||||
cf.normalized_url,
|
||||
cf.host,
|
||||
cf.registrable_domain,
|
||||
cf.site_key
|
||||
cf.site_key,
|
||||
cf.content_source_type,
|
||||
cf.content_source_id
|
||||
FROM question_monitor_runs r
|
||||
JOIN monitoring_citation_facts cf
|
||||
ON cf.tenant_id = r.tenant_id AND cf.run_id = r.id
|
||||
@@ -4220,11 +4270,11 @@ func (s *MonitoringService) loadQuestionCitationAnalysis(ctx context.Context, te
|
||||
)
|
||||
SELECT
|
||||
ff.ai_platform_id,
|
||||
ff.cited_url,
|
||||
ff.normalized_url,
|
||||
COALESCE(dm.mapped_site_name, ff.site_key, ff.registrable_domain) AS site_name,
|
||||
COALESCE(dm.mapped_site_key, ff.site_key) AS site_key,
|
||||
COALESCE(dm.mapped_domain, ff.site_key, ff.registrable_domain) AS site_domain
|
||||
COALESCE(dm.mapped_domain, ff.site_key, ff.registrable_domain) AS site_domain,
|
||||
ff.content_source_type,
|
||||
ff.content_source_id
|
||||
FROM filtered_facts ff
|
||||
LEFT JOIN domain_mappings dm
|
||||
ON dm.host = ff.host
|
||||
@@ -4246,12 +4296,12 @@ func (s *MonitoringService) loadQuestionCitationAnalysis(ctx context.Context, te
|
||||
totals := make(map[string]int64)
|
||||
for rows.Next() {
|
||||
var platformID string
|
||||
var citedURL string
|
||||
var normalizedURL string
|
||||
var siteName string
|
||||
var siteKey string
|
||||
var siteDomain string
|
||||
if scanErr := rows.Scan(&platformID, &citedURL, &normalizedURL, &siteName, &siteKey, &siteDomain); scanErr != nil {
|
||||
var sourceType sql.NullString
|
||||
var sourceID sql.NullInt64
|
||||
if scanErr := rows.Scan(&platformID, &siteName, &siteKey, &siteDomain, &sourceType, &sourceID); scanErr != nil {
|
||||
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse question citation analysis")
|
||||
}
|
||||
platformID = normalizeMonitoringPlatformID(platformID)
|
||||
@@ -4273,10 +4323,8 @@ func (s *MonitoringService) loadQuestionCitationAnalysis(ctx context.Context, te
|
||||
}
|
||||
}
|
||||
item.CitationCount++
|
||||
if platformID != "qwen" {
|
||||
if _, ok := aliasIndex.Match(citedURL, normalizedURL); ok {
|
||||
item.ContentCitationCount++
|
||||
}
|
||||
if platformID != "qwen" && sourceType.Valid && strings.TrimSpace(sourceType.String) != "" && sourceID.Valid {
|
||||
item.ContentCitationCount++
|
||||
}
|
||||
aggregates[key] = item
|
||||
totals[platformID]++
|
||||
@@ -4311,16 +4359,11 @@ func (s *MonitoringService) loadQuestionContentCitations(ctx context.Context, te
|
||||
return []MonitoringQuestionContentCitation{}, nil
|
||||
}
|
||||
|
||||
aliasIndex, err := loadMonitoringPublishedAliasIndex(ctx, s.monitoringPool, tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rows, err := s.monitoringPool.Query(ctx, `
|
||||
SELECT
|
||||
r.ai_platform_id,
|
||||
cf.cited_url,
|
||||
cf.normalized_url
|
||||
cf.content_source_type,
|
||||
cf.content_source_id
|
||||
FROM question_monitor_runs r
|
||||
JOIN monitoring_citation_facts cf
|
||||
ON cf.tenant_id = r.tenant_id AND cf.run_id = r.id
|
||||
@@ -4336,16 +4379,17 @@ func (s *MonitoringService) loadQuestionContentCitations(ctx context.Context, te
|
||||
|
||||
type contentCitationKey struct {
|
||||
PlatformID string
|
||||
ArticleID int64
|
||||
SourceType string
|
||||
SourceID int64
|
||||
}
|
||||
|
||||
aggregates := make(map[contentCitationKey]MonitoringQuestionContentCitation)
|
||||
totals := make(map[string]int64)
|
||||
for rows.Next() {
|
||||
var platformID string
|
||||
var citedURL string
|
||||
var normalizedURL string
|
||||
if scanErr := rows.Scan(&platformID, &citedURL, &normalizedURL); scanErr != nil {
|
||||
var sourceType sql.NullString
|
||||
var sourceID sql.NullInt64
|
||||
if scanErr := rows.Scan(&platformID, &sourceType, &sourceID); scanErr != nil {
|
||||
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse content citations")
|
||||
}
|
||||
platformID = normalizeMonitoringPlatformID(platformID)
|
||||
@@ -4353,22 +4397,22 @@ func (s *MonitoringService) loadQuestionContentCitations(ctx context.Context, te
|
||||
continue
|
||||
}
|
||||
totals[platformID]++
|
||||
alias, ok := aliasIndex.Match(citedURL, normalizedURL)
|
||||
if !ok {
|
||||
normalizedSourceType := strings.TrimSpace(sourceType.String)
|
||||
if !sourceType.Valid || normalizedSourceType == "" || !sourceID.Valid || sourceID.Int64 <= 0 {
|
||||
continue
|
||||
}
|
||||
key := contentCitationKey{
|
||||
PlatformID: platformID,
|
||||
ArticleID: alias.ArticleID,
|
||||
SourceType: normalizedSourceType,
|
||||
SourceID: sourceID.Int64,
|
||||
}
|
||||
item := aggregates[key]
|
||||
if item.ArticleID == 0 {
|
||||
if item.SourceID == 0 {
|
||||
item = MonitoringQuestionContentCitation{
|
||||
ArticleID: alias.ArticleID,
|
||||
ArticleTitle: alias.ArticleTitle,
|
||||
PublishPlatform: alias.PublishPlatform,
|
||||
AIPlatformID: platformID,
|
||||
PlatformName: platformDisplayName(platformID),
|
||||
SourceType: normalizedSourceType,
|
||||
SourceID: sourceID.Int64,
|
||||
AIPlatformID: platformID,
|
||||
PlatformName: platformDisplayName(platformID),
|
||||
}
|
||||
}
|
||||
item.CitationCount++
|
||||
@@ -4377,9 +4421,36 @@ func (s *MonitoringService) loadQuestionContentCitations(ctx context.Context, te
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate content citations")
|
||||
}
|
||||
if len(aggregates) == 0 {
|
||||
return []MonitoringQuestionContentCitation{}, nil
|
||||
}
|
||||
|
||||
sourceRows := make([]monitoringCitedArticleSourceRow, 0, len(aggregates))
|
||||
for key := range aggregates {
|
||||
sourceRows = append(sourceRows, monitoringCitedArticleSourceRow{
|
||||
SourceType: key.SourceType,
|
||||
SourceID: key.SourceID,
|
||||
})
|
||||
}
|
||||
metaByKey, err := s.loadCitedArticleSourceMeta(ctx, tenantID, sourceRows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
items := make([]MonitoringQuestionContentCitation, 0, len(aggregates))
|
||||
for _, item := range aggregates {
|
||||
for key, item := range aggregates {
|
||||
if meta := metaByKey[monitoringContentSourceKey(key.SourceType, key.SourceID)]; meta.SourceID > 0 {
|
||||
item.ArticleID = meta.ArticleID
|
||||
item.ArticleTitle = meta.ArticleTitle
|
||||
item.PublishPlatform = meta.PublishPlatform
|
||||
item.SourceURL = meta.SourceURL
|
||||
}
|
||||
if strings.TrimSpace(item.ArticleTitle) == "" {
|
||||
item.ArticleTitle = "未命名文章"
|
||||
}
|
||||
if strings.TrimSpace(item.PublishPlatform) == "" {
|
||||
item.PublishPlatform = "外部文章"
|
||||
}
|
||||
item.CitationRate = divideAsPointer(item.CitationCount, totals[item.AIPlatformID])
|
||||
items = append(items, item)
|
||||
}
|
||||
@@ -4390,7 +4461,10 @@ func (s *MonitoringService) loadQuestionContentCitations(ctx context.Context, te
|
||||
if items[left].CitationCount != items[right].CitationCount {
|
||||
return items[left].CitationCount > items[right].CitationCount
|
||||
}
|
||||
return items[left].ArticleID < items[right].ArticleID
|
||||
if items[left].SourceType != items[right].SourceType {
|
||||
return items[left].SourceType < items[right].SourceType
|
||||
}
|
||||
return items[left].SourceID < items[right].SourceID
|
||||
})
|
||||
|
||||
return items, nil
|
||||
|
||||
Reference in New Issue
Block a user