feat: add monitoring marked articles and retention updates
Frontend CI / Frontend (push) Successful in 7m47s
Backend CI / Backend (push) Successful in 19m26s

This commit is contained in:
2026-06-17 12:48:41 +08:00
parent 9ed857e159
commit 31c4dd9358
40 changed files with 2373 additions and 488 deletions
@@ -22,9 +22,10 @@ type MonitoringRetentionWorker struct {
}
type retentionTarget struct {
Name string
DeleteSQL string
ProbeSQL string
Name string
DeleteSQL string
ProbeSQL string
PreDeleteSQL string
}
func NewMonitoringRetentionWorker(monitoringPool *pgxpool.Pool, logger *zap.Logger) *MonitoringRetentionWorker {
@@ -136,11 +137,10 @@ func (w *MonitoringRetentionWorker) deleteTarget(ctx context.Context, conn *pgxp
total := int64(0)
batches := 0
for batches < maxBatches {
tag, err := conn.Exec(ctx, target.DeleteSQL, cutoffDate, batchSize)
affected, err := w.deleteTargetBatch(ctx, conn, target, cutoffDate, batchSize)
if err != nil {
return total, batches, err
}
affected := tag.RowsAffected()
if affected == 0 {
break
}
@@ -153,6 +153,40 @@ func (w *MonitoringRetentionWorker) deleteTarget(ctx context.Context, conn *pgxp
return total, batches, nil
}
func (w *MonitoringRetentionWorker) deleteTargetBatch(ctx context.Context, conn *pgxpool.Conn, target retentionTarget, cutoffDate string, batchSize int) (int64, error) {
if strings.TrimSpace(target.PreDeleteSQL) == "" {
tag, err := conn.Exec(ctx, target.DeleteSQL, cutoffDate, batchSize)
if err != nil {
return 0, err
}
return tag.RowsAffected(), nil
}
tx, err := conn.Begin(ctx)
if err != nil {
return 0, fmt.Errorf("begin delete batch: %w", err)
}
committed := false
defer func() {
if !committed {
_ = tx.Rollback(context.Background())
}
}()
if _, err := tx.Exec(ctx, target.PreDeleteSQL, cutoffDate, batchSize); err != nil {
return 0, fmt.Errorf("pre-delete batch: %w", err)
}
tag, err := tx.Exec(ctx, target.DeleteSQL, cutoffDate, batchSize)
if err != nil {
return 0, err
}
if err := tx.Commit(ctx); err != nil {
return 0, fmt.Errorf("commit delete batch: %w", err)
}
committed = true
return tag.RowsAffected(), nil
}
func retentionTargets() []retentionTarget {
return []retentionTarget{
{
@@ -190,6 +224,12 @@ func retentionTargets() []retentionTarget {
ProbeSQL: batchProbeSQL("question_monitor_runs", "id", "business_date < $1::date", "business_date ASC, id ASC"),
DeleteSQL: batchDeleteSQL("question_monitor_runs", "id", "business_date < $1::date", "business_date ASC, id ASC"),
},
{
Name: "monitoring_user_marked_articles",
ProbeSQL: batchProbeSQL("monitoring_user_marked_articles", "id", "$1::date IS NOT NULL AND expires_at < NOW()", "expires_at ASC, id ASC"),
PreDeleteSQL: markedArticleAttributionCleanupSQL(),
DeleteSQL: batchDeleteSQL("monitoring_user_marked_articles", "id", "$1::date IS NOT NULL AND expires_at < NOW()", "expires_at ASC, id ASC"),
},
}
}
@@ -223,3 +263,24 @@ func batchDeleteSQL(table, idColumn, predicate, orderBy string) string {
}
return strings.Join(parts, "\n")
}
func markedArticleAttributionCleanupSQL() string {
parts := []string{
"WITH doomed AS (",
"SELECT tenant_id, id",
"FROM monitoring_user_marked_articles",
"WHERE $1::date IS NOT NULL AND expires_at < NOW()",
"ORDER BY expires_at ASC, id ASC",
"LIMIT $2",
"FOR UPDATE SKIP LOCKED",
")",
"UPDATE monitoring_citation_facts cf",
"SET content_source_type = NULL,",
" content_source_id = NULL",
"FROM doomed",
"WHERE cf.tenant_id = doomed.tenant_id",
" AND cf.content_source_type = 'manual_mark'",
" AND cf.content_source_id = doomed.id",
}
return strings.Join(parts, "\n")
}
@@ -34,3 +34,34 @@ func TestBatchProbeSQLCapsDryRunProbe(t *testing.T) {
t.Fatalf("batchProbeSQL should count only the limited candidate subquery:\n%s", sql)
}
}
func TestMarkedArticleRetentionClearsManualAttributionBeforeDelete(t *testing.T) {
var target retentionTarget
for _, item := range retentionTargets() {
if item.Name == "monitoring_user_marked_articles" {
target = item
break
}
}
if target.Name == "" {
t.Fatal("monitoring_user_marked_articles retention target missing")
}
if target.PreDeleteSQL == "" {
t.Fatal("monitoring_user_marked_articles retention target must clear citation attribution before delete")
}
for _, want := range []string{
"SELECT tenant_id, id",
"UPDATE monitoring_citation_facts cf",
"cf.tenant_id = doomed.tenant_id",
"content_source_type = 'manual_mark'",
"content_source_id = doomed.id",
"SET content_source_type = NULL",
"LIMIT $2",
"FOR UPDATE SKIP LOCKED",
} {
if !strings.Contains(target.PreDeleteSQL, want) {
t.Fatalf("marked article attribution cleanup SQL missing %q in SQL:\n%s", want, target.PreDeleteSQL)
}
}
}
@@ -351,6 +351,10 @@ var routeDocs = map[string]routeDoc{
// --- Tenant:监控(GEO 仪表盘) ---
"GET /api/tenant/monitoring/dashboard/composite": {"监控仪表盘综合视图", "组合返回品牌曝光、引用率、问答覆盖等监控指标。"},
"GET /api/tenant/monitoring/citation-summary": {"引用情况汇总", "按 AI 平台维度返回品牌的引用频次与变化。"},
"GET /api/tenant/monitoring/marked-articles": {"外部文章标记列表", "分页查询用户手动标记的外部文章,可按域名、文章名称和时间范围过滤。"},
"POST /api/tenant/monitoring/marked-articles": {"标记外部文章", "将无法自动对接的外部 URL 标记为外部文章,用于引用归因与排行。"},
"PUT /api/tenant/monitoring/marked-articles/:id": {"更新外部文章标记", "更新外部文章标记的文章名称、原始 URL 或关联品牌。"},
"DELETE /api/tenant/monitoring/marked-articles/:id": {"删除外部文章标记", "删除一条用户手动标记的外部文章记录。"},
"GET /api/tenant/monitoring/brands/:brand_id/questions/:question_id/detail": {"问题级监控详情", "某问题在各 AI 平台的回答样本与引用拆解。"},
"POST /api/tenant/monitoring/brands/:brand_id/collect-now": {"立即触发采集", "对某品牌立即下发一次监控采集任务,跳过定时计划。"},
@@ -334,6 +334,8 @@ func queryParameterNames(route gin.RouteInfo) []string {
add("brand_id", "keyword_id", "question_id", "days", "business_date", "ai_platform_id")
case strings.Contains(path, "/tenant/monitoring/citation-summary"):
add("days", "brand_id", "keyword_id", "question_id", "business_date", "ai_platform_id")
case strings.Contains(path, "/tenant/monitoring/marked-articles"):
add("page", "page_size", "domain", "title", "created_from", "created_to")
case strings.Contains(path, "/tenant/monitoring/brands/"):
add("ai_platform_id", "date_from", "date_to", "question_hash")
case strings.Contains(path, "/tenant/kol/marketplace/packages"):
+25 -11
View File
@@ -96,6 +96,20 @@ type BrandLibrarySummaryResponse struct {
MaxQuestionsPerBrand int `json:"max_questions_per_brand"`
}
func formatBrandTime(value interface{}) string {
switch typed := value.(type) {
case time.Time:
return typed.UTC().Format(time.RFC3339Nano)
case *time.Time:
if typed == nil {
return ""
}
return typed.UTC().Format(time.RFC3339Nano)
default:
return fmt.Sprintf("%v", value)
}
}
func (s *BrandService) List(ctx context.Context) ([]BrandResponse, error) {
actor := auth.MustActor(ctx)
brands, err := sharedcache.LoadJSON(ctx, s.cache, &s.cacheGroup, brandListCacheKey(actor.TenantID), defaultCacheTTL(), func(loadCtx context.Context) ([]BrandResponse, error) {
@@ -197,7 +211,7 @@ func (s *BrandService) Create(ctx context.Context, req BrandRequest) (*BrandResp
KeywordCount: 0,
QuestionCount: 0,
CompetitorCount: 0,
CreatedAt: fmt.Sprintf("%v", ca),
CreatedAt: formatBrandTime(ca),
}, nil
}
@@ -367,7 +381,7 @@ func (s *BrandService) CreateKeyword(ctx context.Context, brandID int64, req Key
return nil, response.ErrInternal(50010, "create_failed", "failed to create keyword")
}
invalidateBrandCaches(ctx, s.cache, actor.TenantID, brandID)
return &KeywordResponse{ID: id, BrandID: brandID, Name: req.Name, Status: "active", CreatedAt: fmt.Sprintf("%v", ca)}, nil
return &KeywordResponse{ID: id, BrandID: brandID, Name: req.Name, Status: "active", CreatedAt: formatBrandTime(ca)}, nil
}
func (s *BrandService) UpdateKeyword(ctx context.Context, brandID, keywordID int64, req KeywordRequest) error {
@@ -587,7 +601,7 @@ func (s *BrandService) CreateQuestion(ctx context.Context, brandID int64, req Qu
KeywordID: req.KeywordID,
QuestionText: req.QuestionText,
Status: "active",
CreatedAt: fmt.Sprintf("%v", ca),
CreatedAt: formatBrandTime(ca),
}, nil
}
@@ -718,7 +732,7 @@ func (s *BrandService) CreateCompetitor(ctx context.Context, brandID int64, req
return nil, response.ErrInternal(50010, "create_failed", "failed to create competitor")
}
invalidateBrandCaches(ctx, s.cache, actor.TenantID, brandID)
return &CompetitorResponse{ID: id, BrandID: brandID, Name: req.Name, Website: req.Website, Description: req.Description, ProductLinesJSON: req.ProductLinesJSON, CreatedAt: fmt.Sprintf("%v", ca)}, nil
return &CompetitorResponse{ID: id, BrandID: brandID, Name: req.Name, Website: req.Website, Description: req.Description, ProductLinesJSON: req.ProductLinesJSON, CreatedAt: formatBrandTime(ca)}, nil
}
func (s *BrandService) UpdateCompetitor(ctx context.Context, brandID, competitorID int64, req CompetitorRequest) error {
@@ -804,8 +818,8 @@ func (s *BrandService) loadBrands(ctx context.Context, tenantID int64) ([]BrandR
if err := rows.Scan(&item.ID, &item.Name, &item.Website, &item.Description, &item.Status, &item.KeywordCount, &item.QuestionCount, &item.CompetitorCount, &createdAt, &updatedAt); err != nil {
return nil, response.ErrInternal(50010, "scan_failed", err.Error())
}
item.CreatedAt = fmt.Sprintf("%v", createdAt)
item.UpdatedAt = fmt.Sprintf("%v", updatedAt)
item.CreatedAt = formatBrandTime(createdAt)
item.UpdatedAt = formatBrandTime(updatedAt)
brands = append(brands, item)
}
return brands, nil
@@ -859,8 +873,8 @@ func (s *BrandService) loadBrandDetail(ctx context.Context, tenantID, brandID in
}
return nil, false, response.ErrInternal(50010, "query_failed", "failed to fetch brand")
}
item.CreatedAt = fmt.Sprintf("%v", createdAt)
item.UpdatedAt = fmt.Sprintf("%v", updatedAt)
item.CreatedAt = formatBrandTime(createdAt)
item.UpdatedAt = formatBrandTime(updatedAt)
return &item, true, nil
}
@@ -885,7 +899,7 @@ func (s *BrandService) loadBrandKeywords(ctx context.Context, tenantID, brandID
if err := rows.Scan(&item.ID, &item.BrandID, &item.Name, &item.Status, &createdAt); err != nil {
return nil, response.ErrInternal(50010, "scan_failed", err.Error())
}
item.CreatedAt = fmt.Sprintf("%v", createdAt)
item.CreatedAt = formatBrandTime(createdAt)
items = append(items, item)
}
return items, nil
@@ -943,7 +957,7 @@ func (s *BrandService) loadBrandQuestions(ctx context.Context, tenantID, brandID
if err := rows.Scan(&item.ID, &item.BrandID, &item.KeywordID, &item.QuestionText, &item.Status, &createdAt); err != nil {
return nil, response.ErrInternal(50010, "scan_failed", err.Error())
}
item.CreatedAt = fmt.Sprintf("%v", createdAt)
item.CreatedAt = formatBrandTime(createdAt)
items = append(items, item)
}
if err := rows.Err(); err != nil {
@@ -978,7 +992,7 @@ func (s *BrandService) loadBrandCompetitors(ctx context.Context, tenantID, brand
if err := rows.Scan(&item.ID, &item.BrandID, &item.Name, &item.Website, &item.Description, &productLinesJSON, &createdAt); err != nil {
return nil, response.ErrInternal(50010, "scan_failed", err.Error())
}
item.CreatedAt = fmt.Sprintf("%v", createdAt)
item.CreatedAt = formatBrandTime(createdAt)
if productLinesJSON != nil {
raw := json.RawMessage(productLinesJSON)
item.ProductLinesJSON = &raw
@@ -0,0 +1,18 @@
package app
import (
"testing"
"time"
)
func TestFormatBrandTimeUsesRFC3339UTC(t *testing.T) {
loc := time.FixedZone("CST", 8*60*60)
value := time.Date(2026, 6, 17, 10, 52, 1, 307551000, loc)
got := formatBrandTime(value)
want := "2026-06-17T02:52:01.307551Z"
if got != want {
t.Fatalf("formatBrandTime() = %q, want %q", got, want)
}
}
@@ -72,6 +72,24 @@ func (s *DesktopTaskService) syncMonitoringArticleAlias(ctx context.Context, inp
nullableString(lastPathSegment),
confidence,
)
if err != nil {
return err
}
if confidence == "high" {
_, err = s.monitoringPool.Exec(ctx, `
UPDATE monitoring_citation_facts
SET content_source_type = 'published_article',
content_source_id = $3,
article_id = COALESCE(article_id, $3),
publish_record_id = COALESCE(publish_record_id, $4)
WHERE tenant_id = $1
AND normalized_url = $2
AND (
content_source_type IS DISTINCT FROM 'published_article'
OR content_source_id IS DISTINCT FROM $3
)
`, input.TenantID, normalizedURL, input.ArticleID, input.PublishRecordID)
}
return err
}
@@ -279,6 +279,8 @@ type monitoringAliasResolution struct {
ArticleID *int64
PublishRecordID *int64
NormalizedURLKey string
SourceType string
SourceID *int64
ResolutionStatus string
ResolutionConfidence string
}
@@ -298,6 +300,9 @@ type monitoringAliasCandidate struct {
MatchKeys []string
ArticleID *int64
PublishRecordID *int64
SourceType string
SourceID *int64
SourcePriority int
RegistrableDomain string
SiteKey string
Host string
@@ -316,6 +321,8 @@ type monitoringCitationFact struct {
SiteKey string
ArticleID *int64
PublishRecordID *int64
ContentSourceType *string
ContentSourceID *int64
ResolutionStatus string
ResolutionConfidence string
}
@@ -2119,7 +2126,7 @@ func (s *MonitoringCallbackService) processTaskResult(ctx context.Context, task
return nil, err
}
citationSourceCount++
if fact.ArticleID != nil {
if fact.ContentSourceType != nil && fact.ContentSourceID != nil {
contentCitationCount++
}
}
@@ -2591,23 +2598,61 @@ func (s *MonitoringCallbackService) loadAliasResolutions(ctx context.Context, tx
}
rows, err := tx.Query(ctx, `
WITH alias_sources AS (
SELECT
normalized_url,
article_id,
publish_record_id,
'published_article' AS source_type,
article_id AS source_id,
1 AS source_priority,
registrable_domain,
site_key,
host,
last_path_segment,
article_title_snapshot,
updated_at,
id
FROM monitoring_article_url_aliases
WHERE tenant_id = $1
AND confidence = 'high'
UNION ALL
SELECT
normalized_url,
NULL AS article_id,
NULL AS publish_record_id,
'manual_mark' AS source_type,
id AS source_id,
2 AS source_priority,
registrable_domain,
site_key,
host,
last_path_segment,
article_title AS article_title_snapshot,
updated_at,
id
FROM monitoring_user_marked_articles
WHERE tenant_id = $1
AND expires_at > NOW()
)
SELECT
normalized_url,
article_id,
publish_record_id,
registrable_domain,
source_type,
source_id,
source_priority,
registrable_domain,
site_key,
host,
last_path_segment,
article_title_snapshot
FROM monitoring_article_url_aliases
WHERE tenant_id = $1
AND (
normalized_url = ANY($2::text[])
OR NULLIF(LOWER(TRIM(registrable_domain)), '') = ANY($3::text[])
OR NULLIF(LOWER(TRIM(site_key)), '') = ANY($3::text[])
OR NULLIF(LOWER(TRIM(host)), '') = ANY($3::text[])
)
FROM alias_sources
WHERE normalized_url = ANY($2::text[])
OR NULLIF(LOWER(TRIM(registrable_domain)), '') = ANY($3::text[])
OR NULLIF(LOWER(TRIM(site_key)), '') = ANY($3::text[])
OR NULLIF(LOWER(TRIM(host)), '') = ANY($3::text[])
ORDER BY updated_at DESC, id DESC
`, tenantID, normalizedURLs, domainKeys)
if err != nil {
return nil, response.ErrInternal(50041, "alias_lookup_failed", "failed to resolve article aliases")
@@ -2619,6 +2664,7 @@ func (s *MonitoringCallbackService) loadAliasResolutions(ctx context.Context, tx
var candidate monitoringAliasCandidate
var articleID sql.NullInt64
var publishRecordID sql.NullInt64
var sourceID sql.NullInt64
var registrableDomain sql.NullString
var siteKey sql.NullString
var host sql.NullString
@@ -2628,6 +2674,9 @@ func (s *MonitoringCallbackService) loadAliasResolutions(ctx context.Context, tx
&candidate.NormalizedURL,
&articleID,
&publishRecordID,
&candidate.SourceType,
&sourceID,
&candidate.SourcePriority,
&registrableDomain,
&siteKey,
&host,
@@ -2638,6 +2687,7 @@ func (s *MonitoringCallbackService) loadAliasResolutions(ctx context.Context, tx
}
candidate.ArticleID = nullableInt64Value(articleID)
candidate.PublishRecordID = nullableInt64Value(publishRecordID)
candidate.SourceID = nullableInt64Value(sourceID)
candidate.RegistrableDomain = strings.ToLower(strings.TrimSpace(nullableStringText(registrableDomain)))
candidate.SiteKey = strings.ToLower(strings.TrimSpace(nullableStringText(siteKey)))
candidate.Host = strings.ToLower(strings.TrimSpace(nullableStringText(host)))
@@ -2665,8 +2715,14 @@ func (s *MonitoringCallbackService) loadAliasResolutions(ctx context.Context, tx
best = candidate
continue
}
if score == bestScore && !sameInt64Pointer(candidate.ArticleID, best.ArticleID) {
bestAmbiguous = true
if score == bestScore && !sameMonitoringAliasCandidateSource(candidate, best) {
if best.SourcePriority == 0 || candidate.SourcePriority == best.SourcePriority {
bestAmbiguous = true
}
if best.SourcePriority == 0 || (candidate.SourcePriority > 0 && candidate.SourcePriority < best.SourcePriority) {
bestAmbiguous = false
best = candidate
}
}
}
@@ -2677,6 +2733,8 @@ func (s *MonitoringCallbackService) loadAliasResolutions(ctx context.Context, tx
ArticleID: cloneInt64(best.ArticleID),
PublishRecordID: cloneInt64(best.PublishRecordID),
NormalizedURLKey: best.NormalizedURL,
SourceType: strings.TrimSpace(best.SourceType),
SourceID: cloneInt64(best.SourceID),
ResolutionStatus: "resolved",
ResolutionConfidence: "high",
}
@@ -2745,19 +2803,28 @@ func sameInt64Pointer(left, right *int64) bool {
return *left == *right
}
func sameMonitoringAliasCandidateSource(left, right monitoringAliasCandidate) bool {
leftType := strings.TrimSpace(left.SourceType)
rightType := strings.TrimSpace(right.SourceType)
if leftType == "" && rightType == "" {
return sameInt64Pointer(left.ArticleID, right.ArticleID)
}
return leftType == rightType && sameInt64Pointer(left.SourceID, right.SourceID)
}
func (s *MonitoringCallbackService) insertCitationFact(ctx context.Context, tx pgx.Tx, task *monitoringCollectTask, runID int64, fact monitoringCitationFact) error {
if _, err := tx.Exec(ctx, `
INSERT INTO monitoring_citation_facts (
tenant_id, run_id, brand_id, ai_platform_id, business_date,
cited_url, cited_title, normalized_url, host, registrable_domain,
subdomain, suffix, site_key, article_id, publish_record_id,
resolution_status, resolution_confidence
content_source_type, content_source_id, resolution_status, resolution_confidence
)
VALUES (
$1, $2, $3, $4, $5::date,
$6, $7, $8, $9, $10,
$11, $12, $13, $14, $15,
$16, $17
$16, $17, $18, $19
)
`,
task.TenantID,
@@ -2775,6 +2842,8 @@ func (s *MonitoringCallbackService) insertCitationFact(ctx context.Context, tx p
fact.SiteKey,
nullableInt64(fact.ArticleID),
nullableInt64(fact.PublishRecordID),
nullableString(fact.ContentSourceType),
nullableInt64(fact.ContentSourceID),
fact.ResolutionStatus,
fact.ResolutionConfidence,
); err != nil {
@@ -2926,10 +2995,10 @@ func (s *MonitoringCallbackService) loadExistingRunSummary(ctx context.Context,
var citationSourceCount int
var contentCitationCount int
if err := s.monitoringPool.QueryRow(ctx, `
SELECT
COUNT(*)::int AS citation_source_count,
COUNT(*) FILTER (WHERE article_id IS NOT NULL AND $3::text <> 'qwen')::int AS content_citation_count
FROM monitoring_citation_facts
SELECT
COUNT(*)::int AS citation_source_count,
COUNT(*) FILTER (WHERE content_source_type IS NOT NULL AND content_source_id IS NOT NULL AND $3::text <> 'qwen')::int AS content_citation_count
FROM monitoring_citation_facts
WHERE tenant_id = $1
AND run_id = $2
`, task.TenantID, runID.Int64, normalizeMonitoringPlatformID(task.AIPlatformID)).Scan(&citationSourceCount, &contentCitationCount); err != nil {
@@ -3173,6 +3242,8 @@ func buildMonitoringCitationFact(input MonitoringSourceItem, aliasMap map[string
var articleID *int64
var publishRecordID *int64
var contentSourceType *string
var contentSourceID *int64
var aliasResolution *monitoringAliasResolution
if resolveContentCitations {
articleID = cloneInt64(input.ArticleID)
@@ -3186,6 +3257,10 @@ func buildMonitoringCitationFact(input MonitoringSourceItem, aliasMap map[string
if publishRecordID == nil {
publishRecordID = cloneInt64(alias.PublishRecordID)
}
if strings.TrimSpace(alias.SourceType) != "" && alias.SourceID != nil {
contentSourceType = optionalString(strings.TrimSpace(alias.SourceType))
contentSourceID = cloneInt64(alias.SourceID)
}
}
}
@@ -3217,6 +3292,8 @@ func buildMonitoringCitationFact(input MonitoringSourceItem, aliasMap map[string
SiteKey: siteKey,
ArticleID: articleID,
PublishRecordID: publishRecordID,
ContentSourceType: contentSourceType,
ContentSourceID: contentSourceID,
ResolutionStatus: resolutionStatus,
ResolutionConfidence: resolutionConfidence,
}, true
@@ -18,6 +18,11 @@ type monitoringPublishedAlias struct {
PublishRecordID *int64
ArticleTitle string
PublishPlatform string
OriginalURL string
NormalizedURL string
SourceType string
SourceID int64
SourcePriority int
Ambiguous bool
}
@@ -29,17 +34,51 @@ type monitoringAliasQuerier interface {
func loadMonitoringPublishedAliasIndex(ctx context.Context, q monitoringAliasQuerier, tenantID int64) (monitoringPublishedAliasIndex, error) {
rows, err := q.Query(ctx, `
WITH alias_sources AS (
SELECT
article_id,
publish_record_id,
COALESCE(article_title_snapshot, '未命名文章') AS article_title,
COALESCE(publish_platform_name_snapshot, '已发布内容') AS publish_platform,
original_url,
normalized_url,
'published_article' AS source_type,
article_id AS source_id,
1 AS source_priority,
updated_at AS sort_updated_at,
id AS sort_id
FROM monitoring_article_url_aliases
WHERE tenant_id = $1
AND confidence = 'high'
UNION ALL
SELECT
NULL AS article_id,
NULL AS publish_record_id,
COALESCE(NULLIF(article_title, ''), '未命名文章') AS article_title,
COALESCE(NULLIF(publish_platform, ''), '外部文章') AS publish_platform,
original_url,
normalized_url,
'manual_mark' AS source_type,
id AS source_id,
2 AS source_priority,
updated_at AS sort_updated_at,
id AS sort_id
FROM monitoring_user_marked_articles
WHERE tenant_id = $1
AND expires_at > NOW()
)
SELECT
article_id,
publish_record_id,
COALESCE(article_title_snapshot, '未命名文章') AS article_title,
COALESCE(publish_platform_name_snapshot, '已发布内容') AS publish_platform,
article_title,
publish_platform,
original_url,
normalized_url
FROM monitoring_article_url_aliases
WHERE tenant_id = $1
AND confidence = 'high'
ORDER BY updated_at DESC, id DESC
normalized_url,
source_type,
source_id,
source_priority
FROM alias_sources
ORDER BY source_priority ASC, sort_updated_at DESC, sort_id DESC
`, tenantID)
if err != nil {
return nil, responseInternalAliasLookupError()
@@ -52,13 +91,36 @@ func loadMonitoringPublishedAliasIndex(ctx context.Context, q monitoringAliasQue
var originalURL string
var normalizedURL string
var publishRecordID sql.NullInt64
if scanErr := rows.Scan(&item.ArticleID, &publishRecordID, &item.ArticleTitle, &item.PublishPlatform, &originalURL, &normalizedURL); scanErr != nil {
var articleID sql.NullInt64
if scanErr := rows.Scan(
&articleID,
&publishRecordID,
&item.ArticleTitle,
&item.PublishPlatform,
&originalURL,
&normalizedURL,
&item.SourceType,
&item.SourceID,
&item.SourcePriority,
); scanErr != nil {
return nil, responseInternalAliasScanError()
}
if articleID.Valid {
item.ArticleID = articleID.Int64
}
item.PublishRecordID = nullableInt64Value(publishRecordID)
item.OriginalURL = originalURL
item.NormalizedURL = normalizedURL
for _, key := range monitoringCitationCandidateKeys(originalURL, normalizedURL) {
existing, exists := index[key]
if exists && existing.ArticleID != item.ArticleID {
if exists && existing.SourcePriority < item.SourcePriority {
continue
}
if exists && item.SourcePriority < existing.SourcePriority {
index[key] = item
continue
}
if exists && (existing.SourceType != item.SourceType || existing.SourceID != item.SourceID) {
existing.Ambiguous = true
index[key] = existing
continue
@@ -0,0 +1,522 @@
package app
import (
"context"
"database/sql"
"fmt"
"net/url"
"strconv"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/geo-platform/tenant-api/internal/shared/auth"
"github.com/geo-platform/tenant-api/internal/shared/response"
)
type MonitoringMarkedArticleListParams struct {
Domain string
Title string
CreatedFrom string
CreatedTo string
Page int
PageSize int
}
type MonitoringMarkedArticleRequest struct {
ArticleTitle string `json:"article_title"`
OriginalURL string `json:"original_url"`
BrandID *int64 `json:"brand_id"`
}
type MonitoringMarkedArticle struct {
ID int64 `json:"id"`
BrandID *int64 `json:"brand_id"`
ArticleTitle string `json:"article_title"`
OriginalURL string `json:"original_url"`
NormalizedURL string `json:"normalized_url"`
Host string `json:"host"`
RegistrableDomain string `json:"registrable_domain"`
SiteKey string `json:"site_key"`
PublishPlatform string `json:"publish_platform"`
MarkedByUserID *int64 `json:"marked_by_user_id"`
MarkedAt string `json:"marked_at"`
ExpiresAt string `json:"expires_at"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
type MonitoringMarkedArticleListResponse struct {
Items []MonitoringMarkedArticle `json:"items"`
Page MonitoringPage `json:"page"`
}
type monitoringMarkedArticleURLMeta struct {
OriginalURL string
NormalizedURL string
Host string
RegistrableDomain string
SiteKey string
LastPathSegment *string
}
func (s *MonitoringService) ListMarkedArticles(ctx context.Context, params MonitoringMarkedArticleListParams) (*MonitoringMarkedArticleListResponse, error) {
actor := auth.MustActor(ctx)
pageReq := normalizeMonitoringPageRequest(MonitoringPageRequest{
Page: params.Page,
PageSize: params.PageSize,
}, defaultMonitoringListPageSize)
createdFrom, createdTo, err := parseMarkedArticleDateRange(params.CreatedFrom, params.CreatedTo)
if err != nil {
return nil, err
}
domain := normalizeMarkedArticleDomain(params.Domain)
title := strings.ToLower(strings.TrimSpace(params.Title))
where, args := buildMarkedArticleListWhere(actor.TenantID, domain, title, createdFrom, createdTo)
var total int64
if err := s.monitoringPool.QueryRow(ctx, `
SELECT COUNT(*)::bigint
FROM monitoring_user_marked_articles
WHERE `+where, args...).Scan(&total); err != nil {
return nil, response.ErrInternal(50041, "marked_article_count_failed", "failed to count marked articles")
}
page := monitoringPageFromRequest(pageReq, total)
if total == 0 {
return &MonitoringMarkedArticleListResponse{Items: []MonitoringMarkedArticle{}, Page: page}, nil
}
listArgs := append([]interface{}{}, args...)
listArgs = append(listArgs, pageReq.PageSize, monitoringPageOffset(pageReq))
limitPlaceholder := len(listArgs) - 1
offsetPlaceholder := len(listArgs)
rows, err := s.monitoringPool.Query(ctx, `
SELECT
id,
brand_id,
article_title,
original_url,
normalized_url,
host,
registrable_domain,
site_key,
publish_platform,
marked_by_user_id,
marked_at,
expires_at,
created_at,
updated_at
FROM monitoring_user_marked_articles
WHERE `+where+`
ORDER BY created_at DESC, id DESC
LIMIT $`+strconv.Itoa(limitPlaceholder)+` OFFSET $`+strconv.Itoa(offsetPlaceholder), listArgs...)
if err != nil {
return nil, response.ErrInternal(50041, "marked_article_query_failed", "failed to load marked articles")
}
defer rows.Close()
items := make([]MonitoringMarkedArticle, 0, pageReq.PageSize)
for rows.Next() {
item, scanErr := scanMonitoringMarkedArticle(rows)
if scanErr != nil {
return nil, scanErr
}
items = append(items, item)
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "marked_article_scan_failed", "failed to iterate marked articles")
}
return &MonitoringMarkedArticleListResponse{Items: items, Page: page}, nil
}
func buildMarkedArticleListWhere(tenantID int64, domain, title string, createdFrom, createdTo *time.Time) (string, []interface{}) {
args := []interface{}{tenantID}
clauses := []string{
"tenant_id = $1",
"expires_at > NOW()",
}
if domain != "" {
args = append(args, strings.ToLower(domain))
placeholder := fmt.Sprintf("$%d", len(args))
clauses = append(clauses, "(registrable_domain = "+placeholder+" OR host = "+placeholder+" OR site_key = "+placeholder+")")
}
if title != "" {
args = append(args, "%"+title+"%")
clauses = append(clauses, "LOWER(article_title) LIKE "+fmt.Sprintf("$%d", len(args)))
}
if createdFrom != nil {
args = append(args, *createdFrom)
clauses = append(clauses, "created_at >= "+fmt.Sprintf("$%d", len(args)))
}
if createdTo != nil {
args = append(args, *createdTo)
clauses = append(clauses, "created_at <= "+fmt.Sprintf("$%d", len(args)))
}
return strings.Join(clauses, " AND "), args
}
func (s *MonitoringService) CreateMarkedArticle(ctx context.Context, req MonitoringMarkedArticleRequest) (*MonitoringMarkedArticle, error) {
actor := auth.MustActor(ctx)
meta, title, brandID, err := s.normalizeMarkedArticleRequest(ctx, actor.TenantID, req)
if err != nil {
return nil, err
}
var id int64
if err := s.monitoringPool.QueryRow(ctx, `
INSERT INTO monitoring_user_marked_articles (
tenant_id, brand_id, marked_by_user_id, article_title, original_url,
normalized_url, host, registrable_domain, site_key, last_path_segment,
publish_platform, expires_at
)
VALUES (
$1, $2, $3, $4, $5,
$6, $7, $8, $9, $10,
'外部文章', NOW() + INTERVAL '6 months'
)
ON CONFLICT (tenant_id, normalized_url)
DO UPDATE SET
brand_id = EXCLUDED.brand_id,
marked_by_user_id = EXCLUDED.marked_by_user_id,
article_title = EXCLUDED.article_title,
original_url = EXCLUDED.original_url,
host = EXCLUDED.host,
registrable_domain = EXCLUDED.registrable_domain,
site_key = EXCLUDED.site_key,
last_path_segment = EXCLUDED.last_path_segment,
publish_platform = EXCLUDED.publish_platform,
marked_at = NOW(),
expires_at = NOW() + INTERVAL '6 months',
updated_at = NOW()
RETURNING id
`,
actor.TenantID,
nullableInt64(brandID),
actor.UserID,
title,
meta.OriginalURL,
meta.NormalizedURL,
meta.Host,
meta.RegistrableDomain,
meta.SiteKey,
nullableString(meta.LastPathSegment),
).Scan(&id); err != nil {
return nil, response.ErrInternal(50041, "marked_article_create_failed", "failed to save marked article")
}
if err := s.syncMarkedArticleCitationFacts(ctx, actor.TenantID, id, meta.NormalizedURL); err != nil {
return nil, err
}
return s.GetMarkedArticle(ctx, id)
}
func (s *MonitoringService) UpdateMarkedArticle(ctx context.Context, id int64, req MonitoringMarkedArticleRequest) (*MonitoringMarkedArticle, error) {
if id <= 0 {
return nil, response.ErrBadRequest(40031, "invalid_marked_article_id", "marked article id must be a positive number")
}
actor := auth.MustActor(ctx)
meta, title, brandID, err := s.normalizeMarkedArticleRequest(ctx, actor.TenantID, req)
if err != nil {
return nil, err
}
var oldNormalizedURL string
if err := s.monitoringPool.QueryRow(ctx, `
SELECT normalized_url
FROM monitoring_user_marked_articles
WHERE id = $1
AND tenant_id = $2
AND expires_at > NOW()
`, id, actor.TenantID).Scan(&oldNormalizedURL); err != nil {
if err == pgx.ErrNoRows {
return nil, response.ErrNotFound(40441, "marked_article_not_found", "marked article not found")
}
return nil, response.ErrInternal(50041, "marked_article_lookup_failed", "failed to load marked article")
}
tag, err := s.monitoringPool.Exec(ctx, `
UPDATE monitoring_user_marked_articles
SET brand_id = $3,
article_title = $4,
original_url = $5,
normalized_url = $6,
host = $7,
registrable_domain = $8,
site_key = $9,
last_path_segment = $10,
expires_at = NOW() + INTERVAL '6 months',
updated_at = NOW()
WHERE id = $1
AND tenant_id = $2
AND expires_at > NOW()
`, id, actor.TenantID, nullableInt64(brandID), title, meta.OriginalURL, meta.NormalizedURL, meta.Host, meta.RegistrableDomain, meta.SiteKey, nullableString(meta.LastPathSegment))
if err != nil {
return nil, response.ErrInternal(50041, "marked_article_update_failed", "failed to update marked article")
}
if tag.RowsAffected() == 0 {
return nil, response.ErrNotFound(40441, "marked_article_not_found", "marked article not found")
}
if oldNormalizedURL != "" && oldNormalizedURL != meta.NormalizedURL {
if err := s.clearMarkedArticleCitationFacts(ctx, actor.TenantID, id, oldNormalizedURL); err != nil {
return nil, err
}
}
if err := s.syncMarkedArticleCitationFacts(ctx, actor.TenantID, id, meta.NormalizedURL); err != nil {
return nil, err
}
return s.GetMarkedArticle(ctx, id)
}
func (s *MonitoringService) DeleteMarkedArticle(ctx context.Context, id int64) error {
if id <= 0 {
return response.ErrBadRequest(40031, "invalid_marked_article_id", "marked article id must be a positive number")
}
actor := auth.MustActor(ctx)
var normalizedURL string
if err := s.monitoringPool.QueryRow(ctx, `
SELECT normalized_url
FROM monitoring_user_marked_articles
WHERE id = $1 AND tenant_id = $2
`, id, actor.TenantID).Scan(&normalizedURL); err != nil {
if err == pgx.ErrNoRows {
return response.ErrNotFound(40441, "marked_article_not_found", "marked article not found")
}
return response.ErrInternal(50041, "marked_article_lookup_failed", "failed to load marked article")
}
tag, err := s.monitoringPool.Exec(ctx, `
DELETE FROM monitoring_user_marked_articles
WHERE id = $1 AND tenant_id = $2
`, id, actor.TenantID)
if err != nil {
return response.ErrInternal(50041, "marked_article_delete_failed", "failed to delete marked article")
}
if tag.RowsAffected() == 0 {
return response.ErrNotFound(40441, "marked_article_not_found", "marked article not found")
}
if err := s.clearMarkedArticleCitationFacts(ctx, actor.TenantID, id, normalizedURL); err != nil {
return err
}
return nil
}
func (s *MonitoringService) GetMarkedArticle(ctx context.Context, id int64) (*MonitoringMarkedArticle, error) {
if id <= 0 {
return nil, response.ErrBadRequest(40031, "invalid_marked_article_id", "marked article id must be a positive number")
}
actor := auth.MustActor(ctx)
row := s.monitoringPool.QueryRow(ctx, `
SELECT
id,
brand_id,
article_title,
original_url,
normalized_url,
host,
registrable_domain,
site_key,
publish_platform,
marked_by_user_id,
marked_at,
expires_at,
created_at,
updated_at
FROM monitoring_user_marked_articles
WHERE id = $1
AND tenant_id = $2
AND expires_at > NOW()
`, id, actor.TenantID)
item, err := scanMonitoringMarkedArticle(row)
if err != nil {
if err == pgx.ErrNoRows {
return nil, response.ErrNotFound(40441, "marked_article_not_found", "marked article not found")
}
return nil, err
}
return &item, nil
}
func (s *MonitoringService) normalizeMarkedArticleRequest(ctx context.Context, tenantID int64, req MonitoringMarkedArticleRequest) (monitoringMarkedArticleURLMeta, string, *int64, error) {
title := strings.TrimSpace(req.ArticleTitle)
if title == "" {
return monitoringMarkedArticleURLMeta{}, "", nil, response.ErrBadRequest(40031, "marked_article_title_required", "article_title is required")
}
meta, err := normalizeMonitoringMarkedArticleURL(req.OriginalURL)
if err != nil {
return monitoringMarkedArticleURLMeta{}, "", nil, err
}
if req.BrandID != nil {
if _, err := s.resolveBrand(ctx, tenantID, *req.BrandID); err != nil {
return monitoringMarkedArticleURLMeta{}, "", nil, err
}
}
return meta, title, req.BrandID, nil
}
func normalizeMonitoringMarkedArticleURL(raw string) (monitoringMarkedArticleURLMeta, error) {
originalURL := strings.TrimSpace(raw)
if originalURL == "" {
return monitoringMarkedArticleURLMeta{}, response.ErrBadRequest(40031, "marked_article_url_required", "original_url is required")
}
parsed, err := url.Parse(originalURL)
if err != nil || parsed.Scheme == "" || parsed.Hostname() == "" {
return monitoringMarkedArticleURLMeta{}, response.ErrBadRequest(40031, "invalid_marked_article_url", "original_url must be an absolute URL")
}
normalizedURL := normalizeCitationURL(originalURL)
if normalizedURL == "" {
return monitoringMarkedArticleURLMeta{}, response.ErrBadRequest(40031, "invalid_marked_article_url", "original_url is invalid")
}
host, registrableDomain, _, _ := deriveCitationURLParts(originalURL, normalizedURL)
if host == "" || registrableDomain == "" {
return monitoringMarkedArticleURLMeta{}, response.ErrBadRequest(40031, "invalid_marked_article_url", "original_url host is invalid")
}
siteKey := host
if siteKey == "" {
siteKey = registrableDomain
}
return monitoringMarkedArticleURLMeta{
OriginalURL: originalURL,
NormalizedURL: normalizedURL,
Host: host,
RegistrableDomain: registrableDomain,
SiteKey: siteKey,
LastPathSegment: monitoringAliasLastPathSegment(normalizedURL),
}, nil
}
func parseMarkedArticleDateRange(from, to string) (*time.Time, *time.Time, error) {
parse := func(value string, endOfDay bool) (*time.Time, error) {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return nil, nil
}
if parsed, err := time.Parse(time.RFC3339, trimmed); err == nil {
value := parsed.UTC()
return &value, nil
}
parsed, err := time.Parse("2006-01-02", trimmed)
if err != nil {
return nil, response.ErrBadRequest(40031, "invalid_marked_article_date_range", "date range must use RFC3339 or YYYY-MM-DD")
}
if endOfDay {
parsed = parsed.Add(24*time.Hour - time.Nanosecond)
}
return &parsed, nil
}
fromTime, err := parse(from, false)
if err != nil {
return nil, nil, err
}
toTime, err := parse(to, true)
if err != nil {
return nil, nil, err
}
if fromTime != nil && toTime != nil && fromTime.After(*toTime) {
return nil, nil, response.ErrBadRequest(40031, "invalid_marked_article_date_range", "created_from must be before created_to")
}
return fromTime, toTime, nil
}
func normalizeMarkedArticleDomain(value string) string {
trimmed := strings.ToLower(strings.TrimSpace(value))
trimmed = strings.TrimPrefix(trimmed, "http://")
trimmed = strings.TrimPrefix(trimmed, "https://")
trimmed = strings.Trim(trimmed, "/")
if idx := strings.IndexAny(trimmed, "/?#"); idx >= 0 {
trimmed = trimmed[:idx]
}
return trimmed
}
func (s *MonitoringService) syncMarkedArticleCitationFacts(ctx context.Context, tenantID, markedArticleID int64, normalizedURL string) error {
if tenantID <= 0 || markedArticleID <= 0 || strings.TrimSpace(normalizedURL) == "" {
return nil
}
if _, err := s.monitoringPool.Exec(ctx, `
UPDATE monitoring_citation_facts
SET content_source_type = 'manual_mark',
content_source_id = $3
WHERE tenant_id = $1
AND normalized_url = $2
AND content_source_type IS NULL
AND content_source_id IS NULL
`, tenantID, strings.TrimSpace(normalizedURL), markedArticleID); err != nil {
return response.ErrInternal(50041, "marked_article_citation_sync_failed", "failed to sync marked article citations")
}
return nil
}
func (s *MonitoringService) clearMarkedArticleCitationFacts(ctx context.Context, tenantID, markedArticleID int64, normalizedURL string) error {
if tenantID <= 0 || markedArticleID <= 0 {
return nil
}
args := []interface{}{tenantID, markedArticleID}
whereURL := ""
if trimmed := strings.TrimSpace(normalizedURL); trimmed != "" {
args = append(args, trimmed)
whereURL = " AND normalized_url = $3"
}
if _, err := s.monitoringPool.Exec(ctx, `
UPDATE monitoring_citation_facts
SET content_source_type = NULL,
content_source_id = NULL
WHERE tenant_id = $1
AND content_source_type = 'manual_mark'
AND content_source_id = $2
`+whereURL, args...); err != nil {
return response.ErrInternal(50041, "marked_article_citation_clear_failed", "failed to clear marked article citations")
}
return nil
}
func nullableTimePtr(value *time.Time) interface{} {
if value == nil {
return nil
}
return *value
}
type monitoringMarkedArticleScanner interface {
Scan(dest ...interface{}) error
}
func scanMonitoringMarkedArticle(row monitoringMarkedArticleScanner) (MonitoringMarkedArticle, error) {
var item MonitoringMarkedArticle
var brandID sql.NullInt64
var markedByUserID sql.NullInt64
var markedAt time.Time
var expiresAt time.Time
var createdAt time.Time
var updatedAt time.Time
if err := row.Scan(
&item.ID,
&brandID,
&item.ArticleTitle,
&item.OriginalURL,
&item.NormalizedURL,
&item.Host,
&item.RegistrableDomain,
&item.SiteKey,
&item.PublishPlatform,
&markedByUserID,
&markedAt,
&expiresAt,
&createdAt,
&updatedAt,
); err != nil {
if err == pgx.ErrNoRows {
return item, err
}
return item, response.ErrInternal(50041, "marked_article_scan_failed", "failed to parse marked article")
}
item.BrandID = nullableInt64Value(brandID)
item.MarkedByUserID = nullableInt64Value(markedByUserID)
item.MarkedAt = markedAt.UTC().Format(time.RFC3339)
item.ExpiresAt = expiresAt.UTC().Format(time.RFC3339)
item.CreatedAt = createdAt.UTC().Format(time.RFC3339)
item.UpdatedAt = updatedAt.UTC().Format(time.RFC3339)
return item, nil
}
@@ -213,7 +213,9 @@ func (s *MonitoringCallbackService) loadMonitoringBrandDailyAggregate(ctx contex
ON r.tenant_id = cf.tenant_id AND r.id = cf.run_id
WHERE cf.tenant_id = $1
AND cf.brand_id = $2
AND cf.article_id IS NOT NULL
AND cf.business_date = $4::date
AND cf.content_source_type IS NOT NULL
AND cf.content_source_id IS NOT NULL
AND r.ai_platform_id <> 'qwen'
AND r.collector_type = $3
AND r.business_date = $4::date
+372 -298
View File
@@ -10,6 +10,7 @@ import (
"fmt"
"math/big"
"sort"
"strconv"
"strings"
"sync"
"time"
@@ -192,9 +193,12 @@ type MonitoringCitationRanking struct {
}
type MonitoringCitedArticle struct {
ArticleID int64 `json:"article_id"`
ArticleID *int64 `json:"article_id"`
ArticleTitle string `json:"article_title"`
PublishPlatform string `json:"publish_platform"`
SourceType string `json:"source_type"`
SourceID int64 `json:"source_id"`
SourceURL string `json:"source_url"`
CitationCount int64 `json:"citation_count"`
CitationRate *float64 `json:"citation_rate"`
SourceShare *float64 `json:"source_share"`
@@ -237,6 +241,9 @@ type MonitoringQuestionDetailCitation struct {
SiteKey string `json:"site_key"`
FaviconURL string `json:"favicon_url"`
ArticleID *int64 `json:"article_id"`
SourceType *string `json:"source_type,omitempty"`
SourceID *int64 `json:"source_id,omitempty"`
SourceURL *string `json:"source_url,omitempty"`
ArticleTitle *string `json:"article_title,omitempty"`
ResolutionStatus string `json:"resolution_status"`
ResolutionConfidence string `json:"resolution_confidence"`
@@ -254,9 +261,12 @@ type MonitoringQuestionCitationStats struct {
}
type MonitoringQuestionContentCitation struct {
ArticleID int64 `json:"article_id"`
ArticleID *int64 `json:"article_id"`
ArticleTitle string `json:"article_title"`
PublishPlatform string `json:"publish_platform"`
SourceType string `json:"source_type"`
SourceID int64 `json:"source_id"`
SourceURL string `json:"source_url"`
AIPlatformID string `json:"ai_platform_id"`
PlatformName string `json:"platform_name"`
CitationCount int64 `json:"citation_count"`
@@ -3458,11 +3468,6 @@ func (s *MonitoringService) loadCitationRanking(
}
platformQueryIDs := monitoringPlatformQueryIDs(aiPlatformID)
aliasIndex, err := loadMonitoringPublishedAliasIndex(ctx, s.monitoringPool, tenantID)
if err != nil {
return nil, err
}
type citationRankingAggregate struct {
CitedAnswerCount int64
CitedArticleCount int64
@@ -3473,18 +3478,57 @@ func (s *MonitoringService) loadCitationRanking(
aggregates := make(map[string]citationRankingAggregate)
rows, err := s.monitoringPool.Query(ctx, `
WITH filtered_runs AS (
SELECT id, ai_platform_id
FROM question_monitor_runs
WHERE tenant_id = $1
AND ($2::bigint = 0 OR brand_id = $2)
AND collector_type = $3
AND status = 'succeeded'
AND business_date BETWEEN $4::date AND $5::date
AND ($6::bigint[] IS NULL OR question_id = ANY($6))
AND ($7::text[] IS NULL OR ai_platform_id = ANY($7))
),
sample_counts AS (
SELECT ai_platform_id, COUNT(*)::bigint AS sample_count
FROM filtered_runs
GROUP BY ai_platform_id
),
citation_counts AS (
SELECT
fr.ai_platform_id,
COUNT(cf.id)::bigint AS citation_source_count,
COUNT(cf.id) FILTER (
WHERE fr.ai_platform_id <> 'qwen'
AND cf.content_source_type IS NOT NULL
AND cf.content_source_id IS NOT NULL
)::bigint AS saas_source_count,
COUNT(DISTINCT cf.run_id) FILTER (
WHERE fr.ai_platform_id <> 'qwen'
AND cf.content_source_type IS NOT NULL
AND cf.content_source_id IS NOT NULL
)::bigint AS cited_answer_count,
COUNT(DISTINCT cf.content_source_type || ':' || cf.content_source_id::text) FILTER (
WHERE fr.ai_platform_id <> 'qwen'
AND cf.content_source_type IS NOT NULL
AND cf.content_source_id IS NOT NULL
)::bigint AS cited_article_count
FROM filtered_runs fr
JOIN monitoring_citation_facts cf
ON cf.tenant_id = $1
AND cf.run_id = fr.id
AND cf.business_date BETWEEN $4::date AND $5::date
GROUP BY fr.ai_platform_id
)
SELECT
r.ai_platform_id,
COUNT(*) AS sample_count
FROM question_monitor_runs r
WHERE r.tenant_id = $1
AND ($2::bigint = 0 OR r.brand_id = $2)
AND r.collector_type = $3
AND r.status = 'succeeded'
AND r.business_date BETWEEN $4::date AND $5::date
AND ($6::bigint[] IS NULL OR r.question_id = ANY($6))
AND ($7::text[] IS NULL OR r.ai_platform_id = ANY($7))
GROUP BY r.ai_platform_id
sc.ai_platform_id,
sc.sample_count,
COALESCE(cc.citation_source_count, 0)::bigint,
COALESCE(cc.saas_source_count, 0)::bigint,
COALESCE(cc.cited_answer_count, 0)::bigint,
COALESCE(cc.cited_article_count, 0)::bigint
FROM sample_counts sc
LEFT JOIN citation_counts cc ON cc.ai_platform_id = sc.ai_platform_id
`, tenantID, brandID, monitoringCollectorType, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), nullableInt64Array(questionIDs), nullableStringArray(platformQueryIDs))
if err != nil {
return nil, response.ErrInternal(50041, "query_failed", "failed to load citation ranking")
@@ -3494,7 +3538,18 @@ func (s *MonitoringService) loadCitationRanking(
for rows.Next() {
var platformID string
var sampleCount int64
if scanErr := rows.Scan(&platformID, &sampleCount); scanErr != nil {
var citationSourceCount int64
var saasSourceCount int64
var citedAnswerCount int64
var citedArticleCount int64
if scanErr := rows.Scan(
&platformID,
&sampleCount,
&citationSourceCount,
&saasSourceCount,
&citedAnswerCount,
&citedArticleCount,
); scanErr != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse citation ranking")
}
platformID = normalizeMonitoringPlatformID(platformID)
@@ -3503,74 +3558,16 @@ func (s *MonitoringService) loadCitationRanking(
}
item := aggregates[platformID]
item.SampleCount += sampleCount
item.CitationSourceCount += citationSourceCount
item.SaaSSourceCount += saasSourceCount
item.CitedAnswerCount += citedAnswerCount
item.CitedArticleCount += citedArticleCount
aggregates[platformID] = item
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate citation ranking")
}
citedRuns := make(map[string]map[int64]struct{})
citedArticles := make(map[string]map[int64]struct{})
rows, err = s.monitoringPool.Query(ctx, `
SELECT
r.ai_platform_id,
cf.run_id,
cf.cited_url,
cf.normalized_url
FROM question_monitor_runs r
JOIN monitoring_citation_facts cf
ON cf.tenant_id = r.tenant_id AND cf.run_id = r.id
WHERE r.tenant_id = $1
AND ($2::bigint = 0 OR r.brand_id = $2)
AND r.collector_type = $3
AND r.status = 'succeeded'
AND r.business_date BETWEEN $4::date AND $5::date
AND ($6::bigint[] IS NULL OR r.question_id = ANY($6))
AND ($7::text[] IS NULL OR r.ai_platform_id = ANY($7))
`, tenantID, brandID, monitoringCollectorType, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), nullableInt64Array(questionIDs), nullableStringArray(platformQueryIDs))
if err != nil {
return nil, response.ErrInternal(50041, "query_failed", "failed to load citation ranking")
}
defer rows.Close()
for rows.Next() {
var platformID string
var runID int64
var citedURL string
var normalizedURL string
if scanErr := rows.Scan(&platformID, &runID, &citedURL, &normalizedURL); scanErr != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse citation ranking")
}
platformID = normalizeMonitoringPlatformID(platformID)
if platformID == "" {
continue
}
item := aggregates[platformID]
item.CitationSourceCount++
if platformID != "qwen" {
if alias, ok := aliasIndex.Match(citedURL, normalizedURL); ok {
item.SaaSSourceCount++
if _, exists := citedRuns[platformID]; !exists {
citedRuns[platformID] = make(map[int64]struct{})
}
citedRuns[platformID][runID] = struct{}{}
if _, exists := citedArticles[platformID]; !exists {
citedArticles[platformID] = make(map[int64]struct{})
}
citedArticles[platformID][alias.ArticleID] = struct{}{}
}
}
aggregates[platformID] = item
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate citation ranking")
}
for platformID, item := range aggregates {
item.CitedAnswerCount = int64(len(citedRuns[platformID]))
item.CitedArticleCount = int64(len(citedArticles[platformID]))
aggregates[platformID] = item
}
items := make([]MonitoringCitationRanking, 0, len(aggregates))
for platformID, aggregate := range aggregates {
if aggregate.SaaSSourceCount == 0 {
@@ -3611,123 +3608,9 @@ func (s *MonitoringService) loadCitedArticles(
startDate, endDate time.Time,
aiPlatformID *string,
) ([]MonitoringCitedArticle, error) {
if questionIDs != nil && len(questionIDs) == 0 {
return []MonitoringCitedArticle{}, nil
}
platformQueryIDs := monitoringPlatformQueryIDs(aiPlatformID)
aliasIndex, err := loadMonitoringPublishedAliasIndex(ctx, s.monitoringPool, tenantID)
if err != nil {
return nil, err
}
var totalSampleCount int64
if err := s.monitoringPool.QueryRow(ctx, `
SELECT COUNT(*)
FROM question_monitor_runs r
WHERE r.tenant_id = $1
AND ($2::bigint = 0 OR r.brand_id = $2)
AND r.collector_type = $3
AND r.status = 'succeeded'
AND r.business_date BETWEEN $4::date AND $5::date
AND ($6::bigint[] IS NULL OR r.question_id = ANY($6))
AND ($7::text[] IS NULL OR r.ai_platform_id = ANY($7))
`, tenantID, brandID, monitoringCollectorType, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), nullableInt64Array(questionIDs), nullableStringArray(platformQueryIDs)).Scan(&totalSampleCount); err != nil {
return nil, response.ErrInternal(50041, "query_failed", "failed to count cited article denominator")
}
rows, err := s.monitoringPool.Query(ctx, `
SELECT
cf.cited_url,
cf.normalized_url,
NULLIF(cf.article_id, 0),
NULLIF(cf.cited_title, '')
FROM monitoring_citation_facts cf
JOIN question_monitor_runs r
ON r.tenant_id = cf.tenant_id AND r.id = cf.run_id
WHERE cf.tenant_id = $1
AND ($2::bigint = 0 OR cf.brand_id = $2)
AND r.ai_platform_id <> 'qwen'
AND r.collector_type = $3
AND r.status = 'succeeded'
AND r.business_date BETWEEN $4::date AND $5::date
AND ($6::bigint[] IS NULL OR r.question_id = ANY($6))
AND ($7::text[] IS NULL OR r.ai_platform_id = ANY($7))
`, tenantID, brandID, monitoringCollectorType, startDate.Format("2006-01-02"), endDate.Format("2006-01-02"), nullableInt64Array(questionIDs), nullableStringArray(platformQueryIDs))
if err != nil {
return nil, response.ErrInternal(50041, "query_failed", "failed to load cited articles")
}
defer rows.Close()
type citedArticleAggregate struct {
ArticleID int64
ArticleTitle string
PublishPlatform string
CitationCount int64
}
aggregates := make(map[int64]citedArticleAggregate)
var totalArticleCitationCount int64
for rows.Next() {
var citedURL string
var normalizedURL string
var factArticleID sql.NullInt64
var factCitedTitle sql.NullString
if scanErr := rows.Scan(&citedURL, &normalizedURL, &factArticleID, &factCitedTitle); scanErr != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse cited articles")
}
alias, ok := aliasIndex.Match(citedURL, normalizedURL)
articleID := factArticleID.Int64
if !factArticleID.Valid && ok {
articleID = alias.ArticleID
}
if articleID <= 0 {
continue
}
item := aggregates[articleID]
if item.ArticleID == 0 {
item.ArticleID = articleID
if ok && strings.TrimSpace(alias.ArticleTitle) != "" {
item.ArticleTitle = alias.ArticleTitle
} else if factCitedTitle.Valid && strings.TrimSpace(factCitedTitle.String) != "" {
item.ArticleTitle = factCitedTitle.String
} else {
item.ArticleTitle = "未命名文章"
}
if ok && strings.TrimSpace(alias.PublishPlatform) != "" {
item.PublishPlatform = alias.PublishPlatform
} else {
item.PublishPlatform = "已发布内容"
}
}
item.CitationCount++
totalArticleCitationCount++
aggregates[articleID] = item
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate cited articles")
}
items := make([]MonitoringCitedArticle, 0, len(aggregates))
for _, item := range aggregates {
items = append(items, MonitoringCitedArticle{
ArticleID: item.ArticleID,
ArticleTitle: item.ArticleTitle,
PublishPlatform: item.PublishPlatform,
CitationCount: item.CitationCount,
CitationRate: divideAsPointer(item.CitationCount, totalSampleCount),
SourceShare: divideAsPointer(item.CitationCount, totalArticleCitationCount),
})
}
sort.Slice(items, func(left, right int) bool {
if items[left].CitationCount != items[right].CitationCount {
return items[left].CitationCount > items[right].CitationCount
}
return items[left].ArticleID < items[right].ArticleID
})
return items, nil
pageReq := MonitoringPageRequest{Page: 1, PageSize: maxMonitoringListPageSize}
items, _, err := s.loadCitedArticlesPage(ctx, tenantID, brandID, questionIDs, startDate, endDate, aiPlatformID, pageReq)
return items, err
}
func (s *MonitoringService) loadCitedArticlesPage(
@@ -3761,94 +3644,238 @@ func (s *MonitoringService) loadCitedArticlesPage(
return nil, MonitoringPage{}, response.ErrInternal(50041, "query_failed", "failed to count cited article denominator")
}
type citedArticleStats struct {
Total int64
TotalCitationCount int64
sourceRows, totalArticleCitationCount, totalSources, err := s.loadCitedArticleSourceRows(ctx, tenantID, brandID, questionIDs, startDateText, endDateText, platformQueryIDs, pageReq)
if err != nil {
return nil, MonitoringPage{}, err
}
var stats citedArticleStats
if err := s.monitoringPool.QueryRow(ctx, `
WITH article_counts AS (
SELECT COALESCE(NULLIF(cf.article_id, 0), alias.article_id) AS article_id,
COUNT(*)::bigint AS citation_count
FROM monitoring_citation_facts cf
JOIN question_monitor_runs r
ON r.tenant_id = cf.tenant_id AND r.id = cf.run_id
LEFT JOIN monitoring_article_url_aliases alias
ON alias.tenant_id = cf.tenant_id
AND alias.normalized_url = cf.normalized_url
AND alias.confidence = 'high'
WHERE cf.tenant_id = $1
AND ($2::bigint = 0 OR cf.brand_id = $2)
AND r.ai_platform_id <> 'qwen'
AND r.collector_type = $3
AND r.status = 'succeeded'
AND r.business_date BETWEEN $4::date AND $5::date
AND ($6::bigint[] IS NULL OR r.question_id = ANY($6))
AND ($7::text[] IS NULL OR r.ai_platform_id = ANY($7))
AND COALESCE(NULLIF(cf.article_id, 0), alias.article_id) IS NOT NULL
GROUP BY COALESCE(NULLIF(cf.article_id, 0), alias.article_id)
)
SELECT COUNT(*)::bigint, COALESCE(SUM(citation_count), 0)::bigint
FROM article_counts
`, tenantID, brandID, monitoringCollectorType, startDateText, endDateText, nullableInt64Array(questionIDs), nullableStringArray(platformQueryIDs)).Scan(&stats.Total, &stats.TotalCitationCount); err != nil {
return nil, MonitoringPage{}, response.ErrInternal(50041, "query_failed", "failed to count cited articles")
}
page := monitoringPageFromRequest(pageReq, stats.Total)
if stats.Total == 0 {
page := monitoringPageFromRequest(pageReq, totalSources)
if len(sourceRows) == 0 {
return []MonitoringCitedArticle{}, page, nil
}
metaByKey, err := s.loadCitedArticleSourceMeta(ctx, tenantID, sourceRows)
if err != nil {
return nil, MonitoringPage{}, err
}
items := make([]MonitoringCitedArticle, 0, len(sourceRows))
for _, source := range sourceRows {
item := metaByKey[monitoringContentSourceKey(source.SourceType, source.SourceID)]
if item.SourceID == 0 {
item = MonitoringCitedArticle{
ArticleTitle: "未命名文章",
PublishPlatform: "外部文章",
SourceType: source.SourceType,
SourceID: source.SourceID,
}
}
item.CitationCount = source.CitationCount
item.CitationRate = divideAsPointer(source.CitationCount, totalSampleCount)
item.SourceShare = divideAsPointer(source.CitationCount, totalArticleCitationCount)
items = append(items, item)
}
return items, page, nil
}
type monitoringCitedArticleSourceRow struct {
SourceType string
SourceID int64
CitationCount int64
}
func (s *MonitoringService) loadCitedArticleSourceRows(
ctx context.Context,
tenantID, brandID int64,
questionIDs []int64,
startDateText, endDateText string,
platformQueryIDs []string,
pageReq MonitoringPageRequest,
) ([]monitoringCitedArticleSourceRow, int64, int64, error) {
rows, err := s.monitoringPool.Query(ctx, `
WITH article_counts AS (
WITH filtered_sources AS (
SELECT
COALESCE(NULLIF(cf.article_id, 0), alias.article_id) AS article_id,
COALESCE(MAX(alias.article_title_snapshot), MAX(NULLIF(cf.cited_title, '')), '未命名文章') AS article_title,
COALESCE(MAX(alias.publish_platform_name_snapshot), '已发布内容') AS publish_platform,
cf.content_source_type,
cf.content_source_id,
COUNT(*)::bigint AS citation_count
FROM monitoring_citation_facts cf
JOIN question_monitor_runs r
ON r.tenant_id = cf.tenant_id AND r.id = cf.run_id
LEFT JOIN monitoring_article_url_aliases alias
ON alias.tenant_id = cf.tenant_id
AND alias.normalized_url = cf.normalized_url
AND alias.confidence = 'high'
ON r.tenant_id = cf.tenant_id
AND r.id = cf.run_id
WHERE cf.tenant_id = $1
AND ($2::bigint = 0 OR cf.brand_id = $2)
AND r.ai_platform_id <> 'qwen'
AND cf.ai_platform_id <> 'qwen'
AND cf.business_date BETWEEN $4::date AND $5::date
AND cf.content_source_type IS NOT NULL
AND cf.content_source_id IS NOT NULL
AND r.collector_type = $3
AND r.status = 'succeeded'
AND r.business_date BETWEEN $4::date AND $5::date
AND ($6::bigint[] IS NULL OR r.question_id = ANY($6))
AND ($7::text[] IS NULL OR r.ai_platform_id = ANY($7))
AND COALESCE(NULLIF(cf.article_id, 0), alias.article_id) IS NOT NULL
GROUP BY COALESCE(NULLIF(cf.article_id, 0), alias.article_id)
GROUP BY cf.content_source_type, cf.content_source_id
),
counts AS (
SELECT
COALESCE(SUM(citation_count), 0)::bigint AS total_citation_count,
COUNT(*)::bigint AS total_source_count
FROM filtered_sources
),
paged_sources AS (
SELECT content_source_type, content_source_id, citation_count
FROM filtered_sources
ORDER BY citation_count DESC, content_source_type ASC, content_source_id ASC
LIMIT $8 OFFSET $9
)
SELECT article_id, article_title, publish_platform, citation_count
FROM article_counts
ORDER BY citation_count DESC, article_id ASC
LIMIT $8 OFFSET $9
SELECT
ps.content_source_type,
ps.content_source_id,
ps.citation_count,
counts.total_citation_count,
counts.total_source_count
FROM counts
LEFT JOIN paged_sources ps ON TRUE
`, tenantID, brandID, monitoringCollectorType, startDateText, endDateText, nullableInt64Array(questionIDs), nullableStringArray(platformQueryIDs), pageReq.PageSize, monitoringPageOffset(pageReq))
if err != nil {
return nil, MonitoringPage{}, response.ErrInternal(50041, "query_failed", "failed to load cited articles")
return nil, 0, 0, response.ErrInternal(50041, "query_failed", "failed to load cited articles")
}
defer rows.Close()
items := make([]MonitoringCitedArticle, 0, pageReq.PageSize)
items := make([]monitoringCitedArticleSourceRow, 0, pageReq.PageSize)
var totalCitationCount int64
var totalSourceCount int64
for rows.Next() {
var item MonitoringCitedArticle
if scanErr := rows.Scan(&item.ArticleID, &item.ArticleTitle, &item.PublishPlatform, &item.CitationCount); scanErr != nil {
return nil, MonitoringPage{}, response.ErrInternal(50041, "scan_failed", "failed to parse cited articles")
var sourceType sql.NullString
var sourceID sql.NullInt64
var citationCount sql.NullInt64
if scanErr := rows.Scan(
&sourceType,
&sourceID,
&citationCount,
&totalCitationCount,
&totalSourceCount,
); scanErr != nil {
return nil, 0, 0, response.ErrInternal(50041, "scan_failed", "failed to parse cited articles")
}
if !sourceType.Valid || !sourceID.Valid || !citationCount.Valid {
continue
}
item := monitoringCitedArticleSourceRow{
SourceType: strings.TrimSpace(sourceType.String),
SourceID: sourceID.Int64,
CitationCount: citationCount.Int64,
}
if item.SourceType == "" || item.SourceID <= 0 {
continue
}
item.CitationRate = divideAsPointer(item.CitationCount, totalSampleCount)
item.SourceShare = divideAsPointer(item.CitationCount, stats.TotalCitationCount)
items = append(items, item)
}
if err := rows.Err(); err != nil {
return nil, MonitoringPage{}, response.ErrInternal(50041, "scan_failed", "failed to iterate cited articles")
return nil, 0, 0, response.ErrInternal(50041, "scan_failed", "failed to iterate cited articles")
}
return items, page, nil
return items, totalCitationCount, totalSourceCount, nil
}
func (s *MonitoringService) loadCitedArticleSourceMeta(ctx context.Context, tenantID int64, sources []monitoringCitedArticleSourceRow) (map[string]MonitoringCitedArticle, error) {
result := make(map[string]MonitoringCitedArticle, len(sources))
publishedIDs := make([]int64, 0, len(sources))
manualIDs := make([]int64, 0, len(sources))
for _, source := range sources {
switch source.SourceType {
case "published_article":
publishedIDs = append(publishedIDs, source.SourceID)
case "manual_mark":
manualIDs = append(manualIDs, source.SourceID)
}
}
if len(publishedIDs) > 0 {
rows, err := s.monitoringPool.Query(ctx, `
SELECT DISTINCT ON (article_id)
article_id,
COALESCE(NULLIF(article_title_snapshot, ''), '未命名文章') AS article_title,
COALESCE(NULLIF(publish_platform_name_snapshot, ''), '已发布内容') AS publish_platform,
COALESCE(NULLIF(original_url, ''), normalized_url) AS source_url
FROM monitoring_article_url_aliases
WHERE tenant_id = $1
AND confidence = 'high'
AND article_id = ANY($2)
ORDER BY article_id, updated_at DESC, id DESC
`, tenantID, publishedIDs)
if err != nil {
return nil, response.ErrInternal(50041, "query_failed", "failed to load cited article metadata")
}
defer rows.Close()
for rows.Next() {
var articleID int64
var item MonitoringCitedArticle
if scanErr := rows.Scan(&articleID, &item.ArticleTitle, &item.PublishPlatform, &item.SourceURL); scanErr != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse cited article metadata")
}
value := articleID
item.ArticleID = &value
item.ArticleTitle = repairMonitoringMojibake(item.ArticleTitle)
item.PublishPlatform = repairMonitoringMojibake(item.PublishPlatform)
item.SourceType = "published_article"
item.SourceID = articleID
result[monitoringContentSourceKey(item.SourceType, item.SourceID)] = item
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate cited article metadata")
}
}
if len(manualIDs) > 0 {
rows, err := s.monitoringPool.Query(ctx, `
SELECT
id,
COALESCE(NULLIF(article_title, ''), '未命名文章') AS article_title,
COALESCE(NULLIF(publish_platform, ''), '外部文章') AS publish_platform,
COALESCE(NULLIF(original_url, ''), normalized_url) AS source_url
FROM monitoring_user_marked_articles
WHERE tenant_id = $1
AND id = ANY($2)
`, tenantID, manualIDs)
if err != nil {
return nil, response.ErrInternal(50041, "query_failed", "failed to load marked article metadata")
}
defer rows.Close()
for rows.Next() {
var item MonitoringCitedArticle
if scanErr := rows.Scan(&item.SourceID, &item.ArticleTitle, &item.PublishPlatform, &item.SourceURL); scanErr != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse marked article metadata")
}
item.ArticleTitle = repairMonitoringMojibake(item.ArticleTitle)
item.PublishPlatform = repairMonitoringMojibake(item.PublishPlatform)
item.SourceType = "manual_mark"
result[monitoringContentSourceKey(item.SourceType, item.SourceID)] = item
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate marked article metadata")
}
}
return result, nil
}
func monitoringContentSourceKey(sourceType string, sourceID int64) string {
return strings.TrimSpace(sourceType) + ":" + strconv.FormatInt(sourceID, 10)
}
func monitoringAliasContentKey(alias monitoringPublishedAlias) string {
sourceType := strings.TrimSpace(alias.SourceType)
sourceID := alias.SourceID
if sourceType == "" {
sourceType = "published_article"
}
if sourceID == 0 && alias.ArticleID > 0 {
sourceID = alias.ArticleID
}
if sourceID > 0 {
return sourceType + ":" + strconv.FormatInt(sourceID, 10)
}
key := strings.TrimSpace(alias.NormalizedURL)
if key == "" {
key = strings.TrimSpace(alias.OriginalURL)
}
return sourceType + ":url:" + key
}
func (s *MonitoringService) loadQuestionText(ctx context.Context, tenantID, brandID, questionID int64) (string, error) {
@@ -4130,10 +4157,35 @@ func (s *MonitoringService) loadCitationsForRuns(ctx context.Context, tenantID i
}
var articleID *int64
var articleTitle *string
var sourceType *string
var sourceID *int64
var sourceURL *string
if alias, ok := aliasIndex.Match(citedURL, normalizedURL); ok {
articleID = &alias.ArticleID
if alias.ArticleID > 0 {
value := alias.ArticleID
articleID = &value
}
title := repairMonitoringMojibake(alias.ArticleTitle)
articleTitle = &title
aliasSourceType := strings.TrimSpace(alias.SourceType)
if aliasSourceType == "" {
aliasSourceType = "published_article"
}
sourceType = &aliasSourceType
aliasSourceID := alias.SourceID
if aliasSourceID == 0 && alias.ArticleID > 0 {
aliasSourceID = alias.ArticleID
}
if aliasSourceID > 0 {
sourceID = &aliasSourceID
}
aliasSourceURL := strings.TrimSpace(alias.OriginalURL)
if aliasSourceURL == "" {
aliasSourceURL = strings.TrimSpace(alias.NormalizedURL)
}
if aliasSourceURL != "" {
sourceURL = &aliasSourceURL
}
resolutionStatus = "resolved"
resolutionConfidence = "high"
}
@@ -4144,6 +4196,9 @@ func (s *MonitoringService) loadCitationsForRuns(ctx context.Context, tenantID i
SiteKey: siteKey,
FaviconURL: faviconURL(siteKey),
ArticleID: articleID,
SourceType: sourceType,
SourceID: sourceID,
SourceURL: sourceURL,
ArticleTitle: articleTitle,
ResolutionStatus: resolutionStatus,
ResolutionConfidence: resolutionConfidence,
@@ -4161,21 +4216,16 @@ func (s *MonitoringService) loadQuestionCitationAnalysis(ctx context.Context, te
return []MonitoringQuestionCitationStats{}, nil
}
aliasIndex, err := loadMonitoringPublishedAliasIndex(ctx, s.monitoringPool, tenantID)
if err != nil {
return nil, err
}
rows, err := s.monitoringPool.Query(ctx, `
WITH filtered_facts AS (
SELECT
r.ai_platform_id,
cf.id,
cf.cited_url,
cf.normalized_url,
cf.host,
cf.registrable_domain,
cf.site_key
cf.site_key,
cf.content_source_type,
cf.content_source_id
FROM question_monitor_runs r
JOIN monitoring_citation_facts cf
ON cf.tenant_id = r.tenant_id AND cf.run_id = r.id
@@ -4220,11 +4270,11 @@ func (s *MonitoringService) loadQuestionCitationAnalysis(ctx context.Context, te
)
SELECT
ff.ai_platform_id,
ff.cited_url,
ff.normalized_url,
COALESCE(dm.mapped_site_name, ff.site_key, ff.registrable_domain) AS site_name,
COALESCE(dm.mapped_site_key, ff.site_key) AS site_key,
COALESCE(dm.mapped_domain, ff.site_key, ff.registrable_domain) AS site_domain
COALESCE(dm.mapped_domain, ff.site_key, ff.registrable_domain) AS site_domain,
ff.content_source_type,
ff.content_source_id
FROM filtered_facts ff
LEFT JOIN domain_mappings dm
ON dm.host = ff.host
@@ -4246,12 +4296,12 @@ func (s *MonitoringService) loadQuestionCitationAnalysis(ctx context.Context, te
totals := make(map[string]int64)
for rows.Next() {
var platformID string
var citedURL string
var normalizedURL string
var siteName string
var siteKey string
var siteDomain string
if scanErr := rows.Scan(&platformID, &citedURL, &normalizedURL, &siteName, &siteKey, &siteDomain); scanErr != nil {
var sourceType sql.NullString
var sourceID sql.NullInt64
if scanErr := rows.Scan(&platformID, &siteName, &siteKey, &siteDomain, &sourceType, &sourceID); scanErr != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse question citation analysis")
}
platformID = normalizeMonitoringPlatformID(platformID)
@@ -4273,10 +4323,8 @@ func (s *MonitoringService) loadQuestionCitationAnalysis(ctx context.Context, te
}
}
item.CitationCount++
if platformID != "qwen" {
if _, ok := aliasIndex.Match(citedURL, normalizedURL); ok {
item.ContentCitationCount++
}
if platformID != "qwen" && sourceType.Valid && strings.TrimSpace(sourceType.String) != "" && sourceID.Valid {
item.ContentCitationCount++
}
aggregates[key] = item
totals[platformID]++
@@ -4311,16 +4359,11 @@ func (s *MonitoringService) loadQuestionContentCitations(ctx context.Context, te
return []MonitoringQuestionContentCitation{}, nil
}
aliasIndex, err := loadMonitoringPublishedAliasIndex(ctx, s.monitoringPool, tenantID)
if err != nil {
return nil, err
}
rows, err := s.monitoringPool.Query(ctx, `
SELECT
r.ai_platform_id,
cf.cited_url,
cf.normalized_url
cf.content_source_type,
cf.content_source_id
FROM question_monitor_runs r
JOIN monitoring_citation_facts cf
ON cf.tenant_id = r.tenant_id AND cf.run_id = r.id
@@ -4336,16 +4379,17 @@ func (s *MonitoringService) loadQuestionContentCitations(ctx context.Context, te
type contentCitationKey struct {
PlatformID string
ArticleID int64
SourceType string
SourceID int64
}
aggregates := make(map[contentCitationKey]MonitoringQuestionContentCitation)
totals := make(map[string]int64)
for rows.Next() {
var platformID string
var citedURL string
var normalizedURL string
if scanErr := rows.Scan(&platformID, &citedURL, &normalizedURL); scanErr != nil {
var sourceType sql.NullString
var sourceID sql.NullInt64
if scanErr := rows.Scan(&platformID, &sourceType, &sourceID); scanErr != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to parse content citations")
}
platformID = normalizeMonitoringPlatformID(platformID)
@@ -4353,22 +4397,22 @@ func (s *MonitoringService) loadQuestionContentCitations(ctx context.Context, te
continue
}
totals[platformID]++
alias, ok := aliasIndex.Match(citedURL, normalizedURL)
if !ok {
normalizedSourceType := strings.TrimSpace(sourceType.String)
if !sourceType.Valid || normalizedSourceType == "" || !sourceID.Valid || sourceID.Int64 <= 0 {
continue
}
key := contentCitationKey{
PlatformID: platformID,
ArticleID: alias.ArticleID,
SourceType: normalizedSourceType,
SourceID: sourceID.Int64,
}
item := aggregates[key]
if item.ArticleID == 0 {
if item.SourceID == 0 {
item = MonitoringQuestionContentCitation{
ArticleID: alias.ArticleID,
ArticleTitle: alias.ArticleTitle,
PublishPlatform: alias.PublishPlatform,
AIPlatformID: platformID,
PlatformName: platformDisplayName(platformID),
SourceType: normalizedSourceType,
SourceID: sourceID.Int64,
AIPlatformID: platformID,
PlatformName: platformDisplayName(platformID),
}
}
item.CitationCount++
@@ -4377,9 +4421,36 @@ func (s *MonitoringService) loadQuestionContentCitations(ctx context.Context, te
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50041, "scan_failed", "failed to iterate content citations")
}
if len(aggregates) == 0 {
return []MonitoringQuestionContentCitation{}, nil
}
sourceRows := make([]monitoringCitedArticleSourceRow, 0, len(aggregates))
for key := range aggregates {
sourceRows = append(sourceRows, monitoringCitedArticleSourceRow{
SourceType: key.SourceType,
SourceID: key.SourceID,
})
}
metaByKey, err := s.loadCitedArticleSourceMeta(ctx, tenantID, sourceRows)
if err != nil {
return nil, err
}
items := make([]MonitoringQuestionContentCitation, 0, len(aggregates))
for _, item := range aggregates {
for key, item := range aggregates {
if meta := metaByKey[monitoringContentSourceKey(key.SourceType, key.SourceID)]; meta.SourceID > 0 {
item.ArticleID = meta.ArticleID
item.ArticleTitle = meta.ArticleTitle
item.PublishPlatform = meta.PublishPlatform
item.SourceURL = meta.SourceURL
}
if strings.TrimSpace(item.ArticleTitle) == "" {
item.ArticleTitle = "未命名文章"
}
if strings.TrimSpace(item.PublishPlatform) == "" {
item.PublishPlatform = "外部文章"
}
item.CitationRate = divideAsPointer(item.CitationCount, totals[item.AIPlatformID])
items = append(items, item)
}
@@ -4390,7 +4461,10 @@ func (s *MonitoringService) loadQuestionContentCitations(ctx context.Context, te
if items[left].CitationCount != items[right].CitationCount {
return items[left].CitationCount > items[right].CitationCount
}
return items[left].ArticleID < items[right].ArticleID
if items[left].SourceType != items[right].SourceType {
return items[left].SourceType < items[right].SourceType
}
return items[left].SourceID < items[right].SourceID
})
return items, nil
@@ -25,6 +25,12 @@ type monitoringCollectNowRequest struct {
TargetClientID *string `json:"target_client_id"`
}
type monitoringMarkedArticleRequest struct {
ArticleTitle string `json:"article_title"`
OriginalURL string `json:"original_url"`
BrandID *int64 `json:"brand_id"`
}
func NewMonitoringHandler(a *bootstrap.App) *MonitoringHandler {
return &MonitoringHandler{
svc: a.MonitoringService,
@@ -178,6 +184,86 @@ func (h *MonitoringHandler) CollectNow(c *gin.Context) {
response.Success(c, data)
}
func (h *MonitoringHandler) ListMarkedArticles(c *gin.Context) {
page, err := parseOptionalInt(c.Query("page"))
if err != nil {
response.Error(c, response.ErrBadRequest(40031, "invalid_page", "page must be a number"))
return
}
pageSize, err := parseOptionalInt(c.Query("page_size"))
if err != nil {
response.Error(c, response.ErrBadRequest(40031, "invalid_page_size", "page_size must be a number"))
return
}
data, svcErr := h.svc.ListMarkedArticles(c.Request.Context(), app.MonitoringMarkedArticleListParams{
Domain: c.Query("domain"),
Title: c.Query("title"),
CreatedFrom: c.Query("created_from"),
CreatedTo: c.Query("created_to"),
Page: page,
PageSize: pageSize,
})
if svcErr != nil {
response.Error(c, svcErr)
return
}
response.Success(c, data)
}
func (h *MonitoringHandler) CreateMarkedArticle(c *gin.Context) {
var req monitoringMarkedArticleRequest
if err := c.ShouldBindJSON(&req); err != nil {
response.Error(c, response.ErrBadRequest(40031, "invalid_marked_article_request", "marked article request is invalid"))
return
}
data, svcErr := h.svc.CreateMarkedArticle(c.Request.Context(), app.MonitoringMarkedArticleRequest{
ArticleTitle: req.ArticleTitle,
OriginalURL: req.OriginalURL,
BrandID: req.BrandID,
})
if svcErr != nil {
response.Error(c, svcErr)
return
}
response.Success(c, data)
}
func (h *MonitoringHandler) UpdateMarkedArticle(c *gin.Context) {
id, err := strconv.ParseInt(c.Param("id"), 10, 64)
if err != nil {
response.Error(c, response.ErrBadRequest(40031, "invalid_marked_article_id", "marked article id must be a number"))
return
}
var req monitoringMarkedArticleRequest
if err := c.ShouldBindJSON(&req); err != nil {
response.Error(c, response.ErrBadRequest(40031, "invalid_marked_article_request", "marked article request is invalid"))
return
}
data, svcErr := h.svc.UpdateMarkedArticle(c.Request.Context(), id, app.MonitoringMarkedArticleRequest{
ArticleTitle: req.ArticleTitle,
OriginalURL: req.OriginalURL,
BrandID: req.BrandID,
})
if svcErr != nil {
response.Error(c, svcErr)
return
}
response.Success(c, data)
}
func (h *MonitoringHandler) DeleteMarkedArticle(c *gin.Context) {
id, err := strconv.ParseInt(c.Param("id"), 10, 64)
if err != nil {
response.Error(c, response.ErrBadRequest(40031, "invalid_marked_article_id", "marked article id must be a number"))
return
}
if svcErr := h.svc.DeleteMarkedArticle(c.Request.Context(), id); svcErr != nil {
response.Error(c, svcErr)
return
}
response.Success(c, gin.H{"ok": true})
}
func parseOptionalInt64(raw string) (int64, error) {
if raw == "" {
return 0, nil
@@ -257,6 +257,10 @@ func RegisterRoutes(app *bootstrap.App) {
monitoringHandler := NewMonitoringHandler(app)
monitoring.GET("/dashboard/composite", monitoringHandler.DashboardComposite)
monitoring.GET("/citation-summary", monitoringHandler.CitationSummary)
monitoring.GET("/marked-articles", monitoringHandler.ListMarkedArticles)
monitoring.POST("/marked-articles", monitoringHandler.CreateMarkedArticle)
monitoring.PUT("/marked-articles/:id", monitoringHandler.UpdateMarkedArticle)
monitoring.DELETE("/marked-articles/:id", monitoringHandler.DeleteMarkedArticle)
monitoring.GET("/brands/:brand_id/questions/:question_id/detail", monitoringHandler.QuestionDetail)
monitoring.POST("/brands/:brand_id/collect-now", monitoringHandler.CollectNow)
@@ -42,6 +42,10 @@ func TestProtectedRoutes_RequireAuth(t *testing.T) {
{http.MethodGet, "/api/tenant/brands"},
{http.MethodGet, "/api/tenant/brands/library-summary"},
{http.MethodGet, "/api/tenant/monitoring/brands/:brand_id/questions/:question_id/detail"},
{http.MethodGet, "/api/tenant/monitoring/marked-articles"},
{http.MethodPost, "/api/tenant/monitoring/marked-articles"},
{http.MethodPut, "/api/tenant/monitoring/marked-articles/:id"},
{http.MethodDelete, "/api/tenant/monitoring/marked-articles/:id"},
{http.MethodPost, "/api/tenant/brands"},
}
@@ -0,0 +1,7 @@
DROP INDEX IF EXISTS idx_monitoring_citation_facts_content_source;
ALTER TABLE monitoring_citation_facts
DROP COLUMN IF EXISTS content_source_id,
DROP COLUMN IF EXISTS content_source_type;
DROP TABLE IF EXISTS monitoring_user_marked_articles;
@@ -0,0 +1,41 @@
CREATE TABLE IF NOT EXISTS monitoring_user_marked_articles (
id BIGSERIAL PRIMARY KEY,
tenant_id BIGINT NOT NULL,
brand_id BIGINT,
marked_by_user_id BIGINT,
article_title TEXT NOT NULL,
original_url TEXT NOT NULL,
normalized_url TEXT NOT NULL,
host VARCHAR(255) NOT NULL,
registrable_domain VARCHAR(255) NOT NULL,
site_key VARCHAR(255) NOT NULL,
last_path_segment VARCHAR(255),
publish_platform VARCHAR(100) NOT NULL DEFAULT '外部文章',
marked_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
expires_at TIMESTAMPTZ NOT NULL DEFAULT (NOW() + INTERVAL '6 months'),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE UNIQUE INDEX IF NOT EXISTS uk_monitoring_user_marked_article_url
ON monitoring_user_marked_articles(tenant_id, normalized_url);
CREATE INDEX IF NOT EXISTS idx_monitoring_user_marked_articles_domain
ON monitoring_user_marked_articles(tenant_id, registrable_domain, created_at DESC, id DESC);
CREATE INDEX IF NOT EXISTS idx_monitoring_user_marked_articles_title
ON monitoring_user_marked_articles USING GIN (to_tsvector('simple', article_title));
CREATE INDEX IF NOT EXISTS idx_monitoring_user_marked_articles_created
ON monitoring_user_marked_articles(tenant_id, created_at DESC, id DESC);
CREATE INDEX IF NOT EXISTS idx_monitoring_user_marked_articles_expires
ON monitoring_user_marked_articles(expires_at, id);
ALTER TABLE monitoring_citation_facts
ADD COLUMN IF NOT EXISTS content_source_type VARCHAR(30),
ADD COLUMN IF NOT EXISTS content_source_id BIGINT;
CREATE INDEX IF NOT EXISTS idx_monitoring_citation_facts_content_source
ON monitoring_citation_facts(tenant_id, content_source_type, content_source_id)
WHERE content_source_type IS NOT NULL AND content_source_id IS NOT NULL;
@@ -0,0 +1,5 @@
DROP INDEX IF EXISTS idx_article_url_alias_article_lookup;
DROP INDEX IF EXISTS idx_citation_facts_content_window;
DROP INDEX IF EXISTS idx_citation_facts_normalized_url;
DROP INDEX IF EXISTS idx_citation_facts_tenant_run;
DROP INDEX IF EXISTS idx_monitor_runs_citation_summary;
@@ -0,0 +1,63 @@
CREATE INDEX IF NOT EXISTS idx_monitor_runs_citation_summary
ON question_monitor_runs (
tenant_id,
brand_id,
collector_type,
status,
business_date,
question_id,
ai_platform_id,
id
);
CREATE INDEX IF NOT EXISTS idx_citation_facts_tenant_run
ON monitoring_citation_facts(tenant_id, run_id);
CREATE INDEX IF NOT EXISTS idx_citation_facts_normalized_url
ON monitoring_citation_facts(tenant_id, normalized_url);
CREATE INDEX IF NOT EXISTS idx_citation_facts_content_window
ON monitoring_citation_facts (
tenant_id,
brand_id,
business_date,
ai_platform_id,
content_source_type,
content_source_id,
run_id
)
WHERE content_source_type IS NOT NULL AND content_source_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_article_url_alias_article_lookup
ON monitoring_article_url_aliases(tenant_id, article_id, updated_at DESC, id DESC)
WHERE confidence = 'high';
UPDATE monitoring_citation_facts cf
SET content_source_type = 'published_article',
content_source_id = alias.article_id
FROM monitoring_article_url_aliases alias
WHERE alias.tenant_id = cf.tenant_id
AND alias.normalized_url = cf.normalized_url
AND alias.confidence = 'high'
AND alias.article_id IS NOT NULL
AND (
cf.content_source_type IS DISTINCT FROM 'published_article'
OR cf.content_source_id IS DISTINCT FROM alias.article_id
);
UPDATE monitoring_citation_facts
SET content_source_type = 'published_article',
content_source_id = article_id
WHERE content_source_type IS NULL
AND content_source_id IS NULL
AND article_id IS NOT NULL;
UPDATE monitoring_citation_facts cf
SET content_source_type = 'manual_mark',
content_source_id = marked.id
FROM monitoring_user_marked_articles marked
WHERE marked.tenant_id = cf.tenant_id
AND marked.normalized_url = cf.normalized_url
AND marked.expires_at > NOW()
AND cf.content_source_type IS NULL
AND cf.content_source_id IS NULL;
@@ -0,0 +1,7 @@
DROP INDEX IF EXISTS idx_marked_articles_title_trgm;
DROP INDEX IF EXISTS idx_marked_articles_domain_trgm;
DROP INDEX IF EXISTS idx_citation_facts_day_content_run;
DROP INDEX IF EXISTS idx_citation_facts_run_content;
CREATE INDEX IF NOT EXISTS idx_monitoring_user_marked_articles_title
ON monitoring_user_marked_articles USING GIN (to_tsvector('simple', article_title));
@@ -0,0 +1,44 @@
CREATE EXTENSION IF NOT EXISTS pg_trgm;
CREATE INDEX IF NOT EXISTS idx_citation_facts_run_content
ON monitoring_citation_facts (
tenant_id,
run_id,
content_source_type,
content_source_id
)
WHERE content_source_type IS NOT NULL AND content_source_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_citation_facts_day_content_run
ON monitoring_citation_facts (
tenant_id,
brand_id,
business_date,
ai_platform_id,
run_id,
content_source_type,
content_source_id
)
WHERE content_source_type IS NOT NULL AND content_source_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_marked_articles_domain_trgm
ON monitoring_user_marked_articles
USING GIN (
lower(registrable_domain) gin_trgm_ops,
lower(host) gin_trgm_ops,
lower(site_key) gin_trgm_ops
);
DROP INDEX IF EXISTS idx_monitoring_user_marked_articles_title;
CREATE INDEX IF NOT EXISTS idx_marked_articles_title_trgm
ON monitoring_user_marked_articles
USING GIN (lower(article_title) gin_trgm_ops);
ALTER TABLE monitoring_user_marked_articles
ALTER COLUMN publish_platform SET DEFAULT '外部文章';
UPDATE monitoring_user_marked_articles
SET publish_platform = '外部文章',
updated_at = NOW()
WHERE publish_platform IN ('自有文章', 'Own Article', 'Owned Article', 'Manual Article');
@@ -0,0 +1,10 @@
DROP INDEX IF EXISTS idx_marked_articles_site_key_lookup;
DROP INDEX IF EXISTS idx_marked_articles_host_lookup;
CREATE INDEX IF NOT EXISTS idx_marked_articles_domain_trgm
ON monitoring_user_marked_articles
USING GIN (
lower(registrable_domain) gin_trgm_ops,
lower(host) gin_trgm_ops,
lower(site_key) gin_trgm_ops
);
@@ -0,0 +1,7 @@
DROP INDEX IF EXISTS idx_marked_articles_domain_trgm;
CREATE INDEX IF NOT EXISTS idx_marked_articles_host_lookup
ON monitoring_user_marked_articles(tenant_id, host, created_at DESC, id DESC);
CREATE INDEX IF NOT EXISTS idx_marked_articles_site_key_lookup
ON monitoring_user_marked_articles(tenant_id, site_key, created_at DESC, id DESC);
@@ -0,0 +1,6 @@
UPDATE ops.scheduler_jobs
SET display_name = '监控历史保留清理',
description = '按保留窗口和索引限批删除最近 30 天以前的监控历史数据。',
config = config - 'external_article_mark_retention' - 'clears_manual_mark_attribution',
updated_at = NOW()
WHERE job_key = 'monitoring_retention_cleanup';
@@ -0,0 +1,6 @@
UPDATE ops.scheduler_jobs
SET display_name = '监控历史与外部文章标记清理',
description = '按保留窗口和索引限批删除监控历史;外部文章标记到期 6 个月后删除,并同步清理引用事实归因。',
config = config || '{"external_article_mark_retention":"6 months","clears_manual_mark_attribution":true}'::jsonb,
updated_at = NOW()
WHERE job_key = 'monitoring_retention_cleanup';