Add monitoring service and database schema

- Implement monitoring service with heartbeat, lease tasks, resume tasks, and task result handling.
- Create monitoring time utilities for business date calculations.
- Add unit tests for date window resolution and business day handling.
- Define database schema for monitoring-related tables including quotas, daily reports, and task management.
- Establish migration scripts for creating and dropping monitoring tables.
This commit is contained in:
2026-04-12 09:56:18 +08:00
parent 9b4dd09780
commit 6066f43a7d
67 changed files with 16312 additions and 125 deletions
@@ -0,0 +1,192 @@
package app
import (
"context"
"time"
"github.com/jackc/pgx/v5"
"github.com/geo-platform/tenant-api/internal/shared/response"
)
const (
monitoringSkipReasonBrandDeleted = "brand_deleted"
monitoringSkipReasonKeywordDeleted = "keyword_deleted"
monitoringSkipReasonQuestionDeleted = "question_deleted"
)
var monitoringProjectionExcludedSkipReasons = []string{
monitoringSkipReasonBrandDeleted,
monitoringSkipReasonKeywordDeleted,
monitoringSkipReasonQuestionDeleted,
}
func (s *BrandService) cleanupMonitoringAfterBrandDelete(ctx context.Context, tenantID, brandID int64) error {
return s.cleanupMonitoringDeletion(ctx, tenantID, brandID, cleanupMonitoringDeletionOptions{
skipReason: monitoringSkipReasonBrandDeleted,
errorMessage: "brand deleted from brand library",
deleteAllByBrand: true,
})
}
func (s *BrandService) cleanupMonitoringAfterKeywordDelete(ctx context.Context, tenantID, brandID, keywordID int64, questionIDs []int64) error {
return s.cleanupMonitoringDeletion(ctx, tenantID, brandID, cleanupMonitoringDeletionOptions{
keywordID: &keywordID,
questionIDs: questionIDs,
skipReason: monitoringSkipReasonKeywordDeleted,
errorMessage: "keyword deleted from brand library",
})
}
func (s *BrandService) cleanupMonitoringAfterQuestionDelete(ctx context.Context, tenantID, brandID, questionID int64) error {
return s.cleanupMonitoringDeletion(ctx, tenantID, brandID, cleanupMonitoringDeletionOptions{
questionIDs: []int64{questionID},
skipReason: monitoringSkipReasonQuestionDeleted,
errorMessage: "question deleted from brand library",
})
}
type cleanupMonitoringDeletionOptions struct {
keywordID *int64
questionIDs []int64
skipReason string
errorMessage string
deleteAllByBrand bool
}
func (s *BrandService) cleanupMonitoringDeletion(ctx context.Context, tenantID, brandID int64, options cleanupMonitoringDeletionOptions) error {
if s.monitoringPool == nil {
return nil
}
tx, err := s.monitoringPool.Begin(ctx)
if err != nil {
return response.ErrInternal(50041, "begin_failed", "failed to start monitoring cleanup")
}
defer tx.Rollback(ctx)
if err := s.markMonitoringQuestionSnapshotsDeleted(ctx, tx, tenantID, brandID, options); err != nil {
return err
}
businessDay := monitoringBusinessToday()
if err := s.skipMonitoringTasksForDeletion(ctx, tx, tenantID, brandID, businessDay, options); err != nil {
return err
}
if err := s.rebuildMonitoringBrandProjection(ctx, tx, tenantID, brandID, businessDay); err != nil {
return err
}
if err := tx.Commit(ctx); err != nil {
return response.ErrInternal(50041, "commit_failed", "failed to commit monitoring cleanup")
}
return nil
}
func (s *BrandService) markMonitoringQuestionSnapshotsDeleted(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, options cleanupMonitoringDeletionOptions) error {
switch {
case options.deleteAllByBrand:
if _, err := tx.Exec(ctx, `
UPDATE monitoring_question_config_snapshots
SET deleted_at = COALESCE(deleted_at, NOW()),
updated_at = NOW()
WHERE tenant_id = $1
AND brand_id = $2
AND deleted_at IS NULL
`, tenantID, brandID); err != nil {
return response.ErrInternal(50041, "snapshot_delete_failed", "failed to invalidate brand monitoring snapshots")
}
case options.keywordID != nil:
if _, err := tx.Exec(ctx, `
UPDATE monitoring_question_config_snapshots
SET deleted_at = COALESCE(deleted_at, NOW()),
updated_at = NOW()
WHERE tenant_id = $1
AND brand_id = $2
AND keyword_id = $3
AND deleted_at IS NULL
`, tenantID, brandID, *options.keywordID); err != nil {
return response.ErrInternal(50041, "snapshot_delete_failed", "failed to invalidate keyword monitoring snapshots")
}
default:
if len(options.questionIDs) == 0 {
return nil
}
if _, err := tx.Exec(ctx, `
UPDATE monitoring_question_config_snapshots
SET deleted_at = COALESCE(deleted_at, NOW()),
updated_at = NOW()
WHERE tenant_id = $1
AND brand_id = $2
AND question_id = ANY($3)
AND deleted_at IS NULL
`, tenantID, brandID, options.questionIDs); err != nil {
return response.ErrInternal(50041, "snapshot_delete_failed", "failed to invalidate question monitoring snapshots")
}
}
return nil
}
func (s *BrandService) skipMonitoringTasksForDeletion(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDay time.Time, options cleanupMonitoringDeletionOptions) error {
dateText := businessDay.Format("2006-01-02")
switch {
case options.deleteAllByBrand:
if _, err := tx.Exec(ctx, `
UPDATE monitoring_collect_tasks
SET status = 'skipped',
lease_token_hash = NULL,
leased_to_executor = NULL,
leased_at = NULL,
lease_expires_at = NULL,
completed_at = COALESCE(completed_at, NOW()),
skip_reason = $4,
error_message = $5,
updated_at = NOW()
WHERE tenant_id = $1
AND brand_id = $2
AND collector_type = $3
AND business_date = $6::date
AND status IN ('pending', 'leased', 'received', 'expired')
`, tenantID, brandID, monitoringCollectorType, options.skipReason, options.errorMessage, dateText); err != nil {
return response.ErrInternal(50041, "task_skip_failed", "failed to invalidate brand monitoring tasks")
}
default:
if len(options.questionIDs) == 0 {
return nil
}
if _, err := tx.Exec(ctx, `
UPDATE monitoring_collect_tasks
SET status = 'skipped',
lease_token_hash = NULL,
leased_to_executor = NULL,
leased_at = NULL,
lease_expires_at = NULL,
completed_at = COALESCE(completed_at, NOW()),
skip_reason = $4,
error_message = $5,
updated_at = NOW()
WHERE tenant_id = $1
AND brand_id = $2
AND collector_type = $3
AND business_date = $6::date
AND question_id = ANY($7)
AND status IN ('pending', 'leased', 'received', 'expired')
`, tenantID, brandID, monitoringCollectorType, options.skipReason, options.errorMessage, dateText, options.questionIDs); err != nil {
return response.ErrInternal(50041, "task_skip_failed", "failed to invalidate question monitoring tasks")
}
}
return nil
}
func (s *BrandService) rebuildMonitoringBrandProjection(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDay time.Time) error {
callbackService := &MonitoringCallbackService{
businessPool: s.pool,
monitoringPool: s.monitoringPool,
}
return callbackService.rebuildBrandDailyProjection(ctx, tx, tenantID, brandID, businessDay)
}
+80 -6
View File
@@ -15,12 +15,13 @@ import (
)
type BrandService struct {
pool *pgxpool.Pool
auditLogs *auditlog.AsyncWriter
pool *pgxpool.Pool
monitoringPool *pgxpool.Pool
auditLogs *auditlog.AsyncWriter
}
func NewBrandService(pool *pgxpool.Pool, auditLogs *auditlog.AsyncWriter) *BrandService {
return &BrandService{pool: pool, auditLogs: auditLogs}
func NewBrandService(pool, monitoringPool *pgxpool.Pool, auditLogs *auditlog.AsyncWriter) *BrandService {
return &BrandService{pool: pool, monitoringPool: monitoringPool, auditLogs: auditLogs}
}
// --- Brand CRUD ---
@@ -174,6 +175,10 @@ func (s *BrandService) Delete(ctx context.Context, id int64) error {
_, _ = tx.Exec(ctx, `UPDATE brand_questions SET deleted_at = NOW(), updated_at = NOW() WHERE brand_id = $1 AND tenant_id = $2 AND deleted_at IS NULL`, id, actor.TenantID)
_, _ = tx.Exec(ctx, `UPDATE competitors SET deleted_at = NOW(), updated_at = NOW() WHERE brand_id = $1 AND tenant_id = $2 AND deleted_at IS NULL`, id, actor.TenantID)
if err := s.cleanupMonitoringAfterBrandDelete(ctx, actor.TenantID, id); err != nil {
return err
}
afterJSON, _ := json.Marshal(map[string]interface{}{"id": id, "action": "cascade_soft_delete"})
if err := tx.Commit(ctx); err != nil {
return err
@@ -265,13 +270,65 @@ func (s *BrandService) UpdateKeyword(ctx context.Context, brandID, keywordID int
func (s *BrandService) DeleteKeyword(ctx context.Context, brandID, keywordID int64) error {
actor := auth.MustActor(ctx)
tag, err := s.pool.Exec(ctx, `
tx, err := s.pool.Begin(ctx)
if err != nil {
return fmt.Errorf("begin tx: %w", err)
}
defer func() {
_ = tx.Rollback(ctx)
}()
questionRows, err := tx.Query(ctx, `
SELECT id
FROM brand_questions
WHERE brand_id = $1
AND keyword_id = $2
AND tenant_id = $3
AND deleted_at IS NULL
`, brandID, keywordID, actor.TenantID)
if err != nil {
return response.ErrInternal(50010, "query_failed", "failed to load keyword questions")
}
questionIDs := make([]int64, 0)
for questionRows.Next() {
var questionID int64
if scanErr := questionRows.Scan(&questionID); scanErr != nil {
return response.ErrInternal(50010, "scan_failed", scanErr.Error())
}
questionIDs = append(questionIDs, questionID)
}
if err := questionRows.Err(); err != nil {
return response.ErrInternal(50010, "scan_failed", err.Error())
}
questionRows.Close()
tag, err := tx.Exec(ctx, `
UPDATE brand_keywords SET deleted_at = NOW(), updated_at = NOW()
WHERE id = $1 AND brand_id = $2 AND tenant_id = $3 AND deleted_at IS NULL
`, keywordID, brandID, actor.TenantID)
if err != nil || tag.RowsAffected() == 0 {
return response.ErrNotFound(40421, "keyword_not_found", "keyword not found")
}
if _, err := tx.Exec(ctx, `
UPDATE brand_questions
SET deleted_at = NOW(), updated_at = NOW()
WHERE brand_id = $1
AND keyword_id = $2
AND tenant_id = $3
AND deleted_at IS NULL
`, brandID, keywordID, actor.TenantID); err != nil {
return response.ErrInternal(50010, "update_failed", "failed to soft delete questions under keyword")
}
if err := s.cleanupMonitoringAfterKeywordDelete(ctx, actor.TenantID, brandID, keywordID, questionIDs); err != nil {
return err
}
if err := tx.Commit(ctx); err != nil {
return err
}
return nil
}
@@ -378,13 +435,30 @@ func (s *BrandService) UpdateQuestion(ctx context.Context, brandID, questionID i
func (s *BrandService) DeleteQuestion(ctx context.Context, brandID, questionID int64) error {
actor := auth.MustActor(ctx)
tag, err := s.pool.Exec(ctx, `
tx, err := s.pool.Begin(ctx)
if err != nil {
return fmt.Errorf("begin tx: %w", err)
}
defer func() {
_ = tx.Rollback(ctx)
}()
tag, err := tx.Exec(ctx, `
UPDATE brand_questions SET deleted_at = NOW(), updated_at = NOW()
WHERE id = $1 AND brand_id = $2 AND tenant_id = $3 AND deleted_at IS NULL
`, questionID, brandID, actor.TenantID)
if err != nil || tag.RowsAffected() == 0 {
return response.ErrNotFound(40422, "question_not_found", "question not found")
}
if err := s.cleanupMonitoringAfterQuestionDelete(ctx, actor.TenantID, brandID, questionID); err != nil {
return err
}
if err := tx.Commit(ctx); err != nil {
return err
}
return nil
}
@@ -288,6 +288,10 @@ func (s *MediaService) RegisterPluginInstallation(ctx context.Context, req Regis
}
}
if err := ensureMonitoringPrimaryInstallation(ctx, tx, actor.TenantID, installationID); err != nil {
return nil, err
}
if err := tx.Commit(ctx); err != nil {
return nil, response.ErrInternal(50065, "installation_commit_failed", "failed to commit plugin installation")
}
@@ -300,6 +304,21 @@ func (s *MediaService) RegisterPluginInstallation(ctx context.Context, req Regis
}, nil
}
func ensureMonitoringPrimaryInstallation(ctx context.Context, tx pgx.Tx, tenantID, installationID int64) error {
if _, err := tx.Exec(ctx, `
INSERT INTO tenant_monitoring_quotas (
tenant_id, primary_installation_id
)
VALUES ($1, $2)
ON CONFLICT (tenant_id) DO UPDATE
SET primary_installation_id = COALESCE(tenant_monitoring_quotas.primary_installation_id, EXCLUDED.primary_installation_id),
updated_at = NOW()
`, tenantID, installationID); err != nil {
return response.ErrInternal(50068, "monitoring_quota_update_failed", "failed to assign monitoring plugin installation")
}
return nil
}
func (s *MediaService) ListPlatformAccounts(ctx context.Context) ([]PlatformAccountResponse, error) {
actor := auth.MustActor(ctx)
rows, err := s.pool.Query(ctx, `
@@ -0,0 +1,173 @@
package app
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
sharedllm "github.com/geo-platform/tenant-api/internal/shared/llm"
)
const (
monitoringAnswerParseLLMModel = "doubao-seed-2-0-lite-260215"
monitoringAnswerParseLLMTimeout = 30 * time.Second
monitoringAnswerParseLLMMaxOutputTokens = 600
)
var monitoringAnswerParseLLMSchema = []byte(`{
"type": "object",
"additionalProperties": false,
"required": [
"brand_mentioned",
"brand_mention_position",
"first_recommended",
"sentiment_label",
"matched_brand_terms"
],
"properties": {
"brand_mentioned": {
"type": "boolean"
},
"brand_mention_position": {
"type": "string",
"enum": ["top1", "mentioned", "not_mentioned"]
},
"first_recommended": {
"type": "boolean"
},
"sentiment_label": {
"type": "string",
"enum": ["positive", "neutral", "negative", "unknown"]
},
"matched_brand_terms": {
"type": "array",
"items": {
"type": "string"
}
}
}
}`)
type monitoringAnswerLLMParsePayload struct {
BrandMentioned bool `json:"brand_mentioned"`
BrandMentionPosition string `json:"brand_mention_position"`
FirstRecommended bool `json:"first_recommended"`
SentimentLabel string `json:"sentiment_label"`
MatchedBrandTerms []string `json:"matched_brand_terms"`
}
func parseMonitoringAnswerWithLLM(
ctx context.Context,
client sharedllm.Client,
answer string,
brandName string,
) (monitoringAnswerParseSummary, error) {
answer = strings.TrimSpace(answer)
brandName = strings.TrimSpace(brandName)
if answer == "" {
return monitoringAnswerParseSummary{}, fmt.Errorf("answer is empty")
}
if brandName == "" {
return monitoringAnswerParseSummary{}, fmt.Errorf("brand name is empty")
}
result, err := client.Generate(ctx, sharedllm.GenerateRequest{
Model: monitoringAnswerParseLLMModel,
Prompt: buildMonitoringAnswerParsePrompt(answer, brandName),
Timeout: monitoringAnswerParseLLMTimeout,
MaxOutputTokens: monitoringAnswerParseLLMMaxOutputTokens,
ResponseFormat: &sharedllm.ResponseFormat{
Type: sharedllm.ResponseFormatTypeJSONSchema,
Name: "monitoring_answer_parse",
Description: "Parse brand mention signals from an AI answer for brand monitoring.",
SchemaJSON: monitoringAnswerParseLLMSchema,
Strict: true,
},
}, nil)
if err != nil {
return monitoringAnswerParseSummary{}, err
}
return decodeMonitoringAnswerLLMParseResult(result.Content)
}
func buildMonitoringAnswerParsePrompt(answer string, brandName string) string {
var builder strings.Builder
builder.WriteString("你是品牌监测解析助手。\n")
builder.WriteString("请只基于给定回答内容,判断目标品牌在回答中的提及、排序、推荐与情感,不要猜测回答外的信息。\n")
builder.WriteString("\n判断规则:\n")
builder.WriteString("1. brand_mentioned:回答中是否明确提到目标品牌,或可明确判断是在指代该品牌。\n")
builder.WriteString("2. brand_mention_position\n")
builder.WriteString(" - top1:目标品牌被排在第一位,或被明确表述为首选/最推荐。\n")
builder.WriteString(" - mentioned:目标品牌被提到,但不是第一位。\n")
builder.WriteString(" - not_mentioned:未提到目标品牌。\n")
builder.WriteString("3. first_recommended:是否明确把目标品牌作为首选推荐。\n")
builder.WriteString("4. sentiment_label:结合品牌相关表述判断 positive / neutral / negative / unknown。\n")
builder.WriteString("5. matched_brand_terms:把回答里实际出现、并用于指代该品牌的词语原样列出;没有就返回空数组。\n")
builder.WriteString("\n只返回 JSON。\n")
builder.WriteString("\n目标品牌:")
builder.WriteString(brandName)
builder.WriteString("\n回答内容:\n")
builder.WriteString(answer)
return builder.String()
}
func decodeMonitoringAnswerLLMParseResult(raw string) (monitoringAnswerParseSummary, error) {
var lastErr error
for _, candidate := range extractJSONCandidates(raw) {
var payload monitoringAnswerLLMParsePayload
if err := json.Unmarshal([]byte(candidate), &payload); err != nil {
lastErr = err
continue
}
return monitoringAnswerParseSummary{
BrandMentioned: payload.BrandMentioned,
BrandMentionPosition: normalizeMonitoringBrandMentionPosition(payload.BrandMentionPosition, payload.BrandMentioned),
FirstRecommended: payload.FirstRecommended,
SentimentLabel: normalizeMonitoringSentiment(payload.SentimentLabel, payload.BrandMentioned),
MatchedBrandTerms: normalizeMonitoringBrandTermList(payload.MatchedBrandTerms),
}, nil
}
if lastErr == nil {
lastErr = fmt.Errorf("empty content")
}
return monitoringAnswerParseSummary{}, fmt.Errorf("decode monitoring answer parse result: %w", lastErr)
}
func normalizeMonitoringBrandMentionPosition(value string, brandMentioned bool) string {
switch strings.ToLower(strings.TrimSpace(value)) {
case "top1":
if !brandMentioned {
return "not_mentioned"
}
return "top1"
case "mentioned":
if !brandMentioned {
return "not_mentioned"
}
return "mentioned"
case "not_mentioned":
return "not_mentioned"
default:
if brandMentioned {
return "mentioned"
}
return "not_mentioned"
}
}
func normalizeMonitoringSentiment(value string, brandMentioned bool) string {
switch strings.ToLower(strings.TrimSpace(value)) {
case "positive", "neutral", "negative":
return strings.ToLower(strings.TrimSpace(value))
case "unknown":
return "unknown"
default:
if brandMentioned {
return "neutral"
}
return "unknown"
}
}
@@ -0,0 +1,246 @@
package app
import (
"strings"
"unicode"
)
type monitoringStoredParseFields struct {
BrandMentioned *bool
BrandMentionPosition *string
FirstRecommended *bool
SentimentLabel *string
MatchedBrandTerms []string
}
type monitoringAnswerParseSummary struct {
BrandMentioned bool
BrandMentionPosition string
FirstRecommended bool
SentimentLabel string
MatchedBrandTerms []string
}
var monitoringPositiveSignals = []string{
"推荐", "首选", "优先", "更推荐", "值得", "靠谱", "专业", "优质", "高端", "不错", "合适", "满意", "口碑", "领先",
}
var monitoringNegativeSignals = []string{
"不推荐", "慎选", "避坑", "不好", "较差", "很差", "问题", "投诉", "负面", "不靠谱", "一般", "短板", "缺点", "不足",
}
func resolveMonitoringAnswerParseSummary(
answer string,
brandName string,
stored monitoringStoredParseFields,
) monitoringAnswerParseSummary {
return overlayMonitoringStoredParseSummary(
deriveMonitoringAnswerParseSummary(answer, brandName),
stored,
)
}
func overlayMonitoringStoredParseSummary(
base monitoringAnswerParseSummary,
stored monitoringStoredParseFields,
) monitoringAnswerParseSummary {
result := base
if stored.BrandMentioned != nil {
result.BrandMentioned = *stored.BrandMentioned
}
if value := strings.TrimSpace(monitoringStringValue(stored.BrandMentionPosition)); value != "" {
result.BrandMentionPosition = value
}
if stored.FirstRecommended != nil {
result.FirstRecommended = *stored.FirstRecommended
}
if value := strings.TrimSpace(monitoringStringValue(stored.SentimentLabel)); value != "" {
result.SentimentLabel = value
}
if len(stored.MatchedBrandTerms) > 0 {
result.MatchedBrandTerms = normalizeMonitoringBrandTermList(stored.MatchedBrandTerms)
}
if result.BrandMentioned && result.BrandMentionPosition == "" {
result.BrandMentionPosition = "mentioned"
}
if !result.BrandMentioned && result.BrandMentionPosition == "" {
result.BrandMentionPosition = "not_mentioned"
}
if result.BrandMentioned && result.SentimentLabel == "" {
result.SentimentLabel = "neutral"
}
if !result.BrandMentioned && result.SentimentLabel == "" {
result.SentimentLabel = "unknown"
}
return result
}
func deriveMonitoringAnswerParseSummary(answer string, brandName string) monitoringAnswerParseSummary {
result := monitoringAnswerParseSummary{
BrandMentionPosition: "not_mentioned",
SentimentLabel: "unknown",
MatchedBrandTerms: []string{},
}
answer = strings.TrimSpace(answer)
if answer == "" {
return result
}
brandTerms := buildMonitoringBrandTerms(brandName)
if len(brandTerms) == 0 {
return result
}
answerLower := strings.ToLower(answer)
answerNormalized := normalizeMonitoringTextForMatch(answer)
matchedTerms := findMonitoringMatchedBrandTerms(answerLower, answerNormalized, brandTerms)
if len(matchedTerms) == 0 {
return result
}
result.BrandMentioned = true
result.BrandMentionPosition = "mentioned"
result.MatchedBrandTerms = matchedTerms
if isMonitoringTop1Mention(answerLower, matchedTerms) {
result.BrandMentionPosition = "top1"
}
result.FirstRecommended = isMonitoringFirstRecommended(answerLower, matchedTerms, result.BrandMentionPosition)
result.SentimentLabel = deriveMonitoringSentimentLabel(answerLower, matchedTerms, result.FirstRecommended)
return result
}
func buildMonitoringBrandTerms(brandName string) []string {
return normalizeMonitoringBrandTermList([]string{strings.TrimSpace(brandName)})
}
func normalizeMonitoringBrandTermList(values []string) []string {
result := make([]string, 0, len(values))
seen := make(map[string]struct{}, len(values))
for _, value := range values {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
continue
}
if _, ok := seen[trimmed]; ok {
continue
}
seen[trimmed] = struct{}{}
result = append(result, trimmed)
}
return result
}
func normalizeMonitoringTextForMatch(value string) string {
var builder strings.Builder
builder.Grow(len(value))
for _, item := range strings.ToLower(value) {
if unicode.IsLetter(item) || unicode.IsNumber(item) {
builder.WriteRune(item)
}
}
return builder.String()
}
func findMonitoringMatchedBrandTerms(answerLower, answerNormalized string, brandTerms []string) []string {
matched := make([]string, 0, len(brandTerms))
for _, term := range brandTerms {
termLower := strings.ToLower(strings.TrimSpace(term))
if termLower == "" {
continue
}
termNormalized := normalizeMonitoringTextForMatch(termLower)
if strings.Contains(answerLower, termLower) || (termNormalized != "" && strings.Contains(answerNormalized, termNormalized)) {
matched = append(matched, term)
}
}
return normalizeMonitoringBrandTermList(matched)
}
func isMonitoringTop1Mention(answerLower string, matchedTerms []string) bool {
segment := strings.ToLower(firstMonitoringRunes(strings.TrimSpace(answerLower), 96))
if segment == "" {
return false
}
for _, term := range matchedTerms {
termLower := strings.ToLower(strings.TrimSpace(term))
if termLower != "" && strings.Contains(segment, termLower) {
return true
}
}
return false
}
func isMonitoringFirstRecommended(answerLower string, matchedTerms []string, mentionPosition string) bool {
if mentionPosition == "top1" {
return true
}
for _, term := range matchedTerms {
termLower := strings.ToLower(strings.TrimSpace(term))
if termLower == "" {
continue
}
patterns := []string{
"推荐" + termLower,
"首选" + termLower,
"优先" + termLower,
"建议选择" + termLower,
"建议选" + termLower,
"更推荐" + termLower,
termLower + "是首选",
termLower + "更合适",
termLower + "更值得",
termLower + "值得推荐",
}
for _, pattern := range patterns {
if strings.Contains(answerLower, pattern) {
return true
}
}
}
return false
}
func deriveMonitoringSentimentLabel(answerLower string, matchedTerms []string, firstRecommended bool) string {
negativeCount := 0
for _, signal := range monitoringNegativeSignals {
if strings.Contains(answerLower, signal) {
negativeCount++
}
}
if negativeCount > 0 {
return "negative"
}
positiveCount := 0
for _, signal := range monitoringPositiveSignals {
if strings.Contains(answerLower, signal) {
positiveCount++
}
}
if firstRecommended || positiveCount > 0 {
return "positive"
}
if len(matchedTerms) > 0 {
return "neutral"
}
return "unknown"
}
func firstMonitoringRunes(value string, limit int) string {
if limit <= 0 {
return ""
}
runes := []rune(value)
if len(runes) <= limit {
return string(runes)
}
return string(runes[:limit])
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,9 @@
package app
import "github.com/geo-platform/tenant-api/internal/shared/response"
func monitoringInternalError(code int, message, detail string, cause error) *response.AppError {
appErr := response.ErrInternal(code, message, detail)
appErr.Cause = cause
return appErr
}
@@ -0,0 +1,79 @@
package app
import (
"context"
"time"
"github.com/jackc/pgx/v5"
)
const (
monitoringLeaseMaxRetryCount = 1
monitoringLeaseExpiredErrPrefix = "monitoring lease expired before result callback"
)
func expireMonitoringLeasedTasks(ctx context.Context, tx pgx.Tx, tenantID *int64, now time.Time) (int64, error) {
tag, err := tx.Exec(ctx, `
UPDATE monitoring_collect_tasks
SET status = 'expired',
lease_token_hash = NULL,
leased_to_executor = NULL,
leased_at = NULL,
lease_expires_at = NULL,
error_message = COALESCE(NULLIF(error_message, ''), $1),
request_payload_json = (
CASE
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
THEN '{}'::jsonb
ELSE request_payload_json
END
) || jsonb_build_object(
'lease_retry_count',
(
CASE
WHEN COALESCE(request_payload_json ->> 'lease_retry_count', '') ~ '^[0-9]+$'
THEN (request_payload_json ->> 'lease_retry_count')::int
ELSE 0
END
) + 1
),
updated_at = NOW()
WHERE collector_type = $2
AND status = 'leased'
AND lease_expires_at IS NOT NULL
AND lease_expires_at < $3
AND ($4::bigint IS NULL OR tenant_id = $4)
`, monitoringLeaseExpiredErrPrefix, monitoringCollectorType, now.UTC(), nullableInt64(tenantID))
if err != nil {
return 0, monitoringInternalError(50041, "lease_expire_failed", "failed to expire overdue monitoring lease tasks", err)
}
return tag.RowsAffected(), nil
}
func requeueMonitoringExpiredTasks(ctx context.Context, tx pgx.Tx, tenantID int64, businessDate time.Time) (int64, error) {
tag, err := tx.Exec(ctx, `
UPDATE monitoring_collect_tasks
SET status = 'pending',
planned_at = NOW(),
callback_received_at = NULL,
completed_at = NULL,
skip_reason = NULL,
error_message = NULL,
updated_at = NOW()
WHERE tenant_id = $1
AND collector_type = $2
AND business_date = $3::date
AND status = 'expired'
AND (
CASE
WHEN COALESCE(request_payload_json ->> 'lease_retry_count', '') ~ '^[0-9]+$'
THEN (request_payload_json ->> 'lease_retry_count')::int
ELSE 0
END
) <= $4
`, tenantID, monitoringCollectorType, businessDate.Format("2006-01-02"), monitoringLeaseMaxRetryCount)
if err != nil {
return 0, monitoringInternalError(50041, "lease_requeue_failed", "failed to recycle expired monitoring lease tasks", err)
}
return tag.RowsAffected(), nil
}
@@ -0,0 +1,116 @@
package app
import (
"context"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"go.uber.org/zap"
"github.com/geo-platform/tenant-api/internal/shared/response"
)
const (
defaultMonitoringLeaseRecoveryInterval = 30 * time.Minute
defaultMonitoringLeaseRecoveryTimeout = 30 * time.Second
)
type MonitoringLeaseRecoveryWorker struct {
monitoringPool *pgxpool.Pool
logger *zap.Logger
interval time.Duration
timeout time.Duration
}
func NewMonitoringLeaseRecoveryWorker(monitoringPool *pgxpool.Pool, logger *zap.Logger) *MonitoringLeaseRecoveryWorker {
return &MonitoringLeaseRecoveryWorker{
monitoringPool: monitoringPool,
logger: logger,
interval: defaultMonitoringLeaseRecoveryInterval,
timeout: defaultMonitoringLeaseRecoveryTimeout,
}
}
func (w *MonitoringLeaseRecoveryWorker) Start(ctx context.Context) {
go w.run(ctx)
}
func (w *MonitoringLeaseRecoveryWorker) run(ctx context.Context) {
w.runOnce(ctx)
ticker := time.NewTicker(w.interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
w.runOnce(ctx)
}
}
}
func (w *MonitoringLeaseRecoveryWorker) runOnce(parent context.Context) {
if w.monitoringPool == nil {
return
}
ctx, cancel := context.WithTimeout(parent, w.timeout)
defer cancel()
tx, err := w.monitoringPool.Begin(ctx)
if err != nil {
w.logger.Warn("monitoring lease recovery begin failed", zap.Error(err))
return
}
defer tx.Rollback(ctx)
expiredCount, err := expireMonitoringLeasedTasks(ctx, tx, nil, time.Now().UTC())
if err != nil {
w.logger.Warn("monitoring lease recovery failed", monitoringWorkerErrorFields(err)...)
return
}
if err := tx.Commit(ctx); err != nil {
w.logger.Warn("monitoring lease recovery commit failed", zap.Error(err))
return
}
if expiredCount > 0 {
w.logger.Info("monitoring lease recovery completed", zap.Int64("expired_task_count", expiredCount))
}
}
func normalizeMonitoringWorkerError(err error) error {
if err == nil {
return nil
}
if appErr, ok := err.(*response.AppError); ok {
return appErr
}
return err
}
func monitoringWorkerErrorFields(err error) []zap.Field {
normalized := normalizeMonitoringWorkerError(err)
if normalized == nil {
return nil
}
if appErr, ok := normalized.(*response.AppError); ok {
fields := []zap.Field{
zap.String("error_code", appErr.Message),
zap.Int("app_code", appErr.Code),
}
if appErr.Detail != "" {
fields = append(fields, zap.String("detail", appErr.Detail))
}
if appErr.Cause != nil {
fields = append(fields, zap.Error(appErr.Cause))
}
return fields
}
return []zap.Field{zap.Error(normalized)}
}
@@ -0,0 +1,540 @@
package app
import (
"context"
"database/sql"
"encoding/json"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
type monitoringBrandDailyAggregate struct {
PlannedSampleCount int64
ActualSampleCount int64
MentionedCount int64
Top1MentionedCount int64
FirstRecommendedCount int64
PositiveMentionedCount int64
CitedAnswerCount int64
SnapshotUpdatedAt sql.NullTime
LatestTriggerSource sql.NullString
}
type monitoringPlatformDailyAggregate struct {
AIPlatformID string
PlannedSampleCount int64
ActualSampleCount int64
MentionedCount int64
Top1MentionedCount int64
LastSampledAt sql.NullTime
AccessStatus string
}
func (s *MonitoringCallbackService) rebuildBrandDailyProjection(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDate time.Time) error {
platforms, err := loadMonitoringProjectionPlatforms(ctx, s.businessPool, tenantID)
if err != nil {
return err
}
enabledQuestionCount, err := s.loadMonitoringEnabledQuestionCount(ctx, tx, tenantID, brandID, businessDate)
if err != nil {
return err
}
brandAgg, err := s.loadMonitoringBrandDailyAggregate(ctx, tx, tenantID, brandID, businessDate)
if err != nil {
return err
}
platformAggs, err := s.loadMonitoringPlatformDailyAggregates(ctx, tx, tenantID, brandID, businessDate, platforms)
if err != nil {
return err
}
desiredSampleCount := enabledQuestionCount * int64(len(platforms))
coverageRate := divideAsPointer(brandAgg.ActualSampleCount, brandAgg.PlannedSampleCount)
mentionRate := averageMonitoringPlatformMentionRate(platformAggs)
top1MentionRate := divideAsPointer(brandAgg.Top1MentionedCount, brandAgg.ActualSampleCount)
firstRecommendRate := divideAsPointer(brandAgg.FirstRecommendedCount, brandAgg.ActualSampleCount)
positiveMentionRate := divideAsPointer(brandAgg.PositiveMentionedCount, brandAgg.ActualSampleCount)
citationRate := divideAsPointer(brandAgg.CitedAnswerCount, brandAgg.ActualSampleCount)
if _, err := tx.Exec(ctx, `
INSERT INTO monitoring_brand_daily (
tenant_id, brand_id, collector_type, business_date, sample_status,
desired_sample_count, planned_sample_count, actual_sample_count,
coverage_rate, confidence_level, trigger_source, snapshot_updated_at,
mentioned_count, top1_mentioned_count, first_recommended_count, positive_mentioned_count,
cited_answer_count, mention_rate, top1_mention_rate, first_recommend_rate,
positive_mention_rate, citation_rate
)
VALUES (
$1, $2, $3, $4::date, $5,
$6, $7, $8,
$9, $10, $11, $12,
$13, $14, $15, $16,
$17, $18, $19, $20,
$21, $22
)
ON CONFLICT (tenant_id, brand_id, collector_type, business_date)
DO UPDATE SET
sample_status = EXCLUDED.sample_status,
desired_sample_count = EXCLUDED.desired_sample_count,
planned_sample_count = EXCLUDED.planned_sample_count,
actual_sample_count = EXCLUDED.actual_sample_count,
coverage_rate = EXCLUDED.coverage_rate,
confidence_level = EXCLUDED.confidence_level,
trigger_source = EXCLUDED.trigger_source,
snapshot_updated_at = EXCLUDED.snapshot_updated_at,
mentioned_count = EXCLUDED.mentioned_count,
top1_mentioned_count = EXCLUDED.top1_mentioned_count,
first_recommended_count = EXCLUDED.first_recommended_count,
positive_mentioned_count = EXCLUDED.positive_mentioned_count,
cited_answer_count = EXCLUDED.cited_answer_count,
mention_rate = EXCLUDED.mention_rate,
top1_mention_rate = EXCLUDED.top1_mention_rate,
first_recommend_rate = EXCLUDED.first_recommend_rate,
positive_mention_rate = EXCLUDED.positive_mention_rate,
citation_rate = EXCLUDED.citation_rate,
updated_at = NOW()
`,
tenantID,
brandID,
monitoringCollectorType,
monitoringBusinessDateText(businessDate),
deriveMonitoringBrandSampleStatus(brandAgg.ActualSampleCount, brandAgg.PlannedSampleCount, coverageRate),
desiredSampleCount,
brandAgg.PlannedSampleCount,
brandAgg.ActualSampleCount,
nullableFloat64(coverageRate),
deriveMonitoringConfidenceLevel(brandAgg.ActualSampleCount, brandAgg.PlannedSampleCount, coverageRate),
nullableString(stringPointer(brandAgg.LatestTriggerSource)),
nullableTime(brandAgg.SnapshotUpdatedAt),
brandAgg.MentionedCount,
brandAgg.Top1MentionedCount,
brandAgg.FirstRecommendedCount,
brandAgg.PositiveMentionedCount,
brandAgg.CitedAnswerCount,
nullableFloat64(mentionRate),
nullableFloat64(top1MentionRate),
nullableFloat64(firstRecommendRate),
nullableFloat64(positiveMentionRate),
nullableFloat64(citationRate),
); err != nil {
return monitoringInternalError(50041, "brand_daily_projection_failed", "failed to persist monitoring brand daily projection", err)
}
if err := s.replaceMonitoringPlatformDailyProjection(ctx, tx, tenantID, brandID, businessDate, platformAggs); err != nil {
return err
}
return nil
}
func loadMonitoringProjectionPlatforms(ctx context.Context, businessPool *pgxpool.Pool, tenantID int64) ([]monitoringPlatformMetadata, error) {
enabled := []string{"deepseek", "qwen", "doubao"}
var enabledJSON []byte
if err := businessPool.QueryRow(ctx, `
SELECT enabled_platforms
FROM tenant_monitoring_quotas
WHERE tenant_id = $1
`, tenantID).Scan(&enabledJSON); err != nil {
if err == pgx.ErrNoRows {
return resolveMonitoringPlatforms(enabled), nil
}
return nil, monitoringInternalError(50041, "quota_lookup_failed", "failed to load monitoring quota", err)
}
if len(enabledJSON) > 0 {
var configured []string
if jsonErr := json.Unmarshal(enabledJSON, &configured); jsonErr == nil && len(configured) > 0 {
enabled = configured
}
}
return resolveMonitoringPlatforms(enabled), nil
}
func (s *MonitoringCallbackService) loadMonitoringEnabledQuestionCount(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDate time.Time) (int64, error) {
dayEnd := monitoringCanonicalBusinessDay(businessDate).AddDate(0, 0, 1)
var count int64
if err := tx.QueryRow(ctx, `
SELECT COUNT(DISTINCT question_id)
FROM monitoring_question_config_snapshots
WHERE tenant_id = $1
AND brand_id = $2
AND monitor_enabled = TRUE
AND projected_at < $3
AND (superseded_at IS NULL OR superseded_at >= $3)
AND (deleted_at IS NULL OR deleted_at >= $3)
`, tenantID, brandID, dayEnd).Scan(&count); err != nil {
return 0, monitoringInternalError(50041, "question_snapshot_lookup_failed", "failed to load monitoring question snapshots", err)
}
return count, nil
}
func (s *MonitoringCallbackService) loadMonitoringBrandDailyAggregate(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDate time.Time) (monitoringBrandDailyAggregate, error) {
agg := monitoringBrandDailyAggregate{}
dateText := monitoringBusinessDateText(businessDate)
if err := tx.QueryRow(ctx, `
SELECT COUNT(*)
FROM monitoring_collect_tasks
WHERE tenant_id = $1
AND brand_id = $2
AND collector_type = $3
AND business_date = $4::date
AND NOT (status = 'skipped' AND skip_reason = ANY($5::text[]))
`, tenantID, brandID, monitoringCollectorType, dateText, monitoringProjectionExcludedSkipReasons).Scan(&agg.PlannedSampleCount); err != nil {
return agg, monitoringInternalError(50041, "task_projection_lookup_failed", "failed to load monitoring task counts", err)
}
if err := tx.QueryRow(ctx, `
SELECT COUNT(*), MAX(completed_at)
FROM question_monitor_runs
WHERE tenant_id = $1
AND brand_id = $2
AND collector_type = $3
AND business_date = $4::date
AND status = 'succeeded'
`, tenantID, brandID, monitoringCollectorType, dateText).Scan(&agg.ActualSampleCount, &agg.SnapshotUpdatedAt); err != nil {
return agg, monitoringInternalError(50041, "run_projection_lookup_failed", "failed to load monitoring run counts", err)
}
if err := tx.QueryRow(ctx, `
SELECT
COUNT(*) FILTER (WHERE p.brand_mentioned IS TRUE)::bigint,
COUNT(*) FILTER (WHERE p.brand_mention_position = 'top1')::bigint,
COUNT(*) FILTER (WHERE p.first_recommended IS TRUE)::bigint,
COUNT(*) FILTER (WHERE p.sentiment_label = 'positive')::bigint
FROM question_monitor_runs r
LEFT JOIN question_monitor_parse_results p
ON p.tenant_id = r.tenant_id AND p.run_id = r.id
WHERE r.tenant_id = $1
AND r.brand_id = $2
AND r.collector_type = $3
AND r.business_date = $4::date
AND r.status = 'succeeded'
`, tenantID, brandID, monitoringCollectorType, dateText).Scan(
&agg.MentionedCount,
&agg.Top1MentionedCount,
&agg.FirstRecommendedCount,
&agg.PositiveMentionedCount,
); err != nil {
return agg, monitoringInternalError(50041, "parse_projection_lookup_failed", "failed to load monitoring parse aggregates", err)
}
if err := tx.QueryRow(ctx, `
SELECT COUNT(DISTINCT cf.run_id)
FROM monitoring_citation_facts cf
JOIN question_monitor_runs r
ON r.tenant_id = cf.tenant_id AND r.id = cf.run_id
WHERE cf.tenant_id = $1
AND cf.brand_id = $2
AND cf.article_id IS NOT NULL
AND r.collector_type = $3
AND r.business_date = $4::date
AND r.status = 'succeeded'
`, tenantID, brandID, monitoringCollectorType, dateText).Scan(&agg.CitedAnswerCount); err != nil {
return agg, monitoringInternalError(50041, "citation_projection_lookup_failed", "failed to load monitoring citation aggregates", err)
}
if err := tx.QueryRow(ctx, `
SELECT trigger_source
FROM question_monitor_runs
WHERE tenant_id = $1
AND brand_id = $2
AND collector_type = $3
AND business_date = $4::date
AND status = 'succeeded'
ORDER BY completed_at DESC NULLS LAST, id DESC
LIMIT 1
`, tenantID, brandID, monitoringCollectorType, dateText).Scan(&agg.LatestTriggerSource); err != nil && err != pgx.ErrNoRows {
return agg, monitoringInternalError(50041, "trigger_source_lookup_failed", "failed to load monitoring trigger source", err)
}
return agg, nil
}
func (s *MonitoringCallbackService) loadMonitoringPlatformDailyAggregates(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDate time.Time, platforms []monitoringPlatformMetadata) ([]monitoringPlatformDailyAggregate, error) {
dateText := monitoringBusinessDateText(businessDate)
taskCounts := make(map[string]int64, len(platforms))
taskRows, err := tx.Query(ctx, `
SELECT ai_platform_id, COUNT(*)
FROM monitoring_collect_tasks
WHERE tenant_id = $1
AND brand_id = $2
AND collector_type = $3
AND business_date = $4::date
AND NOT (status = 'skipped' AND skip_reason = ANY($5::text[]))
GROUP BY ai_platform_id
`, tenantID, brandID, monitoringCollectorType, dateText, monitoringProjectionExcludedSkipReasons)
if err != nil {
return nil, monitoringInternalError(50041, "platform_task_projection_lookup_failed", "failed to load monitoring platform task counts", err)
}
for taskRows.Next() {
var platformID string
var plannedSampleCount int64
if scanErr := taskRows.Scan(&platformID, &plannedSampleCount); scanErr != nil {
taskRows.Close()
return nil, monitoringInternalError(50041, "platform_task_projection_scan_failed", "failed to parse monitoring platform task counts", scanErr)
}
taskCounts[platformID] = plannedSampleCount
}
if err := taskRows.Err(); err != nil {
taskRows.Close()
return nil, monitoringInternalError(50041, "platform_task_projection_scan_failed", "failed to iterate monitoring platform task counts", err)
}
taskRows.Close()
runAggs := make(map[string]monitoringPlatformDailyAggregate, len(platforms))
runRows, err := tx.Query(ctx, `
SELECT
r.ai_platform_id,
COUNT(*)::bigint AS actual_sample_count,
COUNT(*) FILTER (WHERE p.brand_mentioned IS TRUE)::bigint AS mentioned_count,
COUNT(*) FILTER (WHERE p.brand_mention_position = 'top1')::bigint AS top1_mentioned_count,
MAX(r.completed_at) AS last_sampled_at
FROM question_monitor_runs r
LEFT JOIN question_monitor_parse_results p
ON p.tenant_id = r.tenant_id AND p.run_id = r.id
WHERE r.tenant_id = $1
AND r.brand_id = $2
AND r.collector_type = $3
AND r.business_date = $4::date
AND r.status = 'succeeded'
GROUP BY r.ai_platform_id
`, tenantID, brandID, monitoringCollectorType, dateText)
if err != nil {
return nil, monitoringInternalError(50041, "platform_run_projection_lookup_failed", "failed to load monitoring platform run aggregates", err)
}
for runRows.Next() {
var item monitoringPlatformDailyAggregate
if scanErr := runRows.Scan(
&item.AIPlatformID,
&item.ActualSampleCount,
&item.MentionedCount,
&item.Top1MentionedCount,
&item.LastSampledAt,
); scanErr != nil {
runRows.Close()
return nil, monitoringInternalError(50041, "platform_run_projection_scan_failed", "failed to parse monitoring platform run aggregates", scanErr)
}
runAggs[item.AIPlatformID] = item
}
if err := runRows.Err(); err != nil {
runRows.Close()
return nil, monitoringInternalError(50041, "platform_run_projection_scan_failed", "failed to iterate monitoring platform run aggregates", err)
}
runRows.Close()
accessStates := make(map[string]string, len(platforms))
accessRows, err := tx.Query(ctx, `
SELECT DISTINCT ON (ai_platform_id) ai_platform_id, access_status
FROM monitoring_platform_access_snapshots
WHERE tenant_id = $1
AND business_date = $2::date
ORDER BY ai_platform_id ASC, detected_at DESC, id DESC
`, tenantID, dateText)
if err != nil {
return nil, monitoringInternalError(50041, "platform_access_projection_lookup_failed", "failed to load monitoring platform access states", err)
}
for accessRows.Next() {
var platformID string
var accessStatus string
if scanErr := accessRows.Scan(&platformID, &accessStatus); scanErr != nil {
accessRows.Close()
return nil, monitoringInternalError(50041, "platform_access_projection_scan_failed", "failed to parse monitoring platform access states", scanErr)
}
accessStates[platformID] = accessStatus
}
if err := accessRows.Err(); err != nil {
accessRows.Close()
return nil, monitoringInternalError(50041, "platform_access_projection_scan_failed", "failed to iterate monitoring platform access states", err)
}
accessRows.Close()
items := make([]monitoringPlatformDailyAggregate, 0, len(platforms))
for _, platform := range platforms {
item := runAggs[platform.ID]
item.AIPlatformID = platform.ID
item.PlannedSampleCount = taskCounts[platform.ID]
item.AccessStatus = accessStates[platform.ID]
items = append(items, item)
}
return items, nil
}
func averageMonitoringPlatformMentionRate(items []monitoringPlatformDailyAggregate) *float64 {
if len(items) == 0 {
return nil
}
total := 0.0
count := 0
for _, item := range items {
rate := divideAsPointer(item.MentionedCount, item.ActualSampleCount)
if rate == nil {
continue
}
total += *rate
count++
}
if count == 0 {
return nil
}
average := total / float64(count)
return &average
}
func (s *MonitoringCallbackService) replaceMonitoringPlatformDailyProjection(ctx context.Context, tx pgx.Tx, tenantID, brandID int64, businessDate time.Time, items []monitoringPlatformDailyAggregate) error {
dateText := monitoringBusinessDateText(businessDate)
if _, err := tx.Exec(ctx, `
DELETE FROM monitoring_brand_platform_daily
WHERE tenant_id = $1
AND brand_id = $2
AND collector_type = $3
AND business_date = $4::date
`, tenantID, brandID, monitoringCollectorType, dateText); err != nil {
return monitoringInternalError(50041, "platform_daily_projection_cleanup_failed", "failed to reset monitoring platform daily projection", err)
}
for _, item := range items {
mentionRate := divideAsPointer(item.MentionedCount, item.ActualSampleCount)
top1MentionRate := divideAsPointer(item.Top1MentionedCount, item.ActualSampleCount)
if _, err := tx.Exec(ctx, `
INSERT INTO monitoring_brand_platform_daily (
tenant_id, brand_id, ai_platform_id, collector_type, business_date,
platform_sample_status, planned_sample_count, actual_sample_count,
mention_rate, top1_mention_rate, last_sampled_at
)
VALUES (
$1, $2, $3, $4, $5::date,
$6, $7, $8,
$9, $10, $11
)
`,
tenantID,
brandID,
item.AIPlatformID,
monitoringCollectorType,
dateText,
deriveMonitoringPlatformSampleStatus(item.ActualSampleCount, item.PlannedSampleCount, item.AccessStatus),
item.PlannedSampleCount,
item.ActualSampleCount,
nullableFloat64(mentionRate),
nullableFloat64(top1MentionRate),
nullableTime(item.LastSampledAt),
); err != nil {
return monitoringInternalError(50041, "platform_daily_projection_failed", "failed to persist monitoring platform daily projection", err)
}
}
return nil
}
func (s *MonitoringCallbackService) loadMonitoringAffectedBrandIDs(ctx context.Context, tx pgx.Tx, tenantID int64, businessDate time.Time, aiPlatformID string) ([]int64, error) {
rows, err := tx.Query(ctx, `
SELECT DISTINCT brand_id
FROM monitoring_collect_tasks
WHERE tenant_id = $1
AND ai_platform_id = $2
AND collector_type = $3
AND business_date = $4::date
ORDER BY brand_id ASC
`, tenantID, aiPlatformID, monitoringCollectorType, monitoringBusinessDateText(businessDate))
if err != nil {
return nil, monitoringInternalError(50041, "affected_brand_lookup_failed", "failed to load monitoring brands for projection", err)
}
defer rows.Close()
brandIDs := make([]int64, 0)
for rows.Next() {
var brandID int64
if scanErr := rows.Scan(&brandID); scanErr != nil {
return nil, monitoringInternalError(50041, "affected_brand_scan_failed", "failed to parse monitoring brands for projection", scanErr)
}
brandIDs = append(brandIDs, brandID)
}
if err := rows.Err(); err != nil {
return nil, monitoringInternalError(50041, "affected_brand_scan_failed", "failed to iterate monitoring brands for projection", err)
}
return brandIDs, nil
}
func deriveMonitoringBrandSampleStatus(actualSampleCount, plannedSampleCount int64, coverageRate *float64) string {
if actualSampleCount <= 0 {
return "unsampled"
}
if plannedSampleCount <= 0 {
return "sampled"
}
if coverageRate != nil && *coverageRate >= 0.7 {
return "sampled"
}
return "partial"
}
func deriveMonitoringConfidenceLevel(actualSampleCount, plannedSampleCount int64, coverageRate *float64) string {
if actualSampleCount <= 0 {
return "low"
}
if plannedSampleCount <= 0 {
return "high"
}
if coverageRate == nil {
return "low"
}
switch {
case *coverageRate >= 0.7:
return "high"
case *coverageRate >= 0.4:
return "medium"
default:
return "low"
}
}
func deriveMonitoringPlatformSampleStatus(actualSampleCount, plannedSampleCount int64, accessStatus string) string {
if actualSampleCount > 0 {
return "sampled"
}
switch accessStatus {
case "not_logged_in":
return "not_logged_in"
case "unavailable":
return "unavailable"
}
if plannedSampleCount > 0 {
return "pending"
}
if accessStatus == "accessible" {
return "pending"
}
return "pending"
}
func nullableFloat64(value *float64) interface{} {
if value == nil {
return nil
}
return *value
}
func nullableTime(value sql.NullTime) interface{} {
if !value.Valid {
return nil
}
return value.Time
}
@@ -0,0 +1,194 @@
package app
import (
"context"
"encoding/json"
"fmt"
"sort"
"time"
"github.com/geo-platform/tenant-api/internal/shared/response"
"go.uber.org/zap"
)
const (
monitoringProjectionQueueIOTimeout = 5 * time.Second
monitoringProjectionFallbackTimeout = 2 * time.Minute
monitoringProjectionQueueMessageVer = "v1"
)
type monitoringProjectionRebuildEnvelope struct {
Version string `json:"version"`
TenantID int64 `json:"tenant_id"`
BrandID int64 `json:"brand_id"`
BusinessDate string `json:"business_date"`
Source string `json:"source"`
}
func encodeMonitoringProjectionRebuildEnvelope(tenantID, brandID int64, businessDay time.Time, source string) ([]byte, error) {
businessDay = monitoringCanonicalBusinessDay(businessDay)
return json.Marshal(monitoringProjectionRebuildEnvelope{
Version: monitoringProjectionQueueMessageVer,
TenantID: tenantID,
BrandID: brandID,
BusinessDate: businessDay.Format("2006-01-02"),
Source: source,
})
}
func decodeMonitoringProjectionRebuildEnvelope(payload []byte) (*monitoringProjectionRebuildEnvelope, error) {
if len(payload) == 0 {
return nil, fmt.Errorf("monitoring projection rebuild payload is empty")
}
var envelope monitoringProjectionRebuildEnvelope
if err := json.Unmarshal(payload, &envelope); err != nil {
return nil, err
}
if envelope.TenantID <= 0 {
return nil, fmt.Errorf("monitoring projection rebuild payload missing tenant_id")
}
if envelope.BrandID <= 0 {
return nil, fmt.Errorf("monitoring projection rebuild payload missing brand_id")
}
if _, err := parseMonitoringBusinessDateInLocation(envelope.BusinessDate, monitoringBusinessLocation()); err != nil {
return nil, fmt.Errorf("monitoring projection rebuild payload has invalid business_date")
}
return &envelope, nil
}
func (s *MonitoringCallbackService) publishProjectionRebuildEnvelope(ctx context.Context, payload []byte) error {
if s == nil || s.rabbitMQ == nil {
return response.ErrServiceUnavailable(50341, "monitoring_projection_queue_unavailable", "rabbitmq client is not configured")
}
if err := s.rabbitMQ.PublishMonitorProjectionRebuild(ctx, payload); err != nil {
return response.ErrServiceUnavailable(50341, "monitoring_projection_queue_unavailable", err.Error())
}
return nil
}
func (s *MonitoringCallbackService) dispatchMonitoringProjectionRebuilds(tenantID int64, businessDay time.Time, brandIDs map[int64]struct{}, source string) {
if s == nil || len(brandIDs) == 0 {
return
}
sortedBrandIDs := sortedMonitoringProjectionBrandIDs(brandIDs)
failedBrandIDs := make([]int64, 0)
for _, brandID := range sortedBrandIDs {
payload, err := encodeMonitoringProjectionRebuildEnvelope(tenantID, brandID, businessDay, source)
if err != nil {
failedBrandIDs = append(failedBrandIDs, brandID)
if s.logger != nil {
s.logger.Warn("monitoring projection rebuild payload encode failed", zap.Int64("tenant_id", tenantID), zap.Int64("brand_id", brandID), zap.Error(err))
}
continue
}
ctx, cancel := context.WithTimeout(context.Background(), monitoringProjectionQueueIOTimeout)
err = s.publishProjectionRebuildEnvelope(ctx, payload)
cancel()
if err != nil {
failedBrandIDs = append(failedBrandIDs, brandID)
if s.logger != nil {
s.logger.Warn("monitoring projection rebuild enqueue failed", zap.Int64("tenant_id", tenantID), zap.Int64("brand_id", brandID), zap.Error(err))
}
}
}
if len(failedBrandIDs) == 0 {
return
}
ctx, cancel := context.WithTimeout(context.Background(), monitoringProjectionFallbackTimeout)
defer cancel()
if err := s.rebuildBrandDailyProjectionBatch(ctx, tenantID, businessDay, failedBrandIDs); err != nil {
if s.logger != nil {
s.logger.Warn("monitoring projection rebuild fallback failed",
zap.Int64("tenant_id", tenantID),
zap.Int("brand_count", len(failedBrandIDs)),
zap.Error(err),
)
}
return
}
if s.logger != nil {
s.logger.Info("monitoring projection rebuild fallback completed",
zap.Int64("tenant_id", tenantID),
zap.Int("brand_count", len(failedBrandIDs)),
)
}
}
func (s *MonitoringCallbackService) ProcessProjectionRebuildEnvelope(ctx context.Context, envelope monitoringProjectionRebuildEnvelope) error {
businessDay, err := parseMonitoringBusinessDateInLocation(envelope.BusinessDate, monitoringBusinessLocation())
if err != nil {
return response.ErrBadRequest(40041, "invalid_business_date", "projection rebuild business_date is invalid")
}
return s.rebuildBrandDailyProjectionBatch(ctx, envelope.TenantID, businessDay, []int64{envelope.BrandID})
}
func (s *MonitoringCallbackService) rebuildBrandDailyProjectionBatch(ctx context.Context, tenantID int64, businessDay time.Time, brandIDs []int64) error {
if s == nil || s.monitoringPool == nil || len(brandIDs) == 0 {
return nil
}
businessDay = monitoringCanonicalBusinessDay(businessDay)
tx, err := s.monitoringPool.Begin(ctx)
if err != nil {
return response.ErrInternal(50041, "begin_failed", "failed to start monitoring projection rebuild")
}
defer tx.Rollback(ctx)
for _, brandID := range uniqueSortedMonitoringProjectionBrandIDs(brandIDs) {
if err := s.rebuildBrandDailyProjection(ctx, tx, tenantID, brandID, businessDay); err != nil {
return err
}
}
if err := tx.Commit(ctx); err != nil {
return response.ErrInternal(50041, "commit_failed", "failed to commit monitoring projection rebuild")
}
return nil
}
func sortedMonitoringProjectionBrandIDs(items map[int64]struct{}) []int64 {
result := make([]int64, 0, len(items))
for brandID := range items {
result = append(result, brandID)
}
sort.Slice(result, func(i, j int) bool {
return result[i] < result[j]
})
return result
}
func uniqueSortedMonitoringProjectionBrandIDs(items []int64) []int64 {
if len(items) == 0 {
return nil
}
seen := make(map[int64]struct{}, len(items))
result := make([]int64, 0, len(items))
for _, brandID := range items {
if brandID <= 0 {
continue
}
if _, ok := seen[brandID]; ok {
continue
}
seen[brandID] = struct{}{}
result = append(result, brandID)
}
sort.Slice(result, func(i, j int) bool {
return result[i] < result[j]
})
return result
}
@@ -0,0 +1,133 @@
package app
import (
"context"
"errors"
"fmt"
"os"
"time"
amqp "github.com/rabbitmq/amqp091-go"
"go.uber.org/zap"
"github.com/geo-platform/tenant-api/internal/shared/messaging/rabbitmq"
)
const (
defaultMonitoringProjectionRebuildRetryInterval = 5 * time.Second
defaultMonitoringProjectionRebuildProcessTimeout = 2 * time.Minute
)
var errMonitoringProjectionRebuildConsumerClosed = errors.New("monitoring projection rebuild consumer channel closed")
type MonitoringProjectionRebuildWorker struct {
rabbitMQ *rabbitmq.Client
service *MonitoringCallbackService
logger *zap.Logger
retryInterval time.Duration
processTimeout time.Duration
consumerName string
}
func NewMonitoringProjectionRebuildWorker(service *MonitoringCallbackService, rabbitMQClient *rabbitmq.Client, logger *zap.Logger) *MonitoringProjectionRebuildWorker {
return &MonitoringProjectionRebuildWorker{
rabbitMQ: rabbitMQClient,
service: service,
logger: logger,
retryInterval: defaultMonitoringProjectionRebuildRetryInterval,
processTimeout: defaultMonitoringProjectionRebuildProcessTimeout,
consumerName: fmt.Sprintf("tenant-api-monitor-projection-%d", os.Getpid()),
}
}
func (w *MonitoringProjectionRebuildWorker) Start(ctx context.Context) {
go w.run(ctx)
}
func (w *MonitoringProjectionRebuildWorker) run(ctx context.Context) {
if w.rabbitMQ == nil || w.service == nil {
return
}
for {
if err := w.consumeOnce(ctx); err != nil && ctx.Err() == nil {
if w.logger != nil {
if errors.Is(err, errMonitoringProjectionRebuildConsumerClosed) {
w.logger.Info("monitoring projection rebuild consumer channel closed, retrying")
} else {
w.logger.Warn("monitoring projection rebuild worker stopped unexpectedly", zap.Error(err))
}
}
}
select {
case <-ctx.Done():
return
case <-time.After(w.retryInterval):
}
}
}
func (w *MonitoringProjectionRebuildWorker) consumeOnce(ctx context.Context) error {
deliveries, ch, err := w.rabbitMQ.ConsumeMonitorProjectionRebuild(w.consumerName)
if err != nil {
return err
}
defer ch.Close()
for {
select {
case <-ctx.Done():
return nil
case delivery, ok := <-deliveries:
if !ok {
return errMonitoringProjectionRebuildConsumerClosed
}
w.handleDelivery(ctx, delivery)
}
}
}
func (w *MonitoringProjectionRebuildWorker) handleDelivery(parent context.Context, delivery amqp.Delivery) {
envelope, err := decodeMonitoringProjectionRebuildEnvelope(delivery.Body)
if err != nil {
w.rejectDelivery(delivery, false, err, 0, 0)
return
}
ctx, cancel := context.WithTimeout(parent, w.processTimeout)
defer cancel()
if err := w.service.ProcessProjectionRebuildEnvelope(ctx, *envelope); err != nil {
requeue := shouldRequeueMonitoringResultError(err, delivery.Redelivered)
w.rejectDelivery(delivery, requeue, err, envelope.TenantID, envelope.BrandID)
return
}
if err := delivery.Ack(false); err != nil && w.logger != nil {
w.logger.Warn("monitoring projection rebuild ack failed",
zap.Error(err),
zap.Int64("tenant_id", envelope.TenantID),
zap.Int64("brand_id", envelope.BrandID),
)
}
}
func (w *MonitoringProjectionRebuildWorker) rejectDelivery(delivery amqp.Delivery, requeue bool, err error, tenantID, brandID int64) {
if nackErr := delivery.Nack(false, requeue); nackErr != nil && w.logger != nil {
w.logger.Warn("monitoring projection rebuild nack failed",
zap.Error(nackErr),
zap.Int64("tenant_id", tenantID),
zap.Int64("brand_id", brandID),
)
}
if w.logger != nil {
w.logger.Warn("monitoring projection rebuild processing failed",
zap.Error(err),
zap.Bool("requeue", requeue),
zap.Bool("redelivered", delivery.Redelivered),
zap.Int64("tenant_id", tenantID),
zap.Int64("brand_id", brandID),
)
}
}
@@ -0,0 +1,113 @@
package app
import (
"context"
"time"
"github.com/jackc/pgx/v5"
)
const (
defaultMonitoringReceivedAlertThreshold = 15 * time.Minute
defaultMonitoringReceivedInspectLimit = 100
)
type monitoringStaleReceivedTask struct {
ID int64
TenantID int64
BrandID int64
QuestionID int64
AIPlatformID string
CollectorType string
BusinessDate time.Time
CallbackReceivedAt time.Time
}
func markMonitoringStaleReceivedTasks(ctx context.Context, tx pgx.Tx, now time.Time, threshold time.Duration, limit int) ([]monitoringStaleReceivedTask, error) {
if limit <= 0 {
limit = defaultMonitoringReceivedInspectLimit
}
rows, err := tx.Query(ctx, `
WITH candidates AS (
SELECT
id,
tenant_id,
brand_id,
question_id,
ai_platform_id,
collector_type,
business_date,
callback_received_at
FROM monitoring_collect_tasks
WHERE collector_type = $1
AND status = 'received'
AND callback_received_at IS NOT NULL
AND callback_received_at < $2
AND (
request_payload_json IS NULL
OR jsonb_typeof(request_payload_json) <> 'object'
OR COALESCE(NULLIF(request_payload_json ->> 'received_alerted_at', ''), '') = ''
)
ORDER BY callback_received_at ASC, id ASC
FOR UPDATE SKIP LOCKED
LIMIT $3
)
UPDATE monitoring_collect_tasks t
SET request_payload_json = (
CASE
WHEN t.request_payload_json IS NULL OR jsonb_typeof(t.request_payload_json) <> 'object'
THEN '{}'::jsonb
ELSE t.request_payload_json
END
) || jsonb_build_object(
'received_alerted_at', $4::text,
'received_alert_count', (
CASE
WHEN COALESCE(t.request_payload_json ->> 'received_alert_count', '') ~ '^[0-9]+$'
THEN (t.request_payload_json ->> 'received_alert_count')::int
ELSE 0
END
) + 1
),
updated_at = NOW()
FROM candidates c
WHERE t.id = c.id
RETURNING
c.id,
c.tenant_id,
c.brand_id,
c.question_id,
c.ai_platform_id,
c.collector_type,
c.business_date,
c.callback_received_at
`, monitoringCollectorType, now.Add(-threshold), limit, now.UTC().Format(time.RFC3339))
if err != nil {
return nil, monitoringInternalError(50041, "received_watchdog_query_failed", "failed to inspect stale received monitoring tasks", err)
}
defer rows.Close()
result := make([]monitoringStaleReceivedTask, 0)
for rows.Next() {
var item monitoringStaleReceivedTask
if scanErr := rows.Scan(
&item.ID,
&item.TenantID,
&item.BrandID,
&item.QuestionID,
&item.AIPlatformID,
&item.CollectorType,
&item.BusinessDate,
&item.CallbackReceivedAt,
); scanErr != nil {
return nil, monitoringInternalError(50041, "received_watchdog_scan_failed", "failed to parse stale received monitoring tasks", scanErr)
}
result = append(result, item)
}
if err := rows.Err(); err != nil {
return nil, monitoringInternalError(50041, "received_watchdog_scan_failed", "failed to iterate stale received monitoring tasks", err)
}
return result, nil
}
@@ -0,0 +1,105 @@
package app
import (
"context"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"go.uber.org/zap"
)
const (
defaultMonitoringReceivedInspectionInterval = 5 * time.Minute
defaultMonitoringReceivedInspectionTimeout = 30 * time.Second
)
type MonitoringReceivedInspectionWorker struct {
monitoringPool *pgxpool.Pool
logger *zap.Logger
interval time.Duration
timeout time.Duration
threshold time.Duration
limit int
}
func NewMonitoringReceivedInspectionWorker(monitoringPool *pgxpool.Pool, logger *zap.Logger) *MonitoringReceivedInspectionWorker {
return &MonitoringReceivedInspectionWorker{
monitoringPool: monitoringPool,
logger: logger,
interval: defaultMonitoringReceivedInspectionInterval,
timeout: defaultMonitoringReceivedInspectionTimeout,
threshold: defaultMonitoringReceivedAlertThreshold,
limit: defaultMonitoringReceivedInspectLimit,
}
}
func (w *MonitoringReceivedInspectionWorker) Start(ctx context.Context) {
go w.run(ctx)
}
func (w *MonitoringReceivedInspectionWorker) run(ctx context.Context) {
w.runOnce(ctx)
ticker := time.NewTicker(w.interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
w.runOnce(ctx)
}
}
}
func (w *MonitoringReceivedInspectionWorker) runOnce(parent context.Context) {
if w.monitoringPool == nil {
return
}
ctx, cancel := context.WithTimeout(parent, w.timeout)
defer cancel()
tx, err := w.monitoringPool.Begin(ctx)
if err != nil {
w.logger.Warn("monitoring received watchdog begin failed", zap.Error(err))
return
}
defer tx.Rollback(ctx)
now := time.Now().UTC()
items, err := markMonitoringStaleReceivedTasks(ctx, tx, now, w.threshold, w.limit)
if err != nil {
w.logger.Warn("monitoring received watchdog failed", monitoringWorkerErrorFields(err)...)
return
}
if err := tx.Commit(ctx); err != nil {
w.logger.Warn("monitoring received watchdog commit failed", zap.Error(err))
return
}
if len(items) == 0 {
return
}
w.logger.Warn("monitoring received watchdog found overdue tasks",
zap.Int("task_count", len(items)),
zap.Duration("threshold", w.threshold),
)
for _, item := range items {
w.logger.Warn("monitoring task stuck in received state",
zap.Int64("task_id", item.ID),
zap.Int64("tenant_id", item.TenantID),
zap.Int64("brand_id", item.BrandID),
zap.Int64("question_id", item.QuestionID),
zap.String("collector_type", item.CollectorType),
zap.String("ai_platform_id", item.AIPlatformID),
zap.String("business_date", item.BusinessDate.Format("2006-01-02")),
zap.Time("callback_received_at", item.CallbackReceivedAt.UTC()),
zap.Duration("received_lag", now.Sub(item.CallbackReceivedAt.UTC())),
)
}
}
@@ -0,0 +1,134 @@
package app
import (
"context"
"errors"
"fmt"
"os"
"time"
amqp "github.com/rabbitmq/amqp091-go"
"go.uber.org/zap"
"github.com/geo-platform/tenant-api/internal/shared/messaging/rabbitmq"
"github.com/geo-platform/tenant-api/internal/shared/response"
)
const (
defaultMonitoringResultWorkerRetryInterval = 5 * time.Second
defaultMonitoringResultWorkerProcessTimeout = 2 * time.Minute
)
var errMonitoringResultConsumerClosed = errors.New("monitoring result consumer channel closed")
type MonitoringResultIngestWorker struct {
rabbitMQ *rabbitmq.Client
service *MonitoringCallbackService
logger *zap.Logger
retryInterval time.Duration
processTimeout time.Duration
consumerName string
}
func NewMonitoringResultIngestWorker(service *MonitoringCallbackService, rabbitMQClient *rabbitmq.Client, logger *zap.Logger) *MonitoringResultIngestWorker {
return &MonitoringResultIngestWorker{
rabbitMQ: rabbitMQClient,
service: service,
logger: logger,
retryInterval: defaultMonitoringResultWorkerRetryInterval,
processTimeout: defaultMonitoringResultWorkerProcessTimeout,
consumerName: fmt.Sprintf("tenant-api-monitor-result-%d", os.Getpid()),
}
}
func (w *MonitoringResultIngestWorker) Start(ctx context.Context) {
go w.run(ctx)
}
func (w *MonitoringResultIngestWorker) run(ctx context.Context) {
if w.rabbitMQ == nil || w.service == nil {
return
}
for {
if err := w.consumeOnce(ctx); err != nil && ctx.Err() == nil {
if w.logger != nil {
if errors.Is(err, errMonitoringResultConsumerClosed) {
w.logger.Info("monitoring result consumer channel closed, retrying")
} else {
w.logger.Warn("monitoring result worker stopped unexpectedly", zap.Error(err))
}
}
}
select {
case <-ctx.Done():
return
case <-time.After(w.retryInterval):
}
}
}
func (w *MonitoringResultIngestWorker) consumeOnce(ctx context.Context) error {
deliveries, ch, err := w.rabbitMQ.ConsumeMonitorResultIngest(w.consumerName)
if err != nil {
return err
}
defer ch.Close()
for {
select {
case <-ctx.Done():
return nil
case delivery, ok := <-deliveries:
if !ok {
return errMonitoringResultConsumerClosed
}
w.handleDelivery(ctx, delivery)
}
}
}
func (w *MonitoringResultIngestWorker) handleDelivery(parent context.Context, delivery amqp.Delivery) {
envelope, err := decodeMonitoringTaskResultEnvelope(delivery.Body)
if err != nil {
w.rejectDelivery(delivery, false, err, 0)
return
}
ctx, cancel := context.WithTimeout(parent, w.processTimeout)
defer cancel()
if _, err := w.service.ProcessQueuedTaskResult(ctx, *envelope); err != nil {
requeue := shouldRequeueMonitoringResultError(err, delivery.Redelivered)
w.rejectDelivery(delivery, requeue, err, envelope.TaskID)
return
}
if err := delivery.Ack(false); err != nil && w.logger != nil {
w.logger.Warn("monitoring result ack failed", zap.Error(err), zap.Int64("task_id", envelope.TaskID))
}
}
func (w *MonitoringResultIngestWorker) rejectDelivery(delivery amqp.Delivery, requeue bool, err error, taskID int64) {
if nackErr := delivery.Nack(false, requeue); nackErr != nil && w.logger != nil {
w.logger.Warn("monitoring result nack failed", zap.Error(nackErr), zap.Int64("task_id", taskID))
}
if w.logger != nil {
w.logger.Warn("monitoring result processing failed",
zap.Error(err),
zap.Bool("requeue", requeue),
zap.Bool("redelivered", delivery.Redelivered),
zap.Int64("task_id", taskID),
)
}
}
func shouldRequeueMonitoringResultError(err error, redelivered bool) bool {
if redelivered {
return false
}
appErr := response.Normalize(err)
return appErr.HTTPStatus >= 500
}
@@ -0,0 +1,93 @@
package app
import (
"encoding/json"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/geo-platform/tenant-api/internal/shared/response"
)
func TestShouldRequeueMonitoringResultError(t *testing.T) {
t.Run("requeue internal error on first delivery", func(t *testing.T) {
requeue := shouldRequeueMonitoringResultError(
response.ErrInternal(50041, "ingest_failed", "db unavailable"),
false,
)
assert.True(t, requeue)
})
t.Run("do not requeue bad request", func(t *testing.T) {
requeue := shouldRequeueMonitoringResultError(
response.ErrBadRequest(40041, "invalid_status", "status must be succeeded or failed"),
false,
)
assert.False(t, requeue)
})
t.Run("do not requeue redelivered internal error", func(t *testing.T) {
requeue := shouldRequeueMonitoringResultError(
response.ErrInternal(50041, "ingest_failed", "db unavailable"),
true,
)
assert.False(t, requeue)
})
}
func TestNewMonitoringTaskResultEnvelope(t *testing.T) {
receivedAt := time.Date(2026, 4, 10, 12, 0, 0, 0, time.UTC)
req := MonitoringTaskResultRequest{
LeaseToken: "lease-token",
Status: "succeeded",
}
task := &monitoringCollectTask{
ID: 12,
TenantID: 34,
}
envelope := newMonitoringTaskResultEnvelope(task, 56, receivedAt, req)
assert.Equal(t, "v1", envelope.Version)
assert.Equal(t, int64(12), envelope.TaskID)
assert.Equal(t, int64(34), envelope.TenantID)
assert.Equal(t, int64(56), envelope.InstallationID)
assert.Equal(t, receivedAt, envelope.ReceivedAt)
assert.Equal(t, "pending", envelope.QueueStatus)
assert.Nil(t, envelope.QueueEnqueuedAt)
assert.Nil(t, envelope.LastQueuePublishError)
assert.Nil(t, envelope.ReceivedAlertedAt)
assert.Zero(t, envelope.ReceivedAlertCount)
assert.Equal(t, req, envelope.CallbackResult)
}
func TestDecodeMonitoringTaskResultEnvelope(t *testing.T) {
t.Run("decode valid payload", func(t *testing.T) {
raw, err := json.Marshal(monitoringTaskResultEnvelope{
Version: "v1",
TaskID: 12,
TenantID: 34,
InstallationID: 56,
ReceivedAt: time.Date(2026, 4, 10, 12, 0, 0, 0, time.UTC),
QueueStatus: "queued",
CallbackResult: MonitoringTaskResultRequest{LeaseToken: "lease-token", Status: "succeeded"},
})
require.NoError(t, err)
envelope, err := decodeMonitoringTaskResultEnvelope(raw)
require.NoError(t, err)
assert.Equal(t, int64(12), envelope.TaskID)
assert.Equal(t, "queued", envelope.QueueStatus)
})
t.Run("reject payload without task id", func(t *testing.T) {
raw, err := json.Marshal(map[string]interface{}{"version": "v1"})
require.NoError(t, err)
_, err = decodeMonitoringTaskResultEnvelope(raw)
require.Error(t, err)
assert.Contains(t, err.Error(), "task_id")
})
}
@@ -0,0 +1,204 @@
package app
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"go.uber.org/zap"
)
const (
defaultMonitoringResultRecoveryInterval = 1 * time.Minute
defaultMonitoringResultRecoveryTimeout = 30 * time.Second
defaultMonitoringResultRecoveryLimit = 100
)
type MonitoringResultRecoveryWorker struct {
monitoringPool *pgxpool.Pool
service *MonitoringCallbackService
logger *zap.Logger
interval time.Duration
timeout time.Duration
limit int
}
type monitoringRecoverableTaskResult struct {
ID int64
CallbackReceivedAt time.Time
RequestPayloadJSON []byte
}
func NewMonitoringResultRecoveryWorker(monitoringPool *pgxpool.Pool, service *MonitoringCallbackService, logger *zap.Logger) *MonitoringResultRecoveryWorker {
return &MonitoringResultRecoveryWorker{
monitoringPool: monitoringPool,
service: service,
logger: logger,
interval: defaultMonitoringResultRecoveryInterval,
timeout: defaultMonitoringResultRecoveryTimeout,
limit: defaultMonitoringResultRecoveryLimit,
}
}
func (w *MonitoringResultRecoveryWorker) Start(ctx context.Context) {
go w.run(ctx)
}
func (w *MonitoringResultRecoveryWorker) run(ctx context.Context) {
w.runOnce(ctx)
ticker := time.NewTicker(w.interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
w.runOnce(ctx)
}
}
}
func (w *MonitoringResultRecoveryWorker) runOnce(parent context.Context) {
if w.monitoringPool == nil || w.service == nil {
return
}
ctx, cancel := context.WithTimeout(parent, w.timeout)
defer cancel()
tx, err := w.monitoringPool.Begin(ctx)
if err != nil {
w.logger.Warn("monitoring result recovery begin failed", zap.Error(err))
return
}
defer tx.Rollback(ctx)
items, err := loadMonitoringRecoverableTaskResults(ctx, tx, w.limit)
if err != nil {
w.logger.Warn("monitoring result recovery failed", monitoringWorkerErrorFields(err)...)
return
}
if err := tx.Commit(ctx); err != nil {
w.logger.Warn("monitoring result recovery commit failed", zap.Error(err))
return
}
if len(items) == 0 {
return
}
republishedCount := 0
for _, item := range items {
if w.tryRepublish(parent, item) {
republishedCount++
}
}
if republishedCount > 0 && w.logger != nil {
w.logger.Info("monitoring result recovery republished tasks", zap.Int("task_count", republishedCount))
}
}
func (w *MonitoringResultRecoveryWorker) tryRepublish(parent context.Context, item monitoringRecoverableTaskResult) bool {
if _, err := decodeMonitoringTaskResultEnvelope(item.RequestPayloadJSON); err != nil {
w.service.tryPatchTaskResultQueueMeta(item.ID, item.CallbackReceivedAt, monitoringTaskResultQueueMetaPatch{
QueueStatus: "invalid_payload",
QueueClaimedAt: nil,
QueueEnqueuedAt: nil,
LastQueuePublishError: optionalString(err.Error()),
})
if w.logger != nil {
w.logger.Warn("monitoring result recovery skipped invalid payload", zap.Int64("task_id", item.ID), zap.Error(err))
}
return false
}
ctx, cancel := context.WithTimeout(parent, monitoringTaskResultQueueIOTimeout)
defer cancel()
if err := w.service.publishTaskResultEnvelope(ctx, item.RequestPayloadJSON); err != nil {
w.service.tryPatchTaskResultQueueMeta(item.ID, item.CallbackReceivedAt, monitoringTaskResultQueueMetaPatch{
QueueStatus: "publish_failed",
QueueClaimedAt: nil,
QueueEnqueuedAt: nil,
LastQueuePublishError: optionalString(err.Error()),
})
if w.logger != nil {
w.logger.Warn("monitoring result recovery republish failed", zap.Int64("task_id", item.ID), zap.Error(err))
}
return false
}
enqueuedAt := time.Now().UTC().Round(0)
w.service.tryPatchTaskResultQueueMeta(item.ID, item.CallbackReceivedAt, monitoringTaskResultQueueMetaPatch{
QueueStatus: "queued",
QueueClaimedAt: nil,
QueueEnqueuedAt: &enqueuedAt,
LastQueuePublishError: nil,
})
return true
}
func loadMonitoringRecoverableTaskResults(ctx context.Context, tx pgx.Tx, limit int) ([]monitoringRecoverableTaskResult, error) {
if limit <= 0 {
limit = defaultMonitoringResultRecoveryLimit
}
rows, err := tx.Query(ctx, `
SELECT
id,
callback_received_at,
request_payload_json
FROM monitoring_collect_tasks
WHERE collector_type = $1
AND status = 'received'
AND callback_received_at IS NOT NULL
AND request_payload_json IS NOT NULL
AND jsonb_typeof(request_payload_json) = 'object'
AND COALESCE(NULLIF(request_payload_json ->> 'queue_status', ''), 'pending') IN ('pending', 'publish_failed')
ORDER BY callback_received_at ASC, id ASC
FOR UPDATE SKIP LOCKED
LIMIT $2
`, monitoringCollectorType, limit)
if err != nil {
return nil, monitoringInternalError(50041, "result_recovery_query_failed", "failed to load recoverable monitoring results", err)
}
defer rows.Close()
result := make([]monitoringRecoverableTaskResult, 0, limit)
for rows.Next() {
var item monitoringRecoverableTaskResult
if scanErr := rows.Scan(&item.ID, &item.CallbackReceivedAt, &item.RequestPayloadJSON); scanErr != nil {
return nil, monitoringInternalError(50041, "result_recovery_scan_failed", "failed to parse recoverable monitoring results", scanErr)
}
result = append(result, item)
}
if err := rows.Err(); err != nil {
return nil, monitoringInternalError(50041, "result_recovery_scan_failed", "failed to iterate recoverable monitoring results", err)
}
return result, nil
}
func decodeMonitoringTaskResultEnvelope(payload []byte) (*monitoringTaskResultEnvelope, error) {
if len(payload) == 0 {
return nil, fmt.Errorf("monitoring task result payload is empty")
}
var envelope monitoringTaskResultEnvelope
if err := json.Unmarshal(payload, &envelope); err != nil {
return nil, err
}
if envelope.TaskID <= 0 {
return nil, fmt.Errorf("monitoring task result payload missing task_id")
}
return &envelope, nil
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,67 @@
package app
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/geo-platform/tenant-api/internal/shared/response"
)
func TestResolveDashboardDateWindowAtUsesBusinessTimezone(t *testing.T) {
loc := time.FixedZone("CST", 8*60*60)
now := time.Date(2026, 4, 12, 0, 5, 0, 0, loc)
startDate, endDate, err := resolveDashboardDateWindowAt(7, "2026-04-12", now, loc)
require.NoError(t, err)
assert.Equal(t, time.Date(2026, 4, 6, 0, 0, 0, 0, loc), startDate)
assert.Equal(t, time.Date(2026, 4, 12, 0, 0, 0, 0, loc), endDate)
}
func TestResolveDashboardDateWindowAtRejectsFutureBusinessDay(t *testing.T) {
loc := time.FixedZone("CST", 8*60*60)
now := time.Date(2026, 4, 12, 0, 5, 0, 0, loc)
_, _, err := resolveDashboardDateWindowAt(7, "2026-04-13", now, loc)
require.Error(t, err)
appErr := response.Normalize(err)
assert.Equal(t, 40031, appErr.Code)
assert.Equal(t, "invalid_business_date", appErr.Message)
assert.Equal(t, "business_date must not be later than today", appErr.Detail)
}
func TestParseDetailDateRangeAtDefaultsToCurrentBusinessDay(t *testing.T) {
loc := time.FixedZone("CST", 8*60*60)
now := time.Date(2026, 4, 12, 0, 5, 0, 0, loc)
fromDate, toDate, err := parseDetailDateRangeAt("", "", now, loc)
require.NoError(t, err)
expected := time.Date(2026, 4, 12, 0, 0, 0, 0, loc)
assert.Equal(t, expected, fromDate)
assert.Equal(t, expected, toDate)
}
func TestMonitoringBusinessDayAndDateAtUsesShanghaiTimezone(t *testing.T) {
now := time.Date(2026, 4, 11, 17, 19, 0, 0, time.UTC)
businessDay, businessDate := monitoringBusinessDayAndDateAt(now)
assert.Equal(t, time.Date(2026, 4, 12, 0, 0, 0, 0, monitoringBusinessLocation()), businessDay)
assert.Equal(t, "2026-04-12", businessDate)
}
func TestEncodeMonitoringProjectionRebuildEnvelopePreservesBusinessDate(t *testing.T) {
businessDay := time.Date(2026, 4, 12, 0, 0, 0, 0, monitoringBusinessLocation())
payload, err := encodeMonitoringProjectionRebuildEnvelope(1, 2, businessDay, "test")
require.NoError(t, err)
envelope, err := decodeMonitoringProjectionRebuildEnvelope(payload)
require.NoError(t, err)
assert.Equal(t, "2026-04-12", envelope.BusinessDate)
}
@@ -0,0 +1,39 @@
package app
import (
"sync"
"time"
)
const monitoringBusinessTimeZoneName = "Asia/Shanghai"
var (
monitoringBusinessLocationOnce sync.Once
monitoringBusinessLocationValue *time.Location
)
func monitoringBusinessLocation() *time.Location {
monitoringBusinessLocationOnce.Do(func() {
loc, err := time.LoadLocation(monitoringBusinessTimeZoneName)
if err != nil {
monitoringBusinessLocationValue = time.FixedZone("UTC+8", 8*60*60)
return
}
monitoringBusinessLocationValue = loc
})
return monitoringBusinessLocationValue
}
func monitoringCanonicalBusinessDay(day time.Time) time.Time {
return monitoringBusinessDayAt(day, monitoringBusinessLocation())
}
func monitoringBusinessDayAndDateAt(now time.Time) (time.Time, string) {
businessDay := monitoringCanonicalBusinessDay(now)
return businessDay, businessDay.Format("2006-01-02")
}
func monitoringBusinessDateText(businessDay time.Time) string {
return monitoringCanonicalBusinessDay(businessDay).Format("2006-01-02")
}