feat: add brand asset cleanup worker and related functionality
- Implemented a new BrandAssetCleanupWorker to handle the cleanup of brand-related assets after a brand is deleted. - Added SQL queries for cleaning up articles, keywords, questions, competitors, and monitoring data associated with a brand. - Introduced a new API endpoint to delete publish records. - Updated the router to include the new delete publish record endpoint. - Added tests for the BrandAssetCleanupWorker to ensure proper functionality. - Created migration scripts to support soft deletion of publish records and to add the brand asset cleanup scheduler job.
This commit is contained in:
@@ -0,0 +1,940 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"go.uber.org/zap"
|
||||
|
||||
sharedcache "github.com/geo-platform/tenant-api/internal/shared/cache"
|
||||
tenantapp "github.com/geo-platform/tenant-api/internal/tenant/app"
|
||||
)
|
||||
|
||||
const defaultBrandAssetCleanupBatch = 20
|
||||
|
||||
type BrandAssetCleanupWorker struct {
|
||||
pool *pgxpool.Pool
|
||||
monitoringPool *pgxpool.Pool
|
||||
cache sharedcache.Cache
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
type brandCleanupCandidate struct {
|
||||
TenantID int64
|
||||
BrandID int64
|
||||
}
|
||||
|
||||
type brandCleanupStats struct {
|
||||
BrandCount int64
|
||||
ArticleCount int64
|
||||
GenerationTaskCount int64
|
||||
GenerationTaskRecordCount int64
|
||||
KolUsageCount int64
|
||||
QuotaReservationRefundCount int64
|
||||
CancelledPublishRecordCount int64
|
||||
PublishBatchCount int64
|
||||
DesktopPublishJobCount int64
|
||||
DesktopPublishTaskCount int64
|
||||
DesktopMonitorTaskCount int64
|
||||
PromptRuleCount int64
|
||||
PromptRuleGroupCount int64
|
||||
ScheduleTaskCount int64
|
||||
KeywordCount int64
|
||||
QuestionCount int64
|
||||
CompetitorCount int64
|
||||
ImageReferenceCount int64
|
||||
MediaSupplyOrderCount int64
|
||||
MediaSupplyOrderItemCount int64
|
||||
ComplianceReviewJobCount int64
|
||||
ComplianceManualReviewCount int64
|
||||
MonitoringCollectRequestCount int64
|
||||
MonitoringCollectOutboxCount int64
|
||||
MonitoringCollectTaskCount int64
|
||||
MonitoringSnapshotCount int64
|
||||
MonitoringRunCount int64
|
||||
MonitoringParseResultCount int64
|
||||
MonitoringCitationFactCount int64
|
||||
MonitoringBrandDailyCount int64
|
||||
MonitoringBrandPlatformCount int64
|
||||
MonitoringArticleAliasCount int64
|
||||
MonitoringMarkedArticleCount int64
|
||||
MarkedDeletedBrandCount int64
|
||||
CompletedBrandCount int64
|
||||
}
|
||||
|
||||
type monitoringCleanupResult struct {
|
||||
Stats brandCleanupStats
|
||||
MonitorTaskIDs []int64
|
||||
}
|
||||
|
||||
func NewBrandAssetCleanupWorker(pool, monitoringPool *pgxpool.Pool, cache sharedcache.Cache, logger *zap.Logger) *BrandAssetCleanupWorker {
|
||||
return &BrandAssetCleanupWorker{
|
||||
pool: pool,
|
||||
monitoringPool: monitoringPool,
|
||||
cache: cache,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
func (w *BrandAssetCleanupWorker) RunOnce(ctx context.Context, run JobRunContext) (map[string]any, error) {
|
||||
stats := map[string]any{}
|
||||
if w == nil || w.pool == nil {
|
||||
stats["skipped"] = true
|
||||
stats["reason"] = "worker_not_ready"
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
startedAt := time.Now()
|
||||
batchSize := AsInt(run.Config, "batch_size", defaultBrandAssetCleanupBatch)
|
||||
if run.Job != nil && run.Job.BatchSize != nil {
|
||||
batchSize = *run.Job.BatchSize
|
||||
}
|
||||
if batchSize <= 0 {
|
||||
batchSize = defaultBrandAssetCleanupBatch
|
||||
}
|
||||
statementTimeout := AsDuration(run.Config, "statement_timeout", 5*time.Second)
|
||||
dryRun := run.DryRun
|
||||
|
||||
stats["batch_size"] = batchSize
|
||||
stats["dry_run"] = dryRun
|
||||
|
||||
conn, err := w.pool.Acquire(ctx)
|
||||
if err != nil {
|
||||
return stats, fmt.Errorf("acquire tenant connection: %w", err)
|
||||
}
|
||||
defer conn.Release()
|
||||
|
||||
if statementTimeout > 0 {
|
||||
if _, err := conn.Exec(ctx, `SELECT set_config('statement_timeout', $1, false)`, fmt.Sprintf("%dms", statementTimeout.Milliseconds())); err != nil {
|
||||
return stats, fmt.Errorf("set tenant statement timeout: %w", err)
|
||||
}
|
||||
defer conn.Exec(context.Background(), `RESET statement_timeout`)
|
||||
}
|
||||
|
||||
candidates, err := w.claimCandidates(ctx, conn, batchSize)
|
||||
if err != nil {
|
||||
return stats, fmt.Errorf("claim brand cleanup candidates: %w", err)
|
||||
}
|
||||
total := brandCleanupStats{}
|
||||
failures := int64(0)
|
||||
for _, candidate := range candidates {
|
||||
if dryRun {
|
||||
brandStats, err := w.probeBrand(ctx, conn, candidate)
|
||||
if err != nil {
|
||||
failures++
|
||||
w.warn("brand asset cleanup probe failed", candidate, err)
|
||||
continue
|
||||
}
|
||||
total.add(brandStats)
|
||||
continue
|
||||
}
|
||||
|
||||
brandStats, err := w.cleanupBrand(ctx, conn, candidate)
|
||||
if err != nil {
|
||||
failures++
|
||||
w.warn("brand asset cleanup failed", candidate, err)
|
||||
continue
|
||||
}
|
||||
total.add(brandStats)
|
||||
tenantapp.InvalidateBrandDeletionCaches(context.Background(), w.cache, candidate.TenantID, candidate.BrandID)
|
||||
}
|
||||
|
||||
stats["candidate_count"] = len(candidates)
|
||||
stats["failure_count"] = failures
|
||||
total.writeTo(stats)
|
||||
stats["duration_ms"] = time.Since(startedAt).Milliseconds()
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (w *BrandAssetCleanupWorker) claimCandidates(ctx context.Context, conn *pgxpool.Conn, limit int) ([]brandCleanupCandidate, error) {
|
||||
rows, err := conn.Query(ctx, `
|
||||
SELECT tenant_id, id
|
||||
FROM brands
|
||||
WHERE deleted_at IS NOT NULL
|
||||
AND status = 'deleting'
|
||||
ORDER BY deleted_at ASC, id ASC
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
items := make([]brandCleanupCandidate, 0, limit)
|
||||
for rows.Next() {
|
||||
var item brandCleanupCandidate
|
||||
if err := rows.Scan(&item.TenantID, &item.BrandID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (w *BrandAssetCleanupWorker) probeBrand(ctx context.Context, conn *pgxpool.Conn, c brandCleanupCandidate) (brandCleanupStats, error) {
|
||||
stats := brandCleanupStats{BrandCount: 1}
|
||||
if err := conn.QueryRow(ctx, `SELECT COUNT(*)::bigint FROM articles WHERE tenant_id = $1 AND brand_id = $2 AND deleted_at IS NULL`, c.TenantID, c.BrandID).Scan(&stats.ArticleCount); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if err := conn.QueryRow(ctx, `SELECT COUNT(*)::bigint FROM brand_keywords WHERE tenant_id = $1 AND brand_id = $2 AND deleted_at IS NULL`, c.TenantID, c.BrandID).Scan(&stats.KeywordCount); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if err := conn.QueryRow(ctx, `SELECT COUNT(*)::bigint FROM brand_questions WHERE tenant_id = $1 AND brand_id = $2 AND deleted_at IS NULL`, c.TenantID, c.BrandID).Scan(&stats.QuestionCount); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if err := conn.QueryRow(ctx, `SELECT COUNT(*)::bigint FROM competitors WHERE tenant_id = $1 AND brand_id = $2 AND deleted_at IS NULL`, c.TenantID, c.BrandID).Scan(&stats.CompetitorCount); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (w *BrandAssetCleanupWorker) cleanupBrand(ctx context.Context, conn *pgxpool.Conn, c brandCleanupCandidate) (brandCleanupStats, error) {
|
||||
stats := brandCleanupStats{BrandCount: 1}
|
||||
monitorTaskIDs, err := w.loadMonitoringTaskIDsForBrand(ctx, c)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
|
||||
tx, err := conn.Begin(ctx)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
committed := false
|
||||
defer func() {
|
||||
if !committed {
|
||||
_ = tx.Rollback(context.Background())
|
||||
}
|
||||
}()
|
||||
|
||||
articleIDs, err := w.loadBrandArticleIDs(ctx, tx, c)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
articleIDArg := nullableInt64Array(articleIDs)
|
||||
|
||||
generationStats, err := w.cleanupGenerationAssets(ctx, tx, c, articleIDArg)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.GenerationTaskCount += generationStats.GenerationTaskCount
|
||||
stats.GenerationTaskRecordCount += generationStats.GenerationTaskRecordCount
|
||||
stats.KolUsageCount += generationStats.KolUsageCount
|
||||
stats.QuotaReservationRefundCount += generationStats.QuotaReservationRefundCount
|
||||
|
||||
count, err := execRows(ctx, tx, `
|
||||
UPDATE articles
|
||||
SET deleted_at = COALESCE(deleted_at, NOW()),
|
||||
generate_status = CASE WHEN generate_status IN ('pending', 'generating', 'running') THEN 'failed' ELSE generate_status END,
|
||||
updated_at = NOW()
|
||||
WHERE tenant_id = $1
|
||||
AND brand_id = $2
|
||||
AND deleted_at IS NULL
|
||||
`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.ArticleCount += count
|
||||
|
||||
count, err = execRows(ctx, tx, `
|
||||
UPDATE image_asset_references
|
||||
SET updated_at = NOW()
|
||||
WHERE tenant_id = $1
|
||||
AND article_id = ANY($2::bigint[])
|
||||
`, c.TenantID, articleIDArg)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.ImageReferenceCount += count
|
||||
if _, err := tx.Exec(ctx, `
|
||||
DELETE FROM image_asset_references
|
||||
WHERE tenant_id = $1
|
||||
AND article_id = ANY($2::bigint[])
|
||||
`, c.TenantID, articleIDArg); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
|
||||
desktopStats, err := w.cancelDesktopPublishAssets(ctx, tx, c, articleIDArg)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.DesktopPublishJobCount += desktopStats.DesktopPublishJobCount
|
||||
stats.DesktopPublishTaskCount += desktopStats.DesktopPublishTaskCount
|
||||
stats.CancelledPublishRecordCount += desktopStats.CancelledPublishRecordCount
|
||||
stats.PublishBatchCount += desktopStats.PublishBatchCount
|
||||
|
||||
count, err = execRows(ctx, tx, `
|
||||
UPDATE desktop_tasks
|
||||
SET status = 'aborted',
|
||||
error = jsonb_build_object(
|
||||
'code', 'brand_deleted',
|
||||
'message', 'brand deleted before monitor task completion',
|
||||
'source', 'brand_asset_cleanup'
|
||||
),
|
||||
active_attempt_id = NULL,
|
||||
lease_token_hash = NULL,
|
||||
lease_expires_at = NULL,
|
||||
updated_at = NOW()
|
||||
WHERE tenant_id = $1
|
||||
AND kind = 'monitor'
|
||||
AND status IN ('queued', 'in_progress', 'unknown')
|
||||
AND monitor_task_id = ANY($2::bigint[])
|
||||
`, c.TenantID, nullableInt64Array(monitorTaskIDs))
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.DesktopMonitorTaskCount += count
|
||||
|
||||
count, err = execRows(ctx, tx, `
|
||||
UPDATE prompt_rules
|
||||
SET deleted_at = COALESCE(deleted_at, NOW()),
|
||||
updated_at = NOW()
|
||||
WHERE tenant_id = $1
|
||||
AND brand_id = $2
|
||||
AND deleted_at IS NULL
|
||||
`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.PromptRuleCount += count
|
||||
|
||||
count, err = execRows(ctx, tx, `
|
||||
UPDATE prompt_rule_groups
|
||||
SET deleted_at = COALESCE(deleted_at, NOW()),
|
||||
updated_at = NOW()
|
||||
WHERE tenant_id = $1
|
||||
AND brand_id = $2
|
||||
AND deleted_at IS NULL
|
||||
`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.PromptRuleGroupCount += count
|
||||
|
||||
count, err = execRows(ctx, tx, `
|
||||
UPDATE schedule_tasks
|
||||
SET deleted_at = COALESCE(deleted_at, NOW()),
|
||||
status = CASE WHEN status = 'enabled' THEN 'disabled' ELSE status END,
|
||||
updated_at = NOW()
|
||||
WHERE tenant_id = $1
|
||||
AND brand_id = $2
|
||||
AND deleted_at IS NULL
|
||||
`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.ScheduleTaskCount += count
|
||||
|
||||
count, err = execRows(ctx, tx, `
|
||||
UPDATE brand_keywords
|
||||
SET deleted_at = COALESCE(deleted_at, NOW()),
|
||||
updated_at = NOW()
|
||||
WHERE tenant_id = $1
|
||||
AND brand_id = $2
|
||||
AND deleted_at IS NULL
|
||||
`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.KeywordCount += count
|
||||
|
||||
count, err = execRows(ctx, tx, `
|
||||
UPDATE brand_questions
|
||||
SET deleted_at = COALESCE(deleted_at, NOW()),
|
||||
updated_at = NOW()
|
||||
WHERE tenant_id = $1
|
||||
AND brand_id = $2
|
||||
AND deleted_at IS NULL
|
||||
`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.QuestionCount += count
|
||||
|
||||
count, err = execRows(ctx, tx, `
|
||||
UPDATE competitors
|
||||
SET deleted_at = COALESCE(deleted_at, NOW()),
|
||||
updated_at = NOW()
|
||||
WHERE tenant_id = $1
|
||||
AND brand_id = $2
|
||||
AND deleted_at IS NULL
|
||||
`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.CompetitorCount += count
|
||||
|
||||
mediaStats, err := w.cleanupMediaSupply(ctx, tx, c)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.MediaSupplyOrderCount += mediaStats.MediaSupplyOrderCount
|
||||
stats.MediaSupplyOrderItemCount += mediaStats.MediaSupplyOrderItemCount
|
||||
|
||||
complianceStats, err := w.cancelComplianceAssets(ctx, tx, c, articleIDArg)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.ComplianceReviewJobCount += complianceStats.ComplianceReviewJobCount
|
||||
stats.ComplianceManualReviewCount += complianceStats.ComplianceManualReviewCount
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
committed = true
|
||||
|
||||
monitoringCleanup, err := w.cleanupMonitoringBrand(ctx, c, articleIDs)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.add(monitoringCleanup.Stats)
|
||||
|
||||
tag, err := conn.Exec(ctx, `
|
||||
UPDATE brands
|
||||
SET status = 'deleted',
|
||||
updated_at = NOW()
|
||||
WHERE tenant_id = $1
|
||||
AND id = $2
|
||||
AND deleted_at IS NOT NULL
|
||||
AND status = 'deleting'
|
||||
`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.CompletedBrandCount += tag.RowsAffected()
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (w *BrandAssetCleanupWorker) loadBrandArticleIDs(ctx context.Context, tx pgx.Tx, c brandCleanupCandidate) ([]int64, error) {
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT id
|
||||
FROM articles
|
||||
WHERE tenant_id = $1
|
||||
AND brand_id = $2
|
||||
ORDER BY id ASC
|
||||
`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
ids := make([]int64, 0)
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids, rows.Err()
|
||||
}
|
||||
|
||||
func (w *BrandAssetCleanupWorker) loadMonitoringTaskIDsForBrand(ctx context.Context, c brandCleanupCandidate) ([]int64, error) {
|
||||
if w == nil || w.monitoringPool == nil {
|
||||
return nil, nil
|
||||
}
|
||||
conn, err := w.monitoringPool.Acquire(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("acquire monitoring connection: %w", err)
|
||||
}
|
||||
defer conn.Release()
|
||||
return w.loadMonitoringTaskIDs(ctx, conn, c)
|
||||
}
|
||||
|
||||
func (w *BrandAssetCleanupWorker) cleanupGenerationAssets(ctx context.Context, tx pgx.Tx, c brandCleanupCandidate, articleIDs any) (brandCleanupStats, error) {
|
||||
stats := brandCleanupStats{}
|
||||
if err := tx.QueryRow(ctx, `
|
||||
WITH affected_tasks AS (
|
||||
UPDATE generation_tasks
|
||||
SET status = 'failed',
|
||||
error_message = COALESCE(NULLIF(error_message, ''), 'brand_deleted'),
|
||||
completed_at = COALESCE(completed_at, NOW()),
|
||||
lease_token = NULL,
|
||||
lease_owner = NULL,
|
||||
lease_expires_at = NULL,
|
||||
updated_at = NOW()
|
||||
WHERE tenant_id = $1
|
||||
AND article_id = ANY($2::bigint[])
|
||||
AND status IN ('queued', 'running')
|
||||
RETURNING id, quota_reservation_id
|
||||
),
|
||||
updated_task_records AS (
|
||||
UPDATE task_records tr
|
||||
SET status = 'failed',
|
||||
error_message = COALESCE(NULLIF(error_message, ''), 'brand_deleted'),
|
||||
updated_at = NOW()
|
||||
FROM affected_tasks t
|
||||
WHERE tr.tenant_id = $1
|
||||
AND tr.task_type = 'article_generation'
|
||||
AND tr.resource_type = 'generation_task'
|
||||
AND tr.resource_id = t.id
|
||||
AND tr.status IN ('pending', 'queued', 'running')
|
||||
RETURNING tr.id
|
||||
),
|
||||
updated_kol_usage AS (
|
||||
UPDATE kol_usage_logs u
|
||||
SET status = 'failed',
|
||||
error_message = COALESCE(NULLIF(error_message, ''), 'brand_deleted'),
|
||||
finished_at = COALESCE(finished_at, NOW())
|
||||
FROM affected_tasks t
|
||||
WHERE u.tenant_id = $1
|
||||
AND u.generation_task_id = t.id
|
||||
AND u.status IN ('pending', 'running')
|
||||
RETURNING u.id
|
||||
),
|
||||
refunded_reservations AS (
|
||||
UPDATE quota_reservations qr
|
||||
SET status = 'refunded',
|
||||
refunded_amount = reserved_amount,
|
||||
updated_at = NOW()
|
||||
FROM affected_tasks t
|
||||
WHERE qr.id = t.quota_reservation_id
|
||||
AND qr.tenant_id = $1
|
||||
AND qr.status = 'pending'
|
||||
RETURNING qr.id
|
||||
)
|
||||
SELECT
|
||||
(SELECT COUNT(*)::bigint FROM affected_tasks),
|
||||
(SELECT COUNT(*)::bigint FROM updated_task_records),
|
||||
(SELECT COUNT(*)::bigint FROM updated_kol_usage),
|
||||
(SELECT COUNT(*)::bigint FROM refunded_reservations)
|
||||
`, c.TenantID, articleIDs).Scan(
|
||||
&stats.GenerationTaskCount,
|
||||
&stats.GenerationTaskRecordCount,
|
||||
&stats.KolUsageCount,
|
||||
&stats.QuotaReservationRefundCount,
|
||||
); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (w *BrandAssetCleanupWorker) cancelDesktopPublishAssets(ctx context.Context, tx pgx.Tx, c brandCleanupCandidate, articleIDs any) (brandCleanupStats, error) {
|
||||
stats := brandCleanupStats{}
|
||||
if err := tx.QueryRow(ctx, `
|
||||
WITH affected_records AS (
|
||||
UPDATE publish_records pr
|
||||
SET status = 'cancelled',
|
||||
error_message = COALESCE(NULLIF(pr.error_message, ''), 'brand_deleted'),
|
||||
updated_at = NOW()
|
||||
WHERE pr.tenant_id = $1
|
||||
AND pr.article_id = ANY($2::bigint[])
|
||||
AND pr.deleted_at IS NULL
|
||||
AND pr.status IN ('queued', 'publishing', 'pending', 'running')
|
||||
RETURNING pr.publish_batch_id
|
||||
),
|
||||
affected_batches AS (
|
||||
SELECT DISTINCT publish_batch_id
|
||||
FROM affected_records
|
||||
WHERE publish_batch_id IS NOT NULL
|
||||
),
|
||||
updated_batches AS (
|
||||
UPDATE publish_batches pb
|
||||
SET status = CASE
|
||||
WHEN EXISTS (
|
||||
SELECT 1
|
||||
FROM publish_records pr
|
||||
WHERE pr.publish_batch_id = pb.id
|
||||
AND pr.deleted_at IS NULL
|
||||
AND pr.status IN ('queued', 'publishing', 'pending', 'running')
|
||||
) THEN 'publishing'
|
||||
WHEN EXISTS (
|
||||
SELECT 1
|
||||
FROM publish_records pr
|
||||
WHERE pr.publish_batch_id = pb.id
|
||||
AND pr.deleted_at IS NULL
|
||||
AND pr.status IN ('success', 'published', 'publish_success')
|
||||
) AND EXISTS (
|
||||
SELECT 1
|
||||
FROM publish_records pr
|
||||
WHERE pr.publish_batch_id = pb.id
|
||||
AND pr.deleted_at IS NULL
|
||||
AND pr.status NOT IN ('success', 'published', 'publish_success')
|
||||
) THEN 'partial_success'
|
||||
WHEN EXISTS (
|
||||
SELECT 1
|
||||
FROM publish_records pr
|
||||
WHERE pr.publish_batch_id = pb.id
|
||||
AND pr.deleted_at IS NULL
|
||||
AND pr.status IN ('success', 'published', 'publish_success')
|
||||
) THEN 'success'
|
||||
ELSE 'failed'
|
||||
END,
|
||||
updated_at = NOW()
|
||||
FROM affected_batches ab
|
||||
WHERE pb.id = ab.publish_batch_id
|
||||
AND pb.tenant_id = $1
|
||||
RETURNING pb.id
|
||||
)
|
||||
SELECT
|
||||
(SELECT COUNT(*)::bigint FROM affected_records),
|
||||
(SELECT COUNT(*)::bigint FROM updated_batches)
|
||||
`, c.TenantID, articleIDs).Scan(&stats.CancelledPublishRecordCount, &stats.PublishBatchCount); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
|
||||
if err := tx.QueryRow(ctx, `
|
||||
WITH updated_tasks AS (
|
||||
UPDATE desktop_tasks dt
|
||||
SET status = 'aborted',
|
||||
error = jsonb_build_object(
|
||||
'code', 'brand_deleted',
|
||||
'message', 'brand deleted before publish completion',
|
||||
'source', 'brand_asset_cleanup'
|
||||
),
|
||||
active_attempt_id = NULL,
|
||||
lease_token_hash = NULL,
|
||||
lease_expires_at = NULL,
|
||||
updated_at = NOW()
|
||||
FROM desktop_publish_jobs j
|
||||
WHERE dt.job_id = j.desktop_id
|
||||
AND dt.tenant_id = j.tenant_id
|
||||
AND dt.tenant_id = $1
|
||||
AND j.article_id = ANY($2::bigint[])
|
||||
AND dt.kind = 'publish'
|
||||
AND dt.status IN ('queued', 'in_progress', 'unknown')
|
||||
RETURNING dt.job_id
|
||||
),
|
||||
updated_jobs AS (
|
||||
UPDATE desktop_publish_jobs j
|
||||
SET status = 'cancelled',
|
||||
updated_at = NOW()
|
||||
WHERE j.tenant_id = $1
|
||||
AND j.article_id = ANY($2::bigint[])
|
||||
AND j.status IN ('queued', 'blocked_by_compliance')
|
||||
RETURNING j.desktop_id
|
||||
)
|
||||
SELECT
|
||||
(SELECT COUNT(*)::bigint FROM updated_tasks),
|
||||
(SELECT COUNT(*)::bigint FROM updated_jobs)
|
||||
`, c.TenantID, articleIDs).Scan(&stats.DesktopPublishTaskCount, &stats.DesktopPublishJobCount); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (w *BrandAssetCleanupWorker) cleanupMediaSupply(ctx context.Context, tx pgx.Tx, c brandCleanupCandidate) (brandCleanupStats, error) {
|
||||
stats := brandCleanupStats{}
|
||||
if err := tx.QueryRow(ctx, `
|
||||
WITH affected_orders AS (
|
||||
UPDATE media_supply_orders
|
||||
SET status = CASE WHEN status IN ('queued', 'submitting') THEN 'failed' ELSE status END,
|
||||
error_message = CASE
|
||||
WHEN status IN ('queued', 'submitting') THEN COALESCE(NULLIF(error_message, ''), 'brand_deleted')
|
||||
ELSE error_message
|
||||
END,
|
||||
deleted_at = COALESCE(deleted_at, NOW()),
|
||||
completed_at = CASE WHEN status IN ('queued', 'submitting') THEN COALESCE(completed_at, NOW()) ELSE completed_at END,
|
||||
updated_at = NOW()
|
||||
WHERE tenant_id = $1
|
||||
AND brand_id = $2
|
||||
AND deleted_at IS NULL
|
||||
RETURNING id
|
||||
),
|
||||
affected_items AS (
|
||||
UPDATE media_supply_order_items i
|
||||
SET status = CASE WHEN i.status IN ('queued', 'submitting') THEN 'failed' ELSE i.status END,
|
||||
updated_at = NOW()
|
||||
FROM affected_orders o
|
||||
WHERE i.order_id = o.id
|
||||
RETURNING i.id
|
||||
)
|
||||
SELECT
|
||||
(SELECT COUNT(*)::bigint FROM affected_orders),
|
||||
(SELECT COUNT(*)::bigint FROM affected_items)
|
||||
`, c.TenantID, c.BrandID).Scan(&stats.MediaSupplyOrderCount, &stats.MediaSupplyOrderItemCount); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (w *BrandAssetCleanupWorker) cancelComplianceAssets(ctx context.Context, tx pgx.Tx, c brandCleanupCandidate, articleIDs any) (brandCleanupStats, error) {
|
||||
stats := brandCleanupStats{}
|
||||
count, err := execRows(ctx, tx, `
|
||||
UPDATE compliance_review_jobs
|
||||
SET status = 'failed',
|
||||
last_error = COALESCE(NULLIF(last_error, ''), 'brand_deleted'),
|
||||
updated_at = NOW()
|
||||
WHERE tenant_id = $1
|
||||
AND article_id = ANY($2::bigint[])
|
||||
AND status IN ('pending', 'running')
|
||||
`, c.TenantID, articleIDs)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.ComplianceReviewJobCount += count
|
||||
|
||||
if err := tx.QueryRow(ctx, `
|
||||
WITH affected_reviews AS (
|
||||
UPDATE compliance_manual_reviews
|
||||
SET status = 'cancelled',
|
||||
cancelled_by = COALESCE(cancelled_by, requested_by),
|
||||
cancelled_at = COALESCE(cancelled_at, NOW()),
|
||||
cancel_reason = COALESCE(NULLIF(cancel_reason, ''), 'brand_deleted')
|
||||
WHERE tenant_id = $1
|
||||
AND article_id = ANY($2::bigint[])
|
||||
AND status = 'pending'
|
||||
RETURNING article_version_id
|
||||
),
|
||||
updated_versions AS (
|
||||
UPDATE article_versions av
|
||||
SET manual_review_status = 'none',
|
||||
manual_review_id = NULL,
|
||||
manual_review_updated_at = NOW()
|
||||
FROM affected_reviews r
|
||||
WHERE av.id = r.article_version_id
|
||||
RETURNING av.id
|
||||
)
|
||||
SELECT COUNT(*)::bigint FROM affected_reviews
|
||||
`, c.TenantID, articleIDs).Scan(&stats.ComplianceManualReviewCount); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (w *BrandAssetCleanupWorker) cleanupMonitoringBrand(ctx context.Context, c brandCleanupCandidate, articleIDs []int64) (monitoringCleanupResult, error) {
|
||||
result := monitoringCleanupResult{}
|
||||
if w == nil || w.monitoringPool == nil {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
conn, err := w.monitoringPool.Acquire(ctx)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("acquire monitoring connection: %w", err)
|
||||
}
|
||||
defer conn.Release()
|
||||
|
||||
monitorTaskIDs, err := w.loadMonitoringTaskIDs(ctx, conn, c)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.MonitorTaskIDs = monitorTaskIDs
|
||||
|
||||
tx, err := conn.Begin(ctx)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
committed := false
|
||||
defer func() {
|
||||
if !committed {
|
||||
_ = tx.Rollback(context.Background())
|
||||
}
|
||||
}()
|
||||
|
||||
count, err := execRows(ctx, tx, `
|
||||
DELETE FROM monitoring_collect_dispatch_outbox
|
||||
WHERE task_id IN (
|
||||
SELECT id
|
||||
FROM monitoring_collect_tasks
|
||||
WHERE tenant_id = $1 AND brand_id = $2
|
||||
)
|
||||
OR request_id IN (
|
||||
SELECT request_id
|
||||
FROM monitoring_collect_requests
|
||||
WHERE tenant_id = $1 AND brand_id = $2
|
||||
)
|
||||
`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Stats.MonitoringCollectOutboxCount += count
|
||||
|
||||
count, err = execRows(ctx, tx, `DELETE FROM monitoring_collect_requests WHERE tenant_id = $1 AND brand_id = $2`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Stats.MonitoringCollectRequestCount += count
|
||||
|
||||
count, err = execRows(ctx, tx, `DELETE FROM monitoring_citation_facts WHERE tenant_id = $1 AND brand_id = $2`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Stats.MonitoringCitationFactCount += count
|
||||
|
||||
count, err = execRows(ctx, tx, `DELETE FROM question_monitor_parse_results WHERE tenant_id = $1 AND brand_id = $2`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Stats.MonitoringParseResultCount += count
|
||||
|
||||
count, err = execRows(ctx, tx, `DELETE FROM question_monitor_runs WHERE tenant_id = $1 AND brand_id = $2`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Stats.MonitoringRunCount += count
|
||||
|
||||
count, err = execRows(ctx, tx, `DELETE FROM monitoring_collect_tasks WHERE tenant_id = $1 AND brand_id = $2`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Stats.MonitoringCollectTaskCount += count
|
||||
|
||||
count, err = execRows(ctx, tx, `DELETE FROM monitoring_brand_platform_daily WHERE tenant_id = $1 AND brand_id = $2`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Stats.MonitoringBrandPlatformCount += count
|
||||
|
||||
count, err = execRows(ctx, tx, `DELETE FROM monitoring_brand_daily WHERE tenant_id = $1 AND brand_id = $2`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Stats.MonitoringBrandDailyCount += count
|
||||
|
||||
count, err = execRows(ctx, tx, `DELETE FROM monitoring_question_config_snapshots WHERE tenant_id = $1 AND brand_id = $2`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Stats.MonitoringSnapshotCount += count
|
||||
|
||||
count, err = execRows(ctx, tx, `
|
||||
DELETE FROM monitoring_article_url_aliases
|
||||
WHERE tenant_id = $1
|
||||
AND article_id = ANY($2::bigint[])
|
||||
`, c.TenantID, nullableInt64Array(articleIDs))
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Stats.MonitoringArticleAliasCount += count
|
||||
|
||||
count, err = execRows(ctx, tx, `DELETE FROM monitoring_user_marked_articles WHERE tenant_id = $1 AND brand_id = $2`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Stats.MonitoringMarkedArticleCount += count
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return result, err
|
||||
}
|
||||
committed = true
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (w *BrandAssetCleanupWorker) loadMonitoringTaskIDs(ctx context.Context, conn *pgxpool.Conn, c brandCleanupCandidate) ([]int64, error) {
|
||||
rows, err := conn.Query(ctx, `
|
||||
SELECT id
|
||||
FROM monitoring_collect_tasks
|
||||
WHERE tenant_id = $1
|
||||
AND brand_id = $2
|
||||
`, c.TenantID, c.BrandID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
ids := make([]int64, 0)
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids, rows.Err()
|
||||
}
|
||||
|
||||
func execRows(ctx context.Context, tx pgx.Tx, sql string, args ...any) (int64, error) {
|
||||
tag, err := tx.Exec(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
func nullableInt64Array(values []int64) any {
|
||||
if len(values) == 0 {
|
||||
return []int64{-1}
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func (s *brandCleanupStats) add(other brandCleanupStats) {
|
||||
s.BrandCount += other.BrandCount
|
||||
s.ArticleCount += other.ArticleCount
|
||||
s.GenerationTaskCount += other.GenerationTaskCount
|
||||
s.GenerationTaskRecordCount += other.GenerationTaskRecordCount
|
||||
s.KolUsageCount += other.KolUsageCount
|
||||
s.QuotaReservationRefundCount += other.QuotaReservationRefundCount
|
||||
s.CancelledPublishRecordCount += other.CancelledPublishRecordCount
|
||||
s.PublishBatchCount += other.PublishBatchCount
|
||||
s.DesktopPublishJobCount += other.DesktopPublishJobCount
|
||||
s.DesktopPublishTaskCount += other.DesktopPublishTaskCount
|
||||
s.DesktopMonitorTaskCount += other.DesktopMonitorTaskCount
|
||||
s.PromptRuleCount += other.PromptRuleCount
|
||||
s.PromptRuleGroupCount += other.PromptRuleGroupCount
|
||||
s.ScheduleTaskCount += other.ScheduleTaskCount
|
||||
s.KeywordCount += other.KeywordCount
|
||||
s.QuestionCount += other.QuestionCount
|
||||
s.CompetitorCount += other.CompetitorCount
|
||||
s.ImageReferenceCount += other.ImageReferenceCount
|
||||
s.MediaSupplyOrderCount += other.MediaSupplyOrderCount
|
||||
s.MediaSupplyOrderItemCount += other.MediaSupplyOrderItemCount
|
||||
s.ComplianceReviewJobCount += other.ComplianceReviewJobCount
|
||||
s.ComplianceManualReviewCount += other.ComplianceManualReviewCount
|
||||
s.MonitoringCollectRequestCount += other.MonitoringCollectRequestCount
|
||||
s.MonitoringCollectOutboxCount += other.MonitoringCollectOutboxCount
|
||||
s.MonitoringCollectTaskCount += other.MonitoringCollectTaskCount
|
||||
s.MonitoringSnapshotCount += other.MonitoringSnapshotCount
|
||||
s.MonitoringRunCount += other.MonitoringRunCount
|
||||
s.MonitoringParseResultCount += other.MonitoringParseResultCount
|
||||
s.MonitoringCitationFactCount += other.MonitoringCitationFactCount
|
||||
s.MonitoringBrandDailyCount += other.MonitoringBrandDailyCount
|
||||
s.MonitoringBrandPlatformCount += other.MonitoringBrandPlatformCount
|
||||
s.MonitoringArticleAliasCount += other.MonitoringArticleAliasCount
|
||||
s.MonitoringMarkedArticleCount += other.MonitoringMarkedArticleCount
|
||||
s.MarkedDeletedBrandCount += other.MarkedDeletedBrandCount
|
||||
s.CompletedBrandCount += other.CompletedBrandCount
|
||||
}
|
||||
|
||||
func (s brandCleanupStats) writeTo(out map[string]any) {
|
||||
out["brand_count"] = s.BrandCount
|
||||
out["article_count"] = s.ArticleCount
|
||||
out["generation_task_count"] = s.GenerationTaskCount
|
||||
out["generation_task_record_count"] = s.GenerationTaskRecordCount
|
||||
out["kol_usage_count"] = s.KolUsageCount
|
||||
out["quota_reservation_refund_count"] = s.QuotaReservationRefundCount
|
||||
out["cancelled_publish_record_count"] = s.CancelledPublishRecordCount
|
||||
out["publish_batch_count"] = s.PublishBatchCount
|
||||
out["desktop_publish_job_count"] = s.DesktopPublishJobCount
|
||||
out["desktop_publish_task_count"] = s.DesktopPublishTaskCount
|
||||
out["desktop_monitor_task_count"] = s.DesktopMonitorTaskCount
|
||||
out["prompt_rule_count"] = s.PromptRuleCount
|
||||
out["prompt_rule_group_count"] = s.PromptRuleGroupCount
|
||||
out["schedule_task_count"] = s.ScheduleTaskCount
|
||||
out["keyword_count"] = s.KeywordCount
|
||||
out["question_count"] = s.QuestionCount
|
||||
out["competitor_count"] = s.CompetitorCount
|
||||
out["image_reference_count"] = s.ImageReferenceCount
|
||||
out["media_supply_order_count"] = s.MediaSupplyOrderCount
|
||||
out["media_supply_order_item_count"] = s.MediaSupplyOrderItemCount
|
||||
out["compliance_review_job_count"] = s.ComplianceReviewJobCount
|
||||
out["compliance_manual_review_count"] = s.ComplianceManualReviewCount
|
||||
out["monitoring_collect_request_count"] = s.MonitoringCollectRequestCount
|
||||
out["monitoring_collect_outbox_count"] = s.MonitoringCollectOutboxCount
|
||||
out["monitoring_collect_task_count"] = s.MonitoringCollectTaskCount
|
||||
out["monitoring_snapshot_count"] = s.MonitoringSnapshotCount
|
||||
out["monitoring_run_count"] = s.MonitoringRunCount
|
||||
out["monitoring_parse_result_count"] = s.MonitoringParseResultCount
|
||||
out["monitoring_citation_fact_count"] = s.MonitoringCitationFactCount
|
||||
out["monitoring_brand_daily_count"] = s.MonitoringBrandDailyCount
|
||||
out["monitoring_brand_platform_count"] = s.MonitoringBrandPlatformCount
|
||||
out["monitoring_article_alias_count"] = s.MonitoringArticleAliasCount
|
||||
out["monitoring_marked_article_count"] = s.MonitoringMarkedArticleCount
|
||||
out["completed_brand_count"] = s.CompletedBrandCount
|
||||
}
|
||||
|
||||
func (w *BrandAssetCleanupWorker) warn(message string, c brandCleanupCandidate, err error) {
|
||||
if w == nil || w.logger == nil {
|
||||
return
|
||||
}
|
||||
w.logger.Warn(message,
|
||||
zap.Int64("tenant_id", c.TenantID),
|
||||
zap.Int64("brand_id", c.BrandID),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBrandAssetCleanupRunOnceSkipsWhenStoreMissing(t *testing.T) {
|
||||
stats, err := (*BrandAssetCleanupWorker)(nil).RunOnce(context.Background(), JobRunContext{})
|
||||
if err != nil {
|
||||
t.Fatalf("RunOnce returned error: %v", err)
|
||||
}
|
||||
if stats["skipped"] != true {
|
||||
t.Fatalf("stats[skipped] = %v, want true", stats["skipped"])
|
||||
}
|
||||
if stats["reason"] != "worker_not_ready" {
|
||||
t.Fatalf("stats[reason] = %v, want worker_not_ready", stats["reason"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestBrandCleanupStatsWriteToIncludesTerminalMetrics(t *testing.T) {
|
||||
stats := brandCleanupStats{
|
||||
BrandCount: 1,
|
||||
ArticleCount: 2,
|
||||
MonitoringCollectTaskCount: 3,
|
||||
MonitoringCitationFactCount: 4,
|
||||
CompletedBrandCount: 1,
|
||||
}
|
||||
out := map[string]any{}
|
||||
stats.writeTo(out)
|
||||
|
||||
for key, want := range map[string]any{
|
||||
"brand_count": int64(1),
|
||||
"article_count": int64(2),
|
||||
"monitoring_collect_task_count": int64(3),
|
||||
"monitoring_citation_fact_count": int64(4),
|
||||
"completed_brand_count": int64(1),
|
||||
} {
|
||||
if got := out[key]; got != want {
|
||||
t.Fatalf("out[%s] = %v, want %v", key, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user