feat(media-supply): resource cache sync and order flow refinements
Frontend CI / Frontend (push) Successful in 3m38s
Backend CI / Backend (push) Failing after 26m59s

Add a resource cache sync entrypoint (RunMediaResourceSyncOnce) driven
by the new scheduler job, and rework the backlink/sync worker around it.
Tune the meijiequan client and slow the default backlink sync interval
from 10m to 30m to reduce upstream pressure.

Trim unused public/transport surface (handler + router + swagger) and
simplify admin-web order management and ops-web media-supply views,
dropping stale shared-types fields.
This commit is contained in:
2026-06-02 14:50:36 +08:00
parent 842782b3dd
commit 723c3ffb86
14 changed files with 529 additions and 208 deletions
+135 -83
View File
@@ -12,6 +12,8 @@ import (
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"go.uber.org/zap"
"github.com/geo-platform/tenant-api/internal/shared/config"
)
type MediaSupplyWorker struct {
@@ -46,7 +48,12 @@ var (
errMediaSupplyLockedSellBelowCost = errors.New("supplier cost increased above locked sell price")
)
var mediaSupplyOrderCodePattern = regexp.MustCompile(`(?i)\b[A-Z]{1,12}\d{3,}\b`)
var mediaSupplyOrderCodePattern = regexp.MustCompile(`(?i)\bWM\d{3,}\b`)
const (
mediaSupplyValidExternalOrderCodeSQL = `TRIM(COALESCE(external_order_code, '')) ~* '^WM[0-9]{3,}$'`
mediaSupplyBacklinkSyncCooldownKey = "media_supply:backlink_sync:cooldown"
)
func NewMediaSupplyWorker(service *MediaSupplyService, pool *pgxpool.Pool, logger *zap.Logger) *MediaSupplyWorker {
return &MediaSupplyWorker{service: service, pool: pool, logger: logger}
@@ -107,27 +114,14 @@ func (w *MediaSupplyWorker) processOneSyncJob(ctx context.Context) error {
if err != nil || !ok {
return err
}
cfg := w.service.mediaSupplyConfig()
if status, statusErr := w.service.client.SessionStatus(ctx); statusErr != nil || status == nil || !status.Available {
if loginStatus, loginErr := w.service.client.LoginWithConfiguredAccount(ctx); loginErr != nil {
return w.failSyncJob(ctx, job, loginErr)
} else if loginStatus == nil || !loginStatus.Available {
return w.failSyncJob(ctx, job, errMeijiequanSessionMissing)
}
}
modelID := 1
if job.ModelID != nil && *job.ModelID > 0 {
modelID = *job.ModelID
}
totalSynced := 0
stats := map[string]any{"models": map[int]int{}}
modelSynced, syncErr := w.syncModel(ctx, modelID, cfg.Meijiequan.SyncPageSize, cfg.Meijiequan.MaxSyncPages, cfg.Meijiequan.SyncPageDelay)
stats, syncErr := w.service.RunMediaResourceSyncOnce(ctx, modelID)
if syncErr != nil {
return w.failSyncJob(ctx, job, syncErr)
}
totalSynced += modelSynced
stats["models"].(map[int]int)[modelID] = modelSynced
stats["synced"] = totalSynced
raw, _ := json.Marshal(stats)
if _, err := w.pool.Exec(ctx, `
UPDATE media_supply_sync_jobs
@@ -140,32 +134,8 @@ func (w *MediaSupplyWorker) processOneSyncJob(ctx context.Context) error {
}
func (w *MediaSupplyWorker) syncModel(ctx context.Context, modelID, pageSize, maxPages int, pageDelay time.Duration) (int, error) {
total := 0
for page := 1; page <= maxPages; page++ {
if page > 1 && pageDelay > 0 {
select {
case <-ctx.Done():
return total, ctx.Err()
case <-time.After(pageDelay):
}
}
mediaPage, err := w.service.client.ListMedia(ctx, modelID, page, pageSize)
if err != nil {
return total, err
}
count, err := w.service.UpsertSyncedResources(ctx, mediaPage.Items)
if err != nil {
return total, err
}
total += count
if len(mediaPage.Items) == 0 {
break
}
if mediaPage.AllPage > 0 && page >= mediaPage.AllPage {
break
}
}
return total, nil
stats, err := w.service.syncMediaResourceModel(ctx, modelID, pageSize, maxPages, pageDelay)
return stats.Synced, err
}
func (w *MediaSupplyWorker) processOneOrder(ctx context.Context) error {
@@ -331,11 +301,46 @@ func (w *MediaSupplyWorker) refreshSelectedResourcesFromUpstream(ctx context.Con
}
}
if len(pending) > 0 {
return fmt.Errorf("%w for %d selected resources", errMediaSupplySelectedResourcesMissing, len(pending))
return w.validatePendingResourcesFromCache(ctx, modelID, items, pending)
}
return nil
}
func (w *MediaSupplyWorker) validatePendingResourcesFromCache(ctx context.Context, modelID int, items []MediaSupplyOrderItem, pending map[string]struct{}) error {
if len(pending) == 0 {
return nil
}
tx, err := w.pool.Begin(ctx)
if err != nil {
return err
}
defer tx.Rollback(ctx)
missing := 0
for _, item := range items {
resourceID := strings.TrimSpace(item.SupplierResourceID)
if _, ok := pending[resourceID]; !ok {
continue
}
resource, _, _, loadErr := w.service.loadResourceForOrder(ctx, tx, item.ResourceID, modelID, item.PriceType)
if loadErr != nil {
missing++
continue
}
currentCost := costForResourcePriceType(resource, item.PriceType)
if currentCost <= 0 {
missing++
continue
}
if currentCost > item.LockedSellPriceCents {
return fmt.Errorf("%w for resource %s", errMediaSupplyLockedSellBelowCost, item.SupplierResourceID)
}
}
if missing > 0 {
return fmt.Errorf("%w for %d selected resources", errMediaSupplySelectedResourcesMissing, missing)
}
return tx.Commit(ctx)
}
func (w *MediaSupplyWorker) processBacklinkSync(ctx context.Context) error {
cfg := w.service.mediaSupplyConfig()
if cfg.Worker.BacklinkSyncInterval <= 0 {
@@ -375,12 +380,8 @@ func (w *MediaSupplyWorker) processBacklinkSync(ctx context.Context) error {
AND NULLIF(TRIM(COALESCE(item.external_article_url, '')), '') IS NULL
)
)
AND (
supplier_status IS DISTINCT FROM $3
OR updated_at <= NOW() - $4::interval
)
)
`, mediaSupplySupplierMeijiequan, mediaSupplyOrderStatusSubmitted, "published", pgInterval(cfg.Worker.BacklinkSyncInterval)).Scan(&due); err != nil {
`, mediaSupplySupplierMeijiequan, mediaSupplyOrderStatusSubmitted).Scan(&due); err != nil {
return err
}
if err := tx.Commit(ctx); err != nil {
@@ -389,7 +390,18 @@ func (w *MediaSupplyWorker) processBacklinkSync(ctx context.Context) error {
if !due && cleared == 0 {
return nil
}
return w.syncPublishedBacklinks(ctx)
if allowed, err := w.claimBacklinkSyncCooldown(ctx, cfg.Worker.BacklinkSyncInterval); err != nil || !allowed {
return err
}
_, err = w.syncPublishedBacklinksNowUnthrottled(ctx, cfg)
return err
}
func (w *MediaSupplyWorker) claimBacklinkSyncCooldown(ctx context.Context, interval time.Duration) (bool, error) {
if interval <= 0 || w == nil || w.service == nil || w.service.redis == nil {
return true, nil
}
return w.service.redis.SetNX(ctx, mediaSupplyBacklinkSyncCooldownKey, time.Now().UTC().Format(time.RFC3339), interval).Result()
}
func (w *MediaSupplyWorker) clearInvalidBacklinks(ctx context.Context) (int64, error) {
@@ -410,16 +422,21 @@ func (w *MediaSupplyWorker) clearInvalidBacklinks(ctx context.Context) (int64, e
}
func (w *MediaSupplyWorker) syncPublishedBacklinks(ctx context.Context) error {
_, err := w.syncPublishedBacklinksNow(ctx)
cfg := w.service.mediaSupplyConfig()
if allowed, err := w.claimBacklinkSyncCooldown(ctx, cfg.Worker.BacklinkSyncInterval); err != nil {
return err
} else if !allowed {
return nil
}
_, err := w.syncPublishedBacklinksNowUnthrottled(ctx, cfg)
return err
}
func (w *MediaSupplyWorker) syncPublishedBacklinksNow(ctx context.Context) (*SyncMediaSupplyBacklinksResponse, error) {
func (w *MediaSupplyWorker) syncPublishedBacklinksNowUnthrottled(ctx context.Context, cfg config.MediaSupplyConfig) (*SyncMediaSupplyBacklinksResponse, error) {
result := &SyncMediaSupplyBacklinksResponse{}
if _, err := w.clearInvalidBacklinks(ctx); err != nil {
return result, err
}
cfg := w.service.mediaSupplyConfig()
userID := strings.TrimSpace(cfg.Meijiequan.UserID)
if status, statusErr := w.service.client.SessionStatus(ctx); statusErr != nil || status == nil || !status.Available {
loginStatus, loginErr := w.service.client.LoginWithConfiguredAccount(ctx)
@@ -598,7 +615,7 @@ func (w *MediaSupplyWorker) loadOrderCodeSyncCandidates(ctx context.Context, lim
WHERE supplier = $1
AND status = $2
AND deleted_at IS NULL
AND NULLIF(TRIM(COALESCE(external_order_code, '')), '') IS NULL
AND NOT (`+mediaSupplyValidExternalOrderCodeSQL+`)
ORDER BY submitted_at DESC NULLS LAST, id DESC
LIMIT $3
`, mediaSupplySupplierMeijiequan, mediaSupplyOrderStatusSubmitted, limit)
@@ -633,7 +650,7 @@ func (w *MediaSupplyWorker) loadOrderCodeSyncCandidates(ctx context.Context, lim
}
func (w *MediaSupplyWorker) updateOrderCode(ctx context.Context, order mediaSupplySubmitOrder, pending []MeijiequanPublishedArticle) (bool, error) {
if order.ExternalOrderCode != nil && strings.TrimSpace(*order.ExternalOrderCode) != "" {
if order.ExternalOrderCode != nil && isLikelyMeijiequanOrderCode(*order.ExternalOrderCode) {
return false, nil
}
items, err := w.service.listOrderItems(ctx, order.ID)
@@ -661,7 +678,7 @@ func (w *MediaSupplyWorker) updateOrderCode(ctx context.Context, order mediaSupp
END,
updated_at = NOW()
WHERE id = $1
AND NULLIF(TRIM(COALESCE(external_order_code, '')), '') IS NULL
AND NOT (`+mediaSupplyValidExternalOrderCodeSQL+`)
`, order.ID, article.OrderCode, article.OrderID, compactJSON(raw))
if err != nil {
return false, err
@@ -1185,7 +1202,7 @@ func matchMeijiequanPublishedArticles(order mediaSupplySubmitOrder, items []Medi
}
continue
}
itemID := matchMeijiequanPublishedArticleItem(orderTitle, unmatched, article)
itemID := matchMeijiequanPublishedArticleItem(orderTitle, orderCode, unmatched, article)
if itemID <= 0 {
continue
}
@@ -1205,6 +1222,7 @@ func matchMeijiequanOrderCodeArticle(order mediaSupplySubmitOrder, items []Media
return &articles[idx]
}
}
return nil
}
orderTitle := normalizeMeijiequanMatchText(order.Title)
for idx := range articles {
@@ -1212,11 +1230,8 @@ func matchMeijiequanOrderCodeArticle(order mediaSupplySubmitOrder, items []Media
if strings.TrimSpace(article.OrderCode) == "" {
continue
}
articleTitle := normalizeMeijiequanMatchText(article.Title)
if orderTitle != "" && articleTitle != "" && (strings.Contains(articleTitle, orderTitle) || strings.Contains(orderTitle, articleTitle)) {
if len(items) <= 1 || articleMatchesAnyOrderItem(*article, items) {
return article
}
if articleMatchesOrderFallback(orderTitle, *article, items) {
return article
}
}
return nil
@@ -1236,6 +1251,7 @@ func matchMeijiequanProblemArticle(order mediaSupplySubmitOrder, items []MediaSu
return article
}
}
return nil
}
orderTitle := normalizeMeijiequanMatchText(order.Title)
for idx := range articles {
@@ -1243,11 +1259,8 @@ func matchMeijiequanProblemArticle(order mediaSupplySubmitOrder, items []MediaSu
if !isMeijiequanProblemArticle(*article) {
continue
}
articleTitle := normalizeMeijiequanMatchText(article.Title)
if orderTitle != "" && articleTitle != "" && (strings.Contains(articleTitle, orderTitle) || strings.Contains(orderTitle, articleTitle)) {
if len(items) <= 1 || articleMatchesAnyOrderItem(*article, items) {
return article
}
if articleMatchesOrderFallback(orderTitle, *article, items) {
return article
}
}
return nil
@@ -1255,7 +1268,7 @@ func matchMeijiequanProblemArticle(order mediaSupplySubmitOrder, items []MediaSu
func mediaSupplyOrderCodeValue(order mediaSupplySubmitOrder) string {
if order.ExternalOrderCode != nil {
if value := strings.TrimSpace(*order.ExternalOrderCode); value != "" {
if value := strings.TrimSpace(*order.ExternalOrderCode); isLikelyMeijiequanOrderCode(value) {
return value
}
}
@@ -1343,31 +1356,69 @@ func articleMatchesAnyOrderItem(article MeijiequanPublishedArticle, items []Medi
return false
}
func matchMeijiequanPublishedArticleItem(orderTitle string, items map[int64]MediaSupplyOrderItem, article MeijiequanPublishedArticle) int64 {
resourceID := normalizeMeijiequanPublishedResourceID(article.ResourceID)
if resourceID != "" {
for itemID, item := range items {
if strings.EqualFold(strings.TrimSpace(item.SupplierResourceID), resourceID) {
return itemID
}
func articleMatchesOrderFallback(orderTitle string, article MeijiequanPublishedArticle, items []MediaSupplyOrderItem) bool {
if !articleTitleMatchesOrder(orderTitle, article) {
return false
}
if len(items) == 0 {
return false
}
for _, item := range items {
if articleMatchesOrderItemFallback(article, item) {
return true
}
}
return false
}
func articleTitleMatchesOrder(orderTitle string, article MeijiequanPublishedArticle) bool {
articleTitle := normalizeMeijiequanMatchText(article.Title)
if orderTitle != "" && articleTitle != "" && (strings.Contains(articleTitle, orderTitle) || strings.Contains(orderTitle, articleTitle)) {
if len(items) == 1 {
for itemID := range items {
return itemID
}
return orderTitle != "" && articleTitle != "" && (strings.Contains(articleTitle, orderTitle) || strings.Contains(orderTitle, articleTitle))
}
func articleMatchesOrderItemFallback(article MeijiequanPublishedArticle, item MediaSupplyOrderItem) bool {
return articleMatchesOrderItemResource(article, item) && articleMatchesOrderItemPrice(article, item)
}
func articleMatchesOrderItemResource(article MeijiequanPublishedArticle, item MediaSupplyOrderItem) bool {
resourceID := normalizeMeijiequanPublishedResourceID(article.ResourceID)
if resourceID != "" && strings.EqualFold(strings.TrimSpace(item.SupplierResourceID), resourceID) {
return true
}
resourceName := normalizeMeijiequanMatchText(article.ResourceName)
itemName := normalizeMeijiequanMatchText(item.ResourceNameSnapshot)
return resourceName != "" && itemName != "" && (strings.Contains(itemName, resourceName) || strings.Contains(resourceName, itemName))
}
func articleMatchesOrderItemPrice(article MeijiequanPublishedArticle, item MediaSupplyOrderItem) bool {
if article.PriceCents <= 0 {
return false
}
return article.PriceCents == item.LockedCostPriceCents
}
func matchMeijiequanPublishedArticleItem(orderTitle, orderCode string, items map[int64]MediaSupplyOrderItem, article MeijiequanPublishedArticle) int64 {
if orderCode != "" {
if !strings.EqualFold(strings.TrimSpace(article.OrderCode), orderCode) {
return 0
}
resourceName := normalizeMeijiequanMatchText(article.ResourceName)
if resourceName != "" {
for itemID := range items {
return itemID
}
}
if articleTitleMatchesOrder(orderTitle, article) {
if len(items) == 1 {
for itemID, item := range items {
itemName := normalizeMeijiequanMatchText(item.ResourceNameSnapshot)
if itemName != "" && (strings.Contains(itemName, resourceName) || strings.Contains(resourceName, itemName)) {
if articleMatchesOrderItemFallback(article, item) {
return itemID
}
}
}
for itemID, item := range items {
if articleMatchesOrderItemFallback(article, item) {
return itemID
}
}
}
return 0
}
@@ -1463,6 +1514,8 @@ func extractMeijiequanOrderIDFromValue(value any) (string, string) {
if code != "" && !isLikelyMeijiequanOrderCode(code) {
if extracted := firstMeijiequanOrderCode(code); extracted != "" {
code = extracted
} else {
code = ""
}
}
for _, key := range []string{"data", "result", "order", "article", "info", "message", "msg"} {
@@ -1542,4 +1595,3 @@ func (w *MediaSupplyWorker) logWarn(message string, fields ...zap.Field) {
w.logger.Warn(message, fields...)
}
}