feat(media-supply): resource cache sync and order flow refinements
Add a resource cache sync entrypoint (RunMediaResourceSyncOnce) driven by the new scheduler job, and rework the backlink/sync worker around it. Tune the meijiequan client and slow the default backlink sync interval from 10m to 30m to reduce upstream pressure. Trim unused public/transport surface (handler + router + swagger) and simplify admin-web order management and ops-web media-supply views, dropping stale shared-types fields.
This commit is contained in:
@@ -12,6 +12,8 @@ import (
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/config"
|
||||
)
|
||||
|
||||
type MediaSupplyWorker struct {
|
||||
@@ -46,7 +48,12 @@ var (
|
||||
errMediaSupplyLockedSellBelowCost = errors.New("supplier cost increased above locked sell price")
|
||||
)
|
||||
|
||||
var mediaSupplyOrderCodePattern = regexp.MustCompile(`(?i)\b[A-Z]{1,12}\d{3,}\b`)
|
||||
var mediaSupplyOrderCodePattern = regexp.MustCompile(`(?i)\bWM\d{3,}\b`)
|
||||
|
||||
const (
|
||||
mediaSupplyValidExternalOrderCodeSQL = `TRIM(COALESCE(external_order_code, '')) ~* '^WM[0-9]{3,}$'`
|
||||
mediaSupplyBacklinkSyncCooldownKey = "media_supply:backlink_sync:cooldown"
|
||||
)
|
||||
|
||||
func NewMediaSupplyWorker(service *MediaSupplyService, pool *pgxpool.Pool, logger *zap.Logger) *MediaSupplyWorker {
|
||||
return &MediaSupplyWorker{service: service, pool: pool, logger: logger}
|
||||
@@ -107,27 +114,14 @@ func (w *MediaSupplyWorker) processOneSyncJob(ctx context.Context) error {
|
||||
if err != nil || !ok {
|
||||
return err
|
||||
}
|
||||
cfg := w.service.mediaSupplyConfig()
|
||||
if status, statusErr := w.service.client.SessionStatus(ctx); statusErr != nil || status == nil || !status.Available {
|
||||
if loginStatus, loginErr := w.service.client.LoginWithConfiguredAccount(ctx); loginErr != nil {
|
||||
return w.failSyncJob(ctx, job, loginErr)
|
||||
} else if loginStatus == nil || !loginStatus.Available {
|
||||
return w.failSyncJob(ctx, job, errMeijiequanSessionMissing)
|
||||
}
|
||||
}
|
||||
modelID := 1
|
||||
if job.ModelID != nil && *job.ModelID > 0 {
|
||||
modelID = *job.ModelID
|
||||
}
|
||||
totalSynced := 0
|
||||
stats := map[string]any{"models": map[int]int{}}
|
||||
modelSynced, syncErr := w.syncModel(ctx, modelID, cfg.Meijiequan.SyncPageSize, cfg.Meijiequan.MaxSyncPages, cfg.Meijiequan.SyncPageDelay)
|
||||
stats, syncErr := w.service.RunMediaResourceSyncOnce(ctx, modelID)
|
||||
if syncErr != nil {
|
||||
return w.failSyncJob(ctx, job, syncErr)
|
||||
}
|
||||
totalSynced += modelSynced
|
||||
stats["models"].(map[int]int)[modelID] = modelSynced
|
||||
stats["synced"] = totalSynced
|
||||
raw, _ := json.Marshal(stats)
|
||||
if _, err := w.pool.Exec(ctx, `
|
||||
UPDATE media_supply_sync_jobs
|
||||
@@ -140,32 +134,8 @@ func (w *MediaSupplyWorker) processOneSyncJob(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func (w *MediaSupplyWorker) syncModel(ctx context.Context, modelID, pageSize, maxPages int, pageDelay time.Duration) (int, error) {
|
||||
total := 0
|
||||
for page := 1; page <= maxPages; page++ {
|
||||
if page > 1 && pageDelay > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return total, ctx.Err()
|
||||
case <-time.After(pageDelay):
|
||||
}
|
||||
}
|
||||
mediaPage, err := w.service.client.ListMedia(ctx, modelID, page, pageSize)
|
||||
if err != nil {
|
||||
return total, err
|
||||
}
|
||||
count, err := w.service.UpsertSyncedResources(ctx, mediaPage.Items)
|
||||
if err != nil {
|
||||
return total, err
|
||||
}
|
||||
total += count
|
||||
if len(mediaPage.Items) == 0 {
|
||||
break
|
||||
}
|
||||
if mediaPage.AllPage > 0 && page >= mediaPage.AllPage {
|
||||
break
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
stats, err := w.service.syncMediaResourceModel(ctx, modelID, pageSize, maxPages, pageDelay)
|
||||
return stats.Synced, err
|
||||
}
|
||||
|
||||
func (w *MediaSupplyWorker) processOneOrder(ctx context.Context) error {
|
||||
@@ -331,11 +301,46 @@ func (w *MediaSupplyWorker) refreshSelectedResourcesFromUpstream(ctx context.Con
|
||||
}
|
||||
}
|
||||
if len(pending) > 0 {
|
||||
return fmt.Errorf("%w for %d selected resources", errMediaSupplySelectedResourcesMissing, len(pending))
|
||||
return w.validatePendingResourcesFromCache(ctx, modelID, items, pending)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *MediaSupplyWorker) validatePendingResourcesFromCache(ctx context.Context, modelID int, items []MediaSupplyOrderItem, pending map[string]struct{}) error {
|
||||
if len(pending) == 0 {
|
||||
return nil
|
||||
}
|
||||
tx, err := w.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
missing := 0
|
||||
for _, item := range items {
|
||||
resourceID := strings.TrimSpace(item.SupplierResourceID)
|
||||
if _, ok := pending[resourceID]; !ok {
|
||||
continue
|
||||
}
|
||||
resource, _, _, loadErr := w.service.loadResourceForOrder(ctx, tx, item.ResourceID, modelID, item.PriceType)
|
||||
if loadErr != nil {
|
||||
missing++
|
||||
continue
|
||||
}
|
||||
currentCost := costForResourcePriceType(resource, item.PriceType)
|
||||
if currentCost <= 0 {
|
||||
missing++
|
||||
continue
|
||||
}
|
||||
if currentCost > item.LockedSellPriceCents {
|
||||
return fmt.Errorf("%w for resource %s", errMediaSupplyLockedSellBelowCost, item.SupplierResourceID)
|
||||
}
|
||||
}
|
||||
if missing > 0 {
|
||||
return fmt.Errorf("%w for %d selected resources", errMediaSupplySelectedResourcesMissing, missing)
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (w *MediaSupplyWorker) processBacklinkSync(ctx context.Context) error {
|
||||
cfg := w.service.mediaSupplyConfig()
|
||||
if cfg.Worker.BacklinkSyncInterval <= 0 {
|
||||
@@ -375,12 +380,8 @@ func (w *MediaSupplyWorker) processBacklinkSync(ctx context.Context) error {
|
||||
AND NULLIF(TRIM(COALESCE(item.external_article_url, '')), '') IS NULL
|
||||
)
|
||||
)
|
||||
AND (
|
||||
supplier_status IS DISTINCT FROM $3
|
||||
OR updated_at <= NOW() - $4::interval
|
||||
)
|
||||
)
|
||||
`, mediaSupplySupplierMeijiequan, mediaSupplyOrderStatusSubmitted, "published", pgInterval(cfg.Worker.BacklinkSyncInterval)).Scan(&due); err != nil {
|
||||
`, mediaSupplySupplierMeijiequan, mediaSupplyOrderStatusSubmitted).Scan(&due); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
@@ -389,7 +390,18 @@ func (w *MediaSupplyWorker) processBacklinkSync(ctx context.Context) error {
|
||||
if !due && cleared == 0 {
|
||||
return nil
|
||||
}
|
||||
return w.syncPublishedBacklinks(ctx)
|
||||
if allowed, err := w.claimBacklinkSyncCooldown(ctx, cfg.Worker.BacklinkSyncInterval); err != nil || !allowed {
|
||||
return err
|
||||
}
|
||||
_, err = w.syncPublishedBacklinksNowUnthrottled(ctx, cfg)
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *MediaSupplyWorker) claimBacklinkSyncCooldown(ctx context.Context, interval time.Duration) (bool, error) {
|
||||
if interval <= 0 || w == nil || w.service == nil || w.service.redis == nil {
|
||||
return true, nil
|
||||
}
|
||||
return w.service.redis.SetNX(ctx, mediaSupplyBacklinkSyncCooldownKey, time.Now().UTC().Format(time.RFC3339), interval).Result()
|
||||
}
|
||||
|
||||
func (w *MediaSupplyWorker) clearInvalidBacklinks(ctx context.Context) (int64, error) {
|
||||
@@ -410,16 +422,21 @@ func (w *MediaSupplyWorker) clearInvalidBacklinks(ctx context.Context) (int64, e
|
||||
}
|
||||
|
||||
func (w *MediaSupplyWorker) syncPublishedBacklinks(ctx context.Context) error {
|
||||
_, err := w.syncPublishedBacklinksNow(ctx)
|
||||
cfg := w.service.mediaSupplyConfig()
|
||||
if allowed, err := w.claimBacklinkSyncCooldown(ctx, cfg.Worker.BacklinkSyncInterval); err != nil {
|
||||
return err
|
||||
} else if !allowed {
|
||||
return nil
|
||||
}
|
||||
_, err := w.syncPublishedBacklinksNowUnthrottled(ctx, cfg)
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *MediaSupplyWorker) syncPublishedBacklinksNow(ctx context.Context) (*SyncMediaSupplyBacklinksResponse, error) {
|
||||
func (w *MediaSupplyWorker) syncPublishedBacklinksNowUnthrottled(ctx context.Context, cfg config.MediaSupplyConfig) (*SyncMediaSupplyBacklinksResponse, error) {
|
||||
result := &SyncMediaSupplyBacklinksResponse{}
|
||||
if _, err := w.clearInvalidBacklinks(ctx); err != nil {
|
||||
return result, err
|
||||
}
|
||||
cfg := w.service.mediaSupplyConfig()
|
||||
userID := strings.TrimSpace(cfg.Meijiequan.UserID)
|
||||
if status, statusErr := w.service.client.SessionStatus(ctx); statusErr != nil || status == nil || !status.Available {
|
||||
loginStatus, loginErr := w.service.client.LoginWithConfiguredAccount(ctx)
|
||||
@@ -598,7 +615,7 @@ func (w *MediaSupplyWorker) loadOrderCodeSyncCandidates(ctx context.Context, lim
|
||||
WHERE supplier = $1
|
||||
AND status = $2
|
||||
AND deleted_at IS NULL
|
||||
AND NULLIF(TRIM(COALESCE(external_order_code, '')), '') IS NULL
|
||||
AND NOT (`+mediaSupplyValidExternalOrderCodeSQL+`)
|
||||
ORDER BY submitted_at DESC NULLS LAST, id DESC
|
||||
LIMIT $3
|
||||
`, mediaSupplySupplierMeijiequan, mediaSupplyOrderStatusSubmitted, limit)
|
||||
@@ -633,7 +650,7 @@ func (w *MediaSupplyWorker) loadOrderCodeSyncCandidates(ctx context.Context, lim
|
||||
}
|
||||
|
||||
func (w *MediaSupplyWorker) updateOrderCode(ctx context.Context, order mediaSupplySubmitOrder, pending []MeijiequanPublishedArticle) (bool, error) {
|
||||
if order.ExternalOrderCode != nil && strings.TrimSpace(*order.ExternalOrderCode) != "" {
|
||||
if order.ExternalOrderCode != nil && isLikelyMeijiequanOrderCode(*order.ExternalOrderCode) {
|
||||
return false, nil
|
||||
}
|
||||
items, err := w.service.listOrderItems(ctx, order.ID)
|
||||
@@ -661,7 +678,7 @@ func (w *MediaSupplyWorker) updateOrderCode(ctx context.Context, order mediaSupp
|
||||
END,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
AND NULLIF(TRIM(COALESCE(external_order_code, '')), '') IS NULL
|
||||
AND NOT (`+mediaSupplyValidExternalOrderCodeSQL+`)
|
||||
`, order.ID, article.OrderCode, article.OrderID, compactJSON(raw))
|
||||
if err != nil {
|
||||
return false, err
|
||||
@@ -1185,7 +1202,7 @@ func matchMeijiequanPublishedArticles(order mediaSupplySubmitOrder, items []Medi
|
||||
}
|
||||
continue
|
||||
}
|
||||
itemID := matchMeijiequanPublishedArticleItem(orderTitle, unmatched, article)
|
||||
itemID := matchMeijiequanPublishedArticleItem(orderTitle, orderCode, unmatched, article)
|
||||
if itemID <= 0 {
|
||||
continue
|
||||
}
|
||||
@@ -1205,6 +1222,7 @@ func matchMeijiequanOrderCodeArticle(order mediaSupplySubmitOrder, items []Media
|
||||
return &articles[idx]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
orderTitle := normalizeMeijiequanMatchText(order.Title)
|
||||
for idx := range articles {
|
||||
@@ -1212,11 +1230,8 @@ func matchMeijiequanOrderCodeArticle(order mediaSupplySubmitOrder, items []Media
|
||||
if strings.TrimSpace(article.OrderCode) == "" {
|
||||
continue
|
||||
}
|
||||
articleTitle := normalizeMeijiequanMatchText(article.Title)
|
||||
if orderTitle != "" && articleTitle != "" && (strings.Contains(articleTitle, orderTitle) || strings.Contains(orderTitle, articleTitle)) {
|
||||
if len(items) <= 1 || articleMatchesAnyOrderItem(*article, items) {
|
||||
return article
|
||||
}
|
||||
if articleMatchesOrderFallback(orderTitle, *article, items) {
|
||||
return article
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -1236,6 +1251,7 @@ func matchMeijiequanProblemArticle(order mediaSupplySubmitOrder, items []MediaSu
|
||||
return article
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
orderTitle := normalizeMeijiequanMatchText(order.Title)
|
||||
for idx := range articles {
|
||||
@@ -1243,11 +1259,8 @@ func matchMeijiequanProblemArticle(order mediaSupplySubmitOrder, items []MediaSu
|
||||
if !isMeijiequanProblemArticle(*article) {
|
||||
continue
|
||||
}
|
||||
articleTitle := normalizeMeijiequanMatchText(article.Title)
|
||||
if orderTitle != "" && articleTitle != "" && (strings.Contains(articleTitle, orderTitle) || strings.Contains(orderTitle, articleTitle)) {
|
||||
if len(items) <= 1 || articleMatchesAnyOrderItem(*article, items) {
|
||||
return article
|
||||
}
|
||||
if articleMatchesOrderFallback(orderTitle, *article, items) {
|
||||
return article
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -1255,7 +1268,7 @@ func matchMeijiequanProblemArticle(order mediaSupplySubmitOrder, items []MediaSu
|
||||
|
||||
func mediaSupplyOrderCodeValue(order mediaSupplySubmitOrder) string {
|
||||
if order.ExternalOrderCode != nil {
|
||||
if value := strings.TrimSpace(*order.ExternalOrderCode); value != "" {
|
||||
if value := strings.TrimSpace(*order.ExternalOrderCode); isLikelyMeijiequanOrderCode(value) {
|
||||
return value
|
||||
}
|
||||
}
|
||||
@@ -1343,31 +1356,69 @@ func articleMatchesAnyOrderItem(article MeijiequanPublishedArticle, items []Medi
|
||||
return false
|
||||
}
|
||||
|
||||
func matchMeijiequanPublishedArticleItem(orderTitle string, items map[int64]MediaSupplyOrderItem, article MeijiequanPublishedArticle) int64 {
|
||||
resourceID := normalizeMeijiequanPublishedResourceID(article.ResourceID)
|
||||
if resourceID != "" {
|
||||
for itemID, item := range items {
|
||||
if strings.EqualFold(strings.TrimSpace(item.SupplierResourceID), resourceID) {
|
||||
return itemID
|
||||
}
|
||||
func articleMatchesOrderFallback(orderTitle string, article MeijiequanPublishedArticle, items []MediaSupplyOrderItem) bool {
|
||||
if !articleTitleMatchesOrder(orderTitle, article) {
|
||||
return false
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, item := range items {
|
||||
if articleMatchesOrderItemFallback(article, item) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func articleTitleMatchesOrder(orderTitle string, article MeijiequanPublishedArticle) bool {
|
||||
articleTitle := normalizeMeijiequanMatchText(article.Title)
|
||||
if orderTitle != "" && articleTitle != "" && (strings.Contains(articleTitle, orderTitle) || strings.Contains(orderTitle, articleTitle)) {
|
||||
if len(items) == 1 {
|
||||
for itemID := range items {
|
||||
return itemID
|
||||
}
|
||||
return orderTitle != "" && articleTitle != "" && (strings.Contains(articleTitle, orderTitle) || strings.Contains(orderTitle, articleTitle))
|
||||
}
|
||||
|
||||
func articleMatchesOrderItemFallback(article MeijiequanPublishedArticle, item MediaSupplyOrderItem) bool {
|
||||
return articleMatchesOrderItemResource(article, item) && articleMatchesOrderItemPrice(article, item)
|
||||
}
|
||||
|
||||
func articleMatchesOrderItemResource(article MeijiequanPublishedArticle, item MediaSupplyOrderItem) bool {
|
||||
resourceID := normalizeMeijiequanPublishedResourceID(article.ResourceID)
|
||||
if resourceID != "" && strings.EqualFold(strings.TrimSpace(item.SupplierResourceID), resourceID) {
|
||||
return true
|
||||
}
|
||||
resourceName := normalizeMeijiequanMatchText(article.ResourceName)
|
||||
itemName := normalizeMeijiequanMatchText(item.ResourceNameSnapshot)
|
||||
return resourceName != "" && itemName != "" && (strings.Contains(itemName, resourceName) || strings.Contains(resourceName, itemName))
|
||||
}
|
||||
|
||||
func articleMatchesOrderItemPrice(article MeijiequanPublishedArticle, item MediaSupplyOrderItem) bool {
|
||||
if article.PriceCents <= 0 {
|
||||
return false
|
||||
}
|
||||
return article.PriceCents == item.LockedCostPriceCents
|
||||
}
|
||||
|
||||
func matchMeijiequanPublishedArticleItem(orderTitle, orderCode string, items map[int64]MediaSupplyOrderItem, article MeijiequanPublishedArticle) int64 {
|
||||
if orderCode != "" {
|
||||
if !strings.EqualFold(strings.TrimSpace(article.OrderCode), orderCode) {
|
||||
return 0
|
||||
}
|
||||
resourceName := normalizeMeijiequanMatchText(article.ResourceName)
|
||||
if resourceName != "" {
|
||||
for itemID := range items {
|
||||
return itemID
|
||||
}
|
||||
}
|
||||
if articleTitleMatchesOrder(orderTitle, article) {
|
||||
if len(items) == 1 {
|
||||
for itemID, item := range items {
|
||||
itemName := normalizeMeijiequanMatchText(item.ResourceNameSnapshot)
|
||||
if itemName != "" && (strings.Contains(itemName, resourceName) || strings.Contains(resourceName, itemName)) {
|
||||
if articleMatchesOrderItemFallback(article, item) {
|
||||
return itemID
|
||||
}
|
||||
}
|
||||
}
|
||||
for itemID, item := range items {
|
||||
if articleMatchesOrderItemFallback(article, item) {
|
||||
return itemID
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -1463,6 +1514,8 @@ func extractMeijiequanOrderIDFromValue(value any) (string, string) {
|
||||
if code != "" && !isLikelyMeijiequanOrderCode(code) {
|
||||
if extracted := firstMeijiequanOrderCode(code); extracted != "" {
|
||||
code = extracted
|
||||
} else {
|
||||
code = ""
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"data", "result", "order", "article", "info", "message", "msg"} {
|
||||
@@ -1542,4 +1595,3 @@ func (w *MediaSupplyWorker) logWarn(message string, fields ...zap.Field) {
|
||||
w.logger.Warn(message, fields...)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user