fix: allow retry after definitive publish failures
Frontend CI / Frontend (push) Successful in 3m18s
Backend CI / Backend (push) Successful in 16m16s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 26s
Desktop Client Build / Resolve Build Metadata (push) Successful in 26s
Desktop Client Build / Build Desktop Client (push) Successful in 25m45s
Frontend CI / Frontend (push) Successful in 3m18s
Backend CI / Backend (push) Successful in 16m16s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 26s
Desktop Client Build / Resolve Build Metadata (push) Successful in 26s
Desktop Client Build / Build Desktop Client (push) Successful in 25m45s
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -218,6 +219,14 @@ func (s *PublishJobService) Create(ctx context.Context, actor auth.Actor, req Cr
|
||||
if err := lockDesktopPublishDedupKeys(ctx, tx, dedupKeys); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
releasedPublishOutcomes := make([]*desktopPublishSyncOutcome, 0)
|
||||
if len(dedupKeys) > 0 {
|
||||
outcomes, releaseErr := releaseDefinitiveFailedUnknownPublishTasks(ctx, tx, actor.TenantID, actor.PrimaryWorkspaceID, dedupKeys)
|
||||
if releaseErr != nil {
|
||||
return nil, releaseErr
|
||||
}
|
||||
releasedPublishOutcomes = append(releasedPublishOutcomes, outcomes...)
|
||||
}
|
||||
existingTasks, err := loadExistingPublishRecordTargets(ctx, tx, actor.TenantID, actor.PrimaryWorkspaceID, articleID, targets)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -338,6 +347,11 @@ func (s *PublishJobService) Create(ctx context.Context, actor auth.Actor, req Cr
|
||||
return nil, response.ErrInternal(50100, "desktop_publish_job_commit_failed", "failed to commit desktop publish job")
|
||||
}
|
||||
invalidateArticleCaches(ctx, s.cache, actor.TenantID, &articleID)
|
||||
for _, outcome := range releasedPublishOutcomes {
|
||||
if outcome != nil {
|
||||
invalidateArticleCaches(ctx, s.cache, outcome.TenantID, &outcome.ArticleID)
|
||||
}
|
||||
}
|
||||
|
||||
for _, task := range createdTasks {
|
||||
dispatchEvent := DesktopDispatchEventFromTask(task, "task_available")
|
||||
@@ -514,6 +528,155 @@ func lockDesktopPublishDedupKeys(ctx context.Context, tx pgx.Tx, keys []string)
|
||||
return nil
|
||||
}
|
||||
|
||||
func releaseDefinitiveFailedUnknownPublishTasks(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
tenantID int64,
|
||||
workspaceID int64,
|
||||
dedupKeys []string,
|
||||
) ([]*desktopPublishSyncOutcome, error) {
|
||||
if len(dedupKeys) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT desktop_id, tenant_id, workspace_id, platform_id, kind, payload, status, result, error
|
||||
FROM desktop_tasks
|
||||
WHERE tenant_id = $1
|
||||
AND workspace_id = $2
|
||||
AND kind = 'publish'
|
||||
AND status = 'unknown'
|
||||
AND dedup_key = ANY($3::text[])
|
||||
FOR UPDATE
|
||||
`, tenantID, workspaceID, dedupKeys)
|
||||
if err != nil {
|
||||
return nil, response.ErrInternal(50131, "desktop_publish_dedup_cleanup_failed", "failed to inspect stale publish deduplication tasks")
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
tasks := make([]repository.DesktopTask, 0)
|
||||
for rows.Next() {
|
||||
var task repository.DesktopTask
|
||||
if scanErr := rows.Scan(
|
||||
&task.DesktopID,
|
||||
&task.TenantID,
|
||||
&task.WorkspaceID,
|
||||
&task.Platform,
|
||||
&task.Kind,
|
||||
&task.Payload,
|
||||
&task.Status,
|
||||
&task.Result,
|
||||
&task.Error,
|
||||
); scanErr != nil {
|
||||
return nil, response.ErrInternal(50131, "desktop_publish_dedup_cleanup_failed", "failed to parse stale publish deduplication task")
|
||||
}
|
||||
tasks = append(tasks, task)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, response.ErrInternal(50131, "desktop_publish_dedup_cleanup_failed", "failed to iterate stale publish deduplication tasks")
|
||||
}
|
||||
|
||||
outcomes := make([]*desktopPublishSyncOutcome, 0)
|
||||
for index := range tasks {
|
||||
task := tasks[index]
|
||||
if !publishUnknownTaskHasDefinitiveFailure(&task) {
|
||||
continue
|
||||
}
|
||||
updated, updateErr := markUnknownPublishTaskDefinitiveFailed(ctx, tx, &task)
|
||||
if updateErr != nil {
|
||||
return nil, updateErr
|
||||
}
|
||||
outcome, syncErr := syncDesktopPublishTaskState(ctx, tx, updated)
|
||||
if syncErr != nil {
|
||||
return nil, syncErr
|
||||
}
|
||||
if outcome != nil {
|
||||
outcomes = append(outcomes, outcome)
|
||||
}
|
||||
}
|
||||
return outcomes, nil
|
||||
}
|
||||
|
||||
func markUnknownPublishTaskDefinitiveFailed(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
task *repository.DesktopTask,
|
||||
) (*repository.DesktopTask, error) {
|
||||
if task == nil {
|
||||
return nil, response.ErrNotFound(40482, "desktop_task_not_found", "desktop task not found")
|
||||
}
|
||||
row := tx.QueryRow(ctx, `
|
||||
UPDATE desktop_tasks
|
||||
SET status = 'failed',
|
||||
active_attempt_id = NULL,
|
||||
lease_token_hash = NULL,
|
||||
lease_expires_at = NULL,
|
||||
publish_submit_started_at = NULL,
|
||||
updated_at = NOW()
|
||||
WHERE desktop_id = $1
|
||||
AND workspace_id = $2
|
||||
AND status = 'unknown'
|
||||
RETURNING desktop_id, tenant_id, workspace_id, platform_id, kind, payload, status, result, error
|
||||
`, task.DesktopID, task.WorkspaceID)
|
||||
var updated repository.DesktopTask
|
||||
if err := row.Scan(
|
||||
&updated.DesktopID,
|
||||
&updated.TenantID,
|
||||
&updated.WorkspaceID,
|
||||
&updated.Platform,
|
||||
&updated.Kind,
|
||||
&updated.Payload,
|
||||
&updated.Status,
|
||||
&updated.Result,
|
||||
&updated.Error,
|
||||
); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, response.ErrConflict(40985, "desktop_task_reconcile_conflict", "desktop task is not waiting for reconcile")
|
||||
}
|
||||
return nil, response.ErrInternal(50131, "desktop_publish_dedup_cleanup_failed", "failed to release stale publish deduplication task")
|
||||
}
|
||||
return &updated, nil
|
||||
}
|
||||
|
||||
func publishUnknownTaskHasDefinitiveFailure(task *repository.DesktopTask) bool {
|
||||
if task == nil || task.Kind != "publish" || task.Status != "unknown" {
|
||||
return false
|
||||
}
|
||||
errorPayload := unmarshalJSONObject(task.Error)
|
||||
if boolValueFromMap(errorPayload, "publish_submit_uncertain") {
|
||||
return false
|
||||
}
|
||||
combined := strings.ToLower(strings.Join([]string{
|
||||
stringPointerValue(extractStringPointer(errorPayload, "code", "error_code")),
|
||||
stringPointerValue(extractStringPointer(errorPayload, "message", "error_msg", "detail")),
|
||||
}, " "))
|
||||
if combined == "" {
|
||||
return false
|
||||
}
|
||||
return definitivePublishFailurePattern.MatchString(combined)
|
||||
}
|
||||
|
||||
func boolValueFromMap(payload map[string]any, key string) bool {
|
||||
if payload == nil {
|
||||
return false
|
||||
}
|
||||
value, ok := payload[key]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case bool:
|
||||
return typed
|
||||
case string:
|
||||
normalized := strings.TrimSpace(strings.ToLower(typed))
|
||||
return normalized == "true" || normalized == "1" || normalized == "yes"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
var definitivePublishFailurePattern = regexp.MustCompile(`csrf|forbidden|\b403\b|not_logged_in|login_required|token_missing|cookie_missing|challenge_required|risk_control|captcha|验证码|滑块|人机验证|安全验证|风控|风险|参数错误|内容为空|image_fetch_failed|cover_fetch_failed|upload_failed|signature_failed`)
|
||||
|
||||
func loadExistingPublishRecordTargets(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
@@ -567,7 +730,7 @@ func loadExistingPublishRecordTargets(
|
||||
AND pr.article_id = $3
|
||||
AND pr.platform_account_id = ANY($4::bigint[])
|
||||
AND pr.deleted_at IS NULL
|
||||
AND pr.status IN ('queued', 'publishing', 'success')
|
||||
AND (pr.status IN ('queued', 'publishing', 'success') OR dt.status = 'unknown')
|
||||
AND dt.status IN ('queued', 'in_progress', 'succeeded', 'unknown')
|
||||
ORDER BY pr.platform_account_id,
|
||||
CASE
|
||||
@@ -785,10 +948,18 @@ func (s *PublishJobService) requeueUnknownPublishTask(
|
||||
return nil, response.ErrInternal(50094, "desktop_task_reconcile_failed", "failed to reconcile desktop task")
|
||||
}
|
||||
|
||||
publishOutcome, err := syncDesktopPublishTaskState(ctx, tx, requeued)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, response.ErrInternal(50106, "desktop_task_reconcile_commit_failed", "failed to commit desktop task reconcile")
|
||||
}
|
||||
|
||||
if publishOutcome != nil {
|
||||
invalidateArticleCaches(ctx, s.cache, publishOutcome.TenantID, &publishOutcome.ArticleID)
|
||||
}
|
||||
publishDesktopTaskLifecycleEvent(ctx, s.messaging, s.logger, requeued, "task_available")
|
||||
return createPublishJobResponseForRequeuedTask(requeued), nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user