feat(monitoring): scope authorization-failure cancellation to platform across accounts

When a monitor task's final account hits human verification or an
authorization failure, cancel the remaining queued desktop/monitoring
tasks for the whole platform instead of only the same account. Follow-up
tasks are now marked skipped/aborted (not failed) with a
platform_human_verification / platform_authorization_unavailable reason
and a user-facing cancellation message, and emit task_canceled lifecycle
events for monitor kinds.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-14 22:00:41 +08:00
parent b0bf9b4299
commit 1fab7b1e5f
4 changed files with 196 additions and 31 deletions
@@ -36,6 +36,39 @@ type queuedDesktopAuthorizationFailureCandidate struct {
MonitorTaskID *int64 MonitorTaskID *int64
} }
const (
monitoringPlatformHumanVerificationSkipReason = "platform_human_verification"
monitoringPlatformAuthorizationSkipReason = "platform_authorization_unavailable"
)
func authorizationFailureFollowupCancellation(code, message string) (string, string) {
normalized := strings.ToLower(strings.TrimSpace(code + " " + message))
if strings.Contains(normalized, "challenge") ||
strings.Contains(normalized, "captcha") ||
strings.Contains(normalized, "risk_control") ||
strings.Contains(normalized, "人机验证") ||
strings.Contains(normalized, "人工验证") ||
strings.Contains(normalized, "验证码") ||
strings.Contains(normalized, "风控") {
return monitoringPlatformHumanVerificationSkipReason, "该模型没有其他可用账号,因触发人机验证,后续采集任务已自动取消。"
}
return monitoringPlatformAuthorizationSkipReason, "该模型当前没有可用账号,后续采集任务已自动取消。"
}
func desktopAuthorizationFollowupTaskStatus(kind string) string {
if strings.TrimSpace(kind) == "monitor" {
return "aborted"
}
return "failed"
}
func desktopAuthorizationFollowupEventType(task *repository.DesktopTask) string {
if task != nil && task.Kind == "monitor" && task.Status == "aborted" {
return "task_canceled"
}
return "task_completed"
}
func bulkFailQueuedDesktopTasksForAuthorizationFailure( func bulkFailQueuedDesktopTasksForAuthorizationFailure(
ctx context.Context, ctx context.Context,
tx pgx.Tx, tx pgx.Tx,
@@ -62,13 +95,27 @@ func bulkFailQueuedDesktopTasksForAuthorizationFailure(
payload := ensureDesktopAuthorizationFailurePayload(errorPayload, disposition, task, candidate.MonitorTaskID) payload := ensureDesktopAuthorizationFailurePayload(errorPayload, disposition, task, candidate.MonitorTaskID)
payload["blocked_by_authorization_failure_task_id"] = task.DesktopID.String() payload["blocked_by_authorization_failure_task_id"] = task.DesktopID.String()
payload["blocked_by_authorization_failure_platform"] = task.Platform payload["blocked_by_authorization_failure_platform"] = task.Platform
if task.Kind == "monitor" {
skipReason, cancellationMessage := authorizationFailureFollowupCancellation(disposition.Code, disposition.Message)
payload["source_authorization_failure_code"] = payload["code"]
payload["code"] = "desktop_monitor_platform_authorization_canceled"
payload["message"] = cancellationMessage
payload["canceled"] = true
payload["cancellation_reason"] = skipReason
}
candidateErrorJSON, marshalErr := json.Marshal(payload) candidateErrorJSON, marshalErr := json.Marshal(payload)
if marshalErr != nil { if marshalErr != nil {
return nil, response.ErrInternal(50122, "desktop_task_authorization_failure_payload_failed", "failed to encode non-retryable authorization failure") return nil, response.ErrInternal(50122, "desktop_task_authorization_failure_payload_failed", "failed to encode non-retryable authorization failure")
} }
updated, updateErr := failQueuedDesktopTaskForAuthorizationFailure(ctx, tx, candidate.DesktopID, candidateErrorJSON) updated, updateErr := finishQueuedDesktopTaskForAuthorizationFailure(
ctx,
tx,
candidate.DesktopID,
desktopAuthorizationFollowupTaskStatus(task.Kind),
candidateErrorJSON,
)
if updateErr != nil { if updateErr != nil {
return nil, updateErr return nil, updateErr
} }
@@ -107,7 +154,7 @@ func loadQueuedDesktopAuthorizationFailureCandidates(
AND kind = $3 AND kind = $3
AND status = 'queued' AND status = 'queued'
AND target_client_id = $4 AND target_client_id = $4
AND target_account_id = $5 AND (kind = 'monitor' OR target_account_id = $5)
AND platform_id = $6 AND platform_id = $6
AND desktop_id <> $7 AND desktop_id <> $7
ORDER BY COALESCE(enqueued_at, created_at) ASC, created_at ASC, desktop_id ASC ORDER BY COALESCE(enqueued_at, created_at) ASC, created_at ASC, desktop_id ASC
@@ -139,25 +186,28 @@ func loadQueuedDesktopAuthorizationFailureCandidates(
return candidates, nil return candidates, nil
} }
func failQueuedDesktopTaskForAuthorizationFailure( func finishQueuedDesktopTaskForAuthorizationFailure(
ctx context.Context, ctx context.Context,
tx pgx.Tx, tx pgx.Tx,
desktopID uuid.UUID, desktopID uuid.UUID,
status string,
errorJSON []byte, errorJSON []byte,
) (*repository.DesktopTask, error) { ) (*repository.DesktopTask, error) {
row := tx.QueryRow(ctx, ` row := tx.QueryRow(ctx, `
UPDATE desktop_tasks AS t UPDATE desktop_tasks AS t
SET status = 'failed', SET status = $2,
result = NULL, result = NULL,
error = $2, error = $3,
active_attempt_id = NULL, active_attempt_id = NULL,
lease_token_hash = NULL, lease_token_hash = NULL,
lease_expires_at = NULL, lease_expires_at = NULL,
completed_at = NOW(),
updated_at = NOW() updated_at = NOW()
WHERE desktop_id = $1 WHERE desktop_id = $1
AND status = 'queued' AND status = 'queued'
RETURNING `+desktopTaskRepositoryReturningColumns, RETURNING `+desktopTaskRepositoryReturningColumns,
desktopID, desktopID,
status,
errorJSON, errorJSON,
) )
updated, err := scanRepositoryDesktopTask(row) updated, err := scanRepositoryDesktopTask(row)
@@ -165,7 +215,7 @@ func failQueuedDesktopTaskForAuthorizationFailure(
if err == pgx.ErrNoRows { if err == pgx.ErrNoRows {
return nil, nil return nil, nil
} }
return nil, response.ErrInternal(50123, "desktop_task_authorization_failure_update_failed", "failed to fail queued desktop task after authorization failure") return nil, response.ErrInternal(50123, "desktop_task_authorization_failure_update_failed", "failed to finish queued desktop task after authorization failure")
} }
return updated, nil return updated, nil
} }
@@ -311,9 +361,14 @@ func failMonitoringCollectTasksForDesktopAuthorizationFailure(
} }
errorMessage := desktopAuthorizationFailureString(errorPayload["message"]) errorMessage := desktopAuthorizationFailureString(errorPayload["message"])
if errorMessage == "" { skipReason, cancellationMessage := authorizationFailureFollowupCancellation(
errorMessage = "登录态已失效,任务未执行。" desktopAuthorizationFailureString(errorPayload["code"]),
} errorMessage,
)
errorPayload["canceled"] = true
errorPayload["cancellation_reason"] = skipReason
errorPayload["source_authorization_failure_message"] = errorMessage
errorPayload["message"] = cancellationMessage
requestPayloadJSON, err := json.Marshal(map[string]any{ requestPayloadJSON, err := json.Marshal(map[string]any{
"runtime_error": errorPayload, "runtime_error": errorPayload,
"blocked_by_authorization_failure": true, "blocked_by_authorization_failure": true,
@@ -324,28 +379,28 @@ func failMonitoringCollectTasksForDesktopAuthorizationFailure(
if _, err := pool.Exec(ctx, ` if _, err := pool.Exec(ctx, `
UPDATE monitoring_collect_tasks UPDATE monitoring_collect_tasks
SET status = 'failed', SET status = 'skipped',
lease_token_hash = NULL, lease_token_hash = NULL,
leased_to_executor = NULL, leased_to_executor = NULL,
leased_at = NULL, leased_at = NULL,
lease_expires_at = NULL, lease_expires_at = NULL,
callback_received_at = NOW(), callback_received_at = NOW(),
completed_at = NOW(), completed_at = NOW(),
skip_reason = NULL, skip_reason = $2,
error_message = $2, error_message = $3,
request_payload_json = ( request_payload_json = (
CASE CASE
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object' WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
THEN '{}'::jsonb THEN '{}'::jsonb
ELSE request_payload_json ELSE request_payload_json
END END
) || $3::jsonb, ) || $4::jsonb,
updated_at = NOW() updated_at = NOW()
WHERE id = ANY($1::bigint[]) WHERE id = ANY($1::bigint[])
AND callback_received_at IS NULL AND callback_received_at IS NULL
AND status IN ('pending', 'leased', 'expired') AND status IN ('pending', 'leased', 'expired')
`, monitorTaskIDs, errorMessage, requestPayloadJSON); err != nil { `, monitorTaskIDs, skipReason, cancellationMessage, requestPayloadJSON); err != nil {
return response.ErrInternal(50125, "monitoring_authorization_failure_update_failed", "failed to fail monitoring tasks after authorization failure") return response.ErrInternal(50125, "monitoring_authorization_failure_update_failed", "failed to skip monitoring tasks after authorization failure")
} }
return nil return nil
} }
@@ -1269,7 +1269,7 @@ func (s *DesktopTaskService) Complete(ctx context.Context, client *repository.De
s.publishTaskEvent(ctx, task, "task_completed") s.publishTaskEvent(ctx, task, "task_completed")
for _, failedTask := range bulkFailure.Tasks { for _, failedTask := range bulkFailure.Tasks {
s.publishTaskEvent(ctx, failedTask, "task_completed") s.publishTaskEvent(ctx, failedTask, desktopAuthorizationFollowupEventType(failedTask))
} }
view := buildDesktopTaskView(task) view := buildDesktopTaskView(task)
@@ -2683,7 +2683,7 @@ func (s *MonitoringCallbackService) completeLinkedDesktopMonitorTask(
publishDesktopTaskLifecycleEvent(ctx, s.rabbitMQ, s.logger, finalizedTask, "task_completed") publishDesktopTaskLifecycleEvent(ctx, s.rabbitMQ, s.logger, finalizedTask, "task_completed")
for _, failedTask := range bulkFailure.Tasks { for _, failedTask := range bulkFailure.Tasks {
publishDesktopTaskLifecycleEvent(ctx, s.rabbitMQ, s.logger, failedTask, "task_completed") publishDesktopTaskLifecycleEvent(ctx, s.rabbitMQ, s.logger, failedTask, desktopAuthorizationFollowupEventType(failedTask))
} }
return nil return nil
@@ -3260,13 +3260,6 @@ func (s *MonitoringCallbackService) failRemainingMonitoringTasksForAuthorization
runtimeError := buildDesktopMonitorTaskErrorValue(task, req) runtimeError := buildDesktopMonitorTaskErrorValue(task, req)
runtimeError["blocked_by_authorization_failure_task_id"] = task.ID runtimeError["blocked_by_authorization_failure_task_id"] = task.ID
runtimeError["blocked_by_authorization_failure_platform"] = task.AIPlatformID runtimeError["blocked_by_authorization_failure_platform"] = task.AIPlatformID
requestPayloadJSON, err := json.Marshal(map[string]any{
"runtime_error": runtimeError,
"blocked_by_authorization_failure": true,
})
if err != nil {
return 0, response.ErrInternal(50041, "task_payload_encode_failed", "failed to encode monitoring authorization failure payload")
}
message := strings.TrimSpace(optionalStringValue(req.ErrorMessage)) message := strings.TrimSpace(optionalStringValue(req.ErrorMessage))
if message == "" { if message == "" {
@@ -3275,6 +3268,18 @@ func (s *MonitoringCallbackService) failRemainingMonitoringTasksForAuthorization
if message == "" { if message == "" {
message = "登录态已失效,任务未执行。" message = "登录态已失效,任务未执行。"
} }
skipReason, cancellationMessage := authorizationFailureFollowupCancellation(disposition.Code, message)
runtimeError["canceled"] = true
runtimeError["cancellation_reason"] = skipReason
runtimeError["source_authorization_failure_message"] = message
runtimeError["message"] = cancellationMessage
requestPayloadJSON, err := json.Marshal(map[string]any{
"runtime_error": runtimeError,
"blocked_by_authorization_failure": true,
})
if err != nil {
return 0, response.ErrInternal(50041, "task_payload_encode_failed", "failed to encode monitoring authorization cancellation payload")
}
executionOwner := strings.TrimSpace(task.ExecutionOwner) executionOwner := strings.TrimSpace(task.ExecutionOwner)
if executionOwner == "" { if executionOwner == "" {
executionOwner = "legacy" executionOwner = "legacy"
@@ -3282,28 +3287,28 @@ func (s *MonitoringCallbackService) failRemainingMonitoringTasksForAuthorization
tag, err := tx.Exec(ctx, ` tag, err := tx.Exec(ctx, `
UPDATE monitoring_collect_tasks UPDATE monitoring_collect_tasks
SET status = 'failed', SET status = 'skipped',
lease_token_hash = NULL, lease_token_hash = NULL,
leased_to_executor = NULL, leased_to_executor = NULL,
leased_at = NULL, leased_at = NULL,
lease_expires_at = NULL, lease_expires_at = NULL,
callback_received_at = NOW(), callback_received_at = NOW(),
completed_at = $7, completed_at = $7,
skip_reason = NULL, skip_reason = $8,
error_message = $8, error_message = $9,
request_payload_json = ( request_payload_json = (
CASE CASE
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object' WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
THEN '{}'::jsonb THEN '{}'::jsonb
ELSE request_payload_json ELSE request_payload_json
END END
) || $9::jsonb, ) || $10::jsonb,
updated_at = NOW() updated_at = NOW()
WHERE tenant_id = $1 WHERE tenant_id = $1
AND collector_type = $2 AND collector_type = $2
AND business_date = $3::date AND business_date = $3::date
AND ai_platform_id = $4 AND ai_platform_id = $4
AND COALESCE(execution_owner, 'legacy') = $10 AND COALESCE(execution_owner, 'legacy') = $11
AND id <> $5 AND id <> $5
AND status IN ('pending', 'leased', 'expired') AND status IN ('pending', 'leased', 'expired')
AND callback_received_at IS NULL AND callback_received_at IS NULL
@@ -3311,9 +3316,9 @@ func (s *MonitoringCallbackService) failRemainingMonitoringTasksForAuthorization
target_client_id::text = $6 target_client_id::text = $6
OR leased_to_executor = $6 OR leased_to_executor = $6
) )
`, task.TenantID, task.CollectorType, task.BusinessDate.Format("2006-01-02"), task.AIPlatformID, task.ID, targetClientID, completedAt, message, requestPayloadJSON, executionOwner) `, task.TenantID, task.CollectorType, task.BusinessDate.Format("2006-01-02"), task.AIPlatformID, task.ID, targetClientID, completedAt, skipReason, cancellationMessage, requestPayloadJSON, executionOwner)
if err != nil { if err != nil {
return 0, response.ErrInternal(50041, "task_update_failed", "failed to fail remaining monitoring tasks after authorization failure") return 0, response.ErrInternal(50041, "task_update_failed", "failed to skip remaining monitoring tasks after authorization failure")
} }
return tag.RowsAffected(), nil return tag.RowsAffected(), nil
} }
@@ -10,10 +10,12 @@ import (
"time" "time"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jackc/pgx/v5/pgconn"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/geo-platform/tenant-api/internal/shared/config" "github.com/geo-platform/tenant-api/internal/shared/config"
"github.com/geo-platform/tenant-api/internal/tenant/repository"
) )
func TestMonitoringAnswerParseModelUsesConfig(t *testing.T) { func TestMonitoringAnswerParseModelUsesConfig(t *testing.T) {
@@ -635,6 +637,109 @@ func TestMonitoringAuthorizationFailureTargetClientPrefersExplicitTargetClient(t
} }
} }
func TestMonitorAuthorizationFollowupCancellationIsPlatformScopedAcrossAccounts(t *testing.T) {
tx := &captureDailyMonitorTaskTx{}
task := &repository.DesktopTask{
DesktopID: uuid.New(),
TenantID: 77,
WorkspaceID: 88,
Kind: "monitor",
TargetClientID: uuid.New(),
TargetAccountID: uuid.New(),
Platform: "doubao",
}
rows, err := loadQueuedDesktopAuthorizationFailureCandidates(context.Background(), tx, task)
require.NoError(t, err)
assert.Empty(t, rows)
require.True(t, tx.queryCalled)
query := normalizeSQLWhitespace(tx.querySQL)
assert.Contains(t, query, "AND kind = $3")
assert.Contains(t, query, "AND target_client_id = $4")
assert.Contains(t, query, "AND (kind = 'monitor' OR target_account_id = $5)")
assert.Contains(t, query, "AND platform_id = $6")
assert.Equal(t, "aborted", desktopAuthorizationFollowupTaskStatus("monitor"))
assert.Equal(t, "failed", desktopAuthorizationFollowupTaskStatus("publish"))
assert.Equal(t, "task_canceled", desktopAuthorizationFollowupEventType(&repository.DesktopTask{
Kind: "monitor",
Status: "aborted",
}))
}
func TestFailRemainingMonitoringTasksAfterFinalAccountChallengeScopesToModelAndClient(t *testing.T) {
targetClientID := uuid.New().String()
completedAt := time.Date(2026, 7, 14, 9, 30, 0, 0, time.UTC)
message := "豆包账号触发人机验证,请完成验证码后再继续采集。"
tx := &captureDailyMonitorTaskTx{commandTag: pgconn.NewCommandTag("UPDATE 3")}
task := &monitoringCollectTask{
ID: 901,
TenantID: 77,
AIPlatformID: "doubao",
CollectorType: monitoringCollectorType,
BusinessDate: time.Date(2026, 7, 14, 0, 0, 0, 0, time.UTC),
ExecutionOwner: "desktop_tasks",
TargetClientID: sql.NullString{String: targetClientID, Valid: true},
}
req := MonitoringTaskResultRequest{
Status: "failed",
ErrorMessage: &message,
RawResponseJSON: map[string]interface{}{
"runtime_error": map[string]interface{}{
"code": "doubao_challenge_required",
"message": message,
"retryable": false,
"non_retryable": true,
"failure_category": "ai_platform_authorization",
},
},
}
affected, err := (&MonitoringCallbackService{}).failRemainingMonitoringTasksForAuthorizationFailure(
context.Background(),
tx,
task,
req,
completedAt,
)
require.NoError(t, err)
assert.Equal(t, int64(3), affected)
require.True(t, tx.execCalled)
query := normalizeSQLWhitespace(tx.execSQL)
assert.Contains(t, query, "SET status = 'skipped'")
assert.Contains(t, query, "skip_reason = $8")
assert.Contains(t, query, "WHERE tenant_id = $1")
assert.Contains(t, query, "AND business_date = $3::date")
assert.Contains(t, query, "AND ai_platform_id = $4")
assert.Contains(t, query, "AND id <> $5")
assert.Contains(t, query, "target_client_id::text = $6 OR leased_to_executor = $6")
assert.Contains(t, query, "AND status IN ('pending', 'leased', 'expired')")
require.Len(t, tx.execArgs, 11)
assert.Equal(t, int64(77), tx.execArgs[0])
assert.Equal(t, monitoringCollectorType, tx.execArgs[1])
assert.Equal(t, "2026-07-14", tx.execArgs[2])
assert.Equal(t, "doubao", tx.execArgs[3])
assert.Equal(t, int64(901), tx.execArgs[4])
assert.Equal(t, targetClientID, tx.execArgs[5])
assert.Equal(t, completedAt, tx.execArgs[6])
assert.Equal(t, monitoringPlatformHumanVerificationSkipReason, tx.execArgs[7])
assert.Equal(t, "该模型没有其他可用账号,因触发人机验证,后续采集任务已自动取消。", tx.execArgs[8])
assert.Equal(t, "desktop_tasks", tx.execArgs[10])
payloadJSON, ok := tx.execArgs[9].([]byte)
require.True(t, ok)
var payload map[string]interface{}
require.NoError(t, json.Unmarshal(payloadJSON, &payload))
assert.Equal(t, true, payload["blocked_by_authorization_failure"])
runtimeError, ok := payload["runtime_error"].(map[string]interface{})
require.True(t, ok)
assert.Equal(t, "doubao_challenge_required", runtimeError["code"])
assert.Equal(t, "doubao", runtimeError["blocked_by_authorization_failure_platform"])
assert.Equal(t, true, runtimeError["canceled"])
assert.Equal(t, monitoringPlatformHumanVerificationSkipReason, runtimeError["cancellation_reason"])
}
func TestBuildMonitoringRawPayloadQwenIncludesSearchResultsInCitationInputs(t *testing.T) { func TestBuildMonitoringRawPayloadQwenIncludesSearchResultsInCitationInputs(t *testing.T) {
searchTitle := "搜索网页结果" searchTitle := "搜索网页结果"