Files
geo/server/internal/tenant/app/desktop_task_authorization_failure.go
T
root 28633cf570
Desktop Client Build / Resolve Build Metadata (push) Successful in 44s
Backend CI / Backend (push) Successful in 17m5s
Desktop Client Build / Build Desktop Client (push) Successful in 25m13s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 42s
feat(monitoring): fail same-client queued tasks on desktop authorization failure
When a desktop monitoring task fails with a non-retryable authorization failure
(login expired, challenge required, risk control), all pending same-client/same-platform
tasks for the same business day are immediately bulk-failed as non-retryable, avoiding
wasted execution attempts. Adds preflight authorization checks before task execution,
enriches failure payloads with disposition metadata (code, category, retryable flags),
and triggers account health reports on auth state degradation.

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
2026-05-15 23:50:50 +08:00

421 lines
14 KiB
Go

package app
import (
"context"
"encoding/json"
"strings"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"go.uber.org/zap"
"github.com/geo-platform/tenant-api/internal/shared/messaging/rabbitmq"
"github.com/geo-platform/tenant-api/internal/shared/response"
"github.com/geo-platform/tenant-api/internal/shared/stream"
"github.com/geo-platform/tenant-api/internal/tenant/repository"
)
type desktopAuthorizationFailureDisposition struct {
NonRetryable bool
Category string
Code string
Message string
}
type desktopAuthorizationBulkFailure struct {
Tasks []*repository.DesktopTask
PublishOutcomes []*desktopPublishSyncOutcome
MonitorTaskIDs []int64
ErrorPayload map[string]any
}
type queuedDesktopAuthorizationFailureCandidate struct {
DesktopID uuid.UUID
MonitorTaskID *int64
}
func bulkFailQueuedDesktopTasksForAuthorizationFailure(
ctx context.Context,
tx pgx.Tx,
task *repository.DesktopTask,
errorJSON []byte,
) (*desktopAuthorizationBulkFailure, error) {
result := &desktopAuthorizationBulkFailure{}
if tx == nil || task == nil || task.Status != "failed" {
return result, nil
}
errorPayload := unmarshalJSONObject(errorJSON)
disposition, ok := desktopAuthorizationFailureDispositionFromPayload(task, errorPayload)
if !ok || !disposition.NonRetryable {
return result, nil
}
result.ErrorPayload = ensureDesktopAuthorizationFailurePayload(errorPayload, disposition, task, nil)
candidates, err := loadQueuedDesktopAuthorizationFailureCandidates(ctx, tx, task)
if err != nil {
return nil, err
}
for _, candidate := range candidates {
payload := ensureDesktopAuthorizationFailurePayload(errorPayload, disposition, task, candidate.MonitorTaskID)
payload["blocked_by_authorization_failure_task_id"] = task.DesktopID.String()
payload["blocked_by_authorization_failure_platform"] = task.Platform
candidateErrorJSON, marshalErr := json.Marshal(payload)
if marshalErr != nil {
return nil, response.ErrInternal(50122, "desktop_task_authorization_failure_payload_failed", "failed to encode non-retryable authorization failure")
}
updated, updateErr := failQueuedDesktopTaskForAuthorizationFailure(ctx, tx, candidate.DesktopID, candidateErrorJSON)
if updateErr != nil {
return nil, updateErr
}
if updated == nil {
continue
}
result.Tasks = append(result.Tasks, updated)
if updated.Kind == "publish" {
publishOutcome, syncErr := syncDesktopPublishTaskState(ctx, tx, updated)
if syncErr != nil {
return nil, syncErr
}
if publishOutcome != nil {
result.PublishOutcomes = append(result.PublishOutcomes, publishOutcome)
}
}
if updated.Kind == "monitor" && updated.MonitorTaskID != nil {
result.MonitorTaskIDs = append(result.MonitorTaskIDs, *updated.MonitorTaskID)
}
}
return result, nil
}
func loadQueuedDesktopAuthorizationFailureCandidates(
ctx context.Context,
tx pgx.Tx,
task *repository.DesktopTask,
) ([]queuedDesktopAuthorizationFailureCandidate, error) {
rows, err := tx.Query(ctx, `
SELECT desktop_id, monitor_task_id
FROM desktop_tasks
WHERE tenant_id = $1
AND workspace_id = $2
AND kind = $3
AND status = 'queued'
AND target_client_id = $4
AND target_account_id = $5
AND platform_id = $6
AND desktop_id <> $7
ORDER BY COALESCE(enqueued_at, created_at) ASC, created_at ASC, desktop_id ASC
FOR UPDATE SKIP LOCKED
`, task.TenantID, task.WorkspaceID, task.Kind, task.TargetClientID, task.TargetAccountID, task.Platform, task.DesktopID)
if err != nil {
return nil, response.ErrInternal(50121, "desktop_task_authorization_failure_query_failed", "failed to select queued desktop tasks blocked by authorization failure")
}
defer rows.Close()
candidates := make([]queuedDesktopAuthorizationFailureCandidate, 0)
for rows.Next() {
var (
candidate queuedDesktopAuthorizationFailureCandidate
monitorTaskID pgtype.Int8
)
if scanErr := rows.Scan(&candidate.DesktopID, &monitorTaskID); scanErr != nil {
return nil, response.ErrInternal(50121, "desktop_task_authorization_failure_scan_failed", "failed to parse queued desktop tasks blocked by authorization failure")
}
if monitorTaskID.Valid {
value := monitorTaskID.Int64
candidate.MonitorTaskID = &value
}
candidates = append(candidates, candidate)
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50121, "desktop_task_authorization_failure_scan_failed", "failed to iterate queued desktop tasks blocked by authorization failure")
}
return candidates, nil
}
func failQueuedDesktopTaskForAuthorizationFailure(
ctx context.Context,
tx pgx.Tx,
desktopID uuid.UUID,
errorJSON []byte,
) (*repository.DesktopTask, error) {
row := tx.QueryRow(ctx, `
UPDATE desktop_tasks AS t
SET status = 'failed',
result = NULL,
error = $2,
active_attempt_id = NULL,
lease_token_hash = NULL,
lease_expires_at = NULL,
updated_at = NOW()
WHERE desktop_id = $1
AND status = 'queued'
RETURNING `+desktopTaskRepositoryReturningColumns,
desktopID,
errorJSON,
)
updated, err := scanRepositoryDesktopTask(row)
if err != nil {
if err == pgx.ErrNoRows {
return nil, nil
}
return nil, response.ErrInternal(50123, "desktop_task_authorization_failure_update_failed", "failed to fail queued desktop task after authorization failure")
}
return updated, nil
}
func desktopAuthorizationFailureDispositionFromPayload(
task *repository.DesktopTask,
payload map[string]any,
) (desktopAuthorizationFailureDisposition, bool) {
if task == nil || len(payload) == 0 {
return desktopAuthorizationFailureDisposition{}, false
}
disposition := desktopAuthorizationFailureDisposition{
Category: desktopAuthorizationFailureString(payload["failure_category"]),
Code: desktopAuthorizationFailureString(payload["code"]),
Message: desktopAuthorizationFailureString(payload["message"]),
}
hasAuthorizationSignal := desktopAuthorizationFailureCategory(disposition.Category) ||
desktopAuthorizationFailureCode(disposition.Code) ||
desktopAuthorizationFailureMessage(disposition.Message)
if !hasAuthorizationSignal {
return desktopAuthorizationFailureDisposition{}, false
}
disposition.NonRetryable = true
if disposition.Category == "" {
disposition.Category = desktopAuthorizationFailureCategoryForTask(task)
}
if !desktopAuthorizationFailureCategoryMatchesTask(task, disposition.Category) {
return desktopAuthorizationFailureDisposition{}, false
}
return disposition, true
}
func ensureDesktopAuthorizationFailurePayload(
payload map[string]any,
disposition desktopAuthorizationFailureDisposition,
task *repository.DesktopTask,
monitorTaskID *int64,
) map[string]any {
result := make(map[string]any, len(payload)+6)
for key, value := range payload {
result[key] = value
}
if disposition.Code != "" {
result["code"] = disposition.Code
} else if _, exists := result["code"]; !exists {
result["code"] = "desktop_account_auth_expired"
}
if disposition.Message != "" {
result["message"] = disposition.Message
} else if _, exists := result["message"]; !exists {
result["message"] = "登录态已失效,任务未执行。"
}
result["retryable"] = false
result["non_retryable"] = true
if disposition.Category != "" {
result["failure_category"] = disposition.Category
} else if task != nil {
result["failure_category"] = desktopAuthorizationFailureCategoryForTask(task)
}
if task != nil && strings.TrimSpace(task.Platform) != "" {
result["platform"] = strings.TrimSpace(task.Platform)
}
if monitorTaskID != nil {
result["monitoring_task_id"] = *monitorTaskID
}
return result
}
func desktopAuthorizationFailureCategoryForTask(task *repository.DesktopTask) string {
if task != nil && task.Kind == "monitor" {
return "ai_platform_authorization"
}
return "media_account_authorization"
}
func desktopAuthorizationFailureCategoryMatchesTask(task *repository.DesktopTask, category string) bool {
category = strings.TrimSpace(category)
if task != nil && task.Kind == "monitor" {
return category == "ai_platform_authorization"
}
if task != nil && task.Kind == "publish" {
return category == "media_account_authorization"
}
return false
}
func desktopAuthorizationFailureCategory(category string) bool {
category = strings.TrimSpace(category)
return category == "ai_platform_authorization" || category == "media_account_authorization"
}
func desktopAuthorizationFailureCode(code string) bool {
normalized := strings.ToLower(strings.TrimSpace(code))
if normalized == "" {
return false
}
if normalized == "desktop_account_auth_expired" ||
normalized == "desktop_account_challenge_required" ||
normalized == "desktop_account_risk_control" {
return true
}
return strings.HasSuffix(normalized, "_login_required") ||
strings.HasSuffix(normalized, "_login_expired") ||
strings.HasSuffix(normalized, "_not_logged_in") ||
strings.HasSuffix(normalized, "_challenge_required")
}
func desktopAuthorizationFailureMessage(message string) bool {
normalized := strings.ToLower(strings.TrimSpace(message))
if normalized == "" {
return false
}
return strings.Contains(normalized, "login required") ||
strings.Contains(normalized, "login expired") ||
strings.Contains(normalized, "not logged in") ||
strings.Contains(normalized, "unauthorized") ||
strings.Contains(normalized, "登录态失效") ||
strings.Contains(normalized, "登录态已失效") ||
strings.Contains(normalized, "登录已失效") ||
strings.Contains(normalized, "请重新登录") ||
strings.Contains(normalized, "请先登录") ||
strings.Contains(normalized, "未登录")
}
func desktopAuthorizationFailureString(value any) string {
text, ok := value.(string)
if !ok {
return ""
}
return strings.TrimSpace(text)
}
func failMonitoringCollectTasksForDesktopAuthorizationFailure(
ctx context.Context,
pool *pgxpool.Pool,
monitorTaskIDs []int64,
errorPayload map[string]any,
) error {
if pool == nil || len(monitorTaskIDs) == 0 {
return nil
}
errorMessage := desktopAuthorizationFailureString(errorPayload["message"])
if errorMessage == "" {
errorMessage = "登录态已失效,任务未执行。"
}
requestPayloadJSON, err := json.Marshal(map[string]any{
"runtime_error": errorPayload,
"blocked_by_authorization_failure": true,
})
if err != nil {
return response.ErrInternal(50124, "monitoring_authorization_failure_payload_failed", "failed to encode monitoring authorization failure payload")
}
if _, err := pool.Exec(ctx, `
UPDATE monitoring_collect_tasks
SET status = 'failed',
lease_token_hash = NULL,
leased_to_executor = NULL,
leased_at = NULL,
lease_expires_at = NULL,
callback_received_at = NOW(),
completed_at = NOW(),
skip_reason = NULL,
error_message = $2,
request_payload_json = (
CASE
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
THEN '{}'::jsonb
ELSE request_payload_json
END
) || $3::jsonb,
updated_at = NOW()
WHERE id = ANY($1::bigint[])
AND callback_received_at IS NULL
AND status IN ('pending', 'leased', 'expired')
`, monitorTaskIDs, errorMessage, requestPayloadJSON); err != nil {
return response.ErrInternal(50125, "monitoring_authorization_failure_update_failed", "failed to fail monitoring tasks after authorization failure")
}
return nil
}
func publishDesktopTaskLifecycleEvent(
ctx context.Context,
messaging *rabbitmq.Client,
logger *zap.Logger,
task *repository.DesktopTask,
eventType string,
) {
if task == nil {
return
}
title, businessDate, schedulerGroupKey, questionText := deriveDesktopTaskEventMetadataFromPayload(
task.Kind,
task.Payload,
)
event := DesktopTaskEvent{
Type: eventType,
TaskID: task.DesktopID.String(),
JobID: task.JobID.String(),
WorkspaceID: task.WorkspaceID,
TargetAccountID: task.TargetAccountID.String(),
TargetClientID: task.TargetClientID.String(),
Platform: task.Platform,
Title: title,
BusinessDate: businessDate,
SchedulerGroupKey: schedulerGroupKey,
QuestionText: questionText,
Status: task.Status,
Kind: task.Kind,
Priority: task.Priority,
Lane: task.Lane,
InterruptGeneration: task.InterruptGeneration,
UpdatedAt: task.UpdatedAt,
}
if err := publishDesktopTaskEvent(ctx, messaging, event); err != nil && logger != nil {
logger.Warn("desktop task event publish failed",
zap.String("task_id", task.DesktopID.String()),
zap.String("event_type", eventType),
zap.Error(err),
)
}
if err := publishDesktopDispatchEvent(ctx, messaging, logger, stream.DesktopDispatchEvent{
Type: event.Type,
TaskID: event.TaskID,
JobID: event.JobID,
WorkspaceID: event.WorkspaceID,
TargetAccountID: event.TargetAccountID,
TargetClientID: event.TargetClientID,
Platform: event.Platform,
Title: event.Title,
BusinessDate: event.BusinessDate,
SchedulerGroupKey: event.SchedulerGroupKey,
QuestionText: event.QuestionText,
Status: event.Status,
Kind: event.Kind,
Priority: event.Priority,
Lane: event.Lane,
InterruptGeneration: event.InterruptGeneration,
UpdatedAt: event.UpdatedAt,
}); err != nil && logger != nil {
logger.Warn("desktop dispatch mirror publish failed",
zap.String("task_id", task.DesktopID.String()),
zap.String("event_type", eventType),
zap.Error(err),
)
}
}