421 lines
14 KiB
Go
421 lines
14 KiB
Go
|
package app
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"encoding/json"
|
||
|
|
"strings"
|
||
|
|
|
||
|
|
"github.com/google/uuid"
|
||
|
|
"github.com/jackc/pgx/v5"
|
||
|
|
"github.com/jackc/pgx/v5/pgtype"
|
||
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
||
|
|
"go.uber.org/zap"
|
||
|
|
|
||
|
|
"github.com/geo-platform/tenant-api/internal/shared/messaging/rabbitmq"
|
||
|
|
"github.com/geo-platform/tenant-api/internal/shared/response"
|
||
|
|
"github.com/geo-platform/tenant-api/internal/shared/stream"
|
||
|
|
"github.com/geo-platform/tenant-api/internal/tenant/repository"
|
||
|
|
)
|
||
|
|
|
||
|
|
type desktopAuthorizationFailureDisposition struct {
|
||
|
|
NonRetryable bool
|
||
|
|
Category string
|
||
|
|
Code string
|
||
|
|
Message string
|
||
|
|
}
|
||
|
|
|
||
|
|
type desktopAuthorizationBulkFailure struct {
|
||
|
|
Tasks []*repository.DesktopTask
|
||
|
|
PublishOutcomes []*desktopPublishSyncOutcome
|
||
|
|
MonitorTaskIDs []int64
|
||
|
|
ErrorPayload map[string]any
|
||
|
|
}
|
||
|
|
|
||
|
|
type queuedDesktopAuthorizationFailureCandidate struct {
|
||
|
|
DesktopID uuid.UUID
|
||
|
|
MonitorTaskID *int64
|
||
|
|
}
|
||
|
|
|
||
|
|
func bulkFailQueuedDesktopTasksForAuthorizationFailure(
|
||
|
|
ctx context.Context,
|
||
|
|
tx pgx.Tx,
|
||
|
|
task *repository.DesktopTask,
|
||
|
|
errorJSON []byte,
|
||
|
|
) (*desktopAuthorizationBulkFailure, error) {
|
||
|
|
result := &desktopAuthorizationBulkFailure{}
|
||
|
|
if tx == nil || task == nil || task.Status != "failed" {
|
||
|
|
return result, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
errorPayload := unmarshalJSONObject(errorJSON)
|
||
|
|
disposition, ok := desktopAuthorizationFailureDispositionFromPayload(task, errorPayload)
|
||
|
|
if !ok || !disposition.NonRetryable {
|
||
|
|
return result, nil
|
||
|
|
}
|
||
|
|
result.ErrorPayload = ensureDesktopAuthorizationFailurePayload(errorPayload, disposition, task, nil)
|
||
|
|
|
||
|
|
candidates, err := loadQueuedDesktopAuthorizationFailureCandidates(ctx, tx, task)
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
for _, candidate := range candidates {
|
||
|
|
payload := ensureDesktopAuthorizationFailurePayload(errorPayload, disposition, task, candidate.MonitorTaskID)
|
||
|
|
payload["blocked_by_authorization_failure_task_id"] = task.DesktopID.String()
|
||
|
|
payload["blocked_by_authorization_failure_platform"] = task.Platform
|
||
|
|
|
||
|
|
candidateErrorJSON, marshalErr := json.Marshal(payload)
|
||
|
|
if marshalErr != nil {
|
||
|
|
return nil, response.ErrInternal(50122, "desktop_task_authorization_failure_payload_failed", "failed to encode non-retryable authorization failure")
|
||
|
|
}
|
||
|
|
|
||
|
|
updated, updateErr := failQueuedDesktopTaskForAuthorizationFailure(ctx, tx, candidate.DesktopID, candidateErrorJSON)
|
||
|
|
if updateErr != nil {
|
||
|
|
return nil, updateErr
|
||
|
|
}
|
||
|
|
if updated == nil {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
result.Tasks = append(result.Tasks, updated)
|
||
|
|
|
||
|
|
if updated.Kind == "publish" {
|
||
|
|
publishOutcome, syncErr := syncDesktopPublishTaskState(ctx, tx, updated)
|
||
|
|
if syncErr != nil {
|
||
|
|
return nil, syncErr
|
||
|
|
}
|
||
|
|
if publishOutcome != nil {
|
||
|
|
result.PublishOutcomes = append(result.PublishOutcomes, publishOutcome)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if updated.Kind == "monitor" && updated.MonitorTaskID != nil {
|
||
|
|
result.MonitorTaskIDs = append(result.MonitorTaskIDs, *updated.MonitorTaskID)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return result, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func loadQueuedDesktopAuthorizationFailureCandidates(
|
||
|
|
ctx context.Context,
|
||
|
|
tx pgx.Tx,
|
||
|
|
task *repository.DesktopTask,
|
||
|
|
) ([]queuedDesktopAuthorizationFailureCandidate, error) {
|
||
|
|
rows, err := tx.Query(ctx, `
|
||
|
|
SELECT desktop_id, monitor_task_id
|
||
|
|
FROM desktop_tasks
|
||
|
|
WHERE tenant_id = $1
|
||
|
|
AND workspace_id = $2
|
||
|
|
AND kind = $3
|
||
|
|
AND status = 'queued'
|
||
|
|
AND target_client_id = $4
|
||
|
|
AND target_account_id = $5
|
||
|
|
AND platform_id = $6
|
||
|
|
AND desktop_id <> $7
|
||
|
|
ORDER BY COALESCE(enqueued_at, created_at) ASC, created_at ASC, desktop_id ASC
|
||
|
|
FOR UPDATE SKIP LOCKED
|
||
|
|
`, task.TenantID, task.WorkspaceID, task.Kind, task.TargetClientID, task.TargetAccountID, task.Platform, task.DesktopID)
|
||
|
|
if err != nil {
|
||
|
|
return nil, response.ErrInternal(50121, "desktop_task_authorization_failure_query_failed", "failed to select queued desktop tasks blocked by authorization failure")
|
||
|
|
}
|
||
|
|
defer rows.Close()
|
||
|
|
|
||
|
|
candidates := make([]queuedDesktopAuthorizationFailureCandidate, 0)
|
||
|
|
for rows.Next() {
|
||
|
|
var (
|
||
|
|
candidate queuedDesktopAuthorizationFailureCandidate
|
||
|
|
monitorTaskID pgtype.Int8
|
||
|
|
)
|
||
|
|
if scanErr := rows.Scan(&candidate.DesktopID, &monitorTaskID); scanErr != nil {
|
||
|
|
return nil, response.ErrInternal(50121, "desktop_task_authorization_failure_scan_failed", "failed to parse queued desktop tasks blocked by authorization failure")
|
||
|
|
}
|
||
|
|
if monitorTaskID.Valid {
|
||
|
|
value := monitorTaskID.Int64
|
||
|
|
candidate.MonitorTaskID = &value
|
||
|
|
}
|
||
|
|
candidates = append(candidates, candidate)
|
||
|
|
}
|
||
|
|
if err := rows.Err(); err != nil {
|
||
|
|
return nil, response.ErrInternal(50121, "desktop_task_authorization_failure_scan_failed", "failed to iterate queued desktop tasks blocked by authorization failure")
|
||
|
|
}
|
||
|
|
return candidates, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func failQueuedDesktopTaskForAuthorizationFailure(
|
||
|
|
ctx context.Context,
|
||
|
|
tx pgx.Tx,
|
||
|
|
desktopID uuid.UUID,
|
||
|
|
errorJSON []byte,
|
||
|
|
) (*repository.DesktopTask, error) {
|
||
|
|
row := tx.QueryRow(ctx, `
|
||
|
|
UPDATE desktop_tasks AS t
|
||
|
|
SET status = 'failed',
|
||
|
|
result = NULL,
|
||
|
|
error = $2,
|
||
|
|
active_attempt_id = NULL,
|
||
|
|
lease_token_hash = NULL,
|
||
|
|
lease_expires_at = NULL,
|
||
|
|
updated_at = NOW()
|
||
|
|
WHERE desktop_id = $1
|
||
|
|
AND status = 'queued'
|
||
|
|
RETURNING `+desktopTaskRepositoryReturningColumns,
|
||
|
|
desktopID,
|
||
|
|
errorJSON,
|
||
|
|
)
|
||
|
|
updated, err := scanRepositoryDesktopTask(row)
|
||
|
|
if err != nil {
|
||
|
|
if err == pgx.ErrNoRows {
|
||
|
|
return nil, nil
|
||
|
|
}
|
||
|
|
return nil, response.ErrInternal(50123, "desktop_task_authorization_failure_update_failed", "failed to fail queued desktop task after authorization failure")
|
||
|
|
}
|
||
|
|
return updated, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func desktopAuthorizationFailureDispositionFromPayload(
|
||
|
|
task *repository.DesktopTask,
|
||
|
|
payload map[string]any,
|
||
|
|
) (desktopAuthorizationFailureDisposition, bool) {
|
||
|
|
if task == nil || len(payload) == 0 {
|
||
|
|
return desktopAuthorizationFailureDisposition{}, false
|
||
|
|
}
|
||
|
|
|
||
|
|
disposition := desktopAuthorizationFailureDisposition{
|
||
|
|
Category: desktopAuthorizationFailureString(payload["failure_category"]),
|
||
|
|
Code: desktopAuthorizationFailureString(payload["code"]),
|
||
|
|
Message: desktopAuthorizationFailureString(payload["message"]),
|
||
|
|
}
|
||
|
|
hasAuthorizationSignal := desktopAuthorizationFailureCategory(disposition.Category) ||
|
||
|
|
desktopAuthorizationFailureCode(disposition.Code) ||
|
||
|
|
desktopAuthorizationFailureMessage(disposition.Message)
|
||
|
|
if !hasAuthorizationSignal {
|
||
|
|
return desktopAuthorizationFailureDisposition{}, false
|
||
|
|
}
|
||
|
|
|
||
|
|
disposition.NonRetryable = true
|
||
|
|
if disposition.Category == "" {
|
||
|
|
disposition.Category = desktopAuthorizationFailureCategoryForTask(task)
|
||
|
|
}
|
||
|
|
if !desktopAuthorizationFailureCategoryMatchesTask(task, disposition.Category) {
|
||
|
|
return desktopAuthorizationFailureDisposition{}, false
|
||
|
|
}
|
||
|
|
return disposition, true
|
||
|
|
}
|
||
|
|
|
||
|
|
func ensureDesktopAuthorizationFailurePayload(
|
||
|
|
payload map[string]any,
|
||
|
|
disposition desktopAuthorizationFailureDisposition,
|
||
|
|
task *repository.DesktopTask,
|
||
|
|
monitorTaskID *int64,
|
||
|
|
) map[string]any {
|
||
|
|
result := make(map[string]any, len(payload)+6)
|
||
|
|
for key, value := range payload {
|
||
|
|
result[key] = value
|
||
|
|
}
|
||
|
|
if disposition.Code != "" {
|
||
|
|
result["code"] = disposition.Code
|
||
|
|
} else if _, exists := result["code"]; !exists {
|
||
|
|
result["code"] = "desktop_account_auth_expired"
|
||
|
|
}
|
||
|
|
if disposition.Message != "" {
|
||
|
|
result["message"] = disposition.Message
|
||
|
|
} else if _, exists := result["message"]; !exists {
|
||
|
|
result["message"] = "登录态已失效，任务未执行。"
|
||
|
|
}
|
||
|
|
result["retryable"] = false
|
||
|
|
result["non_retryable"] = true
|
||
|
|
if disposition.Category != "" {
|
||
|
|
result["failure_category"] = disposition.Category
|
||
|
|
} else if task != nil {
|
||
|
|
result["failure_category"] = desktopAuthorizationFailureCategoryForTask(task)
|
||
|
|
}
|
||
|
|
if task != nil && strings.TrimSpace(task.Platform) != "" {
|
||
|
|
result["platform"] = strings.TrimSpace(task.Platform)
|
||
|
|
}
|
||
|
|
if monitorTaskID != nil {
|
||
|
|
result["monitoring_task_id"] = *monitorTaskID
|
||
|
|
}
|
||
|
|
return result
|
||
|
|
}
|
||
|
|
|
||
|
|
func desktopAuthorizationFailureCategoryForTask(task *repository.DesktopTask) string {
|
||
|
|
if task != nil && task.Kind == "monitor" {
|
||
|
|
return "ai_platform_authorization"
|
||
|
|
}
|
||
|
|
return "media_account_authorization"
|
||
|
|
}
|
||
|
|
|
||
|
|
func desktopAuthorizationFailureCategoryMatchesTask(task *repository.DesktopTask, category string) bool {
|
||
|
|
category = strings.TrimSpace(category)
|
||
|
|
if task != nil && task.Kind == "monitor" {
|
||
|
|
return category == "ai_platform_authorization"
|
||
|
|
}
|
||
|
|
if task != nil && task.Kind == "publish" {
|
||
|
|
return category == "media_account_authorization"
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
|
||
|
|
func desktopAuthorizationFailureCategory(category string) bool {
|
||
|
|
category = strings.TrimSpace(category)
|
||
|
|
return category == "ai_platform_authorization" || category == "media_account_authorization"
|
||
|
|
}
|
||
|
|
|
||
|
|
func desktopAuthorizationFailureCode(code string) bool {
|
||
|
|
normalized := strings.ToLower(strings.TrimSpace(code))
|
||
|
|
if normalized == "" {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
if normalized == "desktop_account_auth_expired" ||
|
||
|
|
normalized == "desktop_account_challenge_required" ||
|
||
|
|
normalized == "desktop_account_risk_control" {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
return strings.HasSuffix(normalized, "_login_required") ||
|
||
|
|
strings.HasSuffix(normalized, "_login_expired") ||
|
||
|
|
strings.HasSuffix(normalized, "_not_logged_in") ||
|
||
|
|
strings.HasSuffix(normalized, "_challenge_required")
|
||
|
|
}
|
||
|
|
|
||
|
|
func desktopAuthorizationFailureMessage(message string) bool {
|
||
|
|
normalized := strings.ToLower(strings.TrimSpace(message))
|
||
|
|
if normalized == "" {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
return strings.Contains(normalized, "login required") ||
|
||
|
|
strings.Contains(normalized, "login expired") ||
|
||
|
|
strings.Contains(normalized, "not logged in") ||
|
||
|
|
strings.Contains(normalized, "unauthorized") ||
|
||
|
|
strings.Contains(normalized, "登录态失效") ||
|
||
|
|
strings.Contains(normalized, "登录态已失效") ||
|
||
|
|
strings.Contains(normalized, "登录已失效") ||
|
||
|
|
strings.Contains(normalized, "请重新登录") ||
|
||
|
|
strings.Contains(normalized, "请先登录") ||
|
||
|
|
strings.Contains(normalized, "未登录")
|
||
|
|
}
|
||
|
|
|
||
|
|
func desktopAuthorizationFailureString(value any) string {
|
||
|
|
text, ok := value.(string)
|
||
|
|
if !ok {
|
||
|
|
return ""
|
||
|
|
}
|
||
|
|
return strings.TrimSpace(text)
|
||
|
|
}
|
||
|
|
|
||
|
|
func failMonitoringCollectTasksForDesktopAuthorizationFailure(
|
||
|
|
ctx context.Context,
|
||
|
|
pool *pgxpool.Pool,
|
||
|
|
monitorTaskIDs []int64,
|
||
|
|
errorPayload map[string]any,
|
||
|
|
) error {
|
||
|
|
if pool == nil || len(monitorTaskIDs) == 0 {
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
errorMessage := desktopAuthorizationFailureString(errorPayload["message"])
|
||
|
|
if errorMessage == "" {
|
||
|
|
errorMessage = "登录态已失效，任务未执行。"
|
||
|
|
}
|
||
|
|
requestPayloadJSON, err := json.Marshal(map[string]any{
|
||
|
|
"runtime_error": errorPayload,
|
||
|
|
"blocked_by_authorization_failure": true,
|
||
|
|
})
|
||
|
|
if err != nil {
|
||
|
|
return response.ErrInternal(50124, "monitoring_authorization_failure_payload_failed", "failed to encode monitoring authorization failure payload")
|
||
|
|
}
|
||
|
|
|
||
|
|
if _, err := pool.Exec(ctx, `
|
||
|
|
UPDATE monitoring_collect_tasks
|
||
|
|
SET status = 'failed',
|
||
|
|
lease_token_hash = NULL,
|
||
|
|
leased_to_executor = NULL,
|
||
|
|
leased_at = NULL,
|
||
|
|
lease_expires_at = NULL,
|
||
|
|
callback_received_at = NOW(),
|
||
|
|
completed_at = NOW(),
|
||
|
|
skip_reason = NULL,
|
||
|
|
error_message = $2,
|
||
|
|
request_payload_json = (
|
||
|
|
CASE
|
||
|
|
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
|
||
|
|
THEN '{}'::jsonb
|
||
|
|
ELSE request_payload_json
|
||
|
|
END
|
||
|
|
) || $3::jsonb,
|
||
|
|
updated_at = NOW()
|
||
|
|
WHERE id = ANY($1::bigint[])
|
||
|
|
AND callback_received_at IS NULL
|
||
|
|
AND status IN ('pending', 'leased', 'expired')
|
||
|
|
`, monitorTaskIDs, errorMessage, requestPayloadJSON); err != nil {
|
||
|
|
return response.ErrInternal(50125, "monitoring_authorization_failure_update_failed", "failed to fail monitoring tasks after authorization failure")
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func publishDesktopTaskLifecycleEvent(
|
||
|
|
ctx context.Context,
|
||
|
|
messaging *rabbitmq.Client,
|
||
|
|
logger *zap.Logger,
|
||
|
|
task *repository.DesktopTask,
|
||
|
|
eventType string,
|
||
|
|
) {
|
||
|
|
if task == nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
title, businessDate, schedulerGroupKey, questionText := deriveDesktopTaskEventMetadataFromPayload(
|
||
|
|
task.Kind,
|
||
|
|
task.Payload,
|
||
|
|
)
|
||
|
|
event := DesktopTaskEvent{
|
||
|
|
Type: eventType,
|
||
|
|
TaskID: task.DesktopID.String(),
|
||
|
|
JobID: task.JobID.String(),
|
||
|
|
WorkspaceID: task.WorkspaceID,
|
||
|
|
TargetAccountID: task.TargetAccountID.String(),
|
||
|
|
TargetClientID: task.TargetClientID.String(),
|
||
|
|
Platform: task.Platform,
|
||
|
|
Title: title,
|
||
|
|
BusinessDate: businessDate,
|
||
|
|
SchedulerGroupKey: schedulerGroupKey,
|
||
|
|
QuestionText: questionText,
|
||
|
|
Status: task.Status,
|
||
|
|
Kind: task.Kind,
|
||
|
|
Priority: task.Priority,
|
||
|
|
Lane: task.Lane,
|
||
|
|
InterruptGeneration: task.InterruptGeneration,
|
||
|
|
UpdatedAt: task.UpdatedAt,
|
||
|
|
}
|
||
|
|
if err := publishDesktopTaskEvent(ctx, messaging, event); err != nil && logger != nil {
|
||
|
|
logger.Warn("desktop task event publish failed",
|
||
|
|
zap.String("task_id", task.DesktopID.String()),
|
||
|
|
zap.String("event_type", eventType),
|
||
|
|
zap.Error(err),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
if err := publishDesktopDispatchEvent(ctx, messaging, logger, stream.DesktopDispatchEvent{
|
||
|
|
Type: event.Type,
|
||
|
|
TaskID: event.TaskID,
|
||
|
|
JobID: event.JobID,
|
||
|
|
WorkspaceID: event.WorkspaceID,
|
||
|
|
TargetAccountID: event.TargetAccountID,
|
||
|
|
TargetClientID: event.TargetClientID,
|
||
|
|
Platform: event.Platform,
|
||
|
|
Title: event.Title,
|
||
|
|
BusinessDate: event.BusinessDate,
|
||
|
|
SchedulerGroupKey: event.SchedulerGroupKey,
|
||
|
|
QuestionText: event.QuestionText,
|
||
|
|
Status: event.Status,
|
||
|
|
Kind: event.Kind,
|
||
|
|
Priority: event.Priority,
|
||
|
|
Lane: event.Lane,
|
||
|
|
InterruptGeneration: event.InterruptGeneration,
|
||
|
|
UpdatedAt: event.UpdatedAt,
|
||
|
|
}); err != nil && logger != nil {
|
||
|
|
logger.Warn("desktop dispatch mirror publish failed",
|
||
|
|
zap.String("task_id", task.DesktopID.String()),
|
||
|
|
zap.String("event_type", eventType),
|
||
|
|
zap.Error(err),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
}
|