fix: throttle desktop monitoring dispatch
Desktop Client Build / Resolve Build Metadata (push) Successful in 27s
Backend CI / Backend (push) Successful in 15m57s
Desktop Client Build / Build Desktop Client (push) Successful in 22m57s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 27s
Desktop Client Build / Resolve Build Metadata (push) Successful in 27s
Backend CI / Backend (push) Successful in 15m57s
Desktop Client Build / Build Desktop Client (push) Successful in 22m57s
Desktop Client Build / Publish Client Artifacts to NAS (push) Successful in 27s
This commit is contained in:
@@ -138,15 +138,18 @@ func (s *MonitoringService) loadMonitorDesktopTaskSpecs(
|
||||
t.dispatch_priority,
|
||||
t.dispatch_lane,
|
||||
t.interrupt_generation,
|
||||
COALESCE((
|
||||
COALESCE(NULLIF(t.question_text_snapshot, ''), (
|
||||
SELECT question_text_snapshot
|
||||
FROM monitoring_question_config_snapshots s
|
||||
WHERE s.tenant_id = t.tenant_id
|
||||
AND s.brand_id = t.brand_id
|
||||
AND s.question_id = t.question_id
|
||||
AND s.question_hash = t.question_hash
|
||||
AND s.deleted_at IS NULL
|
||||
ORDER BY s.projected_at DESC, s.id DESC
|
||||
AND btrim(s.question_text_snapshot) <> ''
|
||||
ORDER BY
|
||||
CASE WHEN s.deleted_at IS NULL THEN 0 ELSE 1 END,
|
||||
s.projected_at DESC,
|
||||
s.id DESC
|
||||
LIMIT 1
|
||||
), '') AS question_text_snapshot
|
||||
FROM monitoring_collect_tasks t
|
||||
@@ -193,6 +196,10 @@ func (s *MonitoringService) loadMonitorDesktopTaskSpecs(
|
||||
spec.RequestedByUserID = requestedByUserID
|
||||
spec.BusinessDate = businessDay.Format("2006-01-02")
|
||||
spec.QuestionHash = encodeQuestionHash(questionHash)
|
||||
spec.QuestionText = strings.TrimSpace(spec.QuestionText)
|
||||
if spec.QuestionText == "" {
|
||||
continue
|
||||
}
|
||||
spec.SchedulerGroupKey = monitoringSchedulerGroupKey(spec.QuestionHash, spec.QuestionID, spec.QuestionText)
|
||||
result = append(result, spec)
|
||||
}
|
||||
@@ -210,9 +217,13 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
targets, err := s.loadMonitorDesktopTaskTargets(ctx, specs[0].TenantID, specs[0].WorkspaceID, specs[0].RequestedByUserID, specs)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
var targets map[string]monitorDesktopTaskTarget
|
||||
if monitorDesktopTaskSpecsNeedTargetLookup(specs) {
|
||||
var err error
|
||||
targets, err = s.loadMonitorDesktopTaskTargets(ctx, specs[0].TenantID, specs[0].WorkspaceID, specs[0].RequestedByUserID, specs)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
tx, err := s.businessPool.Begin(ctx)
|
||||
@@ -225,13 +236,23 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
|
||||
publishableTasks := make([]*repository.DesktopTask, 0, len(specs))
|
||||
fallbackLegacyTaskIDs := make([]int64, 0)
|
||||
for _, spec := range specs {
|
||||
target, ok := targets[normalizeMonitoringPlatformID(spec.PlatformID)]
|
||||
if !ok || target.AccountID == uuid.Nil || target.ClientID == uuid.Nil {
|
||||
fallbackLegacyTaskIDs = append(fallbackLegacyTaskIDs, spec.MonitorTaskID)
|
||||
if spec.TargetAccountID == uuid.Nil || spec.TargetClientID == uuid.Nil {
|
||||
target, ok := targets[normalizeMonitoringPlatformID(spec.PlatformID)]
|
||||
if !ok || target.AccountID == uuid.Nil || target.ClientID == uuid.Nil {
|
||||
fallbackLegacyTaskIDs = append(fallbackLegacyTaskIDs, spec.MonitorTaskID)
|
||||
continue
|
||||
}
|
||||
spec.TargetAccountID = target.AccountID
|
||||
spec.TargetClientID = target.ClientID
|
||||
}
|
||||
|
||||
available, throttleErr := acquireMonitorDesktopTaskPlatformSlot(ctx, tx, spec)
|
||||
if throttleErr != nil {
|
||||
return nil, nil, throttleErr
|
||||
}
|
||||
if !available {
|
||||
continue
|
||||
}
|
||||
spec.TargetAccountID = target.AccountID
|
||||
spec.TargetClientID = target.ClientID
|
||||
|
||||
task, shouldPublish, fallbackLegacy, upsertErr := s.upsertMonitorDesktopTask(ctx, tx, repo, spec)
|
||||
if upsertErr != nil {
|
||||
@@ -252,6 +273,54 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
|
||||
return publishableTasks, fallbackLegacyTaskIDs, nil
|
||||
}
|
||||
|
||||
func monitorDesktopTaskSpecsNeedTargetLookup(specs []monitorDesktopTaskSpec) bool {
|
||||
for _, spec := range specs {
|
||||
if spec.TargetAccountID == uuid.Nil || spec.TargetClientID == uuid.Nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func acquireMonitorDesktopTaskPlatformSlot(ctx context.Context, tx pgx.Tx, spec monitorDesktopTaskSpec) (bool, error) {
|
||||
if tx == nil || spec.TargetClientID == uuid.Nil || strings.TrimSpace(spec.PlatformID) == "" {
|
||||
return false, nil
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, monitorDesktopTaskPlatformSlotLockKey(spec.TargetClientID, spec.PlatformID)); err != nil {
|
||||
return false, response.ErrInternal(50123, "desktop_task_throttle_lock_failed", "failed to acquire phase2 monitor desktop platform slot")
|
||||
}
|
||||
|
||||
var busy bool
|
||||
if err := tx.QueryRow(ctx, monitorDesktopTaskPlatformSlotBusySQL(), spec.TenantID, spec.WorkspaceID, spec.TargetClientID, normalizeMonitoringPlatformID(spec.PlatformID), spec.MonitorTaskID).Scan(&busy); err != nil {
|
||||
return false, response.ErrInternal(50124, "desktop_task_throttle_lookup_failed", "failed to inspect phase2 monitor desktop platform slot")
|
||||
}
|
||||
return !busy, nil
|
||||
}
|
||||
|
||||
func monitorDesktopTaskPlatformSlotBusySQL() string {
|
||||
return `
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM desktop_tasks
|
||||
WHERE tenant_id = $1
|
||||
AND workspace_id = $2
|
||||
AND kind = 'monitor'
|
||||
AND target_client_id = $3
|
||||
AND platform_id = $4
|
||||
AND status = 'in_progress'
|
||||
AND (
|
||||
$5::bigint <= 0
|
||||
OR monitor_task_id IS NULL
|
||||
OR monitor_task_id <> $5
|
||||
)
|
||||
)
|
||||
`
|
||||
}
|
||||
|
||||
func monitorDesktopTaskPlatformSlotLockKey(clientID uuid.UUID, platformID string) int64 {
|
||||
return int64(monitoringDailyStableHash("monitor_desktop_platform_slot", clientID.String(), normalizeMonitoringPlatformID(platformID)))
|
||||
}
|
||||
|
||||
func (s *MonitoringService) loadMonitorDesktopTaskTargets(
|
||||
ctx context.Context,
|
||||
tenantID, workspaceID, userID int64,
|
||||
@@ -721,6 +790,7 @@ func (s *MonitoringService) requestPhase2MonitorInterrupts(
|
||||
ctx context.Context,
|
||||
targetClientID uuid.UUID,
|
||||
excludedMonitorTaskIDs []int64,
|
||||
excludedPlatformIDs []string,
|
||||
interruptGeneration int,
|
||||
) ([]monitorDesktopInterruptTarget, error) {
|
||||
if s == nil || s.businessPool == nil {
|
||||
@@ -728,6 +798,7 @@ func (s *MonitoringService) requestPhase2MonitorInterrupts(
|
||||
}
|
||||
|
||||
queryArgs := []any{targetClientID, interruptGeneration}
|
||||
nextParam := 3
|
||||
query := `
|
||||
UPDATE desktop_tasks
|
||||
SET control_flags = (
|
||||
@@ -752,8 +823,14 @@ func (s *MonitoringService) requestPhase2MonitorInterrupts(
|
||||
AND monitor_task_id IS NOT NULL
|
||||
`
|
||||
if len(excludedMonitorTaskIDs) > 0 {
|
||||
query += ` AND NOT (monitor_task_id = ANY($3::bigint[]))`
|
||||
query += fmt.Sprintf(` AND NOT (monitor_task_id = ANY($%d::bigint[]))`, nextParam)
|
||||
queryArgs = append(queryArgs, excludedMonitorTaskIDs)
|
||||
nextParam++
|
||||
}
|
||||
excludedPlatformIDs = reconcileEnabledMonitoringPlatforms(excludedPlatformIDs)
|
||||
if len(excludedPlatformIDs) > 0 {
|
||||
query += fmt.Sprintf(` AND NOT (platform_id = ANY($%d::text[]))`, nextParam)
|
||||
queryArgs = append(queryArgs, excludedPlatformIDs)
|
||||
}
|
||||
query += `
|
||||
RETURNING desktop_id::text, monitor_task_id, target_client_id::text, workspace_id, interrupt_generation
|
||||
@@ -981,18 +1058,18 @@ func monitoringSchedulerGroupKey(questionHash string, questionID int64, question
|
||||
}
|
||||
|
||||
func marshalMonitorDesktopTaskPayload(spec monitorDesktopTaskSpec) ([]byte, error) {
|
||||
title := strings.TrimSpace(spec.QuestionText)
|
||||
platformName := platformDisplayName(spec.PlatformID)
|
||||
if title != "" {
|
||||
title = fmt.Sprintf("%s · %s", platformName, title)
|
||||
} else {
|
||||
title = fmt.Sprintf("监控任务 · %s", platformName)
|
||||
questionText := strings.TrimSpace(spec.QuestionText)
|
||||
if questionText == "" {
|
||||
return nil, response.ErrInternal(50041, "missing_question_text_snapshot", "monitor desktop task question text is empty")
|
||||
}
|
||||
title := questionText
|
||||
platformName := platformDisplayName(spec.PlatformID)
|
||||
title = fmt.Sprintf("%s · %s", platformName, title)
|
||||
payload, err := json.Marshal(map[string]any{
|
||||
"title": title,
|
||||
"business_date": spec.BusinessDate,
|
||||
"scheduler_group_key": spec.SchedulerGroupKey,
|
||||
"question_text": spec.QuestionText,
|
||||
"question_text": questionText,
|
||||
"question_id": spec.QuestionID,
|
||||
"question_hash": spec.QuestionHash,
|
||||
"ai_platform_id": spec.PlatformID,
|
||||
|
||||
Reference in New Issue
Block a user