root/geo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix monitoring daily backpressure

Browse Source
This commit is contained in:
liangxu
2026-06-27 15:31:08 +08:00
parent 58fe70268d
commit 940ac23493
10 changed files with 550 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/tenant/app/monitoring_phase2_desktop_tasks.go
+212 -15
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"time"
@@ -203,15 +204,34 @@ func (s *MonitoringService) loadMonitorDesktopTaskSpecs(
func (s *MonitoringService) dispatchMonitorDesktopTasks(
ctx context.Context,
specs []monitorDesktopTaskSpec,
maxClientBacklog ...int,
) ([]*repository.DesktopTask, []int64, error) {
if s == nil || s.businessPool == nil || len(specs) == 0 {
return nil, nil, nil
}
backlogLimit := 0
if len(maxClientBacklog) > 0 && maxClientBacklog[0] > 0 {
backlogLimit = maxClientBacklog[0]
}
var targets map[string]monitorDesktopTaskTarget
if monitorDesktopTaskSpecsNeedTargetLookup(specs) {
var err error
targets, err = s.loadMonitorDesktopTaskTargets(ctx, specs[0].TenantID, specs[0].WorkspaceID, specs[0].RequestedByUserID, specs)
targets, err = s.loadMonitorDesktopTaskTargets(ctx, specs[0].TenantID, specs[0].WorkspaceID, specs[0].RequestedByUserID, specs, backlogLimit)
if err != nil {
return nil, nil, err
}
}
clientBacklog := make(map[uuid.UUID]int)
if backlogLimit > 0 {
var err error
clientBacklog, err = s.loadMonitorDesktopClientBacklog(
ctx,
specs[0].TenantID,
specs[0].WorkspaceID,
monitorDesktopTaskDispatchClientIDs(specs, targets),
backlogLimit,
)
if err != nil {
return nil, nil, err
}
@@ -237,11 +257,22 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
spec.TargetClientID = target.ClientID
}
task, shouldPublish, fallbackLegacy, upsertErr := s.upsertMonitorDesktopTask(ctx, tx, repo, spec)
task, shouldPublish, shouldDefer, upsertErr := s.upsertMonitorDesktopTask(
ctx,
tx,
repo,
spec,
func(oldClientID, newClientID uuid.UUID) bool {
return reserveMonitorDesktopClientBacklog(oldClientID, newClientID, clientBacklog, backlogLimit)
},
func(oldClientID, newClientID uuid.UUID) (bool, error) {
return s.reserveMonitorDesktopClientBacklogTx(ctx, tx, spec.TenantID, spec.WorkspaceID, oldClientID, newClientID, backlogLimit)
},
)
if upsertErr != nil {
return nil, nil, upsertErr
}
if fallbackLegacy {
if shouldDefer {
deferredTaskIDs = append(deferredTaskIDs, spec.MonitorTaskID)
continue
}
@@ -262,6 +293,26 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
return publishableTasks, deferredTaskIDs, nil
}
func monitorDesktopTaskDispatchClientIDs(specs []monitorDesktopTaskSpec, targets map[string]monitorDesktopTaskTarget) []uuid.UUID {
if len(specs) == 0 {
return nil
}
result := make([]uuid.UUID, 0, len(specs)*2)
for _, spec := range specs {
if spec.TargetClientID != uuid.Nil {
result = append(result, spec.TargetClientID)
}
if targets == nil {
continue
}
target, ok := targets[normalizeMonitoringPlatformID(spec.PlatformID)]
if ok && target.ClientID != uuid.Nil {
result = append(result, target.ClientID)
}
}
return uniqueUUIDs(result)
}
func monitorDesktopTaskSpecsNeedTargetLookup(specs []monitorDesktopTaskSpec) bool {
for _, spec := range specs {
if spec.TargetAccountID == uuid.Nil || spec.TargetClientID == uuid.Nil {
@@ -279,11 +330,16 @@ func (s *MonitoringService) loadMonitorDesktopTaskTargets(
ctx context.Context,
tenantID, workspaceID, userID int64,
specs []monitorDesktopTaskSpec,
maxClientBacklog ...int,
) (map[string]monitorDesktopTaskTarget, error) {
platformIDs := uniqueMonitorDesktopTaskPlatformIDs(specs)
if len(platformIDs) == 0 {
return nil, nil
}
backlogLimit := 0
if len(maxClientBacklog) > 0 && maxClientBacklog[0] > 0 {
backlogLimit = maxClientBacklog[0]
}
rows, err := s.businessPool.Query(ctx, `
SELECT
@@ -373,7 +429,11 @@ func (s *MonitoringService) loadMonitorDesktopTaskTargets(
if err != nil {
return nil, err
}
return selectMonitorDesktopTaskTargets(candidates, accountPresence, onlineClients), nil
activeBacklog, err := s.loadMonitorDesktopClientBacklog(ctx, tenantID, workspaceID, clientIDs, backlogLimit)
if err != nil {
return nil, err
}
return selectMonitorDesktopTaskTargets(candidates, accountPresence, onlineClients, activeBacklog, backlogLimit), nil
}
func uniqueMonitorDesktopTaskPlatformIDs(specs []monitorDesktopTaskSpec) []string {
@@ -463,13 +523,58 @@ func (s *MonitoringService) loadOnlineMonitorDesktopClients(ctx context.Context,
return result, nil
}
func (s *MonitoringService) loadMonitorDesktopClientBacklog(
ctx context.Context,
tenantID, workspaceID int64,
clientIDs []uuid.UUID,
limit int,
) (map[uuid.UUID]int, error) {
result := make(map[uuid.UUID]int)
clientIDs = uniqueUUIDs(clientIDs)
if s == nil || s.businessPool == nil || limit <= 0 || len(clientIDs) == 0 {
return result, nil
}
rows, err := s.businessPool.Query(ctx, `
SELECT target_client_id, COUNT(*)::int
FROM desktop_tasks
WHERE tenant_id = $1
AND workspace_id = $2
AND target_client_id = ANY($3::uuid[])
AND kind = 'monitor'
AND status IN ('queued', 'in_progress')
GROUP BY target_client_id
`, tenantID, workspaceID, clientIDs)
if err != nil {
return nil, response.ErrInternal(50117, "desktop_client_backlog_lookup_failed", "failed to resolve monitor desktop client backlog")
}
defer rows.Close()
for rows.Next() {
var (
clientID uuid.UUID
count int
)
if scanErr := rows.Scan(&clientID, &count); scanErr != nil {
return nil, response.ErrInternal(50117, "desktop_client_backlog_lookup_failed", "failed to parse monitor desktop client backlog")
}
result[clientID] = count
}
if err := rows.Err(); err != nil {
return nil, response.ErrInternal(50117, "desktop_client_backlog_lookup_failed", "failed to iterate monitor desktop client backlog")
}
return result, nil
}
func selectMonitorDesktopTaskTargets(
candidates []monitorDesktopAccountCandidate,
accountPresence map[uuid.UUID][]desktopAccountPresenceClient,
onlineClients map[uuid.UUID]bool,
clientBacklog map[uuid.UUID]int,
maxClientBacklog int,
) map[string]monitorDesktopTaskTarget {
result := make(map[string]monitorDesktopTaskTarget)
bestByPlatform := make(map[string]monitorDesktopTaskTargetCandidate)
candidatesByPlatform := make(map[string][]monitorDesktopTaskTargetCandidate)
for _, candidate := range candidates {
platformID := normalizeMonitoringPlatformID(candidate.PlatformID)
if platformID == "" || candidate.AccountID == uuid.Nil {
@@ -495,9 +600,7 @@ func selectMonitorDesktopTaskTargets(
RecordOrder: candidate.RecordOrder,
ClientOrder: index,
}
if current, ok := bestByPlatform[platformID]; !ok || item.betterThan(current) {
bestByPlatform[platformID] = item
}
candidatesByPlatform[platformID] = append(candidatesByPlatform[platformID], item)
}
}
if candidate.DBClientID == nil || *candidate.DBClientID == uuid.Nil || !onlineClients[*candidate.DBClientID] {
@@ -516,16 +619,83 @@ func selectMonitorDesktopTaskTargets(
SourceRank: 1,
RecordOrder: candidate.RecordOrder,
}
if current, ok := bestByPlatform[platformID]; !ok || item.betterThan(current) {
bestByPlatform[platformID] = item
candidatesByPlatform[platformID] = append(candidatesByPlatform[platformID], item)
}
platformIDs := make([]string, 0, len(candidatesByPlatform))
for platformID := range candidatesByPlatform {
platformIDs = append(platformIDs, platformID)
}
sort.Strings(platformIDs)
for _, platformID := range platformIDs {
platformCandidates := candidatesByPlatform[platformID]
sort.SliceStable(platformCandidates, func(i, j int) bool {
return platformCandidates[i].betterThan(platformCandidates[j])
})
for _, candidate := range platformCandidates {
if monitorDesktopClientBacklogFull(candidate.Target.ClientID, clientBacklog, maxClientBacklog) {
continue
}
result[platformID] = candidate.Target
if maxClientBacklog > 0 {
clientBacklog[candidate.Target.ClientID]++
}
break
}
}
for platformID, candidate := range bestByPlatform {
result[platformID] = candidate.Target
}
return result
}
func monitorDesktopClientBacklogFull(clientID uuid.UUID, clientBacklog map[uuid.UUID]int, maxClientBacklog int) bool {
return maxClientBacklog > 0 && clientID != uuid.Nil && clientBacklog[clientID] >= maxClientBacklog
}
func reserveMonitorDesktopClientBacklog(oldClientID, newClientID uuid.UUID, clientBacklog map[uuid.UUID]int, maxClientBacklog int) bool {
if maxClientBacklog <= 0 || newClientID == uuid.Nil || oldClientID == newClientID {
return true
}
if clientBacklog == nil {
return true
}
if monitorDesktopClientBacklogFull(newClientID, clientBacklog, maxClientBacklog) {
return false
}
clientBacklog[newClientID]++
if oldClientID != uuid.Nil && clientBacklog[oldClientID] > 0 {
clientBacklog[oldClientID]--
}
return true
}
func (s *MonitoringService) reserveMonitorDesktopClientBacklogTx(
ctx context.Context,
tx pgx.Tx,
tenantID, workspaceID int64,
oldClientID, newClientID uuid.UUID,
maxClientBacklog int,
) (bool, error) {
if tx == nil || maxClientBacklog <= 0 || newClientID == uuid.Nil || oldClientID == newClientID {
return true, nil
}
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, monitorDesktopTaskClientSlotLockKey(newClientID)); err != nil {
return false, response.ErrInternal(50123, "desktop_client_backlog_lock_failed", "failed to lock monitor desktop client backlog")
}
var backlog int
if err := tx.QueryRow(ctx, `
SELECT COUNT(*)::int
FROM desktop_tasks
WHERE tenant_id = $1
AND workspace_id = $2
AND target_client_id = $3
AND kind = 'monitor'
AND status IN ('queued', 'in_progress')
`, tenantID, workspaceID, newClientID).Scan(&backlog); err != nil {
return false, response.ErrInternal(50124, "desktop_client_backlog_check_failed", "failed to check monitor desktop client backlog")
}
return backlog < maxClientBacklog, nil
}
func (candidate monitorDesktopTaskTargetCandidate) betterThan(current monitorDesktopTaskTargetCandidate) bool {
if candidate.HealthRank != current.HealthRank {
return candidate.HealthRank < current.HealthRank
@@ -559,14 +729,17 @@ func (s *MonitoringService) upsertMonitorDesktopTask(
tx pgx.Tx,
repo repository.DesktopTaskRepository,
spec monitorDesktopTaskSpec,
reserveClientBacklog func(oldClientID, newClientID uuid.UUID) bool,
reserveClientBacklogTx func(oldClientID, newClientID uuid.UUID) (bool, error),
) (*repository.DesktopTask, bool, bool, error) {
var (
existingDesktopID uuid.UUID
existingTargetClientID uuid.UUID
existingStatus string
existingLeaseExpiresAt pgtype.Timestamptz
existingActiveAttempt pgtype.UUID
)
err := tx.QueryRow(ctx, upsertMonitorDesktopTaskActiveLookupSQL(), spec.MonitorTaskID).Scan(&existingDesktopID, &existingStatus, &existingLeaseExpiresAt, &existingActiveAttempt)
err := tx.QueryRow(ctx, upsertMonitorDesktopTaskActiveLookupSQL(), spec.MonitorTaskID).Scan(&existingDesktopID, &existingTargetClientID, &existingStatus, &existingLeaseExpiresAt, &existingActiveAttempt)
switch err {
case nil:
task, getErr := repo.GetByDesktopID(ctx, existingDesktopID, spec.WorkspaceID)
@@ -583,6 +756,18 @@ func (s *MonitoringService) upsertMonitorDesktopTask(
if targetAccountID == uuid.Nil {
targetAccountID = task.TargetAccountID
}
if reserveClientBacklog != nil && !reserveClientBacklog(existingTargetClientID, spec.TargetClientID) {
return nil, false, true, nil
}
if reserveClientBacklogTx != nil {
allowed, reserveErr := reserveClientBacklogTx(existingTargetClientID, spec.TargetClientID)
if reserveErr != nil {
return nil, false, false, reserveErr
}
if !allowed {
return nil, false, true, nil
}
}
payloadJSON, payloadErr := marshalMonitorDesktopTaskPayload(spec)
if payloadErr != nil {
return nil, false, false, payloadErr
@@ -658,6 +843,18 @@ func (s *MonitoringService) upsertMonitorDesktopTask(
if targetAccountID == uuid.Nil || spec.TargetClientID == uuid.Nil {
return nil, false, true, nil
}
if reserveClientBacklog != nil && !reserveClientBacklog(uuid.Nil, spec.TargetClientID) {
return nil, false, true, nil
}
if reserveClientBacklogTx != nil {
allowed, reserveErr := reserveClientBacklogTx(uuid.Nil, spec.TargetClientID)
if reserveErr != nil {
return nil, false, false, reserveErr
}
if !allowed {
return nil, false, true, nil
}
}
payloadJSON, payloadErr := marshalMonitorDesktopTaskPayload(spec)
if payloadErr != nil {
return nil, false, false, payloadErr
@@ -718,7 +915,7 @@ func (s *MonitoringService) upsertMonitorDesktopTask(
func upsertMonitorDesktopTaskActiveLookupSQL() string {
return `
SELECT desktop_id, status, lease_expires_at, active_attempt_id
SELECT desktop_id, target_client_id, status, lease_expires_at, active_attempt_id
FROM desktop_tasks
WHERE kind = 'monitor'
AND monitor_task_id = $1