Files
geo/server/internal/tenant/app/desktop_presence.go
T
root ff86933c24 feat(monitoring): generate daily tasks and optimize heartbeat writes
Adds a MonitoringDailyTaskWorker that materializes and dispatches
daily collect tasks from active subscription/desktop-client state,
with per-run atomic metrics and a Prometheus collector. Dashboard
and question-detail APIs now surface a platform_authorization_status
so the UI can distinguish no-desktop-client, no-authorized-platforms,
and authorized states; quota/access-state lookups resolve by request
workspace instead of tenant.

Hardens heartbeat: a desktop_client_primary_leases table gives
per-(tenant, workspace) sticky primary selection, read-mostly lease
resolution avoids a per-heartbeat transaction, and unchanged platform
access reports skip snapshot upserts outside a 10-minute refresh
window. Adds heartbeat primary/snapshot metrics exposed via
token-protected tenant-api /api/internal/metrics endpoints plus
Prometheus. Monitoring quota seed is removed from dev-seed since
daily plans now derive from desktop bindings, and the projection
uses the canonical six-platform catalog so platforms with zero
samples still render.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-24 22:19:57 +08:00

255 lines
7.0 KiB
Go

package app
import (
"context"
"strconv"
"time"
"github.com/google/uuid"
goredis "github.com/redis/go-redis/v9"
)
// desktopClientPresenceTTL is also used as the monitoring primary-client lease TTL.
// Desktop heartbeat interval must stay below TTL/2, so one missed heartbeat does
// not make another client eligible to steal a still-healthy primary lease.
const desktopClientPresenceTTL = 90 * time.Second
const desktopAccountPresenceTTL = 90 * time.Second
func desktopClientPresenceKey(clientID uuid.UUID) string {
return "desktop:presence:client:" + clientID.String()
}
func desktopAccountPresenceKey(accountID uuid.UUID) string {
return "desktop:presence:account:" + accountID.String()
}
func desktopClientAccountsPresenceKey(clientID uuid.UUID) string {
return "desktop:presence:client_accounts:" + clientID.String()
}
func markDesktopClientPresent(ctx context.Context, redis *goredis.Client, clientID uuid.UUID) {
if redis == nil {
return
}
_ = redis.Set(ctx, desktopClientPresenceKey(clientID), time.Now().UTC().Format(time.RFC3339), desktopClientPresenceTTL).Err()
}
func clearDesktopClientPresence(ctx context.Context, redis *goredis.Client, clientID uuid.UUID) {
if redis == nil {
return
}
_ = redis.Del(ctx, desktopClientPresenceKey(clientID)).Err()
}
func loadDesktopClientPresence(ctx context.Context, redis *goredis.Client, clientIDs []uuid.UUID) map[uuid.UUID]bool {
if redis == nil || len(clientIDs) == 0 {
return nil
}
uniqueIDs := make([]uuid.UUID, 0, len(clientIDs))
seen := make(map[uuid.UUID]struct{}, len(clientIDs))
for _, clientID := range clientIDs {
if _, ok := seen[clientID]; ok {
continue
}
seen[clientID] = struct{}{}
uniqueIDs = append(uniqueIDs, clientID)
}
pipe := redis.Pipeline()
cmds := make(map[uuid.UUID]*goredis.IntCmd, len(uniqueIDs))
for _, clientID := range uniqueIDs {
cmds[clientID] = pipe.Exists(ctx, desktopClientPresenceKey(clientID))
}
if _, err := pipe.Exec(ctx); err != nil && err != goredis.Nil {
return nil
}
result := make(map[uuid.UUID]bool, len(cmds))
for clientID, cmd := range cmds {
result[clientID] = cmd.Val() > 0
}
return result
}
func markDesktopAccountsPresent(ctx context.Context, redis *goredis.Client, clientID uuid.UUID, accountIDs []uuid.UUID) {
if redis == nil {
return
}
uniqueIDs := uniqueUUIDs(accountIDs)
trackedAccountIDs := loadTrackedDesktopClientAccountIDs(ctx, redis, clientID)
nextAccountSet := make(map[uuid.UUID]struct{}, len(uniqueIDs))
for _, accountID := range uniqueIDs {
nextAccountSet[accountID] = struct{}{}
}
now := float64(time.Now().UTC().UnixMilli())
staleBefore := strconv.FormatInt(time.Now().UTC().Add(-desktopAccountPresenceTTL).UnixMilli(), 10)
clientAccountsKey := desktopClientAccountsPresenceKey(clientID)
clientMember := clientID.String()
pipe := redis.Pipeline()
pipe.Del(ctx, clientAccountsKey)
if len(uniqueIDs) > 0 {
members := make([]interface{}, 0, len(uniqueIDs))
for _, accountID := range uniqueIDs {
members = append(members, accountID.String())
}
pipe.SAdd(ctx, clientAccountsKey, members...)
pipe.Expire(ctx, clientAccountsKey, desktopAccountPresenceTTL*4)
}
for _, accountID := range trackedAccountIDs {
if _, ok := nextAccountSet[accountID]; ok {
continue
}
pipe.ZRem(ctx, desktopAccountPresenceKey(accountID), clientMember)
}
for _, accountID := range uniqueIDs {
key := desktopAccountPresenceKey(accountID)
pipe.ZRemRangeByScore(ctx, key, "-inf", staleBefore)
pipe.ZAdd(ctx, key, goredis.Z{
Score: now,
Member: clientMember,
})
pipe.Expire(ctx, key, desktopAccountPresenceTTL*4)
}
_, _ = pipe.Exec(ctx)
cleanupDesktopAccountPresenceKeys(ctx, redis, trackedAccountIDs)
}
func clearDesktopClientPresenceState(ctx context.Context, redis *goredis.Client, clientID uuid.UUID) {
if redis == nil {
return
}
clearDesktopClientPresence(ctx, redis, clientID)
trackedAccountIDs := loadTrackedDesktopClientAccountIDs(ctx, redis, clientID)
if len(trackedAccountIDs) == 0 {
_ = redis.Del(ctx, desktopClientAccountsPresenceKey(clientID)).Err()
return
}
clientMember := clientID.String()
pipe := redis.Pipeline()
for _, accountID := range trackedAccountIDs {
pipe.ZRem(ctx, desktopAccountPresenceKey(accountID), clientMember)
}
pipe.Del(ctx, desktopClientAccountsPresenceKey(clientID))
_, _ = pipe.Exec(ctx)
cleanupDesktopAccountPresenceKeys(ctx, redis, trackedAccountIDs)
}
func loadDesktopAccountPresence(ctx context.Context, redis *goredis.Client, accountIDs []uuid.UUID) map[uuid.UUID]uuid.UUID {
if redis == nil || len(accountIDs) == 0 {
return nil
}
uniqueIDs := uniqueUUIDs(accountIDs)
staleBefore := strconv.FormatInt(time.Now().UTC().Add(-desktopAccountPresenceTTL).UnixMilli(), 10)
pipe := redis.Pipeline()
commands := make(map[uuid.UUID]*goredis.ZSliceCmd, len(uniqueIDs))
for _, accountID := range uniqueIDs {
key := desktopAccountPresenceKey(accountID)
pipe.ZRemRangeByScore(ctx, key, "-inf", staleBefore)
commands[accountID] = pipe.ZRevRangeWithScores(ctx, key, 0, 0)
}
if _, err := pipe.Exec(ctx); err != nil && err != goredis.Nil {
return nil
}
result := make(map[uuid.UUID]uuid.UUID, len(commands))
for accountID, cmd := range commands {
values := cmd.Val()
if len(values) == 0 {
continue
}
clientIDText, ok := values[0].Member.(string)
if !ok || clientIDText == "" {
continue
}
clientID, err := uuid.Parse(clientIDText)
if err != nil {
continue
}
result[accountID] = clientID
}
return result
}
func uniqueUUIDs(values []uuid.UUID) []uuid.UUID {
if len(values) == 0 {
return nil
}
unique := make([]uuid.UUID, 0, len(values))
seen := make(map[uuid.UUID]struct{}, len(values))
for _, value := range values {
if _, ok := seen[value]; ok {
continue
}
seen[value] = struct{}{}
unique = append(unique, value)
}
return unique
}
func loadTrackedDesktopClientAccountIDs(ctx context.Context, redis *goredis.Client, clientID uuid.UUID) []uuid.UUID {
if redis == nil {
return nil
}
values, err := redis.SMembers(ctx, desktopClientAccountsPresenceKey(clientID)).Result()
if err != nil || len(values) == 0 {
return nil
}
accountIDs := make([]uuid.UUID, 0, len(values))
for _, value := range values {
accountID, parseErr := uuid.Parse(value)
if parseErr != nil {
continue
}
accountIDs = append(accountIDs, accountID)
}
return uniqueUUIDs(accountIDs)
}
func cleanupDesktopAccountPresenceKeys(ctx context.Context, redis *goredis.Client, accountIDs []uuid.UUID) {
if redis == nil || len(accountIDs) == 0 {
return
}
uniqueIDs := uniqueUUIDs(accountIDs)
pipe := redis.Pipeline()
commands := make(map[uuid.UUID]*goredis.IntCmd, len(uniqueIDs))
for _, accountID := range uniqueIDs {
commands[accountID] = pipe.ZCard(ctx, desktopAccountPresenceKey(accountID))
}
if _, err := pipe.Exec(ctx); err != nil && err != goredis.Nil {
return
}
deletePipe := redis.Pipeline()
shouldDelete := false
for accountID, cmd := range commands {
if cmd.Val() > 0 {
continue
}
deletePipe.Del(ctx, desktopAccountPresenceKey(accountID))
shouldDelete = true
}
if shouldDelete {
_, _ = deletePipe.Exec(ctx)
}
}