ff86933c24
Adds a MonitoringDailyTaskWorker that materializes and dispatches daily collect tasks from active subscription/desktop-client state, with per-run atomic metrics and a Prometheus collector. Dashboard and question-detail APIs now surface a platform_authorization_status so the UI can distinguish no-desktop-client, no-authorized-platforms, and authorized states; quota/access-state lookups resolve by request workspace instead of tenant. Hardens heartbeat: a desktop_client_primary_leases table gives per-(tenant, workspace) sticky primary selection, read-mostly lease resolution avoids a per-heartbeat transaction, and unchanged platform access reports skip snapshot upserts outside a 10-minute refresh window. Adds heartbeat primary/snapshot metrics exposed via token-protected tenant-api /api/internal/metrics endpoints plus Prometheus. Monitoring quota seed is removed from dev-seed since daily plans now derive from desktop bindings, and the projection uses the canonical six-platform catalog so platforms with zero samples still render. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
255 lines
7.0 KiB
Go
255 lines
7.0 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
goredis "github.com/redis/go-redis/v9"
|
|
)
|
|
|
|
// desktopClientPresenceTTL is also used as the monitoring primary-client lease TTL.
|
|
// Desktop heartbeat interval must stay below TTL/2, so one missed heartbeat does
|
|
// not make another client eligible to steal a still-healthy primary lease.
|
|
const desktopClientPresenceTTL = 90 * time.Second
|
|
const desktopAccountPresenceTTL = 90 * time.Second
|
|
|
|
func desktopClientPresenceKey(clientID uuid.UUID) string {
|
|
return "desktop:presence:client:" + clientID.String()
|
|
}
|
|
|
|
func desktopAccountPresenceKey(accountID uuid.UUID) string {
|
|
return "desktop:presence:account:" + accountID.String()
|
|
}
|
|
|
|
func desktopClientAccountsPresenceKey(clientID uuid.UUID) string {
|
|
return "desktop:presence:client_accounts:" + clientID.String()
|
|
}
|
|
|
|
func markDesktopClientPresent(ctx context.Context, redis *goredis.Client, clientID uuid.UUID) {
|
|
if redis == nil {
|
|
return
|
|
}
|
|
|
|
_ = redis.Set(ctx, desktopClientPresenceKey(clientID), time.Now().UTC().Format(time.RFC3339), desktopClientPresenceTTL).Err()
|
|
}
|
|
|
|
func clearDesktopClientPresence(ctx context.Context, redis *goredis.Client, clientID uuid.UUID) {
|
|
if redis == nil {
|
|
return
|
|
}
|
|
|
|
_ = redis.Del(ctx, desktopClientPresenceKey(clientID)).Err()
|
|
}
|
|
|
|
func loadDesktopClientPresence(ctx context.Context, redis *goredis.Client, clientIDs []uuid.UUID) map[uuid.UUID]bool {
|
|
if redis == nil || len(clientIDs) == 0 {
|
|
return nil
|
|
}
|
|
|
|
uniqueIDs := make([]uuid.UUID, 0, len(clientIDs))
|
|
seen := make(map[uuid.UUID]struct{}, len(clientIDs))
|
|
for _, clientID := range clientIDs {
|
|
if _, ok := seen[clientID]; ok {
|
|
continue
|
|
}
|
|
seen[clientID] = struct{}{}
|
|
uniqueIDs = append(uniqueIDs, clientID)
|
|
}
|
|
|
|
pipe := redis.Pipeline()
|
|
cmds := make(map[uuid.UUID]*goredis.IntCmd, len(uniqueIDs))
|
|
for _, clientID := range uniqueIDs {
|
|
cmds[clientID] = pipe.Exists(ctx, desktopClientPresenceKey(clientID))
|
|
}
|
|
|
|
if _, err := pipe.Exec(ctx); err != nil && err != goredis.Nil {
|
|
return nil
|
|
}
|
|
|
|
result := make(map[uuid.UUID]bool, len(cmds))
|
|
for clientID, cmd := range cmds {
|
|
result[clientID] = cmd.Val() > 0
|
|
}
|
|
return result
|
|
}
|
|
|
|
func markDesktopAccountsPresent(ctx context.Context, redis *goredis.Client, clientID uuid.UUID, accountIDs []uuid.UUID) {
|
|
if redis == nil {
|
|
return
|
|
}
|
|
|
|
uniqueIDs := uniqueUUIDs(accountIDs)
|
|
trackedAccountIDs := loadTrackedDesktopClientAccountIDs(ctx, redis, clientID)
|
|
nextAccountSet := make(map[uuid.UUID]struct{}, len(uniqueIDs))
|
|
for _, accountID := range uniqueIDs {
|
|
nextAccountSet[accountID] = struct{}{}
|
|
}
|
|
|
|
now := float64(time.Now().UTC().UnixMilli())
|
|
staleBefore := strconv.FormatInt(time.Now().UTC().Add(-desktopAccountPresenceTTL).UnixMilli(), 10)
|
|
clientAccountsKey := desktopClientAccountsPresenceKey(clientID)
|
|
clientMember := clientID.String()
|
|
|
|
pipe := redis.Pipeline()
|
|
pipe.Del(ctx, clientAccountsKey)
|
|
if len(uniqueIDs) > 0 {
|
|
members := make([]interface{}, 0, len(uniqueIDs))
|
|
for _, accountID := range uniqueIDs {
|
|
members = append(members, accountID.String())
|
|
}
|
|
pipe.SAdd(ctx, clientAccountsKey, members...)
|
|
pipe.Expire(ctx, clientAccountsKey, desktopAccountPresenceTTL*4)
|
|
}
|
|
|
|
for _, accountID := range trackedAccountIDs {
|
|
if _, ok := nextAccountSet[accountID]; ok {
|
|
continue
|
|
}
|
|
pipe.ZRem(ctx, desktopAccountPresenceKey(accountID), clientMember)
|
|
}
|
|
for _, accountID := range uniqueIDs {
|
|
key := desktopAccountPresenceKey(accountID)
|
|
pipe.ZRemRangeByScore(ctx, key, "-inf", staleBefore)
|
|
pipe.ZAdd(ctx, key, goredis.Z{
|
|
Score: now,
|
|
Member: clientMember,
|
|
})
|
|
pipe.Expire(ctx, key, desktopAccountPresenceTTL*4)
|
|
}
|
|
_, _ = pipe.Exec(ctx)
|
|
|
|
cleanupDesktopAccountPresenceKeys(ctx, redis, trackedAccountIDs)
|
|
}
|
|
|
|
func clearDesktopClientPresenceState(ctx context.Context, redis *goredis.Client, clientID uuid.UUID) {
|
|
if redis == nil {
|
|
return
|
|
}
|
|
|
|
clearDesktopClientPresence(ctx, redis, clientID)
|
|
|
|
trackedAccountIDs := loadTrackedDesktopClientAccountIDs(ctx, redis, clientID)
|
|
if len(trackedAccountIDs) == 0 {
|
|
_ = redis.Del(ctx, desktopClientAccountsPresenceKey(clientID)).Err()
|
|
return
|
|
}
|
|
|
|
clientMember := clientID.String()
|
|
pipe := redis.Pipeline()
|
|
for _, accountID := range trackedAccountIDs {
|
|
pipe.ZRem(ctx, desktopAccountPresenceKey(accountID), clientMember)
|
|
}
|
|
pipe.Del(ctx, desktopClientAccountsPresenceKey(clientID))
|
|
_, _ = pipe.Exec(ctx)
|
|
|
|
cleanupDesktopAccountPresenceKeys(ctx, redis, trackedAccountIDs)
|
|
}
|
|
|
|
func loadDesktopAccountPresence(ctx context.Context, redis *goredis.Client, accountIDs []uuid.UUID) map[uuid.UUID]uuid.UUID {
|
|
if redis == nil || len(accountIDs) == 0 {
|
|
return nil
|
|
}
|
|
|
|
uniqueIDs := uniqueUUIDs(accountIDs)
|
|
staleBefore := strconv.FormatInt(time.Now().UTC().Add(-desktopAccountPresenceTTL).UnixMilli(), 10)
|
|
|
|
pipe := redis.Pipeline()
|
|
commands := make(map[uuid.UUID]*goredis.ZSliceCmd, len(uniqueIDs))
|
|
for _, accountID := range uniqueIDs {
|
|
key := desktopAccountPresenceKey(accountID)
|
|
pipe.ZRemRangeByScore(ctx, key, "-inf", staleBefore)
|
|
commands[accountID] = pipe.ZRevRangeWithScores(ctx, key, 0, 0)
|
|
}
|
|
|
|
if _, err := pipe.Exec(ctx); err != nil && err != goredis.Nil {
|
|
return nil
|
|
}
|
|
|
|
result := make(map[uuid.UUID]uuid.UUID, len(commands))
|
|
for accountID, cmd := range commands {
|
|
values := cmd.Val()
|
|
if len(values) == 0 {
|
|
continue
|
|
}
|
|
clientIDText, ok := values[0].Member.(string)
|
|
if !ok || clientIDText == "" {
|
|
continue
|
|
}
|
|
clientID, err := uuid.Parse(clientIDText)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
result[accountID] = clientID
|
|
}
|
|
return result
|
|
}
|
|
|
|
func uniqueUUIDs(values []uuid.UUID) []uuid.UUID {
|
|
if len(values) == 0 {
|
|
return nil
|
|
}
|
|
|
|
unique := make([]uuid.UUID, 0, len(values))
|
|
seen := make(map[uuid.UUID]struct{}, len(values))
|
|
for _, value := range values {
|
|
if _, ok := seen[value]; ok {
|
|
continue
|
|
}
|
|
seen[value] = struct{}{}
|
|
unique = append(unique, value)
|
|
}
|
|
return unique
|
|
}
|
|
|
|
func loadTrackedDesktopClientAccountIDs(ctx context.Context, redis *goredis.Client, clientID uuid.UUID) []uuid.UUID {
|
|
if redis == nil {
|
|
return nil
|
|
}
|
|
|
|
values, err := redis.SMembers(ctx, desktopClientAccountsPresenceKey(clientID)).Result()
|
|
if err != nil || len(values) == 0 {
|
|
return nil
|
|
}
|
|
|
|
accountIDs := make([]uuid.UUID, 0, len(values))
|
|
for _, value := range values {
|
|
accountID, parseErr := uuid.Parse(value)
|
|
if parseErr != nil {
|
|
continue
|
|
}
|
|
accountIDs = append(accountIDs, accountID)
|
|
}
|
|
return uniqueUUIDs(accountIDs)
|
|
}
|
|
|
|
func cleanupDesktopAccountPresenceKeys(ctx context.Context, redis *goredis.Client, accountIDs []uuid.UUID) {
|
|
if redis == nil || len(accountIDs) == 0 {
|
|
return
|
|
}
|
|
|
|
uniqueIDs := uniqueUUIDs(accountIDs)
|
|
pipe := redis.Pipeline()
|
|
commands := make(map[uuid.UUID]*goredis.IntCmd, len(uniqueIDs))
|
|
for _, accountID := range uniqueIDs {
|
|
commands[accountID] = pipe.ZCard(ctx, desktopAccountPresenceKey(accountID))
|
|
}
|
|
if _, err := pipe.Exec(ctx); err != nil && err != goredis.Nil {
|
|
return
|
|
}
|
|
|
|
deletePipe := redis.Pipeline()
|
|
shouldDelete := false
|
|
for accountID, cmd := range commands {
|
|
if cmd.Val() > 0 {
|
|
continue
|
|
}
|
|
deletePipe.Del(ctx, desktopAccountPresenceKey(accountID))
|
|
shouldDelete = true
|
|
}
|
|
if shouldDelete {
|
|
_, _ = deletePipe.Exec(ctx)
|
|
}
|
|
}
|