feat(monitoring): generate daily tasks and optimize heartbeat writes
Adds a MonitoringDailyTaskWorker that materializes and dispatches daily collect tasks from active subscription/desktop-client state, with per-run atomic metrics and a Prometheus collector. Dashboard and question-detail APIs now surface a platform_authorization_status so the UI can distinguish no-desktop-client, no-authorized-platforms, and authorized states; quota/access-state lookups resolve by request workspace instead of tenant. Hardens heartbeat: a desktop_client_primary_leases table gives per-(tenant, workspace) sticky primary selection, read-mostly lease resolution avoids a per-heartbeat transaction, and unchanged platform access reports skip snapshot upserts outside a 10-minute refresh window. Adds heartbeat primary/snapshot metrics exposed via token-protected tenant-api /api/internal/metrics endpoints plus Prometheus. Monitoring quota seed is removed from dev-seed since daily plans now derive from desktop bindings, and the projection uses the canonical six-platform catalog so platforms with zero samples still render. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -2,9 +2,287 @@ package app
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestDecideHeartbeatPrimaryLease(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
state heartbeatPrimaryLeaseState
|
||||
want heartbeatPrimaryLeaseAction
|
||||
}{
|
||||
{
|
||||
name: "current primary fast path",
|
||||
state: heartbeatPrimaryLeaseState{
|
||||
Found: true,
|
||||
CurrentIsPrimary: true,
|
||||
LeaseLive: true,
|
||||
PrimaryAvailable: true,
|
||||
},
|
||||
want: heartbeatPrimaryLeaseActionPrimaryHit,
|
||||
},
|
||||
{
|
||||
name: "current primary renews when due",
|
||||
state: heartbeatPrimaryLeaseState{
|
||||
Found: true,
|
||||
CurrentIsPrimary: true,
|
||||
LeaseLive: true,
|
||||
RenewDue: true,
|
||||
PrimaryAvailable: true,
|
||||
},
|
||||
want: heartbeatPrimaryLeaseActionRenew,
|
||||
},
|
||||
{
|
||||
name: "non primary does not steal live lease",
|
||||
state: heartbeatPrimaryLeaseState{
|
||||
Found: true,
|
||||
CurrentIsPrimary: false,
|
||||
LeaseLive: true,
|
||||
PrimaryAvailable: true,
|
||||
},
|
||||
want: heartbeatPrimaryLeaseActionNonPrimaryLive,
|
||||
},
|
||||
{
|
||||
name: "expired lease can be claimed",
|
||||
state: heartbeatPrimaryLeaseState{
|
||||
Found: true,
|
||||
CurrentIsPrimary: false,
|
||||
LeaseLive: false,
|
||||
PrimaryAvailable: true,
|
||||
},
|
||||
want: heartbeatPrimaryLeaseActionClaimExpired,
|
||||
},
|
||||
{
|
||||
name: "revoked primary can be replaced",
|
||||
state: heartbeatPrimaryLeaseState{
|
||||
Found: true,
|
||||
CurrentIsPrimary: false,
|
||||
LeaseLive: true,
|
||||
PrimaryAvailable: false,
|
||||
},
|
||||
want: heartbeatPrimaryLeaseActionClaimUnavailable,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := decideHeartbeatPrimaryLease(tt.state)
|
||||
if got.Action != tt.want {
|
||||
t.Fatalf("decideHeartbeatPrimaryLease() = %v, want %v", got.Action, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitoringHeartbeatPrimaryLeaseMetrics(t *testing.T) {
|
||||
resetMonitoringHeartbeatPrimaryLeaseMetricsForTest()
|
||||
t.Cleanup(resetMonitoringHeartbeatPrimaryLeaseMetricsForTest)
|
||||
|
||||
recordHeartbeatPrimaryLeaseMetric("primary_hit", 20*time.Millisecond)
|
||||
recordHeartbeatPrimaryLeaseMetric("contention", 10*time.Millisecond)
|
||||
recordHeartbeatPrimaryLeaseWriteMetric("renew")
|
||||
|
||||
snapshot := MonitoringHeartbeatPrimaryLeaseMetricsSnapshotValue()
|
||||
if snapshot.PrimaryHitTotal != 1 {
|
||||
t.Fatalf("PrimaryHitTotal = %d, want 1", snapshot.PrimaryHitTotal)
|
||||
}
|
||||
if snapshot.ContentionTotal != 1 {
|
||||
t.Fatalf("ContentionTotal = %d, want 1", snapshot.ContentionTotal)
|
||||
}
|
||||
if snapshot.DBRenewTotal != 1 {
|
||||
t.Fatalf("DBRenewTotal = %d, want 1", snapshot.DBRenewTotal)
|
||||
}
|
||||
if snapshot.DurationCount != 2 {
|
||||
t.Fatalf("DurationCount = %d, want 2", snapshot.DurationCount)
|
||||
}
|
||||
|
||||
prometheus := MonitoringHeartbeatPrimaryLeaseMetricsPrometheus()
|
||||
if !strings.Contains(prometheus, `monitoring_heartbeat_primary_lease_resolutions_total{outcome="primary_hit"} 1`) {
|
||||
t.Fatalf("prometheus output missing primary_hit metric: %s", prometheus)
|
||||
}
|
||||
if !strings.Contains(prometheus, `monitoring_heartbeat_primary_lease_db_writes_total{operation="renew"} 1`) {
|
||||
t.Fatalf("prometheus output missing renew write metric: %s", prometheus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecideMonitoringHeartbeatSnapshot(t *testing.T) {
|
||||
now := time.Date(2026, 4, 24, 3, 0, 0, 0, time.UTC)
|
||||
base := monitoringPlatformAccessSnapshotState{
|
||||
AccessStatus: "accessible",
|
||||
Connected: true,
|
||||
Accessible: true,
|
||||
UpdatedAt: now.Add(-time.Minute),
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
previous *monitoringPlatformAccessSnapshotState
|
||||
next monitoringPlatformAccessSnapshotState
|
||||
primary bool
|
||||
want monitoringHeartbeatSnapshotDecision
|
||||
}{
|
||||
{
|
||||
name: "missing snapshot inserts",
|
||||
next: base,
|
||||
primary: true,
|
||||
want: monitoringHeartbeatSnapshotDecision{
|
||||
WriteSnapshot: true,
|
||||
StateChanged: true,
|
||||
SnapshotOperation: "insert",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "fresh unchanged accessible snapshot skips write",
|
||||
previous: &base,
|
||||
next: base,
|
||||
primary: true,
|
||||
want: monitoringHeartbeatSnapshotDecision{
|
||||
SnapshotOperation: "skip",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "stale unchanged snapshot refreshes",
|
||||
previous: &monitoringPlatformAccessSnapshotState{
|
||||
AccessStatus: "accessible",
|
||||
Connected: true,
|
||||
Accessible: true,
|
||||
UpdatedAt: now.Add(-monitoringHeartbeatAccessSnapshotRefreshAfter - time.Second),
|
||||
},
|
||||
next: base,
|
||||
primary: true,
|
||||
want: monitoringHeartbeatSnapshotDecision{
|
||||
WriteSnapshot: true,
|
||||
RefreshDue: true,
|
||||
SnapshotOperation: "refresh",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "changed snapshot updates",
|
||||
previous: &base,
|
||||
next: monitoringPlatformAccessSnapshotState{
|
||||
AccessStatus: "unavailable",
|
||||
Connected: true,
|
||||
Accessible: false,
|
||||
},
|
||||
primary: true,
|
||||
want: monitoringHeartbeatSnapshotDecision{
|
||||
WriteSnapshot: true,
|
||||
StateChanged: true,
|
||||
PendingReconcileCandidate: true,
|
||||
SnapshotOperation: "update",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "fresh unchanged primary inaccessible snapshot checks pending tasks without writing snapshot",
|
||||
previous: &monitoringPlatformAccessSnapshotState{
|
||||
AccessStatus: "not_logged_in",
|
||||
Connected: false,
|
||||
Accessible: false,
|
||||
UpdatedAt: now.Add(-time.Minute),
|
||||
},
|
||||
next: monitoringPlatformAccessSnapshotState{
|
||||
AccessStatus: "not_logged_in",
|
||||
Connected: false,
|
||||
Accessible: false,
|
||||
},
|
||||
primary: true,
|
||||
want: monitoringHeartbeatSnapshotDecision{
|
||||
PendingReconcileCandidate: true,
|
||||
SnapshotOperation: "skip",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "non-primary inaccessible snapshot does not reconcile",
|
||||
previous: &monitoringPlatformAccessSnapshotState{
|
||||
AccessStatus: "not_logged_in",
|
||||
Connected: false,
|
||||
Accessible: false,
|
||||
UpdatedAt: now.Add(-time.Minute),
|
||||
},
|
||||
next: monitoringPlatformAccessSnapshotState{
|
||||
AccessStatus: "not_logged_in",
|
||||
Connected: false,
|
||||
Accessible: false,
|
||||
},
|
||||
primary: false,
|
||||
want: monitoringHeartbeatSnapshotDecision{
|
||||
SnapshotOperation: "skip",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := decideMonitoringHeartbeatSnapshot(tt.previous, tt.next, now, tt.primary)
|
||||
if got.WriteSnapshot != tt.want.WriteSnapshot ||
|
||||
got.StateChanged != tt.want.StateChanged ||
|
||||
got.RefreshDue != tt.want.RefreshDue ||
|
||||
got.PendingReconcileCandidate != tt.want.PendingReconcileCandidate ||
|
||||
got.ReconcilePending != tt.want.ReconcilePending ||
|
||||
got.SnapshotOperation != tt.want.SnapshotOperation {
|
||||
t.Fatalf("decideMonitoringHeartbeatSnapshot() = %+v, want %+v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitoringHeartbeatSnapshotMetrics(t *testing.T) {
|
||||
resetMonitoringHeartbeatSnapshotMetricsForTest()
|
||||
t.Cleanup(resetMonitoringHeartbeatSnapshotMetricsForTest)
|
||||
|
||||
recordMonitoringHeartbeatSnapshotMetrics(monitoringHeartbeatSnapshotMetricsEvent{
|
||||
PlatformCount: 3,
|
||||
TransactionOpened: true,
|
||||
SnapshotInsertedCount: 1,
|
||||
SnapshotSkippedCount: 2,
|
||||
ReconciledSkippedTaskCount: 5,
|
||||
ProjectionRebuildCount: 2,
|
||||
Duration: 20 * time.Millisecond,
|
||||
})
|
||||
recordMonitoringHeartbeatSnapshotMetrics(monitoringHeartbeatSnapshotMetricsEvent{
|
||||
PlatformCount: 1,
|
||||
Duration: 10 * time.Millisecond,
|
||||
Err: errors.New("snapshot failed"),
|
||||
})
|
||||
|
||||
snapshot := MonitoringHeartbeatSnapshotMetricsSnapshotValue()
|
||||
if snapshot.PlatformReportedTotal != 4 {
|
||||
t.Fatalf("PlatformReportedTotal = %d, want 4", snapshot.PlatformReportedTotal)
|
||||
}
|
||||
if snapshot.TransactionOpenedTotal != 1 {
|
||||
t.Fatalf("TransactionOpenedTotal = %d, want 1", snapshot.TransactionOpenedTotal)
|
||||
}
|
||||
if snapshot.SnapshotInsertedTotal != 1 {
|
||||
t.Fatalf("SnapshotInsertedTotal = %d, want 1", snapshot.SnapshotInsertedTotal)
|
||||
}
|
||||
if snapshot.SnapshotSkippedTotal != 2 {
|
||||
t.Fatalf("SnapshotSkippedTotal = %d, want 2", snapshot.SnapshotSkippedTotal)
|
||||
}
|
||||
if snapshot.ReconciledSkippedTaskTotal != 5 {
|
||||
t.Fatalf("ReconciledSkippedTaskTotal = %d, want 5", snapshot.ReconciledSkippedTaskTotal)
|
||||
}
|
||||
if snapshot.ProjectionRebuildTotal != 2 {
|
||||
t.Fatalf("ProjectionRebuildTotal = %d, want 2", snapshot.ProjectionRebuildTotal)
|
||||
}
|
||||
if snapshot.ErrorTotal != 1 {
|
||||
t.Fatalf("ErrorTotal = %d, want 1", snapshot.ErrorTotal)
|
||||
}
|
||||
if snapshot.DurationCount != 2 {
|
||||
t.Fatalf("DurationCount = %d, want 2", snapshot.DurationCount)
|
||||
}
|
||||
|
||||
prometheus := MonitoringHeartbeatSnapshotMetricsPrometheus()
|
||||
if !strings.Contains(prometheus, `monitoring_heartbeat_snapshot_writes_total{operation="insert"} 1`) {
|
||||
t.Fatalf("prometheus output missing insert write metric: %s", prometheus)
|
||||
}
|
||||
if !strings.Contains(prometheus, "monitoring_heartbeat_snapshot_transactions_total 1") {
|
||||
t.Fatalf("prometheus output missing transaction metric: %s", prometheus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMonitoringRawPayloadKimiExcludesSearchResultsFromCitationInputs(t *testing.T) {
|
||||
citationTitle := "真实引用文章"
|
||||
searchTitle := "搜索网页结果"
|
||||
|
||||
Reference in New Issue
Block a user