fix monitoring daily backpressure
This commit is contained in:
@@ -344,20 +344,19 @@ func shutdownSchedulerMetricsServer(server *http.Server, logger interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func monitoringDailyRunOptions(jobCtx internalscheduler.JobRunContext) tenantapp.MonitoringDailyTaskRunOptions {
|
func monitoringDailyRunOptions(jobCtx internalscheduler.JobRunContext) tenantapp.MonitoringDailyTaskRunOptions {
|
||||||
batchSize := 0
|
maxDesktopDispatch := 0
|
||||||
if jobCtx.Job != nil && jobCtx.Job.BatchSize != nil {
|
if jobCtx.Job != nil && jobCtx.Job.BatchSize != nil {
|
||||||
batchSize = *jobCtx.Job.BatchSize
|
maxDesktopDispatch = *jobCtx.Job.BatchSize
|
||||||
}
|
}
|
||||||
if batchSize <= 0 {
|
if maxDesktopDispatch <= 0 {
|
||||||
batchSize = internalscheduler.AsInt(jobCtx.Config, "max_materialize_per_run", 0)
|
maxDesktopDispatch = internalscheduler.AsInt(jobCtx.Config, "max_desktop_dispatch_per_run", 0)
|
||||||
}
|
}
|
||||||
if batchSize <= 0 {
|
if maxDesktopDispatch <= 0 {
|
||||||
batchSize = internalscheduler.AsInt(jobCtx.Config, "batch_size", 0)
|
maxDesktopDispatch = internalscheduler.AsInt(jobCtx.Config, "batch_size", 0)
|
||||||
}
|
}
|
||||||
return tenantapp.MonitoringDailyTaskRunOptions{
|
return tenantapp.MonitoringDailyTaskRunOptions{
|
||||||
MaxMaterializePerRun: batchSize,
|
MaxDesktopDispatchPerRun: maxDesktopDispatch,
|
||||||
MaxMaterializePerPlan: internalscheduler.AsInt(jobCtx.Config, "max_materialize_per_plan", 0),
|
MaxDesktopClientBacklog: internalscheduler.AsInt(jobCtx.Config, "max_desktop_client_backlog", 0),
|
||||||
MaxMaterializePerBrand: internalscheduler.AsInt(jobCtx.Config, "max_materialize_per_brand", 0),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -59,22 +59,29 @@ func TestSchedulerMetricsAuthMiddleware(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMonitoringDailyRunOptionsUsesJobBatchSize(t *testing.T) {
|
func TestMonitoringDailyRunOptionsUsesJobBatchSizeForDesktopDispatchOnly(t *testing.T) {
|
||||||
batchSize := 88
|
batchSize := 88
|
||||||
got := monitoringDailyRunOptions(internalscheduler.JobRunContext{
|
got := monitoringDailyRunOptions(internalscheduler.JobRunContext{
|
||||||
Job: &opsdomain.SchedulerJob{BatchSize: &batchSize},
|
Job: &opsdomain.SchedulerJob{BatchSize: &batchSize},
|
||||||
Config: map[string]any{
|
Config: map[string]any{
|
||||||
"max_materialize_per_run": 999,
|
"max_materialize_per_run": 999,
|
||||||
"max_materialize_per_plan": 12,
|
"max_materialize_per_plan": 12,
|
||||||
"max_materialize_per_brand": 4,
|
"max_materialize_per_brand": 4,
|
||||||
|
"max_desktop_client_backlog": 6,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
if got.MaxMaterializePerRun != 88 {
|
if got.MaxDesktopDispatchPerRun != 88 {
|
||||||
t.Fatalf("MaxMaterializePerRun = %d, want 88", got.MaxMaterializePerRun)
|
t.Fatalf("MaxDesktopDispatchPerRun = %d, want 88", got.MaxDesktopDispatchPerRun)
|
||||||
}
|
}
|
||||||
if got.MaxMaterializePerPlan != 12 || got.MaxMaterializePerBrand != 4 {
|
if got.MaxMaterializePerPlan != 0 || got.MaxMaterializePerBrand != 0 {
|
||||||
t.Fatalf("unexpected plan/brand options: %#v", got)
|
t.Fatalf("legacy plan/brand throttles should be ignored: %#v", got)
|
||||||
|
}
|
||||||
|
if got.MaxMaterializePerRun != 0 {
|
||||||
|
t.Fatalf("batch_size must not throttle daily materialization: %#v", got)
|
||||||
|
}
|
||||||
|
if got.MaxDesktopClientBacklog != 6 {
|
||||||
|
t.Fatalf("MaxDesktopClientBacklog = %d, want 6", got.MaxDesktopClientBacklog)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -19,19 +19,20 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
defaultMonitoringDailyTaskInterval = 5 * time.Minute
|
defaultMonitoringDailyTaskInterval = 5 * time.Minute
|
||||||
defaultMonitoringDailyTaskTimeout = 45 * time.Second
|
defaultMonitoringDailyTaskTimeout = 45 * time.Second
|
||||||
defaultMonitoringDailyTaskHardCap = 36
|
defaultMonitoringDailyTaskHardCap = 36
|
||||||
defaultMonitoringDailyBrandLimit = 1
|
defaultMonitoringDailyBrandLimit = 1
|
||||||
defaultMonitoringDailyLookahead = 10 * time.Minute
|
defaultMonitoringDailyLookahead = 10 * time.Minute
|
||||||
defaultMonitoringDailyMaterializeWindow = 2 * time.Hour
|
defaultMonitoringDailyDesktopRetryCooldown = 15 * time.Minute
|
||||||
defaultMonitoringDailyDesktopRetryCooldown = 15 * time.Minute
|
defaultMonitoringDailyPublishTimeout = 5 * time.Second
|
||||||
defaultMonitoringDailyPublishTimeout = 5 * time.Second
|
defaultMonitoringDailyMaxMaterializePerRun = 0
|
||||||
defaultMonitoringDailyMaxMaterializePerRun = 64
|
defaultMonitoringDailyMaxMaterializePerPlan = 0
|
||||||
defaultMonitoringDailyMaxMaterializePerPlan = 12
|
defaultMonitoringDailyMaxMaterializePerBrand = 0
|
||||||
defaultMonitoringDailyMaxMaterializePerBrand = 4
|
defaultMonitoringDailyMaxDesktopDispatch = 12
|
||||||
monitoringDailyPlatformActiveWindow = 24 * time.Hour
|
defaultMonitoringDailyMaxDesktopClientBacklog = 12
|
||||||
monitoringDailyMaterializeStartOffset = 5 * time.Minute
|
monitoringDailyPlatformActiveWindow = 24 * time.Hour
|
||||||
|
monitoringDailyMaterializeStartOffset = 5 * time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
type MonitoringDailyTaskWorker struct {
|
type MonitoringDailyTaskWorker struct {
|
||||||
@@ -57,16 +58,20 @@ type MonitoringDailyTaskGenerationSummary struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type MonitoringDailyTaskRunOptions struct {
|
type MonitoringDailyTaskRunOptions struct {
|
||||||
MaxMaterializePerRun int
|
MaxMaterializePerRun int
|
||||||
MaxMaterializePerPlan int
|
MaxMaterializePerPlan int
|
||||||
MaxMaterializePerBrand int
|
MaxMaterializePerBrand int
|
||||||
|
MaxDesktopDispatchPerRun int
|
||||||
|
MaxDesktopClientBacklog int
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeMonitoringDailyRunOptions(options ...MonitoringDailyTaskRunOptions) MonitoringDailyTaskRunOptions {
|
func normalizeMonitoringDailyRunOptions(options ...MonitoringDailyTaskRunOptions) MonitoringDailyTaskRunOptions {
|
||||||
out := MonitoringDailyTaskRunOptions{
|
out := MonitoringDailyTaskRunOptions{
|
||||||
MaxMaterializePerRun: defaultMonitoringDailyMaxMaterializePerRun,
|
MaxMaterializePerRun: defaultMonitoringDailyMaxMaterializePerRun,
|
||||||
MaxMaterializePerPlan: defaultMonitoringDailyMaxMaterializePerPlan,
|
MaxMaterializePerPlan: defaultMonitoringDailyMaxMaterializePerPlan,
|
||||||
MaxMaterializePerBrand: defaultMonitoringDailyMaxMaterializePerBrand,
|
MaxMaterializePerBrand: defaultMonitoringDailyMaxMaterializePerBrand,
|
||||||
|
MaxDesktopDispatchPerRun: defaultMonitoringDailyMaxDesktopDispatch,
|
||||||
|
MaxDesktopClientBacklog: defaultMonitoringDailyMaxDesktopClientBacklog,
|
||||||
}
|
}
|
||||||
if len(options) > 0 {
|
if len(options) > 0 {
|
||||||
in := options[0]
|
in := options[0]
|
||||||
@@ -79,11 +84,17 @@ func normalizeMonitoringDailyRunOptions(options ...MonitoringDailyTaskRunOptions
|
|||||||
if in.MaxMaterializePerBrand > 0 {
|
if in.MaxMaterializePerBrand > 0 {
|
||||||
out.MaxMaterializePerBrand = in.MaxMaterializePerBrand
|
out.MaxMaterializePerBrand = in.MaxMaterializePerBrand
|
||||||
}
|
}
|
||||||
|
if in.MaxDesktopDispatchPerRun > 0 {
|
||||||
|
out.MaxDesktopDispatchPerRun = in.MaxDesktopDispatchPerRun
|
||||||
|
}
|
||||||
|
if in.MaxDesktopClientBacklog > 0 {
|
||||||
|
out.MaxDesktopClientBacklog = in.MaxDesktopClientBacklog
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if out.MaxMaterializePerPlan > out.MaxMaterializePerRun {
|
if out.MaxMaterializePerRun > 0 && out.MaxMaterializePerPlan > out.MaxMaterializePerRun {
|
||||||
out.MaxMaterializePerPlan = out.MaxMaterializePerRun
|
out.MaxMaterializePerPlan = out.MaxMaterializePerRun
|
||||||
}
|
}
|
||||||
if out.MaxMaterializePerBrand > out.MaxMaterializePerPlan {
|
if out.MaxMaterializePerPlan > 0 && out.MaxMaterializePerBrand > out.MaxMaterializePerPlan {
|
||||||
out.MaxMaterializePerBrand = out.MaxMaterializePerPlan
|
out.MaxMaterializePerBrand = out.MaxMaterializePerPlan
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
@@ -236,6 +247,8 @@ func (w *MonitoringDailyTaskWorker) RunOnce(parent context.Context, options ...M
|
|||||||
stats["max_materialize_per_run"] = runOptions.MaxMaterializePerRun
|
stats["max_materialize_per_run"] = runOptions.MaxMaterializePerRun
|
||||||
stats["max_materialize_per_plan"] = runOptions.MaxMaterializePerPlan
|
stats["max_materialize_per_plan"] = runOptions.MaxMaterializePerPlan
|
||||||
stats["max_materialize_per_brand"] = runOptions.MaxMaterializePerBrand
|
stats["max_materialize_per_brand"] = runOptions.MaxMaterializePerBrand
|
||||||
|
stats["max_desktop_dispatch_per_run"] = runOptions.MaxDesktopDispatchPerRun
|
||||||
|
stats["max_desktop_client_backlog"] = runOptions.MaxDesktopClientBacklog
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if w.logger != nil {
|
if w.logger != nil {
|
||||||
w.logger.Warn("monitoring daily task generation failed", MonitoringWorkerErrorFields(err)...)
|
w.logger.Warn("monitoring daily task generation failed", MonitoringWorkerErrorFields(err)...)
|
||||||
@@ -267,10 +280,11 @@ func (s *MonitoringService) GenerateDailyMonitoringTasks(
|
|||||||
TenantCount: len(plans),
|
TenantCount: len(plans),
|
||||||
}
|
}
|
||||||
options := normalizeMonitoringDailyRunOptions(runOptions...)
|
options := normalizeMonitoringDailyRunOptions(runOptions...)
|
||||||
globalRemaining := options.MaxMaterializePerRun
|
globalMaterializeRemaining := options.MaxMaterializePerRun
|
||||||
|
desktopDispatchRemaining := options.MaxDesktopDispatchPerRun
|
||||||
|
|
||||||
for _, plan := range plans {
|
for _, plan := range plans {
|
||||||
if globalRemaining <= 0 {
|
if monitoringDailyLimitReached(globalMaterializeRemaining, options.MaxMaterializePerRun) {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
executionOwner, ok, skipReason := s.resolveDailyMonitoringExecutionOwner(plan)
|
executionOwner, ok, skipReason := s.resolveDailyMonitoringExecutionOwner(plan)
|
||||||
@@ -308,10 +322,14 @@ func (s *MonitoringService) GenerateDailyMonitoringTasks(
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
planRunRemaining := minMonitoringDailyInt(options.MaxMaterializePerPlan, globalRemaining)
|
planMaterializeRemaining := options.MaxMaterializePerPlan
|
||||||
|
if options.MaxMaterializePerRun > 0 && (planMaterializeRemaining <= 0 || globalMaterializeRemaining < planMaterializeRemaining) {
|
||||||
|
planMaterializeRemaining = globalMaterializeRemaining
|
||||||
|
}
|
||||||
|
|
||||||
for _, brand := range brands {
|
for _, brand := range brands {
|
||||||
if planRunRemaining <= 0 || globalRemaining <= 0 {
|
if monitoringDailyLimitReached(planMaterializeRemaining, options.MaxMaterializePerPlan) ||
|
||||||
|
monitoringDailyLimitReached(globalMaterializeRemaining, options.MaxMaterializePerRun) {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -334,8 +352,13 @@ func (s *MonitoringService) GenerateDailyMonitoringTasks(
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
dispatchLimit := minMonitoringDailyInt(options.MaxMaterializePerBrand, planRunRemaining, globalRemaining)
|
materializeLimit := monitoringDailyBoundedLimit(
|
||||||
materializeLimit := dispatchLimit
|
len(candidates),
|
||||||
|
options.MaxMaterializePerBrand,
|
||||||
|
planMaterializeRemaining,
|
||||||
|
globalMaterializeRemaining,
|
||||||
|
)
|
||||||
|
dispatchLimit := monitoringDailyDispatchLimit(len(candidates), desktopDispatchRemaining, options.MaxDesktopDispatchPerRun)
|
||||||
createdCount, dueCount, desktopCount, deferredCount, err := s.generateDailyMonitoringTasksForBrand(
|
createdCount, dueCount, desktopCount, deferredCount, err := s.generateDailyMonitoringTasksForBrand(
|
||||||
ctx,
|
ctx,
|
||||||
projectionService,
|
projectionService,
|
||||||
@@ -348,6 +371,7 @@ func (s *MonitoringService) GenerateDailyMonitoringTasks(
|
|||||||
materializeLimit,
|
materializeLimit,
|
||||||
dispatchLimit,
|
dispatchLimit,
|
||||||
executionOwner,
|
executionOwner,
|
||||||
|
options.MaxDesktopClientBacklog,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if abortErr := monitoringDailyAbortErr(ctx); abortErr != nil {
|
if abortErr := monitoringDailyAbortErr(ctx); abortErr != nil {
|
||||||
@@ -358,10 +382,14 @@ func (s *MonitoringService) GenerateDailyMonitoringTasks(
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
processedCount := int(maxMonitoringDailyInt(dueCount, desktopCount+deferredCount))
|
if options.MaxMaterializePerPlan > 0 {
|
||||||
if processedCount > 0 {
|
planMaterializeRemaining -= int(dueCount)
|
||||||
planRunRemaining -= processedCount
|
}
|
||||||
globalRemaining -= processedCount
|
if options.MaxMaterializePerRun > 0 {
|
||||||
|
globalMaterializeRemaining -= int(dueCount)
|
||||||
|
}
|
||||||
|
if options.MaxDesktopDispatchPerRun > 0 {
|
||||||
|
desktopDispatchRemaining -= int(desktopCount + deferredCount)
|
||||||
}
|
}
|
||||||
summary.BrandCount++
|
summary.BrandCount++
|
||||||
summary.PlannedTaskCount += int64(len(candidates))
|
summary.PlannedTaskCount += int64(len(candidates))
|
||||||
@@ -431,6 +459,7 @@ func (s *MonitoringService) generateDailyMonitoringTasksForBrand(
|
|||||||
materializeLimit int,
|
materializeLimit int,
|
||||||
dispatchLimit int,
|
dispatchLimit int,
|
||||||
executionOwner string,
|
executionOwner string,
|
||||||
|
maxDesktopClientBacklog int,
|
||||||
) (int64, int64, int64, int64, error) {
|
) (int64, int64, int64, int64, error) {
|
||||||
if materializeLimit <= 0 && dispatchLimit <= 0 {
|
if materializeLimit <= 0 && dispatchLimit <= 0 {
|
||||||
return 0, 0, 0, 0, nil
|
return 0, 0, 0, 0, nil
|
||||||
@@ -513,7 +542,7 @@ func (s *MonitoringService) generateDailyMonitoringTasksForBrand(
|
|||||||
return createdCount, dueCount, 0, 0, nil
|
return createdCount, dueCount, 0, 0, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
publishedTasks, deferredTaskIDs, err := s.dispatchMonitorDesktopTasks(ctx, specs)
|
publishedTasks, deferredTaskIDs, err := s.dispatchMonitorDesktopTasks(ctx, specs, maxDesktopClientBacklog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if s.logger != nil {
|
if s.logger != nil {
|
||||||
s.logger.Warn("daily monitor desktop dispatch failed; pending tasks will be retried",
|
s.logger.Warn("daily monitor desktop dispatch failed; pending tasks will be retried",
|
||||||
@@ -1178,52 +1207,58 @@ func buildMonitoringDailyTaskCandidates(
|
|||||||
if len(candidates) > limit {
|
if len(candidates) > limit {
|
||||||
candidates = candidates[:limit]
|
candidates = candidates[:limit]
|
||||||
}
|
}
|
||||||
assignMonitoringDailyScheduleTimes(tenantID, brandID, businessDay, candidates)
|
assignMonitoringDailyScheduleTimes(businessDay, candidates)
|
||||||
return candidates
|
return candidates
|
||||||
}
|
}
|
||||||
|
|
||||||
func assignMonitoringDailyScheduleTimes(tenantID, brandID int64, businessDay time.Time, candidates []monitoringDailyTaskCandidate) {
|
func assignMonitoringDailyScheduleTimes(businessDay time.Time, candidates []monitoringDailyTaskCandidate) {
|
||||||
if len(candidates) == 0 {
|
if len(candidates) == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
start := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset)
|
dueAt := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset).UTC()
|
||||||
window := defaultMonitoringDailyMaterializeWindow
|
|
||||||
if window <= 0 {
|
|
||||||
window = time.Hour
|
|
||||||
}
|
|
||||||
slot := window / time.Duration(len(candidates))
|
|
||||||
if slot <= 0 {
|
|
||||||
slot = time.Second
|
|
||||||
}
|
|
||||||
latest := start.Add(window)
|
|
||||||
businessDate := monitoringBusinessDateText(businessDay)
|
|
||||||
|
|
||||||
for i := range candidates {
|
for i := range candidates {
|
||||||
jitter := time.Duration(monitoringDailyStableHash(
|
candidates[i].MaterializeAt = dueAt
|
||||||
"dispatch_jitter",
|
candidates[i].DispatchAfter = dueAt
|
||||||
int64Text(tenantID),
|
|
||||||
int64Text(brandID),
|
|
||||||
businessDate,
|
|
||||||
int64Text(candidates[i].Question.ID),
|
|
||||||
candidates[i].Platform.ID,
|
|
||||||
) % uint64(slot))
|
|
||||||
materializeAt := start.Add(time.Duration(i)*slot + jitter)
|
|
||||||
if materializeAt.After(latest) {
|
|
||||||
materializeAt = latest
|
|
||||||
}
|
|
||||||
candidates[i].MaterializeAt = materializeAt.UTC()
|
|
||||||
candidates[i].DispatchAfter = materializeAt.UTC()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.SliceStable(candidates, func(i, j int) bool {
|
sort.SliceStable(candidates, func(i, j int) bool {
|
||||||
if candidates[i].MaterializeAt.Equal(candidates[j].MaterializeAt) {
|
if candidates[i].sortKey == candidates[j].sortKey {
|
||||||
return candidates[i].sortKey < candidates[j].sortKey
|
if candidates[i].Question.ID == candidates[j].Question.ID {
|
||||||
|
return candidates[i].Platform.ID < candidates[j].Platform.ID
|
||||||
|
}
|
||||||
|
return candidates[i].Question.ID < candidates[j].Question.ID
|
||||||
}
|
}
|
||||||
return candidates[i].MaterializeAt.Before(candidates[j].MaterializeAt)
|
return candidates[i].sortKey < candidates[j].sortKey
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func monitoringDailyLimitReached(remaining, configuredLimit int) bool {
|
||||||
|
return configuredLimit > 0 && remaining <= 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitoringDailyBoundedLimit(limit int, caps ...int) int {
|
||||||
|
if limit <= 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
for _, cap := range caps {
|
||||||
|
if cap > 0 && cap < limit {
|
||||||
|
limit = cap
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return limit
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitoringDailyDispatchLimit(candidateCount, remaining, configuredLimit int) int {
|
||||||
|
if configuredLimit <= 0 {
|
||||||
|
return candidateCount
|
||||||
|
}
|
||||||
|
if remaining <= 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return monitoringDailyBoundedLimit(candidateCount, remaining)
|
||||||
|
}
|
||||||
|
|
||||||
func selectDueMonitoringDailyCandidates(
|
func selectDueMonitoringDailyCandidates(
|
||||||
candidates []monitoringDailyTaskCandidate,
|
candidates []monitoringDailyTaskCandidate,
|
||||||
existing map[monitoringDailyTaskKey]struct{},
|
existing map[monitoringDailyTaskKey]struct{},
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBuildMonitoringDailyTaskCandidatesMaterializesAfterMidnight(t *testing.T) {
|
func TestBuildMonitoringDailyTaskCandidatesMaterializesWholeDayImmediately(t *testing.T) {
|
||||||
businessDay := time.Date(2026, 4, 24, 0, 0, 0, 0, monitoringBusinessLocation())
|
businessDay := time.Date(2026, 4, 24, 0, 0, 0, 0, monitoringBusinessLocation())
|
||||||
questions := []monitoringConfiguredQuestion{
|
questions := []monitoringConfiguredQuestion{
|
||||||
{ID: 101, QuestionHash: []byte("q101")},
|
{ID: 101, QuestionHash: []byte("q101")},
|
||||||
@@ -31,18 +31,85 @@ func TestBuildMonitoringDailyTaskCandidatesMaterializesAfterMidnight(t *testing.
|
|||||||
candidates := buildMonitoringDailyTaskCandidates(7, 11, businessDay, questions, platforms, 4)
|
candidates := buildMonitoringDailyTaskCandidates(7, 11, businessDay, questions, platforms, 4)
|
||||||
|
|
||||||
require.Len(t, candidates, 4)
|
require.Len(t, candidates, 4)
|
||||||
start := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset).UTC()
|
dueAt := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset).UTC()
|
||||||
end := start.Add(defaultMonitoringDailyMaterializeWindow)
|
|
||||||
for i, candidate := range candidates {
|
for i, candidate := range candidates {
|
||||||
assert.False(t, candidate.MaterializeAt.Before(start))
|
assert.Equal(t, dueAt, candidate.MaterializeAt)
|
||||||
assert.False(t, candidate.MaterializeAt.After(end))
|
|
||||||
assert.Equal(t, candidate.MaterializeAt, candidate.DispatchAfter)
|
assert.Equal(t, candidate.MaterializeAt, candidate.DispatchAfter)
|
||||||
if i > 0 {
|
if i > 0 {
|
||||||
assert.False(t, candidate.MaterializeAt.Before(candidates[i-1].MaterializeAt))
|
assert.LessOrEqual(t, candidates[i-1].sortKey, candidate.sortKey)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNormalizeMonitoringDailyRunOptionsDefaultsDoNotThrottleByPlanOrBrand(t *testing.T) {
|
||||||
|
got := normalizeMonitoringDailyRunOptions()
|
||||||
|
|
||||||
|
assert.Equal(t, 0, got.MaxMaterializePerRun)
|
||||||
|
assert.Equal(t, 0, got.MaxMaterializePerPlan)
|
||||||
|
assert.Equal(t, 0, got.MaxMaterializePerBrand)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitoringDailyBoundedLimitTreatsZeroCapsAsUnlimited(t *testing.T) {
|
||||||
|
assert.Equal(t, 102, monitoringDailyBoundedLimit(102, 0, 0, 0))
|
||||||
|
assert.Equal(t, 64, monitoringDailyBoundedLimit(102, 0, 64, 0))
|
||||||
|
assert.Equal(t, 12, monitoringDailyBoundedLimit(102, 0, 64, 12))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitoringDailyDispatchLimitStopsWhenRunCapacityIsExhausted(t *testing.T) {
|
||||||
|
assert.Equal(t, 102, monitoringDailyDispatchLimit(102, 0, 0))
|
||||||
|
assert.Equal(t, 12, monitoringDailyDispatchLimit(102, 12, 12))
|
||||||
|
assert.Equal(t, 0, monitoringDailyDispatchLimit(102, 0, 12))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAutomaticDailySelectionIncludesAllUserQuestionsAndPlatforms(t *testing.T) {
|
||||||
|
businessDay := time.Date(2026, 6, 27, 0, 0, 0, 0, monitoringBusinessLocation())
|
||||||
|
questions := make([]monitoringConfiguredQuestion, 0, 17)
|
||||||
|
for id := int64(1); id <= 17; id++ {
|
||||||
|
questions = append(questions, monitoringConfiguredQuestion{
|
||||||
|
ID: id,
|
||||||
|
QuestionText: "用户问题 " + int64Text(id),
|
||||||
|
QuestionHash: []byte("q" + int64Text(id)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
platforms := []monitoringPlatformMetadata{
|
||||||
|
{ID: "yuanbao"},
|
||||||
|
{ID: "kimi"},
|
||||||
|
{ID: "wenxin"},
|
||||||
|
{ID: "doubao"},
|
||||||
|
{ID: "deepseek"},
|
||||||
|
{ID: "qwen"},
|
||||||
|
}
|
||||||
|
|
||||||
|
candidates := buildMonitoringDailyTaskCandidates(
|
||||||
|
7,
|
||||||
|
7,
|
||||||
|
businessDay,
|
||||||
|
questions,
|
||||||
|
platforms,
|
||||||
|
len(questions)*len(platforms),
|
||||||
|
)
|
||||||
|
limit := monitoringDailyBoundedLimit(len(candidates), 0, 0, 0)
|
||||||
|
due := selectDueMonitoringDailyCandidates(
|
||||||
|
candidates,
|
||||||
|
nil,
|
||||||
|
monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset),
|
||||||
|
defaultMonitoringDailyLookahead,
|
||||||
|
limit,
|
||||||
|
)
|
||||||
|
|
||||||
|
require.Len(t, candidates, 17*6)
|
||||||
|
require.Len(t, due, 17*6)
|
||||||
|
seen := make(map[monitoringDailyTaskKey]struct{}, len(due))
|
||||||
|
for _, candidate := range due {
|
||||||
|
seen[monitoringDailyTaskKey{
|
||||||
|
QuestionID: candidate.Question.ID,
|
||||||
|
PlatformID: candidate.Platform.ID,
|
||||||
|
}] = struct{}{}
|
||||||
|
}
|
||||||
|
assert.Contains(t, seen, monitoringDailyTaskKey{QuestionID: 17, PlatformID: "qwen"})
|
||||||
|
assert.Contains(t, seen, monitoringDailyTaskKey{QuestionID: 17, PlatformID: "deepseek"})
|
||||||
|
}
|
||||||
|
|
||||||
func TestSelectDueMonitoringDailyCandidatesSkipsExistingAndHonorsLimit(t *testing.T) {
|
func TestSelectDueMonitoringDailyCandidatesSkipsExistingAndHonorsLimit(t *testing.T) {
|
||||||
businessDay := time.Date(2026, 4, 24, 0, 0, 0, 0, monitoringBusinessLocation())
|
businessDay := time.Date(2026, 4, 24, 0, 0, 0, 0, monitoringBusinessLocation())
|
||||||
candidates := buildMonitoringDailyTaskCandidates(
|
candidates := buildMonitoringDailyTaskCandidates(
|
||||||
@@ -68,7 +135,7 @@ func TestSelectDueMonitoringDailyCandidatesSkipsExistingAndHonorsLimit(t *testin
|
|||||||
PlatformID: candidates[0].Platform.ID,
|
PlatformID: candidates[0].Platform.ID,
|
||||||
}: {},
|
}: {},
|
||||||
}
|
}
|
||||||
now := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset + defaultMonitoringDailyMaterializeWindow)
|
now := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset)
|
||||||
|
|
||||||
due := selectDueMonitoringDailyCandidates(candidates, existing, now, defaultMonitoringDailyLookahead, 2)
|
due := selectDueMonitoringDailyCandidates(candidates, existing, now, defaultMonitoringDailyLookahead, 2)
|
||||||
|
|
||||||
@@ -110,7 +177,7 @@ func TestSelectDueMonitoringDailyCandidatesContinuesPastExistingTasks(t *testing
|
|||||||
}] = struct{}{}
|
}] = struct{}{}
|
||||||
}
|
}
|
||||||
|
|
||||||
now := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset + defaultMonitoringDailyMaterializeWindow)
|
now := monitoringCanonicalBusinessDay(businessDay).Add(monitoringDailyMaterializeStartOffset)
|
||||||
due := selectDueMonitoringDailyCandidates(candidates, existing, now, defaultMonitoringDailyLookahead, 4)
|
due := selectDueMonitoringDailyCandidates(candidates, existing, now, defaultMonitoringDailyLookahead, 4)
|
||||||
|
|
||||||
require.Len(t, due, 2)
|
require.Len(t, due, 2)
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -203,15 +204,34 @@ func (s *MonitoringService) loadMonitorDesktopTaskSpecs(
|
|||||||
func (s *MonitoringService) dispatchMonitorDesktopTasks(
|
func (s *MonitoringService) dispatchMonitorDesktopTasks(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
specs []monitorDesktopTaskSpec,
|
specs []monitorDesktopTaskSpec,
|
||||||
|
maxClientBacklog ...int,
|
||||||
) ([]*repository.DesktopTask, []int64, error) {
|
) ([]*repository.DesktopTask, []int64, error) {
|
||||||
if s == nil || s.businessPool == nil || len(specs) == 0 {
|
if s == nil || s.businessPool == nil || len(specs) == 0 {
|
||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
backlogLimit := 0
|
||||||
|
if len(maxClientBacklog) > 0 && maxClientBacklog[0] > 0 {
|
||||||
|
backlogLimit = maxClientBacklog[0]
|
||||||
|
}
|
||||||
var targets map[string]monitorDesktopTaskTarget
|
var targets map[string]monitorDesktopTaskTarget
|
||||||
if monitorDesktopTaskSpecsNeedTargetLookup(specs) {
|
if monitorDesktopTaskSpecsNeedTargetLookup(specs) {
|
||||||
var err error
|
var err error
|
||||||
targets, err = s.loadMonitorDesktopTaskTargets(ctx, specs[0].TenantID, specs[0].WorkspaceID, specs[0].RequestedByUserID, specs)
|
targets, err = s.loadMonitorDesktopTaskTargets(ctx, specs[0].TenantID, specs[0].WorkspaceID, specs[0].RequestedByUserID, specs, backlogLimit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
clientBacklog := make(map[uuid.UUID]int)
|
||||||
|
if backlogLimit > 0 {
|
||||||
|
var err error
|
||||||
|
clientBacklog, err = s.loadMonitorDesktopClientBacklog(
|
||||||
|
ctx,
|
||||||
|
specs[0].TenantID,
|
||||||
|
specs[0].WorkspaceID,
|
||||||
|
monitorDesktopTaskDispatchClientIDs(specs, targets),
|
||||||
|
backlogLimit,
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
@@ -237,11 +257,22 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
|
|||||||
spec.TargetClientID = target.ClientID
|
spec.TargetClientID = target.ClientID
|
||||||
}
|
}
|
||||||
|
|
||||||
task, shouldPublish, fallbackLegacy, upsertErr := s.upsertMonitorDesktopTask(ctx, tx, repo, spec)
|
task, shouldPublish, shouldDefer, upsertErr := s.upsertMonitorDesktopTask(
|
||||||
|
ctx,
|
||||||
|
tx,
|
||||||
|
repo,
|
||||||
|
spec,
|
||||||
|
func(oldClientID, newClientID uuid.UUID) bool {
|
||||||
|
return reserveMonitorDesktopClientBacklog(oldClientID, newClientID, clientBacklog, backlogLimit)
|
||||||
|
},
|
||||||
|
func(oldClientID, newClientID uuid.UUID) (bool, error) {
|
||||||
|
return s.reserveMonitorDesktopClientBacklogTx(ctx, tx, spec.TenantID, spec.WorkspaceID, oldClientID, newClientID, backlogLimit)
|
||||||
|
},
|
||||||
|
)
|
||||||
if upsertErr != nil {
|
if upsertErr != nil {
|
||||||
return nil, nil, upsertErr
|
return nil, nil, upsertErr
|
||||||
}
|
}
|
||||||
if fallbackLegacy {
|
if shouldDefer {
|
||||||
deferredTaskIDs = append(deferredTaskIDs, spec.MonitorTaskID)
|
deferredTaskIDs = append(deferredTaskIDs, spec.MonitorTaskID)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -262,6 +293,26 @@ func (s *MonitoringService) dispatchMonitorDesktopTasks(
|
|||||||
return publishableTasks, deferredTaskIDs, nil
|
return publishableTasks, deferredTaskIDs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func monitorDesktopTaskDispatchClientIDs(specs []monitorDesktopTaskSpec, targets map[string]monitorDesktopTaskTarget) []uuid.UUID {
|
||||||
|
if len(specs) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
result := make([]uuid.UUID, 0, len(specs)*2)
|
||||||
|
for _, spec := range specs {
|
||||||
|
if spec.TargetClientID != uuid.Nil {
|
||||||
|
result = append(result, spec.TargetClientID)
|
||||||
|
}
|
||||||
|
if targets == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
target, ok := targets[normalizeMonitoringPlatformID(spec.PlatformID)]
|
||||||
|
if ok && target.ClientID != uuid.Nil {
|
||||||
|
result = append(result, target.ClientID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return uniqueUUIDs(result)
|
||||||
|
}
|
||||||
|
|
||||||
func monitorDesktopTaskSpecsNeedTargetLookup(specs []monitorDesktopTaskSpec) bool {
|
func monitorDesktopTaskSpecsNeedTargetLookup(specs []monitorDesktopTaskSpec) bool {
|
||||||
for _, spec := range specs {
|
for _, spec := range specs {
|
||||||
if spec.TargetAccountID == uuid.Nil || spec.TargetClientID == uuid.Nil {
|
if spec.TargetAccountID == uuid.Nil || spec.TargetClientID == uuid.Nil {
|
||||||
@@ -279,11 +330,16 @@ func (s *MonitoringService) loadMonitorDesktopTaskTargets(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
tenantID, workspaceID, userID int64,
|
tenantID, workspaceID, userID int64,
|
||||||
specs []monitorDesktopTaskSpec,
|
specs []monitorDesktopTaskSpec,
|
||||||
|
maxClientBacklog ...int,
|
||||||
) (map[string]monitorDesktopTaskTarget, error) {
|
) (map[string]monitorDesktopTaskTarget, error) {
|
||||||
platformIDs := uniqueMonitorDesktopTaskPlatformIDs(specs)
|
platformIDs := uniqueMonitorDesktopTaskPlatformIDs(specs)
|
||||||
if len(platformIDs) == 0 {
|
if len(platformIDs) == 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
backlogLimit := 0
|
||||||
|
if len(maxClientBacklog) > 0 && maxClientBacklog[0] > 0 {
|
||||||
|
backlogLimit = maxClientBacklog[0]
|
||||||
|
}
|
||||||
|
|
||||||
rows, err := s.businessPool.Query(ctx, `
|
rows, err := s.businessPool.Query(ctx, `
|
||||||
SELECT
|
SELECT
|
||||||
@@ -373,7 +429,11 @@ func (s *MonitoringService) loadMonitorDesktopTaskTargets(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return selectMonitorDesktopTaskTargets(candidates, accountPresence, onlineClients), nil
|
activeBacklog, err := s.loadMonitorDesktopClientBacklog(ctx, tenantID, workspaceID, clientIDs, backlogLimit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return selectMonitorDesktopTaskTargets(candidates, accountPresence, onlineClients, activeBacklog, backlogLimit), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func uniqueMonitorDesktopTaskPlatformIDs(specs []monitorDesktopTaskSpec) []string {
|
func uniqueMonitorDesktopTaskPlatformIDs(specs []monitorDesktopTaskSpec) []string {
|
||||||
@@ -463,13 +523,58 @@ func (s *MonitoringService) loadOnlineMonitorDesktopClients(ctx context.Context,
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *MonitoringService) loadMonitorDesktopClientBacklog(
|
||||||
|
ctx context.Context,
|
||||||
|
tenantID, workspaceID int64,
|
||||||
|
clientIDs []uuid.UUID,
|
||||||
|
limit int,
|
||||||
|
) (map[uuid.UUID]int, error) {
|
||||||
|
result := make(map[uuid.UUID]int)
|
||||||
|
clientIDs = uniqueUUIDs(clientIDs)
|
||||||
|
if s == nil || s.businessPool == nil || limit <= 0 || len(clientIDs) == 0 {
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := s.businessPool.Query(ctx, `
|
||||||
|
SELECT target_client_id, COUNT(*)::int
|
||||||
|
FROM desktop_tasks
|
||||||
|
WHERE tenant_id = $1
|
||||||
|
AND workspace_id = $2
|
||||||
|
AND target_client_id = ANY($3::uuid[])
|
||||||
|
AND kind = 'monitor'
|
||||||
|
AND status IN ('queued', 'in_progress')
|
||||||
|
GROUP BY target_client_id
|
||||||
|
`, tenantID, workspaceID, clientIDs)
|
||||||
|
if err != nil {
|
||||||
|
return nil, response.ErrInternal(50117, "desktop_client_backlog_lookup_failed", "failed to resolve monitor desktop client backlog")
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
for rows.Next() {
|
||||||
|
var (
|
||||||
|
clientID uuid.UUID
|
||||||
|
count int
|
||||||
|
)
|
||||||
|
if scanErr := rows.Scan(&clientID, &count); scanErr != nil {
|
||||||
|
return nil, response.ErrInternal(50117, "desktop_client_backlog_lookup_failed", "failed to parse monitor desktop client backlog")
|
||||||
|
}
|
||||||
|
result[clientID] = count
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, response.ErrInternal(50117, "desktop_client_backlog_lookup_failed", "failed to iterate monitor desktop client backlog")
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
func selectMonitorDesktopTaskTargets(
|
func selectMonitorDesktopTaskTargets(
|
||||||
candidates []monitorDesktopAccountCandidate,
|
candidates []monitorDesktopAccountCandidate,
|
||||||
accountPresence map[uuid.UUID][]desktopAccountPresenceClient,
|
accountPresence map[uuid.UUID][]desktopAccountPresenceClient,
|
||||||
onlineClients map[uuid.UUID]bool,
|
onlineClients map[uuid.UUID]bool,
|
||||||
|
clientBacklog map[uuid.UUID]int,
|
||||||
|
maxClientBacklog int,
|
||||||
) map[string]monitorDesktopTaskTarget {
|
) map[string]monitorDesktopTaskTarget {
|
||||||
result := make(map[string]monitorDesktopTaskTarget)
|
result := make(map[string]monitorDesktopTaskTarget)
|
||||||
bestByPlatform := make(map[string]monitorDesktopTaskTargetCandidate)
|
candidatesByPlatform := make(map[string][]monitorDesktopTaskTargetCandidate)
|
||||||
for _, candidate := range candidates {
|
for _, candidate := range candidates {
|
||||||
platformID := normalizeMonitoringPlatformID(candidate.PlatformID)
|
platformID := normalizeMonitoringPlatformID(candidate.PlatformID)
|
||||||
if platformID == "" || candidate.AccountID == uuid.Nil {
|
if platformID == "" || candidate.AccountID == uuid.Nil {
|
||||||
@@ -495,9 +600,7 @@ func selectMonitorDesktopTaskTargets(
|
|||||||
RecordOrder: candidate.RecordOrder,
|
RecordOrder: candidate.RecordOrder,
|
||||||
ClientOrder: index,
|
ClientOrder: index,
|
||||||
}
|
}
|
||||||
if current, ok := bestByPlatform[platformID]; !ok || item.betterThan(current) {
|
candidatesByPlatform[platformID] = append(candidatesByPlatform[platformID], item)
|
||||||
bestByPlatform[platformID] = item
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if candidate.DBClientID == nil || *candidate.DBClientID == uuid.Nil || !onlineClients[*candidate.DBClientID] {
|
if candidate.DBClientID == nil || *candidate.DBClientID == uuid.Nil || !onlineClients[*candidate.DBClientID] {
|
||||||
@@ -516,16 +619,83 @@ func selectMonitorDesktopTaskTargets(
|
|||||||
SourceRank: 1,
|
SourceRank: 1,
|
||||||
RecordOrder: candidate.RecordOrder,
|
RecordOrder: candidate.RecordOrder,
|
||||||
}
|
}
|
||||||
if current, ok := bestByPlatform[platformID]; !ok || item.betterThan(current) {
|
candidatesByPlatform[platformID] = append(candidatesByPlatform[platformID], item)
|
||||||
bestByPlatform[platformID] = item
|
}
|
||||||
|
|
||||||
|
platformIDs := make([]string, 0, len(candidatesByPlatform))
|
||||||
|
for platformID := range candidatesByPlatform {
|
||||||
|
platformIDs = append(platformIDs, platformID)
|
||||||
|
}
|
||||||
|
sort.Strings(platformIDs)
|
||||||
|
|
||||||
|
for _, platformID := range platformIDs {
|
||||||
|
platformCandidates := candidatesByPlatform[platformID]
|
||||||
|
sort.SliceStable(platformCandidates, func(i, j int) bool {
|
||||||
|
return platformCandidates[i].betterThan(platformCandidates[j])
|
||||||
|
})
|
||||||
|
for _, candidate := range platformCandidates {
|
||||||
|
if monitorDesktopClientBacklogFull(candidate.Target.ClientID, clientBacklog, maxClientBacklog) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
result[platformID] = candidate.Target
|
||||||
|
if maxClientBacklog > 0 {
|
||||||
|
clientBacklog[candidate.Target.ClientID]++
|
||||||
|
}
|
||||||
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for platformID, candidate := range bestByPlatform {
|
|
||||||
result[platformID] = candidate.Target
|
|
||||||
}
|
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func monitorDesktopClientBacklogFull(clientID uuid.UUID, clientBacklog map[uuid.UUID]int, maxClientBacklog int) bool {
|
||||||
|
return maxClientBacklog > 0 && clientID != uuid.Nil && clientBacklog[clientID] >= maxClientBacklog
|
||||||
|
}
|
||||||
|
|
||||||
|
func reserveMonitorDesktopClientBacklog(oldClientID, newClientID uuid.UUID, clientBacklog map[uuid.UUID]int, maxClientBacklog int) bool {
|
||||||
|
if maxClientBacklog <= 0 || newClientID == uuid.Nil || oldClientID == newClientID {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if clientBacklog == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if monitorDesktopClientBacklogFull(newClientID, clientBacklog, maxClientBacklog) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
clientBacklog[newClientID]++
|
||||||
|
if oldClientID != uuid.Nil && clientBacklog[oldClientID] > 0 {
|
||||||
|
clientBacklog[oldClientID]--
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *MonitoringService) reserveMonitorDesktopClientBacklogTx(
|
||||||
|
ctx context.Context,
|
||||||
|
tx pgx.Tx,
|
||||||
|
tenantID, workspaceID int64,
|
||||||
|
oldClientID, newClientID uuid.UUID,
|
||||||
|
maxClientBacklog int,
|
||||||
|
) (bool, error) {
|
||||||
|
if tx == nil || maxClientBacklog <= 0 || newClientID == uuid.Nil || oldClientID == newClientID {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, monitorDesktopTaskClientSlotLockKey(newClientID)); err != nil {
|
||||||
|
return false, response.ErrInternal(50123, "desktop_client_backlog_lock_failed", "failed to lock monitor desktop client backlog")
|
||||||
|
}
|
||||||
|
var backlog int
|
||||||
|
if err := tx.QueryRow(ctx, `
|
||||||
|
SELECT COUNT(*)::int
|
||||||
|
FROM desktop_tasks
|
||||||
|
WHERE tenant_id = $1
|
||||||
|
AND workspace_id = $2
|
||||||
|
AND target_client_id = $3
|
||||||
|
AND kind = 'monitor'
|
||||||
|
AND status IN ('queued', 'in_progress')
|
||||||
|
`, tenantID, workspaceID, newClientID).Scan(&backlog); err != nil {
|
||||||
|
return false, response.ErrInternal(50124, "desktop_client_backlog_check_failed", "failed to check monitor desktop client backlog")
|
||||||
|
}
|
||||||
|
return backlog < maxClientBacklog, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (candidate monitorDesktopTaskTargetCandidate) betterThan(current monitorDesktopTaskTargetCandidate) bool {
|
func (candidate monitorDesktopTaskTargetCandidate) betterThan(current monitorDesktopTaskTargetCandidate) bool {
|
||||||
if candidate.HealthRank != current.HealthRank {
|
if candidate.HealthRank != current.HealthRank {
|
||||||
return candidate.HealthRank < current.HealthRank
|
return candidate.HealthRank < current.HealthRank
|
||||||
@@ -559,14 +729,17 @@ func (s *MonitoringService) upsertMonitorDesktopTask(
|
|||||||
tx pgx.Tx,
|
tx pgx.Tx,
|
||||||
repo repository.DesktopTaskRepository,
|
repo repository.DesktopTaskRepository,
|
||||||
spec monitorDesktopTaskSpec,
|
spec monitorDesktopTaskSpec,
|
||||||
|
reserveClientBacklog func(oldClientID, newClientID uuid.UUID) bool,
|
||||||
|
reserveClientBacklogTx func(oldClientID, newClientID uuid.UUID) (bool, error),
|
||||||
) (*repository.DesktopTask, bool, bool, error) {
|
) (*repository.DesktopTask, bool, bool, error) {
|
||||||
var (
|
var (
|
||||||
existingDesktopID uuid.UUID
|
existingDesktopID uuid.UUID
|
||||||
|
existingTargetClientID uuid.UUID
|
||||||
existingStatus string
|
existingStatus string
|
||||||
existingLeaseExpiresAt pgtype.Timestamptz
|
existingLeaseExpiresAt pgtype.Timestamptz
|
||||||
existingActiveAttempt pgtype.UUID
|
existingActiveAttempt pgtype.UUID
|
||||||
)
|
)
|
||||||
err := tx.QueryRow(ctx, upsertMonitorDesktopTaskActiveLookupSQL(), spec.MonitorTaskID).Scan(&existingDesktopID, &existingStatus, &existingLeaseExpiresAt, &existingActiveAttempt)
|
err := tx.QueryRow(ctx, upsertMonitorDesktopTaskActiveLookupSQL(), spec.MonitorTaskID).Scan(&existingDesktopID, &existingTargetClientID, &existingStatus, &existingLeaseExpiresAt, &existingActiveAttempt)
|
||||||
switch err {
|
switch err {
|
||||||
case nil:
|
case nil:
|
||||||
task, getErr := repo.GetByDesktopID(ctx, existingDesktopID, spec.WorkspaceID)
|
task, getErr := repo.GetByDesktopID(ctx, existingDesktopID, spec.WorkspaceID)
|
||||||
@@ -583,6 +756,18 @@ func (s *MonitoringService) upsertMonitorDesktopTask(
|
|||||||
if targetAccountID == uuid.Nil {
|
if targetAccountID == uuid.Nil {
|
||||||
targetAccountID = task.TargetAccountID
|
targetAccountID = task.TargetAccountID
|
||||||
}
|
}
|
||||||
|
if reserveClientBacklog != nil && !reserveClientBacklog(existingTargetClientID, spec.TargetClientID) {
|
||||||
|
return nil, false, true, nil
|
||||||
|
}
|
||||||
|
if reserveClientBacklogTx != nil {
|
||||||
|
allowed, reserveErr := reserveClientBacklogTx(existingTargetClientID, spec.TargetClientID)
|
||||||
|
if reserveErr != nil {
|
||||||
|
return nil, false, false, reserveErr
|
||||||
|
}
|
||||||
|
if !allowed {
|
||||||
|
return nil, false, true, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
payloadJSON, payloadErr := marshalMonitorDesktopTaskPayload(spec)
|
payloadJSON, payloadErr := marshalMonitorDesktopTaskPayload(spec)
|
||||||
if payloadErr != nil {
|
if payloadErr != nil {
|
||||||
return nil, false, false, payloadErr
|
return nil, false, false, payloadErr
|
||||||
@@ -658,6 +843,18 @@ func (s *MonitoringService) upsertMonitorDesktopTask(
|
|||||||
if targetAccountID == uuid.Nil || spec.TargetClientID == uuid.Nil {
|
if targetAccountID == uuid.Nil || spec.TargetClientID == uuid.Nil {
|
||||||
return nil, false, true, nil
|
return nil, false, true, nil
|
||||||
}
|
}
|
||||||
|
if reserveClientBacklog != nil && !reserveClientBacklog(uuid.Nil, spec.TargetClientID) {
|
||||||
|
return nil, false, true, nil
|
||||||
|
}
|
||||||
|
if reserveClientBacklogTx != nil {
|
||||||
|
allowed, reserveErr := reserveClientBacklogTx(uuid.Nil, spec.TargetClientID)
|
||||||
|
if reserveErr != nil {
|
||||||
|
return nil, false, false, reserveErr
|
||||||
|
}
|
||||||
|
if !allowed {
|
||||||
|
return nil, false, true, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
payloadJSON, payloadErr := marshalMonitorDesktopTaskPayload(spec)
|
payloadJSON, payloadErr := marshalMonitorDesktopTaskPayload(spec)
|
||||||
if payloadErr != nil {
|
if payloadErr != nil {
|
||||||
return nil, false, false, payloadErr
|
return nil, false, false, payloadErr
|
||||||
@@ -718,7 +915,7 @@ func (s *MonitoringService) upsertMonitorDesktopTask(
|
|||||||
|
|
||||||
func upsertMonitorDesktopTaskActiveLookupSQL() string {
|
func upsertMonitorDesktopTaskActiveLookupSQL() string {
|
||||||
return `
|
return `
|
||||||
SELECT desktop_id, status, lease_expires_at, active_attempt_id
|
SELECT desktop_id, target_client_id, status, lease_expires_at, active_attempt_id
|
||||||
FROM desktop_tasks
|
FROM desktop_tasks
|
||||||
WHERE kind = 'monitor'
|
WHERE kind = 'monitor'
|
||||||
AND monitor_task_id = $1
|
AND monitor_task_id = $1
|
||||||
|
|||||||
@@ -32,6 +32,8 @@ func TestSelectMonitorDesktopTaskTargetsPrefersPresentAccountClient(t *testing.T
|
|||||||
map[uuid.UUID]bool{
|
map[uuid.UUID]bool{
|
||||||
liveClientID: true,
|
liveClientID: true,
|
||||||
},
|
},
|
||||||
|
nil,
|
||||||
|
0,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Equal(t, monitorDesktopTaskTarget{
|
assert.Equal(t, monitorDesktopTaskTarget{
|
||||||
@@ -56,6 +58,8 @@ func TestSelectMonitorDesktopTaskTargetsFallsBackToOnlineDBClient(t *testing.T)
|
|||||||
map[uuid.UUID]bool{
|
map[uuid.UUID]bool{
|
||||||
clientID: true,
|
clientID: true,
|
||||||
},
|
},
|
||||||
|
nil,
|
||||||
|
0,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Equal(t, monitorDesktopTaskTarget{
|
assert.Equal(t, monitorDesktopTaskTarget{
|
||||||
@@ -78,6 +82,8 @@ func TestSelectMonitorDesktopTaskTargetsSkipsOfflineClients(t *testing.T) {
|
|||||||
},
|
},
|
||||||
nil,
|
nil,
|
||||||
map[uuid.UUID]bool{},
|
map[uuid.UUID]bool{},
|
||||||
|
nil,
|
||||||
|
0,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Empty(t, targets)
|
assert.Empty(t, targets)
|
||||||
@@ -111,6 +117,8 @@ func TestSelectMonitorDesktopTaskTargetsPrefersHealthierOnlineCandidate(t *testi
|
|||||||
riskClientID: true,
|
riskClientID: true,
|
||||||
liveClientID: true,
|
liveClientID: true,
|
||||||
},
|
},
|
||||||
|
nil,
|
||||||
|
0,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Equal(t, monitorDesktopTaskTarget{
|
assert.Equal(t, monitorDesktopTaskTarget{
|
||||||
@@ -144,6 +152,8 @@ func TestSelectMonitorDesktopTaskTargetsPrefersEarlierOnlineClientForSameAccount
|
|||||||
earlierClientID: true,
|
earlierClientID: true,
|
||||||
laterClientID: true,
|
laterClientID: true,
|
||||||
},
|
},
|
||||||
|
nil,
|
||||||
|
0,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Equal(t, monitorDesktopTaskTarget{
|
assert.Equal(t, monitorDesktopTaskTarget{
|
||||||
@@ -152,6 +162,119 @@ func TestSelectMonitorDesktopTaskTargetsPrefersEarlierOnlineClientForSameAccount
|
|||||||
}, targets["doubao"])
|
}, targets["doubao"])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSelectMonitorDesktopTaskTargetsAppliesClientBackpressure(t *testing.T) {
|
||||||
|
fullClientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000701")
|
||||||
|
availableClientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000702")
|
||||||
|
fullAccountID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000801")
|
||||||
|
availableAccountID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000802")
|
||||||
|
|
||||||
|
targets := selectMonitorDesktopTaskTargets(
|
||||||
|
[]monitorDesktopAccountCandidate{
|
||||||
|
{
|
||||||
|
PlatformID: "qwen",
|
||||||
|
AccountID: fullAccountID,
|
||||||
|
DBClientID: &fullClientID,
|
||||||
|
HealthRank: 0,
|
||||||
|
RecordOrder: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
PlatformID: "qwen",
|
||||||
|
AccountID: availableAccountID,
|
||||||
|
DBClientID: &availableClientID,
|
||||||
|
HealthRank: 1,
|
||||||
|
RecordOrder: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
nil,
|
||||||
|
map[uuid.UUID]bool{
|
||||||
|
fullClientID: true,
|
||||||
|
availableClientID: true,
|
||||||
|
},
|
||||||
|
map[uuid.UUID]int{
|
||||||
|
fullClientID: 12,
|
||||||
|
},
|
||||||
|
12,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert.Equal(t, monitorDesktopTaskTarget{
|
||||||
|
AccountID: availableAccountID,
|
||||||
|
ClientID: availableClientID,
|
||||||
|
}, targets["qwen"])
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelectMonitorDesktopTaskTargetsConsumesBackpressureWithinRun(t *testing.T) {
|
||||||
|
clientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000703")
|
||||||
|
qwenAccountID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000803")
|
||||||
|
doubaoAccountID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000804")
|
||||||
|
|
||||||
|
targets := selectMonitorDesktopTaskTargets(
|
||||||
|
[]monitorDesktopAccountCandidate{
|
||||||
|
{
|
||||||
|
PlatformID: "doubao",
|
||||||
|
AccountID: doubaoAccountID,
|
||||||
|
DBClientID: &clientID,
|
||||||
|
HealthRank: 0,
|
||||||
|
RecordOrder: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
PlatformID: "qwen",
|
||||||
|
AccountID: qwenAccountID,
|
||||||
|
DBClientID: &clientID,
|
||||||
|
HealthRank: 0,
|
||||||
|
RecordOrder: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
nil,
|
||||||
|
map[uuid.UUID]bool{
|
||||||
|
clientID: true,
|
||||||
|
},
|
||||||
|
map[uuid.UUID]int{
|
||||||
|
clientID: 11,
|
||||||
|
},
|
||||||
|
12,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert.Len(t, targets, 1)
|
||||||
|
assert.Contains(t, targets, "doubao")
|
||||||
|
assert.NotContains(t, targets, "qwen")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReserveMonitorDesktopClientBacklogAllowsExistingSameClientWithoutConsumingSlot(t *testing.T) {
|
||||||
|
clientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000704")
|
||||||
|
backlog := map[uuid.UUID]int{
|
||||||
|
clientID: 12,
|
||||||
|
}
|
||||||
|
|
||||||
|
ok := reserveMonitorDesktopClientBacklog(clientID, clientID, backlog, 12)
|
||||||
|
|
||||||
|
assert.True(t, ok)
|
||||||
|
assert.Equal(t, 12, backlog[clientID])
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReserveMonitorDesktopClientBacklogConsumesCapacityPerTask(t *testing.T) {
|
||||||
|
clientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000705")
|
||||||
|
backlog := map[uuid.UUID]int{
|
||||||
|
clientID: 11,
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.True(t, reserveMonitorDesktopClientBacklog(uuid.Nil, clientID, backlog, 12))
|
||||||
|
assert.False(t, reserveMonitorDesktopClientBacklog(uuid.Nil, clientID, backlog, 12))
|
||||||
|
assert.Equal(t, 12, backlog[clientID])
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReserveMonitorDesktopClientBacklogMovesCapacityWhenRetargeting(t *testing.T) {
|
||||||
|
oldClientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000706")
|
||||||
|
newClientID := mustParseDailyMonitoringUUID(t, "00000000-0000-0000-0000-000000000707")
|
||||||
|
backlog := map[uuid.UUID]int{
|
||||||
|
oldClientID: 4,
|
||||||
|
newClientID: 11,
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.True(t, reserveMonitorDesktopClientBacklog(oldClientID, newClientID, backlog, 12))
|
||||||
|
assert.Equal(t, 3, backlog[oldClientID])
|
||||||
|
assert.Equal(t, 12, backlog[newClientID])
|
||||||
|
}
|
||||||
|
|
||||||
func TestMarshalMonitorDesktopTaskPayloadRequiresQuestionText(t *testing.T) {
|
func TestMarshalMonitorDesktopTaskPayloadRequiresQuestionText(t *testing.T) {
|
||||||
payload, err := marshalMonitorDesktopTaskPayload(monitorDesktopTaskSpec{
|
payload, err := marshalMonitorDesktopTaskPayload(monitorDesktopTaskSpec{
|
||||||
PlatformID: "qwen",
|
PlatformID: "qwen",
|
||||||
|
|||||||
@@ -93,7 +93,7 @@ INSERT INTO ops.scheduler_jobs
|
|||||||
(job_key, display_name, category, description, enabled, schedule_type, interval_seconds, timezone, timeout_seconds, batch_size, config)
|
(job_key, display_name, category, description, enabled, schedule_type, interval_seconds, timezone, timeout_seconds, batch_size, config)
|
||||||
VALUES
|
VALUES
|
||||||
('schedule_dispatch', '内容定时任务派发', 'content', '索引驱动的延迟队列派发,仅认领到期内容任务并投递生成队列。', true, 'interval', 15, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
|
('schedule_dispatch', '内容定时任务派发', 'content', '索引驱动的延迟队列派发,仅认领到期内容任务并投递生成队列。', true, 'interval', 15, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
|
||||||
('monitoring_daily_task', 'AI 监控每日任务生成', 'monitoring', '按业务日小批量物化监控采集任务并派发到桌面任务队列。', true, 'interval', 300, 'Asia/Shanghai', 45, 64, '{"max_materialize_per_plan":12,"max_materialize_per_brand":4}'::jsonb),
|
('monitoring_daily_task', 'AI 监控每日任务生成', 'monitoring', '按业务日全量创建监控采集事实,按容量限速派发到桌面任务队列。', true, 'interval', 300, 'Asia/Shanghai', 45, 12, '{"max_desktop_client_backlog":12}'::jsonb),
|
||||||
('monitoring_result_recovery', '监控结果恢复', 'monitoring', '按回调结果队列索引恢复未成功入队的监控结果。', true, 'interval', 60, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
|
('monitoring_result_recovery', '监控结果恢复', 'monitoring', '按回调结果队列索引恢复未成功入队的监控结果。', true, 'interval', 60, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
|
||||||
('monitoring_lease_recovery', '监控租约回收', 'monitoring', '按过期租约索引限批回收超过租约时间仍未回调的监控任务。', true, 'interval', 1800, 'Asia/Shanghai', 30, 1000, '{}'::jsonb),
|
('monitoring_lease_recovery', '监控租约回收', 'monitoring', '按过期租约索引限批回收超过租约时间仍未回调的监控任务。', true, 'interval', 1800, 'Asia/Shanghai', 30, 1000, '{}'::jsonb),
|
||||||
('monitoring_received_inspection', '监控 received 卡死检查', 'monitoring', '按 received watchdog 索引限批标记长时间未入队的监控任务。', true, 'interval', 300, 'Asia/Shanghai', 30, 100, '{"threshold":"15m"}'::jsonb),
|
('monitoring_received_inspection', '监控 received 卡死检查', 'monitoring', '按 received watchdog 索引限批标记长时间未入队的监控任务。', true, 'interval', 300, 'Asia/Shanghai', 30, 100, '{"threshold":"15m"}'::jsonb),
|
||||||
|
|||||||
@@ -11,10 +11,11 @@ SET description = '索引驱动的延迟队列派发,仅认领到期内容任
|
|||||||
WHERE job_key = 'schedule_dispatch';
|
WHERE job_key = 'schedule_dispatch';
|
||||||
|
|
||||||
UPDATE ops.scheduler_jobs
|
UPDATE ops.scheduler_jobs
|
||||||
SET description = '按业务日小批量物化监控采集任务并派发到桌面任务队列。',
|
SET description = '按业务日全量创建监控采集事实,按容量限速派发到桌面任务队列。',
|
||||||
timeout_seconds = 45,
|
timeout_seconds = 45,
|
||||||
batch_size = 64,
|
batch_size = 12,
|
||||||
config = config || '{"max_materialize_per_plan":12,"max_materialize_per_brand":4}'::jsonb,
|
config = (config - 'max_materialize_per_run' - 'max_materialize_per_plan' - 'max_materialize_per_brand')
|
||||||
|
|| '{"max_desktop_client_backlog":12}'::jsonb,
|
||||||
updated_at = NOW()
|
updated_at = NOW()
|
||||||
WHERE job_key = 'monitoring_daily_task';
|
WHERE job_key = 'monitoring_daily_task';
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,7 @@
|
|||||||
|
UPDATE ops.scheduler_jobs
|
||||||
|
SET description = '按业务日小批量物化监控采集任务并派发到桌面任务队列。',
|
||||||
|
batch_size = 64,
|
||||||
|
config = (config - 'max_desktop_client_backlog')
|
||||||
|
|| '{"max_materialize_per_plan":12,"max_materialize_per_brand":4}'::jsonb,
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE job_key = 'monitoring_daily_task';
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
UPDATE ops.scheduler_jobs
|
||||||
|
SET description = '按业务日全量创建监控采集事实,按容量限速派发到桌面任务队列。',
|
||||||
|
batch_size = 12,
|
||||||
|
config = (config - 'max_materialize_per_run' - 'max_materialize_per_plan' - 'max_materialize_per_brand')
|
||||||
|
|| '{"max_desktop_client_backlog":12}'::jsonb,
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE job_key = 'monitoring_daily_task';
|
||||||
Reference in New Issue
Block a user