fix monitoring online dispatch retries
This commit is contained in:
@@ -40,7 +40,9 @@ const (
|
|||||||
// reports skip writes and only refresh the row at this interval so dashboards
|
// reports skip writes and only refresh the row at this interval so dashboards
|
||||||
// do not depend on per-heartbeat database churn.
|
// do not depend on per-heartbeat database churn.
|
||||||
monitoringHeartbeatAccessSnapshotRefreshAfter = 10 * time.Minute
|
monitoringHeartbeatAccessSnapshotRefreshAfter = 10 * time.Minute
|
||||||
monitoringRuntimeUnknownMaxDispatchAttempts = 3
|
monitoringRuntimeUnknownMaxDispatchAttempts = 2
|
||||||
|
monitoringAutomaticFailureMaxAttempts = 2
|
||||||
|
monitoringAutomaticFailureRetryAfter = 2 * time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
var monitoringCitationMarkerPattern = regexp.MustCompile(`(?i)\[\s*citation\s*:\s*\d+\s*\]`)
|
var monitoringCitationMarkerPattern = regexp.MustCompile(`(?i)\[\s*citation\s*:\s*\d+\s*\]`)
|
||||||
@@ -240,27 +242,28 @@ type monitoringHeartbeatSnapshotDecision struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type monitoringCollectTask struct {
|
type monitoringCollectTask struct {
|
||||||
ID int64
|
ID int64
|
||||||
TenantID int64
|
TenantID int64
|
||||||
BrandID int64
|
BrandID int64
|
||||||
QuestionID int64
|
QuestionID int64
|
||||||
QuestionHash []byte
|
QuestionHash []byte
|
||||||
AIPlatformID string
|
AIPlatformID string
|
||||||
CollectorType string
|
CollectorType string
|
||||||
RunMode string
|
RunMode string
|
||||||
BusinessDate time.Time
|
BusinessDate time.Time
|
||||||
Status string
|
Status string
|
||||||
ExecutionOwner string
|
ExecutionOwner string
|
||||||
LeaseTokenHash sql.NullString
|
LeaseTokenHash sql.NullString
|
||||||
LeasedToExecutor sql.NullString
|
LeasedToExecutor sql.NullString
|
||||||
LeaseExpiresAt sql.NullTime
|
LeaseExpiresAt sql.NullTime
|
||||||
CallbackReceivedAt sql.NullTime
|
CallbackReceivedAt sql.NullTime
|
||||||
CompletedAt sql.NullTime
|
CompletedAt sql.NullTime
|
||||||
SkipReason sql.NullString
|
SkipReason sql.NullString
|
||||||
TriggerSource string
|
TriggerSource string
|
||||||
QuestionText string
|
QuestionText string
|
||||||
TargetClientID sql.NullString
|
TargetClientID sql.NullString
|
||||||
DispatchAttempts int
|
DispatchAttempts int
|
||||||
|
AutomaticFailureCount int
|
||||||
}
|
}
|
||||||
|
|
||||||
type monitoringDesktopExecutionLease struct {
|
type monitoringDesktopExecutionLease struct {
|
||||||
@@ -1062,6 +1065,32 @@ type monitoringSkipOutcome struct {
|
|||||||
RetryAfter time.Duration
|
RetryAfter time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type monitoringFailedResultOutcome struct {
|
||||||
|
TaskStatus string
|
||||||
|
Requeue bool
|
||||||
|
RetryAfter time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitoringFailedResultOutcomeForTask(task *monitoringCollectTask, req MonitoringTaskResultRequest) monitoringFailedResultOutcome {
|
||||||
|
if task == nil {
|
||||||
|
return monitoringFailedResultOutcome{TaskStatus: "failed"}
|
||||||
|
}
|
||||||
|
if !strings.EqualFold(strings.TrimSpace(task.TriggerSource), "automatic") {
|
||||||
|
return monitoringFailedResultOutcome{TaskStatus: "failed"}
|
||||||
|
}
|
||||||
|
if monitoringTaskFailureDispositionFromRequest(req).NonRetryable {
|
||||||
|
return monitoringFailedResultOutcome{TaskStatus: "failed"}
|
||||||
|
}
|
||||||
|
if task.AutomaticFailureCount < monitoringAutomaticFailureMaxAttempts-1 {
|
||||||
|
return monitoringFailedResultOutcome{
|
||||||
|
TaskStatus: "pending",
|
||||||
|
Requeue: true,
|
||||||
|
RetryAfter: monitoringAutomaticFailureRetryAfter,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return monitoringFailedResultOutcome{TaskStatus: "failed"}
|
||||||
|
}
|
||||||
|
|
||||||
func monitoringSkipOutcomeForReason(skipReason string, dispatchAttempts int, triggerSource string) monitoringSkipOutcome {
|
func monitoringSkipOutcomeForReason(skipReason string, dispatchAttempts int, triggerSource string) monitoringSkipOutcome {
|
||||||
switch strings.TrimSpace(skipReason) {
|
switch strings.TrimSpace(skipReason) {
|
||||||
case "runtime_unknown":
|
case "runtime_unknown":
|
||||||
@@ -1562,6 +1591,11 @@ func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantI
|
|||||||
trigger_source,
|
trigger_source,
|
||||||
COALESCE(dispatch_attempts, 0) AS dispatch_attempts,
|
COALESCE(dispatch_attempts, 0) AS dispatch_attempts,
|
||||||
COALESCE(target_client_id::text, '') AS target_client_id,
|
COALESCE(target_client_id::text, '') AS target_client_id,
|
||||||
|
CASE
|
||||||
|
WHEN COALESCE(t.request_payload_json ->> 'automatic_failure_count', '') ~ '^[0-9]+$'
|
||||||
|
THEN (t.request_payload_json ->> 'automatic_failure_count')::int
|
||||||
|
ELSE 0
|
||||||
|
END AS automatic_failure_count,
|
||||||
COALESCE(NULLIF(t.question_text_snapshot, ''), (
|
COALESCE(NULLIF(t.question_text_snapshot, ''), (
|
||||||
SELECT question_text_snapshot
|
SELECT question_text_snapshot
|
||||||
FROM monitoring_question_config_snapshots
|
FROM monitoring_question_config_snapshots
|
||||||
@@ -1600,6 +1634,7 @@ func (s *MonitoringCallbackService) loadCollectTask(ctx context.Context, tenantI
|
|||||||
&item.TriggerSource,
|
&item.TriggerSource,
|
||||||
&item.DispatchAttempts,
|
&item.DispatchAttempts,
|
||||||
&item.TargetClientID,
|
&item.TargetClientID,
|
||||||
|
&item.AutomaticFailureCount,
|
||||||
&item.QuestionText,
|
&item.QuestionText,
|
||||||
); err != nil {
|
); err != nil {
|
||||||
if err == pgx.ErrNoRows {
|
if err == pgx.ErrNoRows {
|
||||||
@@ -1635,6 +1670,11 @@ func (s *MonitoringCallbackService) loadCollectTaskByID(ctx context.Context, tas
|
|||||||
trigger_source,
|
trigger_source,
|
||||||
COALESCE(dispatch_attempts, 0) AS dispatch_attempts,
|
COALESCE(dispatch_attempts, 0) AS dispatch_attempts,
|
||||||
COALESCE(target_client_id::text, '') AS target_client_id,
|
COALESCE(target_client_id::text, '') AS target_client_id,
|
||||||
|
CASE
|
||||||
|
WHEN COALESCE(t.request_payload_json ->> 'automatic_failure_count', '') ~ '^[0-9]+$'
|
||||||
|
THEN (t.request_payload_json ->> 'automatic_failure_count')::int
|
||||||
|
ELSE 0
|
||||||
|
END AS automatic_failure_count,
|
||||||
COALESCE(NULLIF(t.question_text_snapshot, ''), (
|
COALESCE(NULLIF(t.question_text_snapshot, ''), (
|
||||||
SELECT question_text_snapshot
|
SELECT question_text_snapshot
|
||||||
FROM monitoring_question_config_snapshots
|
FROM monitoring_question_config_snapshots
|
||||||
@@ -1672,6 +1712,7 @@ func (s *MonitoringCallbackService) loadCollectTaskByID(ctx context.Context, tas
|
|||||||
&item.TriggerSource,
|
&item.TriggerSource,
|
||||||
&item.DispatchAttempts,
|
&item.DispatchAttempts,
|
||||||
&item.TargetClientID,
|
&item.TargetClientID,
|
||||||
|
&item.AutomaticFailureCount,
|
||||||
&item.QuestionText,
|
&item.QuestionText,
|
||||||
); err != nil {
|
); err != nil {
|
||||||
if err == pgx.ErrNoRows {
|
if err == pgx.ErrNoRows {
|
||||||
@@ -2312,12 +2353,20 @@ func (s *MonitoringCallbackService) processTaskResult(ctx context.Context, task
|
|||||||
}
|
}
|
||||||
defer tx.Rollback(ctx)
|
defer tx.Rollback(ctx)
|
||||||
|
|
||||||
runID, err := s.upsertRun(ctx, tx, task, req, runStatus, completedAt, rawPayload)
|
var failedOutcome monitoringFailedResultOutcome
|
||||||
if err != nil {
|
if runStatus == "failed" {
|
||||||
return nil, err
|
failedOutcome = monitoringFailedResultOutcomeForTask(task, req)
|
||||||
}
|
}
|
||||||
|
|
||||||
if runStatus == "succeeded" {
|
var runID int64
|
||||||
|
if !failedOutcome.Requeue {
|
||||||
|
runID, err = s.upsertRun(ctx, tx, task, req, runStatus, completedAt, rawPayload)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if runStatus == "succeeded" && !failedOutcome.Requeue {
|
||||||
if err := s.upsertParseResult(ctx, tx, task, runID, parseReq, parseVersion); err != nil {
|
if err := s.upsertParseResult(ctx, tx, task, runID, parseReq, parseVersion); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -2325,10 +2374,12 @@ func (s *MonitoringCallbackService) processTaskResult(ctx context.Context, task
|
|||||||
|
|
||||||
citationSourceCount := 0
|
citationSourceCount := 0
|
||||||
contentCitationCount := 0
|
contentCitationCount := 0
|
||||||
if err := s.replaceCitationFacts(ctx, tx, task.TenantID, runID); err != nil {
|
if !failedOutcome.Requeue {
|
||||||
return nil, err
|
if err := s.replaceCitationFacts(ctx, tx, task.TenantID, runID); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if runStatus == "succeeded" {
|
if runStatus == "succeeded" && !failedOutcome.Requeue {
|
||||||
facts, buildErr := s.buildCitationFacts(ctx, tx, task.TenantID, task.AIPlatformID, sourceInputs)
|
facts, buildErr := s.buildCitationFacts(ctx, tx, task.TenantID, task.AIPlatformID, sourceInputs)
|
||||||
if buildErr != nil {
|
if buildErr != nil {
|
||||||
return nil, buildErr
|
return nil, buildErr
|
||||||
@@ -2346,12 +2397,16 @@ func (s *MonitoringCallbackService) processTaskResult(ctx context.Context, task
|
|||||||
|
|
||||||
taskStatus := "completed"
|
taskStatus := "completed"
|
||||||
if runStatus == "failed" {
|
if runStatus == "failed" {
|
||||||
taskStatus = "failed"
|
taskStatus = failedOutcome.TaskStatus
|
||||||
}
|
}
|
||||||
if err := s.updateCollectTaskStatus(ctx, tx, task.ID, taskStatus, completedAt, req.ErrorMessage); err != nil {
|
if failedOutcome.Requeue {
|
||||||
|
if err := s.requeueCollectTaskAfterFailedResult(ctx, tx, task.ID, failedOutcome.RetryAfter, req.ErrorMessage); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
} else if err := s.updateCollectTaskStatus(ctx, tx, task.ID, taskStatus, completedAt, req.ErrorMessage); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if runStatus == "failed" {
|
if runStatus == "failed" && !failedOutcome.Requeue {
|
||||||
if _, err := s.failRemainingMonitoringTasksForAuthorizationFailure(ctx, tx, task, req, completedAt); err != nil {
|
if _, err := s.failRemainingMonitoringTasksForAuthorizationFailure(ctx, tx, task, req, completedAt); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -2365,9 +2420,13 @@ func (s *MonitoringCallbackService) processTaskResult(ctx context.Context, task
|
|||||||
return nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring result ingest")
|
return nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring result ingest")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var runIDPtr *int64
|
||||||
|
if !failedOutcome.Requeue {
|
||||||
|
runIDPtr = &runID
|
||||||
|
}
|
||||||
return &MonitoringTaskResultResponse{
|
return &MonitoringTaskResultResponse{
|
||||||
TaskID: task.ID,
|
TaskID: task.ID,
|
||||||
RunID: &runID,
|
RunID: runIDPtr,
|
||||||
TaskStatus: taskStatus,
|
TaskStatus: taskStatus,
|
||||||
CitationSourceCount: citationSourceCount,
|
CitationSourceCount: citationSourceCount,
|
||||||
ContentCitationCount: contentCitationCount,
|
ContentCitationCount: contentCitationCount,
|
||||||
@@ -3113,6 +3172,51 @@ func (s *MonitoringCallbackService) updateCollectTaskStatus(ctx context.Context,
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *MonitoringCallbackService) requeueCollectTaskAfterFailedResult(ctx context.Context, tx pgx.Tx, taskID int64, retryAfter time.Duration, errorMessage *string) error {
|
||||||
|
if retryAfter <= 0 {
|
||||||
|
retryAfter = monitoringAutomaticFailureRetryAfter
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(ctx, `
|
||||||
|
UPDATE monitoring_collect_tasks
|
||||||
|
SET status = 'pending',
|
||||||
|
lease_token_hash = NULL,
|
||||||
|
leased_to_executor = NULL,
|
||||||
|
leased_at = NULL,
|
||||||
|
lease_expires_at = NULL,
|
||||||
|
callback_received_at = NULL,
|
||||||
|
completed_at = NULL,
|
||||||
|
skip_reason = NULL,
|
||||||
|
last_dispatched_at = NULL,
|
||||||
|
dispatch_after = NOW() + $2::interval,
|
||||||
|
error_message = $3,
|
||||||
|
request_payload_json = (
|
||||||
|
CASE
|
||||||
|
WHEN request_payload_json IS NULL OR jsonb_typeof(request_payload_json) <> 'object'
|
||||||
|
THEN '{}'::jsonb
|
||||||
|
ELSE request_payload_json
|
||||||
|
END
|
||||||
|
) || jsonb_build_object(
|
||||||
|
'automatic_failure_count',
|
||||||
|
(
|
||||||
|
CASE
|
||||||
|
WHEN COALESCE(request_payload_json ->> 'automatic_failure_count', '') ~ '^[0-9]+$'
|
||||||
|
THEN (request_payload_json ->> 'automatic_failure_count')::int
|
||||||
|
ELSE 0
|
||||||
|
END
|
||||||
|
) + 1,
|
||||||
|
'automatic_failure_retry_after_seconds',
|
||||||
|
$4::int,
|
||||||
|
'automatic_failure_last_requeued_at',
|
||||||
|
to_jsonb(NOW())
|
||||||
|
),
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id = $1
|
||||||
|
`, taskID, formatPgInterval(retryAfter), nullableString(errorMessage), int(retryAfter.Seconds())); err != nil {
|
||||||
|
return response.ErrInternal(50041, "task_update_failed", "failed to requeue monitoring task after failed result")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *MonitoringCallbackService) failRemainingMonitoringTasksForAuthorizationFailure(
|
func (s *MonitoringCallbackService) failRemainingMonitoringTasksForAuthorizationFailure(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
tx pgx.Tx,
|
tx pgx.Tx,
|
||||||
|
|||||||
@@ -499,6 +499,73 @@ func TestMonitoringTaskFailureDispositionDetectsAuthorizationFailureFromMessage(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMonitoringFailedResultOutcomeRequeuesAutomaticFirstFailure(t *testing.T) {
|
||||||
|
message := "模型调用超时"
|
||||||
|
got := monitoringFailedResultOutcomeForTask(&monitoringCollectTask{
|
||||||
|
TriggerSource: "automatic",
|
||||||
|
AutomaticFailureCount: 0,
|
||||||
|
}, MonitoringTaskResultRequest{
|
||||||
|
Status: "failed",
|
||||||
|
ErrorMessage: &message,
|
||||||
|
})
|
||||||
|
|
||||||
|
if got.TaskStatus != "pending" {
|
||||||
|
t.Fatalf("TaskStatus = %q, want pending", got.TaskStatus)
|
||||||
|
}
|
||||||
|
if !got.Requeue {
|
||||||
|
t.Fatal("Requeue = false, want true")
|
||||||
|
}
|
||||||
|
if got.RetryAfter != monitoringAutomaticFailureRetryAfter {
|
||||||
|
t.Fatalf("RetryAfter = %v, want %v", got.RetryAfter, monitoringAutomaticFailureRetryAfter)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitoringFailedResultOutcomeFailsAutomaticSecondFailure(t *testing.T) {
|
||||||
|
got := monitoringFailedResultOutcomeForTask(&monitoringCollectTask{
|
||||||
|
TriggerSource: "automatic",
|
||||||
|
AutomaticFailureCount: monitoringAutomaticFailureMaxAttempts - 1,
|
||||||
|
}, MonitoringTaskResultRequest{Status: "failed"})
|
||||||
|
|
||||||
|
if got.TaskStatus != "failed" {
|
||||||
|
t.Fatalf("TaskStatus = %q, want failed", got.TaskStatus)
|
||||||
|
}
|
||||||
|
if got.Requeue {
|
||||||
|
t.Fatal("Requeue = true, want false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitoringFailedResultOutcomeDoesNotRetryAuthorizationFailure(t *testing.T) {
|
||||||
|
message := "登录态已失效,请重新授权后再试。"
|
||||||
|
got := monitoringFailedResultOutcomeForTask(&monitoringCollectTask{
|
||||||
|
TriggerSource: "automatic",
|
||||||
|
AutomaticFailureCount: 0,
|
||||||
|
}, MonitoringTaskResultRequest{
|
||||||
|
Status: "failed",
|
||||||
|
ErrorMessage: &message,
|
||||||
|
})
|
||||||
|
|
||||||
|
if got.TaskStatus != "failed" {
|
||||||
|
t.Fatalf("TaskStatus = %q, want failed", got.TaskStatus)
|
||||||
|
}
|
||||||
|
if got.Requeue {
|
||||||
|
t.Fatal("Requeue = true, want false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitoringFailedResultOutcomeDoesNotRetryManualTask(t *testing.T) {
|
||||||
|
got := monitoringFailedResultOutcomeForTask(&monitoringCollectTask{
|
||||||
|
TriggerSource: "manual",
|
||||||
|
AutomaticFailureCount: 0,
|
||||||
|
}, MonitoringTaskResultRequest{Status: "failed"})
|
||||||
|
|
||||||
|
if got.TaskStatus != "failed" {
|
||||||
|
t.Fatalf("TaskStatus = %q, want failed", got.TaskStatus)
|
||||||
|
}
|
||||||
|
if got.Requeue {
|
||||||
|
t.Fatal("Requeue = true, want false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestMonitoringAuthorizationFailureTargetClientPrefersExplicitTargetClient(t *testing.T) {
|
func TestMonitoringAuthorizationFailureTargetClientPrefersExplicitTargetClient(t *testing.T) {
|
||||||
targetClientID := uuid.New().String()
|
targetClientID := uuid.New().String()
|
||||||
task := &monitoringCollectTask{
|
task := &monitoringCollectTask{
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ type MonitoringDailyTaskMetricsSnapshot struct {
|
|||||||
BrandFailureTotal int64 `json:"brand_failure_total"`
|
BrandFailureTotal int64 `json:"brand_failure_total"`
|
||||||
SkippedPlanTotal int64 `json:"skipped_plan_total"`
|
SkippedPlanTotal int64 `json:"skipped_plan_total"`
|
||||||
PlannedTaskTotal int64 `json:"planned_task_total"`
|
PlannedTaskTotal int64 `json:"planned_task_total"`
|
||||||
|
EligibleTaskTotal int64 `json:"eligible_task_total"`
|
||||||
DueTaskTotal int64 `json:"due_task_total"`
|
DueTaskTotal int64 `json:"due_task_total"`
|
||||||
CreatedTaskTotal int64 `json:"created_task_total"`
|
CreatedTaskTotal int64 `json:"created_task_total"`
|
||||||
DesktopTaskTotal int64 `json:"desktop_task_total"`
|
DesktopTaskTotal int64 `json:"desktop_task_total"`
|
||||||
@@ -44,6 +45,7 @@ type monitoringDailyTaskMetricsRecorder struct {
|
|||||||
brandFailureTotal atomic.Int64
|
brandFailureTotal atomic.Int64
|
||||||
skippedPlanTotal atomic.Int64
|
skippedPlanTotal atomic.Int64
|
||||||
plannedTaskTotal atomic.Int64
|
plannedTaskTotal atomic.Int64
|
||||||
|
eligibleTaskTotal atomic.Int64
|
||||||
dueTaskTotal atomic.Int64
|
dueTaskTotal atomic.Int64
|
||||||
createdTaskTotal atomic.Int64
|
createdTaskTotal atomic.Int64
|
||||||
desktopTaskTotal atomic.Int64
|
desktopTaskTotal atomic.Int64
|
||||||
@@ -98,6 +100,7 @@ func (r *monitoringDailyTaskMetricsRecorder) record(summary *MonitoringDailyTask
|
|||||||
r.brandFailureTotal.Add(int64(summary.FailedBrandCount))
|
r.brandFailureTotal.Add(int64(summary.FailedBrandCount))
|
||||||
r.skippedPlanTotal.Add(int64(summary.SkippedPlanCount))
|
r.skippedPlanTotal.Add(int64(summary.SkippedPlanCount))
|
||||||
r.plannedTaskTotal.Add(summary.PlannedTaskCount)
|
r.plannedTaskTotal.Add(summary.PlannedTaskCount)
|
||||||
|
r.eligibleTaskTotal.Add(summary.EligibleTaskCount)
|
||||||
r.dueTaskTotal.Add(summary.DueTaskCount)
|
r.dueTaskTotal.Add(summary.DueTaskCount)
|
||||||
r.createdTaskTotal.Add(summary.CreatedTaskCount)
|
r.createdTaskTotal.Add(summary.CreatedTaskCount)
|
||||||
r.desktopTaskTotal.Add(summary.DesktopTaskCount)
|
r.desktopTaskTotal.Add(summary.DesktopTaskCount)
|
||||||
@@ -133,6 +136,7 @@ func (r *monitoringDailyTaskMetricsRecorder) snapshot() MonitoringDailyTaskMetri
|
|||||||
BrandFailureTotal: r.brandFailureTotal.Load(),
|
BrandFailureTotal: r.brandFailureTotal.Load(),
|
||||||
SkippedPlanTotal: r.skippedPlanTotal.Load(),
|
SkippedPlanTotal: r.skippedPlanTotal.Load(),
|
||||||
PlannedTaskTotal: r.plannedTaskTotal.Load(),
|
PlannedTaskTotal: r.plannedTaskTotal.Load(),
|
||||||
|
EligibleTaskTotal: r.eligibleTaskTotal.Load(),
|
||||||
DueTaskTotal: r.dueTaskTotal.Load(),
|
DueTaskTotal: r.dueTaskTotal.Load(),
|
||||||
CreatedTaskTotal: r.createdTaskTotal.Load(),
|
CreatedTaskTotal: r.createdTaskTotal.Load(),
|
||||||
DesktopTaskTotal: r.desktopTaskTotal.Load(),
|
DesktopTaskTotal: r.desktopTaskTotal.Load(),
|
||||||
|
|||||||
@@ -50,6 +50,7 @@ type MonitoringDailyTaskGenerationSummary struct {
|
|||||||
FailedPlanCount int
|
FailedPlanCount int
|
||||||
FailedBrandCount int
|
FailedBrandCount int
|
||||||
PlannedTaskCount int64
|
PlannedTaskCount int64
|
||||||
|
EligibleTaskCount int64
|
||||||
DueTaskCount int64
|
DueTaskCount int64
|
||||||
CreatedTaskCount int64
|
CreatedTaskCount int64
|
||||||
DesktopTaskCount int64
|
DesktopTaskCount int64
|
||||||
@@ -238,6 +239,7 @@ func (w *MonitoringDailyTaskWorker) RunOnce(parent context.Context, options ...M
|
|||||||
stats["failed_plan_count"] = summary.FailedPlanCount
|
stats["failed_plan_count"] = summary.FailedPlanCount
|
||||||
stats["failed_brand_count"] = summary.FailedBrandCount
|
stats["failed_brand_count"] = summary.FailedBrandCount
|
||||||
stats["planned_task_count"] = summary.PlannedTaskCount
|
stats["planned_task_count"] = summary.PlannedTaskCount
|
||||||
|
stats["eligible_task_count"] = summary.EligibleTaskCount
|
||||||
stats["due_task_count"] = summary.DueTaskCount
|
stats["due_task_count"] = summary.DueTaskCount
|
||||||
stats["created_task_count"] = summary.CreatedTaskCount
|
stats["created_task_count"] = summary.CreatedTaskCount
|
||||||
stats["desktop_task_count"] = summary.DesktopTaskCount
|
stats["desktop_task_count"] = summary.DesktopTaskCount
|
||||||
@@ -359,7 +361,7 @@ func (s *MonitoringService) GenerateDailyMonitoringTasks(
|
|||||||
globalMaterializeRemaining,
|
globalMaterializeRemaining,
|
||||||
)
|
)
|
||||||
dispatchLimit := monitoringDailyDispatchLimit(len(candidates), desktopDispatchRemaining, options.MaxDesktopDispatchPerRun)
|
dispatchLimit := monitoringDailyDispatchLimit(len(candidates), desktopDispatchRemaining, options.MaxDesktopDispatchPerRun)
|
||||||
createdCount, dueCount, desktopCount, deferredCount, err := s.generateDailyMonitoringTasksForBrand(
|
createdCount, eligibleCount, dueCount, desktopCount, deferredCount, err := s.generateDailyMonitoringTasksForBrand(
|
||||||
ctx,
|
ctx,
|
||||||
projectionService,
|
projectionService,
|
||||||
plan,
|
plan,
|
||||||
@@ -393,6 +395,7 @@ func (s *MonitoringService) GenerateDailyMonitoringTasks(
|
|||||||
}
|
}
|
||||||
summary.BrandCount++
|
summary.BrandCount++
|
||||||
summary.PlannedTaskCount += int64(len(candidates))
|
summary.PlannedTaskCount += int64(len(candidates))
|
||||||
|
summary.EligibleTaskCount += eligibleCount
|
||||||
summary.DueTaskCount += dueCount
|
summary.DueTaskCount += dueCount
|
||||||
summary.CreatedTaskCount += createdCount
|
summary.CreatedTaskCount += createdCount
|
||||||
summary.DesktopTaskCount += desktopCount
|
summary.DesktopTaskCount += desktopCount
|
||||||
@@ -460,86 +463,101 @@ func (s *MonitoringService) generateDailyMonitoringTasksForBrand(
|
|||||||
dispatchLimit int,
|
dispatchLimit int,
|
||||||
executionOwner string,
|
executionOwner string,
|
||||||
maxDesktopClientBacklog int,
|
maxDesktopClientBacklog int,
|
||||||
) (int64, int64, int64, int64, error) {
|
) (int64, int64, int64, int64, int64, error) {
|
||||||
if materializeLimit <= 0 && dispatchLimit <= 0 {
|
if materializeLimit <= 0 && dispatchLimit <= 0 {
|
||||||
return 0, 0, 0, 0, nil
|
return 0, 0, 0, 0, 0, nil
|
||||||
}
|
}
|
||||||
if executionOwner != "desktop_tasks" {
|
if executionOwner != "desktop_tasks" {
|
||||||
return 0, 0, 0, 0, nil
|
return 0, 0, 0, 0, 0, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var createdCount int64
|
var createdCount int64
|
||||||
|
var eligibleCount int64
|
||||||
var dueCount int64
|
var dueCount int64
|
||||||
if materializeLimit > 0 {
|
specs := make([]monitorDesktopTaskSpec, 0)
|
||||||
tx, err := s.monitoringPool.Begin(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, 0, 0, response.ErrInternal(50041, "begin_failed", "failed to start monitoring daily task generation")
|
|
||||||
}
|
|
||||||
defer tx.Rollback(ctx)
|
|
||||||
|
|
||||||
locked, err := acquireMonitoringDailyBrandLock(ctx, tx, plan.TenantID, brand.BrandID, monitoringBusinessDateText(businessDay))
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, 0, 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if locked {
|
|
||||||
existingKeys, err := s.loadDailyMonitoringTaskKeys(ctx, tx, plan.TenantID, brand.BrandID, businessDay)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, 0, 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
dueCandidates := selectDueMonitoringDailyCandidates(candidates, existingKeys, now, defaultMonitoringDailyLookahead, materializeLimit)
|
|
||||||
dueCount = int64(len(dueCandidates))
|
|
||||||
if len(dueCandidates) > 0 {
|
|
||||||
if err := s.syncMonitoringQuestionSnapshots(ctx, tx, plan.TenantID, brand.BrandID, questions, true); err != nil {
|
|
||||||
return 0, 0, 0, 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
createdCount, err = s.ensureDailyMonitoringCollectTasks(ctx, tx, plan, brand, businessDay, dueCandidates, executionOwner)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, 0, 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if projectionService != nil && createdCount > 0 {
|
|
||||||
if err := projectionService.rebuildBrandDailyProjection(ctx, tx, plan.TenantID, brand.BrandID, businessDay); err != nil {
|
|
||||||
return 0, 0, 0, 0, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := tx.Commit(ctx); err != nil {
|
|
||||||
return 0, 0, 0, 0, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring daily task generation")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if dispatchLimit <= 0 {
|
if dispatchLimit <= 0 {
|
||||||
return createdCount, dueCount, 0, 0, nil
|
return createdCount, eligibleCount, dueCount, 0, 0, nil
|
||||||
}
|
}
|
||||||
dispatchablePlatformIDs, err := s.loadDailyMonitorDispatchablePlatformIDs(ctx, plan.TenantID, plan.WorkspaceID, candidates)
|
dispatchablePlatformIDs, err := s.loadDailyMonitorDispatchablePlatformIDs(ctx, plan.TenantID, plan.WorkspaceID, candidates)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return createdCount, dueCount, 0, 0, err
|
return createdCount, eligibleCount, dueCount, 0, 0, err
|
||||||
}
|
}
|
||||||
if len(dispatchablePlatformIDs) == 0 {
|
if len(dispatchablePlatformIDs) == 0 {
|
||||||
return createdCount, dueCount, 0, 0, nil
|
return createdCount, eligibleCount, dueCount, 0, 0, nil
|
||||||
}
|
}
|
||||||
specs, err := s.loadDueDailyMonitorDesktopTaskSpecs(
|
dispatchablePlatformSet := monitoringDailyPlatformSet(dispatchablePlatformIDs)
|
||||||
|
dispatchableCandidates := filterMonitoringDailyCandidatesByPlatform(candidates, dispatchablePlatformSet)
|
||||||
|
eligibleCount = int64(len(dispatchableCandidates))
|
||||||
|
|
||||||
|
retrySpecs, err := s.loadDueDailyMonitorDesktopTaskSpecs(
|
||||||
ctx,
|
ctx,
|
||||||
s.monitoringPool,
|
s.monitoringPool,
|
||||||
plan.TenantID,
|
plan.TenantID,
|
||||||
plan.WorkspaceID,
|
plan.WorkspaceID,
|
||||||
brand.BrandID,
|
brand.BrandID,
|
||||||
businessDay,
|
businessDay,
|
||||||
now.Add(defaultMonitoringDailyLookahead),
|
now,
|
||||||
now.Add(-defaultMonitoringDailyDesktopRetryCooldown),
|
now.Add(-defaultMonitoringDailyDesktopRetryCooldown),
|
||||||
dispatchablePlatformIDs,
|
dispatchablePlatformIDs,
|
||||||
dispatchLimit,
|
dispatchLimit,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return createdCount, dueCount, 0, 0, err
|
return createdCount, eligibleCount, dueCount, 0, 0, err
|
||||||
|
}
|
||||||
|
if len(retrySpecs) > 0 {
|
||||||
|
specs = append(specs, retrySpecs...)
|
||||||
|
createdCount += int64(len(retrySpecs))
|
||||||
|
dueCount += int64(len(retrySpecs))
|
||||||
|
}
|
||||||
|
|
||||||
|
remainingCreateLimit := dispatchLimit - len(specs)
|
||||||
|
if remainingCreateLimit <= 0 {
|
||||||
|
return s.dispatchDailyMonitoringDesktopSpecs(ctx, specs, maxDesktopClientBacklog, createdCount, eligibleCount, dueCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
materializeLimit = monitoringDailyBoundedLimit(materializeLimit, dispatchLimit)
|
||||||
|
if materializeLimit > 0 && len(dispatchableCandidates) > 0 {
|
||||||
|
var materializeErr error
|
||||||
|
var newCreatedCount int64
|
||||||
|
var newDueCount int64
|
||||||
|
var newSpecs []monitorDesktopTaskSpec
|
||||||
|
newCreatedCount, newDueCount, newSpecs, materializeErr = s.createDispatchableDailyMonitoringTasksForBrand(
|
||||||
|
ctx,
|
||||||
|
projectionService,
|
||||||
|
plan,
|
||||||
|
brand,
|
||||||
|
businessDay,
|
||||||
|
now,
|
||||||
|
questions,
|
||||||
|
dispatchableCandidates,
|
||||||
|
monitoringDailyBoundedLimit(materializeLimit, remainingCreateLimit),
|
||||||
|
executionOwner,
|
||||||
|
)
|
||||||
|
if materializeErr != nil {
|
||||||
|
return createdCount, eligibleCount, dueCount, 0, 0, materializeErr
|
||||||
|
}
|
||||||
|
createdCount += newCreatedCount
|
||||||
|
dueCount += newDueCount
|
||||||
|
specs = append(specs, newSpecs...)
|
||||||
}
|
}
|
||||||
if len(specs) == 0 {
|
if len(specs) == 0 {
|
||||||
return createdCount, dueCount, 0, 0, nil
|
return createdCount, eligibleCount, dueCount, 0, 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return s.dispatchDailyMonitoringDesktopSpecs(ctx, specs, maxDesktopClientBacklog, createdCount, eligibleCount, dueCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *MonitoringService) dispatchDailyMonitoringDesktopSpecs(
|
||||||
|
ctx context.Context,
|
||||||
|
specs []monitorDesktopTaskSpec,
|
||||||
|
maxDesktopClientBacklog int,
|
||||||
|
createdCount int64,
|
||||||
|
eligibleCount int64,
|
||||||
|
dueCount int64,
|
||||||
|
) (int64, int64, int64, int64, int64, error) {
|
||||||
|
if len(specs) == 0 {
|
||||||
|
return createdCount, eligibleCount, dueCount, 0, 0, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
publishedTasks, deferredTaskIDs, err := s.dispatchMonitorDesktopTasks(ctx, specs, maxDesktopClientBacklog)
|
publishedTasks, deferredTaskIDs, err := s.dispatchMonitorDesktopTasks(ctx, specs, maxDesktopClientBacklog)
|
||||||
@@ -550,7 +568,7 @@ func (s *MonitoringService) generateDailyMonitoringTasksForBrand(
|
|||||||
zap.Error(err),
|
zap.Error(err),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
return createdCount, dueCount, 0, 0, err
|
return createdCount, eligibleCount, dueCount, 0, 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, task := range publishedTasks {
|
for _, task := range publishedTasks {
|
||||||
@@ -576,7 +594,7 @@ func (s *MonitoringService) generateDailyMonitoringTasksForBrand(
|
|||||||
cancel()
|
cancel()
|
||||||
}
|
}
|
||||||
|
|
||||||
return createdCount, dueCount, int64(len(publishedTasks)), int64(len(deferredTaskIDs)), nil
|
return createdCount, eligibleCount, dueCount, int64(len(publishedTasks)), int64(len(deferredTaskIDs)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MonitoringService) loadDailyMonitoringPlans(ctx context.Context) ([]monitoringDailyPlan, error) {
|
func (s *MonitoringService) loadDailyMonitoringPlans(ctx context.Context) ([]monitoringDailyPlan, error) {
|
||||||
@@ -901,6 +919,67 @@ func (s *MonitoringService) loadDailyMonitoringTaskKeys(ctx context.Context, q m
|
|||||||
return keys, nil
|
return keys, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *MonitoringService) createDispatchableDailyMonitoringTasksForBrand(
|
||||||
|
ctx context.Context,
|
||||||
|
projectionService *MonitoringCallbackService,
|
||||||
|
plan monitoringDailyPlan,
|
||||||
|
brand monitoringDailyBrand,
|
||||||
|
businessDay time.Time,
|
||||||
|
now time.Time,
|
||||||
|
questions []monitoringConfiguredQuestion,
|
||||||
|
candidates []monitoringDailyTaskCandidate,
|
||||||
|
limit int,
|
||||||
|
executionOwner string,
|
||||||
|
) (int64, int64, []monitorDesktopTaskSpec, error) {
|
||||||
|
if limit <= 0 || len(candidates) == 0 {
|
||||||
|
return 0, 0, nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
tx, err := s.monitoringPool.Begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, nil, response.ErrInternal(50041, "begin_failed", "failed to start monitoring daily task generation")
|
||||||
|
}
|
||||||
|
defer tx.Rollback(ctx)
|
||||||
|
|
||||||
|
locked, err := acquireMonitoringDailyBrandLock(ctx, tx, plan.TenantID, brand.BrandID, monitoringBusinessDateText(businessDay))
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, nil, err
|
||||||
|
}
|
||||||
|
if !locked {
|
||||||
|
return 0, 0, nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
existingKeys, err := s.loadDailyMonitoringTaskKeys(ctx, tx, plan.TenantID, brand.BrandID, businessDay)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, nil, err
|
||||||
|
}
|
||||||
|
dueCandidates := selectDueMonitoringDailyCandidates(candidates, existingKeys, now, defaultMonitoringDailyLookahead, limit)
|
||||||
|
dueCount := int64(len(dueCandidates))
|
||||||
|
if len(dueCandidates) == 0 {
|
||||||
|
if err := tx.Commit(ctx); err != nil {
|
||||||
|
return 0, dueCount, nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring daily task generation")
|
||||||
|
}
|
||||||
|
return 0, dueCount, nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.syncMonitoringQuestionSnapshots(ctx, tx, plan.TenantID, brand.BrandID, questions, true); err != nil {
|
||||||
|
return 0, dueCount, nil, err
|
||||||
|
}
|
||||||
|
createdCount, specs, err := s.ensureDailyMonitoringCollectTasks(ctx, tx, plan, brand, businessDay, dueCandidates, executionOwner)
|
||||||
|
if err != nil {
|
||||||
|
return 0, dueCount, nil, err
|
||||||
|
}
|
||||||
|
if projectionService != nil && createdCount > 0 {
|
||||||
|
if err := projectionService.rebuildBrandDailyProjection(ctx, tx, plan.TenantID, brand.BrandID, businessDay); err != nil {
|
||||||
|
return 0, dueCount, nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := tx.Commit(ctx); err != nil {
|
||||||
|
return 0, dueCount, nil, response.ErrInternal(50041, "commit_failed", "failed to commit monitoring daily task generation")
|
||||||
|
}
|
||||||
|
return createdCount, dueCount, specs, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *MonitoringService) loadDailyMonitorDispatchablePlatformIDs(
|
func (s *MonitoringService) loadDailyMonitorDispatchablePlatformIDs(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
tenantID, workspaceID int64,
|
tenantID, workspaceID int64,
|
||||||
@@ -941,6 +1020,30 @@ func (s *MonitoringService) loadDailyMonitorDispatchablePlatformIDs(
|
|||||||
return reconcileEnabledMonitoringPlatforms(platformIDs), nil
|
return reconcileEnabledMonitoringPlatforms(platformIDs), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func monitoringDailyPlatformSet(platformIDs []string) map[string]struct{} {
|
||||||
|
result := make(map[string]struct{}, len(platformIDs))
|
||||||
|
for _, platformID := range platformIDs {
|
||||||
|
platformID = normalizeMonitoringPlatformID(platformID)
|
||||||
|
if platformID != "" {
|
||||||
|
result[platformID] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func filterMonitoringDailyCandidatesByPlatform(candidates []monitoringDailyTaskCandidate, platforms map[string]struct{}) []monitoringDailyTaskCandidate {
|
||||||
|
if len(candidates) == 0 || len(platforms) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
result := make([]monitoringDailyTaskCandidate, 0, len(candidates))
|
||||||
|
for _, candidate := range candidates {
|
||||||
|
if _, ok := platforms[normalizeMonitoringPlatformID(candidate.Platform.ID)]; ok {
|
||||||
|
result = append(result, candidate)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
func (s *MonitoringService) ensureDailyMonitoringCollectTasks(
|
func (s *MonitoringService) ensureDailyMonitoringCollectTasks(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
tx pgx.Tx,
|
tx pgx.Tx,
|
||||||
@@ -949,12 +1052,12 @@ func (s *MonitoringService) ensureDailyMonitoringCollectTasks(
|
|||||||
businessDay time.Time,
|
businessDay time.Time,
|
||||||
candidates []monitoringDailyTaskCandidate,
|
candidates []monitoringDailyTaskCandidate,
|
||||||
executionOwner string,
|
executionOwner string,
|
||||||
) (int64, error) {
|
) (int64, []monitorDesktopTaskSpec, error) {
|
||||||
if len(candidates) == 0 {
|
if len(candidates) == 0 {
|
||||||
return 0, nil
|
return 0, nil, nil
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(executionOwner) != "desktop_tasks" {
|
if strings.TrimSpace(executionOwner) != "desktop_tasks" {
|
||||||
return 0, response.ErrInternal(50041, "invalid_execution_owner", "daily monitoring tasks require desktop task execution")
|
return 0, nil, response.ErrInternal(50041, "invalid_execution_owner", "daily monitoring tasks require desktop task execution")
|
||||||
}
|
}
|
||||||
|
|
||||||
businessDate := monitoringBusinessDateText(businessDay)
|
businessDate := monitoringBusinessDateText(businessDay)
|
||||||
@@ -969,7 +1072,7 @@ func (s *MonitoringService) ensureDailyMonitoringCollectTasks(
|
|||||||
for _, candidate := range candidates {
|
for _, candidate := range candidates {
|
||||||
questionText, textErr := monitoringQuestionTextSnapshot(candidate.Question)
|
questionText, textErr := monitoringQuestionTextSnapshot(candidate.Question)
|
||||||
if textErr != nil {
|
if textErr != nil {
|
||||||
return 0, textErr
|
return 0, nil, textErr
|
||||||
}
|
}
|
||||||
dispatchAfter := candidate.DispatchAfter
|
dispatchAfter := candidate.DispatchAfter
|
||||||
if dispatchAfter.IsZero() {
|
if dispatchAfter.IsZero() {
|
||||||
@@ -982,47 +1085,90 @@ func (s *MonitoringService) ensureDailyMonitoringCollectTasks(
|
|||||||
dispatchAfterValues = append(dispatchAfterValues, dispatchAfter)
|
dispatchAfterValues = append(dispatchAfterValues, dispatchAfter)
|
||||||
}
|
}
|
||||||
|
|
||||||
tag, err := tx.Exec(ctx, `
|
rows, err := tx.Query(ctx, `
|
||||||
INSERT INTO monitoring_collect_tasks (
|
WITH input AS (
|
||||||
tenant_id, brand_id, question_id, question_hash, question_text_snapshot, ai_platform_id,
|
SELECT *
|
||||||
collector_type, trigger_source, run_mode, business_date, planned_at, status,
|
FROM unnest(
|
||||||
dispatch_priority, dispatch_lane, target_client_id, dispatch_after,
|
$7::bigint[],
|
||||||
ingest_shard_key, execution_owner
|
$8::bytea[],
|
||||||
|
$9::text[],
|
||||||
|
$10::text[],
|
||||||
|
$11::timestamptz[]
|
||||||
|
) AS item(question_id, question_hash, question_text, ai_platform_id, dispatch_after)
|
||||||
|
),
|
||||||
|
upserted AS (
|
||||||
|
INSERT INTO monitoring_collect_tasks (
|
||||||
|
tenant_id, brand_id, question_id, question_hash, question_text_snapshot, ai_platform_id,
|
||||||
|
collector_type, trigger_source, run_mode, business_date, planned_at, status,
|
||||||
|
dispatch_priority, dispatch_lane, target_client_id, dispatch_after,
|
||||||
|
ingest_shard_key, execution_owner
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
$1,
|
||||||
|
$2,
|
||||||
|
input.question_id,
|
||||||
|
input.question_hash,
|
||||||
|
input.question_text,
|
||||||
|
input.ai_platform_id,
|
||||||
|
$3,
|
||||||
|
'automatic',
|
||||||
|
'desktop_standard',
|
||||||
|
$4::date,
|
||||||
|
input.dispatch_after,
|
||||||
|
'pending',
|
||||||
|
100,
|
||||||
|
'normal',
|
||||||
|
NULL,
|
||||||
|
input.dispatch_after,
|
||||||
|
$5,
|
||||||
|
$6
|
||||||
|
FROM input
|
||||||
|
ON CONFLICT (
|
||||||
|
tenant_id, brand_id, question_id, ai_platform_id, collector_type, run_mode, business_date
|
||||||
|
)
|
||||||
|
DO UPDATE SET
|
||||||
|
question_hash = EXCLUDED.question_hash,
|
||||||
|
question_text_snapshot = EXCLUDED.question_text_snapshot,
|
||||||
|
dispatch_after = LEAST(
|
||||||
|
COALESCE(monitoring_collect_tasks.dispatch_after, EXCLUDED.dispatch_after),
|
||||||
|
EXCLUDED.dispatch_after
|
||||||
|
),
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE monitoring_collect_tasks.status = 'pending'
|
||||||
|
AND monitoring_collect_tasks.callback_received_at IS NULL
|
||||||
|
RETURNING *
|
||||||
|
),
|
||||||
|
claimed AS (
|
||||||
|
UPDATE monitoring_collect_tasks t
|
||||||
|
SET last_dispatched_at = NOW(),
|
||||||
|
dispatch_attempts = COALESCE(t.dispatch_attempts, 0) + 1,
|
||||||
|
updated_at = NOW()
|
||||||
|
FROM upserted u
|
||||||
|
WHERE t.id = u.id
|
||||||
|
AND t.status = 'pending'
|
||||||
|
AND t.callback_received_at IS NULL
|
||||||
|
AND (t.dispatch_after IS NULL OR t.dispatch_after <= NOW())
|
||||||
|
AND (
|
||||||
|
t.last_dispatched_at IS NULL
|
||||||
|
OR t.last_dispatched_at <= NOW() - $12::interval
|
||||||
|
)
|
||||||
|
RETURNING t.*
|
||||||
)
|
)
|
||||||
SELECT
|
SELECT
|
||||||
$1,
|
t.id,
|
||||||
$2,
|
t.tenant_id,
|
||||||
input.question_id,
|
t.ai_platform_id,
|
||||||
input.question_hash,
|
t.business_date,
|
||||||
input.question_text,
|
t.trigger_source,
|
||||||
input.ai_platform_id,
|
t.run_mode,
|
||||||
$3,
|
t.question_id,
|
||||||
'automatic',
|
t.question_hash,
|
||||||
'desktop_standard',
|
t.dispatch_priority,
|
||||||
$4::date,
|
t.dispatch_lane,
|
||||||
input.dispatch_after,
|
t.interrupt_generation,
|
||||||
'pending',
|
t.question_text_snapshot
|
||||||
100,
|
FROM claimed t
|
||||||
'normal',
|
ORDER BY t.dispatch_priority DESC, t.id ASC
|
||||||
NULL,
|
|
||||||
input.dispatch_after,
|
|
||||||
$5,
|
|
||||||
$6
|
|
||||||
FROM unnest(
|
|
||||||
$7::bigint[],
|
|
||||||
$8::bytea[],
|
|
||||||
$9::text[],
|
|
||||||
$10::text[],
|
|
||||||
$11::timestamptz[]
|
|
||||||
) AS input(question_id, question_hash, question_text, ai_platform_id, dispatch_after)
|
|
||||||
ON CONFLICT (
|
|
||||||
tenant_id, brand_id, question_id, ai_platform_id, collector_type, run_mode, business_date
|
|
||||||
)
|
|
||||||
DO UPDATE SET
|
|
||||||
question_hash = EXCLUDED.question_hash,
|
|
||||||
question_text_snapshot = EXCLUDED.question_text_snapshot,
|
|
||||||
updated_at = NOW()
|
|
||||||
WHERE btrim(COALESCE(monitoring_collect_tasks.question_text_snapshot, '')) = ''
|
|
||||||
`,
|
`,
|
||||||
plan.TenantID,
|
plan.TenantID,
|
||||||
brand.BrandID,
|
brand.BrandID,
|
||||||
@@ -1035,12 +1181,50 @@ func (s *MonitoringService) ensureDailyMonitoringCollectTasks(
|
|||||||
questionTexts,
|
questionTexts,
|
||||||
platformIDs,
|
platformIDs,
|
||||||
dispatchAfterValues,
|
dispatchAfterValues,
|
||||||
|
formatPgInterval(defaultMonitoringDailyDesktopRetryCooldown),
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, response.ErrInternal(50041, "insert_failed", "failed to create daily monitoring tasks")
|
return 0, nil, response.ErrInternal(50041, "insert_failed", "failed to create daily monitoring tasks")
|
||||||
}
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
return tag.RowsAffected(), nil
|
specs := make([]monitorDesktopTaskSpec, 0, len(candidates))
|
||||||
|
for rows.Next() {
|
||||||
|
var (
|
||||||
|
spec monitorDesktopTaskSpec
|
||||||
|
questionHash []byte
|
||||||
|
businessDay time.Time
|
||||||
|
)
|
||||||
|
if scanErr := rows.Scan(
|
||||||
|
&spec.MonitorTaskID,
|
||||||
|
&spec.TenantID,
|
||||||
|
&spec.PlatformID,
|
||||||
|
&businessDay,
|
||||||
|
&spec.TriggerSource,
|
||||||
|
&spec.RunMode,
|
||||||
|
&spec.QuestionID,
|
||||||
|
&questionHash,
|
||||||
|
&spec.Priority,
|
||||||
|
&spec.Lane,
|
||||||
|
&spec.InterruptGeneration,
|
||||||
|
&spec.QuestionText,
|
||||||
|
); scanErr != nil {
|
||||||
|
return 0, nil, response.ErrInternal(50041, "scan_failed", "failed to parse created daily monitoring tasks")
|
||||||
|
}
|
||||||
|
spec.WorkspaceID = plan.WorkspaceID
|
||||||
|
spec.BusinessDate = businessDay.Format("2006-01-02")
|
||||||
|
spec.QuestionHash = encodeQuestionHash(questionHash)
|
||||||
|
spec.QuestionText = strings.TrimSpace(spec.QuestionText)
|
||||||
|
if spec.QuestionText == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
spec.SchedulerGroupKey = monitoringSchedulerGroupKey(spec.QuestionHash, spec.QuestionID, spec.QuestionText)
|
||||||
|
specs = append(specs, spec)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return 0, nil, response.ErrInternal(50041, "scan_failed", "failed to iterate created daily monitoring tasks")
|
||||||
|
}
|
||||||
|
return int64(len(specs)), specs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MonitoringService) loadDueDailyMonitorDesktopTaskSpecs(
|
func (s *MonitoringService) loadDueDailyMonitorDesktopTaskSpecs(
|
||||||
|
|||||||
@@ -225,10 +225,10 @@ func TestEnsureDailyMonitoringCollectTasksPersistsQuestionTextSnapshot(t *testin
|
|||||||
tx := &captureDailyMonitorTaskTx{commandTag: pgconn.NewCommandTag("INSERT 0 2")}
|
tx := &captureDailyMonitorTaskTx{commandTag: pgconn.NewCommandTag("INSERT 0 2")}
|
||||||
businessDay := time.Date(2026, 6, 22, 0, 0, 0, 0, time.UTC)
|
businessDay := time.Date(2026, 6, 22, 0, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
created, err := (&MonitoringService{}).ensureDailyMonitoringCollectTasks(
|
created, specs, err := (&MonitoringService{}).ensureDailyMonitoringCollectTasks(
|
||||||
ctx,
|
ctx,
|
||||||
tx,
|
tx,
|
||||||
monitoringDailyPlan{TenantID: 4},
|
monitoringDailyPlan{TenantID: 4, WorkspaceID: 8},
|
||||||
monitoringDailyBrand{BrandID: 8},
|
monitoringDailyBrand{BrandID: 8},
|
||||||
businessDay,
|
businessDay,
|
||||||
[]monitoringDailyTaskCandidate{
|
[]monitoringDailyTaskCandidate{
|
||||||
@@ -253,23 +253,27 @@ func TestEnsureDailyMonitoringCollectTasksPersistsQuestionTextSnapshot(t *testin
|
|||||||
)
|
)
|
||||||
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, int64(2), created)
|
assert.Equal(t, int64(0), created)
|
||||||
assert.True(t, tx.execCalled)
|
assert.Empty(t, specs)
|
||||||
normalizedSQL := normalizeSQLWhitespace(tx.execSQL)
|
assert.True(t, tx.queryCalled)
|
||||||
|
normalizedSQL := normalizeSQLWhitespace(tx.querySQL)
|
||||||
assert.Contains(t, normalizedSQL, "question_text_snapshot, ai_platform_id")
|
assert.Contains(t, normalizedSQL, "question_text_snapshot, ai_platform_id")
|
||||||
assert.Contains(t, normalizedSQL, "$10::text[]")
|
assert.Contains(t, normalizedSQL, "$10::text[]")
|
||||||
assert.Contains(t, normalizedSQL, "AS input(question_id, question_hash, question_text, ai_platform_id, dispatch_after)")
|
assert.Contains(t, normalizedSQL, "AS item(question_id, question_hash, question_text, ai_platform_id, dispatch_after)")
|
||||||
assert.Contains(t, normalizedSQL, "question_text_snapshot = EXCLUDED.question_text_snapshot")
|
assert.Contains(t, normalizedSQL, "question_text_snapshot = EXCLUDED.question_text_snapshot")
|
||||||
require.Len(t, tx.execArgs, 11)
|
assert.Contains(t, normalizedSQL, "WHERE monitoring_collect_tasks.status = 'pending'")
|
||||||
assert.Equal(t, []string{"幽灵门轨道长度定制", "幽灵门轨道长度定制"}, tx.execArgs[8])
|
assert.Contains(t, normalizedSQL, "t.dispatch_after IS NULL OR t.dispatch_after <= NOW()")
|
||||||
assert.Equal(t, []string{"qwen", "doubao"}, tx.execArgs[9])
|
assert.Contains(t, normalizedSQL, "SET last_dispatched_at = NOW()")
|
||||||
|
require.Len(t, tx.queryArgs, 12)
|
||||||
|
assert.Equal(t, []string{"幽灵门轨道长度定制", "幽灵门轨道长度定制"}, tx.queryArgs[8])
|
||||||
|
assert.Equal(t, []string{"qwen", "doubao"}, tx.queryArgs[9])
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEnsureDailyMonitoringCollectTasksRejectsEmptyQuestionText(t *testing.T) {
|
func TestEnsureDailyMonitoringCollectTasksRejectsEmptyQuestionText(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
tx := &captureDailyMonitorTaskTx{}
|
tx := &captureDailyMonitorTaskTx{}
|
||||||
|
|
||||||
created, err := (&MonitoringService{}).ensureDailyMonitoringCollectTasks(
|
created, specs, err := (&MonitoringService{}).ensureDailyMonitoringCollectTasks(
|
||||||
ctx,
|
ctx,
|
||||||
tx,
|
tx,
|
||||||
monitoringDailyPlan{TenantID: 4},
|
monitoringDailyPlan{TenantID: 4},
|
||||||
@@ -290,10 +294,26 @@ func TestEnsureDailyMonitoringCollectTasksRejectsEmptyQuestionText(t *testing.T)
|
|||||||
|
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
assert.Equal(t, int64(0), created)
|
assert.Equal(t, int64(0), created)
|
||||||
|
assert.Empty(t, specs)
|
||||||
assert.False(t, tx.execCalled)
|
assert.False(t, tx.execCalled)
|
||||||
|
assert.False(t, tx.queryCalled)
|
||||||
assert.Contains(t, err.Error(), "missing_question_text_snapshot")
|
assert.Contains(t, err.Error(), "missing_question_text_snapshot")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFilterMonitoringDailyCandidatesByPlatformKeepsOnlyOnlineDispatchablePlatforms(t *testing.T) {
|
||||||
|
candidates := []monitoringDailyTaskCandidate{
|
||||||
|
{Question: monitoringConfiguredQuestion{ID: 1}, Platform: monitoringPlatformMetadata{ID: "qwen"}},
|
||||||
|
{Question: monitoringConfiguredQuestion{ID: 1}, Platform: monitoringPlatformMetadata{ID: "doubao"}},
|
||||||
|
{Question: monitoringConfiguredQuestion{ID: 2}, Platform: monitoringPlatformMetadata{ID: "kimi"}},
|
||||||
|
}
|
||||||
|
|
||||||
|
filtered := filterMonitoringDailyCandidatesByPlatform(candidates, monitoringDailyPlatformSet([]string{"qwen", "kimi"}))
|
||||||
|
|
||||||
|
require.Len(t, filtered, 2)
|
||||||
|
assert.Equal(t, "qwen", filtered[0].Platform.ID)
|
||||||
|
assert.Equal(t, "kimi", filtered[1].Platform.ID)
|
||||||
|
}
|
||||||
|
|
||||||
func TestDecodeDailyMonitoringEnabledPlatformsReportsMalformedJSON(t *testing.T) {
|
func TestDecodeDailyMonitoringEnabledPlatformsReportsMalformedJSON(t *testing.T) {
|
||||||
platforms, err := decodeDailyMonitoringEnabledPlatforms([]byte(`["qwen"`))
|
platforms, err := decodeDailyMonitoringEnabledPlatforms([]byte(`["qwen"`))
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
@@ -399,6 +419,7 @@ func TestMonitoringDailyTaskMetricsRecordsSummaryCounters(t *testing.T) {
|
|||||||
FailedPlanCount: 1,
|
FailedPlanCount: 1,
|
||||||
FailedBrandCount: 2,
|
FailedBrandCount: 2,
|
||||||
PlannedTaskCount: 20,
|
PlannedTaskCount: 20,
|
||||||
|
EligibleTaskCount: 12,
|
||||||
DueTaskCount: 8,
|
DueTaskCount: 8,
|
||||||
CreatedTaskCount: 6,
|
CreatedTaskCount: 6,
|
||||||
DesktopTaskCount: 4,
|
DesktopTaskCount: 4,
|
||||||
@@ -415,6 +436,7 @@ func TestMonitoringDailyTaskMetricsRecordsSummaryCounters(t *testing.T) {
|
|||||||
assert.Equal(t, int64(2), snapshot.BrandFailureTotal)
|
assert.Equal(t, int64(2), snapshot.BrandFailureTotal)
|
||||||
assert.Equal(t, int64(1), snapshot.SkippedPlanTotal)
|
assert.Equal(t, int64(1), snapshot.SkippedPlanTotal)
|
||||||
assert.Equal(t, int64(20), snapshot.PlannedTaskTotal)
|
assert.Equal(t, int64(20), snapshot.PlannedTaskTotal)
|
||||||
|
assert.Equal(t, int64(12), snapshot.EligibleTaskTotal)
|
||||||
assert.Equal(t, int64(8), snapshot.DueTaskTotal)
|
assert.Equal(t, int64(8), snapshot.DueTaskTotal)
|
||||||
assert.Equal(t, int64(6), snapshot.CreatedTaskTotal)
|
assert.Equal(t, int64(6), snapshot.CreatedTaskTotal)
|
||||||
assert.Equal(t, int64(4), snapshot.DesktopTaskTotal)
|
assert.Equal(t, int64(4), snapshot.DesktopTaskTotal)
|
||||||
@@ -428,6 +450,7 @@ func TestMonitoringDailyTaskMetricsRecordsSummaryCounters(t *testing.T) {
|
|||||||
prometheus := MonitoringDailyTaskMetricsPrometheus()
|
prometheus := MonitoringDailyTaskMetricsPrometheus()
|
||||||
assert.Contains(t, prometheus, `monitoring_daily_task_worker_runs_total{status="completed"} 1`)
|
assert.Contains(t, prometheus, `monitoring_daily_task_worker_runs_total{status="completed"} 1`)
|
||||||
assert.Contains(t, prometheus, `monitoring_daily_task_generation_failures_total{scope="plan"} 1`)
|
assert.Contains(t, prometheus, `monitoring_daily_task_generation_failures_total{scope="plan"} 1`)
|
||||||
|
assert.Contains(t, prometheus, `monitoring_daily_task_generation_tasks_total{stage="eligible"} 12`)
|
||||||
assert.Contains(t, prometheus, `monitoring_daily_task_generation_tasks_total{stage="created"} 6`)
|
assert.Contains(t, prometheus, `monitoring_daily_task_generation_tasks_total{stage="created"} 6`)
|
||||||
assert.Contains(t, prometheus, `monitoring_daily_task_worker_last_business_date_info{business_date="2026-04-24"} 1`)
|
assert.Contains(t, prometheus, `monitoring_daily_task_worker_last_business_date_info{business_date="2026-04-24"} 1`)
|
||||||
}
|
}
|
||||||
@@ -568,12 +591,15 @@ func (r emptyDailyMonitorRows) RawValues() [][]byte { r
|
|||||||
func (r emptyDailyMonitorRows) Conn() *pgx.Conn { return nil }
|
func (r emptyDailyMonitorRows) Conn() *pgx.Conn { return nil }
|
||||||
|
|
||||||
type captureDailyMonitorTaskTx struct {
|
type captureDailyMonitorTaskTx struct {
|
||||||
execCalled bool
|
execCalled bool
|
||||||
execSQL string
|
execSQL string
|
||||||
execArgs []any
|
execArgs []any
|
||||||
execSQLs []string
|
execSQLs []string
|
||||||
execArgses [][]any
|
execArgses [][]any
|
||||||
commandTag pgconn.CommandTag
|
queryCalled bool
|
||||||
|
querySQL string
|
||||||
|
queryArgs []any
|
||||||
|
commandTag pgconn.CommandTag
|
||||||
}
|
}
|
||||||
|
|
||||||
func (tx *captureDailyMonitorTaskTx) Begin(context.Context) (pgx.Tx, error) { return tx, nil }
|
func (tx *captureDailyMonitorTaskTx) Begin(context.Context) (pgx.Tx, error) { return tx, nil }
|
||||||
@@ -597,7 +623,10 @@ func (tx *captureDailyMonitorTaskTx) Exec(_ context.Context, sql string, argumen
|
|||||||
tx.execArgses = append(tx.execArgses, arguments)
|
tx.execArgses = append(tx.execArgses, arguments)
|
||||||
return tx.commandTag, nil
|
return tx.commandTag, nil
|
||||||
}
|
}
|
||||||
func (tx *captureDailyMonitorTaskTx) Query(context.Context, string, ...any) (pgx.Rows, error) {
|
func (tx *captureDailyMonitorTaskTx) Query(_ context.Context, sql string, arguments ...any) (pgx.Rows, error) {
|
||||||
|
tx.queryCalled = true
|
||||||
|
tx.querySQL = sql
|
||||||
|
tx.queryArgs = arguments
|
||||||
return emptyDailyMonitorRows{}, nil
|
return emptyDailyMonitorRows{}, nil
|
||||||
}
|
}
|
||||||
func (tx *captureDailyMonitorTaskTx) QueryRow(context.Context, string, ...any) pgx.Row {
|
func (tx *captureDailyMonitorTaskTx) QueryRow(context.Context, string, ...any) pgx.Row {
|
||||||
|
|||||||
@@ -98,6 +98,7 @@ func (monitoringDailyTaskMetricsCollector) Collect(ch chan<- prometheus.Metric)
|
|||||||
monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.BrandFailureTotal), []string{"scope"}, "brand")
|
monitoringCounter(ch, "monitoring_daily_task_generation_failures_total", "Total non-fatal daily monitoring generation failures by scope.", float64(snapshot.BrandFailureTotal), []string{"scope"}, "brand")
|
||||||
monitoringCounter(ch, "monitoring_daily_task_generation_skipped_plans_total", "Total daily monitoring plans skipped before generation.", float64(snapshot.SkippedPlanTotal), nil)
|
monitoringCounter(ch, "monitoring_daily_task_generation_skipped_plans_total", "Total daily monitoring plans skipped before generation.", float64(snapshot.SkippedPlanTotal), nil)
|
||||||
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.PlannedTaskTotal), []string{"stage"}, "planned")
|
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.PlannedTaskTotal), []string{"stage"}, "planned")
|
||||||
|
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.EligibleTaskTotal), []string{"stage"}, "eligible")
|
||||||
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DueTaskTotal), []string{"stage"}, "due")
|
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DueTaskTotal), []string{"stage"}, "due")
|
||||||
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.CreatedTaskTotal), []string{"stage"}, "created")
|
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.CreatedTaskTotal), []string{"stage"}, "created")
|
||||||
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DesktopTaskTotal), []string{"stage"}, "desktop")
|
monitoringCounter(ch, "monitoring_daily_task_generation_tasks_total", "Total daily monitoring tasks seen by generation stage.", float64(snapshot.DesktopTaskTotal), []string{"stage"}, "desktop")
|
||||||
|
|||||||
@@ -93,7 +93,7 @@ INSERT INTO ops.scheduler_jobs
|
|||||||
(job_key, display_name, category, description, enabled, schedule_type, interval_seconds, timezone, timeout_seconds, batch_size, config)
|
(job_key, display_name, category, description, enabled, schedule_type, interval_seconds, timezone, timeout_seconds, batch_size, config)
|
||||||
VALUES
|
VALUES
|
||||||
('schedule_dispatch', '内容定时任务派发', 'content', '索引驱动的延迟队列派发,仅认领到期内容任务并投递生成队列。', true, 'interval', 15, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
|
('schedule_dispatch', '内容定时任务派发', 'content', '索引驱动的延迟队列派发,仅认领到期内容任务并投递生成队列。', true, 'interval', 15, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
|
||||||
('monitoring_daily_task', 'AI 监控每日任务生成', 'monitoring', '按业务日全量创建监控采集事实,按容量限速派发到桌面任务队列。', true, 'interval', 300, 'Asia/Shanghai', 45, 12, '{"max_desktop_client_backlog":12}'::jsonb),
|
('monitoring_daily_task', 'AI 监控每日任务生成', 'monitoring', '客户端在线时按业务日滚动创建并派发监控采集任务,离线不生成待采积压。', true, 'interval', 300, 'Asia/Shanghai', 45, 12, '{"max_desktop_client_backlog":12}'::jsonb),
|
||||||
('monitoring_result_recovery', '监控结果恢复', 'monitoring', '按回调结果队列索引恢复未成功入队的监控结果。', true, 'interval', 60, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
|
('monitoring_result_recovery', '监控结果恢复', 'monitoring', '按回调结果队列索引恢复未成功入队的监控结果。', true, 'interval', 60, 'Asia/Shanghai', 30, 100, '{}'::jsonb),
|
||||||
('monitoring_lease_recovery', '监控租约回收', 'monitoring', '按过期租约索引限批回收超过租约时间仍未回调的监控任务。', true, 'interval', 1800, 'Asia/Shanghai', 30, 1000, '{}'::jsonb),
|
('monitoring_lease_recovery', '监控租约回收', 'monitoring', '按过期租约索引限批回收超过租约时间仍未回调的监控任务。', true, 'interval', 1800, 'Asia/Shanghai', 30, 1000, '{}'::jsonb),
|
||||||
('monitoring_received_inspection', '监控 received 卡死检查', 'monitoring', '按 received watchdog 索引限批标记长时间未入队的监控任务。', true, 'interval', 300, 'Asia/Shanghai', 30, 100, '{"threshold":"15m"}'::jsonb),
|
('monitoring_received_inspection', '监控 received 卡死检查', 'monitoring', '按 received watchdog 索引限批标记长时间未入队的监控任务。', true, 'interval', 300, 'Asia/Shanghai', 30, 100, '{"threshold":"15m"}'::jsonb),
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ SET description = '索引驱动的延迟队列派发,仅认领到期内容任
|
|||||||
WHERE job_key = 'schedule_dispatch';
|
WHERE job_key = 'schedule_dispatch';
|
||||||
|
|
||||||
UPDATE ops.scheduler_jobs
|
UPDATE ops.scheduler_jobs
|
||||||
SET description = '按业务日全量创建监控采集事实,按容量限速派发到桌面任务队列。',
|
SET description = '客户端在线时按业务日滚动创建并派发监控采集任务,离线不生成待采积压。',
|
||||||
timeout_seconds = 45,
|
timeout_seconds = 45,
|
||||||
batch_size = 12,
|
batch_size = 12,
|
||||||
config = (config - 'max_materialize_per_run' - 'max_materialize_per_plan' - 'max_materialize_per_brand')
|
config = (config - 'max_materialize_per_run' - 'max_materialize_per_plan' - 'max_materialize_per_brand')
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
UPDATE ops.scheduler_jobs
|
UPDATE ops.scheduler_jobs
|
||||||
SET description = '按业务日全量创建监控采集事实,按容量限速派发到桌面任务队列。',
|
SET description = '客户端在线时按业务日滚动创建并派发监控采集任务,离线不生成待采积压。',
|
||||||
batch_size = 12,
|
batch_size = 12,
|
||||||
config = (config - 'max_materialize_per_run' - 'max_materialize_per_plan' - 'max_materialize_per_brand')
|
config = (config - 'max_materialize_per_run' - 'max_materialize_per_plan' - 'max_materialize_per_brand')
|
||||||
|| '{"max_desktop_client_backlog":12}'::jsonb,
|
|| '{"max_desktop_client_backlog":12}'::jsonb,
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
UPDATE ops.scheduler_jobs
|
||||||
|
SET description = '按业务日全量创建监控采集事实,按容量限速派发到桌面任务队列。',
|
||||||
|
batch_size = 12,
|
||||||
|
config = config || '{"max_desktop_client_backlog":12}'::jsonb,
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE job_key = 'monitoring_daily_task';
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
UPDATE ops.scheduler_jobs
|
||||||
|
SET description = '客户端在线时按业务日滚动创建并派发监控采集任务,离线不生成待采积压。',
|
||||||
|
batch_size = 12,
|
||||||
|
config = (config - 'max_materialize_per_run' - 'max_materialize_per_plan' - 'max_materialize_per_brand')
|
||||||
|
|| '{"max_desktop_client_backlog":12}'::jsonb,
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE job_key = 'monitoring_daily_task';
|
||||||
Reference in New Issue
Block a user