harden article generation reliability
This commit is contained in:
@@ -0,0 +1,240 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const GenerationTaskStatusRunning = "running"
|
||||
|
||||
const (
|
||||
GenerationTaskEventTransition = "transition"
|
||||
GenerationTaskEventFailure = "failure"
|
||||
GenerationTaskEventCleanupFailure = "cleanup_failure"
|
||||
GenerationTaskEventQueueFailure = "queue_failure"
|
||||
GenerationTaskEventRecoveryRequeue = "recovery_requeue"
|
||||
GenerationTaskEventRecoveryFinalize = "recovery_finalize"
|
||||
GenerationTaskEventRecoverySkipped = "recovery_skipped"
|
||||
GenerationTaskEventStateAnomaly = "state_anomaly"
|
||||
GenerationTaskEventStateCheckFailure = "state_check_failure"
|
||||
)
|
||||
|
||||
const (
|
||||
GenerationTaskResultSuccess = "success"
|
||||
GenerationTaskResultFailure = "failure"
|
||||
GenerationTaskResultSkipped = "skipped"
|
||||
)
|
||||
|
||||
type GenerationTaskLogContext struct {
|
||||
TaskID int64
|
||||
TenantID int64
|
||||
OperatorID int64
|
||||
ArticleID int64
|
||||
ReservationID int64
|
||||
TaskType string
|
||||
Stage string
|
||||
AttemptCount int
|
||||
LeaseOwner string
|
||||
StatusBefore string
|
||||
StatusAfter string
|
||||
Result string
|
||||
AnomalyType string
|
||||
Severity string
|
||||
Duration time.Duration
|
||||
}
|
||||
|
||||
func GenerationTaskLogFields(ctx GenerationTaskLogContext) []zap.Field {
|
||||
fields := []zap.Field{zap.String("event_domain", "article_generation")}
|
||||
appendStringField := func(key, value string) {
|
||||
if value = strings.TrimSpace(value); value != "" {
|
||||
fields = append(fields, zap.String(key, value))
|
||||
}
|
||||
}
|
||||
appendInt64Field := func(key string, value int64) {
|
||||
if value > 0 {
|
||||
fields = append(fields, zap.Int64(key, value))
|
||||
}
|
||||
}
|
||||
|
||||
appendInt64Field("task_id", ctx.TaskID)
|
||||
appendInt64Field("tenant_id", ctx.TenantID)
|
||||
appendInt64Field("operator_id", ctx.OperatorID)
|
||||
appendInt64Field("article_id", ctx.ArticleID)
|
||||
appendInt64Field("quota_reservation_id", ctx.ReservationID)
|
||||
appendStringField("task_type", ctx.TaskType)
|
||||
appendStringField("stage", ctx.Stage)
|
||||
if ctx.AttemptCount > 0 {
|
||||
fields = append(fields, zap.Int("attempt_count", ctx.AttemptCount))
|
||||
}
|
||||
appendStringField("lease_owner", ctx.LeaseOwner)
|
||||
appendStringField("status_before", ctx.StatusBefore)
|
||||
appendStringField("status_after", ctx.StatusAfter)
|
||||
appendStringField("result", ctx.Result)
|
||||
appendStringField("anomaly_type", ctx.AnomalyType)
|
||||
appendStringField("severity", ctx.Severity)
|
||||
if ctx.Duration > 0 {
|
||||
fields = append(fields, zap.Duration("duration", ctx.Duration))
|
||||
fields = append(fields, zap.Int64("duration_ms", ctx.Duration.Milliseconds()))
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
func LogGenerationTaskInfo(logger *zap.Logger, message string, ctx GenerationTaskLogContext, fields ...zap.Field) {
|
||||
if logger == nil {
|
||||
return
|
||||
}
|
||||
logger.Info(message, append(GenerationTaskLogFields(ctx), fields...)...)
|
||||
}
|
||||
|
||||
func LogGenerationTaskWarn(logger *zap.Logger, message string, ctx GenerationTaskLogContext, fields ...zap.Field) {
|
||||
if logger == nil {
|
||||
return
|
||||
}
|
||||
logger.Warn(message, append(GenerationTaskLogFields(ctx), fields...)...)
|
||||
}
|
||||
|
||||
func LogGenerationTaskError(logger *zap.Logger, message string, ctx GenerationTaskLogContext, fields ...zap.Field) {
|
||||
if logger == nil {
|
||||
return
|
||||
}
|
||||
logger.Error(message, append(GenerationTaskLogFields(ctx), fields...)...)
|
||||
}
|
||||
|
||||
type GenerationTaskMetricsSnapshot struct {
|
||||
TransitionTotal int64 `json:"transition_total"`
|
||||
FailureTotal int64 `json:"failure_total"`
|
||||
CleanupFailureTotal int64 `json:"cleanup_failure_total"`
|
||||
QueueFailureTotal int64 `json:"queue_failure_total"`
|
||||
RecoveryRequeueTotal int64 `json:"recovery_requeue_total"`
|
||||
RecoveryFinalizeTotal int64 `json:"recovery_finalize_total"`
|
||||
StateAnomalyTotal int64 `json:"state_anomaly_total"`
|
||||
StateCheckFailureTotal int64 `json:"state_check_failure_total"`
|
||||
LastEventAt *time.Time `json:"last_event_at,omitempty"`
|
||||
LastEvent string `json:"last_event,omitempty"`
|
||||
LastTaskType string `json:"last_task_type,omitempty"`
|
||||
LastStage string `json:"last_stage,omitempty"`
|
||||
LastResult string `json:"last_result,omitempty"`
|
||||
LastAnomalyType string `json:"last_anomaly_type,omitempty"`
|
||||
LastSeverity string `json:"last_severity,omitempty"`
|
||||
}
|
||||
|
||||
type generationTaskMetricsRecorder struct {
|
||||
transitionTotal atomic.Int64
|
||||
failureTotal atomic.Int64
|
||||
cleanupFailureTotal atomic.Int64
|
||||
queueFailureTotal atomic.Int64
|
||||
recoveryRequeueTotal atomic.Int64
|
||||
recoveryFinalizeTotal atomic.Int64
|
||||
stateAnomalyTotal atomic.Int64
|
||||
stateCheckFailureTotal atomic.Int64
|
||||
lastEventAtUnixNano atomic.Int64
|
||||
lastEvent atomic.Value
|
||||
lastTaskType atomic.Value
|
||||
lastStage atomic.Value
|
||||
lastResult atomic.Value
|
||||
lastAnomalyType atomic.Value
|
||||
lastSeverity atomic.Value
|
||||
}
|
||||
|
||||
var generationTaskMetrics = newGenerationTaskMetricsRecorder()
|
||||
|
||||
func newGenerationTaskMetricsRecorder() *generationTaskMetricsRecorder {
|
||||
return &generationTaskMetricsRecorder{}
|
||||
}
|
||||
|
||||
func RecordGenerationTaskEvent(event string, ctx GenerationTaskLogContext) {
|
||||
generationTaskMetrics.record(event, ctx)
|
||||
}
|
||||
|
||||
func GenerationTaskMetricsSnapshotValue() GenerationTaskMetricsSnapshot {
|
||||
return generationTaskMetrics.snapshot()
|
||||
}
|
||||
|
||||
func resetGenerationTaskMetricsForTest() {
|
||||
generationTaskMetrics = newGenerationTaskMetricsRecorder()
|
||||
}
|
||||
|
||||
func (r *generationTaskMetricsRecorder) record(event string, ctx GenerationTaskLogContext) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
event = strings.TrimSpace(event)
|
||||
if event == "" {
|
||||
event = "unknown"
|
||||
}
|
||||
switch event {
|
||||
case GenerationTaskEventTransition:
|
||||
r.transitionTotal.Add(1)
|
||||
case GenerationTaskEventFailure:
|
||||
r.failureTotal.Add(1)
|
||||
case GenerationTaskEventCleanupFailure:
|
||||
r.cleanupFailureTotal.Add(1)
|
||||
case GenerationTaskEventQueueFailure:
|
||||
r.queueFailureTotal.Add(1)
|
||||
case GenerationTaskEventRecoveryRequeue:
|
||||
r.recoveryRequeueTotal.Add(1)
|
||||
case GenerationTaskEventRecoveryFinalize:
|
||||
r.recoveryFinalizeTotal.Add(1)
|
||||
case GenerationTaskEventStateAnomaly:
|
||||
r.stateAnomalyTotal.Add(1)
|
||||
case GenerationTaskEventStateCheckFailure:
|
||||
r.stateCheckFailureTotal.Add(1)
|
||||
}
|
||||
r.lastEventAtUnixNano.Store(time.Now().UTC().UnixNano())
|
||||
r.lastEvent.Store(event)
|
||||
storeGenerationMetricString(&r.lastTaskType, ctx.TaskType)
|
||||
storeGenerationMetricString(&r.lastStage, ctx.Stage)
|
||||
storeGenerationMetricString(&r.lastResult, ctx.Result)
|
||||
storeGenerationMetricString(&r.lastAnomalyType, ctx.AnomalyType)
|
||||
storeGenerationMetricString(&r.lastSeverity, ctx.Severity)
|
||||
}
|
||||
|
||||
func (r *generationTaskMetricsRecorder) snapshot() GenerationTaskMetricsSnapshot {
|
||||
if r == nil {
|
||||
return GenerationTaskMetricsSnapshot{}
|
||||
}
|
||||
return GenerationTaskMetricsSnapshot{
|
||||
TransitionTotal: r.transitionTotal.Load(),
|
||||
FailureTotal: r.failureTotal.Load(),
|
||||
CleanupFailureTotal: r.cleanupFailureTotal.Load(),
|
||||
QueueFailureTotal: r.queueFailureTotal.Load(),
|
||||
RecoveryRequeueTotal: r.recoveryRequeueTotal.Load(),
|
||||
RecoveryFinalizeTotal: r.recoveryFinalizeTotal.Load(),
|
||||
StateAnomalyTotal: r.stateAnomalyTotal.Load(),
|
||||
StateCheckFailureTotal: r.stateCheckFailureTotal.Load(),
|
||||
LastEventAt: generationMetricUnixNanoTime(r.lastEventAtUnixNano.Load()),
|
||||
LastEvent: loadGenerationMetricString(&r.lastEvent),
|
||||
LastTaskType: loadGenerationMetricString(&r.lastTaskType),
|
||||
LastStage: loadGenerationMetricString(&r.lastStage),
|
||||
LastResult: loadGenerationMetricString(&r.lastResult),
|
||||
LastAnomalyType: loadGenerationMetricString(&r.lastAnomalyType),
|
||||
LastSeverity: loadGenerationMetricString(&r.lastSeverity),
|
||||
}
|
||||
}
|
||||
|
||||
func storeGenerationMetricString(target *atomic.Value, value string) {
|
||||
if target == nil {
|
||||
return
|
||||
}
|
||||
target.Store(strings.TrimSpace(value))
|
||||
}
|
||||
|
||||
func loadGenerationMetricString(value *atomic.Value) string {
|
||||
if value == nil {
|
||||
return ""
|
||||
}
|
||||
raw := value.Load()
|
||||
text, _ := raw.(string)
|
||||
return text
|
||||
}
|
||||
|
||||
func generationMetricUnixNanoTime(value int64) *time.Time {
|
||||
if value <= 0 {
|
||||
return nil
|
||||
}
|
||||
t := time.Unix(0, value).UTC()
|
||||
return &t
|
||||
}
|
||||
Reference in New Issue
Block a user