geo/server/internal/tenant/app/generation_observability.go

package app

import (
	"errors"
	"strings"
	"sync/atomic"
	"time"

	"go.uber.org/zap"

	"github.com/geo-platform/tenant-api/internal/shared/response"
)

const GenerationTaskStatusRunning = "running"

const (
	GenerationTaskEventTransition        = "transition"
	GenerationTaskEventFailure           = "failure"
	GenerationTaskEventCleanupFailure    = "cleanup_failure"
	GenerationTaskEventQueueFailure      = "queue_failure"
	GenerationTaskEventRecoveryRequeue   = "recovery_requeue"
	GenerationTaskEventRecoveryFinalize  = "recovery_finalize"
	GenerationTaskEventRecoverySkipped   = "recovery_skipped"
	GenerationTaskEventStateAnomaly      = "state_anomaly"
	GenerationTaskEventStateCheckFailure = "state_check_failure"
)

const (
	GenerationTaskResultSuccess = "success"
	GenerationTaskResultFailure = "failure"
	GenerationTaskResultSkipped = "skipped"
)

type GenerationTaskLogContext struct {
	TaskID        int64
	TenantID      int64
	OperatorID    int64
	ArticleID     int64
	ReservationID int64
	TaskType      string
	Stage         string
	AttemptCount  int
	LeaseOwner    string
	StatusBefore  string
	StatusAfter   string
	Result        string
	AnomalyType   string
	Severity      string
	Duration      time.Duration
}

func GenerationTaskLogFields(ctx GenerationTaskLogContext) []zap.Field {
	fields := []zap.Field{zap.String("event_domain", "article_generation")}
	appendStringField := func(key, value string) {
		if value = strings.TrimSpace(value); value != "" {
			fields = append(fields, zap.String(key, value))
		}
	}
	appendInt64Field := func(key string, value int64) {
		if value > 0 {
			fields = append(fields, zap.Int64(key, value))
		}
	}

	appendInt64Field("task_id", ctx.TaskID)
	appendInt64Field("tenant_id", ctx.TenantID)
	appendInt64Field("operator_id", ctx.OperatorID)
	appendInt64Field("article_id", ctx.ArticleID)
	appendInt64Field("quota_reservation_id", ctx.ReservationID)
	appendStringField("task_type", ctx.TaskType)
	appendStringField("stage", ctx.Stage)
	if ctx.AttemptCount > 0 {
		fields = append(fields, zap.Int("attempt_count", ctx.AttemptCount))
	}
	appendStringField("lease_owner", ctx.LeaseOwner)
	appendStringField("status_before", ctx.StatusBefore)
	appendStringField("status_after", ctx.StatusAfter)
	appendStringField("result", ctx.Result)
	appendStringField("anomaly_type", ctx.AnomalyType)
	appendStringField("severity", ctx.Severity)
	if ctx.Duration > 0 {
		fields = append(fields, zap.Duration("duration", ctx.Duration))
		fields = append(fields, zap.Int64("duration_ms", ctx.Duration.Milliseconds()))
	}
	return fields
}

func LogGenerationTaskInfo(logger *zap.Logger, message string, ctx GenerationTaskLogContext, fields ...zap.Field) {
	if logger == nil {
		return
	}
	logger.Info(message, append(GenerationTaskLogFields(ctx), fields...)...)
}

func LogGenerationTaskWarn(logger *zap.Logger, message string, ctx GenerationTaskLogContext, fields ...zap.Field) {
	if logger == nil {
		return
	}
	logger.Warn(message, append(GenerationTaskLogFields(ctx), fields...)...)
}

func LogGenerationTaskError(logger *zap.Logger, message string, ctx GenerationTaskLogContext, fields ...zap.Field) {
	if logger == nil {
		return
	}
	logger.Error(message, append(GenerationTaskLogFields(ctx), fields...)...)
}

func GenerationTaskErrorFields(err error) []zap.Field {
	if err == nil {
		return []zap.Field{}
	}

	fields := []zap.Field{zap.Error(err)}
	var appErr *response.AppError
	if errors.As(err, &appErr) {
		if appErr.Code > 0 {
			fields = append(fields, zap.Int("error_code", appErr.Code))
		}
		if appErr.HTTPStatus > 0 {
			fields = append(fields, zap.Int("http_status", appErr.HTTPStatus))
		}
		if message := strings.TrimSpace(appErr.Message); message != "" {
			fields = append(fields, zap.String("error_message", message))
		}
		if detail := strings.TrimSpace(appErr.Detail); detail != "" {
			fields = append(fields, zap.String("error_detail", detail))
		}
	}
	return fields
}

type GenerationTaskMetricsSnapshot struct {
	TransitionTotal        int64      `json:"transition_total"`
	FailureTotal           int64      `json:"failure_total"`
	CleanupFailureTotal    int64      `json:"cleanup_failure_total"`
	QueueFailureTotal      int64      `json:"queue_failure_total"`
	RecoveryRequeueTotal   int64      `json:"recovery_requeue_total"`
	RecoveryFinalizeTotal  int64      `json:"recovery_finalize_total"`
	StateAnomalyTotal      int64      `json:"state_anomaly_total"`
	StateCheckFailureTotal int64      `json:"state_check_failure_total"`
	LastEventAt            *time.Time `json:"last_event_at,omitempty"`
	LastEvent              string     `json:"last_event,omitempty"`
	LastTaskType           string     `json:"last_task_type,omitempty"`
	LastStage              string     `json:"last_stage,omitempty"`
	LastResult             string     `json:"last_result,omitempty"`
	LastAnomalyType        string     `json:"last_anomaly_type,omitempty"`
	LastSeverity           string     `json:"last_severity,omitempty"`
}

type generationTaskMetricsRecorder struct {
	transitionTotal        atomic.Int64
	failureTotal           atomic.Int64
	cleanupFailureTotal    atomic.Int64
	queueFailureTotal      atomic.Int64
	recoveryRequeueTotal   atomic.Int64
	recoveryFinalizeTotal  atomic.Int64
	stateAnomalyTotal      atomic.Int64
	stateCheckFailureTotal atomic.Int64
	lastEventAtUnixNano    atomic.Int64
	lastEvent              atomic.Value
	lastTaskType           atomic.Value
	lastStage              atomic.Value
	lastResult             atomic.Value
	lastAnomalyType        atomic.Value
	lastSeverity           atomic.Value
}

var generationTaskMetrics = newGenerationTaskMetricsRecorder()

func newGenerationTaskMetricsRecorder() *generationTaskMetricsRecorder {
	return &generationTaskMetricsRecorder{}
}

func RecordGenerationTaskEvent(event string, ctx GenerationTaskLogContext) {
	generationTaskMetrics.record(event, ctx)
}

func GenerationTaskMetricsSnapshotValue() GenerationTaskMetricsSnapshot {
	return generationTaskMetrics.snapshot()
}

func resetGenerationTaskMetricsForTest() {
	generationTaskMetrics = newGenerationTaskMetricsRecorder()
}

func (r *generationTaskMetricsRecorder) record(event string, ctx GenerationTaskLogContext) {
	if r == nil {
		return
	}
	event = strings.TrimSpace(event)
	if event == "" {
		event = "unknown"
	}
	switch event {
	case GenerationTaskEventTransition:
		r.transitionTotal.Add(1)
	case GenerationTaskEventFailure:
		r.failureTotal.Add(1)
	case GenerationTaskEventCleanupFailure:
		r.cleanupFailureTotal.Add(1)
	case GenerationTaskEventQueueFailure:
		r.queueFailureTotal.Add(1)
	case GenerationTaskEventRecoveryRequeue:
		r.recoveryRequeueTotal.Add(1)
	case GenerationTaskEventRecoveryFinalize:
		r.recoveryFinalizeTotal.Add(1)
	case GenerationTaskEventStateAnomaly:
		r.stateAnomalyTotal.Add(1)
	case GenerationTaskEventStateCheckFailure:
		r.stateCheckFailureTotal.Add(1)
	}
	r.lastEventAtUnixNano.Store(time.Now().UTC().UnixNano())
	r.lastEvent.Store(event)
	storeGenerationMetricString(&r.lastTaskType, ctx.TaskType)
	storeGenerationMetricString(&r.lastStage, ctx.Stage)
	storeGenerationMetricString(&r.lastResult, ctx.Result)
	storeGenerationMetricString(&r.lastAnomalyType, ctx.AnomalyType)
	storeGenerationMetricString(&r.lastSeverity, ctx.Severity)
}

func (r *generationTaskMetricsRecorder) snapshot() GenerationTaskMetricsSnapshot {
	if r == nil {
		return GenerationTaskMetricsSnapshot{}
	}
	return GenerationTaskMetricsSnapshot{
		TransitionTotal:        r.transitionTotal.Load(),
		FailureTotal:           r.failureTotal.Load(),
		CleanupFailureTotal:    r.cleanupFailureTotal.Load(),
		QueueFailureTotal:      r.queueFailureTotal.Load(),
		RecoveryRequeueTotal:   r.recoveryRequeueTotal.Load(),
		RecoveryFinalizeTotal:  r.recoveryFinalizeTotal.Load(),
		StateAnomalyTotal:      r.stateAnomalyTotal.Load(),
		StateCheckFailureTotal: r.stateCheckFailureTotal.Load(),
		LastEventAt:            generationMetricUnixNanoTime(r.lastEventAtUnixNano.Load()),
		LastEvent:              loadGenerationMetricString(&r.lastEvent),
		LastTaskType:           loadGenerationMetricString(&r.lastTaskType),
		LastStage:              loadGenerationMetricString(&r.lastStage),
		LastResult:             loadGenerationMetricString(&r.lastResult),
		LastAnomalyType:        loadGenerationMetricString(&r.lastAnomalyType),
		LastSeverity:           loadGenerationMetricString(&r.lastSeverity),
	}
}

func storeGenerationMetricString(target *atomic.Value, value string) {
	if target == nil {
		return
	}
	target.Store(strings.TrimSpace(value))
}

func loadGenerationMetricString(value *atomic.Value) string {
	if value == nil {
		return ""
	}
	raw := value.Load()
	text, _ := raw.(string)
	return text
}

func generationMetricUnixNanoTime(value int64) *time.Time {
	if value <= 0 {
		return nil
	}
	t := time.Unix(0, value).UTC()
	return &t
}