Add monitoring service and database schema
- Implement monitoring service with heartbeat, lease tasks, resume tasks, and task result handling. - Create monitoring time utilities for business date calculations. - Add unit tests for date window resolution and business day handling. - Define database schema for monitoring-related tables including quotas, daily reports, and task management. - Establish migration scripts for creating and dropping monitoring tables.
This commit is contained in:
@@ -10,17 +10,19 @@ import (
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Server ServerConfig `mapstructure:"server"`
|
||||
Database DatabaseConfig `mapstructure:"database"`
|
||||
Redis RedisConfig `mapstructure:"redis"`
|
||||
Qdrant QdrantConfig `mapstructure:"qdrant"`
|
||||
ObjectStorage ObjectStorageConfig `mapstructure:"object_storage"`
|
||||
Cache CacheConfig `mapstructure:"cache"`
|
||||
JWT JWTConfig `mapstructure:"jwt"`
|
||||
Log LogConfig `mapstructure:"log"`
|
||||
LLM LLMConfig `mapstructure:"llm"`
|
||||
Retrieval RetrievalConfig `mapstructure:"retrieval"`
|
||||
Generation GenerationConfig `mapstructure:"generation"`
|
||||
Server ServerConfig `mapstructure:"server"`
|
||||
Database DatabaseConfig `mapstructure:"database"`
|
||||
MonitoringDatabase DatabaseConfig `mapstructure:"monitoring_database"`
|
||||
RabbitMQ RabbitMQConfig `mapstructure:"rabbitmq"`
|
||||
Redis RedisConfig `mapstructure:"redis"`
|
||||
Qdrant QdrantConfig `mapstructure:"qdrant"`
|
||||
ObjectStorage ObjectStorageConfig `mapstructure:"object_storage"`
|
||||
Cache CacheConfig `mapstructure:"cache"`
|
||||
JWT JWTConfig `mapstructure:"jwt"`
|
||||
Log LogConfig `mapstructure:"log"`
|
||||
LLM LLMConfig `mapstructure:"llm"`
|
||||
Retrieval RetrievalConfig `mapstructure:"retrieval"`
|
||||
Generation GenerationConfig `mapstructure:"generation"`
|
||||
}
|
||||
|
||||
type ServerConfig struct {
|
||||
@@ -49,6 +51,23 @@ type RedisConfig struct {
|
||||
DB int `mapstructure:"db"`
|
||||
}
|
||||
|
||||
type RabbitMQConfig struct {
|
||||
URL string `mapstructure:"url"`
|
||||
MonitorResultExchange string `mapstructure:"monitor_result_exchange"`
|
||||
MonitorResultRoutingKey string `mapstructure:"monitor_result_routing_key"`
|
||||
MonitorResultQueue string `mapstructure:"monitor_result_queue"`
|
||||
MonitorResultDLX string `mapstructure:"monitor_result_dlx"`
|
||||
MonitorResultDLQ string `mapstructure:"monitor_result_dlq"`
|
||||
MonitorResultDLQRouteKey string `mapstructure:"monitor_result_dlq_routing_key"`
|
||||
MonitorProjectionExchange string `mapstructure:"monitor_projection_exchange"`
|
||||
MonitorProjectionRoutingKey string `mapstructure:"monitor_projection_routing_key"`
|
||||
MonitorProjectionQueue string `mapstructure:"monitor_projection_queue"`
|
||||
MonitorProjectionDLX string `mapstructure:"monitor_projection_dlx"`
|
||||
MonitorProjectionDLQ string `mapstructure:"monitor_projection_dlq"`
|
||||
MonitorProjectionDLQRouteKey string `mapstructure:"monitor_projection_dlq_routing_key"`
|
||||
ConsumerPrefetch int `mapstructure:"consumer_prefetch"`
|
||||
}
|
||||
|
||||
type QdrantConfig struct {
|
||||
URL string `mapstructure:"url"`
|
||||
APIKey string `mapstructure:"api_key"`
|
||||
@@ -139,6 +158,7 @@ func Load(configPath string) (*Config, error) {
|
||||
}
|
||||
|
||||
applyEnvOverrides(&cfg)
|
||||
normalizeRabbitMQConfig(&cfg.RabbitMQ)
|
||||
|
||||
return &cfg, nil
|
||||
}
|
||||
@@ -157,6 +177,9 @@ func applyEnvOverrides(cfg *Config) {
|
||||
if apiKey, ok := lookupNonEmptyEnv("QDRANT_API_KEY"); ok {
|
||||
cfg.Qdrant.APIKey = apiKey
|
||||
}
|
||||
if rabbitmqURL, ok := lookupNonEmptyEnv("RABBITMQ_URL"); ok {
|
||||
cfg.RabbitMQ.URL = rabbitmqURL
|
||||
}
|
||||
if url, ok := lookupNonEmptyEnv("QDRANT_URL"); ok {
|
||||
cfg.Qdrant.URL = url
|
||||
}
|
||||
@@ -201,6 +224,50 @@ func applyEnvOverrides(cfg *Config) {
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeRabbitMQConfig(cfg *RabbitMQConfig) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if strings.TrimSpace(cfg.MonitorResultExchange) == "" {
|
||||
cfg.MonitorResultExchange = "monitor.result"
|
||||
}
|
||||
if strings.TrimSpace(cfg.MonitorResultRoutingKey) == "" {
|
||||
cfg.MonitorResultRoutingKey = "monitor.result.ingest"
|
||||
}
|
||||
if strings.TrimSpace(cfg.MonitorResultQueue) == "" {
|
||||
cfg.MonitorResultQueue = "monitor.result.ingest"
|
||||
}
|
||||
if strings.TrimSpace(cfg.MonitorResultDLX) == "" {
|
||||
cfg.MonitorResultDLX = "monitor.result.dlx"
|
||||
}
|
||||
if strings.TrimSpace(cfg.MonitorResultDLQ) == "" {
|
||||
cfg.MonitorResultDLQ = "monitor.result.ingest.dlq"
|
||||
}
|
||||
if strings.TrimSpace(cfg.MonitorResultDLQRouteKey) == "" {
|
||||
cfg.MonitorResultDLQRouteKey = "monitor.result.ingest.dlq"
|
||||
}
|
||||
|
||||
if strings.TrimSpace(cfg.MonitorProjectionExchange) == "" {
|
||||
cfg.MonitorProjectionExchange = "monitor.projection"
|
||||
}
|
||||
if strings.TrimSpace(cfg.MonitorProjectionRoutingKey) == "" {
|
||||
cfg.MonitorProjectionRoutingKey = "monitor.projection.rebuild"
|
||||
}
|
||||
if strings.TrimSpace(cfg.MonitorProjectionQueue) == "" {
|
||||
cfg.MonitorProjectionQueue = "monitor.projection.rebuild"
|
||||
}
|
||||
if strings.TrimSpace(cfg.MonitorProjectionDLX) == "" {
|
||||
cfg.MonitorProjectionDLX = "monitor.projection.dlx"
|
||||
}
|
||||
if strings.TrimSpace(cfg.MonitorProjectionDLQ) == "" {
|
||||
cfg.MonitorProjectionDLQ = "monitor.projection.rebuild.dlq"
|
||||
}
|
||||
if strings.TrimSpace(cfg.MonitorProjectionDLQRouteKey) == "" {
|
||||
cfg.MonitorProjectionDLQRouteKey = "monitor.projection.rebuild.dlq"
|
||||
}
|
||||
}
|
||||
|
||||
func lookupFirstNonEmptyEnv(keys ...string) (string, bool) {
|
||||
for _, key := range keys {
|
||||
if value, ok := lookupNonEmptyEnv(key); ok {
|
||||
|
||||
@@ -0,0 +1,325 @@
|
||||
package rabbitmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/config"
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
cfg config.RabbitMQConfig
|
||||
mu sync.Mutex
|
||||
conn *amqp.Connection
|
||||
closed bool
|
||||
}
|
||||
|
||||
func New(_ context.Context, cfg config.RabbitMQConfig) (*Client, error) {
|
||||
if strings.TrimSpace(cfg.URL) == "" {
|
||||
return nil, fmt.Errorf("rabbitmq url is required")
|
||||
}
|
||||
|
||||
client := &Client{cfg: cfg}
|
||||
if _, err := client.ensureConnection(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func (c *Client) Close() error {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
c.closed = true
|
||||
conn := c.conn
|
||||
c.conn = nil
|
||||
c.mu.Unlock()
|
||||
|
||||
if conn == nil || conn.IsClosed() {
|
||||
return nil
|
||||
}
|
||||
return conn.Close()
|
||||
}
|
||||
|
||||
func (c *Client) Ping(_ context.Context) error {
|
||||
_, ch, err := c.openChannel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ch.Close()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) PublishMonitorResultIngest(ctx context.Context, body []byte) error {
|
||||
return c.publish(ctx, c.cfg.MonitorResultExchange, c.cfg.MonitorResultRoutingKey, body)
|
||||
}
|
||||
|
||||
func (c *Client) PublishMonitorProjectionRebuild(ctx context.Context, body []byte) error {
|
||||
return c.publish(ctx, c.cfg.MonitorProjectionExchange, c.cfg.MonitorProjectionRoutingKey, body)
|
||||
}
|
||||
|
||||
func (c *Client) publish(ctx context.Context, exchange, routingKey string, body []byte) error {
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
conn, ch, err := c.openChannel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
publishErr := ch.PublishWithContext(ctx, exchange, routingKey, false, false, amqp.Publishing{
|
||||
ContentType: "application/json",
|
||||
DeliveryMode: amqp.Persistent,
|
||||
Body: body,
|
||||
})
|
||||
_ = ch.Close()
|
||||
if publishErr == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
c.invalidateConnection(conn)
|
||||
if attempt == 1 {
|
||||
return fmt.Errorf("publish rabbitmq message: %w", publishErr)
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("publish rabbitmq message: retry exhausted")
|
||||
}
|
||||
|
||||
func (c *Client) ConsumeMonitorResultIngest(consumerName string) (<-chan amqp.Delivery, *amqp.Channel, error) {
|
||||
return c.consume(consumerName, c.cfg.MonitorResultQueue)
|
||||
}
|
||||
|
||||
func (c *Client) ConsumeMonitorProjectionRebuild(consumerName string) (<-chan amqp.Delivery, *amqp.Channel, error) {
|
||||
return c.consume(consumerName, c.cfg.MonitorProjectionQueue)
|
||||
}
|
||||
|
||||
func (c *Client) consume(consumerName, queue string) (<-chan amqp.Delivery, *amqp.Channel, error) {
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
conn, ch, err := c.openChannel()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if c.cfg.ConsumerPrefetch > 0 {
|
||||
if err := ch.Qos(c.cfg.ConsumerPrefetch, 0, false); err != nil {
|
||||
_ = ch.Close()
|
||||
c.invalidateConnection(conn)
|
||||
if attempt == 1 {
|
||||
return nil, nil, fmt.Errorf("configure rabbitmq qos: %w", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
deliveries, err := ch.Consume(
|
||||
queue,
|
||||
consumerName,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
_ = ch.Close()
|
||||
c.invalidateConnection(conn)
|
||||
if attempt == 1 {
|
||||
return nil, nil, fmt.Errorf("consume rabbitmq queue: %w", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
return deliveries, ch, nil
|
||||
}
|
||||
|
||||
return nil, nil, fmt.Errorf("consume rabbitmq queue: retry exhausted")
|
||||
}
|
||||
|
||||
func (c *Client) ensureConnection() (*amqp.Connection, error) {
|
||||
if c == nil {
|
||||
return nil, fmt.Errorf("rabbitmq client is nil")
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if c.closed {
|
||||
return nil, fmt.Errorf("rabbitmq client is closed")
|
||||
}
|
||||
if c.conn != nil && !c.conn.IsClosed() {
|
||||
return c.conn, nil
|
||||
}
|
||||
if c.conn != nil {
|
||||
_ = c.conn.Close()
|
||||
c.conn = nil
|
||||
}
|
||||
|
||||
conn, err := amqp.Dial(c.cfg.URL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dial rabbitmq: %w", err)
|
||||
}
|
||||
if err := c.ensureTopology(conn); err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c.conn = conn
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
func (c *Client) openChannel() (*amqp.Connection, *amqp.Channel, error) {
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
conn, err := c.ensureConnection()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
ch, err := conn.Channel()
|
||||
if err == nil {
|
||||
return conn, ch, nil
|
||||
}
|
||||
|
||||
c.invalidateConnection(conn)
|
||||
if attempt == 1 {
|
||||
return nil, nil, fmt.Errorf("open rabbitmq channel: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil, fmt.Errorf("open rabbitmq channel: retry exhausted")
|
||||
}
|
||||
|
||||
func (c *Client) invalidateConnection(conn *amqp.Connection) {
|
||||
if c == nil || conn == nil {
|
||||
return
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if c.conn != conn {
|
||||
return
|
||||
}
|
||||
c.conn = nil
|
||||
if !conn.IsClosed() {
|
||||
_ = conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) ensureTopology(conn *amqp.Connection) error {
|
||||
ch, err := conn.Channel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("open rabbitmq channel: %w", err)
|
||||
}
|
||||
defer ch.Close()
|
||||
|
||||
if err := c.ensureWorkQueue(
|
||||
ch,
|
||||
c.cfg.MonitorResultExchange,
|
||||
c.cfg.MonitorResultRoutingKey,
|
||||
c.cfg.MonitorResultQueue,
|
||||
c.cfg.MonitorResultDLX,
|
||||
c.cfg.MonitorResultDLQ,
|
||||
c.cfg.MonitorResultDLQRouteKey,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := c.ensureWorkQueue(
|
||||
ch,
|
||||
c.cfg.MonitorProjectionExchange,
|
||||
c.cfg.MonitorProjectionRoutingKey,
|
||||
c.cfg.MonitorProjectionQueue,
|
||||
c.cfg.MonitorProjectionDLX,
|
||||
c.cfg.MonitorProjectionDLQ,
|
||||
c.cfg.MonitorProjectionDLQRouteKey,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) ensureWorkQueue(ch *amqp.Channel, exchange, routingKey, queue, dlx, dlq, dlqRoutingKey string) error {
|
||||
if err := ch.ExchangeDeclare(
|
||||
exchange,
|
||||
"direct",
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
nil,
|
||||
); err != nil {
|
||||
return fmt.Errorf("declare rabbitmq exchange %s: %w", exchange, err)
|
||||
}
|
||||
|
||||
if err := ch.ExchangeDeclare(
|
||||
dlx,
|
||||
"direct",
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
nil,
|
||||
); err != nil {
|
||||
return fmt.Errorf("declare rabbitmq dlx %s: %w", dlx, err)
|
||||
}
|
||||
|
||||
queueArgs := amqp.Table{
|
||||
"x-queue-type": "quorum",
|
||||
"x-dead-letter-exchange": dlx,
|
||||
"x-dead-letter-routing-key": dlqRoutingKey,
|
||||
}
|
||||
if _, err := ch.QueueDeclare(
|
||||
queue,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
queueArgs,
|
||||
); err != nil {
|
||||
return fmt.Errorf("declare rabbitmq queue %s: %w", queue, err)
|
||||
}
|
||||
|
||||
if err := ch.QueueBind(
|
||||
queue,
|
||||
routingKey,
|
||||
exchange,
|
||||
false,
|
||||
nil,
|
||||
); err != nil {
|
||||
return fmt.Errorf("bind rabbitmq queue %s: %w", queue, err)
|
||||
}
|
||||
|
||||
dlqArgs := amqp.Table{
|
||||
"x-queue-type": "quorum",
|
||||
}
|
||||
if _, err := ch.QueueDeclare(
|
||||
dlq,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
dlqArgs,
|
||||
); err != nil {
|
||||
return fmt.Errorf("declare rabbitmq dlq %s: %w", dlq, err)
|
||||
}
|
||||
|
||||
if err := ch.QueueBind(
|
||||
dlq,
|
||||
dlqRoutingKey,
|
||||
dlx,
|
||||
false,
|
||||
nil,
|
||||
); err != nil {
|
||||
return fmt.Errorf("bind rabbitmq dlq %s: %w", dlq, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -31,6 +31,9 @@ func Logger(logger *zap.Logger) gin.HandlerFunc {
|
||||
if appErr.Detail != "" {
|
||||
fields = append(fields, zap.String("app_detail", appErr.Detail))
|
||||
}
|
||||
if appErr.Cause != nil {
|
||||
fields = append(fields, zap.Error(appErr.Cause))
|
||||
}
|
||||
} else if len(c.Errors) > 0 {
|
||||
fields = append(fields, zap.String("error", c.Errors.String()))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user