feat(llm): add configurable request/token rate limiting for LLM client
Introduce process-local and Redis-backed global rate limiting for the shared LLM client, plus a dedicated generation consumer prefetch knob. - Add request (RPM) and token (TPM) rate limits with burst controls, process or global scope, and fail-closed/process-degraded failover - Wire the LLM client through NewWithRedis so limits can be shared across replicas via a Redis token bucket - Add generation_consumer_prefetch to tune RabbitMQ generation consumers independently of the default prefetch - Bump generation worker_concurrency to 24 and promote golang.org/x/time to a direct dependency - Drop t.Parallel() from a few middleware/bootstrap tests Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -7,6 +7,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
goredis "github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/geo-platform/tenant-api/internal/shared/config"
|
||||
)
|
||||
|
||||
@@ -53,15 +55,25 @@ type Client interface {
|
||||
}
|
||||
|
||||
func New(cfg config.LLMConfig) Client {
|
||||
return newClient(cfg, nil)
|
||||
}
|
||||
|
||||
func NewWithRedis(cfg config.LLMConfig, redisClient goredis.Cmdable) Client {
|
||||
return newClient(cfg, redisClient)
|
||||
}
|
||||
|
||||
func newClient(cfg config.LLMConfig, redisClient goredis.Cmdable) Client {
|
||||
provider := strings.ToLower(strings.TrimSpace(cfg.Provider))
|
||||
var client Client
|
||||
switch provider {
|
||||
case "", "disabled":
|
||||
return disabledClient{reason: "llm provider is disabled"}
|
||||
client = disabledClient{reason: "llm provider is disabled"}
|
||||
case "ark":
|
||||
return NewArkClient(cfg)
|
||||
client = NewArkClient(cfg)
|
||||
default:
|
||||
return disabledClient{reason: fmt.Sprintf("unsupported llm provider %q", cfg.Provider)}
|
||||
client = disabledClient{reason: fmt.Sprintf("unsupported llm provider %q", cfg.Provider)}
|
||||
}
|
||||
return NewRateLimitedClientWithRedis(client, cfg, redisClient)
|
||||
}
|
||||
|
||||
type disabledClient struct {
|
||||
|
||||
Reference in New Issue
Block a user