feat(llm): add configurable request/token rate limiting for LLM client
Introduce process-local and Redis-backed global rate limiting for the shared LLM client, plus a dedicated generation consumer prefetch knob. - Add request (RPM) and token (TPM) rate limits with burst controls, process or global scope, and fail-closed/process-degraded failover - Wire the LLM client through NewWithRedis so limits can be shared across replicas via a Redis token bucket - Add generation_consumer_prefetch to tune RabbitMQ generation consumers independently of the default prefetch - Bump generation worker_concurrency to 24 and promote golang.org/x/time to a direct dependency - Drop t.Parallel() from a few middleware/bootstrap tests Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -12,8 +12,6 @@ import (
|
||||
)
|
||||
|
||||
func TestNewEngineUsesForwardedClientIPFromTrustedProxy(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
engine, err := NewEngine(config.ServerConfig{
|
||||
@@ -38,8 +36,6 @@ func TestNewEngineUsesForwardedClientIPFromTrustedProxy(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestNewEngineRejectsInvalidTrustedProxy(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
_, err := NewEngine(config.ServerConfig{
|
||||
|
||||
Reference in New Issue
Block a user