feat(llm): add configurable request/token rate limiting for LLM client
Introduce process-local and Redis-backed global rate limiting for the shared LLM client, plus a dedicated generation consumer prefetch knob. - Add request (RPM) and token (TPM) rate limits with burst controls, process or global scope, and fail-closed/process-degraded failover - Wire the LLM client through NewWithRedis so limits can be shared across replicas via a Redis token bucket - Add generation_consumer_prefetch to tune RabbitMQ generation consumers independently of the default prefetch - Bump generation worker_concurrency to 24 and promote golang.org/x/time to a direct dependency - Drop t.Parallel() from a few middleware/bootstrap tests Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -135,7 +135,7 @@ func New(configPath string) (*App, error) {
|
||||
logger.Warn("tenant login guard disabled by config; brute-force / rate-limit protection is off")
|
||||
}
|
||||
loginGuard := auth.NewLoginGuard(rdb, loginGuardCfg)
|
||||
llmClient := llm.NewReloadableClient(llm.New(cfg.LLM))
|
||||
llmClient := llm.NewReloadableClient(llm.NewWithRedis(cfg.LLM, rdb))
|
||||
retrievalProvider := retrieval.NewReloadableProvider(retrieval.NewProvider(cfg.Retrieval, logger))
|
||||
vectorStore := retrieval.NewReloadableVectorStore(retrieval.NewQdrantStore(cfg.Qdrant, logger))
|
||||
objectStorageClient := objectstorage.NewReloadableClient(objectstorage.New(cfg.ObjectStorage, logger))
|
||||
@@ -294,7 +294,7 @@ func (a *App) applyReloadedConfig(event config.ReloadEvent) {
|
||||
|
||||
a.JWT.Update(event.Current.JWT.Secret, event.Current.JWT.AccessTTL, event.Current.JWT.RefreshTTL)
|
||||
if a.ReloadableLLM != nil {
|
||||
a.ReloadableLLM.Update(llm.New(event.Current.LLM))
|
||||
a.ReloadableLLM.Update(llm.NewWithRedis(event.Current.LLM, a.Redis))
|
||||
}
|
||||
if a.ReloadableRetrieval != nil {
|
||||
a.ReloadableRetrieval.Update(retrieval.NewProvider(event.Current.Retrieval, a.Logger))
|
||||
|
||||
Reference in New Issue
Block a user