feat(llm): add configurable request/token rate limiting for LLM client

Introduce process-local and Redis-backed global rate limiting for the
shared LLM client, plus a dedicated generation consumer prefetch knob.

- Add request (RPM) and token (TPM) rate limits with burst controls,
  process or global scope, and fail-closed/process-degraded failover
- Wire the LLM client through NewWithRedis so limits can be shared
  across replicas via a Redis token bucket
- Add generation_consumer_prefetch to tune RabbitMQ generation
  consumers independently of the default prefetch
- Bump generation worker_concurrency to 24 and promote golang.org/x/time
  to a direct dependency
- Drop t.Parallel() from a few middleware/bootstrap tests

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-10 00:14:40 +08:00
parent 0acd37918e
commit 347cea1296
16 changed files with 1016 additions and 30 deletions
+2 -2
View File
@@ -135,7 +135,7 @@ func New(configPath string) (*App, error) {
logger.Warn("tenant login guard disabled by config; brute-force / rate-limit protection is off")
}
loginGuard := auth.NewLoginGuard(rdb, loginGuardCfg)
llmClient := llm.NewReloadableClient(llm.New(cfg.LLM))
llmClient := llm.NewReloadableClient(llm.NewWithRedis(cfg.LLM, rdb))
retrievalProvider := retrieval.NewReloadableProvider(retrieval.NewProvider(cfg.Retrieval, logger))
vectorStore := retrieval.NewReloadableVectorStore(retrieval.NewQdrantStore(cfg.Qdrant, logger))
objectStorageClient := objectstorage.NewReloadableClient(objectstorage.New(cfg.ObjectStorage, logger))
@@ -294,7 +294,7 @@ func (a *App) applyReloadedConfig(event config.ReloadEvent) {
a.JWT.Update(event.Current.JWT.Secret, event.Current.JWT.AccessTTL, event.Current.JWT.RefreshTTL)
if a.ReloadableLLM != nil {
a.ReloadableLLM.Update(llm.New(event.Current.LLM))
a.ReloadableLLM.Update(llm.NewWithRedis(event.Current.LLM, a.Redis))
}
if a.ReloadableRetrieval != nil {
a.ReloadableRetrieval.Update(retrieval.NewProvider(event.Current.Retrieval, a.Logger))