root/geo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(llm): add configurable request/token rate limiting for LLM client

Browse Source 
Introduce process-local and Redis-backed global rate limiting for the
shared LLM client, plus a dedicated generation consumer prefetch knob.

- Add request (RPM) and token (TPM) rate limits with burst controls,
  process or global scope, and fail-closed/process-degraded failover
- Wire the LLM client through NewWithRedis so limits can be shared
  across replicas via a Redis token bucket
- Add generation_consumer_prefetch to tune RabbitMQ generation
  consumers independently of the default prefetch
- Bump generation worker_concurrency to 24 and promote golang.org/x/time
  to a direct dependency
- Drop t.Parallel() from a few middleware/bootstrap tests

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
liangxu
2026-07-10 00:14:40 +08:00
parent 0acd37918e
commit 347cea1296
16 changed files with 1016 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/shared/messaging/rabbitmq/client.go
+11 -3
View File
@@ -280,7 +280,11 @@ func (c *Client) ConsumeMonitorProjectionRebuild(consumerName string) (<-chan am
}
func (c *Client) ConsumeGenerationTask(consumerName string) (<-chan amqp.Delivery, *amqp.Channel, error) {
return c.consume(consumerName, c.cfg.GenerationQueue)
prefetch := c.cfg.GenerationConsumerPrefetch
if prefetch <= 0 {
prefetch = c.cfg.ConsumerPrefetch
}
return c.consumeWithPrefetch(consumerName, c.cfg.GenerationQueue, prefetch)
}
func (c *Client) ConsumeGenerationEvents(consumerName string) (<-chan amqp.Delivery, *amqp.Channel, error) {
@@ -312,14 +316,18 @@ func (c *Client) ConsumeComplianceReviewTask(consumerName string) (<-chan amqp.D
}
func (c *Client) consume(consumerName, queue string) (<-chan amqp.Delivery, *amqp.Channel, error) {
return c.consumeWithPrefetch(consumerName, queue, c.cfg.ConsumerPrefetch)
}
func (c *Client) consumeWithPrefetch(consumerName, queue string, prefetch int) (<-chan amqp.Delivery, *amqp.Channel, error) {
for attempt := 0; attempt < 2; attempt++ {
conn, ch, err := c.openChannel()
if err != nil {
return nil, nil, err
}
if c.cfg.ConsumerPrefetch > 0 {
if err := ch.Qos(c.cfg.ConsumerPrefetch, 0, false); err != nil {
if prefetch > 0 {
if err := ch.Qos(prefetch, 0, false); err != nil {
_ = ch.Close()
c.invalidateConnection(conn)
if attempt == 1 {