feat(llm): add configurable request/token rate limiting for LLM client

Introduce process-local and Redis-backed global rate limiting for the
shared LLM client, plus a dedicated generation consumer prefetch knob.

- Add request (RPM) and token (TPM) rate limits with burst controls,
  process or global scope, and fail-closed/process-degraded failover
- Wire the LLM client through NewWithRedis so limits can be shared
  across replicas via a Redis token bucket
- Add generation_consumer_prefetch to tune RabbitMQ generation
  consumers independently of the default prefetch
- Bump generation worker_concurrency to 24 and promote golang.org/x/time
  to a direct dependency
- Drop t.Parallel() from a few middleware/bootstrap tests

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-10 00:14:40 +08:00
parent 0acd37918e
commit 347cea1296
16 changed files with 1016 additions and 30 deletions
+15 -3
View File
@@ -7,6 +7,8 @@ import (
"strings"
"time"
goredis "github.com/redis/go-redis/v9"
"github.com/geo-platform/tenant-api/internal/shared/config"
)
@@ -53,15 +55,25 @@ type Client interface {
}
func New(cfg config.LLMConfig) Client {
return newClient(cfg, nil)
}
func NewWithRedis(cfg config.LLMConfig, redisClient goredis.Cmdable) Client {
return newClient(cfg, redisClient)
}
func newClient(cfg config.LLMConfig, redisClient goredis.Cmdable) Client {
provider := strings.ToLower(strings.TrimSpace(cfg.Provider))
var client Client
switch provider {
case "", "disabled":
return disabledClient{reason: "llm provider is disabled"}
client = disabledClient{reason: "llm provider is disabled"}
case "ark":
return NewArkClient(cfg)
client = NewArkClient(cfg)
default:
return disabledClient{reason: fmt.Sprintf("unsupported llm provider %q", cfg.Provider)}
client = disabledClient{reason: fmt.Sprintf("unsupported llm provider %q", cfg.Provider)}
}
return NewRateLimitedClientWithRedis(client, cfg, redisClient)
}
type disabledClient struct {