root/geo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(redis): keep services running when Redis is unreachable

Browse Source 
Add a lazy Redis client constructor and an in-memory fallback for the
refresh-session store, plus tenant-api and ops-api bootstrap that warns
and continues with the lazy client instead of failing. Refresh and
blacklist operations now silently fall back to the in-memory store while
Redis is down so existing sessions stay valid until it recovers.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
liangxu
2026-04-30 01:06:08 +08:00
parent 2aa39d7a11
commit 11bd4e4d4e
5 changed files with 183 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/cmd/ops-api/main.go
+3 -1
View File
@@ -11,6 +11,7 @@ import (
"time"
"github.com/gin-gonic/gin"
"go.uber.org/zap"
"github.com/geo-platform/tenant-api/internal/ops/app"
opsconfig "github.com/geo-platform/tenant-api/internal/ops/config"
@@ -54,7 +55,8 @@ func main() {
rdb, err := redis.NewClient(ctx, cfg.Redis)
if err != nil {
logger.Sugar().Fatalf("ops-api init redis: %v", err)
logger.Warn("redis unavailable during startup; continuing with degraded auth/cache fallback", zap.Error(err))
rdb = redis.NewLazyClient(cfg.Redis)
}
defer func() { _ = rdb.Close() }()
appCache := cache.New(cfg.Cache.Driver, rdb)
server/internal/bootstrap/bootstrap.go
+2 -3
View File
@@ -90,9 +90,8 @@ func New(configPath string) (*App, error) {
rdb, err := redis.NewClient(ctx, cfg.Redis)
if err != nil {
pool.Close()
monitoringPool.Close()
return nil, fmt.Errorf("init redis: %w", err)
logger.Warn("redis unavailable during startup; continuing with degraded auth/cache fallback", zap.Error(err))
rdb = redis.NewLazyClient(cfg.Redis)
}
mqClient, err := rabbitmq.New(ctx, cfg.RabbitMQ)
server/internal/shared/auth/session_store.go
+134 -8
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/redis/go-redis/v9"
@@ -28,32 +29,60 @@ return 1
`)
type SessionStore struct {
rdb *redis.Client
rdb *redis.Client
fallback *memorySessionStore
}
func NewSessionStore(rdb *redis.Client) *SessionStore {
return &SessionStore{rdb: rdb}
return &SessionStore{
rdb: rdb,
fallback: newMemorySessionStore(),
}
}
func (s *SessionStore) SaveRefresh(ctx context.Context, jti string, tokenHash string, ttl time.Duration) error {
key := fmt.Sprintf("refresh:%s", jti)
return s.rdb.Set(ctx, key, tokenHash, ttl).Err()
s.fallback.set(key, tokenHash, ttl)
if s.rdb == nil {
return nil
}
if err := s.rdb.Set(ctx, key, tokenHash, ttl).Err(); err != nil {
return nil
}
return nil
}
func (s *SessionStore) GetRefresh(ctx context.Context, jti string) (string, error) {
key := fmt.Sprintf("refresh:%s", jti)
return s.rdb.Get(ctx, key).Result()
if s.rdb != nil {
value, err := s.rdb.Get(ctx, key).Result()
if err == nil {
return value, nil
}
}
return s.fallback.get(key)
}
func (s *SessionStore) DeleteRefresh(ctx context.Context, jti string) error {
key := fmt.Sprintf("refresh:%s", jti)
return s.rdb.Del(ctx, key).Err()
s.fallback.delete(key)
if s.rdb == nil {
return nil
}
if err := s.rdb.Del(ctx, key).Err(); err != nil {
return nil
}
return nil
}
func (s *SessionStore) RotateRefresh(ctx context.Context, oldJTI, oldHash, newJTI, newHash string, ttl time.Duration) error {
oldKey := fmt.Sprintf("refresh:%s", oldJTI)
newKey := fmt.Sprintf("refresh:%s", newJTI)
if s.rdb == nil {
return s.fallback.rotate(oldKey, oldHash, newKey, newHash, ttl)
}
result, err := rotateRefreshScript.Run(
ctx,
s.rdb,
@@ -63,13 +92,17 @@ func (s *SessionStore) RotateRefresh(ctx context.Context, oldJTI, oldHash, newJT
ttl.Milliseconds(),
).Int()
if err != nil {
return err
return s.fallback.rotate(oldKey, oldHash, newKey, newHash, ttl)
}
switch result {
case 1:
_ = s.fallback.rotate(oldKey, oldHash, newKey, newHash, ttl)
return nil
case 0:
if err := s.fallback.rotate(oldKey, oldHash, newKey, newHash, ttl); err == nil {
return nil
}
return ErrRefreshSessionNotFound
case -1:
return ErrRefreshTokenMismatch
@@ -80,14 +113,107 @@ func (s *SessionStore) RotateRefresh(ctx context.Context, oldJTI, oldHash, newJT
func (s *SessionStore) Blacklist(ctx context.Context, accessJTI string, ttl time.Duration) error {
key := fmt.Sprintf("blacklist:%s", accessJTI)
return s.rdb.Set(ctx, key, "1", ttl).Err()
s.fallback.set(key, "1", ttl)
if s.rdb == nil {
return nil
}
if err := s.rdb.Set(ctx, key, "1", ttl).Err(); err != nil {
return nil
}
return nil
}
func (s *SessionStore) IsBlacklisted(ctx context.Context, accessJTI string) (bool, error) {
key := fmt.Sprintf("blacklist:%s", accessJTI)
if _, err := s.fallback.get(key); err == nil {
return true, nil
}
if s.rdb == nil {
return false, nil
}
n, err := s.rdb.Exists(ctx, key).Result()
if err != nil {
return false, err
return false, nil
}
return n > 0, nil
}
type memorySessionStore struct {
mu sync.Mutex
items map[string]memorySessionItem
}
type memorySessionItem struct {
value string
expiresAt time.Time
}
func newMemorySessionStore() *memorySessionStore {
return &memorySessionStore{items: make(map[string]memorySessionItem)}
}
func (s *memorySessionStore) set(key, value string, ttl time.Duration) {
if s == nil || ttl <= 0 {
return
}
s.mu.Lock()
defer s.mu.Unlock()
s.items[key] = memorySessionItem{
value: value,
expiresAt: time.Now().Add(ttl),
}
}
func (s *memorySessionStore) get(key string) (string, error) {
if s == nil {
return "", redis.Nil
}
s.mu.Lock()
defer s.mu.Unlock()
item, ok := s.items[key]
if !ok {
return "", redis.Nil
}
if !item.expiresAt.After(time.Now()) {
delete(s.items, key)
return "", redis.Nil
}
return item.value, nil
}
func (s *memorySessionStore) delete(key string) {
if s == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
delete(s.items, key)
}
func (s *memorySessionStore) rotate(oldKey, oldHash, newKey, newHash string, ttl time.Duration) error {
if s == nil {
return ErrRefreshSessionNotFound
}
s.mu.Lock()
defer s.mu.Unlock()
item, ok := s.items[oldKey]
if !ok {
return ErrRefreshSessionNotFound
}
if !item.expiresAt.After(time.Now()) {
delete(s.items, oldKey)
return ErrRefreshSessionNotFound
}
if item.value != oldHash {
return ErrRefreshTokenMismatch
}
delete(s.items, oldKey)
if ttl > 0 {
s.items[newKey] = memorySessionItem{
value: newHash,
expiresAt: time.Now().Add(ttl),
}
}
return nil
}
server/internal/shared/auth/session_store_test.go
+35
View File
@@ -11,6 +11,19 @@ import (
"github.com/stretchr/testify/require"
)
func newFailingSessionRedisClient(t *testing.T) *goredis.Client {
t.Helper()
client := goredis.NewClient(&goredis.Options{
Addr: "127.0.0.1:1",
MaxRetries: 0,
DialTimeout: 10 * time.Millisecond,
ReadTimeout: 10 * time.Millisecond,
WriteTimeout: 10 * time.Millisecond,
})
t.Cleanup(func() { _ = client.Close() })
return client
}
func TestSessionStoreRotateRefreshSuccess(t *testing.T) {
mr := miniredis.RunT(t)
client := goredis.NewClient(&goredis.Options{Addr: mr.Addr()})
@@ -50,3 +63,25 @@ func TestSessionStoreRotateRefreshRejectsMismatch(t *testing.T) {
_, getErr = store.GetRefresh(ctx, "new-jti")
assert.Error(t, getErr)
}
func TestSessionStoreUsesMemoryFallbackWhenRedisUnavailable(t *testing.T) {
store := NewSessionStore(newFailingSessionRedisClient(t))
ctx := context.Background()
require.NoError(t, store.SaveRefresh(ctx, "old-jti", "old-hash", time.Minute))
value, err := store.GetRefresh(ctx, "old-jti")
require.NoError(t, err)
assert.Equal(t, "old-hash", value)
require.NoError(t, store.RotateRefresh(ctx, "old-jti", "old-hash", "new-jti", "new-hash", time.Minute))
_, err = store.GetRefresh(ctx, "old-jti")
assert.Error(t, err)
value, err = store.GetRefresh(ctx, "new-jti")
require.NoError(t, err)
assert.Equal(t, "new-hash", value)
require.NoError(t, store.Blacklist(ctx, "access-jti", time.Minute))
revoked, err := store.IsBlacklisted(ctx, "access-jti")
require.NoError(t, err)
assert.True(t, revoked)
}
server/internal/shared/repository/redis/redis.go
+9 -5
View File
@@ -10,14 +10,18 @@ import (
)
func NewClient(ctx context.Context, cfg config.RedisConfig) (*goredis.Client, error) {
client := goredis.NewClient(&goredis.Options{
Addr: cfg.Addr,
DB: cfg.DB,
})
client := NewLazyClient(cfg)
if err := client.Ping(ctx).Err(); err != nil {
return nil, fmt.Errorf("ping redis: %w", err)
}
return client, nil
}
func NewLazyClient(cfg config.RedisConfig) *goredis.Client {
client := goredis.NewClient(&goredis.Options{
Addr: cfg.Addr,
DB: cfg.DB,
})
return client
}