From 3c912949e4a7518a189e38ac1555e956268911ec Mon Sep 17 00:00:00 2001 From: liangxu Date: Tue, 5 May 2026 23:01:25 +0800 Subject: [PATCH] fix client IP handling behind proxies --- apps/admin-web/nginx.conf | 11 ++++- apps/ops-web/nginx.conf | 9 +++- deploy/config.yaml | 6 +++ deploy/k3s/README.md | 10 ++++ deploy/k3s/apps.yaml | 2 + deploy/k3s/config/config.yaml | 6 +++ deploy/k3s/config/ops-config.yaml | 6 +++ deploy/ops-config.yaml | 6 +++ server/cmd/ops-api/main.go | 8 ++-- server/configs/config.local.yaml.example | 6 +++ server/configs/config.yaml | 6 +++ server/configs/ops-config.yaml | 6 +++ server/internal/bootstrap/bootstrap.go | 7 +-- server/internal/ops/config/config.go | 12 +++-- server/internal/shared/bootstrap/http.go | 23 +++++++++ server/internal/shared/bootstrap/http_test.go | 48 +++++++++++++++++++ server/internal/shared/config/config.go | 45 ++++++++++++++++- server/internal/shared/config/config_test.go | 36 ++++++++++++++ server/internal/shared/config/reload.go | 2 +- .../internal/shared/middleware/logger_test.go | 33 +++++++++++++ 20 files changed, 272 insertions(+), 16 deletions(-) create mode 100644 server/internal/shared/bootstrap/http.go create mode 100644 server/internal/shared/bootstrap/http_test.go diff --git a/apps/admin-web/nginx.conf b/apps/admin-web/nginx.conf index 937807f..fa9b170 100644 --- a/apps/admin-web/nginx.conf +++ b/apps/admin-web/nginx.conf @@ -1,3 +1,10 @@ +set_real_ip_from 127.0.0.1; +set_real_ip_from 10.42.0.0/16; +set_real_ip_from 10.43.0.0/16; +set_real_ip_from 172.16.0.0/12; +real_ip_header X-Forwarded-For; +real_ip_recursive on; + map $http_upgrade $connection_upgrade { default upgrade; '' close; @@ -21,7 +28,7 @@ server { proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; @@ -35,7 +42,7 @@ server { proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; diff --git a/apps/ops-web/nginx.conf b/apps/ops-web/nginx.conf index 261964c..1f1a8e7 100644 --- a/apps/ops-web/nginx.conf +++ b/apps/ops-web/nginx.conf @@ -1,3 +1,10 @@ +set_real_ip_from 127.0.0.1; +set_real_ip_from 10.42.0.0/16; +set_real_ip_from 10.43.0.0/16; +set_real_ip_from 172.16.0.0/12; +real_ip_header X-Forwarded-For; +real_ip_recursive on; + server { listen 80; server_name _; @@ -14,7 +21,7 @@ server { proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; proxy_read_timeout 120s; proxy_send_timeout 120s; diff --git a/deploy/config.yaml b/deploy/config.yaml index d6c8b16..32cb9f1 100644 --- a/deploy/config.yaml +++ b/deploy/config.yaml @@ -1,6 +1,12 @@ server: port: 8080 mode: release + trusted_proxies: + - 127.0.0.1 + - ::1 + - 10.42.0.0/16 + - 10.43.0.0/16 + - 172.16.0.0/12 database: host: pgbouncer diff --git a/deploy/k3s/README.md b/deploy/k3s/README.md index 9427c95..884f62f 100644 --- a/deploy/k3s/README.md +++ b/deploy/k3s/README.md @@ -87,6 +87,16 @@ k3s 默认同时支持两种访问方式。 - tenant admin: `http://:30080` - ops admin: `http://:30081` +这两个 NodePort Service 使用 `externalTrafficPolicy: Local`，用于保留客户端源 IP，后端日志和登录审计会读取规范化后的 `X-Forwarded-For` / `X-Real-IP`。 + +如果通过 k3s 内置 Traefik Ingress 对外提供服务，也需要把 Traefik Service 调整为保留源地址： + +```bash +kubectl -n kube-system patch svc traefik -p '{"spec":{"externalTrafficPolicy":"Local"}}' +``` + +临时用 frpc/frps 做 TCP 隧道时，用户源 IP 通常不会穿透到 k3s；这类链路需要 proxy protocol 或 HTTP 反代写入可信 `X-Forwarded-For`，否则日志只能记录隧道出口/NAS 本地转发地址。 + 例如 NAS IP 是 `192.168.100.19`： - tenant admin: `http://192.168.100.19:30080` diff --git a/deploy/k3s/apps.yaml b/deploy/k3s/apps.yaml index 096c7ed..0a280e1 100644 --- a/deploy/k3s/apps.yaml +++ b/deploy/k3s/apps.yaml @@ -471,6 +471,7 @@ metadata: app.kubernetes.io/name: frontend spec: type: NodePort + externalTrafficPolicy: Local selector: app.kubernetes.io/name: frontend ports: @@ -530,6 +531,7 @@ metadata: app.kubernetes.io/name: ops-web spec: type: NodePort + externalTrafficPolicy: Local selector: app.kubernetes.io/name: ops-web ports: diff --git a/deploy/k3s/config/config.yaml b/deploy/k3s/config/config.yaml index e831825..e2f37bc 100644 --- a/deploy/k3s/config/config.yaml +++ b/deploy/k3s/config/config.yaml @@ -1,6 +1,12 @@ server: port: 8080 mode: release + trusted_proxies: + - 127.0.0.1 + - ::1 + - 10.42.0.0/16 + - 10.43.0.0/16 + - 172.16.0.0/12 database: host: pgbouncer diff --git a/deploy/k3s/config/ops-config.yaml b/deploy/k3s/config/ops-config.yaml index 7d2cc50..009f8ed 100644 --- a/deploy/k3s/config/ops-config.yaml +++ b/deploy/k3s/config/ops-config.yaml @@ -1,6 +1,12 @@ server: port: 8090 mode: release + trusted_proxies: + - 127.0.0.1 + - ::1 + - 10.42.0.0/16 + - 10.43.0.0/16 + - 172.16.0.0/12 database: host: pgbouncer diff --git a/deploy/ops-config.yaml b/deploy/ops-config.yaml index 538f734..9a9ce24 100644 --- a/deploy/ops-config.yaml +++ b/deploy/ops-config.yaml @@ -1,6 +1,12 @@ server: port: 8090 mode: release + trusted_proxies: + - 127.0.0.1 + - ::1 + - 10.42.0.0/16 + - 10.43.0.0/16 + - 172.16.0.0/12 database: host: pgbouncer diff --git a/server/cmd/ops-api/main.go b/server/cmd/ops-api/main.go index d6fa47e..cf9e2de 100644 --- a/server/cmd/ops-api/main.go +++ b/server/cmd/ops-api/main.go @@ -10,7 +10,6 @@ import ( "syscall" "time" - "github.com/gin-gonic/gin" "go.uber.org/zap" "github.com/geo-platform/tenant-api/internal/ops/app" @@ -18,6 +17,7 @@ import ( "github.com/geo-platform/tenant-api/internal/ops/repository" "github.com/geo-platform/tenant-api/internal/ops/transport" sharedauth "github.com/geo-platform/tenant-api/internal/shared/auth" + sharedbootstrap "github.com/geo-platform/tenant-api/internal/shared/bootstrap" "github.com/geo-platform/tenant-api/internal/shared/cache" "github.com/geo-platform/tenant-api/internal/shared/observability" "github.com/geo-platform/tenant-api/internal/shared/repository/postgres" @@ -113,10 +113,10 @@ func main() { } }() - if cfg.Server.Mode == "release" { - gin.SetMode(gin.ReleaseMode) + engine, err := sharedbootstrap.NewEngine(cfg.Server) + if err != nil { + logger.Sugar().Fatalf("ops-api init http engine: %v", err) } - engine := gin.New() transport.RegisterRoutes(transport.Deps{ Engine: engine, diff --git a/server/configs/config.local.yaml.example b/server/configs/config.local.yaml.example index bcb3043..7534954 100644 --- a/server/configs/config.local.yaml.example +++ b/server/configs/config.local.yaml.example @@ -4,6 +4,12 @@ server: port: 8080 mode: debug + trusted_proxies: + - 127.0.0.1 + - ::1 + - 10.42.0.0/16 + - 10.43.0.0/16 + - 172.16.0.0/12 database: host: localhost diff --git a/server/configs/config.yaml b/server/configs/config.yaml index fe92a04..d1665cd 100644 --- a/server/configs/config.yaml +++ b/server/configs/config.yaml @@ -1,6 +1,12 @@ server: port: 8080 mode: debug + trusted_proxies: + - 127.0.0.1 + - ::1 + - 10.42.0.0/16 + - 10.43.0.0/16 + - 172.16.0.0/12 database: host: localhost diff --git a/server/configs/ops-config.yaml b/server/configs/ops-config.yaml index 6b8314f..c84d6ef 100644 --- a/server/configs/ops-config.yaml +++ b/server/configs/ops-config.yaml @@ -1,6 +1,12 @@ server: port: 8090 mode: debug + trusted_proxies: + - 127.0.0.1 + - ::1 + - 10.42.0.0/16 + - 10.43.0.0/16 + - 172.16.0.0/12 database: host: localhost diff --git a/server/internal/bootstrap/bootstrap.go b/server/internal/bootstrap/bootstrap.go index 02951d1..f821588 100644 --- a/server/internal/bootstrap/bootstrap.go +++ b/server/internal/bootstrap/bootstrap.go @@ -13,6 +13,7 @@ import ( "github.com/geo-platform/tenant-api/internal/shared/auditlog" "github.com/geo-platform/tenant-api/internal/shared/auth" + sharedbootstrap "github.com/geo-platform/tenant-api/internal/shared/bootstrap" "github.com/geo-platform/tenant-api/internal/shared/cache" "github.com/geo-platform/tenant-api/internal/shared/config" "github.com/geo-platform/tenant-api/internal/shared/llm" @@ -148,10 +149,10 @@ func New(configPath string) (*App, error) { kolAssists := repository.NewKolAssistRepository(pool) kolPromptAsset := tenantapp.NewKolPromptAsset(objectStorageClient) - if cfg.Server.Mode == "release" { - gin.SetMode(gin.ReleaseMode) + engine, err := sharedbootstrap.NewEngine(cfg.Server) + if err != nil { + return nil, fmt.Errorf("init http engine: %w", err) } - engine := gin.New() engine.Use( middleware.Recovery(logger), diff --git a/server/internal/ops/config/config.go b/server/internal/ops/config/config.go index 7ba1834..f90e78b 100644 --- a/server/internal/ops/config/config.go +++ b/server/internal/ops/config/config.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "reflect" "strings" "sync" "time" @@ -216,7 +217,7 @@ func Diff(previous, current *Config) []FieldChange { changes = append(changes, FieldChange{Path: path, Hot: hot}) } - if previous.Server != current.Server { + if !reflect.DeepEqual(previous.Server, current.Server) { add("server", false) } if previous.Database != current.Database { @@ -330,6 +331,7 @@ func normalizeConfig(cfg *Config) { if strings.TrimSpace(cfg.Server.Mode) == "" { cfg.Server.Mode = "debug" } + sharedconfig.NormalizeServerConfig(&cfg.Server) if strings.TrimSpace(cfg.Redis.Addr) == "" { cfg.Redis.Addr = "localhost:6379" } @@ -361,8 +363,9 @@ func normalizeConfig(cfg *Config) { func defaultSettings() map[string]any { return map[string]any{ "server": map[string]any{ - "port": 8090, - "mode": "debug", + "port": 8090, + "mode": "debug", + "trusted_proxies": sharedconfig.DefaultTrustedProxies(), }, "redis": map[string]any{ "addr": "localhost:6379", @@ -417,6 +420,9 @@ func applyEnvOverrides(cfg *Config) error { if v := os.Getenv("OPS_IP_REGION_V6_XDB_PATH"); v != "" { cfg.IPRegion.V6XDBPath = v } + if v := os.Getenv("OPS_SERVER_TRUSTED_PROXIES"); v != "" { + cfg.Server.TrustedProxies = strings.Split(v, ",") + } return nil } diff --git a/server/internal/shared/bootstrap/http.go b/server/internal/shared/bootstrap/http.go new file mode 100644 index 0000000..e5377d2 --- /dev/null +++ b/server/internal/shared/bootstrap/http.go @@ -0,0 +1,23 @@ +package bootstrap + +import ( + "fmt" + + "github.com/gin-gonic/gin" + + "github.com/geo-platform/tenant-api/internal/shared/config" +) + +func NewEngine(server config.ServerConfig) (*gin.Engine, error) { + config.NormalizeServerConfig(&server) + if server.Mode == "release" { + gin.SetMode(gin.ReleaseMode) + } + engine := gin.New() + engine.ForwardedByClientIP = true + engine.RemoteIPHeaders = []string{"X-Forwarded-For", "X-Real-IP"} + if err := engine.SetTrustedProxies(server.TrustedProxies); err != nil { + return nil, fmt.Errorf("set trusted proxies: %w", err) + } + return engine, nil +} diff --git a/server/internal/shared/bootstrap/http_test.go b/server/internal/shared/bootstrap/http_test.go new file mode 100644 index 0000000..c2980fb --- /dev/null +++ b/server/internal/shared/bootstrap/http_test.go @@ -0,0 +1,48 @@ +package bootstrap + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + + "github.com/geo-platform/tenant-api/internal/shared/config" +) + +func TestNewEngineUsesForwardedClientIPFromTrustedProxy(t *testing.T) { + t.Parallel() + + gin.SetMode(gin.TestMode) + + engine, err := NewEngine(config.ServerConfig{ + TrustedProxies: []string{"10.42.0.0/16"}, + }) + require.NoError(t, err) + + engine.GET("/ip", func(c *gin.Context) { + c.String(http.StatusOK, c.ClientIP()) + }) + + req := httptest.NewRequest(http.MethodGet, "/ip", nil) + req.RemoteAddr = "10.42.0.128:51234" + req.Header.Set("X-Forwarded-For", "106.14.89.27") + resp := httptest.NewRecorder() + + engine.ServeHTTP(resp, req) + + require.Equal(t, http.StatusOK, resp.Code) + require.Equal(t, "106.14.89.27", resp.Body.String()) +} + +func TestNewEngineRejectsInvalidTrustedProxy(t *testing.T) { + t.Parallel() + + gin.SetMode(gin.TestMode) + + _, err := NewEngine(config.ServerConfig{ + TrustedProxies: []string{"not-a-cidr"}, + }) + require.Error(t, err) +} diff --git a/server/internal/shared/config/config.go b/server/internal/shared/config/config.go index 04449eb..6558d8a 100644 --- a/server/internal/shared/config/config.go +++ b/server/internal/shared/config/config.go @@ -41,8 +41,9 @@ type Config struct { } type ServerConfig struct { - Port int `mapstructure:"port"` - Mode string `mapstructure:"mode"` + Port int `mapstructure:"port"` + Mode string `mapstructure:"mode"` + TrustedProxies []string `mapstructure:"trusted_proxies"` } type DatabaseConfig struct { @@ -366,6 +367,7 @@ func loadWithFiles(configPath string) (*Config, []string, error) { } applyEnvOverrides(cfg) + NormalizeServerConfig(&cfg.Server) NormalizeRedisConfig(&cfg.Redis) NormalizeCacheConfig(&cfg.Cache) normalizeRabbitMQConfig(&cfg.RabbitMQ) @@ -469,6 +471,42 @@ func ensureMapSetting(settings map[string]any, key string) map[string]any { return next } +func DefaultTrustedProxies() []string { + return []string{ + "127.0.0.1", + "::1", + "10.42.0.0/16", + "10.43.0.0/16", + "172.16.0.0/12", + } +} + +func NormalizeServerConfig(cfg *ServerConfig) { + if cfg == nil { + return + } + cfg.Mode = strings.TrimSpace(cfg.Mode) + if cfg.TrustedProxies == nil { + cfg.TrustedProxies = DefaultTrustedProxies() + return + } + cfg.TrustedProxies = compactStringSlice(cfg.TrustedProxies) +} + +func compactStringSlice(values []string) []string { + if values == nil { + return nil + } + out := make([]string, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value != "" { + out = append(out, value) + } + } + return out +} + func normalizeMonitoringDispatchConfig(cfg *MonitoringDispatchConfig) { if cfg == nil { return @@ -747,6 +785,9 @@ func applyEnvOverrides(cfg *Config) { if token, ok := lookupFirstNonEmptyEnv("SCHEDULER_INTERNAL_METRICS_TOKEN", "SCHEDULER_METRICS_TOKEN"); ok { cfg.Scheduler.InternalMetricsToken = token } + if trustedProxies, ok := lookupNonEmptyEnv("SERVER_TRUSTED_PROXIES"); ok { + cfg.Server.TrustedProxies = strings.Split(trustedProxies, ",") + } } func normalizeRabbitMQConfig(cfg *RabbitMQConfig) { diff --git a/server/internal/shared/config/config_test.go b/server/internal/shared/config/config_test.go index 69ab83f..bf3ed2c 100644 --- a/server/internal/shared/config/config_test.go +++ b/server/internal/shared/config/config_test.go @@ -132,6 +132,42 @@ brand_library: {} } } +func TestLoadAppliesServerTrustedProxyDefaultsAndOverrides(t *testing.T) { + defaultConfigPath := writeTestConfig(t, ` +server: {} +`) + + cfg, err := Load(defaultConfigPath) + if err != nil { + t.Fatalf("Load() default error = %v", err) + } + if got := cfg.Server.TrustedProxies; len(got) != len(DefaultTrustedProxies()) { + t.Fatalf("expected default trusted proxies, got %#v", got) + } + + overrideConfigPath := writeTestConfig(t, ` +server: + trusted_proxies: + - " 10.42.0.0/16 " + - "" + - "192.168.1.10" +`) + + cfg, err = Load(overrideConfigPath) + if err != nil { + t.Fatalf("Load() override error = %v", err) + } + want := []string{"10.42.0.0/16", "192.168.1.10"} + if len(cfg.Server.TrustedProxies) != len(want) { + t.Fatalf("trusted proxies = %#v, want %#v", cfg.Server.TrustedProxies, want) + } + for i := range want { + if cfg.Server.TrustedProxies[i] != want[i] { + t.Fatalf("trusted proxies = %#v, want %#v", cfg.Server.TrustedProxies, want) + } + } +} + func TestLoadAppliesSchedulerHTTPDefaultsAndDisableSentinel(t *testing.T) { defaultConfigPath := writeTestConfig(t, ` scheduler: {} diff --git a/server/internal/shared/config/reload.go b/server/internal/shared/config/reload.go index 52a3d40..8a6b85a 100644 --- a/server/internal/shared/config/reload.go +++ b/server/internal/shared/config/reload.go @@ -17,7 +17,7 @@ func Diff(previous, current *Config) []FieldChange { changes = append(changes, FieldChange{Path: path, Hot: hot}) } - if previous.Server != current.Server { + if !reflect.DeepEqual(previous.Server, current.Server) { addChange("server", false) } if previous.Database != current.Database { diff --git a/server/internal/shared/middleware/logger_test.go b/server/internal/shared/middleware/logger_test.go index b586ce5..0674969 100644 --- a/server/internal/shared/middleware/logger_test.go +++ b/server/internal/shared/middleware/logger_test.go @@ -72,3 +72,36 @@ func TestLoggerIncludesDetailedAuthFailureContext(t *testing.T) { require.Contains(t, logLine, `"auth_parse_error":"token has invalid claims: token is expired"`) require.Contains(t, logLine, `"auth_token_fingerprint":"`) } + +func TestLoggerUsesForwardedClientIPFromTrustedProxy(t *testing.T) { + t.Parallel() + + gin.SetMode(gin.TestMode) + + var logBuffer bytes.Buffer + encoderConfig := zap.NewProductionEncoderConfig() + encoderConfig.TimeKey = "" + logger := zap.New(zapcore.NewCore( + zapcore.NewJSONEncoder(encoderConfig), + zapcore.AddSync(&logBuffer), + zapcore.DebugLevel, + )) + + router := gin.New() + require.NoError(t, router.SetTrustedProxies([]string{"10.42.0.0/16"})) + router.Use(RequestID()) + router.Use(Logger(logger)) + router.GET("/test", func(c *gin.Context) { + c.Status(http.StatusOK) + }) + + req := httptest.NewRequest(http.MethodGet, "/test", nil) + req.RemoteAddr = "10.42.0.128:51234" + req.Header.Set("X-Forwarded-For", "106.14.89.27") + resp := httptest.NewRecorder() + + router.ServeHTTP(resp, req) + + require.Equal(t, http.StatusOK, resp.Code) + require.Contains(t, logBuffer.String(), `"client_ip":"106.14.89.27"`) +}