diff --git a/deploy/k8s/base/control-plane-deployment.yaml b/deploy/k8s/base/control-plane-deployment.yaml index 8c714434..a72096e6 100644 --- a/deploy/k8s/base/control-plane-deployment.yaml +++ b/deploy/k8s/base/control-plane-deployment.yaml @@ -52,6 +52,10 @@ spec: - --rate-limit=120 - --rate-limit-window=1m - --rate-limit-max-keys=10000 + # Ingress NetworkPolicy admits only the labelled edge gateway. + # Cover common private/CGNAT/ULA pod networks; overlays should + # narrow this to their actual gateway CIDR where available. + - --trusted-proxy-cidrs=10.0.0.0/8,100.64.0.0/10,172.16.0.0/12,192.168.0.0/16,fc00::/7 ports: - name: http containerPort: 8080 diff --git a/multiplayer-next.md b/multiplayer-next.md index df8730b0..31dc5799 100644 --- a/multiplayer-next.md +++ b/multiplayer-next.md @@ -1640,3 +1640,5 @@ The production allocator now uses the API it actually implements: Kubernetes cus Drain admission now fails at the handshake boundary: a new `_hello` is rejected with the actual RPC peer ID after `admissions_open` closes. Disconnects no longer perform the admission check (or try to reject an already-gone sender); they always invalidate transport state, release the signed join token, record the reconnect boundary, and remove the roster entry. Godot regressions cover the admission decision and cleanup while draining. Task 8.36's live lifecycle/PDB gates remain open. Allocated team and slot assignments are now immutable after signed admission. MatchNet rejects client `_set_team` requests whenever join authorisation is required, preserving the signed global-slot/team pairing and its derived spawn index; direct/community lobbies retain team switching and its existing unready behavior. The Godot regression asserts both sides of that compatibility boundary. + +Per-IP API limiting now resolves the client behind the edge gateway instead of charging every player to the gateway's socket address. `X-Forwarded-For` is ignored unless the immediate peer belongs to an explicitly configured `--trusted-proxy-cidrs` range; trusted chains are walked from right to left past known proxies, while malformed/oversized chains fail closed to the immediate peer. The base deployment supplies private/CGNAT/ULA pod ranges under its edge-only ingress NetworkPolicy and calls out that production overlays should narrow them to the actual gateway CIDR. Tests cover spoofing from an untrusted peer, chained proxies, malformed input, invalid configuration, and independent clients behind one gateway. diff --git a/server/api/rate_limit.go b/server/api/rate_limit.go index 33f737d2..56acaf6e 100644 --- a/server/api/rate_limit.go +++ b/server/api/rate_limit.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "net/http" + "net/netip" "strings" "sync" "time" @@ -27,6 +28,30 @@ type rateWindow struct { count int } +// ClientIPResolver accepts X-Forwarded-For only from explicitly trusted +// immediate peers. It walks the chain from the application backwards so an +// untrusted client cannot select its own rate-limit identity by prepending a +// forged address. +type ClientIPResolver struct { + trustedProxies []netip.Prefix +} + +func NewClientIPResolver(cidrs string) (*ClientIPResolver, error) { + resolver := &ClientIPResolver{} + for _, raw := range strings.Split(cidrs, ",") { + raw = strings.TrimSpace(raw) + if raw == "" { + continue + } + prefix, err := netip.ParsePrefix(raw) + if err != nil { + return nil, fmt.Errorf("invalid trusted proxy CIDR %q", raw) + } + resolver.trustedProxies = append(resolver.trustedProxies, prefix.Masked()) + } + return resolver, nil +} + func NewRateLimiter(limit int, window time.Duration, maxKeys int) (*RateLimiter, error) { if limit < 1 || window <= 0 || maxKeys < 1 { return nil, fmt.Errorf("invalid rate limiter configuration") @@ -93,26 +118,75 @@ func (l *RateLimiter) AllowKeys(keys []string, now time.Time) bool { return true } -func requestRateKey(r *http.Request) string { - keys := requestRateKeys(r) +func requestRateKey(r *http.Request, resolver *ClientIPResolver) string { + keys := requestRateKeys(r, resolver) if len(keys) == 0 { return "" } return keys[0] } -func requestRateKeys(r *http.Request) []string { +func requestRateKeys(r *http.Request, resolver *ClientIPResolver) []string { keys := make([]string, 0, 2) if authorization := strings.TrimSpace(r.Header.Get("Authorization")); authorization != "" { digest := sha256.Sum256([]byte(authorization)) keys = append(keys, "auth:"+hex.EncodeToString(digest[:])) } - host := r.RemoteAddr - if parsedHost, _, err := net.SplitHostPort(host); err == nil { - host = parsedHost - } + host := requestClientIP(r, resolver) if host == "" { return keys } return append(keys, "ip:"+host) } + +func requestClientIP(r *http.Request, resolver *ClientIPResolver) string { + host := strings.TrimSpace(r.RemoteAddr) + if parsedHost, _, err := net.SplitHostPort(host); err == nil { + host = parsedHost + } + remote, err := netip.ParseAddr(strings.Trim(host, "[]")) + if err != nil { + return host + } + remote = remote.Unmap() + if resolver == nil || !resolver.trusts(remote) { + return remote.String() + } + forwarded := strings.Join(r.Header.Values("X-Forwarded-For"), ",") + if forwarded == "" || len(forwarded) > 2048 { + return remote.String() + } + parts := strings.Split(forwarded, ",") + if len(parts) > 16 { + return remote.String() + } + chain := make([]netip.Addr, 0, len(parts)) + for _, part := range parts { + address, parseErr := netip.ParseAddr(strings.TrimSpace(part)) + if parseErr != nil { + return remote.String() + } + chain = append(chain, address.Unmap()) + } + for index := len(chain) - 1; index >= 0; index-- { + if !resolver.trusts(chain[index]) { + return chain[index].String() + } + } + if len(chain) > 0 { + return chain[0].String() + } + return remote.String() +} + +func (r *ClientIPResolver) trusts(address netip.Addr) bool { + if r == nil || !address.IsValid() { + return false + } + for _, prefix := range r.trustedProxies { + if prefix.Contains(address) { + return true + } + } + return false +} diff --git a/server/api/rate_limit_test.go b/server/api/rate_limit_test.go index 187785b6..a6139005 100644 --- a/server/api/rate_limit_test.go +++ b/server/api/rate_limit_test.go @@ -80,3 +80,73 @@ func TestRateLimitedHTTPBoundaryReturnsGeneric429(t *testing.T) { t.Fatalf("limited request status = %d", response.StatusCode) } } + +func TestClientIPResolverTrustsForwardingOnlyFromConfiguredProxy(t *testing.T) { + resolver, err := NewClientIPResolver("10.0.0.0/8, 2001:db8::/32") + if err != nil { + t.Fatal(err) + } + untrusted := httptest.NewRequest(http.MethodGet, "/", nil) + untrusted.RemoteAddr = "203.0.113.10:1234" + untrusted.Header.Set("X-Forwarded-For", "198.51.100.7") + if got := requestClientIP(untrusted, resolver); got != "203.0.113.10" { + t.Fatalf("untrusted peer selected forwarded IP %q", got) + } + + trusted := httptest.NewRequest(http.MethodGet, "/", nil) + trusted.RemoteAddr = "10.2.3.4:443" + trusted.Header.Set("X-Forwarded-For", "198.51.100.7, 10.9.8.7") + if got := requestClientIP(trusted, resolver); got != "198.51.100.7" { + t.Fatalf("trusted proxy chain resolved to %q", got) + } + trusted.Header["X-Forwarded-For"] = []string{"192.0.2.99", "198.51.100.7, 10.9.8.7"} + if got := requestClientIP(trusted, resolver); got != "198.51.100.7" { + t.Fatalf("repeated forwarded headers bypassed the nearest untrusted address: %q", got) + } + trusted.Header.Set("X-Forwarded-For", "forged, 198.51.100.7") + if got := requestClientIP(trusted, resolver); got != "10.2.3.4" { + t.Fatalf("malformed forwarding did not fail closed to immediate peer: %q", got) + } +} + +func TestClientIPResolverRejectsInvalidCIDRs(t *testing.T) { + if _, err := NewClientIPResolver("10.0.0.0/8,not-a-network"); err == nil { + t.Fatal("invalid trusted proxy CIDR accepted") + } +} + +func TestRateLimiterSeparatesClientsBehindTrustedGateway(t *testing.T) { + limiter, err := NewRateLimiter(1, time.Minute, 8) + if err != nil { + t.Fatal(err) + } + resolver, err := NewClientIPResolver("127.0.0.0/8") + if err != nil { + t.Fatal(err) + } + service := &Service{RateLimiter: limiter, ClientIPs: resolver, Now: func() time.Time { return time.Unix(1000, 0) }} + server := httptest.NewServer(service.Handler()) + defer server.Close() + request := func(forwarded string) int { + req, requestErr := http.NewRequest(http.MethodGet, server.URL+"/healthz", nil) + if requestErr != nil { + t.Fatal(requestErr) + } + req.Header.Set("X-Forwarded-For", forwarded) + response, requestErr := server.Client().Do(req) + if requestErr != nil { + t.Fatal(requestErr) + } + response.Body.Close() + return response.StatusCode + } + if got := request("198.51.100.1"); got != http.StatusOK { + t.Fatalf("first client status = %d", got) + } + if got := request("198.51.100.2"); got != http.StatusOK { + t.Fatalf("second client behind gateway status = %d", got) + } + if got := request("198.51.100.1"); got != http.StatusTooManyRequests { + t.Fatalf("repeated first client status = %d", got) + } +} diff --git a/server/api/service.go b/server/api/service.go index 3b4d42ba..bd038bdc 100644 --- a/server/api/service.go +++ b/server/api/service.go @@ -127,6 +127,7 @@ type Service struct { RankedProfileProvider RankedProfileProvider TierPolicy domain.TierPolicy RateLimiter *RateLimiter + ClientIPs *ClientIPResolver Admission AdmissionController // Log receives a credential-safe structured event for lifecycle-relevant // reads and mutations. Nil @@ -207,7 +208,7 @@ func (s *Service) Handler() http.Handler { var handler http.Handler = mux if s.RateLimiter != nil { handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if !s.RateLimiter.AllowKeys(requestRateKeys(r), s.now()) { + if !s.RateLimiter.AllowKeys(requestRateKeys(r, s.ClientIPs), s.now()) { writeError(w, http.StatusTooManyRequests, "rate_limited") return } diff --git a/server/cmd/control-plane/main.go b/server/cmd/control-plane/main.go index 947225a7..91882dc8 100644 --- a/server/cmd/control-plane/main.go +++ b/server/cmd/control-plane/main.go @@ -33,6 +33,7 @@ func main() { rateLimit := flag.Int("rate-limit", 120, "maximum requests per per-credential/IP fixed window") rateWindow := flag.Duration("rate-limit-window", time.Minute, "fixed window for the per-replica request limiter") rateMaxKeys := flag.Int("rate-limit-max-keys", 10000, "maximum credential/IP keys retained by the per-replica request limiter") + trustedProxyCIDRs := flag.String("trusted-proxy-cidrs", os.Getenv("COSMIC_CLASH_TRUSTED_PROXY_CIDRS"), "comma-separated immediate proxy CIDRs allowed to supply X-Forwarded-For") flag.Parse() if *role != "api" { fatalf("unsupported role %q (only api is implemented)", *role) @@ -47,6 +48,10 @@ func main() { if err != nil { fatalf("invalid request limiter configuration: %v", err) } + clientIPs, err := api.NewClientIPResolver(*trustedProxyCIDRs) + if err != nil { + fatalf("invalid trusted proxy configuration: %v", err) + } db, err := sql.Open("pgx", *dsn) if err != nil { fatalf("open PostgreSQL: %v", err) @@ -72,6 +77,7 @@ func main() { } service := newAPIService(db, *workloadSecret, candidateIndex) service.RateLimiter = rateLimiter + service.ClientIPs = clientIPs admission := api.NewAdmissionGate(*degraded) service.Admission = admission server := &http.Server{Addr: *listen, Handler: service.Handler(), ReadHeaderTimeout: 5 * time.Second} diff --git a/server/security/test_kubernetes_policies.py b/server/security/test_kubernetes_policies.py index 6cc25985..465f00e2 100644 --- a/server/security/test_kubernetes_policies.py +++ b/server/security/test_kubernetes_policies.py @@ -22,6 +22,7 @@ class KubernetesPolicyTest(unittest.TestCase): "readOnlyRootFilesystem: true", "drop: [ALL]", "resources:", "image: ghcr.io/cosmic-clash/control-plane@sha256:", "--rate-limit=120", "--rate-limit-window=1m", "--rate-limit-max-keys=10000", + "--trusted-proxy-cidrs=10.0.0.0/8,100.64.0.0/10,172.16.0.0/12,192.168.0.0/16,fc00::/7", "name: COSMIC_CLASH_POSTGRES_DSN", "key: dsn", "name: COSMIC_CLASH_WORKLOAD_SECRET", "key: secret", ):