From 6366b5e1f6b9362685a4e7963f422fa096d41b05 Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Tue, 1 Sep 2026 19:14:28 +0100 Subject: [PATCH] feat(multiplayer): add degraded admission mode --- docs/MATCHMAKING.md | 6 +++ multiplayer-next.md | 2 +- server/api/admission.go | 70 +++++++++++++++++++++++++ server/api/admission_test.go | 90 ++++++++++++++++++++++++++++++++ server/api/service.go | 11 ++++ server/cmd/control-plane/main.go | 18 +++++++ 6 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 server/api/admission.go create mode 100644 server/api/admission_test.go diff --git a/docs/MATCHMAKING.md b/docs/MATCHMAKING.md index bce211aa..6160a647 100644 --- a/docs/MATCHMAKING.md +++ b/docs/MATCHMAKING.md @@ -411,6 +411,12 @@ Minimum controls: WebSocket handshake/message/idle limits, bounded fan-out and overload shedding. In degraded mode reject new login/queue/allocation work while preserving result ingestion and all live matches. + + The control-plane implements the admission portion of this policy with + `--degraded` at startup, `SIGUSR1` to enable it, and `SIGUSR2` to disable it. + The gate rejects new login, queue, and proposal mutations with `503 + service_degraded`; assignment reads, events, server registration/results, + health, metrics, and other live-match paths remain available. - Images pinned by digest, SBOM generation, dependency/image scanning, signed releases, admission-time signature verification, and a critical-patch SLA. - Structured audit events for auth, queue transitions, allocation, roster diff --git a/multiplayer-next.md b/multiplayer-next.md index e160b14f..e5fa047c 100644 --- a/multiplayer-next.md +++ b/multiplayer-next.md @@ -1191,7 +1191,7 @@ the local/CI/community transport, not a silent production fallback. | 8.9 `[D:8.4,8.7]` | **IN PROGRESS.** Pure Go join policy binds SteamID/player/match/server/team/slot/protocol/expiry, rejects duplicate roster slots, permits same-identity reclaim through backend loss and fences prior server-owned generations | `server/domain/reconnect.go` covers SteamID/server/slot binding, expiry, repeated reclaim, grace boundary and old-generation fencing; signed token issuance/verification, persistent leases, Godot `hello` transport and production integration remain | | 8.10 `[D:8.5,8.31]` | **IN PROGRESS.** Dependency-free workload credential policy validates an adapter-verified signature, issuer/audience/time bounds, namespace/service account, pod UID, GameServer UID, allocation ID, match ID and server ID before result submission; `server/workload` now parses projected compact JWTs, rejects unsigned/malformed/ambiguous tokens, verifies signing input through an injected trust callback, and delegates exact binding to the domain policy; the API exposes the workload-authenticated server result route. **`WorkloadVerify` is now wired for real** -- the blocker this row previously named (`cmd/control-plane/main.go` never wiring it, so `/register` and `/result` 503'd on every real request) is closed by taking a different, equally-valid approach instead of the Kubernetes-JWT one: `serverMutation` only ever compares `WorkloadBinding.ServerID`/`.MatchID` (and `AdvanceServerRegistration` only additionally needs `.AllocationID`) -- nothing downstream requires Namespace/ServiceAcct/PodUID/GameServerUID, so a Kubernetes-issued token was never actually required, just the assumption that it was. `server/workload/signed_token.go` mints and verifies a short-lived HMAC-signed token with a secret only the control plane holds (the same trust model `domain.SessionStore` already uses for player sessions), needing no cluster JWKS/TokenReview to validate -- signature and expiry are fully self-contained. The token binds only `allocation_id` (deliberately, not `match_id`/`server_id` too: it must be requestable in the same Agones request that asks for a server, before Agones has picked one) -- `store.AllocationBindingByAllocationID` resolves `match_id`/`server_id` durably from the `allocations` table at verify time, so a token can never claim a pairing that wasn't actually, durably allocated. `api.WorkloadVerifierFromSignedToken` combines both and is wired into `cmd/control-plane` (`--workload-secret`/`COSMIC_CLASH_WORKLOAD_SECRET`; a startup warning fires and the route stays 503 if it's left unset) and `cmd/testkit-api` (fixed test secret). **The delivery channel is now wired too**: `agones.Client` gains `WorkloadSecret`/`WorkloadTokenTTL` -- when set, `Allocate` mints a token for the allocation and requests it as a third `cosmic-clash.io/workload-token` annotation alongside match-id/allocation-id, wired from `cmd/allocator`'s own `--workload-secret`/`COSMIC_CLASH_WORKLOAD_SECRET` (must match the control plane's); `supervisor.Supervisor.workloadToken()` resolves the bearer credential with an explicit `--workload-token-path` (kept for a possible future Kubernetes-JWT path) always winning, otherwise falling back to that same annotation -- the same fallback pattern `matchID()` already used for `cosmic-clash.io/match-id`. `WorkloadTokenPath` is accordingly no longer required at construction time | `server/domain/workload.go`, `server/workload/jwt.go`, `server/api/service.go` and adversarial tests reject every binding mutation, missing/unverified signature, `none`/malformed JWT, ambiguous audience, server/match mismatch and time boundary; `server/testkit/pipeline_test.go` carries allocation identity through the offline result path; `cmd/control-plane` now wires `ResultSubmitter: store.PostgresResults{DB: db}` (a ready-made adapter that had been referenced from nowhere at all, not even a test); `server/workload/signed_token_test.go` covers round-trip, tampered payload, wrong secret, malformed input and the exact expiry boundary; `server/api/workload_verifier_integration_test.go` (opt-in, real PostgreSQL) covers acceptance against a real allocation row, rejection of an unrecorded allocation, that two distinct real allocations each resolve to their own and only their own match/server pairing, and the previously-503 `Service.WorkloadVerify` field itself now succeeding -- all verified clean with `-race` across multiple runs; `cmd/control-plane/main_test.go`'s `TestServerRoutesRequireWorkloadVerifyToBeWired` now documents and pins the misconfigured-secret case specifically, not the "always unwired" case; `server/agones/allocation_test.go`'s `TestAllocateRequestsAWorkloadTokenAnnotationWhenConfigured` and `server/supervisor/supervisor_test.go`'s new annotation-fallback/fails-closed pair cover the delivery channel end to end (mint -> annotation -> supervisor read -> Authorization header) short of a live cluster. **What's still missing**: this has never run against a real Agones cluster (only HTTP-level fakes), so the exact `object_meta` JSON casing Agones actually returns remains unverified from this sandbox (see `GameServer.ObjectMeta`'s existing caveat); live duplicate/conflict alerting also remains | | 8.11 `[D:8.1]` | **DONE.** Write the threat model: forged clients/replay/queues/results, floods/bots, pod/insider compromise, gameplay and API DDoS, SDR signing-key theft, dependencies and denial-of-wallet | [`docs/THREAT-MODEL.md`](docs/THREAT-MODEL.md) records prevention, detection/response, owner and residual risk for every threat; it separates offline CA/online signer, client/game-server/PostgreSQL/Redis trust boundaries and recovery behavior | -| 8.12 `[D:8.11]` | **IN PROGRESS.** Provider-neutral Kubernetes baseline enforces restricted namespace admission, non-root/read-only/no-capability workloads, separate service accounts, allocator-only RBAC, default-deny networking and explicit edge/data/DNS/Agones/metrics flows; application manifests consume externally populated Secret objects; the Go API and allocator now have bounded per-replica rate/quota boundaries and metrics endpoints | `deploy/k8s/base/` includes hardened control-plane, allocator, and game-server resources; `server/security/test_kubernetes_policies.py`, `server/api/rate_limit.go`, allocator metrics tests and adversarial checks cover static hardening, secret-reference invariants, fixed-window limits, bounded labels and bounded key memory; private-store provisioning, distributed/global quotas, edge DDoS/WAF/origin shielding, WebSocket limits, overload shedding, encrypted backups and live policy/load tests remain | +| 8.12 `[D:8.11]` | **IN PROGRESS.** Provider-neutral Kubernetes baseline enforces restricted namespace admission, non-root/read-only/no-capability workloads, separate service accounts, allocator-only RBAC, default-deny networking and explicit edge/data/DNS/Agones/metrics flows; application manifests consume externally populated Secret objects; the Go API and allocator now have bounded per-replica rate/quota boundaries and metrics endpoints, plus an atomic operator-controlled degraded-mode gate that sheds new login/queue/proposal work while preserving live-match paths | `deploy/k8s/base/` includes hardened control-plane, allocator, and game-server resources; `server/security/test_kubernetes_policies.py`, `server/api/rate_limit.go`, `server/api/admission.go`, allocator metrics tests and adversarial checks cover static hardening, secret-reference invariants, fixed-window limits, bounded labels/key memory, route classification, concurrent toggling and degraded responses; start with control-plane `--degraded`, enable with SIGUSR1, disable with SIGUSR2; private-store provisioning, distributed/global quotas, edge DDoS/WAF/origin shielding, WebSocket limits, encrypted backups and live policy/load tests remain | | 8.13 `[D:8.12]` | **IN PROGRESS.** Docker/Kubernetes references are digest-pinned, a dependency-free checker rejects mutable tags/plaintext credentials and concrete release overlays can reject template digests; release documentation defines SBOM, dependency/image scanning, signing, admission verification and a 24-hour critical-fix SLA | `scripts/verify_supply_chain.py`, `server/security/test_supply_chain.py`, `docs/SUPPLY-CHAIN.md` and `.github/workflows/supply-chain.yml` cover repository policy and provenance requirements; registry SBOM/scan/sign/admission execution and a concrete production overlay remain | #### 8C — Queueing, matchmaking, playlists and rating diff --git a/server/api/admission.go b/server/api/admission.go new file mode 100644 index 00000000..5cfdca26 --- /dev/null +++ b/server/api/admission.go @@ -0,0 +1,70 @@ +package api + +import ( + "strings" + "sync/atomic" +) + +// AdmissionController decides whether a classified public operation may +// start. Implementations must be safe for concurrent requests. +type AdmissionController interface { + Allow(operation string) bool +} + +// AdmissionGate is the operator-controlled overload gate for new matchmaking +// work. Degraded mode is deliberately narrow: existing matches can continue +// to report results and clients can still use read/recovery/event endpoints. +type AdmissionGate struct { + degraded atomic.Bool +} + +func NewAdmissionGate(degraded bool) *AdmissionGate { + gate := &AdmissionGate{} + gate.degraded.Store(degraded) + return gate +} + +func (g *AdmissionGate) SetDegraded(value bool) { + if g != nil { + g.degraded.Store(value) + } +} + +func (g *AdmissionGate) Degraded() bool { + return g != nil && g.degraded.Load() +} + +func (g *AdmissionGate) Allow(operation string) bool { + if !g.Degraded() { + return true + } + switch operation { + case "login", "queue", "proposal", "allocation": + return false + default: + return true + } +} + +func admissionOperation(path, method string) string { + if method == "GET" || method == "HEAD" || method == "OPTIONS" { + return "" + } + path = strings.TrimSuffix(path, "/") + switch { + case path == "/v1/session/steam" || path == "/api/v1/session/steam": + return "login" + case path == "/v1/queue" || path == "/api/v1/queue/tickets": + return "queue" + case underPath(path, "/v1/queue/") || underPath(path, "/api/v1/queue/tickets/"): + return "queue" + case underPath(path, "/v1/proposals/") || underPath(path, "/api/v1/proposals/"): + return "proposal" + default: + return "" + } +} + +func underPath(path, prefix string) bool { + return strings.HasPrefix(path, prefix) && len(path) > len(prefix) +} diff --git a/server/api/admission_test.go b/server/api/admission_test.go new file mode 100644 index 00000000..8559921d --- /dev/null +++ b/server/api/admission_test.go @@ -0,0 +1,90 @@ +package api + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" +) + +func TestAdmissionGateDefaultsToAllowAndBlocksOnlyNewWork(t *testing.T) { + gate := NewAdmissionGate(false) + for _, operation := range []string{"login", "queue", "proposal", "allocation", "result", "events"} { + if !gate.Allow(operation) { + t.Fatalf("normal mode rejected %q", operation) + } + } + gate.SetDegraded(true) + for _, operation := range []string{"login", "queue", "proposal", "allocation"} { + if gate.Allow(operation) { + t.Fatalf("degraded mode allowed %q", operation) + } + } + for _, operation := range []string{"result", "events", "read", ""} { + if !gate.Allow(operation) { + t.Fatalf("degraded mode rejected live-safe operation %q", operation) + } + } +} + +func TestAdmissionOperationClassifiesOnlyMutations(t *testing.T) { + tests := []struct { + path, method, want string + }{ + {"/v1/session/steam", http.MethodPost, "login"}, + {"/api/v1/session/steam/", http.MethodPost, "login"}, + {"/v1/queue", http.MethodPost, "queue"}, + {"/api/v1/queue/tickets/abc/heartbeat", http.MethodPost, "queue"}, + {"/v1/proposals/abc/accept", http.MethodPost, "proposal"}, + {"/api/v1/proposals/abc", http.MethodDelete, "proposal"}, + {"/v1/queue", http.MethodGet, ""}, + {"/v1/queue-not-a-route", http.MethodPost, ""}, + {"/v1/servers/abc/result", http.MethodPost, ""}, + {"/v1/events", http.MethodPost, ""}, + } + for _, test := range tests { + if got := admissionOperation(test.path, test.method); got != test.want { + t.Errorf("admissionOperation(%q, %q) = %q, want %q", test.path, test.method, got, test.want) + } + } +} + +func TestAdmissionGateConcurrentToggle(t *testing.T) { + gate := NewAdmissionGate(false) + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 1000; j++ { + gate.SetDegraded(j%2 == 0) + _ = gate.Allow("queue") + } + }() + } + wg.Wait() +} + +func TestHandlerReturnsDegradedOnlyForNewMatchmakingMutations(t *testing.T) { + service := &Service{Admission: NewAdmissionGate(true)} + tests := []struct { + path, want string + }{ + {"/v1/queue", "service_degraded"}, + {"/api/v1/proposals/proposal-1/accept", "service_degraded"}, + {"/v1/servers/server-1/result", "server_unavailable"}, + {"/v1/events", ""}, + } + for _, test := range tests { + req := httptest.NewRequest(http.MethodPost, test.path, strings.NewReader(`{}`)) + rec := httptest.NewRecorder() + service.Handler().ServeHTTP(rec, req) + if test.want != "" && !strings.Contains(rec.Body.String(), test.want) { + t.Errorf("%s body = %q, want %q", test.path, rec.Body.String(), test.want) + } + if test.want == "service_degraded" && rec.Code != http.StatusServiceUnavailable { + t.Errorf("%s status = %d, want 503", test.path, rec.Code) + } + } +} diff --git a/server/api/service.go b/server/api/service.go index 54671dc7..39a70a73 100644 --- a/server/api/service.go +++ b/server/api/service.go @@ -127,6 +127,7 @@ type Service struct { RankedProfileProvider RankedProfileProvider TierPolicy domain.TierPolicy RateLimiter *RateLimiter + Admission AdmissionController // Log receives a credential-safe structured event for lifecycle-relevant // reads and mutations. Nil // is a valid, silent no-op -- every call site must stay optional so @@ -213,6 +214,16 @@ func (s *Service) Handler() http.Handler { mux.ServeHTTP(w, r) }) } + if s.Admission != nil { + admissionHandler := handler + handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if operation := admissionOperation(r.URL.Path, r.Method); operation != "" && !s.Admission.Allow(operation) { + writeError(w, http.StatusServiceUnavailable, "service_degraded") + return + } + admissionHandler.ServeHTTP(w, r) + }) + } if s.Metrics == nil { return handler } diff --git a/server/cmd/control-plane/main.go b/server/cmd/control-plane/main.go index 28b2a6c2..45cca023 100644 --- a/server/cmd/control-plane/main.go +++ b/server/cmd/control-plane/main.go @@ -29,6 +29,7 @@ func main() { redisPrefix := flag.String("redis-prefix", envOrDefault("COSMIC_CLASH_REDIS_PREFIX", "cosmic-clash"), "Redis key prefix") redisTTL := flag.Duration("redis-ttl", 60*time.Second, "TTL for transient candidate projection entries") workloadSecret := flag.String("workload-secret", os.Getenv("COSMIC_CLASH_WORKLOAD_SECRET"), "HMAC secret for control-plane-issued workload tokens (see workload/signed_token.go); server registration/result submission return 503 until this is set") + degraded := flag.Bool("degraded", false, "start with new login, queue, and proposal mutations rejected; SIGUSR1 enables and SIGUSR2 disables this mode") flag.Parse() if *role != "api" { fatalf("unsupported role %q (only api is implemented)", *role) @@ -63,11 +64,28 @@ func main() { fmt.Fprintln(os.Stderr, "control-plane: warning: --workload-secret / COSMIC_CLASH_WORKLOAD_SECRET is unset; server registration and result submission will return 503") } service := newAPIService(db, *workloadSecret, candidateIndex) + admission := api.NewAdmissionGate(*degraded) + service.Admission = admission server := &http.Server{Addr: *listen, Handler: service.Handler(), ReadHeaderTimeout: 5 * time.Second} serveErr := make(chan error, 1) go func() { serveErr <- server.ListenAndServe() }() ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() + operatorSignals := make(chan os.Signal, 2) + signal.Notify(operatorSignals, syscall.SIGUSR1, syscall.SIGUSR2) + defer signal.Stop(operatorSignals) + go func() { + for sig := range operatorSignals { + switch sig { + case syscall.SIGUSR1: + admission.SetDegraded(true) + fmt.Fprintln(os.Stderr, "control-plane: degraded admission enabled") + case syscall.SIGUSR2: + admission.SetDegraded(false) + fmt.Fprintln(os.Stderr, "control-plane: degraded admission disabled") + } + } + }() go api.RunProposalOutboxDispatcher(ctx, db, service) go api.RunResultOutboxDispatcher(ctx, db, service) go api.RunStateOutboxDispatcher(ctx, db, service)