From 72d4a604c20cb760e81ede87e52cc18ae3a49f34 Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:59:51 +0100 Subject: [PATCH] feat: spread game fleet across on-demand zones --- deploy/k8s/base/fleet.yaml | 9 +++++++++ multiplayer-next.md | 7 +++++-- multiplayer-todo.md | 2 +- server/security/test_fleet_manifests.py | 7 +++++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/deploy/k8s/base/fleet.yaml b/deploy/k8s/base/fleet.yaml index a4b84bfe..1204fef4 100644 --- a/deploy/k8s/base/fleet.yaml +++ b/deploy/k8s/base/fleet.yaml @@ -29,6 +29,15 @@ spec: failureThreshold: 3 template: spec: + nodeSelector: + cosmic-clash.io/capacity-type: on-demand + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: game-server serviceAccountName: match-server automountServiceAccountToken: false securityContext: diff --git a/multiplayer-next.md b/multiplayer-next.md index 2f316024..89e41c72 100644 --- a/multiplayer-next.md +++ b/multiplayer-next.md @@ -132,8 +132,11 @@ product policy are in [`docs/MATCHMAKING.md`](docs/MATCHMAKING.md). fall to zero. A provider-neutral Agones FleetAutoscaler now encodes a two-process Ready buffer and six-process warm cap; regional node pools, pre-pull rollout and measured N+1 capacity remain. -- [ ] Spread on-demand capacity across zones with N+1 headroom; do not place - live matches on interruptible nodes. +- [ ] **IN PROGRESS:** Spread on-demand capacity across zones with N+1 + headroom; do not place live matches on interruptible nodes. The Fleet now + requires the on-demand capacity label and uses a zone topology spread + constraint; force-loss testing of the largest node and measured headroom + remain. - [ ] Benchmark native x86_64 boot, p99 CPU/RSS/network and tick health; set requests/limits and node density from measurements plus 30% headroom. - [ ] Add 30 s no-show handling, Go PID-1 TERM/drain supervision, PDB/Fleet diff --git a/multiplayer-todo.md b/multiplayer-todo.md index 2cd00bbe..0ea007af 100644 --- a/multiplayer-todo.md +++ b/multiplayer-todo.md @@ -1215,7 +1215,7 @@ the local/CI/community transport, not a silent production fallback. | 8.30 `[D:8.18,8.26,8.28,8.29]` | **IN PROGRESS.** Pure Go allocator filters Ready servers by region/build/protocol/transport, atomically claims one with idempotent allocation replay, and now owns the assignment-publication boundary | `server/domain/allocator.go` covers deterministic compatible selection, exhaustion, conflicting/identical allocation replay, unknown allocations, and assignment replay/conflict; Agones `GameServerAllocation`, signed roster metadata, bounded cross-replica retry and live integration remain | | 8.31 `[D:8.9,8.30]` | **IN PROGRESS.** Pure Go assignment gate requires Allocated state, exact allocation ID/match/server/region/build/protocol/transport compatibility, non-empty hosted endpoint and verified manifest signature before exposure; allocator publication cannot expose Ready state | `server/domain/assignment.go` and `allocator.go` plus adversarial fixtures cover early-connect, tampered signature/manifest, wrong compatibility, empty endpoint, unknown allocation and post-publication mutation rejection; Agones metadata watch, hosted-address registration, production signer and client-ticket publication remain | | 8.32 `[D:8.2,8.26,8.30]` | **IN PROGRESS.** Provider-neutral FleetAutoscaler baseline preserves a two-process Ready buffer, caps warm capacity, and leaves Allocated scale-down independent of the Ready floor; Fleet image references remain digest-pinned for current/rollback pre-pull | `deploy/k8s/base/fleet-autoscaler.yaml` and manifest tests cover Fleet ownership, Buffer policy and floor/cap invariants; regional on-demand node pools/failure domains, pre-pull rollout, warm-allocation p95/p99 and N+1 certification remain | -| 8.33 `[D:8.26,8.32]` | On-demand-only live capacity and measured N+1: loss of largest node leaves two Ready slots plus headroom for surviving Allocated matches | Interruptible nodes cannot receive live matches; forced node loss neither overloads survivors nor prevents the next allocation | +| 8.33 `[D:8.26,8.32]` | **IN PROGRESS.** Fleet scheduling now requires on-demand capacity and spreads Ready processes across zones with skew 1; the autoscaler preserves the two-process Ready floor | `deploy/k8s/base/fleet.yaml` and manifest tests reject interruptible placement and single-zone concentration structurally; regional node pools, forced node-loss testing and measured N+1 headroom remain | | 8.34 `[D:8.28,8.29]` | Native x86_64 benchmark of boot-to-process-ready and assignment-ready, p99 CPU/RSS/network and 60 Hz ticks; limits/node cap with 30% headroom | Measurements replace old estimates and certify density with no tick backlog | | 8.35 `[D:8.17,8.19,8.20,8.30,8.31]` | **IN PROGRESS.** Pure Go initial-connect policy decides ranked 30 s no-show cancellation with abandon ladder and casual 60 s bot start only when each team has a human; empty-team casual allocations cancel | `server/domain/noshow.go` covers wait/deadline boundaries, deterministic no-show/innocent ordering, ranked cooldown history and no pre-live rating action; persistent ticket restoration, allocation shutdown, bot spawn and live integration remain | | 8.36 `[D:8.10,8.25,8.28,8.30]` | **IN PROGRESS.** Supervisor exposes an authenticated loopback-only drain request boundary that never places the token in command arguments/logs and rejects remote/partial/query-bearing configurations | `server/supervisor/` covers bearer-token enforcement, loopback URL validation, secret-safe configuration and rejection of missing drain credentials; TERM signal handling, 300 s/285 s lifecycle, PDB/Fleet drain and infrastructure-abort classification remain | diff --git a/server/security/test_fleet_manifests.py b/server/security/test_fleet_manifests.py index 72642e16..3bfd544d 100644 --- a/server/security/test_fleet_manifests.py +++ b/server/security/test_fleet_manifests.py @@ -19,6 +19,13 @@ class FleetManifestTest(unittest.TestCase): self.assertIn(label, fleet) for hardening in ("runAsNonRoot: true", "automountServiceAccountToken: false", "readOnlyRootFilesystem: true", "allowPrivilegeEscalation: false"): self.assertIn(hardening, fleet) + for scheduling in ( + "cosmic-clash.io/capacity-type: on-demand", + "topologyKey: topology.kubernetes.io/zone", + "whenUnsatisfiable: DoNotSchedule", + "maxSkew: 1", + ): + self.assertIn(scheduling, fleet) def test_autoscaler_preserves_ready_floor_and_owns_fleet(self): autoscaler = self.read("base/fleet-autoscaler.yaml")