From 2e1c010fa2c8e686bdac6a442b4a6e843881bcac Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:48:51 +0100 Subject: [PATCH] feat(multiplayer): deploy allocator role --- deploy/k8s/base/allocator-deployment.yaml | 61 +++++++++++++++++++++ deploy/k8s/base/kustomization.yaml | 1 + deploy/k8s/base/network-policies.yaml | 52 ++++++++++++++++++ deploy/k8s/base/service-accounts.yaml | 8 ++- multiplayer-next.md | 4 +- server/security/test_kubernetes_policies.py | 21 +++++++ 6 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 deploy/k8s/base/allocator-deployment.yaml diff --git a/deploy/k8s/base/allocator-deployment.yaml b/deploy/k8s/base/allocator-deployment.yaml new file mode 100644 index 00000000..6e7899dd --- /dev/null +++ b/deploy/k8s/base/allocator-deployment.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: allocator + namespace: cosmic-clash + labels: + app.kubernetes.io/name: allocator + app.kubernetes.io/component: allocator +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: allocator + template: + metadata: + labels: + app.kubernetes.io/name: allocator + app.kubernetes.io/component: allocator + spec: + serviceAccountName: allocator + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: allocator + image: ghcr.io/cosmic-clash/allocator@sha256:0000000000000000000000000000000000000000000000000000000000000000 + args: + - --dsn=$(COSMIC_CLASH_POSTGRES_DSN) + - --agones-url=https://agones-allocator.agones-system.svc.cluster.local + - --agones-namespace=cosmic-clash + - --metrics-addr=:9091 + ports: + - name: metrics + containerPort: 9091 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 1 + memory: 512Mi + env: + - name: COSMIC_CLASH_POSTGRES_DSN + valueFrom: + secretKeyRef: + name: cosmic-clash-database + key: dsn + - name: COSMIC_CLASH_WORKLOAD_SECRET + valueFrom: + secretKeyRef: + name: cosmic-clash-workload + key: secret diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml index 5ca4150e..2c8f29b2 100644 --- a/deploy/k8s/base/kustomization.yaml +++ b/deploy/k8s/base/kustomization.yaml @@ -7,6 +7,7 @@ resources: - network-policies.yaml - control-plane-deployment.yaml - control-plane-service.yaml + - allocator-deployment.yaml - allocator-service.yaml - fleet.yaml - fleet-autoscaler.yaml diff --git a/deploy/k8s/base/network-policies.yaml b/deploy/k8s/base/network-policies.yaml index 1e9e39e2..a76ece70 100644 --- a/deploy/k8s/base/network-policies.yaml +++ b/deploy/k8s/base/network-policies.yaml @@ -97,3 +97,55 @@ spec: podSelector: matchLabels: k8s-app: kube-dns +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allocator-allowed-flows + namespace: cosmic-clash +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: allocator + policyTypes: [Ingress, Egress] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus + ports: + - protocol: TCP + port: 9091 + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: data + podSelector: + matchLabels: + app.kubernetes.io/name: postgres + ports: + - protocol: TCP + port: 5432 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: agones-system + ports: + - protocol: TCP + port: 443 + - ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns diff --git a/deploy/k8s/base/service-accounts.yaml b/deploy/k8s/base/service-accounts.yaml index 8e701794..8b45bb4d 100644 --- a/deploy/k8s/base/service-accounts.yaml +++ b/deploy/k8s/base/service-accounts.yaml @@ -11,4 +11,10 @@ metadata: name: match-server namespace: cosmic-clash automountServiceAccountToken: false - +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: allocator + namespace: cosmic-clash +automountServiceAccountToken: false diff --git a/multiplayer-next.md b/multiplayer-next.md index 4f8155b4..61f498dd 100644 --- a/multiplayer-next.md +++ b/multiplayer-next.md @@ -1191,7 +1191,7 @@ the local/CI/community transport, not a silent production fallback. | 8.9 `[D:8.4,8.7]` | **IN PROGRESS.** Pure Go join policy binds SteamID/player/match/server/team/slot/protocol/expiry, rejects duplicate roster slots, permits same-identity reclaim through backend loss and fences prior server-owned generations | `server/domain/reconnect.go` covers SteamID/server/slot binding, expiry, repeated reclaim, grace boundary and old-generation fencing; signed token issuance/verification, persistent leases, Godot `hello` transport and production integration remain | | 8.10 `[D:8.5,8.31]` | **IN PROGRESS.** Dependency-free workload credential policy validates an adapter-verified signature, issuer/audience/time bounds, namespace/service account, pod UID, GameServer UID, allocation ID, match ID and server ID before result submission; `server/workload` now parses projected compact JWTs, rejects unsigned/malformed/ambiguous tokens, verifies signing input through an injected trust callback, and delegates exact binding to the domain policy; the API exposes the workload-authenticated server result route. **`WorkloadVerify` is now wired for real** -- the blocker this row previously named (`cmd/control-plane/main.go` never wiring it, so `/register` and `/result` 503'd on every real request) is closed by taking a different, equally-valid approach instead of the Kubernetes-JWT one: `serverMutation` only ever compares `WorkloadBinding.ServerID`/`.MatchID` (and `AdvanceServerRegistration` only additionally needs `.AllocationID`) -- nothing downstream requires Namespace/ServiceAcct/PodUID/GameServerUID, so a Kubernetes-issued token was never actually required, just the assumption that it was. `server/workload/signed_token.go` mints and verifies a short-lived HMAC-signed token with a secret only the control plane holds (the same trust model `domain.SessionStore` already uses for player sessions), needing no cluster JWKS/TokenReview to validate -- signature and expiry are fully self-contained. The token binds only `allocation_id` (deliberately, not `match_id`/`server_id` too: it must be requestable in the same Agones request that asks for a server, before Agones has picked one) -- `store.AllocationBindingByAllocationID` resolves `match_id`/`server_id` durably from the `allocations` table at verify time, so a token can never claim a pairing that wasn't actually, durably allocated. `api.WorkloadVerifierFromSignedToken` combines both and is wired into `cmd/control-plane` (`--workload-secret`/`COSMIC_CLASH_WORKLOAD_SECRET`; a startup warning fires and the route stays 503 if it's left unset) and `cmd/testkit-api` (fixed test secret). **The delivery channel is now wired too**: `agones.Client` gains `WorkloadSecret`/`WorkloadTokenTTL` -- when set, `Allocate` mints a token for the allocation and requests it as a third `cosmic-clash.io/workload-token` annotation alongside match-id/allocation-id, wired from `cmd/allocator`'s own `--workload-secret`/`COSMIC_CLASH_WORKLOAD_SECRET` (must match the control plane's); `supervisor.Supervisor.workloadToken()` resolves the bearer credential with an explicit `--workload-token-path` (kept for a possible future Kubernetes-JWT path) always winning, otherwise falling back to that same annotation -- the same fallback pattern `matchID()` already used for `cosmic-clash.io/match-id`. `WorkloadTokenPath` is accordingly no longer required at construction time | `server/domain/workload.go`, `server/workload/jwt.go`, `server/api/service.go` and adversarial tests reject every binding mutation, missing/unverified signature, `none`/malformed JWT, ambiguous audience, server/match mismatch and time boundary; `server/testkit/pipeline_test.go` carries allocation identity through the offline result path; `cmd/control-plane` now wires `ResultSubmitter: store.PostgresResults{DB: db}` (a ready-made adapter that had been referenced from nowhere at all, not even a test); `server/workload/signed_token_test.go` covers round-trip, tampered payload, wrong secret, malformed input and the exact expiry boundary; `server/api/workload_verifier_integration_test.go` (opt-in, real PostgreSQL) covers acceptance against a real allocation row, rejection of an unrecorded allocation, that two distinct real allocations each resolve to their own and only their own match/server pairing, and the previously-503 `Service.WorkloadVerify` field itself now succeeding -- all verified clean with `-race` across multiple runs; `cmd/control-plane/main_test.go`'s `TestServerRoutesRequireWorkloadVerifyToBeWired` now documents and pins the misconfigured-secret case specifically, not the "always unwired" case; `server/agones/allocation_test.go`'s `TestAllocateRequestsAWorkloadTokenAnnotationWhenConfigured` and `server/supervisor/supervisor_test.go`'s new annotation-fallback/fails-closed pair cover the delivery channel end to end (mint -> annotation -> supervisor read -> Authorization header) short of a live cluster. **What's still missing**: this has never run against a real Agones cluster (only HTTP-level fakes), so the exact `object_meta` JSON casing Agones actually returns remains unverified from this sandbox (see `GameServer.ObjectMeta`'s existing caveat); live duplicate/conflict alerting also remains | | 8.11 `[D:8.1]` | **DONE.** Write the threat model: forged clients/replay/queues/results, floods/bots, pod/insider compromise, gameplay and API DDoS, SDR signing-key theft, dependencies and denial-of-wallet | [`docs/THREAT-MODEL.md`](docs/THREAT-MODEL.md) records prevention, detection/response, owner and residual risk for every threat; it separates offline CA/online signer, client/game-server/PostgreSQL/Redis trust boundaries and recovery behavior | -| 8.12 `[D:8.11]` | **IN PROGRESS.** Provider-neutral Kubernetes baseline enforces restricted namespace admission, non-root/read-only/no-capability workloads, separate service accounts, allocator-only RBAC, default-deny networking and explicit edge/data/DNS/Agones flows; application manifests consume externally populated Secret objects; the Go API now has an optional bounded per-replica rate-limit/429 boundary | `deploy/k8s/base/` plus `server/security/test_kubernetes_policies.py`, `server/api/rate_limit.go` and adversarial tests cover static hardening, secret-reference invariants, fixed-window limits and bounded key memory; private-store provisioning, distributed/global quotas, edge DDoS/WAF/origin shielding, WebSocket limits, overload shedding, encrypted backups and live policy/load tests remain | +| 8.12 `[D:8.11]` | **IN PROGRESS.** Provider-neutral Kubernetes baseline enforces restricted namespace admission, non-root/read-only/no-capability workloads, separate service accounts, allocator-only RBAC, default-deny networking and explicit edge/data/DNS/Agones/metrics flows; application manifests consume externally populated Secret objects; the Go API and allocator now have bounded per-replica rate/quota boundaries and metrics endpoints | `deploy/k8s/base/` includes hardened control-plane, allocator, and game-server resources; `server/security/test_kubernetes_policies.py`, `server/api/rate_limit.go`, allocator metrics tests and adversarial checks cover static hardening, secret-reference invariants, fixed-window limits, bounded labels and bounded key memory; private-store provisioning, distributed/global quotas, edge DDoS/WAF/origin shielding, WebSocket limits, overload shedding, encrypted backups and live policy/load tests remain | | 8.13 `[D:8.12]` | **IN PROGRESS.** Docker/Kubernetes references are digest-pinned, a dependency-free checker rejects mutable tags/plaintext credentials and concrete release overlays can reject template digests; release documentation defines SBOM, dependency/image scanning, signing, admission verification and a 24-hour critical-fix SLA | `scripts/verify_supply_chain.py`, `server/security/test_supply_chain.py`, `docs/SUPPLY-CHAIN.md` and `.github/workflows/supply-chain.yml` cover repository policy and provenance requirements; registry SBOM/scan/sign/admission execution and a concrete production overlay remain | #### 8C — Queueing, matchmaking, playlists and rating @@ -1251,7 +1251,7 @@ the local/CI/community transport, not a silent production fallback. | 8.49 `[D:8.25,8.26,8.28,8.29,8.30,8.31,8.35,8.36]` | **IN PROGRESS.** `scripts/verify_kind_agones.sh` creates a disposable kind cluster, installs pinned Agones, loads the real `game-server` image, applies the Fleet in an explicitly separate Agones-only supervisor/UDP readiness mode, and verifies readiness plus allocation of a dynamic UDP endpoint; `.github/workflows/agones-integration.yml` runs it for infrastructure changes and on demand | The cloud-free runner is committed and fails clearly when Docker/kind/Helm are unavailable. CI/live evidence for production control-plane registration, roster/no-show, both readiness stages, races, multi-match node, result-pending reconciliation, drain, and rollback remains open | | 8.50 `[D:8.25,8.37,8.43,8.49]` | **IN PROGRESS.** `make verify-chaos-recovery` provides a disposable PostgreSQL + real testkit API + real maintenance flow: it restarts the API, injects a stale allocation, and verifies no-penalty requeue plus a durable participant-targeted lifecycle event | The API-restart/stalled-allocation slice is implemented and documented; 100 ms RTT/jitter/loss, matcher/client restart, game-pod death, node drain, Redis failover, control-plane loss, and live chaos evidence remain | | 8.51 `[D:8.17,8.18,8.30,8.31,8.45]` | **IN PROGRESS.** The opt-in `make verify-multiplayer-load` gate drives 10,000 real HTTP queue-create requests through the service with 256 in flight and records p95/p99, plus 100 concurrent proposal formations through the real matcher/domain path; the handler and in-process ownership boundary are exercised without weakening normal tests | Local API load passes at p95 <250 ms in normal and race runs, and the matcher forms 100 unique proposals; PostgreSQL saturation, durable matcher fencing under load, forecast launch concurrency x2, and replica scaling remain live infrastructure gates | -| 8.52 `[D:8.32,8.34,8.45,8.51]` | **IN PROGRESS.** Allocator supports both an opt-in per-replica fixed-window quota (`--allocation-quota` / `--allocation-quota-window`) and an optional PostgreSQL-backed EU/NA quota table consumed inside the serializable allocation transaction before any provider call; idempotent replays do not double-count. The allocator now exposes bounded EU/NA Prometheus counters at an explicit `/metrics` listener (`--metrics-addr`), including quota denials; the checked-in rule warns on regional denial activity, and Kubernetes Service/ServiceMonitor resources provide the discovery contract | Normal/race/vet tests cover local quota exhaustion, window reset, region isolation, invalid input, atomic concurrent consumption, bounded metrics labels, and read-only endpoint behavior; migration/SQL and manifest coverage define the shared quota/metrics boundaries; allocator Deployment provisioning, measured regional cost model, threshold tuning, and denial-of-wallet rehearsal remain | +| 8.52 `[D:8.32,8.34,8.45,8.51]` | **IN PROGRESS.** Allocator supports both an opt-in per-replica fixed-window quota (`--allocation-quota` / `--allocation-quota-window`) and an optional PostgreSQL-backed EU/NA quota table consumed inside the serializable allocation transaction before any provider call; idempotent replays do not double-count. The allocator now exposes bounded EU/NA Prometheus counters at an explicit `/metrics` listener (`--metrics-addr`), including quota denials; the checked-in rule warns on regional denial activity, and hardened Kubernetes Deployment/Service/ServiceMonitor resources provide the provisioning and discovery contract | Normal/race/vet tests cover local quota exhaustion, window reset, region isolation, invalid input, atomic concurrent consumption, bounded metrics labels, read-only endpoint behavior, and hardened deployment/network policy invariants; migration/SQL and manifest coverage define the shared quota/metrics boundaries; real image digest/secrets, measured regional cost model, threshold tuning, and denial-of-wallet rehearsal remain | | 8.53 `[D:7.8,8.13,8.38,8.45,8.46,8.48,8.49,8.50,8.51,8.52]` | **IN PROGRESS.** `scripts/verify_release_gate.py` provides a fail-closed promotion check for the ordered development → internal → casual canary → casual → provisional ranked → ranked stages, requiring an evidence report for SLO, security, cost, rollback, EU+NA playtests, and both legacy gates | Validator and adversarial tests cover skipped stages, unknown stages, missing gates, non-boolean gate values, and blank release IDs; the actual reports, production rollback rehearsal, regional playtests, and live promotion remain open | Implementation invariants for every task above: diff --git a/server/security/test_kubernetes_policies.py b/server/security/test_kubernetes_policies.py index 5ee421d7..3c826be6 100644 --- a/server/security/test_kubernetes_policies.py +++ b/server/security/test_kubernetes_policies.py @@ -26,6 +26,27 @@ class KubernetesPolicyTest(unittest.TestCase): self.assertNotRegex(deployment, r"(?im)^\s*(password|token|private.?key):\s*[^\n]+$") self.assertIn("secretKeyRef:", deployment) + def test_allocator_is_hardened_and_uses_only_external_secrets(self): + deployment = self.read("allocator-deployment.yaml") + for required in ( + "runAsNonRoot: true", "type: RuntimeDefault", "allowPrivilegeEscalation: false", + "readOnlyRootFilesystem: true", "drop: [ALL]", "resources:", + "image: ghcr.io/cosmic-clash/allocator@sha256:", + "--metrics-addr=:9091", "containerPort: 9091", + "key: dsn", "key: secret", "automountServiceAccountToken: false", + ): + self.assertIn(required, deployment) + self.assertNotRegex(deployment, r"(?im)^\s*(password|token|private.?key):\s*[^\n]+$") + + def test_allocator_network_policy_has_only_metrics_data_agones_and_dns_flows(self): + policies = self.read("network-policies.yaml") + allocator = policies.split("name: allocator-allowed-flows", 1)[-1] + self.assertIn("port: 9091", allocator) + for port in ("port: 5432", "port: 443", "port: 53"): + self.assertIn(port, allocator) + self.assertNotIn("port: 8080", allocator) + self.assertNotIn("ipBlock:", allocator) + def test_rbac_is_scoped_to_allocator_create(self): rbac = self.read("rbac.yaml") self.assertIn("namespace: agones-system", rbac)