From d40344a2c06107c6d59e565b17e15fed96c8a558 Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Sat, 5 Sep 2026 10:52:52 +0100 Subject: [PATCH] fix(deploy): ship runnable control-plane and matcher workloads The Kubernetes base deployed a control-plane image the Dockerfile never built -- cmd/control-plane was absent from the Go build stage and no target existed -- while the Dockerfile built a matcher image no manifest ever deployed. Applying the checked-in base therefore could not produce the advertised topology: one required workload had no repository-defined artifact, and nothing consumed queued tickets. Tickets could be created but never became proposals. Add the production control-plane build and image target, explicitly not the testkit-api target, which injects a fake login accepting any ticket. Add casual and ranked matcher Deployments as separate workloads: they have different match sizes, and separating them keeps a ranked backlog from delaying casual formation. One replica each -- CreateProposal's SKIP LOCKED fences make more replicas safe, but they would halve the candidate pool each worker sees per poll and worsen formation for no throughput gain at this scale. Their PDB uses maxUnavailable, since minAvailable against a single replica blocks node drains outright. Also fix both blocked traffic directions. No ingress policy admitted UDP/7777 to game-server pods, so an allocated server was unreachable from the internet under the namespace-wide default deny. And control-plane ingress admitted only edge-gateway pods, so roster fetch, registration, connection receipts, shutdown and result submission from game servers were dropped even inside the cluster, despite their egress being permitted. The default deny stays. Manifest tests now assert every required role is deployed, both playlists are scheduled, every referenced image maps to a real Dockerfile target, and both traffic directions are permitted. Each was verified to fail against the defect it covers. The control-plane image was built and run to confirm the target works. --- Dockerfile | 17 +++ deploy/k8s/base/kustomization.yaml | 2 + deploy/k8s/base/matcher-deployment.yaml | 147 ++++++++++++++++++++ deploy/k8s/base/matcher-pdb.yaml | 15 ++ deploy/k8s/base/network-policies.yaml | 84 +++++++++++ deploy/k8s/base/service-accounts.yaml | 7 + server/security/test_kubernetes_policies.py | 68 +++++++++ 7 files changed, 340 insertions(+) create mode 100644 deploy/k8s/base/matcher-deployment.yaml create mode 100644 deploy/k8s/base/matcher-pdb.yaml diff --git a/Dockerfile b/Dockerfile index f56d9277..82b3f22a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,6 +57,7 @@ COPY server/go.mod server/go.sum ./ RUN go mod download COPY server/ ./ RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/game-server-supervisor ./cmd/game-server-supervisor +RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/control-plane ./cmd/control-plane RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/testkit-api ./cmd/testkit-api RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/matcher ./cmd/matcher RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/allocator ./cmd/allocator @@ -76,6 +77,22 @@ COPY --from=supervisor-build /opt/cosmic-clash/game-server-supervisor /opt/cosmi RUN chmod 0755 /opt/cosmic-clash/game-server-supervisor ENTRYPOINT ["/opt/cosmic-clash/game-server-supervisor"] +# The production control-plane API. deploy/k8s/base/control-plane-deployment.yaml +# has always referenced this image, but nothing built it: cmd/control-plane was +# absent from the Go build stage and no target existed, so the checked-in +# Kubernetes base could not produce its own advertised topology. +# +# This must never be substituted with the testkit-api target below, which +# injects a fake login provider that accepts any ticket string. +FROM server AS control-plane +COPY --from=supervisor-build /opt/cosmic-clash/control-plane /opt/cosmic-clash/control-plane +COPY server/migrations /opt/cosmic-clash/migrations +RUN chmod 0755 /opt/cosmic-clash/control-plane +EXPOSE 8080 +ENTRYPOINT ["/opt/cosmic-clash/control-plane"] + +# TEST ONLY. Supplies a fake Steam login that accepts any ticket; never deploy +# this in place of the control-plane target above. FROM server AS testkit-api COPY --from=supervisor-build /opt/cosmic-clash/testkit-api /opt/cosmic-clash/testkit-api COPY server/migrations /opt/cosmic-clash/migrations diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml index dbcaa033..cbbf6982 100644 --- a/deploy/k8s/base/kustomization.yaml +++ b/deploy/k8s/base/kustomization.yaml @@ -11,6 +11,8 @@ resources: - allocator-deployment.yaml - allocator-service.yaml - allocator-pdb.yaml + - matcher-deployment.yaml + - matcher-pdb.yaml - maintenance-deployment.yaml - maintenance-pdb.yaml - fleet.yaml diff --git a/deploy/k8s/base/matcher-deployment.yaml b/deploy/k8s/base/matcher-deployment.yaml new file mode 100644 index 00000000..11a4945e --- /dev/null +++ b/deploy/k8s/base/matcher-deployment.yaml @@ -0,0 +1,147 @@ +# cmd/matcher is a standalone poll loop that turns queued tickets into +# proposals. It was built as an image but had no Deployment anywhere in this +# base, so applying the checked-in manifests produced a cluster where tickets +# could be created but nothing ever consumed them. +# +# Casual and ranked run as separate Deployments rather than one process with +# two loops: they have different match sizes, and separating them means a +# ranked backlog cannot delay casual formation (and vice versa). Each worker +# reads its own playlist-scoped Redis namespace. +# +# Exactly one replica each. The matcher claims tickets through CreateProposal's +# SKIP LOCKED fences so a second replica would be safe, but it would also halve +# the candidate pool each worker sees per poll and make formation quality worse +# for no throughput gain at this scale. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: matcher-casual + namespace: cosmic-clash + labels: + app.kubernetes.io/name: matcher + app.kubernetes.io/component: matcher + cosmic-clash.io/playlist: casual +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: matcher + cosmic-clash.io/playlist: casual + template: + metadata: + labels: + app.kubernetes.io/name: matcher + app.kubernetes.io/component: matcher + cosmic-clash.io/playlist: casual + spec: + terminationGracePeriodSeconds: 10 + serviceAccountName: matcher + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: matcher + image: ghcr.io/cosmic-clash/matcher@sha256:0000000000000000000000000000000000000000000000000000000000000000 + args: + - --dsn=$(COSMIC_CLASH_POSTGRES_DSN) + - --playlist=casual + - --size=4 + - --interval=1s + - --redis-addr=$(COSMIC_CLASH_REDIS_ADDR) + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 1 + memory: 512Mi + env: + - name: COSMIC_CLASH_POSTGRES_DSN + valueFrom: + secretKeyRef: + name: cosmic-clash-database + key: dsn + - name: COSMIC_CLASH_REDIS_ADDR + valueFrom: + secretKeyRef: + name: cosmic-clash-redis + key: addr +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: matcher-ranked + namespace: cosmic-clash + labels: + app.kubernetes.io/name: matcher + app.kubernetes.io/component: matcher + cosmic-clash.io/playlist: ranked +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: matcher + cosmic-clash.io/playlist: ranked + template: + metadata: + labels: + app.kubernetes.io/name: matcher + app.kubernetes.io/component: matcher + cosmic-clash.io/playlist: ranked + spec: + terminationGracePeriodSeconds: 10 + serviceAccountName: matcher + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: matcher + image: ghcr.io/cosmic-clash/matcher@sha256:0000000000000000000000000000000000000000000000000000000000000000 + args: + - --dsn=$(COSMIC_CLASH_POSTGRES_DSN) + # Ranked is strictly 3v3; domain.AllocateAcceptedProposal rejects a + # ranked proposal that is not exactly six players. + - --playlist=ranked + - --size=6 + - --interval=1s + - --redis-addr=$(COSMIC_CLASH_REDIS_ADDR) + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 1 + memory: 512Mi + env: + - name: COSMIC_CLASH_POSTGRES_DSN + valueFrom: + secretKeyRef: + name: cosmic-clash-database + key: dsn + - name: COSMIC_CLASH_REDIS_ADDR + valueFrom: + secretKeyRef: + name: cosmic-clash-redis + key: addr diff --git a/deploy/k8s/base/matcher-pdb.yaml b/deploy/k8s/base/matcher-pdb.yaml new file mode 100644 index 00000000..ac381110 --- /dev/null +++ b/deploy/k8s/base/matcher-pdb.yaml @@ -0,0 +1,15 @@ +# Each playlist runs a single matcher, so maxUnavailable rather than +# minAvailable: minAvailable: 1 against a one-replica Deployment blocks every +# voluntary eviction, including node drains. Allowing one keeps drains possible; +# formation simply pauses for the restart, and queued tickets are unaffected +# because the matcher holds no state of its own. +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: matcher + namespace: cosmic-clash +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: matcher diff --git a/deploy/k8s/base/network-policies.yaml b/deploy/k8s/base/network-policies.yaml index 8a67bded..17222c60 100644 --- a/deploy/k8s/base/network-policies.yaml +++ b/deploy/k8s/base/network-policies.yaml @@ -26,6 +26,18 @@ spec: ports: - protocol: TCP port: 8080 + # Allocated game servers are control-plane clients too: roster fetch, + # registration, connection receipts, shutdown acknowledgement and result + # submission all target this port. Their egress was already permitted, but + # without a matching ingress rule every one of those calls was dropped, so + # no allocated match could complete even inside the cluster. + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: game-server + ports: + - protocol: TCP + port: 8080 egress: - to: - namespaceSelector: @@ -175,3 +187,75 @@ spec: podSelector: matchLabels: k8s-app: kube-dns +--- +# Public players connect straight to the allocated GameServer's UDP port; the +# control plane only ever hands out its address. The namespace-wide default +# deny blocked that ingress entirely, so an allocated server was unreachable +# from the internet and no matchmade game could be joined. +# +# The source cannot be narrowed by selector: these peers are player machines +# outside the cluster. It is narrowed instead to exactly one protocol and port +# on exactly the game-server pods, and the game server admits a peer only with +# a valid signed join authorisation for its own match. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: game-server-allowed-ingress + namespace: cosmic-clash +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: game-server + policyTypes: + - Ingress + ingress: + - ports: + - protocol: UDP + port: 7777 +--- +# The matcher reads queued candidates and writes proposals. It exposes nothing +# and talks to nobody but its two datastores. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: matcher-allowed-egress + namespace: cosmic-clash +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: matcher + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: data + podSelector: + matchLabels: + app.kubernetes.io/name: postgres + ports: + - protocol: TCP + port: 5432 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: data + podSelector: + matchLabels: + app.kubernetes.io/name: redis + ports: + - protocol: TCP + port: 6379 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 diff --git a/deploy/k8s/base/service-accounts.yaml b/deploy/k8s/base/service-accounts.yaml index 488e5750..e02605fc 100644 --- a/deploy/k8s/base/service-accounts.yaml +++ b/deploy/k8s/base/service-accounts.yaml @@ -25,3 +25,10 @@ metadata: name: maintenance namespace: cosmic-clash automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: matcher + namespace: cosmic-clash +automountServiceAccountToken: false diff --git a/server/security/test_kubernetes_policies.py b/server/security/test_kubernetes_policies.py index f039a529..a606918d 100644 --- a/server/security/test_kubernetes_policies.py +++ b/server/security/test_kubernetes_policies.py @@ -160,6 +160,74 @@ class KubernetesPolicyTest(unittest.TestCase): self.assertIn(port, maintenance) self.assertNotIn("port: 8080", maintenance) + def test_every_required_workload_role_is_deployed(self): + # The base deployed a control-plane image nothing built, and built a + # matcher image nothing deployed -- so applying it produced a cluster + # where tickets could be created but never consumed. Assert the + # advertised topology is actually complete. + kustomization = self.read("kustomization.yaml") + rendered = "".join( + self.read(name.strip("- ").strip()) + for name in kustomization.splitlines() + if name.strip().startswith("- ") and name.strip().endswith(".yaml") + ) + for role in ("control-plane", "allocator", "maintenance", "matcher"): + self.assertIn(f"app.kubernetes.io/name: {role}", rendered, role) + self.assertIn("kind: Fleet", rendered) + # Casual and ranked must both be scheduled; one matcher process serves + # exactly one playlist. + self.assertIn("name: matcher-casual", rendered) + self.assertIn("name: matcher-ranked", rendered) + self.assertIn("--playlist=casual", rendered) + self.assertIn("--playlist=ranked", rendered) + # Ranked is strictly 3v3; AllocateAcceptedProposal rejects anything else. + self.assertIn("--size=6", rendered) + + def test_every_referenced_image_maps_to_a_real_dockerfile_target(self): + dockerfile = (Path(__file__).parents[2] / "Dockerfile").read_text() + targets = set(re.findall(r"(?mi)^FROM\s+.*?\bAS\s+(\S+)\s*$", dockerfile)) + # Guard the guard: if the target regex stops matching, every image + # below would "pass" vacuously. + self.assertIn("server", targets) + referenced = set() + for path in sorted(BASE.glob("*.yaml")): + for image in re.findall(r"image:\s*ghcr\.io/cosmic-clash/([\w.-]+)@", path.read_text()): + referenced.add(image) + self.assertTrue(referenced, "no images were found to check") + self.assertEqual(set(), referenced - targets, "manifests reference images this repo cannot build") + # testkit-api injects a fake login provider that accepts any ticket. + self.assertNotIn("testkit-api", referenced) + + def test_game_traffic_and_workload_callbacks_are_permitted(self): + policies = self.read("network-policies.yaml") + # Public players reach the allocated server directly over UDP; the + # namespace-wide default deny blocked that entirely. + ingress = policies.split("name: game-server-allowed-ingress", 1) + self.assertEqual(len(ingress), 2, "game-server ingress policy is missing") + game_ingress = ingress[1] + self.assertIn("app.kubernetes.io/name: game-server", game_ingress) + self.assertIn("protocol: UDP", game_ingress) + self.assertIn("port: 7777", game_ingress) + + # Game servers are control-plane clients: roster fetch, registration, + # connection receipts, shutdown and result submission. Their egress was + # allowed but the matching control-plane ingress was not. + control_plane = policies.split("name: control-plane-allowed-flows", 1)[-1].split("---", 1)[0] + self.assertIn("app.kubernetes.io/name: game-server", control_plane) + self.assertIn("app.kubernetes.io/name: edge-gateway", control_plane) + + # The default deny must survive all of this. + self.assertIn("name: default-deny-ingress-egress", policies) + + def test_matcher_network_policy_only_allows_its_datastores_and_dns(self): + policies = self.read("network-policies.yaml") + matcher = policies.split("name: matcher-allowed-egress", 1)[-1] + self.assertIn("app.kubernetes.io/name: matcher", matcher) + for port in ("port: 5432", "port: 6379", "port: 53"): + self.assertIn(port, matcher) + # The matcher never calls the control plane's API. + self.assertNotIn("port: 8080", matcher) + if __name__ == "__main__": unittest.main()