diff --git a/Dockerfile b/Dockerfile index f56d9277..82b3f22a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,6 +57,7 @@ COPY server/go.mod server/go.sum ./ RUN go mod download COPY server/ ./ RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/game-server-supervisor ./cmd/game-server-supervisor +RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/control-plane ./cmd/control-plane RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/testkit-api ./cmd/testkit-api RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/matcher ./cmd/matcher RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/allocator ./cmd/allocator @@ -76,6 +77,22 @@ COPY --from=supervisor-build /opt/cosmic-clash/game-server-supervisor /opt/cosmi RUN chmod 0755 /opt/cosmic-clash/game-server-supervisor ENTRYPOINT ["/opt/cosmic-clash/game-server-supervisor"] +# The production control-plane API. deploy/k8s/base/control-plane-deployment.yaml +# has always referenced this image, but nothing built it: cmd/control-plane was +# absent from the Go build stage and no target existed, so the checked-in +# Kubernetes base could not produce its own advertised topology. +# +# This must never be substituted with the testkit-api target below, which +# injects a fake login provider that accepts any ticket string. +FROM server AS control-plane +COPY --from=supervisor-build /opt/cosmic-clash/control-plane /opt/cosmic-clash/control-plane +COPY server/migrations /opt/cosmic-clash/migrations +RUN chmod 0755 /opt/cosmic-clash/control-plane +EXPOSE 8080 +ENTRYPOINT ["/opt/cosmic-clash/control-plane"] + +# TEST ONLY. Supplies a fake Steam login that accepts any ticket; never deploy +# this in place of the control-plane target above. FROM server AS testkit-api COPY --from=supervisor-build /opt/cosmic-clash/testkit-api /opt/cosmic-clash/testkit-api COPY server/migrations /opt/cosmic-clash/migrations diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml index dbcaa033..cbbf6982 100644 --- a/deploy/k8s/base/kustomization.yaml +++ b/deploy/k8s/base/kustomization.yaml @@ -11,6 +11,8 @@ resources: - allocator-deployment.yaml - allocator-service.yaml - allocator-pdb.yaml + - matcher-deployment.yaml + - matcher-pdb.yaml - maintenance-deployment.yaml - maintenance-pdb.yaml - fleet.yaml diff --git a/deploy/k8s/base/matcher-deployment.yaml b/deploy/k8s/base/matcher-deployment.yaml new file mode 100644 index 00000000..11a4945e --- /dev/null +++ b/deploy/k8s/base/matcher-deployment.yaml @@ -0,0 +1,147 @@ +# cmd/matcher is a standalone poll loop that turns queued tickets into +# proposals. It was built as an image but had no Deployment anywhere in this +# base, so applying the checked-in manifests produced a cluster where tickets +# could be created but nothing ever consumed them. +# +# Casual and ranked run as separate Deployments rather than one process with +# two loops: they have different match sizes, and separating them means a +# ranked backlog cannot delay casual formation (and vice versa). Each worker +# reads its own playlist-scoped Redis namespace. +# +# Exactly one replica each. The matcher claims tickets through CreateProposal's +# SKIP LOCKED fences so a second replica would be safe, but it would also halve +# the candidate pool each worker sees per poll and make formation quality worse +# for no throughput gain at this scale. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: matcher-casual + namespace: cosmic-clash + labels: + app.kubernetes.io/name: matcher + app.kubernetes.io/component: matcher + cosmic-clash.io/playlist: casual +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: matcher + cosmic-clash.io/playlist: casual + template: + metadata: + labels: + app.kubernetes.io/name: matcher + app.kubernetes.io/component: matcher + cosmic-clash.io/playlist: casual + spec: + terminationGracePeriodSeconds: 10 + serviceAccountName: matcher + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: matcher + image: ghcr.io/cosmic-clash/matcher@sha256:0000000000000000000000000000000000000000000000000000000000000000 + args: + - --dsn=$(COSMIC_CLASH_POSTGRES_DSN) + - --playlist=casual + - --size=4 + - --interval=1s + - --redis-addr=$(COSMIC_CLASH_REDIS_ADDR) + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 1 + memory: 512Mi + env: + - name: COSMIC_CLASH_POSTGRES_DSN + valueFrom: + secretKeyRef: + name: cosmic-clash-database + key: dsn + - name: COSMIC_CLASH_REDIS_ADDR + valueFrom: + secretKeyRef: + name: cosmic-clash-redis + key: addr +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: matcher-ranked + namespace: cosmic-clash + labels: + app.kubernetes.io/name: matcher + app.kubernetes.io/component: matcher + cosmic-clash.io/playlist: ranked +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: matcher + cosmic-clash.io/playlist: ranked + template: + metadata: + labels: + app.kubernetes.io/name: matcher + app.kubernetes.io/component: matcher + cosmic-clash.io/playlist: ranked + spec: + terminationGracePeriodSeconds: 10 + serviceAccountName: matcher + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: matcher + image: ghcr.io/cosmic-clash/matcher@sha256:0000000000000000000000000000000000000000000000000000000000000000 + args: + - --dsn=$(COSMIC_CLASH_POSTGRES_DSN) + # Ranked is strictly 3v3; domain.AllocateAcceptedProposal rejects a + # ranked proposal that is not exactly six players. + - --playlist=ranked + - --size=6 + - --interval=1s + - --redis-addr=$(COSMIC_CLASH_REDIS_ADDR) + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 1 + memory: 512Mi + env: + - name: COSMIC_CLASH_POSTGRES_DSN + valueFrom: + secretKeyRef: + name: cosmic-clash-database + key: dsn + - name: COSMIC_CLASH_REDIS_ADDR + valueFrom: + secretKeyRef: + name: cosmic-clash-redis + key: addr diff --git a/deploy/k8s/base/matcher-pdb.yaml b/deploy/k8s/base/matcher-pdb.yaml new file mode 100644 index 00000000..ac381110 --- /dev/null +++ b/deploy/k8s/base/matcher-pdb.yaml @@ -0,0 +1,15 @@ +# Each playlist runs a single matcher, so maxUnavailable rather than +# minAvailable: minAvailable: 1 against a one-replica Deployment blocks every +# voluntary eviction, including node drains. Allowing one keeps drains possible; +# formation simply pauses for the restart, and queued tickets are unaffected +# because the matcher holds no state of its own. +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: matcher + namespace: cosmic-clash +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: matcher diff --git a/deploy/k8s/base/network-policies.yaml b/deploy/k8s/base/network-policies.yaml index 8a67bded..17222c60 100644 --- a/deploy/k8s/base/network-policies.yaml +++ b/deploy/k8s/base/network-policies.yaml @@ -26,6 +26,18 @@ spec: ports: - protocol: TCP port: 8080 + # Allocated game servers are control-plane clients too: roster fetch, + # registration, connection receipts, shutdown acknowledgement and result + # submission all target this port. Their egress was already permitted, but + # without a matching ingress rule every one of those calls was dropped, so + # no allocated match could complete even inside the cluster. + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: game-server + ports: + - protocol: TCP + port: 8080 egress: - to: - namespaceSelector: @@ -175,3 +187,75 @@ spec: podSelector: matchLabels: k8s-app: kube-dns +--- +# Public players connect straight to the allocated GameServer's UDP port; the +# control plane only ever hands out its address. The namespace-wide default +# deny blocked that ingress entirely, so an allocated server was unreachable +# from the internet and no matchmade game could be joined. +# +# The source cannot be narrowed by selector: these peers are player machines +# outside the cluster. It is narrowed instead to exactly one protocol and port +# on exactly the game-server pods, and the game server admits a peer only with +# a valid signed join authorisation for its own match. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: game-server-allowed-ingress + namespace: cosmic-clash +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: game-server + policyTypes: + - Ingress + ingress: + - ports: + - protocol: UDP + port: 7777 +--- +# The matcher reads queued candidates and writes proposals. It exposes nothing +# and talks to nobody but its two datastores. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: matcher-allowed-egress + namespace: cosmic-clash +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: matcher + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: data + podSelector: + matchLabels: + app.kubernetes.io/name: postgres + ports: + - protocol: TCP + port: 5432 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: data + podSelector: + matchLabels: + app.kubernetes.io/name: redis + ports: + - protocol: TCP + port: 6379 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 diff --git a/deploy/k8s/base/service-accounts.yaml b/deploy/k8s/base/service-accounts.yaml index 488e5750..e02605fc 100644 --- a/deploy/k8s/base/service-accounts.yaml +++ b/deploy/k8s/base/service-accounts.yaml @@ -25,3 +25,10 @@ metadata: name: maintenance namespace: cosmic-clash automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: matcher + namespace: cosmic-clash +automountServiceAccountToken: false diff --git a/server/security/test_kubernetes_policies.py b/server/security/test_kubernetes_policies.py index f039a529..a606918d 100644 --- a/server/security/test_kubernetes_policies.py +++ b/server/security/test_kubernetes_policies.py @@ -160,6 +160,74 @@ class KubernetesPolicyTest(unittest.TestCase): self.assertIn(port, maintenance) self.assertNotIn("port: 8080", maintenance) + def test_every_required_workload_role_is_deployed(self): + # The base deployed a control-plane image nothing built, and built a + # matcher image nothing deployed -- so applying it produced a cluster + # where tickets could be created but never consumed. Assert the + # advertised topology is actually complete. + kustomization = self.read("kustomization.yaml") + rendered = "".join( + self.read(name.strip("- ").strip()) + for name in kustomization.splitlines() + if name.strip().startswith("- ") and name.strip().endswith(".yaml") + ) + for role in ("control-plane", "allocator", "maintenance", "matcher"): + self.assertIn(f"app.kubernetes.io/name: {role}", rendered, role) + self.assertIn("kind: Fleet", rendered) + # Casual and ranked must both be scheduled; one matcher process serves + # exactly one playlist. + self.assertIn("name: matcher-casual", rendered) + self.assertIn("name: matcher-ranked", rendered) + self.assertIn("--playlist=casual", rendered) + self.assertIn("--playlist=ranked", rendered) + # Ranked is strictly 3v3; AllocateAcceptedProposal rejects anything else. + self.assertIn("--size=6", rendered) + + def test_every_referenced_image_maps_to_a_real_dockerfile_target(self): + dockerfile = (Path(__file__).parents[2] / "Dockerfile").read_text() + targets = set(re.findall(r"(?mi)^FROM\s+.*?\bAS\s+(\S+)\s*$", dockerfile)) + # Guard the guard: if the target regex stops matching, every image + # below would "pass" vacuously. + self.assertIn("server", targets) + referenced = set() + for path in sorted(BASE.glob("*.yaml")): + for image in re.findall(r"image:\s*ghcr\.io/cosmic-clash/([\w.-]+)@", path.read_text()): + referenced.add(image) + self.assertTrue(referenced, "no images were found to check") + self.assertEqual(set(), referenced - targets, "manifests reference images this repo cannot build") + # testkit-api injects a fake login provider that accepts any ticket. + self.assertNotIn("testkit-api", referenced) + + def test_game_traffic_and_workload_callbacks_are_permitted(self): + policies = self.read("network-policies.yaml") + # Public players reach the allocated server directly over UDP; the + # namespace-wide default deny blocked that entirely. + ingress = policies.split("name: game-server-allowed-ingress", 1) + self.assertEqual(len(ingress), 2, "game-server ingress policy is missing") + game_ingress = ingress[1] + self.assertIn("app.kubernetes.io/name: game-server", game_ingress) + self.assertIn("protocol: UDP", game_ingress) + self.assertIn("port: 7777", game_ingress) + + # Game servers are control-plane clients: roster fetch, registration, + # connection receipts, shutdown and result submission. Their egress was + # allowed but the matching control-plane ingress was not. + control_plane = policies.split("name: control-plane-allowed-flows", 1)[-1].split("---", 1)[0] + self.assertIn("app.kubernetes.io/name: game-server", control_plane) + self.assertIn("app.kubernetes.io/name: edge-gateway", control_plane) + + # The default deny must survive all of this. + self.assertIn("name: default-deny-ingress-egress", policies) + + def test_matcher_network_policy_only_allows_its_datastores_and_dns(self): + policies = self.read("network-policies.yaml") + matcher = policies.split("name: matcher-allowed-egress", 1)[-1] + self.assertIn("app.kubernetes.io/name: matcher", matcher) + for port in ("port: 5432", "port: 6379", "port: 53"): + self.assertIn(port, matcher) + # The matcher never calls the control plane's API. + self.assertNotIn("port: 8080", matcher) + if __name__ == "__main__": unittest.main()