fix(deploy): ship runnable control-plane and matcher workloads

The Kubernetes base deployed a control-plane image the Dockerfile never
built -- cmd/control-plane was absent from the Go build stage and no
target existed -- while the Dockerfile built a matcher image no manifest
ever deployed. Applying the checked-in base therefore could not produce
the advertised topology: one required workload had no repository-defined
artifact, and nothing consumed queued tickets. Tickets could be created
but never became proposals.

Add the production control-plane build and image target, explicitly not
the testkit-api target, which injects a fake login accepting any ticket.
Add casual and ranked matcher Deployments as separate workloads: they
have different match sizes, and separating them keeps a ranked backlog
from delaying casual formation. One replica each -- CreateProposal's
SKIP LOCKED fences make more replicas safe, but they would halve the
candidate pool each worker sees per poll and worsen formation for no
throughput gain at this scale. Their PDB uses maxUnavailable, since
minAvailable against a single replica blocks node drains outright.

Also fix both blocked traffic directions. No ingress policy admitted
UDP/7777 to game-server pods, so an allocated server was unreachable
from the internet under the namespace-wide default deny. And
control-plane ingress admitted only edge-gateway pods, so roster fetch,
registration, connection receipts, shutdown and result submission from
game servers were dropped even inside the cluster, despite their egress
being permitted. The default deny stays.

Manifest tests now assert every required role is deployed, both
playlists are scheduled, every referenced image maps to a real
Dockerfile target, and both traffic directions are permitted. Each was
verified to fail against the defect it covers. The control-plane image
was built and run to confirm the target works.
This commit is contained in:
Josh Creek
2026-09-05 10:52:52 +01:00
parent 801fca7cb0
commit d40344a2c0
7 changed files with 340 additions and 0 deletions
+17
View File
@@ -57,6 +57,7 @@ COPY server/go.mod server/go.sum ./
RUN go mod download RUN go mod download
COPY server/ ./ COPY server/ ./
RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/game-server-supervisor ./cmd/game-server-supervisor RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/game-server-supervisor ./cmd/game-server-supervisor
RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/control-plane ./cmd/control-plane
RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/testkit-api ./cmd/testkit-api RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/testkit-api ./cmd/testkit-api
RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/matcher ./cmd/matcher RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/matcher ./cmd/matcher
RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/allocator ./cmd/allocator RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/allocator ./cmd/allocator
@@ -76,6 +77,22 @@ COPY --from=supervisor-build /opt/cosmic-clash/game-server-supervisor /opt/cosmi
RUN chmod 0755 /opt/cosmic-clash/game-server-supervisor RUN chmod 0755 /opt/cosmic-clash/game-server-supervisor
ENTRYPOINT ["/opt/cosmic-clash/game-server-supervisor"] ENTRYPOINT ["/opt/cosmic-clash/game-server-supervisor"]
# The production control-plane API. deploy/k8s/base/control-plane-deployment.yaml
# has always referenced this image, but nothing built it: cmd/control-plane was
# absent from the Go build stage and no target existed, so the checked-in
# Kubernetes base could not produce its own advertised topology.
#
# This must never be substituted with the testkit-api target below, which
# injects a fake login provider that accepts any ticket string.
FROM server AS control-plane
COPY --from=supervisor-build /opt/cosmic-clash/control-plane /opt/cosmic-clash/control-plane
COPY server/migrations /opt/cosmic-clash/migrations
RUN chmod 0755 /opt/cosmic-clash/control-plane
EXPOSE 8080
ENTRYPOINT ["/opt/cosmic-clash/control-plane"]
# TEST ONLY. Supplies a fake Steam login that accepts any ticket; never deploy
# this in place of the control-plane target above.
FROM server AS testkit-api FROM server AS testkit-api
COPY --from=supervisor-build /opt/cosmic-clash/testkit-api /opt/cosmic-clash/testkit-api COPY --from=supervisor-build /opt/cosmic-clash/testkit-api /opt/cosmic-clash/testkit-api
COPY server/migrations /opt/cosmic-clash/migrations COPY server/migrations /opt/cosmic-clash/migrations
+2
View File
@@ -11,6 +11,8 @@ resources:
- allocator-deployment.yaml - allocator-deployment.yaml
- allocator-service.yaml - allocator-service.yaml
- allocator-pdb.yaml - allocator-pdb.yaml
- matcher-deployment.yaml
- matcher-pdb.yaml
- maintenance-deployment.yaml - maintenance-deployment.yaml
- maintenance-pdb.yaml - maintenance-pdb.yaml
- fleet.yaml - fleet.yaml
+147
View File
@@ -0,0 +1,147 @@
# cmd/matcher is a standalone poll loop that turns queued tickets into
# proposals. It was built as an image but had no Deployment anywhere in this
# base, so applying the checked-in manifests produced a cluster where tickets
# could be created but nothing ever consumed them.
#
# Casual and ranked run as separate Deployments rather than one process with
# two loops: they have different match sizes, and separating them means a
# ranked backlog cannot delay casual formation (and vice versa). Each worker
# reads its own playlist-scoped Redis namespace.
#
# Exactly one replica each. The matcher claims tickets through CreateProposal's
# SKIP LOCKED fences so a second replica would be safe, but it would also halve
# the candidate pool each worker sees per poll and make formation quality worse
# for no throughput gain at this scale.
apiVersion: apps/v1
kind: Deployment
metadata:
name: matcher-casual
namespace: cosmic-clash
labels:
app.kubernetes.io/name: matcher
app.kubernetes.io/component: matcher
cosmic-clash.io/playlist: casual
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app.kubernetes.io/name: matcher
cosmic-clash.io/playlist: casual
template:
metadata:
labels:
app.kubernetes.io/name: matcher
app.kubernetes.io/component: matcher
cosmic-clash.io/playlist: casual
spec:
terminationGracePeriodSeconds: 10
serviceAccountName: matcher
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
seccompProfile:
type: RuntimeDefault
containers:
- name: matcher
image: ghcr.io/cosmic-clash/matcher@sha256:0000000000000000000000000000000000000000000000000000000000000000
args:
- --dsn=$(COSMIC_CLASH_POSTGRES_DSN)
- --playlist=casual
- --size=4
- --interval=1s
- --redis-addr=$(COSMIC_CLASH_REDIS_ADDR)
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: [ALL]
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 1
memory: 512Mi
env:
- name: COSMIC_CLASH_POSTGRES_DSN
valueFrom:
secretKeyRef:
name: cosmic-clash-database
key: dsn
- name: COSMIC_CLASH_REDIS_ADDR
valueFrom:
secretKeyRef:
name: cosmic-clash-redis
key: addr
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: matcher-ranked
namespace: cosmic-clash
labels:
app.kubernetes.io/name: matcher
app.kubernetes.io/component: matcher
cosmic-clash.io/playlist: ranked
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app.kubernetes.io/name: matcher
cosmic-clash.io/playlist: ranked
template:
metadata:
labels:
app.kubernetes.io/name: matcher
app.kubernetes.io/component: matcher
cosmic-clash.io/playlist: ranked
spec:
terminationGracePeriodSeconds: 10
serviceAccountName: matcher
automountServiceAccountToken: false
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
seccompProfile:
type: RuntimeDefault
containers:
- name: matcher
image: ghcr.io/cosmic-clash/matcher@sha256:0000000000000000000000000000000000000000000000000000000000000000
args:
- --dsn=$(COSMIC_CLASH_POSTGRES_DSN)
# Ranked is strictly 3v3; domain.AllocateAcceptedProposal rejects a
# ranked proposal that is not exactly six players.
- --playlist=ranked
- --size=6
- --interval=1s
- --redis-addr=$(COSMIC_CLASH_REDIS_ADDR)
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: [ALL]
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 1
memory: 512Mi
env:
- name: COSMIC_CLASH_POSTGRES_DSN
valueFrom:
secretKeyRef:
name: cosmic-clash-database
key: dsn
- name: COSMIC_CLASH_REDIS_ADDR
valueFrom:
secretKeyRef:
name: cosmic-clash-redis
key: addr
+15
View File
@@ -0,0 +1,15 @@
# Each playlist runs a single matcher, so maxUnavailable rather than
# minAvailable: minAvailable: 1 against a one-replica Deployment blocks every
# voluntary eviction, including node drains. Allowing one keeps drains possible;
# formation simply pauses for the restart, and queued tickets are unaffected
# because the matcher holds no state of its own.
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: matcher
namespace: cosmic-clash
spec:
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: matcher
+84
View File
@@ -26,6 +26,18 @@ spec:
ports: ports:
- protocol: TCP - protocol: TCP
port: 8080 port: 8080
# Allocated game servers are control-plane clients too: roster fetch,
# registration, connection receipts, shutdown acknowledgement and result
# submission all target this port. Their egress was already permitted, but
# without a matching ingress rule every one of those calls was dropped, so
# no allocated match could complete even inside the cluster.
- from:
- podSelector:
matchLabels:
app.kubernetes.io/name: game-server
ports:
- protocol: TCP
port: 8080
egress: egress:
- to: - to:
- namespaceSelector: - namespaceSelector:
@@ -175,3 +187,75 @@ spec:
podSelector: podSelector:
matchLabels: matchLabels:
k8s-app: kube-dns k8s-app: kube-dns
---
# Public players connect straight to the allocated GameServer's UDP port; the
# control plane only ever hands out its address. The namespace-wide default
# deny blocked that ingress entirely, so an allocated server was unreachable
# from the internet and no matchmade game could be joined.
#
# The source cannot be narrowed by selector: these peers are player machines
# outside the cluster. It is narrowed instead to exactly one protocol and port
# on exactly the game-server pods, and the game server admits a peer only with
# a valid signed join authorisation for its own match.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: game-server-allowed-ingress
namespace: cosmic-clash
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: game-server
policyTypes:
- Ingress
ingress:
- ports:
- protocol: UDP
port: 7777
---
# The matcher reads queued candidates and writes proposals. It exposes nothing
# and talks to nobody but its two datastores.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: matcher-allowed-egress
namespace: cosmic-clash
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: matcher
policyTypes:
- Egress
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: data
podSelector:
matchLabels:
app.kubernetes.io/name: postgres
ports:
- protocol: TCP
port: 5432
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: data
podSelector:
matchLabels:
app.kubernetes.io/name: redis
ports:
- protocol: TCP
port: 6379
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
+7
View File
@@ -25,3 +25,10 @@ metadata:
name: maintenance name: maintenance
namespace: cosmic-clash namespace: cosmic-clash
automountServiceAccountToken: false automountServiceAccountToken: false
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: matcher
namespace: cosmic-clash
automountServiceAccountToken: false
@@ -160,6 +160,74 @@ class KubernetesPolicyTest(unittest.TestCase):
self.assertIn(port, maintenance) self.assertIn(port, maintenance)
self.assertNotIn("port: 8080", maintenance) self.assertNotIn("port: 8080", maintenance)
def test_every_required_workload_role_is_deployed(self):
# The base deployed a control-plane image nothing built, and built a
# matcher image nothing deployed -- so applying it produced a cluster
# where tickets could be created but never consumed. Assert the
# advertised topology is actually complete.
kustomization = self.read("kustomization.yaml")
rendered = "".join(
self.read(name.strip("- ").strip())
for name in kustomization.splitlines()
if name.strip().startswith("- ") and name.strip().endswith(".yaml")
)
for role in ("control-plane", "allocator", "maintenance", "matcher"):
self.assertIn(f"app.kubernetes.io/name: {role}", rendered, role)
self.assertIn("kind: Fleet", rendered)
# Casual and ranked must both be scheduled; one matcher process serves
# exactly one playlist.
self.assertIn("name: matcher-casual", rendered)
self.assertIn("name: matcher-ranked", rendered)
self.assertIn("--playlist=casual", rendered)
self.assertIn("--playlist=ranked", rendered)
# Ranked is strictly 3v3; AllocateAcceptedProposal rejects anything else.
self.assertIn("--size=6", rendered)
def test_every_referenced_image_maps_to_a_real_dockerfile_target(self):
dockerfile = (Path(__file__).parents[2] / "Dockerfile").read_text()
targets = set(re.findall(r"(?mi)^FROM\s+.*?\bAS\s+(\S+)\s*$", dockerfile))
# Guard the guard: if the target regex stops matching, every image
# below would "pass" vacuously.
self.assertIn("server", targets)
referenced = set()
for path in sorted(BASE.glob("*.yaml")):
for image in re.findall(r"image:\s*ghcr\.io/cosmic-clash/([\w.-]+)@", path.read_text()):
referenced.add(image)
self.assertTrue(referenced, "no images were found to check")
self.assertEqual(set(), referenced - targets, "manifests reference images this repo cannot build")
# testkit-api injects a fake login provider that accepts any ticket.
self.assertNotIn("testkit-api", referenced)
def test_game_traffic_and_workload_callbacks_are_permitted(self):
policies = self.read("network-policies.yaml")
# Public players reach the allocated server directly over UDP; the
# namespace-wide default deny blocked that entirely.
ingress = policies.split("name: game-server-allowed-ingress", 1)
self.assertEqual(len(ingress), 2, "game-server ingress policy is missing")
game_ingress = ingress[1]
self.assertIn("app.kubernetes.io/name: game-server", game_ingress)
self.assertIn("protocol: UDP", game_ingress)
self.assertIn("port: 7777", game_ingress)
# Game servers are control-plane clients: roster fetch, registration,
# connection receipts, shutdown and result submission. Their egress was
# allowed but the matching control-plane ingress was not.
control_plane = policies.split("name: control-plane-allowed-flows", 1)[-1].split("---", 1)[0]
self.assertIn("app.kubernetes.io/name: game-server", control_plane)
self.assertIn("app.kubernetes.io/name: edge-gateway", control_plane)
# The default deny must survive all of this.
self.assertIn("name: default-deny-ingress-egress", policies)
def test_matcher_network_policy_only_allows_its_datastores_and_dns(self):
policies = self.read("network-policies.yaml")
matcher = policies.split("name: matcher-allowed-egress", 1)[-1]
self.assertIn("app.kubernetes.io/name: matcher", matcher)
for port in ("port: 5432", "port: 6379", "port: 53"):
self.assertIn(port, matcher)
# The matcher never calls the control plane's API.
self.assertNotIn("port: 8080", matcher)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()