jcreek/CosmicClash
1
0
Fork 0
mirror of https://github.com/jcreek/CosmicClash.git synced 2026-09-10 16:04:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(multiplayer): wire allocated fleet runtime

Browse Source
This commit is contained in:
Josh Creek
2026-09-01 16:00:03 +01:00
parent 68e76b5feb
commit c0861bfcad
10 changed files with 196 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/k8s/base/control-plane-service.yaml
+14
View File
@@ -0,0 +1,14 @@
apiVersion: v1
kind: Service
metadata:
name: control-plane
namespace: cosmic-clash
labels:
app.kubernetes.io/name: control-plane
spec:
selector:
app.kubernetes.io/name: control-plane
ports:
- name: http
port: 8080
targetPort: http
deploy/k8s/base/fleet.yaml
+57 -1
View File
@@ -17,6 +17,11 @@ spec:
cosmic-clash.io/build: build-1
cosmic-clash.io/protocol: "1"
cosmic-clash.io/transport: enet
annotations:
# The release process replaces this with the immutable image digest;
# the Downward API passes the same value to the supervisor so the
# allocated child can validate its assignment manifest.
cosmic-clash.io/image-digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
spec:
ports:
- name: game
@@ -49,7 +54,49 @@ spec:
containers:
- name: game-server
image: ghcr.io/cosmic-clash/game-server@sha256:0000000000000000000000000000000000000000000000000000000000000000
args: ["--port=7777"]
args:
- --sdk-base-url=http://127.0.0.1:9357
- --ready-url=http://127.0.0.1:7780/ready
- --drain-url=http://127.0.0.1:7780/drain
- --drain-token-env=COSMIC_CLASH_DRAIN_TOKEN
- --control-plane-url=http://control-plane.cosmic-clash.svc.cluster.local:8080
- --server-id-env=COSMIC_CLASH_SERVER_ID
- --image-digest-env=COSMIC_CLASH_IMAGE_DIGEST
- --roster-path=/run/cosmic-clash/join-roster.json
- --transport=enet
- --
- --allocated-mode
- --match-id=allocation-placeholder
- --server-id=allocation-placeholder
- --playlist-version=casual
- --client-build=build-1
- --assignment-expiry-unix=1
- --server-image-digest=sha256:0000000000000000000000000000000000000000000000000000000000000000
- --transport=enet
- --region=EU
- --join-authorisations-file=/run/cosmic-clash/join-roster.json
- --join-authorisations-key-file=/run/secrets/cosmic-clash/join-signing-key
- --readiness-port=7780
env:
- name: COSMIC_CLASH_SERVER_ID
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: COSMIC_CLASH_IMAGE_DIGEST
valueFrom:
fieldRef:
fieldPath: metadata.annotations['cosmic-clash.io/image-digest']
- name: COSMIC_CLASH_DRAIN_TOKEN
valueFrom:
secretKeyRef:
name: cosmic-clash-game-server
key: drain-token
volumeMounts:
- name: allocated-roster
mountPath: /run/cosmic-clash
- name: join-signing-key
mountPath: /run/secrets/cosmic-clash
readOnly: true
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
@@ -62,3 +109,12 @@ spec:
limits:
cpu: 1
memory: 512Mi
volumes:
- name: allocated-roster
emptyDir: {}
- name: join-signing-key
secret:
secretName: cosmic-clash-game-server
items:
- key: join-signing-key
path: join-signing-key
deploy/k8s/base/kustomization.yaml
+1
View File
@@ -6,6 +6,7 @@ resources:
- rbac.yaml
- network-policies.yaml
- control-plane-deployment.yaml
- control-plane-service.yaml
- fleet.yaml
- fleet-autoscaler.yaml
- game-server-pdb.yaml
deploy/k8s/base/network-policies.yaml
+31
View File
@@ -66,3 +66,34 @@ spec:
podSelector:
matchLabels:
k8s-app: kube-dns
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: game-server-allowed-egress
namespace: cosmic-clash
spec:
podSelector:
matchLabels:
app.kubernetes.io/name: game-server
policyTypes: [Egress]
egress:
- to:
- podSelector:
matchLabels:
app.kubernetes.io/name: control-plane
ports:
- protocol: TCP
port: 8080
- ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
podSelector:
matchLabels:
k8s-app: kube-dns
multiplayer-next.md
+4 -1
View File
File diff suppressed because one or more lines are too long
server/api/service_test.go
+1 -1
View File
@@ -343,7 +343,7 @@ func TestServerRosterRequiresWorkloadBindingAndReturnsRawSignedEnvelopes(t *test
if binding.ServerID != "server-1" || binding.MatchID != "match-1" || !at.Equal(now) {
t.Fatal("unexpected roster binding")
}
return [][]byte{[]byte(`{"authorisation":{"player_id":"player-1"},"signature":"sig"}`)}, nil
return [][]byte{[]byte(`{"authorisation":{"player_id":"player-1","expires_at":"1970-01-01T00:33:20Z"},"signature":"sig"}`)}, nil
},
}
server := httptest.NewServer(service.Handler())
server/security/test_fleet_manifests.py
+22
View File
@@ -19,6 +19,18 @@ class FleetManifestTest(unittest.TestCase):
self.assertIn(label, fleet)
for hardening in ("runAsNonRoot: true", "automountServiceAccountToken: false", "readOnlyRootFilesystem: true", "allowPrivilegeEscalation: false"):
self.assertIn(hardening, fleet)
for runtime in (
"ghcr.io/cosmic-clash/game-server@sha256:",
"--sdk-base-url=http://127.0.0.1:9357",
"--control-plane-url=http://control-plane.cosmic-clash.svc.cluster.local:8080",
"--roster-path=/run/cosmic-clash/join-roster.json",
"--allocated-mode",
"--join-authorisations-key-file=/run/secrets/cosmic-clash/join-signing-key",
"fieldPath: metadata.annotations['cosmic-clash.io/image-digest']",
"secretName: cosmic-clash-game-server",
"emptyDir: {}",
):
self.assertIn(runtime, fleet)
for scheduling in (
"cosmic-clash.io/capacity-type: on-demand",
"topologyKey: topology.kubernetes.io/zone",
@@ -60,6 +72,16 @@ class FleetManifestTest(unittest.TestCase):
self.assertNotIn("namespace: cosmic-clash", base)
self.assertIn("namespace: agones-system", rbac)
def test_control_plane_service_and_game_server_egress_are_declared(self):
service = self.read("base/control-plane-service.yaml")
network = self.read("base/network-policies.yaml")
base = self.read("base/kustomization.yaml")
for field in ("kind: Service", "name: control-plane", "port: 8080", "targetPort: http"):
self.assertIn(field, service)
for field in ("name: game-server-allowed-egress", "app.kubernetes.io/name: game-server", "port: 8080"):
self.assertIn(field, network)
self.assertIn("control-plane-service.yaml", base)
if __name__ == "__main__":
unittest.main()
server/supervisor/supervisor.go
+64 -18
View File
@@ -182,10 +182,12 @@ func (s *Supervisor) Start(ctx context.Context) error {
if s.config.Transport == "steam_sdr" {
env = append(env, "SDR_LISTEN_PORT="+strconv.Itoa(port), "SDR_IP="+address+":"+strconv.Itoa(port))
}
if err := s.fetchRoster(ctx); err != nil {
rosterExpiry, err := s.fetchRoster(ctx)
if err != nil {
return err
}
command := withPort(s.config.Command, port)
command := withAllocatedConfig(s.config.Command, s.matchID(), s.config.ServerID, s.config.ImageDigest, rosterExpiry)
command = withPort(command, port)
s.cmd = exec.CommandContext(ctx, command[0], command[1:]...)
} else {
s.cmd = exec.CommandContext(ctx, s.config.Command[0], s.config.Command[1:]...)
@@ -218,68 +220,112 @@ func (s *Supervisor) Start(ctx context.Context) error {
return nil
}
func (s *Supervisor) fetchRoster(ctx context.Context) error {
func (s *Supervisor) fetchRoster(ctx context.Context) (time.Time, error) {
if s.config.RosterPath == "" {
return nil
return time.Time{}, nil
}
matchID := s.matchID()
if matchID == "" {
return fmt.Errorf("roster fetch has no match ID")
return time.Time{}, fmt.Errorf("roster fetch has no match ID")
}
token, err := s.workloadToken()
if err != nil {
return err
return time.Time{}, err
}
rosterURL := strings.TrimRight(s.config.ControlPlaneURL, "/") + "/v1/servers/" + url.PathEscape(s.config.ServerID) + "/roster"
request, err := http.NewRequestWithContext(ctx, http.MethodGet, rosterURL, nil)
if err != nil {
return err
return time.Time{}, err
}
request.Header.Set("Authorization", "Bearer "+token)
response, err := s.client.Do(request)
if err != nil {
return err
return time.Time{}, err
}
defer response.Body.Close()
if response.StatusCode/100 != 2 {
return fmt.Errorf("control-plane roster returned %s", response.Status)
return time.Time{}, fmt.Errorf("control-plane roster returned %s", response.Status)
}
var roster []json.RawMessage
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&roster); err != nil || len(roster) == 0 {
if err == nil {
err = fmt.Errorf("empty roster")
}
return fmt.Errorf("decode control-plane roster: %w", err)
return time.Time{}, fmt.Errorf("decode control-plane roster: %w", err)
}
var expiry time.Time
for _, envelope := range roster {
if len(envelope) == 0 || string(envelope) == "null" {
return fmt.Errorf("control-plane roster contains an invalid envelope")
return time.Time{}, fmt.Errorf("control-plane roster contains an invalid envelope")
}
var decoded struct {
Authorisation struct {
ExpiresAt time.Time `json:"expires_at"`
} `json:"authorisation"`
}
if err := json.Unmarshal(envelope, &decoded); err != nil || decoded.Authorisation.ExpiresAt.IsZero() {
return time.Time{}, fmt.Errorf("control-plane roster contains an envelope without expiry")
}
if expiry.IsZero() || decoded.Authorisation.ExpiresAt.Before(expiry) {
expiry = decoded.Authorisation.ExpiresAt
}
}
contents, err := json.Marshal(roster)
if err != nil {
return fmt.Errorf("encode roster: %w", err)
return time.Time{}, fmt.Errorf("encode roster: %w", err)
}
directory := filepath.Dir(s.config.RosterPath)
temporary, err := os.CreateTemp(directory, ".cosmic-clash-roster-*")
if err != nil {
return fmt.Errorf("create roster file: %w", err)
return time.Time{}, fmt.Errorf("create roster file: %w", err)
}
temporaryName := temporary.Name()
defer os.Remove(temporaryName)
if err := temporary.Chmod(0600); err == nil {
_, err = temporary.Write(contents)
if err := temporary.Chmod(0600); err != nil {
_ = temporary.Close()
return time.Time{}, fmt.Errorf("secure roster file: %w", err)
}
_, err = temporary.Write(contents)
if closeErr := temporary.Close(); err == nil {
err = closeErr
}
if err != nil {
return fmt.Errorf("write roster file: %w", err)
return time.Time{}, fmt.Errorf("write roster file: %w", err)
}
if err := os.Rename(temporaryName, s.config.RosterPath); err != nil {
return fmt.Errorf("install roster file: %w", err)
return time.Time{}, fmt.Errorf("install roster file: %w", err)
}
return nil
return expiry, nil
}
func withAllocatedConfig(command []string, matchID, serverID, imageDigest string, rosterExpiry time.Time) []string {
result := append([]string(nil), command...)
values := map[string]string{
"match-id": matchID,
"server-id": serverID,
"server-image-digest": imageDigest,
}
if !rosterExpiry.IsZero() {
values["assignment-expiry-unix"] = strconv.FormatInt(rosterExpiry.Unix(), 10)
}
for key, value := range values {
if value == "" {
continue
}
prefix := "--" + key + "="
replaced := false
for i, arg := range result {
if strings.HasPrefix(arg, prefix) {
result[i] = prefix + value
replaced = true
break
}
}
if !replaced {
result = append(result, prefix+value)
}
}
return result
}
// reportAssignmentReady is best-effort: process-ready has already succeeded,
server/supervisor/supervisor_integration_test.go
+1 -1
View File
@@ -80,7 +80,7 @@ func TestRealSupervisorRegistersAllocatedServerThroughControlPlane(t *testing.T)
if err := store.SaveAssignment(ctx, db, store.DurableAssignment{
MatchID: "supervisor-live-match", PlayerID: player, AllocationID: request.AllocationID, ServerID: "supervisor-live-server", Slot: index,
Region: "EU", ClientBuild: "build-1", ProtocolVersion: 1, Transport: "enet", Endpoint: "127.0.0.1:7777",
JoinAuthorisation: base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf(`{"authorisation":{"match_id":"supervisor-live-match","server_id":"supervisor-live-server","player_id":%q},"signature":"sig"}`, player))), ManifestDigest: []byte{0, 1, 2, 3}, ExpiresAt: now.Add(time.Hour), Revision: 1,
JoinAuthorisation: base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf(`{"authorisation":{"match_id":"supervisor-live-match","server_id":"supervisor-live-server","player_id":%q,"expires_at":%q},"signature":"sig"}`, player, now.Add(time.Hour).Format(time.RFC3339)))), ManifestDigest: []byte{0, 1, 2, 3}, ExpiresAt: now.Add(time.Hour), Revision: 1,
}); err != nil {
t.Fatal(err)
}
server/supervisor/supervisor_test.go
+1 -1
View File
@@ -87,7 +87,7 @@ func TestAllocatedStartMaterializesWorkloadAuthenticatedRosterBeforeChild(t *tes
w.WriteHeader(http.StatusUnauthorized)
return
}
_, _ = w.Write([]byte(`[{"authorisation":{"player_id":"player-1"},"signature":"sig"}]`))
_, _ = w.Write([]byte(`[{"authorisation":{"player_id":"player-1","expires_at":"2030-01-01T00:00:00Z"},"signature":"sig"}]`))
return
}
if r.URL.Path == "/v1/servers/server-1/register" {