feat(multiplayer): wire allocated fleet runtime

deploy/k8s/base/control-plane-service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: control-plane
+  namespace: cosmic-clash
+  labels:
+    app.kubernetes.io/name: control-plane
+spec:
+  selector:
+    app.kubernetes.io/name: control-plane
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http

deploy/k8s/base/fleet.yaml

@@ -17,6 +17,11 @@ spec:
         cosmic-clash.io/build: build-1
         cosmic-clash.io/protocol: "1"
         cosmic-clash.io/transport: enet
+      annotations:
+        # The release process replaces this with the immutable image digest;
+        # the Downward API passes the same value to the supervisor so the
+        # allocated child can validate its assignment manifest.
+        cosmic-clash.io/image-digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
     spec:
       ports:
         - name: game
@@ -49,7 +54,49 @@ spec:
           containers:
             - name: game-server
               image: ghcr.io/cosmic-clash/game-server@sha256:0000000000000000000000000000000000000000000000000000000000000000
-              args: ["--port=7777"]
+              args:
+                - --sdk-base-url=http://127.0.0.1:9357
+                - --ready-url=http://127.0.0.1:7780/ready
+                - --drain-url=http://127.0.0.1:7780/drain
+                - --drain-token-env=COSMIC_CLASH_DRAIN_TOKEN
+                - --control-plane-url=http://control-plane.cosmic-clash.svc.cluster.local:8080
+                - --server-id-env=COSMIC_CLASH_SERVER_ID
+                - --image-digest-env=COSMIC_CLASH_IMAGE_DIGEST
+                - --roster-path=/run/cosmic-clash/join-roster.json
+                - --transport=enet
+                - --
+                - --allocated-mode
+                - --match-id=allocation-placeholder
+                - --server-id=allocation-placeholder
+                - --playlist-version=casual
+                - --client-build=build-1
+                - --assignment-expiry-unix=1
+                - --server-image-digest=sha256:0000000000000000000000000000000000000000000000000000000000000000
+                - --transport=enet
+                - --region=EU
+                - --join-authorisations-file=/run/cosmic-clash/join-roster.json
+                - --join-authorisations-key-file=/run/secrets/cosmic-clash/join-signing-key
+                - --readiness-port=7780
+              env:
+                - name: COSMIC_CLASH_SERVER_ID
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: metadata.name
+                - name: COSMIC_CLASH_IMAGE_DIGEST
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: metadata.annotations['cosmic-clash.io/image-digest']
+                - name: COSMIC_CLASH_DRAIN_TOKEN
+                  valueFrom:
+                    secretKeyRef:
+                      name: cosmic-clash-game-server
+                      key: drain-token
+              volumeMounts:
+                - name: allocated-roster
+                  mountPath: /run/cosmic-clash
+                - name: join-signing-key
+                  mountPath: /run/secrets/cosmic-clash
+                  readOnly: true
               securityContext:
                 allowPrivilegeEscalation: false
                 readOnlyRootFilesystem: true
@@ -62,3 +109,12 @@ spec:
                 limits:
                   cpu: 1
                   memory: 512Mi
+          volumes:
+            - name: allocated-roster
+              emptyDir: {}
+            - name: join-signing-key
+              secret:
+                secretName: cosmic-clash-game-server
+                items:
+                  - key: join-signing-key
+                    path: join-signing-key

deploy/k8s/base/kustomization.yaml

@@ -6,6 +6,7 @@ resources:
   - rbac.yaml
   - network-policies.yaml
   - control-plane-deployment.yaml
+  - control-plane-service.yaml
   - fleet.yaml
   - fleet-autoscaler.yaml
   - game-server-pdb.yaml

deploy/k8s/base/network-policies.yaml

@@ -66,3 +66,34 @@ spec:
           podSelector:
             matchLabels:
               k8s-app: kube-dns
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: game-server-allowed-egress
+  namespace: cosmic-clash
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: game-server
+  policyTypes: [Egress]
+  egress:
+    - to:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: control-plane
+      ports:
+        - protocol: TCP
+          port: 8080
+    - ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+      to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+          podSelector:
+            matchLabels:
+              k8s-app: kube-dns

server/api/service_test.go

@@ -343,7 +343,7 @@ func TestServerRosterRequiresWorkloadBindingAndReturnsRawSignedEnvelopes(t *test
 			if binding.ServerID != "server-1" || binding.MatchID != "match-1" || !at.Equal(now) {
 				t.Fatal("unexpected roster binding")
 			}
-			return [][]byte{[]byte(`{"authorisation":{"player_id":"player-1"},"signature":"sig"}`)}, nil
+			return [][]byte{[]byte(`{"authorisation":{"player_id":"player-1","expires_at":"1970-01-01T00:33:20Z"},"signature":"sig"}`)}, nil
 		},
 	}
 	server := httptest.NewServer(service.Handler())

server/security/test_fleet_manifests.py

@@ -19,6 +19,18 @@ class FleetManifestTest(unittest.TestCase):
             self.assertIn(label, fleet)
         for hardening in ("runAsNonRoot: true", "automountServiceAccountToken: false", "readOnlyRootFilesystem: true", "allowPrivilegeEscalation: false"):
             self.assertIn(hardening, fleet)
+        for runtime in (
+            "ghcr.io/cosmic-clash/game-server@sha256:",
+            "--sdk-base-url=http://127.0.0.1:9357",
+            "--control-plane-url=http://control-plane.cosmic-clash.svc.cluster.local:8080",
+            "--roster-path=/run/cosmic-clash/join-roster.json",
+            "--allocated-mode",
+            "--join-authorisations-key-file=/run/secrets/cosmic-clash/join-signing-key",
+            "fieldPath: metadata.annotations['cosmic-clash.io/image-digest']",
+            "secretName: cosmic-clash-game-server",
+            "emptyDir: {}",
+        ):
+            self.assertIn(runtime, fleet)
         for scheduling in (
             "cosmic-clash.io/capacity-type: on-demand",
             "topologyKey: topology.kubernetes.io/zone",
@@ -60,6 +72,16 @@ class FleetManifestTest(unittest.TestCase):
         self.assertNotIn("namespace: cosmic-clash", base)
         self.assertIn("namespace: agones-system", rbac)
 
+    def test_control_plane_service_and_game_server_egress_are_declared(self):
+        service = self.read("base/control-plane-service.yaml")
+        network = self.read("base/network-policies.yaml")
+        base = self.read("base/kustomization.yaml")
+        for field in ("kind: Service", "name: control-plane", "port: 8080", "targetPort: http"):
+            self.assertIn(field, service)
+        for field in ("name: game-server-allowed-egress", "app.kubernetes.io/name: game-server", "port: 8080"):
+            self.assertIn(field, network)
+        self.assertIn("control-plane-service.yaml", base)
+
 
 if __name__ == "__main__":
     unittest.main()

server/supervisor/supervisor.go

@@ -182,10 +182,12 @@ func (s *Supervisor) Start(ctx context.Context) error {
 		if s.config.Transport == "steam_sdr" {
 			env = append(env, "SDR_LISTEN_PORT="+strconv.Itoa(port), "SDR_IP="+address+":"+strconv.Itoa(port))
 		}
-		if err := s.fetchRoster(ctx); err != nil {
+		rosterExpiry, err := s.fetchRoster(ctx)
+		if err != nil {
 			return err
 		}
-		command := withPort(s.config.Command, port)
+		command := withAllocatedConfig(s.config.Command, s.matchID(), s.config.ServerID, s.config.ImageDigest, rosterExpiry)
+		command = withPort(command, port)
 		s.cmd = exec.CommandContext(ctx, command[0], command[1:]...)
 	} else {
 		s.cmd = exec.CommandContext(ctx, s.config.Command[0], s.config.Command[1:]...)
@@ -218,68 +220,112 @@ func (s *Supervisor) Start(ctx context.Context) error {
 	return nil
 }
 
-func (s *Supervisor) fetchRoster(ctx context.Context) error {
+func (s *Supervisor) fetchRoster(ctx context.Context) (time.Time, error) {
 	if s.config.RosterPath == "" {
-		return nil
+		return time.Time{}, nil
 	}
 	matchID := s.matchID()
 	if matchID == "" {
-		return fmt.Errorf("roster fetch has no match ID")
+		return time.Time{}, fmt.Errorf("roster fetch has no match ID")
 	}
 	token, err := s.workloadToken()
 	if err != nil {
-		return err
+		return time.Time{}, err
 	}
 	rosterURL := strings.TrimRight(s.config.ControlPlaneURL, "/") + "/v1/servers/" + url.PathEscape(s.config.ServerID) + "/roster"
 	request, err := http.NewRequestWithContext(ctx, http.MethodGet, rosterURL, nil)
 	if err != nil {
-		return err
+		return time.Time{}, err
 	}
 	request.Header.Set("Authorization", "Bearer "+token)
 	response, err := s.client.Do(request)
 	if err != nil {
-		return err
+		return time.Time{}, err
 	}
 	defer response.Body.Close()
 	if response.StatusCode/100 != 2 {
-		return fmt.Errorf("control-plane roster returned %s", response.Status)
+		return time.Time{}, fmt.Errorf("control-plane roster returned %s", response.Status)
 	}
 	var roster []json.RawMessage
 	if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&roster); err != nil || len(roster) == 0 {
 		if err == nil {
 			err = fmt.Errorf("empty roster")
 		}
-		return fmt.Errorf("decode control-plane roster: %w", err)
+		return time.Time{}, fmt.Errorf("decode control-plane roster: %w", err)
 	}
+	var expiry time.Time
 	for _, envelope := range roster {
 		if len(envelope) == 0 || string(envelope) == "null" {
-			return fmt.Errorf("control-plane roster contains an invalid envelope")
+			return time.Time{}, fmt.Errorf("control-plane roster contains an invalid envelope")
+		}
+		var decoded struct {
+			Authorisation struct {
+				ExpiresAt time.Time `json:"expires_at"`
+			} `json:"authorisation"`
+		}
+		if err := json.Unmarshal(envelope, &decoded); err != nil || decoded.Authorisation.ExpiresAt.IsZero() {
+			return time.Time{}, fmt.Errorf("control-plane roster contains an envelope without expiry")
+		}
+		if expiry.IsZero() || decoded.Authorisation.ExpiresAt.Before(expiry) {
+			expiry = decoded.Authorisation.ExpiresAt
 		}
 	}
 	contents, err := json.Marshal(roster)
 	if err != nil {
-		return fmt.Errorf("encode roster: %w", err)
+		return time.Time{}, fmt.Errorf("encode roster: %w", err)
 	}
 	directory := filepath.Dir(s.config.RosterPath)
 	temporary, err := os.CreateTemp(directory, ".cosmic-clash-roster-*")
 	if err != nil {
-		return fmt.Errorf("create roster file: %w", err)
+		return time.Time{}, fmt.Errorf("create roster file: %w", err)
 	}
 	temporaryName := temporary.Name()
 	defer os.Remove(temporaryName)
-	if err := temporary.Chmod(0600); err == nil {
+	if err := temporary.Chmod(0600); err != nil {
-		_, err = temporary.Write(contents)
+		_ = temporary.Close()
+		return time.Time{}, fmt.Errorf("secure roster file: %w", err)
 	}
+	_, err = temporary.Write(contents)
 	if closeErr := temporary.Close(); err == nil {
 		err = closeErr
 	}
 	if err != nil {
-		return fmt.Errorf("write roster file: %w", err)
+		return time.Time{}, fmt.Errorf("write roster file: %w", err)
 	}
 	if err := os.Rename(temporaryName, s.config.RosterPath); err != nil {
-		return fmt.Errorf("install roster file: %w", err)
+		return time.Time{}, fmt.Errorf("install roster file: %w", err)
 	}
-	return nil
+	return expiry, nil
+}
+
+func withAllocatedConfig(command []string, matchID, serverID, imageDigest string, rosterExpiry time.Time) []string {
+	result := append([]string(nil), command...)
+	values := map[string]string{
+		"match-id":            matchID,
+		"server-id":           serverID,
+		"server-image-digest": imageDigest,
+	}
+	if !rosterExpiry.IsZero() {
+		values["assignment-expiry-unix"] = strconv.FormatInt(rosterExpiry.Unix(), 10)
+	}
+	for key, value := range values {
+		if value == "" {
+			continue
+		}
+		prefix := "--" + key + "="
+		replaced := false
+		for i, arg := range result {
+			if strings.HasPrefix(arg, prefix) {
+				result[i] = prefix + value
+				replaced = true
+				break
+			}
+		}
+		if !replaced {
+			result = append(result, prefix+value)
+		}
+	}
+	return result
 }
 
 // reportAssignmentReady is best-effort: process-ready has already succeeded,

server/supervisor/supervisor_integration_test.go

@@ -80,7 +80,7 @@ func TestRealSupervisorRegistersAllocatedServerThroughControlPlane(t *testing.T)
 		if err := store.SaveAssignment(ctx, db, store.DurableAssignment{
 			MatchID: "supervisor-live-match", PlayerID: player, AllocationID: request.AllocationID, ServerID: "supervisor-live-server", Slot: index,
 			Region: "EU", ClientBuild: "build-1", ProtocolVersion: 1, Transport: "enet", Endpoint: "127.0.0.1:7777",
-			JoinAuthorisation: base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf(`{"authorisation":{"match_id":"supervisor-live-match","server_id":"supervisor-live-server","player_id":%q},"signature":"sig"}`, player))), ManifestDigest: []byte{0, 1, 2, 3}, ExpiresAt: now.Add(time.Hour), Revision: 1,
+			JoinAuthorisation: base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf(`{"authorisation":{"match_id":"supervisor-live-match","server_id":"supervisor-live-server","player_id":%q,"expires_at":%q},"signature":"sig"}`, player, now.Add(time.Hour).Format(time.RFC3339)))), ManifestDigest: []byte{0, 1, 2, 3}, ExpiresAt: now.Add(time.Hour), Revision: 1,
 		}); err != nil {
 			t.Fatal(err)
 		}

server/supervisor/supervisor_test.go

@@ -87,7 +87,7 @@ func TestAllocatedStartMaterializesWorkloadAuthenticatedRosterBeforeChild(t *tes
 				w.WriteHeader(http.StatusUnauthorized)
 				return
 			}
-			_, _ = w.Write([]byte(`[{"authorisation":{"player_id":"player-1"},"signature":"sig"}]`))
+			_, _ = w.Write([]byte(`[{"authorisation":{"player_id":"player-1","expires_at":"2030-01-01T00:00:00Z"},"signature":"sig"}]`))
 			return
 		}
 		if r.URL.Path == "/v1/servers/server-1/register" {