fix(agones): make the kind gate's Agones lifecycle actually work

Several independent causes, all of which had to be right before the
Fleet could reach Ready.

The supervisor pointed --sdk-base-url at 127.0.0.1:9357, which is the
Agones sidecar's gRPC port; its HTTP surface is 9358, and that is what
AGONES_SDK_HTTP_PORT carries and what agones_sdk.gd reads. An HTTP
client against the gRPC port could never have worked, in kind or in
production.

The supervisor also treated the sidecar's first incomplete /gameserver
response as fatal. The sidecar accepts requests before the controller
populates status.address and status.ports, so this produced a restart
loop precisely during normal Agones startup. It now polls until the
endpoint is assigned or ReadyTimeout elapses.

server_boot.gd started ServerControl and the Agones SDK only under
--allocated-mode, but the kind smoke deliberately strips that flag, so
nothing served the readiness probe and the GameServer could never become
Ready. Lifecycle now keys on AGONES_SDK_HTTP_PORT, which Agones injects
into every managed container, while allocation and roster semantics stay
tied to --allocated-mode. The SDK node is added to the tree
non-deferred, since start_health() creates a Timer immediately.

Fleet: Agones assigns its own SDK service account and masks that token
from the game container while keeping it for the injected sidecar, so
the manifest must not pin serviceAccountName or
automountServiceAccountToken. Godot stores user:// under HOME, so HOME
points at the writable runtime volume to keep the root filesystem
read-only, and fsGroup makes that volume writable for the non-root user.

Namespace: Agones' Dynamic port policy injects a hostPort, which both
the baseline and restricted Pod Security Standards forbid, so the
workload namespace enforces privileged while continuing to audit and
warn against restricted.

NetworkPolicy: the injected sidecar reaches the Kubernetes API over
HTTPS, and NetworkPolicy applies to the whole Pod rather than to the
container whose token was masked.

The kind runner creates the namespace before Helm so Agones can install
its per-namespace SDK RBAC, scopes gameservers.namespaces to it, forces
the allocator and ping Services to ClusterIP because LoadBalancer
ingress never becomes ready in plain kind, and labels the node so the
production Fleet's on-demand/zone constraints are exercised rather than
edited out of the rendered manifest.
This commit is contained in:
Josh Creek
2026-09-05 20:50:01 +01:00
parent 8aa4af3a3a
commit ca70568fad
11 changed files with 159 additions and 40 deletions
+22 -4
View File
@@ -18,11 +18,19 @@ class FleetManifestTest(unittest.TestCase):
"protocol: UDP", "containerPort: 7777", "replicas: 2",
):
self.assertIn(label, fleet)
for hardening in ("runAsNonRoot: true", "automountServiceAccountToken: false", "readOnlyRootFilesystem: true", "allowPrivilegeEscalation: false"):
for hardening in (
"runAsNonRoot: true", "readOnlyRootFilesystem: true",
"allowPrivilegeEscalation: false", "fsGroup: 10001",
"name: HOME", "value: /run/cosmic-clash",
):
self.assertIn(hardening, fleet)
# Agones must assign its SDK service account so it can keep the token
# available to its injected sidecar while masking it from the game.
self.assertNotIn("serviceAccountName:", fleet)
self.assertNotIn("automountServiceAccountToken:", fleet)
for runtime in (
"ghcr.io/cosmic-clash/game-server@sha256:",
"--sdk-base-url=http://127.0.0.1:9357",
"--sdk-base-url=http://127.0.0.1:9358",
"--control-plane-url=http://control-plane.cosmic-clash.svc.cluster.local:8080",
"--protocol-version=1",
"/opt/cosmic-clash/CosmicClashServer.x86_64",
@@ -88,13 +96,23 @@ class FleetManifestTest(unittest.TestCase):
base = self.read("base/kustomization.yaml")
for field in ("kind: Service", "name: control-plane", "port: 8080", "targetPort: http"):
self.assertIn(field, service)
for field in ("name: game-server-allowed-egress", "app.kubernetes.io/name: game-server", "port: 8080"):
self.assertIn(field, network)
game_server_egress = network.split("name: game-server-allowed-egress", 1)[-1].split("---", 1)[0]
for field in ("app.kubernetes.io/name: game-server", "port: 8080", "port: 443"):
self.assertIn(field, game_server_egress)
self.assertIn("control-plane-service.yaml", base)
def test_kind_runner_is_explicitly_separate_from_production_roster_flow(self):
runner = (ROOT / "scripts/verify_kind_agones.sh").read_text()
self.assertIn("Agones lifecycle smoke", runner)
self.assertIn("gameservers.namespaces[0]=cosmic-clash", runner)
for service in (
"agones.allocator.service.serviceType=ClusterIP",
"agones.ping.http.serviceType=ClusterIP",
"agones.ping.udp.serviceType=ClusterIP",
):
self.assertIn(service, runner)
self.assertIn("cosmic-clash.io/capacity-type=on-demand", runner)
self.assertIn("topology.kubernetes.io/zone=kind-smoke", runner)
self.assertIn("--control-plane-url=", runner)
self.assertIn("--allocated-mode", runner)
validator = (ROOT / "scripts/verify_agones_allocation_response.py").read_text()
+4 -2
View File
@@ -10,10 +10,12 @@ class KubernetesPolicyTest(unittest.TestCase):
def read(self, name):
return (BASE / name).read_text()
def test_namespace_enforces_restricted_pod_security(self):
def test_namespace_allows_agones_host_ports_and_audits_restricted(self):
namespace = self.read("namespace.yaml")
for key in ("enforce", "audit", "warn"):
self.assertIn("pod-security.kubernetes.io/enforce: privileged", namespace)
for key in ("audit", "warn"):
self.assertIn(f"pod-security.kubernetes.io/{key}: restricted", namespace)
self.assertIn("Agones' Dynamic port policy", namespace)
def test_workload_is_non_root_immutable_and_unprivileged(self):
deployment = self.read("control-plane-deployment.yaml")
+25 -1
View File
@@ -199,7 +199,7 @@ func (s *Supervisor) Start(ctx context.Context) error {
env := append([]string(nil), os.Environ()...)
env = append(env, s.config.Environment...)
if s.config.SDKBaseURL != "" {
port, address, err := s.assignedEndpoint(ctx)
port, address, err := s.waitAssignedEndpoint(ctx)
if err != nil {
return err
}
@@ -269,6 +269,30 @@ func (s *Supervisor) Start(ctx context.Context) error {
return nil
}
// waitAssignedEndpoint covers the short interval between the SDK sidecar
// accepting requests and the GameServer controller populating status.address
// and status.ports. Treating the sidecar's first incomplete response as fatal
// creates a restart loop precisely while Agones is finishing normal startup.
func (s *Supervisor) waitAssignedEndpoint(ctx context.Context) (int, string, error) {
deadline := time.NewTimer(s.config.ReadyTimeout)
defer deadline.Stop()
var lastErr error
for {
port, address, err := s.assignedEndpoint(ctx)
if err == nil {
return port, address, nil
}
lastErr = err
select {
case <-ctx.Done():
return 0, "", ctx.Err()
case <-deadline.C:
return 0, "", fmt.Errorf("assigned endpoint timed out: %w", lastErr)
case <-time.After(s.config.PollInterval):
}
}
}
func (s *Supervisor) signalInitialConnectReady(ctx context.Context) error {
if s.config.AdmissionURL == "" {
return nil
+38
View File
@@ -165,6 +165,44 @@ func TestAllocatedStartInjectsDynamicEndpointAndCallsReadyAfterProbe(t *testing.
}
}
func TestAllocatedStartWaitsForAgonesToAssignEndpoint(t *testing.T) {
requests := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/gameserver":
requests++
if requests == 1 {
_, _ = w.Write([]byte(`{"status":{}}`))
return
}
_, _ = w.Write([]byte(`{"status":{"address":"203.0.113.9","ports":[{"name":"game","port":31001}]}}`))
case "/ready-probe", "/ready":
w.WriteHeader(http.StatusOK)
default:
w.WriteHeader(http.StatusNotFound)
}
}))
defer server.Close()
s, err := New(Config{
Command: []string{"/bin/sh", "-c", "true"}, SDKBaseURL: server.URL,
ReadyURL: server.URL + "/ready-probe", ReadyTimeout: time.Second,
PollInterval: time.Millisecond,
})
if err != nil {
t.Fatal(err)
}
if err := s.Start(context.Background()); err != nil {
t.Fatal(err)
}
if err := s.Wait(); err != nil {
t.Fatal(err)
}
if requests < 2 {
t.Fatalf("gameserver requests = %d, want at least 2", requests)
}
}
func TestAllocatedStartMaterializesWorkloadAuthenticatedRosterBeforeChild(t *testing.T) {
rosterPath := filepath.Join(t.TempDir(), "join-roster.json")
sdk := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {