diff --git a/deploy/k8s/base/allocator-deployment.yaml b/deploy/k8s/base/allocator-deployment.yaml index 537a3a60..87e3e203 100644 --- a/deploy/k8s/base/allocator-deployment.yaml +++ b/deploy/k8s/base/allocator-deployment.yaml @@ -24,7 +24,9 @@ spec: spec: terminationGracePeriodSeconds: 10 serviceAccountName: allocator - automountServiceAccountToken: false + # This role calls Agones CRDs through the Kubernetes API. The client + # rereads the short-lived projected token on every request. + automountServiceAccountToken: true topologySpreadConstraints: - maxSkew: 1 topologyKey: topology.kubernetes.io/zone @@ -52,8 +54,9 @@ spec: image: ghcr.io/cosmic-clash/allocator@sha256:0000000000000000000000000000000000000000000000000000000000000000 args: - --dsn=$(COSMIC_CLASH_POSTGRES_DSN) - - --agones-url=https://agones-allocator.agones-system.svc.cluster.local + - --agones-url=https://kubernetes.default.svc - --agones-namespace=cosmic-clash + - --provider-timeout=10s - --metrics-addr=:9091 ports: - name: metrics diff --git a/deploy/k8s/base/network-policies.yaml b/deploy/k8s/base/network-policies.yaml index a76ece70..a8df97fd 100644 --- a/deploy/k8s/base/network-policies.yaml +++ b/deploy/k8s/base/network-policies.yaml @@ -47,13 +47,6 @@ spec: ports: - protocol: TCP port: 6379 - - to: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: agones-system - ports: - - protocol: TCP - port: 443 - ports: - protocol: UDP port: 53 @@ -130,11 +123,10 @@ spec: ports: - protocol: TCP port: 5432 - - to: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: agones-system - ports: + # The kubernetes.default Service endpoint is implementation-specific and + # may be a control-plane/node IP that cannot be selected by pod labels. + # Keep API egress portable while limiting it to TLS only. + - ports: - protocol: TCP port: 443 - ports: diff --git a/deploy/k8s/base/rbac.yaml b/deploy/k8s/base/rbac.yaml index f440c7ad..059c5735 100644 --- a/deploy/k8s/base/rbac.yaml +++ b/deploy/k8s/base/rbac.yaml @@ -1,9 +1,12 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: control-plane-agones-allocator - namespace: agones-system + name: allocator-agones-api + namespace: cosmic-clash rules: + - apiGroups: ["agones.dev"] + resources: ["gameservers"] + verbs: ["list"] - apiGroups: ["allocation.agones.dev"] resources: ["gameserverallocations"] verbs: ["create"] @@ -11,14 +14,13 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: cosmic-clash-control-plane-agones-allocator - namespace: agones-system + name: allocator-agones-api + namespace: cosmic-clash subjects: - kind: ServiceAccount - name: control-plane + name: allocator namespace: cosmic-clash roleRef: kind: Role - name: control-plane-agones-allocator + name: allocator-agones-api apiGroup: rbac.authorization.k8s.io - diff --git a/multiplayer-next.md b/multiplayer-next.md index af847673..5953755b 100644 --- a/multiplayer-next.md +++ b/multiplayer-next.md @@ -1218,7 +1218,7 @@ the local/CI/community transport, not a silent production fallback. | # | Task | Acceptance | |---|---|---| -| 8.26 `[D:8.1,8.6,8.12]` | **IN PROGRESS.** Provider-neutral Kustomize base now defines a restricted Agones Fleet with region/build/protocol/transport labels and UDP game port, plus distinct EU/NA overlays; the base avoids rewriting cross-namespace Agones RBAC and now wires the digest-pinned supervisor image, control-plane Service, dynamic roster volume, signing/drain secret references and required network flow | `deploy/k8s/base/fleet.yaml`, `control-plane-service.yaml`, `network-policies.yaml`, `overlays/eu`, `overlays/na` and `server/security/test_fleet_manifests.py` cover labels, replica floor, UDP declaration, pod hardening, supervisor/runtime arguments, Service selection, egress policy, overlay distinction, Kustomize rendering and RBAC namespace safety; operator secret/image replacement, second-provider fixtures, edge/DNS and SDR POP/cert/public-UDP overlays remain | +| 8.26 `[D:8.1,8.6,8.12]` | **IN PROGRESS.** Provider-neutral Kustomize base now defines a restricted Agones Fleet with region/build/protocol/transport labels and UDP game port, plus distinct EU/NA overlays; the base grants the allocator only namespaced Agones CRD access through the Kubernetes API and wires the digest-pinned supervisor image, control-plane Service, dynamic roster volume, signing/drain secret references and required network flow | `deploy/k8s/base/fleet.yaml`, `control-plane-service.yaml`, `network-policies.yaml`, `rbac.yaml`, `overlays/eu`, `overlays/na` and the manifest policy tests cover labels, replica floor, UDP declaration, pod hardening, supervisor/runtime arguments, Service selection, egress policy, overlay distinction, Kustomize rendering and allocator-only RBAC; operator secret/image replacement, second-provider fixtures, edge/DNS and SDR POP/cert/public-UDP overlays remain | | 8.27 `[D:8.26]` | **IN PROGRESS.** Go supervisor package provides local-safe Agones REST discovery, validates assigned address/port data, injects dynamic `SDR_LISTEN_PORT`/`SDR_IP`, performs explicit process-ready probing and Ready transition; direct mode bypasses Agones | `server/supervisor/` covers allocated/direct startup, invalid endpoint rejection, dynamic endpoint/Ready ordering and authenticated drain; allocated Godot now supplies a loopback readiness/drain control surface and `agones_sdk.gd` supplies sidecar Health/Ready/Shutdown/annotation REST operations; metadata watch, real Agones annotation/shutdown confirmation and emulator integration remain | | 8.28 `[D:8.6,8.27]` | **IN PROGRESS.** Supervisor separates explicit process-ready from Agones Ready and never scrapes stdout; allocated mode refuses to mark Ready without a configured readiness probe. The durable control-plane counterpart now exists: a workload-authenticated `POST /v1/servers/{id}/register` (and its `/api/v1` contract alias) advances a match's `ALLOCATING -> PROCESS_READY -> ASSIGNMENT_READY` state, and every participant's queue ticket with it, as one idempotent SERIALIZABLE transaction, gated on every participant already holding a live, unexpired assignment. The supervisor now calls it: once Agones Ready succeeds, it POSTs process-ready (`assignment_ready=false`) using a workload token read fresh from disk each call (matching kubelet's in-place rotation of a projected service-account token), and a registration failure kills the child rather than leaving an Agones-Ready-but-control-plane-unregistered process running; `ControlPlaneURL` unset (the default) is a total no-op. It then reports assignment-ready too: `server_boot.gd` already verifies its mounted roster synchronously before `/ready` is ever exposed (so process-ready implies the roster was valid), and the API's `ASSIGNMENT_READY` gate checks only durable `assignments` rows server-side — so no new Godot-side state was needed, correcting an earlier overcautious note here. The supervisor retries assignment-ready (default 5 attempts, 2s apart, configurable) since those durable rows may lag process-ready slightly; a persistent failure there does not kill the child, unlike process-ready. Per-allocation data (currently `match-id`) now has a real channel to an already-Ready pod: `server/agones.Client.Allocate` requests `cosmic-clash.io/match-id`/`cosmic-clash.io/allocation-id` as `GameServerAllocation.spec.metadata.annotations` (Agones applies these to the allocated GameServer's own `object_meta` — the only channel that exists post-allocation, since env vars are fixed at pod creation), and the supervisor reads them back from its existing `/gameserver` SDK call, falling back to them only when `MatchID` isn't explicitly configured. The image now exists: a new `Dockerfile` `game-server` target packages the supervisor as ENTRYPOINT alongside the same dedicated-server export `server` produces; the base Fleet now invokes that target with the control-plane URL, server/image Downward API identity, roster/signing/drain material, and exported Godot executable. | `server/supervisor/` tests prove Ready follows the probe and direct mode remains functional; `server_control.gd`, `agones_sdk.gd` and process-level smokes prove loopback `/ready`, `/health`, bearer-protected `/drain`, sidecar-shaped Health/Ready calls and drain admission fencing; `server/api/service.go`, `server/store/allocation_match_sql.go` and adversarial tests cover the registration route, digest/protocol validation, idempotent replay/conflict and the assignment-count gate; `server/supervisor/supervisor_test.go` covers opt-in registration, the workload-token/body/idempotency-key shape, the kill-on-failure path, the match-ID annotation fallback (both that it's used and that its absence fails closed before any HTTP call), the full process-ready-then-assignment-ready sequence and body shapes, and a retry test that fails assignment-ready twice with 409 before succeeding, asserting `Start()` still succeeds and the child is never killed; `server/agones/allocation_test.go` covers the requested annotations. `docker build --target game-server` verified for real: both binaries present, correct permissions, supervisor prints its usage; `server/store/stalled_allocation_sql.go`/`_test.go` and a live `TestPostgreSQLStalledAllocationsAreReclaimedWithoutPenalisingPlayers` cover the deadline boundary (a recent match must survive untouched), the no-penalty requeue and refreshed expiry, participant release, and idempotence against a second pass. **Fixed in passing**: `Dockerfile`'s `server` stage's `ubuntu` base digest had gone dead on Docker Hub (`docker pull` returned "not found", verified independently) — `make verify-phase6` was silently broken for a clean build before the re-pin; confirmed fixed with a full `make verify-phase6` run (arenas rotated, both goals observed, clean teardown). `deploy/cosmic-clash-server` now wraps its exec in `stdbuf -oL -eL` (falling back to unwrapped if unavailable), fixing a real, live-confirmed bug: a genuinely detached (`docker run -d`) container showed zero `docker logs` output — not even the startup line — for 20+ seconds while the process ran normally, and `docker stop`'s SIGTERM lost that buffered output permanently rather than delaying it; re-verified fixed against the real launcher script, then a full `make verify-phase6` re-run confirmed no regression. Health-reclaim now exists: `store.ExpireStalledAllocations` reclaims a match stuck in `ALLOCATING`/`PROCESS_READY`/`ASSIGNMENT_READY` past a deadline (server crashed, or was reclaimed by Agones as unhealthy, before ever registering) by failing the match and requeuing every participant to `QUEUED` with a fresh expiry rather than penalising them — task 8.50's own "infrastructure-caused cases cannot penalise affected players" criterion directly settles the requeue-vs-fail design question this had been blocked on. Wired into `cmd/maintenance` alongside the season-rollover sweep (`--stalled-allocation-deadline` default 2m, `--stalled-allocation-batch`). **Superseding the `fleet.yaml` framing above**: §8.10's `WorkloadVerify` blocker, and its delivery channel, are both now closed — a control-plane-self-issued signed token (not a Kubernetes JWT), minted by `cmd/allocator` into a `cosmic-clash.io/workload-token` annotation and read back by the supervisor, exactly the way `match-id` already worked — see §8.10. `/register` and `/result` no longer 503 unconditionally once every `--workload-secret` (control plane, allocator) is set consistently. The remaining gates are live Agones annotation/shutdown behavior and production cluster readiness; those are covered by §8.49 and remain explicitly open. | | 8.29 `[D:8.26,8.27]` | **IN PROGRESS.** Supervisor discovers and validates the Agones endpoint, propagates the actual dynamic `--port`, and exports `SDR_LISTEN_PORT`/`SDR_IP` only for Hosted-SDR while preserving an isolated ENet path | `server/supervisor/` tests cover invalid address/port rejection, dynamic port argument/env propagation and SDR-vs-ENet separation; real Agones dynamic/passthrough mapping, POP/cert/firewall/NAT and multi-match fixture remain | @@ -1634,3 +1634,5 @@ The video-settings TODO is likewise locally implemented: presets, vsync, refresh Assignment handoff now has a non-circular recovery path. Match-scoped lifecycle events are no longer misapplied as queue-ticket resources: they trigger owner-scoped ticket recovery, and recovered active tickets include their durable `match_id`. An `ASSIGNMENT_READY` event or recovered ticket can therefore drive `GET /assignments/{matchId}` without already having fetched that assignment. Owner-scoped REST ticket snapshots may cross missed revisions only along a reachable forward lifecycle path, while incremental WebSocket updates remain strictly contiguous and neither path can rewind state. The OpenAPI queue projection includes the optional active match identity, Go tests cover the store/API projection, and the 199-test Godot harness covers match-resource separation, malformed identities, missed-revision recovery, illegal rewinds, and assignment-fetch scheduling. The real PostgreSQL assertion is committed with the store integration suite; rerunning it in this workspace is temporarily blocked by Docker storage exhaustion (`initdb` cannot create `pg_wal`), so live SQL evidence remains open rather than being claimed from the static/unit gates. Replica-independent client convergence now supersedes the earlier "at-least-once WebSocket delivery" wording in tasks 8.25/8.40 and the allocation-outbox progress notes. The database outbox guarantees ordered, replayable invocation of a replica's transient publication adapter, not receipt by a socket that may be absent or attached to another replica. Active clients now perform bounded five-second owner-scoped REST recovery; ticket recovery exposes the active `proposal_id` or `match_id`, so a missed proposal, allocation, assignment, or result notification cannot strand the client without the next resource key. A terminal proposal can be replaced by a later recovered proposal identity, while an open proposal cannot be overwritten. Network and malformed-JSON failures during recovery remain visible and retryable instead of falsely terminating matchmaking. WebSocket events remain the low-latency path; REST snapshots are the correctness path. Store/API and the 200-test Godot harness cover projection, transient failure, replacement, and hostile identity/state combinations, with live PostgreSQL execution still subject to the Docker storage gate recorded above. + +The production allocator now uses the API it actually implements: Kubernetes custom-resource paths at `https://kubernetes.default.svc`, rather than sending those paths to the distinct mTLS Agones Allocator Service. Its HTTPS client trusts the mounted cluster CA, rereads the projected service-account token for every request so rotation is honored, applies a ten-second request timeout, and refuses to forward the credential to another origin. The allocator pod explicitly mounts its token; namespaced RBAC permits only GameServer `list` and GameServerAllocation `create`; and its default-deny policy permits portable API-server egress only on TCP 443. Focused Go/auth, static policy, and `kubectl kustomize` checks pass. The real kind/Agones runtime gate remains open because kind and Helm are unavailable here and Docker storage is exhausted; no live-cluster success is claimed. diff --git a/server/agones/kubernetes_client.go b/server/agones/kubernetes_client.go new file mode 100644 index 00000000..b0088580 --- /dev/null +++ b/server/agones/kubernetes_client.go @@ -0,0 +1,67 @@ +package agones + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +// NewKubernetesHTTPClient builds an in-cluster client for the Kubernetes API. +// The bearer token is read for every request so kubelet token rotation does not +// leave a long-running allocator with an expired credential. +func NewKubernetesHTTPClient(baseURL, tokenPath, caPath string, timeout time.Duration) (*http.Client, error) { + origin, err := url.Parse(baseURL) + if err != nil || origin.Scheme != "https" || origin.Host == "" || origin.User != nil || origin.Path != "" || origin.RawQuery != "" || origin.Fragment != "" { + return nil, fmt.Errorf("Kubernetes API base URL must be an HTTPS origin") + } + if strings.TrimSpace(tokenPath) == "" || strings.TrimSpace(caPath) == "" || timeout <= 0 { + return nil, fmt.Errorf("Kubernetes API token path, CA path, and positive timeout are required") + } + caPEM, err := os.ReadFile(caPath) + if err != nil { + return nil, fmt.Errorf("read Kubernetes API CA: %w", err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("Kubernetes API CA contains no certificates") + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12} + return &http.Client{ + Timeout: timeout, + Transport: bearerTokenTransport{ + tokenPath: tokenPath, + expectedOrigin: origin.Scheme + "://" + origin.Host, + base: transport, + }, + }, nil +} + +type bearerTokenTransport struct { + tokenPath string + expectedOrigin string + base http.RoundTripper +} + +func (t bearerTokenTransport) RoundTrip(request *http.Request) (*http.Response, error) { + if request.URL.Scheme+"://"+request.URL.Host != t.expectedOrigin { + return nil, fmt.Errorf("refusing to send Kubernetes API credential to unexpected origin") + } + tokenBytes, err := os.ReadFile(t.tokenPath) + if err != nil { + return nil, fmt.Errorf("read Kubernetes API bearer token: %w", err) + } + token := strings.TrimSpace(string(tokenBytes)) + if token == "" || strings.ContainsAny(token, " \t\r\n") { + return nil, fmt.Errorf("Kubernetes API bearer token is empty or malformed") + } + cloned := request.Clone(request.Context()) + cloned.Header = request.Header.Clone() + cloned.Header.Set("Authorization", "Bearer "+token) + return t.base.RoundTrip(cloned) +} diff --git a/server/agones/kubernetes_client_test.go b/server/agones/kubernetes_client_test.go new file mode 100644 index 00000000..87c33142 --- /dev/null +++ b/server/agones/kubernetes_client_test.go @@ -0,0 +1,125 @@ +package agones + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" +) + +func TestKubernetesHTTPClientTrustsCAAddsAndRotatesBearerToken(t *testing.T) { + var seen []string + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = append(seen, r.Header.Get("Authorization")) + w.WriteHeader(http.StatusNoContent) + })) + server.TLS = testTLSConfig(t) + server.StartTLS() + defer server.Close() + + directory := t.TempDir() + caPath := filepath.Join(directory, "ca.crt") + tokenPath := filepath.Join(directory, "token") + certificate := server.Certificate() + if err := os.WriteFile(caPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificate.Raw}), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(tokenPath, []byte("first-token\n"), 0o600); err != nil { + t.Fatal(err) + } + client, err := NewKubernetesHTTPClient(server.URL, tokenPath, caPath, time.Second) + if err != nil { + t.Fatal(err) + } + for _, token := range []string{"first-token", "rotated-token"} { + if err := os.WriteFile(tokenPath, []byte(token), 0o600); err != nil { + t.Fatal(err) + } + response, err := client.Get(server.URL) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + } + if len(seen) != 2 || seen[0] != "Bearer first-token" || seen[1] != "Bearer rotated-token" { + t.Fatalf("authorization headers = %v", seen) + } +} + +func TestKubernetesHTTPClientRejectsInvalidConfigurationAndToken(t *testing.T) { + directory := t.TempDir() + caPath := filepath.Join(directory, "ca.crt") + tokenPath := filepath.Join(directory, "token") + if _, err := NewKubernetesHTTPClient("http://kubernetes.default.svc", tokenPath, caPath, time.Second); err == nil { + t.Fatal("non-TLS API origin accepted") + } + if _, err := NewKubernetesHTTPClient("https://kubernetes.default.svc", tokenPath, caPath, time.Second); err == nil { + t.Fatal("missing CA accepted") + } + if err := os.WriteFile(caPath, []byte("not a certificate"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := NewKubernetesHTTPClient("https://kubernetes.default.svc", tokenPath, caPath, time.Second); err == nil { + t.Fatal("invalid CA accepted") + } +} + +func TestKubernetesHTTPClientDoesNotForwardCredentialAcrossOrigins(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + defer server.Close() + directory := t.TempDir() + caPath := filepath.Join(directory, "ca.crt") + tokenPath := filepath.Join(directory, "token") + if err := os.WriteFile(caPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(tokenPath, []byte("secret-token"), 0o600); err != nil { + t.Fatal(err) + } + client, err := NewKubernetesHTTPClient(server.URL, tokenPath, caPath, time.Second) + if err != nil { + t.Fatal(err) + } + if _, err := client.Get("https://example.invalid/"); err == nil { + t.Fatal("credentialed request to another origin was not rejected") + } +} + +func testTLSConfig(t *testing.T) *tls.Config { + t.Helper() + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "127.0.0.1"}, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + } + der, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + if err != nil { + t.Fatal(err) + } + certificate, err := tls.X509KeyPair( + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), + pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)}), + ) + if err != nil { + t.Fatal(err) + } + return &tls.Config{Certificates: []tls.Certificate{certificate}, MinVersion: tls.VersionTLS12} +} diff --git a/server/cmd/allocator/main.go b/server/cmd/allocator/main.go index 8754e9b5..f490d190 100644 --- a/server/cmd/allocator/main.go +++ b/server/cmd/allocator/main.go @@ -23,6 +23,9 @@ func main() { migrationDir := flag.String("migrations", "migrations", "directory containing numbered SQL migrations") agonesURL := flag.String("agones-url", os.Getenv("COSMIC_CLASH_AGONES_URL"), "Agones allocation API base URL") namespace := flag.String("agones-namespace", envOrDefault("COSMIC_CLASH_AGONES_NAMESPACE", "default"), "Agones namespace") + kubernetesTokenPath := flag.String("kubernetes-token-path", envOrDefault("COSMIC_CLASH_KUBERNETES_TOKEN_PATH", "/var/run/secrets/kubernetes.io/serviceaccount/token"), "rotating Kubernetes service-account bearer token") + kubernetesCAPath := flag.String("kubernetes-ca-path", envOrDefault("COSMIC_CLASH_KUBERNETES_CA_PATH", "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"), "Kubernetes API cluster CA bundle") + providerTimeout := flag.Duration("provider-timeout", 10*time.Second, "timeout for each Kubernetes/Agones API request") transport := flag.String("transport", envOrDefault("COSMIC_CLASH_TRANSPORT", "enet"), "game transport: enet or steam_sdr") interval := flag.Duration("interval", time.Second, "allocation poll interval") workloadSecret := flag.String("workload-secret", os.Getenv("COSMIC_CLASH_WORKLOAD_SECRET"), "HMAC secret for control-plane-issued workload tokens (see workload/signed_token.go); must match cmd/control-plane's own --workload-secret. Unset skips minting a cosmic-clash.io/workload-token annotation entirely") @@ -33,8 +36,8 @@ func main() { if *dsn == "" || *agonesURL == "" { fatalf("--dsn/COSMIC_CLASH_POSTGRES_DSN and --agones-url/COSMIC_CLASH_AGONES_URL are required") } - if (*transport != "enet" && *transport != "steam_sdr") || *interval <= 0 { - fatalf("--transport must be enet or steam_sdr and --interval must be positive") + if (*transport != "enet" && *transport != "steam_sdr") || *interval <= 0 || *providerTimeout <= 0 { + fatalf("--transport must be enet or steam_sdr and --interval/--provider-timeout must be positive") } if *allocationQuota < 0 || *allocationQuotaWindow <= 0 { fatalf("--allocation-quota must be non-negative and --allocation-quota-window must be positive") @@ -65,7 +68,11 @@ func main() { log.Printf("allocator: enabled per-replica regional allocation quota=%d window=%s", *allocationQuota, *allocationQuotaWindow) } metrics := allocator.NewMetrics() - client := agones.Client{BaseURL: *agonesURL, Namespace: *namespace, WorkloadSecret: []byte(*workloadSecret)} + providerHTTP, err := agones.NewKubernetesHTTPClient(*agonesURL, *kubernetesTokenPath, *kubernetesCAPath, *providerTimeout) + if err != nil { + fatalf("configure Kubernetes API client: %v", err) + } + client := agones.Client{BaseURL: *agonesURL, Namespace: *namespace, HTTP: providerHTTP, WorkloadSecret: []byte(*workloadSecret)} worker := allocator.Worker{ Claims: store.AllocatingMatchClaims{DB: db, Transport: *transport}, Service: allocator.Service{ diff --git a/server/security/test_fleet_manifests.py b/server/security/test_fleet_manifests.py index e38282c2..d2b4d0ac 100644 --- a/server/security/test_fleet_manifests.py +++ b/server/security/test_fleet_manifests.py @@ -74,11 +74,13 @@ class FleetManifestTest(unittest.TestCase): for document in (eu, na): self.assertIn("namespace: cosmic-clash", document) - def test_kustomization_does_not_rewrite_cross_namespace_agones_rbac(self): + def test_allocator_agones_rbac_is_in_the_game_server_namespace(self): base = self.read("base/kustomization.yaml") rbac = self.read("base/rbac.yaml") self.assertNotIn("namespace: cosmic-clash", base) - self.assertIn("namespace: agones-system", rbac) + self.assertNotIn("namespace: agones-system", rbac) + self.assertGreaterEqual(rbac.count("namespace: cosmic-clash"), 3) + self.assertIn("name: allocator", rbac) def test_control_plane_service_and_game_server_egress_are_declared(self): service = self.read("base/control-plane-service.yaml") diff --git a/server/security/test_kubernetes_policies.py b/server/security/test_kubernetes_policies.py index 3b108d63..6cc25985 100644 --- a/server/security/test_kubernetes_policies.py +++ b/server/security/test_kubernetes_policies.py @@ -36,7 +36,8 @@ class KubernetesPolicyTest(unittest.TestCase): "readOnlyRootFilesystem: true", "drop: [ALL]", "resources:", "image: ghcr.io/cosmic-clash/allocator@sha256:", "--metrics-addr=:9091", "containerPort: 9091", - "key: dsn", "key: secret", "automountServiceAccountToken: false", + "key: dsn", "key: secret", "automountServiceAccountToken: true", + "--agones-url=https://kubernetes.default.svc", "--provider-timeout=10s", ): self.assertIn(required, deployment) self.assertNotRegex(deployment, r"(?im)^\s*(password|token|private.?key):\s*[^\n]+$") @@ -86,7 +87,7 @@ class KubernetesPolicyTest(unittest.TestCase): self.assertIn(required, deployment) self.assertGreaterEqual(deployment.count("app.kubernetes.io/name: allocator"), 4) - def test_allocator_network_policy_has_only_metrics_data_agones_and_dns_flows(self): + def test_allocator_network_policy_has_only_metrics_data_kubernetes_api_and_dns_flows(self): policies = self.read("network-policies.yaml") allocator = policies.split("name: allocator-allowed-flows", 1)[-1] self.assertIn("port: 9091", allocator) @@ -94,6 +95,7 @@ class KubernetesPolicyTest(unittest.TestCase): self.assertIn(port, allocator) self.assertNotIn("port: 8080", allocator) self.assertNotIn("ipBlock:", allocator) + self.assertNotIn("agones-system", allocator) def test_allocator_pdb_preserves_one_replica_during_voluntary_disruption(self): pdb = self.read("allocator-pdb.yaml") @@ -104,12 +106,16 @@ class KubernetesPolicyTest(unittest.TestCase): ): self.assertIn(required, pdb) - def test_rbac_is_scoped_to_allocator_create(self): + def test_rbac_is_scoped_to_allocator_agones_operations(self): rbac = self.read("rbac.yaml") - self.assertIn("namespace: agones-system", rbac) + self.assertNotIn("namespace: agones-system", rbac) + self.assertGreaterEqual(rbac.count("namespace: cosmic-clash"), 3) + self.assertIn('resources: ["gameservers"]', rbac) + self.assertIn('verbs: ["list"]', rbac) self.assertIn('resources: ["gameserverallocations"]', rbac) self.assertIn('verbs: ["create"]', rbac) - self.assertNotRegex(rbac, r"verbs:.*\b(get|list|watch|update|patch|delete|\*)\b") + self.assertIn("name: allocator", rbac) + self.assertNotRegex(rbac, r"verbs:.*\b(watch|update|patch|delete|\*)\b") self.assertNotIn('resources: ["*"]', rbac) def test_default_deny_and_only_declared_data_dns_edge_flows_exist(self):