fix(multiplayer): authenticate Agones Kubernetes API

deploy/k8s/base/allocator-deployment.yaml

@@ -24,7 +24,9 @@ spec:
     spec:
       terminationGracePeriodSeconds: 10
       serviceAccountName: allocator
-      automountServiceAccountToken: false
+      # This role calls Agones CRDs through the Kubernetes API. The client
+      # rereads the short-lived projected token on every request.
+      automountServiceAccountToken: true
       topologySpreadConstraints:
         - maxSkew: 1
           topologyKey: topology.kubernetes.io/zone
@@ -52,8 +54,9 @@ spec:
           image: ghcr.io/cosmic-clash/allocator@sha256:0000000000000000000000000000000000000000000000000000000000000000
           args:
             - --dsn=$(COSMIC_CLASH_POSTGRES_DSN)
-            - --agones-url=https://agones-allocator.agones-system.svc.cluster.local
+            - --agones-url=https://kubernetes.default.svc
             - --agones-namespace=cosmic-clash
+            - --provider-timeout=10s
             - --metrics-addr=:9091
           ports:
             - name: metrics

deploy/k8s/base/network-policies.yaml

@@ -47,13 +47,6 @@ spec:
       ports:
         - protocol: TCP
           port: 6379
-    - to:
-        - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: agones-system
-      ports:
-        - protocol: TCP
-          port: 443
     - ports:
         - protocol: UDP
           port: 53
@@ -130,11 +123,10 @@ spec:
       ports:
         - protocol: TCP
           port: 5432
-    - to:
+    # The kubernetes.default Service endpoint is implementation-specific and
-        - namespaceSelector:
+    # may be a control-plane/node IP that cannot be selected by pod labels.
-            matchLabels:
+    # Keep API egress portable while limiting it to TLS only.
-              kubernetes.io/metadata.name: agones-system
+    - ports:
-      ports:
         - protocol: TCP
           port: 443
     - ports:

deploy/k8s/base/rbac.yaml

@@ -1,9 +1,12 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: control-plane-agones-allocator
+  name: allocator-agones-api
-  namespace: agones-system
+  namespace: cosmic-clash
 rules:
+  - apiGroups: ["agones.dev"]
+    resources: ["gameservers"]
+    verbs: ["list"]
   - apiGroups: ["allocation.agones.dev"]
     resources: ["gameserverallocations"]
     verbs: ["create"]
@@ -11,14 +14,13 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: cosmic-clash-control-plane-agones-allocator
+  name: allocator-agones-api
-  namespace: agones-system
+  namespace: cosmic-clash
 subjects:
   - kind: ServiceAccount
-    name: control-plane
+    name: allocator
     namespace: cosmic-clash
 roleRef:
   kind: Role
-  name: control-plane-agones-allocator
+  name: allocator-agones-api
   apiGroup: rbac.authorization.k8s.io
-

server/agones/kubernetes_client.go

@@ -0,0 +1,67 @@
+package agones
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+)
+
+// NewKubernetesHTTPClient builds an in-cluster client for the Kubernetes API.
+// The bearer token is read for every request so kubelet token rotation does not
+// leave a long-running allocator with an expired credential.
+func NewKubernetesHTTPClient(baseURL, tokenPath, caPath string, timeout time.Duration) (*http.Client, error) {
+	origin, err := url.Parse(baseURL)
+	if err != nil || origin.Scheme != "https" || origin.Host == "" || origin.User != nil || origin.Path != "" || origin.RawQuery != "" || origin.Fragment != "" {
+		return nil, fmt.Errorf("Kubernetes API base URL must be an HTTPS origin")
+	}
+	if strings.TrimSpace(tokenPath) == "" || strings.TrimSpace(caPath) == "" || timeout <= 0 {
+		return nil, fmt.Errorf("Kubernetes API token path, CA path, and positive timeout are required")
+	}
+	caPEM, err := os.ReadFile(caPath)
+	if err != nil {
+		return nil, fmt.Errorf("read Kubernetes API CA: %w", err)
+	}
+	roots := x509.NewCertPool()
+	if !roots.AppendCertsFromPEM(caPEM) {
+		return nil, fmt.Errorf("Kubernetes API CA contains no certificates")
+	}
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.TLSClientConfig = &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}
+	return &http.Client{
+		Timeout: timeout,
+		Transport: bearerTokenTransport{
+			tokenPath:      tokenPath,
+			expectedOrigin: origin.Scheme + "://" + origin.Host,
+			base:           transport,
+		},
+	}, nil
+}
+
+type bearerTokenTransport struct {
+	tokenPath      string
+	expectedOrigin string
+	base           http.RoundTripper
+}
+
+func (t bearerTokenTransport) RoundTrip(request *http.Request) (*http.Response, error) {
+	if request.URL.Scheme+"://"+request.URL.Host != t.expectedOrigin {
+		return nil, fmt.Errorf("refusing to send Kubernetes API credential to unexpected origin")
+	}
+	tokenBytes, err := os.ReadFile(t.tokenPath)
+	if err != nil {
+		return nil, fmt.Errorf("read Kubernetes API bearer token: %w", err)
+	}
+	token := strings.TrimSpace(string(tokenBytes))
+	if token == "" || strings.ContainsAny(token, " \t\r\n") {
+		return nil, fmt.Errorf("Kubernetes API bearer token is empty or malformed")
+	}
+	cloned := request.Clone(request.Context())
+	cloned.Header = request.Header.Clone()
+	cloned.Header.Set("Authorization", "Bearer "+token)
+	return t.base.RoundTrip(cloned)
+}

server/agones/kubernetes_client_test.go

@@ -0,0 +1,125 @@
+package agones
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestKubernetesHTTPClientTrustsCAAddsAndRotatesBearerToken(t *testing.T) {
+	var seen []string
+	server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		seen = append(seen, r.Header.Get("Authorization"))
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	server.TLS = testTLSConfig(t)
+	server.StartTLS()
+	defer server.Close()
+
+	directory := t.TempDir()
+	caPath := filepath.Join(directory, "ca.crt")
+	tokenPath := filepath.Join(directory, "token")
+	certificate := server.Certificate()
+	if err := os.WriteFile(caPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificate.Raw}), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(tokenPath, []byte("first-token\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	client, err := NewKubernetesHTTPClient(server.URL, tokenPath, caPath, time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, token := range []string{"first-token", "rotated-token"} {
+		if err := os.WriteFile(tokenPath, []byte(token), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		response, err := client.Get(server.URL)
+		if err != nil {
+			t.Fatal(err)
+		}
+		response.Body.Close()
+	}
+	if len(seen) != 2 || seen[0] != "Bearer first-token" || seen[1] != "Bearer rotated-token" {
+		t.Fatalf("authorization headers = %v", seen)
+	}
+}
+
+func TestKubernetesHTTPClientRejectsInvalidConfigurationAndToken(t *testing.T) {
+	directory := t.TempDir()
+	caPath := filepath.Join(directory, "ca.crt")
+	tokenPath := filepath.Join(directory, "token")
+	if _, err := NewKubernetesHTTPClient("http://kubernetes.default.svc", tokenPath, caPath, time.Second); err == nil {
+		t.Fatal("non-TLS API origin accepted")
+	}
+	if _, err := NewKubernetesHTTPClient("https://kubernetes.default.svc", tokenPath, caPath, time.Second); err == nil {
+		t.Fatal("missing CA accepted")
+	}
+	if err := os.WriteFile(caPath, []byte("not a certificate"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := NewKubernetesHTTPClient("https://kubernetes.default.svc", tokenPath, caPath, time.Second); err == nil {
+		t.Fatal("invalid CA accepted")
+	}
+}
+
+func TestKubernetesHTTPClientDoesNotForwardCredentialAcrossOrigins(t *testing.T) {
+	server := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
+	defer server.Close()
+	directory := t.TempDir()
+	caPath := filepath.Join(directory, "ca.crt")
+	tokenPath := filepath.Join(directory, "token")
+	if err := os.WriteFile(caPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(tokenPath, []byte("secret-token"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	client, err := NewKubernetesHTTPClient(server.URL, tokenPath, caPath, time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := client.Get("https://example.invalid/"); err == nil {
+		t.Fatal("credentialed request to another origin was not rejected")
+	}
+}
+
+func testTLSConfig(t *testing.T) *tls.Config {
+	t.Helper()
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatal(err)
+	}
+	template := x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject:      pkix.Name{CommonName: "127.0.0.1"},
+		NotBefore:    time.Now().Add(-time.Minute),
+		NotAfter:     time.Now().Add(time.Hour),
+		KeyUsage:     x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		IPAddresses:  []net.IP{net.ParseIP("127.0.0.1")},
+	}
+	der, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	certificate, err := tls.X509KeyPair(
+		pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}),
+		pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)}),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return &tls.Config{Certificates: []tls.Certificate{certificate}, MinVersion: tls.VersionTLS12}
+}

server/cmd/allocator/main.go

@@ -23,6 +23,9 @@ func main() {
 	migrationDir := flag.String("migrations", "migrations", "directory containing numbered SQL migrations")
 	agonesURL := flag.String("agones-url", os.Getenv("COSMIC_CLASH_AGONES_URL"), "Agones allocation API base URL")
 	namespace := flag.String("agones-namespace", envOrDefault("COSMIC_CLASH_AGONES_NAMESPACE", "default"), "Agones namespace")
+	kubernetesTokenPath := flag.String("kubernetes-token-path", envOrDefault("COSMIC_CLASH_KUBERNETES_TOKEN_PATH", "/var/run/secrets/kubernetes.io/serviceaccount/token"), "rotating Kubernetes service-account bearer token")
+	kubernetesCAPath := flag.String("kubernetes-ca-path", envOrDefault("COSMIC_CLASH_KUBERNETES_CA_PATH", "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"), "Kubernetes API cluster CA bundle")
+	providerTimeout := flag.Duration("provider-timeout", 10*time.Second, "timeout for each Kubernetes/Agones API request")
 	transport := flag.String("transport", envOrDefault("COSMIC_CLASH_TRANSPORT", "enet"), "game transport: enet or steam_sdr")
 	interval := flag.Duration("interval", time.Second, "allocation poll interval")
 	workloadSecret := flag.String("workload-secret", os.Getenv("COSMIC_CLASH_WORKLOAD_SECRET"), "HMAC secret for control-plane-issued workload tokens (see workload/signed_token.go); must match cmd/control-plane's own --workload-secret. Unset skips minting a cosmic-clash.io/workload-token annotation entirely")
@@ -33,8 +36,8 @@ func main() {
 	if *dsn == "" || *agonesURL == "" {
 		fatalf("--dsn/COSMIC_CLASH_POSTGRES_DSN and --agones-url/COSMIC_CLASH_AGONES_URL are required")
 	}
-	if (*transport != "enet" && *transport != "steam_sdr") || *interval <= 0 {
+	if (*transport != "enet" && *transport != "steam_sdr") || *interval <= 0 || *providerTimeout <= 0 {
-		fatalf("--transport must be enet or steam_sdr and --interval must be positive")
+		fatalf("--transport must be enet or steam_sdr and --interval/--provider-timeout must be positive")
 	}
 	if *allocationQuota < 0 || *allocationQuotaWindow <= 0 {
 		fatalf("--allocation-quota must be non-negative and --allocation-quota-window must be positive")
@@ -65,7 +68,11 @@ func main() {
 		log.Printf("allocator: enabled per-replica regional allocation quota=%d window=%s", *allocationQuota, *allocationQuotaWindow)
 	}
 	metrics := allocator.NewMetrics()
-	client := agones.Client{BaseURL: *agonesURL, Namespace: *namespace, WorkloadSecret: []byte(*workloadSecret)}
+	providerHTTP, err := agones.NewKubernetesHTTPClient(*agonesURL, *kubernetesTokenPath, *kubernetesCAPath, *providerTimeout)
+	if err != nil {
+		fatalf("configure Kubernetes API client: %v", err)
+	}
+	client := agones.Client{BaseURL: *agonesURL, Namespace: *namespace, HTTP: providerHTTP, WorkloadSecret: []byte(*workloadSecret)}
 	worker := allocator.Worker{
 		Claims: store.AllocatingMatchClaims{DB: db, Transport: *transport},
 		Service: allocator.Service{

server/security/test_fleet_manifests.py

@@ -74,11 +74,13 @@ class FleetManifestTest(unittest.TestCase):
         for document in (eu, na):
             self.assertIn("namespace: cosmic-clash", document)
 
-    def test_kustomization_does_not_rewrite_cross_namespace_agones_rbac(self):
+    def test_allocator_agones_rbac_is_in_the_game_server_namespace(self):
         base = self.read("base/kustomization.yaml")
         rbac = self.read("base/rbac.yaml")
         self.assertNotIn("namespace: cosmic-clash", base)
-        self.assertIn("namespace: agones-system", rbac)
+        self.assertNotIn("namespace: agones-system", rbac)
+        self.assertGreaterEqual(rbac.count("namespace: cosmic-clash"), 3)
+        self.assertIn("name: allocator", rbac)
 
     def test_control_plane_service_and_game_server_egress_are_declared(self):
         service = self.read("base/control-plane-service.yaml")

server/security/test_kubernetes_policies.py

@@ -36,7 +36,8 @@ class KubernetesPolicyTest(unittest.TestCase):
             "readOnlyRootFilesystem: true", "drop: [ALL]", "resources:",
             "image: ghcr.io/cosmic-clash/allocator@sha256:",
             "--metrics-addr=:9091", "containerPort: 9091",
-            "key: dsn", "key: secret", "automountServiceAccountToken: false",
+            "key: dsn", "key: secret", "automountServiceAccountToken: true",
+            "--agones-url=https://kubernetes.default.svc", "--provider-timeout=10s",
         ):
             self.assertIn(required, deployment)
         self.assertNotRegex(deployment, r"(?im)^\s*(password|token|private.?key):\s*[^\n]+$")
@@ -86,7 +87,7 @@ class KubernetesPolicyTest(unittest.TestCase):
             self.assertIn(required, deployment)
         self.assertGreaterEqual(deployment.count("app.kubernetes.io/name: allocator"), 4)
 
-    def test_allocator_network_policy_has_only_metrics_data_agones_and_dns_flows(self):
+    def test_allocator_network_policy_has_only_metrics_data_kubernetes_api_and_dns_flows(self):
         policies = self.read("network-policies.yaml")
         allocator = policies.split("name: allocator-allowed-flows", 1)[-1]
         self.assertIn("port: 9091", allocator)
@@ -94,6 +95,7 @@ class KubernetesPolicyTest(unittest.TestCase):
             self.assertIn(port, allocator)
         self.assertNotIn("port: 8080", allocator)
         self.assertNotIn("ipBlock:", allocator)
+        self.assertNotIn("agones-system", allocator)
 
     def test_allocator_pdb_preserves_one_replica_during_voluntary_disruption(self):
         pdb = self.read("allocator-pdb.yaml")
@@ -104,12 +106,16 @@ class KubernetesPolicyTest(unittest.TestCase):
         ):
             self.assertIn(required, pdb)
 
-    def test_rbac_is_scoped_to_allocator_create(self):
+    def test_rbac_is_scoped_to_allocator_agones_operations(self):
         rbac = self.read("rbac.yaml")
-        self.assertIn("namespace: agones-system", rbac)
+        self.assertNotIn("namespace: agones-system", rbac)
+        self.assertGreaterEqual(rbac.count("namespace: cosmic-clash"), 3)
+        self.assertIn('resources: ["gameservers"]', rbac)
+        self.assertIn('verbs: ["list"]', rbac)
         self.assertIn('resources: ["gameserverallocations"]', rbac)
         self.assertIn('verbs: ["create"]', rbac)
-        self.assertNotRegex(rbac, r"verbs:.*\b(get|list|watch|update|patch|delete|\*)\b")
+        self.assertIn("name: allocator", rbac)
+        self.assertNotRegex(rbac, r"verbs:.*\b(watch|update|patch|delete|\*)\b")
         self.assertNotIn('resources: ["*"]', rbac)
 
     def test_default_deny_and_only_declared_data_dns_edge_flows_exist(self):