From 931e51a647f98032f02d11b6c0f9f61b7aa0af4a Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Tue, 1 Sep 2026 09:49:09 +0100 Subject: [PATCH] feat: add Agones allocation client --- multiplayer-next.md | 4 +- multiplayer-todo.md | 2 +- server/agones/allocation.go | 147 +++++++++++++++++++++++++++++++ server/agones/allocation_test.go | 72 +++++++++++++++ 4 files changed, 223 insertions(+), 2 deletions(-) create mode 100644 server/agones/allocation.go create mode 100644 server/agones/allocation_test.go diff --git a/multiplayer-next.md b/multiplayer-next.md index ce8305a7..84b0dd4c 100644 --- a/multiplayer-next.md +++ b/multiplayer-next.md @@ -55,7 +55,9 @@ product policy are in [`docs/MATCHMAKING.md`](docs/MATCHMAKING.md). acquisition and multi-region probe population remain. - [ ] **IN PROGRESS:** Durable allocator registry now records READY GameServer projections and atomically claims compatible capacity with replay/conflict - fencing; provider allocation and assignment publication remain. + fencing; `server/agones` now submits and validates namespaced + `GameServerAllocation` responses, including dynamic address/port data; + provider-to-durable claim reconciliation and assignment publication remain. - [ ] **IN PROGRESS:** Run the Go control plane against PostgreSQL/Redis with independently runnable API, matcher, allocator and maintenance roles. The `cmd/control-plane` API role now opens PostgreSQL, applies migrations, wires diff --git a/multiplayer-todo.md b/multiplayer-todo.md index 82597a13..9b604343 100644 --- a/multiplayer-todo.md +++ b/multiplayer-todo.md @@ -1213,7 +1213,7 @@ the local/CI/community transport, not a silent production fallback. | 8.27 `[D:8.26]` | **IN PROGRESS.** Go supervisor package provides local-safe Agones REST discovery, validates assigned address/port data, injects dynamic `SDR_LISTEN_PORT`/`SDR_IP`, performs explicit process-ready probing and Ready transition; direct mode bypasses Agones | `server/supervisor/` covers allocated/direct startup, invalid endpoint rejection, dynamic endpoint/Ready ordering and authenticated drain; allocated Godot now supplies a loopback readiness/drain control surface and `agones_sdk.gd` supplies sidecar Health/Ready/Shutdown/annotation REST operations; metadata watch, real Agones annotation/shutdown confirmation and emulator integration remain | | 8.28 `[D:8.6,8.27]` | **IN PROGRESS.** Supervisor separates explicit process-ready from Agones Ready and never scrapes stdout; allocated mode refuses to mark Ready without a configured readiness probe | `server/supervisor/` tests prove Ready follows the probe and direct mode remains functional; `server_control.gd`, `agones_sdk.gd` and process-level smokes prove loopback `/ready`, `/health`, bearer-protected `/drain`, sidecar-shaped Health/Ready calls and drain admission fencing; detached-container and Health-reclaim integration remain | | 8.29 `[D:8.26,8.27]` | **IN PROGRESS.** Supervisor discovers and validates the Agones endpoint, propagates the actual dynamic `--port`, and exports `SDR_LISTEN_PORT`/`SDR_IP` only for Hosted-SDR while preserving an isolated ENet path | `server/supervisor/` tests cover invalid address/port rejection, dynamic port argument/env propagation and SDR-vs-ENet separation; real Agones dynamic/passthrough mapping, POP/cert/firewall/NAT and multi-match fixture remain | -| 8.30 `[D:8.18,8.26,8.28,8.29]` | **IN PROGRESS.** Pure Go allocator filters Ready servers by region/build/protocol/transport, atomically claims one with idempotent allocation replay, and now owns the assignment-publication boundary; PostgreSQL adds durable GameServer registration and compatible `SKIP LOCKED` claims with request-digest fencing | `server/domain/allocator.go`, `server/store/allocator_sql.go`, `server/migrations/0004_allocator_registry.sql` and tests cover deterministic compatible selection, exhaustion, conflicting/identical allocation replay, unknown allocations, SQL claim ordering, invalid input and assignment replay/conflict; `TestPostgreSQLAllocatorClaimReplayAndCapacityFence` now covers live registration/selection/replay/conflict/no-capacity when the disposable database gate is run; Agones `GameServerAllocation`, signed roster metadata, bounded cross-replica retry and live integration remain | +| 8.30 `[D:8.18,8.26,8.28,8.29]` | **IN PROGRESS.** Pure Go allocator filters Ready servers by region/build/protocol/transport, atomically claims one with idempotent allocation replay, and now owns the assignment-publication boundary; PostgreSQL adds durable GameServer registration and compatible `SKIP LOCKED` claims with request-digest fencing; `server/agones` submits and validates namespaced `GameServerAllocation` responses and dynamic endpoints | `server/domain/allocator.go`, `server/store/allocator_sql.go`, `server/agones/allocation.go`, `server/migrations/0004_allocator_registry.sql` and tests cover deterministic compatible selection, exhaustion, conflicting/identical allocation replay, unknown allocations, SQL claim ordering, invalid input, provider error/malformed response/IPv6 endpoint handling and assignment replay/conflict; `TestPostgreSQLAllocatorClaimReplayAndCapacityFence` now covers live registration/selection/replay/conflict/no-capacity when the disposable database gate is run; provider-to-durable claim reconciliation, signed roster metadata, bounded cross-replica retry and live integration remain | | 8.31 `[D:8.9,8.30]` | **IN PROGRESS.** Pure Go assignment gate requires Allocated state, exact allocation ID/match/server/region/build/protocol/transport compatibility, non-empty hosted endpoint and verified manifest signature before exposure; allocator publication cannot expose Ready state | `server/domain/assignment.go` and `allocator.go` plus adversarial fixtures cover early-connect, tampered signature/manifest, wrong compatibility, empty endpoint, unknown allocation and post-publication mutation rejection; Agones metadata watch, hosted-address registration, production signer and client-ticket publication remain | | 8.32 `[D:8.2,8.26,8.30]` | **IN PROGRESS.** Provider-neutral FleetAutoscaler baseline preserves a two-process Ready buffer, caps warm capacity, and leaves Allocated scale-down independent of the Ready floor; Fleet image references remain digest-pinned for current/rollback pre-pull | `deploy/k8s/base/fleet-autoscaler.yaml` and manifest tests cover Fleet ownership, Buffer policy and floor/cap invariants; regional on-demand node pools/failure domains, pre-pull rollout, warm-allocation p95/p99 and N+1 certification remain | | 8.33 `[D:8.26,8.32]` | **IN PROGRESS.** Fleet scheduling now requires on-demand capacity and spreads Ready processes across zones with skew 1; the autoscaler preserves the two-process Ready floor | `deploy/k8s/base/fleet.yaml` and manifest tests reject interruptible placement and single-zone concentration structurally; regional node pools, forced node-loss testing and measured N+1 headroom remain | diff --git a/server/agones/allocation.go b/server/agones/allocation.go new file mode 100644 index 00000000..7c63870d --- /dev/null +++ b/server/agones/allocation.go @@ -0,0 +1,147 @@ +// Package agones contains the narrow provider adapter used by the allocator. +// Domain policy and durable allocation records remain outside this package. +package agones + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/cosmic-clash/cosmic-clash/server/domain" +) + +type Client struct { + BaseURL string + Namespace string + HTTP *http.Client +} + +type AllocatedServer struct { + Allocation domain.Allocation + Endpoint string + GameServer string +} + +type allocationRequest struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Spec struct { + Selectors []struct { + MatchLabels map[string]string `json:"matchLabels"` + } `json:"selectors"` + } `json:"spec"` +} + +type allocationResponse struct { + Status struct { + State string `json:"state"` + GameServerName string `json:"gameServerName"` + Address string `json:"address"` + Ports []struct { + Name string `json:"name"` + Port int `json:"port"` + } `json:"ports"` + } `json:"status"` +} + +func (c Client) Allocate(ctx context.Context, request domain.AllocationRequest, labels map[string]string, now time.Time) (AllocatedServer, error) { + if c.HTTP == nil { + c.HTTP = http.DefaultClient + } + base, err := c.endpoint() + if err != nil { + return AllocatedServer{}, err + } + if request.AllocationID == "" || request.MatchID == "" || (request.Region != "EU" && request.Region != "NA") || request.Build == "" || request.Protocol <= 0 || (request.Transport != "enet" && request.Transport != "steam_sdr") || now.IsZero() { + return AllocatedServer{}, domain.ErrAllocationInput + } + if len(labels) == 0 { + return AllocatedServer{}, fmt.Errorf("allocation labels are required") + } + for key, value := range labels { + if key == "" || value == "" || strings.ContainsAny(key+value, "\r\n") { + return AllocatedServer{}, fmt.Errorf("invalid allocation label") + } + } + var body allocationRequest + body.APIVersion = "allocation.agones.dev/v1" + body.Kind = "GameServerAllocation" + body.Spec.Selectors = []struct { + MatchLabels map[string]string `json:"matchLabels"` + }{{MatchLabels: cloneLabels(labels)}} + encoded, err := json.Marshal(body) + if err != nil { + return AllocatedServer{}, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, base+"/apis/allocation.agones.dev/v1/namespaces/"+url.PathEscape(c.Namespace)+"/gameserverallocations", bytes.NewReader(encoded)) + if err != nil { + return AllocatedServer{}, err + } + req.Header.Set("Content-Type", "application/json") + response, err := c.HTTP.Do(req) + if err != nil { + return AllocatedServer{}, err + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode >= 300 { + return AllocatedServer{}, fmt.Errorf("Agones allocation returned %s", response.Status) + } + var decoded allocationResponse + decoder := json.NewDecoder(io.LimitReader(response.Body, 64<<10)) + if err := decoder.Decode(&decoded); err != nil { + return AllocatedServer{}, fmt.Errorf("decode Agones allocation: %w", err) + } + if decoded.Status.State != "Allocated" || decoded.Status.GameServerName == "" || decoded.Status.Address == "" { + return AllocatedServer{}, fmt.Errorf("Agones allocation is incomplete") + } + port, err := selectPort(decoded.Status.Ports) + if err != nil { + return AllocatedServer{}, err + } + return AllocatedServer{Allocation: domain.Allocation{AllocationID: request.AllocationID, MatchID: request.MatchID, ServerID: decoded.Status.GameServerName, Region: request.Region, Build: request.Build, Protocol: request.Protocol, Transport: request.Transport, State: domain.ServerAllocated, AllocatedAt: now}, Endpoint: net.JoinHostPort(decoded.Status.Address, strconv.Itoa(port)), GameServer: decoded.Status.GameServerName}, nil +} + +func (c Client) endpoint() (string, error) { + if c.Namespace == "" || strings.ContainsAny(c.Namespace, "/\r\n") { + return "", fmt.Errorf("invalid Agones namespace") + } + u, err := url.Parse(c.BaseURL) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.RawQuery != "" || u.Fragment != "" || u.Path != "" { + return "", fmt.Errorf("invalid Agones base URL") + } + return strings.TrimRight(c.BaseURL, "/"), nil +} + +func selectPort(ports []struct { + Name string `json:"name"` + Port int `json:"port"` +}) (int, error) { + for _, port := range ports { + if port.Name == "default" { + if port.Port < 1 || port.Port > 65535 { + return 0, fmt.Errorf("Agones returned invalid default port") + } + return port.Port, nil + } + } + if len(ports) != 1 || ports[0].Port < 1 || ports[0].Port > 65535 { + return 0, fmt.Errorf("Agones returned no usable game port") + } + return ports[0].Port, nil +} + +func cloneLabels(labels map[string]string) map[string]string { + copy := make(map[string]string, len(labels)) + for key, value := range labels { + copy[key] = value + } + return copy +} diff --git a/server/agones/allocation_test.go b/server/agones/allocation_test.go new file mode 100644 index 00000000..e08125a4 --- /dev/null +++ b/server/agones/allocation_test.go @@ -0,0 +1,72 @@ +package agones + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/cosmic-clash/cosmic-clash/server/domain" +) + +func request() domain.AllocationRequest { + return domain.AllocationRequest{AllocationID: "allocation-1", MatchID: "match-1", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet"} +} + +func TestAllocateBuildsStrictGameServerAllocationAndEndpoint(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/apis/allocation.agones.dev/v1/namespaces/games/gameserverallocations" { + t.Fatalf("request=%s %s", r.Method, r.URL.Path) + } + var body allocationRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatal(err) + } + if body.APIVersion != "allocation.agones.dev/v1" || body.Kind != "GameServerAllocation" || len(body.Spec.Selectors) != 1 || body.Spec.Selectors[0].MatchLabels["cosmic-clash/region"] != "EU" { + t.Fatalf("body=%+v", body) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"status":{"state":"Allocated","gameServerName":"gs-a","address":"2001:db8::1","ports":[{"name":"default","port":7777}]}}`)) + })) + defer server.Close() + got, err := (Client{BaseURL: server.URL, Namespace: "games", HTTP: server.Client()}).Allocate(context.Background(), request(), map[string]string{"cosmic-clash/region": "EU", "cosmic-clash/build": "build-1"}, time.Unix(1000, 0)) + if err != nil { + t.Fatal(err) + } + if got.GameServer != "gs-a" || got.Endpoint != "[2001:db8::1]:7777" || got.Allocation.State != domain.ServerAllocated { + t.Fatalf("allocation=%+v", got) + } +} + +func TestAllocateFailsClosedOnMalformedProviderResponses(t *testing.T) { + cases := []string{ + `{"status":{"state":"UnAllocated","gameServerName":"gs","address":"127.0.0.1","ports":[{"name":"default","port":7777}]}}`, + `{"status":{"state":"Allocated","gameServerName":"gs","address":"127.0.0.1","ports":[]}}`, + `{"status":{"state":"Allocated","gameServerName":"gs","address":"127.0.0.1","ports":[{"name":"default","port":70000}]}}`, + } + for _, payload := range cases { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(payload)) })) + _, err := (Client{BaseURL: server.URL, Namespace: "games", HTTP: server.Client()}).Allocate(context.Background(), request(), map[string]string{"region": "EU"}, time.Unix(1000, 0)) + server.Close() + if err == nil { + t.Fatalf("malformed response accepted: %s", payload) + } + } +} + +func TestAllocateRejectsUnsafeConfigurationAndProviderFailure(t *testing.T) { + for _, client := range []Client{{BaseURL: "http://127.0.0.1:1/path", Namespace: "games"}, {BaseURL: "http://127.0.0.1:1", Namespace: "games/other"}} { + if _, err := client.Allocate(context.Background(), request(), map[string]string{"region": "EU"}, time.Unix(1000, 0)); err == nil { + t.Fatal("unsafe client configuration accepted") + } + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "no capacity", http.StatusConflict) })) + defer server.Close() + _, err := (Client{BaseURL: server.URL, Namespace: "games", HTTP: server.Client()}).Allocate(context.Background(), request(), map[string]string{"region": "EU"}, time.Unix(1000, 0)) + if err == nil || !strings.Contains(err.Error(), "409") { + t.Fatalf("provider failure err=%v", err) + } +}