feat(multiplayer): propagate match ID to an already-allocated pod via annotations

Investigated the Fleet-manifest wiring task flagged last commit and
found a deeper, previously-undesigned gap: Kubernetes env vars are
fixed at pod creation, but Agones allocates a match to an already-
running Ready pod well after it starts -- so there was no channel at
all for match-specific data (match ID) to reach that pod's processes.

Close it using the Agones GameServerAllocation API's documented
spec.metadata.annotations field, which Agones applies to the allocated
GameServer's own object_meta on success: server/agones.Client.Allocate
now requests cosmic-clash.io/match-id and cosmic-clash.io/allocation-id
annotations, and the supervisor reads them back from the same
/gameserver SDK call it already makes for the assigned port/address
(GameServer.ObjectMeta.Annotations), falling back to them for its own
control-plane registration only when MatchID isn't explicitly
configured -- an explicit value always wins, and a match ID resolvable
from neither source fails Start() closed before any HTTP call.

The exact object_meta vs objectMeta JSON key from a live Agones SDK
sidecar is not independently verified from this sandbox; documented
inline, and the fallback degrades safely (empty annotations map, same
as before this change) if it turns out to be wrong.

Covered by two new tests: the annotation actually flowing through to
the registration body, and fail-closed with neither config nor
annotation supplying a match ID (registerCalled stays false, not just
that Start() errors).
This commit is contained in:
Josh Creek
2026-09-01 13:34:08 +01:00
parent 4278c04d60
commit fd18cf6ac0
5 changed files with 152 additions and 14 deletions
+16
View File
@@ -37,6 +37,18 @@ type allocationRequest struct {
Selectors []struct {
MatchLabels map[string]string `json:"matchLabels"`
} `json:"selectors"`
// Metadata.Annotations is applied to the allocated GameServer's own
// object_meta by Agones on successful allocation (a documented part
// of the GameServerAllocation spec, independent of the Selectors
// used to find capacity). This is the only way match-specific data
// reaches an already-Ready pod after allocation: Kubernetes env vars
// are fixed at pod creation, long before Agones assigns a match to
// that pod, so there is no other channel for it. The allocated
// process reads these back via the SDK's own GameServer call
// (server/supervisor's existing /gameserver request).
Metadata struct {
Annotations map[string]string `json:"annotations"`
} `json:"metadata"`
} `json:"spec"`
}
@@ -139,6 +151,10 @@ func (c Client) Allocate(ctx context.Context, request domain.AllocationRequest,
body.Spec.Selectors = []struct {
MatchLabels map[string]string `json:"matchLabels"`
}{{MatchLabels: cloneLabels(labels)}}
body.Spec.Metadata.Annotations = map[string]string{
"cosmic-clash.io/match-id": request.MatchID,
"cosmic-clash.io/allocation-id": request.AllocationID,
}
encoded, err := json.Marshal(body)
if err != nil {
return AllocatedServer{}, err
+3
View File
@@ -28,6 +28,9 @@ func TestAllocateBuildsStrictGameServerAllocationAndEndpoint(t *testing.T) {
if body.APIVersion != "allocation.agones.dev/v1" || body.Kind != "GameServerAllocation" || len(body.Spec.Selectors) != 1 || body.Spec.Selectors[0].MatchLabels["cosmic-clash/region"] != "EU" {
t.Fatalf("body=%+v", body)
}
if body.Spec.Metadata.Annotations["cosmic-clash.io/match-id"] != "match-1" || body.Spec.Metadata.Annotations["cosmic-clash.io/allocation-id"] != "allocation-1" {
t.Fatalf("allocation did not request match/allocation ID annotations on the GameServer: %+v", body.Spec.Metadata.Annotations)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"status":{"state":"Allocated","gameServerName":"gs-a","address":"2001:db8::1","ports":[{"name":"default","port":7777}]}}`))
}))
+1 -1
View File
@@ -44,7 +44,7 @@ func main() {
controlPlaneURL := options.String("control-plane-url", "", "matchmaking control-plane base URL; empty skips process-ready registration entirely")
workloadTokenPath := options.String("workload-token-path", "", "path to the projected workload service-account token, read fresh on every registration call")
serverIDEnv := options.String("server-id-env", "COSMIC_CLASH_SERVER_ID", "environment variable containing this GameServer's control-plane server ID (populate via the Kubernetes Downward API, fieldRef: metadata.name)")
matchIDEnv := options.String("match-id-env", "COSMIC_CLASH_MATCH_ID", "environment variable containing the allocated match ID")
matchIDEnv := options.String("match-id-env", "COSMIC_CLASH_MATCH_ID", "environment variable containing the allocated match ID; if unset/empty, falls back to the cosmic-clash.io/match-id annotation on the allocated GameServer")
protocolVersion := options.Int("protocol-version", 0, "protocol version reported at registration")
imageDigestEnv := options.String("image-digest-env", "COSMIC_CLASH_IMAGE_DIGEST", "environment variable containing this build's sha256 image digest")
if err := options.Parse(args[:separator]); err != nil {
+55 -13
View File
@@ -19,6 +19,19 @@ import (
)
type GameServer struct {
// ObjectMeta.Annotations carries per-allocation data the agones package
// requests on the GameServerAllocation (server/agones/allocation.go) --
// currently cosmic-clash.io/match-id and cosmic-clash.io/allocation-id.
// This is the only channel for match-specific config to reach an
// already-Ready pod: env vars are fixed at pod creation, before Agones
// assigns a match to it. NOTE: the exact JSON key for this field
// (object_meta vs objectMeta) is not independently verified against a
// live Agones SDK sidecar from this sandbox; if it turns out wrong,
// annotationMatchID simply returns "" and callers fall back to whatever
// was explicitly configured, so this degrades safely either way.
ObjectMeta struct {
Annotations map[string]string `json:"annotations"`
} `json:"object_meta"`
Status struct {
Address string `json:"address"`
Ports []struct {
@@ -46,10 +59,11 @@ type Config struct {
// exactly -- direct/Compose mode and allocated-without-control-plane mode
// are both unaffected. WorkloadTokenPath is read fresh on every call
// rather than cached, matching how a Kubernetes projected service account
// token is rotated in place by kubelet before it expires; ServerID,
// MatchID and ImageDigest are expected to be populated from the pod spec
// (Downward API / mounted build metadata), not guessed at from the
// Agones SDK's own GameServer response.
// token is rotated in place by kubelet before it expires; ServerID and
// ImageDigest are expected to be populated from the pod spec (Downward
// API / mounted build metadata). MatchID may be left empty here and is
// then read from the allocated GameServer's own annotations (see
// GameServer.ObjectMeta above) -- an explicit value here always wins.
ControlPlaneURL string
WorkloadTokenPath string
ServerID string
@@ -59,9 +73,10 @@ type Config struct {
}
type Supervisor struct {
config Config
client *http.Client
cmd *exec.Cmd
config Config
client *http.Client
cmd *exec.Cmd
lastGameServer GameServer
}
const DefaultDrainGrace = 285 * time.Second
@@ -93,9 +108,13 @@ func New(config Config) (*Supervisor, error) {
return nil, err
}
}
if config.ControlPlaneURL != "" && (config.WorkloadTokenPath == "" || config.ServerID == "" || config.MatchID == "" || config.ProtocolVersion < 1 || config.ImageDigest == "") {
return nil, fmt.Errorf("control-plane registration requires a workload token path, server ID, match ID, protocol version and image digest")
if config.ControlPlaneURL != "" && (config.WorkloadTokenPath == "" || config.ServerID == "" || config.ProtocolVersion < 1 || config.ImageDigest == "") {
return nil, fmt.Errorf("control-plane registration requires a workload token path, server ID, protocol version and image digest")
}
// MatchID is deliberately not required here: it can also be resolved at
// Start time from the allocated GameServer's own annotations (see
// registerControlPlane). It is validated to actually be resolvable
// there, not silently skipped.
return &Supervisor{config: config, client: config.HTTPClient}, nil
}
@@ -168,10 +187,26 @@ func (s *Supervisor) Start(ctx context.Context) error {
// a Kubernetes projected service account token is rotated in place by
// kubelet before it expires, so caching it risks presenting a stale one on a
// long-lived process.
// matchID resolves the match ID for control-plane registration: an
// explicitly configured value always wins, otherwise it falls back to the
// cosmic-clash.io/match-id annotation Agones applied to this GameServer at
// allocation time (see server/agones.Client.Allocate). Empty if neither is
// available.
func (s *Supervisor) matchID() string {
if s.config.MatchID != "" {
return s.config.MatchID
}
return s.lastGameServer.ObjectMeta.Annotations["cosmic-clash.io/match-id"]
}
func (s *Supervisor) registerControlPlane(ctx context.Context, assignmentReady bool) error {
if s.config.ControlPlaneURL == "" {
return nil
}
matchID := s.matchID()
if matchID == "" {
return fmt.Errorf("control-plane registration has no match ID: not configured, and no cosmic-clash.io/match-id annotation was present on the allocated GameServer")
}
tokenBytes, err := os.ReadFile(s.config.WorkloadTokenPath)
if err != nil {
return fmt.Errorf("read workload token: %w", err)
@@ -185,7 +220,7 @@ func (s *Supervisor) registerControlPlane(ctx context.Context, assignmentReady b
ProtocolVersion int `json:"protocol_version"`
ImageDigest string `json:"image_digest"`
AssignmentReady bool `json:"assignment_ready"`
}{s.config.MatchID, s.config.ProtocolVersion, s.config.ImageDigest, assignmentReady})
}{matchID, s.config.ProtocolVersion, s.config.ImageDigest, assignmentReady})
if err != nil {
return err
}
@@ -196,9 +231,15 @@ func (s *Supervisor) registerControlPlane(ctx context.Context, assignmentReady b
}
request.Header.Set("Content-Type", "application/json")
request.Header.Set("Authorization", "Bearer "+token)
// Idempotent per (server, readiness stage): a supervisor restart or a
// dropped response retrying this exact call must replay, not conflict.
request.Header.Set("Idempotency-Key", "supervisor-register-"+s.config.ServerID+"-"+strconv.FormatBool(assignmentReady))
// Idempotent per (server, match, readiness stage): a supervisor restart
// or a dropped response retrying this exact call must replay, not
// conflict. The API enforces a 16-128 byte key; ServerID and MatchID are
// both already required non-empty by this point.
key := "supervisor-register-" + s.config.ServerID + "-" + matchID + "-" + strconv.FormatBool(assignmentReady)
if len(key) > 128 {
key = key[:128]
}
request.Header.Set("Idempotency-Key", key)
response, err := s.client.Do(request)
if err != nil {
return err
@@ -308,6 +349,7 @@ func (s *Supervisor) assignedEndpoint(ctx context.Context) (int, string, error)
if err := s.sdkGet(ctx, "/gameserver", &server); err != nil {
return 0, "", err
}
s.lastGameServer = server
if len(server.Status.Ports) == 0 || strings.TrimSpace(server.Status.Address) == "" || strings.ContainsAny(server.Status.Address, " \t\r\n") {
return 0, "", fmt.Errorf("Agones returned no assigned endpoint")
}
+77
View File
@@ -127,6 +127,83 @@ func TestControlPlaneRegistrationReportsProcessReadyWithWorkloadToken(t *testing
}
}
func TestControlPlaneRegistrationFallsBackToGameServerAnnotationForMatchID(t *testing.T) {
var gotBody string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/gameserver":
_, _ = w.Write([]byte(`{"object_meta":{"annotations":{"cosmic-clash.io/match-id":"match-from-annotation","cosmic-clash.io/allocation-id":"allocation-xyz"}},"status":{"address":"203.0.113.9","ports":[{"name":"game","port":31001}]}}`))
case "/ready-probe", "/ready":
w.WriteHeader(http.StatusOK)
case "/v1/servers/server-1/register":
body, _ := io.ReadAll(r.Body)
gotBody = string(body)
w.WriteHeader(http.StatusNoContent)
default:
w.WriteHeader(http.StatusNotFound)
}
}))
defer server.Close()
tokenPath := filepath.Join(t.TempDir(), "token")
if err := os.WriteFile(tokenPath, []byte("workload-jwt"), 0o600); err != nil {
t.Fatal(err)
}
// Deliberately no MatchID in config -- only the GameServer's own
// annotation supplies it, proving the fallback path itself, not just
// that an explicitly configured value gets sent.
s, err := New(Config{
Command: []string{"/bin/sh", "-c", "true"}, SDKBaseURL: server.URL, ReadyURL: server.URL + "/ready-probe", ReadyTimeout: time.Second, PollInterval: time.Millisecond,
ControlPlaneURL: server.URL, WorkloadTokenPath: tokenPath, ServerID: "server-1", ProtocolVersion: 1, ImageDigest: "sha256:aa",
})
if err != nil {
t.Fatal(err)
}
if err := s.Start(context.Background()); err != nil {
t.Fatal(err)
}
_ = s.Wait()
if !strings.Contains(gotBody, `"match_id":"match-from-annotation"`) {
t.Fatalf("register body did not use the GameServer annotation's match ID: %s", gotBody)
}
}
func TestControlPlaneRegistrationWithoutMatchIDOrAnnotationFailsClosed(t *testing.T) {
registerCalled := false
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/gameserver":
_, _ = w.Write([]byte(`{"status":{"address":"203.0.113.9","ports":[{"name":"game","port":31001}]}}`))
case "/ready-probe", "/ready":
w.WriteHeader(http.StatusOK)
case "/v1/servers/server-1/register":
registerCalled = true
w.WriteHeader(http.StatusNoContent)
default:
w.WriteHeader(http.StatusNotFound)
}
}))
defer server.Close()
tokenPath := filepath.Join(t.TempDir(), "token")
if err := os.WriteFile(tokenPath, []byte("workload-jwt"), 0o600); err != nil {
t.Fatal(err)
}
s, err := New(Config{
Command: []string{"/bin/sh", "-c", "sleep 30"}, SDKBaseURL: server.URL, ReadyURL: server.URL + "/ready-probe", ReadyTimeout: time.Second, PollInterval: time.Millisecond,
ControlPlaneURL: server.URL, WorkloadTokenPath: tokenPath, ServerID: "server-1", ProtocolVersion: 1, ImageDigest: "sha256:aa",
})
if err != nil {
t.Fatal(err)
}
if err := s.Start(context.Background()); err == nil {
t.Fatal("Start succeeded with no match ID available from either config or annotations")
}
if registerCalled {
t.Fatal("register was called despite having no match ID to send")
}
}
func TestControlPlaneRegistrationFailureKillsChildRatherThanRunningUnregistered(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {