diff --git a/server/agones/allocation.go b/server/agones/allocation.go index 13027bb9..1dc369af 100644 --- a/server/agones/allocation.go +++ b/server/agones/allocation.go @@ -37,6 +37,18 @@ type allocationRequest struct { Selectors []struct { MatchLabels map[string]string `json:"matchLabels"` } `json:"selectors"` + // Metadata.Annotations is applied to the allocated GameServer's own + // object_meta by Agones on successful allocation (a documented part + // of the GameServerAllocation spec, independent of the Selectors + // used to find capacity). This is the only way match-specific data + // reaches an already-Ready pod after allocation: Kubernetes env vars + // are fixed at pod creation, long before Agones assigns a match to + // that pod, so there is no other channel for it. The allocated + // process reads these back via the SDK's own GameServer call + // (server/supervisor's existing /gameserver request). + Metadata struct { + Annotations map[string]string `json:"annotations"` + } `json:"metadata"` } `json:"spec"` } @@ -139,6 +151,10 @@ func (c Client) Allocate(ctx context.Context, request domain.AllocationRequest, body.Spec.Selectors = []struct { MatchLabels map[string]string `json:"matchLabels"` }{{MatchLabels: cloneLabels(labels)}} + body.Spec.Metadata.Annotations = map[string]string{ + "cosmic-clash.io/match-id": request.MatchID, + "cosmic-clash.io/allocation-id": request.AllocationID, + } encoded, err := json.Marshal(body) if err != nil { return AllocatedServer{}, err diff --git a/server/agones/allocation_test.go b/server/agones/allocation_test.go index 59126e83..aa9831b3 100644 --- a/server/agones/allocation_test.go +++ b/server/agones/allocation_test.go @@ -28,6 +28,9 @@ func TestAllocateBuildsStrictGameServerAllocationAndEndpoint(t *testing.T) { if body.APIVersion != "allocation.agones.dev/v1" || body.Kind != "GameServerAllocation" || len(body.Spec.Selectors) != 1 || body.Spec.Selectors[0].MatchLabels["cosmic-clash/region"] != "EU" { t.Fatalf("body=%+v", body) } + if body.Spec.Metadata.Annotations["cosmic-clash.io/match-id"] != "match-1" || body.Spec.Metadata.Annotations["cosmic-clash.io/allocation-id"] != "allocation-1" { + t.Fatalf("allocation did not request match/allocation ID annotations on the GameServer: %+v", body.Spec.Metadata.Annotations) + } w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"status":{"state":"Allocated","gameServerName":"gs-a","address":"2001:db8::1","ports":[{"name":"default","port":7777}]}}`)) })) diff --git a/server/cmd/game-server-supervisor/main.go b/server/cmd/game-server-supervisor/main.go index 71c741fb..b273306b 100644 --- a/server/cmd/game-server-supervisor/main.go +++ b/server/cmd/game-server-supervisor/main.go @@ -44,7 +44,7 @@ func main() { controlPlaneURL := options.String("control-plane-url", "", "matchmaking control-plane base URL; empty skips process-ready registration entirely") workloadTokenPath := options.String("workload-token-path", "", "path to the projected workload service-account token, read fresh on every registration call") serverIDEnv := options.String("server-id-env", "COSMIC_CLASH_SERVER_ID", "environment variable containing this GameServer's control-plane server ID (populate via the Kubernetes Downward API, fieldRef: metadata.name)") - matchIDEnv := options.String("match-id-env", "COSMIC_CLASH_MATCH_ID", "environment variable containing the allocated match ID") + matchIDEnv := options.String("match-id-env", "COSMIC_CLASH_MATCH_ID", "environment variable containing the allocated match ID; if unset/empty, falls back to the cosmic-clash.io/match-id annotation on the allocated GameServer") protocolVersion := options.Int("protocol-version", 0, "protocol version reported at registration") imageDigestEnv := options.String("image-digest-env", "COSMIC_CLASH_IMAGE_DIGEST", "environment variable containing this build's sha256 image digest") if err := options.Parse(args[:separator]); err != nil { diff --git a/server/supervisor/supervisor.go b/server/supervisor/supervisor.go index 6b44ba69..5e4c00e6 100644 --- a/server/supervisor/supervisor.go +++ b/server/supervisor/supervisor.go @@ -19,6 +19,19 @@ import ( ) type GameServer struct { + // ObjectMeta.Annotations carries per-allocation data the agones package + // requests on the GameServerAllocation (server/agones/allocation.go) -- + // currently cosmic-clash.io/match-id and cosmic-clash.io/allocation-id. + // This is the only channel for match-specific config to reach an + // already-Ready pod: env vars are fixed at pod creation, before Agones + // assigns a match to it. NOTE: the exact JSON key for this field + // (object_meta vs objectMeta) is not independently verified against a + // live Agones SDK sidecar from this sandbox; if it turns out wrong, + // annotationMatchID simply returns "" and callers fall back to whatever + // was explicitly configured, so this degrades safely either way. + ObjectMeta struct { + Annotations map[string]string `json:"annotations"` + } `json:"object_meta"` Status struct { Address string `json:"address"` Ports []struct { @@ -46,10 +59,11 @@ type Config struct { // exactly -- direct/Compose mode and allocated-without-control-plane mode // are both unaffected. WorkloadTokenPath is read fresh on every call // rather than cached, matching how a Kubernetes projected service account - // token is rotated in place by kubelet before it expires; ServerID, - // MatchID and ImageDigest are expected to be populated from the pod spec - // (Downward API / mounted build metadata), not guessed at from the - // Agones SDK's own GameServer response. + // token is rotated in place by kubelet before it expires; ServerID and + // ImageDigest are expected to be populated from the pod spec (Downward + // API / mounted build metadata). MatchID may be left empty here and is + // then read from the allocated GameServer's own annotations (see + // GameServer.ObjectMeta above) -- an explicit value here always wins. ControlPlaneURL string WorkloadTokenPath string ServerID string @@ -59,9 +73,10 @@ type Config struct { } type Supervisor struct { - config Config - client *http.Client - cmd *exec.Cmd + config Config + client *http.Client + cmd *exec.Cmd + lastGameServer GameServer } const DefaultDrainGrace = 285 * time.Second @@ -93,9 +108,13 @@ func New(config Config) (*Supervisor, error) { return nil, err } } - if config.ControlPlaneURL != "" && (config.WorkloadTokenPath == "" || config.ServerID == "" || config.MatchID == "" || config.ProtocolVersion < 1 || config.ImageDigest == "") { - return nil, fmt.Errorf("control-plane registration requires a workload token path, server ID, match ID, protocol version and image digest") + if config.ControlPlaneURL != "" && (config.WorkloadTokenPath == "" || config.ServerID == "" || config.ProtocolVersion < 1 || config.ImageDigest == "") { + return nil, fmt.Errorf("control-plane registration requires a workload token path, server ID, protocol version and image digest") } + // MatchID is deliberately not required here: it can also be resolved at + // Start time from the allocated GameServer's own annotations (see + // registerControlPlane). It is validated to actually be resolvable + // there, not silently skipped. return &Supervisor{config: config, client: config.HTTPClient}, nil } @@ -168,10 +187,26 @@ func (s *Supervisor) Start(ctx context.Context) error { // a Kubernetes projected service account token is rotated in place by // kubelet before it expires, so caching it risks presenting a stale one on a // long-lived process. +// matchID resolves the match ID for control-plane registration: an +// explicitly configured value always wins, otherwise it falls back to the +// cosmic-clash.io/match-id annotation Agones applied to this GameServer at +// allocation time (see server/agones.Client.Allocate). Empty if neither is +// available. +func (s *Supervisor) matchID() string { + if s.config.MatchID != "" { + return s.config.MatchID + } + return s.lastGameServer.ObjectMeta.Annotations["cosmic-clash.io/match-id"] +} + func (s *Supervisor) registerControlPlane(ctx context.Context, assignmentReady bool) error { if s.config.ControlPlaneURL == "" { return nil } + matchID := s.matchID() + if matchID == "" { + return fmt.Errorf("control-plane registration has no match ID: not configured, and no cosmic-clash.io/match-id annotation was present on the allocated GameServer") + } tokenBytes, err := os.ReadFile(s.config.WorkloadTokenPath) if err != nil { return fmt.Errorf("read workload token: %w", err) @@ -185,7 +220,7 @@ func (s *Supervisor) registerControlPlane(ctx context.Context, assignmentReady b ProtocolVersion int `json:"protocol_version"` ImageDigest string `json:"image_digest"` AssignmentReady bool `json:"assignment_ready"` - }{s.config.MatchID, s.config.ProtocolVersion, s.config.ImageDigest, assignmentReady}) + }{matchID, s.config.ProtocolVersion, s.config.ImageDigest, assignmentReady}) if err != nil { return err } @@ -196,9 +231,15 @@ func (s *Supervisor) registerControlPlane(ctx context.Context, assignmentReady b } request.Header.Set("Content-Type", "application/json") request.Header.Set("Authorization", "Bearer "+token) - // Idempotent per (server, readiness stage): a supervisor restart or a - // dropped response retrying this exact call must replay, not conflict. - request.Header.Set("Idempotency-Key", "supervisor-register-"+s.config.ServerID+"-"+strconv.FormatBool(assignmentReady)) + // Idempotent per (server, match, readiness stage): a supervisor restart + // or a dropped response retrying this exact call must replay, not + // conflict. The API enforces a 16-128 byte key; ServerID and MatchID are + // both already required non-empty by this point. + key := "supervisor-register-" + s.config.ServerID + "-" + matchID + "-" + strconv.FormatBool(assignmentReady) + if len(key) > 128 { + key = key[:128] + } + request.Header.Set("Idempotency-Key", key) response, err := s.client.Do(request) if err != nil { return err @@ -308,6 +349,7 @@ func (s *Supervisor) assignedEndpoint(ctx context.Context) (int, string, error) if err := s.sdkGet(ctx, "/gameserver", &server); err != nil { return 0, "", err } + s.lastGameServer = server if len(server.Status.Ports) == 0 || strings.TrimSpace(server.Status.Address) == "" || strings.ContainsAny(server.Status.Address, " \t\r\n") { return 0, "", fmt.Errorf("Agones returned no assigned endpoint") } diff --git a/server/supervisor/supervisor_test.go b/server/supervisor/supervisor_test.go index 64ddde80..f7b5e3d2 100644 --- a/server/supervisor/supervisor_test.go +++ b/server/supervisor/supervisor_test.go @@ -127,6 +127,83 @@ func TestControlPlaneRegistrationReportsProcessReadyWithWorkloadToken(t *testing } } +func TestControlPlaneRegistrationFallsBackToGameServerAnnotationForMatchID(t *testing.T) { + var gotBody string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/gameserver": + _, _ = w.Write([]byte(`{"object_meta":{"annotations":{"cosmic-clash.io/match-id":"match-from-annotation","cosmic-clash.io/allocation-id":"allocation-xyz"}},"status":{"address":"203.0.113.9","ports":[{"name":"game","port":31001}]}}`)) + case "/ready-probe", "/ready": + w.WriteHeader(http.StatusOK) + case "/v1/servers/server-1/register": + body, _ := io.ReadAll(r.Body) + gotBody = string(body) + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + + tokenPath := filepath.Join(t.TempDir(), "token") + if err := os.WriteFile(tokenPath, []byte("workload-jwt"), 0o600); err != nil { + t.Fatal(err) + } + // Deliberately no MatchID in config -- only the GameServer's own + // annotation supplies it, proving the fallback path itself, not just + // that an explicitly configured value gets sent. + s, err := New(Config{ + Command: []string{"/bin/sh", "-c", "true"}, SDKBaseURL: server.URL, ReadyURL: server.URL + "/ready-probe", ReadyTimeout: time.Second, PollInterval: time.Millisecond, + ControlPlaneURL: server.URL, WorkloadTokenPath: tokenPath, ServerID: "server-1", ProtocolVersion: 1, ImageDigest: "sha256:aa", + }) + if err != nil { + t.Fatal(err) + } + if err := s.Start(context.Background()); err != nil { + t.Fatal(err) + } + _ = s.Wait() + if !strings.Contains(gotBody, `"match_id":"match-from-annotation"`) { + t.Fatalf("register body did not use the GameServer annotation's match ID: %s", gotBody) + } +} + +func TestControlPlaneRegistrationWithoutMatchIDOrAnnotationFailsClosed(t *testing.T) { + registerCalled := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/gameserver": + _, _ = w.Write([]byte(`{"status":{"address":"203.0.113.9","ports":[{"name":"game","port":31001}]}}`)) + case "/ready-probe", "/ready": + w.WriteHeader(http.StatusOK) + case "/v1/servers/server-1/register": + registerCalled = true + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + + tokenPath := filepath.Join(t.TempDir(), "token") + if err := os.WriteFile(tokenPath, []byte("workload-jwt"), 0o600); err != nil { + t.Fatal(err) + } + s, err := New(Config{ + Command: []string{"/bin/sh", "-c", "sleep 30"}, SDKBaseURL: server.URL, ReadyURL: server.URL + "/ready-probe", ReadyTimeout: time.Second, PollInterval: time.Millisecond, + ControlPlaneURL: server.URL, WorkloadTokenPath: tokenPath, ServerID: "server-1", ProtocolVersion: 1, ImageDigest: "sha256:aa", + }) + if err != nil { + t.Fatal(err) + } + if err := s.Start(context.Background()); err == nil { + t.Fatal("Start succeeded with no match ID available from either config or annotations") + } + if registerCalled { + t.Fatal("register was called despite having no match ID to send") + } +} + func TestControlPlaneRegistrationFailureKillsChildRatherThanRunningUnregistered(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path {