mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
feat(multiplayer): propagate match ID to an already-allocated pod via annotations
Investigated the Fleet-manifest wiring task flagged last commit and found a deeper, previously-undesigned gap: Kubernetes env vars are fixed at pod creation, but Agones allocates a match to an already- running Ready pod well after it starts -- so there was no channel at all for match-specific data (match ID) to reach that pod's processes. Close it using the Agones GameServerAllocation API's documented spec.metadata.annotations field, which Agones applies to the allocated GameServer's own object_meta on success: server/agones.Client.Allocate now requests cosmic-clash.io/match-id and cosmic-clash.io/allocation-id annotations, and the supervisor reads them back from the same /gameserver SDK call it already makes for the assigned port/address (GameServer.ObjectMeta.Annotations), falling back to them for its own control-plane registration only when MatchID isn't explicitly configured -- an explicit value always wins, and a match ID resolvable from neither source fails Start() closed before any HTTP call. The exact object_meta vs objectMeta JSON key from a live Agones SDK sidecar is not independently verified from this sandbox; documented inline, and the fallback degrades safely (empty annotations map, same as before this change) if it turns out to be wrong. Covered by two new tests: the annotation actually flowing through to the registration body, and fail-closed with neither config nor annotation supplying a match ID (registerCalled stays false, not just that Start() errors).
This commit is contained in:
@@ -37,6 +37,18 @@ type allocationRequest struct {
|
||||
Selectors []struct {
|
||||
MatchLabels map[string]string `json:"matchLabels"`
|
||||
} `json:"selectors"`
|
||||
// Metadata.Annotations is applied to the allocated GameServer's own
|
||||
// object_meta by Agones on successful allocation (a documented part
|
||||
// of the GameServerAllocation spec, independent of the Selectors
|
||||
// used to find capacity). This is the only way match-specific data
|
||||
// reaches an already-Ready pod after allocation: Kubernetes env vars
|
||||
// are fixed at pod creation, long before Agones assigns a match to
|
||||
// that pod, so there is no other channel for it. The allocated
|
||||
// process reads these back via the SDK's own GameServer call
|
||||
// (server/supervisor's existing /gameserver request).
|
||||
Metadata struct {
|
||||
Annotations map[string]string `json:"annotations"`
|
||||
} `json:"metadata"`
|
||||
} `json:"spec"`
|
||||
}
|
||||
|
||||
@@ -139,6 +151,10 @@ func (c Client) Allocate(ctx context.Context, request domain.AllocationRequest,
|
||||
body.Spec.Selectors = []struct {
|
||||
MatchLabels map[string]string `json:"matchLabels"`
|
||||
}{{MatchLabels: cloneLabels(labels)}}
|
||||
body.Spec.Metadata.Annotations = map[string]string{
|
||||
"cosmic-clash.io/match-id": request.MatchID,
|
||||
"cosmic-clash.io/allocation-id": request.AllocationID,
|
||||
}
|
||||
encoded, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return AllocatedServer{}, err
|
||||
|
||||
@@ -28,6 +28,9 @@ func TestAllocateBuildsStrictGameServerAllocationAndEndpoint(t *testing.T) {
|
||||
if body.APIVersion != "allocation.agones.dev/v1" || body.Kind != "GameServerAllocation" || len(body.Spec.Selectors) != 1 || body.Spec.Selectors[0].MatchLabels["cosmic-clash/region"] != "EU" {
|
||||
t.Fatalf("body=%+v", body)
|
||||
}
|
||||
if body.Spec.Metadata.Annotations["cosmic-clash.io/match-id"] != "match-1" || body.Spec.Metadata.Annotations["cosmic-clash.io/allocation-id"] != "allocation-1" {
|
||||
t.Fatalf("allocation did not request match/allocation ID annotations on the GameServer: %+v", body.Spec.Metadata.Annotations)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"status":{"state":"Allocated","gameServerName":"gs-a","address":"2001:db8::1","ports":[{"name":"default","port":7777}]}}`))
|
||||
}))
|
||||
|
||||
@@ -44,7 +44,7 @@ func main() {
|
||||
controlPlaneURL := options.String("control-plane-url", "", "matchmaking control-plane base URL; empty skips process-ready registration entirely")
|
||||
workloadTokenPath := options.String("workload-token-path", "", "path to the projected workload service-account token, read fresh on every registration call")
|
||||
serverIDEnv := options.String("server-id-env", "COSMIC_CLASH_SERVER_ID", "environment variable containing this GameServer's control-plane server ID (populate via the Kubernetes Downward API, fieldRef: metadata.name)")
|
||||
matchIDEnv := options.String("match-id-env", "COSMIC_CLASH_MATCH_ID", "environment variable containing the allocated match ID")
|
||||
matchIDEnv := options.String("match-id-env", "COSMIC_CLASH_MATCH_ID", "environment variable containing the allocated match ID; if unset/empty, falls back to the cosmic-clash.io/match-id annotation on the allocated GameServer")
|
||||
protocolVersion := options.Int("protocol-version", 0, "protocol version reported at registration")
|
||||
imageDigestEnv := options.String("image-digest-env", "COSMIC_CLASH_IMAGE_DIGEST", "environment variable containing this build's sha256 image digest")
|
||||
if err := options.Parse(args[:separator]); err != nil {
|
||||
|
||||
@@ -19,6 +19,19 @@ import (
|
||||
)
|
||||
|
||||
type GameServer struct {
|
||||
// ObjectMeta.Annotations carries per-allocation data the agones package
|
||||
// requests on the GameServerAllocation (server/agones/allocation.go) --
|
||||
// currently cosmic-clash.io/match-id and cosmic-clash.io/allocation-id.
|
||||
// This is the only channel for match-specific config to reach an
|
||||
// already-Ready pod: env vars are fixed at pod creation, before Agones
|
||||
// assigns a match to it. NOTE: the exact JSON key for this field
|
||||
// (object_meta vs objectMeta) is not independently verified against a
|
||||
// live Agones SDK sidecar from this sandbox; if it turns out wrong,
|
||||
// annotationMatchID simply returns "" and callers fall back to whatever
|
||||
// was explicitly configured, so this degrades safely either way.
|
||||
ObjectMeta struct {
|
||||
Annotations map[string]string `json:"annotations"`
|
||||
} `json:"object_meta"`
|
||||
Status struct {
|
||||
Address string `json:"address"`
|
||||
Ports []struct {
|
||||
@@ -46,10 +59,11 @@ type Config struct {
|
||||
// exactly -- direct/Compose mode and allocated-without-control-plane mode
|
||||
// are both unaffected. WorkloadTokenPath is read fresh on every call
|
||||
// rather than cached, matching how a Kubernetes projected service account
|
||||
// token is rotated in place by kubelet before it expires; ServerID,
|
||||
// MatchID and ImageDigest are expected to be populated from the pod spec
|
||||
// (Downward API / mounted build metadata), not guessed at from the
|
||||
// Agones SDK's own GameServer response.
|
||||
// token is rotated in place by kubelet before it expires; ServerID and
|
||||
// ImageDigest are expected to be populated from the pod spec (Downward
|
||||
// API / mounted build metadata). MatchID may be left empty here and is
|
||||
// then read from the allocated GameServer's own annotations (see
|
||||
// GameServer.ObjectMeta above) -- an explicit value here always wins.
|
||||
ControlPlaneURL string
|
||||
WorkloadTokenPath string
|
||||
ServerID string
|
||||
@@ -59,9 +73,10 @@ type Config struct {
|
||||
}
|
||||
|
||||
type Supervisor struct {
|
||||
config Config
|
||||
client *http.Client
|
||||
cmd *exec.Cmd
|
||||
config Config
|
||||
client *http.Client
|
||||
cmd *exec.Cmd
|
||||
lastGameServer GameServer
|
||||
}
|
||||
|
||||
const DefaultDrainGrace = 285 * time.Second
|
||||
@@ -93,9 +108,13 @@ func New(config Config) (*Supervisor, error) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if config.ControlPlaneURL != "" && (config.WorkloadTokenPath == "" || config.ServerID == "" || config.MatchID == "" || config.ProtocolVersion < 1 || config.ImageDigest == "") {
|
||||
return nil, fmt.Errorf("control-plane registration requires a workload token path, server ID, match ID, protocol version and image digest")
|
||||
if config.ControlPlaneURL != "" && (config.WorkloadTokenPath == "" || config.ServerID == "" || config.ProtocolVersion < 1 || config.ImageDigest == "") {
|
||||
return nil, fmt.Errorf("control-plane registration requires a workload token path, server ID, protocol version and image digest")
|
||||
}
|
||||
// MatchID is deliberately not required here: it can also be resolved at
|
||||
// Start time from the allocated GameServer's own annotations (see
|
||||
// registerControlPlane). It is validated to actually be resolvable
|
||||
// there, not silently skipped.
|
||||
return &Supervisor{config: config, client: config.HTTPClient}, nil
|
||||
}
|
||||
|
||||
@@ -168,10 +187,26 @@ func (s *Supervisor) Start(ctx context.Context) error {
|
||||
// a Kubernetes projected service account token is rotated in place by
|
||||
// kubelet before it expires, so caching it risks presenting a stale one on a
|
||||
// long-lived process.
|
||||
// matchID resolves the match ID for control-plane registration: an
|
||||
// explicitly configured value always wins, otherwise it falls back to the
|
||||
// cosmic-clash.io/match-id annotation Agones applied to this GameServer at
|
||||
// allocation time (see server/agones.Client.Allocate). Empty if neither is
|
||||
// available.
|
||||
func (s *Supervisor) matchID() string {
|
||||
if s.config.MatchID != "" {
|
||||
return s.config.MatchID
|
||||
}
|
||||
return s.lastGameServer.ObjectMeta.Annotations["cosmic-clash.io/match-id"]
|
||||
}
|
||||
|
||||
func (s *Supervisor) registerControlPlane(ctx context.Context, assignmentReady bool) error {
|
||||
if s.config.ControlPlaneURL == "" {
|
||||
return nil
|
||||
}
|
||||
matchID := s.matchID()
|
||||
if matchID == "" {
|
||||
return fmt.Errorf("control-plane registration has no match ID: not configured, and no cosmic-clash.io/match-id annotation was present on the allocated GameServer")
|
||||
}
|
||||
tokenBytes, err := os.ReadFile(s.config.WorkloadTokenPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read workload token: %w", err)
|
||||
@@ -185,7 +220,7 @@ func (s *Supervisor) registerControlPlane(ctx context.Context, assignmentReady b
|
||||
ProtocolVersion int `json:"protocol_version"`
|
||||
ImageDigest string `json:"image_digest"`
|
||||
AssignmentReady bool `json:"assignment_ready"`
|
||||
}{s.config.MatchID, s.config.ProtocolVersion, s.config.ImageDigest, assignmentReady})
|
||||
}{matchID, s.config.ProtocolVersion, s.config.ImageDigest, assignmentReady})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -196,9 +231,15 @@ func (s *Supervisor) registerControlPlane(ctx context.Context, assignmentReady b
|
||||
}
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
request.Header.Set("Authorization", "Bearer "+token)
|
||||
// Idempotent per (server, readiness stage): a supervisor restart or a
|
||||
// dropped response retrying this exact call must replay, not conflict.
|
||||
request.Header.Set("Idempotency-Key", "supervisor-register-"+s.config.ServerID+"-"+strconv.FormatBool(assignmentReady))
|
||||
// Idempotent per (server, match, readiness stage): a supervisor restart
|
||||
// or a dropped response retrying this exact call must replay, not
|
||||
// conflict. The API enforces a 16-128 byte key; ServerID and MatchID are
|
||||
// both already required non-empty by this point.
|
||||
key := "supervisor-register-" + s.config.ServerID + "-" + matchID + "-" + strconv.FormatBool(assignmentReady)
|
||||
if len(key) > 128 {
|
||||
key = key[:128]
|
||||
}
|
||||
request.Header.Set("Idempotency-Key", key)
|
||||
response, err := s.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -308,6 +349,7 @@ func (s *Supervisor) assignedEndpoint(ctx context.Context) (int, string, error)
|
||||
if err := s.sdkGet(ctx, "/gameserver", &server); err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
s.lastGameServer = server
|
||||
if len(server.Status.Ports) == 0 || strings.TrimSpace(server.Status.Address) == "" || strings.ContainsAny(server.Status.Address, " \t\r\n") {
|
||||
return 0, "", fmt.Errorf("Agones returned no assigned endpoint")
|
||||
}
|
||||
|
||||
@@ -127,6 +127,83 @@ func TestControlPlaneRegistrationReportsProcessReadyWithWorkloadToken(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestControlPlaneRegistrationFallsBackToGameServerAnnotationForMatchID(t *testing.T) {
|
||||
var gotBody string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/gameserver":
|
||||
_, _ = w.Write([]byte(`{"object_meta":{"annotations":{"cosmic-clash.io/match-id":"match-from-annotation","cosmic-clash.io/allocation-id":"allocation-xyz"}},"status":{"address":"203.0.113.9","ports":[{"name":"game","port":31001}]}}`))
|
||||
case "/ready-probe", "/ready":
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case "/v1/servers/server-1/register":
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
gotBody = string(body)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
tokenPath := filepath.Join(t.TempDir(), "token")
|
||||
if err := os.WriteFile(tokenPath, []byte("workload-jwt"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Deliberately no MatchID in config -- only the GameServer's own
|
||||
// annotation supplies it, proving the fallback path itself, not just
|
||||
// that an explicitly configured value gets sent.
|
||||
s, err := New(Config{
|
||||
Command: []string{"/bin/sh", "-c", "true"}, SDKBaseURL: server.URL, ReadyURL: server.URL + "/ready-probe", ReadyTimeout: time.Second, PollInterval: time.Millisecond,
|
||||
ControlPlaneURL: server.URL, WorkloadTokenPath: tokenPath, ServerID: "server-1", ProtocolVersion: 1, ImageDigest: "sha256:aa",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Start(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = s.Wait()
|
||||
if !strings.Contains(gotBody, `"match_id":"match-from-annotation"`) {
|
||||
t.Fatalf("register body did not use the GameServer annotation's match ID: %s", gotBody)
|
||||
}
|
||||
}
|
||||
|
||||
func TestControlPlaneRegistrationWithoutMatchIDOrAnnotationFailsClosed(t *testing.T) {
|
||||
registerCalled := false
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/gameserver":
|
||||
_, _ = w.Write([]byte(`{"status":{"address":"203.0.113.9","ports":[{"name":"game","port":31001}]}}`))
|
||||
case "/ready-probe", "/ready":
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case "/v1/servers/server-1/register":
|
||||
registerCalled = true
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
tokenPath := filepath.Join(t.TempDir(), "token")
|
||||
if err := os.WriteFile(tokenPath, []byte("workload-jwt"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err := New(Config{
|
||||
Command: []string{"/bin/sh", "-c", "sleep 30"}, SDKBaseURL: server.URL, ReadyURL: server.URL + "/ready-probe", ReadyTimeout: time.Second, PollInterval: time.Millisecond,
|
||||
ControlPlaneURL: server.URL, WorkloadTokenPath: tokenPath, ServerID: "server-1", ProtocolVersion: 1, ImageDigest: "sha256:aa",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Start(context.Background()); err == nil {
|
||||
t.Fatal("Start succeeded with no match ID available from either config or annotations")
|
||||
}
|
||||
if registerCalled {
|
||||
t.Fatal("register was called despite having no match ID to send")
|
||||
}
|
||||
}
|
||||
|
||||
func TestControlPlaneRegistrationFailureKillsChildRatherThanRunningUnregistered(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
|
||||
Reference in New Issue
Block a user