diff --git a/server/agones/allocation.go b/server/agones/allocation.go index 1dc369af..1cd47c6b 100644 --- a/server/agones/allocation.go +++ b/server/agones/allocation.go @@ -16,12 +16,30 @@ import ( "time" "github.com/cosmic-clash/cosmic-clash/server/domain" + "github.com/cosmic-clash/cosmic-clash/server/workload" ) type Client struct { BaseURL string Namespace string HTTP *http.Client + + // WorkloadSecret, when set, mints a control-plane-self-issued signed + // workload token (server/workload/signed_token.go) for every allocation + // and requests it as the cosmic-clash.io/workload-token annotation + // alongside match-id/allocation-id -- the delivery channel + // supervisor.Supervisor.workloadToken() reads from. It must be the same + // secret cmd/control-plane verifies with (--workload-secret / + // COSMIC_CLASH_WORKLOAD_SECRET). Left unset, Allocate behaves exactly as + // before: no workload-token annotation is requested, matching how a + // deployment not yet using this delivery path (e.g. one still building + // toward a Kubernetes-JWT WorkloadVerify) is unaffected. + WorkloadSecret []byte + // WorkloadTokenTTL bounds how long the minted token remains valid; it + // must comfortably exceed the time between allocation and this + // GameServer completing process-ready/assignment-ready registration. + // Zero defaults to 30 minutes. + WorkloadTokenTTL time.Duration } type AllocatedServer struct { @@ -155,6 +173,17 @@ func (c Client) Allocate(ctx context.Context, request domain.AllocationRequest, "cosmic-clash.io/match-id": request.MatchID, "cosmic-clash.io/allocation-id": request.AllocationID, } + if len(c.WorkloadSecret) > 0 { + ttl := c.WorkloadTokenTTL + if ttl <= 0 { + ttl = 30 * time.Minute + } + token, err := workload.IssueSignedWorkloadToken(c.WorkloadSecret, request.AllocationID, now, ttl) + if err != nil { + return AllocatedServer{}, fmt.Errorf("issue workload token: %w", err) + } + body.Spec.Metadata.Annotations["cosmic-clash.io/workload-token"] = token + } encoded, err := json.Marshal(body) if err != nil { return AllocatedServer{}, err diff --git a/server/agones/allocation_test.go b/server/agones/allocation_test.go index aa9831b3..c8d83dc7 100644 --- a/server/agones/allocation_test.go +++ b/server/agones/allocation_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/cosmic-clash/cosmic-clash/server/domain" + "github.com/cosmic-clash/cosmic-clash/server/workload" ) func request() domain.AllocationRequest { @@ -44,6 +45,56 @@ func TestAllocateBuildsStrictGameServerAllocationAndEndpoint(t *testing.T) { } } +// TestAllocateRequestsAWorkloadTokenAnnotationWhenConfigured proves the +// delivery-channel wiring for the control-plane's self-issued signed token +// (server/workload/signed_token.go): with WorkloadSecret set, Allocate +// requests a cosmic-clash.io/workload-token annotation whose value actually +// parses and verifies against that same secret and names this allocation's +// ID -- the exact thing supervisor.Supervisor.workloadToken() reads back +// and cmd/control-plane's WorkloadVerify checks. With WorkloadSecret unset +// (the default), no such annotation is requested at all, leaving deployments +// not yet using this delivery path unaffected. +func TestAllocateRequestsAWorkloadTokenAnnotationWhenConfigured(t *testing.T) { + secret := []byte("agones-integration-secret") + var gotAnnotations map[string]string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body allocationRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatal(err) + } + gotAnnotations = body.Spec.Metadata.Annotations + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"status":{"state":"Allocated","gameServerName":"gs-a","address":"203.0.113.9","ports":[{"name":"default","port":7777}]}}`)) + })) + defer server.Close() + + now := time.Unix(1000, 0) + client := Client{BaseURL: server.URL, Namespace: "games", HTTP: server.Client(), WorkloadSecret: secret} + if _, err := client.Allocate(context.Background(), request(), map[string]string{"cosmic-clash/region": "EU"}, now); err != nil { + t.Fatal(err) + } + token := gotAnnotations["cosmic-clash.io/workload-token"] + if token == "" { + t.Fatal("Allocate did not request a cosmic-clash.io/workload-token annotation with WorkloadSecret configured") + } + claims, err := workload.ParseSignedWorkloadToken(secret, token, now.Add(time.Second)) + if err != nil { + t.Fatalf("minted token does not verify against the same secret: %v", err) + } + if claims.AllocationID != "allocation-1" { + t.Fatalf("token names allocation %q, want %q", claims.AllocationID, "allocation-1") + } + + gotAnnotations = nil + unsigned := Client{BaseURL: server.URL, Namespace: "games", HTTP: server.Client()} + if _, err := unsigned.Allocate(context.Background(), request(), map[string]string{"cosmic-clash/region": "EU"}, now); err != nil { + t.Fatal(err) + } + if _, ok := gotAnnotations["cosmic-clash.io/workload-token"]; ok { + t.Fatal("Allocate requested a workload-token annotation with no WorkloadSecret configured") + } +} + func TestAllocateFailsClosedOnMalformedProviderResponses(t *testing.T) { cases := []string{ `{"status":{"state":"UnAllocated","gameServerName":"gs","address":"127.0.0.1","ports":[{"name":"default","port":7777}]}}`, diff --git a/server/cmd/allocator/main.go b/server/cmd/allocator/main.go index 11298bea..ac93ab1a 100644 --- a/server/cmd/allocator/main.go +++ b/server/cmd/allocator/main.go @@ -24,6 +24,7 @@ func main() { namespace := flag.String("agones-namespace", envOrDefault("COSMIC_CLASH_AGONES_NAMESPACE", "default"), "Agones namespace") transport := flag.String("transport", envOrDefault("COSMIC_CLASH_TRANSPORT", "enet"), "game transport: enet or steam_sdr") interval := flag.Duration("interval", time.Second, "allocation poll interval") + workloadSecret := flag.String("workload-secret", os.Getenv("COSMIC_CLASH_WORKLOAD_SECRET"), "HMAC secret for control-plane-issued workload tokens (see workload/signed_token.go); must match cmd/control-plane's own --workload-secret. Unset skips minting a cosmic-clash.io/workload-token annotation entirely") flag.Parse() if *dsn == "" || *agonesURL == "" { fatalf("--dsn/COSMIC_CLASH_POSTGRES_DSN and --agones-url/COSMIC_CLASH_AGONES_URL are required") @@ -44,8 +45,11 @@ func main() { if err := migrations.Apply(startupCtx, db, *migrationDir); err != nil { fatalf("apply migrations: %v", err) } + if *workloadSecret == "" { + log.Printf("allocator: warning: --workload-secret / COSMIC_CLASH_WORKLOAD_SECRET is unset; allocated GameServers will receive no cosmic-clash.io/workload-token annotation, and control-plane registration will fail unless a --workload-token-path is separately configured on the supervisor") + } now := func() time.Time { return time.Now().UTC() } - client := agones.Client{BaseURL: *agonesURL, Namespace: *namespace} + client := agones.Client{BaseURL: *agonesURL, Namespace: *namespace, WorkloadSecret: []byte(*workloadSecret)} worker := allocator.Worker{ Claims: store.AllocatingMatchClaims{DB: db, Transport: *transport}, Service: allocator.Service{ diff --git a/server/cmd/game-server-supervisor/main.go b/server/cmd/game-server-supervisor/main.go index 6a335a82..5655a382 100644 --- a/server/cmd/game-server-supervisor/main.go +++ b/server/cmd/game-server-supervisor/main.go @@ -42,7 +42,7 @@ func main() { transport := options.String("transport", "enet", "enet or steam_sdr") grace := options.Duration("drain-grace", supervisor.DefaultDrainGrace, "maximum graceful drain duration") controlPlaneURL := options.String("control-plane-url", "", "matchmaking control-plane base URL; empty skips process-ready registration entirely") - workloadTokenPath := options.String("workload-token-path", "", "path to the projected workload service-account token, read fresh on every registration call") + workloadTokenPath := options.String("workload-token-path", "", "path to a projected workload service-account token, read fresh on every registration call; if unset, falls back to the cosmic-clash.io/workload-token annotation Agones applied to this GameServer at allocation time") serverIDEnv := options.String("server-id-env", "COSMIC_CLASH_SERVER_ID", "environment variable containing this GameServer's control-plane server ID (populate via the Kubernetes Downward API, fieldRef: metadata.name)") matchIDEnv := options.String("match-id-env", "COSMIC_CLASH_MATCH_ID", "environment variable containing the allocated match ID; if unset/empty, falls back to the cosmic-clash.io/match-id annotation on the allocated GameServer") protocolVersion := options.Int("protocol-version", 0, "protocol version reported at registration") diff --git a/server/supervisor/supervisor.go b/server/supervisor/supervisor.go index 4b156915..f5a1d712 100644 --- a/server/supervisor/supervisor.go +++ b/server/supervisor/supervisor.go @@ -57,13 +57,20 @@ type Config struct { // matchmaking control plane (multiplayer-next.md task 8.28) once Agones // Ready succeeds. Leaving it empty preserves every existing behavior // exactly -- direct/Compose mode and allocated-without-control-plane mode - // are both unaffected. WorkloadTokenPath is read fresh on every call - // rather than cached, matching how a Kubernetes projected service account - // token is rotated in place by kubelet before it expires; ServerID and - // ImageDigest are expected to be populated from the pod spec (Downward - // API / mounted build metadata). MatchID may be left empty here and is - // then read from the allocated GameServer's own annotations (see - // GameServer.ObjectMeta above) -- an explicit value here always wins. + // are both unaffected. WorkloadTokenPath, if set, is read fresh on every + // call rather than cached, matching how a Kubernetes projected service + // account token is rotated in place by kubelet before it expires -- this + // is for a future Kubernetes-JWT-based WorkloadVerify (server/workload/ + // jwt.go), not yet wired server-side. Today the control plane instead + // verifies a self-issued signed token (server/workload/signed_token.go), + // which reaches this process via the cosmic-clash.io/workload-token + // annotation Agones applies to the allocated GameServer (see + // server/agones.Client.Allocate) -- see workloadToken() for the + // precedence between the two sources. ServerID and ImageDigest are + // expected to be populated from the pod spec (Downward API / mounted + // build metadata). MatchID may be left empty here and is then read from + // the allocated GameServer's own annotations (see GameServer.ObjectMeta + // above) -- an explicit value here always wins. ControlPlaneURL string WorkloadTokenPath string ServerID string @@ -126,13 +133,13 @@ func New(config Config) (*Supervisor, error) { return nil, err } } - if config.ControlPlaneURL != "" && (config.WorkloadTokenPath == "" || config.ServerID == "" || config.ProtocolVersion < 1 || config.ImageDigest == "") { - return nil, fmt.Errorf("control-plane registration requires a workload token path, server ID, protocol version and image digest") + if config.ControlPlaneURL != "" && (config.ServerID == "" || config.ProtocolVersion < 1 || config.ImageDigest == "") { + return nil, fmt.Errorf("control-plane registration requires a server ID, protocol version and image digest") } - // MatchID is deliberately not required here: it can also be resolved at - // Start time from the allocated GameServer's own annotations (see - // registerControlPlane). It is validated to actually be resolvable - // there, not silently skipped. + // Neither MatchID nor WorkloadTokenPath is required here: both can + // instead be resolved at Start time from the allocated GameServer's own + // annotations (see registerControlPlane/workloadToken/matchID). They are + // validated to actually be resolvable there, not silently skipped. return &Supervisor{config: config, client: config.HTTPClient}, nil } @@ -243,6 +250,34 @@ func (s *Supervisor) matchID() string { return s.lastGameServer.ObjectMeta.Annotations["cosmic-clash.io/match-id"] } +// workloadToken resolves the bearer credential for control-plane +// registration. WorkloadTokenPath, when configured, always wins -- it is +// for a future Kubernetes-projected-JWT WorkloadVerify path (see the Config +// field's doc comment) and an operator who explicitly set it presumably +// wants it used. Otherwise it falls back to the cosmic-clash.io/workload- +// token annotation Agones applied to this GameServer at allocation time +// (server/agones.Client.Allocate, verified by +// api.WorkloadVerifierFromSignedToken today) -- the same annotation-fallback +// pattern matchID already uses for cosmic-clash.io/match-id. +func (s *Supervisor) workloadToken() (string, error) { + if s.config.WorkloadTokenPath != "" { + tokenBytes, err := os.ReadFile(s.config.WorkloadTokenPath) + if err != nil { + return "", fmt.Errorf("read workload token: %w", err) + } + token := strings.TrimSpace(string(tokenBytes)) + if token == "" { + return "", fmt.Errorf("workload token file %q is empty", s.config.WorkloadTokenPath) + } + return token, nil + } + token := s.lastGameServer.ObjectMeta.Annotations["cosmic-clash.io/workload-token"] + if token == "" { + return "", fmt.Errorf("control-plane registration has no workload token: no --workload-token-path configured, and no cosmic-clash.io/workload-token annotation was present on the allocated GameServer") + } + return token, nil +} + func (s *Supervisor) registerControlPlane(ctx context.Context, assignmentReady bool) error { if s.config.ControlPlaneURL == "" { return nil @@ -251,13 +286,9 @@ func (s *Supervisor) registerControlPlane(ctx context.Context, assignmentReady b if matchID == "" { return fmt.Errorf("control-plane registration has no match ID: not configured, and no cosmic-clash.io/match-id annotation was present on the allocated GameServer") } - tokenBytes, err := os.ReadFile(s.config.WorkloadTokenPath) + token, err := s.workloadToken() if err != nil { - return fmt.Errorf("read workload token: %w", err) - } - token := strings.TrimSpace(string(tokenBytes)) - if token == "" { - return fmt.Errorf("workload token file %q is empty", s.config.WorkloadTokenPath) + return err } body, err := json.Marshal(struct { MatchID string `json:"match_id"` diff --git a/server/supervisor/supervisor_test.go b/server/supervisor/supervisor_test.go index dafb58b4..5f99c12e 100644 --- a/server/supervisor/supervisor_test.go +++ b/server/supervisor/supervisor_test.go @@ -71,13 +71,22 @@ func TestAllocatedStartInjectsDynamicEndpointAndCallsReadyAfterProbe(t *testing. func TestControlPlaneRegistrationRejectsIncompleteConfig(t *testing.T) { base := Config{Command: []string{"/bin/true"}, ControlPlaneURL: "https://control-plane.invalid"} if _, err := New(base); err == nil { - t.Fatal("registration enabled with no token path/server/match/digest was accepted") + t.Fatal("registration enabled with no server/protocol/digest was accepted") } complete := base - complete.WorkloadTokenPath, complete.ServerID, complete.MatchID, complete.ProtocolVersion, complete.ImageDigest = "/tmp/token", "server-1", "match-1", 1, "sha256:aa" + complete.ServerID, complete.MatchID, complete.ProtocolVersion, complete.ImageDigest = "server-1", "match-1", 1, "sha256:aa" if _, err := New(complete); err != nil { t.Fatalf("fully configured registration rejected: %v", err) } + // WorkloadTokenPath is deliberately not required at construction time -- + // it can instead be resolved at Start time from the GameServer's own + // cosmic-clash.io/workload-token annotation (see workloadToken() and + // TestControlPlaneRegistrationFallsBackToGameServerAnnotationForWorkloadToken). + withTokenPath := complete + withTokenPath.WorkloadTokenPath = "/tmp/token" + if _, err := New(withTokenPath); err != nil { + t.Fatalf("configured token path rejected: %v", err) + } } func TestControlPlaneRegistrationReportsProcessReadyThenAssignmentReady(t *testing.T) { @@ -277,6 +286,84 @@ func TestControlPlaneRegistrationWithoutMatchIDOrAnnotationFailsClosed(t *testin } } +// TestControlPlaneRegistrationFallsBackToGameServerAnnotationForWorkloadToken +// proves the primary intended delivery channel for the control-plane's +// self-issued signed token (server/workload/signed_token.go): with no +// --workload-token-path configured at all, a token arriving only via the +// cosmic-clash.io/workload-token annotation Agones applies to this +// GameServer (server/agones.Client.Allocate) is what gets sent as the +// Authorization bearer. +func TestControlPlaneRegistrationFallsBackToGameServerAnnotationForWorkloadToken(t *testing.T) { + var gotAuth string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/gameserver": + _, _ = w.Write([]byte(`{"object_meta":{"annotations":{"cosmic-clash.io/match-id":"match-1","cosmic-clash.io/workload-token":"signed-token-from-annotation"}},"status":{"address":"203.0.113.9","ports":[{"name":"game","port":31001}]}}`)) + case "/ready-probe", "/ready": + w.WriteHeader(http.StatusOK) + case "/v1/servers/server-1/register": + gotAuth = r.Header.Get("Authorization") + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + + // Deliberately no WorkloadTokenPath -- only the GameServer annotation + // supplies a token, proving the fallback path itself. + s, err := New(Config{ + Command: []string{"/bin/sh", "-c", "true"}, SDKBaseURL: server.URL, ReadyURL: server.URL + "/ready-probe", ReadyTimeout: time.Second, PollInterval: time.Millisecond, + ControlPlaneURL: server.URL, ServerID: "server-1", ProtocolVersion: 1, ImageDigest: "sha256:aa", + }) + if err != nil { + t.Fatal(err) + } + if err := s.Start(context.Background()); err != nil { + t.Fatal(err) + } + _ = s.Wait() + if gotAuth != "Bearer signed-token-from-annotation" { + t.Fatalf("Authorization header = %q, want the annotation-sourced token", gotAuth) + } +} + +// TestControlPlaneRegistrationWithoutWorkloadTokenPathOrAnnotationFailsClosed +// is the workload-token counterpart to the match-ID fails-closed test above: +// with neither a configured token path nor an annotation present, Start must +// fail rather than register unauthenticated or with an empty token. +func TestControlPlaneRegistrationWithoutWorkloadTokenPathOrAnnotationFailsClosed(t *testing.T) { + registerCalled := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/gameserver": + _, _ = w.Write([]byte(`{"object_meta":{"annotations":{"cosmic-clash.io/match-id":"match-1"}},"status":{"address":"203.0.113.9","ports":[{"name":"game","port":31001}]}}`)) + case "/ready-probe", "/ready": + w.WriteHeader(http.StatusOK) + case "/v1/servers/server-1/register": + registerCalled = true + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + + s, err := New(Config{ + Command: []string{"/bin/sh", "-c", "sleep 30"}, SDKBaseURL: server.URL, ReadyURL: server.URL + "/ready-probe", ReadyTimeout: time.Second, PollInterval: time.Millisecond, + ControlPlaneURL: server.URL, ServerID: "server-1", ProtocolVersion: 1, ImageDigest: "sha256:aa", + }) + if err != nil { + t.Fatal(err) + } + if err := s.Start(context.Background()); err == nil { + t.Fatal("Start succeeded with no workload token available from either config or annotations") + } + if registerCalled { + t.Fatal("register was called despite having no workload token to send") + } +} + func TestControlPlaneRegistrationFailureKillsChildRatherThanRunningUnregistered(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path {