feat(multiplayer): deliver signed workload tokens via Agones allocation annotation

Closes the remaining gap the previous two commits left open: WorkloadVerify
itself worked, but nothing minted a real token at allocation time or handed
it to a running pod, so it had no real caller yet.

agones.Client gains WorkloadSecret/WorkloadTokenTTL. When set, Allocate
mints a signed workload token for the allocation (allocation_id is known at
request-construction time, before Agones has picked a server -- see the
previous commit for why that's the only identifier the token can bind) and
requests it as a third cosmic-clash.io/workload-token annotation, alongside
the existing match-id/allocation-id ones. Left unset (the default), Allocate
requests no such annotation, so a deployment not yet using this path is
unaffected. cmd/allocator wires it from a new --workload-secret /
COSMIC_CLASH_WORKLOAD_SECRET flag (must match cmd/control-plane's own), with
a startup warning if left unset.

supervisor.Supervisor.workloadToken() resolves the bearer credential for
control-plane registration: an explicitly configured --workload-token-path
always wins (kept for a future Kubernetes-projected-JWT WorkloadVerify path,
not yet wired server-side), otherwise it falls back to the
cosmic-clash.io/workload-token annotation on the allocated GameServer --
the same annotation-fallback pattern matchID already used for
cosmic-clash.io/match-id. WorkloadTokenPath is accordingly no longer
required at construction time when ControlPlaneURL is set.

Verified: new agones test proves the annotation is requested (and parses/
verifies against the same secret, naming the right allocation) when
WorkloadSecret is configured, and that it's absent when it isn't; new
supervisor tests prove the annotation-sourced token is what's actually sent
as the Authorization bearer, and that Start fails closed with neither a
configured path nor an annotation present. Full
`go build ./... && go vet ./... && gofmt -l . && go test ./... -race` and
`go test -tags integration ./... -race` both clean.
This commit is contained in:
Josh Creek
2026-09-01 14:57:45 +01:00
parent d588898f5d
commit 544f76c502
6 changed files with 225 additions and 23 deletions
+29
View File
@@ -16,12 +16,30 @@ import (
"time"
"github.com/cosmic-clash/cosmic-clash/server/domain"
"github.com/cosmic-clash/cosmic-clash/server/workload"
)
type Client struct {
BaseURL string
Namespace string
HTTP *http.Client
// WorkloadSecret, when set, mints a control-plane-self-issued signed
// workload token (server/workload/signed_token.go) for every allocation
// and requests it as the cosmic-clash.io/workload-token annotation
// alongside match-id/allocation-id -- the delivery channel
// supervisor.Supervisor.workloadToken() reads from. It must be the same
// secret cmd/control-plane verifies with (--workload-secret /
// COSMIC_CLASH_WORKLOAD_SECRET). Left unset, Allocate behaves exactly as
// before: no workload-token annotation is requested, matching how a
// deployment not yet using this delivery path (e.g. one still building
// toward a Kubernetes-JWT WorkloadVerify) is unaffected.
WorkloadSecret []byte
// WorkloadTokenTTL bounds how long the minted token remains valid; it
// must comfortably exceed the time between allocation and this
// GameServer completing process-ready/assignment-ready registration.
// Zero defaults to 30 minutes.
WorkloadTokenTTL time.Duration
}
type AllocatedServer struct {
@@ -155,6 +173,17 @@ func (c Client) Allocate(ctx context.Context, request domain.AllocationRequest,
"cosmic-clash.io/match-id": request.MatchID,
"cosmic-clash.io/allocation-id": request.AllocationID,
}
if len(c.WorkloadSecret) > 0 {
ttl := c.WorkloadTokenTTL
if ttl <= 0 {
ttl = 30 * time.Minute
}
token, err := workload.IssueSignedWorkloadToken(c.WorkloadSecret, request.AllocationID, now, ttl)
if err != nil {
return AllocatedServer{}, fmt.Errorf("issue workload token: %w", err)
}
body.Spec.Metadata.Annotations["cosmic-clash.io/workload-token"] = token
}
encoded, err := json.Marshal(body)
if err != nil {
return AllocatedServer{}, err
+51
View File
@@ -10,6 +10,7 @@ import (
"time"
"github.com/cosmic-clash/cosmic-clash/server/domain"
"github.com/cosmic-clash/cosmic-clash/server/workload"
)
func request() domain.AllocationRequest {
@@ -44,6 +45,56 @@ func TestAllocateBuildsStrictGameServerAllocationAndEndpoint(t *testing.T) {
}
}
// TestAllocateRequestsAWorkloadTokenAnnotationWhenConfigured proves the
// delivery-channel wiring for the control-plane's self-issued signed token
// (server/workload/signed_token.go): with WorkloadSecret set, Allocate
// requests a cosmic-clash.io/workload-token annotation whose value actually
// parses and verifies against that same secret and names this allocation's
// ID -- the exact thing supervisor.Supervisor.workloadToken() reads back
// and cmd/control-plane's WorkloadVerify checks. With WorkloadSecret unset
// (the default), no such annotation is requested at all, leaving deployments
// not yet using this delivery path unaffected.
func TestAllocateRequestsAWorkloadTokenAnnotationWhenConfigured(t *testing.T) {
secret := []byte("agones-integration-secret")
var gotAnnotations map[string]string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var body allocationRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatal(err)
}
gotAnnotations = body.Spec.Metadata.Annotations
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"status":{"state":"Allocated","gameServerName":"gs-a","address":"203.0.113.9","ports":[{"name":"default","port":7777}]}}`))
}))
defer server.Close()
now := time.Unix(1000, 0)
client := Client{BaseURL: server.URL, Namespace: "games", HTTP: server.Client(), WorkloadSecret: secret}
if _, err := client.Allocate(context.Background(), request(), map[string]string{"cosmic-clash/region": "EU"}, now); err != nil {
t.Fatal(err)
}
token := gotAnnotations["cosmic-clash.io/workload-token"]
if token == "" {
t.Fatal("Allocate did not request a cosmic-clash.io/workload-token annotation with WorkloadSecret configured")
}
claims, err := workload.ParseSignedWorkloadToken(secret, token, now.Add(time.Second))
if err != nil {
t.Fatalf("minted token does not verify against the same secret: %v", err)
}
if claims.AllocationID != "allocation-1" {
t.Fatalf("token names allocation %q, want %q", claims.AllocationID, "allocation-1")
}
gotAnnotations = nil
unsigned := Client{BaseURL: server.URL, Namespace: "games", HTTP: server.Client()}
if _, err := unsigned.Allocate(context.Background(), request(), map[string]string{"cosmic-clash/region": "EU"}, now); err != nil {
t.Fatal(err)
}
if _, ok := gotAnnotations["cosmic-clash.io/workload-token"]; ok {
t.Fatal("Allocate requested a workload-token annotation with no WorkloadSecret configured")
}
}
func TestAllocateFailsClosedOnMalformedProviderResponses(t *testing.T) {
cases := []string{
`{"status":{"state":"UnAllocated","gameServerName":"gs","address":"127.0.0.1","ports":[{"name":"default","port":7777}]}}`,
+5 -1
View File
@@ -24,6 +24,7 @@ func main() {
namespace := flag.String("agones-namespace", envOrDefault("COSMIC_CLASH_AGONES_NAMESPACE", "default"), "Agones namespace")
transport := flag.String("transport", envOrDefault("COSMIC_CLASH_TRANSPORT", "enet"), "game transport: enet or steam_sdr")
interval := flag.Duration("interval", time.Second, "allocation poll interval")
workloadSecret := flag.String("workload-secret", os.Getenv("COSMIC_CLASH_WORKLOAD_SECRET"), "HMAC secret for control-plane-issued workload tokens (see workload/signed_token.go); must match cmd/control-plane's own --workload-secret. Unset skips minting a cosmic-clash.io/workload-token annotation entirely")
flag.Parse()
if *dsn == "" || *agonesURL == "" {
fatalf("--dsn/COSMIC_CLASH_POSTGRES_DSN and --agones-url/COSMIC_CLASH_AGONES_URL are required")
@@ -44,8 +45,11 @@ func main() {
if err := migrations.Apply(startupCtx, db, *migrationDir); err != nil {
fatalf("apply migrations: %v", err)
}
if *workloadSecret == "" {
log.Printf("allocator: warning: --workload-secret / COSMIC_CLASH_WORKLOAD_SECRET is unset; allocated GameServers will receive no cosmic-clash.io/workload-token annotation, and control-plane registration will fail unless a --workload-token-path is separately configured on the supervisor")
}
now := func() time.Time { return time.Now().UTC() }
client := agones.Client{BaseURL: *agonesURL, Namespace: *namespace}
client := agones.Client{BaseURL: *agonesURL, Namespace: *namespace, WorkloadSecret: []byte(*workloadSecret)}
worker := allocator.Worker{
Claims: store.AllocatingMatchClaims{DB: db, Transport: *transport},
Service: allocator.Service{
+1 -1
View File
@@ -42,7 +42,7 @@ func main() {
transport := options.String("transport", "enet", "enet or steam_sdr")
grace := options.Duration("drain-grace", supervisor.DefaultDrainGrace, "maximum graceful drain duration")
controlPlaneURL := options.String("control-plane-url", "", "matchmaking control-plane base URL; empty skips process-ready registration entirely")
workloadTokenPath := options.String("workload-token-path", "", "path to the projected workload service-account token, read fresh on every registration call")
workloadTokenPath := options.String("workload-token-path", "", "path to a projected workload service-account token, read fresh on every registration call; if unset, falls back to the cosmic-clash.io/workload-token annotation Agones applied to this GameServer at allocation time")
serverIDEnv := options.String("server-id-env", "COSMIC_CLASH_SERVER_ID", "environment variable containing this GameServer's control-plane server ID (populate via the Kubernetes Downward API, fieldRef: metadata.name)")
matchIDEnv := options.String("match-id-env", "COSMIC_CLASH_MATCH_ID", "environment variable containing the allocated match ID; if unset/empty, falls back to the cosmic-clash.io/match-id annotation on the allocated GameServer")
protocolVersion := options.Int("protocol-version", 0, "protocol version reported at registration")
+50 -19
View File
@@ -57,13 +57,20 @@ type Config struct {
// matchmaking control plane (multiplayer-next.md task 8.28) once Agones
// Ready succeeds. Leaving it empty preserves every existing behavior
// exactly -- direct/Compose mode and allocated-without-control-plane mode
// are both unaffected. WorkloadTokenPath is read fresh on every call
// rather than cached, matching how a Kubernetes projected service account
// token is rotated in place by kubelet before it expires; ServerID and
// ImageDigest are expected to be populated from the pod spec (Downward
// API / mounted build metadata). MatchID may be left empty here and is
// then read from the allocated GameServer's own annotations (see
// GameServer.ObjectMeta above) -- an explicit value here always wins.
// are both unaffected. WorkloadTokenPath, if set, is read fresh on every
// call rather than cached, matching how a Kubernetes projected service
// account token is rotated in place by kubelet before it expires -- this
// is for a future Kubernetes-JWT-based WorkloadVerify (server/workload/
// jwt.go), not yet wired server-side. Today the control plane instead
// verifies a self-issued signed token (server/workload/signed_token.go),
// which reaches this process via the cosmic-clash.io/workload-token
// annotation Agones applies to the allocated GameServer (see
// server/agones.Client.Allocate) -- see workloadToken() for the
// precedence between the two sources. ServerID and ImageDigest are
// expected to be populated from the pod spec (Downward API / mounted
// build metadata). MatchID may be left empty here and is then read from
// the allocated GameServer's own annotations (see GameServer.ObjectMeta
// above) -- an explicit value here always wins.
ControlPlaneURL string
WorkloadTokenPath string
ServerID string
@@ -126,13 +133,13 @@ func New(config Config) (*Supervisor, error) {
return nil, err
}
}
if config.ControlPlaneURL != "" && (config.WorkloadTokenPath == "" || config.ServerID == "" || config.ProtocolVersion < 1 || config.ImageDigest == "") {
return nil, fmt.Errorf("control-plane registration requires a workload token path, server ID, protocol version and image digest")
if config.ControlPlaneURL != "" && (config.ServerID == "" || config.ProtocolVersion < 1 || config.ImageDigest == "") {
return nil, fmt.Errorf("control-plane registration requires a server ID, protocol version and image digest")
}
// MatchID is deliberately not required here: it can also be resolved at
// Start time from the allocated GameServer's own annotations (see
// registerControlPlane). It is validated to actually be resolvable
// there, not silently skipped.
// Neither MatchID nor WorkloadTokenPath is required here: both can
// instead be resolved at Start time from the allocated GameServer's own
// annotations (see registerControlPlane/workloadToken/matchID). They are
// validated to actually be resolvable there, not silently skipped.
return &Supervisor{config: config, client: config.HTTPClient}, nil
}
@@ -243,6 +250,34 @@ func (s *Supervisor) matchID() string {
return s.lastGameServer.ObjectMeta.Annotations["cosmic-clash.io/match-id"]
}
// workloadToken resolves the bearer credential for control-plane
// registration. WorkloadTokenPath, when configured, always wins -- it is
// for a future Kubernetes-projected-JWT WorkloadVerify path (see the Config
// field's doc comment) and an operator who explicitly set it presumably
// wants it used. Otherwise it falls back to the cosmic-clash.io/workload-
// token annotation Agones applied to this GameServer at allocation time
// (server/agones.Client.Allocate, verified by
// api.WorkloadVerifierFromSignedToken today) -- the same annotation-fallback
// pattern matchID already uses for cosmic-clash.io/match-id.
func (s *Supervisor) workloadToken() (string, error) {
if s.config.WorkloadTokenPath != "" {
tokenBytes, err := os.ReadFile(s.config.WorkloadTokenPath)
if err != nil {
return "", fmt.Errorf("read workload token: %w", err)
}
token := strings.TrimSpace(string(tokenBytes))
if token == "" {
return "", fmt.Errorf("workload token file %q is empty", s.config.WorkloadTokenPath)
}
return token, nil
}
token := s.lastGameServer.ObjectMeta.Annotations["cosmic-clash.io/workload-token"]
if token == "" {
return "", fmt.Errorf("control-plane registration has no workload token: no --workload-token-path configured, and no cosmic-clash.io/workload-token annotation was present on the allocated GameServer")
}
return token, nil
}
func (s *Supervisor) registerControlPlane(ctx context.Context, assignmentReady bool) error {
if s.config.ControlPlaneURL == "" {
return nil
@@ -251,13 +286,9 @@ func (s *Supervisor) registerControlPlane(ctx context.Context, assignmentReady b
if matchID == "" {
return fmt.Errorf("control-plane registration has no match ID: not configured, and no cosmic-clash.io/match-id annotation was present on the allocated GameServer")
}
tokenBytes, err := os.ReadFile(s.config.WorkloadTokenPath)
token, err := s.workloadToken()
if err != nil {
return fmt.Errorf("read workload token: %w", err)
}
token := strings.TrimSpace(string(tokenBytes))
if token == "" {
return fmt.Errorf("workload token file %q is empty", s.config.WorkloadTokenPath)
return err
}
body, err := json.Marshal(struct {
MatchID string `json:"match_id"`
+89 -2
View File
@@ -71,13 +71,22 @@ func TestAllocatedStartInjectsDynamicEndpointAndCallsReadyAfterProbe(t *testing.
func TestControlPlaneRegistrationRejectsIncompleteConfig(t *testing.T) {
base := Config{Command: []string{"/bin/true"}, ControlPlaneURL: "https://control-plane.invalid"}
if _, err := New(base); err == nil {
t.Fatal("registration enabled with no token path/server/match/digest was accepted")
t.Fatal("registration enabled with no server/protocol/digest was accepted")
}
complete := base
complete.WorkloadTokenPath, complete.ServerID, complete.MatchID, complete.ProtocolVersion, complete.ImageDigest = "/tmp/token", "server-1", "match-1", 1, "sha256:aa"
complete.ServerID, complete.MatchID, complete.ProtocolVersion, complete.ImageDigest = "server-1", "match-1", 1, "sha256:aa"
if _, err := New(complete); err != nil {
t.Fatalf("fully configured registration rejected: %v", err)
}
// WorkloadTokenPath is deliberately not required at construction time --
// it can instead be resolved at Start time from the GameServer's own
// cosmic-clash.io/workload-token annotation (see workloadToken() and
// TestControlPlaneRegistrationFallsBackToGameServerAnnotationForWorkloadToken).
withTokenPath := complete
withTokenPath.WorkloadTokenPath = "/tmp/token"
if _, err := New(withTokenPath); err != nil {
t.Fatalf("configured token path rejected: %v", err)
}
}
func TestControlPlaneRegistrationReportsProcessReadyThenAssignmentReady(t *testing.T) {
@@ -277,6 +286,84 @@ func TestControlPlaneRegistrationWithoutMatchIDOrAnnotationFailsClosed(t *testin
}
}
// TestControlPlaneRegistrationFallsBackToGameServerAnnotationForWorkloadToken
// proves the primary intended delivery channel for the control-plane's
// self-issued signed token (server/workload/signed_token.go): with no
// --workload-token-path configured at all, a token arriving only via the
// cosmic-clash.io/workload-token annotation Agones applies to this
// GameServer (server/agones.Client.Allocate) is what gets sent as the
// Authorization bearer.
func TestControlPlaneRegistrationFallsBackToGameServerAnnotationForWorkloadToken(t *testing.T) {
var gotAuth string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/gameserver":
_, _ = w.Write([]byte(`{"object_meta":{"annotations":{"cosmic-clash.io/match-id":"match-1","cosmic-clash.io/workload-token":"signed-token-from-annotation"}},"status":{"address":"203.0.113.9","ports":[{"name":"game","port":31001}]}}`))
case "/ready-probe", "/ready":
w.WriteHeader(http.StatusOK)
case "/v1/servers/server-1/register":
gotAuth = r.Header.Get("Authorization")
w.WriteHeader(http.StatusNoContent)
default:
w.WriteHeader(http.StatusNotFound)
}
}))
defer server.Close()
// Deliberately no WorkloadTokenPath -- only the GameServer annotation
// supplies a token, proving the fallback path itself.
s, err := New(Config{
Command: []string{"/bin/sh", "-c", "true"}, SDKBaseURL: server.URL, ReadyURL: server.URL + "/ready-probe", ReadyTimeout: time.Second, PollInterval: time.Millisecond,
ControlPlaneURL: server.URL, ServerID: "server-1", ProtocolVersion: 1, ImageDigest: "sha256:aa",
})
if err != nil {
t.Fatal(err)
}
if err := s.Start(context.Background()); err != nil {
t.Fatal(err)
}
_ = s.Wait()
if gotAuth != "Bearer signed-token-from-annotation" {
t.Fatalf("Authorization header = %q, want the annotation-sourced token", gotAuth)
}
}
// TestControlPlaneRegistrationWithoutWorkloadTokenPathOrAnnotationFailsClosed
// is the workload-token counterpart to the match-ID fails-closed test above:
// with neither a configured token path nor an annotation present, Start must
// fail rather than register unauthenticated or with an empty token.
func TestControlPlaneRegistrationWithoutWorkloadTokenPathOrAnnotationFailsClosed(t *testing.T) {
registerCalled := false
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/gameserver":
_, _ = w.Write([]byte(`{"object_meta":{"annotations":{"cosmic-clash.io/match-id":"match-1"}},"status":{"address":"203.0.113.9","ports":[{"name":"game","port":31001}]}}`))
case "/ready-probe", "/ready":
w.WriteHeader(http.StatusOK)
case "/v1/servers/server-1/register":
registerCalled = true
w.WriteHeader(http.StatusNoContent)
default:
w.WriteHeader(http.StatusNotFound)
}
}))
defer server.Close()
s, err := New(Config{
Command: []string{"/bin/sh", "-c", "sleep 30"}, SDKBaseURL: server.URL, ReadyURL: server.URL + "/ready-probe", ReadyTimeout: time.Second, PollInterval: time.Millisecond,
ControlPlaneURL: server.URL, ServerID: "server-1", ProtocolVersion: 1, ImageDigest: "sha256:aa",
})
if err != nil {
t.Fatal(err)
}
if err := s.Start(context.Background()); err == nil {
t.Fatal("Start succeeded with no workload token available from either config or annotations")
}
if registerCalled {
t.Fatal("register was called despite having no workload token to send")
}
}
func TestControlPlaneRegistrationFailureKillsChildRatherThanRunningUnregistered(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {