fix(allocator): publish signed assignment rosters before servers start

The root blocker (issue #14). The worker bound the provider allocation
and stopped. Service.PublishRoster and store.SaveVerifiedAssignmentRoster
both existed, fully tested, with zero non-test callers, and the
production allocator configured neither a roster store nor a signing
key. Nothing ever wrote the assignments table.

The allocated supervisor fetches a non-empty roster before it launches
the game child, so every real allocation failed at that fetch: no match
could reach ASSIGNMENT_READY or accept a player. Existing tests seeded
assignments directly, which is exactly why the missing hand-off went
unnoticed.

The worker now builds one join authorisation per durable participant,
signs each with the active key, and publishes them. Participants are
read through the same query SaveVerifiedAssignmentRoster re-validates
against, so the allocator cannot construct a roster the persistence
boundary would reject. The manifest commits to a digest over the whole
roster, so a server cannot be handed a truncated roster whose surviving
entries are each individually valid.

Persist the provider endpoint on the allocation: it arrived on the
provider response and was never stored, so a worker crashing between
allocating and publishing had no endpoint to recover and would have
stranded the match permanently. Republishing is idempotent, so that
crash now simply retries.

cmd/allocator refuses to start without key material rather than running
an allocator that binds allocations and silently strands every match.
The k8s allocator Deployment mounts the same key set the Fleet does, and
both now take the JSON key map so a rotation can publish several.

New integration test drives the real worker through to the supervisor's
own roster read path without seeding the assignments table. Verified it
fails with "assignments = 0, want 2" when the publish step is removed.
This commit is contained in:
Josh Creek
2026-09-05 10:42:31 +01:00
parent b8bcc1f3c1
commit 5765532409
19 changed files with 786 additions and 17 deletions
+47 -1
View File
@@ -3,7 +3,10 @@ package main
import (
"context"
"database/sql"
"encoding/base64"
"encoding/json"
"flag"
"fmt"
"log"
"net/http"
"os"
@@ -34,6 +37,8 @@ func main() {
allocationQuota := flag.Int("allocation-quota", 0, "optional per-replica allocation attempts per region per quota window; zero disables this local guard")
allocationQuotaWindow := flag.Duration("allocation-quota-window", time.Minute, "window for --allocation-quota")
metricsAddr := flag.String("metrics-addr", envOrDefault("COSMIC_CLASH_ALLOCATOR_METRICS_ADDR", ":9091"), "allocator Prometheus metrics address; empty disables metrics")
joinKeyFile := flag.String("join-authorisations-key-file", os.Getenv("COSMIC_CLASH_JOIN_SIGNING_KEY_FILE"), "JSON file mapping join-signing key ID to base64 key; the same material allocated game servers mount. Required: without it no assignment roster is published and no allocated match can start")
joinKeyID := flag.String("join-authorisations-key-id", os.Getenv("COSMIC_CLASH_JOIN_SIGNING_KEY_ID"), "which key in --join-authorisations-key-file signs new authorisations; other keys stay valid for verification so a rotation does not break in-flight matches")
flag.Parse()
if *dsn == "" || *agonesURL == "" {
fatalf("--dsn/COSMIC_CLASH_POSTGRES_DSN and --agones-url/COSMIC_CLASH_AGONES_URL are required")
@@ -47,6 +52,16 @@ func main() {
if *allocationQuota < 0 || *allocationQuotaWindow <= 0 || *workloadTokenTTL <= 0 {
fatalf("--allocation-quota must be non-negative and --allocation-quota-window/--workload-token-ttl must be positive")
}
// Refuse to start without signing material rather than running an
// allocator that binds allocations and silently never publishes a roster,
// which strands every match short of ASSIGNMENT_READY.
if *joinKeyFile == "" || *joinKeyID == "" {
fatalf("--join-authorisations-key-file/COSMIC_CLASH_JOIN_SIGNING_KEY_FILE and --join-authorisations-key-id/COSMIC_CLASH_JOIN_SIGNING_KEY_ID are required; without them allocated matches can never become joinable")
}
joinKeys, err := loadJoinSigningKeys(*joinKeyFile, *joinKeyID)
if err != nil {
fatalf("load join signing keys: %v", err)
}
db, err := sql.Open("pgx", *dsn)
if err != nil {
fatalf("open PostgreSQL: %v", err)
@@ -89,7 +104,9 @@ func main() {
Metrics: metrics,
Now: now,
},
Now: now,
Now: now,
Roster: store.AssignmentRosters{DB: db},
Keys: joinKeys,
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
@@ -152,3 +169,32 @@ func fatalf(format string, args ...any) {
log.Printf("allocator: "+format, args...)
os.Exit(1)
}
// loadJoinSigningKeys reads the key ID to base64 key map shared with allocated
// game servers. Every key in the file stays valid for verification; only the
// named one signs, so rotation is: publish the new key everywhere, then point
// --join-authorisations-key-id at it, then drop the old key once no live match
// can still reference it.
func loadJoinSigningKeys(path, activeKeyID string) (allocator.JoinSigningKeys, error) {
raw, err := os.ReadFile(path)
if err != nil {
return allocator.JoinSigningKeys{}, err
}
var encoded map[string]string
if err := json.Unmarshal(raw, &encoded); err != nil {
return allocator.JoinSigningKeys{}, fmt.Errorf("expected a JSON object of key ID to base64 key: %w", err)
}
keys := make(map[string][]byte, len(encoded))
for keyID, value := range encoded {
key, err := base64.StdEncoding.DecodeString(value)
if err != nil || len(key) == 0 {
return allocator.JoinSigningKeys{}, fmt.Errorf("join signing key %q is not valid base64", keyID)
}
keys[keyID] = key
}
result := allocator.JoinSigningKeys{ActiveKeyID: activeKeyID, Keys: keys}
if len(keys[activeKeyID]) == 0 {
return allocator.JoinSigningKeys{}, fmt.Errorf("active key ID %q is not present in %s", activeKeyID, path)
}
return result, nil
}