Files
CosmicClash/server/domain/allocator.go
T
Josh Creek 5765532409 fix(allocator): publish signed assignment rosters before servers start
The root blocker (issue #14). The worker bound the provider allocation
and stopped. Service.PublishRoster and store.SaveVerifiedAssignmentRoster
both existed, fully tested, with zero non-test callers, and the
production allocator configured neither a roster store nor a signing
key. Nothing ever wrote the assignments table.

The allocated supervisor fetches a non-empty roster before it launches
the game child, so every real allocation failed at that fetch: no match
could reach ASSIGNMENT_READY or accept a player. Existing tests seeded
assignments directly, which is exactly why the missing hand-off went
unnoticed.

The worker now builds one join authorisation per durable participant,
signs each with the active key, and publishes them. Participants are
read through the same query SaveVerifiedAssignmentRoster re-validates
against, so the allocator cannot construct a roster the persistence
boundary would reject. The manifest commits to a digest over the whole
roster, so a server cannot be handed a truncated roster whose surviving
entries are each individually valid.

Persist the provider endpoint on the allocation: it arrived on the
provider response and was never stored, so a worker crashing between
allocating and publishing had no endpoint to recover and would have
stranded the match permanently. Republishing is idempotent, so that
crash now simply retries.

cmd/allocator refuses to start without key material rather than running
an allocator that binds allocations and silently strands every match.
The k8s allocator Deployment mounts the same key set the Fleet does, and
both now take the JSON key map so a rotation can publish several.

New integration test drives the real worker through to the supervisor's
own roster read path without seeding the assignments table. Verified it
fails with "assignments = 0, want 2" when the publish step is removed.
2026-09-05 10:42:31 +01:00

153 lines
5.2 KiB
Go

package domain
import (
"crypto/sha256"
"fmt"
"sort"
"sync"
"time"
)
type ServerLifecycle string
const (
ServerReady ServerLifecycle = "READY"
ServerAllocated ServerLifecycle = "ALLOCATED"
)
type ReadyServer struct {
ServerID string
Region string
Build string
Protocol int
Transport string
State ServerLifecycle
}
type AllocationRequest struct {
AllocationID string
MatchID string
Playlist Playlist
Region string
Build string
Protocol int
ArenaPath string
Transport string
}
type Allocation struct {
AllocationID string
MatchID string
ServerID string
Region string
Build string
Protocol int
ArenaPath string
Transport string
State ServerLifecycle
AllocatedAt time.Time
// Endpoint is the client-facing address the provider returned. It is
// persisted so a worker that crashes between allocating and publishing the
// assignment roster can recover it instead of stranding the match.
Endpoint string
}
type Allocator struct {
mu sync.Mutex
servers map[string]ReadyServer
allocations map[string]Allocation
assignments map[string]Assignment
requestHashes map[string][32]byte
}
var (
ErrNoCapacity = fmt.Errorf("no compatible ready server")
ErrAllocationInput = fmt.Errorf("invalid allocation request")
ErrAllocationNotFound = fmt.Errorf("allocation not found")
)
func NewAllocator(servers []ReadyServer) (*Allocator, error) {
a := &Allocator{servers: make(map[string]ReadyServer, len(servers)), allocations: make(map[string]Allocation), assignments: make(map[string]Assignment), requestHashes: make(map[string][32]byte)}
for _, server := range servers {
if server.ServerID == "" || server.Region == "" || server.Build == "" || server.Protocol <= 0 || (server.Transport != "enet" && server.Transport != "steam_sdr") || server.State != ServerReady {
return nil, fmt.Errorf("%w: invalid ready server", ErrAllocationInput)
}
if _, exists := a.servers[server.ServerID]; exists {
return nil, fmt.Errorf("%w: duplicate server", ErrAllocationInput)
}
a.servers[server.ServerID] = server
}
return a, nil
}
// Allocate is the in-process equivalent of a GameServerAllocation. The mutex
// represents the durable allocator transaction; the PostgreSQL/Agones adapter
// must preserve this claim-before-assignment ordering across replicas.
func (a *Allocator) Allocate(request AllocationRequest, now time.Time) (Allocation, error) {
if err := validateAllocationRequest(request); err != nil {
return Allocation{}, err
}
digest := allocationDigest(request)
a.mu.Lock()
defer a.mu.Unlock()
if prior, ok := a.allocations[request.AllocationID]; ok {
if a.requestHashes[request.AllocationID] != digest {
return Allocation{}, ErrConflict
}
return prior, nil
}
ids := make([]string, 0)
for id, server := range a.servers {
if server.State == ServerReady && server.Region == request.Region && server.Build == request.Build && server.Protocol == request.Protocol && server.Transport == request.Transport {
ids = append(ids, id)
}
}
if len(ids) == 0 {
return Allocation{}, ErrNoCapacity
}
sort.Strings(ids)
server := a.servers[ids[0]]
server.State = ServerAllocated
a.servers[server.ServerID] = server
allocation := Allocation{AllocationID: request.AllocationID, MatchID: request.MatchID, ServerID: server.ServerID, Region: server.Region, Build: server.Build, Protocol: server.Protocol, ArenaPath: request.ArenaPath, Transport: server.Transport, State: ServerAllocated, AllocatedAt: now}
a.allocations[request.AllocationID] = allocation
a.requestHashes[request.AllocationID] = digest
return allocation, nil
}
// PublishAssignment is the allocation-to-client boundary. It holds the same
// allocator lock as the claim and exposes no assignment until the allocated
// server, complete compatibility tuple, endpoint, and manifest signature all
// verify. The returned assignment is stable across an identical retry.
func (a *Allocator) PublishAssignment(allocationID string, manifest AllocationManifest, endpoint string, signature []byte, verify func([]byte, []byte) bool) (Assignment, error) {
a.mu.Lock()
defer a.mu.Unlock()
allocation, ok := a.allocations[allocationID]
if !ok {
return Assignment{}, ErrAllocationNotFound
}
assignment, err := VerifyAssignment(allocation, manifest, endpoint, signature, verify)
if err != nil {
return Assignment{}, err
}
if prior, exists := a.assignments[allocationID]; exists {
if prior != assignment {
return Assignment{}, ErrConflict
}
return prior, nil
}
a.assignments[allocationID] = assignment
return assignment, nil
}
func validateAllocationRequest(request AllocationRequest) error {
if request.AllocationID == "" || request.MatchID == "" || request.Region == "" || request.Build == "" || request.Protocol <= 0 || (request.Transport != "enet" && request.Transport != "steam_sdr") || (request.ArenaPath != "" && !IsRankedArenaPath(request.ArenaPath)) {
return ErrAllocationInput
}
return nil
}
func allocationDigest(request AllocationRequest) [32]byte {
return sha256.Sum256([]byte(fmt.Sprintf("%s\x00%s\x00%s\x00%s\x00%d\x00%s\x00%s", request.AllocationID, request.MatchID, request.Region, request.Build, request.Protocol, request.Transport, request.ArenaPath)))
}