Files
CosmicClash/server/store/assignment_sql.go
T
Josh Creek 5765532409 fix(allocator): publish signed assignment rosters before servers start
The root blocker (issue #14). The worker bound the provider allocation
and stopped. Service.PublishRoster and store.SaveVerifiedAssignmentRoster
both existed, fully tested, with zero non-test callers, and the
production allocator configured neither a roster store nor a signing
key. Nothing ever wrote the assignments table.

The allocated supervisor fetches a non-empty roster before it launches
the game child, so every real allocation failed at that fetch: no match
could reach ASSIGNMENT_READY or accept a player. Existing tests seeded
assignments directly, which is exactly why the missing hand-off went
unnoticed.

The worker now builds one join authorisation per durable participant,
signs each with the active key, and publishes them. Participants are
read through the same query SaveVerifiedAssignmentRoster re-validates
against, so the allocator cannot construct a roster the persistence
boundary would reject. The manifest commits to a digest over the whole
roster, so a server cannot be handed a truncated roster whose surviving
entries are each individually valid.

Persist the provider endpoint on the allocation: it arrived on the
provider response and was never stored, so a worker crashing between
allocating and publishing had no endpoint to recover and would have
stranded the match permanently. Republishing is idempotent, so that
crash now simply retries.

cmd/allocator refuses to start without key material rather than running
an allocator that binds allocations and silently strands every match.
The k8s allocator Deployment mounts the same key set the Fleet does, and
both now take the JSON key map so a rotation can publish several.

New integration test drives the real worker through to the supervisor's
own roster read path without seeding the assignments table. Verified it
fails with "assignments = 0, want 2" when the publish step is removed.
2026-09-05 10:42:31 +01:00

376 lines
16 KiB
Go

package store
import (
"context"
"database/sql"
"encoding/base64"
"encoding/json"
"fmt"
"strconv"
"time"
"github.com/cosmic-clash/cosmic-clash/server/domain"
)
// DurableAssignment is the persistence form of a verified assignment-ready
// projection. It deliberately keeps player ownership in the primary key and
// query predicate so another participant cannot recover its join material.
type DurableAssignment struct {
MatchID string
PlayerID string
AllocationID string
ServerID string
Slot int
Region string
ClientBuild string
ProtocolVersion int
Transport string
Endpoint string
JoinAuthorisation string
ManifestDigest []byte
ExpiresAt time.Time
Revision uint64
}
type PostgresRosterStore struct{ DB *sql.DB }
func (s PostgresRosterStore) PublishRoster(ctx context.Context, assignment domain.Assignment, roster []domain.SignedJoinAuthorisation, verify func([]byte, []byte) bool) error {
return SaveVerifiedAssignmentRoster(ctx, s.DB, assignment, roster, verify)
}
const AssignmentUpsertSQL = `INSERT INTO assignments
(match_id, player_id, allocation_id, server_id, slot, region, client_build,
protocol_version, transport, endpoint, join_authorisation, manifest_digest,
expires_at, revision)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
ON CONFLICT (match_id, player_id) DO UPDATE SET
allocation_id = EXCLUDED.allocation_id, server_id = EXCLUDED.server_id,
slot = EXCLUDED.slot, region = EXCLUDED.region, client_build = EXCLUDED.client_build,
protocol_version = EXCLUDED.protocol_version, transport = EXCLUDED.transport,
endpoint = EXCLUDED.endpoint, join_authorisation = EXCLUDED.join_authorisation,
manifest_digest = EXCLUDED.manifest_digest, expires_at = EXCLUDED.expires_at,
revision = EXCLUDED.revision
WHERE assignments.allocation_id = EXCLUDED.allocation_id
AND assignments.server_id = EXCLUDED.server_id
AND assignments.slot = EXCLUDED.slot
AND assignments.region = EXCLUDED.region
AND assignments.client_build = EXCLUDED.client_build
AND assignments.protocol_version = EXCLUDED.protocol_version
AND assignments.transport = EXCLUDED.transport
AND assignments.endpoint = EXCLUDED.endpoint
AND assignments.join_authorisation = EXCLUDED.join_authorisation
AND assignments.manifest_digest = EXCLUDED.manifest_digest
AND assignments.expires_at = EXCLUDED.expires_at
AND assignments.revision = EXCLUDED.revision`
const AssignmentSelectSQL = `SELECT a.match_id, a.player_id, a.allocation_id, a.server_id,
a.slot, a.region, a.client_build, a.protocol_version, a.transport, a.endpoint,
a.join_authorisation, a.manifest_digest, a.expires_at, a.revision
FROM assignments a
JOIN matches m ON m.match_id = a.match_id AND m.server_id = a.server_id
WHERE a.match_id = $1 AND a.player_id = $2 AND a.expires_at > $3
AND m.state IN ('ASSIGNMENT_READY', 'ASSIGNED', 'CONNECTING', 'LIVE')`
const AssignmentRosterSelectSQL = `SELECT allocation_id, server_id, join_authorisation
FROM assignments
WHERE match_id = $1 AND server_id = $2 AND expires_at > $3
ORDER BY slot, player_id`
const AssignmentExpectedRosterSQL = `SELECT mp.player_id, i.steam_id, mp.slot, mp.team
FROM match_participants mp
JOIN identities i ON i.player_id = mp.player_id
JOIN matches m ON m.match_id = mp.match_id
JOIN allocations a ON a.allocation_id = m.allocation_id AND a.match_id = m.match_id AND a.server_id = m.server_id
WHERE mp.match_id = $1 AND m.allocation_id = $2 AND m.server_id = $3
AND m.region = $4 AND m.protocol_version = $5
AND a.region = $4 AND a.build = $6 AND a.protocol_version = $5 AND a.transport = $7
AND a.state = 'ALLOCATED' AND mp.participation_active
ORDER BY mp.player_id
FOR UPDATE OF mp`
func validateDurableAssignment(assignment DurableAssignment) error {
if assignment.MatchID == "" || assignment.PlayerID == "" || assignment.AllocationID == "" || assignment.ServerID == "" || assignment.Slot < 0 || assignment.Slot > 5 || (assignment.Region != "EU" && assignment.Region != "NA") || assignment.ClientBuild == "" || assignment.ProtocolVersion < 1 || (assignment.Transport != "enet" && assignment.Transport != "steam_sdr") || assignment.Endpoint == "" || assignment.JoinAuthorisation == "" || len(assignment.ManifestDigest) == 0 || assignment.ExpiresAt.IsZero() || assignment.Revision == 0 {
return fmt.Errorf("invalid durable assignment")
}
return nil
}
func SaveAssignment(ctx context.Context, db *sql.DB, assignment DurableAssignment) error {
if db == nil {
return fmt.Errorf("invalid assignment database")
}
if err := validateDurableAssignment(assignment); err != nil {
return err
}
result, err := db.ExecContext(ctx, AssignmentUpsertSQL, assignment.MatchID, assignment.PlayerID, assignment.AllocationID, assignment.ServerID, assignment.Slot, assignment.Region, assignment.ClientBuild, assignment.ProtocolVersion, assignment.Transport, assignment.Endpoint, assignment.JoinAuthorisation, assignment.ManifestDigest, assignment.ExpiresAt, assignment.Revision)
if err != nil {
return err
}
changed, err := result.RowsAffected()
if err != nil {
return err
}
if changed != 1 {
return fmt.Errorf("assignment persistence conflict")
}
return nil
}
// SaveAssignments publishes a complete signed roster atomically. Assignment
// readiness is a match boundary: exposing only some players would let the
// control plane tell different participants incompatible stories after a
// transient database failure.
func SaveAssignments(ctx context.Context, db *sql.DB, assignments []DurableAssignment) error {
if db == nil || len(assignments) == 0 {
return fmt.Errorf("invalid assignment batch")
}
if err := validateAssignmentBatch(assignments); err != nil {
return err
}
return RunSerializable(ctx, db, DefaultSerializableAttempts, func(ctx context.Context, tx *sql.Tx) error {
return saveAssignmentsTx(ctx, tx, assignments)
})
}
func validateAssignmentBatch(assignments []DurableAssignment) error {
if len(assignments) == 0 {
return fmt.Errorf("invalid assignment batch")
}
first := assignments[0]
seen := make(map[string]struct{}, len(assignments))
seenSlots := make(map[int]struct{}, len(assignments))
for _, assignment := range assignments {
if err := validateDurableAssignment(assignment); err != nil {
return err
}
if assignment.MatchID != first.MatchID || assignment.AllocationID != first.AllocationID || assignment.ServerID != first.ServerID || assignment.Region != first.Region || assignment.ClientBuild != first.ClientBuild || assignment.ProtocolVersion != first.ProtocolVersion || assignment.Transport != first.Transport || assignment.Endpoint != first.Endpoint || string(assignment.ManifestDigest) != string(first.ManifestDigest) || assignment.Revision != first.Revision {
return fmt.Errorf("mixed assignment batch")
}
if _, ok := seen[assignment.PlayerID]; ok {
return fmt.Errorf("duplicate assignment in batch")
}
if _, ok := seenSlots[assignment.Slot]; ok {
return fmt.Errorf("duplicate assignment slot in batch")
}
seen[assignment.PlayerID] = struct{}{}
seenSlots[assignment.Slot] = struct{}{}
}
return nil
}
func saveAssignmentsTx(ctx context.Context, tx *sql.Tx, assignments []DurableAssignment) error {
for _, assignment := range assignments {
result, err := tx.ExecContext(ctx, AssignmentUpsertSQL, assignment.MatchID, assignment.PlayerID, assignment.AllocationID, assignment.ServerID, assignment.Slot, assignment.Region, assignment.ClientBuild, assignment.ProtocolVersion, assignment.Transport, assignment.Endpoint, assignment.JoinAuthorisation, assignment.ManifestDigest, assignment.ExpiresAt, assignment.Revision)
if err != nil {
return err
}
changed, err := result.RowsAffected()
if err != nil {
return err
}
if changed != 1 {
return fmt.Errorf("assignment persistence conflict")
}
}
return nil
}
// SaveVerifiedAssignmentRoster converts the backend-verified signed roster to
// player-scoped rows. It rechecks the claims at this persistence boundary so a
// caller cannot accidentally publish a token for another match or slot.
func SaveVerifiedAssignmentRoster(ctx context.Context, db *sql.DB, assignment domain.Assignment, roster []domain.SignedJoinAuthorisation, verify func([]byte, []byte) bool) error {
if assignment.Allocation.State != domain.ServerAllocated || len(roster) == 0 || verify == nil {
return fmt.Errorf("invalid verified assignment roster")
}
digest := domain.ManifestDigest(assignment.Manifest)
rows := make([]DurableAssignment, 0, len(roster))
seenPlayers := make(map[string]struct{}, len(roster))
seenSlots := make(map[int]struct{}, len(roster))
for _, signed := range roster {
auth := signed.Authorisation
if err := validateSignedRosterEntry(assignment, signed, verify); err != nil {
return err
}
if _, exists := seenPlayers[auth.PlayerID]; exists {
return fmt.Errorf("invalid signed assignment roster: duplicate player")
}
if _, exists := seenSlots[auth.Slot]; exists {
return fmt.Errorf("invalid signed assignment roster: duplicate slot")
}
seenPlayers[auth.PlayerID] = struct{}{}
seenSlots[auth.Slot] = struct{}{}
envelope, err := json.Marshal(signed)
if err != nil {
return fmt.Errorf("encode signed assignment roster: %w", err)
}
rows = append(rows, DurableAssignment{
MatchID: assignment.Allocation.MatchID, PlayerID: auth.PlayerID,
AllocationID: assignment.Allocation.AllocationID, ServerID: assignment.Allocation.ServerID,
Slot: auth.Slot, Region: assignment.Allocation.Region, ClientBuild: assignment.Allocation.Build,
ProtocolVersion: assignment.Allocation.Protocol, Transport: assignment.Allocation.Transport,
Endpoint: assignment.Endpoint, JoinAuthorisation: base64.RawURLEncoding.EncodeToString(envelope),
ManifestDigest: digest[:], ExpiresAt: auth.ExpiresAt, Revision: 1,
})
}
if db == nil {
return fmt.Errorf("invalid assignment database")
}
if err := validateAssignmentBatch(rows); err != nil {
return err
}
return RunSerializable(ctx, db, DefaultSerializableAttempts, func(ctx context.Context, tx *sql.Tx) error {
if err := validateExpectedAssignmentRoster(ctx, tx, assignment, roster); err != nil {
return err
}
return saveAssignmentsTx(ctx, tx, rows)
})
}
func validateExpectedAssignmentRoster(ctx context.Context, tx *sql.Tx, assignment domain.Assignment, roster []domain.SignedJoinAuthorisation) error {
rows, err := tx.QueryContext(ctx, AssignmentExpectedRosterSQL,
assignment.Allocation.MatchID, assignment.Allocation.AllocationID, assignment.Allocation.ServerID,
assignment.Allocation.Region, assignment.Allocation.Protocol, assignment.Allocation.Build, assignment.Allocation.Transport)
if err != nil {
return err
}
defer rows.Close()
type expectedPlayer struct {
steamID string
slot int
team int
}
expected := make(map[string]expectedPlayer, len(roster))
for rows.Next() {
var playerID string
var player expectedPlayer
if err := rows.Scan(&playerID, &player.steamID, &player.slot, &player.team); err != nil {
return err
}
expected[playerID] = player
}
if err := rows.Err(); err != nil {
return err
}
if err := rows.Close(); err != nil {
return err
}
if len(expected) == 0 || len(expected) != len(roster) {
return fmt.Errorf("signed assignment roster is incomplete")
}
for _, signed := range roster {
auth := signed.Authorisation
player, ok := expected[auth.PlayerID]
if !ok || player.steamID != auth.SteamID || player.slot != auth.Slot || player.team != auth.Team {
return fmt.Errorf("signed assignment roster does not match durable participants")
}
}
return nil
}
func validateSignedRosterEntry(assignment domain.Assignment, signed domain.SignedJoinAuthorisation, verify func([]byte, []byte) bool) error {
auth := signed.Authorisation
if len(signed.Signature) == 0 || verify == nil || !verify(domain.JoinAuthorisationBytes(auth), signed.Signature) || auth.MatchID != assignment.Allocation.MatchID || auth.ServerID != assignment.Allocation.ServerID || auth.Protocol != strconv.Itoa(assignment.Allocation.Protocol) || auth.PlayerID == "" || auth.Slot < 0 || auth.Slot > 5 || auth.Team < 0 || auth.Team > 1 || auth.Slot/3 != auth.Team || auth.ExpiresAt.IsZero() {
return fmt.Errorf("invalid signed assignment roster")
}
return nil
}
func GetAssignment(ctx context.Context, db *sql.DB, playerID, matchID string, now time.Time) (DurableAssignment, error) {
if db == nil || playerID == "" || matchID == "" || now.IsZero() {
return DurableAssignment{}, fmt.Errorf("invalid assignment recovery arguments")
}
var assignment DurableAssignment
err := db.QueryRowContext(ctx, AssignmentSelectSQL, matchID, playerID, now).Scan(&assignment.MatchID, &assignment.PlayerID, &assignment.AllocationID, &assignment.ServerID, &assignment.Slot, &assignment.Region, &assignment.ClientBuild, &assignment.ProtocolVersion, &assignment.Transport, &assignment.Endpoint, &assignment.JoinAuthorisation, &assignment.ManifestDigest, &assignment.ExpiresAt, &assignment.Revision)
if err != nil {
return DurableAssignment{}, err
}
if err := validateDurableAssignment(assignment); err != nil {
return DurableAssignment{}, err
}
return assignment, nil
}
// GetAssignmentRoster returns the complete signed roster for an allocated
// server. It is intentionally server-scoped rather than player-scoped and is
// called only after workload authentication at the API boundary. All rows
// must belong to one allocation; a partial or mixed allocation is unsafe to
// hand to the game process.
func GetAssignmentRoster(ctx context.Context, db *sql.DB, matchID, serverID string, now time.Time) ([][]byte, error) {
if db == nil || matchID == "" || serverID == "" || now.IsZero() {
return nil, fmt.Errorf("invalid assignment roster arguments")
}
rows, err := db.QueryContext(ctx, AssignmentRosterSelectSQL, matchID, serverID, now)
if err != nil {
return nil, err
}
defer rows.Close()
var allocationID string
var roster [][]byte
for rows.Next() {
var rowAllocation, rowServer, encoded string
if err := rows.Scan(&rowAllocation, &rowServer, &encoded); err != nil {
return nil, err
}
if rowServer != serverID || rowAllocation == "" || (allocationID != "" && allocationID != rowAllocation) {
return nil, fmt.Errorf("assignment roster contains mixed allocation")
}
decoded, err := base64.RawURLEncoding.DecodeString(encoded)
if err != nil || len(decoded) == 0 {
return nil, fmt.Errorf("assignment roster contains invalid envelope")
}
allocationID = rowAllocation
roster = append(roster, decoded)
}
if err := rows.Err(); err != nil {
return nil, err
}
if len(roster) == 0 {
return nil, sql.ErrNoRows
}
return roster, nil
}
// LoadAssignmentParticipants reads the authoritative roster for an allocated
// match. It reuses AssignmentExpectedRosterSQL -- the same query
// SaveVerifiedAssignmentRoster re-validates against -- so the allocator cannot
// build a roster the persistence boundary would then reject for disagreeing
// with the durable participants.
func LoadAssignmentParticipants(ctx context.Context, db *sql.DB, allocation domain.Allocation) ([]domain.AssignmentParticipant, error) {
if db == nil || allocation.MatchID == "" || allocation.AllocationID == "" || allocation.ServerID == "" {
return nil, fmt.Errorf("invalid assignment participant arguments")
}
rows, err := db.QueryContext(ctx, AssignmentExpectedRosterSQL,
allocation.MatchID, allocation.AllocationID, allocation.ServerID,
allocation.Region, allocation.Protocol, allocation.Build, allocation.Transport)
if err != nil {
return nil, err
}
defer rows.Close()
var participants []domain.AssignmentParticipant
for rows.Next() {
var participant domain.AssignmentParticipant
if err := rows.Scan(&participant.PlayerID, &participant.SteamID, &participant.Slot, &participant.Team); err != nil {
return nil, err
}
if participant.PlayerID == "" || participant.SteamID == "" || participant.Slot < 0 || participant.Slot > 5 || participant.Team < 0 || participant.Team > 1 || participant.Slot/3 != participant.Team {
return nil, fmt.Errorf("assignment participant %q has an invalid slot/team", participant.PlayerID)
}
participants = append(participants, participant)
}
if err := rows.Err(); err != nil {
return nil, err
}
if len(participants) == 0 {
return nil, fmt.Errorf("allocated match %s has no durable participants", allocation.MatchID)
}
return participants, nil
}
// AssignmentRosters adapts the participant loader to the allocator's
// AssignmentRosterSource interface.
type AssignmentRosters struct{ DB *sql.DB }
func (a AssignmentRosters) LoadAssignmentParticipants(ctx context.Context, allocation domain.Allocation) ([]domain.AssignmentParticipant, error) {
return LoadAssignmentParticipants(ctx, a.DB, allocation)
}