From e6733bd6cbdf28fd34be73568736168f3e677abd Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:43:31 +0100 Subject: [PATCH] fix(multiplayer): repair PostgreSQL integration invariants --- multiplayer-next.md | 6 ++-- server/store/postgres_integration_test.go | 40 +++++++++++++++++++---- server/store/season_sql.go | 6 +++- server/store/stalled_allocation_sql.go | 3 +- 4 files changed, 43 insertions(+), 12 deletions(-) diff --git a/multiplayer-next.md b/multiplayer-next.md index b449dba2..4ac45cd7 100644 --- a/multiplayer-next.md +++ b/multiplayer-next.md @@ -1499,13 +1499,13 @@ returns `ErrProposalClosed`; deterministic penalty IDs preserve replay safety, future/corrupt cooldown events are ignored, and reopening an old declined proposal cannot create false timeout penalties for its innocent participants. -### Current local completion index (2026-09-02) +### Current local completion index (2026-09-04) The following Phase 8 slices have local implementation and verification evidence in this document: 8.29 dynamic allocated launch flags and endpoint handling; 8.30 allocator claim/reconciliation including provider-outcome recovery fencing and durable arena identity (migrations 0008–0009); 8.31 signed assignment/roster validation; 8.35 initial-connect no-show and casual bot policy; 8.36 controlled drain and shutdown acknowledgment; 8.39–8.43 client state, assignment, profile, recovery, and idempotent action retry; 8.44 structured observability and content-aware redaction; 8.45 bounded API metrics export plus optional Prometheus scrape/alert rules; 8.46 normal/race/vet/fuzz coverage; 8.47–8.48 offline/testkit/Compose coverage; 8.50 atomic stalled-allocation recovery notifications; 8.51 the 10,000-client API load boundary; 8.52 the opt-in per-replica plus shared PostgreSQL regional allocator quota; and 8.53 the fail-closed promotion validator. Their remaining acceptance text is infrastructure or production dependent where explicitly noted below the corresponding row. -The following are not locally certifiable from this workspace and remain open prerequisites rather than silently “done”: Valve/GodotSteam credentials and hosted SDR (7.1–7.8), reruns of disposable PostgreSQL/Redis gates while Docker storage is exhausted, live Agones/kind lifecycle (8.30–8.38, 8.49), public-network chaos/load/cost/release gates (8.50–8.53), and real-hardware graphics profiling (0.15b onward). `make verify-kind-agones` is the committed runner for 8.49; its response validator is unit-tested against malformed/ambiguous allocation payloads, but this machine still lacks kind and Helm and cannot initialize another Docker database until storage is reclaimed. `TODO.md`’s AI-training and presentation tasks remain separate from multiplayer and are not marked by this index. +The following are not locally certifiable from this workspace and remain open prerequisites rather than silently “done”: Valve/GodotSteam credentials and hosted SDR (7.1–7.8), live Agones/kind lifecycle (8.30–8.38, 8.49), public-network chaos/load/cost/release gates (8.50–8.53), and real-hardware graphics profiling (0.15b onward). `make verify-kind-agones` is the committed runner for 8.49; its response validator is unit-tested against malformed/ambiguous allocation payloads, but this machine still lacks kind and Helm. `TODO.md`’s AI-training and presentation tasks remain separate from multiplayer and are not marked by this index. -The live control-plane integration was retried on 2026-09-01 after Docker Desktop became available, but the disposable `postgres:17-alpine` container failed during `initdb` with `No space left on device`; Docker reported 10.2 GB of images and 3.3 GB of volumes. No live integration pass is claimed until storage is reclaimed and the gate completes. +The live control-plane integration was retried on 2026-09-01 after Docker Desktop became available, but the disposable `postgres:17-alpine` container failed during `initdb` with `No space left on device`; Docker reported 10.2 GB of images and 3.3 GB of volumes. The user approved pruning the disposable volumes on 2026-09-04 (3.3 GB reclaimed), and `scripts/run_postgres_integration.sh` then passed against real PostgreSQL. That run caught and repaired a stalled-allocation outbox CTE without `RETURNING`, an untyped JSON timestamp parameter, a season-rollover scan arity mismatch, lifecycle-incompatible fixtures, and a rollback-test step count that did not actually reach migration 0006. The deferred teamplay TODO prerequisite is now implemented locally but not enabled: team-touch credit is opt-in and the evaluator can run paired 2v2 diff --git a/server/store/postgres_integration_test.go b/server/store/postgres_integration_test.go index d19972cb..1b507d9c 100644 --- a/server/store/postgres_integration_test.go +++ b/server/store/postgres_integration_test.go @@ -6,6 +6,7 @@ import ( "context" "crypto/sha256" "database/sql" + "encoding/json" "errors" "fmt" "os" @@ -288,7 +289,7 @@ func TestPostgreSQLAllocationMatchClaimLeaseAndBindFence(t *testing.T) { } } claim, found, err := ClaimAllocatingMatch(ctx, db, "enet", now) - if err != nil || !found || claim.Request != (domain.AllocationRequest{AllocationID: "allocation-allocation-match", MatchID: "allocation-match", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet"}) { + if err != nil || !found || claim.Request != (domain.AllocationRequest{AllocationID: "allocation-allocation-match", MatchID: "allocation-match", Playlist: domain.Casual, Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet"}) { t.Fatalf("claim=%+v found=%t err=%v", claim, found, err) } if err := ReleaseAllocatedMatchClaim(ctx, db, claim.Request.MatchID, "different-allocation"); err != domain.ErrConflict { @@ -325,7 +326,18 @@ func TestPostgreSQLAllocationMatchClaimLeaseAndBindFence(t *testing.T) { if err := db.QueryRowContext(ctx, `SELECT event_type, payload FROM outbox WHERE aggregate_id = 'allocation-match' AND event_type = 'state_changed'`).Scan(&eventType, &eventPayload); err != nil { t.Fatalf("allocation outbox event: %v", err) } - if eventType != "state_changed" || !strings.Contains(string(eventPayload), `"state":"ALLOCATING"`) || !strings.Contains(string(eventPayload), `"allocation-match-a"`) || !strings.Contains(string(eventPayload), `"allocation-match-b"`) { + var event struct { + State string `json:"state"` + PlayerIDs []string `json:"player_ids"` + } + if err := json.Unmarshal(eventPayload, &event); err != nil { + t.Fatalf("decode allocation outbox event: %v", err) + } + players := make(map[string]bool, len(event.PlayerIDs)) + for _, playerID := range event.PlayerIDs { + players[playerID] = true + } + if eventType != "state_changed" || event.State != "ALLOCATING" || !players["allocation-match-a"] || !players["allocation-match-b"] { t.Fatalf("allocation outbox event = %s", eventPayload) } if _, found, err := ClaimAllocatingMatch(ctx, db, "enet", now.Add(2*time.Second)); err != nil || found { @@ -588,12 +600,15 @@ func TestPostgreSQLConnectionReceiptsStartCompleteRelaxedCasualRoster(t *testing if _, err := db.ExecContext(ctx, `INSERT INTO game_servers (server_id, region, build, protocol_version, transport, state) VALUES ('connect-server', 'EU', 'integration-build', 1, 'enet', 'ALLOCATED')`); err != nil { t.Fatal(err) } - if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, allocation_id, initial_connect_ready_at) VALUES ('connect-match', 'casual', 'ASSIGNMENT_READY', 'EU', 1, 'connect-server', 'connect-allocation', $1)`, now); err != nil { + if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version) VALUES ('connect-match', 'casual', 'ALLOCATING', 'EU', 1)`); err != nil { t.Fatal(err) } if _, err := db.ExecContext(ctx, `INSERT INTO allocations (allocation_id, match_id, server_id, region, build, protocol_version, transport, request_digest, state, allocated_at) VALUES ('connect-allocation', 'connect-match', 'connect-server', 'EU', 'integration-build', 1, 'enet', $1, 'ALLOCATED', $2)`, []byte("request"), now); err != nil { t.Fatal(err) } + if _, err := db.ExecContext(ctx, `UPDATE matches SET state = 'ASSIGNMENT_READY', server_id = 'connect-server', allocation_id = 'connect-allocation', allocation_claimed_at = $1, initial_connect_ready_at = $1 WHERE match_id = 'connect-match'`, now); err != nil { + t.Fatal(err) + } for i := 0; i < 2; i++ { playerID := fmt.Sprintf("connect-player-%d", i) ticketID := fmt.Sprintf("connect-ticket-%d", i) @@ -662,7 +677,7 @@ func TestPostgreSQLLiveReconnectGraceExpiryPersistsAbandonmentWithoutReleasingRe t.Fatal(err) } } - if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id) VALUES ('live-abandon-match', 'ranked', 'LIVE', 'EU', 1, 'live-abandon-server')`); err != nil { + if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, arena_path) VALUES ('live-abandon-match', 'ranked', 'LIVE', 'EU', 1, 'live-abandon-server', 'res://scenes/arena_01.tscn')`); err != nil { t.Fatal(err) } if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team, connection_generation, connected_at, disconnected_at) VALUES @@ -1523,7 +1538,18 @@ func TestPostgreSQLStalledAllocationsAreReclaimedWithoutPenalisingPlayers(t *tes if err := db.QueryRow(`SELECT event_type, payload FROM outbox WHERE event_id = 'stalled-allocation:stalled-match:1'`).Scan(&eventType, &eventPayload); err != nil { t.Fatalf("stalled allocation state event missing: %v", err) } - if eventType != "state_changed" || !strings.Contains(string(eventPayload), `"state":"FAILED"`) || !strings.Contains(string(eventPayload), `"stall-player-a"`) { + var event struct { + State string `json:"state"` + PlayerIDs []string `json:"player_ids"` + } + if err := json.Unmarshal(eventPayload, &event); err != nil { + t.Fatalf("decode stalled allocation outbox event: %v", err) + } + players := make(map[string]bool, len(event.PlayerIDs)) + for _, playerID := range event.PlayerIDs { + players[playerID] = true + } + if eventType != "state_changed" || event.State != "FAILED" || !players["stall-player-a"] { t.Fatalf("stalled allocation event = %s %s, want FAILED state and affected player IDs", eventType, eventPayload) } @@ -1653,8 +1679,8 @@ func TestPostgreSQLMigrationsRollBackAndReapplyCleanly(t *testing.T) { // Roll back every migration one at a time, in reverse, checking each // down file actually undoes what its forward file created — not just // that Rollback returns nil. - if err := migrations.Rollback(context.Background(), db, dir, 4); err != nil { - t.Fatalf("rollback 0010 through 0007: %v", err) + if err := migrations.Rollback(context.Background(), db, dir, 7); err != nil { + t.Fatalf("rollback 0013 through 0007: %v", err) } var hasInitialConnectReadyColumn bool if err := db.QueryRow(`SELECT count(*) > 0 FROM information_schema.columns WHERE table_name = 'matches' AND column_name = 'initial_connect_ready_at'`).Scan(&hasInitialConnectReadyColumn); err != nil { diff --git a/server/store/season_sql.go b/server/store/season_sql.go index 536bc8f7..2d03530c 100644 --- a/server/store/season_sql.go +++ b/server/store/season_sql.go @@ -39,10 +39,14 @@ func ApplyRankedSeasonRollover(ctx context.Context, db *sql.DB, playerID, season applied := false err := RunSerializable(ctx, db, DefaultSerializableAttempts, func(ctx context.Context, tx *sql.Tx) error { var locked domain.RankedProfile + var lockedPlayerID string var revision int64 - if err := tx.QueryRowContext(ctx, SeasonRatingLockSQL, playerID).Scan(&locked.Value, &locked.RD, &locked.Volatility, &locked.RankedGames, &revision); err != nil { + if err := tx.QueryRowContext(ctx, SeasonRatingLockSQL, playerID).Scan(&lockedPlayerID, &locked.Value, &locked.RD, &locked.Volatility, &locked.RankedGames, &revision); err != nil { return err } + if lockedPlayerID != playerID { + return fmt.Errorf("locked unexpected rating row") + } var err error updated, _, err = domain.ApplySeasonRollover(locked, seasonID) if err != nil { diff --git a/server/store/stalled_allocation_sql.go b/server/store/stalled_allocation_sql.go index b68dee2d..4e34f002 100644 --- a/server/store/stalled_allocation_sql.go +++ b/server/store/stalled_allocation_sql.go @@ -44,7 +44,7 @@ const ExpireStalledAllocationsSQL = `WITH stalled AS ( 'event', 'state_changed', 'revision', failed.revision, 'resource_id', failed.match_id, - 'occurred_at', $4, + 'occurred_at', $4::timestamptz, 'state', 'FAILED', 'match_id', failed.match_id, 'player_ids', COALESCE(( @@ -54,6 +54,7 @@ const ExpireStalledAllocationsSQL = `WITH stalled AS ( ) FROM failed ON CONFLICT DO NOTHING + RETURNING event_id ) SELECT (SELECT count(*) FROM failed), (SELECT count(*) FROM requeued), (SELECT count(*) FROM events)`