test(multiplayer): add real concurrent proposal-claim integration test

Every existing Postgres integration test runs strictly one transaction
at a time, so none of them exercise the SERIALIZABLE retry-and-fence
path CreateProposal actually depends on for correctness under real
matcher-replica contention -- only concurrent goroutines against a
real connection can. Add a test that races two goroutines each
proposing a formation that shares one contested ticket (a realistic
scenario: nothing stops two matcher replicas reading the same QUEUED
ticket in the same poll window), and asserts exactly one proposal
commits, the loser's proposal and participant rows are fully rolled
back, the contested ticket ends up claimed by the winner, and -- the
part a single-threaded test can't show -- the loser's OWN uncontested
ticket also rolls back to QUEUED rather than being left stranded as
PROPOSED with no surviving proposal.

Adversarial review of my own first draft: it initially failed
deterministically (5/5 runs), but the failure was in the test itself
-- the winner/loser branch picking the loser's uncontested ticket had
the two branches swapped, so it was checking the WINNER's ticket
against the QUEUED expectation. Fixed and re-verified clean across 8
runs with -race, plus the full integration suite.
This commit is contained in:
Josh Creek
2026-09-01 13:13:16 +01:00
parent e9d9f59a6a
commit 63eb43d50a
+103
View File
@@ -9,6 +9,7 @@ import (
"fmt"
"os"
"path/filepath"
"sync"
"testing"
"time"
@@ -394,6 +395,108 @@ func TestPostgreSQLProposalCreationRollsBackPartialClaims(t *testing.T) {
}
}
// TestPostgreSQLConcurrentProposalCreationClaimsContestedTicketOnce is the
// live counterpart to TestPostgreSQLProposalCreationRollsBackPartialClaims:
// every other proposal test in this file (and the whole matcher/allocator
// suite) runs its transactions strictly one at a time, so none of them can
// actually exercise the SERIALIZABLE retry-and-fence path CreateProposal
// relies on -- only two goroutines racing a real connection pool can. Two
// matchers independently form a proposal that both include the same waiting
// player's ticket (a real scenario: nothing stops two matcher replicas from
// reading the same QUEUED ticket in the same poll window); exactly one
// CreateProposal must win, the other must fail with its whole transaction
// rolled back, not a database/sql panic, deadlock, or a half-inserted row.
func TestPostgreSQLConcurrentProposalCreationClaimsContestedTicketOnce(t *testing.T) {
db := openIntegrationPostgres(t)
applyIntegrationMigrations(t, db)
now := time.Now().UTC().Truncate(time.Microsecond)
ctx := context.Background()
for _, player := range []string{"race-player-a", "race-player-b", "race-player-c"} {
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
t.Fatal(err)
}
}
tickets := map[string]string{"race-player-a": "race-ticket-a", "race-player-b": "race-ticket-b", "race-player-c": "race-ticket-c"}
for player, ticket := range tickets {
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'QUEUED', 'integration-build', 1, $3, $4)`, ticket, player, now, now.Add(time.Minute)); err != nil {
t.Fatal(err)
}
}
proposalA, err := domain.NewProposal("race-proposal-a", domain.Casual, []string{"race-player-a", "race-player-b"}, now)
if err != nil {
t.Fatal(err)
}
proposalB, err := domain.NewProposal("race-proposal-b", domain.Casual, []string{"race-player-b", "race-player-c"}, now)
if err != nil {
t.Fatal(err)
}
var wg sync.WaitGroup
errs := make([]error, 2)
wg.Add(2)
go func() {
defer wg.Done()
errs[0] = CreateProposal(ctx, db, proposalA, map[string]string{"race-player-a": tickets["race-player-a"], "race-player-b": tickets["race-player-b"]}, now)
}()
go func() {
defer wg.Done()
errs[1] = CreateProposal(ctx, db, proposalB, map[string]string{"race-player-b": tickets["race-player-b"], "race-player-c": tickets["race-player-c"]}, now)
}()
wg.Wait()
succeeded := errs[0] == nil
if succeeded == (errs[1] == nil) {
t.Fatalf("exactly one contested proposal must win, got errA=%v errB=%v", errs[0], errs[1])
}
winner, loser := "race-proposal-a", "race-proposal-b"
if !succeeded {
winner, loser = "race-proposal-b", "race-proposal-a"
}
var winnerRows, loserRows, loserParticipants int
if err := db.QueryRow(`SELECT count(*) FROM proposals WHERE proposal_id = $1`, winner).Scan(&winnerRows); err != nil {
t.Fatal(err)
}
if err := db.QueryRow(`SELECT count(*) FROM proposals WHERE proposal_id = $1`, loser).Scan(&loserRows); err != nil {
t.Fatal(err)
}
if err := db.QueryRow(`SELECT count(*) FROM proposal_participants WHERE proposal_id = $1`, loser).Scan(&loserParticipants); err != nil {
t.Fatal(err)
}
if winnerRows != 1 {
t.Fatalf("winning proposal %s was not persisted", winner)
}
if loserRows != 0 || loserParticipants != 0 {
t.Fatalf("losing proposal %s was not fully rolled back: proposals=%d participants=%d", loser, loserRows, loserParticipants)
}
var contestedState string
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = $1`, tickets["race-player-b"]).Scan(&contestedState); err != nil {
t.Fatal(err)
}
if contestedState != "PROPOSED" {
t.Fatalf("contested ticket should be claimed by the winner, got state=%s", contestedState)
}
// The loser's OWN uncontested ticket (a or c) must have rolled back to
// QUEUED too -- CreateProposal is one transaction per proposal, so a
// contested loss on one participant must not leave another participant's
// ticket stranded as PROPOSED with no surviving proposal to reference it.
// A (player-a + contested player-b) won iff succeeded, in which case B's
// own uncontested ticket (player-c) is the one that must have rolled
// back; if A lost, it's A's own uncontested ticket (player-a) instead.
loserOnlyTicket := tickets["race-player-c"]
if !succeeded {
loserOnlyTicket = tickets["race-player-a"]
}
var loserOnlyState string
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = $1`, loserOnlyTicket).Scan(&loserOnlyState); err != nil {
t.Fatal(err)
}
if loserOnlyState != "QUEUED" {
t.Fatalf("loser's uncontested ticket %s should have rolled back to QUEUED, got %s", loserOnlyTicket, loserOnlyState)
}
}
func TestPostgreSQLResultCompletionAndOutboxAreAtomicAndReplayable(t *testing.T) {
db := openIntegrationPostgres(t)
applyIntegrationMigrations(t, db)