mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
801fca7cb0
domain.validCandidate hard-requires a non-empty PredictedRTT map, but
CreateQueueTicket persisted an empty one and the only endpoint that
could fill it returned 503 in every real binary, because Service.Probe
was assigned nowhere outside api tests. No client-created ticket could
ever be selected by the matcher. The Godot client had no probe method at
all, so even a wired backend was unreachable from the game.
Four distinct defects had to be fixed for this path to work:
Nothing issued the nonce ProbeProvider was meant to compare against, so
the contract could not be satisfied even in principle. Add
POST /v1/probes/{region}/challenge, backed by a durable single-use
challenge -- durable because any replica may serve the answer for a
challenge another replica issued. RTT is the interval between issuing
and receiving, so no client-reported latency reaches placement.
CreateQueueTicket marshalled a nil map to JSON `null`, a JSONB scalar
rather than an object, and jsonb_set rejects that with "cannot set path
in scalar". RecordProbe would have failed at runtime even once wired.
Persist an object, and normalise non-object values in the update for
rows already written.
A nil ProbeRecorder made the handler report success while persisting
nothing, which silently leaves the ticket unmatchable. That is a
misconfiguration, not a successful probe; it now returns 503.
A successful probe updated PostgreSQL only. The candidate inserted at
enqueue time carries an empty RTT map, and the Redis keyspace has its
TTL continually refreshed, so the stale entry need never repair itself.
Refresh that player's projection after the probe commits.
Client side: add the challenge/answer round trip and have the
matchmaking screen collect evidence before creating a ticket, since
queueing first produces a search that can never match. Probing every
region fully is not required -- placement uses whichever regions
answered -- but queueing with none is refused rather than silently
stalling.
New integration test drives the real enqueue and probe paths and then
asks the actual matcher predicate, rather than hand-building a candidate
the way the unit tests do -- which is exactly why they missed this.
Also make the integration schema reset drop the whole public schema: the
enumerated table list silently broke with each new migration.
2280 lines
109 KiB
Go
2280 lines
109 KiB
Go
//go:build integration
|
|
|
|
package store
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/cosmic-clash/cosmic-clash/server/domain"
|
|
"github.com/cosmic-clash/cosmic-clash/server/migrations"
|
|
_ "github.com/jackc/pgx/v5/stdlib"
|
|
)
|
|
|
|
// This binary is deliberately opt-in. It requires a disposable PostgreSQL
|
|
// instance supplied by scripts/run_postgres_integration.sh.
|
|
func openIntegrationPostgres(t *testing.T) *sql.DB {
|
|
t.Helper()
|
|
dsn := os.Getenv("COSMIC_CLASH_POSTGRES_DSN")
|
|
if dsn == "" {
|
|
t.Skip("COSMIC_CLASH_POSTGRES_DSN is not set")
|
|
}
|
|
db, err := sql.Open("pgx", dsn)
|
|
if err != nil {
|
|
t.Fatalf("open PostgreSQL: %v", err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if err := db.PingContext(ctx); err != nil {
|
|
db.Close()
|
|
t.Fatalf("ping PostgreSQL: %v", err)
|
|
}
|
|
t.Cleanup(func() { db.Close() })
|
|
return db
|
|
}
|
|
|
|
func applyIntegrationMigrations(t *testing.T, db *sql.DB) {
|
|
t.Helper()
|
|
if _, err := db.ExecContext(context.Background(), `DROP SCHEMA public CASCADE; CREATE SCHEMA public`); err != nil {
|
|
t.Fatalf("reset PostgreSQL schema: %v", err)
|
|
}
|
|
if err := migrations.Apply(context.Background(), db, filepath.Join("..", "migrations")); err != nil {
|
|
t.Fatalf("apply migrations: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLAllocatorClaimReplayAndCapacityFence(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
servers := []domain.ReadyServer{
|
|
{ServerID: "allocator-server-b", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet", State: domain.ServerReady},
|
|
{ServerID: "allocator-server-a", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet", State: domain.ServerReady},
|
|
}
|
|
for _, server := range servers {
|
|
if err := RegisterReadyServer(ctx, db, server, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
request := domain.AllocationRequest{AllocationID: "allocation-integration-1", MatchID: "match-integration-1", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet"}
|
|
allocation, err := ClaimAllocation(ctx, db, request, now)
|
|
if err != nil {
|
|
t.Fatalf("claim: %v", err)
|
|
}
|
|
if allocation.ServerID != "allocator-server-a" || allocation.State != domain.ServerAllocated {
|
|
t.Fatalf("allocation=%+v", allocation)
|
|
}
|
|
if err := RegisterReadyServer(ctx, db, servers[1], now.Add(time.Second)); err != nil {
|
|
t.Fatalf("stale Ready projection: %v", err)
|
|
}
|
|
var lifecycle string
|
|
if err := db.QueryRowContext(ctx, `SELECT state FROM game_servers WHERE server_id = 'allocator-server-a'`).Scan(&lifecycle); err != nil || lifecycle != "ALLOCATED" {
|
|
t.Fatalf("stale Ready projection reopened allocation state=%q err=%v", lifecycle, err)
|
|
}
|
|
replay, err := ClaimAllocation(ctx, db, request, now.Add(time.Second))
|
|
if err != nil || replay.ServerID != allocation.ServerID || !replay.AllocatedAt.Equal(allocation.AllocatedAt) {
|
|
t.Fatalf("replay=%+v err=%v", replay, err)
|
|
}
|
|
conflict := request
|
|
conflict.MatchID = "match-integration-other"
|
|
if _, err := ClaimAllocation(ctx, db, conflict, now); err != domain.ErrConflict {
|
|
t.Fatalf("conflicting replay err=%v", err)
|
|
}
|
|
second := request
|
|
second.AllocationID = "allocation-integration-2"
|
|
second.MatchID = "match-integration-2"
|
|
if _, err := ClaimAllocation(ctx, db, second, now); err != nil {
|
|
t.Fatalf("second claim: %v", err)
|
|
}
|
|
third := second
|
|
third.AllocationID = "allocation-integration-3"
|
|
third.MatchID = "match-integration-3"
|
|
if _, err := ClaimAllocation(ctx, db, third, now); err != domain.ErrNoCapacity {
|
|
t.Fatalf("capacity err=%v", err)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLSharedAllocationQuotaFencesClaimsAndReplays(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
if err := SetAllocationQuota(ctx, db, "EU", 1, time.Minute, now); err != nil {
|
|
t.Fatalf("set quota: %v", err)
|
|
}
|
|
for _, server := range []domain.ReadyServer{
|
|
{ServerID: "quota-server-a", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet", State: domain.ServerReady},
|
|
{ServerID: "quota-server-b", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet", State: domain.ServerReady},
|
|
} {
|
|
if err := RegisterReadyServer(ctx, db, server, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
first := domain.AllocationRequest{AllocationID: "quota-allocation-1", MatchID: "quota-match-1", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet"}
|
|
if _, err := ClaimAllocation(ctx, db, first, now); err != nil {
|
|
t.Fatalf("first claim: %v", err)
|
|
}
|
|
if _, err := ClaimAllocation(ctx, db, first, now.Add(time.Second)); err != nil {
|
|
t.Fatalf("idempotent replay was fenced: %v", err)
|
|
}
|
|
second := domain.AllocationRequest{AllocationID: "quota-allocation-2", MatchID: "quota-match-2", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet"}
|
|
if _, err := ClaimAllocation(ctx, db, second, now.Add(2*time.Second)); !errors.Is(err, ErrAllocationQuotaExceeded) {
|
|
t.Fatalf("second claim err=%v, want shared quota fence", err)
|
|
}
|
|
if err := SetAllocationQuota(ctx, db, "EU", 1, time.Minute, now.Add(time.Minute)); err != nil {
|
|
t.Fatalf("reset quota: %v", err)
|
|
}
|
|
if _, err := ClaimAllocation(ctx, db, second, now.Add(time.Minute)); err != nil {
|
|
t.Fatalf("claim after quota window reset: %v", err)
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLConcurrentAllocationClaimNeverDoubleBooksAReadyServer is the
|
|
// live counterpart to TestPostgreSQLAllocatorClaimReplayAndCapacityFence: that
|
|
// test claims strictly one request at a time, so it cannot show what happens
|
|
// when two allocator replicas race for the same compatible capacity, which is
|
|
// exactly the scenario 8.30's "bounded cross-replica retry" is about. Register
|
|
// fewer Ready servers than concurrent requests and fire them all at once;
|
|
// exactly as many must win as there was capacity, each winner must get a
|
|
// distinct server, and every loser must fail with ErrNoCapacity rather than a
|
|
// raw serialization error, a duplicate claim, or a hang.
|
|
func TestPostgreSQLConcurrentAllocationClaimNeverDoubleBooksAReadyServer(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
const capacity = 3
|
|
const contenders = 6
|
|
for i := 0; i < capacity; i++ {
|
|
server := domain.ReadyServer{ServerID: fmt.Sprintf("race-server-%d", i), Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet", State: domain.ServerReady}
|
|
if err := RegisterReadyServer(ctx, db, server, now); err != nil {
|
|
t.Fatalf("register %s: %v", server.ServerID, err)
|
|
}
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
allocations := make([]domain.Allocation, contenders)
|
|
errs := make([]error, contenders)
|
|
wg.Add(contenders)
|
|
for i := 0; i < contenders; i++ {
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
request := domain.AllocationRequest{AllocationID: fmt.Sprintf("race-allocation-%d", i), MatchID: fmt.Sprintf("race-match-%d", i), Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet"}
|
|
allocations[i], errs[i] = ClaimAllocation(ctx, db, request, now)
|
|
}(i)
|
|
}
|
|
wg.Wait()
|
|
|
|
wonServers := map[string]int{}
|
|
won, lost := 0, 0
|
|
for i, err := range errs {
|
|
switch {
|
|
case err == nil:
|
|
won++
|
|
if allocations[i].ServerID == "" {
|
|
t.Fatalf("claim %d succeeded with no server", i)
|
|
}
|
|
wonServers[allocations[i].ServerID]++
|
|
case errors.Is(err, domain.ErrNoCapacity):
|
|
lost++
|
|
default:
|
|
t.Fatalf("claim %d failed with unexpected error: %v", i, err)
|
|
}
|
|
}
|
|
if won != capacity || lost != contenders-capacity {
|
|
t.Fatalf("won=%d lost=%d, want won=%d lost=%d", won, lost, capacity, contenders-capacity)
|
|
}
|
|
if len(wonServers) != capacity {
|
|
t.Fatalf("expected %d distinct servers claimed, got %d: %v", capacity, len(wonServers), wonServers)
|
|
}
|
|
for server, count := range wonServers {
|
|
if count != 1 {
|
|
t.Fatalf("server %s was claimed %d times", server, count)
|
|
}
|
|
}
|
|
var allocatedCount int
|
|
if err := db.QueryRowContext(ctx, `SELECT count(*) FROM game_servers WHERE state = 'ALLOCATED'`).Scan(&allocatedCount); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if allocatedCount != capacity {
|
|
t.Fatalf("durable ALLOCATED server count = %d, want %d", allocatedCount, capacity)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLAcceptedProposalPromotesOneAtomicAllocatingMatch(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"promote-a", "promote-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for index, player := range []string{"promote-a", "promote-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'PROPOSED', 'build-1', 1, $3, $4)`, fmt.Sprintf("promote-ticket-%d", index), player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO proposals (proposal_id, playlist, state, expires_at, revision) VALUES ('promote-proposal', 'casual', 'ACCEPTED', $1, 2)`, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for index, player := range []string{"promote-a", "promote-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO proposal_participants (proposal_id, player_id, ticket_id, response) VALUES ('promote-proposal', $1, $2, 'ACCEPTED')`, player, fmt.Sprintf("promote-ticket-%d", index)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
plan := AcceptedMatchPlan{MatchID: "promote-match", ProposalID: "promote-proposal", Region: "EU", Protocol: 1, Players: []MatchPlayer{{PlayerID: "promote-a", Team: 0, Slot: 0}, {PlayerID: "promote-b", Team: 1, Slot: 3}}}
|
|
if err := CreateMatchFromAcceptedProposal(ctx, db, plan, now); err != nil {
|
|
t.Fatalf("promote accepted proposal: %v", err)
|
|
}
|
|
var state string
|
|
if err := db.QueryRowContext(ctx, `SELECT state FROM matches WHERE match_id = 'promote-match'`).Scan(&state); err != nil || state != "ALLOCATING" {
|
|
t.Fatalf("match state=%q err=%v", state, err)
|
|
}
|
|
var acceptedTickets, participantCount int
|
|
if err := db.QueryRowContext(ctx, `SELECT count(*) FROM queue_tickets WHERE ticket_id LIKE 'promote-ticket-%' AND state = 'ACCEPTED'`).Scan(&acceptedTickets); err != nil || acceptedTickets != 2 {
|
|
t.Fatalf("accepted tickets=%d err=%v", acceptedTickets, err)
|
|
}
|
|
if err := db.QueryRowContext(ctx, `SELECT count(*) FROM match_participants WHERE match_id = 'promote-match'`).Scan(&participantCount); err != nil || participantCount != 2 {
|
|
t.Fatalf("participants=%d err=%v", participantCount, err)
|
|
}
|
|
if err := CreateMatchFromAcceptedProposal(ctx, db, plan, now.Add(time.Second)); err != nil {
|
|
t.Fatalf("identical match promotion replay: %v", err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `UPDATE matches SET state = 'LIVE' WHERE match_id = 'promote-match'`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := CreateMatchFromAcceptedProposal(ctx, db, plan, now.Add(2*time.Second)); err != nil {
|
|
t.Fatalf("promotion replay after match lifecycle advanced: %v", err)
|
|
}
|
|
conflict := plan
|
|
conflict.Players = append([]MatchPlayer(nil), plan.Players...)
|
|
conflict.Players[1].Slot = 4
|
|
if err := CreateMatchFromAcceptedProposal(ctx, db, conflict, now.Add(3*time.Second)); err == nil {
|
|
t.Fatal("conflicting match promotion replay was accepted")
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLAllocationMatchClaimLeaseAndBindFence(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for index, player := range []string{"allocation-match-a", "allocation-match-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'ACCEPTED', 'build-1', 1, $3, $4)`, fmt.Sprintf("allocation-match-ticket-%d", index), player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version) VALUES ('allocation-match', 'casual', 'ALLOCATING', 'EU', 1)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for index, player := range []string{"allocation-match-a", "allocation-match-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) VALUES ('allocation-match', $1, $2, $3, $4)`, player, fmt.Sprintf("allocation-match-ticket-%d", index), index*3, index); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
claim, found, err := ClaimAllocatingMatch(ctx, db, "enet", now)
|
|
if err != nil || !found || claim.Request != (domain.AllocationRequest{AllocationID: "allocation-allocation-match", MatchID: "allocation-match", Playlist: domain.Casual, Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet"}) {
|
|
t.Fatalf("claim=%+v found=%t err=%v", claim, found, err)
|
|
}
|
|
if err := ReleaseAllocatedMatchClaim(ctx, db, claim.Request.MatchID, "different-allocation"); err != domain.ErrConflict {
|
|
t.Fatalf("wrong-claim release err=%v", err)
|
|
}
|
|
if err := ReleaseAllocatedMatchClaim(ctx, db, claim.Request.MatchID, claim.Request.AllocationID); err != nil {
|
|
t.Fatalf("release claim: %v", err)
|
|
}
|
|
reclaimed, found, err := ClaimAllocatingMatch(ctx, db, "enet", now.Add(time.Second))
|
|
if err != nil || !found || reclaimed.Request.AllocationID != claim.Request.AllocationID {
|
|
t.Fatalf("reclaimed=%+v found=%t err=%v", reclaimed, found, err)
|
|
}
|
|
server := domain.ReadyServer{ServerID: "allocation-server", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet", State: domain.ServerReady}
|
|
if err := RegisterReadyServer(ctx, db, server, now); err != nil {
|
|
t.Fatalf("register allocation server: %v", err)
|
|
}
|
|
allocation, err := ClaimAllocation(ctx, db, reclaimed.Request, now.Add(time.Second))
|
|
if err != nil {
|
|
t.Fatalf("record provider allocation: %v", err)
|
|
}
|
|
if err := BindAllocatedMatch(ctx, db, allocation); err != nil {
|
|
t.Fatalf("bind allocation: %v", err)
|
|
}
|
|
recoveredTicket, err := GetQueueTicket(ctx, db, "allocation-match-a", "allocation-match-ticket-0", now.Add(2*time.Second))
|
|
if err != nil || recoveredTicket.MatchID != "allocation-match" {
|
|
t.Fatalf("recovered ticket match=%q err=%v", recoveredTicket.MatchID, err)
|
|
}
|
|
var allocatingTickets int
|
|
if err := db.QueryRowContext(ctx, `SELECT count(*) FROM queue_tickets WHERE ticket_id LIKE 'allocation-match-ticket-%' AND state = 'ALLOCATING'`).Scan(&allocatingTickets); err != nil || allocatingTickets != 2 {
|
|
t.Fatalf("allocating tickets=%d err=%v", allocatingTickets, err)
|
|
}
|
|
var eventType string
|
|
var eventPayload []byte
|
|
if err := db.QueryRowContext(ctx, `SELECT event_type, payload FROM outbox WHERE aggregate_id = 'allocation-match' AND event_type = 'state_changed'`).Scan(&eventType, &eventPayload); err != nil {
|
|
t.Fatalf("allocation outbox event: %v", err)
|
|
}
|
|
var event struct {
|
|
State string `json:"state"`
|
|
PlayerIDs []string `json:"player_ids"`
|
|
}
|
|
if err := json.Unmarshal(eventPayload, &event); err != nil {
|
|
t.Fatalf("decode allocation outbox event: %v", err)
|
|
}
|
|
players := make(map[string]bool, len(event.PlayerIDs))
|
|
for _, playerID := range event.PlayerIDs {
|
|
players[playerID] = true
|
|
}
|
|
if eventType != "state_changed" || event.State != "ALLOCATING" || !players["allocation-match-a"] || !players["allocation-match-b"] {
|
|
t.Fatalf("allocation outbox event = %s", eventPayload)
|
|
}
|
|
if _, found, err := ClaimAllocatingMatch(ctx, db, "enet", now.Add(2*time.Second)); err != nil || found {
|
|
t.Fatalf("bound match re-claimed found=%t err=%v", found, err)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLQueueAdapterAgainstRealDatabase(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('integration-player', 'integration-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
spec := domain.QueueSpec{Playlist: domain.Casual, ClientBuild: "integration-build", ProtocolVersion: 1}
|
|
ticket, err := CreateQueueTicket(ctx, db, "integration-ticket", "integration-player", "integration-create-0001", spec, now)
|
|
if err != nil {
|
|
t.Fatalf("create queue ticket: %v", err)
|
|
}
|
|
if ticket.State != domain.Queued || ticket.Revision != 0 {
|
|
t.Fatalf("unexpected ticket: %+v", ticket)
|
|
}
|
|
replay, err := CreateQueueTicket(ctx, db, "integration-ticket", "integration-player", "integration-create-0001", spec, now.Add(time.Second))
|
|
if err != nil {
|
|
t.Fatalf("idempotent queue replay: %v", err)
|
|
}
|
|
if replay.TicketID != ticket.TicketID || !replay.ExpiresAt.Equal(ticket.ExpiresAt) {
|
|
t.Fatalf("replay changed durable result: %+v vs %+v", replay, ticket)
|
|
}
|
|
if _, err := CreateQueueTicket(ctx, db, "integration-ticket-2", "integration-player", "integration-create-0002", spec, now); err == nil {
|
|
t.Fatal("second active player ticket was accepted")
|
|
}
|
|
if _, err := GetQueueTicket(ctx, db, "integration-player", "integration-ticket", now); err != nil {
|
|
t.Fatalf("owner recovery: %v", err)
|
|
}
|
|
if _, err := GetQueueTicket(ctx, db, "other-player", "integration-ticket", now); err == nil {
|
|
t.Fatal("non-owner recovered queue ticket")
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLQueueHeartbeatAndCancelAreRevisionFenced(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('heartbeat-player', 'heartbeat-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
spec := domain.QueueSpec{Playlist: domain.Ranked, ClientBuild: "integration-build", ProtocolVersion: 1}
|
|
if _, err := CreateQueueTicket(ctx, db, "heartbeat-ticket", "heartbeat-player", "heartbeat-create-0001", spec, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
heartbeat, err := HeartbeatQueueTicket(ctx, db, "heartbeat-player", "heartbeat-ticket", "heartbeat-op-0000001", 0, now.Add(5*time.Second))
|
|
if err != nil {
|
|
t.Fatalf("heartbeat: %v", err)
|
|
}
|
|
if heartbeat.Revision != 1 || !heartbeat.ExpiresAt.Equal(now.Add(35*time.Second)) {
|
|
t.Fatalf("unexpected heartbeat result: %+v", heartbeat)
|
|
}
|
|
if _, err := HeartbeatQueueTicket(ctx, db, "heartbeat-player", "heartbeat-ticket", "heartbeat-op-0000002", 0, now.Add(6*time.Second)); !errors.Is(err, domain.ErrStaleRevision) {
|
|
t.Fatalf("stale heartbeat error = %v, want ErrStaleRevision", err)
|
|
}
|
|
cancelled, err := CancelQueueTicket(ctx, db, "heartbeat-player", "heartbeat-ticket", "heartbeat-op-0000003", 1, now.Add(7*time.Second))
|
|
if err != nil {
|
|
t.Fatalf("cancel: %v", err)
|
|
}
|
|
if cancelled.State != domain.Cancelled || cancelled.Revision != 2 {
|
|
t.Fatalf("unexpected cancellation result: %+v", cancelled)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('live-cancel-player', 'live-cancel-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ('live-cancel-ticket', 'live-cancel-player', 'ranked', 'LIVE', 'integration-build', 1, $1, $2)`, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := CancelQueueTicket(ctx, db, "live-cancel-player", "live-cancel-ticket", "live-cancel-op-0001", 0, now.Add(8*time.Second)); !errors.Is(err, domain.ErrConflict) {
|
|
t.Fatalf("live cancellation error = %v, want ErrConflict", err)
|
|
}
|
|
var liveState string
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'live-cancel-ticket'`).Scan(&liveState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if liveState != "LIVE" {
|
|
t.Fatalf("live ticket state = %s after cancellation attempt", liveState)
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLConcurrentQueueHeartbeatIsRevisionFencedUnderRealRace is the
|
|
// live counterpart to the sequential stale-heartbeat check above: calling the
|
|
// second heartbeat only after the first has already committed proves the SQL
|
|
// predicate is correct, but not that it actually fences two requests that
|
|
// genuinely overlap at the database. A client can legitimately double-send a
|
|
// heartbeat (a slow response triggering a client-side retry, or two tabs/
|
|
// processes for the same player), and both requests can reach PostgreSQL
|
|
// truly concurrently -- this races that directly.
|
|
func TestPostgreSQLConcurrentQueueHeartbeatIsRevisionFencedUnderRealRace(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('race-heartbeat-player', 'race-heartbeat-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
spec := domain.QueueSpec{Playlist: domain.Casual, ClientBuild: "integration-build", ProtocolVersion: 1}
|
|
if _, err := CreateQueueTicket(ctx, db, "race-heartbeat-ticket", "race-heartbeat-player", "race-heartbeat-create-01", spec, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
const attempts = 5
|
|
var wg sync.WaitGroup
|
|
tickets := make([]domain.QueueTicket, attempts)
|
|
errs := make([]error, attempts)
|
|
wg.Add(attempts)
|
|
for i := 0; i < attempts; i++ {
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
tickets[i], errs[i] = HeartbeatQueueTicket(ctx, db, "race-heartbeat-player", "race-heartbeat-ticket", fmt.Sprintf("race-heartbeat-op-%08d", i), 0, now.Add(time.Duration(i)*time.Millisecond))
|
|
}(i)
|
|
}
|
|
wg.Wait()
|
|
|
|
won, lost := 0, 0
|
|
for i, err := range errs {
|
|
if err == nil {
|
|
won++
|
|
if tickets[i].Revision != 1 {
|
|
t.Fatalf("winning heartbeat %d landed at revision %d, want 1", i, tickets[i].Revision)
|
|
}
|
|
continue
|
|
}
|
|
lost++
|
|
}
|
|
if won != 1 {
|
|
t.Fatalf("won=%d, want exactly 1 of %d concurrent heartbeats at the same expected revision to win", won, attempts)
|
|
}
|
|
if lost != attempts-1 {
|
|
t.Fatalf("lost=%d, want %d", lost, attempts-1)
|
|
}
|
|
var revision uint64
|
|
if err := db.QueryRow(`SELECT revision FROM queue_tickets WHERE ticket_id = 'race-heartbeat-ticket'`).Scan(&revision); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if revision != 1 {
|
|
t.Fatalf("durable revision = %d, want exactly 1 (a stale winner re-applying would leave it higher)", revision)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLAssignmentPersistenceIsPlayerScopedAndExpiryBound(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"assignment-player", "assignment-other"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ('assignment-ticket', 'assignment-player', 'casual', 'ASSIGNED', 'integration-build', 1, $1, $2)`, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, initial_connect_ready_at) VALUES ('assignment-match', 'casual', 'ASSIGNED', 'EU', 1, 'assignment-server', $1)`, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) VALUES ('assignment-match', 'assignment-player', 'assignment-ticket', 0, 0)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assignment := DurableAssignment{MatchID: "assignment-match", PlayerID: "assignment-player", AllocationID: "allocation-1", ServerID: "assignment-server", Slot: 0, Region: "EU", ClientBuild: "integration-build", ProtocolVersion: 1, Transport: "enet", Endpoint: "127.0.0.1:7777", JoinAuthorisation: "join-token", ManifestDigest: []byte("manifest"), ExpiresAt: now.Add(time.Minute), Revision: 1}
|
|
if err := SaveAssignment(ctx, db, assignment); err != nil {
|
|
t.Fatalf("save assignment: %v", err)
|
|
}
|
|
got, err := GetAssignment(ctx, db, assignment.PlayerID, assignment.MatchID, now)
|
|
if err != nil {
|
|
t.Fatalf("recover assignment: %v", err)
|
|
}
|
|
if got.JoinAuthorisation != assignment.JoinAuthorisation || got.Slot != assignment.Slot {
|
|
t.Fatalf("assignment changed on round trip: %+v", got)
|
|
}
|
|
if _, err := GetAssignment(ctx, db, "assignment-other", assignment.MatchID, now); err == nil {
|
|
t.Fatal("non-owner recovered assignment")
|
|
}
|
|
if _, err := GetAssignment(ctx, db, assignment.PlayerID, assignment.MatchID, now.Add(2*time.Minute)); err == nil {
|
|
t.Fatal("expired assignment was recovered")
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLVerifiedAssignmentRosterMustMatchDurableParticipants(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for index, player := range []string{"roster-player-a", "roster-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $2)`, player, "steam-"+player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'ALLOCATING', 'build-1', 1, $3, $4)`, fmt.Sprintf("roster-ticket-%d", index), player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO game_servers (server_id, region, build, protocol_version, transport, state, updated_at) VALUES ('roster-server', 'EU', 'build-1', 1, 'enet', 'ALLOCATED', $1)`, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO allocations (allocation_id, match_id, server_id, region, build, protocol_version, transport, request_digest, state, allocated_at) VALUES ('roster-allocation', 'roster-match', 'roster-server', 'EU', 'build-1', 1, 'enet', 'digest', 'ALLOCATED', $1)`, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, allocation_id, allocation_claimed_at) VALUES ('roster-match', 'casual', 'ALLOCATING', 'EU', 1, 'roster-server', 'roster-allocation', $1)`, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for index, player := range []string{"roster-player-a", "roster-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) VALUES ('roster-match', $1, $2, $3, $4)`, player, fmt.Sprintf("roster-ticket-%d", index), index*3, index); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
allocation := domain.Allocation{AllocationID: "roster-allocation", MatchID: "roster-match", ServerID: "roster-server", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet", State: domain.ServerAllocated, AllocatedAt: now}
|
|
assignment := domain.Assignment{Allocation: allocation, Manifest: domain.AllocationManifest{AllocationID: allocation.AllocationID, MatchID: allocation.MatchID, ServerID: allocation.ServerID, Region: allocation.Region, Build: allocation.Build, Protocol: allocation.Protocol, Transport: allocation.Transport, RosterDigest: "roster-digest"}, Endpoint: "127.0.0.1:7777"}
|
|
roster := []domain.SignedJoinAuthorisation{
|
|
{Authorisation: domain.JoinAuthorisation{MatchID: "roster-match", ServerID: "roster-server", PlayerID: "roster-player-a", SteamID: "steam-roster-player-a", Slot: 0, Team: 0, Protocol: "1", Generation: 1, ExpiresAt: now.Add(time.Minute)}, Signature: []byte("sig-a")},
|
|
{Authorisation: domain.JoinAuthorisation{MatchID: "roster-match", ServerID: "roster-server", PlayerID: "roster-player-b", SteamID: "steam-roster-player-b", Slot: 3, Team: 1, Protocol: "1", Generation: 1, ExpiresAt: now.Add(time.Minute)}, Signature: []byte("sig-b")},
|
|
}
|
|
verify := func([]byte, []byte) bool { return true }
|
|
if err := SaveVerifiedAssignmentRoster(ctx, db, assignment, roster[:1], verify); err == nil {
|
|
t.Fatal("partial signed roster was accepted")
|
|
}
|
|
var count int
|
|
if err := db.QueryRow(`SELECT count(*) FROM assignments WHERE match_id = 'roster-match'`).Scan(&count); err != nil || count != 0 {
|
|
t.Fatalf("partial roster persisted rows=%d err=%v", count, err)
|
|
}
|
|
if err := SaveVerifiedAssignmentRoster(ctx, db, assignment, roster, verify); err != nil {
|
|
t.Fatalf("complete durable roster rejected: %v", err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM assignments WHERE match_id = 'roster-match'`).Scan(&count); err != nil || count != 2 {
|
|
t.Fatalf("complete roster rows=%d err=%v", count, err)
|
|
}
|
|
forged := append([]domain.SignedJoinAuthorisation(nil), roster...)
|
|
forged[1].Authorisation.SteamID = "steam-other"
|
|
if err := SaveVerifiedAssignmentRoster(ctx, db, assignment, forged, verify); err == nil {
|
|
t.Fatal("signed roster with wrong durable Steam identity was accepted")
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLConnectionReceiptsStartCompleteRelaxedCasualRoster(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
|
|
for i := 0; i < 2; i++ {
|
|
playerID := fmt.Sprintf("connect-player-%d", i)
|
|
ticketID := fmt.Sprintf("connect-ticket-%d", i)
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, playerID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'ASSIGNMENT_READY', 'integration-build', 1, $3, $4)`, ticketID, playerID, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO game_servers (server_id, region, build, protocol_version, transport, state) VALUES ('connect-server', 'EU', 'integration-build', 1, 'enet', 'ALLOCATED')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version) VALUES ('connect-match', 'casual', 'ALLOCATING', 'EU', 1)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO allocations (allocation_id, match_id, server_id, region, build, protocol_version, transport, request_digest, state, allocated_at) VALUES ('connect-allocation', 'connect-match', 'connect-server', 'EU', 'integration-build', 1, 'enet', $1, 'ALLOCATED', $2)`, []byte("request"), now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `UPDATE matches SET state = 'ASSIGNMENT_READY', server_id = 'connect-server', allocation_id = 'connect-allocation', allocation_claimed_at = $1, initial_connect_ready_at = $1 WHERE match_id = 'connect-match'`, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 2; i++ {
|
|
playerID := fmt.Sprintf("connect-player-%d", i)
|
|
ticketID := fmt.Sprintf("connect-ticket-%d", i)
|
|
slot := i * 3
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) VALUES ('connect-match', $1, $2, $3, $4)`, playerID, ticketID, slot, i); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO assignments (match_id, player_id, allocation_id, server_id, slot, region, client_build, protocol_version, transport, endpoint, join_authorisation, manifest_digest, expires_at) VALUES ('connect-match', $1, 'connect-allocation', 'connect-server', $2, 'EU', 'integration-build', 1, 'enet', '127.0.0.1:7777', 'join-token', $3, $4)`, playerID, slot, []byte("manifest"), now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
binding := domain.WorkloadBinding{AllocationID: "connect-allocation", MatchID: "connect-match", ServerID: "connect-server"}
|
|
if generation, err := ClaimPlayerConnection(ctx, db, binding, "connect-player-0", 0, "connect-receipt-key-0000", now); err != nil || generation != 1 {
|
|
t.Fatalf("first receipt: %v", err)
|
|
}
|
|
if _, err := ClaimPlayerConnection(ctx, db, domain.WorkloadBinding{AllocationID: "forged-allocation", MatchID: "connect-match", ServerID: "connect-server"}, "connect-player-1", 0, "connect-receipt-key-forged", now); !errors.Is(err, domain.ErrConflict) {
|
|
t.Fatalf("forged binding err=%v, want conflict", err)
|
|
}
|
|
if generation, err := ClaimPlayerConnection(ctx, db, binding, "connect-player-1", 0, "connect-receipt-key-0001", now); err != nil || generation != 1 {
|
|
t.Fatalf("second receipt: %v", err)
|
|
}
|
|
reconciled, err := ReconcileInitialConnect(ctx, db, now.Add(time.Second), 10)
|
|
if err != nil || reconciled != 1 {
|
|
t.Fatalf("reconcile count=%d err=%v", reconciled, err)
|
|
}
|
|
var state string
|
|
if err := db.QueryRowContext(ctx, `SELECT state FROM matches WHERE match_id = 'connect-match'`).Scan(&state); err != nil || state != string(domain.Live) {
|
|
t.Fatalf("match state=%q err=%v", state, err)
|
|
}
|
|
var liveTickets int
|
|
if err := db.QueryRowContext(ctx, `SELECT count(*) FROM queue_tickets WHERE state = 'LIVE'`).Scan(&liveTickets); err != nil || liveTickets != 2 {
|
|
t.Fatalf("live tickets=%d err=%v", liveTickets, err)
|
|
}
|
|
// A lost 204 can be retried after assignment expiry because the exact
|
|
// durable receipt is replayed before checking the now-expired assignment.
|
|
if generation, err := ClaimPlayerConnection(ctx, db, binding, "connect-player-0", 0, "connect-receipt-key-0000", now.Add(2*time.Minute)); err != nil || generation != 1 {
|
|
t.Fatalf("durable receipt replay: %v", err)
|
|
}
|
|
if _, err := ClaimPlayerConnection(ctx, db, binding, "connect-player-0", 1, "connect-active-duplicate", now.Add(2*time.Second)); !errors.Is(err, domain.ErrConflict) {
|
|
t.Fatalf("active duplicate err=%v, want conflict", err)
|
|
}
|
|
if err := RecordPlayerDisconnected(ctx, db, binding, "connect-player-0", 1, "disconnect-receipt-0000", now.Add(3*time.Second)); err != nil {
|
|
t.Fatalf("disconnect receipt: %v", err)
|
|
}
|
|
if _, err := ClaimPlayerConnection(ctx, db, binding, "connect-player-0", 0, "connect-receipt-key-0000", now.Add(4*time.Second)); !errors.Is(err, domain.ErrConflict) {
|
|
t.Fatalf("stale connect replay err=%v, want conflict", err)
|
|
}
|
|
if generation, err := ClaimPlayerConnection(ctx, db, binding, "connect-player-0", 0, "reconnect-receipt-0000", now.Add(63*time.Second)); err != nil || generation != 2 {
|
|
t.Fatalf("grace-boundary reconnect generation=%d err=%v", generation, err)
|
|
}
|
|
if err := RecordPlayerDisconnected(ctx, db, binding, "connect-player-0", 1, "stale-disconnect-0000", now.Add(64*time.Second)); !errors.Is(err, domain.ErrConflict) {
|
|
t.Fatalf("stale disconnect err=%v, want conflict", err)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLLiveReconnectGraceExpiryPersistsAbandonmentWithoutReleasingResultRoster(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
ctx := context.Background()
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
for _, playerID := range []string{"live-abandon-player", "live-present-player"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, playerID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'ranked', 'LIVE', 'integration-build', 1, $3, $4)`, "live-abandon-ticket-"+playerID, playerID, now, now.Add(time.Hour)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, arena_path) VALUES ('live-abandon-match', 'ranked', 'LIVE', 'EU', 1, 'live-abandon-server', 'res://scenes/arena_01.tscn')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team, connection_generation, connected_at, disconnected_at) VALUES
|
|
('live-abandon-match', 'live-abandon-player', 'live-abandon-ticket-live-abandon-player', 0, 0, 1, $1, $2),
|
|
('live-abandon-match', 'live-present-player', 'live-abandon-ticket-live-present-player', 3, 1, 1, $1, NULL)`, now.Add(-2*time.Minute), now.Add(-domain.RankedReconnectGrace-time.Nanosecond)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
reconciled, err := ReconcileLiveAbandonments(ctx, db, now, 10)
|
|
if err != nil || reconciled != 1 {
|
|
t.Fatalf("reconciled=%d err=%v", reconciled, err)
|
|
}
|
|
var active bool
|
|
var abandonedAt sql.NullTime
|
|
if err := db.QueryRowContext(ctx, `SELECT participation_active, abandoned_at FROM match_participants WHERE match_id = 'live-abandon-match' AND player_id = 'live-abandon-player'`).Scan(&active, &abandonedAt); err != nil || !active || !abandonedAt.Valid || !abandonedAt.Time.Equal(now) {
|
|
t.Fatalf("participant active=%t abandoned=%v err=%v", active, abandonedAt, err)
|
|
}
|
|
var ticketState string
|
|
if err := db.QueryRowContext(ctx, `SELECT state FROM queue_tickets WHERE ticket_id = 'live-abandon-ticket-live-abandon-player'`).Scan(&ticketState); err != nil || ticketState != "LIVE" {
|
|
t.Fatalf("ticket state=%q err=%v", ticketState, err)
|
|
}
|
|
var endsAt time.Time
|
|
if err := db.QueryRowContext(ctx, `SELECT ends_at FROM penalties WHERE player_id = 'live-abandon-player' AND kind = 'MATCH_ABANDONED'`).Scan(&endsAt); err != nil || !endsAt.Equal(now.Add(5*time.Minute)) {
|
|
t.Fatalf("penalty ends=%v err=%v", endsAt, err)
|
|
}
|
|
var outboxCount, revision int
|
|
if err := db.QueryRowContext(ctx, `SELECT count(*) FROM outbox WHERE aggregate_id = 'live-abandon-match' AND event_type = 'state_changed'`).Scan(&outboxCount); err != nil || outboxCount != 1 {
|
|
t.Fatalf("outbox=%d err=%v", outboxCount, err)
|
|
}
|
|
if err := db.QueryRowContext(ctx, `SELECT revision FROM matches WHERE match_id = 'live-abandon-match'`).Scan(&revision); err != nil || revision != 1 {
|
|
t.Fatalf("revision=%d err=%v", revision, err)
|
|
}
|
|
if reconciled, err = ReconcileLiveAbandonments(ctx, db, now.Add(time.Minute), 10); err != nil || reconciled != 0 {
|
|
t.Fatalf("replay reconciled=%d err=%v", reconciled, err)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLProposalClaimAndResponseAreAtomic(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"proposal-player-a", "proposal-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for i, player := range []string{"proposal-player-a", "proposal-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'QUEUED', 'integration-build', 1, $3, $4)`, fmt.Sprintf("proposal-ticket-%d", i), player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
proposal, err := domain.NewProposal("proposal-integration", domain.Casual, []string{"proposal-player-a", "proposal-player-b"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
proposal.Region = "EU"
|
|
proposal.Protocol = 1
|
|
for index := range proposal.Participants {
|
|
if proposal.Participants[index].PlayerID == "proposal-player-a" {
|
|
proposal.Participants[index].Team = 0
|
|
proposal.Participants[index].Slot = 0
|
|
} else {
|
|
proposal.Participants[index].Team = 1
|
|
proposal.Participants[index].Slot = 3
|
|
}
|
|
}
|
|
if err := CreateProposal(ctx, db, proposal, map[string]string{"proposal-player-a": "proposal-ticket-0", "proposal-player-b": "proposal-ticket-1"}, now); err != nil {
|
|
t.Fatalf("create proposal: %v", err)
|
|
}
|
|
var proposed int
|
|
if err := db.QueryRow(`SELECT count(*) FROM queue_tickets WHERE state = 'PROPOSED'`).Scan(&proposed); err != nil || proposed != 2 {
|
|
t.Fatalf("proposed queue tickets = %d, err = %v", proposed, err)
|
|
}
|
|
recoveredTicket, err := GetQueueTicket(ctx, db, "proposal-player-a", "proposal-ticket-0", now)
|
|
if err != nil || recoveredTicket.ProposalID != proposal.ProposalID {
|
|
t.Fatalf("recovered ticket proposal=%q err=%v", recoveredTicket.ProposalID, err)
|
|
}
|
|
recovered, err := GetProposal(ctx, db, "proposal-player-a", proposal.ProposalID, now)
|
|
if err != nil {
|
|
t.Fatalf("recover proposal: %v", err)
|
|
}
|
|
if len(recovered.Participants) != 2 || recovered.Revision != 0 {
|
|
t.Fatalf("unexpected recovered proposal: %+v", recovered)
|
|
}
|
|
accepted, err := RespondToProposal(ctx, db, "proposal-player-a", proposal.ProposalID, "proposal-response-a-0001", true, 0, now)
|
|
if err != nil {
|
|
t.Fatalf("first proposal acceptance: %v", err)
|
|
}
|
|
if accepted.Revision != 1 || accepted.State != domain.Open {
|
|
t.Fatalf("unexpected first acceptance: %+v", accepted)
|
|
}
|
|
accepted, err = RespondToProposal(ctx, db, "proposal-player-b", proposal.ProposalID, "proposal-response-b-0001", true, 1, now)
|
|
if err != nil {
|
|
t.Fatalf("second proposal acceptance: %v", err)
|
|
}
|
|
if accepted.State != domain.Accepted || accepted.Revision != 2 {
|
|
t.Fatalf("proposal did not close after unanimous acceptance: %+v", accepted)
|
|
}
|
|
// The API's post-commit recovery promoter must accept the nullable casual
|
|
// arena path left by the atomic response transaction and converge on the
|
|
// already-created match.
|
|
if err := PromoteStoredAcceptedProposal(ctx, db, proposal.ProposalID, now.Add(time.Second)); err != nil {
|
|
t.Fatalf("replay persisted casual promotion: %v", err)
|
|
}
|
|
var matchState string
|
|
if err := db.QueryRow(`SELECT state FROM matches WHERE match_id = 'match-proposal-integration'`).Scan(&matchState); err != nil {
|
|
t.Fatalf("atomic accepted match: %v", err)
|
|
}
|
|
var acceptedTickets, matchPlayers int
|
|
if err := db.QueryRow(`SELECT count(*) FROM queue_tickets WHERE ticket_id LIKE 'proposal-ticket-%' AND state = 'ACCEPTED'`).Scan(&acceptedTickets); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM match_participants WHERE match_id = 'match-proposal-integration'`).Scan(&matchPlayers); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if matchState != "ALLOCATING" || acceptedTickets != 2 || matchPlayers != 2 {
|
|
t.Fatalf("acceptance did not atomically materialize match: state=%s tickets=%d players=%d", matchState, acceptedTickets, matchPlayers)
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLProposalDeclineCancelsOffenderAndRequeuesInnocent protects the
|
|
// durable decline boundary: the offender's ticket becomes terminal while
|
|
// every innocent ticket keeps its original queue precedence.
|
|
func TestPostgreSQLProposalDeclineCancelsOffenderAndRequeuesInnocent(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"decline-player-a", "decline-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for i, player := range []string{"decline-player-a", "decline-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'QUEUED', 'integration-build', 1, $3, $4)`, fmt.Sprintf("decline-ticket-%d", i), player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
proposal, err := domain.NewProposal("decline-proposal", domain.Casual, []string{"decline-player-a", "decline-player-b"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := CreateProposal(ctx, db, proposal, map[string]string{"decline-player-a": "decline-ticket-0", "decline-player-b": "decline-ticket-1"}, now); err != nil {
|
|
t.Fatalf("create proposal: %v", err)
|
|
}
|
|
|
|
// player-a declines; player-b never responded at all -- the bug affects
|
|
// even a participant who was never asked to do anything wrong.
|
|
declined, err := RespondToProposal(ctx, db, "decline-player-a", proposal.ProposalID, "decline-response-a-0001", false, 0, now)
|
|
if err != nil {
|
|
t.Fatalf("decline: %v", err)
|
|
}
|
|
if declined.State != domain.Declined {
|
|
t.Fatalf("proposal did not close on decline: %+v", declined)
|
|
}
|
|
|
|
var stateA, stateB string
|
|
var expiresB time.Time
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'decline-ticket-0'`).Scan(&stateA); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT state, expires_at FROM queue_tickets WHERE ticket_id = 'decline-ticket-1'`).Scan(&stateB, &expiresB); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if stateA != "CANCELLED" {
|
|
t.Fatalf("decliner's own ticket state = %s, want CANCELLED", stateA)
|
|
}
|
|
if stateB != "QUEUED" {
|
|
t.Fatalf("uninvolved participant's ticket state = %s, want QUEUED -- they must not be stranded by someone else's decline", stateB)
|
|
}
|
|
if !expiresB.After(now) {
|
|
t.Fatalf("requeued ticket expiry %v was not refreshed forward from %v", expiresB, now)
|
|
}
|
|
|
|
// Only the innocent player can be selected again. A durable cooldown also
|
|
// rejects a new ticket from the decliner until the policy window ends.
|
|
candidates, err := ListQueuedCandidates(ctx, db, domain.Casual, now, 10)
|
|
if err != nil {
|
|
t.Fatalf("list queued candidates: %v", err)
|
|
}
|
|
found := map[string]bool{}
|
|
for _, candidate := range candidates {
|
|
found[candidate.PlayerID] = true
|
|
}
|
|
if found["decline-player-a"] || !found["decline-player-b"] {
|
|
t.Fatalf("matcher did not isolate offender from innocent: %+v", candidates)
|
|
}
|
|
var cooldownEnd time.Time
|
|
if err := db.QueryRow(`SELECT ends_at FROM penalties WHERE player_id = 'decline-player-a' AND kind = 'PROPOSAL_DECLINED'`).Scan(&cooldownEnd); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if want := now.Add(30 * time.Second); !cooldownEnd.Equal(want) {
|
|
t.Fatalf("decline cooldown end = %v, want %v", cooldownEnd, want)
|
|
}
|
|
// Recovering the closed proposal after its old deadline must not convert
|
|
// the innocent participant's PENDING response into a timeout penalty.
|
|
if _, err := GetProposal(ctx, db, "decline-player-b", proposal.ProposalID, now.Add(domain.ProposalWindow+time.Second)); err != nil {
|
|
t.Fatalf("recover declined proposal: %v", err)
|
|
}
|
|
var innocentTimeouts int
|
|
if err := db.QueryRow(`SELECT count(*) FROM penalties WHERE player_id = 'decline-player-b' AND kind = 'PROPOSAL_TIMEOUT'`).Scan(&innocentTimeouts); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if innocentTimeouts != 0 {
|
|
t.Fatalf("innocent participant received %d timeout penalties after decline", innocentTimeouts)
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLProposalTimeoutExpiresOffenderAndRequeuesAccepted protects the
|
|
// timeout sibling: accepted participants retain precedence, while no-shows
|
|
// receive a terminal ticket and cooldown.
|
|
func TestPostgreSQLProposalTimeoutExpiresOffenderAndRequeuesAccepted(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"timeout-player-a", "timeout-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for i, player := range []string{"timeout-player-a", "timeout-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'QUEUED', 'integration-build', 1, $3, $4)`, fmt.Sprintf("timeout-ticket-%d", i), player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
proposal, err := domain.NewProposal("timeout-proposal", domain.Casual, []string{"timeout-player-a", "timeout-player-b"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := CreateProposal(ctx, db, proposal, map[string]string{"timeout-player-a": "timeout-ticket-0", "timeout-player-b": "timeout-ticket-1"}, now); err != nil {
|
|
t.Fatalf("create proposal: %v", err)
|
|
}
|
|
if _, err := RespondToProposal(ctx, db, "timeout-player-a", proposal.ProposalID, "timeout-accept-a-0001", true, 0, now.Add(time.Second)); err != nil {
|
|
t.Fatalf("accept proposal: %v", err)
|
|
}
|
|
|
|
// player-b never responds; recover well after the response window.
|
|
afterExpiry := now.Add(domain.ProposalWindow + time.Second)
|
|
recovered, err := GetProposal(ctx, db, "timeout-player-a", proposal.ProposalID, afterExpiry)
|
|
if err != nil {
|
|
t.Fatalf("recover expired proposal: %v", err)
|
|
}
|
|
if recovered.State != domain.Expired {
|
|
t.Fatalf("proposal did not expire: %+v", recovered)
|
|
}
|
|
|
|
var stateA, stateB string
|
|
var expiresA time.Time
|
|
if err := db.QueryRow(`SELECT state, expires_at FROM queue_tickets WHERE ticket_id = 'timeout-ticket-0'`).Scan(&stateA, &expiresA); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'timeout-ticket-1'`).Scan(&stateB); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if stateA != "QUEUED" || stateB != "EXPIRED" {
|
|
t.Fatalf("timeout did not split accepted and offender tickets: a=%s b=%s", stateA, stateB)
|
|
}
|
|
if !expiresA.After(afterExpiry) {
|
|
t.Fatalf("requeued ticket expiry %v was not refreshed forward from %v", expiresA, afterExpiry)
|
|
}
|
|
candidates, err := ListQueuedCandidates(ctx, db, domain.Casual, afterExpiry, 10)
|
|
if err != nil {
|
|
t.Fatalf("list queued candidates: %v", err)
|
|
}
|
|
found := map[string]bool{}
|
|
for _, candidate := range candidates {
|
|
found[candidate.PlayerID] = true
|
|
}
|
|
if !found["timeout-player-a"] || found["timeout-player-b"] {
|
|
t.Fatalf("matcher did not isolate timeout offender: %+v", candidates)
|
|
}
|
|
var cooldownEnd time.Time
|
|
if err := db.QueryRow(`SELECT ends_at FROM penalties WHERE player_id = 'timeout-player-b' AND kind = 'PROPOSAL_TIMEOUT'`).Scan(&cooldownEnd); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if want := afterExpiry.Add(60 * time.Second); !cooldownEnd.Equal(want) {
|
|
t.Fatalf("timeout cooldown end = %v, want %v", cooldownEnd, want)
|
|
}
|
|
}
|
|
|
|
// A late response must report a closed proposal only after committing the
|
|
// expiry recovery. Returning that domain error from inside RunSerializable
|
|
// used to roll every recovery write back.
|
|
func TestPostgreSQLLateProposalResponseCommitsExpiryRecovery(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"late-player-a", "late-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for i, player := range []string{"late-player-a", "late-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'QUEUED', 'integration-build', 1, $3, $4)`, fmt.Sprintf("late-ticket-%d", i), player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
proposal, err := domain.NewProposal("late-proposal", domain.Casual, []string{"late-player-a", "late-player-b"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := CreateProposal(ctx, db, proposal, map[string]string{"late-player-a": "late-ticket-0", "late-player-b": "late-ticket-1"}, now); err != nil {
|
|
t.Fatalf("create proposal: %v", err)
|
|
}
|
|
|
|
late := now.Add(domain.ProposalWindow + time.Second)
|
|
_, err = RespondToProposal(ctx, db, "late-player-a", proposal.ProposalID, "late-response-a-0001", true, 0, late)
|
|
if !errors.Is(err, domain.ErrProposalClosed) {
|
|
t.Fatalf("late response error = %v, want ErrProposalClosed", err)
|
|
}
|
|
var proposalState, ticketA, ticketB string
|
|
if err := db.QueryRow(`SELECT state FROM proposals WHERE proposal_id = 'late-proposal'`).Scan(&proposalState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'late-ticket-0'`).Scan(&ticketA); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'late-ticket-1'`).Scan(&ticketB); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if proposalState != "EXPIRED" || ticketA != "EXPIRED" || ticketB != "EXPIRED" {
|
|
t.Fatalf("late recovery was not committed: proposal=%s tickets=%s,%s", proposalState, ticketA, ticketB)
|
|
}
|
|
var penalties, idempotencyRows int
|
|
if err := db.QueryRow(`SELECT count(*) FROM penalties WHERE kind = 'PROPOSAL_TIMEOUT' AND player_id IN ('late-player-a', 'late-player-b')`).Scan(&penalties); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM idempotency_keys WHERE scope = $1 AND idempotency_key = 'late-response-a-0001'`, ProposalResponseIdempotencyScope).Scan(&idempotencyRows); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if penalties != 2 || idempotencyRows != 0 {
|
|
t.Fatalf("late recovery side effects: penalties=%d idempotency_rows=%d", penalties, idempotencyRows)
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLConcurrentProposalExpiryRecoveryAppliesCooldownsExactlyOnce
|
|
// covers the race §8.46 flagged as still open: multiple concurrent recovery
|
|
// paths (a read-side GetProposal from each participant polling for an
|
|
// update, and a RespondToProposal arriving right at the same boundary) can
|
|
// all observe the same past-expiry proposal simultaneously. Every one of
|
|
// them runs the identical expiry-advance SQL in its own transaction, so this
|
|
// proves that racing recovery does not multiply the durable side effects: a
|
|
// PROPOSAL_TIMEOUT cooldown must land exactly once per offending player, not
|
|
// once per racing transaction that happened to perform the PENDING ->
|
|
// TIMED_OUT flip. The design's own defense is that ProposalParticipantExpireSQL
|
|
// only ever flips a still-PENDING row once, and recordProposalTimeoutCooldowns
|
|
// only cooldowns participants whose responded_at equals this transaction's
|
|
// own `now` -- so a loser transaction's `now` simply matches nothing.
|
|
func TestPostgreSQLConcurrentProposalExpiryRecoveryAppliesCooldownsExactlyOnce(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"race-expiry-a", "race-expiry-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for i, player := range []string{"race-expiry-a", "race-expiry-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'QUEUED', 'integration-build', 1, $3, $4)`, fmt.Sprintf("race-expiry-ticket-%d", i), player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
proposal, err := domain.NewProposal("race-expiry-proposal", domain.Casual, []string{"race-expiry-a", "race-expiry-b"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := CreateProposal(ctx, db, proposal, map[string]string{"race-expiry-a": "race-expiry-ticket-0", "race-expiry-b": "race-expiry-ticket-1"}, now); err != nil {
|
|
t.Fatalf("create proposal: %v", err)
|
|
}
|
|
|
|
late := now.Add(domain.ProposalWindow + time.Second)
|
|
const racers = 8
|
|
var wg sync.WaitGroup
|
|
errs := make([]error, racers)
|
|
wg.Add(racers)
|
|
for i := 0; i < racers; i++ {
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
// Each racer's `now` is distinct (and every one is past expiry), so a
|
|
// real implementation bug would show up as several of them believing
|
|
// they were the one that performed the PENDING -> TIMED_OUT flip.
|
|
racerNow := late.Add(time.Duration(i) * time.Millisecond)
|
|
switch i % 3 {
|
|
case 0:
|
|
_, errs[i] = GetProposal(ctx, db, "race-expiry-a", proposal.ProposalID, racerNow)
|
|
case 1:
|
|
_, errs[i] = GetProposal(ctx, db, "race-expiry-b", proposal.ProposalID, racerNow)
|
|
default:
|
|
_, errs[i] = RespondToProposal(ctx, db, "race-expiry-a", proposal.ProposalID, fmt.Sprintf("race-expiry-key-%04d", i), true, 0, racerNow)
|
|
}
|
|
}(i)
|
|
}
|
|
wg.Wait()
|
|
for i, err := range errs {
|
|
// GetProposal never errors on an already-expired proposal (it's a pure
|
|
// read-with-recovery); RespondToProposal on an already-closed proposal
|
|
// must report exactly ErrProposalClosed, nothing else.
|
|
if err != nil && !errors.Is(err, domain.ErrProposalClosed) {
|
|
t.Fatalf("racer %d: unexpected error %v", i, err)
|
|
}
|
|
}
|
|
|
|
var proposalState string
|
|
if err := db.QueryRow(`SELECT state FROM proposals WHERE proposal_id = 'race-expiry-proposal'`).Scan(&proposalState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if proposalState != "EXPIRED" {
|
|
t.Fatalf("proposal state = %s, want EXPIRED", proposalState)
|
|
}
|
|
var ticketA, ticketB string
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'race-expiry-ticket-0'`).Scan(&ticketA); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'race-expiry-ticket-1'`).Scan(&ticketB); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if ticketA != "EXPIRED" || ticketB != "EXPIRED" {
|
|
t.Fatalf("tickets not expired exactly once: a=%s b=%s", ticketA, ticketB)
|
|
}
|
|
// The crux of the race: exactly one PROPOSAL_TIMEOUT penalty per player,
|
|
// however many transactions raced to observe the expiry.
|
|
var penaltiesA, penaltiesB int
|
|
if err := db.QueryRow(`SELECT count(*) FROM penalties WHERE kind = 'PROPOSAL_TIMEOUT' AND player_id = 'race-expiry-a'`).Scan(&penaltiesA); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM penalties WHERE kind = 'PROPOSAL_TIMEOUT' AND player_id = 'race-expiry-b'`).Scan(&penaltiesB); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if penaltiesA != 1 || penaltiesB != 1 {
|
|
t.Fatalf("cooldown was not applied exactly once per player: a=%d b=%d", penaltiesA, penaltiesB)
|
|
}
|
|
var idempotencyRows int
|
|
if err := db.QueryRow(`SELECT count(*) FROM idempotency_keys WHERE scope = $1`, ProposalResponseIdempotencyScope).Scan(&idempotencyRows); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if idempotencyRows != 0 {
|
|
t.Fatalf("closed-proposal responses left stray idempotency rows: %d", idempotencyRows)
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLCancellingAProposedTicketImmediatelyRequeuesTheOtherParticipant
|
|
// covers the responsiveness gap the decline/timeout fixes above left bounded
|
|
// but not closed: cancelling a ticket that's part of an OPEN proposal used
|
|
// to leave the OTHER participant waiting out the full 10s window for
|
|
// something the system already knew couldn't happen (their proposal partner
|
|
// just walked away). CascadeCancelToOpenProposal declines and requeues that
|
|
// proposal in the same transaction as the cancel itself.
|
|
func TestPostgreSQLCancellingAProposedTicketImmediatelyRequeuesTheOtherParticipant(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"cancel-cascade-a", "cancel-cascade-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for i, player := range []string{"cancel-cascade-a", "cancel-cascade-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'QUEUED', 'integration-build', 1, $3, $4)`, fmt.Sprintf("cancel-cascade-ticket-%d", i), player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
proposal, err := domain.NewProposal("cancel-cascade-proposal", domain.Casual, []string{"cancel-cascade-a", "cancel-cascade-b"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := CreateProposal(ctx, db, proposal, map[string]string{"cancel-cascade-a": "cancel-cascade-ticket-0", "cancel-cascade-b": "cancel-cascade-ticket-1"}, now); err != nil {
|
|
t.Fatalf("create proposal: %v", err)
|
|
}
|
|
|
|
// player-a cancels their own ticket directly, well within the response
|
|
// window -- not a decline, not a timeout, just abandoning the queue.
|
|
// CreateProposal's own QueueTicketProposeSQL already bumped the ticket's
|
|
// revision from 0 to 1, so the cancel's expected revision is 1, not 0.
|
|
cancelled, err := CancelQueueTicket(ctx, db, "cancel-cascade-a", "cancel-cascade-ticket-0", "cancel-cascade-key-0001", 1, now.Add(time.Second))
|
|
if err != nil {
|
|
t.Fatalf("cancel: %v", err)
|
|
}
|
|
if cancelled.State != domain.Cancelled {
|
|
t.Fatalf("ticket did not cancel: %+v", cancelled)
|
|
}
|
|
|
|
var proposalState string
|
|
if err := db.QueryRow(`SELECT state FROM proposals WHERE proposal_id = 'cancel-cascade-proposal'`).Scan(&proposalState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if proposalState != "DECLINED" {
|
|
t.Fatalf("proposal state = %s, want DECLINED immediately, not left OPEN to time out", proposalState)
|
|
}
|
|
var stateB string
|
|
var expiresB time.Time
|
|
if err := db.QueryRow(`SELECT state, expires_at FROM queue_tickets WHERE ticket_id = 'cancel-cascade-ticket-1'`).Scan(&stateB, &expiresB); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if stateB != "QUEUED" {
|
|
t.Fatalf("other participant's ticket state = %s, want QUEUED immediately", stateB)
|
|
}
|
|
if !expiresB.After(now.Add(time.Second)) {
|
|
t.Fatalf("requeued ticket expiry %v was not refreshed forward", expiresB)
|
|
}
|
|
// The cancelling player's own ticket must stay CANCELLED, not get swept
|
|
// back up into the requeue meant for the other participant.
|
|
var stateA string
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'cancel-cascade-ticket-0'`).Scan(&stateA); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if stateA != "CANCELLED" {
|
|
t.Fatalf("cancelling player's own ticket state = %s, want it to stay CANCELLED", stateA)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLProposalCreationRollsBackPartialClaims(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('rollback-player', 'rollback-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ('rollback-ticket', 'rollback-player', 'casual', 'QUEUED', 'integration-build', 1, $1, $2)`, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
proposal, err := domain.NewProposal("rollback-proposal", domain.Casual, []string{"rollback-player", "missing-player"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := CreateProposal(ctx, db, proposal, map[string]string{"rollback-player": "rollback-ticket"}, now); err == nil {
|
|
t.Fatal("proposal with missing ticket mapping was accepted")
|
|
}
|
|
var proposals, participants, proposed int
|
|
if err := db.QueryRow(`SELECT count(*) FROM proposals WHERE proposal_id = 'rollback-proposal'`).Scan(&proposals); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM proposal_participants WHERE proposal_id = 'rollback-proposal'`).Scan(&participants); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM queue_tickets WHERE ticket_id = 'rollback-ticket' AND state = 'PROPOSED'`).Scan(&proposed); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if proposals != 0 || participants != 0 || proposed != 0 {
|
|
t.Fatalf("partial proposal claim was not rolled back: proposals=%d participants=%d proposed=%d", proposals, participants, proposed)
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLConcurrentProposalCreationClaimsContestedTicketOnce is the
|
|
// live counterpart to TestPostgreSQLProposalCreationRollsBackPartialClaims:
|
|
// every other proposal test in this file (and the whole matcher/allocator
|
|
// suite) runs its transactions strictly one at a time, so none of them can
|
|
// actually exercise the SERIALIZABLE retry-and-fence path CreateProposal
|
|
// relies on -- only two goroutines racing a real connection pool can. Two
|
|
// matchers independently form a proposal that both include the same waiting
|
|
// player's ticket (a real scenario: nothing stops two matcher replicas from
|
|
// reading the same QUEUED ticket in the same poll window); exactly one
|
|
// CreateProposal must win, the other must fail with its whole transaction
|
|
// rolled back, not a database/sql panic, deadlock, or a half-inserted row.
|
|
func TestPostgreSQLConcurrentProposalCreationClaimsContestedTicketOnce(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"race-player-a", "race-player-b", "race-player-c"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
tickets := map[string]string{"race-player-a": "race-ticket-a", "race-player-b": "race-ticket-b", "race-player-c": "race-ticket-c"}
|
|
for player, ticket := range tickets {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'QUEUED', 'integration-build', 1, $3, $4)`, ticket, player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
proposalA, err := domain.NewProposal("race-proposal-a", domain.Casual, []string{"race-player-a", "race-player-b"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
proposalB, err := domain.NewProposal("race-proposal-b", domain.Casual, []string{"race-player-b", "race-player-c"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
errs := make([]error, 2)
|
|
wg.Add(2)
|
|
go func() {
|
|
defer wg.Done()
|
|
errs[0] = CreateProposal(ctx, db, proposalA, map[string]string{"race-player-a": tickets["race-player-a"], "race-player-b": tickets["race-player-b"]}, now)
|
|
}()
|
|
go func() {
|
|
defer wg.Done()
|
|
errs[1] = CreateProposal(ctx, db, proposalB, map[string]string{"race-player-b": tickets["race-player-b"], "race-player-c": tickets["race-player-c"]}, now)
|
|
}()
|
|
wg.Wait()
|
|
|
|
succeeded := errs[0] == nil
|
|
if succeeded == (errs[1] == nil) {
|
|
t.Fatalf("exactly one contested proposal must win, got errA=%v errB=%v", errs[0], errs[1])
|
|
}
|
|
|
|
winner, loser := "race-proposal-a", "race-proposal-b"
|
|
if !succeeded {
|
|
winner, loser = "race-proposal-b", "race-proposal-a"
|
|
}
|
|
var winnerRows, loserRows, loserParticipants int
|
|
if err := db.QueryRow(`SELECT count(*) FROM proposals WHERE proposal_id = $1`, winner).Scan(&winnerRows); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM proposals WHERE proposal_id = $1`, loser).Scan(&loserRows); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM proposal_participants WHERE proposal_id = $1`, loser).Scan(&loserParticipants); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if winnerRows != 1 {
|
|
t.Fatalf("winning proposal %s was not persisted", winner)
|
|
}
|
|
if loserRows != 0 || loserParticipants != 0 {
|
|
t.Fatalf("losing proposal %s was not fully rolled back: proposals=%d participants=%d", loser, loserRows, loserParticipants)
|
|
}
|
|
var contestedState string
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = $1`, tickets["race-player-b"]).Scan(&contestedState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if contestedState != "PROPOSED" {
|
|
t.Fatalf("contested ticket should be claimed by the winner, got state=%s", contestedState)
|
|
}
|
|
// The loser's OWN uncontested ticket (a or c) must have rolled back to
|
|
// QUEUED too -- CreateProposal is one transaction per proposal, so a
|
|
// contested loss on one participant must not leave another participant's
|
|
// ticket stranded as PROPOSED with no surviving proposal to reference it.
|
|
// A (player-a + contested player-b) won iff succeeded, in which case B's
|
|
// own uncontested ticket (player-c) is the one that must have rolled
|
|
// back; if A lost, it's A's own uncontested ticket (player-a) instead.
|
|
loserOnlyTicket := tickets["race-player-c"]
|
|
if !succeeded {
|
|
loserOnlyTicket = tickets["race-player-a"]
|
|
}
|
|
var loserOnlyState string
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = $1`, loserOnlyTicket).Scan(&loserOnlyState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if loserOnlyState != "QUEUED" {
|
|
t.Fatalf("loser's uncontested ticket %s should have rolled back to QUEUED, got %s", loserOnlyTicket, loserOnlyState)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLResultCompletionAndOutboxAreAtomicAndReplayable(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id) VALUES ('result-match', 'casual', 'RESULT_PENDING', 'NA', 1, 'result-server')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
payload := []byte(`{"match_id":"result-match","team0_score":2,"team1_score":1}`)
|
|
digest := sha256.Sum256(payload)
|
|
receipt := domain.ResultReceipt{ResultID: "result-receipt", MatchID: "result-match", ResultNonce: "result-nonce-123456", PayloadDigest: digest, IntegrityState: domain.IntegrityCertified, ReceivedAt: now}
|
|
if err := CompleteResult(ctx, db, receipt, "result-server", "result-event", payload, now); err != nil {
|
|
t.Fatalf("complete result: %v", err)
|
|
}
|
|
var state string
|
|
if err := db.QueryRow(`SELECT state FROM matches WHERE match_id = 'result-match'`).Scan(&state); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if state != "COMPLETED" {
|
|
t.Fatalf("result match state = %s", state)
|
|
}
|
|
events, err := ReadUnpublishedOutbox(ctx, db, 10)
|
|
if err != nil || len(events) != 1 || events[0].EventID != "result-event" {
|
|
t.Fatalf("unpublished result events = %+v, err = %v", events, err)
|
|
}
|
|
if err := MarkOutboxPublished(ctx, db, events[0].EventID, now.Add(time.Second)); err != nil {
|
|
t.Fatalf("ack result event: %v", err)
|
|
}
|
|
if remaining, err := ReadUnpublishedOutbox(ctx, db, 10); err != nil || len(remaining) != 0 {
|
|
t.Fatalf("outbox after ack = %+v, err = %v", remaining, err)
|
|
}
|
|
if err := CompleteResult(ctx, db, receipt, "result-server", "result-event-retry", payload, now.Add(time.Second)); err != nil {
|
|
t.Fatalf("identical completed result replay: %v", err)
|
|
}
|
|
conflict := receipt
|
|
conflict.ResultID = "different-result"
|
|
if err := CompleteResult(ctx, db, conflict, "result-server", "different-event", []byte(`{"conflict":true}`), now.Add(2*time.Second)); err == nil {
|
|
t.Fatal("conflicting completed result was accepted")
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLConcurrentConflictingResultSubmissionsKeepOneReceipt
|
|
// exercises the other side of the result race: retries with different payloads
|
|
// must not let the winner's durable receipt be overwritten or create a second
|
|
// completion event.
|
|
func TestPostgreSQLConcurrentConflictingResultSubmissionsKeepOneReceipt(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"result-conflict-a", "result-conflict-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id) VALUES ('result-conflict-match', 'casual', 'RESULT_PENDING', 'NA', 1, 'result-conflict-server')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ('result-conflict-ticket-a', 'result-conflict-a', 'casual', 'LIVE', 'build-1', 1, $1, $2), ('result-conflict-ticket-b', 'result-conflict-b', 'casual', 'LIVE', 'build-1', 1, $1, $2)`, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) VALUES ('result-conflict-match', 'result-conflict-a', 'result-conflict-ticket-a', 0, 0), ('result-conflict-match', 'result-conflict-b', 'result-conflict-ticket-b', 1, 1)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
base := domain.MatchResult{MatchID: "result-conflict-match", ServerID: "result-conflict-server", IntegrityState: domain.IntegritySuppressed}
|
|
results := []domain.MatchResult{
|
|
{MatchID: base.MatchID, ServerID: base.ServerID, ResultNonce: "result-conflict-nonce-a-123456", Team0Score: 2, Team1Score: 1, IntegrityState: base.IntegrityState},
|
|
{MatchID: base.MatchID, ServerID: base.ServerID, ResultNonce: "result-conflict-nonce-b-123456", Team0Score: 1, Team1Score: 2, IntegrityState: base.IntegrityState},
|
|
}
|
|
errs := make([]error, 2)
|
|
var wg sync.WaitGroup
|
|
wg.Add(2)
|
|
for i := range results {
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
digest := domain.ResultDigest(results[i])
|
|
receipt := domain.ResultReceipt{ResultID: fmt.Sprintf("result-conflict-receipt-%d", i), MatchID: results[i].MatchID, ResultNonce: results[i].ResultNonce, PayloadDigest: digest, IntegrityState: results[i].IntegrityState, ReceivedAt: now}
|
|
errs[i] = CompleteResult(ctx, db, receipt, results[i].ServerID, fmt.Sprintf("result-conflict-event-%d", i), []byte(fmt.Sprintf(`{"nonce":%q}`, results[i].ResultNonce)), now)
|
|
}(i)
|
|
}
|
|
wg.Wait()
|
|
wins := 0
|
|
for _, err := range errs {
|
|
if err == nil {
|
|
wins++
|
|
} else if !strings.Contains(err.Error(), "conflict") {
|
|
t.Fatalf("non-conflict error in conflicting race: %v", err)
|
|
}
|
|
}
|
|
if wins != 1 {
|
|
t.Fatalf("successful conflicting submissions = %d, want exactly one; errors=%v", wins, errs)
|
|
}
|
|
var receipts, events int
|
|
if err := db.QueryRow(`SELECT count(*) FROM result_receipts WHERE match_id = 'result-conflict-match'`).Scan(&receipts); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM outbox WHERE aggregate_id = 'result-conflict-match' AND event_type = 'match_completed'`).Scan(&events); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if receipts != 1 || events != 1 {
|
|
t.Fatalf("durable conflict race left receipts=%d events=%d, want one of each", receipts, events)
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLConcurrentIdenticalResultSubmissionAppliesRatingsExactlyOnce
|
|
// races real concurrent duplicate result submissions -- the scenario behind
|
|
// task 8.25's "identical duplicates idempotent" claim, which every other
|
|
// result test in this file (and the mocked-driver unit tests) only exercises
|
|
// sequentially. A game server can legitimately retry an unacknowledged
|
|
// result POST, and two such retries can land at PostgreSQL genuinely
|
|
// concurrently; every one of them must succeed (this is the identical-replay
|
|
// path, not a conflict), the match must complete exactly once, and -- the
|
|
// part that matters -- the rating update inside applyResultRatings must not
|
|
// run twice just because it raced.
|
|
func TestPostgreSQLConcurrentIdenticalResultSubmissionAppliesRatingsExactlyOnce(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"result-race-winner", "result-race-loser"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO ratings (player_id, rating, deviation, volatility, ranked_games) VALUES ($1, 1500, 350, 0.06, 0)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id) VALUES ('result-race-match', 'casual', 'LIVE', 'NA', 1, 'result-race-server')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ('result-race-ticket-w', 'result-race-winner', 'casual', 'LIVE', 'build-1', 1, $1, $2), ('result-race-ticket-l', 'result-race-loser', 'casual', 'LIVE', 'build-1', 1, $1, $2)`, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) VALUES ('result-race-match', 'result-race-winner', 'result-race-ticket-w', 0, 0), ('result-race-match', 'result-race-loser', 'result-race-ticket-l', 1, 1)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
result := domain.MatchResult{MatchID: "result-race-match", ServerID: "result-race-server", ResultNonce: "result-race-nonce-123456", Team0Score: 3, Team1Score: 1, IntegrityState: domain.IntegrityCertified}
|
|
digest := domain.ResultDigest(result)
|
|
receipt := domain.ResultReceipt{ResultID: "result-race-receipt", MatchID: result.MatchID, ResultNonce: result.ResultNonce, PayloadDigest: digest, IntegrityState: result.IntegrityState, ReceivedAt: now}
|
|
payload := []byte(`{"match_id":"result-race-match"}`)
|
|
|
|
const attempts = 5
|
|
var wg sync.WaitGroup
|
|
errs := make([]error, attempts)
|
|
wg.Add(attempts)
|
|
for i := 0; i < attempts; i++ {
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
errs[i] = CompleteResultWithResult(ctx, db, receipt, result.ServerID, fmt.Sprintf("result-race-event-%d", i), payload, result, now)
|
|
}(i)
|
|
}
|
|
wg.Wait()
|
|
for i, err := range errs {
|
|
if err != nil {
|
|
t.Fatalf("identical concurrent submission %d failed: %v", i, err)
|
|
}
|
|
}
|
|
|
|
var state string
|
|
if err := db.QueryRow(`SELECT state FROM matches WHERE match_id = 'result-race-match'`).Scan(&state); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if state != "COMPLETED" {
|
|
t.Fatalf("match state = %s, want COMPLETED", state)
|
|
}
|
|
var completedTickets int
|
|
if err := db.QueryRow(`SELECT count(*) FROM queue_tickets WHERE ticket_id IN ('result-race-ticket-w', 'result-race-ticket-l') AND state = 'COMPLETED'`).Scan(&completedTickets); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if completedTickets != 2 {
|
|
t.Fatalf("completed tickets = %d, want 2", completedTickets)
|
|
}
|
|
var winnerGames, loserGames int
|
|
var winnerRating, loserRating float64
|
|
if err := db.QueryRow(`SELECT ranked_games, rating FROM ratings WHERE player_id = 'result-race-winner'`).Scan(&winnerGames, &winnerRating); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT ranked_games, rating FROM ratings WHERE player_id = 'result-race-loser'`).Scan(&loserGames, &loserRating); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Casual results never increment ranked_games by design (rankedIncrement
|
|
// is unconditionally 0 for domain.Casual in applyResultRatings) -- that's
|
|
// not what this test is verifying. What proves "applied exactly once, not
|
|
// N times under the race" is the rating VALUE: a second application would
|
|
// recompute from the already-updated current rating and compound further
|
|
// away from 1500, so an exact match against a single, independently
|
|
// computed application is the assertion that actually falsifies a double
|
|
// application (unlike an inequality check, which a doubled update would
|
|
// still satisfy).
|
|
if winnerGames != 0 || loserGames != 0 {
|
|
t.Fatalf("casual result should never touch ranked_games: winner=%d loser=%d", winnerGames, loserGames)
|
|
}
|
|
baseline := domain.Rating{Value: 1500, RD: 350, Volatility: 0.06}
|
|
winnerOpponents, err := domain.CasualOpponents([]domain.Opponent{{PlayerID: "result-race-loser", Rating: baseline, Score: 1}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wantWinner, err := domain.UpdateRating(baseline, winnerOpponents, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
loserOpponents, err := domain.CasualOpponents([]domain.Opponent{{PlayerID: "result-race-winner", Rating: baseline, Score: 0}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wantLoser, err := domain.UpdateRating(baseline, loserOpponents, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if winnerRating != wantWinner.Value {
|
|
t.Fatalf("winner rating = %v, want exactly %v (a value between these would indicate a partial/compounded update)", winnerRating, wantWinner.Value)
|
|
}
|
|
if loserRating != wantLoser.Value {
|
|
t.Fatalf("loser rating = %v, want exactly %v", loserRating, wantLoser.Value)
|
|
}
|
|
if winnerRating <= loserRating {
|
|
t.Fatalf("winner rating %v should exceed loser rating %v after a certified result", winnerRating, loserRating)
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLStalledAllocationsAreReclaimedWithoutPenalisingPlayers is the
|
|
// live counterpart to the SQL fragment test: it proves the actual data
|
|
// movement against a real database, not just that the right substrings are
|
|
// present. Two matches: one genuinely stalled (old enough to reclaim), one
|
|
// recent (must survive untouched) -- the deadline boundary and the
|
|
// no-penalty requeue are both meaningless without a real row to check.
|
|
func TestPostgreSQLStalledAllocationsAreReclaimedWithoutPenalisingPlayers(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
ctx := context.Background()
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
stalledCreatedAt := now.Add(-10 * time.Minute)
|
|
recentCreatedAt := now.Add(-5 * time.Second)
|
|
|
|
for _, player := range []string{"stall-player-a", "stall-player-b", "recent-player"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
insertTicket := func(ticketID, playerID, state string, expiresAt time.Time) {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', $3, 'integration-build', 1, $4, $5)`, ticketID, playerID, state, now, expiresAt); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
insertTicket("stall-ticket-a", "stall-player-a", "PROCESS_READY", now.Add(time.Hour))
|
|
insertTicket("stall-ticket-b", "stall-player-b", "PROCESS_READY", now.Add(time.Hour))
|
|
insertTicket("recent-ticket", "recent-player", "ALLOCATING", now.Add(time.Hour))
|
|
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, created_at) VALUES ('stalled-match', 'casual', 'PROCESS_READY', 'NA', 1, 'stalled-server', $1)`, stalledCreatedAt); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, created_at) VALUES ('recent-match', 'casual', 'ALLOCATING', 'NA', 1, $1)`, recentCreatedAt); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) VALUES ('stalled-match', 'stall-player-a', 'stall-ticket-a', 0, 0), ('stalled-match', 'stall-player-b', 'stall-ticket-b', 1, 1)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) VALUES ('recent-match', 'recent-player', 'recent-ticket', 0, 0)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
reclaimed, err := ExpireStalledAllocations(ctx, db, now, 2*time.Minute, 10)
|
|
if err != nil {
|
|
t.Fatalf("expire stalled allocations: %v", err)
|
|
}
|
|
if reclaimed != 1 {
|
|
t.Fatalf("reclaimed = %d, want exactly 1 (the recent match must survive)", reclaimed)
|
|
}
|
|
|
|
var stalledState, recentState string
|
|
if err := db.QueryRow(`SELECT state FROM matches WHERE match_id = 'stalled-match'`).Scan(&stalledState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT state FROM matches WHERE match_id = 'recent-match'`).Scan(&recentState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if stalledState != "FAILED" {
|
|
t.Fatalf("stalled match state = %s, want FAILED", stalledState)
|
|
}
|
|
if recentState != "ALLOCATING" {
|
|
t.Fatalf("recent match state = %s, want untouched ALLOCATING", recentState)
|
|
}
|
|
|
|
var ticketAState, ticketBState, recentTicketState string
|
|
var ticketAExpiry time.Time
|
|
if err := db.QueryRow(`SELECT state, expires_at FROM queue_tickets WHERE ticket_id = 'stall-ticket-a'`).Scan(&ticketAState, &ticketAExpiry); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'stall-ticket-b'`).Scan(&ticketBState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'recent-ticket'`).Scan(&recentTicketState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if ticketAState != "QUEUED" || ticketBState != "QUEUED" {
|
|
t.Fatalf("stalled participants' tickets = %s, %s -- want both requeued to QUEUED, not failed/left behind", ticketAState, ticketBState)
|
|
}
|
|
if !ticketAExpiry.After(now) {
|
|
t.Fatalf("requeued ticket expiry %v was not refreshed forward from %v", ticketAExpiry, now)
|
|
}
|
|
if recentTicketState != "ALLOCATING" {
|
|
t.Fatalf("recent match's ticket state = %s, want untouched ALLOCATING", recentTicketState)
|
|
}
|
|
|
|
var activeParticipants int
|
|
if err := db.QueryRow(`SELECT count(*) FROM match_participants WHERE match_id = 'stalled-match' AND participation_active`).Scan(&activeParticipants); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if activeParticipants != 0 {
|
|
t.Fatalf("stalled match still has %d active participants, want 0 (so the player can be matched again)", activeParticipants)
|
|
}
|
|
var eventType string
|
|
var eventPayload []byte
|
|
if err := db.QueryRow(`SELECT event_type, payload FROM outbox WHERE event_id = 'stalled-allocation:stalled-match:1'`).Scan(&eventType, &eventPayload); err != nil {
|
|
t.Fatalf("stalled allocation state event missing: %v", err)
|
|
}
|
|
var event struct {
|
|
State string `json:"state"`
|
|
PlayerIDs []string `json:"player_ids"`
|
|
}
|
|
if err := json.Unmarshal(eventPayload, &event); err != nil {
|
|
t.Fatalf("decode stalled allocation outbox event: %v", err)
|
|
}
|
|
players := make(map[string]bool, len(event.PlayerIDs))
|
|
for _, playerID := range event.PlayerIDs {
|
|
players[playerID] = true
|
|
}
|
|
if eventType != "state_changed" || event.State != "FAILED" || !players["stall-player-a"] {
|
|
t.Fatalf("stalled allocation event = %s %s, want FAILED state and affected player IDs", eventType, eventPayload)
|
|
}
|
|
|
|
// Idempotent: the match is now FAILED, not one of the three reclaimable
|
|
// states, so a second pass must not touch it again.
|
|
reclaimedAgain, err := ExpireStalledAllocations(ctx, db, now.Add(time.Minute), 2*time.Minute, 10)
|
|
if err != nil {
|
|
t.Fatalf("second expire pass: %v", err)
|
|
}
|
|
if reclaimedAgain != 0 {
|
|
t.Fatalf("second pass reclaimed %d matches, want 0 (already-FAILED match must not be reprocessed)", reclaimedAgain)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLRankedSeasonRolloverIsExactlyOnce(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('season-player', 'season-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO ratings (player_id, rating, deviation, volatility, ranked_games) VALUES ('season-player', 2000, 100, 0.12, 25)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO seasons (season_id, playlist, starts_at, ends_at) VALUES ('season-1', 'ranked', $1, $2)`, now.Add(-12*7*24*time.Hour), now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
profile := domain.RankedProfile{Rating: domain.Rating{Value: 1900, RD: 100, Volatility: 0.12}, RankedGames: 25}
|
|
updated, applied, err := ApplyRankedSeasonRollover(ctx, db, "season-player", "season-1", profile, now)
|
|
if err != nil || !applied {
|
|
t.Fatalf("first season rollover = %+v applied=%v err=%v", updated, applied, err)
|
|
}
|
|
if updated.Value != 1875 || updated.RD != 200 {
|
|
t.Fatalf("unexpected rolled rating: %+v", updated)
|
|
}
|
|
var rating float64
|
|
var markers int
|
|
if err := db.QueryRow(`SELECT rating FROM ratings WHERE player_id = 'season-player'`).Scan(&rating); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM ranked_season_rollovers WHERE player_id = 'season-player' AND season_id = 'season-1'`).Scan(&markers); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if rating != 1875 || markers != 1 {
|
|
t.Fatalf("durable rollover state rating=%v markers=%d", rating, markers)
|
|
}
|
|
duplicate, applied, err := ApplyRankedSeasonRollover(ctx, db, "season-player", "season-1", profile, now.Add(time.Second))
|
|
if err != nil || applied || duplicate.Value != 1875 {
|
|
t.Fatalf("duplicate rollover = %+v applied=%v err=%v", duplicate, applied, err)
|
|
}
|
|
if err := db.QueryRow(`SELECT rating FROM ratings WHERE player_id = 'season-player'`).Scan(&rating); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if rating != 1875 {
|
|
t.Fatalf("duplicate rollover changed rating to %v", rating)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLEmptyRankedSeasonIsMarkedRolledOver(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO seasons (season_id, playlist, starts_at, ends_at) VALUES ('empty-season', 'ranked', $1, $2)`, now.Add(-12*7*24*time.Hour), now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if count, err := RolloverDueSeasons(ctx, db, now, 100); err != nil || count != 0 {
|
|
t.Fatalf("empty-season maintenance count=%d err=%v", count, err)
|
|
}
|
|
var rolledAt sql.NullTime
|
|
if err := db.QueryRowContext(ctx, `SELECT rolled_over_at FROM seasons WHERE season_id = 'empty-season'`).Scan(&rolledAt); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !rolledAt.Valid {
|
|
t.Fatal("empty ranked season was not marked rolled over")
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLRankedProfileProjectsActiveSeason(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('profile-season-player', 'profile-season-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO ratings (player_id, rating, deviation, volatility, ranked_games) VALUES ('profile-season-player', 1600, 200, 0.06, 10)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO seasons (season_id, playlist, starts_at, ends_at) VALUES ('profile-season-current', 'ranked', $1, $2)`, now.Add(-time.Hour), now.Add(time.Hour)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
profile, found, err := (PostgresRankedProfiles{DB: db}).Get(ctx, "profile-season-player")
|
|
if err != nil || !found || profile.CurrentSeasonID != "profile-season-current" || !profile.CurrentSeasonEndsAt.Equal(now.Add(time.Hour)) {
|
|
t.Fatalf("profile=%+v found=%t err=%v", profile, found, err)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLMigrationsAreForwardExecutable(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
var tableCount int
|
|
if err := db.QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'assignments'`).Scan(&tableCount); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if tableCount != 1 {
|
|
t.Fatal("assignments migration did not create its table")
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLMigrationsRollBackAndReapplyCleanly(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
dir := filepath.Join("..", "migrations")
|
|
tableExists := func(table string) bool {
|
|
var count int
|
|
if err := db.QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_name = $1`, table).Scan(&count); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return count == 1
|
|
}
|
|
if !tableExists("assignments") || !tableExists("allocations") {
|
|
t.Fatal("expected forward-applied schema before rollback")
|
|
}
|
|
|
|
// Roll back every migration one at a time, in reverse, checking each
|
|
// down file actually undoes what its forward file created — not just
|
|
// that Rollback returns nil.
|
|
// Derived, not hardcoded: every added migration shifts this count, and a
|
|
// stale literal silently makes the fixed-count rollbacks below target the
|
|
// wrong files (the failure then surfaces as a confusing "0006 rollback did
|
|
// not drop matches.allocation_id").
|
|
aboveMigration0006 := countMigrationsAbove(t, dir, 6)
|
|
if err := migrations.Rollback(context.Background(), db, dir, aboveMigration0006); err != nil {
|
|
t.Fatalf("rollback everything above 0006: %v", err)
|
|
}
|
|
var hasInitialConnectReadyColumn bool
|
|
if err := db.QueryRow(`SELECT count(*) > 0 FROM information_schema.columns WHERE table_name = 'matches' AND column_name = 'initial_connect_ready_at'`).Scan(&hasInitialConnectReadyColumn); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if hasInitialConnectReadyColumn {
|
|
t.Fatal("0010 rollback did not drop matches.initial_connect_ready_at")
|
|
}
|
|
|
|
if err := migrations.Rollback(context.Background(), db, dir, 1); err != nil {
|
|
t.Fatalf("rollback 0006: %v", err)
|
|
}
|
|
var hasAllocationClaimColumn bool
|
|
if err := db.QueryRow(`SELECT count(*) > 0 FROM information_schema.columns WHERE table_name = 'matches' AND column_name = 'allocation_id'`).Scan(&hasAllocationClaimColumn); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if hasAllocationClaimColumn {
|
|
t.Fatal("0006 rollback did not drop matches.allocation_id")
|
|
}
|
|
|
|
if err := migrations.Rollback(context.Background(), db, dir, 4); err != nil {
|
|
t.Fatalf("rollback 0005 through 0002: %v", err)
|
|
}
|
|
if tableExists("assignments") || tableExists("allocations") || tableExists("game_servers") {
|
|
t.Fatal("rollback left later-migration tables behind")
|
|
}
|
|
|
|
if err := migrations.Rollback(context.Background(), db, dir, 1); err != nil {
|
|
t.Fatalf("rollback 0001: %v", err)
|
|
}
|
|
if tableExists("identities") || tableExists("matches") {
|
|
t.Fatal("0001 rollback did not drop its own tables")
|
|
}
|
|
var remaining int
|
|
if err := db.QueryRow(`SELECT count(*) FROM schema_migrations`).Scan(&remaining); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if remaining != 0 {
|
|
t.Fatalf("expected schema_migrations empty after full rollback, got %d rows", remaining)
|
|
}
|
|
|
|
// Reapplying from a fully rolled-back state must reach the same schema,
|
|
// proving down files don't leave orphaned state that trips a forward
|
|
// re-run (e.g. a constraint or index Apply then tries to recreate).
|
|
if err := migrations.Apply(context.Background(), db, dir); err != nil {
|
|
t.Fatalf("reapply after full rollback: %v", err)
|
|
}
|
|
if !tableExists("assignments") || !tableExists("allocations") {
|
|
t.Fatal("reapply after rollback did not recreate the schema")
|
|
}
|
|
}
|
|
|
|
// Ranked matchmaking reads Candidate.Rating for tolerance, selection scoring
|
|
// and team partitioning. The candidate projection did not join the ratings
|
|
// table and its scan never set the field, so every PostgreSQL-sourced ranked
|
|
// candidate arrived with Go's zero value and the matcher treated a 900-rated
|
|
// player as identical to a 2100-rated one. Unit tests missed this because they
|
|
// construct candidates with ratings already populated.
|
|
func TestPostgreSQLQueuedCandidatesCarryAuthoritativeRatings(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
// "rated-low" and "rated-high" are deliberately far apart; "unrated" has no
|
|
// ratings row at all and must fall back to the new-profile default.
|
|
seeded := map[string]float64{"rated-low": 900, "rated-high": 2100}
|
|
for _, playerID := range []string{"rated-low", "rated-high", "unrated"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $2)`, playerID, "steam-"+playerID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if rating, ok := seeded[playerID]; ok {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO ratings (player_id, rating) VALUES ($1, $2)`, playerID, rating); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
spec := domain.QueueSpec{Playlist: domain.Ranked, ClientBuild: "rating-build", ProtocolVersion: 1}
|
|
ticket, err := CreateQueueTicket(ctx, db, "ticket-"+playerID, playerID, "rating-create-"+playerID+"-01", spec, now)
|
|
if err != nil {
|
|
t.Fatalf("create queue ticket for %s: %v", playerID, err)
|
|
}
|
|
// The Redis projection is seeded from this candidate rather than from
|
|
// the query below, so it must carry the same rating or the two
|
|
// projections disagree about who is comparable to whom.
|
|
want := domain.GlickoInitialRating
|
|
if rating, ok := seeded[playerID]; ok {
|
|
want = rating
|
|
}
|
|
if ticket.Candidate.Rating != want {
|
|
t.Fatalf("%s enqueue candidate rating = %v, want %v", playerID, ticket.Candidate.Rating, want)
|
|
}
|
|
}
|
|
|
|
candidates, err := ListQueuedCandidates(ctx, db, domain.Ranked, now, 100)
|
|
if err != nil {
|
|
t.Fatalf("list queued candidates: %v", err)
|
|
}
|
|
if len(candidates) != 3 {
|
|
t.Fatalf("expected 3 candidates, got %d", len(candidates))
|
|
}
|
|
got := make(map[string]float64, len(candidates))
|
|
for _, candidate := range candidates {
|
|
got[candidate.PlayerID] = candidate.Rating
|
|
}
|
|
for playerID, want := range map[string]float64{
|
|
"rated-low": 900, "rated-high": 2100, "unrated": domain.GlickoInitialRating,
|
|
} {
|
|
if got[playerID] != want {
|
|
t.Fatalf("%s durable candidate rating = %v, want %v", playerID, got[playerID], want)
|
|
}
|
|
}
|
|
|
|
// The whole point of loading the rating is that the matcher can tell these
|
|
// players apart. Assert the spread survives into team partitioning rather
|
|
// than only that the field is non-zero.
|
|
if got["rated-high"]-got["rated-low"] != 1200 {
|
|
t.Fatalf("rating spread collapsed: %v", got)
|
|
}
|
|
}
|
|
|
|
// banned_until and ban_reason existed in the schema from day one but no
|
|
// production query ever read them: the only ban check was an in-memory map on
|
|
// domain.TicketVerifier used by domain tests. A banned identity therefore kept
|
|
// working through every already-issued session until expiry, and could obtain
|
|
// new ones, defeating the server-authoritative anti-abuse boundary.
|
|
func TestPostgreSQLIdentityBanIsEnforcedOnIssuanceAndAuthentication(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
ctx := context.Background()
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('ban-player', 'ban-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Two independently constructed stores stand in for two control-plane
|
|
// replicas: the ban must hold on the replica that did not apply it.
|
|
replicaA := PostgresSessions{DB: db}
|
|
replicaB := PostgresSessions{DB: db}
|
|
|
|
session, token, err := replicaA.Issue(ctx, "ban-player", time.Hour, now)
|
|
if err != nil {
|
|
t.Fatalf("issue before ban: %v", err)
|
|
}
|
|
if _, err := replicaB.Authenticate(ctx, session.SessionID, token, now); err != nil {
|
|
t.Fatalf("authenticate before ban: %v", err)
|
|
}
|
|
|
|
if err := ApplyIdentityBan(ctx, db, "ban-player", "cheating", now.Add(24*time.Hour), now); err != nil {
|
|
t.Fatalf("apply ban: %v", err)
|
|
}
|
|
|
|
// The pre-existing session must stop working immediately, on a replica
|
|
// that never saw the ban being applied -- not at session expiry.
|
|
if _, err := replicaB.Authenticate(ctx, session.SessionID, token, now.Add(time.Minute)); err == nil {
|
|
t.Fatal("banned identity still authenticated with its existing session")
|
|
}
|
|
// ... and no new session may be minted.
|
|
if _, _, err := replicaB.Issue(ctx, "ban-player", time.Hour, now.Add(time.Minute)); err == nil {
|
|
t.Fatal("banned identity was issued a new session")
|
|
}
|
|
|
|
// The ban is time-bounded: once it lapses, sign-in works again.
|
|
afterBan := now.Add(25 * time.Hour)
|
|
revived, revivedToken, err := replicaA.Issue(ctx, "ban-player", time.Hour, afterBan)
|
|
if err != nil {
|
|
t.Fatalf("issue after ban expiry: %v", err)
|
|
}
|
|
if _, err := replicaB.Authenticate(ctx, revived.SessionID, revivedToken, afterBan); err != nil {
|
|
t.Fatalf("authenticate after ban expiry: %v", err)
|
|
}
|
|
|
|
// An explicit unban clears the state without resurrecting revoked sessions.
|
|
if err := ApplyIdentityBan(ctx, db, "ban-player", "", time.Time{}, now); err != nil {
|
|
t.Fatalf("clear ban: %v", err)
|
|
}
|
|
if _, err := replicaB.Authenticate(ctx, session.SessionID, token, now.Add(time.Minute)); err == nil {
|
|
t.Fatal("unban resurrected a session revoked by the ban")
|
|
}
|
|
}
|
|
|
|
// ApplyIdentityBan revokes sessions, so revocation alone would mask a missing
|
|
// ban check in Authenticate. A ban applied by any other path -- an admin tool,
|
|
// a future adapter, a direct operational UPDATE -- does not revoke anything,
|
|
// and must still take effect on the very next authenticated request.
|
|
func TestPostgreSQLBanWithoutRevocationStillBlocksExistingSessions(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
ctx := context.Background()
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('raw-ban-player', 'raw-ban-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sessions := PostgresSessions{DB: db}
|
|
session, token, err := sessions.Issue(ctx, "raw-ban-player", time.Hour, now)
|
|
if err != nil {
|
|
t.Fatalf("issue: %v", err)
|
|
}
|
|
if _, err := sessions.Authenticate(ctx, session.SessionID, token, now); err != nil {
|
|
t.Fatalf("authenticate before ban: %v", err)
|
|
}
|
|
// Set the ban only; deliberately leave every session unrevoked.
|
|
if _, err := db.ExecContext(ctx, `UPDATE identities SET banned_until = $2, ban_reason = 'manual' WHERE player_id = $1`, "raw-ban-player", now.Add(time.Hour)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := sessions.Authenticate(ctx, session.SessionID, token, now.Add(time.Minute)); err == nil {
|
|
t.Fatal("a ban applied without revocation left the existing session usable")
|
|
}
|
|
}
|
|
|
|
// The WebSocket hub is per-process, but any control-plane replica may drain a
|
|
// given outbox row, and the winner sets the single global published_at even
|
|
// with no matching local subscriber. Fan-out through LISTEN/NOTIFY is what
|
|
// lets the replica that actually owns the connection deliver the event.
|
|
func TestPostgreSQLControlPlaneEventFanoutReachesEveryReplica(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
dsn := os.Getenv("COSMIC_CLASH_POSTGRES_DSN")
|
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
|
defer cancel()
|
|
|
|
// Two listeners stand in for two replicas; neither is the one that will
|
|
// perform the notify.
|
|
type replica struct {
|
|
name string
|
|
received chan []byte
|
|
}
|
|
replicas := []*replica{
|
|
{name: "replica-a", received: make(chan []byte, 4)},
|
|
{name: "replica-b", received: make(chan []byte, 4)},
|
|
}
|
|
for _, r := range replicas {
|
|
target := r
|
|
go ListenControlPlaneEvents(ctx, dsn, func(payload []byte) {
|
|
select {
|
|
case target.received <- payload:
|
|
default:
|
|
}
|
|
}, nil)
|
|
}
|
|
|
|
// LISTEN is asynchronous; retry the notify until both listeners are
|
|
// attached rather than sleeping an arbitrary amount.
|
|
payload := []byte(`{"event":"state_changed","resource_id":"match-fanout","player_id":"player-fanout"}`)
|
|
deadline := time.Now().Add(15 * time.Second)
|
|
pending := map[string]bool{"replica-a": true, "replica-b": true}
|
|
for len(pending) > 0 && time.Now().Before(deadline) {
|
|
if err := NotifyControlPlaneEvent(ctx, db, payload); err != nil {
|
|
t.Fatalf("notify: %v", err)
|
|
}
|
|
for _, r := range replicas {
|
|
select {
|
|
case got := <-r.received:
|
|
if string(got) != string(payload) {
|
|
t.Fatalf("%s received %q, want %q", r.name, got, payload)
|
|
}
|
|
delete(pending, r.name)
|
|
case <-time.After(200 * time.Millisecond):
|
|
}
|
|
}
|
|
}
|
|
if len(pending) > 0 {
|
|
t.Fatalf("replicas never received the fanned-out event: %v", pending)
|
|
}
|
|
}
|
|
|
|
func TestNotifyControlPlaneEventRejectsOversizedPayloads(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
oversized := make([]byte, MaxNotifyPayloadBytes+1)
|
|
for i := range oversized {
|
|
oversized[i] = 'x'
|
|
}
|
|
if err := NotifyControlPlaneEvent(context.Background(), db, oversized); err == nil {
|
|
t.Fatal("payload over the NOTIFY limit was accepted")
|
|
}
|
|
}
|
|
|
|
// Each 10s queue heartbeat mints a fresh idempotency key and permanently
|
|
// inserts a row; published outbox rows and expired sessions were never purged
|
|
// either. At 10,000 queued players heartbeats alone add roughly 60,000 durable
|
|
// rows per minute, so table and index growth, vacuum pressure, backup size and
|
|
// recovery time were all unbounded on a horizontally-scaled service.
|
|
func TestPostgreSQLRetentionPurgesExpiredRecordsInBoundedBatches(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
ctx := context.Background()
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
stale := now.Add(-IdempotencyKeyRetention - time.Hour)
|
|
fresh := now.Add(-time.Minute)
|
|
|
|
// 120 stale keys (purgeable) and 5 fresh ones (must survive: a client may
|
|
// still retry those mutations).
|
|
for i := 0; i < 120; i++ {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO idempotency_keys (scope, idempotency_key, payload_digest, result, created_at) VALUES ('queue.mutate', $1, '\x00', '{}'::jsonb, $2)`,
|
|
fmt.Sprintf("stale-key-%030d", i), stale); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for i := 0; i < 5; i++ {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO idempotency_keys (scope, idempotency_key, payload_digest, result, created_at) VALUES ('queue.mutate', $1, '\x00', '{}'::jsonb, $2)`,
|
|
fmt.Sprintf("fresh-key-%030d", i), fresh); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
countKeys := func() int {
|
|
var count int
|
|
if err := db.QueryRowContext(ctx, `SELECT count(*) FROM idempotency_keys`).Scan(&count); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return count
|
|
}
|
|
if countKeys() != 125 {
|
|
t.Fatalf("seed failed: %d keys", countKeys())
|
|
}
|
|
|
|
// One pass must delete at most the batch size, not the whole backlog: a
|
|
// purge that took an unbounded lock would stall live traffic.
|
|
report, err := PurgeExpiredRecords(ctx, db, now, 50)
|
|
if err != nil {
|
|
t.Fatalf("purge: %v", err)
|
|
}
|
|
if report.IdempotencyKeys != 50 {
|
|
t.Fatalf("first pass deleted %d keys, want the 50-row batch bound", report.IdempotencyKeys)
|
|
}
|
|
if countKeys() != 75 {
|
|
t.Fatalf("after one bounded pass: %d keys", countKeys())
|
|
}
|
|
|
|
// Repeated passes converge on exactly the fresh rows and then stop.
|
|
for i := 0; i < 5; i++ {
|
|
if _, err := PurgeExpiredRecords(ctx, db, now, 50); err != nil {
|
|
t.Fatalf("purge pass %d: %v", i, err)
|
|
}
|
|
}
|
|
if countKeys() != 5 {
|
|
t.Fatalf("steady state left %d keys, want only the 5 fresh ones", countKeys())
|
|
}
|
|
final, err := PurgeExpiredRecords(ctx, db, now, 50)
|
|
if err != nil {
|
|
t.Fatalf("final purge: %v", err)
|
|
}
|
|
if final.Total() != 0 {
|
|
t.Fatalf("purge deleted %d rows that were still within their retention window", final.Total())
|
|
}
|
|
backlog, err := RetentionBacklog(ctx, db, now)
|
|
if err != nil {
|
|
t.Fatalf("backlog: %v", err)
|
|
}
|
|
if backlog != 0 {
|
|
t.Fatalf("deletion lag is %d after draining the backlog", backlog)
|
|
}
|
|
}
|
|
|
|
// countMigrationsAbove reports how many forward migrations have a number
|
|
// greater than the given one, so rollback step counts in this file track new
|
|
// migrations automatically instead of needing a manual bump.
|
|
func countMigrationsAbove(t *testing.T, dir string, number int) int {
|
|
t.Helper()
|
|
entries, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
t.Fatalf("read migrations: %v", err)
|
|
}
|
|
count := 0
|
|
for _, entry := range entries {
|
|
name := entry.Name()
|
|
if entry.IsDir() || !strings.HasSuffix(name, ".sql") || len(name) < 4 {
|
|
continue
|
|
}
|
|
index, err := strconv.Atoi(name[:4])
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if index > number {
|
|
count++
|
|
}
|
|
}
|
|
if count == 0 {
|
|
t.Fatalf("no migrations found above %04d in %s", number, dir)
|
|
}
|
|
return count
|
|
}
|
|
|
|
// The second fatal blocker. domain.validCandidate hard-requires a non-empty
|
|
// PredictedRTT map, but CreateQueueTicket persisted an empty one and the only
|
|
// endpoint that could fill it returned 503 in every real binary because
|
|
// Service.Probe was never wired. No client-created ticket could ever be
|
|
// selected by the matcher.
|
|
//
|
|
// This drives the real enqueue and probe paths and then asks the actual
|
|
// matcher predicate, rather than hand-building a domain.Candidate the way the
|
|
// unit tests do -- which is precisely why they missed it.
|
|
func TestPostgreSQLProbedTicketBecomesSelectableByTheMatcher(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
ctx := context.Background()
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('probe-player', 'probe-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
spec := domain.QueueSpec{Playlist: domain.Casual, ClientBuild: "probe-build", ProtocolVersion: 1}
|
|
if _, err := CreateQueueTicket(ctx, db, "probe-ticket", "probe-player", "probe-create-000001", spec, now); err != nil {
|
|
t.Fatalf("create queue ticket: %v", err)
|
|
}
|
|
|
|
// Freshly queued: no RTT evidence yet, so the matcher must not consider it.
|
|
candidates, err := ListQueuedCandidates(ctx, db, domain.Casual, now, 100)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(candidates) != 1 {
|
|
t.Fatalf("expected the queued ticket, got %d candidates", len(candidates))
|
|
}
|
|
if len(candidates[0].PredictedRTT) != 0 {
|
|
t.Fatalf("a freshly queued ticket already has RTT evidence: %v", candidates[0].PredictedRTT)
|
|
}
|
|
if _, err := domain.SelectCandidates(candidates[0], candidates, 1, now); err == nil {
|
|
t.Fatal("a candidate with no RTT evidence was accepted by the matcher")
|
|
}
|
|
|
|
// The real challenge/answer round trip: the backend issues the nonce and
|
|
// derives RTT from its own timestamps, never from a client-reported value.
|
|
nonce, err := IssueProbeChallenge(ctx, db, "probe-player", "EU", now)
|
|
if err != nil {
|
|
t.Fatalf("issue challenge: %v", err)
|
|
}
|
|
if len(nonce) != ProbeNonceBytes {
|
|
t.Fatalf("challenge nonce is %d bytes", len(nonce))
|
|
}
|
|
answeredAt := now.Add(40 * time.Millisecond)
|
|
evidence, expectedNonce, err := ProbeEvidenceFromChallenge(ctx, db, "probe-player", "EU", []byte("opaque-location"), nonce, answeredAt)
|
|
if err != nil {
|
|
t.Fatalf("probe evidence: %v", err)
|
|
}
|
|
if err := domain.ValidateProbe(evidence, expectedNonce, answeredAt); err != nil {
|
|
t.Fatalf("backend-derived evidence failed its own validation: %v", err)
|
|
}
|
|
if evidence.ServerRTT != 40*time.Millisecond {
|
|
t.Fatalf("server-derived RTT = %v, want the 40ms round trip", evidence.ServerRTT)
|
|
}
|
|
|
|
// A challenge is single-use, so a captured answer cannot be replayed to
|
|
// refresh a stale RTT.
|
|
if _, _, err := ProbeEvidenceFromChallenge(ctx, db, "probe-player", "EU", []byte("opaque-location"), nonce, answeredAt); err == nil {
|
|
t.Fatal("a probe challenge was answerable twice")
|
|
}
|
|
|
|
if err := (PostgresQueue{DB: db}).RecordProbe(ctx, "probe-player", "EU", evidence.ServerRTT, answeredAt); err != nil {
|
|
t.Fatalf("record probe: %v", err)
|
|
}
|
|
|
|
// Now the same ticket, read through the same production query, is
|
|
// selectable.
|
|
candidates, err = ListQueuedCandidates(ctx, db, domain.Casual, answeredAt, 100)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(candidates) != 1 || candidates[0].PredictedRTT["EU"] == 0 {
|
|
t.Fatalf("probe did not reach the candidate projection: %+v", candidates)
|
|
}
|
|
if _, err := domain.SelectCandidates(candidates[0], candidates, 1, answeredAt); err != nil {
|
|
t.Fatalf("a probed, client-created ticket is still not selectable by the matcher: %v", err)
|
|
}
|
|
|
|
// The same candidate must survive the Redis path, which is seeded from the
|
|
// per-player refresh the probe handler performs.
|
|
refreshed, queued, err := FindQueuedCandidateByPlayer(ctx, db, "probe-player", answeredAt)
|
|
if err != nil || !queued {
|
|
t.Fatalf("per-player candidate refresh: queued=%t err=%v", queued, err)
|
|
}
|
|
if refreshed.PredictedRTT["EU"] == 0 {
|
|
t.Fatal("the refreshed candidate still carries an empty RTT map, so Redis would keep a stale entry")
|
|
}
|
|
}
|