mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
f628ccfd35
newAPIService never supplied SteamLogin, so POST /v1/session/steam always returned 503 auth_unavailable in production. The only implementation was cmd/testkit-api's fake, which derives an identity from the ticket string itself and accepts anything -- so the passing integration path was neither deployable nor secure. On the client side the game started with an empty token and a loopback base URL, and no production code called configure() or login_steam(); the menu entered matchmaking directly, so every request failed ERR_UNAUTHORIZED before reaching the network. Add a real ISteamUserAuth/AuthenticateUserTicket adapter behind an interface, so the production login path is testable with only the Valve call stubbed. It rejects family-shared copies (the account playing does not own the app) and, by default, VAC- or publisher-banned accounts, and refuses malformed tickets locally rather than forwarding them. Crucially it separates our faults from the player's: a Valve outage or a revoked publisher key returns 503, not 401. Answering 401 would tell a legitimate player their login failed and send them to fix an account that is fine while the real fault went unnoticed. A banned identity now returns 403 rather than a misleading 503. Sign-in is configuration-gated on the publisher key and App ID: without them the endpoint keeps returning 503, since silently accepting an unverified ticket would be worse than refusing to authenticate. A returning player keeps the player ID they already had, so ratings, penalties and bans follow the account rather than the session. Client side: acquire a web-API ticket through GodotSteam's async signal -- requesting one returns a handle, not a ticket -- using the existing dynamic-call pattern so stock Godot still parses the project. The endpoint is configurable for release builds, and matchmaking completes sign-in before it will queue. Verified against real PostgreSQL; 232 Godot tests pass.
2341 lines
111 KiB
Go
2341 lines
111 KiB
Go
//go:build integration
|
|
|
|
package store
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/cosmic-clash/cosmic-clash/server/domain"
|
|
"github.com/cosmic-clash/cosmic-clash/server/migrations"
|
|
_ "github.com/jackc/pgx/v5/stdlib"
|
|
)
|
|
|
|
// This binary is deliberately opt-in. It requires a disposable PostgreSQL
|
|
// instance supplied by scripts/run_postgres_integration.sh.
|
|
func openIntegrationPostgres(t *testing.T) *sql.DB {
|
|
t.Helper()
|
|
dsn := os.Getenv("COSMIC_CLASH_POSTGRES_DSN")
|
|
if dsn == "" {
|
|
t.Skip("COSMIC_CLASH_POSTGRES_DSN is not set")
|
|
}
|
|
db, err := sql.Open("pgx", dsn)
|
|
if err != nil {
|
|
t.Fatalf("open PostgreSQL: %v", err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if err := db.PingContext(ctx); err != nil {
|
|
db.Close()
|
|
t.Fatalf("ping PostgreSQL: %v", err)
|
|
}
|
|
t.Cleanup(func() { db.Close() })
|
|
return db
|
|
}
|
|
|
|
func applyIntegrationMigrations(t *testing.T, db *sql.DB) {
|
|
t.Helper()
|
|
if _, err := db.ExecContext(context.Background(), `DROP SCHEMA public CASCADE; CREATE SCHEMA public`); err != nil {
|
|
t.Fatalf("reset PostgreSQL schema: %v", err)
|
|
}
|
|
if err := migrations.Apply(context.Background(), db, filepath.Join("..", "migrations")); err != nil {
|
|
t.Fatalf("apply migrations: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLAllocatorClaimReplayAndCapacityFence(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
servers := []domain.ReadyServer{
|
|
{ServerID: "allocator-server-b", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet", State: domain.ServerReady},
|
|
{ServerID: "allocator-server-a", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet", State: domain.ServerReady},
|
|
}
|
|
for _, server := range servers {
|
|
if err := RegisterReadyServer(ctx, db, server, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
request := domain.AllocationRequest{AllocationID: "allocation-integration-1", MatchID: "match-integration-1", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet"}
|
|
allocation, err := ClaimAllocation(ctx, db, request, now)
|
|
if err != nil {
|
|
t.Fatalf("claim: %v", err)
|
|
}
|
|
if allocation.ServerID != "allocator-server-a" || allocation.State != domain.ServerAllocated {
|
|
t.Fatalf("allocation=%+v", allocation)
|
|
}
|
|
if err := RegisterReadyServer(ctx, db, servers[1], now.Add(time.Second)); err != nil {
|
|
t.Fatalf("stale Ready projection: %v", err)
|
|
}
|
|
var lifecycle string
|
|
if err := db.QueryRowContext(ctx, `SELECT state FROM game_servers WHERE server_id = 'allocator-server-a'`).Scan(&lifecycle); err != nil || lifecycle != "ALLOCATED" {
|
|
t.Fatalf("stale Ready projection reopened allocation state=%q err=%v", lifecycle, err)
|
|
}
|
|
replay, err := ClaimAllocation(ctx, db, request, now.Add(time.Second))
|
|
if err != nil || replay.ServerID != allocation.ServerID || !replay.AllocatedAt.Equal(allocation.AllocatedAt) {
|
|
t.Fatalf("replay=%+v err=%v", replay, err)
|
|
}
|
|
conflict := request
|
|
conflict.MatchID = "match-integration-other"
|
|
if _, err := ClaimAllocation(ctx, db, conflict, now); err != domain.ErrConflict {
|
|
t.Fatalf("conflicting replay err=%v", err)
|
|
}
|
|
second := request
|
|
second.AllocationID = "allocation-integration-2"
|
|
second.MatchID = "match-integration-2"
|
|
if _, err := ClaimAllocation(ctx, db, second, now); err != nil {
|
|
t.Fatalf("second claim: %v", err)
|
|
}
|
|
third := second
|
|
third.AllocationID = "allocation-integration-3"
|
|
third.MatchID = "match-integration-3"
|
|
if _, err := ClaimAllocation(ctx, db, third, now); err != domain.ErrNoCapacity {
|
|
t.Fatalf("capacity err=%v", err)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLSharedAllocationQuotaFencesClaimsAndReplays(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
if err := SetAllocationQuota(ctx, db, "EU", 1, time.Minute, now); err != nil {
|
|
t.Fatalf("set quota: %v", err)
|
|
}
|
|
for _, server := range []domain.ReadyServer{
|
|
{ServerID: "quota-server-a", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet", State: domain.ServerReady},
|
|
{ServerID: "quota-server-b", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet", State: domain.ServerReady},
|
|
} {
|
|
if err := RegisterReadyServer(ctx, db, server, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
first := domain.AllocationRequest{AllocationID: "quota-allocation-1", MatchID: "quota-match-1", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet"}
|
|
if _, err := ClaimAllocation(ctx, db, first, now); err != nil {
|
|
t.Fatalf("first claim: %v", err)
|
|
}
|
|
if _, err := ClaimAllocation(ctx, db, first, now.Add(time.Second)); err != nil {
|
|
t.Fatalf("idempotent replay was fenced: %v", err)
|
|
}
|
|
second := domain.AllocationRequest{AllocationID: "quota-allocation-2", MatchID: "quota-match-2", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet"}
|
|
if _, err := ClaimAllocation(ctx, db, second, now.Add(2*time.Second)); !errors.Is(err, ErrAllocationQuotaExceeded) {
|
|
t.Fatalf("second claim err=%v, want shared quota fence", err)
|
|
}
|
|
if err := SetAllocationQuota(ctx, db, "EU", 1, time.Minute, now.Add(time.Minute)); err != nil {
|
|
t.Fatalf("reset quota: %v", err)
|
|
}
|
|
if _, err := ClaimAllocation(ctx, db, second, now.Add(time.Minute)); err != nil {
|
|
t.Fatalf("claim after quota window reset: %v", err)
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLConcurrentAllocationClaimNeverDoubleBooksAReadyServer is the
|
|
// live counterpart to TestPostgreSQLAllocatorClaimReplayAndCapacityFence: that
|
|
// test claims strictly one request at a time, so it cannot show what happens
|
|
// when two allocator replicas race for the same compatible capacity, which is
|
|
// exactly the scenario 8.30's "bounded cross-replica retry" is about. Register
|
|
// fewer Ready servers than concurrent requests and fire them all at once;
|
|
// exactly as many must win as there was capacity, each winner must get a
|
|
// distinct server, and every loser must fail with ErrNoCapacity rather than a
|
|
// raw serialization error, a duplicate claim, or a hang.
|
|
func TestPostgreSQLConcurrentAllocationClaimNeverDoubleBooksAReadyServer(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
const capacity = 3
|
|
const contenders = 6
|
|
for i := 0; i < capacity; i++ {
|
|
server := domain.ReadyServer{ServerID: fmt.Sprintf("race-server-%d", i), Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet", State: domain.ServerReady}
|
|
if err := RegisterReadyServer(ctx, db, server, now); err != nil {
|
|
t.Fatalf("register %s: %v", server.ServerID, err)
|
|
}
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
allocations := make([]domain.Allocation, contenders)
|
|
errs := make([]error, contenders)
|
|
wg.Add(contenders)
|
|
for i := 0; i < contenders; i++ {
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
request := domain.AllocationRequest{AllocationID: fmt.Sprintf("race-allocation-%d", i), MatchID: fmt.Sprintf("race-match-%d", i), Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet"}
|
|
allocations[i], errs[i] = ClaimAllocation(ctx, db, request, now)
|
|
}(i)
|
|
}
|
|
wg.Wait()
|
|
|
|
wonServers := map[string]int{}
|
|
won, lost := 0, 0
|
|
for i, err := range errs {
|
|
switch {
|
|
case err == nil:
|
|
won++
|
|
if allocations[i].ServerID == "" {
|
|
t.Fatalf("claim %d succeeded with no server", i)
|
|
}
|
|
wonServers[allocations[i].ServerID]++
|
|
case errors.Is(err, domain.ErrNoCapacity):
|
|
lost++
|
|
default:
|
|
t.Fatalf("claim %d failed with unexpected error: %v", i, err)
|
|
}
|
|
}
|
|
if won != capacity || lost != contenders-capacity {
|
|
t.Fatalf("won=%d lost=%d, want won=%d lost=%d", won, lost, capacity, contenders-capacity)
|
|
}
|
|
if len(wonServers) != capacity {
|
|
t.Fatalf("expected %d distinct servers claimed, got %d: %v", capacity, len(wonServers), wonServers)
|
|
}
|
|
for server, count := range wonServers {
|
|
if count != 1 {
|
|
t.Fatalf("server %s was claimed %d times", server, count)
|
|
}
|
|
}
|
|
var allocatedCount int
|
|
if err := db.QueryRowContext(ctx, `SELECT count(*) FROM game_servers WHERE state = 'ALLOCATED'`).Scan(&allocatedCount); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if allocatedCount != capacity {
|
|
t.Fatalf("durable ALLOCATED server count = %d, want %d", allocatedCount, capacity)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLAcceptedProposalPromotesOneAtomicAllocatingMatch(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"promote-a", "promote-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for index, player := range []string{"promote-a", "promote-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'PROPOSED', 'build-1', 1, $3, $4)`, fmt.Sprintf("promote-ticket-%d", index), player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO proposals (proposal_id, playlist, state, expires_at, revision) VALUES ('promote-proposal', 'casual', 'ACCEPTED', $1, 2)`, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for index, player := range []string{"promote-a", "promote-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO proposal_participants (proposal_id, player_id, ticket_id, response) VALUES ('promote-proposal', $1, $2, 'ACCEPTED')`, player, fmt.Sprintf("promote-ticket-%d", index)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
plan := AcceptedMatchPlan{MatchID: "promote-match", ProposalID: "promote-proposal", Region: "EU", Protocol: 1, Players: []MatchPlayer{{PlayerID: "promote-a", Team: 0, Slot: 0}, {PlayerID: "promote-b", Team: 1, Slot: 3}}}
|
|
if err := CreateMatchFromAcceptedProposal(ctx, db, plan, now); err != nil {
|
|
t.Fatalf("promote accepted proposal: %v", err)
|
|
}
|
|
var state string
|
|
if err := db.QueryRowContext(ctx, `SELECT state FROM matches WHERE match_id = 'promote-match'`).Scan(&state); err != nil || state != "ALLOCATING" {
|
|
t.Fatalf("match state=%q err=%v", state, err)
|
|
}
|
|
var acceptedTickets, participantCount int
|
|
if err := db.QueryRowContext(ctx, `SELECT count(*) FROM queue_tickets WHERE ticket_id LIKE 'promote-ticket-%' AND state = 'ACCEPTED'`).Scan(&acceptedTickets); err != nil || acceptedTickets != 2 {
|
|
t.Fatalf("accepted tickets=%d err=%v", acceptedTickets, err)
|
|
}
|
|
if err := db.QueryRowContext(ctx, `SELECT count(*) FROM match_participants WHERE match_id = 'promote-match'`).Scan(&participantCount); err != nil || participantCount != 2 {
|
|
t.Fatalf("participants=%d err=%v", participantCount, err)
|
|
}
|
|
if err := CreateMatchFromAcceptedProposal(ctx, db, plan, now.Add(time.Second)); err != nil {
|
|
t.Fatalf("identical match promotion replay: %v", err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `UPDATE matches SET state = 'LIVE' WHERE match_id = 'promote-match'`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := CreateMatchFromAcceptedProposal(ctx, db, plan, now.Add(2*time.Second)); err != nil {
|
|
t.Fatalf("promotion replay after match lifecycle advanced: %v", err)
|
|
}
|
|
conflict := plan
|
|
conflict.Players = append([]MatchPlayer(nil), plan.Players...)
|
|
conflict.Players[1].Slot = 4
|
|
if err := CreateMatchFromAcceptedProposal(ctx, db, conflict, now.Add(3*time.Second)); err == nil {
|
|
t.Fatal("conflicting match promotion replay was accepted")
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLAllocationMatchClaimLeaseAndBindFence(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for index, player := range []string{"allocation-match-a", "allocation-match-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'ACCEPTED', 'build-1', 1, $3, $4)`, fmt.Sprintf("allocation-match-ticket-%d", index), player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version) VALUES ('allocation-match', 'casual', 'ALLOCATING', 'EU', 1)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for index, player := range []string{"allocation-match-a", "allocation-match-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) VALUES ('allocation-match', $1, $2, $3, $4)`, player, fmt.Sprintf("allocation-match-ticket-%d", index), index*3, index); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
claim, found, err := ClaimAllocatingMatch(ctx, db, "enet", now)
|
|
if err != nil || !found || claim.Request != (domain.AllocationRequest{AllocationID: "allocation-allocation-match", MatchID: "allocation-match", Playlist: domain.Casual, Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet"}) {
|
|
t.Fatalf("claim=%+v found=%t err=%v", claim, found, err)
|
|
}
|
|
if err := ReleaseAllocatedMatchClaim(ctx, db, claim.Request.MatchID, "different-allocation"); err != domain.ErrConflict {
|
|
t.Fatalf("wrong-claim release err=%v", err)
|
|
}
|
|
if err := ReleaseAllocatedMatchClaim(ctx, db, claim.Request.MatchID, claim.Request.AllocationID); err != nil {
|
|
t.Fatalf("release claim: %v", err)
|
|
}
|
|
reclaimed, found, err := ClaimAllocatingMatch(ctx, db, "enet", now.Add(time.Second))
|
|
if err != nil || !found || reclaimed.Request.AllocationID != claim.Request.AllocationID {
|
|
t.Fatalf("reclaimed=%+v found=%t err=%v", reclaimed, found, err)
|
|
}
|
|
server := domain.ReadyServer{ServerID: "allocation-server", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet", State: domain.ServerReady}
|
|
if err := RegisterReadyServer(ctx, db, server, now); err != nil {
|
|
t.Fatalf("register allocation server: %v", err)
|
|
}
|
|
allocation, err := ClaimAllocation(ctx, db, reclaimed.Request, now.Add(time.Second))
|
|
if err != nil {
|
|
t.Fatalf("record provider allocation: %v", err)
|
|
}
|
|
if err := BindAllocatedMatch(ctx, db, allocation); err != nil {
|
|
t.Fatalf("bind allocation: %v", err)
|
|
}
|
|
recoveredTicket, err := GetQueueTicket(ctx, db, "allocation-match-a", "allocation-match-ticket-0", now.Add(2*time.Second))
|
|
if err != nil || recoveredTicket.MatchID != "allocation-match" {
|
|
t.Fatalf("recovered ticket match=%q err=%v", recoveredTicket.MatchID, err)
|
|
}
|
|
var allocatingTickets int
|
|
if err := db.QueryRowContext(ctx, `SELECT count(*) FROM queue_tickets WHERE ticket_id LIKE 'allocation-match-ticket-%' AND state = 'ALLOCATING'`).Scan(&allocatingTickets); err != nil || allocatingTickets != 2 {
|
|
t.Fatalf("allocating tickets=%d err=%v", allocatingTickets, err)
|
|
}
|
|
var eventType string
|
|
var eventPayload []byte
|
|
if err := db.QueryRowContext(ctx, `SELECT event_type, payload FROM outbox WHERE aggregate_id = 'allocation-match' AND event_type = 'state_changed'`).Scan(&eventType, &eventPayload); err != nil {
|
|
t.Fatalf("allocation outbox event: %v", err)
|
|
}
|
|
var event struct {
|
|
State string `json:"state"`
|
|
PlayerIDs []string `json:"player_ids"`
|
|
}
|
|
if err := json.Unmarshal(eventPayload, &event); err != nil {
|
|
t.Fatalf("decode allocation outbox event: %v", err)
|
|
}
|
|
players := make(map[string]bool, len(event.PlayerIDs))
|
|
for _, playerID := range event.PlayerIDs {
|
|
players[playerID] = true
|
|
}
|
|
if eventType != "state_changed" || event.State != "ALLOCATING" || !players["allocation-match-a"] || !players["allocation-match-b"] {
|
|
t.Fatalf("allocation outbox event = %s", eventPayload)
|
|
}
|
|
if _, found, err := ClaimAllocatingMatch(ctx, db, "enet", now.Add(2*time.Second)); err != nil || found {
|
|
t.Fatalf("bound match re-claimed found=%t err=%v", found, err)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLQueueAdapterAgainstRealDatabase(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('integration-player', 'integration-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
spec := domain.QueueSpec{Playlist: domain.Casual, ClientBuild: "integration-build", ProtocolVersion: 1}
|
|
ticket, err := CreateQueueTicket(ctx, db, "integration-ticket", "integration-player", "integration-create-0001", spec, now)
|
|
if err != nil {
|
|
t.Fatalf("create queue ticket: %v", err)
|
|
}
|
|
if ticket.State != domain.Queued || ticket.Revision != 0 {
|
|
t.Fatalf("unexpected ticket: %+v", ticket)
|
|
}
|
|
replay, err := CreateQueueTicket(ctx, db, "integration-ticket", "integration-player", "integration-create-0001", spec, now.Add(time.Second))
|
|
if err != nil {
|
|
t.Fatalf("idempotent queue replay: %v", err)
|
|
}
|
|
if replay.TicketID != ticket.TicketID || !replay.ExpiresAt.Equal(ticket.ExpiresAt) {
|
|
t.Fatalf("replay changed durable result: %+v vs %+v", replay, ticket)
|
|
}
|
|
if _, err := CreateQueueTicket(ctx, db, "integration-ticket-2", "integration-player", "integration-create-0002", spec, now); err == nil {
|
|
t.Fatal("second active player ticket was accepted")
|
|
}
|
|
if _, err := GetQueueTicket(ctx, db, "integration-player", "integration-ticket", now); err != nil {
|
|
t.Fatalf("owner recovery: %v", err)
|
|
}
|
|
if _, err := GetQueueTicket(ctx, db, "other-player", "integration-ticket", now); err == nil {
|
|
t.Fatal("non-owner recovered queue ticket")
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLQueueHeartbeatAndCancelAreRevisionFenced(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('heartbeat-player', 'heartbeat-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
spec := domain.QueueSpec{Playlist: domain.Ranked, ClientBuild: "integration-build", ProtocolVersion: 1}
|
|
if _, err := CreateQueueTicket(ctx, db, "heartbeat-ticket", "heartbeat-player", "heartbeat-create-0001", spec, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
heartbeat, err := HeartbeatQueueTicket(ctx, db, "heartbeat-player", "heartbeat-ticket", "heartbeat-op-0000001", 0, now.Add(5*time.Second))
|
|
if err != nil {
|
|
t.Fatalf("heartbeat: %v", err)
|
|
}
|
|
if heartbeat.Revision != 1 || !heartbeat.ExpiresAt.Equal(now.Add(35*time.Second)) {
|
|
t.Fatalf("unexpected heartbeat result: %+v", heartbeat)
|
|
}
|
|
if _, err := HeartbeatQueueTicket(ctx, db, "heartbeat-player", "heartbeat-ticket", "heartbeat-op-0000002", 0, now.Add(6*time.Second)); !errors.Is(err, domain.ErrStaleRevision) {
|
|
t.Fatalf("stale heartbeat error = %v, want ErrStaleRevision", err)
|
|
}
|
|
cancelled, err := CancelQueueTicket(ctx, db, "heartbeat-player", "heartbeat-ticket", "heartbeat-op-0000003", 1, now.Add(7*time.Second))
|
|
if err != nil {
|
|
t.Fatalf("cancel: %v", err)
|
|
}
|
|
if cancelled.State != domain.Cancelled || cancelled.Revision != 2 {
|
|
t.Fatalf("unexpected cancellation result: %+v", cancelled)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('live-cancel-player', 'live-cancel-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ('live-cancel-ticket', 'live-cancel-player', 'ranked', 'LIVE', 'integration-build', 1, $1, $2)`, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := CancelQueueTicket(ctx, db, "live-cancel-player", "live-cancel-ticket", "live-cancel-op-0001", 0, now.Add(8*time.Second)); !errors.Is(err, domain.ErrConflict) {
|
|
t.Fatalf("live cancellation error = %v, want ErrConflict", err)
|
|
}
|
|
var liveState string
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'live-cancel-ticket'`).Scan(&liveState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if liveState != "LIVE" {
|
|
t.Fatalf("live ticket state = %s after cancellation attempt", liveState)
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLConcurrentQueueHeartbeatIsRevisionFencedUnderRealRace is the
|
|
// live counterpart to the sequential stale-heartbeat check above: calling the
|
|
// second heartbeat only after the first has already committed proves the SQL
|
|
// predicate is correct, but not that it actually fences two requests that
|
|
// genuinely overlap at the database. A client can legitimately double-send a
|
|
// heartbeat (a slow response triggering a client-side retry, or two tabs/
|
|
// processes for the same player), and both requests can reach PostgreSQL
|
|
// truly concurrently -- this races that directly.
|
|
func TestPostgreSQLConcurrentQueueHeartbeatIsRevisionFencedUnderRealRace(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('race-heartbeat-player', 'race-heartbeat-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
spec := domain.QueueSpec{Playlist: domain.Casual, ClientBuild: "integration-build", ProtocolVersion: 1}
|
|
if _, err := CreateQueueTicket(ctx, db, "race-heartbeat-ticket", "race-heartbeat-player", "race-heartbeat-create-01", spec, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
const attempts = 5
|
|
var wg sync.WaitGroup
|
|
tickets := make([]domain.QueueTicket, attempts)
|
|
errs := make([]error, attempts)
|
|
wg.Add(attempts)
|
|
for i := 0; i < attempts; i++ {
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
tickets[i], errs[i] = HeartbeatQueueTicket(ctx, db, "race-heartbeat-player", "race-heartbeat-ticket", fmt.Sprintf("race-heartbeat-op-%08d", i), 0, now.Add(time.Duration(i)*time.Millisecond))
|
|
}(i)
|
|
}
|
|
wg.Wait()
|
|
|
|
won, lost := 0, 0
|
|
for i, err := range errs {
|
|
if err == nil {
|
|
won++
|
|
if tickets[i].Revision != 1 {
|
|
t.Fatalf("winning heartbeat %d landed at revision %d, want 1", i, tickets[i].Revision)
|
|
}
|
|
continue
|
|
}
|
|
lost++
|
|
}
|
|
if won != 1 {
|
|
t.Fatalf("won=%d, want exactly 1 of %d concurrent heartbeats at the same expected revision to win", won, attempts)
|
|
}
|
|
if lost != attempts-1 {
|
|
t.Fatalf("lost=%d, want %d", lost, attempts-1)
|
|
}
|
|
var revision uint64
|
|
if err := db.QueryRow(`SELECT revision FROM queue_tickets WHERE ticket_id = 'race-heartbeat-ticket'`).Scan(&revision); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if revision != 1 {
|
|
t.Fatalf("durable revision = %d, want exactly 1 (a stale winner re-applying would leave it higher)", revision)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLAssignmentPersistenceIsPlayerScopedAndExpiryBound(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"assignment-player", "assignment-other"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ('assignment-ticket', 'assignment-player', 'casual', 'ASSIGNED', 'integration-build', 1, $1, $2)`, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, initial_connect_ready_at) VALUES ('assignment-match', 'casual', 'ASSIGNED', 'EU', 1, 'assignment-server', $1)`, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) VALUES ('assignment-match', 'assignment-player', 'assignment-ticket', 0, 0)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
assignment := DurableAssignment{MatchID: "assignment-match", PlayerID: "assignment-player", AllocationID: "allocation-1", ServerID: "assignment-server", Slot: 0, Region: "EU", ClientBuild: "integration-build", ProtocolVersion: 1, Transport: "enet", Endpoint: "127.0.0.1:7777", JoinAuthorisation: "join-token", ManifestDigest: []byte("manifest"), ExpiresAt: now.Add(time.Minute), Revision: 1}
|
|
if err := SaveAssignment(ctx, db, assignment); err != nil {
|
|
t.Fatalf("save assignment: %v", err)
|
|
}
|
|
got, err := GetAssignment(ctx, db, assignment.PlayerID, assignment.MatchID, now)
|
|
if err != nil {
|
|
t.Fatalf("recover assignment: %v", err)
|
|
}
|
|
if got.JoinAuthorisation != assignment.JoinAuthorisation || got.Slot != assignment.Slot {
|
|
t.Fatalf("assignment changed on round trip: %+v", got)
|
|
}
|
|
if _, err := GetAssignment(ctx, db, "assignment-other", assignment.MatchID, now); err == nil {
|
|
t.Fatal("non-owner recovered assignment")
|
|
}
|
|
if _, err := GetAssignment(ctx, db, assignment.PlayerID, assignment.MatchID, now.Add(2*time.Minute)); err == nil {
|
|
t.Fatal("expired assignment was recovered")
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLVerifiedAssignmentRosterMustMatchDurableParticipants(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for index, player := range []string{"roster-player-a", "roster-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $2)`, player, "steam-"+player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'ALLOCATING', 'build-1', 1, $3, $4)`, fmt.Sprintf("roster-ticket-%d", index), player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO game_servers (server_id, region, build, protocol_version, transport, state, updated_at) VALUES ('roster-server', 'EU', 'build-1', 1, 'enet', 'ALLOCATED', $1)`, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO allocations (allocation_id, match_id, server_id, region, build, protocol_version, transport, request_digest, state, allocated_at) VALUES ('roster-allocation', 'roster-match', 'roster-server', 'EU', 'build-1', 1, 'enet', 'digest', 'ALLOCATED', $1)`, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, allocation_id, allocation_claimed_at) VALUES ('roster-match', 'casual', 'ALLOCATING', 'EU', 1, 'roster-server', 'roster-allocation', $1)`, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for index, player := range []string{"roster-player-a", "roster-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) VALUES ('roster-match', $1, $2, $3, $4)`, player, fmt.Sprintf("roster-ticket-%d", index), index*3, index); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
allocation := domain.Allocation{AllocationID: "roster-allocation", MatchID: "roster-match", ServerID: "roster-server", Region: "EU", Build: "build-1", Protocol: 1, Transport: "enet", State: domain.ServerAllocated, AllocatedAt: now}
|
|
assignment := domain.Assignment{Allocation: allocation, Manifest: domain.AllocationManifest{AllocationID: allocation.AllocationID, MatchID: allocation.MatchID, ServerID: allocation.ServerID, Region: allocation.Region, Build: allocation.Build, Protocol: allocation.Protocol, Transport: allocation.Transport, RosterDigest: "roster-digest"}, Endpoint: "127.0.0.1:7777"}
|
|
roster := []domain.SignedJoinAuthorisation{
|
|
{Authorisation: domain.JoinAuthorisation{MatchID: "roster-match", ServerID: "roster-server", PlayerID: "roster-player-a", SteamID: "steam-roster-player-a", Slot: 0, Team: 0, Protocol: "1", Generation: 1, ExpiresAt: now.Add(time.Minute)}, Signature: []byte("sig-a")},
|
|
{Authorisation: domain.JoinAuthorisation{MatchID: "roster-match", ServerID: "roster-server", PlayerID: "roster-player-b", SteamID: "steam-roster-player-b", Slot: 3, Team: 1, Protocol: "1", Generation: 1, ExpiresAt: now.Add(time.Minute)}, Signature: []byte("sig-b")},
|
|
}
|
|
verify := func([]byte, []byte) bool { return true }
|
|
if err := SaveVerifiedAssignmentRoster(ctx, db, assignment, roster[:1], verify); err == nil {
|
|
t.Fatal("partial signed roster was accepted")
|
|
}
|
|
var count int
|
|
if err := db.QueryRow(`SELECT count(*) FROM assignments WHERE match_id = 'roster-match'`).Scan(&count); err != nil || count != 0 {
|
|
t.Fatalf("partial roster persisted rows=%d err=%v", count, err)
|
|
}
|
|
if err := SaveVerifiedAssignmentRoster(ctx, db, assignment, roster, verify); err != nil {
|
|
t.Fatalf("complete durable roster rejected: %v", err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM assignments WHERE match_id = 'roster-match'`).Scan(&count); err != nil || count != 2 {
|
|
t.Fatalf("complete roster rows=%d err=%v", count, err)
|
|
}
|
|
forged := append([]domain.SignedJoinAuthorisation(nil), roster...)
|
|
forged[1].Authorisation.SteamID = "steam-other"
|
|
if err := SaveVerifiedAssignmentRoster(ctx, db, assignment, forged, verify); err == nil {
|
|
t.Fatal("signed roster with wrong durable Steam identity was accepted")
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLConnectionReceiptsStartCompleteRelaxedCasualRoster(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
|
|
for i := 0; i < 2; i++ {
|
|
playerID := fmt.Sprintf("connect-player-%d", i)
|
|
ticketID := fmt.Sprintf("connect-ticket-%d", i)
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, playerID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'ASSIGNMENT_READY', 'integration-build', 1, $3, $4)`, ticketID, playerID, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO game_servers (server_id, region, build, protocol_version, transport, state) VALUES ('connect-server', 'EU', 'integration-build', 1, 'enet', 'ALLOCATED')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version) VALUES ('connect-match', 'casual', 'ALLOCATING', 'EU', 1)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO allocations (allocation_id, match_id, server_id, region, build, protocol_version, transport, request_digest, state, allocated_at) VALUES ('connect-allocation', 'connect-match', 'connect-server', 'EU', 'integration-build', 1, 'enet', $1, 'ALLOCATED', $2)`, []byte("request"), now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `UPDATE matches SET state = 'ASSIGNMENT_READY', server_id = 'connect-server', allocation_id = 'connect-allocation', allocation_claimed_at = $1, initial_connect_ready_at = $1 WHERE match_id = 'connect-match'`, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 2; i++ {
|
|
playerID := fmt.Sprintf("connect-player-%d", i)
|
|
ticketID := fmt.Sprintf("connect-ticket-%d", i)
|
|
slot := i * 3
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) VALUES ('connect-match', $1, $2, $3, $4)`, playerID, ticketID, slot, i); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO assignments (match_id, player_id, allocation_id, server_id, slot, region, client_build, protocol_version, transport, endpoint, join_authorisation, manifest_digest, expires_at) VALUES ('connect-match', $1, 'connect-allocation', 'connect-server', $2, 'EU', 'integration-build', 1, 'enet', '127.0.0.1:7777', 'join-token', $3, $4)`, playerID, slot, []byte("manifest"), now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
binding := domain.WorkloadBinding{AllocationID: "connect-allocation", MatchID: "connect-match", ServerID: "connect-server"}
|
|
if generation, err := ClaimPlayerConnection(ctx, db, binding, "connect-player-0", 0, "connect-receipt-key-0000", now); err != nil || generation != 1 {
|
|
t.Fatalf("first receipt: %v", err)
|
|
}
|
|
if _, err := ClaimPlayerConnection(ctx, db, domain.WorkloadBinding{AllocationID: "forged-allocation", MatchID: "connect-match", ServerID: "connect-server"}, "connect-player-1", 0, "connect-receipt-key-forged", now); !errors.Is(err, domain.ErrConflict) {
|
|
t.Fatalf("forged binding err=%v, want conflict", err)
|
|
}
|
|
if generation, err := ClaimPlayerConnection(ctx, db, binding, "connect-player-1", 0, "connect-receipt-key-0001", now); err != nil || generation != 1 {
|
|
t.Fatalf("second receipt: %v", err)
|
|
}
|
|
reconciled, err := ReconcileInitialConnect(ctx, db, now.Add(time.Second), 10)
|
|
if err != nil || reconciled != 1 {
|
|
t.Fatalf("reconcile count=%d err=%v", reconciled, err)
|
|
}
|
|
var state string
|
|
if err := db.QueryRowContext(ctx, `SELECT state FROM matches WHERE match_id = 'connect-match'`).Scan(&state); err != nil || state != string(domain.Live) {
|
|
t.Fatalf("match state=%q err=%v", state, err)
|
|
}
|
|
var liveTickets int
|
|
if err := db.QueryRowContext(ctx, `SELECT count(*) FROM queue_tickets WHERE state = 'LIVE'`).Scan(&liveTickets); err != nil || liveTickets != 2 {
|
|
t.Fatalf("live tickets=%d err=%v", liveTickets, err)
|
|
}
|
|
// A lost 204 can be retried after assignment expiry because the exact
|
|
// durable receipt is replayed before checking the now-expired assignment.
|
|
if generation, err := ClaimPlayerConnection(ctx, db, binding, "connect-player-0", 0, "connect-receipt-key-0000", now.Add(2*time.Minute)); err != nil || generation != 1 {
|
|
t.Fatalf("durable receipt replay: %v", err)
|
|
}
|
|
if _, err := ClaimPlayerConnection(ctx, db, binding, "connect-player-0", 1, "connect-active-duplicate", now.Add(2*time.Second)); !errors.Is(err, domain.ErrConflict) {
|
|
t.Fatalf("active duplicate err=%v, want conflict", err)
|
|
}
|
|
if err := RecordPlayerDisconnected(ctx, db, binding, "connect-player-0", 1, "disconnect-receipt-0000", now.Add(3*time.Second)); err != nil {
|
|
t.Fatalf("disconnect receipt: %v", err)
|
|
}
|
|
if _, err := ClaimPlayerConnection(ctx, db, binding, "connect-player-0", 0, "connect-receipt-key-0000", now.Add(4*time.Second)); !errors.Is(err, domain.ErrConflict) {
|
|
t.Fatalf("stale connect replay err=%v, want conflict", err)
|
|
}
|
|
if generation, err := ClaimPlayerConnection(ctx, db, binding, "connect-player-0", 0, "reconnect-receipt-0000", now.Add(63*time.Second)); err != nil || generation != 2 {
|
|
t.Fatalf("grace-boundary reconnect generation=%d err=%v", generation, err)
|
|
}
|
|
if err := RecordPlayerDisconnected(ctx, db, binding, "connect-player-0", 1, "stale-disconnect-0000", now.Add(64*time.Second)); !errors.Is(err, domain.ErrConflict) {
|
|
t.Fatalf("stale disconnect err=%v, want conflict", err)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLLiveReconnectGraceExpiryPersistsAbandonmentWithoutReleasingResultRoster(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
ctx := context.Background()
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
for _, playerID := range []string{"live-abandon-player", "live-present-player"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, playerID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'ranked', 'LIVE', 'integration-build', 1, $3, $4)`, "live-abandon-ticket-"+playerID, playerID, now, now.Add(time.Hour)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, arena_path) VALUES ('live-abandon-match', 'ranked', 'LIVE', 'EU', 1, 'live-abandon-server', 'res://scenes/arena_01.tscn')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team, connection_generation, connected_at, disconnected_at) VALUES
|
|
('live-abandon-match', 'live-abandon-player', 'live-abandon-ticket-live-abandon-player', 0, 0, 1, $1, $2),
|
|
('live-abandon-match', 'live-present-player', 'live-abandon-ticket-live-present-player', 3, 1, 1, $1, NULL)`, now.Add(-2*time.Minute), now.Add(-domain.RankedReconnectGrace-time.Nanosecond)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
reconciled, err := ReconcileLiveAbandonments(ctx, db, now, 10)
|
|
if err != nil || reconciled != 1 {
|
|
t.Fatalf("reconciled=%d err=%v", reconciled, err)
|
|
}
|
|
var active bool
|
|
var abandonedAt sql.NullTime
|
|
if err := db.QueryRowContext(ctx, `SELECT participation_active, abandoned_at FROM match_participants WHERE match_id = 'live-abandon-match' AND player_id = 'live-abandon-player'`).Scan(&active, &abandonedAt); err != nil || !active || !abandonedAt.Valid || !abandonedAt.Time.Equal(now) {
|
|
t.Fatalf("participant active=%t abandoned=%v err=%v", active, abandonedAt, err)
|
|
}
|
|
var ticketState string
|
|
if err := db.QueryRowContext(ctx, `SELECT state FROM queue_tickets WHERE ticket_id = 'live-abandon-ticket-live-abandon-player'`).Scan(&ticketState); err != nil || ticketState != "LIVE" {
|
|
t.Fatalf("ticket state=%q err=%v", ticketState, err)
|
|
}
|
|
var endsAt time.Time
|
|
if err := db.QueryRowContext(ctx, `SELECT ends_at FROM penalties WHERE player_id = 'live-abandon-player' AND kind = 'MATCH_ABANDONED'`).Scan(&endsAt); err != nil || !endsAt.Equal(now.Add(5*time.Minute)) {
|
|
t.Fatalf("penalty ends=%v err=%v", endsAt, err)
|
|
}
|
|
var outboxCount, revision int
|
|
if err := db.QueryRowContext(ctx, `SELECT count(*) FROM outbox WHERE aggregate_id = 'live-abandon-match' AND event_type = 'state_changed'`).Scan(&outboxCount); err != nil || outboxCount != 1 {
|
|
t.Fatalf("outbox=%d err=%v", outboxCount, err)
|
|
}
|
|
if err := db.QueryRowContext(ctx, `SELECT revision FROM matches WHERE match_id = 'live-abandon-match'`).Scan(&revision); err != nil || revision != 1 {
|
|
t.Fatalf("revision=%d err=%v", revision, err)
|
|
}
|
|
if reconciled, err = ReconcileLiveAbandonments(ctx, db, now.Add(time.Minute), 10); err != nil || reconciled != 0 {
|
|
t.Fatalf("replay reconciled=%d err=%v", reconciled, err)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLProposalClaimAndResponseAreAtomic(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"proposal-player-a", "proposal-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for i, player := range []string{"proposal-player-a", "proposal-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'QUEUED', 'integration-build', 1, $3, $4)`, fmt.Sprintf("proposal-ticket-%d", i), player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
proposal, err := domain.NewProposal("proposal-integration", domain.Casual, []string{"proposal-player-a", "proposal-player-b"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
proposal.Region = "EU"
|
|
proposal.Protocol = 1
|
|
for index := range proposal.Participants {
|
|
if proposal.Participants[index].PlayerID == "proposal-player-a" {
|
|
proposal.Participants[index].Team = 0
|
|
proposal.Participants[index].Slot = 0
|
|
} else {
|
|
proposal.Participants[index].Team = 1
|
|
proposal.Participants[index].Slot = 3
|
|
}
|
|
}
|
|
if err := CreateProposal(ctx, db, proposal, map[string]string{"proposal-player-a": "proposal-ticket-0", "proposal-player-b": "proposal-ticket-1"}, now); err != nil {
|
|
t.Fatalf("create proposal: %v", err)
|
|
}
|
|
var proposed int
|
|
if err := db.QueryRow(`SELECT count(*) FROM queue_tickets WHERE state = 'PROPOSED'`).Scan(&proposed); err != nil || proposed != 2 {
|
|
t.Fatalf("proposed queue tickets = %d, err = %v", proposed, err)
|
|
}
|
|
recoveredTicket, err := GetQueueTicket(ctx, db, "proposal-player-a", "proposal-ticket-0", now)
|
|
if err != nil || recoveredTicket.ProposalID != proposal.ProposalID {
|
|
t.Fatalf("recovered ticket proposal=%q err=%v", recoveredTicket.ProposalID, err)
|
|
}
|
|
recovered, err := GetProposal(ctx, db, "proposal-player-a", proposal.ProposalID, now)
|
|
if err != nil {
|
|
t.Fatalf("recover proposal: %v", err)
|
|
}
|
|
if len(recovered.Participants) != 2 || recovered.Revision != 0 {
|
|
t.Fatalf("unexpected recovered proposal: %+v", recovered)
|
|
}
|
|
accepted, err := RespondToProposal(ctx, db, "proposal-player-a", proposal.ProposalID, "proposal-response-a-0001", true, 0, now)
|
|
if err != nil {
|
|
t.Fatalf("first proposal acceptance: %v", err)
|
|
}
|
|
if accepted.Revision != 1 || accepted.State != domain.Open {
|
|
t.Fatalf("unexpected first acceptance: %+v", accepted)
|
|
}
|
|
accepted, err = RespondToProposal(ctx, db, "proposal-player-b", proposal.ProposalID, "proposal-response-b-0001", true, 1, now)
|
|
if err != nil {
|
|
t.Fatalf("second proposal acceptance: %v", err)
|
|
}
|
|
if accepted.State != domain.Accepted || accepted.Revision != 2 {
|
|
t.Fatalf("proposal did not close after unanimous acceptance: %+v", accepted)
|
|
}
|
|
// The API's post-commit recovery promoter must accept the nullable casual
|
|
// arena path left by the atomic response transaction and converge on the
|
|
// already-created match.
|
|
if err := PromoteStoredAcceptedProposal(ctx, db, proposal.ProposalID, now.Add(time.Second)); err != nil {
|
|
t.Fatalf("replay persisted casual promotion: %v", err)
|
|
}
|
|
var matchState string
|
|
if err := db.QueryRow(`SELECT state FROM matches WHERE match_id = 'match-proposal-integration'`).Scan(&matchState); err != nil {
|
|
t.Fatalf("atomic accepted match: %v", err)
|
|
}
|
|
var acceptedTickets, matchPlayers int
|
|
if err := db.QueryRow(`SELECT count(*) FROM queue_tickets WHERE ticket_id LIKE 'proposal-ticket-%' AND state = 'ACCEPTED'`).Scan(&acceptedTickets); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM match_participants WHERE match_id = 'match-proposal-integration'`).Scan(&matchPlayers); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if matchState != "ALLOCATING" || acceptedTickets != 2 || matchPlayers != 2 {
|
|
t.Fatalf("acceptance did not atomically materialize match: state=%s tickets=%d players=%d", matchState, acceptedTickets, matchPlayers)
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLProposalDeclineCancelsOffenderAndRequeuesInnocent protects the
|
|
// durable decline boundary: the offender's ticket becomes terminal while
|
|
// every innocent ticket keeps its original queue precedence.
|
|
func TestPostgreSQLProposalDeclineCancelsOffenderAndRequeuesInnocent(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"decline-player-a", "decline-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for i, player := range []string{"decline-player-a", "decline-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'QUEUED', 'integration-build', 1, $3, $4)`, fmt.Sprintf("decline-ticket-%d", i), player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
proposal, err := domain.NewProposal("decline-proposal", domain.Casual, []string{"decline-player-a", "decline-player-b"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := CreateProposal(ctx, db, proposal, map[string]string{"decline-player-a": "decline-ticket-0", "decline-player-b": "decline-ticket-1"}, now); err != nil {
|
|
t.Fatalf("create proposal: %v", err)
|
|
}
|
|
|
|
// player-a declines; player-b never responded at all -- the bug affects
|
|
// even a participant who was never asked to do anything wrong.
|
|
declined, err := RespondToProposal(ctx, db, "decline-player-a", proposal.ProposalID, "decline-response-a-0001", false, 0, now)
|
|
if err != nil {
|
|
t.Fatalf("decline: %v", err)
|
|
}
|
|
if declined.State != domain.Declined {
|
|
t.Fatalf("proposal did not close on decline: %+v", declined)
|
|
}
|
|
|
|
var stateA, stateB string
|
|
var expiresB time.Time
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'decline-ticket-0'`).Scan(&stateA); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT state, expires_at FROM queue_tickets WHERE ticket_id = 'decline-ticket-1'`).Scan(&stateB, &expiresB); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if stateA != "CANCELLED" {
|
|
t.Fatalf("decliner's own ticket state = %s, want CANCELLED", stateA)
|
|
}
|
|
if stateB != "QUEUED" {
|
|
t.Fatalf("uninvolved participant's ticket state = %s, want QUEUED -- they must not be stranded by someone else's decline", stateB)
|
|
}
|
|
if !expiresB.After(now) {
|
|
t.Fatalf("requeued ticket expiry %v was not refreshed forward from %v", expiresB, now)
|
|
}
|
|
|
|
// Only the innocent player can be selected again. A durable cooldown also
|
|
// rejects a new ticket from the decliner until the policy window ends.
|
|
candidates, err := ListQueuedCandidates(ctx, db, domain.Casual, now, 10)
|
|
if err != nil {
|
|
t.Fatalf("list queued candidates: %v", err)
|
|
}
|
|
found := map[string]bool{}
|
|
for _, candidate := range candidates {
|
|
found[candidate.PlayerID] = true
|
|
}
|
|
if found["decline-player-a"] || !found["decline-player-b"] {
|
|
t.Fatalf("matcher did not isolate offender from innocent: %+v", candidates)
|
|
}
|
|
var cooldownEnd time.Time
|
|
if err := db.QueryRow(`SELECT ends_at FROM penalties WHERE player_id = 'decline-player-a' AND kind = 'PROPOSAL_DECLINED'`).Scan(&cooldownEnd); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if want := now.Add(30 * time.Second); !cooldownEnd.Equal(want) {
|
|
t.Fatalf("decline cooldown end = %v, want %v", cooldownEnd, want)
|
|
}
|
|
// Recovering the closed proposal after its old deadline must not convert
|
|
// the innocent participant's PENDING response into a timeout penalty.
|
|
if _, err := GetProposal(ctx, db, "decline-player-b", proposal.ProposalID, now.Add(domain.ProposalWindow+time.Second)); err != nil {
|
|
t.Fatalf("recover declined proposal: %v", err)
|
|
}
|
|
var innocentTimeouts int
|
|
if err := db.QueryRow(`SELECT count(*) FROM penalties WHERE player_id = 'decline-player-b' AND kind = 'PROPOSAL_TIMEOUT'`).Scan(&innocentTimeouts); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if innocentTimeouts != 0 {
|
|
t.Fatalf("innocent participant received %d timeout penalties after decline", innocentTimeouts)
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLProposalTimeoutExpiresOffenderAndRequeuesAccepted protects the
|
|
// timeout sibling: accepted participants retain precedence, while no-shows
|
|
// receive a terminal ticket and cooldown.
|
|
func TestPostgreSQLProposalTimeoutExpiresOffenderAndRequeuesAccepted(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"timeout-player-a", "timeout-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for i, player := range []string{"timeout-player-a", "timeout-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'QUEUED', 'integration-build', 1, $3, $4)`, fmt.Sprintf("timeout-ticket-%d", i), player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
proposal, err := domain.NewProposal("timeout-proposal", domain.Casual, []string{"timeout-player-a", "timeout-player-b"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := CreateProposal(ctx, db, proposal, map[string]string{"timeout-player-a": "timeout-ticket-0", "timeout-player-b": "timeout-ticket-1"}, now); err != nil {
|
|
t.Fatalf("create proposal: %v", err)
|
|
}
|
|
if _, err := RespondToProposal(ctx, db, "timeout-player-a", proposal.ProposalID, "timeout-accept-a-0001", true, 0, now.Add(time.Second)); err != nil {
|
|
t.Fatalf("accept proposal: %v", err)
|
|
}
|
|
|
|
// player-b never responds; recover well after the response window.
|
|
afterExpiry := now.Add(domain.ProposalWindow + time.Second)
|
|
recovered, err := GetProposal(ctx, db, "timeout-player-a", proposal.ProposalID, afterExpiry)
|
|
if err != nil {
|
|
t.Fatalf("recover expired proposal: %v", err)
|
|
}
|
|
if recovered.State != domain.Expired {
|
|
t.Fatalf("proposal did not expire: %+v", recovered)
|
|
}
|
|
|
|
var stateA, stateB string
|
|
var expiresA time.Time
|
|
if err := db.QueryRow(`SELECT state, expires_at FROM queue_tickets WHERE ticket_id = 'timeout-ticket-0'`).Scan(&stateA, &expiresA); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'timeout-ticket-1'`).Scan(&stateB); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if stateA != "QUEUED" || stateB != "EXPIRED" {
|
|
t.Fatalf("timeout did not split accepted and offender tickets: a=%s b=%s", stateA, stateB)
|
|
}
|
|
if !expiresA.After(afterExpiry) {
|
|
t.Fatalf("requeued ticket expiry %v was not refreshed forward from %v", expiresA, afterExpiry)
|
|
}
|
|
candidates, err := ListQueuedCandidates(ctx, db, domain.Casual, afterExpiry, 10)
|
|
if err != nil {
|
|
t.Fatalf("list queued candidates: %v", err)
|
|
}
|
|
found := map[string]bool{}
|
|
for _, candidate := range candidates {
|
|
found[candidate.PlayerID] = true
|
|
}
|
|
if !found["timeout-player-a"] || found["timeout-player-b"] {
|
|
t.Fatalf("matcher did not isolate timeout offender: %+v", candidates)
|
|
}
|
|
var cooldownEnd time.Time
|
|
if err := db.QueryRow(`SELECT ends_at FROM penalties WHERE player_id = 'timeout-player-b' AND kind = 'PROPOSAL_TIMEOUT'`).Scan(&cooldownEnd); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if want := afterExpiry.Add(60 * time.Second); !cooldownEnd.Equal(want) {
|
|
t.Fatalf("timeout cooldown end = %v, want %v", cooldownEnd, want)
|
|
}
|
|
}
|
|
|
|
// A late response must report a closed proposal only after committing the
|
|
// expiry recovery. Returning that domain error from inside RunSerializable
|
|
// used to roll every recovery write back.
|
|
func TestPostgreSQLLateProposalResponseCommitsExpiryRecovery(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"late-player-a", "late-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for i, player := range []string{"late-player-a", "late-player-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'QUEUED', 'integration-build', 1, $3, $4)`, fmt.Sprintf("late-ticket-%d", i), player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
proposal, err := domain.NewProposal("late-proposal", domain.Casual, []string{"late-player-a", "late-player-b"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := CreateProposal(ctx, db, proposal, map[string]string{"late-player-a": "late-ticket-0", "late-player-b": "late-ticket-1"}, now); err != nil {
|
|
t.Fatalf("create proposal: %v", err)
|
|
}
|
|
|
|
late := now.Add(domain.ProposalWindow + time.Second)
|
|
_, err = RespondToProposal(ctx, db, "late-player-a", proposal.ProposalID, "late-response-a-0001", true, 0, late)
|
|
if !errors.Is(err, domain.ErrProposalClosed) {
|
|
t.Fatalf("late response error = %v, want ErrProposalClosed", err)
|
|
}
|
|
var proposalState, ticketA, ticketB string
|
|
if err := db.QueryRow(`SELECT state FROM proposals WHERE proposal_id = 'late-proposal'`).Scan(&proposalState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'late-ticket-0'`).Scan(&ticketA); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'late-ticket-1'`).Scan(&ticketB); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if proposalState != "EXPIRED" || ticketA != "EXPIRED" || ticketB != "EXPIRED" {
|
|
t.Fatalf("late recovery was not committed: proposal=%s tickets=%s,%s", proposalState, ticketA, ticketB)
|
|
}
|
|
var penalties, idempotencyRows int
|
|
if err := db.QueryRow(`SELECT count(*) FROM penalties WHERE kind = 'PROPOSAL_TIMEOUT' AND player_id IN ('late-player-a', 'late-player-b')`).Scan(&penalties); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM idempotency_keys WHERE scope = $1 AND idempotency_key = 'late-response-a-0001'`, ProposalResponseIdempotencyScope).Scan(&idempotencyRows); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if penalties != 2 || idempotencyRows != 0 {
|
|
t.Fatalf("late recovery side effects: penalties=%d idempotency_rows=%d", penalties, idempotencyRows)
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLConcurrentProposalExpiryRecoveryAppliesCooldownsExactlyOnce
|
|
// covers the race §8.46 flagged as still open: multiple concurrent recovery
|
|
// paths (a read-side GetProposal from each participant polling for an
|
|
// update, and a RespondToProposal arriving right at the same boundary) can
|
|
// all observe the same past-expiry proposal simultaneously. Every one of
|
|
// them runs the identical expiry-advance SQL in its own transaction, so this
|
|
// proves that racing recovery does not multiply the durable side effects: a
|
|
// PROPOSAL_TIMEOUT cooldown must land exactly once per offending player, not
|
|
// once per racing transaction that happened to perform the PENDING ->
|
|
// TIMED_OUT flip. The design's own defense is that ProposalParticipantExpireSQL
|
|
// only ever flips a still-PENDING row once, and recordProposalTimeoutCooldowns
|
|
// only cooldowns participants whose responded_at equals this transaction's
|
|
// own `now` -- so a loser transaction's `now` simply matches nothing.
|
|
func TestPostgreSQLConcurrentProposalExpiryRecoveryAppliesCooldownsExactlyOnce(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"race-expiry-a", "race-expiry-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for i, player := range []string{"race-expiry-a", "race-expiry-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'QUEUED', 'integration-build', 1, $3, $4)`, fmt.Sprintf("race-expiry-ticket-%d", i), player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
proposal, err := domain.NewProposal("race-expiry-proposal", domain.Casual, []string{"race-expiry-a", "race-expiry-b"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := CreateProposal(ctx, db, proposal, map[string]string{"race-expiry-a": "race-expiry-ticket-0", "race-expiry-b": "race-expiry-ticket-1"}, now); err != nil {
|
|
t.Fatalf("create proposal: %v", err)
|
|
}
|
|
|
|
late := now.Add(domain.ProposalWindow + time.Second)
|
|
const racers = 8
|
|
var wg sync.WaitGroup
|
|
errs := make([]error, racers)
|
|
wg.Add(racers)
|
|
for i := 0; i < racers; i++ {
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
// Each racer's `now` is distinct (and every one is past expiry), so a
|
|
// real implementation bug would show up as several of them believing
|
|
// they were the one that performed the PENDING -> TIMED_OUT flip.
|
|
racerNow := late.Add(time.Duration(i) * time.Millisecond)
|
|
switch i % 3 {
|
|
case 0:
|
|
_, errs[i] = GetProposal(ctx, db, "race-expiry-a", proposal.ProposalID, racerNow)
|
|
case 1:
|
|
_, errs[i] = GetProposal(ctx, db, "race-expiry-b", proposal.ProposalID, racerNow)
|
|
default:
|
|
_, errs[i] = RespondToProposal(ctx, db, "race-expiry-a", proposal.ProposalID, fmt.Sprintf("race-expiry-key-%04d", i), true, 0, racerNow)
|
|
}
|
|
}(i)
|
|
}
|
|
wg.Wait()
|
|
for i, err := range errs {
|
|
// GetProposal never errors on an already-expired proposal (it's a pure
|
|
// read-with-recovery); RespondToProposal on an already-closed proposal
|
|
// must report exactly ErrProposalClosed, nothing else.
|
|
if err != nil && !errors.Is(err, domain.ErrProposalClosed) {
|
|
t.Fatalf("racer %d: unexpected error %v", i, err)
|
|
}
|
|
}
|
|
|
|
var proposalState string
|
|
if err := db.QueryRow(`SELECT state FROM proposals WHERE proposal_id = 'race-expiry-proposal'`).Scan(&proposalState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if proposalState != "EXPIRED" {
|
|
t.Fatalf("proposal state = %s, want EXPIRED", proposalState)
|
|
}
|
|
var ticketA, ticketB string
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'race-expiry-ticket-0'`).Scan(&ticketA); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'race-expiry-ticket-1'`).Scan(&ticketB); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if ticketA != "EXPIRED" || ticketB != "EXPIRED" {
|
|
t.Fatalf("tickets not expired exactly once: a=%s b=%s", ticketA, ticketB)
|
|
}
|
|
// The crux of the race: exactly one PROPOSAL_TIMEOUT penalty per player,
|
|
// however many transactions raced to observe the expiry.
|
|
var penaltiesA, penaltiesB int
|
|
if err := db.QueryRow(`SELECT count(*) FROM penalties WHERE kind = 'PROPOSAL_TIMEOUT' AND player_id = 'race-expiry-a'`).Scan(&penaltiesA); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM penalties WHERE kind = 'PROPOSAL_TIMEOUT' AND player_id = 'race-expiry-b'`).Scan(&penaltiesB); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if penaltiesA != 1 || penaltiesB != 1 {
|
|
t.Fatalf("cooldown was not applied exactly once per player: a=%d b=%d", penaltiesA, penaltiesB)
|
|
}
|
|
var idempotencyRows int
|
|
if err := db.QueryRow(`SELECT count(*) FROM idempotency_keys WHERE scope = $1`, ProposalResponseIdempotencyScope).Scan(&idempotencyRows); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if idempotencyRows != 0 {
|
|
t.Fatalf("closed-proposal responses left stray idempotency rows: %d", idempotencyRows)
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLCancellingAProposedTicketImmediatelyRequeuesTheOtherParticipant
|
|
// covers the responsiveness gap the decline/timeout fixes above left bounded
|
|
// but not closed: cancelling a ticket that's part of an OPEN proposal used
|
|
// to leave the OTHER participant waiting out the full 10s window for
|
|
// something the system already knew couldn't happen (their proposal partner
|
|
// just walked away). CascadeCancelToOpenProposal declines and requeues that
|
|
// proposal in the same transaction as the cancel itself.
|
|
func TestPostgreSQLCancellingAProposedTicketImmediatelyRequeuesTheOtherParticipant(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"cancel-cascade-a", "cancel-cascade-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for i, player := range []string{"cancel-cascade-a", "cancel-cascade-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'QUEUED', 'integration-build', 1, $3, $4)`, fmt.Sprintf("cancel-cascade-ticket-%d", i), player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
proposal, err := domain.NewProposal("cancel-cascade-proposal", domain.Casual, []string{"cancel-cascade-a", "cancel-cascade-b"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := CreateProposal(ctx, db, proposal, map[string]string{"cancel-cascade-a": "cancel-cascade-ticket-0", "cancel-cascade-b": "cancel-cascade-ticket-1"}, now); err != nil {
|
|
t.Fatalf("create proposal: %v", err)
|
|
}
|
|
|
|
// player-a cancels their own ticket directly, well within the response
|
|
// window -- not a decline, not a timeout, just abandoning the queue.
|
|
// CreateProposal's own QueueTicketProposeSQL already bumped the ticket's
|
|
// revision from 0 to 1, so the cancel's expected revision is 1, not 0.
|
|
cancelled, err := CancelQueueTicket(ctx, db, "cancel-cascade-a", "cancel-cascade-ticket-0", "cancel-cascade-key-0001", 1, now.Add(time.Second))
|
|
if err != nil {
|
|
t.Fatalf("cancel: %v", err)
|
|
}
|
|
if cancelled.State != domain.Cancelled {
|
|
t.Fatalf("ticket did not cancel: %+v", cancelled)
|
|
}
|
|
|
|
var proposalState string
|
|
if err := db.QueryRow(`SELECT state FROM proposals WHERE proposal_id = 'cancel-cascade-proposal'`).Scan(&proposalState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if proposalState != "DECLINED" {
|
|
t.Fatalf("proposal state = %s, want DECLINED immediately, not left OPEN to time out", proposalState)
|
|
}
|
|
var stateB string
|
|
var expiresB time.Time
|
|
if err := db.QueryRow(`SELECT state, expires_at FROM queue_tickets WHERE ticket_id = 'cancel-cascade-ticket-1'`).Scan(&stateB, &expiresB); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if stateB != "QUEUED" {
|
|
t.Fatalf("other participant's ticket state = %s, want QUEUED immediately", stateB)
|
|
}
|
|
if !expiresB.After(now.Add(time.Second)) {
|
|
t.Fatalf("requeued ticket expiry %v was not refreshed forward", expiresB)
|
|
}
|
|
// The cancelling player's own ticket must stay CANCELLED, not get swept
|
|
// back up into the requeue meant for the other participant.
|
|
var stateA string
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'cancel-cascade-ticket-0'`).Scan(&stateA); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if stateA != "CANCELLED" {
|
|
t.Fatalf("cancelling player's own ticket state = %s, want it to stay CANCELLED", stateA)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLProposalCreationRollsBackPartialClaims(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('rollback-player', 'rollback-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ('rollback-ticket', 'rollback-player', 'casual', 'QUEUED', 'integration-build', 1, $1, $2)`, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
proposal, err := domain.NewProposal("rollback-proposal", domain.Casual, []string{"rollback-player", "missing-player"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := CreateProposal(ctx, db, proposal, map[string]string{"rollback-player": "rollback-ticket"}, now); err == nil {
|
|
t.Fatal("proposal with missing ticket mapping was accepted")
|
|
}
|
|
var proposals, participants, proposed int
|
|
if err := db.QueryRow(`SELECT count(*) FROM proposals WHERE proposal_id = 'rollback-proposal'`).Scan(&proposals); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM proposal_participants WHERE proposal_id = 'rollback-proposal'`).Scan(&participants); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM queue_tickets WHERE ticket_id = 'rollback-ticket' AND state = 'PROPOSED'`).Scan(&proposed); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if proposals != 0 || participants != 0 || proposed != 0 {
|
|
t.Fatalf("partial proposal claim was not rolled back: proposals=%d participants=%d proposed=%d", proposals, participants, proposed)
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLConcurrentProposalCreationClaimsContestedTicketOnce is the
|
|
// live counterpart to TestPostgreSQLProposalCreationRollsBackPartialClaims:
|
|
// every other proposal test in this file (and the whole matcher/allocator
|
|
// suite) runs its transactions strictly one at a time, so none of them can
|
|
// actually exercise the SERIALIZABLE retry-and-fence path CreateProposal
|
|
// relies on -- only two goroutines racing a real connection pool can. Two
|
|
// matchers independently form a proposal that both include the same waiting
|
|
// player's ticket (a real scenario: nothing stops two matcher replicas from
|
|
// reading the same QUEUED ticket in the same poll window); exactly one
|
|
// CreateProposal must win, the other must fail with its whole transaction
|
|
// rolled back, not a database/sql panic, deadlock, or a half-inserted row.
|
|
func TestPostgreSQLConcurrentProposalCreationClaimsContestedTicketOnce(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"race-player-a", "race-player-b", "race-player-c"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
tickets := map[string]string{"race-player-a": "race-ticket-a", "race-player-b": "race-ticket-b", "race-player-c": "race-ticket-c"}
|
|
for player, ticket := range tickets {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', 'QUEUED', 'integration-build', 1, $3, $4)`, ticket, player, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
proposalA, err := domain.NewProposal("race-proposal-a", domain.Casual, []string{"race-player-a", "race-player-b"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
proposalB, err := domain.NewProposal("race-proposal-b", domain.Casual, []string{"race-player-b", "race-player-c"}, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
errs := make([]error, 2)
|
|
wg.Add(2)
|
|
go func() {
|
|
defer wg.Done()
|
|
errs[0] = CreateProposal(ctx, db, proposalA, map[string]string{"race-player-a": tickets["race-player-a"], "race-player-b": tickets["race-player-b"]}, now)
|
|
}()
|
|
go func() {
|
|
defer wg.Done()
|
|
errs[1] = CreateProposal(ctx, db, proposalB, map[string]string{"race-player-b": tickets["race-player-b"], "race-player-c": tickets["race-player-c"]}, now)
|
|
}()
|
|
wg.Wait()
|
|
|
|
succeeded := errs[0] == nil
|
|
if succeeded == (errs[1] == nil) {
|
|
t.Fatalf("exactly one contested proposal must win, got errA=%v errB=%v", errs[0], errs[1])
|
|
}
|
|
|
|
winner, loser := "race-proposal-a", "race-proposal-b"
|
|
if !succeeded {
|
|
winner, loser = "race-proposal-b", "race-proposal-a"
|
|
}
|
|
var winnerRows, loserRows, loserParticipants int
|
|
if err := db.QueryRow(`SELECT count(*) FROM proposals WHERE proposal_id = $1`, winner).Scan(&winnerRows); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM proposals WHERE proposal_id = $1`, loser).Scan(&loserRows); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM proposal_participants WHERE proposal_id = $1`, loser).Scan(&loserParticipants); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if winnerRows != 1 {
|
|
t.Fatalf("winning proposal %s was not persisted", winner)
|
|
}
|
|
if loserRows != 0 || loserParticipants != 0 {
|
|
t.Fatalf("losing proposal %s was not fully rolled back: proposals=%d participants=%d", loser, loserRows, loserParticipants)
|
|
}
|
|
var contestedState string
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = $1`, tickets["race-player-b"]).Scan(&contestedState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if contestedState != "PROPOSED" {
|
|
t.Fatalf("contested ticket should be claimed by the winner, got state=%s", contestedState)
|
|
}
|
|
// The loser's OWN uncontested ticket (a or c) must have rolled back to
|
|
// QUEUED too -- CreateProposal is one transaction per proposal, so a
|
|
// contested loss on one participant must not leave another participant's
|
|
// ticket stranded as PROPOSED with no surviving proposal to reference it.
|
|
// A (player-a + contested player-b) won iff succeeded, in which case B's
|
|
// own uncontested ticket (player-c) is the one that must have rolled
|
|
// back; if A lost, it's A's own uncontested ticket (player-a) instead.
|
|
loserOnlyTicket := tickets["race-player-c"]
|
|
if !succeeded {
|
|
loserOnlyTicket = tickets["race-player-a"]
|
|
}
|
|
var loserOnlyState string
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = $1`, loserOnlyTicket).Scan(&loserOnlyState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if loserOnlyState != "QUEUED" {
|
|
t.Fatalf("loser's uncontested ticket %s should have rolled back to QUEUED, got %s", loserOnlyTicket, loserOnlyState)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLResultCompletionAndOutboxAreAtomicAndReplayable(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id) VALUES ('result-match', 'casual', 'RESULT_PENDING', 'NA', 1, 'result-server')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
payload := []byte(`{"match_id":"result-match","team0_score":2,"team1_score":1}`)
|
|
digest := sha256.Sum256(payload)
|
|
receipt := domain.ResultReceipt{ResultID: "result-receipt", MatchID: "result-match", ResultNonce: "result-nonce-123456", PayloadDigest: digest, IntegrityState: domain.IntegrityCertified, ReceivedAt: now}
|
|
if err := CompleteResult(ctx, db, receipt, "result-server", "result-event", payload, now); err != nil {
|
|
t.Fatalf("complete result: %v", err)
|
|
}
|
|
var state string
|
|
if err := db.QueryRow(`SELECT state FROM matches WHERE match_id = 'result-match'`).Scan(&state); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if state != "COMPLETED" {
|
|
t.Fatalf("result match state = %s", state)
|
|
}
|
|
events, err := ReadUnpublishedOutbox(ctx, db, 10)
|
|
if err != nil || len(events) != 1 || events[0].EventID != "result-event" {
|
|
t.Fatalf("unpublished result events = %+v, err = %v", events, err)
|
|
}
|
|
if err := MarkOutboxPublished(ctx, db, events[0].EventID, now.Add(time.Second)); err != nil {
|
|
t.Fatalf("ack result event: %v", err)
|
|
}
|
|
if remaining, err := ReadUnpublishedOutbox(ctx, db, 10); err != nil || len(remaining) != 0 {
|
|
t.Fatalf("outbox after ack = %+v, err = %v", remaining, err)
|
|
}
|
|
if err := CompleteResult(ctx, db, receipt, "result-server", "result-event-retry", payload, now.Add(time.Second)); err != nil {
|
|
t.Fatalf("identical completed result replay: %v", err)
|
|
}
|
|
conflict := receipt
|
|
conflict.ResultID = "different-result"
|
|
if err := CompleteResult(ctx, db, conflict, "result-server", "different-event", []byte(`{"conflict":true}`), now.Add(2*time.Second)); err == nil {
|
|
t.Fatal("conflicting completed result was accepted")
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLConcurrentConflictingResultSubmissionsKeepOneReceipt
|
|
// exercises the other side of the result race: retries with different payloads
|
|
// must not let the winner's durable receipt be overwritten or create a second
|
|
// completion event.
|
|
func TestPostgreSQLConcurrentConflictingResultSubmissionsKeepOneReceipt(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"result-conflict-a", "result-conflict-b"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id) VALUES ('result-conflict-match', 'casual', 'RESULT_PENDING', 'NA', 1, 'result-conflict-server')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ('result-conflict-ticket-a', 'result-conflict-a', 'casual', 'LIVE', 'build-1', 1, $1, $2), ('result-conflict-ticket-b', 'result-conflict-b', 'casual', 'LIVE', 'build-1', 1, $1, $2)`, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) VALUES ('result-conflict-match', 'result-conflict-a', 'result-conflict-ticket-a', 0, 0), ('result-conflict-match', 'result-conflict-b', 'result-conflict-ticket-b', 1, 1)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
base := domain.MatchResult{MatchID: "result-conflict-match", ServerID: "result-conflict-server", IntegrityState: domain.IntegritySuppressed}
|
|
results := []domain.MatchResult{
|
|
{MatchID: base.MatchID, ServerID: base.ServerID, ResultNonce: "result-conflict-nonce-a-123456", Team0Score: 2, Team1Score: 1, IntegrityState: base.IntegrityState},
|
|
{MatchID: base.MatchID, ServerID: base.ServerID, ResultNonce: "result-conflict-nonce-b-123456", Team0Score: 1, Team1Score: 2, IntegrityState: base.IntegrityState},
|
|
}
|
|
errs := make([]error, 2)
|
|
var wg sync.WaitGroup
|
|
wg.Add(2)
|
|
for i := range results {
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
digest := domain.ResultDigest(results[i])
|
|
receipt := domain.ResultReceipt{ResultID: fmt.Sprintf("result-conflict-receipt-%d", i), MatchID: results[i].MatchID, ResultNonce: results[i].ResultNonce, PayloadDigest: digest, IntegrityState: results[i].IntegrityState, ReceivedAt: now}
|
|
errs[i] = CompleteResult(ctx, db, receipt, results[i].ServerID, fmt.Sprintf("result-conflict-event-%d", i), []byte(fmt.Sprintf(`{"nonce":%q}`, results[i].ResultNonce)), now)
|
|
}(i)
|
|
}
|
|
wg.Wait()
|
|
wins := 0
|
|
for _, err := range errs {
|
|
if err == nil {
|
|
wins++
|
|
} else if !strings.Contains(err.Error(), "conflict") {
|
|
t.Fatalf("non-conflict error in conflicting race: %v", err)
|
|
}
|
|
}
|
|
if wins != 1 {
|
|
t.Fatalf("successful conflicting submissions = %d, want exactly one; errors=%v", wins, errs)
|
|
}
|
|
var receipts, events int
|
|
if err := db.QueryRow(`SELECT count(*) FROM result_receipts WHERE match_id = 'result-conflict-match'`).Scan(&receipts); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM outbox WHERE aggregate_id = 'result-conflict-match' AND event_type = 'match_completed'`).Scan(&events); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if receipts != 1 || events != 1 {
|
|
t.Fatalf("durable conflict race left receipts=%d events=%d, want one of each", receipts, events)
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLConcurrentIdenticalResultSubmissionAppliesRatingsExactlyOnce
|
|
// races real concurrent duplicate result submissions -- the scenario behind
|
|
// task 8.25's "identical duplicates idempotent" claim, which every other
|
|
// result test in this file (and the mocked-driver unit tests) only exercises
|
|
// sequentially. A game server can legitimately retry an unacknowledged
|
|
// result POST, and two such retries can land at PostgreSQL genuinely
|
|
// concurrently; every one of them must succeed (this is the identical-replay
|
|
// path, not a conflict), the match must complete exactly once, and -- the
|
|
// part that matters -- the rating update inside applyResultRatings must not
|
|
// run twice just because it raced.
|
|
func TestPostgreSQLConcurrentIdenticalResultSubmissionAppliesRatingsExactlyOnce(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
for _, player := range []string{"result-race-winner", "result-race-loser"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO ratings (player_id, rating, deviation, volatility, ranked_games) VALUES ($1, 1500, 350, 0.06, 0)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id) VALUES ('result-race-match', 'casual', 'LIVE', 'NA', 1, 'result-race-server')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ('result-race-ticket-w', 'result-race-winner', 'casual', 'LIVE', 'build-1', 1, $1, $2), ('result-race-ticket-l', 'result-race-loser', 'casual', 'LIVE', 'build-1', 1, $1, $2)`, now, now.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) VALUES ('result-race-match', 'result-race-winner', 'result-race-ticket-w', 0, 0), ('result-race-match', 'result-race-loser', 'result-race-ticket-l', 1, 1)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
result := domain.MatchResult{MatchID: "result-race-match", ServerID: "result-race-server", ResultNonce: "result-race-nonce-123456", Team0Score: 3, Team1Score: 1, IntegrityState: domain.IntegrityCertified}
|
|
digest := domain.ResultDigest(result)
|
|
receipt := domain.ResultReceipt{ResultID: "result-race-receipt", MatchID: result.MatchID, ResultNonce: result.ResultNonce, PayloadDigest: digest, IntegrityState: result.IntegrityState, ReceivedAt: now}
|
|
payload := []byte(`{"match_id":"result-race-match"}`)
|
|
|
|
const attempts = 5
|
|
var wg sync.WaitGroup
|
|
errs := make([]error, attempts)
|
|
wg.Add(attempts)
|
|
for i := 0; i < attempts; i++ {
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
errs[i] = CompleteResultWithResult(ctx, db, receipt, result.ServerID, fmt.Sprintf("result-race-event-%d", i), payload, result, now)
|
|
}(i)
|
|
}
|
|
wg.Wait()
|
|
for i, err := range errs {
|
|
if err != nil {
|
|
t.Fatalf("identical concurrent submission %d failed: %v", i, err)
|
|
}
|
|
}
|
|
|
|
var state string
|
|
if err := db.QueryRow(`SELECT state FROM matches WHERE match_id = 'result-race-match'`).Scan(&state); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if state != "COMPLETED" {
|
|
t.Fatalf("match state = %s, want COMPLETED", state)
|
|
}
|
|
var completedTickets int
|
|
if err := db.QueryRow(`SELECT count(*) FROM queue_tickets WHERE ticket_id IN ('result-race-ticket-w', 'result-race-ticket-l') AND state = 'COMPLETED'`).Scan(&completedTickets); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if completedTickets != 2 {
|
|
t.Fatalf("completed tickets = %d, want 2", completedTickets)
|
|
}
|
|
var winnerGames, loserGames int
|
|
var winnerRating, loserRating float64
|
|
if err := db.QueryRow(`SELECT ranked_games, rating FROM ratings WHERE player_id = 'result-race-winner'`).Scan(&winnerGames, &winnerRating); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT ranked_games, rating FROM ratings WHERE player_id = 'result-race-loser'`).Scan(&loserGames, &loserRating); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Casual results never increment ranked_games by design (rankedIncrement
|
|
// is unconditionally 0 for domain.Casual in applyResultRatings) -- that's
|
|
// not what this test is verifying. What proves "applied exactly once, not
|
|
// N times under the race" is the rating VALUE: a second application would
|
|
// recompute from the already-updated current rating and compound further
|
|
// away from 1500, so an exact match against a single, independently
|
|
// computed application is the assertion that actually falsifies a double
|
|
// application (unlike an inequality check, which a doubled update would
|
|
// still satisfy).
|
|
if winnerGames != 0 || loserGames != 0 {
|
|
t.Fatalf("casual result should never touch ranked_games: winner=%d loser=%d", winnerGames, loserGames)
|
|
}
|
|
baseline := domain.Rating{Value: 1500, RD: 350, Volatility: 0.06}
|
|
winnerOpponents, err := domain.CasualOpponents([]domain.Opponent{{PlayerID: "result-race-loser", Rating: baseline, Score: 1}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wantWinner, err := domain.UpdateRating(baseline, winnerOpponents, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
loserOpponents, err := domain.CasualOpponents([]domain.Opponent{{PlayerID: "result-race-winner", Rating: baseline, Score: 0}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wantLoser, err := domain.UpdateRating(baseline, loserOpponents, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if winnerRating != wantWinner.Value {
|
|
t.Fatalf("winner rating = %v, want exactly %v (a value between these would indicate a partial/compounded update)", winnerRating, wantWinner.Value)
|
|
}
|
|
if loserRating != wantLoser.Value {
|
|
t.Fatalf("loser rating = %v, want exactly %v", loserRating, wantLoser.Value)
|
|
}
|
|
if winnerRating <= loserRating {
|
|
t.Fatalf("winner rating %v should exceed loser rating %v after a certified result", winnerRating, loserRating)
|
|
}
|
|
}
|
|
|
|
// TestPostgreSQLStalledAllocationsAreReclaimedWithoutPenalisingPlayers is the
|
|
// live counterpart to the SQL fragment test: it proves the actual data
|
|
// movement against a real database, not just that the right substrings are
|
|
// present. Two matches: one genuinely stalled (old enough to reclaim), one
|
|
// recent (must survive untouched) -- the deadline boundary and the
|
|
// no-penalty requeue are both meaningless without a real row to check.
|
|
func TestPostgreSQLStalledAllocationsAreReclaimedWithoutPenalisingPlayers(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
ctx := context.Background()
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
stalledCreatedAt := now.Add(-10 * time.Minute)
|
|
recentCreatedAt := now.Add(-5 * time.Second)
|
|
|
|
for _, player := range []string{"stall-player-a", "stall-player-b", "recent-player"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
insertTicket := func(ticketID, playerID, state string, expiresAt time.Time) {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', $3, 'integration-build', 1, $4, $5)`, ticketID, playerID, state, now, expiresAt); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
insertTicket("stall-ticket-a", "stall-player-a", "PROCESS_READY", now.Add(time.Hour))
|
|
insertTicket("stall-ticket-b", "stall-player-b", "PROCESS_READY", now.Add(time.Hour))
|
|
insertTicket("recent-ticket", "recent-player", "ALLOCATING", now.Add(time.Hour))
|
|
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, created_at) VALUES ('stalled-match', 'casual', 'PROCESS_READY', 'NA', 1, 'stalled-server', $1)`, stalledCreatedAt); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, created_at) VALUES ('recent-match', 'casual', 'ALLOCATING', 'NA', 1, $1)`, recentCreatedAt); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) VALUES ('stalled-match', 'stall-player-a', 'stall-ticket-a', 0, 0), ('stalled-match', 'stall-player-b', 'stall-ticket-b', 1, 1)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) VALUES ('recent-match', 'recent-player', 'recent-ticket', 0, 0)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
reclaimed, err := ExpireStalledAllocations(ctx, db, now, 2*time.Minute, 10)
|
|
if err != nil {
|
|
t.Fatalf("expire stalled allocations: %v", err)
|
|
}
|
|
if reclaimed != 1 {
|
|
t.Fatalf("reclaimed = %d, want exactly 1 (the recent match must survive)", reclaimed)
|
|
}
|
|
|
|
var stalledState, recentState string
|
|
if err := db.QueryRow(`SELECT state FROM matches WHERE match_id = 'stalled-match'`).Scan(&stalledState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT state FROM matches WHERE match_id = 'recent-match'`).Scan(&recentState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if stalledState != "FAILED" {
|
|
t.Fatalf("stalled match state = %s, want FAILED", stalledState)
|
|
}
|
|
if recentState != "ALLOCATING" {
|
|
t.Fatalf("recent match state = %s, want untouched ALLOCATING", recentState)
|
|
}
|
|
|
|
var ticketAState, ticketBState, recentTicketState string
|
|
var ticketAExpiry time.Time
|
|
if err := db.QueryRow(`SELECT state, expires_at FROM queue_tickets WHERE ticket_id = 'stall-ticket-a'`).Scan(&ticketAState, &ticketAExpiry); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'stall-ticket-b'`).Scan(&ticketBState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'recent-ticket'`).Scan(&recentTicketState); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if ticketAState != "QUEUED" || ticketBState != "QUEUED" {
|
|
t.Fatalf("stalled participants' tickets = %s, %s -- want both requeued to QUEUED, not failed/left behind", ticketAState, ticketBState)
|
|
}
|
|
if !ticketAExpiry.After(now) {
|
|
t.Fatalf("requeued ticket expiry %v was not refreshed forward from %v", ticketAExpiry, now)
|
|
}
|
|
if recentTicketState != "ALLOCATING" {
|
|
t.Fatalf("recent match's ticket state = %s, want untouched ALLOCATING", recentTicketState)
|
|
}
|
|
|
|
var activeParticipants int
|
|
if err := db.QueryRow(`SELECT count(*) FROM match_participants WHERE match_id = 'stalled-match' AND participation_active`).Scan(&activeParticipants); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if activeParticipants != 0 {
|
|
t.Fatalf("stalled match still has %d active participants, want 0 (so the player can be matched again)", activeParticipants)
|
|
}
|
|
var eventType string
|
|
var eventPayload []byte
|
|
if err := db.QueryRow(`SELECT event_type, payload FROM outbox WHERE event_id = 'stalled-allocation:stalled-match:1'`).Scan(&eventType, &eventPayload); err != nil {
|
|
t.Fatalf("stalled allocation state event missing: %v", err)
|
|
}
|
|
var event struct {
|
|
State string `json:"state"`
|
|
PlayerIDs []string `json:"player_ids"`
|
|
}
|
|
if err := json.Unmarshal(eventPayload, &event); err != nil {
|
|
t.Fatalf("decode stalled allocation outbox event: %v", err)
|
|
}
|
|
players := make(map[string]bool, len(event.PlayerIDs))
|
|
for _, playerID := range event.PlayerIDs {
|
|
players[playerID] = true
|
|
}
|
|
if eventType != "state_changed" || event.State != "FAILED" || !players["stall-player-a"] {
|
|
t.Fatalf("stalled allocation event = %s %s, want FAILED state and affected player IDs", eventType, eventPayload)
|
|
}
|
|
|
|
// Idempotent: the match is now FAILED, not one of the three reclaimable
|
|
// states, so a second pass must not touch it again.
|
|
reclaimedAgain, err := ExpireStalledAllocations(ctx, db, now.Add(time.Minute), 2*time.Minute, 10)
|
|
if err != nil {
|
|
t.Fatalf("second expire pass: %v", err)
|
|
}
|
|
if reclaimedAgain != 0 {
|
|
t.Fatalf("second pass reclaimed %d matches, want 0 (already-FAILED match must not be reprocessed)", reclaimedAgain)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLRankedSeasonRolloverIsExactlyOnce(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('season-player', 'season-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO ratings (player_id, rating, deviation, volatility, ranked_games) VALUES ('season-player', 2000, 100, 0.12, 25)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO seasons (season_id, playlist, starts_at, ends_at) VALUES ('season-1', 'ranked', $1, $2)`, now.Add(-12*7*24*time.Hour), now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
profile := domain.RankedProfile{Rating: domain.Rating{Value: 1900, RD: 100, Volatility: 0.12}, RankedGames: 25}
|
|
updated, applied, err := ApplyRankedSeasonRollover(ctx, db, "season-player", "season-1", profile, now)
|
|
if err != nil || !applied {
|
|
t.Fatalf("first season rollover = %+v applied=%v err=%v", updated, applied, err)
|
|
}
|
|
if updated.Value != 1875 || updated.RD != 200 {
|
|
t.Fatalf("unexpected rolled rating: %+v", updated)
|
|
}
|
|
var rating float64
|
|
var markers int
|
|
if err := db.QueryRow(`SELECT rating FROM ratings WHERE player_id = 'season-player'`).Scan(&rating); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.QueryRow(`SELECT count(*) FROM ranked_season_rollovers WHERE player_id = 'season-player' AND season_id = 'season-1'`).Scan(&markers); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if rating != 1875 || markers != 1 {
|
|
t.Fatalf("durable rollover state rating=%v markers=%d", rating, markers)
|
|
}
|
|
duplicate, applied, err := ApplyRankedSeasonRollover(ctx, db, "season-player", "season-1", profile, now.Add(time.Second))
|
|
if err != nil || applied || duplicate.Value != 1875 {
|
|
t.Fatalf("duplicate rollover = %+v applied=%v err=%v", duplicate, applied, err)
|
|
}
|
|
if err := db.QueryRow(`SELECT rating FROM ratings WHERE player_id = 'season-player'`).Scan(&rating); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if rating != 1875 {
|
|
t.Fatalf("duplicate rollover changed rating to %v", rating)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLEmptyRankedSeasonIsMarkedRolledOver(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO seasons (season_id, playlist, starts_at, ends_at) VALUES ('empty-season', 'ranked', $1, $2)`, now.Add(-12*7*24*time.Hour), now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if count, err := RolloverDueSeasons(ctx, db, now, 100); err != nil || count != 0 {
|
|
t.Fatalf("empty-season maintenance count=%d err=%v", count, err)
|
|
}
|
|
var rolledAt sql.NullTime
|
|
if err := db.QueryRowContext(ctx, `SELECT rolled_over_at FROM seasons WHERE season_id = 'empty-season'`).Scan(&rolledAt); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !rolledAt.Valid {
|
|
t.Fatal("empty ranked season was not marked rolled over")
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLRankedProfileProjectsActiveSeason(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('profile-season-player', 'profile-season-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO ratings (player_id, rating, deviation, volatility, ranked_games) VALUES ('profile-season-player', 1600, 200, 0.06, 10)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO seasons (season_id, playlist, starts_at, ends_at) VALUES ('profile-season-current', 'ranked', $1, $2)`, now.Add(-time.Hour), now.Add(time.Hour)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
profile, found, err := (PostgresRankedProfiles{DB: db}).Get(ctx, "profile-season-player")
|
|
if err != nil || !found || profile.CurrentSeasonID != "profile-season-current" || !profile.CurrentSeasonEndsAt.Equal(now.Add(time.Hour)) {
|
|
t.Fatalf("profile=%+v found=%t err=%v", profile, found, err)
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLMigrationsAreForwardExecutable(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
var tableCount int
|
|
if err := db.QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'assignments'`).Scan(&tableCount); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if tableCount != 1 {
|
|
t.Fatal("assignments migration did not create its table")
|
|
}
|
|
}
|
|
|
|
func TestPostgreSQLMigrationsRollBackAndReapplyCleanly(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
dir := filepath.Join("..", "migrations")
|
|
tableExists := func(table string) bool {
|
|
var count int
|
|
if err := db.QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_name = $1`, table).Scan(&count); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return count == 1
|
|
}
|
|
if !tableExists("assignments") || !tableExists("allocations") {
|
|
t.Fatal("expected forward-applied schema before rollback")
|
|
}
|
|
|
|
// Roll back every migration one at a time, in reverse, checking each
|
|
// down file actually undoes what its forward file created — not just
|
|
// that Rollback returns nil.
|
|
// Derived, not hardcoded: every added migration shifts this count, and a
|
|
// stale literal silently makes the fixed-count rollbacks below target the
|
|
// wrong files (the failure then surfaces as a confusing "0006 rollback did
|
|
// not drop matches.allocation_id").
|
|
aboveMigration0006 := countMigrationsAbove(t, dir, 6)
|
|
if err := migrations.Rollback(context.Background(), db, dir, aboveMigration0006); err != nil {
|
|
t.Fatalf("rollback everything above 0006: %v", err)
|
|
}
|
|
var hasInitialConnectReadyColumn bool
|
|
if err := db.QueryRow(`SELECT count(*) > 0 FROM information_schema.columns WHERE table_name = 'matches' AND column_name = 'initial_connect_ready_at'`).Scan(&hasInitialConnectReadyColumn); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if hasInitialConnectReadyColumn {
|
|
t.Fatal("0010 rollback did not drop matches.initial_connect_ready_at")
|
|
}
|
|
|
|
if err := migrations.Rollback(context.Background(), db, dir, 1); err != nil {
|
|
t.Fatalf("rollback 0006: %v", err)
|
|
}
|
|
var hasAllocationClaimColumn bool
|
|
if err := db.QueryRow(`SELECT count(*) > 0 FROM information_schema.columns WHERE table_name = 'matches' AND column_name = 'allocation_id'`).Scan(&hasAllocationClaimColumn); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if hasAllocationClaimColumn {
|
|
t.Fatal("0006 rollback did not drop matches.allocation_id")
|
|
}
|
|
|
|
if err := migrations.Rollback(context.Background(), db, dir, 4); err != nil {
|
|
t.Fatalf("rollback 0005 through 0002: %v", err)
|
|
}
|
|
if tableExists("assignments") || tableExists("allocations") || tableExists("game_servers") {
|
|
t.Fatal("rollback left later-migration tables behind")
|
|
}
|
|
|
|
if err := migrations.Rollback(context.Background(), db, dir, 1); err != nil {
|
|
t.Fatalf("rollback 0001: %v", err)
|
|
}
|
|
if tableExists("identities") || tableExists("matches") {
|
|
t.Fatal("0001 rollback did not drop its own tables")
|
|
}
|
|
var remaining int
|
|
if err := db.QueryRow(`SELECT count(*) FROM schema_migrations`).Scan(&remaining); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if remaining != 0 {
|
|
t.Fatalf("expected schema_migrations empty after full rollback, got %d rows", remaining)
|
|
}
|
|
|
|
// Reapplying from a fully rolled-back state must reach the same schema,
|
|
// proving down files don't leave orphaned state that trips a forward
|
|
// re-run (e.g. a constraint or index Apply then tries to recreate).
|
|
if err := migrations.Apply(context.Background(), db, dir); err != nil {
|
|
t.Fatalf("reapply after full rollback: %v", err)
|
|
}
|
|
if !tableExists("assignments") || !tableExists("allocations") {
|
|
t.Fatal("reapply after rollback did not recreate the schema")
|
|
}
|
|
}
|
|
|
|
// Ranked matchmaking reads Candidate.Rating for tolerance, selection scoring
|
|
// and team partitioning. The candidate projection did not join the ratings
|
|
// table and its scan never set the field, so every PostgreSQL-sourced ranked
|
|
// candidate arrived with Go's zero value and the matcher treated a 900-rated
|
|
// player as identical to a 2100-rated one. Unit tests missed this because they
|
|
// construct candidates with ratings already populated.
|
|
func TestPostgreSQLQueuedCandidatesCarryAuthoritativeRatings(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
ctx := context.Background()
|
|
// "rated-low" and "rated-high" are deliberately far apart; "unrated" has no
|
|
// ratings row at all and must fall back to the new-profile default.
|
|
seeded := map[string]float64{"rated-low": 900, "rated-high": 2100}
|
|
for _, playerID := range []string{"rated-low", "rated-high", "unrated"} {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $2)`, playerID, "steam-"+playerID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if rating, ok := seeded[playerID]; ok {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO ratings (player_id, rating) VALUES ($1, $2)`, playerID, rating); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
spec := domain.QueueSpec{Playlist: domain.Ranked, ClientBuild: "rating-build", ProtocolVersion: 1}
|
|
ticket, err := CreateQueueTicket(ctx, db, "ticket-"+playerID, playerID, "rating-create-"+playerID+"-01", spec, now)
|
|
if err != nil {
|
|
t.Fatalf("create queue ticket for %s: %v", playerID, err)
|
|
}
|
|
// The Redis projection is seeded from this candidate rather than from
|
|
// the query below, so it must carry the same rating or the two
|
|
// projections disagree about who is comparable to whom.
|
|
want := domain.GlickoInitialRating
|
|
if rating, ok := seeded[playerID]; ok {
|
|
want = rating
|
|
}
|
|
if ticket.Candidate.Rating != want {
|
|
t.Fatalf("%s enqueue candidate rating = %v, want %v", playerID, ticket.Candidate.Rating, want)
|
|
}
|
|
}
|
|
|
|
candidates, err := ListQueuedCandidates(ctx, db, domain.Ranked, now, 100)
|
|
if err != nil {
|
|
t.Fatalf("list queued candidates: %v", err)
|
|
}
|
|
if len(candidates) != 3 {
|
|
t.Fatalf("expected 3 candidates, got %d", len(candidates))
|
|
}
|
|
got := make(map[string]float64, len(candidates))
|
|
for _, candidate := range candidates {
|
|
got[candidate.PlayerID] = candidate.Rating
|
|
}
|
|
for playerID, want := range map[string]float64{
|
|
"rated-low": 900, "rated-high": 2100, "unrated": domain.GlickoInitialRating,
|
|
} {
|
|
if got[playerID] != want {
|
|
t.Fatalf("%s durable candidate rating = %v, want %v", playerID, got[playerID], want)
|
|
}
|
|
}
|
|
|
|
// The whole point of loading the rating is that the matcher can tell these
|
|
// players apart. Assert the spread survives into team partitioning rather
|
|
// than only that the field is non-zero.
|
|
if got["rated-high"]-got["rated-low"] != 1200 {
|
|
t.Fatalf("rating spread collapsed: %v", got)
|
|
}
|
|
}
|
|
|
|
// banned_until and ban_reason existed in the schema from day one but no
|
|
// production query ever read them: the only ban check was an in-memory map on
|
|
// domain.TicketVerifier used by domain tests. A banned identity therefore kept
|
|
// working through every already-issued session until expiry, and could obtain
|
|
// new ones, defeating the server-authoritative anti-abuse boundary.
|
|
func TestPostgreSQLIdentityBanIsEnforcedOnIssuanceAndAuthentication(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
ctx := context.Background()
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('ban-player', 'ban-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Two independently constructed stores stand in for two control-plane
|
|
// replicas: the ban must hold on the replica that did not apply it.
|
|
replicaA := PostgresSessions{DB: db}
|
|
replicaB := PostgresSessions{DB: db}
|
|
|
|
session, token, err := replicaA.Issue(ctx, "ban-player", time.Hour, now)
|
|
if err != nil {
|
|
t.Fatalf("issue before ban: %v", err)
|
|
}
|
|
if _, err := replicaB.Authenticate(ctx, session.SessionID, token, now); err != nil {
|
|
t.Fatalf("authenticate before ban: %v", err)
|
|
}
|
|
|
|
if err := ApplyIdentityBan(ctx, db, "ban-player", "cheating", now.Add(24*time.Hour), now); err != nil {
|
|
t.Fatalf("apply ban: %v", err)
|
|
}
|
|
|
|
// The pre-existing session must stop working immediately, on a replica
|
|
// that never saw the ban being applied -- not at session expiry.
|
|
if _, err := replicaB.Authenticate(ctx, session.SessionID, token, now.Add(time.Minute)); err == nil {
|
|
t.Fatal("banned identity still authenticated with its existing session")
|
|
}
|
|
// ... and no new session may be minted.
|
|
if _, _, err := replicaB.Issue(ctx, "ban-player", time.Hour, now.Add(time.Minute)); err == nil {
|
|
t.Fatal("banned identity was issued a new session")
|
|
}
|
|
|
|
// The ban is time-bounded: once it lapses, sign-in works again.
|
|
afterBan := now.Add(25 * time.Hour)
|
|
revived, revivedToken, err := replicaA.Issue(ctx, "ban-player", time.Hour, afterBan)
|
|
if err != nil {
|
|
t.Fatalf("issue after ban expiry: %v", err)
|
|
}
|
|
if _, err := replicaB.Authenticate(ctx, revived.SessionID, revivedToken, afterBan); err != nil {
|
|
t.Fatalf("authenticate after ban expiry: %v", err)
|
|
}
|
|
|
|
// An explicit unban clears the state without resurrecting revoked sessions.
|
|
if err := ApplyIdentityBan(ctx, db, "ban-player", "", time.Time{}, now); err != nil {
|
|
t.Fatalf("clear ban: %v", err)
|
|
}
|
|
if _, err := replicaB.Authenticate(ctx, session.SessionID, token, now.Add(time.Minute)); err == nil {
|
|
t.Fatal("unban resurrected a session revoked by the ban")
|
|
}
|
|
}
|
|
|
|
// ApplyIdentityBan revokes sessions, so revocation alone would mask a missing
|
|
// ban check in Authenticate. A ban applied by any other path -- an admin tool,
|
|
// a future adapter, a direct operational UPDATE -- does not revoke anything,
|
|
// and must still take effect on the very next authenticated request.
|
|
func TestPostgreSQLBanWithoutRevocationStillBlocksExistingSessions(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
ctx := context.Background()
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('raw-ban-player', 'raw-ban-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sessions := PostgresSessions{DB: db}
|
|
session, token, err := sessions.Issue(ctx, "raw-ban-player", time.Hour, now)
|
|
if err != nil {
|
|
t.Fatalf("issue: %v", err)
|
|
}
|
|
if _, err := sessions.Authenticate(ctx, session.SessionID, token, now); err != nil {
|
|
t.Fatalf("authenticate before ban: %v", err)
|
|
}
|
|
// Set the ban only; deliberately leave every session unrevoked.
|
|
if _, err := db.ExecContext(ctx, `UPDATE identities SET banned_until = $2, ban_reason = 'manual' WHERE player_id = $1`, "raw-ban-player", now.Add(time.Hour)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := sessions.Authenticate(ctx, session.SessionID, token, now.Add(time.Minute)); err == nil {
|
|
t.Fatal("a ban applied without revocation left the existing session usable")
|
|
}
|
|
}
|
|
|
|
// The WebSocket hub is per-process, but any control-plane replica may drain a
|
|
// given outbox row, and the winner sets the single global published_at even
|
|
// with no matching local subscriber. Fan-out through LISTEN/NOTIFY is what
|
|
// lets the replica that actually owns the connection deliver the event.
|
|
func TestPostgreSQLControlPlaneEventFanoutReachesEveryReplica(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
dsn := os.Getenv("COSMIC_CLASH_POSTGRES_DSN")
|
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
|
defer cancel()
|
|
|
|
// Two listeners stand in for two replicas; neither is the one that will
|
|
// perform the notify.
|
|
type replica struct {
|
|
name string
|
|
received chan []byte
|
|
}
|
|
replicas := []*replica{
|
|
{name: "replica-a", received: make(chan []byte, 4)},
|
|
{name: "replica-b", received: make(chan []byte, 4)},
|
|
}
|
|
for _, r := range replicas {
|
|
target := r
|
|
go ListenControlPlaneEvents(ctx, dsn, func(payload []byte) {
|
|
select {
|
|
case target.received <- payload:
|
|
default:
|
|
}
|
|
}, nil)
|
|
}
|
|
|
|
// LISTEN is asynchronous; retry the notify until both listeners are
|
|
// attached rather than sleeping an arbitrary amount.
|
|
payload := []byte(`{"event":"state_changed","resource_id":"match-fanout","player_id":"player-fanout"}`)
|
|
deadline := time.Now().Add(15 * time.Second)
|
|
pending := map[string]bool{"replica-a": true, "replica-b": true}
|
|
for len(pending) > 0 && time.Now().Before(deadline) {
|
|
if err := NotifyControlPlaneEvent(ctx, db, payload); err != nil {
|
|
t.Fatalf("notify: %v", err)
|
|
}
|
|
for _, r := range replicas {
|
|
select {
|
|
case got := <-r.received:
|
|
if string(got) != string(payload) {
|
|
t.Fatalf("%s received %q, want %q", r.name, got, payload)
|
|
}
|
|
delete(pending, r.name)
|
|
case <-time.After(200 * time.Millisecond):
|
|
}
|
|
}
|
|
}
|
|
if len(pending) > 0 {
|
|
t.Fatalf("replicas never received the fanned-out event: %v", pending)
|
|
}
|
|
}
|
|
|
|
func TestNotifyControlPlaneEventRejectsOversizedPayloads(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
oversized := make([]byte, MaxNotifyPayloadBytes+1)
|
|
for i := range oversized {
|
|
oversized[i] = 'x'
|
|
}
|
|
if err := NotifyControlPlaneEvent(context.Background(), db, oversized); err == nil {
|
|
t.Fatal("payload over the NOTIFY limit was accepted")
|
|
}
|
|
}
|
|
|
|
// Each 10s queue heartbeat mints a fresh idempotency key and permanently
|
|
// inserts a row; published outbox rows and expired sessions were never purged
|
|
// either. At 10,000 queued players heartbeats alone add roughly 60,000 durable
|
|
// rows per minute, so table and index growth, vacuum pressure, backup size and
|
|
// recovery time were all unbounded on a horizontally-scaled service.
|
|
func TestPostgreSQLRetentionPurgesExpiredRecordsInBoundedBatches(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
ctx := context.Background()
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
stale := now.Add(-IdempotencyKeyRetention - time.Hour)
|
|
fresh := now.Add(-time.Minute)
|
|
|
|
// 120 stale keys (purgeable) and 5 fresh ones (must survive: a client may
|
|
// still retry those mutations).
|
|
for i := 0; i < 120; i++ {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO idempotency_keys (scope, idempotency_key, payload_digest, result, created_at) VALUES ('queue.mutate', $1, '\x00', '{}'::jsonb, $2)`,
|
|
fmt.Sprintf("stale-key-%030d", i), stale); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for i := 0; i < 5; i++ {
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO idempotency_keys (scope, idempotency_key, payload_digest, result, created_at) VALUES ('queue.mutate', $1, '\x00', '{}'::jsonb, $2)`,
|
|
fmt.Sprintf("fresh-key-%030d", i), fresh); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
countKeys := func() int {
|
|
var count int
|
|
if err := db.QueryRowContext(ctx, `SELECT count(*) FROM idempotency_keys`).Scan(&count); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return count
|
|
}
|
|
if countKeys() != 125 {
|
|
t.Fatalf("seed failed: %d keys", countKeys())
|
|
}
|
|
|
|
// One pass must delete at most the batch size, not the whole backlog: a
|
|
// purge that took an unbounded lock would stall live traffic.
|
|
report, err := PurgeExpiredRecords(ctx, db, now, 50)
|
|
if err != nil {
|
|
t.Fatalf("purge: %v", err)
|
|
}
|
|
if report.IdempotencyKeys != 50 {
|
|
t.Fatalf("first pass deleted %d keys, want the 50-row batch bound", report.IdempotencyKeys)
|
|
}
|
|
if countKeys() != 75 {
|
|
t.Fatalf("after one bounded pass: %d keys", countKeys())
|
|
}
|
|
|
|
// Repeated passes converge on exactly the fresh rows and then stop.
|
|
for i := 0; i < 5; i++ {
|
|
if _, err := PurgeExpiredRecords(ctx, db, now, 50); err != nil {
|
|
t.Fatalf("purge pass %d: %v", i, err)
|
|
}
|
|
}
|
|
if countKeys() != 5 {
|
|
t.Fatalf("steady state left %d keys, want only the 5 fresh ones", countKeys())
|
|
}
|
|
final, err := PurgeExpiredRecords(ctx, db, now, 50)
|
|
if err != nil {
|
|
t.Fatalf("final purge: %v", err)
|
|
}
|
|
if final.Total() != 0 {
|
|
t.Fatalf("purge deleted %d rows that were still within their retention window", final.Total())
|
|
}
|
|
backlog, err := RetentionBacklog(ctx, db, now)
|
|
if err != nil {
|
|
t.Fatalf("backlog: %v", err)
|
|
}
|
|
if backlog != 0 {
|
|
t.Fatalf("deletion lag is %d after draining the backlog", backlog)
|
|
}
|
|
}
|
|
|
|
// countMigrationsAbove reports how many forward migrations have a number
|
|
// greater than the given one, so rollback step counts in this file track new
|
|
// migrations automatically instead of needing a manual bump.
|
|
func countMigrationsAbove(t *testing.T, dir string, number int) int {
|
|
t.Helper()
|
|
entries, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
t.Fatalf("read migrations: %v", err)
|
|
}
|
|
count := 0
|
|
for _, entry := range entries {
|
|
name := entry.Name()
|
|
if entry.IsDir() || !strings.HasSuffix(name, ".sql") || len(name) < 4 {
|
|
continue
|
|
}
|
|
index, err := strconv.Atoi(name[:4])
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if index > number {
|
|
count++
|
|
}
|
|
}
|
|
if count == 0 {
|
|
t.Fatalf("no migrations found above %04d in %s", number, dir)
|
|
}
|
|
return count
|
|
}
|
|
|
|
// The second fatal blocker. domain.validCandidate hard-requires a non-empty
|
|
// PredictedRTT map, but CreateQueueTicket persisted an empty one and the only
|
|
// endpoint that could fill it returned 503 in every real binary because
|
|
// Service.Probe was never wired. No client-created ticket could ever be
|
|
// selected by the matcher.
|
|
//
|
|
// This drives the real enqueue and probe paths and then asks the actual
|
|
// matcher predicate, rather than hand-building a domain.Candidate the way the
|
|
// unit tests do -- which is precisely why they missed it.
|
|
func TestPostgreSQLProbedTicketBecomesSelectableByTheMatcher(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
ctx := context.Background()
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ('probe-player', 'probe-steam')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
spec := domain.QueueSpec{Playlist: domain.Casual, ClientBuild: "probe-build", ProtocolVersion: 1}
|
|
if _, err := CreateQueueTicket(ctx, db, "probe-ticket", "probe-player", "probe-create-000001", spec, now); err != nil {
|
|
t.Fatalf("create queue ticket: %v", err)
|
|
}
|
|
|
|
// Freshly queued: no RTT evidence yet, so the matcher must not consider it.
|
|
candidates, err := ListQueuedCandidates(ctx, db, domain.Casual, now, 100)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(candidates) != 1 {
|
|
t.Fatalf("expected the queued ticket, got %d candidates", len(candidates))
|
|
}
|
|
if len(candidates[0].PredictedRTT) != 0 {
|
|
t.Fatalf("a freshly queued ticket already has RTT evidence: %v", candidates[0].PredictedRTT)
|
|
}
|
|
if _, err := domain.SelectCandidates(candidates[0], candidates, 1, now); err == nil {
|
|
t.Fatal("a candidate with no RTT evidence was accepted by the matcher")
|
|
}
|
|
|
|
// The real challenge/answer round trip: the backend issues the nonce and
|
|
// derives RTT from its own timestamps, never from a client-reported value.
|
|
nonce, err := IssueProbeChallenge(ctx, db, "probe-player", "EU", now)
|
|
if err != nil {
|
|
t.Fatalf("issue challenge: %v", err)
|
|
}
|
|
if len(nonce) != ProbeNonceBytes {
|
|
t.Fatalf("challenge nonce is %d bytes", len(nonce))
|
|
}
|
|
answeredAt := now.Add(40 * time.Millisecond)
|
|
evidence, expectedNonce, err := ProbeEvidenceFromChallenge(ctx, db, "probe-player", "EU", []byte("opaque-location"), nonce, answeredAt)
|
|
if err != nil {
|
|
t.Fatalf("probe evidence: %v", err)
|
|
}
|
|
if err := domain.ValidateProbe(evidence, expectedNonce, answeredAt); err != nil {
|
|
t.Fatalf("backend-derived evidence failed its own validation: %v", err)
|
|
}
|
|
if evidence.ServerRTT != 40*time.Millisecond {
|
|
t.Fatalf("server-derived RTT = %v, want the 40ms round trip", evidence.ServerRTT)
|
|
}
|
|
|
|
// A challenge is single-use, so a captured answer cannot be replayed to
|
|
// refresh a stale RTT.
|
|
if _, _, err := ProbeEvidenceFromChallenge(ctx, db, "probe-player", "EU", []byte("opaque-location"), nonce, answeredAt); err == nil {
|
|
t.Fatal("a probe challenge was answerable twice")
|
|
}
|
|
|
|
if err := (PostgresQueue{DB: db}).RecordProbe(ctx, "probe-player", "EU", evidence.ServerRTT, answeredAt); err != nil {
|
|
t.Fatalf("record probe: %v", err)
|
|
}
|
|
|
|
// Now the same ticket, read through the same production query, is
|
|
// selectable.
|
|
candidates, err = ListQueuedCandidates(ctx, db, domain.Casual, answeredAt, 100)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(candidates) != 1 || candidates[0].PredictedRTT["EU"] == 0 {
|
|
t.Fatalf("probe did not reach the candidate projection: %+v", candidates)
|
|
}
|
|
if _, err := domain.SelectCandidates(candidates[0], candidates, 1, answeredAt); err != nil {
|
|
t.Fatalf("a probed, client-created ticket is still not selectable by the matcher: %v", err)
|
|
}
|
|
|
|
// The same candidate must survive the Redis path, which is seeded from the
|
|
// per-player refresh the probe handler performs.
|
|
refreshed, queued, err := FindQueuedCandidateByPlayer(ctx, db, "probe-player", answeredAt)
|
|
if err != nil || !queued {
|
|
t.Fatalf("per-player candidate refresh: queued=%t err=%v", queued, err)
|
|
}
|
|
if refreshed.PredictedRTT["EU"] == 0 {
|
|
t.Fatal("the refreshed candidate still carries an empty RTT map, so Redis would keep a stale entry")
|
|
}
|
|
}
|
|
|
|
// Production sign-in, exercised through the real SteamLogin provider with only
|
|
// the Valve HTTP call stubbed. newAPIService never supplied SteamLogin, so
|
|
// POST /v1/session/steam always returned 503 auth_unavailable; the only
|
|
// implementation was cmd/testkit-api's fake, which accepts any ticket string
|
|
// and therefore proves nothing about the deployable path.
|
|
func TestPostgreSQLSteamLoginResolvesDurableIdentities(t *testing.T) {
|
|
db := openIntegrationPostgres(t)
|
|
applyIntegrationMigrations(t, db)
|
|
|
|
ctx := context.Background()
|
|
const steamID = "76561198000000001"
|
|
|
|
// First sign-in creates the identity.
|
|
playerID, err := ResolveSteamIdentity(ctx, db, steamID, "player-first")
|
|
if err != nil {
|
|
t.Fatalf("first sign-in: %v", err)
|
|
}
|
|
if playerID != "player-first" {
|
|
t.Fatalf("first sign-in player = %q", playerID)
|
|
}
|
|
|
|
// A returning player must keep the player ID they already had, or their
|
|
// ratings, penalties and bans would silently detach from their account.
|
|
returning, err := ResolveSteamIdentity(ctx, db, steamID, "player-different-proposal")
|
|
if err != nil {
|
|
t.Fatalf("returning sign-in: %v", err)
|
|
}
|
|
if returning != "player-first" {
|
|
t.Fatalf("returning player was given a new ID %q", returning)
|
|
}
|
|
|
|
var identities int
|
|
if err := db.QueryRowContext(ctx, `SELECT count(*) FROM identities WHERE steam_id = $1`, steamID).Scan(&identities); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if identities != 1 {
|
|
t.Fatalf("one Steam ID produced %d identity rows", identities)
|
|
}
|
|
|
|
// The resolved identity must be usable for session issuance, which is the
|
|
// step that was unreachable in production.
|
|
sessions := PostgresSessions{DB: db}
|
|
now := time.Now().UTC().Truncate(time.Microsecond)
|
|
session, token, err := sessions.Issue(ctx, returning, time.Hour, now)
|
|
if err != nil {
|
|
t.Fatalf("issue session for a freshly resolved identity: %v", err)
|
|
}
|
|
if _, err := sessions.Authenticate(ctx, session.SessionID, token, now); err != nil {
|
|
t.Fatalf("authenticate freshly issued session: %v", err)
|
|
}
|
|
|
|
// And a banned account cannot sign in, tying the real login path to the
|
|
// durable ban enforcement rather than leaving it adapter-specific.
|
|
if err := ApplyIdentityBan(ctx, db, returning, "cheating", now.Add(time.Hour), now); err != nil {
|
|
t.Fatalf("apply ban: %v", err)
|
|
}
|
|
if _, _, err := sessions.Issue(ctx, returning, time.Hour, now.Add(time.Minute)); err == nil {
|
|
t.Fatal("a banned identity signed in through the production path")
|
|
}
|
|
}
|