feat(multiplayer): reclaim stalled allocations without penalising players

Closes the last named item on task 8.28 (Health-reclaim): nothing
currently detects or cleans up a match stuck in
ALLOCATING/PROCESS_READY/ASSIGNMENT_READY forever because its server
crashed or was reclaimed by Agones as unhealthy before ever
registering -- players would wait indefinitely for a match that was
never coming.

The design question this was blocked on -- does an abandoned match
auto-requeue its players, or fail and make them re-queue -- isn't
actually open: task 8.50's own stated acceptance criterion already
answers it ("infrastructure-caused cases cannot penalise affected
players"). A server-side crash/reclaim is exactly that, not player
behaviour, so store.ExpireStalledAllocations fails the match but
requeues every participant's ticket to QUEUED with a fresh expiry
(matching the ordinary 30s queue window), releases their
match_participants row (participation_active = false, so they're
matchable again immediately), all inside one FOR UPDATE SKIP LOCKED
pass so a second maintenance replica continues past whatever a
concurrent one is already reclaiming.

Wired into cmd/maintenance alongside the existing season-rollover
sweep: --stalled-allocation-deadline (default 2m) and
--stalled-allocation-batch (default 100).

Covered by a SQL-fragment test and a real PostgreSQL integration test:
two matches (one genuinely stalled, one recent), confirming the
deadline boundary is respected (recent match untouched), both
stranded participants' tickets requeue with a refreshed expiry, the
match_participants row releases, and a second pass doesn't reprocess
an already-FAILED match. Verified clean across 5 runs, plus the full
integration and unit suites.

server/store/postgres_integration_test.go

@@ -781,6 +781,109 @@ func TestPostgreSQLConcurrentIdenticalResultSubmissionAppliesRatingsExactlyOnce(
 	}
 }
 
+// TestPostgreSQLStalledAllocationsAreReclaimedWithoutPenalisingPlayers is the
+// live counterpart to the SQL fragment test: it proves the actual data
+// movement against a real database, not just that the right substrings are
+// present. Two matches: one genuinely stalled (old enough to reclaim), one
+// recent (must survive untouched) -- the deadline boundary and the
+// no-penalty requeue are both meaningless without a real row to check.
+func TestPostgreSQLStalledAllocationsAreReclaimedWithoutPenalisingPlayers(t *testing.T) {
+	db := openIntegrationPostgres(t)
+	applyIntegrationMigrations(t, db)
+	ctx := context.Background()
+	now := time.Now().UTC().Truncate(time.Microsecond)
+	stalledCreatedAt := now.Add(-10 * time.Minute)
+	recentCreatedAt := now.Add(-5 * time.Second)
+
+	for _, player := range []string{"stall-player-a", "stall-player-b", "recent-player"} {
+		if _, err := db.ExecContext(ctx, `INSERT INTO identities (player_id, steam_id) VALUES ($1, $1)`, player); err != nil {
+			t.Fatal(err)
+		}
+	}
+	insertTicket := func(ticketID, playerID, state string, expiresAt time.Time) {
+		if _, err := db.ExecContext(ctx, `INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) VALUES ($1, $2, 'casual', $3, 'integration-build', 1, $4, $5)`, ticketID, playerID, state, now, expiresAt); err != nil {
+			t.Fatal(err)
+		}
+	}
+	insertTicket("stall-ticket-a", "stall-player-a", "PROCESS_READY", now.Add(time.Hour))
+	insertTicket("stall-ticket-b", "stall-player-b", "PROCESS_READY", now.Add(time.Hour))
+	insertTicket("recent-ticket", "recent-player", "ALLOCATING", now.Add(time.Hour))
+
+	if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, created_at) VALUES ('stalled-match', 'casual', 'PROCESS_READY', 'NA', 1, 'stalled-server', $1)`, stalledCreatedAt); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.ExecContext(ctx, `INSERT INTO matches (match_id, playlist, state, region, protocol_version, created_at) VALUES ('recent-match', 'casual', 'ALLOCATING', 'NA', 1, $1)`, recentCreatedAt); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) VALUES ('stalled-match', 'stall-player-a', 'stall-ticket-a', 0, 0), ('stalled-match', 'stall-player-b', 'stall-ticket-b', 1, 1)`); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.ExecContext(ctx, `INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) VALUES ('recent-match', 'recent-player', 'recent-ticket', 0, 0)`); err != nil {
+		t.Fatal(err)
+	}
+
+	reclaimed, err := ExpireStalledAllocations(ctx, db, now, 2*time.Minute, 10)
+	if err != nil {
+		t.Fatalf("expire stalled allocations: %v", err)
+	}
+	if reclaimed != 1 {
+		t.Fatalf("reclaimed = %d, want exactly 1 (the recent match must survive)", reclaimed)
+	}
+
+	var stalledState, recentState string
+	if err := db.QueryRow(`SELECT state FROM matches WHERE match_id = 'stalled-match'`).Scan(&stalledState); err != nil {
+		t.Fatal(err)
+	}
+	if err := db.QueryRow(`SELECT state FROM matches WHERE match_id = 'recent-match'`).Scan(&recentState); err != nil {
+		t.Fatal(err)
+	}
+	if stalledState != "FAILED" {
+		t.Fatalf("stalled match state = %s, want FAILED", stalledState)
+	}
+	if recentState != "ALLOCATING" {
+		t.Fatalf("recent match state = %s, want untouched ALLOCATING", recentState)
+	}
+
+	var ticketAState, ticketBState, recentTicketState string
+	var ticketAExpiry time.Time
+	if err := db.QueryRow(`SELECT state, expires_at FROM queue_tickets WHERE ticket_id = 'stall-ticket-a'`).Scan(&ticketAState, &ticketAExpiry); err != nil {
+		t.Fatal(err)
+	}
+	if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'stall-ticket-b'`).Scan(&ticketBState); err != nil {
+		t.Fatal(err)
+	}
+	if err := db.QueryRow(`SELECT state FROM queue_tickets WHERE ticket_id = 'recent-ticket'`).Scan(&recentTicketState); err != nil {
+		t.Fatal(err)
+	}
+	if ticketAState != "QUEUED" || ticketBState != "QUEUED" {
+		t.Fatalf("stalled participants' tickets = %s, %s -- want both requeued to QUEUED, not failed/left behind", ticketAState, ticketBState)
+	}
+	if !ticketAExpiry.After(now) {
+		t.Fatalf("requeued ticket expiry %v was not refreshed forward from %v", ticketAExpiry, now)
+	}
+	if recentTicketState != "ALLOCATING" {
+		t.Fatalf("recent match's ticket state = %s, want untouched ALLOCATING", recentTicketState)
+	}
+
+	var activeParticipants int
+	if err := db.QueryRow(`SELECT count(*) FROM match_participants WHERE match_id = 'stalled-match' AND participation_active`).Scan(&activeParticipants); err != nil {
+		t.Fatal(err)
+	}
+	if activeParticipants != 0 {
+		t.Fatalf("stalled match still has %d active participants, want 0 (so the player can be matched again)", activeParticipants)
+	}
+
+	// Idempotent: the match is now FAILED, not one of the three reclaimable
+	// states, so a second pass must not touch it again.
+	reclaimedAgain, err := ExpireStalledAllocations(ctx, db, now.Add(time.Minute), 2*time.Minute, 10)
+	if err != nil {
+		t.Fatalf("second expire pass: %v", err)
+	}
+	if reclaimedAgain != 0 {
+		t.Fatalf("second pass reclaimed %d matches, want 0 (already-FAILED match must not be reprocessed)", reclaimedAgain)
+	}
+}
+
 func TestPostgreSQLRankedSeasonRolloverIsExactlyOnce(t *testing.T) {
 	db := openIntegrationPostgres(t)
 	applyIntegrationMigrations(t, db)