feat(multiplayer): reclaim stalled allocations without penalising players

Closes the last named item on task 8.28 (Health-reclaim): nothing
currently detects or cleans up a match stuck in
ALLOCATING/PROCESS_READY/ASSIGNMENT_READY forever because its server
crashed or was reclaimed by Agones as unhealthy before ever
registering -- players would wait indefinitely for a match that was
never coming.

The design question this was blocked on -- does an abandoned match
auto-requeue its players, or fail and make them re-queue -- isn't
actually open: task 8.50's own stated acceptance criterion already
answers it ("infrastructure-caused cases cannot penalise affected
players"). A server-side crash/reclaim is exactly that, not player
behaviour, so store.ExpireStalledAllocations fails the match but
requeues every participant's ticket to QUEUED with a fresh expiry
(matching the ordinary 30s queue window), releases their
match_participants row (participation_active = false, so they're
matchable again immediately), all inside one FOR UPDATE SKIP LOCKED
pass so a second maintenance replica continues past whatever a
concurrent one is already reclaiming.

Wired into cmd/maintenance alongside the existing season-rollover
sweep: --stalled-allocation-deadline (default 2m) and
--stalled-allocation-batch (default 100).

Covered by a SQL-fragment test and a real PostgreSQL integration test:
two matches (one genuinely stalled, one recent), confirming the
deadline boundary is respected (recent match untouched), both
stranded participants' tickets requeue with a refreshed expiry, the
match_participants row releases, and a second pass doesn't reprocess
an already-FAILED match. Verified clean across 5 runs, plus the full
integration and unit suites.
This commit is contained in:
Josh Creek
2026-09-01 13:54:54 +01:00
parent d8245047a9
commit fc2faf9723
4 changed files with 217 additions and 1 deletions
+14 -1
View File
@@ -21,6 +21,8 @@ func main() {
migrationDir := flag.String("migrations", "migrations", "directory containing numbered SQL migrations")
interval := flag.Duration("interval", time.Minute, "maintenance poll interval")
batch := flag.Int("batch", 100, "maximum player rollovers per pass")
stalledAllocationDeadline := flag.Duration("stalled-allocation-deadline", 2*time.Minute, "reclaim a match stuck in ALLOCATING/PROCESS_READY/ASSIGNMENT_READY (server crashed or was reclaimed before registering) after this long, requeuing every participant without penalty")
stalledAllocationBatch := flag.Int("stalled-allocation-batch", 100, "maximum stalled matches reclaimed per pass")
flag.Parse()
if *dsn == "" {
fatalf("--dsn or COSMIC_CLASH_POSTGRES_DSN is required")
@@ -28,6 +30,9 @@ func main() {
if *interval <= 0 || *batch < 1 || *batch > 1000 {
fatalf("invalid interval or batch")
}
if *stalledAllocationDeadline <= 0 || *stalledAllocationBatch < 1 || *stalledAllocationBatch > 1000 {
fatalf("invalid stalled-allocation deadline or batch")
}
db, err := sql.Open("pgx", *dsn)
if err != nil {
fatalf("open PostgreSQL: %v", err)
@@ -44,13 +49,21 @@ func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
for {
count, err := store.RolloverDueSeasons(ctx, db, time.Now().UTC(), *batch)
now := time.Now().UTC()
count, err := store.RolloverDueSeasons(ctx, db, now, *batch)
if err != nil {
fatalf("season maintenance: %v", err)
}
if count > 0 {
log.Printf("applied %d ranked season rollovers", count)
}
reclaimed, err := store.ExpireStalledAllocations(ctx, db, now, *stalledAllocationDeadline, *stalledAllocationBatch)
if err != nil {
fatalf("stalled-allocation maintenance: %v", err)
}
if reclaimed > 0 {
log.Printf("reclaimed %d stalled allocations, requeuing their participants", reclaimed)
}
timer := time.NewTimer(*interval)
select {
case <-ctx.Done():