From 76c1c3d600f4a38196130659f4d05ff44290b629 Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:19:28 +0100 Subject: [PATCH] fix(multiplayer): publish stalled allocation recovery --- multiplayer-next.md | 8 ++++++ server/store/postgres_integration_test.go | 8 ++++++ server/store/stalled_allocation_sql.go | 29 ++++++++++++++++++--- server/store/stalled_allocation_sql_test.go | 6 +++++ 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/multiplayer-next.md b/multiplayer-next.md index 2d578256..6fb6d632 100644 --- a/multiplayer-next.md +++ b/multiplayer-next.md @@ -1480,4 +1480,12 @@ The three declared domain fuzz targets have now each completed a bounded 4-secon The real ENet integration gate now passes with `GODOT_BIN=/Applications/Godot.app/Contents/MacOS/Godot bash scripts/verify_enet_integration.sh`, covering the `net`, `match-net`, `clock`, `lobby`, and `networked match` process scenarios. The default `GODOT_BIN` remains the portable `godot` PATH lookup for CI; this machine requires the explicit app-bundle path. +Stalled-allocation recovery now emits a participant-targeted `state_changed` +outbox event in the same serializable transaction that fails the abandoned +match, releases its participants, and requeues their tickets. The maintenance +adapter verifies that every reclaimed match produced its durable event, so an +API/WebSocket restart cannot turn a successful infrastructure recovery into a +silent client-side stale state. Normal, race, vet, and SQL-shape checks pass; +the live PostgreSQL chaos/restart gate remains part of 8.50. + The ENet gate now auto-detects `/Applications/Godot.app/Contents/MacOS/Godot` when no PATH executable or `GODOT_BIN` override exists, while retaining explicit override precedence. The same gate passes without an environment override on this macOS host. diff --git a/server/store/postgres_integration_test.go b/server/store/postgres_integration_test.go index 67443849..6bde46b8 100644 --- a/server/store/postgres_integration_test.go +++ b/server/store/postgres_integration_test.go @@ -1154,6 +1154,14 @@ func TestPostgreSQLStalledAllocationsAreReclaimedWithoutPenalisingPlayers(t *tes if activeParticipants != 0 { t.Fatalf("stalled match still has %d active participants, want 0 (so the player can be matched again)", activeParticipants) } + var eventType string + var eventPayload []byte + if err := db.QueryRow(`SELECT event_type, payload FROM outbox WHERE event_id = 'stalled-allocation:stalled-match:1'`).Scan(&eventType, &eventPayload); err != nil { + t.Fatalf("stalled allocation state event missing: %v", err) + } + if eventType != "state_changed" || !strings.Contains(string(eventPayload), `"state":"FAILED"`) || !strings.Contains(string(eventPayload), `"stall-player-a"`) { + t.Fatalf("stalled allocation event = %s %s, want FAILED state and affected player IDs", eventType, eventPayload) + } // Idempotent: the match is now FAILED, not one of the three reclaimable // states, so a second pass must not touch it again. diff --git a/server/store/stalled_allocation_sql.go b/server/store/stalled_allocation_sql.go index 0e9aecfc..b68dee2d 100644 --- a/server/store/stalled_allocation_sql.go +++ b/server/store/stalled_allocation_sql.go @@ -27,7 +27,7 @@ const ExpireStalledAllocationsSQL = `WITH stalled AS ( ), failed AS ( UPDATE matches SET state = 'FAILED', revision = revision + 1 WHERE match_id IN (SELECT match_id FROM stalled) - RETURNING match_id + RETURNING match_id, revision ), released AS ( UPDATE match_participants SET participation_active = FALSE WHERE match_id IN (SELECT match_id FROM failed) AND participation_active @@ -36,8 +36,26 @@ const ExpireStalledAllocationsSQL = `WITH stalled AS ( UPDATE queue_tickets SET state = 'QUEUED', expires_at = $3, revision = revision + 1 WHERE ticket_id IN (SELECT ticket_id FROM released) RETURNING ticket_id +), events AS ( + INSERT INTO outbox (event_id, aggregate_type, aggregate_id, revision, event_type, payload) + SELECT 'stalled-allocation:' || failed.match_id || ':' || failed.revision, + 'match', failed.match_id, failed.revision, 'state_changed', + jsonb_build_object( + 'event', 'state_changed', + 'revision', failed.revision, + 'resource_id', failed.match_id, + 'occurred_at', $4, + 'state', 'FAILED', + 'match_id', failed.match_id, + 'player_ids', COALESCE(( + SELECT jsonb_agg(mp.player_id ORDER BY mp.player_id) + FROM match_participants mp WHERE mp.match_id = failed.match_id + ), '[]'::jsonb) + ) + FROM failed + ON CONFLICT DO NOTHING ) -SELECT (SELECT count(*) FROM failed), (SELECT count(*) FROM requeued)` +SELECT (SELECT count(*) FROM failed), (SELECT count(*) FROM requeued), (SELECT count(*) FROM events)` // ExpireStalledAllocations reclaims up to `limit` matches whose // created_at is at or before `now - deadline` and are still stuck in one of @@ -48,12 +66,15 @@ func ExpireStalledAllocations(ctx context.Context, db *sql.DB, now time.Time, de if db == nil || now.IsZero() || deadline <= 0 || limit < 1 || limit > 1000 { return 0, fmt.Errorf("invalid stalled-allocation maintenance arguments") } - var matches, requeued int + var matches, requeued, events int err := RunSerializable(ctx, db, DefaultSerializableAttempts, func(ctx context.Context, tx *sql.Tx) error { - return tx.QueryRowContext(ctx, ExpireStalledAllocationsSQL, now.Add(-deadline), limit, now.Add(domain.QueueExpiryWindow)).Scan(&matches, &requeued) + return tx.QueryRowContext(ctx, ExpireStalledAllocationsSQL, now.Add(-deadline), limit, now.Add(domain.QueueExpiryWindow), now).Scan(&matches, &requeued, &events) }) if err != nil { return 0, err } + if events != matches { + return 0, fmt.Errorf("stalled-allocation outbox count %d does not match reclaimed matches %d", events, matches) + } return matches, nil } diff --git a/server/store/stalled_allocation_sql_test.go b/server/store/stalled_allocation_sql_test.go index 6360d811..6e77ed05 100644 --- a/server/store/stalled_allocation_sql_test.go +++ b/server/store/stalled_allocation_sql_test.go @@ -14,11 +14,17 @@ func TestExpireStalledAllocationsSQLFencesAndRequeuesWithoutPenalty(t *testing.T "SET state = 'FAILED'", "SET participation_active = FALSE", "SET state = 'QUEUED'", + "INSERT INTO outbox", + "'state_changed'", + "'stalled-allocation:'", } { if !strings.Contains(ExpireStalledAllocationsSQL, fragment) { t.Fatalf("ExpireStalledAllocationsSQL missing fragment %q:\n%s", fragment, ExpireStalledAllocationsSQL) } } + if !strings.Contains(ExpireStalledAllocationsSQL, "occurred_at', $4") { + t.Fatalf("stalled allocation event timestamp is not bound") + } } func TestExpireStalledAllocationsRejectsInvalidArgumentsWithoutDatabase(t *testing.T) {