fix(multiplayer): publish stalled allocation recovery

This commit is contained in:
Josh Creek
2026-09-01 18:19:28 +01:00
parent 7c044b7094
commit 76c1c3d600
4 changed files with 47 additions and 4 deletions
+8
View File
@@ -1480,4 +1480,12 @@ The three declared domain fuzz targets have now each completed a bounded 4-secon
The real ENet integration gate now passes with `GODOT_BIN=/Applications/Godot.app/Contents/MacOS/Godot bash scripts/verify_enet_integration.sh`, covering the `net`, `match-net`, `clock`, `lobby`, and `networked match` process scenarios. The default `GODOT_BIN` remains the portable `godot` PATH lookup for CI; this machine requires the explicit app-bundle path.
Stalled-allocation recovery now emits a participant-targeted `state_changed`
outbox event in the same serializable transaction that fails the abandoned
match, releases its participants, and requeues their tickets. The maintenance
adapter verifies that every reclaimed match produced its durable event, so an
API/WebSocket restart cannot turn a successful infrastructure recovery into a
silent client-side stale state. Normal, race, vet, and SQL-shape checks pass;
the live PostgreSQL chaos/restart gate remains part of 8.50.
The ENet gate now auto-detects `/Applications/Godot.app/Contents/MacOS/Godot` when no PATH executable or `GODOT_BIN` override exists, while retaining explicit override precedence. The same gate passes without an environment override on this macOS host.
@@ -1154,6 +1154,14 @@ func TestPostgreSQLStalledAllocationsAreReclaimedWithoutPenalisingPlayers(t *tes
if activeParticipants != 0 {
t.Fatalf("stalled match still has %d active participants, want 0 (so the player can be matched again)", activeParticipants)
}
var eventType string
var eventPayload []byte
if err := db.QueryRow(`SELECT event_type, payload FROM outbox WHERE event_id = 'stalled-allocation:stalled-match:1'`).Scan(&eventType, &eventPayload); err != nil {
t.Fatalf("stalled allocation state event missing: %v", err)
}
if eventType != "state_changed" || !strings.Contains(string(eventPayload), `"state":"FAILED"`) || !strings.Contains(string(eventPayload), `"stall-player-a"`) {
t.Fatalf("stalled allocation event = %s %s, want FAILED state and affected player IDs", eventType, eventPayload)
}
// Idempotent: the match is now FAILED, not one of the three reclaimable
// states, so a second pass must not touch it again.
+25 -4
View File
@@ -27,7 +27,7 @@ const ExpireStalledAllocationsSQL = `WITH stalled AS (
), failed AS (
UPDATE matches SET state = 'FAILED', revision = revision + 1
WHERE match_id IN (SELECT match_id FROM stalled)
RETURNING match_id
RETURNING match_id, revision
), released AS (
UPDATE match_participants SET participation_active = FALSE
WHERE match_id IN (SELECT match_id FROM failed) AND participation_active
@@ -36,8 +36,26 @@ const ExpireStalledAllocationsSQL = `WITH stalled AS (
UPDATE queue_tickets SET state = 'QUEUED', expires_at = $3, revision = revision + 1
WHERE ticket_id IN (SELECT ticket_id FROM released)
RETURNING ticket_id
), events AS (
INSERT INTO outbox (event_id, aggregate_type, aggregate_id, revision, event_type, payload)
SELECT 'stalled-allocation:' || failed.match_id || ':' || failed.revision,
'match', failed.match_id, failed.revision, 'state_changed',
jsonb_build_object(
'event', 'state_changed',
'revision', failed.revision,
'resource_id', failed.match_id,
'occurred_at', $4,
'state', 'FAILED',
'match_id', failed.match_id,
'player_ids', COALESCE((
SELECT jsonb_agg(mp.player_id ORDER BY mp.player_id)
FROM match_participants mp WHERE mp.match_id = failed.match_id
), '[]'::jsonb)
)
FROM failed
ON CONFLICT DO NOTHING
)
SELECT (SELECT count(*) FROM failed), (SELECT count(*) FROM requeued)`
SELECT (SELECT count(*) FROM failed), (SELECT count(*) FROM requeued), (SELECT count(*) FROM events)`
// ExpireStalledAllocations reclaims up to `limit` matches whose
// created_at is at or before `now - deadline` and are still stuck in one of
@@ -48,12 +66,15 @@ func ExpireStalledAllocations(ctx context.Context, db *sql.DB, now time.Time, de
if db == nil || now.IsZero() || deadline <= 0 || limit < 1 || limit > 1000 {
return 0, fmt.Errorf("invalid stalled-allocation maintenance arguments")
}
var matches, requeued int
var matches, requeued, events int
err := RunSerializable(ctx, db, DefaultSerializableAttempts, func(ctx context.Context, tx *sql.Tx) error {
return tx.QueryRowContext(ctx, ExpireStalledAllocationsSQL, now.Add(-deadline), limit, now.Add(domain.QueueExpiryWindow)).Scan(&matches, &requeued)
return tx.QueryRowContext(ctx, ExpireStalledAllocationsSQL, now.Add(-deadline), limit, now.Add(domain.QueueExpiryWindow), now).Scan(&matches, &requeued, &events)
})
if err != nil {
return 0, err
}
if events != matches {
return 0, fmt.Errorf("stalled-allocation outbox count %d does not match reclaimed matches %d", events, matches)
}
return matches, nil
}
@@ -14,11 +14,17 @@ func TestExpireStalledAllocationsSQLFencesAndRequeuesWithoutPenalty(t *testing.T
"SET state = 'FAILED'",
"SET participation_active = FALSE",
"SET state = 'QUEUED'",
"INSERT INTO outbox",
"'state_changed'",
"'stalled-allocation:'",
} {
if !strings.Contains(ExpireStalledAllocationsSQL, fragment) {
t.Fatalf("ExpireStalledAllocationsSQL missing fragment %q:\n%s", fragment, ExpireStalledAllocationsSQL)
}
}
if !strings.Contains(ExpireStalledAllocationsSQL, "occurred_at', $4") {
t.Fatalf("stalled allocation event timestamp is not bound")
}
}
func TestExpireStalledAllocationsRejectsInvalidArgumentsWithoutDatabase(t *testing.T) {