mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-11 00:14:00 +00:00
feat(multiplayer): reclaim stalled allocations without penalising players
Closes the last named item on task 8.28 (Health-reclaim): nothing
currently detects or cleans up a match stuck in
ALLOCATING/PROCESS_READY/ASSIGNMENT_READY forever because its server
crashed or was reclaimed by Agones as unhealthy before ever
registering -- players would wait indefinitely for a match that was
never coming.
The design question this was blocked on -- does an abandoned match
auto-requeue its players, or fail and make them re-queue -- isn't
actually open: task 8.50's own stated acceptance criterion already
answers it ("infrastructure-caused cases cannot penalise affected
players"). A server-side crash/reclaim is exactly that, not player
behaviour, so store.ExpireStalledAllocations fails the match but
requeues every participant's ticket to QUEUED with a fresh expiry
(matching the ordinary 30s queue window), releases their
match_participants row (participation_active = false, so they're
matchable again immediately), all inside one FOR UPDATE SKIP LOCKED
pass so a second maintenance replica continues past whatever a
concurrent one is already reclaiming.
Wired into cmd/maintenance alongside the existing season-rollover
sweep: --stalled-allocation-deadline (default 2m) and
--stalled-allocation-batch (default 100).
Covered by a SQL-fragment test and a real PostgreSQL integration test:
two matches (one genuinely stalled, one recent), confirming the
deadline boundary is respected (recent match untouched), both
stranded participants' tickets requeue with a refreshed expiry, the
match_participants row releases, and a second pass doesn't reprocess
an already-FAILED match. Verified clean across 5 runs, plus the full
integration and unit suites.
This commit is contained in:
@@ -0,0 +1,59 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/cosmic-clash/cosmic-clash/server/domain"
|
||||
)
|
||||
|
||||
// ExpireStalledAllocationsSQL reclaims a match that has sat in
|
||||
// ALLOCATING/PROCESS_READY/ASSIGNMENT_READY past the deadline -- its server
|
||||
// crashed, was reclaimed by Agones as unhealthy, or otherwise never finished
|
||||
// registering. Requeues every participant instead of just failing the match:
|
||||
// task 8.50's own stated acceptance criterion is that "infrastructure-caused
|
||||
// cases cannot penalise affected players", and a server-side failure here is
|
||||
// exactly that, not player behaviour. FOR UPDATE SKIP LOCKED lets a second
|
||||
// maintenance replica continue past whatever a concurrent one is already
|
||||
// reclaiming rather than blocking on it.
|
||||
const ExpireStalledAllocationsSQL = `WITH stalled AS (
|
||||
SELECT match_id FROM matches
|
||||
WHERE state IN ('ALLOCATING', 'PROCESS_READY', 'ASSIGNMENT_READY') AND created_at <= $1
|
||||
ORDER BY created_at, match_id
|
||||
LIMIT $2
|
||||
FOR UPDATE SKIP LOCKED
|
||||
), failed AS (
|
||||
UPDATE matches SET state = 'FAILED', revision = revision + 1
|
||||
WHERE match_id IN (SELECT match_id FROM stalled)
|
||||
RETURNING match_id
|
||||
), released AS (
|
||||
UPDATE match_participants SET participation_active = FALSE
|
||||
WHERE match_id IN (SELECT match_id FROM failed) AND participation_active
|
||||
RETURNING ticket_id
|
||||
), requeued AS (
|
||||
UPDATE queue_tickets SET state = 'QUEUED', expires_at = $3, revision = revision + 1
|
||||
WHERE ticket_id IN (SELECT ticket_id FROM released)
|
||||
RETURNING ticket_id
|
||||
)
|
||||
SELECT (SELECT count(*) FROM failed), (SELECT count(*) FROM requeued)`
|
||||
|
||||
// ExpireStalledAllocations reclaims up to `limit` matches whose
|
||||
// created_at is at or before `now - deadline` and are still stuck in one of
|
||||
// the pre-live allocation states, failing the match and requeuing every
|
||||
// participant's ticket with a fresh expiry rather than penalising them. It
|
||||
// returns the number of matches reclaimed.
|
||||
func ExpireStalledAllocations(ctx context.Context, db *sql.DB, now time.Time, deadline time.Duration, limit int) (int, error) {
|
||||
if db == nil || now.IsZero() || deadline <= 0 || limit < 1 || limit > 1000 {
|
||||
return 0, fmt.Errorf("invalid stalled-allocation maintenance arguments")
|
||||
}
|
||||
var matches, requeued int
|
||||
err := RunSerializable(ctx, db, DefaultSerializableAttempts, func(ctx context.Context, tx *sql.Tx) error {
|
||||
return tx.QueryRowContext(ctx, ExpireStalledAllocationsSQL, now.Add(-deadline), limit, now.Add(domain.QueueExpiryWindow)).Scan(&matches, &requeued)
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return matches, nil
|
||||
}
|
||||
Reference in New Issue
Block a user