mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-11 22:13:44 +00:00
Josh Creek
fc2faf9723
Closes the last named item on task 8.28 (Health-reclaim): nothing
currently detects or cleans up a match stuck in
ALLOCATING/PROCESS_READY/ASSIGNMENT_READY forever because its server
crashed or was reclaimed by Agones as unhealthy before ever
registering -- players would wait indefinitely for a match that was
never coming.
The design question this was blocked on -- does an abandoned match
auto-requeue its players, or fail and make them re-queue -- isn't
actually open: task 8.50's own stated acceptance criterion already
answers it ("infrastructure-caused cases cannot penalise affected
players"). A server-side crash/reclaim is exactly that, not player
behaviour, so store.ExpireStalledAllocations fails the match but
requeues every participant's ticket to QUEUED with a fresh expiry
(matching the ordinary 30s queue window), releases their
match_participants row (participation_active = false, so they're
matchable again immediately), all inside one FOR UPDATE SKIP LOCKED
pass so a second maintenance replica continues past whatever a
concurrent one is already reclaiming.
Wired into cmd/maintenance alongside the existing season-rollover
sweep: --stalled-allocation-deadline (default 2m) and
--stalled-allocation-batch (default 100).
Covered by a SQL-fragment test and a real PostgreSQL integration test:
two matches (one genuinely stalled, one recent), confirming the
deadline boundary is respected (recent match untouched), both
stranded participants' tickets requeue with a refreshed expiry, the
match_participants row releases, and a second pass doesn't reprocess
an already-FAILED match. Verified clean across 5 runs, plus the full
integration and unit suites.
60 lines
2.5 KiB
Go
60 lines
2.5 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/cosmic-clash/cosmic-clash/server/domain"
|
|
)
|
|
|
|
// ExpireStalledAllocationsSQL reclaims a match that has sat in
|
|
// ALLOCATING/PROCESS_READY/ASSIGNMENT_READY past the deadline -- its server
|
|
// crashed, was reclaimed by Agones as unhealthy, or otherwise never finished
|
|
// registering. Requeues every participant instead of just failing the match:
|
|
// task 8.50's own stated acceptance criterion is that "infrastructure-caused
|
|
// cases cannot penalise affected players", and a server-side failure here is
|
|
// exactly that, not player behaviour. FOR UPDATE SKIP LOCKED lets a second
|
|
// maintenance replica continue past whatever a concurrent one is already
|
|
// reclaiming rather than blocking on it.
|
|
const ExpireStalledAllocationsSQL = `WITH stalled AS (
|
|
SELECT match_id FROM matches
|
|
WHERE state IN ('ALLOCATING', 'PROCESS_READY', 'ASSIGNMENT_READY') AND created_at <= $1
|
|
ORDER BY created_at, match_id
|
|
LIMIT $2
|
|
FOR UPDATE SKIP LOCKED
|
|
), failed AS (
|
|
UPDATE matches SET state = 'FAILED', revision = revision + 1
|
|
WHERE match_id IN (SELECT match_id FROM stalled)
|
|
RETURNING match_id
|
|
), released AS (
|
|
UPDATE match_participants SET participation_active = FALSE
|
|
WHERE match_id IN (SELECT match_id FROM failed) AND participation_active
|
|
RETURNING ticket_id
|
|
), requeued AS (
|
|
UPDATE queue_tickets SET state = 'QUEUED', expires_at = $3, revision = revision + 1
|
|
WHERE ticket_id IN (SELECT ticket_id FROM released)
|
|
RETURNING ticket_id
|
|
)
|
|
SELECT (SELECT count(*) FROM failed), (SELECT count(*) FROM requeued)`
|
|
|
|
// ExpireStalledAllocations reclaims up to `limit` matches whose
|
|
// created_at is at or before `now - deadline` and are still stuck in one of
|
|
// the pre-live allocation states, failing the match and requeuing every
|
|
// participant's ticket with a fresh expiry rather than penalising them. It
|
|
// returns the number of matches reclaimed.
|
|
func ExpireStalledAllocations(ctx context.Context, db *sql.DB, now time.Time, deadline time.Duration, limit int) (int, error) {
|
|
if db == nil || now.IsZero() || deadline <= 0 || limit < 1 || limit > 1000 {
|
|
return 0, fmt.Errorf("invalid stalled-allocation maintenance arguments")
|
|
}
|
|
var matches, requeued int
|
|
err := RunSerializable(ctx, db, DefaultSerializableAttempts, func(ctx context.Context, tx *sql.Tx) error {
|
|
return tx.QueryRowContext(ctx, ExpireStalledAllocationsSQL, now.Add(-deadline), limit, now.Add(domain.QueueExpiryWindow)).Scan(&matches, &requeued)
|
|
})
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return matches, nil
|
|
}
|