mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-11 08:23:45 +00:00
Josh Creek
fc2faf9723
Closes the last named item on task 8.28 (Health-reclaim): nothing
currently detects or cleans up a match stuck in
ALLOCATING/PROCESS_READY/ASSIGNMENT_READY forever because its server
crashed or was reclaimed by Agones as unhealthy before ever
registering -- players would wait indefinitely for a match that was
never coming.
The design question this was blocked on -- does an abandoned match
auto-requeue its players, or fail and make them re-queue -- isn't
actually open: task 8.50's own stated acceptance criterion already
answers it ("infrastructure-caused cases cannot penalise affected
players"). A server-side crash/reclaim is exactly that, not player
behaviour, so store.ExpireStalledAllocations fails the match but
requeues every participant's ticket to QUEUED with a fresh expiry
(matching the ordinary 30s queue window), releases their
match_participants row (participation_active = false, so they're
matchable again immediately), all inside one FOR UPDATE SKIP LOCKED
pass so a second maintenance replica continues past whatever a
concurrent one is already reclaiming.
Wired into cmd/maintenance alongside the existing season-rollover
sweep: --stalled-allocation-deadline (default 2m) and
--stalled-allocation-batch (default 100).
Covered by a SQL-fragment test and a real PostgreSQL integration test:
two matches (one genuinely stalled, one recent), confirming the
deadline boundary is respected (recent match untouched), both
stranded participants' tickets requeue with a refreshed expiry, the
match_participants row releases, and a second pass doesn't reprocess
an already-FAILED match. Verified clean across 5 runs, plus the full
integration and unit suites.
81 lines
2.6 KiB
Go
81 lines
2.6 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/cosmic-clash/cosmic-clash/server/migrations"
|
|
"github.com/cosmic-clash/cosmic-clash/server/store"
|
|
_ "github.com/jackc/pgx/v5/stdlib"
|
|
)
|
|
|
|
func main() {
|
|
dsn := flag.String("dsn", os.Getenv("COSMIC_CLASH_POSTGRES_DSN"), "PostgreSQL connection string")
|
|
migrationDir := flag.String("migrations", "migrations", "directory containing numbered SQL migrations")
|
|
interval := flag.Duration("interval", time.Minute, "maintenance poll interval")
|
|
batch := flag.Int("batch", 100, "maximum player rollovers per pass")
|
|
stalledAllocationDeadline := flag.Duration("stalled-allocation-deadline", 2*time.Minute, "reclaim a match stuck in ALLOCATING/PROCESS_READY/ASSIGNMENT_READY (server crashed or was reclaimed before registering) after this long, requeuing every participant without penalty")
|
|
stalledAllocationBatch := flag.Int("stalled-allocation-batch", 100, "maximum stalled matches reclaimed per pass")
|
|
flag.Parse()
|
|
if *dsn == "" {
|
|
fatalf("--dsn or COSMIC_CLASH_POSTGRES_DSN is required")
|
|
}
|
|
if *interval <= 0 || *batch < 1 || *batch > 1000 {
|
|
fatalf("invalid interval or batch")
|
|
}
|
|
if *stalledAllocationDeadline <= 0 || *stalledAllocationBatch < 1 || *stalledAllocationBatch > 1000 {
|
|
fatalf("invalid stalled-allocation deadline or batch")
|
|
}
|
|
db, err := sql.Open("pgx", *dsn)
|
|
if err != nil {
|
|
fatalf("open PostgreSQL: %v", err)
|
|
}
|
|
defer db.Close()
|
|
startupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
if err := db.PingContext(startupCtx); err != nil {
|
|
fatalf("ping PostgreSQL: %v", err)
|
|
}
|
|
if err := migrations.Apply(startupCtx, db, *migrationDir); err != nil {
|
|
fatalf("apply migrations: %v", err)
|
|
}
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
for {
|
|
now := time.Now().UTC()
|
|
count, err := store.RolloverDueSeasons(ctx, db, now, *batch)
|
|
if err != nil {
|
|
fatalf("season maintenance: %v", err)
|
|
}
|
|
if count > 0 {
|
|
log.Printf("applied %d ranked season rollovers", count)
|
|
}
|
|
reclaimed, err := store.ExpireStalledAllocations(ctx, db, now, *stalledAllocationDeadline, *stalledAllocationBatch)
|
|
if err != nil {
|
|
fatalf("stalled-allocation maintenance: %v", err)
|
|
}
|
|
if reclaimed > 0 {
|
|
log.Printf("reclaimed %d stalled allocations, requeuing their participants", reclaimed)
|
|
}
|
|
timer := time.NewTimer(*interval)
|
|
select {
|
|
case <-ctx.Done():
|
|
timer.Stop()
|
|
return
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
func fatalf(format string, args ...any) {
|
|
fmt.Fprintf(os.Stderr, "maintenance: "+format+"\n", args...)
|
|
os.Exit(1)
|
|
}
|