test(multiplayer): add chaos recovery smoke

.github/workflows/multiplayer-chaos.yml

@@ -0,0 +1,26 @@
+name: Multiplayer Chaos Recovery
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - Dockerfile
+      - Makefile
+      - compose.chaos-smoke.yml
+      - server/cmd/maintenance/**
+      - server/store/**
+      - server/migrations/**
+      - scripts/verify_chaos_recovery.sh
+      - .github/workflows/multiplayer-chaos.yml
+
+permissions:
+  contents: read
+
+jobs:
+  api-restart-recovery:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+      - name: Verify API restart and stalled-allocation recovery
+        run: make verify-chaos-recovery

Dockerfile

@@ -60,6 +60,7 @@ RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/game-server-supervisor ./cmd/gam
 RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/testkit-api ./cmd/testkit-api
 RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/matcher ./cmd/matcher
 RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/allocator ./cmd/allocator
+RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/maintenance ./cmd/maintenance
 
 # Agones-allocated fleet image: the same dedicated-server export as `server`
 # (unchanged above; make verify-phase6 exercises that target exactly as
@@ -92,3 +93,9 @@ COPY --from=supervisor-build /opt/cosmic-clash/allocator /opt/cosmic-clash/alloc
 COPY server/migrations /opt/cosmic-clash/migrations
 RUN chmod 0755 /opt/cosmic-clash/allocator
 ENTRYPOINT ["/opt/cosmic-clash/allocator"]
+
+FROM server AS maintenance
+COPY --from=supervisor-build /opt/cosmic-clash/maintenance /opt/cosmic-clash/maintenance
+COPY server/migrations /opt/cosmic-clash/migrations
+RUN chmod 0755 /opt/cosmic-clash/maintenance
+ENTRYPOINT ["/opt/cosmic-clash/maintenance"]

Makefile

@@ -1,4 +1,4 @@
-.PHONY: verify-phase6 verify-enet-integration verify-steam-templates verify-supply-chain verify-kind-agones verify-allocated-compose verify-multiplayer-local verify-multiplayer-load verify-release-gate
+.PHONY: verify-phase6 verify-enet-integration verify-steam-templates verify-supply-chain verify-kind-agones verify-allocated-compose verify-chaos-recovery verify-multiplayer-local verify-multiplayer-load verify-release-gate
 
 verify-multiplayer-local:
 	bash scripts/verify_multiplayer_local.sh
@@ -27,3 +27,6 @@ verify-kind-agones:
 
 verify-allocated-compose:
 	bash scripts/verify_allocated_compose.sh
+
+verify-chaos-recovery:
+	bash scripts/verify_chaos_recovery.sh

compose.chaos-smoke.yml

@@ -0,0 +1,37 @@
+services:
+  database:
+    image: postgres:17-alpine
+    environment:
+      POSTGRES_DB: cosmic_clash_test
+      POSTGRES_USER: cosmic_clash_test
+      POSTGRES_PASSWORD: cosmic_clash_test
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U cosmic_clash_test -d cosmic_clash_test"]
+      interval: 1s
+      timeout: 3s
+      retries: 30
+
+  control-plane:
+    build:
+      context: .
+      target: testkit-api
+    environment:
+      COSMIC_CLASH_POSTGRES_DSN: postgres://cosmic_clash_test:cosmic_clash_test@database:5432/cosmic_clash_test?sslmode=disable
+      COSMIC_CLASH_WORKLOAD_SECRET: chaos-workload-secret
+    command: ["--listen=0.0.0.0:8080", "--migrations=/opt/cosmic-clash/migrations"]
+    depends_on:
+      database:
+        condition: service_healthy
+    ports:
+      - "18082:8080"
+
+  maintenance:
+    build:
+      context: .
+      target: maintenance
+    environment:
+      COSMIC_CLASH_POSTGRES_DSN: postgres://cosmic_clash_test:cosmic_clash_test@database:5432/cosmic_clash_test?sslmode=disable
+    command: ["--dsn=postgres://cosmic_clash_test:cosmic_clash_test@database:5432/cosmic_clash_test?sslmode=disable", "--migrations=/opt/cosmic-clash/migrations", "--interval=1s", "--stalled-allocation-deadline=1s", "--stalled-allocation-batch=10", "--initial-connect-batch=1"]
+    depends_on:
+      database:
+        condition: service_healthy

multiplayer-next.md

@@ -1249,7 +1249,7 @@ the local/CI/community transport, not a silent production fallback.
 | 8.47 `[D:8.7,8.30]` | **IN PROGRESS.** Offline testkit provides deterministic fake Steam verification and fake allocation with forced failure injection; the independent Compose runner drives fake-Steam session issuance, real HTTP queue create/heartbeat/cancel, matcher-backed six-player proposal formation/acceptance, and idempotency-conflict checks | `server/testkit/` covers verified identity/replay, unknown identity, wrong App ID, expiry, no capacity, compatibility-key conflict, idempotent allocation replay and cloud-free forced allocation failure; the Compose API/matcher slice is wired into CI, while live exhaustive matrix and production Steam remain |
 | 8.48 `[D:8.10,8.14,8.17,8.18,8.27,8.31,8.35,8.47]` | **IN PROGRESS.** Offline testkit exercises verified queue projection → ranked six-player proposal → ENet allocation → assignment-ready manifest → certified durable result receipt; `compose.allocated-smoke.yml` independently runs the real testkit API, matcher, allocator, Agones-shaped provider, PostgreSQL, and game-server supervisor with a generated signed roster, verifying queue/proposal/allocation binding, authenticated result, idempotent retry, shutdown acknowledgment, durable receipt/audit rows, and SIGTERM-driven game-process drain | `.github/workflows/allocated-compose.yml` runs `make verify-allocated-compose`. Live Docker evidence from this workspace and legacy fixture non-regression remain open |
 | 8.49 `[D:8.25,8.26,8.28,8.29,8.30,8.31,8.35,8.36]` | **IN PROGRESS.** `scripts/verify_kind_agones.sh` creates a disposable kind cluster, installs pinned Agones, loads the real `game-server` image, applies the Fleet in an explicitly separate Agones-only supervisor/UDP readiness mode, and verifies readiness plus allocation of a dynamic UDP endpoint; `.github/workflows/agones-integration.yml` runs it for infrastructure changes and on demand | The cloud-free runner is committed and fails clearly when Docker/kind/Helm are unavailable. CI/live evidence for production control-plane registration, roster/no-show, both readiness stages, races, multi-match node, result-pending reconciliation, drain, and rollback remains open |
-| 8.50 `[D:8.25,8.37,8.43,8.49]` | Network/chaos suite: 100 ms RTT, jitter/loss, client/API/matcher restart, game-pod death, node drain, Redis failover and control-plane loss | System recovers to a defined state; infrastructure-caused cases cannot penalise affected players |
+| 8.50 `[D:8.25,8.37,8.43,8.49]` | **IN PROGRESS.** `make verify-chaos-recovery` provides a disposable PostgreSQL + real testkit API + real maintenance flow: it restarts the API, injects a stale allocation, and verifies no-penalty requeue plus a durable participant-targeted lifecycle event | The API-restart/stalled-allocation slice is implemented and documented; 100 ms RTT/jitter/loss, matcher/client restart, game-pod death, node drain, Redis failover, control-plane loss, and live chaos evidence remain |
 | 8.51 `[D:8.17,8.18,8.30,8.31,8.45]` | **IN PROGRESS.** The opt-in `make verify-multiplayer-load` gate drives 10,000 real HTTP queue-create requests through the service with 256 in flight and records p95/p99; the handler and in-process ownership boundary are exercised without weakening normal tests | Local API load passes at p95 <250 ms in normal and race runs; PostgreSQL saturation, >=100 proposals/s, durable matcher fencing under load, forecast launch concurrency x2, and replica scaling remain live infrastructure gates |
 | 8.52 `[D:8.32,8.34,8.45,8.51]` | **IN PROGRESS.** Allocator now supports an opt-in, per-replica fixed-window allocation quota per EU/NA region (`--allocation-quota` / `--allocation-quota-window`), checked before any provider call and safe under concurrent attempts | Normal/race/vet tests cover quota exhaustion, window reset, region isolation, invalid input, and atomic concurrent consumption; measured regional cost model, shared/global quota, budget alerts, and denial-of-wallet production rehearsal remain |
 | 8.53 `[D:7.8,8.13,8.38,8.45,8.46,8.48,8.49,8.50,8.51,8.52]` | **IN PROGRESS.** `scripts/verify_release_gate.py` provides a fail-closed promotion check for the ordered development → internal → casual canary → casual → provisional ranked → ranked stages, requiring an evidence report for SLO, security, cost, rollback, EU+NA playtests, and both legacy gates | Validator and adversarial tests cover skipped stages, unknown stages, missing gates, non-boolean gate values, and blank release IDs; the actual reports, production rollback rehearsal, regional playtests, and live promotion remain open |

scripts/verify_chaos_recovery.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Disposable 8.50 recovery smoke. It proves the API can restart while durable
+# maintenance reclaims an infrastructure-stalled allocation without player
+# penalties and publishes a replayable lifecycle event.
+root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+project="${COMPOSE_PROJECT_NAME:-cosmic-clash-chaos-smoke}"
+compose=(docker compose -p "$project" -f "$root_dir/compose.chaos-smoke.yml")
+
+cleanup() {
+  local rc=$?
+  "${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true
+  exit "$rc"
+}
+trap cleanup EXIT
+
+command -v docker >/dev/null 2>&1 || { echo "Docker is required for 8.50" >&2; exit 2; }
+docker info >/dev/null 2>&1 || { echo "A running Docker daemon is required for 8.50" >&2; exit 2; }
+
+"${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true
+"${compose[@]}" up -d --build database control-plane
+
+for attempt in $(seq 1 60); do
+  if curl -fsS http://127.0.0.1:18082/healthz >/dev/null 2>&1; then break; fi
+  [[ "$attempt" == 60 ]] && { "${compose[@]}" logs >&2; echo "control plane did not become ready" >&2; exit 1; }
+  sleep 1
+done
+
+# Restart the API before seeding the failure, proving durable state is not
+# tied to the process that first opened the database connection.
+"${compose[@]}" restart control-plane >/dev/null
+for attempt in $(seq 1 30); do
+  if curl -fsS http://127.0.0.1:18082/healthz >/dev/null 2>&1; then break; fi
+  [[ "$attempt" == 30 ]] && { echo "control plane did not recover after restart" >&2; exit 1; }
+  sleep 1
+done
+
+"${compose[@]}" exec -T database psql -v ON_ERROR_STOP=1 -U cosmic_clash_test -d cosmic_clash_test <<'SQL'
+INSERT INTO identities (player_id, steam_id) VALUES ('chaos-player', 'chaos-steam');
+INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at)
+VALUES ('chaos-ticket', 'chaos-player', 'casual', 'ALLOCATING', 'build-1', 1, now() - interval '10 minutes', now() + interval '10 minutes');
+INSERT INTO matches (match_id, playlist, state, region, protocol_version, created_at)
+VALUES ('chaos-match', 'casual', 'ALLOCATING', 'EU', 1, now() - interval '10 minutes');
+INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team)
+VALUES ('chaos-match', 'chaos-player', 'chaos-ticket', 0, 0);
+SQL
+
+"${compose[@]}" up -d maintenance
+for attempt in $(seq 1 30); do
+  state="$(${compose[@]} exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT state FROM matches WHERE match_id = 'chaos-match'" | tr -d '\r')"
+  [[ "$state" == "FAILED" ]] && break
+  [[ "$attempt" == 30 ]] && { "${compose[@]}" logs maintenance >&2; echo "stalled allocation was not reclaimed" >&2; exit 1; }
+  sleep 1
+done
+
+ticket_state="$(${compose[@]} exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT state FROM queue_tickets WHERE ticket_id = 'chaos-ticket'" | tr -d '\r')"
+[[ "$ticket_state" == "QUEUED" ]]
+active="$(${compose[@]} exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM match_participants WHERE match_id = 'chaos-match' AND participation_active" | tr -d '\r')"
+[[ "$active" == "0" ]]
+events="$(${compose[@]} exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM outbox WHERE event_id = 'stalled-allocation:chaos-match:1' AND event_type = 'state_changed'" | tr -d '\r')"
+[[ "$events" == "1" ]]
+echo "8.50 PASS: API restart and stalled-allocation recovery preserved player eligibility and emitted a durable event"

server/security/test_compose_manifests.py

@@ -39,6 +39,14 @@ class ComposeManifestTest(unittest.TestCase):
         self.assertIn("target: allocator", allocated)
         self.assertIn("agones-provider", allocated)
 
+    def test_chaos_fixture_has_real_maintenance_and_restart_boundary(self):
+        chaos = (ROOT / "compose.chaos-smoke.yml").read_text()
+        runner = (ROOT / "scripts/verify_chaos_recovery.sh").read_text()
+        self.assertIn("target: maintenance", chaos)
+        self.assertIn("restart control-plane", runner)
+        self.assertIn("stalled-allocation:", runner)
+        self.assertIn("down --volumes --remove-orphans", runner)
+
     def test_kind_runner_uses_strict_allocation_response_validation(self):
         runner = (ROOT / "scripts/verify_kind_agones.sh").read_text()
         self.assertIn("verify_agones_allocation_response.py", runner)