test(multiplayer): add chaos recovery smoke

scripts/verify_chaos_recovery.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Disposable 8.50 recovery smoke. It proves the API can restart while durable
+# maintenance reclaims an infrastructure-stalled allocation without player
+# penalties and publishes a replayable lifecycle event.
+root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+project="${COMPOSE_PROJECT_NAME:-cosmic-clash-chaos-smoke}"
+compose=(docker compose -p "$project" -f "$root_dir/compose.chaos-smoke.yml")
+
+cleanup() {
+  local rc=$?
+  "${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true
+  exit "$rc"
+}
+trap cleanup EXIT
+
+command -v docker >/dev/null 2>&1 || { echo "Docker is required for 8.50" >&2; exit 2; }
+docker info >/dev/null 2>&1 || { echo "A running Docker daemon is required for 8.50" >&2; exit 2; }
+
+"${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true
+"${compose[@]}" up -d --build database control-plane
+
+for attempt in $(seq 1 60); do
+  if curl -fsS http://127.0.0.1:18082/healthz >/dev/null 2>&1; then break; fi
+  [[ "$attempt" == 60 ]] && { "${compose[@]}" logs >&2; echo "control plane did not become ready" >&2; exit 1; }
+  sleep 1
+done
+
+# Restart the API before seeding the failure, proving durable state is not
+# tied to the process that first opened the database connection.
+"${compose[@]}" restart control-plane >/dev/null
+for attempt in $(seq 1 30); do
+  if curl -fsS http://127.0.0.1:18082/healthz >/dev/null 2>&1; then break; fi
+  [[ "$attempt" == 30 ]] && { echo "control plane did not recover after restart" >&2; exit 1; }
+  sleep 1
+done
+
+"${compose[@]}" exec -T database psql -v ON_ERROR_STOP=1 -U cosmic_clash_test -d cosmic_clash_test <<'SQL'
+INSERT INTO identities (player_id, steam_id) VALUES ('chaos-player', 'chaos-steam');
+INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at)
+VALUES ('chaos-ticket', 'chaos-player', 'casual', 'ALLOCATING', 'build-1', 1, now() - interval '10 minutes', now() + interval '10 minutes');
+INSERT INTO matches (match_id, playlist, state, region, protocol_version, created_at)
+VALUES ('chaos-match', 'casual', 'ALLOCATING', 'EU', 1, now() - interval '10 minutes');
+INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team)
+VALUES ('chaos-match', 'chaos-player', 'chaos-ticket', 0, 0);
+SQL
+
+"${compose[@]}" up -d maintenance
+for attempt in $(seq 1 30); do
+  state="$(${compose[@]} exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT state FROM matches WHERE match_id = 'chaos-match'" | tr -d '\r')"
+  [[ "$state" == "FAILED" ]] && break
+  [[ "$attempt" == 30 ]] && { "${compose[@]}" logs maintenance >&2; echo "stalled allocation was not reclaimed" >&2; exit 1; }
+  sleep 1
+done
+
+ticket_state="$(${compose[@]} exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT state FROM queue_tickets WHERE ticket_id = 'chaos-ticket'" | tr -d '\r')"
+[[ "$ticket_state" == "QUEUED" ]]
+active="$(${compose[@]} exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM match_participants WHERE match_id = 'chaos-match' AND participation_active" | tr -d '\r')"
+[[ "$active" == "0" ]]
+events="$(${compose[@]} exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM outbox WHERE event_id = 'stalled-allocation:chaos-match:1' AND event_type = 'state_changed'" | tr -d '\r')"
+[[ "$events" == "1" ]]
+echo "8.50 PASS: API restart and stalled-allocation recovery preserved player eligibility and emitted a durable event"