diff --git a/.github/workflows/multiplayer-chaos.yml b/.github/workflows/multiplayer-chaos.yml new file mode 100644 index 00000000..5b24c317 --- /dev/null +++ b/.github/workflows/multiplayer-chaos.yml @@ -0,0 +1,26 @@ +name: Multiplayer Chaos Recovery + +on: + workflow_dispatch: + pull_request: + paths: + - Dockerfile + - Makefile + - compose.chaos-smoke.yml + - server/cmd/maintenance/** + - server/store/** + - server/migrations/** + - scripts/verify_chaos_recovery.sh + - .github/workflows/multiplayer-chaos.yml + +permissions: + contents: read + +jobs: + api-restart-recovery: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - name: Verify API restart and stalled-allocation recovery + run: make verify-chaos-recovery diff --git a/Dockerfile b/Dockerfile index 16ba3494..06ece0f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,6 +60,7 @@ RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/game-server-supervisor ./cmd/gam RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/testkit-api ./cmd/testkit-api RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/matcher ./cmd/matcher RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/allocator ./cmd/allocator +RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/maintenance ./cmd/maintenance # Agones-allocated fleet image: the same dedicated-server export as `server` # (unchanged above; make verify-phase6 exercises that target exactly as @@ -92,3 +93,9 @@ COPY --from=supervisor-build /opt/cosmic-clash/allocator /opt/cosmic-clash/alloc COPY server/migrations /opt/cosmic-clash/migrations RUN chmod 0755 /opt/cosmic-clash/allocator ENTRYPOINT ["/opt/cosmic-clash/allocator"] + +FROM server AS maintenance +COPY --from=supervisor-build /opt/cosmic-clash/maintenance /opt/cosmic-clash/maintenance +COPY server/migrations /opt/cosmic-clash/migrations +RUN chmod 0755 /opt/cosmic-clash/maintenance +ENTRYPOINT ["/opt/cosmic-clash/maintenance"] diff --git a/Makefile b/Makefile index de74fd18..cdd71269 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: verify-phase6 verify-enet-integration verify-steam-templates verify-supply-chain verify-kind-agones verify-allocated-compose verify-multiplayer-local verify-multiplayer-load verify-release-gate +.PHONY: verify-phase6 verify-enet-integration verify-steam-templates verify-supply-chain verify-kind-agones verify-allocated-compose verify-chaos-recovery verify-multiplayer-local verify-multiplayer-load verify-release-gate verify-multiplayer-local: bash scripts/verify_multiplayer_local.sh @@ -27,3 +27,6 @@ verify-kind-agones: verify-allocated-compose: bash scripts/verify_allocated_compose.sh + +verify-chaos-recovery: + bash scripts/verify_chaos_recovery.sh diff --git a/compose.chaos-smoke.yml b/compose.chaos-smoke.yml new file mode 100644 index 00000000..8b3a28e1 --- /dev/null +++ b/compose.chaos-smoke.yml @@ -0,0 +1,37 @@ +services: + database: + image: postgres:17-alpine + environment: + POSTGRES_DB: cosmic_clash_test + POSTGRES_USER: cosmic_clash_test + POSTGRES_PASSWORD: cosmic_clash_test + healthcheck: + test: ["CMD-SHELL", "pg_isready -U cosmic_clash_test -d cosmic_clash_test"] + interval: 1s + timeout: 3s + retries: 30 + + control-plane: + build: + context: . + target: testkit-api + environment: + COSMIC_CLASH_POSTGRES_DSN: postgres://cosmic_clash_test:cosmic_clash_test@database:5432/cosmic_clash_test?sslmode=disable + COSMIC_CLASH_WORKLOAD_SECRET: chaos-workload-secret + command: ["--listen=0.0.0.0:8080", "--migrations=/opt/cosmic-clash/migrations"] + depends_on: + database: + condition: service_healthy + ports: + - "18082:8080" + + maintenance: + build: + context: . + target: maintenance + environment: + COSMIC_CLASH_POSTGRES_DSN: postgres://cosmic_clash_test:cosmic_clash_test@database:5432/cosmic_clash_test?sslmode=disable + command: ["--dsn=postgres://cosmic_clash_test:cosmic_clash_test@database:5432/cosmic_clash_test?sslmode=disable", "--migrations=/opt/cosmic-clash/migrations", "--interval=1s", "--stalled-allocation-deadline=1s", "--stalled-allocation-batch=10", "--initial-connect-batch=1"] + depends_on: + database: + condition: service_healthy diff --git a/multiplayer-next.md b/multiplayer-next.md index 38c3492a..edcb7209 100644 --- a/multiplayer-next.md +++ b/multiplayer-next.md @@ -1249,7 +1249,7 @@ the local/CI/community transport, not a silent production fallback. | 8.47 `[D:8.7,8.30]` | **IN PROGRESS.** Offline testkit provides deterministic fake Steam verification and fake allocation with forced failure injection; the independent Compose runner drives fake-Steam session issuance, real HTTP queue create/heartbeat/cancel, matcher-backed six-player proposal formation/acceptance, and idempotency-conflict checks | `server/testkit/` covers verified identity/replay, unknown identity, wrong App ID, expiry, no capacity, compatibility-key conflict, idempotent allocation replay and cloud-free forced allocation failure; the Compose API/matcher slice is wired into CI, while live exhaustive matrix and production Steam remain | | 8.48 `[D:8.10,8.14,8.17,8.18,8.27,8.31,8.35,8.47]` | **IN PROGRESS.** Offline testkit exercises verified queue projection → ranked six-player proposal → ENet allocation → assignment-ready manifest → certified durable result receipt; `compose.allocated-smoke.yml` independently runs the real testkit API, matcher, allocator, Agones-shaped provider, PostgreSQL, and game-server supervisor with a generated signed roster, verifying queue/proposal/allocation binding, authenticated result, idempotent retry, shutdown acknowledgment, durable receipt/audit rows, and SIGTERM-driven game-process drain | `.github/workflows/allocated-compose.yml` runs `make verify-allocated-compose`. Live Docker evidence from this workspace and legacy fixture non-regression remain open | | 8.49 `[D:8.25,8.26,8.28,8.29,8.30,8.31,8.35,8.36]` | **IN PROGRESS.** `scripts/verify_kind_agones.sh` creates a disposable kind cluster, installs pinned Agones, loads the real `game-server` image, applies the Fleet in an explicitly separate Agones-only supervisor/UDP readiness mode, and verifies readiness plus allocation of a dynamic UDP endpoint; `.github/workflows/agones-integration.yml` runs it for infrastructure changes and on demand | The cloud-free runner is committed and fails clearly when Docker/kind/Helm are unavailable. CI/live evidence for production control-plane registration, roster/no-show, both readiness stages, races, multi-match node, result-pending reconciliation, drain, and rollback remains open | -| 8.50 `[D:8.25,8.37,8.43,8.49]` | Network/chaos suite: 100 ms RTT, jitter/loss, client/API/matcher restart, game-pod death, node drain, Redis failover and control-plane loss | System recovers to a defined state; infrastructure-caused cases cannot penalise affected players | +| 8.50 `[D:8.25,8.37,8.43,8.49]` | **IN PROGRESS.** `make verify-chaos-recovery` provides a disposable PostgreSQL + real testkit API + real maintenance flow: it restarts the API, injects a stale allocation, and verifies no-penalty requeue plus a durable participant-targeted lifecycle event | The API-restart/stalled-allocation slice is implemented and documented; 100 ms RTT/jitter/loss, matcher/client restart, game-pod death, node drain, Redis failover, control-plane loss, and live chaos evidence remain | | 8.51 `[D:8.17,8.18,8.30,8.31,8.45]` | **IN PROGRESS.** The opt-in `make verify-multiplayer-load` gate drives 10,000 real HTTP queue-create requests through the service with 256 in flight and records p95/p99; the handler and in-process ownership boundary are exercised without weakening normal tests | Local API load passes at p95 <250 ms in normal and race runs; PostgreSQL saturation, >=100 proposals/s, durable matcher fencing under load, forecast launch concurrency x2, and replica scaling remain live infrastructure gates | | 8.52 `[D:8.32,8.34,8.45,8.51]` | **IN PROGRESS.** Allocator now supports an opt-in, per-replica fixed-window allocation quota per EU/NA region (`--allocation-quota` / `--allocation-quota-window`), checked before any provider call and safe under concurrent attempts | Normal/race/vet tests cover quota exhaustion, window reset, region isolation, invalid input, and atomic concurrent consumption; measured regional cost model, shared/global quota, budget alerts, and denial-of-wallet production rehearsal remain | | 8.53 `[D:7.8,8.13,8.38,8.45,8.46,8.48,8.49,8.50,8.51,8.52]` | **IN PROGRESS.** `scripts/verify_release_gate.py` provides a fail-closed promotion check for the ordered development → internal → casual canary → casual → provisional ranked → ranked stages, requiring an evidence report for SLO, security, cost, rollback, EU+NA playtests, and both legacy gates | Validator and adversarial tests cover skipped stages, unknown stages, missing gates, non-boolean gate values, and blank release IDs; the actual reports, production rollback rehearsal, regional playtests, and live promotion remain open | diff --git a/scripts/verify_chaos_recovery.sh b/scripts/verify_chaos_recovery.sh new file mode 100644 index 00000000..6e124e74 --- /dev/null +++ b/scripts/verify_chaos_recovery.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Disposable 8.50 recovery smoke. It proves the API can restart while durable +# maintenance reclaims an infrastructure-stalled allocation without player +# penalties and publishes a replayable lifecycle event. +root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +project="${COMPOSE_PROJECT_NAME:-cosmic-clash-chaos-smoke}" +compose=(docker compose -p "$project" -f "$root_dir/compose.chaos-smoke.yml") + +cleanup() { + local rc=$? + "${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true + exit "$rc" +} +trap cleanup EXIT + +command -v docker >/dev/null 2>&1 || { echo "Docker is required for 8.50" >&2; exit 2; } +docker info >/dev/null 2>&1 || { echo "A running Docker daemon is required for 8.50" >&2; exit 2; } + +"${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true +"${compose[@]}" up -d --build database control-plane + +for attempt in $(seq 1 60); do + if curl -fsS http://127.0.0.1:18082/healthz >/dev/null 2>&1; then break; fi + [[ "$attempt" == 60 ]] && { "${compose[@]}" logs >&2; echo "control plane did not become ready" >&2; exit 1; } + sleep 1 +done + +# Restart the API before seeding the failure, proving durable state is not +# tied to the process that first opened the database connection. +"${compose[@]}" restart control-plane >/dev/null +for attempt in $(seq 1 30); do + if curl -fsS http://127.0.0.1:18082/healthz >/dev/null 2>&1; then break; fi + [[ "$attempt" == 30 ]] && { echo "control plane did not recover after restart" >&2; exit 1; } + sleep 1 +done + +"${compose[@]}" exec -T database psql -v ON_ERROR_STOP=1 -U cosmic_clash_test -d cosmic_clash_test <<'SQL' +INSERT INTO identities (player_id, steam_id) VALUES ('chaos-player', 'chaos-steam'); +INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at) +VALUES ('chaos-ticket', 'chaos-player', 'casual', 'ALLOCATING', 'build-1', 1, now() - interval '10 minutes', now() + interval '10 minutes'); +INSERT INTO matches (match_id, playlist, state, region, protocol_version, created_at) +VALUES ('chaos-match', 'casual', 'ALLOCATING', 'EU', 1, now() - interval '10 minutes'); +INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team) +VALUES ('chaos-match', 'chaos-player', 'chaos-ticket', 0, 0); +SQL + +"${compose[@]}" up -d maintenance +for attempt in $(seq 1 30); do + state="$(${compose[@]} exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT state FROM matches WHERE match_id = 'chaos-match'" | tr -d '\r')" + [[ "$state" == "FAILED" ]] && break + [[ "$attempt" == 30 ]] && { "${compose[@]}" logs maintenance >&2; echo "stalled allocation was not reclaimed" >&2; exit 1; } + sleep 1 +done + +ticket_state="$(${compose[@]} exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT state FROM queue_tickets WHERE ticket_id = 'chaos-ticket'" | tr -d '\r')" +[[ "$ticket_state" == "QUEUED" ]] +active="$(${compose[@]} exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM match_participants WHERE match_id = 'chaos-match' AND participation_active" | tr -d '\r')" +[[ "$active" == "0" ]] +events="$(${compose[@]} exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM outbox WHERE event_id = 'stalled-allocation:chaos-match:1' AND event_type = 'state_changed'" | tr -d '\r')" +[[ "$events" == "1" ]] +echo "8.50 PASS: API restart and stalled-allocation recovery preserved player eligibility and emitted a durable event" diff --git a/server/security/test_compose_manifests.py b/server/security/test_compose_manifests.py index f94226ad..2aeafa6b 100644 --- a/server/security/test_compose_manifests.py +++ b/server/security/test_compose_manifests.py @@ -39,6 +39,14 @@ class ComposeManifestTest(unittest.TestCase): self.assertIn("target: allocator", allocated) self.assertIn("agones-provider", allocated) + def test_chaos_fixture_has_real_maintenance_and_restart_boundary(self): + chaos = (ROOT / "compose.chaos-smoke.yml").read_text() + runner = (ROOT / "scripts/verify_chaos_recovery.sh").read_text() + self.assertIn("target: maintenance", chaos) + self.assertIn("restart control-plane", runner) + self.assertIn("stalled-allocation:", runner) + self.assertIn("down --volumes --remove-orphans", runner) + def test_kind_runner_uses_strict_allocation_response_validation(self): runner = (ROOT / "scripts/verify_kind_agones.sh").read_text() self.assertIn("verify_agones_allocation_response.py", runner)