test(multiplayer): drive compose proposal orchestration

This commit is contained in:
Josh Creek
2026-09-01 17:58:37 +01:00
parent 00d6b1edd3
commit 063dff463d
5 changed files with 66 additions and 1 deletions
+7
View File
@@ -58,6 +58,7 @@ RUN go mod download
COPY server/ ./
RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/game-server-supervisor ./cmd/game-server-supervisor
RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/testkit-api ./cmd/testkit-api
RUN CGO_ENABLED=0 go build -o /opt/cosmic-clash/matcher ./cmd/matcher
# Agones-allocated fleet image: the same dedicated-server export as `server`
# (unchanged above; make verify-phase6 exercises that target exactly as
@@ -78,3 +79,9 @@ COPY --from=supervisor-build /opt/cosmic-clash/testkit-api /opt/cosmic-clash/tes
COPY server/migrations /opt/cosmic-clash/migrations
RUN chmod 0755 /opt/cosmic-clash/testkit-api
ENTRYPOINT ["/opt/cosmic-clash/testkit-api"]
FROM server AS matcher
COPY --from=supervisor-build /opt/cosmic-clash/matcher /opt/cosmic-clash/matcher
COPY server/migrations /opt/cosmic-clash/migrations
RUN chmod 0755 /opt/cosmic-clash/matcher
ENTRYPOINT ["/opt/cosmic-clash/matcher"]
+11
View File
@@ -25,6 +25,17 @@ services:
ports:
- "18080:8080"
matcher:
build:
context: .
target: matcher
environment:
COSMIC_CLASH_POSTGRES_DSN: postgres://cosmic_clash_test:cosmic_clash_test@database:5432/cosmic_clash_test?sslmode=disable
command: ["--dsn=postgres://cosmic_clash_test:cosmic_clash_test@database:5432/cosmic_clash_test?sslmode=disable", "--migrations=/opt/cosmic-clash/migrations", "--playlist=casual", "--size=6", "--interval=1s"]
depends_on:
database:
condition: service_healthy
game-server:
build:
context: .
+1 -1
View File
@@ -1246,7 +1246,7 @@ the local/CI/community transport, not a silent production fallback.
| 8.44 `[D:8.3,8.4,8.28,8.31]` | **LOCAL COMPLETE; PRODUCTION GATE OPEN.** Go observability package encodes queue/proposal/match/server IDs and lifecycle stage while recursively redacting auth/relay tokens and credentials. `Service.Log` is wired to mutation and read routes at every outcome, and `cmd/control-plane` writes those events as JSON lines to stderr | `server/observability/` covers correlation fields, nested secret redaction, content-aware credential canaries and unnamed-event rejection; API tests cover lifecycle event wiring without logging error text. A production metrics/traces backend and dashboard/alert routing remain open; the local logger is intentionally stderr-only |
| 8.45 `[D:8.2,8.44]` | **LOCAL COMPLETE; PRODUCTION GATE OPEN.** Go observability package turns the documented RTT, allocation/connect latency, result-success, API-latency and tick/headroom thresholds into executable window checks; the API exporter emits a bounded cumulative latency histogram suitable for querying the documented p95 API SLO | `server/observability/slo.go`, `metrics.go`, `deploy/observability/prometheus-rules.yaml` and adversarial tests cover healthy/violating/empty windows, fixed operation/status labels, cumulative bucket boundaries, arbitrary-path cardinality safety, and optional API p95/5xx alerts. Production scrape configuration, alert routing, wait/MMR/proposal/flood/cost series and runbooks remain |
| 8.46 `[D:8.5,8.7,8.9,8.10,8.14,8.18,8.21,8.23,8.25]` | **IN PROGRESS.** Go unit/race coverage spans the current domain/store/supervisor policies, and fuzz targets now exercise queue input, result payload hashing and revision events | `server/domain/*_test.go`, `server/store/*_test.go`, `server/supervisor/*_test.go`, `server/migrations/*_test.go` and `server/domain/fuzz_test.go` pass normal/race suites; `go test -race ./...` passes across API, domain, migrations, observability, store, supervisor and testkit; `go vet ./...` passes; each of the three declared domain fuzz targets passes a bounded 4-second run; PostgreSQL live migration execution now runs clean (§8.5), and four real-concurrency cases are covered against a live database with `-race`: §8.14's queue-heartbeat revision race, §8.18's two-matcher contested-ticket race, §8.30's cross-allocator-replica capacity race, and §8.21/§8.25's concurrent identical-result-submission race; the "lost Redis" fixture is covered live against a real server (§8.14: real TTL expiry, repair-after-`FLUSHALL`; fake Steam/allocator fixtures are §8.47's testkit, already done). Further transaction fixtures (e.g. concurrent proposal-recovery expiry races, live Redis failover mid-write under load) remain |
| 8.47 `[D:8.7,8.30]` | **IN PROGRESS.** Offline testkit provides deterministic fake Steam verification and fake allocation with forced failure injection; the independent Compose runner now also drives fake-Steam session issuance and the real HTTP queue create/heartbeat/cancel boundary with an idempotency-conflict check | `server/testkit/` covers verified identity/replay, unknown identity, wrong App ID, expiry, no capacity, compatibility-key conflict, idempotent allocation replay and cloud-free forced allocation failure; the Compose API slice is wired into CI, while live exhaustive matrix and production Steam remain |
| 8.47 `[D:8.7,8.30]` | **IN PROGRESS.** Offline testkit provides deterministic fake Steam verification and fake allocation with forced failure injection; the independent Compose runner drives fake-Steam session issuance, real HTTP queue create/heartbeat/cancel, matcher-backed six-player proposal formation/acceptance, and idempotency-conflict checks | `server/testkit/` covers verified identity/replay, unknown identity, wrong App ID, expiry, no capacity, compatibility-key conflict, idempotent allocation replay and cloud-free forced allocation failure; the Compose API/matcher slice is wired into CI, while live exhaustive matrix and production Steam remain |
| 8.48 `[D:8.10,8.14,8.17,8.18,8.27,8.31,8.35,8.47]` | **IN PROGRESS.** Offline testkit exercises verified queue projection → ranked six-player proposal → ENet allocation → assignment-ready manifest → certified durable result receipt; `compose.allocated-smoke.yml` independently runs the real testkit API against PostgreSQL and the real game-server supervisor with a generated signed roster, verifying authenticated result, idempotent retry, shutdown acknowledgment, durable receipt/audit rows, and SIGTERM-driven game-process drain | `.github/workflows/allocated-compose.yml` runs `make verify-allocated-compose`. Full queue/proposal/allocation orchestration over HTTP, live Docker evidence from this workspace, and legacy fixture non-regression remain open |
| 8.49 `[D:8.25,8.26,8.28,8.29,8.30,8.31,8.35,8.36]` | **IN PROGRESS.** `scripts/verify_kind_agones.sh` creates a disposable kind cluster, installs pinned Agones, loads the real `game-server` image, applies the Fleet in an explicitly separate Agones-only supervisor/UDP readiness mode, and verifies readiness plus allocation of a dynamic UDP endpoint; `.github/workflows/agones-integration.yml` runs it for infrastructure changes and on demand | The cloud-free runner is committed and fails clearly when Docker/kind/Helm are unavailable. CI/live evidence for production control-plane registration, roster/no-show, both readiness stages, races, multi-match node, result-pending reconciliation, drain, and rollback remains open |
| 8.50 `[D:8.25,8.37,8.43,8.49]` | Network/chaos suite: 100 ms RTT, jitter/loss, client/API/matcher restart, game-pod death, node drain, Redis failover and control-plane loss | System recovers to a defined state; infrastructure-caused cases cannot penalise affected players |
+43
View File
@@ -94,6 +94,49 @@ curl -fsS -o /dev/null -X POST "$api_url/v1/queue/compose-queue-ticket/cancel" \
-H 'Idempotency-Key: compose-cancel-key-123456' \
-H "If-Match-Revision: $heartbeat_revision"
# Drive six independent authenticated players through the real queue boundary;
# the matcher service consumes the durable rows below and creates the proposal.
match_tokens=()
for player in 1 2 3 4 5 6; do
player_session="$(curl -fsS -X POST "$api_url/v1/session/steam" \
-H 'Content-Type: application/json' -d "{\"web_api_ticket\":\"compose-match-player-${player}\"}")"
match_tokens+=("$(python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])' <<<"$player_session")")
curl -fsS -o /dev/null -X POST "$api_url/v1/queue" \
-H "Authorization: Bearer ${match_tokens[$((player - 1))]}" \
-H "Idempotency-Key: compose-match-queue-key-${player}-123456" \
-H 'Content-Type: application/json' \
-d "{\"ticket_id\":\"compose-match-ticket-${player}\",\"playlist\":\"casual\",\"client_build\":\"build-1\",\"protocol_version\":1}"
done
"${compose[@]}" exec -T database psql -v ON_ERROR_STOP=1 -U cosmic_clash_test -d cosmic_clash_test -c \
"UPDATE queue_tickets SET predicted_rtt = '{\"EU\":30}'::jsonb WHERE ticket_id LIKE 'compose-match-ticket-%'" >/dev/null
proposal_id=""
for attempt in $(seq 1 30); do
proposal_id="$("${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT proposal_id FROM proposals WHERE state = 'OPEN' ORDER BY created_at DESC LIMIT 1" | tr -d '\r')"
if [[ -n "$proposal_id" ]]; then break; fi
[[ "$attempt" == 30 ]] && { echo "matcher did not create a proposal" >&2; exit 1; }
sleep 1
done
proposal_json="$(curl -fsS -H "Authorization: Bearer ${match_tokens[0]}" "$api_url/v1/proposals/$proposal_id")"
python3 - "$proposal_json" <<'PY'
import json, sys
proposal = json.loads(sys.argv[1])
assert proposal["state"] == "OPEN"
assert len(proposal["participants"]) == 6
print("proposal formation check passed")
PY
proposal_revision=0
for player in 1 2 3 4 5 6; do
proposal_json="$(curl -fsS -X POST "$api_url/v1/proposals/$proposal_id/accept" \
-H "Authorization: Bearer ${match_tokens[$((player - 1))]}" \
-H "Idempotency-Key: compose-proposal-accept-${player}-123456" \
-H "If-Match-Revision: $proposal_revision")"
proposal_revision="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["revision"])' <<<"$proposal_json")"
done
[[ "$("${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT state FROM proposals WHERE proposal_id = '$proposal_id'" | tr -d '\r')" == ACCEPTED ]]
[[ "$("${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM matches WHERE state = 'ALLOCATING'" | tr -d '\r')" == 1 ]]
# Model the durable state produced by the allocator, then use the real HTTP
# workload authentication and mutation boundaries for every action below.
"${compose[@]}" exec -T database psql -v ON_ERROR_STOP=1 -U cosmic_clash_test -d cosmic_clash_test <<'SQL'
@@ -20,6 +20,7 @@ class ComposeManifestTest(unittest.TestCase):
def test_allocated_runner_checks_durable_retry_and_shutdown(self):
runner = (ROOT / "scripts/verify_allocated_compose.sh").read_text()
allocated = (ROOT / "compose.allocated-smoke.yml").read_text()
for marker in (
"/v1/servers/compose-server/result",
"compose-result-key-123456",
@@ -29,9 +30,12 @@ class ComposeManifestTest(unittest.TestCase):
"/v1/session/steam",
"compose-queue-key-123456",
"/v1/queue/compose-queue-ticket/heartbeat",
"/v1/proposals/$proposal_id/accept",
"compose-match-ticket-",
"down --volumes --remove-orphans",
):
self.assertIn(marker, runner)
self.assertIn("target: matcher", allocated)
if __name__ == "__main__":