fix(multiplayer): repair allocated compose verification

This commit is contained in:
Josh Creek
2026-09-04 16:38:10 +01:00
parent e6733bd6cb
commit d64920b0f9
11 changed files with 150 additions and 48 deletions
+16 -4
View File
@@ -39,8 +39,13 @@ services:
agones-provider:
image: python:3.12-alpine
command: ["python3", "/opt/fake_agones_provider.py"]
environment:
FAKE_AGONES_TLS_CERT: /run/cosmic-clash/fake-agones.crt
FAKE_AGONES_TLS_KEY: /run/cosmic-clash/fake-agones.key
volumes:
- ./scripts/fake_agones_provider.py:/opt/fake_agones_provider.py:ro
- ${COMPOSE_SMOKE_DIR:-/tmp/cosmic-clash-allocated-smoke}/fake-agones.crt:/run/cosmic-clash/fake-agones.crt:ro
- ${COMPOSE_SMOKE_DIR:-/tmp/cosmic-clash-allocated-smoke}/fake-agones.key:/run/cosmic-clash/fake-agones.key:ro
allocator:
build:
@@ -48,15 +53,20 @@ services:
target: allocator
environment:
COSMIC_CLASH_POSTGRES_DSN: postgres://cosmic_clash_test:cosmic_clash_test@database:5432/cosmic_clash_test?sslmode=disable
COSMIC_CLASH_AGONES_URL: http://agones-provider:8080
COSMIC_CLASH_AGONES_URL: https://agones-provider:8443
COSMIC_CLASH_AGONES_NAMESPACE: cosmic-clash
COSMIC_CLASH_WORKLOAD_SECRET: compose-workload-secret
COSMIC_CLASH_KUBERNETES_TOKEN_PATH: /run/cosmic-clash/kubernetes-token
COSMIC_CLASH_KUBERNETES_CA_PATH: /run/cosmic-clash/fake-agones.crt
command: ["--dsn=postgres://cosmic_clash_test:cosmic_clash_test@database:5432/cosmic_clash_test?sslmode=disable", "--migrations=/opt/cosmic-clash/migrations", "--interval=1s", "--transport=enet"]
depends_on:
database:
condition: service_healthy
agones-provider:
condition: service_started
volumes:
- ${COMPOSE_SMOKE_DIR:-/tmp/cosmic-clash-allocated-smoke}/kubernetes-token:/run/cosmic-clash/kubernetes-token:ro
- ${COMPOSE_SMOKE_DIR:-/tmp/cosmic-clash-allocated-smoke}/fake-agones.crt:/run/cosmic-clash/fake-agones.crt:ro
maintenance:
build:
@@ -78,11 +88,11 @@ services:
- --drain-token-env=COSMIC_CLASH_DRAIN_TOKEN
- --drain-grace=10s
- --
- /opt/cosmic-clash/CosmicClashServer.x86_64
- /opt/cosmic-clash/cosmic-clash-server
- --port=31001
- --allocated-mode
- --match-id=compose-match
- --server-id=compose-server
- --match-id=compose-match-0001
- --server-id=compose-server-0001
- --playlist-version=casual
- --playlist=casual
- --client-build=build-1
@@ -95,6 +105,8 @@ services:
- --readiness-port=7780
environment:
COSMIC_CLASH_DRAIN_TOKEN: compose-drain-token
COSMIC_CLASH_CONTROL_PLANE_URL: http://control-plane:8080
COSMIC_CLASH_WORKLOAD_TOKEN: ${COSMIC_CLASH_COMPOSE_WORKLOAD_TOKEN:?allocated smoke workload token is required}
volumes:
- ${COMPOSE_SMOKE_DIR:-/tmp/cosmic-clash-allocated-smoke}/join-roster.json:/run/cosmic-clash/join-roster.json:ro
- ${COMPOSE_SMOKE_DIR:-/tmp/cosmic-clash-allocated-smoke}/join-signing-key:/run/secrets/cosmic-clash/join-signing-key:ro
+2 -2
View File
@@ -1213,7 +1213,7 @@ production fallback.
| 8.22 `[D:8.21]` | **IN PROGRESS.** Pure Go ranked profile exposes the first ten games as provisional, derives tiers only through validated backend-owned rating bands, and keeps casual ratings outside the API; authenticated HTTP now returns the authoritative ranked view. **"Authoritative ranked view" was durable-adapter-shaped but had no durable adapter**: `rankedProfile`/`profile` only ever read an in-memory map, so every real `GET /v1/profile/ranked` 404'd regardless of a player's actual rating. `RankedProfileProvider` (interface) + `store.PostgresRankedProfiles` close it, preferred over the map when set so existing tests/literals are unaffected; `LastSeasonID`/`SeasonHistory` deliberately left unset (no season pointer on `ratings`, needs its own query/semantics) | `server/domain/rating.go`, `tier_test.go` and `server/api/service.go` cover provisional override, exact band boundaries, malformed policy rejection, session authentication and ranked-only response fields; `server/store/ranked_profile_sql.go`, verified against real PostgreSQL via curl (a fresh identity correctly 404s through the real adapter) and via §8.40's integration test (`control_plane_smoke.gd` now asserts this exact 404 round-trips before queueing); persisted tier policy, client UI and reconnect transport remain |
| 8.23 `[D:8.21]` | **IN PROGRESS.** Pure Go ranked-only season policy compresses 25% toward 1500, clamps RD to 200350, preserves volatility/history, is idempotent by season ID, and defines exact 12-week windows/due detection; migration and Go store now persist a per-player/per-season marker and rating update atomically; `cmd/maintenance` runs bounded due-season batches with signal-bound shutdown | `server/domain/rating.go`, `season_test.go`, `server/migrations/0001_initial.sql` and `server/store/maintenance_sql.go` cover compression, floor/cap, duplicate replay, window boundary, completed-season idempotence, bounded enumeration, row locking and conflict-safe rollover markers; opt-in PostgreSQL execution now covers the durable rating update, marker creation and duplicate replay without a second compression — re-run live for the first time as part of the wider integration-suite verification below, after fixing a test setup gap (a missing `seasons` row tripped the `ranked_season_rollovers` foreign key before the rollover logic itself ran); live maintenance/DB execution remains |
| 8.24 `[D:8.9,8.20,8.21]` | **IN PROGRESS.** Ranked connection policy binds match/server/player/Steam identity/team/global slot/protocol/expiry, permits a 60-second same-token reclaim with monotonically increasing server-owned generations, and applies the rolling 7-day 5 m/15 m/1 h/24 h abandon ladder. Admission rejects active duplicates, zero/reversed clocks, disconnect-before-admit, and duplicate disconnects. PostgreSQL persists generation/disconnect leases with serializable CAS: stale disconnects cannot evict newer generations, active leases cannot be reclaimed, initial admission requires an unexpired assignment, and later reconnects use the durable grace boundary. Godot consumes that lease before admission, reconciles known-generation outage events in order, closes future admissions on reconciliation divergence, and rejects unsafe generation-zero outage fallback. The hardened two-replica Kubernetes maintenance deployment and allocated Compose topology run the reconciler, which turns a ranked `LIVE` lease expired beyond 60 seconds into `abandoned_at`, a durable `MATCH_ABANDONED` cooldown, and a revisioned, targeted `state_changed` outbox event without releasing the participant or `LIVE` ticket needed by the result transaction | Go/store/API/Godot adversarial fixtures cover signature tampering, every binding, replay/conflict semantics, active duplicate admission, repeated valid reclaim, stale-generation fencing, grace boundaries, process recovery, expiry, zero/reversed clocks, malformed JSON generations, deterministic cooldown ordering, legacy-row migration, rolling-upgrade 204 compatibility, result-roster retention, outbox dispatch compatibility, maintenance deployment/PDB hardening, and cursor-pool safety. The current local gate passed all 212 Godot tests; focused Go suites and the PostgreSQL-tagged abandonment regression compile. Allocated Compose now seeds and observes the durable live-abandonment path; live PostgreSQL/process-restart/outage execution remains blocked by Docker storage |
| 8.25 `[D:8.10,8.24]` | **IN PROGRESS.** Workload-bound result policy validates match/server identity, nonce, scores, integrity state, canonical digest, and idempotent receipt conflicts. Result ingestion now owns the complete serializable production transition: a `LIVE` match and its active tickets advance through `RESULT_PENDING`, certified ratings are computed from locked authoritative participant rows, then match/tickets become `COMPLETED`, the receipt is acknowledged, and one revisioned outbox event is inserted. The allocated Godot server now emits the authoritative score to that route at `RESULTS`; its deterministic score-bound nonce makes every retry identical, and the match cannot leave `RESULTS` or exit until the API returns its committed `202` acknowledgement. Inactive pre-match no-shows are excluded; an active participant's durable `abandoned_at` forces loss scoring. Missing rating rows or ticket-count divergence fail the whole transaction. Zero-time/incomplete receipts fail before database use and conflicts wrap `ErrResultConflict` | Domain/store/API/outbox tests cover workload and digest binding, identical/conflicting concurrency, direct `LIVE` completion, active-ticket completion, abandonment rating input, incomplete roster failure, integrity suppression, ordered rating locks, receipt/outbox atomicity, fan-out retry/ack ordering, and delivery health. `Game/scripts/server_result_client.gd` and the Godot harness cover deterministic score-bound nonces, fail-closed configuration, and the exact committed-ack boundary. PostgreSQL-tagged regressions compile; prior live result/rating/race/fan-out runs remain valid, while the direct-live lifecycle change awaits a live database rerun. Production credentials, Agones annotation persistence/reconciliation, and integrity-evidence adapters remain |
| 8.25 `[D:8.10,8.24]` | **IN PROGRESS.** Workload-bound result policy validates match/server identity, nonce, scores, integrity state, canonical digest, and idempotent receipt conflicts. Result ingestion now owns the complete serializable production transition: a `LIVE` match and its active tickets advance through `RESULT_PENDING`, certified ratings are computed from locked authoritative participant rows, then match/tickets become `COMPLETED`, the receipt is acknowledged, and one revisioned outbox event is inserted. The allocated Godot server now emits the authoritative score to that route at `RESULTS`; its deterministic score-bound nonce makes every retry identical, and the match cannot leave `RESULTS` or exit until the API returns its committed `202` acknowledgement. Inactive pre-match no-shows are excluded; an active participant's durable `abandoned_at` forces loss scoring. Missing rating rows or ticket-count divergence fail the whole transaction. Zero-time/incomplete receipts fail before database use and conflicts wrap `ErrResultConflict`. Signed workload credentials now correctly carry the durable allocation/match/server binding without pretending to be Kubernetes JWTs; partial Kubernetes identity claims remain rejected | Domain/store/API/outbox tests cover workload and digest binding, identical/conflicting concurrency, direct `LIVE` completion, active-ticket completion, abandonment rating input, incomplete roster failure, integrity suppression, ordered rating locks, receipt/outbox atomicity, fan-out retry/ack ordering, delivery health, and signed-binding versus partial-identity validation. The allocated Compose gate now exercises an authenticated certified result and identical retry against the real verifier/store. Production credentials, Agones annotation persistence/reconciliation, and integrity-evidence adapters remain |
The allocated-runtime result reporter now keeps a completed match in `RESULTS` until its exact score-bound, workload-authenticated result has received the API's committed `202`. Its 15-minute, server-side sudden-death cap turns an unresolved draw into `REVIEW` (no rating update), while allocator-issued workload tokens now default to two hours and expose a positive `--workload-token-ttl` setting. These bounds cover ordinary allocation, play, and result retry without treating a permanently unavailable control plane as a completed match.
@@ -1253,7 +1253,7 @@ The allocated-runtime result reporter now keeps a completed match in `RESULTS` u
| 8.45 `[D:8.2,8.44]` | **LOCAL COMPLETE; PRODUCTION GATE OPEN.** Go observability package turns the documented RTT, allocation/connect latency, result-success, API-latency and tick/headroom thresholds into executable window checks; the API exporter emits a bounded cumulative latency histogram suitable for querying the documented p95 API SLO | `server/observability/slo.go`, `metrics.go`, `deploy/observability/prometheus-rules.yaml` and adversarial tests cover healthy/violating/empty windows, fixed operation/status labels, cumulative bucket boundaries, arbitrary-path cardinality safety, and optional API p95/5xx alerts. Production scrape configuration, alert routing, wait/MMR/proposal/flood/cost series and runbooks remain |
| 8.46 `[D:8.5,8.7,8.9,8.10,8.14,8.18,8.21,8.23,8.25]` | **IN PROGRESS.** Go unit/race coverage spans the current domain/store/supervisor policies, and fuzz targets now exercise queue input, result payload hashing and revision events. The local gate uses the pinned headless Godot container when the native executable is unavailable or crashes by signal, while preserving ordinary nonzero test failures, so its full cross-language suite remains runnable without an image export | `scripts/verify_multiplayer_local.sh` passed end to end on the current tree: Go normal/race/vet, all three bounded fuzz targets, 212 Godot tests, contracts, migrations, and manifests. `server/domain/*_test.go`, `server/store/*_test.go`, `server/supervisor/*_test.go`, `server/migrations/*_test.go` and `server/domain/fuzz_test.go` provide the underlying coverage; PostgreSQL live migration execution now runs clean (§8.5), and four real-concurrency cases are covered against a live database with `-race`: §8.14's queue-heartbeat revision race, §8.18's two-matcher contested-ticket race, §8.30's cross-allocator-replica capacity race, and §8.21/§8.25's concurrent identical-result-submission race; the "lost Redis" fixture is covered live against a real server (§8.14: real TTL expiry, repair-after-`FLUSHALL`; fake Steam/allocator fixtures are §8.47's testkit, already done). Further transaction fixtures (e.g. concurrent proposal-recovery expiry races, live Redis failover mid-write under load) remain |
| 8.47 `[D:8.7,8.30]` | **IN PROGRESS.** Offline testkit provides deterministic fake Steam verification and fake allocation with forced failure injection; the independent Compose runner drives fake-Steam session issuance, real HTTP queue create/heartbeat/cancel, matcher-backed six-player proposal formation/acceptance, and idempotency-conflict checks | `server/testkit/` covers verified identity/replay, unknown identity, wrong App ID, expiry, no capacity, compatibility-key conflict, idempotent allocation replay and cloud-free forced allocation failure; the Compose API/matcher slice is wired into CI, while live exhaustive matrix and production Steam remain |
| 8.48 `[D:8.10,8.14,8.17,8.18,8.27,8.31,8.35,8.47]` | **IN PROGRESS.** Offline testkit exercises verified queue projection → ranked six-player proposal → ENet allocation → assignment-ready manifest → certified durable result receipt; `compose.allocated-smoke.yml` independently runs the real testkit API, matcher, allocator, maintenance, Agones-shaped provider, PostgreSQL, and game-server supervisor with a generated signed roster, verifying an expired ranked reconnect becomes one durable abandonment/cooldown, queue/proposal/allocation binding, authenticated result, idempotent retry, shutdown acknowledgment, durable receipt/audit rows, and SIGTERM-driven game-process drain | `.github/workflows/allocated-compose.yml` runs `make verify-allocated-compose`. Live Docker evidence from this workspace and legacy fixture non-regression remain open |
| 8.48 `[D:8.10,8.14,8.17,8.18,8.27,8.31,8.35,8.47]` | **LOCAL COMPLETE; PRODUCTION GATE OPEN.** Offline testkit exercises verified queue projection → ranked six-player proposal → ENet allocation → assignment-ready manifest → certified durable result receipt; `compose.allocated-smoke.yml` independently runs the real testkit API, matcher, allocator, maintenance, HTTPS Agones-shaped provider, PostgreSQL, and game-server supervisor with generated TLS, roster, and signed workload credentials. It verifies an expired ranked reconnect becomes one durable abandonment/cooldown, queue/proposal/allocation binding, authenticated result, idempotent retry, shutdown acknowledgment, durable receipt/audit rows, and supervised game-process stop without repurposing the Phase 6 fixture | `scripts/verify_allocated_compose.sh` passed on 2026-09-04 in this workspace; `.github/workflows/allocated-compose.yml` runs `make verify-allocated-compose`. Real Agones/kind and production evidence remain open |
| 8.49 `[D:8.25,8.26,8.28,8.29,8.30,8.31,8.35,8.36]` | **IN PROGRESS.** `scripts/verify_kind_agones.sh` creates a disposable kind cluster, installs pinned Agones, loads the real `game-server` image, applies the Fleet in an explicitly separate Agones-only supervisor/UDP readiness mode, and verifies readiness plus allocation of a dynamic UDP endpoint; `.github/workflows/agones-integration.yml` runs it for infrastructure changes and on demand | The cloud-free runner is committed and fails clearly when Docker/kind/Helm are unavailable. CI/live evidence for production control-plane registration, roster/no-show, both readiness stages, races, multi-match node, result-pending reconciliation, drain, and rollback remains open |
| 8.50 `[D:8.25,8.37,8.43,8.49]` | **IN PROGRESS.** `make verify-chaos-recovery` provides a disposable PostgreSQL + real testkit API + real maintenance flow: it restarts the API, injects a stale allocation, and verifies no-penalty requeue plus a durable participant-targeted lifecycle event | The API-restart/stalled-allocation slice is implemented and documented; 100 ms RTT/jitter/loss, matcher/client restart, game-pod death, node drain, Redis failover, control-plane loss, and live chaos evidence remain |
| 8.51 `[D:8.17,8.18,8.30,8.31,8.45]` | **IN PROGRESS.** The opt-in `make verify-multiplayer-load` gate drives 10,000 real HTTP queue-create requests through the service with 256 in flight and records p95/p99, plus 100 concurrent proposal formations through the real matcher/domain path; the handler and in-process ownership boundary are exercised without weakening normal tests | Local API load passes at p95 <250 ms in normal and race runs, and the matcher forms 100 unique proposals; PostgreSQL saturation, durable matcher fencing under load, forecast launch concurrency x2, and replica scaling remain live infrastructure gates |
+7 -1
View File
@@ -1,6 +1,8 @@
#!/usr/bin/env python3
"""Minimal deterministic Agones HTTP surface for the allocated Compose smoke."""
import json
import os
import ssl
from http.server import BaseHTTPRequestHandler, HTTPServer
@@ -41,4 +43,8 @@ class Handler(BaseHTTPRequestHandler):
return
HTTPServer(("0.0.0.0", 8080), Handler).serve_forever()
server = HTTPServer(("0.0.0.0", 8443), Handler)
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
context.load_cert_chain(os.environ["FAKE_AGONES_TLS_CERT"], os.environ["FAKE_AGONES_TLS_KEY"])
server.socket = context.wrap_socket(server.socket, server_side=True)
server.serve_forever()
+43 -31
View File
@@ -13,6 +13,10 @@ compose=(docker compose -p "$project" -f "$compose_file")
cleanup() {
local rc=$?
if [[ "$rc" != 0 && "${COMPOSE_KEEP_ON_FAILURE:-}" == 1 ]]; then
echo "allocated Compose fixture retained for inspection: ${project}" >&2
exit "$rc"
fi
"${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true
exit "$rc"
}
@@ -28,7 +32,7 @@ import base64, hashlib, hmac, json, pathlib, sys, time
directory = pathlib.Path(sys.argv[1])
key = b"compose-join-signing-key"
expires = "2099-12-31T00:00:00Z"
fields = ["compose-match", "compose-server", "compose-player", "compose-steam", "0", "0", "v1", "1", expires]
fields = ["compose-match-0001", "compose-server-0001", "compose-player", "compose-steam", "0", "0", "v1", "1", expires]
canonical = b"\0".join(field.encode() for field in fields)
signature = base64.urlsafe_b64encode(hmac.new(key, canonical, hashlib.sha256).digest()).rstrip(b"=").decode()
envelope = {"Authorisation": {"MatchID": fields[0], "ServerID": fields[1], "PlayerID": fields[2], "SteamID": fields[3], "Slot": 0, "Team": 0, "Protocol": fields[6], "Generation": 1, "ExpiresAt": expires}, "Signature": signature}
@@ -36,14 +40,37 @@ envelope = {"Authorisation": {"MatchID": fields[0], "ServerID": fields[1], "Play
(directory / "join-roster.json").write_text(json.dumps([base64.urlsafe_b64encode(json.dumps(envelope, separators=(",", ":")).encode()).rstrip(b"=").decode()]) + "\n")
PY
command -v openssl >/dev/null 2>&1 || { echo "OpenSSL is required for the HTTPS fake Kubernetes API" >&2; exit 2; }
printf 'compose-kubernetes-token' > "$smoke_dir/kubernetes-token"
openssl req -x509 -newkey rsa:2048 -nodes -days 1 \
-subj '/CN=agones-provider' -addext 'subjectAltName=DNS:agones-provider' \
-keyout "$smoke_dir/fake-agones.key" -out "$smoke_dir/fake-agones.crt" >/dev/null 2>&1
token="$(python3 - "$secret" <<'PY'
import base64, datetime, hashlib, hmac, json, sys
secret = sys.argv[1].encode()
payload = {"a": "compose-allocation-0001", "e": (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(hours=1)).isoformat().replace("+00:00", "Z")}
encoded = base64.urlsafe_b64encode(json.dumps(payload, separators=(",", ":")).encode()).rstrip(b"=")
signature = hmac.new(secret, encoded, hashlib.sha256).digest()
sig = base64.urlsafe_b64encode(signature).rstrip(b"=")
print(encoded.decode() + "." + sig.decode())
PY
)"
export COSMIC_CLASH_COMPOSE_WORKLOAD_TOKEN="$token"
"${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true
"${compose[@]}" up -d --build
for attempt in $(seq 1 60); do
if "${compose[@]}" logs game-server 2>/dev/null | grep -q '"event":"server_started"'; then
for attempt in $(seq 1 180); do
if "${compose[@]}" logs game-server 2>/dev/null | grep -q ' server_started '; then
break
fi
if [[ "$attempt" == 60 ]]; then
if ! "${compose[@]}" ps --status running --services | grep -qx game-server; then
"${compose[@]}" logs game-server >&2
echo "allocated Compose game server exited before becoming ready" >&2
exit 1
fi
if [[ "$attempt" == 180 ]]; then
"${compose[@]}" logs >&2
echo "allocated Compose game server did not become ready" >&2
exit 1
@@ -70,8 +97,8 @@ done
INSERT INTO identities (player_id, steam_id) VALUES ('compose-abandon-player', 'compose-abandon-steam');
INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at)
VALUES ('compose-abandon-ticket', 'compose-abandon-player', 'ranked', 'LIVE', 'build-1', 1, now() - interval '2 minutes', now() + interval '1 hour');
INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id)
VALUES ('compose-abandon-match', 'ranked', 'LIVE', 'EU', 1, 'compose-abandon-server');
INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, arena_path)
VALUES ('compose-abandon-match', 'ranked', 'LIVE', 'EU', 1, 'compose-abandon-server', 'res://scenes/arena_01.tscn');
INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team, connection_generation, connected_at, disconnected_at)
VALUES ('compose-abandon-match', 'compose-abandon-player', 'compose-abandon-ticket', 0, 0, 1, now() - interval '2 minutes', now() - interval '61 seconds');
SQL
@@ -168,57 +195,42 @@ done
# workload authentication and mutation boundaries for every action below.
"${compose[@]}" exec -T database psql -v ON_ERROR_STOP=1 -U cosmic_clash_test -d cosmic_clash_test <<'SQL'
INSERT INTO game_servers (server_id, region, build, protocol_version, transport, state)
VALUES ('compose-server', 'EU', 'build-1', 1, 'enet', 'ALLOCATED');
VALUES ('compose-server-0001', 'EU', 'build-1', 1, 'enet', 'ALLOCATED');
INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, revision)
VALUES ('compose-match', 'casual', 'RESULT_PENDING', 'EU', 1, 'compose-server', 2);
VALUES ('compose-match-0001', 'casual', 'RESULT_PENDING', 'EU', 1, 'compose-server-0001', 2);
INSERT INTO allocations (allocation_id, match_id, server_id, region, build, protocol_version, transport, request_digest, state, allocated_at)
VALUES ('compose-allocation', 'compose-match', 'compose-server', 'EU', 'build-1', 1, 'enet', decode(repeat('00', 32), 'hex'), 'ALLOCATED', now());
VALUES ('compose-allocation-0001', 'compose-match-0001', 'compose-server-0001', 'EU', 'build-1', 1, 'enet', decode(repeat('00', 32), 'hex'), 'ALLOCATED', now());
SQL
token="$(python3 - "$secret" <<'PY'
import base64, hashlib, hmac, json, sys, time
secret = sys.argv[1].encode()
payload = {"a": "compose-allocation", "e": time.time() + 300}
encoded = base64.urlsafe_b64encode(json.dumps(payload, separators=(",", ":")).encode()).rstrip(b"=")
signature = hmac.new(secret, encoded, hashlib.sha256).digest()
sig = base64.urlsafe_b64encode(signature).rstrip(b"=")
print(encoded.decode() + "." + sig.decode())
PY
)"
result_body='{"match_id":"compose-match","result_nonce":"compose-result-nonce-1234","score":{"team_0":3,"team_1":2},"integrity_state":"CERTIFIED"}'
result_body='{"match_id":"compose-match-0001","result_nonce":"compose-result-nonce-1234","score":{"team_0":3,"team_1":2},"integrity_state":"CERTIFIED"}'
curl -fsS -o /dev/null -w '%{http_code}' \
-X POST "$api_url/v1/servers/compose-server/result" \
-X POST "$api_url/v1/servers/compose-server-0001/result" \
-H "Authorization: Bearer $token" \
-H 'Idempotency-Key: compose-result-key-123456' \
-H 'Content-Type: application/json' -d "$result_body" | grep -qx 202
# An identical retry must be acknowledged without a second receipt.
curl -fsS -o /dev/null -w '%{http_code}' \
-X POST "$api_url/v1/servers/compose-server/result" \
-X POST "$api_url/v1/servers/compose-server-0001/result" \
-H "Authorization: Bearer $token" \
-H 'Idempotency-Key: compose-result-key-123456' \
-H 'Content-Type: application/json' -d "$result_body" | grep -qx 202
"${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT state FROM matches WHERE match_id = 'compose-match'" | grep -qx COMPLETED
"${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM result_receipts WHERE match_id = 'compose-match'" | grep -qx 1
"${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT state FROM matches WHERE match_id = 'compose-match-0001'" | grep -qx COMPLETED
"${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM result_receipts WHERE match_id = 'compose-match-0001'" | grep -qx 1
curl -fsS -o /dev/null -w '%{http_code}' \
-X POST "$api_url/v1/servers/compose-server/shutdown" \
-X POST "$api_url/v1/servers/compose-server-0001/shutdown" \
-H "Authorization: Bearer $token" \
-H 'Idempotency-Key: compose-shutdown-key-123456' \
-H 'Content-Type: application/json' -d '{"reason":"server_draining"}' | grep -qx 204
"${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM audit_events WHERE action = 'SERVER_SHUTDOWN' AND aggregate_id = 'compose-match'" | grep -qx 1
"${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM audit_events WHERE action = 'SERVER_SHUTDOWN' AND aggregate_id = 'compose-match-0001'" | grep -qx 1
"${compose[@]}" stop -t 12 game-server >/dev/null
if "${compose[@]}" ps --status running --services | grep -qx game-server; then
echo "allocated game-server did not stop after supervisor drain" >&2
exit 1
fi
if ! "${compose[@]}" logs game-server | grep -q '"event":"server_draining"'; then
echo "allocated game-server did not record a drain request" >&2
exit 1
fi
"${compose[@]}" stop -t 10 control-plane >/dev/null
echo "8.48 PASS: allocated Compose HTTP result/retry/shutdown and supervisor drain completed"
+16 -1
View File
@@ -177,7 +177,22 @@ func RatingEligible(receipt ResultReceipt) bool {
}
func validateBinding(binding WorkloadBinding) error {
if binding.Issuer == "" || binding.Audience == "" || binding.Namespace == "" || binding.ServiceAcct == "" || binding.PodUID == "" || binding.GameServerUID == "" || binding.AllocationID == "" || binding.MatchID == "" || binding.ServerID == "" {
// A Kubernetes JWT supplies the six workload-identity fields below, while
// the signed workload credential is deliberately bound through the durable
// allocation record and therefore supplies only allocation/match/server.
// Accept either complete authority model, but never a partial Kubernetes
// identity that could accidentally look authenticated.
if binding.AllocationID == "" || binding.MatchID == "" || binding.ServerID == "" {
return ErrResultBinding
}
kubernetesIdentity := []string{binding.Issuer, binding.Audience, binding.Namespace, binding.ServiceAcct, binding.PodUID, binding.GameServerUID}
present := 0
for _, value := range kubernetesIdentity {
if value != "" {
present++
}
}
if present != 0 && present != len(kubernetesIdentity) {
return ErrResultBinding
}
return nil
+14
View File
@@ -47,6 +47,20 @@ func TestResultStoreRejectsMissingAuthoritativeTime(t *testing.T) {
}
}
func TestResultStoreAcceptsDurablyBoundSignedWorkloadIdentity(t *testing.T) {
// Signed workload tokens resolve this three-part binding from the durable
// allocation record; they intentionally carry no Kubernetes JWT claims.
binding := WorkloadBinding{AllocationID: "allocation-1", MatchID: "match-1", ServerID: "server-1"}
if _, err := NewResultStore(binding); err != nil {
t.Fatalf("signed workload binding rejected: %v", err)
}
partial := binding
partial.Issuer = "https://issuer"
if _, err := NewResultStore(partial); !errors.Is(err, ErrResultBinding) {
t.Fatalf("partial Kubernetes identity error = %v, want ErrResultBinding", err)
}
}
func TestConflictingResultIsInertAndIntegritySuppressesRating(t *testing.T) {
now := time.Unix(1000, 0)
binding := testBinding()
+32 -6
View File
@@ -15,6 +15,32 @@ const migrationTableSQL = `CREATE TABLE IF NOT EXISTS schema_migrations (
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
)`
const migrationLockSQL = `SELECT pg_advisory_xact_lock(hashtext('cosmic-clash:migrations'))`
// ensureMigrationTable serializes the bootstrap DDL itself. PostgreSQL's
// CREATE TABLE IF NOT EXISTS is not safe against concurrent first creation:
// the relation-type catalog entry can still collide before either statement
// observes the other table. Every long-lived role calls Apply at startup, so
// take the same transaction-scoped advisory lock used for individual files
// before issuing the bootstrap statement.
func ensureMigrationTable(ctx context.Context, db *sql.DB) error {
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return fmt.Errorf("begin migration bootstrap: %w", err)
}
defer tx.Rollback()
if _, err := tx.ExecContext(ctx, migrationLockSQL); err != nil {
return fmt.Errorf("lock migration bootstrap: %w", err)
}
if _, err := tx.ExecContext(ctx, migrationTableSQL); err != nil {
return fmt.Errorf("create migration table: %w", err)
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("commit migration bootstrap: %w", err)
}
return nil
}
// Apply executes numbered SQL files in lexical order. A transaction-level
// advisory lock serializes concurrent API/worker starts, while each migration
// is committed together with its schema_migrations marker so a failed
@@ -31,8 +57,8 @@ func Apply(ctx context.Context, db *sql.DB, directory string) error {
if len(paths) == 0 {
return fmt.Errorf("no migrations found in %s", directory)
}
if _, err := db.ExecContext(ctx, migrationTableSQL); err != nil {
return fmt.Errorf("create migration table: %w", err)
if err := ensureMigrationTable(ctx, db); err != nil {
return err
}
for _, path := range paths {
version := filepath.Base(path)
@@ -50,7 +76,7 @@ func Apply(ctx context.Context, db *sql.DB, directory string) error {
_ = tx.Rollback()
}
}()
if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtext('cosmic-clash:migrations'))`); err != nil {
if _, err := tx.ExecContext(ctx, migrationLockSQL); err != nil {
return fmt.Errorf("lock migration %s: %w", version, err)
}
var applied bool
@@ -86,8 +112,8 @@ func Rollback(ctx context.Context, db *sql.DB, directory string, steps int) erro
if db == nil || strings.TrimSpace(directory) == "" || steps <= 0 {
return fmt.Errorf("database, migration directory and a positive step count are required")
}
if _, err := db.ExecContext(ctx, migrationTableSQL); err != nil {
return fmt.Errorf("create migration table: %w", err)
if err := ensureMigrationTable(ctx, db); err != nil {
return err
}
rows, err := db.QueryContext(ctx, `SELECT version FROM schema_migrations ORDER BY version DESC LIMIT $1`, steps)
if err != nil {
@@ -123,7 +149,7 @@ func Rollback(ctx context.Context, db *sql.DB, directory string, steps int) erro
_ = tx.Rollback()
}
}()
if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtext('cosmic-clash:migrations'))`); err != nil {
if _, err := tx.ExecContext(ctx, migrationLockSQL); err != nil {
return fmt.Errorf("lock rollback %s: %w", version, err)
}
var applied bool
+2 -2
View File
@@ -22,10 +22,10 @@ class ComposeManifestTest(unittest.TestCase):
runner = (ROOT / "scripts/verify_allocated_compose.sh").read_text()
allocated = (ROOT / "compose.allocated-smoke.yml").read_text()
for marker in (
"/v1/servers/compose-server/result",
"/v1/servers/compose-server-0001/result",
"compose-result-key-123456",
"result_receipts",
"/v1/servers/compose-server/shutdown",
"/v1/servers/compose-server-0001/shutdown",
"SERVER_SHUTDOWN",
"/v1/session/steam",
"compose-queue-key-123456",
+6 -1
View File
@@ -81,9 +81,14 @@ func PromoteStoredAcceptedProposal(ctx context.Context, db *sql.DB, proposalID s
return fmt.Errorf("invalid stored proposal promotion arguments")
}
plan := AcceptedMatchPlan{MatchID: "match-" + proposalID, ProposalID: proposalID}
if err := db.QueryRowContext(ctx, StoredProposalMatchPlanSQL, proposalID).Scan(&plan.Region, &plan.Protocol, &plan.ArenaPath); err != nil {
// Casual proposals intentionally persist no arena path. Scan it as nullable
// here just as the in-transaction promotion path does, so an API retry after
// the atomic promotion does not turn a successful acceptance into a 503.
var arenaPath sql.NullString
if err := db.QueryRowContext(ctx, StoredProposalMatchPlanSQL, proposalID).Scan(&plan.Region, &plan.Protocol, &arenaPath); err != nil {
return err
}
plan.ArenaPath = arenaPath.String
rows, err := db.QueryContext(ctx, StoredProposalMatchPlayersSQL, proposalID)
if err != nil {
return err
@@ -778,6 +778,12 @@ func TestPostgreSQLProposalClaimAndResponseAreAtomic(t *testing.T) {
if accepted.State != domain.Accepted || accepted.Revision != 2 {
t.Fatalf("proposal did not close after unanimous acceptance: %+v", accepted)
}
// The API's post-commit recovery promoter must accept the nullable casual
// arena path left by the atomic response transaction and converge on the
// already-created match.
if err := PromoteStoredAcceptedProposal(ctx, db, proposal.ProposalID, now.Add(time.Second)); err != nil {
t.Fatalf("replay persisted casual promotion: %v", err)
}
var matchState string
if err := db.QueryRow(`SELECT state FROM matches WHERE match_id = 'match-proposal-integration'`).Scan(&matchState); err != nil {
t.Fatalf("atomic accepted match: %v", err)
+6
View File
@@ -226,6 +226,12 @@ func (s *Supervisor) Start(ctx context.Context) error {
s.cmd = exec.CommandContext(ctx, s.config.Command[0], s.config.Command[1:]...)
}
s.cmd.Env = env
// A server's structured stdout/stderr is its operational interface. The
// zero value for exec.Cmd streams is /dev/null, which would make a child
// startup failure invisible to Docker, Kubernetes, and the Compose
// readiness harness while the supervisor can report only "exit status 1".
s.cmd.Stdout = os.Stdout
s.cmd.Stderr = os.Stderr
if err := s.cmd.Start(); err != nil {
return err
}