Files
CosmicClash/scripts/verify_allocated_compose.sh
T
Josh Creek 5765532409 fix(allocator): publish signed assignment rosters before servers start
The root blocker (issue #14). The worker bound the provider allocation
and stopped. Service.PublishRoster and store.SaveVerifiedAssignmentRoster
both existed, fully tested, with zero non-test callers, and the
production allocator configured neither a roster store nor a signing
key. Nothing ever wrote the assignments table.

The allocated supervisor fetches a non-empty roster before it launches
the game child, so every real allocation failed at that fetch: no match
could reach ASSIGNMENT_READY or accept a player. Existing tests seeded
assignments directly, which is exactly why the missing hand-off went
unnoticed.

The worker now builds one join authorisation per durable participant,
signs each with the active key, and publishes them. Participants are
read through the same query SaveVerifiedAssignmentRoster re-validates
against, so the allocator cannot construct a roster the persistence
boundary would reject. The manifest commits to a digest over the whole
roster, so a server cannot be handed a truncated roster whose surviving
entries are each individually valid.

Persist the provider endpoint on the allocation: it arrived on the
provider response and was never stored, so a worker crashing between
allocating and publishing had no endpoint to recover and would have
stranded the match permanently. Republishing is idempotent, so that
crash now simply retries.

cmd/allocator refuses to start without key material rather than running
an allocator that binds allocations and silently strands every match.
The k8s allocator Deployment mounts the same key set the Fleet does, and
both now take the JSON key map so a rotation can publish several.

New integration test drives the real worker through to the supervisor's
own roster read path without seeding the assignments table. Verified it
fails with "assignments = 0, want 2" when the publish step is removed.
2026-09-05 10:42:31 +01:00

241 lines
14 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
# Independent allocated-flow fixture for multiplayer-next.md §8.48. This
# intentionally does not call compose.phase6-smoke.yml or reuse its ports.
root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
compose_file="$root_dir/compose.allocated-smoke.yml"
project="${COMPOSE_PROJECT_NAME:-cosmic-clash-allocated-smoke}"
api_url="http://127.0.0.1:18080"
secret="compose-workload-secret"
smoke_dir="${COMPOSE_SMOKE_DIR:-/tmp/cosmic-clash-allocated-smoke}"
compose=(docker compose -p "$project" -f "$compose_file")
cleanup() {
local rc=$?
if [[ "$rc" != 0 && "${COMPOSE_KEEP_ON_FAILURE:-}" == 1 ]]; then
echo "allocated Compose fixture retained for inspection: ${project}" >&2
exit "$rc"
fi
"${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true
exit "$rc"
}
trap cleanup EXIT
command -v docker >/dev/null 2>&1 || { echo "Docker is required for 8.48" >&2; exit 2; }
docker info >/dev/null 2>&1 || { echo "A running Docker daemon is required for 8.48" >&2; exit 2; }
mkdir -p "$smoke_dir"
python3 - "$smoke_dir" <<'PY'
import base64, hashlib, hmac, json, pathlib, sys, time
directory = pathlib.Path(sys.argv[1])
key = b"compose-join-signing-key"
key_id = "compose-key-1"
expires = "2099-12-31T00:00:00Z"
# Field order and the trailing key ID must match
# server/domain.JoinAuthorisationBytes and Game/scripts/match_net.gd.
fields = ["compose-match-0001", "compose-server-0001", "compose-player", "compose-steam", "0", "0", "v1", "1", expires, key_id]
canonical = b"\0".join(field.encode() for field in fields)
signature = base64.urlsafe_b64encode(hmac.new(key, canonical, hashlib.sha256).digest()).rstrip(b"=").decode()
envelope = {"Authorisation": {"MatchID": fields[0], "ServerID": fields[1], "PlayerID": fields[2], "SteamID": fields[3], "Slot": 0, "Team": 0, "Protocol": fields[6], "Generation": 1, "ExpiresAt": expires, "KeyID": key_id}, "Signature": signature}
# The key file maps key ID -> base64 key so a rotation can publish several.
(directory / "join-signing-keys.json").write_text(json.dumps({key_id: base64.b64encode(key).decode()}) + "\n")
(directory / "join-roster.json").write_text(json.dumps([base64.urlsafe_b64encode(json.dumps(envelope, separators=(",", ":")).encode()).rstrip(b"=").decode()]) + "\n")
PY
command -v openssl >/dev/null 2>&1 || { echo "OpenSSL is required for the HTTPS fake Kubernetes API" >&2; exit 2; }
printf 'compose-kubernetes-token' > "$smoke_dir/kubernetes-token"
openssl req -x509 -newkey rsa:2048 -nodes -days 1 \
-subj '/CN=agones-provider' -addext 'subjectAltName=DNS:agones-provider' \
-keyout "$smoke_dir/fake-agones.key" -out "$smoke_dir/fake-agones.crt" >/dev/null 2>&1
token="$(python3 - "$secret" <<'PY'
import base64, datetime, hashlib, hmac, json, sys
secret = sys.argv[1].encode()
payload = {"a": "compose-allocation-0001", "e": (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(hours=1)).isoformat().replace("+00:00", "Z")}
encoded = base64.urlsafe_b64encode(json.dumps(payload, separators=(",", ":")).encode()).rstrip(b"=")
signature = hmac.new(secret, encoded, hashlib.sha256).digest()
sig = base64.urlsafe_b64encode(signature).rstrip(b"=")
print(encoded.decode() + "." + sig.decode())
PY
)"
export COSMIC_CLASH_COMPOSE_WORKLOAD_TOKEN="$token"
"${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true
"${compose[@]}" up -d --build
for attempt in $(seq 1 180); do
if "${compose[@]}" logs game-server 2>/dev/null | grep -q ' server_started '; then
break
fi
if ! "${compose[@]}" ps --status running --services | grep -qx game-server; then
"${compose[@]}" logs game-server >&2
echo "allocated Compose game server exited before becoming ready" >&2
exit 1
fi
if [[ "$attempt" == 180 ]]; then
"${compose[@]}" logs >&2
echo "allocated Compose game server did not become ready" >&2
exit 1
fi
sleep 1
done
for attempt in $(seq 1 60); do
if curl -fsS "$api_url/healthz" >/dev/null 2>&1; then
break
fi
if [[ "$attempt" == 60 ]]; then
"${compose[@]}" logs >&2
echo "allocated Compose control plane did not become ready" >&2
exit 1
fi
sleep 1
done
# The deployed maintenance service owns the ranked reconnect deadline. Seed an
# already-expired durable lease and require the real Compose process to record
# its cooldown before exercising the rest of the allocation flow.
"${compose[@]}" exec -T database psql -v ON_ERROR_STOP=1 -U cosmic_clash_test -d cosmic_clash_test <<'SQL'
INSERT INTO identities (player_id, steam_id) VALUES ('compose-abandon-player', 'compose-abandon-steam');
INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at)
VALUES ('compose-abandon-ticket', 'compose-abandon-player', 'ranked', 'LIVE', 'build-1', 1, now() - interval '2 minutes', now() + interval '1 hour');
INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, arena_path)
VALUES ('compose-abandon-match', 'ranked', 'LIVE', 'EU', 1, 'compose-abandon-server', 'res://scenes/arena_01.tscn');
INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team, connection_generation, connected_at, disconnected_at)
VALUES ('compose-abandon-match', 'compose-abandon-player', 'compose-abandon-ticket', 0, 0, 1, now() - interval '2 minutes', now() - interval '61 seconds');
SQL
for attempt in $(seq 1 30); do
abandoned="$(${compose[@]} exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM match_participants WHERE match_id = 'compose-abandon-match' AND abandoned_at IS NOT NULL" | tr -d '\r')"
[[ "$abandoned" == 1 ]] && break
[[ "$attempt" == 30 ]] && { "${compose[@]}" logs maintenance >&2; echo "maintenance did not record an expired ranked reconnect lease" >&2; exit 1; }
sleep 1
done
"${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM penalties WHERE match_id = 'compose-abandon-match' AND kind = 'MATCH_ABANDONED'" | grep -qx 1
session_json="$(curl -fsS -X POST "$api_url/v1/session/steam" \
-H 'Content-Type: application/json' -d '{"web_api_ticket":"compose-queue-ticket"}')"
access_token="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])' <<<"$session_json")"
queue_body='{"ticket_id":"compose-queue-ticket","playlist":"casual","client_build":"build-1","protocol_version":1}'
queue_json="$(curl -fsS -X POST "$api_url/v1/queue" \
-H "Authorization: Bearer $access_token" \
-H 'Idempotency-Key: compose-queue-key-123456' \
-H 'Content-Type: application/json' -d "$queue_body")"
queue_revision="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["revision"])' <<<"$queue_json")"
[[ "$queue_revision" == 0 ]]
# Reusing a queue idempotency key with different command material must not
# silently turn into a second ticket or a successful replay.
conflict_status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST "$api_url/v1/queue" \
-H "Authorization: Bearer $access_token" \
-H 'Idempotency-Key: compose-queue-key-123456' \
-H 'Content-Type: application/json' \
-d '{"ticket_id":"compose-other-ticket","playlist":"casual","client_build":"build-1","protocol_version":1}')"
[[ "$conflict_status" == 409 ]]
heartbeat_json="$(curl -fsS -X POST "$api_url/v1/queue/compose-queue-ticket/heartbeat" \
-H "Authorization: Bearer $access_token" \
-H 'Idempotency-Key: compose-heartbeat-key-123456' \
-H 'If-Match-Revision: 0')"
heartbeat_revision="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["revision"])' <<<"$heartbeat_json")"
[[ "$heartbeat_revision" == 1 ]]
curl -fsS -o /dev/null -X POST "$api_url/v1/queue/compose-queue-ticket/cancel" \
-H "Authorization: Bearer $access_token" \
-H 'Idempotency-Key: compose-cancel-key-123456' \
-H "If-Match-Revision: $heartbeat_revision"
# Drive six independent authenticated players through the real queue boundary;
# the matcher service consumes the durable rows below and creates the proposal.
match_tokens=()
for player in 1 2 3 4 5 6; do
player_session="$(curl -fsS -X POST "$api_url/v1/session/steam" \
-H 'Content-Type: application/json' -d "{\"web_api_ticket\":\"compose-match-player-${player}\"}")"
match_tokens+=("$(python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])' <<<"$player_session")")
curl -fsS -o /dev/null -X POST "$api_url/v1/queue" \
-H "Authorization: Bearer ${match_tokens[$((player - 1))]}" \
-H "Idempotency-Key: compose-match-queue-key-${player}-123456" \
-H 'Content-Type: application/json' \
-d "{\"ticket_id\":\"compose-match-ticket-${player}\",\"playlist\":\"casual\",\"client_build\":\"build-1\",\"protocol_version\":1}"
done
"${compose[@]}" exec -T database psql -v ON_ERROR_STOP=1 -U cosmic_clash_test -d cosmic_clash_test -c \
"UPDATE queue_tickets SET predicted_rtt = '{\"EU\":30}'::jsonb WHERE ticket_id LIKE 'compose-match-ticket-%'" >/dev/null
proposal_id=""
for attempt in $(seq 1 30); do
proposal_id="$("${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT proposal_id FROM proposals WHERE state = 'OPEN' ORDER BY created_at DESC LIMIT 1" | tr -d '\r')"
if [[ -n "$proposal_id" ]]; then break; fi
[[ "$attempt" == 30 ]] && { echo "matcher did not create a proposal" >&2; exit 1; }
sleep 1
done
proposal_json="$(curl -fsS -H "Authorization: Bearer ${match_tokens[0]}" "$api_url/v1/proposals/$proposal_id")"
python3 - "$proposal_json" <<'PY'
import json, sys
proposal = json.loads(sys.argv[1])
assert proposal["state"] == "OPEN"
assert len(proposal["participants"]) == 6
print("proposal formation check passed")
PY
proposal_revision=0
for player in 1 2 3 4 5 6; do
proposal_json="$(curl -fsS -X POST "$api_url/v1/proposals/$proposal_id/accept" \
-H "Authorization: Bearer ${match_tokens[$((player - 1))]}" \
-H "Idempotency-Key: compose-proposal-accept-${player}-123456" \
-H "If-Match-Revision: $proposal_revision")"
proposal_revision="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["revision"])' <<<"$proposal_json")"
done
[[ "$("${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT state FROM proposals WHERE proposal_id = '$proposal_id'" | tr -d '\r')" == ACCEPTED ]]
[[ "$("${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM matches WHERE state = 'ALLOCATING'" | tr -d '\r')" == 1 ]]
for attempt in $(seq 1 30); do
allocation_count="$("${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM allocations WHERE match_id LIKE 'match-%'" | tr -d '\r')"
if [[ "$allocation_count" == 1 ]]; then break; fi
[[ "$attempt" == 30 ]] && { echo "allocator did not bind a provider allocation" >&2; exit 1; }
sleep 1
done
[[ "$("${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT server_id FROM matches WHERE state = 'ALLOCATING'" | tr -d '\r')" == allocator-ready-1 ]]
# Model the durable state produced by the allocator, then use the real HTTP
# workload authentication and mutation boundaries for every action below.
"${compose[@]}" exec -T database psql -v ON_ERROR_STOP=1 -U cosmic_clash_test -d cosmic_clash_test <<'SQL'
INSERT INTO game_servers (server_id, region, build, protocol_version, transport, state)
VALUES ('compose-server-0001', 'EU', 'build-1', 1, 'enet', 'ALLOCATED');
INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, revision)
VALUES ('compose-match-0001', 'casual', 'RESULT_PENDING', 'EU', 1, 'compose-server-0001', 2);
INSERT INTO allocations (allocation_id, match_id, server_id, region, build, protocol_version, transport, request_digest, state, allocated_at)
VALUES ('compose-allocation-0001', 'compose-match-0001', 'compose-server-0001', 'EU', 'build-1', 1, 'enet', decode(repeat('00', 32), 'hex'), 'ALLOCATED', now());
SQL
result_body='{"match_id":"compose-match-0001","result_nonce":"compose-result-nonce-1234","score":{"team_0":3,"team_1":2},"integrity_state":"CERTIFIED"}'
curl -fsS -o /dev/null -w '%{http_code}' \
-X POST "$api_url/v1/servers/compose-server-0001/result" \
-H "Authorization: Bearer $token" \
-H 'Idempotency-Key: compose-result-key-123456' \
-H 'Content-Type: application/json' -d "$result_body" | grep -qx 202
# An identical retry must be acknowledged without a second receipt.
curl -fsS -o /dev/null -w '%{http_code}' \
-X POST "$api_url/v1/servers/compose-server-0001/result" \
-H "Authorization: Bearer $token" \
-H 'Idempotency-Key: compose-result-key-123456' \
-H 'Content-Type: application/json' -d "$result_body" | grep -qx 202
"${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT state FROM matches WHERE match_id = 'compose-match-0001'" | grep -qx COMPLETED
"${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM result_receipts WHERE match_id = 'compose-match-0001'" | grep -qx 1
curl -fsS -o /dev/null -w '%{http_code}' \
-X POST "$api_url/v1/servers/compose-server-0001/shutdown" \
-H "Authorization: Bearer $token" \
-H 'Idempotency-Key: compose-shutdown-key-123456' \
-H 'Content-Type: application/json' -d '{"reason":"server_draining"}' | grep -qx 204
"${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM audit_events WHERE action = 'SERVER_SHUTDOWN' AND aggregate_id = 'compose-match-0001'" | grep -qx 1
"${compose[@]}" stop -t 12 game-server >/dev/null
if "${compose[@]}" ps --status running --services | grep -qx game-server; then
echo "allocated game-server did not stop after supervisor drain" >&2
exit 1
fi
"${compose[@]}" stop -t 10 control-plane >/dev/null
echo "8.48 PASS: allocated Compose HTTP result/retry/shutdown and supervisor drain completed"