mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
5765532409
The root blocker (issue #14). The worker bound the provider allocation and stopped. Service.PublishRoster and store.SaveVerifiedAssignmentRoster both existed, fully tested, with zero non-test callers, and the production allocator configured neither a roster store nor a signing key. Nothing ever wrote the assignments table. The allocated supervisor fetches a non-empty roster before it launches the game child, so every real allocation failed at that fetch: no match could reach ASSIGNMENT_READY or accept a player. Existing tests seeded assignments directly, which is exactly why the missing hand-off went unnoticed. The worker now builds one join authorisation per durable participant, signs each with the active key, and publishes them. Participants are read through the same query SaveVerifiedAssignmentRoster re-validates against, so the allocator cannot construct a roster the persistence boundary would reject. The manifest commits to a digest over the whole roster, so a server cannot be handed a truncated roster whose surviving entries are each individually valid. Persist the provider endpoint on the allocation: it arrived on the provider response and was never stored, so a worker crashing between allocating and publishing had no endpoint to recover and would have stranded the match permanently. Republishing is idempotent, so that crash now simply retries. cmd/allocator refuses to start without key material rather than running an allocator that binds allocations and silently strands every match. The k8s allocator Deployment mounts the same key set the Fleet does, and both now take the JSON key map so a rotation can publish several. New integration test drives the real worker through to the supervisor's own roster read path without seeding the assignments table. Verified it fails with "assignments = 0, want 2" when the publish step is removed.
241 lines
14 KiB
Bash
Executable File
241 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Independent allocated-flow fixture for multiplayer-next.md §8.48. This
|
|
# intentionally does not call compose.phase6-smoke.yml or reuse its ports.
|
|
root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
compose_file="$root_dir/compose.allocated-smoke.yml"
|
|
project="${COMPOSE_PROJECT_NAME:-cosmic-clash-allocated-smoke}"
|
|
api_url="http://127.0.0.1:18080"
|
|
secret="compose-workload-secret"
|
|
smoke_dir="${COMPOSE_SMOKE_DIR:-/tmp/cosmic-clash-allocated-smoke}"
|
|
compose=(docker compose -p "$project" -f "$compose_file")
|
|
|
|
cleanup() {
|
|
local rc=$?
|
|
if [[ "$rc" != 0 && "${COMPOSE_KEEP_ON_FAILURE:-}" == 1 ]]; then
|
|
echo "allocated Compose fixture retained for inspection: ${project}" >&2
|
|
exit "$rc"
|
|
fi
|
|
"${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true
|
|
exit "$rc"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
command -v docker >/dev/null 2>&1 || { echo "Docker is required for 8.48" >&2; exit 2; }
|
|
docker info >/dev/null 2>&1 || { echo "A running Docker daemon is required for 8.48" >&2; exit 2; }
|
|
|
|
mkdir -p "$smoke_dir"
|
|
python3 - "$smoke_dir" <<'PY'
|
|
import base64, hashlib, hmac, json, pathlib, sys, time
|
|
|
|
directory = pathlib.Path(sys.argv[1])
|
|
key = b"compose-join-signing-key"
|
|
key_id = "compose-key-1"
|
|
expires = "2099-12-31T00:00:00Z"
|
|
# Field order and the trailing key ID must match
|
|
# server/domain.JoinAuthorisationBytes and Game/scripts/match_net.gd.
|
|
fields = ["compose-match-0001", "compose-server-0001", "compose-player", "compose-steam", "0", "0", "v1", "1", expires, key_id]
|
|
canonical = b"\0".join(field.encode() for field in fields)
|
|
signature = base64.urlsafe_b64encode(hmac.new(key, canonical, hashlib.sha256).digest()).rstrip(b"=").decode()
|
|
envelope = {"Authorisation": {"MatchID": fields[0], "ServerID": fields[1], "PlayerID": fields[2], "SteamID": fields[3], "Slot": 0, "Team": 0, "Protocol": fields[6], "Generation": 1, "ExpiresAt": expires, "KeyID": key_id}, "Signature": signature}
|
|
# The key file maps key ID -> base64 key so a rotation can publish several.
|
|
(directory / "join-signing-keys.json").write_text(json.dumps({key_id: base64.b64encode(key).decode()}) + "\n")
|
|
(directory / "join-roster.json").write_text(json.dumps([base64.urlsafe_b64encode(json.dumps(envelope, separators=(",", ":")).encode()).rstrip(b"=").decode()]) + "\n")
|
|
PY
|
|
|
|
command -v openssl >/dev/null 2>&1 || { echo "OpenSSL is required for the HTTPS fake Kubernetes API" >&2; exit 2; }
|
|
printf 'compose-kubernetes-token' > "$smoke_dir/kubernetes-token"
|
|
openssl req -x509 -newkey rsa:2048 -nodes -days 1 \
|
|
-subj '/CN=agones-provider' -addext 'subjectAltName=DNS:agones-provider' \
|
|
-keyout "$smoke_dir/fake-agones.key" -out "$smoke_dir/fake-agones.crt" >/dev/null 2>&1
|
|
|
|
token="$(python3 - "$secret" <<'PY'
|
|
import base64, datetime, hashlib, hmac, json, sys
|
|
secret = sys.argv[1].encode()
|
|
payload = {"a": "compose-allocation-0001", "e": (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(hours=1)).isoformat().replace("+00:00", "Z")}
|
|
encoded = base64.urlsafe_b64encode(json.dumps(payload, separators=(",", ":")).encode()).rstrip(b"=")
|
|
signature = hmac.new(secret, encoded, hashlib.sha256).digest()
|
|
sig = base64.urlsafe_b64encode(signature).rstrip(b"=")
|
|
print(encoded.decode() + "." + sig.decode())
|
|
PY
|
|
)"
|
|
export COSMIC_CLASH_COMPOSE_WORKLOAD_TOKEN="$token"
|
|
|
|
"${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true
|
|
"${compose[@]}" up -d --build
|
|
|
|
for attempt in $(seq 1 180); do
|
|
if "${compose[@]}" logs game-server 2>/dev/null | grep -q ' server_started '; then
|
|
break
|
|
fi
|
|
if ! "${compose[@]}" ps --status running --services | grep -qx game-server; then
|
|
"${compose[@]}" logs game-server >&2
|
|
echo "allocated Compose game server exited before becoming ready" >&2
|
|
exit 1
|
|
fi
|
|
if [[ "$attempt" == 180 ]]; then
|
|
"${compose[@]}" logs >&2
|
|
echo "allocated Compose game server did not become ready" >&2
|
|
exit 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
for attempt in $(seq 1 60); do
|
|
if curl -fsS "$api_url/healthz" >/dev/null 2>&1; then
|
|
break
|
|
fi
|
|
if [[ "$attempt" == 60 ]]; then
|
|
"${compose[@]}" logs >&2
|
|
echo "allocated Compose control plane did not become ready" >&2
|
|
exit 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
# The deployed maintenance service owns the ranked reconnect deadline. Seed an
|
|
# already-expired durable lease and require the real Compose process to record
|
|
# its cooldown before exercising the rest of the allocation flow.
|
|
"${compose[@]}" exec -T database psql -v ON_ERROR_STOP=1 -U cosmic_clash_test -d cosmic_clash_test <<'SQL'
|
|
INSERT INTO identities (player_id, steam_id) VALUES ('compose-abandon-player', 'compose-abandon-steam');
|
|
INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at)
|
|
VALUES ('compose-abandon-ticket', 'compose-abandon-player', 'ranked', 'LIVE', 'build-1', 1, now() - interval '2 minutes', now() + interval '1 hour');
|
|
INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, arena_path)
|
|
VALUES ('compose-abandon-match', 'ranked', 'LIVE', 'EU', 1, 'compose-abandon-server', 'res://scenes/arena_01.tscn');
|
|
INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team, connection_generation, connected_at, disconnected_at)
|
|
VALUES ('compose-abandon-match', 'compose-abandon-player', 'compose-abandon-ticket', 0, 0, 1, now() - interval '2 minutes', now() - interval '61 seconds');
|
|
SQL
|
|
for attempt in $(seq 1 30); do
|
|
abandoned="$(${compose[@]} exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM match_participants WHERE match_id = 'compose-abandon-match' AND abandoned_at IS NOT NULL" | tr -d '\r')"
|
|
[[ "$abandoned" == 1 ]] && break
|
|
[[ "$attempt" == 30 ]] && { "${compose[@]}" logs maintenance >&2; echo "maintenance did not record an expired ranked reconnect lease" >&2; exit 1; }
|
|
sleep 1
|
|
done
|
|
"${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM penalties WHERE match_id = 'compose-abandon-match' AND kind = 'MATCH_ABANDONED'" | grep -qx 1
|
|
|
|
session_json="$(curl -fsS -X POST "$api_url/v1/session/steam" \
|
|
-H 'Content-Type: application/json' -d '{"web_api_ticket":"compose-queue-ticket"}')"
|
|
access_token="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])' <<<"$session_json")"
|
|
queue_body='{"ticket_id":"compose-queue-ticket","playlist":"casual","client_build":"build-1","protocol_version":1}'
|
|
queue_json="$(curl -fsS -X POST "$api_url/v1/queue" \
|
|
-H "Authorization: Bearer $access_token" \
|
|
-H 'Idempotency-Key: compose-queue-key-123456' \
|
|
-H 'Content-Type: application/json' -d "$queue_body")"
|
|
queue_revision="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["revision"])' <<<"$queue_json")"
|
|
[[ "$queue_revision" == 0 ]]
|
|
|
|
# Reusing a queue idempotency key with different command material must not
|
|
# silently turn into a second ticket or a successful replay.
|
|
conflict_status="$(curl -sS -o /dev/null -w '%{http_code}' -X POST "$api_url/v1/queue" \
|
|
-H "Authorization: Bearer $access_token" \
|
|
-H 'Idempotency-Key: compose-queue-key-123456' \
|
|
-H 'Content-Type: application/json' \
|
|
-d '{"ticket_id":"compose-other-ticket","playlist":"casual","client_build":"build-1","protocol_version":1}')"
|
|
[[ "$conflict_status" == 409 ]]
|
|
|
|
heartbeat_json="$(curl -fsS -X POST "$api_url/v1/queue/compose-queue-ticket/heartbeat" \
|
|
-H "Authorization: Bearer $access_token" \
|
|
-H 'Idempotency-Key: compose-heartbeat-key-123456' \
|
|
-H 'If-Match-Revision: 0')"
|
|
heartbeat_revision="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["revision"])' <<<"$heartbeat_json")"
|
|
[[ "$heartbeat_revision" == 1 ]]
|
|
curl -fsS -o /dev/null -X POST "$api_url/v1/queue/compose-queue-ticket/cancel" \
|
|
-H "Authorization: Bearer $access_token" \
|
|
-H 'Idempotency-Key: compose-cancel-key-123456' \
|
|
-H "If-Match-Revision: $heartbeat_revision"
|
|
|
|
# Drive six independent authenticated players through the real queue boundary;
|
|
# the matcher service consumes the durable rows below and creates the proposal.
|
|
match_tokens=()
|
|
for player in 1 2 3 4 5 6; do
|
|
player_session="$(curl -fsS -X POST "$api_url/v1/session/steam" \
|
|
-H 'Content-Type: application/json' -d "{\"web_api_ticket\":\"compose-match-player-${player}\"}")"
|
|
match_tokens+=("$(python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])' <<<"$player_session")")
|
|
curl -fsS -o /dev/null -X POST "$api_url/v1/queue" \
|
|
-H "Authorization: Bearer ${match_tokens[$((player - 1))]}" \
|
|
-H "Idempotency-Key: compose-match-queue-key-${player}-123456" \
|
|
-H 'Content-Type: application/json' \
|
|
-d "{\"ticket_id\":\"compose-match-ticket-${player}\",\"playlist\":\"casual\",\"client_build\":\"build-1\",\"protocol_version\":1}"
|
|
done
|
|
"${compose[@]}" exec -T database psql -v ON_ERROR_STOP=1 -U cosmic_clash_test -d cosmic_clash_test -c \
|
|
"UPDATE queue_tickets SET predicted_rtt = '{\"EU\":30}'::jsonb WHERE ticket_id LIKE 'compose-match-ticket-%'" >/dev/null
|
|
|
|
proposal_id=""
|
|
for attempt in $(seq 1 30); do
|
|
proposal_id="$("${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT proposal_id FROM proposals WHERE state = 'OPEN' ORDER BY created_at DESC LIMIT 1" | tr -d '\r')"
|
|
if [[ -n "$proposal_id" ]]; then break; fi
|
|
[[ "$attempt" == 30 ]] && { echo "matcher did not create a proposal" >&2; exit 1; }
|
|
sleep 1
|
|
done
|
|
proposal_json="$(curl -fsS -H "Authorization: Bearer ${match_tokens[0]}" "$api_url/v1/proposals/$proposal_id")"
|
|
python3 - "$proposal_json" <<'PY'
|
|
import json, sys
|
|
proposal = json.loads(sys.argv[1])
|
|
assert proposal["state"] == "OPEN"
|
|
assert len(proposal["participants"]) == 6
|
|
print("proposal formation check passed")
|
|
PY
|
|
|
|
proposal_revision=0
|
|
for player in 1 2 3 4 5 6; do
|
|
proposal_json="$(curl -fsS -X POST "$api_url/v1/proposals/$proposal_id/accept" \
|
|
-H "Authorization: Bearer ${match_tokens[$((player - 1))]}" \
|
|
-H "Idempotency-Key: compose-proposal-accept-${player}-123456" \
|
|
-H "If-Match-Revision: $proposal_revision")"
|
|
proposal_revision="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["revision"])' <<<"$proposal_json")"
|
|
done
|
|
[[ "$("${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT state FROM proposals WHERE proposal_id = '$proposal_id'" | tr -d '\r')" == ACCEPTED ]]
|
|
[[ "$("${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM matches WHERE state = 'ALLOCATING'" | tr -d '\r')" == 1 ]]
|
|
for attempt in $(seq 1 30); do
|
|
allocation_count="$("${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM allocations WHERE match_id LIKE 'match-%'" | tr -d '\r')"
|
|
if [[ "$allocation_count" == 1 ]]; then break; fi
|
|
[[ "$attempt" == 30 ]] && { echo "allocator did not bind a provider allocation" >&2; exit 1; }
|
|
sleep 1
|
|
done
|
|
[[ "$("${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT server_id FROM matches WHERE state = 'ALLOCATING'" | tr -d '\r')" == allocator-ready-1 ]]
|
|
|
|
# Model the durable state produced by the allocator, then use the real HTTP
|
|
# workload authentication and mutation boundaries for every action below.
|
|
"${compose[@]}" exec -T database psql -v ON_ERROR_STOP=1 -U cosmic_clash_test -d cosmic_clash_test <<'SQL'
|
|
INSERT INTO game_servers (server_id, region, build, protocol_version, transport, state)
|
|
VALUES ('compose-server-0001', 'EU', 'build-1', 1, 'enet', 'ALLOCATED');
|
|
INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, revision)
|
|
VALUES ('compose-match-0001', 'casual', 'RESULT_PENDING', 'EU', 1, 'compose-server-0001', 2);
|
|
INSERT INTO allocations (allocation_id, match_id, server_id, region, build, protocol_version, transport, request_digest, state, allocated_at)
|
|
VALUES ('compose-allocation-0001', 'compose-match-0001', 'compose-server-0001', 'EU', 'build-1', 1, 'enet', decode(repeat('00', 32), 'hex'), 'ALLOCATED', now());
|
|
SQL
|
|
|
|
result_body='{"match_id":"compose-match-0001","result_nonce":"compose-result-nonce-1234","score":{"team_0":3,"team_1":2},"integrity_state":"CERTIFIED"}'
|
|
curl -fsS -o /dev/null -w '%{http_code}' \
|
|
-X POST "$api_url/v1/servers/compose-server-0001/result" \
|
|
-H "Authorization: Bearer $token" \
|
|
-H 'Idempotency-Key: compose-result-key-123456' \
|
|
-H 'Content-Type: application/json' -d "$result_body" | grep -qx 202
|
|
|
|
# An identical retry must be acknowledged without a second receipt.
|
|
curl -fsS -o /dev/null -w '%{http_code}' \
|
|
-X POST "$api_url/v1/servers/compose-server-0001/result" \
|
|
-H "Authorization: Bearer $token" \
|
|
-H 'Idempotency-Key: compose-result-key-123456' \
|
|
-H 'Content-Type: application/json' -d "$result_body" | grep -qx 202
|
|
|
|
"${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT state FROM matches WHERE match_id = 'compose-match-0001'" | grep -qx COMPLETED
|
|
"${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM result_receipts WHERE match_id = 'compose-match-0001'" | grep -qx 1
|
|
|
|
curl -fsS -o /dev/null -w '%{http_code}' \
|
|
-X POST "$api_url/v1/servers/compose-server-0001/shutdown" \
|
|
-H "Authorization: Bearer $token" \
|
|
-H 'Idempotency-Key: compose-shutdown-key-123456' \
|
|
-H 'Content-Type: application/json' -d '{"reason":"server_draining"}' | grep -qx 204
|
|
|
|
"${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM audit_events WHERE action = 'SERVER_SHUTDOWN' AND aggregate_id = 'compose-match-0001'" | grep -qx 1
|
|
|
|
"${compose[@]}" stop -t 12 game-server >/dev/null
|
|
if "${compose[@]}" ps --status running --services | grep -qx game-server; then
|
|
echo "allocated game-server did not stop after supervisor drain" >&2
|
|
exit 1
|
|
fi
|
|
"${compose[@]}" stop -t 10 control-plane >/dev/null
|
|
echo "8.48 PASS: allocated Compose HTTP result/retry/shutdown and supervisor drain completed"
|