CosmicClash/scripts/verify_allocated_compose.sh

#!/usr/bin/env bash
set -Eeuo pipefail

# Independent allocated-flow fixture for multiplayer-next.md §8.48. This
# intentionally does not call compose.phase6-smoke.yml or reuse its ports.
root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
compose_file="$root_dir/compose.allocated-smoke.yml"
project="${COMPOSE_PROJECT_NAME:-cosmic-clash-allocated-smoke}"
api_url="http://127.0.0.1:18080"
secret="compose-workload-secret"
smoke_dir="${COMPOSE_SMOKE_DIR:-/tmp/cosmic-clash-allocated-smoke}"
compose=(docker compose -p "$project" -f "$compose_file")

# Most of this script is `curl -fsS` and bare [[ ]] assertions under `set -e`,
# which abort with no message at all. That is fine locally, where the fixture
# is still up to poke at, but in CI it produces a failed run whose log contains
# nothing but "make: *** Error 1" -- undiagnosable without re-running by hand.
# Report where it stopped, and dump the service logs, so a CI failure explains
# itself on the first occurrence.
failed_line=""
on_error() {
  failed_line="$1"
  echo "allocated Compose fixture failed at ${BASH_SOURCE[0]}:${failed_line}" >&2
  echo "--- failing command: ${BASH_COMMAND}" >&2
}
trap 'on_error "$LINENO"' ERR

cleanup() {
  local rc=$?
  if [[ "$rc" != 0 ]]; then
    echo "--- allocated Compose service logs follow (exit ${rc}) ---" >&2
    "${compose[@]}" ps >&2 2>/dev/null || true
    "${compose[@]}" logs --no-color --tail=80 >&2 2>/dev/null || true
  fi
  if [[ "$rc" != 0 && "${COMPOSE_KEEP_ON_FAILURE:-}" == 1 ]]; then
    echo "allocated Compose fixture retained for inspection: ${project}" >&2
    exit "$rc"
  fi
  "${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true
  exit "$rc"
}
trap cleanup EXIT

command -v docker >/dev/null 2>&1 || { echo "Docker is required for 8.48" >&2; exit 2; }
docker info >/dev/null 2>&1 || { echo "A running Docker daemon is required for 8.48" >&2; exit 2; }

mkdir -p "$smoke_dir"
python3 - "$smoke_dir" <<'PY'
import base64, hashlib, hmac, json, pathlib, sys, time

directory = pathlib.Path(sys.argv[1])
key = b"compose-join-signing-key"
key_id = "compose-key-1"
expires = "2099-12-31T00:00:00Z"
# Field order and the trailing key ID must match
# server/domain.JoinAuthorisationBytes and Game/scripts/match_net.gd.
fields = ["compose-match-0001", "compose-server-0001", "compose-player", "compose-steam", "0", "0", "v1", "1", expires, key_id]
canonical = b"\0".join(field.encode() for field in fields)
signature = base64.urlsafe_b64encode(hmac.new(key, canonical, hashlib.sha256).digest()).rstrip(b"=").decode()
envelope = {"Authorisation": {"MatchID": fields[0], "ServerID": fields[1], "PlayerID": fields[2], "SteamID": fields[3], "Slot": 0, "Team": 0, "Protocol": fields[6], "Generation": 1, "ExpiresAt": expires, "KeyID": key_id}, "Signature": signature}
# The key file maps key ID -> base64 key so a rotation can publish several.
(directory / "join-signing-keys.json").write_text(json.dumps({key_id: base64.b64encode(key).decode()}) + "\n")
(directory / "join-roster.json").write_text(json.dumps([base64.urlsafe_b64encode(json.dumps(envelope, separators=(",", ":")).encode()).rstrip(b"=").decode()]) + "\n")
PY

command -v openssl >/dev/null 2>&1 || { echo "OpenSSL is required for the HTTPS fake Kubernetes API" >&2; exit 2; }
printf 'compose-kubernetes-token' > "$smoke_dir/kubernetes-token"
openssl req -x509 -newkey rsa:2048 -nodes -days 1 \
  -subj '/CN=agones-provider' -addext 'subjectAltName=DNS:agones-provider' \
  -keyout "$smoke_dir/fake-agones.key" -out "$smoke_dir/fake-agones.crt" >/dev/null 2>&1

token="$(python3 - "$secret" <<'PY'
import base64, datetime, hashlib, hmac, json, sys
secret = sys.argv[1].encode()
payload = {"a": "compose-allocation-0001", "e": (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(hours=1)).isoformat().replace("+00:00", "Z")}
encoded = base64.urlsafe_b64encode(json.dumps(payload, separators=(",", ":")).encode()).rstrip(b"=")
signature = hmac.new(secret, encoded, hashlib.sha256).digest()
sig = base64.urlsafe_b64encode(signature).rstrip(b"=")
print(encoded.decode() + "." + sig.decode())
PY
)"
export COSMIC_CLASH_COMPOSE_WORKLOAD_TOKEN="$token"

"${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true
"${compose[@]}" up -d --build

for attempt in $(seq 1 180); do
  if "${compose[@]}" logs game-server 2>/dev/null | grep -q ' server_started '; then
    break
  fi
  # Ask whether it EXITED, not whether it is absent from the running list.
  # Those differ: a container that has been created but has not started yet is
  # missing from --status running too, so the previous check called a
  # still-starting server dead on the first poll. It failed intermittently
  # against a game server whose own logs showed a clean `server_started`.
  if "${compose[@]}" ps -a --status exited --services 2>/dev/null | grep -qx game-server; then
    "${compose[@]}" logs game-server >&2
    echo "allocated Compose game server exited before becoming ready" >&2
    exit 1
  fi
  if [[ "$attempt" == 180 ]]; then
    "${compose[@]}" logs >&2
    echo "allocated Compose game server did not become ready" >&2
    exit 1
  fi
  sleep 1
done

for attempt in $(seq 1 60); do
  if curl -fsS "$api_url/healthz" >/dev/null 2>&1; then
    break
  fi
  if [[ "$attempt" == 60 ]]; then
    "${compose[@]}" logs >&2
    echo "allocated Compose control plane did not become ready" >&2
    exit 1
  fi
  sleep 1
done

# The deployed maintenance service owns the ranked reconnect deadline. Seed an
# already-expired durable lease and require the real Compose process to record
# its cooldown before exercising the rest of the allocation flow.
"${compose[@]}" exec -T database psql -v ON_ERROR_STOP=1 -U cosmic_clash_test -d cosmic_clash_test <<'SQL'
INSERT INTO identities (player_id, steam_id) VALUES ('compose-abandon-player', 'compose-abandon-steam');
INSERT INTO queue_tickets (ticket_id, player_id, playlist, state, client_build, protocol_version, enqueued_at, expires_at)
VALUES ('compose-abandon-ticket', 'compose-abandon-player', 'ranked', 'LIVE', 'build-1', 1, now() - interval '2 minutes', now() + interval '1 hour');
INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, arena_path)
VALUES ('compose-abandon-match', 'ranked', 'LIVE', 'EU', 1, 'compose-abandon-server', 'res://scenes/arena_01.tscn');
INSERT INTO match_participants (match_id, player_id, ticket_id, slot, team, connection_generation, connected_at, disconnected_at)
VALUES ('compose-abandon-match', 'compose-abandon-player', 'compose-abandon-ticket', 0, 0, 1, now() - interval '2 minutes', now() - interval '61 seconds');
SQL
for attempt in $(seq 1 30); do
  abandoned="$(${compose[@]} exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM match_participants WHERE match_id = 'compose-abandon-match' AND abandoned_at IS NOT NULL" | tr -d '\r')"
  [[ "$abandoned" == 1 ]] && break
  [[ "$attempt" == 30 ]] && { "${compose[@]}" logs maintenance >&2; echo "maintenance did not record an expired ranked reconnect lease" >&2; exit 1; }
  sleep 1
done
"${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM penalties WHERE match_id = 'compose-abandon-match' AND kind = 'MATCH_ABANDONED'" | grep -qx 1

session_json="$(curl -fsS -X POST "$api_url/v1/session/steam" \
  -H 'Content-Type: application/json' -d '{"web_api_ticket":"compose-queue-ticket"}')"
access_token="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])' <<<"$session_json")"
queue_body='{"ticket_id":"compose-queue-ticket","playlist":"casual","client_build":"build-1","protocol_version":1}'
queue_json="$(curl -fsS -X POST "$api_url/v1/queue" \
  -H "Authorization: Bearer $access_token" \
  -H 'Idempotency-Key: compose-queue-key-123456' \
  -H 'Content-Type: application/json' -d "$queue_body")"
queue_revision="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["revision"])' <<<"$queue_json")"
[[ "$queue_revision" == 0 ]]

# Reusing a queue idempotency key with different command material must not
# silently turn into a second ticket or a successful replay.
conflict_body="$(mktemp)"
conflict_status="$(curl -sS -o "$conflict_body" -w '%{http_code}' -X POST "$api_url/v1/queue" \
  -H "Authorization: Bearer $access_token" \
  -H 'Idempotency-Key: compose-queue-key-123456' \
  -H 'Content-Type: application/json' \
  -d '{"ticket_id":"compose-other-ticket","playlist":"casual","client_build":"build-1","protocol_version":1}')"
if [[ "$conflict_status" != 409 ]]; then
  # Report what actually came back. A bare [[ ]] here just aborts, which is
  # how this assertion failed in CI three times without ever saying what the
  # status was.
  echo "idempotency conflict returned ${conflict_status}, want 409; body:" >&2
  cat "$conflict_body" >&2 || true
  echo >&2
  rm -f "$conflict_body"
  exit 1
fi
rm -f "$conflict_body"

heartbeat_json="$(curl -fsS -X POST "$api_url/v1/queue/compose-queue-ticket/heartbeat" \
  -H "Authorization: Bearer $access_token" \
  -H 'Idempotency-Key: compose-heartbeat-key-123456' \
  -H 'If-Match-Revision: 0')"
heartbeat_revision="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["revision"])' <<<"$heartbeat_json")"
[[ "$heartbeat_revision" == 1 ]]
curl -fsS -o /dev/null -X POST "$api_url/v1/queue/compose-queue-ticket/cancel" \
  -H "Authorization: Bearer $access_token" \
  -H 'Idempotency-Key: compose-cancel-key-123456' \
  -H "If-Match-Revision: $heartbeat_revision"

# Drive six independent authenticated players through the real queue boundary;
# the matcher service consumes the durable rows below and creates the proposal.
match_tokens=()
for player in 1 2 3 4 5 6; do
  player_session="$(curl -fsS -X POST "$api_url/v1/session/steam" \
    -H 'Content-Type: application/json' -d "{\"web_api_ticket\":\"compose-match-player-${player}\"}")"
  match_tokens+=("$(python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])' <<<"$player_session")")
  curl -fsS -o /dev/null -X POST "$api_url/v1/queue" \
    -H "Authorization: Bearer ${match_tokens[$((player - 1))]}" \
    -H "Idempotency-Key: compose-match-queue-key-${player}-123456" \
    -H 'Content-Type: application/json' \
    -d "{\"ticket_id\":\"compose-match-ticket-${player}\",\"playlist\":\"casual\",\"client_build\":\"build-1\",\"protocol_version\":1}"
done
"${compose[@]}" exec -T database psql -v ON_ERROR_STOP=1 -U cosmic_clash_test -d cosmic_clash_test -c \
  "UPDATE queue_tickets SET predicted_rtt = '{\"EU\":30}'::jsonb WHERE ticket_id LIKE 'compose-match-ticket-%'" >/dev/null

proposal_id=""
for attempt in $(seq 1 30); do
  proposal_id="$("${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT proposal_id FROM proposals WHERE state = 'OPEN' ORDER BY created_at DESC LIMIT 1" | tr -d '\r')"
  if [[ -n "$proposal_id" ]]; then break; fi
  [[ "$attempt" == 30 ]] && { echo "matcher did not create a proposal" >&2; exit 1; }
  sleep 1
done
proposal_json="$(curl -fsS -H "Authorization: Bearer ${match_tokens[0]}" "$api_url/v1/proposals/$proposal_id")"
python3 - "$proposal_json" <<'PY'
import json, sys
proposal = json.loads(sys.argv[1])
assert proposal["state"] == "OPEN"
assert len(proposal["participants"]) == 6
print("proposal formation check passed")
PY

proposal_revision=0
for player in 1 2 3 4 5 6; do
  proposal_json="$(curl -fsS -X POST "$api_url/v1/proposals/$proposal_id/accept" \
    -H "Authorization: Bearer ${match_tokens[$((player - 1))]}" \
    -H "Idempotency-Key: compose-proposal-accept-${player}-123456" \
    -H "If-Match-Revision: $proposal_revision")"
  proposal_revision="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["revision"])' <<<"$proposal_json")"
done
[[ "$("${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT state FROM proposals WHERE proposal_id = '$proposal_id'" | tr -d '\r')" == ACCEPTED ]]
[[ "$("${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM matches WHERE state = 'ALLOCATING'" | tr -d '\r')" == 1 ]]
for attempt in $(seq 1 30); do
  allocation_count="$("${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM allocations WHERE match_id LIKE 'match-%'" | tr -d '\r')"
  if [[ "$allocation_count" == 1 ]]; then break; fi
  [[ "$attempt" == 30 ]] && { echo "allocator did not bind a provider allocation" >&2; exit 1; }
  sleep 1
done
[[ "$("${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT server_id FROM matches WHERE state = 'ALLOCATING'" | tr -d '\r')" == allocator-ready-1 ]]

# Model the durable state produced by the allocator, then use the real HTTP
# workload authentication and mutation boundaries for every action below.
"${compose[@]}" exec -T database psql -v ON_ERROR_STOP=1 -U cosmic_clash_test -d cosmic_clash_test <<'SQL'
INSERT INTO game_servers (server_id, region, build, protocol_version, transport, state)
VALUES ('compose-server-0001', 'EU', 'build-1', 1, 'enet', 'ALLOCATED');
INSERT INTO matches (match_id, playlist, state, region, protocol_version, server_id, revision)
VALUES ('compose-match-0001', 'casual', 'RESULT_PENDING', 'EU', 1, 'compose-server-0001', 2);
INSERT INTO allocations (allocation_id, match_id, server_id, region, build, protocol_version, transport, request_digest, state, allocated_at)
VALUES ('compose-allocation-0001', 'compose-match-0001', 'compose-server-0001', 'EU', 'build-1', 1, 'enet', decode(repeat('00', 32), 'hex'), 'ALLOCATED', now());
SQL

result_body='{"match_id":"compose-match-0001","result_nonce":"compose-result-nonce-1234","score":{"team_0":3,"team_1":2},"integrity_state":"CERTIFIED"}'
curl -fsS -o /dev/null -w '%{http_code}' \
  -X POST "$api_url/v1/servers/compose-server-0001/result" \
  -H "Authorization: Bearer $token" \
  -H 'Idempotency-Key: compose-result-key-123456' \
  -H 'Content-Type: application/json' -d "$result_body" | grep -qx 202

# An identical retry must be acknowledged without a second receipt.
curl -fsS -o /dev/null -w '%{http_code}' \
  -X POST "$api_url/v1/servers/compose-server-0001/result" \
  -H "Authorization: Bearer $token" \
  -H 'Idempotency-Key: compose-result-key-123456' \
  -H 'Content-Type: application/json' -d "$result_body" | grep -qx 202

"${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT state FROM matches WHERE match_id = 'compose-match-0001'" | grep -qx COMPLETED
"${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM result_receipts WHERE match_id = 'compose-match-0001'" | grep -qx 1

curl -fsS -o /dev/null -w '%{http_code}' \
  -X POST "$api_url/v1/servers/compose-server-0001/shutdown" \
  -H "Authorization: Bearer $token" \
  -H 'Idempotency-Key: compose-shutdown-key-123456' \
  -H 'Content-Type: application/json' -d '{"reason":"server_draining"}' | grep -qx 204

"${compose[@]}" exec -T database psql -At -U cosmic_clash_test -d cosmic_clash_test -c "SELECT count(*) FROM audit_events WHERE action = 'SERVER_SHUTDOWN' AND aggregate_id = 'compose-match-0001'" | grep -qx 1

"${compose[@]}" stop -t 12 game-server >/dev/null
if "${compose[@]}" ps --status running --services | grep -qx game-server; then
  echo "allocated game-server did not stop after supervisor drain" >&2
  exit 1
fi
"${compose[@]}" stop -t 10 control-plane >/dev/null
echo "8.48 PASS: allocated Compose HTTP result/retry/shutdown and supervisor drain completed"