CosmicClash/scripts/verify_kind_agones.sh

#!/usr/bin/env bash
set -euo pipefail

# Disposable integration gate for multiplayer-next.md §8.49. This deliberately
# does not touch an existing cluster: kind creates an isolated cluster and the
# EXIT trap removes only that named cluster.
root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$root_dir"

cluster_name="${KIND_CLUSTER_NAME:-cosmic-clash-agones-smoke}"
agones_version="${AGONES_VERSION:-1.49.0}"
game_server_image="${GAME_SERVER_IMAGE:-cosmic-clash-game-server:kind}"
kind_node_image="${KIND_NODE_IMAGE:-kindest/node:v1.33.1}"
work_dir="$(mktemp -d "${TMPDIR:-/tmp}/cosmic-clash-agones.XXXXXX")"

# This gate fails in CI with nothing but Helm's "context deadline exceeded",
# and the EXIT trap then deletes the cluster, so there is no way to learn why
# the pods never became Available. Dump enough cluster state on failure that a
# CI run explains itself without needing a local reproduction -- which is not
# equivalent anyway, since a developer machine has different resources and a
# different container runtime.
#
# Set KIND_KEEP_ON_FAILURE=1 to retain the cluster for interactive inspection.
on_error() {
  echo "kind/Agones gate failed at ${BASH_SOURCE[0]}:$1" >&2
  echo "--- failing command: ${BASH_COMMAND}" >&2
}
trap 'on_error "$LINENO"' ERR

dump_cluster_state() {
  echo "=== node capacity and conditions ===" >&2
  kubectl get nodes -o wide >&2 2>&1 || true
  kubectl describe nodes 2>&1 | grep -A 12 -E "Allocated resources|Conditions:" >&2 || true
  for ns in agones-system cosmic-clash; do
    echo "=== namespace ${ns}: pods ===" >&2
    kubectl -n "$ns" get pods -o wide >&2 2>&1 || true
    echo "=== namespace ${ns}: services ===" >&2
    kubectl -n "$ns" get services -o wide >&2 2>&1 || true
    # Events explain scheduling/image/probe failures that pod status alone
    # does not: FailedScheduling, ImagePullBackOff, readiness probe errors.
    echo "=== namespace ${ns}: recent events ===" >&2
    kubectl -n "$ns" get events --sort-by=.lastTimestamp 2>&1 | tail -40 >&2 || true
    # Log EVERY pod, not only the not-ready ones. A GameServer that reaches
    # Ready and is then recycled on a health check leaves no unready pod
    # behind: the failures are already deleted and the survivors read 2/2
    # Running, so filtering on readiness dumped nothing useful and the game
    # server's own output went unseen for several CI runs.
    for pod in $(kubectl -n "$ns" get pods -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null); do
      ready="$(kubectl -n "$ns" get pod "$pod" -o jsonpath='{.status.containerStatuses[*].ready}' 2>/dev/null || true)"
      echo "=== ${ns}/${pod} (ready=${ready:-unknown}) ===" >&2
      kubectl -n "$ns" describe pod "$pod" 2>&1 | tail -35 >&2 || true
      # Per container, not --all-containers: the Agones sidecar is far chattier
      # than the game server, so a shared tail hides exactly the output needed,
      # and --previous without -c resolves to a container that never restarted.
      for container in $(kubectl -n "$ns" get pod "$pod" -o jsonpath='{range .spec.containers[*]}{.name}{"\n"}{end}' 2>/dev/null); do
        echo "--- ${ns}/${pod}[${container}] logs (current) ---" >&2
        kubectl -n "$ns" logs "$pod" -c "$container" --tail=60 >&2 2>&1 || true
        echo "--- ${ns}/${pod}[${container}] logs (previous, if it restarted) ---" >&2
        kubectl -n "$ns" logs "$pod" -c "$container" --previous --tail=60 >&2 2>&1 || true
      done
    done
  done
  # Agones' own view: a GameServer can be Unhealthy while its Pod looks fine,
  # which is precisely the shape of a failed health check.
  echo "=== Agones GameServers and Fleets ===" >&2
  kubectl get gameservers --all-namespaces -o wide >&2 2>&1 || true
  kubectl get fleets --all-namespaces -o wide >&2 2>&1 || true
  echo "=== helm releases ===" >&2
  helm list --all-namespaces >&2 2>&1 || true
}

cleanup() {
  local status=$?
  # No reachability guard here: every command inside dump_cluster_state is
  # already `|| true`, so a gone cluster costs a few harmless errors, whereas
  # a guard that misjudges reachability silently suppresses the whole dump --
  # which is exactly what happened on its first run.
  if [[ "$status" != 0 ]]; then
    dump_cluster_state
  fi
  if [[ "$status" != 0 && "${KIND_KEEP_ON_FAILURE:-}" == 1 ]]; then
    echo "kind cluster retained for inspection: kind-${cluster_name} (delete with: kind delete cluster --name ${cluster_name})" >&2
    rm -rf "$work_dir"
    exit "$status"
  fi
  kind delete cluster --name "$cluster_name" >/dev/null 2>&1 || true
  rm -rf "$work_dir"
  exit "$status"
}
trap cleanup EXIT

for tool in docker kind kubectl helm; do
  command -v "$tool" >/dev/null 2>&1 || {
    echo "8.49 requires '$tool'; install Docker, kind, kubectl, and Helm to run the disposable gate" >&2
    exit 2
  }
done

if ! docker info >/dev/null 2>&1; then
  echo "8.49 requires a running Docker daemon" >&2
  exit 2
fi

kind delete cluster --name "$cluster_name" >/dev/null 2>&1 || true

# Build by default. Reusing whatever happens to be tagged locally silently
# verifies stale code: a developer fixes the game server, reruns this gate, and
# it exercises the previous build because the tag already exists. CI never hits
# that because a fresh runner has no image, which is precisely how a local pass
# and a CI failure can disagree about the same commit.
if [[ "${KIND_REUSE_GAME_SERVER_IMAGE:-}" == 1 ]] && docker image inspect "$game_server_image" >/dev/null 2>&1; then
  echo "Reusing existing $game_server_image (KIND_REUSE_GAME_SERVER_IMAGE=1); it may not contain local changes"
else
  echo "Building $game_server_image from the pinned game-server target"
  docker build --target game-server -t "$game_server_image" .
fi

kind create cluster --name "$cluster_name" --image "$kind_node_image" --wait 120s
kind load docker-image "$game_server_image" --name "$cluster_name"

# Agones creates its SDK service account and namespaced RBAC in each configured
# GameServer namespace. The namespace must therefore exist before Helm runs.
kubectl apply -f deploy/k8s/base/namespace.yaml

helm repo add agones https://agones.dev/chart/stable >/dev/null
helm repo update >/dev/null
# Agones 1.49 otherwise requests 10,100 MiB of ephemeral storage for both its
# controller and extensions pods, which exceeds a default single-node kind
# cluster before the Fleet can be exercised. Its allocator and ping Services
# also default to LoadBalancer, whose ingress never becomes ready in plain kind.
# These are smoke-only bounds; production sizing and exposure remain
# deployment-owned.
helm upgrade --install agones agones/agones \
  --namespace agones-system --create-namespace \
  --version "$agones_version" \
  --set 'gameservers.namespaces[0]=cosmic-clash' \
  --set agones.crds.cleanup.enabled=true \
  --set agones.controller.replicas=1 \
  --set agones.controller.resources.requests.ephemeral-storage=128Mi \
  --set agones.controller.resources.limits.ephemeral-storage=512Mi \
  --set agones.extensions.replicas=1 \
  --set agones.extensions.resources.requests.ephemeral-storage=128Mi \
  --set agones.extensions.resources.limits.ephemeral-storage=512Mi \
  --set agones.allocator.replicas=1 \
  --set agones.allocator.service.serviceType=ClusterIP \
  --set agones.ping.http.serviceType=ClusterIP \
  --set agones.ping.udp.serviceType=ClusterIP \
  --wait --timeout 5m

kubectl wait --for=condition=available deployment/agones-controller \
  -n agones-system --timeout=180s
kubectl wait --for=condition=available deployment/agones-allocator \
  -n agones-system --timeout=180s

# The production Fleet only schedules on explicitly on-demand, zoned nodes.
# Give the disposable node equivalent labels so this gate exercises those
# constraints instead of rewriting them out of the rendered Fleet.
kubectl label nodes --all \
  cosmic-clash.io/capacity-type=on-demand \
  topology.kubernetes.io/zone=kind-smoke \
  --overwrite

# The base Fleet intentionally carries a release-time digest placeholder. For
# this isolated run only, replace that exact placeholder with the image loaded
# into kind. No repository manifest is modified and no mutable image is used
# outside the disposable cluster.
#
# This runner is intentionally an Agones lifecycle smoke, not a substitute for
# the production control-plane gate: there is no PostgreSQL/API/roster backend
# in this disposable cluster. Disable only those production-only child paths so
# the real supervisor can validate the assigned endpoint, launch the exported
# server, and call the Agones SDK Ready endpoint.
zero_digest="$(printf '0%.0s' {1..64})"
sed -e "s|ghcr.io/cosmic-clash/game-server@sha256:${zero_digest}|$game_server_image|" \
    -e 's|--control-plane-url=http://control-plane.cosmic-clash.svc.cluster.local:8080|--control-plane-url=|' \
    -e '/- --roster-path=\/run\/cosmic-clash\/join-roster.json/d' \
    -e '/- --allocated-mode$/d' \
  deploy/k8s/base/fleet.yaml > "$work_dir/fleet.yaml"

kubectl -n cosmic-clash create secret generic cosmic-clash-game-server \
  --from-literal=drain-token=kind-smoke-drain-token \
  --from-literal=join-signing-keys.json='{"kind-smoke-key":"a2luZC1zbW9rZS1zaWduaW5nLWtleQ=="}' \
  --from-literal=join-signing-key-id=kind-smoke-key \
  --dry-run=client -o yaml | kubectl apply -f -
kubectl apply -f deploy/k8s/base/service-accounts.yaml
kubectl apply -f "$work_dir/fleet.yaml"

# The field is readyReplicas, not ready: an Agones Fleet's status carries
# replicas/readyReplicas/reservedReplicas/allocatedReplicas, and the READY
# column printed by kubectl is readyReplicas. Waiting on `.status.ready` could
# never match however healthy the Fleet was, which masked itself as "the Fleet
# never became ready" and sent three separate investigations after the game
# server instead of the assertion.
kubectl wait --for=jsonpath='{.status.readyReplicas}'=2 \
  fleet/cosmic-clash-game -n cosmic-clash --timeout=5m

cat > "$work_dir/allocation.yaml" <<'EOF'
apiVersion: allocation.agones.dev/v1
kind: GameServerAllocation
metadata:
  generateName: cosmic-clash-smoke-
  namespace: cosmic-clash
spec:
  fleet:
    name: cosmic-clash-game
EOF
kubectl create -f "$work_dir/allocation.yaml" -o json > "$work_dir/allocation.json"

python3 scripts/verify_agones_allocation_response.py "$work_dir/allocation.json"