test(kind): dump cluster state before the Agones gate deletes its cluster

This gate fails with nothing but Helm's "context deadline exceeded" and
three Deployments reporting Available: 0/1, then the EXIT trap deletes
the cluster -- so there is no way to learn why the pods never became
ready. Both CI runs and a local run are equally uninformative.

Dump node capacity and conditions, pods and recent events for
agones-system and cosmic-clash, and describe plus current/previous logs
for every not-ready pod, on any failure and before deletion. Events
matter as much as pod status here: FailedScheduling, ImagePullBackOff
and probe failures are all invisible in a status column.

KIND_KEEP_ON_FAILURE=1 retains the cluster for interactive inspection.

Same approach that just found the allocated-Compose cause, where a
silent assertion had hidden a real 422-instead-of-409 API bug across
several CI runs.
This commit is contained in:
Josh Creek
2026-09-05 19:57:05 +01:00
parent 14da286e11
commit 9ab1bec89a
+51
View File
@@ -13,8 +13,59 @@ game_server_image="${GAME_SERVER_IMAGE:-cosmic-clash-game-server:kind}"
kind_node_image="${KIND_NODE_IMAGE:-kindest/node:v1.33.1}"
work_dir="$(mktemp -d "${TMPDIR:-/tmp}/cosmic-clash-agones.XXXXXX")"
# This gate fails in CI with nothing but Helm's "context deadline exceeded",
# and the EXIT trap then deletes the cluster, so there is no way to learn why
# the pods never became Available. Dump enough cluster state on failure that a
# CI run explains itself without needing a local reproduction -- which is not
# equivalent anyway, since a developer machine has different resources and a
# different container runtime.
#
# Set KIND_KEEP_ON_FAILURE=1 to retain the cluster for interactive inspection.
on_error() {
echo "kind/Agones gate failed at ${BASH_SOURCE[0]}:$1" >&2
echo "--- failing command: ${BASH_COMMAND}" >&2
}
trap 'on_error "$LINENO"' ERR
dump_cluster_state() {
echo "=== node capacity and conditions ===" >&2
kubectl get nodes -o wide >&2 2>&1 || true
kubectl describe nodes 2>&1 | grep -A 12 -E "Allocated resources|Conditions:" >&2 || true
for ns in agones-system cosmic-clash; do
echo "=== namespace ${ns}: pods ===" >&2
kubectl -n "$ns" get pods -o wide >&2 2>&1 || true
# Events explain scheduling/image/probe failures that pod status alone
# does not: FailedScheduling, ImagePullBackOff, readiness probe errors.
echo "=== namespace ${ns}: recent events ===" >&2
kubectl -n "$ns" get events --sort-by=.lastTimestamp 2>&1 | tail -40 >&2 || true
for pod in $(kubectl -n "$ns" get pods -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null); do
ready="$(kubectl -n "$ns" get pod "$pod" -o jsonpath='{.status.containerStatuses[*].ready}' 2>/dev/null || true)"
case "$ready" in
*false*|"")
echo "=== ${ns}/${pod} is not ready (ready=${ready:-unknown}) ===" >&2
kubectl -n "$ns" describe pod "$pod" 2>&1 | tail -35 >&2 || true
echo "--- ${ns}/${pod} logs (current) ---" >&2
kubectl -n "$ns" logs "$pod" --all-containers --tail=40 >&2 2>&1 || true
echo "--- ${ns}/${pod} logs (previous, if it restarted) ---" >&2
kubectl -n "$ns" logs "$pod" --all-containers --previous --tail=40 >&2 2>&1 || true
;;
esac
done
done
echo "=== helm releases ===" >&2
helm list --all-namespaces >&2 2>&1 || true
}
cleanup() {
local status=$?
if [[ "$status" != 0 ]] && kubectl cluster-info --context "kind-${cluster_name}" >/dev/null 2>&1; then
dump_cluster_state
fi
if [[ "$status" != 0 && "${KIND_KEEP_ON_FAILURE:-}" == 1 ]]; then
echo "kind cluster retained for inspection: kind-${cluster_name} (delete with: kind delete cluster --name ${cluster_name})" >&2
rm -rf "$work_dir"
exit "$status"
fi
kind delete cluster --name "$cluster_name" >/dev/null 2>&1 || true
rm -rf "$work_dir"
exit "$status"