mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
test(kind): dump cluster state before the Agones gate deletes its cluster
This gate fails with nothing but Helm's "context deadline exceeded" and three Deployments reporting Available: 0/1, then the EXIT trap deletes the cluster -- so there is no way to learn why the pods never became ready. Both CI runs and a local run are equally uninformative. Dump node capacity and conditions, pods and recent events for agones-system and cosmic-clash, and describe plus current/previous logs for every not-ready pod, on any failure and before deletion. Events matter as much as pod status here: FailedScheduling, ImagePullBackOff and probe failures are all invisible in a status column. KIND_KEEP_ON_FAILURE=1 retains the cluster for interactive inspection. Same approach that just found the allocated-Compose cause, where a silent assertion had hidden a real 422-instead-of-409 API bug across several CI runs.
This commit is contained in:
@@ -13,8 +13,59 @@ game_server_image="${GAME_SERVER_IMAGE:-cosmic-clash-game-server:kind}"
|
||||
kind_node_image="${KIND_NODE_IMAGE:-kindest/node:v1.33.1}"
|
||||
work_dir="$(mktemp -d "${TMPDIR:-/tmp}/cosmic-clash-agones.XXXXXX")"
|
||||
|
||||
# This gate fails in CI with nothing but Helm's "context deadline exceeded",
|
||||
# and the EXIT trap then deletes the cluster, so there is no way to learn why
|
||||
# the pods never became Available. Dump enough cluster state on failure that a
|
||||
# CI run explains itself without needing a local reproduction -- which is not
|
||||
# equivalent anyway, since a developer machine has different resources and a
|
||||
# different container runtime.
|
||||
#
|
||||
# Set KIND_KEEP_ON_FAILURE=1 to retain the cluster for interactive inspection.
|
||||
on_error() {
|
||||
echo "kind/Agones gate failed at ${BASH_SOURCE[0]}:$1" >&2
|
||||
echo "--- failing command: ${BASH_COMMAND}" >&2
|
||||
}
|
||||
trap 'on_error "$LINENO"' ERR
|
||||
|
||||
dump_cluster_state() {
|
||||
echo "=== node capacity and conditions ===" >&2
|
||||
kubectl get nodes -o wide >&2 2>&1 || true
|
||||
kubectl describe nodes 2>&1 | grep -A 12 -E "Allocated resources|Conditions:" >&2 || true
|
||||
for ns in agones-system cosmic-clash; do
|
||||
echo "=== namespace ${ns}: pods ===" >&2
|
||||
kubectl -n "$ns" get pods -o wide >&2 2>&1 || true
|
||||
# Events explain scheduling/image/probe failures that pod status alone
|
||||
# does not: FailedScheduling, ImagePullBackOff, readiness probe errors.
|
||||
echo "=== namespace ${ns}: recent events ===" >&2
|
||||
kubectl -n "$ns" get events --sort-by=.lastTimestamp 2>&1 | tail -40 >&2 || true
|
||||
for pod in $(kubectl -n "$ns" get pods -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null); do
|
||||
ready="$(kubectl -n "$ns" get pod "$pod" -o jsonpath='{.status.containerStatuses[*].ready}' 2>/dev/null || true)"
|
||||
case "$ready" in
|
||||
*false*|"")
|
||||
echo "=== ${ns}/${pod} is not ready (ready=${ready:-unknown}) ===" >&2
|
||||
kubectl -n "$ns" describe pod "$pod" 2>&1 | tail -35 >&2 || true
|
||||
echo "--- ${ns}/${pod} logs (current) ---" >&2
|
||||
kubectl -n "$ns" logs "$pod" --all-containers --tail=40 >&2 2>&1 || true
|
||||
echo "--- ${ns}/${pod} logs (previous, if it restarted) ---" >&2
|
||||
kubectl -n "$ns" logs "$pod" --all-containers --previous --tail=40 >&2 2>&1 || true
|
||||
;;
|
||||
esac
|
||||
done
|
||||
done
|
||||
echo "=== helm releases ===" >&2
|
||||
helm list --all-namespaces >&2 2>&1 || true
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local status=$?
|
||||
if [[ "$status" != 0 ]] && kubectl cluster-info --context "kind-${cluster_name}" >/dev/null 2>&1; then
|
||||
dump_cluster_state
|
||||
fi
|
||||
if [[ "$status" != 0 && "${KIND_KEEP_ON_FAILURE:-}" == 1 ]]; then
|
||||
echo "kind cluster retained for inspection: kind-${cluster_name} (delete with: kind delete cluster --name ${cluster_name})" >&2
|
||||
rm -rf "$work_dir"
|
||||
exit "$status"
|
||||
fi
|
||||
kind delete cluster --name "$cluster_name" >/dev/null 2>&1 || true
|
||||
rm -rf "$work_dir"
|
||||
exit "$status"
|
||||
|
||||
Reference in New Issue
Block a user