mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
test(kind): dump cluster state before the Agones gate deletes its cluster
This gate fails with nothing but Helm's "context deadline exceeded" and three Deployments reporting Available: 0/1, then the EXIT trap deletes the cluster -- so there is no way to learn why the pods never became ready. Both CI runs and a local run are equally uninformative. Dump node capacity and conditions, pods and recent events for agones-system and cosmic-clash, and describe plus current/previous logs for every not-ready pod, on any failure and before deletion. Events matter as much as pod status here: FailedScheduling, ImagePullBackOff and probe failures are all invisible in a status column. KIND_KEEP_ON_FAILURE=1 retains the cluster for interactive inspection. Same approach that just found the allocated-Compose cause, where a silent assertion had hidden a real 422-instead-of-409 API bug across several CI runs.
This commit is contained in:
@@ -13,8 +13,59 @@ game_server_image="${GAME_SERVER_IMAGE:-cosmic-clash-game-server:kind}"
|
|||||||
kind_node_image="${KIND_NODE_IMAGE:-kindest/node:v1.33.1}"
|
kind_node_image="${KIND_NODE_IMAGE:-kindest/node:v1.33.1}"
|
||||||
work_dir="$(mktemp -d "${TMPDIR:-/tmp}/cosmic-clash-agones.XXXXXX")"
|
work_dir="$(mktemp -d "${TMPDIR:-/tmp}/cosmic-clash-agones.XXXXXX")"
|
||||||
|
|
||||||
|
# This gate fails in CI with nothing but Helm's "context deadline exceeded",
|
||||||
|
# and the EXIT trap then deletes the cluster, so there is no way to learn why
|
||||||
|
# the pods never became Available. Dump enough cluster state on failure that a
|
||||||
|
# CI run explains itself without needing a local reproduction -- which is not
|
||||||
|
# equivalent anyway, since a developer machine has different resources and a
|
||||||
|
# different container runtime.
|
||||||
|
#
|
||||||
|
# Set KIND_KEEP_ON_FAILURE=1 to retain the cluster for interactive inspection.
|
||||||
|
on_error() {
|
||||||
|
echo "kind/Agones gate failed at ${BASH_SOURCE[0]}:$1" >&2
|
||||||
|
echo "--- failing command: ${BASH_COMMAND}" >&2
|
||||||
|
}
|
||||||
|
trap 'on_error "$LINENO"' ERR
|
||||||
|
|
||||||
|
dump_cluster_state() {
|
||||||
|
echo "=== node capacity and conditions ===" >&2
|
||||||
|
kubectl get nodes -o wide >&2 2>&1 || true
|
||||||
|
kubectl describe nodes 2>&1 | grep -A 12 -E "Allocated resources|Conditions:" >&2 || true
|
||||||
|
for ns in agones-system cosmic-clash; do
|
||||||
|
echo "=== namespace ${ns}: pods ===" >&2
|
||||||
|
kubectl -n "$ns" get pods -o wide >&2 2>&1 || true
|
||||||
|
# Events explain scheduling/image/probe failures that pod status alone
|
||||||
|
# does not: FailedScheduling, ImagePullBackOff, readiness probe errors.
|
||||||
|
echo "=== namespace ${ns}: recent events ===" >&2
|
||||||
|
kubectl -n "$ns" get events --sort-by=.lastTimestamp 2>&1 | tail -40 >&2 || true
|
||||||
|
for pod in $(kubectl -n "$ns" get pods -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null); do
|
||||||
|
ready="$(kubectl -n "$ns" get pod "$pod" -o jsonpath='{.status.containerStatuses[*].ready}' 2>/dev/null || true)"
|
||||||
|
case "$ready" in
|
||||||
|
*false*|"")
|
||||||
|
echo "=== ${ns}/${pod} is not ready (ready=${ready:-unknown}) ===" >&2
|
||||||
|
kubectl -n "$ns" describe pod "$pod" 2>&1 | tail -35 >&2 || true
|
||||||
|
echo "--- ${ns}/${pod} logs (current) ---" >&2
|
||||||
|
kubectl -n "$ns" logs "$pod" --all-containers --tail=40 >&2 2>&1 || true
|
||||||
|
echo "--- ${ns}/${pod} logs (previous, if it restarted) ---" >&2
|
||||||
|
kubectl -n "$ns" logs "$pod" --all-containers --previous --tail=40 >&2 2>&1 || true
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
done
|
||||||
|
echo "=== helm releases ===" >&2
|
||||||
|
helm list --all-namespaces >&2 2>&1 || true
|
||||||
|
}
|
||||||
|
|
||||||
cleanup() {
|
cleanup() {
|
||||||
local status=$?
|
local status=$?
|
||||||
|
if [[ "$status" != 0 ]] && kubectl cluster-info --context "kind-${cluster_name}" >/dev/null 2>&1; then
|
||||||
|
dump_cluster_state
|
||||||
|
fi
|
||||||
|
if [[ "$status" != 0 && "${KIND_KEEP_ON_FAILURE:-}" == 1 ]]; then
|
||||||
|
echo "kind cluster retained for inspection: kind-${cluster_name} (delete with: kind delete cluster --name ${cluster_name})" >&2
|
||||||
|
rm -rf "$work_dir"
|
||||||
|
exit "$status"
|
||||||
|
fi
|
||||||
kind delete cluster --name "$cluster_name" >/dev/null 2>&1 || true
|
kind delete cluster --name "$cluster_name" >/dev/null 2>&1 || true
|
||||||
rm -rf "$work_dir"
|
rm -rf "$work_dir"
|
||||||
exit "$status"
|
exit "$status"
|
||||||
|
|||||||
Reference in New Issue
Block a user