Files
CosmicClash/Game/tests/agones_sdk_smoke.gd
T
Josh Creek 0de97381b7 fix(agones): stop a dead health loop from passing as a healthy server
Every allocated GameServer reached Ready and was recycled by Agones ~20s
later. Health pings are the game process's job by design -- the
supervisor has no health implementation at all -- so a server that stops
pinging is exactly what Agones is built to reclaim.

start_health() armed a Timer on a node that might not be inside the
SceneTree. A Timer only ticks inside the tree, so the node reported
itself configured, sent nothing, and said nothing about it. It now
returns a bool, refuses loudly when unconfigured, and defers to _ready()
when called before parenting, so the SDK arms its own timer and no
caller has to get the ordering right. server_boot.gd defers the add like
every sibling does (§9 gotcha 27) and logs when AGONES_SDK_HTTP_PORT is
missing, which previously read identically to a healthy start.

Also bounded the in-flight latch: it is set across an await, so a request
that never completes would silence health permanently. Defence in depth
rather than an observed fault.

Tests target the contract rather than the mechanism: a test that parents
the SDK correctly and asserts pings passes with the bug present, because
the defect was in the wiring. The unit tests assert start_health()
cannot claim success out of tree, and were confirmed to fail against the
previous code. The smoke gains a counting sidecar and asserts a
*repeating* ping -- it reports "health pings in 3.0s = 1, want at least
2" when the loop is broken, which is the production symptom exactly. It
is also now actually run: nothing referenced it before.

Two diagnostic fixes, both of which changed conclusions during this work:

The kind gate only built the game-server image when the tag was absent,
so a local rerun silently verified whatever was built last. That is why
local runs and CI disagreed about the same commit. It now builds by
default, with KIND_REUSE_GAME_SERVER_IMAGE=1 as the opt-in fast path.

The failure dump logged only not-ready pods, and used --all-containers
with a shared tail. A GameServer recycled after reaching Ready leaves no
unready pod behind, and the Agones sidecar out-logs the game server, so
the relevant output was never captured. It now dumps every pod, per
container, current and previous, plus the GameServer and Fleet resources
-- Agones' own state machine is what rejects these.
2026-09-05 21:35:50 +01:00

131 lines
4.5 KiB
GDScript

extends SceneTree
# Headless smoke for the Agones SDK bridge. Run by
# scripts/verify_multiplayer_local.sh:
# godot --headless --path Game --script res://tests/agones_sdk_smoke.gd
#
# Phase 1 drives each REST call directly. Phase 2 covers what phase 1 cannot:
# that start_health() produces a *repeating* ping. That is the property Agones
# actually enforces -- one ping proves nothing, because the Fleet recycles any
# GameServer that stops pinging for periodSeconds * failureThreshold -- and its
# absence is what silently recycled every allocated server.
const ServerControlScript = preload("res://scripts/server_control.gd")
const AgonesSDKScript = preload("res://scripts/agones_sdk.gd")
const PORT := 18081
const HEALTH_PORT := 18082
# start_health() pings every 2s, so three seconds must contain at least two.
const HEALTH_OBSERVATION_SECONDS := 3.0
const MINIMUM_EXPECTED_PINGS := 2
# Counting stand-in for the Agones sidecar. ServerControl answers /health but
# cannot report how often it was called, and asserting repetition is the whole
# point here, so this counts rather than changing production code for a test.
class CountingSidecar extends Node:
var health_pings := 0
var _listener := TCPServer.new()
var _peers: Array = []
func start(port: int) -> Error:
return _listener.listen(port, "127.0.0.1")
func stop() -> void:
_listener.stop()
for peer in _peers:
if is_instance_valid(peer):
peer.disconnect_from_host()
_peers.clear()
func _process(_delta: float) -> void:
while _listener.is_connection_available():
_peers.append(_listener.take_connection())
for i in range(_peers.size() - 1, -1, -1):
var peer: StreamPeerTCP = _peers[i]
if peer.get_status() != StreamPeerTCP.STATUS_CONNECTED:
_peers.remove_at(i)
continue
var available := peer.get_available_bytes()
if available <= 0:
continue
var request := peer.get_utf8_string(available)
if "\r\n\r\n" not in request:
continue
if request.begins_with("POST /health"):
health_pings += 1
var body := "{}"
peer.put_data(("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: %d\r\nConnection: close\r\n\r\n%s" % [body.length(), body]).to_utf8_buffer())
peer.disconnect_from_host()
_peers.remove_at(i)
func _init() -> void:
if not await _direct_calls_smoke():
quit(1)
return
if not await _repeating_health_smoke():
quit(1)
return
print("Agones SDK smoke passed")
quit(0)
func _direct_calls_smoke() -> bool:
var fake_sidecar = ServerControlScript.new()
root.add_child(fake_sidecar)
if fake_sidecar.start(PORT) != OK:
printerr("fake sidecar failed to bind")
return false
fake_sidecar.set_process_ready(true)
var sdk = AgonesSDKScript.new()
root.add_child(sdk)
if not sdk.configure_for_testing("http://127.0.0.1:%d" % PORT):
printerr("SDK test configuration failed")
return false
await process_frame
var health_status := await sdk.health()
var ready_status := await sdk.mark_ready()
var annotation_status := await sdk.set_annotation("match", "result")
var shutdown_status := await sdk.shutdown()
fake_sidecar.stop()
fake_sidecar.queue_free()
sdk.queue_free()
if health_status != 200 or ready_status != 200 or annotation_status < 400 or shutdown_status < 400:
printerr("Agones SDK smoke statuses health=%d ready=%d annotation=%d shutdown=%d" % [health_status, ready_status, annotation_status, shutdown_status])
return false
return true
func _repeating_health_smoke() -> bool:
var sidecar := CountingSidecar.new()
root.add_child(sidecar)
if sidecar.start(HEALTH_PORT) != OK:
printerr("counting sidecar failed to bind")
return false
# Configure before parenting and let the node arm its own timer on _ready(),
# which is exactly how server_boot.gd wires it in an allocated pod.
var sdk = AgonesSDKScript.new()
if not sdk.configure_for_testing("http://127.0.0.1:%d" % HEALTH_PORT):
printerr("health SDK configuration failed")
return false
if sdk.start_health():
printerr("start_health() reported success while the node was outside the tree")
return false
root.add_child(sdk)
await process_frame
if not sdk.health_is_running():
printerr("health loop did not arm once the node entered the tree")
return false
await create_timer(HEALTH_OBSERVATION_SECONDS).timeout
var observed := sidecar.health_pings
sdk.stop_health()
sidecar.stop()
sdk.queue_free()
sidecar.queue_free()
if observed < MINIMUM_EXPECTED_PINGS:
printerr("Agones health pings in %.1fs = %d, want at least %d; the health loop is not repeating" % [HEALTH_OBSERVATION_SECONDS, observed, MINIMUM_EXPECTED_PINGS])
return false
return true