Files
CosmicClash/Game/scripts/agones_sdk.gd
T
Josh Creek 0de97381b7 fix(agones): stop a dead health loop from passing as a healthy server
Every allocated GameServer reached Ready and was recycled by Agones ~20s
later. Health pings are the game process's job by design -- the
supervisor has no health implementation at all -- so a server that stops
pinging is exactly what Agones is built to reclaim.

start_health() armed a Timer on a node that might not be inside the
SceneTree. A Timer only ticks inside the tree, so the node reported
itself configured, sent nothing, and said nothing about it. It now
returns a bool, refuses loudly when unconfigured, and defers to _ready()
when called before parenting, so the SDK arms its own timer and no
caller has to get the ordering right. server_boot.gd defers the add like
every sibling does (§9 gotcha 27) and logs when AGONES_SDK_HTTP_PORT is
missing, which previously read identically to a healthy start.

Also bounded the in-flight latch: it is set across an await, so a request
that never completes would silence health permanently. Defence in depth
rather than an observed fault.

Tests target the contract rather than the mechanism: a test that parents
the SDK correctly and asserts pings passes with the bug present, because
the defect was in the wiring. The unit tests assert start_health()
cannot claim success out of tree, and were confirmed to fail against the
previous code. The smoke gains a counting sidecar and asserts a
*repeating* ping -- it reports "health pings in 3.0s = 1, want at least
2" when the loop is broken, which is the production symptom exactly. It
is also now actually run: nothing referenced it before.

Two diagnostic fixes, both of which changed conclusions during this work:

The kind gate only built the game-server image when the tag was absent,
so a local rerun silently verified whatever was built last. That is why
local runs and CI disagreed about the same commit. It now builds by
default, with KIND_REUSE_GAME_SERVER_IMAGE=1 as the opt-in fast path.

The failure dump logged only not-ready pods, and used --all-containers
with a shared tail. A GameServer recycled after reaching Ready leaves no
unready pod behind, and the Agones sidecar out-logs the game server, so
the relevant output was never captured. It now dumps every pod, per
container, current and previous, plus the GameServer and Fleet resources
-- Agones' own state machine is what rejects these.
2026-09-05 21:35:50 +01:00

153 lines
5.1 KiB
GDScript

class_name AgonesSDK
extends Node
# Dependency-free REST bridge for the Agones sidecar. The Go supervisor owns
# the process-ready probe and /ready transition; this node owns the game
# process's periodic Health pings and terminal Shutdown/annotation calls.
const HEALTH_INTERVAL_SECONDS := 2.0
const REQUEST_TIMEOUT_SECONDS := 2.0
const MAX_ANNOTATION_VALUE_LENGTH := 4096
var _base_url := ""
var _health_timer: Timer = null
var _health_in_flight := false
var _health_started_msec := 0
# Set when start_health() is called before this node is inside the tree, so
# _ready() can arm the timer at the first moment it is legal to do so.
var _health_pending := false
# Health is armed here rather than by the caller. A Timer only ticks while its
# owner is inside the SceneTree, so arming it from a caller that has not yet
# parented this node produces a node that looks configured and never pings —
# which is exactly how every allocated GameServer silently failed its Agones
# health check and was recycled.
func _ready() -> void:
if _health_pending:
_health_pending = false
_arm_health()
func configure_from_environment() -> bool:
var port := OS.get_environment("AGONES_SDK_HTTP_PORT")
if port.is_empty() or not port.is_valid_int() or int(port) < 1 or int(port) > 65535:
return false
_base_url = "http://127.0.0.1:%d" % int(port)
return true
func configure_for_testing(base_url: String) -> bool:
if not base_url.begins_with("http://127.0.0.1:") and not base_url.begins_with("http://localhost:"):
return false
_base_url = base_url.trim_suffix("/")
return true
func is_available() -> bool:
return not _base_url.is_empty()
# Returns whether health pings are running. It is a bool rather than void
# because every way this can fail used to be silent, and a game server that
# believes it is healthy while sending nothing is worse than one that refuses
# to start: Agones recycles the former every ~20 seconds forever.
func start_health() -> bool:
if not is_available():
push_error("AgonesSDK: start_health() before configuration; no health pings will be sent")
return false
if _health_timer != null:
return true
if not is_inside_tree():
# Deferred rather than fatal: the caller may legitimately configure
# before parenting. _ready() arms it. Still reported, because if the
# node is never parented this is the whole failure.
_health_pending = true
push_warning("AgonesSDK: start_health() called outside the tree; deferring until ready")
return false
_arm_health()
return true
func _arm_health() -> void:
if _health_timer != null:
return
_health_timer = Timer.new()
_health_timer.name = "AgonesHealth"
_health_timer.wait_time = HEALTH_INTERVAL_SECONDS
_health_timer.one_shot = false
_health_timer.timeout.connect(_send_health)
add_child(_health_timer)
_health_timer.start()
_send_health()
func health_is_running() -> bool:
return _health_timer != null and is_inside_tree()
func stop_health() -> void:
if _health_timer != null:
_health_timer.stop()
_health_timer.queue_free()
_health_timer = null
func health() -> int:
return await _request(HTTPClient.METHOD_POST, "/health", {})
func mark_ready() -> int:
return await _request(HTTPClient.METHOD_POST, "/ready", {})
func shutdown() -> int:
return await _request(HTTPClient.METHOD_POST, "/shutdown", {})
func set_annotation(key: String, value: String) -> int:
if not annotation_is_valid(key, value):
return 400
return await _request(HTTPClient.METHOD_PUT, "/metadata/annotation", {"key": key, "value": value})
static func annotation_is_valid(key: String, value: String) -> bool:
return not (key.is_empty() or value.is_empty() or value.length() > MAX_ANNOTATION_VALUE_LENGTH or "\n" in key or "\r" in key or "\n" in value or "\r" in value)
func _send_health() -> void:
if not is_available():
return
# The latch stops overlapping requests, but it must never become permanent.
# It is set across an await, and a request that never completes would
# otherwise silence health for the lifetime of the process. HTTPRequest's
# own timeout normally resolves this; the elapsed check is the backstop for
# the case where request_completed never fires at all.
if _health_in_flight:
var stuck_for := Time.get_ticks_msec() - _health_started_msec
if stuck_for < int(REQUEST_TIMEOUT_SECONDS * 2.0 * 1000.0):
return
push_warning("Agones health ping did not complete in %dms; sending another" % stuck_for)
_health_in_flight = true
_health_started_msec = Time.get_ticks_msec()
var status := await health()
_health_in_flight = false
if status < 200 or status >= 300:
push_warning("Agones health ping failed (%d)" % status)
func _request(method: int, path: String, payload: Dictionary) -> int:
if not is_available() or not path.begins_with("/"):
return 408
var request := HTTPRequest.new()
request.timeout = REQUEST_TIMEOUT_SECONDS
add_child(request)
var body := JSON.stringify(payload)
var err := request.request(_base_url + path, PackedStringArray(["Content-Type: application/json"]), method, body)
if err != OK:
request.queue_free()
return 599
var result = await request.request_completed
request.queue_free()
return int(result[1])