mirror of
https://github.com/jcreek/CosmicClash.git
synced 2026-09-10 16:04:04 +00:00
0de97381b7
Every allocated GameServer reached Ready and was recycled by Agones ~20s later. Health pings are the game process's job by design -- the supervisor has no health implementation at all -- so a server that stops pinging is exactly what Agones is built to reclaim. start_health() armed a Timer on a node that might not be inside the SceneTree. A Timer only ticks inside the tree, so the node reported itself configured, sent nothing, and said nothing about it. It now returns a bool, refuses loudly when unconfigured, and defers to _ready() when called before parenting, so the SDK arms its own timer and no caller has to get the ordering right. server_boot.gd defers the add like every sibling does (§9 gotcha 27) and logs when AGONES_SDK_HTTP_PORT is missing, which previously read identically to a healthy start. Also bounded the in-flight latch: it is set across an await, so a request that never completes would silence health permanently. Defence in depth rather than an observed fault. Tests target the contract rather than the mechanism: a test that parents the SDK correctly and asserts pings passes with the bug present, because the defect was in the wiring. The unit tests assert start_health() cannot claim success out of tree, and were confirmed to fail against the previous code. The smoke gains a counting sidecar and asserts a *repeating* ping -- it reports "health pings in 3.0s = 1, want at least 2" when the loop is broken, which is the production symptom exactly. It is also now actually run: nothing referenced it before. Two diagnostic fixes, both of which changed conclusions during this work: The kind gate only built the game-server image when the tag was absent, so a local rerun silently verified whatever was built last. That is why local runs and CI disagreed about the same commit. It now builds by default, with KIND_REUSE_GAME_SERVER_IMAGE=1 as the opt-in fast path. The failure dump logged only not-ready pods, and used --all-containers with a shared tail. A GameServer recycled after reaching Ready leaves no unready pod behind, and the Agones sidecar out-logs the game server, so the relevant output was never captured. It now dumps every pod, per container, current and previous, plus the GameServer and Fleet resources -- Agones' own state machine is what rejects these.
299 lines
15 KiB
GDScript
299 lines
15 KiB
GDScript
extends Node
|
|
|
|
const NetCodec = preload("res://scripts/net_codec.gd")
|
|
const ServerControlScript = preload("res://scripts/server_control.gd")
|
|
const AgonesSDKScript = preload("res://scripts/agones_sdk.gd")
|
|
const AssignmentState = preload("res://scripts/assignment_state.gd")
|
|
const ConnectionLeaseClientScript = preload("res://scripts/connection_lease_client.gd")
|
|
const ServerResultClientScript = preload("res://scripts/server_result_client.gd")
|
|
|
|
# Headless dedicated server entry point (task 1.6). Parses CLI args, hosts
|
|
# via NetworkManager, logs structured lines, and watches for physics-tick
|
|
# overrun (§9 gotcha 9: Engine.max_physics_steps_per_frame defaults to 8;
|
|
# a tick overrunning 16.7ms backs up the accumulator and the next frame
|
|
# runs multiple ticks, spiking CPU further — worth logging, not just
|
|
# silently absorbing).
|
|
#
|
|
# Run: godot --headless --path Game res://scenes/server_boot.tscn -- --port=7777
|
|
#
|
|
# Dedicated *exports* reach this scene without the CLI argument above, via the
|
|
# `run/main_scene.dedicated_server` feature override in project.godot — the same
|
|
# project-setting mechanism the training export uses for training.tscn. That
|
|
# override is deliberately uncommented in project.godot: Godot's ConfigFile
|
|
# writer does not round-trip comments, and a `#` block directly above a setting
|
|
# can be spliced into the setting's own line on rewrite, silently commenting it
|
|
# out and sending dedicated builds to the interactive main menu instead of here.
|
|
# `tests/cases/test_project_settings.gd` fails loudly if that ever happens.
|
|
#
|
|
# Deliberately does not spawn a match yet — that's Phase 2's networked_match
|
|
# scene. This is just the process shell: listen, log, idle cheaply.
|
|
|
|
var _last_physics_frame := 0
|
|
var config: ServerConfig = null
|
|
var _watchdog_armed := false # skip the first _process(): engine startup scheduling can batch several physics frames before the first idle frame runs, which isn't a real overrun
|
|
var _control: ServerControl = null
|
|
var _match_loop: ServerMatchLoop = null
|
|
var _agones = null
|
|
var _connection_leases = null
|
|
var _result_client = null
|
|
var _drain_requested := false
|
|
|
|
|
|
func _ready() -> void:
|
|
Engine.max_fps = 60 # a server never renders; this just caps the idle-frame poll rate so it doesn't spin
|
|
|
|
# Task 6.3. This process owns the whole command line, so it parses STRICTLY:
|
|
# an unknown flag or an out-of-range value stops the server with a message
|
|
# rather than starting one that silently ignores half of what it was told.
|
|
config = ServerConfig.parse(OS.get_cmdline_user_args())
|
|
if config.help_requested:
|
|
print(ServerConfig.help_text())
|
|
get_tree().quit(0)
|
|
return
|
|
if not config.is_valid():
|
|
# Straight to stderr-ish plain print rather than through _log: the log
|
|
# level itself may be one of the things that failed to parse, and an
|
|
# operator running this by hand needs to see every problem at once, not
|
|
# the first one.
|
|
printerr("cosmic-clash-server: refusing to start")
|
|
for problem in config.errors:
|
|
printerr(" %s" % problem)
|
|
printerr("try --help")
|
|
get_tree().quit(1)
|
|
return
|
|
ServerLog.configure(String(config.get_value("log-level")))
|
|
var port := int(config.get_value("port"))
|
|
var max_clients := int(config.get_value("max-clients"))
|
|
var allocated_mode := bool(config.get_value("allocated-mode"))
|
|
var assigned_transport := String(config.get_value("transport"))
|
|
# Hosted SDR is not wired into the Godot transport layer yet. Refuse the
|
|
# allocated launch rather than silently opening an ENet endpoint that does
|
|
# not match the signed assignment's transport contract.
|
|
if allocated_mode and assigned_transport != NetworkManager.TRANSPORT_ENET:
|
|
printerr("cosmic-clash-server: allocated transport '%s' is not supported by this build" % assigned_transport)
|
|
get_tree().quit(1)
|
|
return
|
|
# Agones injects its HTTP port into every managed game-server container.
|
|
# Keep lifecycle readiness and health active in the reduced kind smoke even
|
|
# though that environment intentionally omits allocation/roster semantics.
|
|
var agones_managed := not OS.get_environment("AGONES_SDK_HTTP_PORT").is_empty()
|
|
if allocated_mode or agones_managed:
|
|
_control = ServerControlScript.new()
|
|
_control.name = "ServerControl"
|
|
_control.drain_requested.connect(_on_drain_requested)
|
|
_control.initial_connect_ready.connect(_on_initial_connect_ready)
|
|
get_tree().root.add_child.call_deferred(_control)
|
|
var control_err := _control.start(int(config.get_value("readiness-port")), OS.get_environment(String(config.get_value("drain-token-env"))))
|
|
if control_err != OK:
|
|
printerr("cosmic-clash-server: refusing to start with invalid readiness control port")
|
|
get_tree().quit(1)
|
|
return
|
|
if agones_managed:
|
|
_agones = AgonesSDKScript.new()
|
|
_agones.name = "AgonesSDK"
|
|
# Configure before parenting, then request health and defer the add like
|
|
# every other node here (§9 gotcha 27: add_child() on get_tree().root
|
|
# from inside _ready() is refused because the tree is still attaching
|
|
# this very node, and the refusal is not catchable from GDScript). The
|
|
# SDK arms its own timer in _ready(), so nothing depends on the order
|
|
# these deferred calls happen to flush in.
|
|
if _agones.configure_from_environment():
|
|
_agones.start_health()
|
|
else:
|
|
# Never silent: without this the log looks identical to a healthy
|
|
# server right up until Agones recycles it.
|
|
printerr("cosmic-clash-server: AGONES_SDK_HTTP_PORT is missing or invalid; Agones health pings are disabled")
|
|
get_tree().root.add_child.call_deferred(_agones)
|
|
if allocated_mode:
|
|
var roster_file := String(config.get_value("join-authorisations-file"))
|
|
var key_file := String(config.get_value("join-authorisations-key-file"))
|
|
var roster_json := FileAccess.get_file_as_string(roster_file)
|
|
var signing_keys := _load_join_signing_keys(key_file)
|
|
var roster_tokens = JSON.parse_string(roster_json)
|
|
if not roster_tokens is Array or roster_tokens.is_empty() or signing_keys.is_empty() or not MatchNet.configure_join_authorisations(roster_tokens, {
|
|
"match_id": String(config.get_value("match-id")),
|
|
"server_id": String(config.get_value("server-id")),
|
|
"protocol": str(NetCodec.PROTOCOL_VERSION),
|
|
"protocol_version": NetCodec.PROTOCOL_VERSION,
|
|
}, signing_keys) or MatchNet.assigned_player_slots().size() != roster_tokens.size():
|
|
printerr("cosmic-clash-server: refusing to start with invalid join-authorisations-file")
|
|
get_tree().quit(1)
|
|
return
|
|
# An allocated process owns exactly the roster issued for this match.
|
|
# Never let the general-purpose direct-server default (one player) start
|
|
# an allocated match with only a partial assignment admitted.
|
|
config.values["min-players"] = required_min_players(true, roster_tokens.size(), int(config.get_value("min-players")))
|
|
_connection_leases = ConnectionLeaseClientScript.new()
|
|
_connection_leases.name = "ConnectionLeases"
|
|
var lease_url := OS.get_environment("COSMIC_CLASH_CONTROL_PLANE_URL")
|
|
var lease_token := OS.get_environment("COSMIC_CLASH_WORKLOAD_TOKEN")
|
|
if _connection_leases.configure(lease_url, lease_token, String(config.get_value("match-id")), String(config.get_value("server-id"))):
|
|
_connection_leases.reconciliation_failed.connect(_on_connection_lease_reconciliation_failed)
|
|
get_tree().root.add_child.call_deferred(_connection_leases)
|
|
MatchNet.configure_connection_lease_callbacks(_connection_leases.claim, _connection_leases.record_disconnect)
|
|
else:
|
|
_connection_leases.queue_free()
|
|
_connection_leases = null
|
|
# Allocated matches must never fall back to an in-memory connection
|
|
# generation. Doing so would admit a player without the durable fence
|
|
# that prevents a second process (or a stale peer) from owning the same
|
|
# ranked slot. Direct/community servers do not enter this branch.
|
|
printerr("cosmic-clash-server: refusing allocated startup without connection-lease configuration")
|
|
get_tree().quit(1)
|
|
return
|
|
_result_client = ServerResultClientScript.new()
|
|
_result_client.name = "ServerResults"
|
|
if not _result_client.configure(lease_url, lease_token, String(config.get_value("match-id")), String(config.get_value("server-id"))):
|
|
printerr("cosmic-clash-server: refusing allocated startup without result-submission configuration")
|
|
get_tree().quit(1)
|
|
return
|
|
_result_client.accepted.connect(func(): MatchNet.result_submission_accepted.emit())
|
|
_result_client.retrying.connect(func(http_code): MatchNet.result_submission_retrying.emit(http_code))
|
|
get_tree().root.add_child.call_deferred(_result_client)
|
|
MatchNet.configure_result_submission(_result_client.submit)
|
|
|
|
NetworkManager.client_connected.connect(_on_client_connected)
|
|
NetworkManager.client_disconnected.connect(_on_client_disconnected)
|
|
MatchNet.player_joined.connect(_on_player_joined)
|
|
MatchNet.player_left.connect(_on_player_left)
|
|
|
|
var err := NetworkManager.host(port, max_clients)
|
|
if err != OK:
|
|
ServerLog.error("server_boot_failed", {"port": port, "error": error_string(err)})
|
|
get_tree().quit(1)
|
|
return
|
|
if _control != null:
|
|
_control.set_process_ready(true)
|
|
_install_match_loop()
|
|
ServerLog.info("server_started", {
|
|
"port": port, "max_clients": max_clients, "log_level": ServerLog.level_name(),
|
|
"min_players": int(config.get_value("min-players")),
|
|
"max_matches": 1 if allocated_mode else int(config.get_value("max-matches")),
|
|
"arena_rotation": String(config.get_value("arena-rotation")),
|
|
"allocated_mode": allocated_mode,
|
|
"match_id": String(config.get_value("match-id")) if allocated_mode else "",
|
|
"server_id": String(config.get_value("server-id")) if allocated_mode else "",
|
|
"region": String(config.get_value("region")) if allocated_mode else "",
|
|
"transport": assigned_transport if allocated_mode else NetworkManager.TRANSPORT_ENET,
|
|
})
|
|
_last_physics_frame = Engine.get_physics_frames()
|
|
|
|
|
|
# Task 6.5. Parented to the ROOT rather than to this node: the loop calls
|
|
# change_scene_to_file, which frees the current scene — and this boot scene IS
|
|
# the current scene, so a loop parented here would be freed by the first match
|
|
# it started. Same constraint the smoke-test hooks document.
|
|
func _install_match_loop() -> void:
|
|
var loop := ServerMatchLoop.new()
|
|
_match_loop = loop
|
|
loop.name = "ServerMatchLoop"
|
|
loop.min_players = int(config.get_value("min-players"))
|
|
loop.start_countdown_seconds = float(config.get_value("start-countdown"))
|
|
loop.max_matches = 1 if bool(config.get_value("allocated-mode")) else int(config.get_value("max-matches"))
|
|
loop.rotation_mode = String(config.get_value("arena-rotation"))
|
|
loop.allocated_mode = bool(config.get_value("allocated-mode"))
|
|
loop.allocated_playlist = String(config.get_value("playlist"))
|
|
loop.allocated_roster_size = MatchNet.assigned_player_slots().size() if loop.allocated_mode else 0
|
|
loop.allocated_arena_path = String(config.get_value("arena-path"))
|
|
# The backend's fair timeout starts only after durable assignment-ready.
|
|
# When a control plane is present, the supervisor arms this loop through
|
|
# the authenticated local control endpoint after that transition commits.
|
|
loop.allocated_admission_armed = not loop.allocated_mode or OS.get_environment("COSMIC_CLASH_INITIAL_CONNECT_SIGNAL_REQUIRED") != "1"
|
|
get_tree().root.add_child.call_deferred(loop)
|
|
|
|
|
|
func _process(_delta: float) -> void:
|
|
NetworkManager.poll()
|
|
if _drain_requested:
|
|
var scene := get_tree().current_scene
|
|
if not (is_instance_valid(scene) and scene.is_in_group("game")) and MatchNet.roster.is_empty():
|
|
get_tree().quit(0)
|
|
var current := Engine.get_physics_frames()
|
|
var steps := current - _last_physics_frame
|
|
_last_physics_frame = current
|
|
# §9 gotcha 6: with physics_jitter_fix = 0.0, frames legitimately
|
|
# alternate between 0 and 2 ticks even on an idle, healthy server —
|
|
# that's expected quantisation, not backlog. A real overrun is the
|
|
# accumulator failing to drain back down, i.e. 3+ ticks in one frame.
|
|
if steps > 2 and _watchdog_armed:
|
|
ServerLog.warn("physics_overrun", {"steps": steps})
|
|
_watchdog_armed = true
|
|
|
|
|
|
func _physics_process(_delta: float) -> void:
|
|
NetworkManager.poll()
|
|
|
|
|
|
func _on_client_connected(peer_id: int) -> void:
|
|
ServerLog.debug("peer_connected", {"peer_id": peer_id})
|
|
|
|
|
|
func _on_client_disconnected(peer_id: int) -> void:
|
|
ServerLog.debug("peer_disconnected", {"peer_id": peer_id})
|
|
|
|
|
|
func _on_player_joined(peer_id: int, player_name: String) -> void:
|
|
ServerLog.info("player_joined", {"peer_id": peer_id, "name": player_name, "roster": MatchNet.roster.size()})
|
|
|
|
|
|
func _on_player_left(peer_id: int) -> void:
|
|
ServerLog.info("player_left", {"peer_id": peer_id, "roster": MatchNet.roster.size()})
|
|
|
|
|
|
func _on_drain_requested() -> void:
|
|
_drain_requested = true
|
|
MatchNet.admissions_open = false
|
|
MatchNet.broadcast_server_shutdown("server_draining")
|
|
ServerLog.info("server_draining", {"reason": "control_request"})
|
|
|
|
|
|
func _on_initial_connect_ready() -> void:
|
|
if _match_loop != null and is_instance_valid(_match_loop):
|
|
_match_loop.arm_allocated_admission()
|
|
ServerLog.info("initial_connect_window_started", {"match_id": String(config.get_value("match-id"))})
|
|
|
|
|
|
func _on_connection_lease_reconciliation_failed(reason: String) -> void:
|
|
# A durable/local divergence means this process can no longer prove that a
|
|
# future generation is globally current. Preserve the live match but close
|
|
# admission so it cannot mint additional ambiguous leases.
|
|
MatchNet.admissions_open = false
|
|
ServerLog.error("connection_lease_reconciliation_failed", {"reason": reason})
|
|
|
|
|
|
static func valid_connection_report_configuration(base_url: String, workload_token: String, match_id: String, server_id: String, player_id: String) -> bool:
|
|
return ConnectionLeaseClientScript.valid_configuration(base_url, workload_token, match_id, server_id) and AssignmentState.is_valid_opaque_id(player_id)
|
|
|
|
|
|
static func required_min_players(allocated: bool, roster_size: int, configured: int) -> int:
|
|
if allocated and roster_size > 0:
|
|
return roster_size
|
|
return configured
|
|
|
|
|
|
# The join-signing key file maps key ID -> base64 raw key, so the allocator can
|
|
# rotate the signing key without invalidating authorisations already issued for
|
|
# in-flight matches: a rotation publishes the new key alongside the old, and the
|
|
# old one is dropped only once no live match can still reference it.
|
|
#
|
|
# A file containing raw key bytes (no JSON object) is accepted as a single key
|
|
# under the empty ID, which is what an unrotated deployment and the local smoke
|
|
# fixtures use.
|
|
static func _load_join_signing_keys(key_file: String) -> Dictionary:
|
|
var raw := FileAccess.get_file_as_bytes(key_file)
|
|
if raw.is_empty():
|
|
return {}
|
|
var parsed = JSON.parse_string(raw.get_string_from_utf8())
|
|
if not parsed is Dictionary or (parsed as Dictionary).is_empty():
|
|
return {"": raw}
|
|
var keys := {}
|
|
for key_id in parsed:
|
|
var encoded = parsed[key_id]
|
|
if not encoded is String or String(encoded).is_empty():
|
|
return {}
|
|
var decoded := Marshalls.base64_to_raw(String(encoded))
|
|
if decoded.is_empty():
|
|
return {}
|
|
keys[str(key_id)] = decoded
|
|
return keys
|