fix(multiplayer): reconcile authoritative initial connections

This commit is contained in:
Josh Creek
2026-09-03 00:02:04 +01:00
parent 781cbc35aa
commit aa446cfbfe
40 changed files with 809 additions and 87 deletions
+66
View File
@@ -3,6 +3,7 @@ extends Node
const NetCodec = preload("res://scripts/net_codec.gd")
const ServerControlScript = preload("res://scripts/server_control.gd")
const AgonesSDKScript = preload("res://scripts/agones_sdk.gd")
const AssignmentState = preload("res://scripts/assignment_state.gd")
# Headless dedicated server entry point (task 1.6). Parses CLI args, hosts
# via NetworkManager, logs structured lines, and watches for physics-tick
@@ -29,8 +30,11 @@ var _last_physics_frame := 0
var config: ServerConfig = null
var _watchdog_armed := false # skip the first _process(): engine startup scheduling can batch several physics frames before the first idle frame runs, which isn't a real overrun
var _control: ServerControl = null
var _match_loop: ServerMatchLoop = null
var _agones = null
var _drain_requested := false
var _connection_reports_inflight: Dictionary = {}
var _connection_reports_complete: Dictionary = {}
func _ready() -> void:
@@ -89,6 +93,7 @@ func _ready() -> void:
config.values["min-players"] = required_min_players(true, roster_tokens.size(), int(config.get_value("min-players")))
_control.name = "ServerControl"
_control.drain_requested.connect(_on_drain_requested)
_control.initial_connect_ready.connect(_on_initial_connect_ready)
get_tree().root.add_child.call_deferred(_control)
var control_err := _control.start(int(config.get_value("readiness-port")), OS.get_environment(String(config.get_value("drain-token-env"))))
if control_err != OK:
@@ -134,6 +139,7 @@ func _ready() -> void:
# it started. Same constraint the smoke-test hooks document.
func _install_match_loop() -> void:
var loop := ServerMatchLoop.new()
_match_loop = loop
loop.name = "ServerMatchLoop"
loop.min_players = int(config.get_value("min-players"))
loop.start_countdown_seconds = float(config.get_value("start-countdown"))
@@ -143,6 +149,10 @@ func _install_match_loop() -> void:
loop.allocated_playlist = String(config.get_value("playlist"))
loop.allocated_roster_size = MatchNet.assigned_player_slots().size() if loop.allocated_mode else 0
loop.allocated_arena_path = String(config.get_value("arena-path"))
# The backend's fair timeout starts only after durable assignment-ready.
# When a control plane is present, the supervisor arms this loop through
# the authenticated local control endpoint after that transition commits.
loop.allocated_admission_armed = not loop.allocated_mode or OS.get_environment("COSMIC_CLASH_INITIAL_CONNECT_SIGNAL_REQUIRED") != "1"
get_tree().root.add_child.call_deferred(loop)
@@ -178,6 +188,8 @@ func _on_client_disconnected(peer_id: int) -> void:
func _on_player_joined(peer_id: int, player_name: String) -> void:
ServerLog.info("player_joined", {"peer_id": peer_id, "name": player_name, "roster": MatchNet.roster.size()})
if config != null and bool(config.get_value("allocated-mode")):
_report_player_connected(MatchNet.player_identity(peer_id))
func _on_player_left(peer_id: int) -> void:
@@ -191,6 +203,60 @@ func _on_drain_requested() -> void:
ServerLog.info("server_draining", {"reason": "control_request"})
func _on_initial_connect_ready() -> void:
if _match_loop != null and is_instance_valid(_match_loop):
_match_loop.arm_allocated_admission()
ServerLog.info("initial_connect_window_started", {"match_id": String(config.get_value("match-id"))})
func _report_player_connected(player_id: String) -> void:
var base_url := OS.get_environment("COSMIC_CLASH_CONTROL_PLANE_URL").strip_edges().trim_suffix("/")
var workload_token := OS.get_environment("COSMIC_CLASH_WORKLOAD_TOKEN").strip_edges()
var server_id := String(config.get_value("server-id"))
var match_id := String(config.get_value("match-id"))
if not valid_connection_report_configuration(base_url, workload_token, match_id, server_id, player_id) or _connection_reports_inflight.has(player_id) or _connection_reports_complete.has(player_id):
return
_connection_reports_inflight[player_id] = true
var endpoint := "%s/v1/servers/%s/connect" % [base_url, server_id.uri_encode()]
# A player can join many matches. Scope the durable key to this match so a
# later valid report cannot conflict with an earlier match's stored digest.
var idempotency_key := "server-connect-" + (match_id + "\n" + player_id).sha256_text()
var payload := JSON.stringify({"player_id": player_id})
for attempt in range(5):
var request := HTTPRequest.new()
request.timeout = 5.0
add_child(request)
var start_error := request.request(endpoint, [
"Authorization: Bearer " + workload_token,
"Content-Type: application/json",
"Idempotency-Key: " + idempotency_key,
], HTTPClient.METHOD_POST, payload)
var response_code := 0
if start_error == OK:
var response: Array = await request.request_completed
response_code = int(response[1])
request.queue_free()
if response_code == 204:
_connection_reports_complete[player_id] = true
_connection_reports_inflight.erase(player_id)
ServerLog.debug("player_connection_recorded", {"player_id": player_id})
return
if response_code in [400, 401, 404, 409, 422]:
break
if attempt < 4 and is_inside_tree():
await get_tree().create_timer(1.0).timeout
_connection_reports_inflight.erase(player_id)
ServerLog.warn("player_connection_report_failed", {"player_id": player_id})
static func valid_connection_report_configuration(base_url: String, workload_token: String, match_id: String, server_id: String, player_id: String) -> bool:
if not (base_url.begins_with("http://") or base_url.begins_with("https://")) or base_url.contains("\n") or base_url.contains("\r") or base_url.contains("?") or base_url.contains("#"):
return false
if workload_token.is_empty() or workload_token.contains("\n") or workload_token.contains("\r"):
return false
return AssignmentState.is_valid_opaque_id(match_id) and AssignmentState.is_valid_opaque_id(server_id) and AssignmentState.is_valid_opaque_id(player_id)
static func required_min_players(allocated: bool, roster_size: int, configured: int) -> int:
if allocated and roster_size > 0:
return roster_size
+13
View File
@@ -6,6 +6,7 @@ extends Node
# controlled termination. Direct/community servers do not start this node.
signal drain_requested
signal initial_connect_ready
var _listener := TCPServer.new()
var _peers: Array = []
@@ -87,6 +88,18 @@ func _respond(peer: StreamPeerTCP, request: String) -> void:
drain_requested.emit()
status = 202
reason = "Accepted"
elif method == "POST" and path == "/initial-connect-ready":
var supplied := ""
for line in lines:
if line.begins_with("Authorization: Bearer "):
supplied = line.substr("Authorization: Bearer ".length())
if _drain_token.is_empty() or not _constant_time_equal(supplied, _drain_token):
status = 401
reason = "Unauthorized"
else:
initial_connect_ready.emit()
status = 202
reason = "Accepted"
else:
status = 405 if method in ["GET", "POST"] else 400
reason = "Method Not Allowed" if status == 405 else "Bad Request"
+18 -2
View File
@@ -48,6 +48,7 @@ var allocated_mode := false
var allocated_playlist := ""
var allocated_roster_size := 0
var allocated_arena_path := ""
var allocated_admission_armed := true
var matches_completed := 0
var _countdown_started_ms := -1
@@ -74,6 +75,8 @@ func _process(_delta: float) -> void:
func _poll_allocated_match_start(now: int) -> void:
if not allocated_admission_armed:
return
if _allocated_connect_started_ms < 0:
_allocated_connect_started_ms = now
var connected := MatchNet.roster.size()
@@ -95,14 +98,27 @@ func _poll_allocated_match_start(now: int) -> void:
_poll_match_start(now)
func arm_allocated_admission() -> void:
allocated_admission_armed = true
_allocated_connect_started_ms = -1
static func allocated_initial_connect_action(playlist: String, elapsed_ms: int, connected: int, expected: int, has_team_zero: bool, has_team_one: bool) -> String:
if elapsed_ms < 0 or connected < 0 or expected < 1:
return ALLOCATED_CANCEL
if connected >= expected:
return ALLOCATED_READY
if playlist == "ranked":
if expected != 6:
return ALLOCATED_CANCEL
if connected >= expected:
return ALLOCATED_READY
return ALLOCATED_CANCEL if elapsed_ms >= 30000 else ALLOCATED_WAIT
if playlist == "casual":
if expected < 2 or expected > 6:
return ALLOCATED_CANCEL
if connected >= expected:
if expected == 6:
return ALLOCATED_READY
return ALLOCATED_START_WITH_BOTS if has_team_zero and has_team_one else ALLOCATED_CANCEL
if elapsed_ms < 60000:
return ALLOCATED_WAIT
return ALLOCATED_START_WITH_BOTS if connected >= 2 and has_team_zero and has_team_one else ALLOCATED_CANCEL
+9
View File
@@ -173,3 +173,12 @@ func test_allocated_start_floor_is_the_verified_roster_size() -> void:
assert_eq(boot.required_min_players(true, 6, 1), 6, "allocated six-player roster cannot start with one player")
assert_eq(boot.required_min_players(true, 2, 6), 2, "allocated casual roster uses its complete size")
assert_eq(boot.required_min_players(false, 1, 1), 1, "direct server keeps its configured floor")
func test_connection_reporting_requires_safe_workload_configuration() -> void:
var boot = preload("res://scripts/server_boot.gd")
assert_true(boot.valid_connection_report_configuration("http://control-plane:8080", "signed-token", "match-1234567890", "server-123456789", "player-123456789"), "allocated workload configuration is accepted")
assert_true(not boot.valid_connection_report_configuration("http://control-plane:8080?token=leak", "signed-token", "match-1234567890", "server-123456789", "player-123456789"), "query-bearing control-plane URL is rejected")
assert_true(not boot.valid_connection_report_configuration("http://control-plane:8080", "token\nforged", "match-1234567890", "server-123456789", "player-123456789"), "header injection token is rejected")
assert_true(not boot.valid_connection_report_configuration("http://control-plane:8080", "signed-token", "short", "server-123456789", "player-123456789"), "non-opaque match identity is rejected")
assert_true(not boot.valid_connection_report_configuration("http://control-plane:8080", "signed-token", "match-1234567890", "server-123456789", "short"), "non-opaque player identity is rejected")
@@ -6,6 +6,14 @@ func test_allocated_initial_connect_policy_has_explicit_boundaries() -> void:
assert_eq(loop.allocated_initial_connect_action("ranked", 30000, 5, 6, true, true), loop.ALLOCATED_CANCEL, "ranked cancels at 30 seconds")
assert_eq(loop.allocated_initial_connect_action("casual", 59999, 2, 6, true, true), loop.ALLOCATED_WAIT, "casual waits before 60 seconds")
assert_eq(loop.allocated_initial_connect_action("casual", 60000, 2, 6, true, true), loop.ALLOCATED_START_WITH_BOTS, "casual starts with bots when both teams are represented")
assert_eq(loop.allocated_initial_connect_action("casual", 1000, 2, 2, true, true), loop.ALLOCATED_START_WITH_BOTS, "complete relaxed casual roster starts with disclosed bots immediately")
assert_eq(loop.allocated_initial_connect_action("casual", 1000, 2, 2, true, false), loop.ALLOCATED_CANCEL, "malformed relaxed casual roster fails closed")
assert_eq(loop.allocated_initial_connect_action("casual", 60000, 2, 6, true, false), loop.ALLOCATED_CANCEL, "casual cancels when one team is empty")
assert_eq(loop.allocated_initial_connect_action("casual", 1000, 6, 6, true, true), loop.ALLOCATED_READY, "complete roster is ready immediately")
assert_eq(loop.allocated_initial_connect_action("ranked", 1000, 5, 5, true, true), loop.ALLOCATED_CANCEL, "ranked cannot shrink its expected roster")
assert_eq(loop.allocated_initial_connect_action("other", 0, 1, 6, true, true), loop.ALLOCATED_CANCEL, "unknown allocated playlist fails closed")
var instance = loop.new()
instance.allocated_admission_armed = false
instance.arm_allocated_admission()
assert_true(instance.allocated_admission_armed, "durable readiness signal arms the local timeout")
instance.free()
+12
View File
@@ -12,6 +12,8 @@ func _init() -> void:
quit(1)
return
control.set_process_ready(true)
control.set_meta("admission_armed", false)
control.initial_connect_ready.connect(func() -> void: control.set_meta("admission_armed", true))
await process_frame
var ready_response := await _request("GET", "/ready", [])
if ready_response != 200:
@@ -23,6 +25,16 @@ func _init() -> void:
printerr("unauthorized drain response was %d" % unauthorized)
quit(1)
return
var unauthorized_admission := await _request("POST", "/initial-connect-ready", ["Authorization: Bearer wrong"])
if unauthorized_admission != 401 or bool(control.get_meta("admission_armed")):
printerr("unauthorized initial-connect response/state was %d/%s" % [unauthorized_admission, control.get_meta("admission_armed")])
quit(1)
return
var admitted := await _request("POST", "/initial-connect-ready", ["Authorization: Bearer drain-secret"])
if admitted != 202 or not bool(control.get_meta("admission_armed")):
printerr("authorized initial-connect response/state was %d/%s" % [admitted, control.get_meta("admission_armed")])
quit(1)
return
var drained := await _request("POST", "/drain", ["Authorization: Bearer drain-secret"])
if drained != 202 or not control.is_draining():
printerr("authorized drain response/state was %d/%s" % [drained, control.is_draining()])