feat(multiplayer): bind admissions to durable leases

2026-09-11 08:23:45 +00:00 · 2026-09-03 13:45:39 +01:00
parent 3e0022ce9c
commit aac81c89b6
9 changed files with 301 additions and 64 deletions
@@ -0,0 +1,153 @@
+class_name ConnectionLeaseClient
+extends Node
+
+const AssignmentState = preload("res://scripts/assignment_state.gd")
+
+signal reconciliation_failed(reason: String)
+
+const CLAIMED := "claimed"
+const UNAVAILABLE := "unavailable"
+const REJECTED := "rejected"
+
+var _base_url := ""
+var _workload_token := ""
+var _match_id := ""
+var _server_id := ""
+var _pending: Array[Dictionary] = []
+var _processing := false
+
+
+func configure(base_url: String, workload_token: String, match_id: String, server_id: String) -> bool:
+	base_url = base_url.strip_edges().trim_suffix("/")
+	workload_token = workload_token.strip_edges()
+	if not valid_configuration(base_url, workload_token, match_id, server_id):
+		return false
+	_base_url = base_url
+	_workload_token = workload_token
+	_match_id = match_id
+	_server_id = server_id
+	return true
+
+
+# Admission awaits one bounded request only. If the control plane is down, the
+# same process may continue using its local generation and this exact event is
+# retained ahead of every later disconnect/reconnect for ordered reconciliation.
+func claim(player_id: String, expected_generation: int) -> Dictionary:
+	if not AssignmentState.is_valid_opaque_id(player_id) or expected_generation < 0:
+		return {"status": REJECTED}
+	var event := _connect_event(player_id, expected_generation)
+	if _processing or not _pending.is_empty():
+		if expected_generation == 0:
+			return {"status": REJECTED}
+		_pending.append(event)
+		_start_processing()
+		return {"status": UNAVAILABLE, "generation": expected_generation + 1}
+	var response := await _send(event, true)
+	if String(response.get("status", "")) == UNAVAILABLE:
+		if expected_generation == 0:
+			return {"status": REJECTED}
+		_pending.append(event)
+		_start_processing()
+		return {"status": UNAVAILABLE, "generation": expected_generation + 1}
+	return response
+
+
+func record_disconnect(player_id: String, generation: int) -> void:
+	if not AssignmentState.is_valid_opaque_id(player_id) or generation < 1:
+		return
+	_pending.append(_disconnect_event(player_id, generation))
+	_start_processing()
+
+
+func _start_processing() -> void:
+	if _processing or _pending.is_empty() or not is_inside_tree():
+		return
+	_process_pending()
+
+
+func _process_pending() -> void:
+	_processing = true
+	while not _pending.is_empty() and is_inside_tree():
+		var event := _pending[0]
+		var response := await _send(event)
+		var status := String(response.get("status", ""))
+		if status == CLAIMED:
+			_pending.pop_front()
+			continue
+		if status == REJECTED:
+			reconciliation_failed.emit("durable connection lease conflict")
+			_processing = false
+			return
+		await get_tree().create_timer(1.0).timeout
+	_processing = false
+
+
+func _send(event: Dictionary, allow_recovery := false) -> Dictionary:
+	var request := HTTPRequest.new()
+	request.timeout = 1.0
+	add_child(request)
+	var operation := String(event["operation"])
+	var endpoint := "%s/v1/servers/%s/%s" % [_base_url, _server_id.uri_encode(), operation]
+	var start_error := request.request(endpoint, [
+		"Authorization: Bearer " + _workload_token,
+		"Content-Type: application/json",
+		"Idempotency-Key: " + String(event["key"]),
+	], HTTPClient.METHOD_POST, JSON.stringify(event["payload"]))
+	if start_error != OK:
+		request.queue_free()
+		return {"status": UNAVAILABLE}
+	var raw: Array = await request.request_completed
+	request.queue_free()
+	return classify_response(operation, int(event["generation"]), int(raw[0]), int(raw[1]), raw[3], allow_recovery)
+
+
+func _connect_event(player_id: String, expected_generation: int) -> Dictionary:
+	return {
+		"operation": "connect",
+		"generation": expected_generation,
+		"key": event_key(_match_id, player_id, "connect", expected_generation),
+		"payload": {"player_id": player_id, "expected_generation": expected_generation},
+	}
+
+
+func _disconnect_event(player_id: String, generation: int) -> Dictionary:
+	return {
+		"operation": "disconnect",
+		"generation": generation,
+		"key": event_key(_match_id, player_id, "disconnect", generation),
+		"payload": {"player_id": player_id, "generation": generation},
+	}
+
+
+static func classify_response(operation: String, generation: int, request_result: int, response_code: int, body: PackedByteArray, allow_recovery := false) -> Dictionary:
+	if request_result != HTTPRequest.RESULT_SUCCESS or response_code == 0 or response_code == 429 or response_code >= 500:
+		return {"status": UNAVAILABLE}
+	if operation == "disconnect" and response_code == 204:
+		return {"status": CLAIMED, "generation": generation}
+	if operation == "connect" and response_code == 200:
+		var decoded = JSON.parse_string(body.get_string_from_utf8())
+		if decoded is Dictionary and _valid_generation(decoded.get("generation")):
+			var claimed_generation := int(decoded["generation"])
+			if claimed_generation == generation + 1 or (allow_recovery and generation == 0 and claimed_generation > 1):
+				return {"status": CLAIMED, "generation": claimed_generation}
+	return {"status": REJECTED}
+
+
+static func _valid_generation(value: Variant) -> bool:
+	if value is int:
+		return int(value) >= 1
+	if value is float:
+		return is_finite(float(value)) and float(value) >= 1.0 and float(value) == floor(float(value)) and float(value) <= 9007199254740991.0
+	return false
+
+
+static func event_key(match_id: String, player_id: String, operation: String, generation: int) -> String:
+	return "server-lease-" + (match_id + "\n" + player_id + "\n" + operation + "\n" + str(generation)).sha256_text()
+
+
+static func valid_configuration(base_url: String, workload_token: String, match_id: String, server_id: String) -> bool:
+	if not (base_url.begins_with("http://") or base_url.begins_with("https://")) or base_url.contains("\n") or base_url.contains("\r") or base_url.contains("?") or base_url.contains("#"):
+		return false
+	if workload_token.is_empty() or workload_token.contains("\n") or workload_token.contains("\r"):
+		return false
+	return AssignmentState.is_valid_opaque_id(match_id) and AssignmentState.is_valid_opaque_id(server_id)
@@ -66,6 +66,8 @@ var _active_join_peers: Dictionary = {} # opaque authorisation -> peer_id
 var _join_history: Dictionary = {} # token -> {generation, lost_at}
 var _join_authorisation_context: Dictionary = {}
 var _join_signing_key := PackedByteArray()
+var _connection_lease_claim := Callable()
+var _connection_lease_disconnect := Callable()

 # Test hook (tests/match_net_smoke.gd): set false before connecting to
 # suppress the automatic real hello, so a test can send a deliberately
@@ -105,6 +107,8 @@ func _on_shutting_down() -> void:
 	_join_history.clear()
 	_join_authorisation_context.clear()
 	_join_signing_key = PackedByteArray()
+	_connection_lease_claim = Callable()
+	_connection_lease_disconnect = Callable()
 	require_join_authorisation = false
 	admissions_open = true

@@ -184,6 +188,8 @@ func _remove_player(peer_id: int) -> void:
 			var history: Dictionary = _join_history.get(token, {})
 			history["lost_at"] = Time.get_unix_time_from_system()
 			_join_history[token] = history
+			if _connection_lease_disconnect.is_valid():
+				_connection_lease_disconnect.call(_join_identity(token), int(history.get("generation", 0)))
 			break
 	if not roster.has(peer_id):
 		return
@@ -286,9 +292,9 @@ func _hello(protocol_version: int, tick_hz: int, player_name: String, supplied_j
 		return
 	var join_generation := 1
 	if require_join_authorisation:
-		join_generation = _reserve_join_authorisation(supplied_join_authorisation, peer_id)
+		join_generation = await _claim_join_authorisation(supplied_join_authorisation, peer_id)
 		if join_generation < 0:
-			await _reject(peer_id, "join authorisation reclaim expired")
+			await _reject(peer_id, "join authorisation lease rejected")
 			return
 	if player_name.length() > MAX_INPUT_LENGTH:
 		await _reject(peer_id, "player name too long")
@@ -432,16 +438,68 @@ func is_join_authorisation_active(token: String) -> bool:
 	return not token.is_empty() and _active_join_peers.has(token)


-func _reserve_join_authorisation(token: String, peer_id: int) -> int:
+func configure_connection_lease_callbacks(claim: Callable, disconnect: Callable) -> void:
+	_connection_lease_claim = claim
+	_connection_lease_disconnect = disconnect
+
+
+func _claim_join_authorisation(token: String, peer_id: int) -> int:
+	var expected_generation := _available_join_generation(token)
+	if expected_generation < 0:
+		return -1
+	var generation := expected_generation + 1
+	if _connection_lease_claim.is_valid():
+		var response = await _connection_lease_claim.call(_join_identity(token), expected_generation)
+		generation = lease_claim_generation(response, expected_generation)
+		if generation < 0:
+			return -1
+	# The await above deliberately allows one bounded control-plane request.
+	# Re-evaluate every local fact that can change during that suspension before
+	# publishing the reservation. If a durable claim succeeded, close it again.
+	# A concurrent same-token hello can receive the same idempotent claim; its
+	# loser must not close the generation now owned by the local winner.
+	if _active_join_peers.has(token):
+		return -1
+	if not admissions_open or not _valid_join_authorisation(token) or peer_id not in multiplayer.get_peers():
+		if _connection_lease_disconnect.is_valid():
+			_connection_lease_disconnect.call(_join_identity(token), generation)
+		return -1
+	_join_history[token] = {"generation": generation, "lost_at": 0.0}
+	_active_join_peers[token] = peer_id
+	return generation
+
+
+func _available_join_generation(token: String) -> int:
 	if token.is_empty() or _active_join_peers.has(token):
 		return -1
 	var now := Time.get_unix_time_from_system()
 	var history: Dictionary = _join_history.get(token, {})
 	var lost_at := float(history.get("lost_at", 0.0))
-	if lost_at > 0.0:
-		if now < lost_at or now - lost_at > RECONNECT_GRACE_SECONDS:
-			return -1
-	var generation := int(history.get("generation", 0)) + 1
+	if lost_at > 0.0 and (now < lost_at or now - lost_at > RECONNECT_GRACE_SECONDS):
+		return -1
+	return int(history.get("generation", 0))
+
+
+static func lease_claim_generation(response, expected_generation: int) -> int:
+	if not response is Dictionary or expected_generation < 0:
+		return -1
+	var status := String(response.get("status", ""))
+	if status not in ["claimed", "unavailable"] or not response.get("generation") is int:
+		return -1
+	var generation := int(response["generation"])
+	if status == "unavailable":
+		return generation if expected_generation > 0 and generation == expected_generation + 1 else -1
+	# A durable backend may return a later generation only to a fresh process
+	# recovering an already-disconnected lease. Locally known generations never
+	# skip, and outage fallback never invents a jump.
+	return generation if generation == expected_generation + 1 or (expected_generation == 0 and generation > 1) else -1
+
+
+func _reserve_join_authorisation(token: String, peer_id: int) -> int:
+	var expected_generation := _available_join_generation(token)
+	if expected_generation < 0:
+		return -1
+	var generation := expected_generation + 1
 	_join_history[token] = {"generation": generation, "lost_at": 0.0}
 	_active_join_peers[token] = peer_id
 	return generation
@@ -4,6 +4,7 @@ const NetCodec = preload("res://scripts/net_codec.gd")
 const ServerControlScript = preload("res://scripts/server_control.gd")
 const AgonesSDKScript = preload("res://scripts/agones_sdk.gd")
 const AssignmentState = preload("res://scripts/assignment_state.gd")
+const ConnectionLeaseClientScript = preload("res://scripts/connection_lease_client.gd")

 # Headless dedicated server entry point (task 1.6). Parses CLI args, hosts
 # via NetworkManager, logs structured lines, and watches for physics-tick
@@ -32,9 +33,8 @@ var _watchdog_armed := false   # skip the first _process(): engine startup sched
 var _control: ServerControl = null
 var _match_loop: ServerMatchLoop = null
 var _agones = null
+var _connection_leases = null
 var _drain_requested := false
-var _connection_reports_inflight: Dictionary = {}
-var _connection_reports_complete: Dictionary = {}


 func _ready() -> void:
@@ -105,6 +105,18 @@ func _ready() -> void:
 		get_tree().root.add_child.call_deferred(_agones)
 		if _agones.configure_from_environment():
 			_agones.start_health()
+		_connection_leases = ConnectionLeaseClientScript.new()
+		_connection_leases.name = "ConnectionLeases"
+		var lease_url := OS.get_environment("COSMIC_CLASH_CONTROL_PLANE_URL")
+		var lease_token := OS.get_environment("COSMIC_CLASH_WORKLOAD_TOKEN")
+		if _connection_leases.configure(lease_url, lease_token, String(config.get_value("match-id")), String(config.get_value("server-id"))):
+			_connection_leases.reconciliation_failed.connect(_on_connection_lease_reconciliation_failed)
+			get_tree().root.add_child.call_deferred(_connection_leases)
+			MatchNet.configure_connection_lease_callbacks(_connection_leases.claim, _connection_leases.record_disconnect)
+		else:
+			_connection_leases.queue_free()
+			_connection_leases = null
+			ServerLog.warn("connection_lease_backend_unavailable", {"reason": "invalid_or_missing_configuration"})

 	NetworkManager.client_connected.connect(_on_client_connected)
 	NetworkManager.client_disconnected.connect(_on_client_disconnected)
@@ -188,8 +200,6 @@ func _on_client_disconnected(peer_id: int) -> void:

 func _on_player_joined(peer_id: int, player_name: String) -> void:
 	ServerLog.info("player_joined", {"peer_id": peer_id, "name": player_name, "roster": MatchNet.roster.size()})
-	if config != null and bool(config.get_value("allocated-mode")):
-		_report_player_connected(MatchNet.player_identity(peer_id))


 func _on_player_left(peer_id: int) -> void:
@@ -209,52 +219,16 @@ func _on_initial_connect_ready() -> void:
 		ServerLog.info("initial_connect_window_started", {"match_id": String(config.get_value("match-id"))})


-func _report_player_connected(player_id: String) -> void:
-	var base_url := OS.get_environment("COSMIC_CLASH_CONTROL_PLANE_URL").strip_edges().trim_suffix("/")
-	var workload_token := OS.get_environment("COSMIC_CLASH_WORKLOAD_TOKEN").strip_edges()
-	var server_id := String(config.get_value("server-id"))
-	var match_id := String(config.get_value("match-id"))
-	if not valid_connection_report_configuration(base_url, workload_token, match_id, server_id, player_id) or _connection_reports_inflight.has(player_id) or _connection_reports_complete.has(player_id):
-		return
-	_connection_reports_inflight[player_id] = true
-	var endpoint := "%s/v1/servers/%s/connect" % [base_url, server_id.uri_encode()]
-	# A player can join many matches. Scope the durable key to this match so a
-	# later valid report cannot conflict with an earlier match's stored digest.
-	var idempotency_key := "server-connect-" + (match_id + "\n" + player_id).sha256_text()
-	var payload := JSON.stringify({"player_id": player_id})
-	for attempt in range(5):
-		var request := HTTPRequest.new()
-		request.timeout = 5.0
-		add_child(request)
-		var start_error := request.request(endpoint, [
-			"Authorization: Bearer " + workload_token,
-			"Content-Type: application/json",
-			"Idempotency-Key: " + idempotency_key,
-		], HTTPClient.METHOD_POST, payload)
-		var response_code := 0
-		if start_error == OK:
-			var response: Array = await request.request_completed
-			response_code = int(response[1])
-		request.queue_free()
-		if response_code == 204:
-			_connection_reports_complete[player_id] = true
-			_connection_reports_inflight.erase(player_id)
-			ServerLog.debug("player_connection_recorded", {"player_id": player_id})
-			return
-		if response_code in [400, 401, 404, 409, 422]:
-			break
-		if attempt < 4 and is_inside_tree():
-			await get_tree().create_timer(1.0).timeout
-	_connection_reports_inflight.erase(player_id)
-	ServerLog.warn("player_connection_report_failed", {"player_id": player_id})
+func _on_connection_lease_reconciliation_failed(reason: String) -> void:
+	# A durable/local divergence means this process can no longer prove that a
+	# future generation is globally current. Preserve the live match but close
+	# admission so it cannot mint additional ambiguous leases.
+	MatchNet.admissions_open = false
+	ServerLog.error("connection_lease_reconciliation_failed", {"reason": reason})


 static func valid_connection_report_configuration(base_url: String, workload_token: String, match_id: String, server_id: String, player_id: String) -> bool:
-	if not (base_url.begins_with("http://") or base_url.begins_with("https://")) or base_url.contains("\n") or base_url.contains("\r") or base_url.contains("?") or base_url.contains("#"):
-		return false
-	if workload_token.is_empty() or workload_token.contains("\n") or workload_token.contains("\r"):
-		return false
-	return AssignmentState.is_valid_opaque_id(match_id) and AssignmentState.is_valid_opaque_id(server_id) and AssignmentState.is_valid_opaque_id(player_id)
+	return ConnectionLeaseClientScript.valid_configuration(base_url, workload_token, match_id, server_id) and AssignmentState.is_valid_opaque_id(player_id)


 static func required_min_players(allocated: bool, roster_size: int, configured: int) -> int:
@@ -0,0 +1,36 @@
+extends "res://tests/test_case.gd"
+
+const LeaseClient = preload("res://scripts/connection_lease_client.gd")
+
+
+func test_connection_lease_response_classification_is_fail_closed() -> void:
+	var success_body := JSON.stringify({"generation": 2}).to_utf8_buffer()
+	assert_eq(LeaseClient.classify_response("connect", 1, HTTPRequest.RESULT_SUCCESS, 200, success_body), {"status": "claimed", "generation": 2}, "exact next generation is accepted")
+	assert_eq(LeaseClient.classify_response("connect", 1, HTTPRequest.RESULT_SUCCESS, 200, JSON.stringify({"generation": 3}).to_utf8_buffer())["status"], "rejected", "skipped generation is rejected")
+	assert_eq(LeaseClient.classify_response("connect", 0, HTTPRequest.RESULT_SUCCESS, 200, JSON.stringify({"generation": 3}).to_utf8_buffer(), true), {"status": "claimed", "generation": 3}, "a fresh process accepts a durable recovery generation")
+	assert_eq(LeaseClient.classify_response("connect", 0, HTTPRequest.RESULT_SUCCESS, 200, JSON.stringify({"generation": 3}).to_utf8_buffer())["status"], "rejected", "queued outage reconciliation cannot skip generations")
+	assert_eq(LeaseClient.classify_response("connect", 1, HTTPRequest.RESULT_SUCCESS, 200, JSON.stringify({"generation": "2"}).to_utf8_buffer())["status"], "rejected", "string generation is not coerced")
+	assert_eq(LeaseClient.classify_response("connect", 1, HTTPRequest.RESULT_CANT_CONNECT, 0, PackedByteArray())["status"], "unavailable", "transport outage permits bounded local fallback")
+	assert_eq(LeaseClient.classify_response("connect", 1, HTTPRequest.RESULT_SUCCESS, 503, PackedByteArray())["status"], "unavailable", "service outage permits bounded local fallback")
+	assert_eq(LeaseClient.classify_response("connect", 1, HTTPRequest.RESULT_SUCCESS, 409, PackedByteArray())["status"], "rejected", "durable conflict is terminal")
+	assert_eq(LeaseClient.classify_response("disconnect", 2, HTTPRequest.RESULT_SUCCESS, 204, PackedByteArray()), {"status": "claimed", "generation": 2}, "disconnect acknowledgement preserves exact generation")
+
+
+func test_connection_lease_configuration_and_keys_are_bound() -> void:
+	assert_true(LeaseClient.valid_configuration("https://control.invalid", "workload-token", "match-1234567890", "server-123456789"), "valid workload configuration is accepted")
+	assert_true(not LeaseClient.valid_configuration("https://control.invalid?token=leak", "workload-token", "match-1234567890", "server-123456789"), "query-bearing endpoint is rejected")
+	assert_true(not LeaseClient.valid_configuration("https://control.invalid", "bad\ntoken", "match-1234567890", "server-123456789"), "header injection is rejected")
+	var initial := LeaseClient.event_key("match-123456789", "player-12345678", "connect", 0)
+	assert_true(initial != LeaseClient.event_key("match-123456789", "player-12345678", "disconnect", 1), "operation and generation bind the key")
+	assert_true(initial != LeaseClient.event_key("match-000000000", "player-12345678", "connect", 0), "match identity binds the key")
+
+
+func test_match_net_rejects_malformed_or_skipped_backend_generations() -> void:
+	assert_eq(MatchNet.lease_claim_generation({"status": "claimed", "generation": 2}, 1), 2, "exact backend generation is accepted")
+	assert_eq(MatchNet.lease_claim_generation({"status": "unavailable", "generation": 2}, 1), 2, "local fallback retains the exact next generation")
+	assert_eq(MatchNet.lease_claim_generation({"status": "unavailable", "generation": 1}, 0), -1, "a fresh process cannot guess a generation during an outage")
+	assert_eq(MatchNet.lease_claim_generation({"status": "rejected", "generation": 2}, 1), -1, "backend conflict rejects admission")
+	assert_eq(MatchNet.lease_claim_generation({"status": "claimed", "generation": 3}, 1), -1, "generation skips are fenced")
+	assert_eq(MatchNet.lease_claim_generation({"status": "claimed", "generation": 3}, 0), 3, "fresh process adopts durable recovery generation")
+	assert_eq(MatchNet.lease_claim_generation({"status": "unavailable", "generation": 3}, 0), -1, "offline fallback cannot invent a skipped generation")
+	assert_eq(MatchNet.lease_claim_generation({"status": "claimed", "generation": "2"}, 1), -1, "string generation is fenced")
@@ -1192,7 +1192,7 @@ production fallback.
 |---|---|---|
 | 8.7 `[D:7.6,8.3]` | **IN PROGRESS.** Pure Go ticket policy binds the expected App ID and verified identity, rejects expiry/replay/wrong app/malformed tickets, and consumes each ticket once. `cmd/control-plane` now wires `SessionIssuer: store.PostgresSessions{DB: db}` (same discovery/fix pattern as §8.10's `ResultSubmitter`: the adapter already correctly implemented `Issue`, just wasn't wired, so `/v1/session/steam` 503'd even before considering whether `SteamLogin` — the real, still-correctly-unwired Steam blocker — was available) | `server/domain/auth.go` covers single-use and binding invariants; real `AuthenticateUserTicket` backend adapter, bans, publisher secret store and Steam verification remain. A separate test-only `server/cmd/testkit-api` binary (never referenced by any Dockerfile/K8s manifest) substitutes a fake Steam login accepting any non-empty ticket, enabling §8.40's real Go+Postgres+Godot integration test |
 | 8.8 `[D:8.7]` | **IN PROGRESS.** Pure Go session policy issues opaque short-lived tokens, stores only digests, authenticates by verified player identity and supports revocation; the production control-plane uses bounded atomic account+IP request limits | `server/domain/auth.go` covers wrong-token/expiry/revocation behavior; `server/store/session_sql.go` provides durable digest/revocation persistence and `server/api/rate_limit.go` plus `cmd/control-plane` provide per-replica request limiting; distributed revocation coordination and live Steam/session integration remain |
-| 8.9 `[D:8.4,8.7]` | **IN PROGRESS.** Pure Go join policy binds SteamID/player/match/server/team/slot/protocol/expiry, rejects duplicate roster slots, permits same-identity reclaim through backend loss and fences prior server-owned generations. A workload-authenticated durable lease API now atomically claims the next exact generation and records exact-generation disconnects against the allocation/match/server/participant roster | `server/domain/reconnect.go`, `server/store/server_connection_sql.go`, migration 0011, `/servers/{serverId}/{connect|disconnect}`, and adversarial tests cover active duplicate claims, stale disconnect fencing, exact 60-second reclaim, wrong binding, initial assignment expiry, retry-safe receipts, and migration backfill. Godot still uses its local lease during admission; pre-admission durable claim/fallback reconciliation and cross-process runtime verification remain |
+| 8.9 `[D:8.4,8.7]` | **IN PROGRESS.** Pure Go join policy binds SteamID/player/match/server/team/slot/protocol/expiry, rejects duplicate roster slots, permits same-identity reclaim through backend loss and fences prior server-owned generations. A workload-authenticated durable lease API atomically claims generations and records exact-generation disconnects against the allocation/match/server/participant roster. Allocated Godot admission now awaits one bounded durable claim before publishing the roster entry; definitive conflicts fail closed, while a known nonzero same-process generation may reconnect during an outage and queues its connect/disconnect sequence for ordered reconciliation. A fresh process never guesses generation one offline and can adopt a later backend generation only from a durably disconnected lease | `server/domain/reconnect.go`, `server/store/server_connection_sql.go`, migration 0011, `/servers/{serverId}/{connect|disconnect}`, `connection_lease_client.gd`, and adversarial tests cover active duplicate claims, stale disconnect fencing, exact 60-second reclaim, process recovery, wrong binding, initial assignment expiry, malformed/skipped responses, ordered outage rules, retry-safe active receipts, and migration backfill. Admission rechecks drain, token expiry, and peer presence after the awaited claim and releases a claim that became unusable. Live PostgreSQL/Godot process-restart and outage recovery verification remains |
 | 8.10 `[D:8.5,8.31]` | **IN PROGRESS.** Dependency-free workload credential policy validates an adapter-verified signature, issuer/audience/time bounds, namespace/service account, pod UID, GameServer UID, allocation ID, match ID and server ID before result submission; `server/workload` now parses projected compact JWTs, rejects unsigned/malformed/ambiguous tokens, verifies signing input through an injected trust callback, and delegates exact binding to the domain policy; the API exposes the workload-authenticated server result route. **`WorkloadVerify` is now wired for real** -- the blocker this row previously named (`cmd/control-plane/main.go` never wiring it, so `/register` and `/result` 503'd on every real request) is closed by taking a different, equally-valid approach instead of the Kubernetes-JWT one: `serverMutation` only ever compares `WorkloadBinding.ServerID`/`.MatchID` (and `AdvanceServerRegistration` only additionally needs `.AllocationID`) -- nothing downstream requires Namespace/ServiceAcct/PodUID/GameServerUID, so a Kubernetes-issued token was never actually required, just the assumption that it was. `server/workload/signed_token.go` mints and verifies a short-lived HMAC-signed token with a secret only the control plane holds (the same trust model `domain.SessionStore` already uses for player sessions), needing no cluster JWKS/TokenReview to validate -- signature and expiry are fully self-contained. The token binds only `allocation_id` (deliberately, not `match_id`/`server_id` too: it must be requestable in the same Agones request that asks for a server, before Agones has picked one) -- `store.AllocationBindingByAllocationID` resolves `match_id`/`server_id` durably from the `allocations` table at verify time, so a token can never claim a pairing that wasn't actually, durably allocated. `api.WorkloadVerifierFromSignedToken` combines both and is wired into `cmd/control-plane` (`--workload-secret`/`COSMIC_CLASH_WORKLOAD_SECRET`; a startup warning fires and the route stays 503 if it's left unset) and `cmd/testkit-api` (fixed test secret). **The delivery channel is now wired too**: `agones.Client` gains `WorkloadSecret`/`WorkloadTokenTTL` -- when set, `Allocate` mints a token for the allocation and requests it as a third `cosmic-clash.io/workload-token` annotation alongside match-id/allocation-id, wired from `cmd/allocator`'s own `--workload-secret`/`COSMIC_CLASH_WORKLOAD_SECRET` (must match the control plane's); `supervisor.Supervisor.workloadToken()` resolves the bearer credential with an explicit `--workload-token-path` (kept for a possible future Kubernetes-JWT path) always winning, otherwise falling back to that same annotation -- the same fallback pattern `matchID()` already used for `cosmic-clash.io/match-id`. `WorkloadTokenPath` is accordingly no longer required at construction time | `server/domain/workload.go`, `server/workload/jwt.go`, `server/api/service.go` and adversarial tests reject every binding mutation, missing/unverified signature, `none`/malformed JWT, ambiguous audience, server/match mismatch and time boundary; `server/testkit/pipeline_test.go` carries allocation identity through the offline result path; `cmd/control-plane` now wires `ResultSubmitter: store.PostgresResults{DB: db}` (a ready-made adapter that had been referenced from nowhere at all, not even a test); `server/workload/signed_token_test.go` covers round-trip, tampered payload, wrong secret, malformed input and the exact expiry boundary; `server/api/workload_verifier_integration_test.go` (opt-in, real PostgreSQL) covers acceptance against a real allocation row, rejection of an unrecorded allocation, that two distinct real allocations each resolve to their own and only their own match/server pairing, and the previously-503 `Service.WorkloadVerify` field itself now succeeding -- all verified clean with `-race` across multiple runs; `cmd/control-plane/main_test.go`'s `TestServerRoutesRequireWorkloadVerifyToBeWired` now documents and pins the misconfigured-secret case specifically, not the "always unwired" case; `server/agones/allocation_test.go`'s `TestAllocateRequestsAWorkloadTokenAnnotationWhenConfigured` and `server/supervisor/supervisor_test.go`'s new annotation-fallback/fails-closed pair cover the delivery channel end to end (mint -> annotation -> supervisor read -> Authorization header) short of a live cluster. **What's still missing**: this has never run against a real Agones cluster (only HTTP-level fakes), so the exact `object_meta` JSON casing Agones actually returns remains unverified from this sandbox (see `GameServer.ObjectMeta`'s existing caveat); live duplicate/conflict alerting also remains |
 | 8.11 `[D:8.1]` | **DONE.** Write the threat model: forged clients/replay/queues/results, floods/bots, pod/insider compromise, gameplay and API DDoS, SDR signing-key theft, dependencies and denial-of-wallet | [`docs/THREAT-MODEL.md`](docs/THREAT-MODEL.md) records prevention, detection/response, owner and residual risk for every threat; it separates offline CA/online signer, client/game-server/PostgreSQL/Redis trust boundaries and recovery behavior |
 | 8.12 `[D:8.11]` | **IN PROGRESS.** Provider-neutral Kubernetes baseline enforces restricted namespace admission, non-root/read-only/no-capability workloads, separate service accounts, default-deny networking and explicit edge/data/DNS/Agones/metrics flows; application manifests consume externally populated Secret objects; the Go API and allocator now have bounded per-replica account+IP rate/quota boundaries and metrics endpoints, plus an atomic operator-controlled degraded-mode gate that sheds new login/queue/proposal work while preserving live-match paths | `deploy/k8s/base/` includes hardened control-plane, allocator, and game-server resources; the control-plane now has health probes, a disruption budget, explicit zero-unavailable/one-surge rolling updates with graceful termination, failure-domain spreading/anti-affinity, and Secret-backed `COSMIC_CLASH_POSTGRES_DSN`/`COSMIC_CLASH_WORKLOAD_SECRET` runtime wiring; the authenticated WebSocket now requires RFC 6455 version 13, enforces a bounded 64 KiB frame size, two-minute idle deadline, 120-message/minute inbound budget, and bounded per-player fan-out; `server/security/test_kubernetes_policies.py`, `server/api/rate_limit.go`, `server/api/admission.go`, allocator metrics tests and adversarial checks cover static hardening, secret-reference invariants, fixed-window limits, bounded labels/key memory, atomic account+IP charging, route classification, concurrent toggling and degraded responses; production `cmd/control-plane` now wires the bounded limiter by default with `--rate-limit`, `--rate-limit-window`, and `--rate-limit-max-keys`, and the base Deployment declares those defaults explicitly; start with control-plane `--degraded`, enable with SIGUSR1, disable with SIGUSR2; private-store provisioning, distributed/global quotas, edge DDoS/WAF/origin shielding, encrypted backups and live policy/load tests remain |
@@ -1212,7 +1212,7 @@ production fallback.
 | 8.21 `[D:8.5,8.20]` | **IN PROGRESS.** Pure Go rating core implements canonical Glicko-2, daily inactivity, ranked 1/3 and casual 1/N human-opponent weights, deterministic opponent ordering, and authoritative draw/overtime/abandon scoring; certified result completion now applies per-player updates inside the durable transaction with lexical row locks and revision increments | `server/domain/rating.go`, `server/store/result_sql.go` and tests cover canonical/inactivity/weight/invalid-input, draw/OT/abandon, ordered participant snapshots, lock/value re-read and rating update SQL; live PostgreSQL rating and seasons execution now covered (§8.23), and concurrent result transaction cases are covered: `TestPostgreSQLConcurrentIdenticalResultSubmissionAppliesRatingsExactlyOnce` races 5 identical submissions and confirms one rating application, while `TestPostgreSQLConcurrentConflictingResultSubmissionsKeepOneReceipt` races different payloads and confirms exactly one winner, one conflict, one receipt and one completion event; live maintenance/DB execution remains |
 | 8.22 `[D:8.21]` | **IN PROGRESS.** Pure Go ranked profile exposes the first ten games as provisional, derives tiers only through validated backend-owned rating bands, and keeps casual ratings outside the API; authenticated HTTP now returns the authoritative ranked view. **"Authoritative ranked view" was durable-adapter-shaped but had no durable adapter**: `rankedProfile`/`profile` only ever read an in-memory map, so every real `GET /v1/profile/ranked` 404'd regardless of a player's actual rating. `RankedProfileProvider` (interface) + `store.PostgresRankedProfiles` close it, preferred over the map when set so existing tests/literals are unaffected; `LastSeasonID`/`SeasonHistory` deliberately left unset (no season pointer on `ratings`, needs its own query/semantics) | `server/domain/rating.go`, `tier_test.go` and `server/api/service.go` cover provisional override, exact band boundaries, malformed policy rejection, session authentication and ranked-only response fields; `server/store/ranked_profile_sql.go`, verified against real PostgreSQL via curl (a fresh identity correctly 404s through the real adapter) and via §8.40's integration test (`control_plane_smoke.gd` now asserts this exact 404 round-trips before queueing); persisted tier policy, client UI and reconnect transport remain |
 | 8.23 `[D:8.21]` | **IN PROGRESS.** Pure Go ranked-only season policy compresses 25% toward 1500, clamps RD to 200–350, preserves volatility/history, is idempotent by season ID, and defines exact 12-week windows/due detection; migration and Go store now persist a per-player/per-season marker and rating update atomically; `cmd/maintenance` runs bounded due-season batches with signal-bound shutdown | `server/domain/rating.go`, `season_test.go`, `server/migrations/0001_initial.sql` and `server/store/maintenance_sql.go` cover compression, floor/cap, duplicate replay, window boundary, completed-season idempotence, bounded enumeration, row locking and conflict-safe rollover markers; opt-in PostgreSQL execution now covers the durable rating update, marker creation and duplicate replay without a second compression — re-run live for the first time as part of the wider integration-suite verification below, after fixing a test setup gap (a missing `seasons` row tripped the `ranked_season_rollovers` foreign key before the rollover logic itself ran); live maintenance/DB execution remains |
-| 8.24 `[D:8.9,8.20,8.21]` | **IN PROGRESS.** Ranked connection policy binds match/server/player/Steam identity/team/global slot/protocol/expiry, permits a 60-second same-token reclaim with monotonically increasing server-owned generations, and applies the rolling 7-day 5 m/15 m/1 h/24 h abandon ladder. Admission rejects active duplicates, zero/reversed clocks, disconnect-before-admit, and duplicate disconnects. PostgreSQL now persists generation/disconnect leases with serializable exact-generation CAS: a stale process cannot disconnect a newer generation, an active lease cannot be reclaimed, initial admission requires an unexpired assignment, and later reconnects use the durable grace boundary rather than the short publication expiry | Go/store/API/Godot adversarial fixtures cover signature tampering, every binding, replay/conflict semantics, active duplicate admission, repeated valid reclaim, stale-generation fencing, exact grace boundary, expiry, zero/reversed clocks, deterministic cooldown ordering, and legacy-row migration. The full local gate passes. Godot pre-admission use of the durable API, outage reconciliation, abandonment persistence, and live PostgreSQL/runtime execution remain |
+| 8.24 `[D:8.9,8.20,8.21]` | **IN PROGRESS.** Ranked connection policy binds match/server/player/Steam identity/team/global slot/protocol/expiry, permits a 60-second same-token reclaim with monotonically increasing server-owned generations, and applies the rolling 7-day 5 m/15 m/1 h/24 h abandon ladder. Admission rejects active duplicates, zero/reversed clocks, disconnect-before-admit, and duplicate disconnects. PostgreSQL persists generation/disconnect leases with serializable CAS: stale disconnects cannot evict newer generations, active leases cannot be reclaimed, initial admission requires an unexpired assignment, and later reconnects use the durable grace boundary. Godot now consumes that lease before admission, reconciles known-generation outage events in order, closes future admissions on reconciliation divergence, and rejects unsafe generation-zero outage fallback | Go/store/API/Godot adversarial fixtures cover signature tampering, every binding, replay/conflict semantics, active duplicate admission, repeated valid reclaim, stale-generation fencing, exact grace boundary, process recovery, expiry, zero/reversed clocks, malformed JSON generations, deterministic cooldown ordering, legacy-row migration, and rolling-upgrade 204 compatibility. The 207-test Godot harness and focused Go suites pass. Abandonment persistence and live PostgreSQL/process-restart/outage execution remain |
 | 8.25 `[D:8.10,8.24]` | **IN PROGRESS.** Workload-bound result policy validates match/server identity, nonce, scores, integrity state, canonical digest, and idempotent receipt conflicts. Result ingestion now owns the complete serializable production transition: a `LIVE` match and its active tickets advance through `RESULT_PENDING`, certified ratings are computed from locked authoritative participant rows, then match/tickets become `COMPLETED`, the receipt is acknowledged, and one revisioned outbox event is inserted. Inactive pre-match no-shows are excluded; an active participant's durable `abandoned_at` forces loss scoring. Missing rating rows or ticket-count divergence fail the whole transaction. Zero-time/incomplete receipts fail before database use and conflicts wrap `ErrResultConflict` | Domain/store/API/outbox tests cover workload and digest binding, identical/conflicting concurrency, direct `LIVE` completion, active-ticket completion, abandonment rating input, incomplete roster failure, integrity suppression, ordered rating locks, receipt/outbox atomicity, fan-out retry/ack ordering, and delivery health. PostgreSQL-tagged regressions compile; prior live result/rating/race/fan-out runs remain valid, while the direct-live lifecycle change awaits a live database rerun. Production credentials, Agones annotation persistence/reconciliation, and integrity-evidence adapters remain |

 #### 8D — Agones, allocation and regional scaling
@@ -669,9 +669,9 @@ func (s *Service) serverMutation(w http.ResponseWriter, r *http.Request) {
 	}
 	if parts[1] == "connect" || parts[1] == "disconnect" {
 		var input struct {
-			PlayerID           string `json:"player_id"`
-			Generation         uint64 `json:"generation,omitempty"`
-			ExpectedGeneration uint64 `json:"expected_generation,omitempty"`
+			PlayerID           string  `json:"player_id"`
+			Generation         uint64  `json:"generation,omitempty"`
+			ExpectedGeneration *uint64 `json:"expected_generation,omitempty"`
 		}
 		if !decodeBody(w, r, &input) {
 			return
@@ -687,9 +687,13 @@ func (s *Service) serverMutation(w http.ResponseWriter, r *http.Request) {
 				writeError(w, http.StatusUnprocessableEntity, "invalid_request")
 				return
 			}
-			generation, err = s.ServerConnections.ClaimPlayerConnection(r.Context(), binding, input.PlayerID, input.ExpectedGeneration, key, now)
+			expectedGeneration := uint64(0)
+			if input.ExpectedGeneration != nil {
+				expectedGeneration = *input.ExpectedGeneration
+			}
+			generation, err = s.ServerConnections.ClaimPlayerConnection(r.Context(), binding, input.PlayerID, expectedGeneration, key, now)
 		} else {
-			if input.Generation == 0 || input.ExpectedGeneration != 0 {
+			if input.Generation == 0 || input.ExpectedGeneration != nil {
 				writeError(w, http.StatusUnprocessableEntity, "invalid_request")
 				return
 			}
@@ -717,6 +721,12 @@ func (s *Service) serverMutation(w http.ResponseWriter, r *http.Request) {
 			w.WriteHeader(http.StatusNoContent)
 			return
 		}
+		if input.ExpectedGeneration == nil {
+			// Rolling-upgrade compatibility for the pre-lease reporter. New
+			// servers always send expected_generation and consume the JSON lease.
+			w.WriteHeader(http.StatusNoContent)
+			return
+		}
 		w.Header().Set("Content-Type", "application/json")
 		_ = json.NewEncoder(w).Encode(map[string]uint64{"generation": generation})
 		return
@@ -1513,13 +1513,16 @@ func TestServerConnectionAPIRequiresBoundWorkloadAndOpaqueAssignedPlayer(t *test
 	if recorder.connectCalls != 1 || recorder.binding != binding || recorder.playerID != "player-123456789" || recorder.expectedGeneration != 0 || recorder.key != "connect-player-123456789" {
 		t.Fatalf("connection receipt = %+v", recorder)
 	}
+	if got, body := request("connect", binding.ServerID, "player-legacy-123456", "workload-token", "connect-legacy-123456", ""); got != http.StatusNoContent || body != "" {
+		t.Fatalf("legacy connection status=%d body=%q", got, body)
+	}
 	if got, _ := request("connect", "server-000000000", "player-123456789", "workload-token", "connect-player-123456789", ""); got != http.StatusUnauthorized {
 		t.Fatalf("wrong server status = %d", got)
 	}
 	if got, _ := request("connect", binding.ServerID, "short", "workload-token", "connect-player-short-123", ""); got != http.StatusUnprocessableEntity {
 		t.Fatalf("short player status = %d", got)
 	}
-	if recorder.connectCalls != 1 {
+	if recorder.connectCalls != 2 {
 		t.Fatalf("invalid receipts reached backend: %d", recorder.connectCalls)
 	}
 	if got, _ := request("disconnect", binding.ServerID, "player-123456789", "workload-token", "disconnect-player-123456789", `,"generation":1`); got != http.StatusNoContent {
@@ -641,7 +641,7 @@ func TestPostgreSQLConnectionReceiptsStartCompleteRelaxedCasualRoster(t *testing
 	if _, err := ClaimPlayerConnection(ctx, db, binding, "connect-player-0", 0, "connect-receipt-key-0000", now.Add(4*time.Second)); !errors.Is(err, domain.ErrConflict) {
 		t.Fatalf("stale connect replay err=%v, want conflict", err)
 	}
-	if generation, err := ClaimPlayerConnection(ctx, db, binding, "connect-player-0", 1, "reconnect-receipt-0000", now.Add(63*time.Second)); err != nil || generation != 2 {
+	if generation, err := ClaimPlayerConnection(ctx, db, binding, "connect-player-0", 0, "reconnect-receipt-0000", now.Add(63*time.Second)); err != nil || generation != 2 {
 		t.Fatalf("grace-boundary reconnect generation=%d err=%v", generation, err)
 	}
 	if err := RecordPlayerDisconnected(ctx, db, binding, "connect-player-0", 1, "stale-disconnect-0000", now.Add(64*time.Second)); !errors.Is(err, domain.ErrConflict) {
@@ -78,7 +78,10 @@ func ClaimPlayerConnection(ctx context.Context, db *sql.DB, binding domain.Workl
 		if err != nil {
 			return err
 		}
-		if current != expectedGeneration || expectedGeneration == ^uint64(0) {
+		// A fresh process has no in-memory generation. It may recover only a
+		// durably disconnected lease; an active row still fences it. All
+		// nonzero expectations remain exact CAS operations.
+		if (current != expectedGeneration && !(expectedGeneration == 0 && current > 0 && disconnectedAt.Valid)) || expectedGeneration == ^uint64(0) {
 			return domain.ErrConflict
 		}
 		if current == 0 {