fix(matchmaking): make regional RTT evidence obtainable end to end

domain.validCandidate hard-requires a non-empty PredictedRTT map, but
CreateQueueTicket persisted an empty one and the only endpoint that
could fill it returned 503 in every real binary, because Service.Probe
was assigned nowhere outside api tests. No client-created ticket could
ever be selected by the matcher. The Godot client had no probe method at
all, so even a wired backend was unreachable from the game.

Four distinct defects had to be fixed for this path to work:

Nothing issued the nonce ProbeProvider was meant to compare against, so
the contract could not be satisfied even in principle. Add
POST /v1/probes/{region}/challenge, backed by a durable single-use
challenge -- durable because any replica may serve the answer for a
challenge another replica issued. RTT is the interval between issuing
and receiving, so no client-reported latency reaches placement.

CreateQueueTicket marshalled a nil map to JSON `null`, a JSONB scalar
rather than an object, and jsonb_set rejects that with "cannot set path
in scalar". RecordProbe would have failed at runtime even once wired.
Persist an object, and normalise non-object values in the update for
rows already written.

A nil ProbeRecorder made the handler report success while persisting
nothing, which silently leaves the ticket unmatchable. That is a
misconfiguration, not a successful probe; it now returns 503.

A successful probe updated PostgreSQL only. The candidate inserted at
enqueue time carries an empty RTT map, and the Redis keyspace has its
TTL continually refreshed, so the stale entry need never repair itself.
Refresh that player's projection after the probe commits.

Client side: add the challenge/answer round trip and have the
matchmaking screen collect evidence before creating a ticket, since
queueing first produces a search that can never match. Probing every
region fully is not required -- placement uses whichever regions
answered -- but queueing with none is refused rather than silently
stalling.

New integration test drives the real enqueue and probe paths and then
asks the actual matcher predicate, rather than hand-building a candidate
the way the unit tests do -- which is exactly why they missed this.

Also make the integration schema reset drop the whole public schema: the
enumerated table list silently broke with each new migration.
This commit is contained in:
Josh Creek
2026-09-05 10:49:28 +01:00
parent 5765532409
commit 801fca7cb0
16 changed files with 1729 additions and 71 deletions
+57
View File
@@ -6,6 +6,8 @@ extends Node
signal request_succeeded(operation: String, payload: Dictionary)
signal request_failed(operation: String, http_code: int, detail: String)
signal session_expired()
signal probe_challenge_received(region: String, nonce_base64: String)
signal probe_recorded(region: String, server_rtt_ms: int)
signal session_changed(player_id: String)
signal websocket_event(event: Dictionary)
signal websocket_status_changed(status: String)
@@ -215,6 +217,50 @@ func queue_create(ticket_id: String, playlist: String, client_build: String, pro
return err
# Regional latency probing. The backend issues a single-use nonce, the client
# echoes it back with its opaque platform location, and the backend derives the
# round trip from its own timestamps -- no client-measured latency is accepted.
#
# Until a ticket has RTT evidence for at least one region the matcher will not
# consider it (server/domain.validCandidate requires a non-empty map), so this
# has to complete before searching is meaningful.
const PROBE_REGIONS := ["EU", "NA"]
func request_probe_challenge(region: String) -> Error:
if not is_valid_probe_region(region):
return ERR_INVALID_PARAMETER
return _start_request("probe_challenge_" + region, HTTPClient.METHOD_POST, "/v1/probes/%s/challenge" % region, {}, "")
func submit_probe_answer(region: String, nonce_base64: String, opaque_location_base64: String) -> Error:
if not is_valid_probe_region(region) or nonce_base64.is_empty() or opaque_location_base64.is_empty():
return ERR_INVALID_PARAMETER
return _start_request("probe_answer_" + region, HTTPClient.METHOD_POST, "/v1/probes/%s" % region, {
"nonce": nonce_base64,
"opaque_location": opaque_location_base64,
}, "")
static func is_valid_probe_region(region: String) -> bool:
return region == "EU" or region == "NA"
# The platform location is opaque to us by design: the backend treats it as a
# blob and never derives placement from anything the client measured. Without a
# Steam runtime there is nothing to report, so send a stable non-empty marker
# rather than failing the probe -- the RTT is what actually matters and that is
# measured by the backend either way.
static func opaque_location_payload() -> String:
if Engine.has_singleton("Steam"):
var steam := Engine.get_singleton("Steam")
if steam.has_method("getLocalPingLocation"):
var location = steam.call("getLocalPingLocation")
if location is String and not String(location).is_empty():
return Marshalls.utf8_to_base64(String(location))
return Marshalls.utf8_to_base64("no-platform-ping-location")
func login_steam(web_api_ticket: String) -> Error:
if not is_valid_web_api_ticket(web_api_ticket):
return ERR_INVALID_PARAMETER
@@ -592,6 +638,17 @@ func _on_request_completed(result: HTTPRequest.Result, response_code: int, _head
if not assignment.apply(payload, player_id):
request_failed.emit(operation, response_code, assignment.error_message)
return
elif operation.begins_with("probe_challenge_"):
# Answer immediately: the nonce is single-use and short-lived, and the
# interval to this answer is exactly what the backend measures.
var challenge_region := operation.trim_prefix("probe_challenge_")
var nonce := String(payload.get("nonce", ""))
if nonce.is_empty():
request_failed.emit(operation, response_code, "probe challenge did not include a nonce")
return
probe_challenge_received.emit(challenge_region, nonce)
elif operation.begins_with("probe_answer_"):
probe_recorded.emit(operation.trim_prefix("probe_answer_"), int(payload.get("server_rtt_ms", -1)))
request_succeeded.emit(operation, payload)
if not _pending_resync_resource_id.is_empty():
call_deferred("_run_pending_resync")
+70
View File
@@ -18,6 +18,12 @@ const RECOVERY_POLL_SECONDS := 2.0
var _elapsed_seconds := 0.0
var _heartbeat_seconds := 0.0
var _recovery_poll_seconds := 0.0
# Regions still awaiting RTT evidence, and the queue request deferred until at
# least one lands. The matcher ignores a ticket with no predicted RTT, so
# queueing before probing produces a search that can never match.
var _pending_probe_regions: Array[String] = []
var _probed_regions: Array[String] = []
var _deferred_queue := {}
func _ready() -> void:
@@ -31,6 +37,8 @@ func _ready() -> void:
ControlPlaneClient.request_failed.connect(_on_request_failed)
ControlPlaneClient.request_succeeded.connect(_on_request_succeeded)
ControlPlaneClient.session_expired.connect(_on_session_expired)
ControlPlaneClient.probe_challenge_received.connect(_on_probe_challenge_received)
ControlPlaneClient.probe_recorded.connect(_on_probe_recorded)
_refresh_ranked_profile()
_render(ControlPlaneClient.state.snapshot())
@@ -71,11 +79,73 @@ func _on_queue_pressed() -> void:
_recovery_poll_seconds = 0.0
var playlist := String(playlist_dropdown.get_selected_metadata())
var ticket_id := "ticket-%s-%s" % [str(Time.get_ticks_usec()), str(randi())]
# A ticket with no regional RTT evidence is invisible to the matcher, so
# collect it first and queue once the first region reports.
if _probed_regions.is_empty():
_deferred_queue = {"ticket_id": ticket_id, "playlist": playlist}
_start_probe_collection()
return
var err := ControlPlaneClient.queue_create(ticket_id, playlist, CLIENT_BUILD, PROTOCOL_VERSION)
if err != OK:
_on_local_error("Could not start matchmaking: %s" % error_string(err))
func _start_probe_collection() -> void:
_pending_probe_regions = []
for region in ControlPlaneClient.PROBE_REGIONS:
_pending_probe_regions.append(String(region))
ControlPlaneClient.state.set_notice("Measuring connection quality...")
_request_next_probe()
# One request at a time: the client serialises HTTP through a single
# HTTPRequest, so a second call would return ERR_BUSY.
func _request_next_probe() -> void:
if _pending_probe_regions.is_empty():
_finish_probe_collection()
return
var region := _pending_probe_regions[0]
var err := ControlPlaneClient.request_probe_challenge(region)
if err != OK and err != ERR_BUSY:
# A region we cannot probe is not fatal; placement just uses the
# regions that did respond.
_pending_probe_regions.remove_at(0)
_request_next_probe()
func _on_probe_challenge_received(region: String, nonce_base64: String) -> void:
var err := ControlPlaneClient.submit_probe_answer(region, nonce_base64, ControlPlaneClient.opaque_location_payload())
if err != OK:
_drop_pending_probe(region)
func _on_probe_recorded(region: String, _server_rtt_ms: int) -> void:
if not _probed_regions.has(region):
_probed_regions.append(region)
_drop_pending_probe(region)
func _drop_pending_probe(region: String) -> void:
var index := _pending_probe_regions.find(region)
if index >= 0:
_pending_probe_regions.remove_at(index)
_request_next_probe()
func _finish_probe_collection() -> void:
if _deferred_queue.is_empty():
return
var queued := _deferred_queue
_deferred_queue = {}
if _probed_regions.is_empty():
# Queueing now would create a ticket the matcher can never select.
_on_local_error("Could not measure connection quality to any region; matchmaking is unavailable")
return
var err := ControlPlaneClient.queue_create(String(queued["ticket_id"]), String(queued["playlist"]), CLIENT_BUILD, PROTOCOL_VERSION)
if err != OK:
_on_local_error("Could not start matchmaking: %s" % error_string(err))
func _on_cancel_pressed() -> void:
if not ControlPlaneClient.state.can_cancel():
return