diff --git a/Game/scripts/control_plane_client.gd b/Game/scripts/control_plane_client.gd index 0f54df9b..0777d7df 100644 --- a/Game/scripts/control_plane_client.gd +++ b/Game/scripts/control_plane_client.gd @@ -62,6 +62,32 @@ func _ready() -> void: _request.request_completed.connect(_on_request_completed) state.resync_required.connect(_on_resync_required) _websocket = WebSocketPeer.new() + assignment_connection_failed.connect(_on_assignment_connection_failed) + NetworkManager.connection_failed.connect(_on_network_connection_failed) + + +# Covers §8.43's "failed reconnect UX": connect_to_assignment()'s own +# synchronous failures (assignment missing/expired, invalid endpoint, +# NetworkManager.join() erroring immediately) previously only emitted +# assignment_connection_failed -- a signal nothing in the client actually +# listened to. state.phase would stay stuck at ASSIGNED, the UI would keep +# showing "Your match server is ready" forever, and there was no way back to +# a fresh search. +func _on_assignment_connection_failed(detail: String) -> void: + state.fail(detail) + + +# The likelier real-world failure than the synchronous one above: +# NetworkManager.join() returns OK immediately (the attempt started), but the +# actual ENet handshake fails asynchronously later -- unreachable server, +# refused connection, ENet's own ~5s connect timeout. This is exactly the gap +# main_menu.gd's own _on_connection_failed exists to cover for the direct-join +# flow (see its header comment); nothing covered it for a matchmaking-driven +# connect. Guarded to CONNECTING so this never reacts to an unrelated +# connection_failed, such as one belonging to main_menu.gd's own direct join. +func _on_network_connection_failed() -> void: + if state.phase == MatchmakingState.CONNECTING: + state.fail("Unable to connect to the match server") func _process(_delta: float) -> void: diff --git a/Game/tests/cases/test_control_plane_client.gd b/Game/tests/cases/test_control_plane_client.gd index 1eec9320..4aef55ee 100644 --- a/Game/tests/cases/test_control_plane_client.gd +++ b/Game/tests/cases/test_control_plane_client.gd @@ -310,6 +310,62 @@ func test_client_defers_the_connect_until_the_assignment_fetch_completes() -> vo client.free() +# Covers §8.43's "failed reconnect UX": connect_to_assignment()'s own +# synchronous failures previously only emitted assignment_connection_failed, +# a signal nothing in the client listened to -- state.phase stayed stuck at +# ASSIGNED, the UI kept showing "Your match server is ready" forever, and +# there was no way back to a fresh search. +func test_synchronous_assignment_connection_failure_surfaces_as_a_failed_search() -> void: + var client := ControlPlaneClient.new() + client._ready() + assert_true(client.state.begin_queue("ticket-connect-unavailable", "casual"), "queue setup succeeds") + # client.assignment is still the default, unavailable one. + var err := client.connect_to_assignment() + assert_eq(err, ERR_UNAUTHORIZED, "connect fails closed when the assignment isn't ready") + assert_eq(client.state.phase, MatchmakingState.FAILED, "the failure is surfaced as a failed search rather than leaving the UI stuck at ASSIGNED") + assert_true(client.state.message.to_lower().contains("unavailable") or client.state.message.to_lower().contains("expired"), "the failure detail is retained: %s" % client.state.message) + client.free() + + +# The likelier real-world failure than the synchronous one above: +# NetworkManager.join() returns OK immediately (the attempt started), but the +# actual ENet handshake fails asynchronously later -- unreachable server, +# refused connection, ENet's own ~5s connect timeout. This is exactly the gap +# main_menu.gd's own _on_connection_failed exists to cover for the +# direct-join flow; nothing covered it for a matchmaking-driven connect. +func test_async_network_connection_failure_after_assignment_ready_surfaces_as_a_failed_search() -> void: + var client := ControlPlaneClient.new() + client._ready() + assert_true(client.configure("https://match.example", "session-id:opaque-token"), "client configures") + client.player_id = "player_1234567890" + assert_true(client.state.begin_queue("ticket-connect-asyncfail", "casual"), "queue setup succeeds") + client._operation = "assignment" + var assignment_payload := {"match_id": "match_asyncfail_1234567890", "server_id": "server_asyncfail_1234567890", "player_id": "player_1234567890", "slot": 0, "expires_at": "2099-08-31T12:00:00Z", "protocol_version": 1, "transport": "enet", "endpoint": "127.0.0.1:65501", "join_authorisation": "opaque-join-token"} + client._on_request_completed(HTTPRequest.RESULT_SUCCESS, 200, PackedStringArray(), JSON.stringify(assignment_payload).to_utf8_buffer()) + client._operation = "queue_recover" + var ticket_payload := {"ticket_id": "ticket-connect-asyncfail", "player_id": "player_1234567890", "playlist": "casual", "revision": 5, "state": "ASSIGNED", "match_id": "match_asyncfail_1234567890", "enqueued_at": "2026-08-31T12:00:00Z", "expires_at": "2099-08-31T12:00:00Z"} + client._on_request_completed(HTTPRequest.RESULT_SUCCESS, 200, PackedStringArray(), JSON.stringify(ticket_payload).to_utf8_buffer()) + assert_eq(client.state.phase, MatchmakingState.CONNECTING, "the transport attempt started") + + NetworkManager.connection_failed.emit() + assert_eq(client.state.phase, MatchmakingState.FAILED, "the async handshake failure is surfaced rather than leaving CONNECTING stuck forever") + + NetworkManager.shutdown() + client.free() + + +func test_network_connection_failure_is_ignored_outside_a_matchmaking_driven_connect() -> void: + var client := ControlPlaneClient.new() + client._ready() + assert_true(client.state.begin_queue("ticket-unrelated-failure", "casual"), "queue setup succeeds") + # state.phase is QUEUED, not CONNECTING -- this connection_failed belongs + # to something else (e.g. main_menu.gd's own direct-join flow) and must + # not be misattributed to matchmaking. + NetworkManager.connection_failed.emit() + assert_eq(client.state.phase, MatchmakingState.QUEUED, "an unrelated connection_failed does not fail an active queue search") + client.free() + + func test_rest_resource_identifiers_use_the_opaque_contract_shape() -> void: assert_true(ControlPlaneClient.is_valid_resource_id("ticket_1234567890"), "contract-sized resource id is accepted") assert_true(not ControlPlaneClient.is_valid_resource_id("ticket-1"), "short resource id is rejected") diff --git a/multiplayer-next.md b/multiplayer-next.md index aa469f81..a95a0f33 100644 --- a/multiplayer-next.md +++ b/multiplayer-next.md @@ -1243,7 +1243,7 @@ The allocated-runtime result reporter now keeps a completed match in `RESULTS` u | 8.40 `[D:8.3,8.14]` | **IN PROGRESS.** Pure Go revisioned replica reducer rejects gaps for REST resync, makes duplicate/out-of-order events idempotent, and resumes from the authoritative snapshot revision; authenticated queue-ticket recovery now has an owner-checked REST read; Godot client projection persists non-secret ticket/proposal state, forces authoritative recovery after restart, and can replay a lost queue-create response with the original ticket/idempotency key; the Go API now exposes an authenticated `/v1/events` WebSocket with bounded per-player queues, strict upgrade/vocabulary validation and REST-resync-safe slow-client failure; Godot `ControlPlaneClient` can connect to the stream using Godot 4.7's handshake-header API, consume validated events, automatically reconnect with bounded backoff, and trigger REST recovery on projection gaps and stream return, including a deferred recovery when an event arrives during an in-flight HTTP mutation; `OutboxDispatcher` now provides ordered at-least-once delivery after durable commit; assignment recovery events carry the durable assignment revision without leaking it into REST JSON; prediction startup now distinguishes the sequence-0 warm-up acknowledgement from genuine missing/evicted history, preventing a false hard-snap during a live match; proposal creation writes one durable targeted `proposal_changed` outbox event per proposal aggregate revision, while result completion writes `match_completed` and production `cmd/control-plane` plus the test-only API harness dispatch both event types through separate filtered consumers | `server/domain/sync.go`, `server/api/events.go`, `server/api/outbox.go`, `server/api/service.go`, `server/store/outbox.go`, `server/store/proposal_sql.go`, `server/cmd/control-plane`, `server/cmd/testkit-api`, `service_test.go`, `outbox_test.go`, `matchmaking_state.gd`, `control_plane_client.gd`, `local_prediction_history.gd`, `net_ship_predictor.gd` and tests cover gap, snapshot, replay, same-revision conflict, owner-only ticket recovery, expired-ticket terminal handling, malformed restart snapshots, API-level duplicate-create replay/conflict, authenticated handshake/key rejection, targeted event delivery, exactly-once slow-subscriber closure, invalid-event rejection, delivery-before-ack failure ordering, assignment event/response revision separation, prediction warm-up/hard-resync separation, deferred proposal recovery and proposal/result outbox filtering and delivery failures; Godot 4.7.1 headless project parse and 150-test unit harness pass with compatibility rendering, and a two-process ENet authoritative match smoke passes spawn, movement, prediction, 0% snapshot loss and 0 hard snaps; `cmd/control-plane` provides a signal-bound API role, `cmd/matcher` now supports casual and explicitly enabled ranked roles with durable identity lookup, and `cmd/maintenance` provides bounded season maintenance. **Live multi-process control-plane/game verification now exists**: `scripts/verify_control_plane_integration.sh` runs a real `postgres:17-alpine`, the real `api.Service` (via the new test-only `server/cmd/testkit-api`, wired identically to `cmd/control-plane` except for a fake Steam login — see §8.7), and a real headless Godot client (`control_plane_smoke.gd`) round-tripping login → `fetch_ranked_profile` (expect 404, §8.22) → `queue_create` → heartbeat → `cancel_queue` over an actual network connection — the first time this boundary was exercised end to end rather than against a mock on either side. **The two-player proposal extension now also passes**: `scripts/verify_control_plane_proposal_integration.sh` starts isolated PostgreSQL, the real matcher and testkit API, launches two headless Godot clients, seeds deterministic RTT projections, observes the real WebSocket `OPEN` proposal, accepts from both clients, and verifies both durable tickets plus the proposal reach `ACCEPTED` (including authoritative recovery when the concurrent revision advances). `scripts/run_result_fanout_integration.sh` additionally verifies a real PostgreSQL-backed authenticated WebSocket receives a completed-match event; allocator and Redis fan-out live verification remain | | 8.41 `[D:7.8,8.9,8.31,8.40]` | **IN PROGRESS.** Authenticated `GET /v1/assignments/{matchId}` now exposes only a validated, player-scoped assignment view including the hosted endpoint; Godot `AssignmentState`/`ControlPlaneClient.fetch_assignment()` bind the response to the authenticated player, recheck expiry, validate and retain the endpoint, and `connect_to_assignment()` starts only the validated ENet/Steam transport after assignment readiness. **Found and fixed a severe gap this row had previously described as already closed**: `connect_to_assignment()` existed, fully validated, with its own `assignment_connection_started`/`assignment_connection_failed` signals -- but nothing anywhere in the client ever called it. A player who completed the entire queue -> proposal -> allocate -> assign pipeline would reach `ASSIGNED` and see "Your match server is ready" and then simply sit there forever; the transport was never actually started. `ControlPlaneClient._connect_when_assigned()` now calls it automatically the moment `state.phase` reaches `ASSIGNED` (wired into the one call site every queue-shaped HTTP response -- heartbeat, recover, and resync-triggered recover -- already shares, so both the REST poll and the WebSocket-triggered-resync path are covered without a second call site), deferring via `_pending_connect_match_id` if the assignment fetch triggered earlier by `ASSIGNMENT_READY` hasn't completed yet, and guarding against a duplicate/replayed `ASSIGNED` event reattempting the connection. The opaque join authorisation is carried in the MatchNet hello payload rather than the endpoint URL; allocated Godot servers now fail closed unless an operator-mounted JSON roster of control-plane signed envelopes and an HMAC-SHA256 key are present, and MatchNet verifies the canonical Go claim bytes, checks exact token membership plus match/server/protocol/slot/expiry claims before admitting a peer, rejects concurrent reuse of an active token, tracks server-owned reconnect generations across the 60-second reclaim window, fences expired reclaims and invalidates old peers; migration 0002 and the Go store adapter now persist/recover the complete player-scoped assignment projection with conflict-safe identical replay; `AssignmentProviderFromStore` wires that durable projection into the API injection point; `SaveAssignments` publishes a complete signed roster atomically instead of allowing partial player visibility; `SaveVerifiedAssignmentRoster` rechecks signed claims before deriving player rows; a workload-authenticated `GET /v1/servers/{serverId}/roster` now returns the complete signed envelope set only for the bound allocation, and the allocated supervisor atomically materializes it before launching Godot; live authenticated Godot/PostgreSQL assignment verification now exists via `scripts/verify_assignment_integration.sh`, which seeds a durable player-scoped `ASSIGNMENT_READY` row behind the real testkit API and verifies the returned endpoint/join authorisation over HTTP/JSON (including the live numeric-decoding boundary); `scripts/run_supervisor_integration.sh` additionally verifies the allocated-server process-ready → roster fetch/materialization → assignment-ready registration path against real PostgreSQL and the real API, with the workload token recovered from an Agones-shaped GameServer annotation | `server/api/service.go`, `store_adapters.go`, `service_test.go`, `assignment_state.gd`, `control_plane_client.gd`, `match_net.gd`, `server_boot.gd`, `server_config.gd`, `test_assignment_state.gd`, `test_control_plane_client.gd`, `test_match_net.gd`, `server/contracts/v1/openapi.json`, `server/migrations/0002_assignments.sql`, `server/allocator/allocator_integration_test.go`, `server/supervisor/supervisor_integration_test.go`, `server/supervisor/supervisor.go` and `server/store/assignment_sql.go` cover participant/identity/expiry/shape/transport/endpoint boundaries, player-scoped schema keys, expiry-filtered reads, assignment upsert conflict handling, atomic batch validation, signed-claim binding, strict endpoint splitting, allowlisted claim rejection, canonical HMAC interoperability, forged-signature rejection, duplicate active-token rejection, reclaim generation/expiry behavior, workload-authenticated roster delivery, atomic file installation, real allocator reconciliation, real supervisor registration and 150-test Godot compatibility coverage; the three-process disconnect/reclaim smoke passes ship retention, replacement drive and old-peer invalidation; two new `test_control_plane_client.gd` tests cover the connect-wiring fix directly: `test_client_starts_the_transport_once_the_ticket_reaches_assigned` proves a ready, fresh assignment plus an `ASSIGNED` ticket update actually starts the transport (`state.phase` advances to `CONNECTING`, `connect_to_assignment`'s own signal fires) and that a duplicate attempt is refused, `test_client_defers_the_connect_until_the_assignment_fetch_completes` proves the opposite ordering (an `ASSIGNED` update before the assignment fetch completes) defers rather than either connecting with stale data or erroring; verified against the real Godot 4.7.1 binary (216/216, no crash), the full local gate and the ENet integration suite. **"Dynamic per-match launch flags" was stale, corrected in §8.16**: `agones.Client.Allocate` already requests arena-path/playlist/region/build/protocol/transport as Agones annotations and `supervisor.withAllocatedCompatibility` already overlays them onto the launch command, fully tested and wired into `Supervisor.Start()`. SDR relay-ticket installation and live Agones cluster integration remain | | 8.42 `[D:8.22,8.23,8.24,8.40]` | **IN PROGRESS.** `RankedProfileState` and `ControlPlaneClient.fetch_ranked_profile()` expose the backend-authoritative rating/RD/volatility/games/tier/provisional/season view; matchmaking UI displays provisional/tier status without client-side rating math; runnable API binaries now wire the backend-owned default tier policy; live authenticated profile verification now exists via `scripts/verify_ranked_profile_integration.sh`, which seeds a durable rating and verifies the populated profile over real PostgreSQL, Go HTTP, and Godot JSON | `test_control_plane_client.gd` validates profile shape, numeric safety and provisional display; the real populated profile path now covers durable rating/games/tier/provisional decoding and caught the missing runtime tier-policy wiring; committed revision after reconnect, abandon status and season countdown remain dependent on live auth/backend events and Godot runtime verification | -| 8.43 `[D:8.39,8.40,8.41]` | **IN PROGRESS.** Matchmaking client now distinguishes expired queue recovery, session expiry, missing records and retryable control-plane outages; a 401 clears the in-memory token, emits `session_expired` and disables retry until a new session is configured; terminal messages remain visible and active searches are not falsely failed on transient errors; stale proposal accept/decline conflicts now automatically schedule an authoritative proposal recovery instead of requiring each UI caller to implement 409 handling | `MatchmakingState` and `ControlPlaneClient` tests cover explicit expiry and the existing terminal/retry-safe state paths; the real two-player proposal smoke now exercises the client-owned conflict-recovery path. **This row's own "remain" list was stale**: `matchmaking.gd`'s decline button/handler already existed (`%DeclineButton`, `_on_decline_pressed`, visibility toggled by `MatchmakingState.PROPOSED` alongside accept) with no test gap; `ControlPlaneClient.can_retry_last_mutation()`/`retry_last_mutation()` -- the generic "duplicate-action recovery beyond proposals" and "regional outage retry" mechanism -- also already existed (any mutation, not just a proposal response, becomes retryable on a transport failure or 503/408/429, and the matchmaking queue button already fell back to it), it simply had zero test coverage proving the transition actually happens for a non-proposal mutation; two new tests close that (`test_generic_mutation_retry_recovers_after_a_transient_failure`, `test_generic_mutation_retry_is_not_offered_for_unsafe_failures`). `MatchmakingClient`'s real dispatch (`HTTPRequest.request()`) needs a live SceneTree that `test_runner.tscn`'s synchronous single-`_ready()` execution model cannot provide mid-suite, so the two new tests exercise the `can_retry_last_mutation()` decision boundary and the `ERR_INVALID_DATA` fail-closed path rather than the literal network call; verified against the real Godot 4.7.1 binary (214/214 tests, no crash, no engine-level error), plus a full `make verify-multiplayer-local` re-run. **Version-mismatch messaging is now built**: before this, there was no server-side protocol rejection at all -- `queue_create` accepted any `protocol_version >= 1` unconditionally, so an outdated client could only ever discover the mismatch by waiting forever unmatched (the matcher's own compatibility check requires every formed player to share an identical `protocol_version`), with no error and no explanation. `Service.MinProtocolVersion` (opt-in, zero by default) now rejects a below-floor `queue_create` with `426 Upgrade Required`/`client_outdated` before ever reaching the candidate provider, wired via `cmd/control-plane`'s `--min-protocol-version` flag; `ControlPlaneClient` recognises 426 on `queue_create` specifically and sets a distinct "Your client is out of date -- please update to continue searching" message, clearing `_last_queue_create` so the generally-available "Retry Search" affordance is never offered for a failure retrying can't fix. `TestQueueCreateEnforcesMinProtocolVersion`/`TestQueueCreateMinProtocolVersionZeroIsDisabled` (Go) and `test_outdated_client_receives_a_distinct_message_and_no_retry_offer` (Godot) cover the floor end to end: below-floor rejection before the candidate provider is ever reached, exactly-at-floor acceptance, the opt-in zero-disables-it default, the client message and the suppressed retry. Failed-reconnect UX and long-running worker integration (§8.16) remain | +| 8.43 `[D:8.39,8.40,8.41]` | **IN PROGRESS.** Matchmaking client now distinguishes expired queue recovery, session expiry, missing records and retryable control-plane outages; a 401 clears the in-memory token, emits `session_expired` and disables retry until a new session is configured; terminal messages remain visible and active searches are not falsely failed on transient errors; stale proposal accept/decline conflicts now automatically schedule an authoritative proposal recovery instead of requiring each UI caller to implement 409 handling | `MatchmakingState` and `ControlPlaneClient` tests cover explicit expiry and the existing terminal/retry-safe state paths; the real two-player proposal smoke now exercises the client-owned conflict-recovery path. **This row's own "remain" list was stale**: `matchmaking.gd`'s decline button/handler already existed (`%DeclineButton`, `_on_decline_pressed`, visibility toggled by `MatchmakingState.PROPOSED` alongside accept) with no test gap; `ControlPlaneClient.can_retry_last_mutation()`/`retry_last_mutation()` -- the generic "duplicate-action recovery beyond proposals" and "regional outage retry" mechanism -- also already existed (any mutation, not just a proposal response, becomes retryable on a transport failure or 503/408/429, and the matchmaking queue button already fell back to it), it simply had zero test coverage proving the transition actually happens for a non-proposal mutation; two new tests close that (`test_generic_mutation_retry_recovers_after_a_transient_failure`, `test_generic_mutation_retry_is_not_offered_for_unsafe_failures`). `MatchmakingClient`'s real dispatch (`HTTPRequest.request()`) needs a live SceneTree that `test_runner.tscn`'s synchronous single-`_ready()` execution model cannot provide mid-suite, so the two new tests exercise the `can_retry_last_mutation()` decision boundary and the `ERR_INVALID_DATA` fail-closed path rather than the literal network call; verified against the real Godot 4.7.1 binary (214/214 tests, no crash, no engine-level error), plus a full `make verify-multiplayer-local` re-run. **Version-mismatch messaging is now built**: before this, there was no server-side protocol rejection at all -- `queue_create` accepted any `protocol_version >= 1` unconditionally, so an outdated client could only ever discover the mismatch by waiting forever unmatched (the matcher's own compatibility check requires every formed player to share an identical `protocol_version`), with no error and no explanation. `Service.MinProtocolVersion` (opt-in, zero by default) now rejects a below-floor `queue_create` with `426 Upgrade Required`/`client_outdated` before ever reaching the candidate provider, wired via `cmd/control-plane`'s `--min-protocol-version` flag; `ControlPlaneClient` recognises 426 on `queue_create` specifically and sets a distinct "Your client is out of date -- please update to continue searching" message, clearing `_last_queue_create` so the generally-available "Retry Search" affordance is never offered for a failure retrying can't fix. `TestQueueCreateEnforcesMinProtocolVersion`/`TestQueueCreateMinProtocolVersionZeroIsDisabled` (Go) and `test_outdated_client_receives_a_distinct_message_and_no_retry_offer` (Godot) cover the floor end to end: below-floor rejection before the candidate provider is ever reached, exactly-at-floor acceptance, the opt-in zero-disables-it default, the client message and the suppressed retry. **Failed-reconnect UX is now built too**: `connect_to_assignment()`'s synchronous failures (assignment missing/expired, invalid endpoint, `NetworkManager.join()` erroring immediately) only ever emitted `assignment_connection_failed` -- a signal nothing in the client listened to, leaving `state.phase` stuck at `ASSIGNED` and the UI showing "Your match server is ready" forever with no way back to a fresh search. Worse, the likelier real-world failure -- `NetworkManager.join()` returning `OK` immediately while the actual ENet handshake fails asynchronously later (unreachable server, refused connection, ENet's own ~5s connect timeout) -- had no handler at all for a matchmaking-driven connect, even though `main_menu.gd`'s own `_on_connection_failed` exists specifically to cover this exact async gap for the direct-join flow. `ControlPlaneClient` now connects both `assignment_connection_failed` and (guarded to `state.phase == CONNECTING`, so it never misattributes an unrelated direct-join failure) `NetworkManager.connection_failed` to `state.fail(...)`, so either failure mode now surfaces as a failed search the player can retry from, instead of a silent hang. `test_synchronous_assignment_connection_failure_surfaces_as_a_failed_search`, `test_async_network_connection_failure_after_assignment_ready_surfaces_as_a_failed_search` and `test_network_connection_failure_is_ignored_outside_a_matchmaking_driven_connect` cover both failure modes and the CONNECTING guard; verified against the real Godot 4.7.1 binary (220/220, no crash, stable across repeated runs), the full local gate and the ENet integration suite, zero new crash reports. Long-running worker integration (§8.16) remains | #### 8F — Observability, verification, cost and rollout