fix(multiplayer): resolve composition regression from second adversarial review

A second adversarial review of the previous fix commit found two of its
nine fixes silently defeated each other: the seq-range guard (fix for a
MEDIUM epoch-mismatch finding) capped the exact variable the ring-overflow
resync (fix for the original CRITICAL finding) depends on, making the
resync unreachable in production and recreating permanent input death at
a lower failure threshold, reachable via ordinary server tick loss alone.

- CRITICAL: rebind the seq-range guard to InputJitterBuffer's own
  highest_ingested_seq (now public) instead of the consumer-side
  last_applied_seq, so it tracks the client's send epoch rather than a
  value that can lag arbitrarily far behind during a stall.
- HIGH: InputLeadController's release logic still ANDed the old
  `lead > LEAD_MIN` gate onto the new depth-driven condition, so a
  backlog the controller never caused still couldn't drain. Split into
  two independent decisions: the seq-duplicate action follows real
  depth alone; lead's own bookkeeping separately never drops below its
  floor.
- MEDIUM: widen the CI driver's movement/stalled sampling margin
  (run_seconds - 2.0, was - 0.5) and assert the peer is still in
  multiplayer.get_peers() at sample time, since the old margin let the
  check pass on residual starvation grace after a bot had already
  disconnected.
- LOW: measure horizontal-only displacement in the human smoke test's
  movement check — the old 3D-distance bar was beatable by pure
  gravity settling with fully dead input.
- LOW: fix a real "clean stderr" violation (match_net.gd broadcasting
  a departure notice to a peer whose ENet channels are already torn
  down, including a second peer disconnecting in the same poll batch)
  by deferring the notification to the next idle frame.
- Wire the server's per-slot stalled bit into the client debug overlay
  for real — a prior commit message claimed this already reached the
  overlay when only the CI gate actually read it.

Re-verified end-to-end against the real production RPC path (not just
unit tests in isolation, which is how the composition bug got past the
first round): a 2-bot CI match with a 1.5s host SIGSTOP freeze injected
mid-run, well past the 0.6s threshold the review reproduced the bug at,
now recovers cleanly on repeated runs with zero stderr noise.
This commit is contained in:
Josh Creek
2026-08-20 18:26:12 +01:00
parent 2325313ad2
commit cf73074e27
8 changed files with 180 additions and 53 deletions
+25 -12
View File
@@ -67,12 +67,18 @@ func test_release_requires_both_clean_surplus_and_its_own_interval() -> void:
func test_release_stops_at_minimum() -> void:
var c := InputLeadController.new()
# Sustained surplus depth, but lead is already at LEAD_MIN — must never
# push it below the floor regardless of how much surplus is reported.
# Sustained surplus depth with lead already at LEAD_MIN: `lead` itself
# must never drop below the floor, but release must still fire
# (duplicate a seq) once its own timing conditions are met, since a
# real reported surplus at floor lead is exactly the "backlog this
# controller never caused" case — capping `lead` is cosmetic, it must
# not also block the seq-duplicate action that drains real depth.
var released := false
for i in InputLeadController.CLEAN_SURPLUS_TICKS * 3:
var delta := c.update(InputLeadController.TARGET_DEPTH + 1)
assert_true(delta == 1, "lead already at minimum, never duplicates a seq trying to release further, tick %d" % i)
assert_eq(c.lead, InputLeadController.LEAD_MIN, "stays at minimum")
if c.update(InputLeadController.TARGET_DEPTH + 1) == 0:
released = true
assert_eq(c.lead, InputLeadController.LEAD_MIN, "lead never drops below the floor, tick %d" % i)
assert_true(released, "release still fires (duplicates a seq) even though lead itself is pinned at minimum")
func test_starve_resets_clean_surplus_counter() -> void:
@@ -99,14 +105,18 @@ func test_starve_resets_clean_surplus_counter() -> void:
assert_eq(c.lead, lead_after_attack - 1, "release finally fires once a full fresh clean window has elapsed since the interruption")
# An adversarial review found the original release gate was `lead >
# A first attempt at fixing this gated the whole release branch on `lead >
# LEAD_MIN` — this controller's own memory of past attacks — so a backlog
# it did NOT itself create (a server hitch, persistent client/server clock
# drift, a burst re-delivery) was never drained: lead stayed at 1 forever
# even while the server kept reporting a deep, real backlog. This
# reproduces that scenario directly: lead never attacks (depth is never
# reported as a starve, <= 0), yet release must still fire from sustained
# real surplus alone.
# even while the server kept reporting a deep, real backlog, and — because
# that gate blocked the seq-duplicate action too, not just lead's own
# bookkeeping — the actual buffered depth was never drained either. A
# second adversarial review caught that the depth check added alongside
# it didn't remove the old gate, just sat next to it. This reproduces the
# scenario directly: lead never attacks (depth is never reported as a
# starve, <= 0), yet release must still fire from sustained real surplus
# alone, even while lead itself stays pinned at its floor throughout.
func test_release_drains_a_backlog_it_never_caused_itself() -> void:
var c := InputLeadController.new()
assert_eq(c.lead, InputLeadController.LEAD_MIN, "starts at minimum, never attacked")
@@ -114,9 +124,12 @@ func test_release_drains_a_backlog_it_never_caused_itself() -> void:
# A large, externally-caused surplus (e.g. right after the server's own
# ring-overflow resync) reported for well over 2s — lead never moves
# via attack since depth is never <= 0.
var released_at_floor := false
for i in InputLeadController.CLEAN_SURPLUS_TICKS + InputLeadController.RELEASE_INTERVAL_TICKS:
c.update(10)
assert_eq(c.lead, InputLeadController.LEAD_MIN, "lead cannot release below its own floor even under large surplus")
if c.update(10) == 0:
released_at_floor = true
assert_eq(c.lead, InputLeadController.LEAD_MIN, "lead's own bookkeeping never drops below its floor")
assert_true(released_at_floor, "release still fires (duplicates a seq, actually draining real depth) even while lead is pinned at the floor")
# Raise it above the floor via one real attack, then confirm sustained
# external surplus (not self-caused) still drains it back down.
+31 -10
View File
@@ -98,8 +98,16 @@ func run_client_check(settle_seconds: float, drive_seconds: float) -> void:
var end_position: Vector3 = my_slot.ship.visual.global_position
var moved := start_position.distance_to(end_position)
print("SMOKE INFO: client ship moved %.2fm (start=%s end=%s) while holding forward thrust for %.1fs" % [
moved, str(start_position), str(end_position), drive_seconds
# Horizontal-only (XZ), not full 3D distance: an adversarial review
# found a 1.2s window of completely dead input still registers ~1.07m
# of pure gravity settling on the Y axis alone (spawn height dropping
# to the floor), which sat ABOVE the old moved > 1.0 bar — only
# thrust_z_ok caught that failure, not moved. Forward thrust is a
# horizontal force (see ship.gd), so measuring XZ displacement can't
# be satisfied by gravity alone, regardless of spawn height or timing.
var moved_horizontal := Vector2(end_position.x, end_position.z).distance_to(Vector2(start_position.x, start_position.z))
print("SMOKE INFO: client ship moved %.2fm (%.2fm horizontal) (start=%s end=%s) while holding forward thrust for %.1fs" % [
moved, moved_horizontal, str(start_position), str(end_position), drive_seconds
])
# thrust_power 150 / mass 5 = 30 m/s^2 nominal acceleration (see ship.gd) —
@@ -107,9 +115,9 @@ func run_client_check(settle_seconds: float, drive_seconds: float) -> void:
# A generous, not-tuned-to-the-decimal bound: this is a wiring smoke
# test, not a physics-accuracy test (net_codec's own tests already cover
# quantisation precision).
var success := moved > 1.0 and thrust_z_ok
print("SMOKE %s: client observed %.2fm of server-authoritative movement via interpolation, thrust_z_ok=%s" % [
"PASS" if success else "FAIL", moved, str(thrust_z_ok)
var success := moved_horizontal > 1.0 and thrust_z_ok
print("SMOKE %s: client observed %.2fm horizontal of server-authoritative movement via interpolation, thrust_z_ok=%s" % [
"PASS" if success else "FAIL", moved_horizontal, str(thrust_z_ok)
])
await get_tree().create_timer(0.3).timeout
NetworkManager.shutdown()
@@ -268,20 +276,33 @@ func run_ci_host_check(run_seconds: float) -> void:
# connected and playing, not after their run finishes — a client's own
# (legitimate, expected) disconnect at the end of its run naturally
# starves its jitter buffer too, which looks identical to the ring-
# overflow bug this check exists to catch if sampled too late. Clients
# start ~1.5s after the host and finish their own run_seconds shortly
# before disconnecting, so sample just ahead of that, not after.
var movement_check_delay := maxf(1.0, run_seconds - 0.5)
# overflow bug this check exists to catch if sampled too late. A first
# attempt used a 0.5s margin (run_seconds - 0.5); a second adversarial
# review instrumented multiplayer.get_peers() at sample time and found
# it was already EMPTY — both bots had legitimately disconnected before
# the sample ran, and the check was only passing on the ~200ms of
# residual STARVE_ZERO_TICKS starvation grace, not because it was
# genuinely still connected as this print used to claim. Widen the
# margin AND assert connectivity directly at sample time, rather than
# inferring it from timing, so a future regression in either direction
# (margin too tight again, or client run_seconds changing) fails loudly
# here instead of silently passing on residual grace.
var movement_check_delay := maxf(1.0, run_seconds - 2.0)
await get_tree().create_timer(movement_check_delay).timeout
var connected_peers := multiplayer.get_peers()
var input_reached_server := true
for slot in match_scene._slots:
var still_connected: bool = slot.peer_id in connected_peers
if not still_connected:
input_reached_server = false
print("SMOKE FAIL: peer %d already disconnected at movement-sample time (connected_peers=%s) — margin too tight" % [slot.peer_id, str(connected_peers)])
if not is_instance_valid(slot.ship) or not start_positions.has(slot.peer_id):
input_reached_server = false
print("SMOKE FAIL: peer %d has no valid ship to check movement on" % slot.peer_id)
continue
var moved: float = start_positions[slot.peer_id].distance_to(slot.ship.global_position)
var stalled: bool = slot.jitter_buffer.stalled
print("SMOKE INFO: peer %d moved %.2fm server-side (while still connected), stalled=%s" % [slot.peer_id, moved, str(stalled)])
print("SMOKE INFO: peer %d moved %.2fm server-side (connected=%s), stalled=%s" % [slot.peer_id, moved, str(still_connected), str(stalled)])
if moved <= 0.5 or stalled:
input_reached_server = false