Files
CosmicClash/Game/tests/networked_match_test_hooks.gd
T
Josh Creek cf73074e27 fix(multiplayer): resolve composition regression from second adversarial review
A second adversarial review of the previous fix commit found two of its
nine fixes silently defeated each other: the seq-range guard (fix for a
MEDIUM epoch-mismatch finding) capped the exact variable the ring-overflow
resync (fix for the original CRITICAL finding) depends on, making the
resync unreachable in production and recreating permanent input death at
a lower failure threshold, reachable via ordinary server tick loss alone.

- CRITICAL: rebind the seq-range guard to InputJitterBuffer's own
  highest_ingested_seq (now public) instead of the consumer-side
  last_applied_seq, so it tracks the client's send epoch rather than a
  value that can lag arbitrarily far behind during a stall.
- HIGH: InputLeadController's release logic still ANDed the old
  `lead > LEAD_MIN` gate onto the new depth-driven condition, so a
  backlog the controller never caused still couldn't drain. Split into
  two independent decisions: the seq-duplicate action follows real
  depth alone; lead's own bookkeeping separately never drops below its
  floor.
- MEDIUM: widen the CI driver's movement/stalled sampling margin
  (run_seconds - 2.0, was - 0.5) and assert the peer is still in
  multiplayer.get_peers() at sample time, since the old margin let the
  check pass on residual starvation grace after a bot had already
  disconnected.
- LOW: measure horizontal-only displacement in the human smoke test's
  movement check — the old 3D-distance bar was beatable by pure
  gravity settling with fully dead input.
- LOW: fix a real "clean stderr" violation (match_net.gd broadcasting
  a departure notice to a peer whose ENet channels are already torn
  down, including a second peer disconnecting in the same poll batch)
  by deferring the notification to the next idle frame.
- Wire the server's per-slot stalled bit into the client debug overlay
  for real — a prior commit message claimed this already reached the
  overlay when only the CI gate actually read it.

Re-verified end-to-end against the real production RPC path (not just
unit tests in isolation, which is how the composition bug got past the
first round): a 2-bot CI match with a 1.5s host SIGSTOP freeze injected
mid-run, well past the 0.6s threshold the review reproduced the bug at,
now recovers cleanly on repeated runs with zero stderr noise.
2026-08-20 18:26:12 +01:00

378 lines
18 KiB
GDScript

extends Node
# Test-only helper (tests/networked_match_smoke.gd). Not a project autoload
# — production code never references this. Same reason as
# tests/lobby_test_hooks.gd: networked_match.tscn is loaded via
# change_scene_to_file(), which frees whatever node initiated the load, so
# a driver can't keep orchestrating from a node that just got freed. The
# smoke test add_child()s this directly under get_tree().root instead (a
# sibling of current_scene, not a descendant of it), so it survives the swap.
#
# Uses preload(), not the bare `NetworkedMatch` class_name, and leaves
# `match_scene` itself untyped (Node) throughout — same global-script-class-
# cache-timing reason as tests/test_case.gd, plus every member access off an
# untyped Node returns Variant, which then needs explicit `: Type`
# annotations wherever `:=` would otherwise fail to infer one.
const NetworkedMatchScript = preload("res://scripts/networked_match.gd")
func _is_networked_match(node: Node) -> bool:
return node != null and node.get_script() == NetworkedMatchScript
func run_host_check(lifetime_seconds: float) -> void:
await get_tree().create_timer(lifetime_seconds * 0.4).timeout
var match_scene := get_tree().current_scene
var ok := _is_networked_match(match_scene)
var ship_count := 0
var ball_ok := false
var arena_name := "null"
if ok:
ship_count = match_scene.ships.size()
ball_ok = is_instance_valid(match_scene.ball)
if match_scene.arena:
arena_name = match_scene.arena.name
print("SMOKE INFO: host is_networked_match=%s ship_count=%d ball_ok=%s arena=%s" % [
str(ok), ship_count, str(ball_ok), arena_name
])
var success := ok and ship_count == 1 and ball_ok
print("SMOKE %s: host spawn check (ship_count=%d, ball_ok=%s)" % ["PASS" if success else "FAIL", ship_count, str(ball_ok)])
await get_tree().create_timer(lifetime_seconds * 0.6).timeout
if _is_networked_match(match_scene) and not match_scene.ships.is_empty():
var ship: Ship = match_scene.ships[0]
print("SMOKE INFO: host ship final position=%s (spawned, driven by client input if any arrived)" % str(ship.global_position))
NetworkManager.shutdown()
get_tree().quit(0 if success else 1)
func run_client_check(settle_seconds: float, drive_seconds: float) -> void:
await get_tree().create_timer(settle_seconds).timeout
var match_scene := get_tree().current_scene
if not _is_networked_match(match_scene):
print("SMOKE FAIL: current_scene is not NetworkedMatch after %.1fs" % settle_seconds)
get_tree().quit(1)
return
var slots_ok: bool = match_scene._slots.size() == 1
var ball_ok: bool = is_instance_valid(match_scene.ball)
var my_slot = match_scene._my_slot
var my_slot_ok: bool = my_slot != null and is_instance_valid(my_slot.ship)
var camera_ok: bool = is_instance_valid(match_scene._camera_rig)
var hud_ok: bool = is_instance_valid(match_scene.hud)
var start_position := Vector3.ZERO
if my_slot_ok:
start_position = my_slot.ship.visual.global_position
print("SMOKE INFO: client slots_ok=%s ball_ok=%s my_slot_ok=%s camera_ok=%s hud_ok=%s start_pos=%s" % [
str(slots_ok), str(ball_ok), str(my_slot_ok), str(camera_ok), str(hud_ok), str(start_position)
])
if not (slots_ok and ball_ok and my_slot_ok and camera_ok and hud_ok):
print("SMOKE FAIL: spawn/wiring check failed")
get_tree().quit(1)
return
# Drive forward thrust (a real, held key state — exercises the actual
# client input path, not a synthetic RPC call) and confirm the ship
# the CLIENT renders (its interpolated $Visual, not a raw snapshot
# value) actually moved — proving input reached the server, the server
# applied real thruster force, broadcast it back, and the client's
# interpolator produced smooth motion from it.
Input.action_press("move_forward")
await get_tree().create_timer(drive_seconds * 0.5).timeout
# Task 2.6: the server-computed thrust_z it broadcast in the snapshot
# should have reached this client's interpolator and be readable off
# the latest sample — this is what set_visual_action's engine-flame
# wiring actually reads, so it's the real thing to check, not just
# "the ship physically moved" (which 2.6 doesn't claim on its own).
var latest_state = my_slot.interpolator.latest()
var thrust_z_ok: bool = latest_state != null and latest_state.thrust_z > 0.5
print("SMOKE INFO: mid-drive thrust_z=%.2f (expect >0.5 while holding forward)" % (latest_state.thrust_z if latest_state != null else -1.0))
await get_tree().create_timer(drive_seconds * 0.5).timeout
Input.action_release("move_forward")
var end_position: Vector3 = my_slot.ship.visual.global_position
var moved := start_position.distance_to(end_position)
# Horizontal-only (XZ), not full 3D distance: an adversarial review
# found a 1.2s window of completely dead input still registers ~1.07m
# of pure gravity settling on the Y axis alone (spawn height dropping
# to the floor), which sat ABOVE the old moved > 1.0 bar — only
# thrust_z_ok caught that failure, not moved. Forward thrust is a
# horizontal force (see ship.gd), so measuring XZ displacement can't
# be satisfied by gravity alone, regardless of spawn height or timing.
var moved_horizontal := Vector2(end_position.x, end_position.z).distance_to(Vector2(start_position.x, start_position.z))
print("SMOKE INFO: client ship moved %.2fm (%.2fm horizontal) (start=%s end=%s) while holding forward thrust for %.1fs" % [
moved, moved_horizontal, str(start_position), str(end_position), drive_seconds
])
# thrust_power 150 / mass 5 = 30 m/s^2 nominal acceleration (see ship.gd) —
# over 2s even with drag/ramp-up this should clear a couple of metres.
# A generous, not-tuned-to-the-decimal bound: this is a wiring smoke
# test, not a physics-accuracy test (net_codec's own tests already cover
# quantisation precision).
var success := moved_horizontal > 1.0 and thrust_z_ok
print("SMOKE %s: client observed %.2fm horizontal of server-authoritative movement via interpolation, thrust_z_ok=%s" % [
"PASS" if success else "FAIL", moved_horizontal, str(thrust_z_ok)
])
await get_tree().create_timer(0.3).timeout
NetworkManager.shutdown()
get_tree().quit(0 if success else 1)
# task 3.4: MatchSim._recv_input must count malformed packets and disconnect
# after MALFORMED_LIMIT_TO_DISCONNECT (20) of them. Calls the RPC directly
# with garbage bytes rather than going through networked_match.gd's own
# honest encoder — this IS what a hostile custom client sending raw ENet
# packets would look like, so bypassing the normal send path is the point,
# not a shortcut.
func run_malformed_abuse_check() -> void:
await get_tree().create_timer(1.0).timeout
# A single-element Array, not a plain bool: GDScript lambdas capture
# outer local variables BY VALUE at creation time, not by reference, so
# `disconnected = true` inside the lambda below would silently mutate
# only the lambda's own captured copy — invisible to this function's
# own `disconnected` if it were a plain bool. Mutating an Array's
# CONTENTS from inside the lambda works because the Array object
# itself (not a copy of it) is what got captured.
var disconnected := [false]
NetworkManager.disconnected_from_server.connect(func() -> void: disconnected[0] = true)
for i in 25:
MatchSim._recv_input.rpc_id(1, PackedByteArray([1, 2, 3])) # far too short to even hold a header
NetworkManager.poll()
await get_tree().physics_frame
await get_tree().create_timer(1.0).timeout
NetworkManager.poll()
print("SMOKE %s: 25 malformed packets %s" % [
"PASS" if disconnected[0] else "FAIL",
"resulted in disconnect" if disconnected[0] else "did NOT disconnect the abusive peer",
])
get_tree().quit(0 if disconnected[0] else 1)
# task 3.4: MatchSim._recv_input must rate-limit and disconnect a sustained
# continuous flood well above RATE_LIMIT_PACKETS_PER_SEC (110/s) via the
# leaky-bucket excess accumulator (RATE_LIMIT_EXCESS_PACKETS_TO_DISCONNECT).
# Every packet here is individually well-formed (a real NetCodec.pack_input
# payload) — only the SEND RATE is abusive, confirming the rate limiter
# fires independently of the malformed-packet counter, not as a side
# effect of it.
func run_rate_limit_abuse_check() -> void:
await get_tree().create_timer(1.0).timeout
var disconnected := [false] # see run_malformed_abuse_check's comment on why not a plain bool
NetworkManager.disconnected_from_server.connect(func() -> void: disconnected[0] = true)
var net_codec := preload("res://scripts/net_codec.gd")
var ship_action_script := preload("res://scripts/ship_action.gd")
var bytes: PackedByteArray = net_codec.pack_input(1, 0, Time.get_ticks_msec(), [ship_action_script.new()])
var deadline_ms := Time.get_ticks_msec() + 4000
while Time.get_ticks_msec() < deadline_ms and not disconnected[0]:
for i in 40: # well above 110/s once summed across a frame's worth of iterations
MatchSim._recv_input.rpc_id(1, bytes)
NetworkManager.poll()
await get_tree().process_frame
await get_tree().create_timer(0.5).timeout
NetworkManager.poll()
print("SMOKE %s: sustained packet flood %s" % [
"PASS" if disconnected[0] else "FAIL",
"resulted in disconnect" if disconnected[0] else "did NOT disconnect the abusive peer",
])
get_tree().quit(0 if disconnected[0] else 1)
# Regression test for a real bug an adversarial review found and this
# session fixed: the ORIGINAL rate limiter tracked "N consecutive
# over-budget seconds" and hard-reset that streak to 0 on any single clean
# window — so a burst-then-idle duty cycle (flood hard, go quiet for one
# window, repeat) evaded it indefinitely. Reproduced against the real
# MatchSim._recv_input: ~33x the packet budget sustained for 28.5s with
# zero disconnect warnings. The fix (a leaky-bucket excess accumulator
# that grows by the window's actual total and drains by only one window's
# worth of budget, every window) doesn't care how the excess is
# distributed in time. This test reproduces the exact attack shape.
func run_duty_cycle_flood_abuse_check() -> void:
await get_tree().create_timer(1.0).timeout
var disconnected := [false]
NetworkManager.disconnected_from_server.connect(func() -> void: disconnected[0] = true)
var net_codec := preload("res://scripts/net_codec.gd")
var ship_action_script := preload("res://scripts/ship_action.gd")
var bytes: PackedByteArray = net_codec.pack_input(1, 0, Time.get_ticks_msec(), [ship_action_script.new()])
const CYCLE_SECONDS := 3.0
const BURST_SECONDS := 0.35
const TEST_SECONDS := 6.0 # the leaky bucket trips within the first cycle; no need for a long soak
const TRICKLE_HZ := 60 # legitimate-shaped background rate, well under budget alone
var deadline_ms := Time.get_ticks_msec() + int(TEST_SECONDS * 1000.0)
var cycle_start_ms := Time.get_ticks_msec()
while Time.get_ticks_msec() < deadline_ms and not disconnected[0]:
var t_in_cycle := float(Time.get_ticks_msec() - cycle_start_ms) / 1000.0
if t_in_cycle >= CYCLE_SECONDS:
cycle_start_ms = Time.get_ticks_msec()
t_in_cycle = 0.0
if t_in_cycle < BURST_SECONDS:
for i in 200: # a hard burst, far above budget
MatchSim._recv_input.rpc_id(1, bytes)
else:
for i in maxi(1, TRICKLE_HZ / 60): # ~60/s trickle, keeps the window rolling and stays under budget alone
MatchSim._recv_input.rpc_id(1, bytes)
NetworkManager.poll()
await get_tree().process_frame
await get_tree().create_timer(0.5).timeout
NetworkManager.poll()
print("SMOKE %s: duty-cycled flood (burst %.2fs / cycle %.1fs) %s" % [
"PASS" if disconnected[0] else "FAIL", BURST_SECONDS, CYCLE_SECONDS,
"resulted in disconnect" if disconnected[0] else "evaded rate limiting entirely",
])
get_tree().quit(0 if disconnected[0] else 1)
# task 3.6, host role: waits for both bots' scenes to settle, forces a
# deterministic goal (bot-vs-bot scoring isn't reliable enough within a
# short CI run to gate on), then compares the server's own final score
# against what each client independently wrote to disk (run_ci_client_check
# below) — genuine cross-peer agreement, not just "the server thinks so".
func run_ci_host_check(run_seconds: float) -> void:
await get_tree().create_timer(2.0).timeout
var match_scene := get_tree().current_scene
if not _is_networked_match(match_scene):
print("SMOKE FAIL: host scene is not NetworkedMatch")
NetworkManager.shutdown()
get_tree().quit(1)
return
print("SMOKE INFO: host ship_count=%d slot_count=%d" % [match_scene.ships.size(), match_scene._slots.size()])
# An adversarial review found this driver's original checks (snapshot
# count, a server-FORCED goal's cross-peer score agreement) don't
# depend on client input ever reaching the server at all — it kept
# reporting PASS with the input pipeline completely dead (verified by
# injecting the ring-overflow bug this session's critical fix
# addresses, mid-run). Record each ship's starting position now, before
# anything moves, so real server-side movement over the run can be
# checked directly — the same signal run_client_check already uses for
# a human client, applied here per-bot instead of just for "my own ship".
var start_positions: Dictionary = {}
for slot in match_scene._slots:
if is_instance_valid(slot.ship):
start_positions[slot.peer_id] = slot.ship.global_position
var goals: Array = match_scene.arena.get_goals() if match_scene.arena else []
if is_instance_valid(match_scene.ball) and not goals.is_empty():
match_scene.ball.linear_velocity = Vector3.ZERO
match_scene.ball.global_position = goals[0].global_position
print("SMOKE INFO: host forced a goal for the cross-peer score agreement check")
# Movement/stalled must be checked WHILE clients are still actively
# connected and playing, not after their run finishes — a client's own
# (legitimate, expected) disconnect at the end of its run naturally
# starves its jitter buffer too, which looks identical to the ring-
# overflow bug this check exists to catch if sampled too late. A first
# attempt used a 0.5s margin (run_seconds - 0.5); a second adversarial
# review instrumented multiplayer.get_peers() at sample time and found
# it was already EMPTY — both bots had legitimately disconnected before
# the sample ran, and the check was only passing on the ~200ms of
# residual STARVE_ZERO_TICKS starvation grace, not because it was
# genuinely still connected as this print used to claim. Widen the
# margin AND assert connectivity directly at sample time, rather than
# inferring it from timing, so a future regression in either direction
# (margin too tight again, or client run_seconds changing) fails loudly
# here instead of silently passing on residual grace.
var movement_check_delay := maxf(1.0, run_seconds - 2.0)
await get_tree().create_timer(movement_check_delay).timeout
var connected_peers := multiplayer.get_peers()
var input_reached_server := true
for slot in match_scene._slots:
var still_connected: bool = slot.peer_id in connected_peers
if not still_connected:
input_reached_server = false
print("SMOKE FAIL: peer %d already disconnected at movement-sample time (connected_peers=%s) — margin too tight" % [slot.peer_id, str(connected_peers)])
if not is_instance_valid(slot.ship) or not start_positions.has(slot.peer_id):
input_reached_server = false
print("SMOKE FAIL: peer %d has no valid ship to check movement on" % slot.peer_id)
continue
var moved: float = start_positions[slot.peer_id].distance_to(slot.ship.global_position)
var stalled: bool = slot.jitter_buffer.stalled
print("SMOKE INFO: peer %d moved %.2fm server-side (connected=%s), stalled=%s" % [slot.peer_id, moved, str(still_connected), str(stalled)])
if moved <= 0.5 or stalled:
input_reached_server = false
# Extra buffer beyond run_seconds: clients run for their own run_seconds
# measured from THEIR (later) start, so waiting only run_seconds here
# would race their score files not being written yet.
await get_tree().create_timer(run_seconds + 5.0 - movement_check_delay).timeout
print("SMOKE INFO: host final score=%s" % str(match_scene.score))
var slots_ok: bool = match_scene._slots.size() == 2
var scores_agree := true
var scores_seen := 0
for slot in match_scene._slots:
var path := "/tmp/cosmicclash_ci_score_%d.txt" % slot.peer_id
if not FileAccess.file_exists(path):
print("SMOKE FAIL: no score file from peer %d at %s" % [slot.peer_id, path])
scores_agree = false
continue
var f := FileAccess.open(path, FileAccess.READ)
var client_score := f.get_as_text()
f.close()
scores_seen += 1
var expected := JSON.stringify(match_scene.score)
if client_score != expected:
print("SMOKE FAIL: peer %d saw score %s, server has %s" % [slot.peer_id, client_score, expected])
scores_agree = false
var success: bool = slots_ok and scores_agree and scores_seen == 2 and input_reached_server
print("SMOKE %s: CI host run (slots_ok=%s scores_agree=%s scores_seen=%d/2 input_reached_server=%s)" % [
"PASS" if success else "FAIL", str(slots_ok), str(scores_agree), scores_seen, str(input_reached_server),
])
NetworkManager.shutdown()
get_tree().quit(0 if success else 1)
# task 3.6, client-bot role: counts real snapshots received over run_seconds
# (proving steady traffic, not just a handshake) and writes this peer's own
# final server-authoritative score to a peer-id-keyed file for the host to
# compare against the other bot's (run_ci_host_check above).
func run_ci_client_check(run_seconds: float) -> void:
var snapshot_count := [0]
MatchSim.snapshot_received.connect(func(_decoded: Dictionary) -> void: snapshot_count[0] += 1)
await get_tree().create_timer(1.0).timeout
var match_scene := get_tree().current_scene
if not _is_networked_match(match_scene):
print("SMOKE FAIL: client-bot scene is not NetworkedMatch")
get_tree().quit(1)
return
await get_tree().create_timer(run_seconds).timeout
var slots_ok: bool = not match_scene._slots.is_empty()
# 60Hz nominal; generous margin for connection/scene-load settle time
# eaten out of run_seconds and for the odd dropped/simulated-lossy tick.
var min_expected := int((run_seconds - 2.0) * 30.0)
var snapshot_count_ok: bool = snapshot_count[0] >= min_expected
var my_id := multiplayer.get_unique_id()
var score_path := "/tmp/cosmicclash_ci_score_%d.txt" % my_id
var f := FileAccess.open(score_path, FileAccess.WRITE)
f.store_string(JSON.stringify(match_scene.score))
f.close()
print("SMOKE INFO: client-bot snapshot_count=%d (want >= %d) slots_ok=%s final_score=%s" % [
snapshot_count[0], min_expected, str(slots_ok), str(match_scene.score),
])
var success: bool = slots_ok and snapshot_count_ok
print("SMOKE %s: CI client-bot run" % ("PASS" if success else "FAIL"))
await get_tree().create_timer(0.3).timeout
NetworkManager.shutdown()
get_tree().quit(0 if success else 1)