Files
CosmicClash/Game/tests/networked_match_test_hooks.gd
T
Josh Creek 2325313ad2 fix(multiplayer): adversarial review fixes for Phase 3
An Opus subagent's adversarial review of Phase 3 found a critical, silent,
permanent bug plus eight smaller real issues, all empirically verified
with real two- and three-process runs:

CRITICAL: InputJitterBuffer's 32-entry ring permanently bricked a
player's input once the un-consumed backlog exceeded the ring's
capacity - a fresh arrival would land in the exact slot consume() was
still waiting on, and since both counters only ever advance, the gap
never closed. Reproduced with a real SIGSTOP/SIGCONT host freeze:
client movement dropped from ~26m to 0.00m at ~0.7s, worse under real
loss (a lossy link lowered the fatal threshold to ~400ms), and
reachable via ordinary clock drift with no external trigger at all.
Fixed by tracking the highest seq ever ingested and having consume()
jump directly to what the ring can still provide once the gap exceeds
capacity, instead of starving through an unrecoverable span. Re-verified
with a 3s freeze (well past the original threshold): full recovery.

HIGH: InputLeadController's release logic was gated on its own past
attacks (lead > LEAD_MIN) rather than the real server-reported depth, so
a backlog it didn't itself cause was never drained. Fixed to gate on
actual depth vs target.

MEDIUM-HIGH: the rate limiter's "N consecutive over-budget seconds"
streak hard-reset to 0 on any clean window, letting a duty-cycled flood
(burst, one clean window, repeat) sustain ~33x budget indefinitely with
zero warnings. Replaced with a leaky-bucket accumulator immune to the
same evasion by construction.

MEDIUM: the seq > server_tick + 20 guard compared two unrelated clock
epochs (server process uptime vs. client's own from-zero seq numbering),
so it never actually protected anything on a long-running server and
could silently drop an honest client's input forever. Bound against the
buffer's own last_applied_seq instead.

MEDIUM: InputJitterBuffer.stalled was computed but never reached the
wire - the one signal that would have made the ring-overflow bug visible
anywhere. Now wired through _ship_to_net_body_state.

MEDIUM: task 3.6's CI driver's assertions didn't depend on client input
reaching the server at all, so it kept passing with the ring-overflow
bug actively triggered. Added real ship-movement and non-stalled checks,
sampled while bots are still connected (an initial attempt sampled after
their own legitimate disconnect, which starves identically to the bug).

LOW-MEDIUM: a lead change silently mislabelled _input_history's older
entries, since the wire format has no per-entry seq field. Fixed by
handling each delta case (ordinary/release/attack) on its own terms.

LOW: bandwidth and snapshot-loss overlay metrics froze at their last
value during a total outage instead of decaying - exactly when they
matter most. Both now report honest post-outage values.

LOW: a guard comment on NetworkManager._ping misdescribed the actual
disconnect_peer() arguments in use. Corrected.

New permanent regression tests: test_ring_overflow_resyncs_to_fresh_data
_instead_of_starving_forever, test_release_drains_a_backlog_it_never_
caused_itself, and client-abuse-flood-dutycycle (reproduces the exact
duty-cycle evasion). Full regression suite, including the net-sim-latency
milestone gate, all abuse roles, and the CI driver, re-run clean after
every fix.
2026-08-20 15:28:44 +01:00

357 lines
17 KiB
GDScript

extends Node
# Test-only helper (tests/networked_match_smoke.gd). Not a project autoload
# — production code never references this. Same reason as
# tests/lobby_test_hooks.gd: networked_match.tscn is loaded via
# change_scene_to_file(), which frees whatever node initiated the load, so
# a driver can't keep orchestrating from a node that just got freed. The
# smoke test add_child()s this directly under get_tree().root instead (a
# sibling of current_scene, not a descendant of it), so it survives the swap.
#
# Uses preload(), not the bare `NetworkedMatch` class_name, and leaves
# `match_scene` itself untyped (Node) throughout — same global-script-class-
# cache-timing reason as tests/test_case.gd, plus every member access off an
# untyped Node returns Variant, which then needs explicit `: Type`
# annotations wherever `:=` would otherwise fail to infer one.
const NetworkedMatchScript = preload("res://scripts/networked_match.gd")
func _is_networked_match(node: Node) -> bool:
return node != null and node.get_script() == NetworkedMatchScript
func run_host_check(lifetime_seconds: float) -> void:
await get_tree().create_timer(lifetime_seconds * 0.4).timeout
var match_scene := get_tree().current_scene
var ok := _is_networked_match(match_scene)
var ship_count := 0
var ball_ok := false
var arena_name := "null"
if ok:
ship_count = match_scene.ships.size()
ball_ok = is_instance_valid(match_scene.ball)
if match_scene.arena:
arena_name = match_scene.arena.name
print("SMOKE INFO: host is_networked_match=%s ship_count=%d ball_ok=%s arena=%s" % [
str(ok), ship_count, str(ball_ok), arena_name
])
var success := ok and ship_count == 1 and ball_ok
print("SMOKE %s: host spawn check (ship_count=%d, ball_ok=%s)" % ["PASS" if success else "FAIL", ship_count, str(ball_ok)])
await get_tree().create_timer(lifetime_seconds * 0.6).timeout
if _is_networked_match(match_scene) and not match_scene.ships.is_empty():
var ship: Ship = match_scene.ships[0]
print("SMOKE INFO: host ship final position=%s (spawned, driven by client input if any arrived)" % str(ship.global_position))
NetworkManager.shutdown()
get_tree().quit(0 if success else 1)
func run_client_check(settle_seconds: float, drive_seconds: float) -> void:
await get_tree().create_timer(settle_seconds).timeout
var match_scene := get_tree().current_scene
if not _is_networked_match(match_scene):
print("SMOKE FAIL: current_scene is not NetworkedMatch after %.1fs" % settle_seconds)
get_tree().quit(1)
return
var slots_ok: bool = match_scene._slots.size() == 1
var ball_ok: bool = is_instance_valid(match_scene.ball)
var my_slot = match_scene._my_slot
var my_slot_ok: bool = my_slot != null and is_instance_valid(my_slot.ship)
var camera_ok: bool = is_instance_valid(match_scene._camera_rig)
var hud_ok: bool = is_instance_valid(match_scene.hud)
var start_position := Vector3.ZERO
if my_slot_ok:
start_position = my_slot.ship.visual.global_position
print("SMOKE INFO: client slots_ok=%s ball_ok=%s my_slot_ok=%s camera_ok=%s hud_ok=%s start_pos=%s" % [
str(slots_ok), str(ball_ok), str(my_slot_ok), str(camera_ok), str(hud_ok), str(start_position)
])
if not (slots_ok and ball_ok and my_slot_ok and camera_ok and hud_ok):
print("SMOKE FAIL: spawn/wiring check failed")
get_tree().quit(1)
return
# Drive forward thrust (a real, held key state — exercises the actual
# client input path, not a synthetic RPC call) and confirm the ship
# the CLIENT renders (its interpolated $Visual, not a raw snapshot
# value) actually moved — proving input reached the server, the server
# applied real thruster force, broadcast it back, and the client's
# interpolator produced smooth motion from it.
Input.action_press("move_forward")
await get_tree().create_timer(drive_seconds * 0.5).timeout
# Task 2.6: the server-computed thrust_z it broadcast in the snapshot
# should have reached this client's interpolator and be readable off
# the latest sample — this is what set_visual_action's engine-flame
# wiring actually reads, so it's the real thing to check, not just
# "the ship physically moved" (which 2.6 doesn't claim on its own).
var latest_state = my_slot.interpolator.latest()
var thrust_z_ok: bool = latest_state != null and latest_state.thrust_z > 0.5
print("SMOKE INFO: mid-drive thrust_z=%.2f (expect >0.5 while holding forward)" % (latest_state.thrust_z if latest_state != null else -1.0))
await get_tree().create_timer(drive_seconds * 0.5).timeout
Input.action_release("move_forward")
var end_position: Vector3 = my_slot.ship.visual.global_position
var moved := start_position.distance_to(end_position)
print("SMOKE INFO: client ship moved %.2fm (start=%s end=%s) while holding forward thrust for %.1fs" % [
moved, str(start_position), str(end_position), drive_seconds
])
# thrust_power 150 / mass 5 = 30 m/s^2 nominal acceleration (see ship.gd) —
# over 2s even with drag/ramp-up this should clear a couple of metres.
# A generous, not-tuned-to-the-decimal bound: this is a wiring smoke
# test, not a physics-accuracy test (net_codec's own tests already cover
# quantisation precision).
var success := moved > 1.0 and thrust_z_ok
print("SMOKE %s: client observed %.2fm of server-authoritative movement via interpolation, thrust_z_ok=%s" % [
"PASS" if success else "FAIL", moved, str(thrust_z_ok)
])
await get_tree().create_timer(0.3).timeout
NetworkManager.shutdown()
get_tree().quit(0 if success else 1)
# task 3.4: MatchSim._recv_input must count malformed packets and disconnect
# after MALFORMED_LIMIT_TO_DISCONNECT (20) of them. Calls the RPC directly
# with garbage bytes rather than going through networked_match.gd's own
# honest encoder — this IS what a hostile custom client sending raw ENet
# packets would look like, so bypassing the normal send path is the point,
# not a shortcut.
func run_malformed_abuse_check() -> void:
await get_tree().create_timer(1.0).timeout
# A single-element Array, not a plain bool: GDScript lambdas capture
# outer local variables BY VALUE at creation time, not by reference, so
# `disconnected = true` inside the lambda below would silently mutate
# only the lambda's own captured copy — invisible to this function's
# own `disconnected` if it were a plain bool. Mutating an Array's
# CONTENTS from inside the lambda works because the Array object
# itself (not a copy of it) is what got captured.
var disconnected := [false]
NetworkManager.disconnected_from_server.connect(func() -> void: disconnected[0] = true)
for i in 25:
MatchSim._recv_input.rpc_id(1, PackedByteArray([1, 2, 3])) # far too short to even hold a header
NetworkManager.poll()
await get_tree().physics_frame
await get_tree().create_timer(1.0).timeout
NetworkManager.poll()
print("SMOKE %s: 25 malformed packets %s" % [
"PASS" if disconnected[0] else "FAIL",
"resulted in disconnect" if disconnected[0] else "did NOT disconnect the abusive peer",
])
get_tree().quit(0 if disconnected[0] else 1)
# task 3.4: MatchSim._recv_input must rate-limit and disconnect a sustained
# continuous flood well above RATE_LIMIT_PACKETS_PER_SEC (110/s) via the
# leaky-bucket excess accumulator (RATE_LIMIT_EXCESS_PACKETS_TO_DISCONNECT).
# Every packet here is individually well-formed (a real NetCodec.pack_input
# payload) — only the SEND RATE is abusive, confirming the rate limiter
# fires independently of the malformed-packet counter, not as a side
# effect of it.
func run_rate_limit_abuse_check() -> void:
await get_tree().create_timer(1.0).timeout
var disconnected := [false] # see run_malformed_abuse_check's comment on why not a plain bool
NetworkManager.disconnected_from_server.connect(func() -> void: disconnected[0] = true)
var net_codec := preload("res://scripts/net_codec.gd")
var ship_action_script := preload("res://scripts/ship_action.gd")
var bytes: PackedByteArray = net_codec.pack_input(1, 0, Time.get_ticks_msec(), [ship_action_script.new()])
var deadline_ms := Time.get_ticks_msec() + 4000
while Time.get_ticks_msec() < deadline_ms and not disconnected[0]:
for i in 40: # well above 110/s once summed across a frame's worth of iterations
MatchSim._recv_input.rpc_id(1, bytes)
NetworkManager.poll()
await get_tree().process_frame
await get_tree().create_timer(0.5).timeout
NetworkManager.poll()
print("SMOKE %s: sustained packet flood %s" % [
"PASS" if disconnected[0] else "FAIL",
"resulted in disconnect" if disconnected[0] else "did NOT disconnect the abusive peer",
])
get_tree().quit(0 if disconnected[0] else 1)
# Regression test for a real bug an adversarial review found and this
# session fixed: the ORIGINAL rate limiter tracked "N consecutive
# over-budget seconds" and hard-reset that streak to 0 on any single clean
# window — so a burst-then-idle duty cycle (flood hard, go quiet for one
# window, repeat) evaded it indefinitely. Reproduced against the real
# MatchSim._recv_input: ~33x the packet budget sustained for 28.5s with
# zero disconnect warnings. The fix (a leaky-bucket excess accumulator
# that grows by the window's actual total and drains by only one window's
# worth of budget, every window) doesn't care how the excess is
# distributed in time. This test reproduces the exact attack shape.
func run_duty_cycle_flood_abuse_check() -> void:
await get_tree().create_timer(1.0).timeout
var disconnected := [false]
NetworkManager.disconnected_from_server.connect(func() -> void: disconnected[0] = true)
var net_codec := preload("res://scripts/net_codec.gd")
var ship_action_script := preload("res://scripts/ship_action.gd")
var bytes: PackedByteArray = net_codec.pack_input(1, 0, Time.get_ticks_msec(), [ship_action_script.new()])
const CYCLE_SECONDS := 3.0
const BURST_SECONDS := 0.35
const TEST_SECONDS := 6.0 # the leaky bucket trips within the first cycle; no need for a long soak
const TRICKLE_HZ := 60 # legitimate-shaped background rate, well under budget alone
var deadline_ms := Time.get_ticks_msec() + int(TEST_SECONDS * 1000.0)
var cycle_start_ms := Time.get_ticks_msec()
while Time.get_ticks_msec() < deadline_ms and not disconnected[0]:
var t_in_cycle := float(Time.get_ticks_msec() - cycle_start_ms) / 1000.0
if t_in_cycle >= CYCLE_SECONDS:
cycle_start_ms = Time.get_ticks_msec()
t_in_cycle = 0.0
if t_in_cycle < BURST_SECONDS:
for i in 200: # a hard burst, far above budget
MatchSim._recv_input.rpc_id(1, bytes)
else:
for i in maxi(1, TRICKLE_HZ / 60): # ~60/s trickle, keeps the window rolling and stays under budget alone
MatchSim._recv_input.rpc_id(1, bytes)
NetworkManager.poll()
await get_tree().process_frame
await get_tree().create_timer(0.5).timeout
NetworkManager.poll()
print("SMOKE %s: duty-cycled flood (burst %.2fs / cycle %.1fs) %s" % [
"PASS" if disconnected[0] else "FAIL", BURST_SECONDS, CYCLE_SECONDS,
"resulted in disconnect" if disconnected[0] else "evaded rate limiting entirely",
])
get_tree().quit(0 if disconnected[0] else 1)
# task 3.6, host role: waits for both bots' scenes to settle, forces a
# deterministic goal (bot-vs-bot scoring isn't reliable enough within a
# short CI run to gate on), then compares the server's own final score
# against what each client independently wrote to disk (run_ci_client_check
# below) — genuine cross-peer agreement, not just "the server thinks so".
func run_ci_host_check(run_seconds: float) -> void:
await get_tree().create_timer(2.0).timeout
var match_scene := get_tree().current_scene
if not _is_networked_match(match_scene):
print("SMOKE FAIL: host scene is not NetworkedMatch")
NetworkManager.shutdown()
get_tree().quit(1)
return
print("SMOKE INFO: host ship_count=%d slot_count=%d" % [match_scene.ships.size(), match_scene._slots.size()])
# An adversarial review found this driver's original checks (snapshot
# count, a server-FORCED goal's cross-peer score agreement) don't
# depend on client input ever reaching the server at all — it kept
# reporting PASS with the input pipeline completely dead (verified by
# injecting the ring-overflow bug this session's critical fix
# addresses, mid-run). Record each ship's starting position now, before
# anything moves, so real server-side movement over the run can be
# checked directly — the same signal run_client_check already uses for
# a human client, applied here per-bot instead of just for "my own ship".
var start_positions: Dictionary = {}
for slot in match_scene._slots:
if is_instance_valid(slot.ship):
start_positions[slot.peer_id] = slot.ship.global_position
var goals: Array = match_scene.arena.get_goals() if match_scene.arena else []
if is_instance_valid(match_scene.ball) and not goals.is_empty():
match_scene.ball.linear_velocity = Vector3.ZERO
match_scene.ball.global_position = goals[0].global_position
print("SMOKE INFO: host forced a goal for the cross-peer score agreement check")
# Movement/stalled must be checked WHILE clients are still actively
# connected and playing, not after their run finishes — a client's own
# (legitimate, expected) disconnect at the end of its run naturally
# starves its jitter buffer too, which looks identical to the ring-
# overflow bug this check exists to catch if sampled too late. Clients
# start ~1.5s after the host and finish their own run_seconds shortly
# before disconnecting, so sample just ahead of that, not after.
var movement_check_delay := maxf(1.0, run_seconds - 0.5)
await get_tree().create_timer(movement_check_delay).timeout
var input_reached_server := true
for slot in match_scene._slots:
if not is_instance_valid(slot.ship) or not start_positions.has(slot.peer_id):
input_reached_server = false
print("SMOKE FAIL: peer %d has no valid ship to check movement on" % slot.peer_id)
continue
var moved: float = start_positions[slot.peer_id].distance_to(slot.ship.global_position)
var stalled: bool = slot.jitter_buffer.stalled
print("SMOKE INFO: peer %d moved %.2fm server-side (while still connected), stalled=%s" % [slot.peer_id, moved, str(stalled)])
if moved <= 0.5 or stalled:
input_reached_server = false
# Extra buffer beyond run_seconds: clients run for their own run_seconds
# measured from THEIR (later) start, so waiting only run_seconds here
# would race their score files not being written yet.
await get_tree().create_timer(run_seconds + 5.0 - movement_check_delay).timeout
print("SMOKE INFO: host final score=%s" % str(match_scene.score))
var slots_ok: bool = match_scene._slots.size() == 2
var scores_agree := true
var scores_seen := 0
for slot in match_scene._slots:
var path := "/tmp/cosmicclash_ci_score_%d.txt" % slot.peer_id
if not FileAccess.file_exists(path):
print("SMOKE FAIL: no score file from peer %d at %s" % [slot.peer_id, path])
scores_agree = false
continue
var f := FileAccess.open(path, FileAccess.READ)
var client_score := f.get_as_text()
f.close()
scores_seen += 1
var expected := JSON.stringify(match_scene.score)
if client_score != expected:
print("SMOKE FAIL: peer %d saw score %s, server has %s" % [slot.peer_id, client_score, expected])
scores_agree = false
var success: bool = slots_ok and scores_agree and scores_seen == 2 and input_reached_server
print("SMOKE %s: CI host run (slots_ok=%s scores_agree=%s scores_seen=%d/2 input_reached_server=%s)" % [
"PASS" if success else "FAIL", str(slots_ok), str(scores_agree), scores_seen, str(input_reached_server),
])
NetworkManager.shutdown()
get_tree().quit(0 if success else 1)
# task 3.6, client-bot role: counts real snapshots received over run_seconds
# (proving steady traffic, not just a handshake) and writes this peer's own
# final server-authoritative score to a peer-id-keyed file for the host to
# compare against the other bot's (run_ci_host_check above).
func run_ci_client_check(run_seconds: float) -> void:
var snapshot_count := [0]
MatchSim.snapshot_received.connect(func(_decoded: Dictionary) -> void: snapshot_count[0] += 1)
await get_tree().create_timer(1.0).timeout
var match_scene := get_tree().current_scene
if not _is_networked_match(match_scene):
print("SMOKE FAIL: client-bot scene is not NetworkedMatch")
get_tree().quit(1)
return
await get_tree().create_timer(run_seconds).timeout
var slots_ok: bool = not match_scene._slots.is_empty()
# 60Hz nominal; generous margin for connection/scene-load settle time
# eaten out of run_seconds and for the odd dropped/simulated-lossy tick.
var min_expected := int((run_seconds - 2.0) * 30.0)
var snapshot_count_ok: bool = snapshot_count[0] >= min_expected
var my_id := multiplayer.get_unique_id()
var score_path := "/tmp/cosmicclash_ci_score_%d.txt" % my_id
var f := FileAccess.open(score_path, FileAccess.WRITE)
f.store_string(JSON.stringify(match_scene.score))
f.close()
print("SMOKE INFO: client-bot snapshot_count=%d (want >= %d) slots_ok=%s final_score=%s" % [
snapshot_count[0], min_expected, str(slots_ok), str(match_scene.score),
])
var success: bool = slots_ok and snapshot_count_ok
print("SMOKE %s: CI client-bot run" % ("PASS" if success else "FAIL"))
await get_tree().create_timer(0.3).timeout
NetworkManager.shutdown()
get_tree().quit(0 if success else 1)