fix(multiplayer): server no longer rate-limits a backlog it caused itself

Closes task 5.10's three recording gaps, and the gap-closing found a real
input-loss bug.

Replay log: a failed write now ends the log permanently instead of
desyncing every later record's framing; close() is called from _exit_tree
with a summary, since the RefCounted destructor closes it implicitly but
never says whether the log is complete; rejected packets are recorded
with their reason in the kind byte (framing unchanged, FORMAT_VERSION 2
so "no rejects" differs from "this build never recorded them"). Recording
is capped at 8 per peer per window - uncapped, the diagnostic is a remote
disk-fill amplifier, since the attacker picks the packet rate. Uncapped
totals live on MatchSim and survive the peer's disconnect.

The bug: a 2s host stall has the client sending at 60Hz throughout, and
ENet delivers that whole backlog in the first window after resume - 70 of
an honest client's packets rejected as "rate limit exceeded". Redundancy
does not cover it, because the dropped packets are contiguous: 0 of 70
rescued, and 82 of 923 sequences (8.88%, ~1.4s of input) never reached
the server, against 0.00% with no stall. Every prediction gate passed.

Fixed by granting each already-tracked peer a capped, two-window packet
grace when the server detects its own wall-clock stall. Rate-limit
rejects 70 -> 0, sequences missing 8.88% -> 0.00%, seq-guard rejects
9 -> 0. Controls on the unfixed build lost 4.34/7.52/7.86%. All three
abuse roles still disconnect and no flood induced a stall, so the grace
cannot be farmed.

Also corrects an earlier wrong conclusion: the reviewer's free-flight
p95 0.688 is real and reproduces on two processes with 0.0% snapshot
loss. The plain --role=client drive fails the 0.5 free-flight bound in
3 of 8 runs because that drive is mostly a contact test - the harness
comment already said so - leaving a cohort as small as 12 samples.
Near-surface error is genuinely several times open-air error, so the
calibrated bound now belongs to --exercise-free-flight alone and the
plain role asserts the always-well-sampled all-cohort percentiles at
1.2/2.0, printing the free-flight numbers as reported-not-asserted.
6/6 plain runs pass where 3/7 failed; tightening to 0.3 still fails.

tools/replay_dump.gd reads a log back: counts by kind, plus how much of
the input sequence stream reached the server once redundancy is counted.
This commit is contained in:
Josh Creek
2026-08-21 16:10:43 +01:00
parent e51dc765a2
commit 866efa0d9b
7 changed files with 469 additions and 12 deletions
+70
View File
@@ -108,6 +108,76 @@ func test_a_truncated_log_yields_its_intact_records() -> void:
DirAccess.remove_absolute(ProjectSettings.globalize_path(path))
func test_rejected_packets_are_recorded_and_distinguishable_by_reason() -> void:
# The log exists to answer "my input did nothing", and the packets that
# explain that are exactly the ones the server threw away. Recording them
# is only useful if the reason survives too, so a reader can tell a
# malformed packet from one the rate limiter dropped.
var path := _temp_path("rejects")
var log_writer = ReplayLogScript.new()
log_writer.open_for_write(path)
var malformed := PackedByteArray([0x01, 0x02])
var flooded := PackedByteArray([0x09, 0x08, 0x07])
var far_future := PackedByteArray([0x11, 0x22, 0x33, 0x44])
log_writer.record_rejected_input(ReplayLogScript.RecordKind.REJECTED_MALFORMED, 10, 5, malformed)
log_writer.record_rejected_input(ReplayLogScript.RecordKind.REJECTED_RATE_LIMIT, 11, 5, flooded)
log_writer.record_rejected_input(ReplayLogScript.RecordKind.REJECTED_SEQ_GUARD, 12, 6, far_future)
log_writer.record_input(13, 5, PackedByteArray([0xAA]))
log_writer.close()
var records: Array = ReplayLogScript.read_all(path).get("records", [])
assert_eq(records.size(), 4, "rejects and accepted input share one ordered stream")
assert_eq(records[0]["kind"], ReplayLogScript.RecordKind.REJECTED_MALFORMED, "malformed reason survives")
assert_eq(records[0]["payload"], malformed, "and so do the bytes that caused it")
assert_eq(records[1]["kind"], ReplayLogScript.RecordKind.REJECTED_RATE_LIMIT, "rate-limit reason survives")
assert_eq(records[1]["payload"], flooded, "with its own payload")
assert_eq(records[2]["kind"], ReplayLogScript.RecordKind.REJECTED_SEQ_GUARD, "seq-guard reason survives")
assert_eq(records[2]["peer_id"], 6, "attributed to the peer that sent it")
assert_eq(records[3]["kind"], ReplayLogScript.RecordKind.INPUT, "an accepted packet is still its own kind")
DirAccess.remove_absolute(ProjectSettings.globalize_path(path))
func test_a_failed_write_stops_the_log_instead_of_corrupting_it() -> void:
# A half-written record desyncs the framing of everything after it, turning
# "the disk filled up" into "the file is garbage". Forcing a real ENOSPC is
# out of scope for a unit test, so the failure is injected directly — this
# covers the guard and the accounting, not the detection, which is a
# get_error() check on the real FileAccess.
var path := _temp_path("writefail")
var log_writer = ReplayLogScript.new()
log_writer.open_for_write(path)
log_writer.record_input(1, 1, PackedByteArray([1, 2, 3, 4]))
log_writer.write_failed = true
log_writer.record_input(2, 1, PackedByteArray([5, 6, 7, 8]))
log_writer.record_snapshot(3, PackedByteArray([9]))
assert_eq(log_writer.records_written, 1, "nothing is written after a failure")
log_writer.close()
var records: Array = ReplayLogScript.read_all(path).get("records", [])
assert_eq(records.size(), 1, "what was written before the failure is still readable")
assert_eq(records[0]["payload"], PackedByteArray([1, 2, 3, 4]), "and intact")
DirAccess.remove_absolute(ProjectSettings.globalize_path(path))
func test_an_oversized_payload_is_dropped_without_breaking_later_records() -> void:
# `length` is a u16. Truncating an over-64KB payload to fit would leave the
# reader parsing the payload's own tail as the next record header.
var path := _temp_path("oversized")
var log_writer = ReplayLogScript.new()
log_writer.open_for_write(path)
var oversized := PackedByteArray()
oversized.resize(0x10000)
log_writer.record_input(1, 1, oversized)
assert_eq(log_writer.records_dropped, 1, "the oversized record is counted as dropped")
log_writer.record_input(2, 1, PackedByteArray([7, 7]))
log_writer.close()
var records: Array = ReplayLogScript.read_all(path).get("records", [])
assert_eq(records.size(), 1, "only the well-sized record is present")
assert_eq(records[0]["tick"], 2, "and it is the one that came after the drop, correctly framed")
DirAccess.remove_absolute(ProjectSettings.globalize_path(path))
func test_writing_to_an_unopened_log_is_a_no_op() -> void:
# --replay-log is optional, so every record_* call happens behind a null
# check in production — but the class must not corrupt or crash if that
+48
View File
@@ -353,6 +353,42 @@ func run_client_check(settle_seconds: float, drive_seconds: float, exercise_ball
# predictions for an attack's filled gap sequences.
const MAX_ACTION_MARKER_MISMATCH_RATE := 0.05
var action_label_ok: bool = marker_samples_ok and not server_stalled and marker_rate < MAX_ACTION_MARKER_MISMATCH_RATE
# The 0.5/2.0 free-flight bounds belong to --exercise-free-flight and ONLY
# to it, because that mode is the only one that produces the profile they
# were calibrated on. _run_free_flight_trace exists precisely because, in
# its own words, "a straight forward trace reaches the goal/wall in seconds
# and turns the supposed free-flight QA run into a contact test" — yet the
# plain role went on asserting the open-volume bounds against whatever
# free-flight samples that contact-heavy drive happened to leave behind.
#
# Measured over 8 plain-role runs on an idle machine: the free-flight
# cohort ranged from 12 to 257 samples and its p95 from 0.275 to 0.726,
# failing the 0.5 bound in 3 of 8 — a ~37% flake rate with no defect
# present. That is the p95 0.688 an adversarial review reported and I first
# mis-attributed to three-process CPU contention: it reproduces on two
# processes, on an idle box, with 0.0% snapshot loss. The mechanism is not
# noise — error near the arena's surface-pull field is genuinely several
# times higher than in open air (--exercise-free-flight measures 0.084-0.111
# on the same build) — but a gate that fires a third of the time is worse
# than no gate, and calibrating one bound for both profiles cannot work.
#
# So the plain role asserts the ALL-COHORT percentiles instead. They are
# always well-sampled (545-696 samples across those same runs, versus a
# free-flight cohort that can collapse to 12) and much tighter in spread:
# raw_p95 0.354-0.609, raw_p99 0.362-0.742. The bounds below sit ~2x above
# the worst observed. A free-flight-cohort regression still cannot hide:
# free_flight_hard_snaps is asserted in both modes, and anything past 2.0m
# IS a hard snap by definition.
#
# The 100-sample floor is not the plain drive's number (545-696) but
# --exercise-match-state's: its forced goal suspends prediction for the
# whole GOAL_PAUSE, so an 8s run yields ~153. Still five times what the
# old free-flight floor accepted.
const NEAR_SURFACE_P95 := 1.2
const NEAR_SURFACE_P99 := 2.0
var overall_p95: float = float(prediction_stats.get("position_error_p95", INF))
var overall_p99: float = float(prediction_stats.get("position_error_p99", INF))
var overall_samples := int(prediction_stats.get("sample_count", 0))
var prediction_quality_ok: bool = action_label_ok if exercise_input_transitions else \
prediction_stats.get("hard_snap_count", 99) < 4 if exercise_ball_contact else \
quality_samples >= 30 \
@@ -362,7 +398,19 @@ func run_client_check(settle_seconds: float, drive_seconds: float, exercise_ball
and raw_rotation_p99 < 15.0 \
and quality_p95 < 0.5 \
and quality_p99 < 2.0 \
and free_flight_hard_snaps == 0 \
if exercise_free_flight else \
overall_samples >= 100 \
and overall_p95 < NEAR_SURFACE_P95 \
and overall_p99 < NEAR_SURFACE_P99 \
and raw_rotation_p95 < 5.0 \
and raw_rotation_p99 < 15.0 \
and free_flight_hard_snaps == 0
if not exercise_free_flight and not exercise_input_transitions and not exercise_ball_contact:
print("SMOKE INFO: near-surface profile — asserting all-cohort p95=%.3f/p99=%.3f (bounds %.1f/%.1f, %d samples); free-flight cohort p95=%.3f/p99=%.3f over %d samples is REPORTED, NOT ASSERTED (see --exercise-free-flight for the calibrated gate)" % [
overall_p95, overall_p99, NEAR_SURFACE_P95, NEAR_SURFACE_P99, overall_samples,
raw_quality_p95, raw_quality_p99, quality_samples,
])
# ball_proxy_moved_before_authority counts ticks where the predicted proxy
# had visibly moved BEFORE the next authoritative ball state arrived. That
# is only a meaningful — or even achievable — claim when there is real RTT