feat(training): add wall and rebound curriculum states

This commit is contained in:
Josh Creek
2026-09-01 17:37:52 +01:00
parent 7170400f49
commit e376675fa6
6 changed files with 68 additions and 11 deletions
+41 -4
View File
@@ -70,6 +70,11 @@ const SimConstants = preload("res://scripts/sim_constants.gd")
# a real goal and ships start low behind/lateral to it, so a useful touch is
# naturally reinforced by the existing goal-directed ball rewards.
@export_range(0.0, 1.0) var air_intercept_chance := 0.0
# Wall-play and rebound starts are separate: wall-play begins beside a wall
# with the ball travelling inward, while rebound begins just before an
# outward wall impact. Both default off to preserve existing distributions.
@export_range(0.0, 1.0) var wall_play_chance := 0.0
@export_range(0.0, 1.0) var rebound_chance := 0.0
# Ground-start branch for the generation-5 handling stage: ships spawn level
# and resting on the floor with a low, floor-level ball. Every other branch
# samples ship Y uniformly across the full 18m volume (see _random_position),
@@ -82,8 +87,8 @@ const SimConstants = preload("res://scripts/sim_constants.gd")
# Ships per team. Default 1 preserves every existing curriculum script's 1v1
# behaviour unchanged; up to 5 matches ShipObservations.MAX_TEAMMATES/
# MAX_OPPONENTS. Plumbing only for this pass — no 2v2+ curriculum/reward
# design has been done, so a run above 1 is untested territory.
# MAX_OPPONENTS. Team-credit reward and paired 2v2 evaluation are opt-in;
# no teamplay training stage is enabled by default.
@export_range(1, 5) var team_size: int = 1
# Placement bounds for randomized episode starts, derived from the standard
@@ -100,6 +105,9 @@ const FIELD_MIN_Y := 1.5
# spawning interpenetrated with it.
const GROUND_START_Y := 0.35
const GROUND_START_BALL_Y := 0.55
const WALL_PLAY_BALL_CLEARANCE := 1.0
const REBOUND_BALL_CLEARANCE := 0.75
const WALL_PLAY_SPEED := Vector2(4.0, 9.0)
const FIELD_MAX_Y := ArenaBoundary.INNER_HEIGHT - SPAWN_INSET
# The corner curves reach at most their chord plane |x| + |z| = INNER_HALF_X
# + INNER_HALF_Z - CORNER_RADIUS; spawns keep the same SPAWN_INSET clearance
@@ -264,6 +272,7 @@ const TRAINING_MODE_OVERRIDES := [
"goal_reward", "draw_penalty", "kickoff_state_chance",
"ball_near_goal_chance", "attack_goal_bias", "air_drill_chance",
"air_intercept_chance", "ground_start_chance", "team_size",
"wall_play_chance", "rebound_chance",
]
# ShipAIController @export names a curriculum run may override, read as
# --ai_<name>=<value> to avoid colliding with the names above.
@@ -292,8 +301,8 @@ func _parse_curriculum_args() -> void:
if args.has(name):
set(name, _typed_like(args[name], get(name)))
var start_probability := kickoff_state_chance + ball_near_goal_chance \
+ air_drill_chance + air_intercept_chance
start_probability += ground_start_chance
+ air_drill_chance + air_intercept_chance + ground_start_chance \
+ wall_play_chance + rebound_chance
if start_probability > 1.0:
push_error("TrainingMode: episode-start probabilities sum to %.3f (> 1.0)" % start_probability)
@@ -455,6 +464,12 @@ func _reset_episode() -> void:
elif roll < kickoff_state_chance + ball_near_goal_chance + air_drill_chance \
+ air_intercept_chance + ground_start_chance:
_place_ground_start()
elif roll < kickoff_state_chance + ball_near_goal_chance + air_drill_chance \
+ air_intercept_chance + ground_start_chance + wall_play_chance:
_place_wall_state(false)
elif roll < kickoff_state_chance + ball_near_goal_chance + air_drill_chance \
+ air_intercept_chance + ground_start_chance + wall_play_chance + rebound_chance:
_place_wall_state(true)
else:
_place_ships_random()
_place_ball_random()
@@ -564,6 +579,28 @@ func _place_ground_start() -> void:
)
# Wall-play/rebound states (see wall_play_chance/rebound_chance). The ball is
# placed against a side wall, never in a corner or goal sensor. A wall-play
# state starts after the bounce and sends the ball inward; a rebound state
# starts before contact and sends it outward so the physics engine supplies
# the reflected trajectory. Ships use the ordinary randomized placement, so
# the policy has to read the wall/rebound context instead of memorising a
# fixed attacker spawn.
func _place_wall_state(rebound: bool) -> void:
_place_ships_random()
var side := -1.0 if randf() < 0.5 else 1.0
var clearance := REBOUND_BALL_CLEARANCE if rebound else WALL_PLAY_BALL_CLEARANCE
var ball_position := Vector3(
side * (ArenaBoundary.INNER_HALF_X - clearance),
randf_range(1.0, minf(FIELD_MAX_Y, 7.0)),
randf_range(-FIELD_HALF_Z, FIELD_HALF_Z)
)
var toward_field := Vector3(-side, randf_range(-0.15, 0.15), randf_range(-0.15, 0.15)).normalized()
var direction := -toward_field if rebound else toward_field
var velocity := direction * randf_range(WALL_PLAY_SPEED.x, WALL_PLAY_SPEED.y)
_place_body(ball, Transform3D(Basis.IDENTITY, ball_position), velocity, Vector3.ZERO)
# Air-intercept drill geometry. These six ranges are not free tuning knobs —
# together they decide whether the drill is solvable at all, and the original
# values made it arithmetically impossible (see the Round 9 note in
+1 -1
View File
@@ -7,7 +7,7 @@ Deferred work, in rough priority order. The current architecture (ShipAction/Shi
The training pipeline is built — see `TRAINING.md` (self-play PPO via the vendored godot_rl_agents bridge, JSON policy export, in-game GDScript inference, eval ladder). Remaining:
- [ ] Run the generation-5 handling/intercepts/league/teamplay curriculum described in `TRAINING.md`; promote later checkpoints as `medium`/`hard` only after they clear the match and behaviour gates. The orchestrator now requires three independent paired evaluation seeds for each promotion decision; the current Stage 6 league run remains blocked on its recorded regression/telemetry results.
- [ ] Extend generation 5's moving aerial-intercept states with wall plays and rebound scenarios after Stage 5 establishes a productive-air-touch baseline.
- [ ] Extend generation 5's moving aerial-intercept states with wall plays and rebound scenarios after Stage 5 establishes a productive-air-touch baseline. The opt-in wall-play/rebound state generator is now implemented and enabled for the next Stage 6 league command; training evidence is still required.
- [x] Design team-credit rewards and paired 2v2 evaluation before enabling the deferred teamplay stage. `team_touch_credit_weight` is zero by default and `evaluate.py --team-size=2` provides the opt-in paired evaluator; Stage 7 remains disabled pending recorded 2v2 behaviour gates.
## Presentation / AAA polish
+6 -5
View File
@@ -665,10 +665,9 @@ can be based on evidence instead of a single watched match.
| 6 — `league` | Live policy against a frozen opponent sampled per episode from Stage 3, Stage 4, and Stage 5 | 100M (~10h) | Prevent a narrow self-play equilibrium and consolidate ground handling, aerial interception, attack, and defence against distinct styles. | No clear head-to-head regression against any pool member plus conservative handling/aerial telemetry floors. Promote the passing result to `medium.json` after these recorded evaluations support it. |
Stage 7 teamplay remains deliberately unconfigured. The fixed roster
observation and `team_size` plumbing can run 2v2, but there is no paired 2v2
evaluation or team-credit reward yet; spending 120M steps without those gates
would make a pass meaningless. The prerequisites are now implemented but
remain opt-in: `ShipAIController.team_touch_credit_weight` shares a bounded
observation and `team_size` plumbing can run 2v2. The team-credit reward and
paired 2v2 evaluation prerequisites are implemented but remain opt-in:
`ShipAIController.team_touch_credit_weight` shares a bounded
fraction of a touch payout across same-team agents (default `0.0` preserves
all existing curricula), and `evaluate.py --team-size=2` runs the same policy
as a two-ship team with the existing paired side swap. Stage 7 stays
@@ -860,7 +859,9 @@ real tail, the same way this one now has been.
Stage 6's `league` opponent mode samples a historical exported policy at each
episode reset. Each later stage preserves the preceding shaping and adds one
new difficulty. The generation-5 orchestrator evaluates every candidate
new difficulty. Stage 6 now also reserves 10% each for wall-play and
pre-rebound states; these starts are generated by `TrainingMode` and are not
present in Stages 45. The generation-5 orchestrator evaluates every candidate
against every reference on three independent paired seeds (`1, 19, 43`) before
advancing; pass `--evaluation-seeds` only when deliberately running a
different, recorded experiment. This avoids promoting a policy from a single
+6
View File
@@ -1463,6 +1463,12 @@ matches with `--team-size=2`. No Stage 7 training run or promotion is claimed;
teamplay still needs recorded behaviour thresholds and a working Godot runtime
for its end-to-end evaluation.
The generation-5 environment now also has opt-in wall-play and pre-rebound
episode starts. Stage 6's next command enables each at 10% after the Stage 5
aerial baseline; Stages 45 retain their prior distributions. The state
generator and configuration are covered statically, but no training pass or
promotion is claimed until the Godot runtime and telemetry gates are available.
The three declared domain fuzz targets have now each completed a bounded 4-second run (`FuzzQueueCreateDoesNotPanic`, `FuzzResultDigestIsDeterministic`, and `FuzzSyncEventApplicationDoesNotPanic`) with no failures; this closes the locally runnable fuzz portion of 8.46. Live Redis failover, further transaction races, and cloud/runtime gates remain explicitly unverified.
The real ENet integration gate now passes with `GODOT_BIN=/Applications/Godot.app/Contents/MacOS/Godot bash scripts/verify_enet_integration.sh`, covering the `net`, `match-net`, `clock`, `lobby`, and `networked match` process scenarios. The default `GODOT_BIN` remains the portable `godot` PATH lookup for CI; this machine requires the explicit app-bundle path.
+5
View File
@@ -442,6 +442,11 @@ STAGES = [
"--near-goal-chance", "0.25",
"--air-drill-chance", "0.15",
"--air-intercept-chance", "0.25",
# Stage 5 established the aerial baseline; Stage 6 adds a
# measured opportunity for wall/rebound decisions without
# changing the preceding stages' distributions.
"--wall-play-chance", "0.10",
"--rebound-chance", "0.10",
*HANDLING_REWARD_FLAGS,
],
"telemetry_floors": {
+9 -1
View File
@@ -27,12 +27,14 @@ class Generation5ConfigTests(unittest.TestCase):
for stage in generation5.STAGES:
flags = stage["flags"]
total = sum(
float(flag_value(flags, name))
float(flag_value(flags, name)) if name in flags else 0.0
for name in (
"--kickoff-chance",
"--near-goal-chance",
"--air-drill-chance",
"--air-intercept-chance",
"--wall-play-chance",
"--rebound-chance",
)
)
with self.subTest(stage=stage["name"]):
@@ -45,6 +47,12 @@ class Generation5ConfigTests(unittest.TestCase):
)
self.assertEqual(flag_value(generation5.STAGES[2]["flags"], "--opponent-mode"), "league")
def test_league_stage_enables_wall_and_rebound_states_after_intercepts(self) -> None:
self.assertEqual(flag_value(generation5.STAGES[2]["flags"], "--wall-play-chance"), "0.10")
self.assertEqual(flag_value(generation5.STAGES[2]["flags"], "--rebound-chance"), "0.10")
self.assertNotIn("--wall-play-chance", generation5.STAGES[0]["flags"])
self.assertNotIn("--rebound-chance", generation5.STAGES[1]["flags"])
def test_telemetry_floors_fail_closed_on_missing_metric(self) -> None:
ok, failures = generation5.telemetry_passes(
generation5.STAGES[0], {"rollout/upright_fraction": 1.0}