From 1811e9333e9d0e6efa2ca7a2982f1077d1967701 Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Tue, 4 Aug 2026 23:27:57 +0100 Subject: [PATCH] =?UTF-8?q?feat(training):=20curriculum=20generation=204?= =?UTF-8?q?=20=E2=80=94=20MultiDiscrete=20action=20space=20redesign?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three curriculum generations (2026-07-21 through 2026-08-04) all tried gating *when* the policy could use vertical thrust/pitch-roll on top of a continuous Gaussian action space, and all three failed the same way: PPO's action-distribution std collapsed within ~10% of steps and never recovered, landing at a 15-32% win rate vs the grounded reference regardless of mechanism (hard mask, then a gradual ramp). Generation 3's final attempt just landed at 24% — the worst of the three. Root cause, verified against this project's own physics: hovering this ship requires *holding* thrust.y ~= 0.408 continuously (mass 5.0, vertical_thrust 120, gravity 9.8). A collapsed near-zero-mean Gaussian can brush that value but never sustain it long enough to earn the reward gradient that would move the mean — no amount of gating *when* the axis acts fixes a problem in *how* the policy represents a decision on it. This also independently found and fixes a real bug: godot_rl never marks an episode timeout as a truncation, so PPO was bootstrapping V(s)=0 on every 30s draw in every generation to date. - Game/scripts/ship_action_codec.gd (new): single source of truth for a per-axis MultiDiscrete action space (7 heads, nvec [5,5,5,5,5,5,2]) shared by training and in-game inference, replacing the continuous Gaussian. thrust_y's bins are deliberately asymmetric so a random policy drifts through the volume instead of floor-pinning. Legacy continuous decode (ai_ship_controller.gd's old logic) preserved verbatim so every pre-generation-4 export (e.g. Game/bots/promoted/easy.json) keeps working unchanged via an optional "action_space" JSON field. - ship_observations.gd: append own contact state (SIZE 31 -> 35, append-only) so the value function can see what wall_contact_penalty fires on. - ship_ai_controller.gd: action space/decode via the codec; drop the vertical_ramp/pitch_roll_ramp mechanism entirely; tilt_penalty default lowered 4x (aerial approaches require pitching); flight telemetry (airborne_fraction, mean_altitude, air_touch_fraction, vertical_thrust_mean) and truncation-snapshot fields on get_info(). - training_mode.gd: new air_drill_chance state-setter branch (ball spawned high, ships low, kept clear of walls) so aerial practice is forced by the environment instead of relying on reward-driven exploration alone; snapshot terminal observations before a timeout reset for the truncation fix. - cosmic_env.py: remap ShipAIController's truncated/terminal_obs info into SB3's TimeLimit.truncated/terminal_observation keys. - train.py: --reset-logits (+ --reset-logits-heads) replaces the now-meaningless --reset-std; new EntropyFloorCallback (a persistent per-rollout ent_coef controller replacing the one-shot std-reset shock) and per-head entropy logging; FlightTelemetryCallback; --air-drill-chance/ --tilt-penalty flags; optional AbortIfCallback kill-criterion. - export_policy.py: writes the action_space block for MultiDiscrete models; index-level parity check (argmax per head) instead of comparing floats. - curriculum.py: full rewrite — 3 stages (bootstrap/selfplay/gauntlet), no grounded stage, full action space live from step 1; deletes generation 1-3's checkpoint-lineage machinery (nothing to resume from); final report evaluates against both promoted/easy.json and the new promoted/reference-grounded.json (a copy of curric-s5-aggression, the strongest grounded-era artifact, kept as a fixed yardstick). - run_training.sh/.gitignore: commit only final.zip, not the ~2400 intermediate checkpoint files a single stage was writing (~500MB -> ~0.2MB per run); requirements.txt pinned (behaviour here now depends on specific library internals, not just public APIs). - test_action_space.py (new): offline rung-0 check catching a head-order mismatch before it silently corrupts 24h of training. Validated: GDScript compiles clean (Godot --headless --import + script validation), free_play.tscn and training.tscn both boot headless without errors, offline action-space assertions pass. Not yet run: the actual smoke-training/A-B validation ladder steps in TRAINING.md's "Generation 4" section, before committing to the full ~32h curriculum. See TRAINING.md's "Generation 4" section for the full design writeup. --- .gitignore | 7 + Game/bots/promoted/reference-grounded.json | 1 + Game/scripts/ai_ship_controller.gd | 49 ++- Game/scripts/policy_network.gd | 19 +- Game/scripts/ship.gd | 10 + Game/scripts/ship_action_codec.gd | 128 +++++++ Game/scripts/ship_action_codec.gd.uid | 1 + Game/scripts/ship_ai_controller.gd | 150 +++++--- Game/scripts/ship_observations.gd | 46 ++- Game/scripts/training_mode.gd | 68 +++- TRAINING.md | 239 ++++++++++--- training/cosmic_env.py | 34 +- training/curriculum.py | 397 ++++++++++----------- training/eval_history.json | 10 + training/export_policy.py | 83 ++++- training/requirements.txt | 20 +- training/run_training.sh | 7 +- training/test_action_space.py | 105 ++++++ training/train.py | 254 ++++++++++++- 19 files changed, 1259 insertions(+), 369 deletions(-) create mode 100644 Game/bots/promoted/reference-grounded.json create mode 100644 Game/scripts/ship_action_codec.gd create mode 100644 Game/scripts/ship_action_codec.gd.uid create mode 100644 training/test_action_space.py diff --git a/.gitignore b/.gitignore index 7817b09d..0efa2175 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,13 @@ training/.venv/ training/smoke_run.log training/__pycache__/ +# Intermediate PPO checkpoints: only final.zip is ever committed (see +# run_training.sh) — --resume only ever points at final.zip, and a single +# experiment's intermediate checkpoints were 2401 files / ~500MB, of which +# final.zip was ~0.2MB. This is the training-results-survive-any-machine +# property from ~2500x less data, not a relaxation of it. +training/checkpoints/*/ppo_*_steps.zip + # Exported training binary: a regenerable build artifact (rebuilt by # export_linux.sh / run_training.sh), not a training result. training/build/ diff --git a/Game/bots/promoted/reference-grounded.json b/Game/bots/promoted/reference-grounded.json new file mode 100644 index 00000000..07a313c6 --- /dev/null +++ b/Game/bots/promoted/reference-grounded.json @@ -0,0 +1 @@ +{"input_size": 31, "layers": [{"weights": [[0.2132105529308319, -0.5505000948905945, -0.6299572587013245, 1.101082682609558, 1.0697838068008423, -0.38019272685050964, -0.19478388130664825, 0.6982545852661133, 0.06301423162221909, 0.46840742230415344, 0.44514384865760803, -0.8079569339752197, 0.2460804283618927, -0.493038535118103, 0.32830461859703064, -1.0333590507507324, -1.504453182220459, -0.1586839109659195, -0.5961207151412964, -0.6666276454925537, 0.18814708292484283, 0.1651584357023239, -1.0349421501159668, -0.22046735882759094, 0.12252813577651978, 0.14373619854450226, 0.7280840277671814, 0.19836586713790894, 0.46903347969055176, 0.49104923009872437, -0.1864822804927826], [-0.4460527002811432, -0.4951425790786743, -0.3486891984939575, -0.016519125550985336, -0.23766790330410004, -0.2331138253211975, 0.6212438941001892, 0.2845558822154999, -0.7765030860900879, -1.237285852432251, -0.8231315016746521, 0.5390392541885376, 0.7447232007980347, -0.7502881288528442, -0.36507049202919006, 2.2791948318481445, -0.006740210577845573, -0.19338341057300568, 1.782576560974121, -0.05935865640640259, 0.1898214966058731, -0.06066112592816353, -0.06848990172147751, 0.07098804414272308, 0.3501407504081726, 0.1068389043211937, 0.3070503771305084, 0.28725096583366394, 0.7638812065124512, 0.05608776584267616, -0.36970797181129456], [0.16663919389247894, 1.0087212324142456, -0.12791062891483307, -1.5273956060409546, 0.5117733478546143, -0.011997949331998825, 0.4588538408279419, -0.36254268884658813, 0.3834598660469055, -0.8435989022254944, 0.5330172777175903, 0.08633169531822205, -0.4592211842536926, 0.16750074923038483, 0.4255237877368927, 1.4952901601791382, -0.09250624477863312, 0.7272112369537354, -0.3249727785587311, 0.27654188871383667, -1.5549042224884033, -0.023184752091765404, -0.01334468461573124, 0.4287340044975281, -0.41425472497940063, 0.11185535788536072, 0.19514864683151245, -0.34884488582611084, -1.6414152383804321, -0.16382308304309845, 0.14191773533821106], [0.37455734610557556, -0.2276124209165573, -0.30163946747779846, -0.3815499246120453, -0.463408499956131, -0.44171127676963806, 0.38873058557510376, -0.15567438304424286, 0.016486983746290207, -1.083630919456482, 0.413242369890213, -0.956475019454956, -0.16837967932224274, -0.7088351845741272, -0.9181639552116394, -0.9060636758804321, -1.539887547492981, -0.9702157378196716, -2.1830265522003174, -2.152541399002075, 1.1429919004440308, -0.46659547090530396, -0.7576267123222351, -0.28280362486839294, -1.026808261871338, 0.45270127058029175, -0.3587704300880432, -0.6678931713104248, 0.3521687984466553, 0.2714675962924957, -0.1927388608455658], [-0.9500450491905212, 0.3190913796424866, -0.08662700653076172, 0.3540552854537964, -1.5242657661437988, -0.08563567698001862, 0.6659894585609436, -0.1681835949420929, -0.11354732513427734, 2.377159595489502, -0.2634127736091614, -0.35974395275115967, 0.6345059275627136, 0.26066723465919495, 0.9594666361808777, -0.8117844462394714, 0.8088960647583008, 0.27260449528694153, -1.2117085456848145, 0.6557010412216187, -1.0643246173858643, 0.07832551002502441, 0.8069413304328918, 0.017750974744558334, -0.7365855574607849, -0.20129647850990295, 0.7792006731033325, 0.45474573969841003, -0.4191882312297821, 0.03246185556054115, -0.050275348126888275], [0.7344499826431274, 0.44900161027908325, 0.16023188829421997, -0.321226567029953, -0.8210815787315369, -0.1505436897277832, 0.12789365649223328, -0.5361554026603699, -0.7803841829299927, 0.4905050992965698, 0.3338911831378937, 0.31390485167503357, 0.43749529123306274, 0.2700745463371277, 1.2102490663528442, 0.391421914100647, -0.9092614054679871, 1.5652027130126953, -2.1053221225738525, -0.1750103384256363, 0.5332126617431641, -0.1967775672674179, 0.5124427676200867, -0.06064920127391815, -1.1131091117858887, 0.8027903437614441, 0.4946668744087219, -0.8241100311279297, -0.8981505036354065, 0.003694050945341587, 0.19664861261844635], [-0.17490260303020477, -0.31722205877304077, 0.8307740688323975, 0.3279227912425995, -0.29292672872543335, -0.33404913544654846, 0.45560455322265625, -0.23229072988033295, 0.6025132536888123, -2.253107786178589, 0.20882490277290344, 0.2648746371269226, 0.3480221927165985, 0.007984754629433155, -0.21552079916000366, 1.859559416770935, -0.21106217801570892, 0.1428244411945343, 0.464933305978775, 0.5790753364562988, 0.17377373576164246, -0.929498553276062, 0.7654732465744019, 0.3526759743690491, -0.9805470108985901, -0.3762736916542053, 0.7167382836341858, 0.5031139850616455, 0.17538395524024963, -0.1269184798002243, 0.029666511341929436], [0.27864617109298706, -0.11816706508398056, 0.18067435920238495, 0.5626689195632935, 0.5899884700775146, -0.3229883015155792, -0.44114843010902405, -0.5411189794540405, -0.13134963810443878, -1.2640666961669922, 0.5258992910385132, 1.9763816595077515, 0.5916152596473694, 0.14050202071666718, 0.15208055078983307, -1.2476837635040283, 0.8024832606315613, -0.5255927443504333, -1.7240465879440308, 1.2391303777694702, -0.838809609413147, 0.05874824896454811, 0.7073816061019897, 0.18527552485466003, -0.5016698837280273, -0.6942269206047058, 1.9170135259628296, 0.002948430832475424, 0.015289627015590668, 0.1358039230108261, 0.06304434686899185], [0.4288829565048218, -0.11334791779518127, 0.13698813319206238, 0.8560556769371033, -0.5878512263298035, -0.08874566853046417, 0.5969183444976807, -0.20484282076358795, -0.2562952935695648, 2.3878040313720703, -0.4969453811645508, -0.07563065737485886, -0.7439992427825928, 0.1334875226020813, -0.6293483376502991, -0.1000913679599762, 0.142692431807518, -0.5956588983535767, 0.611894428730011, -0.279428094625473, 0.30990591645240784, -0.5901391506195068, -0.26930588483810425, 0.914375364780426, -0.8271951675415039, 1.2447254657745361, 0.5970252752304077, -0.3425673246383667, 0.31649741530418396, -0.020532481372356415, 0.045367028564214706], [0.3299916088581085, 0.15130621194839478, -0.8379265666007996, -1.7740552425384521, -0.12167522311210632, 0.044675394892692566, 0.057895511388778687, -0.02901553362607956, 0.0025225782301276922, 0.4424581527709961, 0.8757785558700562, 0.08258272707462311, -0.38553547859191895, 0.21754609048366547, -1.6254210472106934, 0.3842737078666687, -0.7047141194343567, -0.625278890132904, -1.1356866359710693, -1.3144681453704834, -0.3702086806297302, 0.3921129107475281, 0.2642570734024048, 0.23995286226272583, 0.21371087431907654, -0.029204998165369034, 0.4899316728115082, -0.012755795381963253, 0.30883312225341797, 0.17661282420158386, -0.1464177817106247], [-0.2750246226787567, -0.42322656512260437, 0.11301738023757935, 0.2800047695636749, 0.6426944136619568, -0.3778299391269684, -0.16732758283615112, -0.24563419818878174, 0.5626721978187561, 2.005563497543335, 0.4134571850299835, 0.04481896385550499, -0.662806510925293, 0.5413726568222046, 0.4773791432380676, 0.4843907356262207, -1.9212958812713623, -1.778074026107788, 0.33394595980644226, -0.07241053879261017, -1.009761095046997, -0.10664412379264832, -0.20580819249153137, 0.2804328501224518, 0.09670223295688629, -0.5183238983154297, 0.23046250641345978, 0.2614084780216217, 0.1585039496421814, 0.07257378101348877, 0.3082593083381653], [-0.07141581922769547, -0.08366688340902328, -0.4315843880176544, -1.4596396684646606, -0.07523322850465775, 0.2191641479730606, -0.807645320892334, 0.15748970210552216, -0.17931587994098663, -0.7560264468193054, -0.06257321685552597, 1.0923439264297485, -0.9454646110534668, 0.22581075131893158, -0.5657235980033875, 0.29912903904914856, 0.15823443233966827, -0.12748092412948608, -0.19321030378341675, -0.14543674886226654, -0.8292683362960815, 0.8816763162612915, 0.18552176654338837, -0.32859155535697937, 0.24396519362926483, 0.6105519533157349, -0.4852618873119354, -0.5110201835632324, -0.16323520243167877, 0.25166118144989014, 0.24786292016506195], [0.21523956954479218, 0.1562301218509674, 0.0221262127161026, -0.6087404489517212, 0.46063607931137085, -0.22778768837451935, 1.1153236627578735, -0.07135883718729019, 0.5272279977798462, -0.7476999163627625, -0.09008852392435074, -0.771179735660553, 0.8229042887687683, 0.4366285800933838, 0.11759470403194427, 2.4032702445983887, 0.19325625896453857, 1.4736907482147217, 1.397333025932312, 0.3277589678764343, 0.09186668694019318, -0.07355167716741562, -0.3139346241950989, 0.07797830551862717, 0.08198469132184982, -0.15765316784381866, -0.6290473341941833, -0.2553953528404236, -0.5131102204322815, 0.0704541802406311, -0.07626032084226608], [0.21483288705348969, -0.5825058221817017, -0.7902686595916748, 0.7512765526771545, 0.41720467805862427, -0.5540686845779419, 0.7735106348991394, -0.22908470034599304, -0.5620445013046265, -0.02319338731467724, 0.005882989149540663, 1.2394624948501587, 1.653592824935913, -0.30428504943847656, -0.4428202509880066, -0.12983369827270508, 0.1864069402217865, -0.11130665242671967, 0.10901036113500595, -0.27872201800346375, -0.4714524447917938, 0.2462785243988037, 1.1345648765563965, -0.5250326991081238, -0.5664281845092773, -0.5138376951217651, 0.39596718549728394, -0.40804052352905273, 0.6452508568763733, -0.08483627438545227, -0.002257945481687784], [0.5506195425987244, -0.3956407308578491, 0.3537728488445282, 0.5782803893089294, -0.9234032034873962, -0.5335764288902283, 0.6904617547988892, -0.2740117609500885, 0.38703808188438416, -0.27440956234931946, 2.113621473312378, 0.5639458298683167, 0.22581587731838226, 1.4223661422729492, 0.00976602640002966, -1.1552176475524902, 2.0022995471954346, 0.013152356259524822, -0.14697788655757904, 0.5560394525527954, 0.8351344466209412, -0.4368472099304199, -0.6299958825111389, -0.1166602298617363, 0.16454081237316132, -1.188342571258545, -1.0871034860610962, -0.36139702796936035, 0.29247498512268066, -0.10740535706281662, 0.12439137697219849], [0.15858498215675354, -0.3089373707771301, 0.1203608289361, -0.6152315735816956, -0.13064149022102356, 0.15337948501110077, -0.2839452028274536, 0.1619073450565338, 0.1773609220981598, 1.8561214208602905, 0.42279499769210815, -0.8818660974502563, 0.49074411392211914, 0.10039592534303665, -0.38606658577919006, -1.6269742250442505, 0.7172017693519592, 1.278944492340088, -1.067112922668457, 0.3465897738933563, 0.6781565546989441, 0.09660379588603973, 0.6463130712509155, 0.31459519267082214, 0.6088874340057373, 0.7052118182182312, -0.7811537384986877, 0.04141373559832573, 0.3025234341621399, -0.0419432669878006, 0.11746691912412643], [-0.048144031316041946, -0.27650946378707886, 0.47251471877098083, -0.1464807391166687, -0.8889816999435425, 0.11264384537935257, 0.7397831678390503, 0.0005442112451419234, -0.8854065537452698, -1.9628329277038574, 1.414870262145996, 0.3343764841556549, 0.9638422727584839, 1.0498201847076416, -0.2579888105392456, -1.0939849615097046, 0.2806292772293091, -0.4307694137096405, -1.0267916917800903, 1.129167079925537, -0.935599684715271, 0.2978683412075043, 1.0454447269439697, -0.5543971061706543, -0.5025802254676819, 0.16941571235656738, 0.9752537608146667, -0.1902124434709549, 0.76014244556427, -0.2305237203836441, 0.2535443902015686], [-0.3575473427772522, -0.4492252767086029, 0.38518282771110535, 0.1777973473072052, 0.006381429266184568, 0.11884448677301407, 1.2545849084854126, 0.004657002631574869, -0.7935553789138794, -0.5084020495414734, -0.4133960008621216, 0.43182191252708435, -0.4280209541320801, 0.17573869228363037, 0.9975442886352539, 1.747553825378418, 0.042458295822143555, 0.5982169508934021, 1.458974003791809, 0.42343586683273315, 1.0318281650543213, -1.1827878952026367, 0.5987748503684998, -0.5067319273948669, 0.5470690131187439, 0.5551016926765442, 0.47158151865005493, 0.1258331835269928, 0.4674302339553833, -0.2764972150325775, 0.1811937391757965], [0.04613569751381874, 0.6367710828781128, 0.24329431354999542, 0.5319068431854248, -0.6258873343467712, -0.1334078460931778, -0.2625136971473694, 0.03469282016158104, -1.0083165168762207, 1.4533424377441406, 0.28859612345695496, -0.1485857516527176, -0.005053929518908262, -0.5198847651481628, 0.08681698888540268, -2.0698859691619873, 1.0620553493499756, 1.4042739868164062, -1.5031594038009644, 0.9145076870918274, 0.02047746628522873, 0.18322204053401947, 0.002196536399424076, 0.013853883370757103, -0.3710305690765381, -0.8285648822784424, 0.9766260385513306, 0.13140995800495148, -0.6376929879188538, 0.16438259184360504, -0.09338130801916122], [0.34514838457107544, 0.6067920923233032, 0.03374651074409485, 0.00441536121070385, 0.30811241269111633, 0.1924552321434021, 0.6282570958137512, 0.7567582726478577, -0.8620604276657104, -0.3870491087436676, -0.5399124622344971, 0.18058091402053833, -0.4668331742286682, -0.27139654755592346, -0.3524084687232971, 0.8194896578788757, -0.5191629528999329, 0.7277465462684631, -0.31682512164115906, 0.5592758059501648, 0.8800665140151978, -0.5043678283691406, -0.9082348942756653, 0.15767262876033783, -0.7737700343132019, -1.0040363073349, -0.7502061128616333, -0.2531436085700989, -0.6493576169013977, -0.2837606370449066, 0.31464695930480957], [-0.6942869424819946, 0.09817755222320557, 0.09316311031579971, -0.6193087697029114, 0.07577309757471085, 0.32056745886802673, -0.791151762008667, 0.1279391199350357, -0.7477056384086609, -0.5956919193267822, 0.43228909373283386, 0.804003894329071, -2.106433868408203, -0.5097801089286804, -0.3200743794441223, -1.9558981657028198, 0.15056496858596802, 0.15553626418113708, -0.9098101854324341, 0.5521074533462524, 0.04138537123799324, -0.07762231677770615, 0.06645286828279495, -0.505439817905426, -0.7230849862098694, 1.71471107006073, -1.2639193534851074, 0.6948148012161255, -0.15441125631332397, -0.012134360149502754, -0.09230659157037735], [0.10544876009225845, -0.3217163681983948, 0.3409155309200287, -0.05179629102349281, -0.11420193314552307, 0.197198286652565, 0.47831031680107117, -0.09577081352472305, 0.3124120533466339, 1.0411791801452637, 0.38966134190559387, 1.3746219873428345, -0.19504019618034363, -0.5691676735877991, 1.5548425912857056, -0.9700558185577393, -0.3559514880180359, -1.062670111656189, 0.03043489158153534, -0.7535311579704285, -0.6286920309066772, 0.858180820941925, 0.8123920559883118, 0.03491126373410225, 1.2232165336608887, 0.926016092300415, -0.6569399833679199, -0.18672248721122742, 0.20801173150539398, -0.2607758641242981, 0.3991272449493408], [1.36765456199646, -0.3129424452781677, 0.1500723510980606, -1.134727120399475, -0.02680778130888939, 0.11964192986488342, -0.10965193808078766, -0.0007834864663891494, -0.4120284914970398, -0.9514195919036865, -0.28768742084503174, -0.7559131383895874, 0.8819848895072937, 0.6489817500114441, -1.4502633810043335, 0.5602828860282898, 0.461474746465683, -0.46204251050949097, -0.007783589884638786, -0.11963685601949692, 1.1479274034500122, -1.1846191883087158, -0.8645534515380859, -0.3626865744590759, -0.39708226919174194, 0.5060600638389587, 0.11990253627300262, -1.6674041748046875, 0.5383869409561157, -0.16197995841503143, 0.19358383119106293], [0.4536041021347046, -0.0026997143868356943, 0.44831472635269165, -0.09836556017398834, 0.14857910573482513, 0.13567973673343658, -0.12254010885953903, 0.11211569607257843, -0.154966339468956, 0.5181134343147278, -0.15424063801765442, 0.5072258710861206, 1.378092646598816, -0.31099456548690796, 0.8755845427513123, 1.6339304447174072, -0.6203537583351135, 0.27565571665763855, -2.2021734714508057, 0.7715266942977905, 0.691386342048645, -0.26966825127601624, 0.8774563074111938, 0.4758078455924988, 0.2446833997964859, -0.9826756119728088, 0.903386652469635, -0.47751814126968384, -0.1251213103532791, -0.2451823651790619, 0.21213629841804504], [0.39142778515815735, -0.4066818058490753, -0.5251694321632385, -1.5113228559494019, -2.0825436115264893, 0.3072400391101837, 0.6058724522590637, 0.261303186416626, -0.9675375819206238, 2.693213701248169, 1.4720042943954468, 1.0757704973220825, -0.8771958947181702, 0.4792841672897339, -0.8264062404632568, -0.07283894717693329, -0.1665881872177124, 0.12549284100532532, -0.20103318989276886, 0.2738268971443176, -0.5030565857887268, 0.0341813899576664, -0.06950643658638, -0.09999190270900726, -0.33353230357170105, -0.21550782024860382, -0.0280232485383749, -0.1998167186975479, 0.30597132444381714, -0.011612622067332268, 0.1356598287820816], [-0.29214179515838623, -0.0622551254928112, -0.1515011042356491, -0.7277367115020752, -0.7841710448265076, -0.14336422085762024, -0.6140298247337341, -0.6693986654281616, -0.9395642876625061, -0.7057627439498901, 0.13200163841247559, -0.5926122069358826, 0.4485284090042114, -0.11713109165430069, -1.0402933359146118, 2.018364906311035, -0.6918347477912903, -1.5002237558364868, 0.4463595747947693, -0.11621084064245224, -0.7771536111831665, 0.32500964403152466, 0.029153717681765556, -0.5401647090911865, 0.10460993647575378, 1.1849581003189087, -0.0835174173116684, 0.2844874858856201, -0.17409363389015198, 0.1649567186832428, 0.262765109539032], [-0.576680064201355, -0.07009746879339218, -0.6833460330963135, 0.0020045775454491377, -0.027892256155610085, -0.006819072645157576, -0.14601320028305054, -0.23412387073040009, -0.963516891002655, -0.9453272819519043, -0.4035561680793762, -0.2697681486606598, -0.40226903557777405, 0.24423234164714813, 0.6054315567016602, 0.6507388949394226, 0.17035609483718872, -0.18423941731452942, -0.7043160796165466, 1.156758189201355, 0.9585304260253906, 1.0228400230407715, 0.42332637310028076, -0.3804110586643219, 1.2443982362747192, 1.6576685905456543, 0.3021104037761688, 0.4378035366535187, 0.4118233323097229, 0.4497647285461426, -0.38520026206970215], [-0.387575626373291, -0.04514460265636444, -0.2656518816947937, 0.31849852204322815, -0.5850221514701843, -0.5161714553833008, -1.0058727264404297, -0.0755247175693512, -0.18515783548355103, -1.5444148778915405, -0.4897160828113556, -0.30640241503715515, -0.8111675977706909, 0.3902054727077484, -0.4658120572566986, 2.1852924823760986, -0.09316900372505188, 1.1216487884521484, 0.5297088623046875, 0.30224791169166565, 0.46636638045310974, -0.19908049702644348, -0.20835109055042267, 0.9026744961738586, -0.20766912400722504, -0.6462790369987488, 0.5700953602790833, 0.6439032554626465, 0.09409524500370026, 0.03293928503990173, -0.15031886100769043], [-0.32719311118125916, 0.027429739013314247, 0.16720907390117645, -0.681341826915741, 1.2983540296554565, -0.1241360530257225, -0.7683018445968628, 0.6445281505584717, -0.04669317603111267, 0.14816820621490479, -0.07645556330680847, -0.3070893883705139, 0.7824313044548035, 0.8403593897819519, 1.3767330646514893, 0.5843608975410461, -0.13834968209266663, -0.18852739036083221, -0.2392747849225998, -2.398667097091675, 0.11197268217802048, 0.032774560153484344, -0.24869859218597412, 0.3618611991405487, 0.2470080703496933, -2.5172815322875977, 0.006227733101695776, 0.16578197479248047, 0.13258086144924164, -0.29544365406036377, 0.16136786341667175], [-0.05500642955303192, 1.2475649118423462, 0.11381109803915024, -0.05138897895812988, -0.4788208305835724, 0.09767657518386841, 0.4894331395626068, -0.01572480797767639, -0.5039882659912109, -1.0099929571151733, 0.49912428855895996, 0.34670108556747437, 0.2269955426454544, -0.21685369312763214, -0.31023871898651123, 3.434086322784424, 0.6320929527282715, 1.3945839405059814, 0.593915581703186, -0.5473356246948242, 0.3186118006706238, 0.17811298370361328, -0.5212305784225464, 0.03551964461803436, 0.05821860954165459, 0.24057915806770325, -0.1604190319776535, 0.25033482909202576, -1.674814224243164, -0.24526137113571167, 0.16689333319664001], [-0.0627739280462265, -0.4370183050632477, 0.9478804469108582, 0.6989040970802307, -0.015529388561844826, 0.2396281361579895, 0.9101287722587585, 0.3104791045188904, -0.5690582394599915, -1.0597362518310547, 0.3974674642086029, 0.12136056274175644, 1.4560863971710205, -0.005939011927694082, 0.06041634455323219, -0.8156495094299316, -0.48844051361083984, 0.8700518012046814, -0.02514316700398922, 1.6782280206680298, -0.4777756631374359, -0.017148956656455994, -0.35132071375846863, -0.9042287468910217, -0.5762501955032349, -1.5720937252044678, -0.2751193046569824, 0.1476137787103653, 0.3710356056690216, -0.06549927592277527, 0.6820142269134521], [0.16711841523647308, -0.8365080952644348, -0.10026721656322479, -0.3550814092159271, -0.20056352019309998, 0.15391318500041962, 0.39037516713142395, 0.14569269120693207, -0.15990225970745087, -0.6147333979606628, -0.915063738822937, 0.4064595103263855, 0.870851993560791, -0.02866234816610813, 1.2971161603927612, 1.7681567668914795, 0.01051651407033205, -1.1288716793060303, 0.6947669386863708, 0.5482943058013916, -0.27685514092445374, 0.31324687600135803, 0.33905795216560364, -0.590002715587616, 1.324366569519043, -0.39793694019317627, 0.3282362222671509, -0.040452491492033005, 0.27380770444869995, 0.21238598227500916, -0.6219302415847778], [0.30060645937919617, 0.2996620237827301, -0.3211168348789215, 0.1687331348657608, 0.6492659449577332, -0.12555454671382904, 0.06819744408130646, -0.6411682963371277, 0.49712321162223816, 0.3487483561038971, 0.25776079297065735, 1.7213290929794312, 1.4579806327819824, 0.2865139842033386, -0.708747148513794, 0.14949354529380798, 0.8147575259208679, -0.5075638890266418, 0.15066714584827423, 1.3954932689666748, -0.07912800461053848, -0.4580004811286926, 0.12642763555049896, -0.2472696453332901, 1.22947096824646, 1.0794901847839355, -0.25976529717445374, 0.1033957451581955, 0.028262877836823463, -0.08810719847679138, 0.19214332103729248], [-0.44671836495399475, 0.05847730487585068, 0.12384235858917236, 0.08104486018419266, 0.20325486361980438, -0.07116208225488663, -0.21403275430202484, 0.08530040085315704, -0.5283556580543518, 0.7272126078605652, -0.24671365320682526, -1.5935354232788086, 0.7061281204223633, 0.16595587134361267, -0.04807766526937485, 0.44538989663124084, 0.7358726859092712, 1.2682894468307495, 0.7265345454216003, 0.589333176612854, -0.08940710127353668, -0.46815919876098633, 0.692786693572998, -0.3982551395893097, 0.16104309260845184, -0.1876053363084793, -0.5362364649772644, 0.2589689791202545, -0.02588633820414543, -0.08308960497379303, 0.05077908933162689], [0.39994314312934875, -0.14967921376228333, 0.1618608683347702, 0.7263690829277039, 0.17942318320274353, -0.3970875144004822, 1.4547969102859497, 0.6467104554176331, -0.48395657539367676, 0.07611531019210815, -0.04506843537092209, 0.3606020510196686, -0.2885378897190094, -0.7488252520561218, 0.8493556380271912, 0.9254662990570068, -0.463605135679245, 0.7422187924385071, 0.9587868452072144, -0.10687801986932755, 0.011608106084167957, 0.4432542026042938, 0.11442688852548599, -0.0073565468192100525, 1.2243587970733643, 0.3589254915714264, 0.30356478691101074, -0.49892762303352356, 0.4168878495693207, -0.329427570104599, 0.10920578241348267], [0.0624539852142334, 0.15387940406799316, -0.714148223400116, 0.9624636173248291, 0.6449174284934998, -0.09682285785675049, 0.13327117264270782, -0.4456203877925873, -0.385953426361084, -0.4813072979450226, 0.3807862102985382, 0.9575257301330566, -0.07176758348941803, -0.4715876579284668, -0.5012513399124146, -0.8205323815345764, 0.9834148287773132, 1.8883239030838013, -0.4197019338607788, 0.3541316092014313, 0.7495369911193848, -0.6937528848648071, -0.39541834592819214, 0.33907702565193176, 0.44089677929878235, -1.0045816898345947, 1.4028699398040771, -0.18781064450740814, -0.5633153915405273, -0.07524692267179489, -0.058812182396650314], [-0.026482203975319862, -1.322697401046753, -0.16445015370845795, -0.4334000051021576, -0.5730889439582825, -0.4075861871242523, -0.46657291054725647, -0.33804741501808167, -0.2891969382762909, -1.8056706190109253, 0.2737622857093811, 0.7437800765037537, -0.011731550097465515, 0.5347179174423218, -0.11594793945550919, 2.6284632682800293, -0.8722235560417175, -1.3141937255859375, 0.6577720046043396, -0.665094256401062, 0.1447727233171463, 0.1016981452703476, 0.3292398750782013, -0.007864343002438545, 0.29762160778045654, -0.4886038899421692, -0.22902393341064453, -0.15344911813735962, 1.2391823530197144, -0.13144226372241974, -0.06795002520084381], [0.299752801656723, -0.2900698184967041, -0.2782038748264313, -0.3295152187347412, -0.7015608549118042, 0.476704478263855, 0.14678703248500824, 0.0017168765189126134, -1.167356014251709, 0.905274510383606, -0.5296701192855835, -0.57451993227005, 1.1962013244628906, 0.4220272898674011, 1.6016789674758911, 0.11343099176883698, 2.296884298324585, 1.2632757425308228, -0.026845524087548256, 0.8721913695335388, 0.7804331183433533, 0.14089460670948029, 1.4706528186798096, -0.4856940805912018, 0.26580914855003357, 0.37718379497528076, -0.7073254585266113, 0.44664421677589417, 0.09802479296922684, 0.08296879380941391, 0.05520201101899147], [-0.45397764444351196, 0.25146016478538513, 0.40223824977874756, -1.1515171527862549, -0.4438095688819885, 0.0289212204515934, -0.25279128551483154, -0.08980973809957504, -0.5780074596405029, 0.03861148655414581, -0.4670255184173584, -0.6882158517837524, -0.14815768599510193, 0.9708629846572876, 0.24342510104179382, 0.39881590008735657, 0.5831748843193054, 0.1913965344429016, 1.2602286338806152, -0.5349177718162537, -1.6357223987579346, 0.3923565745353699, -1.2737274169921875, 0.3402162790298462, -0.02517903782427311, 1.1747589111328125, -0.5656527876853943, 0.5517640709877014, -0.4905766248703003, 0.030687639489769936, 0.2465113401412964], [-0.23079513013362885, -0.2116118222475052, 0.4062027633190155, 0.6578575968742371, 0.47084924578666687, -0.0696568414568901, -0.25866323709487915, 0.04367390647530556, -1.1487891674041748, -0.47884228825569153, 1.2280105352401733, 0.6548188328742981, -0.6322630047798157, 0.5513315200805664, -0.21304167807102203, -2.2135744094848633, 0.8213816285133362, -0.46365824341773987, -0.933225154876709, 0.4770141839981079, 1.1014049053192139, 0.016720080748200417, -0.25608551502227783, 0.548501193523407, 0.975741982460022, -0.054120298475027084, -0.08818386495113373, 0.051787570118904114, 0.7068743109703064, -0.19094549119472504, 0.22622178494930267], [0.011844945140182972, 0.3112587630748749, -0.5386514663696289, 1.6982405185699463, 1.3107391595840454, -0.3550209105014801, 0.36662691831588745, 0.009313693270087242, 1.008508563041687, 0.05664806067943573, 0.29652127623558044, 1.903110146522522, -0.0835442841053009, 0.18204835057258606, -0.48104560375213623, 0.14213266968727112, 0.13274599611759186, 0.2971539795398712, 0.41568300127983093, 0.9609764814376831, -0.3459359407424927, -0.3248620331287384, -0.27998122572898865, -0.37872079014778137, 0.08157225698232651, -1.2874311208724976, 0.24673360586166382, -0.0728776752948761, -0.00906410627067089, 0.46703487634658813, -0.16061510145664215], [-0.9938454031944275, -0.012515855021774769, 0.02996119111776352, -0.3482325077056885, 0.9975041151046753, 0.25347399711608887, 0.38461893796920776, 0.572554349899292, 0.28873804211616516, 0.8657961487770081, -0.1437607854604721, 0.5426943302154541, 1.0856488943099976, -0.45868661999702454, 1.1411188840866089, -1.8646793365478516, -0.7675983905792236, 0.39370250701904297, -0.26422446966171265, 0.49514877796173096, -0.09639129042625427, -0.1947592943906784, -0.7764107584953308, -0.8911095857620239, -0.7438493371009827, -0.700161874294281, 0.3598264157772064, 0.29361581802368164, -0.1970219761133194, -0.20227044820785522, 0.22024299204349518], [0.40509918332099915, -0.16562114655971527, -0.08524385839700699, 1.5268088579177856, 0.4778102934360504, -0.25189927220344543, 0.6014324426651001, 0.04876195639371872, 0.3737357556819916, 0.5383155941963196, 0.6340929865837097, -0.51346355676651, 0.16278983652591705, -1.3440645933151245, 0.038382504135370255, -1.4778213500976562, 0.23579977452754974, 0.12468844652175903, -0.5668957233428955, -0.20124536752700806, 0.1279638558626175, -0.1863238513469696, 0.906355619430542, 0.5041385889053345, 0.20389388501644135, -0.29495224356651306, -0.06372905522584915, -0.07770907133817673, -0.1456836462020874, 0.007741913665086031, 0.2676427960395813], [-0.277328222990036, 0.36922988295555115, 0.5017278790473938, -0.03392766788601875, -0.6130388975143433, 0.28455591201782227, 0.4535404443740845, -0.401658296585083, 0.4146745502948761, -0.09382172673940659, -1.440410852432251, -0.20516154170036316, 0.5222387909889221, 0.15507325530052185, 0.9687291383743286, 1.316907286643982, 1.1820597648620605, -0.26438623666763306, 1.8520128726959229, 0.51289302110672, 1.6609636545181274, 0.8655974864959717, 0.5632660388946533, -0.7787572145462036, 0.42163750529289246, 1.7640106678009033, -0.12956367433071136, 0.3172037899494171, -0.6251618266105652, -0.17396940290927887, 0.1476900279521942], [0.040034934878349304, -0.15747937560081482, 0.1570352464914322, -0.3151501715183258, 0.664653480052948, -0.27492842078208923, -0.7150352597236633, 0.5181596875190735, -0.5957698822021484, -0.011149944737553596, 0.5846840143203735, 1.546970248222351, -1.0364559888839722, 0.6723319292068481, -0.19322849810123444, -1.4825794696807861, -0.3810608386993408, 0.6944361925125122, -1.297455906867981, 0.8620111346244812, 0.5539892911911011, -0.2135775089263916, -0.5743125677108765, 0.027590056881308556, -0.029860971495509148, -1.8412617444992065, 0.5647971630096436, -0.01061893068253994, 0.13486824929714203, -0.1616760790348053, -0.29181545972824097], [0.40082040429115295, -0.51335209608078, 0.05622901767492294, -1.9816757440567017, -0.2749852240085602, 0.5477635860443115, 0.11329694092273712, 0.5719118714332581, -0.2696360945701599, -0.03898360952734947, -0.7569354176521301, -0.23236173391342163, -0.8354728817939758, 0.5556238889694214, 0.45291775465011597, 0.8073603510856628, 0.6306107640266418, 0.8113075494766235, 0.6830958724021912, -0.22841662168502808, 0.20933692157268524, 0.8030779361724854, 0.0830773338675499, 0.5500664710998535, -0.1640903204679489, 0.08459538221359253, -0.3895355463027954, -0.26907023787498474, 0.06422176957130432, 0.09721243381500244, 0.09754937887191772], [-0.3321678042411804, 1.2692234516143799, -0.3601268231868744, 0.1281515657901764, -0.21838733553886414, 0.31470558047294617, -0.393319696187973, -0.6673617362976074, -0.04733317717909813, 0.7207376956939697, -0.7639656066894531, 0.8206013441085815, 1.5568263530731201, 0.9966807961463928, 0.5850076079368591, -1.8558508157730103, 1.22233247756958, -0.8989970088005066, -1.2047154903411865, 0.5223557949066162, -1.0085957050323486, -0.19046495854854584, -0.7744905352592468, 0.1573782116174698, 0.14419017732143402, -0.17084576189517975, 0.14788271486759186, 0.5025849938392639, -1.987717866897583, -0.01846572570502758, 0.13367833197116852], [0.19677750766277313, 0.2638705372810364, 0.26520031690597534, -0.004340855870395899, 0.31601831316947937, 0.03234419599175453, -0.10841498523950577, 0.41397398710250854, 0.43411684036254883, 1.4707121849060059, 0.26229485869407654, -0.494173526763916, -1.1139297485351562, 0.7170564532279968, -0.3261699378490448, -2.4232702255249023, 1.2306244373321533, 0.10754088312387466, -1.8558048009872437, -0.24558956921100616, 1.5805302858352661, -0.5472484827041626, 0.15063555538654327, 0.46920764446258545, 0.10391564667224884, -0.1915907859802246, 0.7591761350631714, -0.5905429124832153, -0.17792420089244843, -0.215016171336174, 0.33103424310684204], [0.48095986247062683, -0.6185263991355896, 0.21382993459701538, 1.0371853113174438, 0.5895959138870239, -0.3100070357322693, 0.3175446689128876, -0.15176433324813843, 0.19946163892745972, 0.2723955512046814, 1.2826857566833496, 1.964006781578064, 0.07146865874528885, -0.21487553417682648, -0.17858588695526123, -1.1723555326461792, -0.20069445669651031, -0.3023582100868225, -0.3761823773384094, -0.6605634689331055, -0.24269050359725952, 0.2325814664363861, 0.19730079174041748, -0.20893637835979462, 0.2197975516319275, -2.412583589553833, 0.7909851670265198, -0.4697912633419037, 0.736172616481781, 0.01872575841844082, 0.4556511342525482], [0.4587743580341339, 0.35857734084129333, -0.2478468269109726, 0.45369213819503784, -0.47050002217292786, -0.1504405438899994, 0.06195719912648201, 0.06592250615358353, 0.5452407598495483, -1.002233624458313, 0.20096619427204132, -0.1737622767686844, -1.619133710861206, -0.39694148302078247, -1.092629075050354, 2.8384761810302734, 0.9287282824516296, 0.27462562918663025, 3.1199655532836914, -0.2077256739139557, 0.03277574107050896, -0.10055871307849884, -0.9589976668357849, 0.02807142399251461, 0.3107658326625824, 0.5630953311920166, -0.08052612841129303, -0.3036925494670868, -0.04417308419942856, 0.16055649518966675, 0.20623105764389038], [-0.3598407208919525, 0.6064196825027466, -0.0678451880812645, -0.31300315260887146, 0.8527018427848816, 0.023714415729045868, 0.4722001254558563, 0.27708420157432556, 0.6946168541908264, -0.01960093528032303, -0.5727085471153259, 0.3491816222667694, 0.9196710586547852, 0.2250647395849228, -0.5483614206314087, 2.3658666610717773, 0.30255135893821716, 1.0150226354599, 0.42475876212120056, 0.5399816036224365, 0.8158000707626343, -0.46099168062210083, 0.4150732755661011, 0.5506529211997986, 0.18872885406017303, -1.0659923553466797, 0.04852084070444107, 0.23829659819602966, -0.47052326798439026, 0.22022360563278198, -0.23213237524032593], [-0.18726392090320587, -0.2825382351875305, -0.1454131156206131, -1.0098870992660522, -0.35972657799720764, 0.2590763568878174, -0.3801027834415436, -0.43649622797966003, -1.013580083847046, 2.0092885494232178, -0.06832530349493027, -0.5455363988876343, 0.431570440530777, -0.28315478563308716, 1.2990570068359375, -1.082703948020935, 0.41023433208465576, -0.07709380239248276, 0.8532571792602539, -0.8189753293991089, 0.4045141935348511, -1.0841602087020874, 0.07912339270114899, -0.3498631715774536, 1.2760621309280396, 2.301055669784546, -0.2853253185749054, 0.17643170058727264, 0.5799521803855896, -0.3420849144458771, -0.08938970416784286], [0.2965988516807556, -0.20754987001419067, -0.20522546768188477, 0.18979373574256897, -0.5389218926429749, -0.23372547328472137, 0.5217804908752441, -0.11481441557407379, -1.100899338722229, 0.6622284054756165, -0.11675318330526352, -2.19977068901062, 0.4493687152862549, 0.12901192903518677, -0.15399575233459473, 0.7258388996124268, -0.8950110673904419, -0.00522695342078805, 1.395017147064209, 1.0530394315719604, -1.2930097579956055, -0.538140058517456, 0.5495323538780212, 0.6563804149627686, -0.5166490077972412, 1.0521517992019653, -0.33348166942596436, -0.26138967275619507, 0.6307269930839539, -0.4252305030822754, -0.08824671804904938], [-0.3756205439567566, 0.7676253318786621, -0.3592883050441742, -0.961834192276001, -0.6801121830940247, 0.6756430864334106, -0.8311035633087158, -0.4341951608657837, -0.49564456939697266, 0.6780841946601868, 0.5290892720222473, 0.15360967814922333, -0.27392566204071045, 0.6539314389228821, -0.8392751812934875, -0.4186549782752991, 0.7104243636131287, -0.33786964416503906, -0.24407795071601868, -0.8177807927131653, -0.3436790406703949, 0.12490251660346985, -0.02482438087463379, -0.09776843339204788, -0.5940395593643188, 0.8383199572563171, -1.3535815477371216, 0.08947484940290451, -0.32441088557243347, 0.3499828577041626, 0.03920093923807144], [-0.3157261610031128, -0.14430463314056396, 0.05148475244641304, -0.06065061315894127, -0.5173096060752869, -0.05461962893605232, -0.16935475170612335, -0.23754185438156128, -0.529109537601471, -0.43856558203697205, 0.2725905179977417, 2.1360690593719482, 0.06223596632480621, -0.9345031380653381, -0.3875099718570709, -0.9674027562141418, -1.1001081466674805, -0.2407207190990448, -1.1918736696243286, -0.7326985597610474, -0.046593114733695984, 0.5121049284934998, 0.022880874574184418, 0.4757077991962433, 0.18793950974941254, -0.1036168783903122, 0.4658242166042328, 0.16946899890899658, 0.3295442461967468, -0.19106772541999817, 0.06743134558200836], [0.1539204716682434, -1.165246605873108, 0.023515857756137848, 0.3240848481655121, -0.12335315346717834, -0.4485209882259369, 0.2839837670326233, 0.2342517077922821, -0.22575122117996216, -0.1920219510793686, 0.14570219814777374, -2.084298849105835, -0.8449127078056335, -0.7453968524932861, -0.3369138836860657, 0.2731589376926422, -1.2485454082489014, -0.3568277359008789, 1.09275484085083, -1.7320209741592407, 0.1704191416501999, -0.663579523563385, 0.2746066749095917, 0.2059839814901352, 0.576666533946991, 0.17524315416812897, -0.0005326811806298792, -0.32102808356285095, 1.4665803909301758, 0.3218451142311096, 0.06438750773668289], [0.01534208282828331, -0.05332767218351364, -0.13395357131958008, -1.0242971181869507, 0.9258811473846436, -0.04480831325054169, -0.7344211339950562, 0.13050220906734467, -0.6013354063034058, -0.2854117751121521, 0.02627275511622429, 0.07066972553730011, -0.4198996424674988, 1.1619129180908203, -0.8883883357048035, 0.11203129589557648, 0.2532823979854584, -0.16977302730083466, -0.12181545794010162, -0.8572608232498169, 0.24530071020126343, 0.20859050750732422, -0.14728304743766785, -0.33673691749572754, 0.4649946987628937, -1.4662491083145142, 0.6589832305908203, -0.12103752046823502, -0.30933356285095215, -0.022711215540766716, -0.696141242980957], [0.13923688232898712, -0.18616436421871185, 0.29413676261901855, 0.6033438444137573, 0.5557142496109009, 0.2081630825996399, 0.17353063821792603, 0.33324214816093445, 0.08338551223278046, 1.2611217498779297, 0.8895794749259949, -0.6021537780761719, -0.6485137343406677, -0.30661770701408386, -0.6076834797859192, -3.7919719219207764, -0.04128427430987358, 0.6609193682670593, -0.619981586933136, -0.6633102893829346, -0.9690878987312317, 0.07167359441518784, -0.7334836721420288, 0.6496936678886414, -0.2595572769641876, -0.376991868019104, 0.4212815463542938, 0.0705559179186821, -0.18393059074878693, -0.17973577976226807, -0.03478838875889778], [0.22597967088222504, 0.01818777807056904, 0.060780253261327744, -0.6247227191925049, -0.10452599078416824, -0.44304758310317993, -0.3837253749370575, 0.35488128662109375, 0.29858994483947754, 0.04297943785786629, -0.5857251882553101, -0.6826701164245605, 0.5145366191864014, -0.6375901103019714, 0.3319736123085022, 2.5816540718078613, 0.19963152706623077, 0.6455979943275452, 1.359352946281433, -0.6374908089637756, 0.15965603291988373, -0.3524719178676605, -0.2671850323677063, 0.22335858643054962, -0.007677319925278425, 0.056682053953409195, -0.4912680983543396, -0.10269548743963242, 0.0679328665137291, 0.32296639680862427, -0.23712454736232758], [0.58661949634552, 0.3340369760990143, -0.23812738060951233, -1.1941797733306885, -0.33435729146003723, 0.017771244049072266, -0.166326105594635, 0.33277082443237305, 0.29999205470085144, 0.7744258046150208, 0.3493824303150177, -0.711145281791687, 0.5207378268241882, 1.0355663299560547, 1.3783022165298462, 0.7541822195053101, 0.5294140577316284, 0.20038627088069916, -0.05234625190496445, -2.0346896648406982, 0.24231983721256256, -0.5993454456329346, -0.2905706465244293, 0.2545454502105713, -1.4946434497833252, -0.5670071244239807, 0.8627631664276123, -0.07708344608545303, 0.3194708824157715, -0.27439454197883606, 0.20520402491092682], [0.286390095949173, 0.27995163202285767, -0.3815981149673462, 1.3211658000946045, 1.0008825063705444, -0.581589937210083, 0.37772664427757263, -0.38058021664619446, -0.14917223155498505, 0.8413388729095459, -0.20084892213344574, 0.15395407378673553, 1.3351577520370483, -0.932884931564331, 0.5855576992034912, -0.657626211643219, -0.7724206447601318, 0.4470459520816803, -0.39814385771751404, 0.06809782981872559, -0.34879082441329956, -0.15133178234100342, -0.8320281505584717, -0.31409138441085815, -0.09600373357534409, -0.16835179924964905, -1.1024341583251953, -0.42651572823524475, -0.3539661765098572, 0.21452979743480682, 0.2666531205177307], [0.5348305702209473, 0.3083992898464203, -0.2120555341243744, -0.8079414963722229, -0.46015170216560364, 0.5605082511901855, -1.5329409837722778, 0.2608286142349243, 0.07972618192434311, 0.5420815944671631, -0.7620843648910522, 0.2970089316368103, -0.5762823820114136, -0.44860726594924927, -0.9677515029907227, 0.37197089195251465, 1.073244571685791, -0.257574200630188, 0.7422356009483337, -0.029116343706846237, 0.25026118755340576, 0.04226864129304886, 0.546045184135437, 0.21040606498718262, 0.6786608099937439, 1.497604250907898, -0.1571328490972519, -0.5515083074569702, -0.3088379502296448, 0.10137271136045456, 0.3218660056591034], [0.02727934718132019, -0.43159249424934387, -0.020553581416606903, 0.6562311053276062, 0.30349358916282654, 0.006238221190869808, -0.1067642942070961, -0.1854034960269928, 0.12234057486057281, -0.7358368635177612, -0.04125338792800903, -1.8640720844268799, 0.6369256377220154, -0.09335672110319138, 0.24708430469036102, 1.298158884048462, 0.013507522642612457, 1.3078311681747437, 0.8402671217918396, 0.7311627864837646, -0.4523691236972809, 0.46723127365112305, 0.8308722376823425, 0.18069547414779663, -0.4565419554710388, 1.0693905353546143, -0.7553561925888062, -0.17456857860088348, 0.47746050357818604, -0.3435596525669098, 0.1064993292093277], [0.4013546109199524, 0.6251990795135498, -0.02499288320541382, 0.6756789088249207, 0.43834206461906433, -0.21210432052612305, -1.7826850414276123, 0.005986116826534271, 0.7902331948280334, -1.0270030498504639, 0.537825882434845, 0.42931780219078064, 0.721985936164856, 0.13535191118717194, -0.2664039731025696, -0.7899031639099121, 0.9139006733894348, 1.9249160289764404, -0.9179577827453613, 0.544310450553894, 1.9887957572937012, -0.41038352251052856, 0.05930642783641815, 0.6602637767791748, 0.9257289171218872, 0.06849502772092819, -0.09985154122114182, -0.5547024011611938, -0.7194960117340088, 0.05670616403222084, 0.28175145387649536]], "biases": [-0.025711851194500923, 0.008000257425010204, 0.18931297957897186, 0.004886779468506575, -0.03522408753633499, -0.18955233693122864, -0.20740428566932678, -0.16725489497184753, -0.09479349851608276, 0.07989947497844696, -0.1047869473695755, -0.027113312855362892, 0.04336683824658394, 0.04042784869670868, -0.15986718237400055, 0.003534190123900771, 0.20955967903137207, -0.048015762120485306, -0.11728587001562119, -0.027835316956043243, -0.1408730447292328, 0.14913134276866913, 0.030613895505666733, -0.05880972370505333, 0.17396685481071472, 0.0921584740281105, 0.1776161640882492, -0.000747458660043776, -0.10042879730463028, 0.20053179562091827, -0.08324036002159119, -0.23362548649311066, 0.14012248814105988, 0.12735573947429657, -0.005815979093313217, 0.06403452903032303, -0.05040629208087921, -0.04221174120903015, 0.012285836040973663, 0.11316249519586563, 0.08832170069217682, -0.02654496394097805, -0.07300498336553574, -0.15756511688232422, 0.014100714586675167, -0.002897772006690502, -0.00041322316974401474, 0.184908926486969, 0.08990196883678436, 0.06896142661571503, 0.014281596057116985, -0.011095487512648106, 0.08119723945856094, -0.17926709353923798, -0.08810196071863174, -0.07826272398233414, 0.0402865968644619, -0.09521450847387314, -0.15919390320777893, 0.151686429977417, 0.22526469826698303, 0.25461331009864807, -0.009615182876586914, -0.060651250183582306], "activation": "tanh"}, {"weights": [[-0.26858991384506226, -0.8803926110267639, -0.175493061542511, 0.7287434339523315, 0.36532652378082275, 1.1836124658584595, -0.4841229319572449, 1.52265202999115, 0.39340463280677795, 0.1926397681236267, -0.47678467631340027, -0.0333169661462307, -0.6910607218742371, 0.2653118073940277, 0.4091465473175049, 0.3971278667449951, 0.27625688910484314, -0.275641530752182, 1.0361688137054443, -0.9559720158576965, 0.27262723445892334, 0.5019906759262085, -0.019868772476911545, -0.08028484880924225, 0.31985101103782654, -0.035836778581142426, 0.08682826906442642, -0.6149583458900452, -0.8810451030731201, -0.9215654730796814, -0.6511422991752625, -0.32047829031944275, 0.46253520250320435, -0.7656152248382568, -0.4325225055217743, 0.9982739090919495, -0.10263596475124359, 0.3474268913269043, -0.07130088657140732, 0.6933578252792358, -0.2526565492153168, 0.1417969912290573, -0.11346657574176788, -0.262228786945343, 0.006330885924398899, -0.5310674905776978, 0.47205349802970886, 0.895899772644043, 0.39758822321891785, -0.3366633951663971, -0.25187021493911743, 0.24264319241046906, -0.3669900596141815, 0.9376022219657898, 0.6403771638870239, -0.7811033725738525, -0.25189876556396484, 0.33114078640937805, -0.7034898400306702, 0.31005463004112244, -0.07840621471405029, 0.27593526244163513, -0.7162179946899414, 1.0166771411895752], [-0.8566887378692627, -0.41204315423965454, 0.47861695289611816, -1.2662694454193115, -0.3088938295841217, -0.011365991085767746, 0.03202860429883003, 0.10407065600156784, -0.4794103503227234, -0.14227208495140076, -0.40040820837020874, 0.5128381848335266, -0.07816825062036514, -0.7312759160995483, 0.2938980758190155, 0.6675187349319458, 0.030490444973111153, 0.3125169575214386, 0.047048576176166534, 0.0314808115363121, 0.6543048024177551, -0.02916112169623375, -0.2390885055065155, -0.13056574761867523, 0.29387351870536804, -1.3122382164001465, -0.6558917760848999, -0.21544016897678375, 0.43868741393089294, 0.5676335692405701, -0.07462131232023239, -0.29872065782546997, 0.23546253144741058, 0.8677670955657959, 0.1192568987607956, 0.5486143231391907, -0.5613360404968262, 0.9827077984809875, -0.040784548968076706, 0.3484368324279785, 0.3423377275466919, 0.8649640679359436, 0.13743233680725098, -0.08346492052078247, 0.8684433698654175, 0.48778682947158813, -0.2964746057987213, 0.7357478141784668, 0.3901081383228302, 0.15477874875068665, 0.38438156247138977, 0.5414373874664307, -0.6301776170730591, 0.23017337918281555, -0.40268057584762573, -1.1719963550567627, 0.08528807759284973, 0.3487979471683502, -0.019539669156074524, 0.6149886846542358, -1.0213730335235596, 0.6337916254997253, 0.3280608355998993, 1.129940390586853], [0.5646363496780396, 0.41582584381103516, -0.3541962206363678, 0.4570062458515167, -0.18046283721923828, -0.39785605669021606, -0.8927167057991028, 0.12192066758871078, 0.35218819975852966, 0.1633521467447281, -0.43834608793258667, -0.14895819127559662, -1.0543633699417114, 0.05572378262877464, 0.2966940104961395, 0.618480920791626, 0.2579405903816223, -0.6334313750267029, 0.7328931093215942, 0.47830891609191895, 0.541700541973114, 0.032475121319293976, -0.33461177349090576, 0.009322594851255417, 0.1192246824502945, -0.42246460914611816, -0.23509570956230164, 0.3338738679885864, 0.10330743342638016, -0.3582238554954529, 0.23560136556625366, -0.7932535409927368, -0.14652007818222046, -0.1366889923810959, 0.5633927583694458, 1.0249627828598022, -0.6042234301567078, -0.29170525074005127, 0.15648309886455536, 1.054022192955017, 0.41406118869781494, 0.3533244729042053, 0.43958884477615356, -1.0122023820877075, 0.11884120851755142, -0.22367708384990692, 0.1288568377494812, 0.5337108373641968, 0.3428105115890503, -0.49691522121429443, -0.47040876746177673, 0.26872384548187256, 0.3438851833343506, -0.5013525485992432, 0.6850315928459167, 0.6747360229492188, -0.21487128734588623, 1.6077131032943726, -0.7491133213043213, -0.44259366393089294, 0.5075628161430359, 0.283733993768692, 0.5827034711837769, 0.16205254197120667], [0.4027978181838989, 0.3253728151321411, -0.17025315761566162, 0.3124324381351471, -0.25998052954673767, -0.2446150928735733, -0.7423665523529053, -0.7246736288070679, 0.04845931753516197, 0.4155547618865967, -0.6077316999435425, 0.01104936096817255, -0.09177826344966888, 0.4484623670578003, -0.7781371474266052, 0.5740388631820679, -0.19635972380638123, -0.16096331179141998, 0.024337584152817726, 0.2811517119407654, -0.8594304323196411, -0.0252555999904871, 1.4476044178009033, 0.11917807906866074, 1.3414875268936157, -0.07902497053146362, -0.49849259853363037, -0.6425914764404297, 0.22321480512619019, -0.09706287086009979, 0.03834932669997215, -0.36339184641838074, 0.33482587337493896, 0.6031606793403625, 0.6835650205612183, -0.5302177667617798, -0.8156614303588867, -0.17521622776985168, -0.6963517665863037, 0.04710537567734718, -0.5350344181060791, 0.1979687362909317, 0.6426767110824585, -0.2412528097629547, 0.17167609930038452, 0.3215070366859436, -0.5309687852859497, 0.6220839023590088, 0.06829074770212173, 0.14818856120109558, 0.27622437477111816, 0.3155226409435272, 0.3151553273200989, -0.3996555507183075, -0.5457673072814941, 0.5034137964248657, 0.019960591569542885, 1.0657920837402344, 0.5516753196716309, 0.9381482005119324, -0.018273483961820602, 0.6886007785797119, -0.09257584065198898, -0.31578877568244934], [-0.2125994861125946, 0.05444193258881569, 0.30943772196769714, -0.3350163996219635, -0.6922782063484192, 0.24909448623657227, 0.9027829766273499, -0.3008936047554016, -0.40761443972587585, -1.174436092376709, -0.013786116614937782, -0.27806851267814636, 0.3367643654346466, -0.5879755616188049, -0.5748070478439331, 0.1985870897769928, -0.04024413600564003, -0.15514157712459564, 0.13030779361724854, 0.12008842080831528, -0.4612916111946106, -0.3030804693698883, -0.8993830680847168, 0.8155161142349243, -1.5988601446151733, 0.1117967888712883, 0.10286469757556915, 1.4179191589355469, 0.056067630648612976, 0.7113032341003418, 0.2922031879425049, -0.8518880605697632, -0.5214735865592957, 0.5868503451347351, 0.030887847766280174, 0.4658340513706207, -0.4698723256587982, 0.49092695116996765, -0.0628746747970581, 0.32429298758506775, -0.605805516242981, 0.07401999831199646, 0.7374993562698364, 0.08160802721977234, -0.5222500562667847, -0.4881954491138458, 0.465261310338974, -0.7472295761108398, -0.09401661902666092, 0.3217161297798157, 0.6386223435401917, -0.5981647968292236, 0.3988063335418701, -0.9629002213478088, -0.31324687600135803, 0.3095210790634155, -0.8191533088684082, -0.1859177201986313, -0.10450097173452377, -0.6233547925949097, 0.06060885265469551, 0.18944519758224487, 1.282225489616394, 1.3311444520950317], [0.003775170771405101, 0.09823580831289291, 0.8350533246994019, -0.3673534691333771, 1.0958622694015503, 0.6802770495414734, -0.22584182024002075, 0.34335586428642273, -0.08937883377075195, 0.23139886558055878, -0.26835566759109497, 0.12104400992393494, -0.24807195365428925, 0.34882059693336487, -0.7456581592559814, 0.1715012639760971, 0.4587622582912445, 0.3261015713214874, 1.1416634321212769, -0.15692883729934692, 0.319159597158432, 0.09236141294240952, -0.17009487748146057, 0.8044149279594421, 0.8256756067276001, -0.2543458044528961, -0.184882253408432, -0.2749890685081482, 0.5714864134788513, 0.3407129645347595, 0.3683856725692749, 0.16746097803115845, -1.039034128189087, 0.8504417538642883, 0.18184645473957062, -0.015001035295426846, -0.8573463559150696, 0.9233683347702026, 0.16250485181808472, 0.12365644425153732, -0.8259143829345703, 0.5863765478134155, 0.4321041405200958, -0.49444255232810974, 0.5721136331558228, 0.06824494153261185, -0.13121344149112701, 0.2838122844696045, -0.6946507692337036, -0.999738872051239, -0.8344466686248779, 0.5695533156394958, 1.4886512756347656, -0.7567752003669739, -0.6680445075035095, 0.4310517907142639, 0.028771372511982918, 0.042167458683252335, -0.13798758387565613, 0.9748432636260986, -0.11305967718362808, -0.14889082312583923, 0.28767916560173035, -0.7876322269439697], [-0.48003309965133667, 0.39516717195510864, -0.2685529589653015, -0.18864187598228455, 0.4046672284603119, -0.15926121175289154, 1.4651644229888916, 0.5096290111541748, -0.7860770225524902, -1.2421352863311768, -0.32840049266815186, -0.3288649916648865, -0.2866443991661072, -0.09884621202945709, -0.2619222402572632, -0.2602021396160126, 0.3416216969490051, 0.4109889566898346, -0.21051256358623505, -0.053046759217977524, 0.4876067042350769, 0.7328717708587646, 0.05311858654022217, 1.3043338060379028, -0.7166907787322998, -0.4539499878883362, 0.04589148983359337, -0.3975794017314911, -0.5854105353355408, 0.19520510733127594, 0.3594616651535034, 0.46624401211738586, 0.5296103954315186, -0.4203190207481384, -0.14461037516593933, 0.36466550827026367, 0.04790627956390381, 0.48972490429878235, -1.4397653341293335, -0.2050728052854538, 0.38622307777404785, 0.34415575861930847, 0.5902184247970581, 0.19964216649532318, 0.17516788840293884, -0.5557155013084412, 0.7478794455528259, 0.09298194199800491, -0.13862410187721252, -0.41843801736831665, 0.38277512788772583, -0.23899796605110168, -0.683428943157196, -0.39847415685653687, 0.45258280634880066, -0.66070955991745, -0.92625492811203, -0.9695783257484436, -0.10105044394731522, 0.03224828094244003, -0.02239028550684452, -0.09155071526765823, -0.10313675552606583, 0.4439442753791809], [1.3217147588729858, 0.30924808979034424, -0.7456761002540588, -0.0034375234972685575, 0.010829851031303406, -1.1578855514526367, -0.12194743007421494, -0.31743505597114563, -0.5697538256645203, -0.28490057587623596, 0.5191757082939148, -0.2259858399629593, 0.48473671078681946, 0.031025182455778122, -0.07815086096525192, 0.13629820942878723, -0.2366734892129898, 0.1811540275812149, -0.601107656955719, 0.9703599214553833, -0.28240570425987244, -0.365397572517395, 0.08066046983003616, 0.5569601058959961, 0.016297010704874992, -0.27615976333618164, -0.38810020685195923, 0.7204360365867615, 0.7454237937927246, 0.062144167721271515, 0.2875519394874573, 0.1630610227584839, -0.0004730704822577536, 0.46478208899497986, 0.3961910307407379, -0.25442102551460266, 0.1252402514219284, -0.14739663898944855, -0.43142274022102356, -0.2396543174982071, 0.24154850840568542, 0.7358044385910034, -0.01867605373263359, -0.37675169110298157, 0.4716220200061798, -0.04856584966182709, -1.0316909551620483, 0.10856419801712036, -0.10276629775762558, 0.4995826780796051, 0.30061355233192444, -0.647284984588623, -0.0744803324341774, -0.9621489644050598, -0.8087669014930725, 0.1950668841600418, -0.33742624521255493, -0.6142959594726562, 0.9577800631523132, -0.35320940613746643, 0.9515750408172607, -0.43459638953208923, 0.576178252696991, 0.7844505310058594], [-0.1761978566646576, -0.2093510776758194, 0.864495038986206, 1.2909678220748901, -0.07054934650659561, 0.08234713971614838, 0.2581261396408081, 0.004136879462748766, 0.22832323610782623, 0.6690795421600342, 0.11042491346597672, -0.6774465441703796, 0.42133063077926636, 0.2759993076324463, 0.5736172199249268, 0.17741811275482178, 0.23122790455818176, -0.7166213393211365, -0.1543615311384201, 0.07376374304294586, -0.9135354161262512, -0.8151782155036926, 0.9969415068626404, -0.6088348031044006, 0.5734524130821228, 0.08781924843788147, -0.47134295105934143, 0.08440618216991425, -0.6573649048805237, 0.7837097644805908, -0.22640222311019897, -0.3079908788204193, 0.9298328757286072, -0.14190702140331268, -0.1262531876564026, 0.3744608461856842, -0.1457434892654419, -1.2795438766479492, -0.012158608995378017, -0.2986009120941162, -0.36691099405288696, -0.4102608859539032, 0.13442948460578918, -0.6698336005210876, -0.06563404202461243, -0.5647316575050354, -0.151301309466362, 0.5096102356910706, 0.5371527671813965, 1.1243364810943604, 0.23394176363945007, -1.3362460136413574, -0.10795833170413971, -0.33997491002082825, -0.13878968358039856, 0.15063056349754333, 0.017893288284540176, 0.3740904927253723, 0.5152573585510254, 0.6152447462081909, 0.182191401720047, 0.07762917131185532, 0.15642349421977997, 0.2981356382369995], [-0.3598978817462921, -0.2651832401752472, 0.7985305190086365, -0.27903440594673157, -0.3228992521762848, 0.435697466135025, 1.315330982208252, 0.8500860333442688, -0.6226081848144531, 0.06103057414293289, -0.29164260625839233, -0.10773163288831711, -0.2074843943119049, -0.036593079566955566, -0.8362637162208557, -0.13404522836208344, 0.6765540242195129, 0.4358043074607849, 0.31866249442100525, 0.04298859462141991, 0.05777592584490776, 0.12373366206884384, 0.07434165477752686, 1.733417272567749, -0.10549211502075195, -0.49581462144851685, -0.3874030113220215, -0.5681264996528625, 0.1759418398141861, 0.4977153539657593, 0.9058506488800049, -0.18056637048721313, -0.13901832699775696, -0.3961583077907562, 0.6235479712486267, 0.3142733871936798, -0.5061337947845459, -0.20528428256511688, -0.564799964427948, -0.005147840827703476, 0.47281721234321594, 0.5750415921211243, 0.31354424357414246, -0.09422186017036438, 0.346468985080719, -0.7029070258140564, -0.36778995394706726, -0.584640383720398, 0.5063212513923645, -0.9984685778617859, 0.015378973446786404, -0.013910328969359398, 0.09368997067213058, -0.9923146367073059, 1.4137916564941406, -0.24499006569385529, -0.39884474873542786, 0.5210859179496765, -0.6167714595794678, 0.24001973867416382, 0.03526124358177185, 0.009253960102796555, 0.18738466501235962, 0.4463522732257843], [-0.1386459916830063, -0.22764889895915985, -0.658611536026001, 0.11767987906932831, -0.8299256563186646, -0.21054016053676605, 0.41098690032958984, 0.8674471378326416, 0.7407540082931519, -0.22123458981513977, 0.10849392414093018, 0.45266014337539673, -0.9281235337257385, -0.20639103651046753, 0.5931476950645447, -0.6712300777435303, -0.01696900837123394, 0.14926958084106445, -0.3995753824710846, -0.365936279296875, 0.6106324195861816, -0.09836333990097046, -0.16556020081043243, -0.6961102485656738, -0.7938899993896484, 1.387540340423584, -0.10404998064041138, -0.42124372720718384, -0.5993583798408508, 0.02640095353126526, -0.10316020995378494, -0.8431614637374878, 0.5244629383087158, -0.9646133184432983, -0.5358114242553711, 0.004121113568544388, 0.8561708927154541, -1.3822896480560303, 0.11268208175897598, -0.17912232875823975, 0.22016096115112305, -0.49490299820899963, -0.2987746000289917, 0.322122186422348, -0.21729597449302673, -1.1279057264328003, 0.39423826336860657, -1.14548659324646, 0.26377472281455994, 0.8095209002494812, -0.5205593705177307, -0.43877848982810974, -0.5263779759407043, 0.35918280482292175, 0.821033775806427, -0.20325054228305817, -0.4055081903934479, -0.18223948776721954, -0.5561245083808899, -0.879960834980011, 0.5339585542678833, 0.08129139244556427, 0.07276921719312668, -0.2816557288169861], [0.6304290294647217, 0.8965656757354736, -0.19580304622650146, 0.893065333366394, -0.22178244590759277, 0.4058596193790436, -0.008223539218306541, -0.5571658611297607, 0.4488414227962494, 0.14028140902519226, -0.08726854622364044, 0.0987008586525917, -0.00543258199468255, -0.5215153098106384, -0.5377306342124939, -1.0041744709014893, -0.8691244125366211, -0.34031108021736145, 0.08861534297466278, 0.04003084823489189, -0.362122505903244, -1.118872046470642, 0.3544963598251343, 0.3976443409919739, -0.40338170528411865, 0.7243332266807556, 0.30709466338157654, 0.7945301532745361, -0.020458975806832314, 0.5459760427474976, 0.005937198176980019, 0.4043121635913849, -0.6752533912658691, -0.47180309891700745, 0.0656568631529808, -0.5193695425987244, 0.7708470225334167, -0.247676283121109, 0.4245411157608032, -0.3722025156021118, 0.04980752617120743, 0.14122870564460754, -0.47835034132003784, -0.10157536715269089, 0.034872546792030334, 0.4646393656730652, -0.6772860288619995, -1.4337592124938965, -0.5562114715576172, 0.5091385245323181, -0.25420743227005005, -0.7220991253852844, 0.013439088128507137, -0.6707005500793457, 0.06456340104341507, 0.2722185254096985, 0.3950416147708893, -0.5270026922225952, 0.9720513820648193, 0.46436381340026855, 0.31740713119506836, 0.03579586744308472, -0.014101865701377392, -0.3806242048740387], [-0.6652542352676392, -0.3637858033180237, 0.8001901507377625, -0.21060742437839508, -0.7240539789199829, 0.4374763071537018, 0.23751138150691986, 0.25961095094680786, -0.8096344470977783, 0.5054351687431335, -0.13176609575748444, 1.137427568435669, 0.16038060188293457, -0.762116551399231, -0.6022933721542358, -0.12482460588216782, 0.4861498177051544, -0.7871569395065308, -0.0506085604429245, -0.06104448810219765, 1.101694941520691, -1.3067433834075928, 0.08886674791574478, -0.8275467157363892, -0.3743020296096802, 0.2416035234928131, 0.41958194971084595, 0.15633562207221985, -0.1362897902727127, -0.20074017345905304, -0.3454587161540985, -0.7505072355270386, -0.25030091404914856, -0.13653086125850677, -1.064268708229065, 0.13285961747169495, 0.29066362977027893, -0.2860463261604309, 1.0166065692901611, 0.16210663318634033, -0.45280659198760986, -0.6503522992134094, -0.5801578760147095, 0.19230811297893524, 0.3279672861099243, 0.40113702416419983, 0.07344949245452881, 0.17090217769145966, -0.48115476965904236, 0.11579103767871857, -0.3037651479244232, -0.48535311222076416, -0.3497332036495209, 1.4014322757720947, 0.14921866357326508, -0.8125302791595459, 0.9620984196662903, 0.3065279722213745, -0.26258906722068787, -0.1378108412027359, -0.6519872546195984, 0.054463498294353485, -0.45285236835479736, 0.42124226689338684], [0.3708036541938782, 0.23292405903339386, -0.43770042061805725, -0.11641791462898254, -0.6288750767707825, -0.4847656786441803, -0.03338715434074402, 0.28073689341545105, 0.09253836423158646, -0.45566168427467346, 0.9921374320983887, -0.3331068456172943, -0.5605544447898865, -0.381259024143219, 0.08445922285318375, -0.791993260383606, -0.42519423365592957, -0.582714855670929, -0.5646874308586121, -0.20791377127170563, -0.6631632447242737, -0.25106289982795715, 0.08478628844022751, 0.1287967413663864, -0.7817326188087463, 0.9138734340667725, -0.18425463140010834, 0.771760880947113, 0.25184789299964905, -0.11953064054250717, 0.032466258853673935, 0.07212796062231064, 0.340670108795166, -0.19187457859516144, -0.14861957728862762, 0.17926551401615143, 2.1510071754455566, -1.3109771013259888, -0.019689291715621948, -0.19972874224185944, 0.3543807864189148, -0.29588064551353455, -0.3647492527961731, -0.4020490050315857, -0.22731398046016693, -0.9288572072982788, 0.020886559039354324, -0.6522638201713562, 0.2898416817188263, 0.7584104537963867, 0.2062104046344757, 0.10721538215875626, 0.1442621648311615, -0.027779284864664078, -0.07905196398496628, -0.027129217982292175, 0.3672861158847809, -0.9283031225204468, 0.6015563607215881, 0.1584503948688507, 0.694511890411377, -0.030383434146642685, -0.05433735251426697, -0.0423051081597805], [-0.7025123834609985, 0.6983584761619568, 0.34922468662261963, -0.5098455548286438, -0.2723993957042694, -0.2360234409570694, 0.26561444997787476, -0.22185179591178894, 0.29410865902900696, 0.3394968807697296, -0.7019520401954651, 0.6018360257148743, 0.13163943588733673, -0.0010307823540642858, -0.5456092357635498, -0.6385861039161682, -0.0383104644715786, 0.5345087051391602, 0.17117899656295776, 0.47133105993270874, -0.5150694251060486, -0.219855397939682, 0.3379783034324646, 0.05242983624339104, -0.039113618433475494, 0.6359801292419434, 0.06200128793716431, 0.4456958770751953, 0.5637887716293335, 1.2007410526275635, -0.16138657927513123, 0.541334867477417, -0.1671958714723587, 0.0322665311396122, 0.2764253318309784, 0.07359366118907928, 1.1604249477386475, 0.5436397194862366, 0.3788605332374573, -0.1460762321949005, -0.14427295327186584, -0.32717230916023254, -0.5623636245727539, 0.2641547918319702, -0.3708305358886719, 0.5042265057563782, -0.5520554184913635, -0.5232001543045044, 0.004113742150366306, 1.4857311248779297, 0.1224885880947113, -0.235642671585083, -0.21868589520454407, 0.18326108157634735, -0.1234179139137268, -0.1647324413061142, 0.33304503560066223, -1.1906172037124634, 0.16593125462532043, 0.2791943848133087, -0.4230016767978668, 0.5963057279586792, 0.47881871461868286, -0.39054957032203674], [-0.39284205436706543, -0.22271563112735748, -0.939502477645874, -0.851894199848175, 0.923736572265625, 0.053742360323667526, -0.08077467232942581, 0.09774808585643768, 0.4958094656467438, -0.17591150104999542, 0.4586237668991089, -0.11696186661720276, -0.023676978424191475, -0.09369927644729614, 0.4550985097885132, 0.7151793837547302, 0.416155606508255, 0.19667384028434753, 0.4929186999797821, 0.5067948698997498, -0.10491368919610977, 0.9076743125915527, -0.12793314456939697, 0.5819640755653381, 0.6501755714416504, -0.5192146897315979, 0.5234456062316895, -0.1495836228132248, -0.06083110347390175, 0.1852961778640747, 0.6067084074020386, -0.1287086308002472, 0.8008235692977905, 0.1879565268754959, 0.038062237203121185, -0.47785934805870056, -0.6230278611183167, 1.339801549911499, -0.2278483808040619, 0.45497146248817444, 0.7222754955291748, 0.64890056848526, -0.251110702753067, 0.5251243114471436, 0.06451268494129181, 0.04713406786322594, 0.6269079446792603, 0.6391866207122803, 0.6458538174629211, -0.8792214393615723, 0.42027145624160767, -0.4205353558063507, -0.018162574619054794, -0.1771552711725235, 0.04631173610687256, -0.7007535099983215, -0.6266342997550964, -0.08523166924715042, -0.7570703625679016, -0.1347590535879135, -0.054714467376470566, -0.21908186376094818, -0.1638973206281662, -0.4625527262687683], [0.2585522532463074, 0.11897498369216919, -0.46131739020347595, 0.6607919931411743, -0.588764488697052, -0.7253416180610657, 0.8262731432914734, 0.30029600858688354, 0.9357724785804749, -0.5930327773094177, 0.45220476388931274, -0.3175692856311798, -0.2291436642408371, 0.15105853974819183, 0.7800558805465698, -0.12555062770843506, 0.6225930452346802, 0.06919818371534348, 0.08239271491765976, -0.09832339733839035, -0.23812046647071838, -0.015669794753193855, 0.8524523377418518, 0.03767891600728035, -0.1390768438577652, 0.1756116896867752, 0.29563063383102417, 0.37629643082618713, -0.5332661271095276, -0.1897030472755432, 0.20786035060882568, 0.1466423124074936, -0.09165585786104202, 0.35447895526885986, 0.2296847552061081, 0.27892351150512695, 0.8841118812561035, -0.5661367177963257, -0.8409940600395203, 0.5617172718048096, 0.25819823145866394, -0.988825798034668, 1.0199412107467651, 0.12383918464183807, -0.3393443822860718, -0.9240338802337646, -0.7389121651649475, 0.29310309886932373, 1.0241934061050415, 0.6401276588439941, -0.4948936402797699, 0.13937215507030487, 0.7174156904220581, -1.0820813179016113, -0.3073943257331848, 0.9950302839279175, 0.001144477748312056, 0.18918487429618835, 0.13553546369075775, -0.04535220190882683, -0.301140695810318, -0.050814636051654816, 0.7651317119598389, -0.030771177262067795], [-0.5387178659439087, 0.7310386300086975, 0.9516421556472778, -0.8588510155677795, 0.6523151993751526, 0.5264340043067932, 0.10629453510046005, -0.23580338060855865, -0.3790130317211151, -0.8280941247940063, -0.1797424852848053, 0.10251504927873611, 1.0447332859039307, 0.224382221698761, 0.38959911465644836, 0.22210419178009033, -0.3485901653766632, 0.4468747079372406, 0.5064748525619507, -0.01913878507912159, 0.013203524053096771, 0.28879082202911377, -0.23068460822105408, 0.45573946833610535, 0.10447162389755249, -0.3600275218486786, 0.11314214766025543, 0.11657577753067017, 0.131100594997406, 1.154347538948059, 0.14175565540790558, -0.13315236568450928, 0.14427106082439423, 0.7112497687339783, 0.4852129817008972, 0.3916344940662384, -0.5826886296272278, 0.8200644254684448, 0.8100209832191467, -0.22608385980129242, 0.15363597869873047, -0.052533622831106186, -0.1424766331911087, 0.5927441120147705, -0.3407948315143585, 0.5208740234375, 0.21003437042236328, -0.11751116812229156, 0.13644789159297943, 0.5914900898933411, 0.22473162412643433, 0.6816579699516296, 0.20974138379096985, -0.6649277210235596, 0.16856001317501068, -0.9426714181900024, -0.4941462576389313, -0.7340720891952515, 0.7566215395927429, 0.40461087226867676, 0.3321739435195923, -0.07139068096876144, 0.8918262720108032, 0.14147256314754486], [-0.2685970067977905, 0.32916784286499023, 0.041001688688993454, 0.22886396944522858, -0.25532981753349304, -0.4259527325630188, 0.1579858511686325, -0.9989697933197021, 0.013118773698806763, 0.5953852534294128, 0.33912599086761475, 0.3638560473918915, 0.2033698707818985, -0.09872011095285416, -0.3392525613307953, -0.8694298267364502, 0.49687138199806213, 0.8474025130271912, -0.951632022857666, -0.3727366626262665, 0.30513277649879456, 0.1532832682132721, 0.38489314913749695, -0.4754294455051422, 0.24618852138519287, 1.4482697248458862, 0.29521894454956055, -0.017034612596035004, -0.10139741003513336, -0.2690942883491516, -0.46576178073883057, 0.46108224987983704, 0.1386849582195282, -0.10793212056159973, 0.10143640637397766, -1.5047508478164673, 0.48667529225349426, -0.17600800096988678, -0.7283354997634888, -0.6623680591583252, -0.7444618940353394, -0.6663853526115417, 0.053398456424474716, 0.5313229560852051, -1.0038553476333618, 0.4230891466140747, -0.9337508082389832, -0.7512387633323669, -0.5977792143821716, 0.4352271258831024, -1.040827751159668, 0.3872038424015045, 0.8886813521385193, 0.6730218529701233, -0.4209226369857788, 1.1733174324035645, -0.4905450642108917, -0.917957603931427, 0.28688690066337585, 0.2335217297077179, -0.41738077998161316, 0.1265222132205963, 0.39003822207450867, -1.3620489835739136], [1.271164894104004, -0.2576178312301636, -1.2140181064605713, 0.7647606134414673, -0.5941591858863831, -0.3592756688594818, -0.8397913575172424, -0.44995683431625366, -0.14366357028484344, -0.4771307408809662, -0.03319024667143822, -0.10590031743049622, -0.4234999120235443, -0.7475995421409607, 0.4515496492385864, -0.48213714361190796, -0.40291574597358704, -0.3475067615509033, -0.15977531671524048, -0.1628476232290268, 0.38104239106178284, -0.8231481909751892, -0.33800360560417175, -0.41143983602523804, -1.4002342224121094, -0.14764560759067535, 1.0183604955673218, 0.5728771686553955, -0.01258512306958437, -0.2623874545097351, -0.6656293869018555, -0.22224920988082886, -0.6581847667694092, -0.7198300957679749, -0.25544530153274536, 0.6380367279052734, -0.7365259528160095, -0.3110143840312958, -0.40580615401268005, -0.0028267991729080677, 0.3249225616455078, -0.28962084650993347, 0.6299553513526917, -0.21236294507980347, -0.40750929713249207, -0.5423155426979065, 0.08888185769319534, 0.6338714361190796, -0.3204336166381836, 0.1709064543247223, -0.10044346004724503, -0.6283197402954102, -1.1382255554199219, -0.3255874812602997, -0.06220864877104759, 0.482295960187912, 0.40090399980545044, 0.33012887835502625, -0.39887574315071106, -0.35540711879730225, 0.23421533405780792, -0.31752341985702515, -0.3490537703037262, 0.5251176357269287], [-0.3838750422000885, -0.3464794456958771, 0.04445662349462509, 0.38457998633384705, -0.6304214596748352, 0.575380265712738, -0.6295069456100464, 0.3761866092681885, 0.5598293542861938, 1.0134470462799072, -0.3798341751098633, 0.8673900961875916, 0.005599422380328178, 0.0838640034198761, -0.22313782572746277, 0.11349967867136002, -0.22296230494976044, -0.41534075140953064, -0.6863978505134583, -0.37345728278160095, 0.05937840789556503, 0.06889112293720245, 1.228554368019104, -0.30897292494773865, 0.5081403255462646, 0.04321329668164253, 0.5428298115730286, 0.4266011714935303, -0.6040518879890442, 0.4401458501815796, -0.7956951260566711, 0.05719742551445961, 0.07436684519052505, -0.30088719725608826, -0.27779051661491394, 0.31457480788230896, 0.8996129035949707, -0.47696205973625183, 0.31807735562324524, -0.05514683946967125, -0.03267462179064751, -1.0507527589797974, -0.5217869281768799, -0.7206051349639893, 0.27166420221328735, 0.9863677620887756, -0.07333730161190033, 0.38157418370246887, 0.5100879669189453, 0.5208761692047119, -0.45111340284347534, 0.05710633844137192, 0.17876124382019043, 0.9851651787757874, 0.15171033143997192, -0.13469776511192322, 0.20598724484443665, -0.2216276079416275, 0.08494116365909576, 0.17193758487701416, -0.39078718423843384, 0.19489789009094238, -0.06370758265256882, -0.008569148369133472], [0.7055855989456177, -0.147139310836792, -0.029739556834101677, 0.9023157954216003, 0.5806978940963745, 0.3323290944099426, -0.5082922577857971, -0.6935105919837952, -0.36287033557891846, 1.0008084774017334, 0.5386233925819397, 0.6264660954475403, 0.009568152949213982, -0.25171974301338196, -0.9085391163825989, 0.45126262307167053, -1.5468491315841675, -0.9370296597480774, 0.3595731556415558, 0.04019434377551079, -0.09854790568351746, -0.36438336968421936, -0.46934446692466736, -0.04274361580610275, 0.7566497325897217, 0.40109848976135254, -0.2681743800640106, 0.9453946352005005, 0.34548038244247437, -0.127247616648674, -0.8318287134170532, -0.05153957009315491, -0.262356698513031, 0.46095871925354004, -0.23895202577114105, -0.036379508674144745, -0.5429402589797974, -0.7846524715423584, -0.2218683958053589, -1.0662976503372192, 0.3162086308002472, -0.05057026445865631, 0.5487393140792847, -0.9117118120193481, 0.2023186832666397, -0.2740163803100586, -0.5773319602012634, 0.17067907750606537, -0.4142027497291565, 0.508520781993866, 0.7457674145698547, 0.1622556447982788, -0.4769080579280853, 0.3593216836452484, -0.169098362326622, 0.019652748480439186, 0.5216022729873657, 0.4811527132987976, 1.2413527965545654, 0.21533219516277313, 0.485055536031723, 0.4879413843154907, -0.2092064768075943, 0.011410479433834553], [-1.027523159980774, -1.281586766242981, 0.16262784600257874, -0.4451788365840912, -0.2773549258708954, 0.11916576325893402, -0.745164155960083, -0.41692203283309937, 0.18259063363075256, -0.5601967573165894, 0.1800476610660553, 0.2828071117401123, -0.27336522936820984, -1.431519627571106, 0.49186062812805176, 0.5278029441833496, -0.728144109249115, 0.43643757700920105, 0.2675011157989502, -0.3496343195438385, 1.0037888288497925, 0.7298439145088196, 0.5661534667015076, 0.2338842749595642, 0.28354504704475403, -0.21576005220413208, -0.16472241282463074, -0.7045645713806152, -0.2326323539018631, -0.0730404332280159, 0.30125898122787476, 0.3536487817764282, 0.4012005031108856, -0.4688602387905121, -0.43186622858047485, -0.5805065035820007, -0.8575382828712463, 0.44032639265060425, 0.6607264280319214, 0.40167301893234253, -0.08760847896337509, 0.7478721141815186, -0.6405034065246582, 1.0029443502426147, 0.14445309340953827, 0.4896925389766693, 0.5654675960540771, 0.6782181262969971, -0.1255560666322708, -0.15915848314762115, -0.0250496044754982, 0.4760352671146393, -0.7641479969024658, 0.8621911406517029, 0.2994719445705414, -0.8252661228179932, -0.36118990182876587, 0.06736321747303009, -0.6546920537948608, -0.060330744832754135, -0.8260552883148193, 0.4238300025463104, -0.784392774105072, 0.42618855834007263], [-0.4332399070262909, -0.9207371473312378, 0.2030828893184662, -0.37390539050102234, 0.4135376811027527, 0.14498765766620636, -0.7018439769744873, 0.02935212105512619, 0.0462443046271801, -0.07534195482730865, 0.3932335674762726, -0.42789578437805176, 0.7201423645019531, -0.31489893794059753, 0.701328456401825, 0.7467517256736755, 0.12730906903743744, -0.041822634637355804, -0.08252021670341492, 0.2217322140932083, 0.8763081431388855, -0.3509371280670166, 0.5266558527946472, -0.32161128520965576, 0.8191807866096497, -0.39134305715560913, 0.015499345026910305, -0.2927330732345581, 0.26286792755126953, -0.6698457598686218, 0.40686601400375366, -0.6831849217414856, 0.18342246115207672, 0.8523324728012085, -0.9688096642494202, 0.1527654528617859, -0.46816331148147583, 0.7676944136619568, 0.7075058221817017, 0.7822736501693726, 0.15195046365261078, 0.17847049236297607, -1.1211520433425903, 0.41273418068885803, 0.381879985332489, -0.027089755982160568, 0.9119658470153809, 0.6190710067749023, 0.4800119400024414, -0.6446691155433655, -0.02632838673889637, 0.6401652097702026, 0.09827125817537308, 0.900861382484436, -0.774557888507843, -0.9228671193122864, 0.5855491757392883, 0.19118347764015198, -0.7535254955291748, 0.748316764831543, -0.07759760320186615, -0.11002073436975479, 0.35379016399383545, 0.6134330630302429], [0.4464288353919983, -0.4417898654937744, -0.23113077878952026, 0.9088426828384399, -0.5904891490936279, 1.1005812883377075, -0.15831980109214783, 0.34406593441963196, 0.38152915239334106, 0.6373175382614136, 0.4053706228733063, 0.07656826823949814, -0.5730303525924683, -0.44736501574516296, -0.5610087513923645, 0.6903757452964783, -0.3479428291320801, -0.020340070128440857, 0.43562552332878113, 0.42584875226020813, -0.62751305103302, 0.1791808307170868, 0.6405635476112366, 1.062111735343933, 0.6047086715698242, -0.657614529132843, -0.23538166284561157, -0.03636014834046364, 0.13577121496200562, -0.33791279792785645, -0.28615802526474, -0.3652345836162567, -0.7084136605262756, -0.5160958170890808, 0.3167743682861328, -0.007184249348938465, 0.2725076675415039, -0.5113470554351807, -0.8281272649765015, -0.28375276923179626, -0.12085319310426712, -0.22113682329654694, -0.15777862071990967, -1.5700557231903076, 1.1112587451934814, 0.2958138585090637, -0.7117188572883606, 0.9524408578872681, 0.6261690258979797, -0.41950157284736633, 0.16155481338500977, 0.004975464660674334, 0.06481941789388657, 0.03804583474993706, 0.0023587339092046022, 0.03509090840816498, 0.27969688177108765, 0.05062885209918022, -0.22132906317710876, 0.6717921495437622, -0.15477702021598816, 0.4045460820198059, -0.015616126358509064, 0.10704846680164337], [-0.542816698551178, 0.34408828616142273, -0.565493106842041, -0.08484142273664474, -0.21919022500514984, -0.5583694577217102, -0.46190738677978516, -0.298538476228714, -0.13238531351089478, 0.6113324165344238, 0.28890255093574524, 0.7814954519271851, 0.11101332306861877, -0.6105642318725586, 0.8689151406288147, 0.3186928331851959, 0.796379029750824, -0.19674929976463318, -0.3974653482437134, 0.4532467722892761, 0.16185712814331055, -0.6482434868812561, 0.16859789192676544, -0.3715789318084717, 0.9527479410171509, 0.05778547748923302, 0.08069447427988052, 0.3483573794364929, 0.7610460519790649, -0.5198812484741211, 0.09010478854179382, 0.3018307685852051, -0.31034988164901733, 0.5862675309181213, 0.10265029966831207, -0.26094329357147217, 0.921902596950531, 0.20766185224056244, 0.8676112294197083, 0.7450262904167175, -0.7547044157981873, -0.6195504665374756, -0.8369206190109253, -0.45509594678878784, 0.7858018279075623, 0.8640417456626892, 0.35240116715431213, 0.6314951777458191, -0.07011836022138596, -0.5471429824829102, -0.2157461941242218, 0.009643683210015297, 0.057967983186244965, 0.5041096210479736, 0.1800425946712494, 0.0035090255551040173, 0.7296944260597229, 0.24281945824623108, 0.00982947088778019, 0.6714699268341064, -1.2430399656295776, 0.4854595363140106, -0.2581583261489868, 0.28523361682891846], [-0.09120086580514908, 0.3666396141052246, -0.40073850750923157, -0.006819271016865969, 0.5119044780731201, -0.6574500203132629, 0.018828241154551506, 0.37045782804489136, 0.2935572564601898, 0.6947765946388245, 0.8398568034172058, 0.2419973909854889, -1.11231529712677, 0.046609506011009216, -0.7008976936340332, -0.11571992188692093, -0.36950618028640747, 0.12081947922706604, -0.5571078062057495, -0.8827386498451233, 0.31762534379959106, 0.8284326791763306, -0.5120101571083069, 0.6072613596916199, -0.3647273778915405, 1.0343632698059082, 0.7757814526557922, -0.021735254675149918, 0.04957577958703041, -1.1064434051513672, -1.206855297088623, 1.0873278379440308, 0.4921233654022217, -0.5713328719139099, -0.10108524560928345, -0.4253828227519989, 0.95694500207901, -0.36473342776298523, -0.11420898139476776, 0.011894558556377888, -0.055340759456157684, -1.0715631246566772, 0.17789259552955627, 0.38632920384407043, -0.22371412813663483, -0.18697945773601532, 0.35060954093933105, 0.14368665218353271, 0.129171222448349, 0.04460059851408005, 0.18344037234783173, 0.4581518769264221, -0.1388651430606842, -0.17327213287353516, 0.6216182708740234, 0.25236374139785767, 0.34761950373649597, -1.1949466466903687, 0.36656785011291504, 0.18124383687973022, -0.4834268093109131, 0.13867075741291046, -0.24993585050106049, -0.5101149082183838], [-0.10339122265577316, -0.643669843673706, -0.7127878069877625, 0.1288256049156189, 0.08758962154388428, 0.15436922013759613, -0.32193198800086975, 0.7461262345314026, -0.2767865061759949, -0.38992249965667725, -0.8664074540138245, 0.11864102631807327, -1.0348682403564453, 0.8089736104011536, 0.38037657737731934, 0.9965166449546814, 0.0035995689686387777, -0.369406133890152, 0.9299968481063843, -0.47633790969848633, 1.0243937969207764, -0.2517385482788086, 0.05364159494638443, -0.6667627096176147, -0.6082610487937927, -0.34533530473709106, 0.16464102268218994, 0.19315776228904724, -0.3550718426704407, -0.4111051857471466, 0.07818825542926788, -0.4655970335006714, -0.2645418345928192, 0.7461053133010864, -0.7741556167602539, 1.1233545541763306, -0.6799180507659912, 0.5343706011772156, -0.16635505855083466, -0.08594333380460739, -0.18985998630523682, -0.5147362351417542, 0.3428046405315399, -0.3966163694858551, 0.26671481132507324, 0.009832949377596378, 0.2470255196094513, 0.08410829305648804, 0.7025597095489502, -0.44697749614715576, -0.23329097032546997, -0.5561825037002563, -0.6884645819664001, 0.5998959541320801, 0.3412735164165497, -2.139293428626843e-06, -0.10720498859882355, 0.9785444736480713, -0.07999642938375473, -0.8375494480133057, 0.4918002784252167, 0.04800960794091225, 0.5442028045654297, 0.9903136491775513], [0.274934321641922, 0.7588880658149719, 0.7010481357574463, -0.08934442698955536, -0.3000774085521698, -0.14382076263427734, 0.9903869032859802, -0.5194559097290039, -0.8620035648345947, 0.1655164659023285, -0.6871246695518494, -0.04564711079001427, 1.476210594177246, -0.3073875606060028, -0.29741474986076355, -0.34506306052207947, 0.37793105840682983, 0.5732699632644653, 0.18118199706077576, 0.49319401383399963, 0.03747629374265671, -1.066258192062378, 0.2996906340122223, 0.13475477695465088, -0.5750162601470947, -0.08643440157175064, -0.3991602957248688, -0.26355814933776855, 0.012758790515363216, 0.5143492817878723, 1.1905717849731445, 0.08792611956596375, -0.05091102793812752, -0.1938418596982956, 0.07488937675952911, 0.22394917905330658, -0.6285423040390015, 0.21267341077327728, 0.5607821345329285, -0.04314781725406647, -0.006355451885610819, 1.0624213218688965, -0.9866064786911011, -0.46760183572769165, 0.5057851076126099, 1.0308090448379517, -0.0886741653084755, -0.06787145137786865, -0.8133926391601562, 0.44020724296569824, 0.3673814535140991, -0.7171485424041748, 0.009170420467853546, -0.6797433495521545, -0.3695051372051239, 0.02960808575153351, 0.2434273213148117, 0.34799304604530334, 0.42415231466293335, 0.3165286183357239, -0.7440662384033203, -0.5811270475387573, 0.28173384070396423, -0.2804275453090668], [-0.42578527331352234, -0.017406350001692772, -0.07271943986415863, -0.11181091517210007, -0.12116913497447968, 0.23560714721679688, -0.13179028034210205, -1.1305732727050781, 0.10032549500465393, 0.4737175703048706, -0.4754630923271179, 0.3414557874202728, 0.8340891599655151, -0.08972420543432236, 0.5392658114433289, 0.7920632362365723, -0.2586025893688202, -0.15201404690742493, -0.1753500998020172, -0.015457035973668098, -0.31512653827667236, -0.9772701263427734, 0.6051205396652222, -0.4192015528678894, 0.3154431879520416, 0.41602882742881775, 0.19603173434734344, -0.06422678381204605, -0.31663084030151367, 0.3690532147884369, -0.37593474984169006, -0.1050119623541832, -0.32274121046066284, 0.29563242197036743, -0.7888069748878479, -0.6678032279014587, -0.06681971251964569, 0.9038295149803162, 0.12171735614538193, -0.6608976721763611, -0.27877363562583923, -0.44717493653297424, -0.5031247735023499, 0.679966926574707, -0.1897604763507843, 1.019742727279663, -0.39247676730155945, 0.26334795355796814, -0.8467605710029602, 0.48049813508987427, 0.5794020295143127, -0.2862943410873413, 0.6578640341758728, 0.6224981546401978, -0.7446714043617249, 0.11683692038059235, 0.3819051682949066, -1.0555604696273804, 0.7187362313270569, 0.7112518548965454, -0.3980990946292877, 0.2809048891067505, 0.4775415062904358, -0.0869731605052948], [1.005798578262329, 0.4057684540748596, -0.3589532971382141, 0.179491326212883, -0.17408506572246552, -0.2984258234500885, -0.16376103460788727, -1.1421051025390625, -0.9929838180541992, 0.9104849100112915, 0.7508656978607178, 0.2010727971792221, 0.6269676089286804, -0.3248359262943268, -0.44133028388023376, 0.05425494536757469, -0.17488351464271545, 0.23358196020126343, -1.2309669256210327, 0.09106719493865967, -0.6802250146865845, -0.3444984555244446, 0.05007018893957138, 0.455375999212265, 0.29558926820755005, 0.6281906366348267, 0.5850049257278442, -0.1502465009689331, 0.6675586104393005, 0.19244179129600525, 0.004418295342475176, 0.48002099990844727, 0.07020702213048935, 0.4043445289134979, -0.11913637816905975, -1.851346492767334, 0.7152024507522583, 0.2249455749988556, 0.34333211183547974, -0.4586624205112457, -0.2311183363199234, 0.2965913712978363, -1.0463272333145142, -0.37712112069129944, -0.0003262011450715363, 0.6641138195991516, -0.10586994886398315, -0.8274921774864197, -0.8862664699554443, -0.4350810945034027, 0.14577387273311615, 0.1301998496055603, 0.20619115233421326, -0.3010488748550415, -0.6017985343933105, 0.03226141631603241, 0.5156689286231995, -1.463017463684082, 0.6954513788223267, 0.5181238055229187, -0.21400028467178345, -0.731307864189148, 0.7908226847648621, -0.6646506190299988], [-0.2737045884132385, 0.4555564522743225, -0.2992802858352661, 0.5215930938720703, -0.7729761600494385, -0.39055606722831726, 0.04605691134929657, -0.7021504044532776, -0.647597074508667, 0.4965752363204956, -1.1301329135894775, 0.9241412281990051, -0.23594258725643158, -0.31270039081573486, -0.31375348567962646, 0.6121459007263184, -0.12649869918823242, 0.23551055788993835, -0.30844634771347046, 0.7274677753448486, 0.9472172856330872, 0.5721473097801208, 0.4441540837287903, -0.1087048277258873, 0.3537577986717224, -0.44819822907447815, -0.3803856670856476, 0.29648518562316895, -0.732856273651123, -0.16242201626300812, 0.7144838571548462, 0.6347333788871765, -0.8564469218254089, 0.4370782673358917, 0.7887919545173645, -0.13262298703193665, 0.5775727033615112, -0.10680454969406128, -0.6161509156227112, 0.012770798057317734, -0.593368649482727, -0.22375927865505219, 0.45005860924720764, -0.31143373250961304, 0.6614726781845093, 1.1679552793502808, -2.246492624282837, 0.24938072264194489, 0.0036716503091156483, 0.17503106594085693, -0.1571873426437378, 0.13901086151599884, 0.49184173345565796, -0.48696649074554443, 0.21219119429588318, 0.525384247303009, -0.18565918505191803, 0.24398502707481384, -0.20861667394638062, -0.7784566879272461, -0.40198177099227905, 0.34562957286834717, 0.4465624988079071, -0.13209159672260284], [-0.34592413902282715, 0.07974724471569061, 0.40894123911857605, -0.9370200037956238, 0.25156471133232117, -0.1972983181476593, 0.2576397657394409, 1.1818033456802368, -0.9022714495658875, -0.48572397232055664, 0.247512087225914, 0.1692168414592743, 0.45594584941864014, 0.8851481676101685, 0.7293418645858765, -0.16536115109920502, 0.39510542154312134, -0.3767394721508026, -0.6506462097167969, -0.3109314739704132, -0.43584001064300537, -0.4457772672176361, -0.366525262594223, -0.20519457757472992, -0.5822686553001404, 0.4538267254829407, 0.4156906008720398, 0.2558766305446625, 0.4934338629245758, -0.6074575781822205, 0.04603958874940872, 1.202193021774292, 1.2097718715667725, 0.1280829906463623, -1.2882434129714966, -0.0469360277056694, 0.4669188857078552, -0.04616187885403633, 0.5970274209976196, 0.014825080521404743, 0.06750144064426422, 0.007504566106945276, -0.8239827156066895, 0.5136610269546509, -0.2184344232082367, 0.06707336753606796, 1.4909416437149048, -0.5183647871017456, 0.22330904006958008, -0.124917171895504, 0.8142050504684448, -0.5867356657981873, -0.35974621772766113, 0.07187546044588089, -0.686166524887085, -1.177422046661377, 0.8615736961364746, -1.1718188524246216, -0.31102943420410156, 0.028031757101416588, -0.08594360202550888, -0.17333416640758514, -0.216001957654953, 0.18555086851119995], [0.17797087132930756, -0.2488003969192505, -0.32546326518058777, 0.24328966438770294, -0.9467374086380005, 0.27854546904563904, -0.3505866229534149, -0.1444218009710312, 0.265537828207016, -0.5871498584747314, -0.47046732902526855, -0.254096120595932, -0.5189715623855591, -0.802935004234314, -0.44501230120658875, 0.3129514157772064, 0.10989624261856079, 0.7169875502586365, 0.35605764389038086, 0.5044398903846741, -0.5887882113456726, 0.991136372089386, 0.8482059240341187, 0.7566862106323242, 0.10884984582662582, -0.4808894395828247, -1.6818040609359741, -0.4421393871307373, 0.7245400547981262, -0.008257654495537281, -0.02344083972275257, -0.49513259530067444, -0.25277242064476013, 0.3579031527042389, 0.8268052935600281, 0.1943066418170929, -1.1260392665863037, 0.21079909801483154, -0.47643911838531494, 0.7051896452903748, -0.7927025556564331, 0.42713505029678345, 0.8594772815704346, 0.07030113786458969, 0.4064992070198059, 0.49976423382759094, 0.14504797756671906, 0.85970139503479, -0.11188521981239319, -0.26417791843414307, -1.0027376413345337, 1.595901370048523, 0.674561083316803, -0.04527711495757103, -0.02641657367348671, 0.19899605214595795, -0.29856008291244507, 0.5225307941436768, 0.2772959768772125, 0.7928931713104248, 0.4053063690662384, 0.7897830009460449, -0.34333109855651855, 0.1963350623846054], [0.05964265391230583, 0.7030603289604187, 0.6138596534729004, -0.03554347902536392, -0.030871368944644928, 0.7668542861938477, 1.3526618480682373, -0.11578856408596039, 0.5359909534454346, -0.28801971673965454, -0.0011024662526324391, -0.3655877709388733, 1.0802346467971802, -0.383544385433197, 0.5204489231109619, -0.6352495551109314, 0.4651540517807007, 0.7578370571136475, -0.21795666217803955, 0.5845386385917664, -0.22336804866790771, 0.07058241963386536, -0.3653830289840698, 0.9266320466995239, -0.7214633822441101, -0.14691726863384247, -0.7778204679489136, 0.8109776973724365, 0.45875707268714905, 1.3661366701126099, 0.3608860969543457, -0.49505704641342163, -0.017201703041791916, -0.1543070673942566, 0.5686652660369873, 0.04316089302301407, 0.5909399390220642, -0.7183049917221069, -0.060052789747714996, 0.6039858460426331, -0.6415796875953674, -0.7810484170913696, 0.3310546576976776, -0.38190677762031555, -0.13559967279434204, -0.4216844439506531, -0.795308530330658, 0.03780033439397812, -0.2501181364059448, 0.399631530046463, 0.3326980173587799, -0.4550950527191162, 0.25333598256111145, -0.6925356984138489, 0.2253946214914322, 0.5769964456558228, -0.8842695951461792, -0.5824788808822632, 0.40801164507865906, 0.43575385212898254, -0.17334957420825958, -0.24565792083740234, 0.7222055196762085, 1.0547703504562378], [-0.11799461394548416, -0.2689751088619232, 1.4544463157653809, -0.05471291393041611, -0.1995777040719986, 0.30070728063583374, -0.16480708122253418, 0.19565626978874207, 0.08914172649383545, 0.5700005292892456, 0.6338801980018616, 1.0466291904449463, -0.39324530959129333, -0.09012342244386673, -0.43484994769096375, -0.7965660095214844, 0.29926788806915283, -0.06138494610786438, 0.23435111343860626, 1.015907645225525, -0.34583619236946106, 0.4712706208229065, 1.2460873126983643, 0.768368124961853, 1.2073793411254883, 0.008447762578725815, -0.8153574466705322, -0.7392880916595459, 0.35063329339027405, 0.3581577241420746, 0.34539833664894104, 0.053259436041116714, -0.39354583621025085, -0.5578526854515076, 0.4266386032104492, -0.44102874398231506, -0.4610752761363983, -0.31116005778312683, -0.20285995304584503, -0.1335127204656601, -0.50963294506073, 0.13059811294078827, -0.5480132699012756, -0.5484479665756226, 1.3535218238830566, 0.15925179421901703, 0.6536794304847717, 0.4822673797607422, 0.09774903953075409, -0.37905359268188477, -1.095834732055664, 0.3006003797054291, -0.16394385695457458, -0.00866282545030117, -0.21280382573604584, -0.5922619700431824, 0.13647884130477905, -0.07745594531297684, -0.1241244226694107, 0.9934437274932861, -0.11328441649675369, 0.37392690777778625, -1.0672051906585693, -0.3763633966445923], [-0.10021627694368362, -0.5175090432167053, -0.21336597204208374, 0.046981532126665115, -0.35001152753829956, -0.9023317694664001, 0.1667780876159668, -0.1717481166124344, -0.6716023087501526, -0.2192569226026535, 0.16088464856147766, 0.8105601668357849, 0.4115423560142517, -0.9217320680618286, 0.000571776763536036, 0.738050639629364, 0.0844358429312706, -0.33673250675201416, -0.05655111372470856, -0.49671345949172974, -0.4470941424369812, -0.093831866979599, 0.9415104985237122, 0.206846684217453, 0.16697704792022705, -0.9732581377029419, -0.3464633822441101, 0.39856261014938354, 0.18962740898132324, -0.6089619398117065, -0.3248484134674072, -0.43199390172958374, -0.27864861488342285, 0.2553609609603882, 0.07292850315570831, -0.3281621038913727, -0.6258571743965149, -0.7697728872299194, -0.0997215136885643, 0.0678459107875824, -0.7717106938362122, -0.32952919602394104, 0.28336575627326965, -0.022789660841226578, 0.05593981221318245, 0.03001582622528076, -0.6326587796211243, 0.9997230172157288, 0.0025530103594064713, 0.4127023220062256, -0.2925960123538971, -0.1595761626958847, 0.30582961440086365, 0.09621044248342514, -1.2673789262771606, 0.4527612030506134, 0.5090910792350769, 0.6979734897613525, -0.3611203134059906, 0.5215694904327393, -1.3308252096176147, 0.9486982822418213, 0.5176920890808105, 0.6403636932373047], [-0.1259659230709076, -0.21370206773281097, -0.15881472826004028, 0.057511407881975174, -0.08685591071844101, -0.4819299876689911, -0.5445441603660583, 0.19833317399024963, 0.3507222831249237, 0.04657300189137459, -0.06719401478767395, 0.6675125956535339, -0.24646277725696564, 0.293374240398407, 0.11027964949607849, 0.27163246273994446, -0.18338918685913086, -0.6527784466743469, -0.3782504200935364, -0.11191020160913467, 0.19812285900115967, 1.3761894702911377, -0.10828462243080139, 1.109838843345642, -0.040374353528022766, -0.709572434425354, 0.2190210372209549, 0.10509777069091797, 0.19045676290988922, -1.0282657146453857, -0.9836556315422058, 0.28694307804107666, -0.2086615264415741, -0.9125812649726868, 0.4962603747844696, -0.817804753780365, 0.02967022731900215, 0.4872666597366333, -0.6462298035621643, -0.19748300313949585, 0.14924843609333038, -0.18636448681354523, 0.46554917097091675, -0.7068954110145569, 0.16421690583229065, 0.8029150366783142, -0.9228299856185913, 0.3548417091369629, 0.2674908936023712, -0.22553446888923645, 0.1764993667602539, -0.5013752579689026, -0.2090548872947693, -0.21286185085773468, 0.1661110520362854, 0.18723006546497345, 0.246041402220726, 0.37142473459243774, 0.8012164831161499, 0.11276601999998093, -0.5011789798736572, 0.45008403062820435, 0.08985231071710587, 0.2554093599319458], [-0.2016085386276245, 1.060929775238037, -0.37717628479003906, 0.09663815051317215, -0.7022382616996765, -0.297048956155777, 0.9020997285842896, 0.15988245606422424, -1.2227954864501953, 0.08935626596212387, 0.142256960272789, 0.0784631296992302, 0.03538769483566284, 0.5643885731697083, -0.32862967252731323, -0.4385095536708832, 0.6736993789672852, 0.42273426055908203, -0.3527325987815857, 0.3222692906856537, 0.0031526817474514246, 0.33314961194992065, 0.17172668874263763, 0.9589053988456726, -0.26251697540283203, 0.4922948181629181, 0.38683009147644043, -0.5519113540649414, 1.0342414379119873, -0.4550269842147827, 0.8802828192710876, 1.247750997543335, 0.16215722262859344, 0.1553674191236496, 0.1646706610918045, -1.1036185026168823, 0.6218135356903076, 0.716867983341217, -0.1898859292268753, 0.44437289237976074, -0.06136217713356018, 0.7140162587165833, -0.22839343547821045, -0.3439713418483734, 0.6141719222068787, -0.10127527266740799, -0.10834163427352905, -1.0744975805282593, -0.1397140473127365, -1.6077934503555298, -0.037792932242155075, 0.25197580456733704, 0.20273615419864655, -0.6287218332290649, 0.2656780779361725, -0.0954289510846138, 0.3447573781013489, -0.7751062512397766, 0.459320992231369, 0.22653333842754364, 0.2775416970252991, -0.7667480707168579, 0.36614418029785156, -0.4042637348175049], [-0.5834426283836365, 0.2163291871547699, -1.2452566623687744, 0.7110536694526672, 0.40769025683403015, 0.6603903770446777, 0.12698571383953094, 0.2616727650165558, 0.21370850503444672, -0.04920312017202377, -0.9713714718818665, 0.10502494126558304, -0.5562537908554077, 0.7392582893371582, -0.27433130145072937, 0.3416609466075897, 0.10611512511968613, 0.46829965710639954, 0.43324390053749084, 0.05153181776404381, 0.6071379780769348, 0.18251866102218628, 0.32169583439826965, 0.4175863564014435, -0.16950397193431854, 0.6989021897315979, 0.266042023897171, -0.24629898369312286, -0.4568333029747009, 0.22879062592983246, 0.1856808364391327, 0.6505733728408813, -0.06439702957868576, -0.17795012891292572, 0.26929324865341187, -0.10294340550899506, 0.0004663344589062035, 1.2951844930648804, -0.5326679944992065, 0.10192829370498657, -0.055664706975221634, 0.17929601669311523, 0.15820921957492828, 0.4450390338897705, -0.5592989921569824, 0.4360351860523224, 0.2775111198425293, -0.3382914960384369, -0.28989729285240173, -0.3525812029838562, -0.498345285654068, 0.9255052804946899, -0.7942175269126892, 0.17641115188598633, 0.35715073347091675, -0.1880110204219818, 0.36462321877479553, -0.7752276659011841, -0.40664657950401306, 0.30274102091789246, -0.5975717902183533, 1.0398021936416626, -0.6051531434059143, -0.36000490188598633], [0.7334619164466858, 0.33653053641319275, -0.8361110091209412, 0.2947935163974762, -0.4040071666240692, -0.5431703329086304, 0.09347796440124512, -0.49902093410491943, -0.07003802806138992, -0.3075691759586334, 0.7785559892654419, -0.0640406683087349, -0.4755573272705078, 0.6091006398200989, -0.8723800182342529, -1.3789870738983154, 0.38004231452941895, 0.2939225435256958, -1.134297490119934, 0.06901304423809052, -0.020259184762835503, 0.47431516647338867, 0.38659489154815674, 0.17130018770694733, -0.9856992959976196, 0.6279326677322388, -0.4167436957359314, 0.2749173045158386, 0.6167796850204468, 0.010756921023130417, -0.030533432960510254, 0.38447752594947815, -0.2369336038827896, -0.25681591033935547, 0.746517539024353, -0.2597847878932953, 0.33570849895477295, -0.5206649899482727, -0.32888197898864746, 0.38505321741104126, -0.20378224551677704, 0.43010517954826355, -0.20952631533145905, 0.19081436097621918, 0.2441702038049698, -0.43741536140441895, -1.2768640518188477, -0.2751924991607666, 0.6209731101989746, -0.0717310756444931, -0.9828211069107056, -0.1454026997089386, 0.09427965432405472, -1.1997382640838623, 0.34838929772377014, 1.1661959886550903, 0.294253408908844, -0.43883422017097473, -0.1452711820602417, 0.054011568427085876, 0.03459887206554413, -0.6135363578796387, 0.4410812556743622, -0.8104044795036316], [-0.038494981825351715, -0.16885888576507568, 0.8694888353347778, -0.5306839346885681, 0.26037925481796265, 1.1166319847106934, -0.23500345647335052, -0.2038806527853012, -0.07382607460021973, 0.08063439279794693, -1.0047404766082764, 0.55631422996521, 0.37777236104011536, -0.03377107158303261, -1.0094002485275269, 0.9935961365699768, -0.47786518931388855, -0.7164338827133179, 0.5461480617523193, -0.14783895015716553, 0.5341114401817322, -0.5065426826477051, -0.543846070766449, 0.9012907147407532, 0.12297298014163971, 0.5897815227508545, 0.4095466434955597, 1.0314232110977173, -0.47519078850746155, -0.16162750124931335, -0.38224396109580994, -0.009175102226436138, 0.3167394995689392, 0.5384993553161621, -0.034285563975572586, 0.9019371867179871, 0.03450877591967583, 0.15134580433368683, -0.31626009941101074, -1.0822967290878296, 0.9416297078132629, 0.10257942229509354, 0.10170561820268631, -0.23870541155338287, -0.4632558524608612, -0.06180119514465332, -0.3827403485774994, -0.37079134583473206, -0.5934602618217468, 0.08363328874111176, 1.0027247667312622, -0.6074113249778748, -0.014093275181949139, 0.1086650863289833, 0.5699244737625122, -0.6759028434753418, 0.010390968061983585, -0.5294843316078186, 0.35931485891342163, -0.8336284756660461, 0.43543732166290283, 0.6588752269744873, 0.8553584218025208, 0.48608770966529846], [0.6474120020866394, 0.29944536089897156, -1.0836313962936401, -0.13587988913059235, -0.27633345127105713, -1.027634859085083, -0.40673840045928955, 0.7050645351409912, 0.1720106303691864, -0.006359919440001249, 0.28383204340934753, -0.19011804461479187, -0.2592189610004425, 1.0448166131973267, -0.04593576490879059, -0.752178966999054, 0.4914501905441284, 0.21262511610984802, -0.44307082891464233, -0.04343158379197121, -0.38117823004722595, 0.9469054937362671, -0.17892035841941833, 0.293608158826828, -0.5248786211013794, 0.13960479199886322, -0.11662022769451141, -1.5060964822769165, 0.09828726947307587, -0.6388605833053589, 0.07088136672973633, 0.5360326170921326, 1.071517825126648, -0.7002942562103271, -0.06437385827302933, -0.05786276236176491, 0.11194022744894028, -0.2840512692928314, -1.1769764423370361, 0.8051267862319946, 0.5023530721664429, 0.5703234672546387, 0.6400531530380249, 0.10898065567016602, -0.2537892162799835, -0.9435917139053345, 0.7047411799430847, -0.2532200217247009, 1.044724702835083, 0.4354463815689087, -0.5151786208152771, 0.5873066186904907, -0.4452798366546631, -0.9769890904426575, 0.2721918225288391, 0.29100069403648376, 0.436188280582428, -0.3082008957862854, -0.7393777370452881, -0.4088820219039917, 0.2711525857448578, -0.15845705568790436, -0.13771486282348633, -0.39224931597709656], [-0.06836137175559998, -0.7316828370094299, 0.2888129651546478, -0.13811540603637695, 0.22125065326690674, -0.23837807774543762, -0.39272215962409973, 0.9982982277870178, 0.32905104756355286, -0.09150709211826324, -0.7652115821838379, -0.4846635162830353, 0.15387961268424988, 0.3491412103176117, -0.23685576021671295, -0.4507599472999573, -0.4528835117816925, -1.189422369003296, 0.6095165610313416, -0.3744533658027649, -0.14557282626628876, -0.5997023582458496, -0.3514750003814697, -0.22286294400691986, -0.22484201192855835, 0.029457632452249527, 0.050452832132577896, 0.32035762071609497, -1.3260602951049805, -0.18678279221057892, -0.29519838094711304, -0.21789570152759552, 0.6996891498565674, 0.13195306062698364, -0.2750512361526489, 1.854920506477356, -0.9069437980651855, -0.10503503680229187, -0.1791025549173355, -1.1528801918029785, 1.5858166217803955, -0.6309536695480347, 0.3444491922855377, -0.0647740364074707, -0.1776837557554245, -1.1020194292068481, 1.1350361108779907, -0.4707305133342743, 0.3939964175224304, 0.25451385974884033, 0.6484439373016357, -0.753540575504303, -0.04630051925778389, 0.084856778383255, 0.31603941321372986, -0.6911497116088867, -0.5409302711486816, 0.6318641304969788, -0.7698169946670532, -0.32930314540863037, 0.8428332805633545, 0.4278671443462372, 0.48264750838279724, 0.40973255038261414], [-0.05685556307435036, -0.1692235916852951, 0.00032228135387413204, 0.14436927437782288, 0.4219925105571747, -0.11748003214597702, 0.5835523009300232, 0.7918893098831177, 0.21060535311698914, 0.20831741392612457, -0.05462144315242767, -0.6236609220504761, -0.4542822539806366, -0.48276033997535706, -0.9544159770011902, -0.6470900774002075, -0.32117894291877747, -0.05621514841914177, 0.20759642124176025, -0.010192385874688625, -0.07799972593784332, 0.0681605115532875, -0.058317169547080994, 0.026707781478762627, 0.28332212567329407, -0.2751551568508148, -0.4082344174385071, -0.055011823773384094, 0.5791856646537781, -0.7033698558807373, 0.28897878527641296, -0.46957942843437195, -0.19320811331272125, -0.6189641356468201, -0.49471428990364075, 0.3282245695590973, -0.4379984438419342, -0.8857600092887878, -0.13988693058490753, 0.09403324127197266, 0.11170144379138947, 0.9709576964378357, -0.5228478908538818, -0.26991626620292664, 0.20518897473812103, 0.22945182025432587, 0.2303556352853775, 0.5176925659179688, 0.6239179372787476, -1.2897430658340454, 0.6880218982696533, -0.48061537742614746, -0.6610395908355713, 0.2934274971485138, 0.15968169271945953, -0.36795979738235474, 0.1563272774219513, 0.2615383267402649, -0.6182337999343872, 0.6548645496368408, -1.183724045753479, -0.2923392355442047, -0.8332907557487488, -0.13418987393379211], [-0.0252973809838295, -0.3515413999557495, -0.6153445839881897, -0.3199360966682434, 0.6922181844711304, 0.47450903058052063, 0.646771252155304, 0.218846395611763, 0.6977770328521729, -1.5989755392074585, -0.5295665264129639, -0.8174709677696228, -0.1664728969335556, -0.29737597703933716, 0.44898298382759094, 0.6472498774528503, -0.3061273992061615, -0.22972196340560913, 0.7984771132469177, -0.01708739995956421, 0.1396062672138214, -0.2649041414260864, -0.6356479525566101, -0.23884187638759613, -0.5546197891235352, -1.12370765209198, -0.07920782268047333, 0.625324010848999, -0.34497973322868347, -0.13288046419620514, 0.09810785949230194, -0.6844922304153442, -0.8664050698280334, 0.4998316764831543, 0.36442214250564575, 0.6201989650726318, -0.7018576860427856, 0.73735511302948, 0.23832844197750092, 0.4749808609485626, 0.5232864022254944, -0.17496450245380402, 0.8861621022224426, 0.7697163224220276, -0.6044033169746399, 0.20715677738189697, -0.5059547424316406, 0.6648250222206116, 0.18984302878379822, 0.39027225971221924, 0.025043901056051254, -0.3482627868652344, -0.08803233504295349, -0.0232449509203434, -0.2964612543582916, 0.03268628194928169, -0.5041012167930603, 1.0417218208312988, -0.12290089577436447, -0.45545434951782227, -0.31125518679618835, -0.14599286019802094, 0.9162153601646423, 0.28433629870414734], [-0.18071278929710388, 0.48269838094711304, 0.13448749482631683, -0.9971837401390076, 0.06469575315713882, 0.053705453872680664, 0.3996802568435669, 0.7820818424224854, -0.8161417841911316, 0.8480843901634216, 0.1713218241930008, 0.3002285063266754, -0.29203829169273376, -0.15241080522537231, -0.9367872476577759, -0.6438299417495728, 0.5121896862983704, -0.06481534242630005, 0.23612380027770996, -0.1560322493314743, -0.65212082862854, 0.0864052101969719, -0.16518299281597137, 0.23462225496768951, -0.3408971428871155, -0.1304962933063507, -0.12540555000305176, 1.1941763162612915, 0.23662398755550385, 0.5991743206977844, 0.09125328809022903, -0.07916940748691559, -0.16307403147220612, -0.9362355470657349, 0.19400310516357422, -0.04916994273662567, 0.5469874143600464, 0.09714331477880478, 0.3422161936759949, -0.12468203157186508, 0.05730811879038811, 0.05512893572449684, -1.0022605657577515, 0.3507091999053955, 0.2597799599170685, 0.23594653606414795, 0.09168508648872375, -0.4632396996021271, 0.4498501121997833, 0.49396374821662903, 0.404538094997406, -1.0807530879974365, -1.134214997291565, -0.19639571011066437, 1.0486472845077515, -0.9644426107406616, 0.5596848130226135, -0.6089821457862854, 0.1430729776620865, -0.16444562375545502, -0.4789261519908905, -0.16146698594093323, -0.41191616654396057, 0.7273581027984619], [0.9276934266090393, -0.11138655990362167, -0.4531767666339874, 0.16085070371627808, 0.689923882484436, 0.9541415572166443, -0.49877217411994934, -0.13829931616783142, 0.3592127859592438, -0.36711686849594116, 0.8482348322868347, -0.3724347949028015, -0.19676947593688965, -0.06596912443637848, -0.556250810623169, 0.18723024427890778, -0.4177734851837158, -0.4777909517288208, 0.08316469937562943, -0.4203415513038635, -0.3969753384590149, -0.2448168694972992, -0.48114845156669617, 0.15823735296726227, -0.9939629435539246, 0.6336735486984253, 0.5169137716293335, -0.1150844469666481, -0.027417436242103577, -0.4929037392139435, -0.07716315984725952, 0.14392955601215363, -0.1507815718650818, 0.61855149269104, -0.2846549153327942, -0.4523222744464874, -0.04362458735704422, 1.0281639099121094, 0.4296605885028839, -0.8871081471443176, -0.5653639435768127, -0.0015127743827179074, 0.11075558513402939, 0.3458997905254364, -1.1554309129714966, -0.5235943794250488, 0.7895398736000061, -0.5943877696990967, -0.4326883554458618, -0.6983180642127991, 0.014480728656053543, 0.638466477394104, 0.561663806438446, 0.27498555183410645, 0.2548449635505676, 0.07370290905237198, -0.37771639227867126, -0.12653212249279022, 0.37957262992858887, -0.4520462155342102, 0.9485391974449158, -0.11641042679548264, 0.8965917229652405, -0.06384959816932678], [0.7358308434486389, -0.04473140463232994, -0.12576554715633392, -0.10413195192813873, 0.3705751895904541, 0.14008496701717377, -0.21363915503025055, -0.295592725276947, -0.42783647775650024, -0.29918524622917175, 0.6414058208465576, 0.19878512620925903, -1.256304383277893, 0.9892602562904358, 0.30404043197631836, -0.7129643559455872, 0.09352399408817291, -0.2385772168636322, -0.35798871517181396, 0.7956852912902832, 0.0642816573381424, 0.741752564907074, -0.808360755443573, 0.3883063495159149, 0.1924230009317398, -0.20139546692371368, -0.6958039999008179, -0.766547679901123, 0.11754381656646729, 0.04891948401927948, 0.859000563621521, -0.5244534611701965, 0.03577416017651558, -0.2764841318130493, 0.08720790594816208, -0.8510231375694275, -0.2996276617050171, 0.19510513544082642, -0.751408040523529, -0.09985597431659698, 0.43645694851875305, 1.0587769746780396, 0.6109675765037537, -0.6991297006607056, 0.4773726463317871, -0.851989209651947, 0.059462420642375946, -0.558180570602417, 0.6720613241195679, 0.17402128875255585, -1.1943389177322388, 0.592326819896698, -0.05116623267531395, -0.24868948757648468, 0.5850926041603088, 0.11226030439138412, -0.55926114320755, -0.03026479296386242, 0.11080653965473175, 0.11958466470241547, 1.333645224571228, -0.07826178520917892, -0.09041960537433624, -0.20847870409488678], [-0.0019813133403658867, 0.11129608750343323, -0.7275233864784241, 0.002096317010000348, 1.0956306457519531, 0.34638914465904236, 0.7096289396286011, 1.269976258277893, 0.4014279842376709, 0.08994029462337494, 0.45089074969291687, -0.8348850011825562, -0.10854816436767578, 1.3284660577774048, 0.3141551613807678, -0.5579777359962463, 0.4060502350330353, 0.48890987038612366, -0.06806202977895737, -0.019260108470916748, -0.5687779188156128, -0.6999167799949646, -0.26355838775634766, 0.05127460137009621, -0.29797306656837463, 0.4032156467437744, 0.6427865028381348, 0.5994659066200256, -0.25229260325431824, -0.31470489501953125, 0.9260333180427551, 0.16540874540805817, -0.1008651927113533, 0.22503621876239777, 0.07405941188335419, 0.4979080855846405, 0.3372790813446045, 0.8528563976287842, 0.06699123978614807, 0.09495018422603607, 1.2435060739517212, -0.29252639412879944, -0.05714443325996399, -0.21268446743488312, 0.08816405385732651, -0.4939711093902588, -0.25487497448921204, 0.07348392903804779, 0.7312441468238831, -0.4599204659461975, 0.36414793133735657, -1.1062514781951904, 0.19945332407951355, -0.5456030964851379, -0.1616053283214569, -0.4232404828071594, 0.3686276972293854, -0.0391964353621006, -1.2111610174179077, -0.3680465817451477, -0.1380445659160614, -1.1531215906143188, 0.3300223648548126, -0.1047426164150238], [0.22194340825080872, 0.33674702048301697, -0.20980888605117798, -0.4314627945423126, 0.648333728313446, -1.065690279006958, -0.45174840092658997, 0.23428916931152344, -0.10648973286151886, -0.3264976441860199, -0.4975073039531708, 0.774644672870636, -0.8620310425758362, -0.32352402806282043, -0.5875492691993713, -0.15247568488121033, 0.2209480106830597, -0.9419780373573303, -0.1490849256515503, -0.40894201397895813, 1.4494750499725342, 0.33689144253730774, -1.6717416048049927, -1.5804096460342407, -0.47116538882255554, 0.3363557457923889, 0.3954857885837555, 0.0749143436551094, -0.5839167833328247, -0.16136528551578522, -0.003125283168628812, 0.3584856688976288, 0.25048795342445374, -0.3065134286880493, -0.06117599457502365, -0.7662073373794556, -0.9998608231544495, -0.37930554151535034, 0.6940529346466064, -0.06198402866721153, 0.4786189794540405, -0.05210179090499878, 0.1146869882941246, 0.3645413815975189, -0.8771081566810608, -0.09670481085777283, 0.7215465307235718, -0.4248121976852417, -0.5316145420074463, 0.9620856642723083, -0.17884279787540436, -0.5056113004684448, -0.5809733867645264, 0.1725773811340332, -0.20993047952651978, 0.0699734017252922, -0.15476086735725403, 0.48134365677833557, -0.0168449729681015, -0.9248344302177429, -0.2007608711719513, 0.7580259442329407, 0.43352317810058594, -0.8879064321517944], [0.24360014498233795, -0.2385111004114151, 0.4618881940841675, 0.2287561595439911, 0.2349425107240677, 0.4200951159000397, -0.0797441229224205, -0.20457567274570465, -0.48007291555404663, -0.16790771484375, -0.32148948311805725, -0.14890335500240326, 0.08183165639638901, -0.36396557092666626, 0.982158899307251, -0.07901015877723694, 1.2218464612960815, 0.11663999408483505, 0.08621620386838913, 0.720994234085083, -0.3081475794315338, -0.6054086685180664, 0.8448964953422546, -0.17594514787197113, -0.040392037481069565, -0.5912017226219177, -0.5954295992851257, -0.15525402128696442, 0.603515625, -0.8149049878120422, 1.0148459672927856, -0.5033754706382751, -0.9320169687271118, 0.5497316718101501, 0.22025062143802643, -0.3818660080432892, -0.4964947998523712, 0.08905918151140213, 0.8099433779716492, 0.48016414046287537, -0.669983983039856, 0.44703927636146545, 0.275235116481781, -0.5627390742301941, 0.20415674149990082, 0.1329127550125122, 0.23817500472068787, -0.12171664088964462, -0.25977131724357605, -0.2404487431049347, -0.9967672824859619, -0.012141400948166847, 0.22996382415294647, -0.29459884762763977, -0.46927905082702637, -0.24774938821792603, -0.6297982931137085, 0.5569151043891907, 0.05093250796198845, 0.7592196464538574, 0.4864167273044586, -1.3476295471191406, 0.6372496485710144, -0.14086821675300598], [0.40861132740974426, 0.3992466926574707, -0.18572288751602173, -0.38445407152175903, 0.651986837387085, -0.6893773078918457, 0.0558413602411747, -0.010077334009110928, 0.922711193561554, -0.25377166271209717, 0.8593665361404419, 0.1685279756784439, 0.12539875507354736, 0.628257691860199, 0.2827606797218323, 0.021303677931427956, 0.23754701018333435, 0.32539379596710205, -0.15658514201641083, 0.40702852606773376, -0.33045920729637146, 0.3789345920085907, -0.48298192024230957, -0.5127674341201782, 0.047412894666194916, -0.18004824221134186, -0.615779459476471, 0.9969949722290039, 0.3020975589752197, -0.2518087327480316, 0.37330496311187744, -0.29318439960479736, -0.27480295300483704, 0.18953537940979004, 1.1214263439178467, -0.03467397764325142, 0.17175927758216858, -0.15296487510204315, 0.02577519789338112, 0.21472157537937164, 0.3990453779697418, 0.5800144672393799, 0.7931126952171326, -0.3929918706417084, 0.7217679619789124, -0.28168290853500366, 0.07218557596206665, 0.3473726809024811, 0.7048264741897583, 0.0649348646402359, -0.5933091640472412, -0.3558711111545563, 1.571296215057373, -1.081658124923706, -0.39226993918418884, 0.9028757810592651, -0.2191924750804901, 0.41394269466400146, -0.15143908560276031, 0.2596852779388428, -0.13515785336494446, -0.9929466247558594, 0.5877224206924438, -0.7289437651634216], [0.09794001281261444, 0.9287726283073425, -0.21108470857143402, -0.33191466331481934, 0.2295602709054947, -0.817850649356842, 0.12044117599725723, -0.6431976556777954, 0.34154051542282104, -0.26740944385528564, 0.1723853349685669, -0.39848288893699646, 0.3300703763961792, 0.4119032919406891, -0.24467453360557556, -0.32879263162612915, 0.5573919415473938, 0.7773308157920837, -0.03097798489034176, 0.4626595973968506, -0.09653320163488388, 0.27529263496398926, -0.3030340075492859, 0.3276619017124176, -0.5465219616889954, -0.2979937195777893, -0.6176669597625732, -0.7350994944572449, -0.002076179953292012, -0.3578450083732605, 1.050199270248413, -0.3656791150569916, 0.36314868927001953, 0.5415497422218323, 0.9204831719398499, -0.8429520130157471, -0.6376948356628418, -0.43330901861190796, 0.16645480692386627, -0.13157996535301208, 0.18723420798778534, 0.2741236090660095, 0.6814454793930054, 0.13765062391757965, -0.6449400186538696, -0.7152114510536194, -0.12903547286987305, -0.2814684212207794, 0.27892348170280457, 0.8894116282463074, -0.10986921191215515, -1.1853939294815063, 0.47955822944641113, -0.5117124319076538, 0.025147175416350365, -0.5666738152503967, -0.6357588171958923, 0.6804415583610535, 0.44465091824531555, -0.3572807312011719, 0.4864681363105774, -0.48468244075775146, 1.0255770683288574, -1.0066710710525513], [0.05770503357052803, -0.6230103373527527, 0.3008553683757782, 0.700778067111969, -0.23137590289115906, 0.7827839851379395, -0.6102397441864014, -0.3633263111114502, -1.1446058750152588, 0.8901513814926147, -0.186383455991745, 0.4977416396141052, -0.10958819836378098, -0.6674988269805908, -0.18718597292900085, 0.12958693504333496, -0.05919546261429787, -0.4850984513759613, 1.3064475059509277, 0.8312985301017761, 0.6098862886428833, -0.14159449934959412, 0.6574477553367615, 0.5494109988212585, 0.2709214687347412, -0.34458398818969727, -0.16292475163936615, 0.3004809021949768, 1.1533586978912354, 0.8246363997459412, 0.3213283121585846, -0.09584169834852219, -1.0890392065048218, 0.26786333322525024, 0.7944603562355042, 0.2107256054878235, -0.7638033032417297, 0.26210930943489075, 0.005641198717057705, 0.08920398354530334, -0.42203906178474426, 0.5760370492935181, 0.1450107842683792, 0.17913198471069336, 0.967456042766571, 0.3919832706451416, 0.4613155722618103, 0.49122560024261475, -0.15921902656555176, -0.3946307301521301, -0.18610769510269165, -0.2094126045703888, -0.9990876913070679, -0.05119280889630318, -0.09563865512609482, -0.6589800715446472, 0.41401487588882446, 0.18428277969360352, -0.0932610034942627, 1.0334553718566895, 0.398200660943985, -0.27189189195632935, -0.6920416951179504, 0.32091885805130005], [0.480016827583313, -0.4966762065887451, -0.6216842532157898, -0.17486752569675446, 0.4065338969230652, 0.2364184558391571, 0.04619290307164192, 1.1559572219848633, -0.6852723360061646, -0.15186041593551636, 0.20321451127529144, 0.1616412252187729, -0.8609560132026672, 0.4534081816673279, 0.39640846848487854, -0.16475680470466614, 0.23010346293449402, -0.09106466174125671, 0.9350199699401855, -0.05104558542370796, 1.0571930408477783, 0.0938928872346878, -0.0757032185792923, -0.2632450759410858, 0.200464129447937, -0.26112017035484314, 0.15032601356506348, 0.6020817160606384, 0.14238165318965912, -0.640011191368103, 0.33530744910240173, -0.35045793652534485, -0.1834123283624649, -0.03397836536169052, -0.10759087651968002, 0.16168224811553955, -0.749823272228241, 0.11354149132966995, -0.6681593060493469, 1.1718990802764893, 0.5051043629646301, 0.6819053292274475, -0.3346115052700043, -0.824557900428772, 1.1555382013320923, -0.6036047339439392, 0.6641510725021362, 0.4331534802913666, 0.8481924533843994, -0.8618602156639099, -0.623171329498291, 0.41918763518333435, -0.4472488462924957, -0.8382503390312195, 1.1528452634811401, 0.23573680222034454, 0.42212098836898804, 0.3871491849422455, -0.7743521928787231, -0.3600543141365051, 0.41426125168800354, -0.44027742743492126, -1.0944935083389282, 0.023068005219101906], [-0.21388790011405945, -0.18497273325920105, -0.3929170072078705, 0.4975011646747589, 0.8662815093994141, 0.03837203234434128, -0.6963260173797607, 0.28382664918899536, 0.36132678389549255, -0.04393327236175537, -0.21545900404453278, 0.07933919876813889, -1.1991897821426392, -0.5685876607894897, -0.5713104009628296, 0.5966396331787109, -0.4780424237251282, -1.1405079364776611, 0.7828341722488403, -0.5174704790115356, 0.5301441550254822, -0.23891158401966095, -0.6498485803604126, -0.23966097831726074, -0.22862114012241364, 0.8562759160995483, 0.5389225482940674, 0.6350173354148865, -0.1354895383119583, 0.17611315846443176, -1.3996529579162598, -0.05931141972541809, 0.40485480427742004, -0.3515543043613434, -1.1220325231552124, 0.31156015396118164, -0.24926787614822388, 0.18284015357494354, 0.1774812489748001, 0.36077961325645447, -0.5464267730712891, -0.40910184383392334, 0.24088287353515625, -0.19668802618980408, -0.014876516535878181, -0.5822754502296448, 0.6667584180831909, 0.22053031623363495, -0.6996209025382996, 0.29021990299224854, -0.5719208121299744, 0.5057847499847412, 0.1709074079990387, 0.9240687489509583, 0.7705442905426025, -0.3908136785030365, -0.6339319348335266, 0.46695777773857117, -0.34571248292922974, -0.4572354257106781, -0.09087732434272766, 0.7956113815307617, -0.3313543498516083, 0.372792512178421], [1.1593502759933472, 0.715041995048523, -0.7307782173156738, 0.0862768292427063, 0.9552000761032104, -0.3832303285598755, -0.49132686853408813, -0.4487711787223816, 1.2171645164489746, 0.021966008469462395, 0.5442203283309937, -0.9468206167221069, 0.3732682764530182, 0.7671979665756226, 0.14049650728702545, -0.7436248660087585, -0.24039502441883087, -0.15944980084896088, 0.055511776357889175, -0.008506640791893005, -0.4168757200241089, -0.421895295381546, -0.33525550365448, -0.19540949165821075, 0.36081913113594055, 0.04115384444594383, 0.539640486240387, 0.31833699345588684, -0.5083988308906555, -0.35098493099212646, 0.09123615175485611, 0.07705049961805344, -0.5577001571655273, -0.15480546653270721, 0.30376002192497253, 0.6932672262191772, -0.8979170918464661, -0.017431840300559998, -0.1302023082971573, -0.3640531599521637, 0.8465628027915955, 0.2551203668117523, -0.13798540830612183, -0.30096831917762756, -0.5106556415557861, 0.18894556164741516, 0.4536186158657074, -0.08505122363567352, -0.3183814287185669, 1.199530005455017, 0.5420256853103638, 0.04993048310279846, -0.4049569368362427, -0.4227733910083771, 0.026943648234009743, 0.38899093866348267, -0.8860475420951843, 0.3529926836490631, 0.03989556431770325, 0.2017202526330948, 0.36863574385643005, -0.5858064889907837, -0.5384843945503235, -0.475842148065567], [-0.12116055935621262, -0.5594345331192017, 0.7835156321525574, -0.6626426577568054, 0.2502936124801636, -0.11214301735162735, -0.3410862386226654, -0.4433029592037201, -0.12884309887886047, 0.13628751039505005, 0.6052929162979126, 0.32422420382499695, -0.8229010701179504, -1.350551724433899, -0.7743116617202759, -0.24484877288341522, 0.5274329781532288, -0.7571449279785156, 0.6517723798751831, -0.23527830839157104, 0.33640405535697937, -0.19096505641937256, -0.11285470426082611, -0.07850939780473709, -0.16634482145309448, 0.5014060735702515, -0.5922567844390869, -0.7554278373718262, 0.4653310477733612, 0.4455394446849823, 0.037630632519721985, -0.2439545840024948, -0.5258448123931885, 0.10922110825777054, -0.6118710041046143, -0.8599111437797546, -0.7419331073760986, -0.05461408942937851, 0.867716372013092, -0.18059395253658295, -1.3168834447860718, 0.5266565680503845, -0.6323597431182861, 0.3670352101325989, -0.23038390278816223, 0.4366312325000763, 0.4974013566970825, -0.05733490735292435, -0.5425690412521362, -0.27704715728759766, -1.0124439001083374, 0.08083433657884598, 0.10403703153133392, 0.4149165749549866, 0.1855139136314392, -0.0011599318822845817, 0.09112785756587982, 1.0679422616958618, -0.519433319568634, 0.12203653901815414, -0.663739800453186, 0.26234978437423706, 0.01649359241127968, -1.0918643474578857], [-0.3640797436237335, -0.1732790768146515, 0.4910027086734772, 0.23659950494766235, -0.19449849426746368, -0.32745039463043213, -0.8289140462875366, 0.2333650290966034, 0.11344248801469803, 0.7399993538856506, 0.2659689486026764, 1.1396100521087646, 0.34871724247932434, 0.17195075750350952, 0.6550496220588684, 0.7179338932037354, -0.6234434247016907, -0.655917227268219, -0.5866730213165283, 0.4134211838245392, 0.9228956699371338, -0.32486552000045776, 0.35369300842285156, -0.6651487946510315, 0.9455316662788391, 0.18856282532215118, -0.6530244946479797, -0.5583948493003845, -0.36561182141304016, 0.04357195273041725, -0.6842544674873352, -0.7457671761512756, 0.4607809782028198, 0.280464231967926, -1.0098261833190918, -0.20899800956249237, 0.24553421139717102, -0.18199019134044647, 0.5205492973327637, -0.0008640672313049436, 0.14568935334682465, -0.23763400316238403, -0.5173323750495911, -0.8948543667793274, 0.6496553421020508, 0.6634525656700134, 0.6762828230857849, 0.34214651584625244, -0.04225274547934532, -0.6489904522895813, 0.33977991342544556, -0.3425113558769226, -0.20471350848674774, 0.9770987033843994, -0.16717077791690826, -0.48245769739151, 0.40142926573753357, 0.36253249645233154, -0.34387147426605225, 0.7480267286300659, 0.39323660731315613, 0.16972558200359344, -0.5094162225723267, 0.0008092848584055901], [0.8484907150268555, 0.19094082713127136, -0.2450607717037201, 0.1605413258075714, -0.4736824035644531, -0.03770367056131363, -0.7974322438240051, 0.5939814448356628, -0.08765934407711029, 0.2740066945552826, -0.023351246491074562, 0.7229105234146118, -0.4082370400428772, -0.04870263487100601, -0.26119980216026306, -0.14231997728347778, -0.5109091997146606, -0.49628642201423645, 0.2947596311569214, 0.12119103223085403, 0.32910189032554626, 0.8392416834831238, -0.45104968547821045, 0.10541196912527084, 0.6395463943481445, 0.3664642870426178, -0.22877347469329834, -0.748519778251648, 0.10554099828004837, 0.4086017310619354, -0.118429034948349, -0.05373374745249748, 0.4410092532634735, -0.8845661878585815, 0.3016798496246338, 0.7323253154754639, -0.1926407516002655, -1.1392238140106201, -0.5515409708023071, 0.15265502035617828, 0.03783063590526581, 0.2745957672595978, 0.5089574456214905, -0.7893738150596619, 0.5169968605041504, -0.21054886281490326, 0.2245378941297531, -0.21156303584575653, 1.292554497718811, 0.2166682779788971, -0.4629489779472351, -0.5472886562347412, -1.1899515390396118, -0.010594169609248638, 1.1970235109329224, -0.06680554151535034, 0.14750434458255768, 0.15806743502616882, -0.3404342532157898, -0.6911656260490417, 1.0136187076568604, 0.035791248083114624, -0.7959699034690857, 0.025452574715018272], [0.19943402707576752, 0.33907395601272583, -0.15230399370193481, 0.38882458209991455, -1.3126572370529175, 0.028511976823210716, -0.3473533093929291, -0.2892816364765167, 0.3594416081905365, -0.03545379638671875, -0.39885082840919495, -0.22812992334365845, -0.05079912394285202, -0.5635113716125488, -0.8981558084487915, -0.012223981320858002, -2.062713623046875, -1.084430456161499, -0.12032528221607208, -0.17799700796604156, 0.6961796879768372, 0.1534489095211029, 0.3448081314563751, -0.37660807371139526, -1.2363204956054688, -0.2057289332151413, -0.7457383275032043, 0.20391662418842316, -0.5737360119819641, -0.22034090757369995, -0.8651507496833801, -0.10893575102090836, 0.0418969951570034, 0.044649139046669006, -0.29232776165008545, 0.6683505177497864, -0.26981237530708313, -0.8048123717308044, -0.17099152505397797, -0.6177659630775452, 0.31493961811065674, -0.25504252314567566, 0.4862963855266571, 0.0860016793012619, 0.17822349071502686, -0.3271331787109375, -0.7215312719345093, -0.03442690521478653, -0.1594781130552292, 0.5512779951095581, 0.30746689438819885, -0.31441086530685425, -0.06723792105913162, 0.12119562923908234, 0.4477638304233551, 1.0430430173873901, -0.6188620328903198, 0.2571992576122284, 0.8266279101371765, -0.8477853536605835, 1.0017346143722534, 0.7014250159263611, 0.4149007499217987, 0.119094118475914], [-0.4001998007297516, 0.001980650005862117, -0.9797636866569519, -0.6347225308418274, -0.707625687122345, -0.07034291326999664, 0.15912330150604248, 1.0063282251358032, 0.10920552164316177, -1.3882744312286377, -0.34387078881263733, -0.21261705458164215, -0.7416880130767822, 0.15390364825725555, -0.03457193076610565, 0.22269010543823242, -0.15747077763080597, 0.8132011294364929, 1.1635217666625977, -0.3354403078556061, 0.7110434174537659, -0.5160247087478638, -0.39597806334495544, -0.08564357459545135, -0.3451279401779175, 0.04655157029628754, 0.022550925612449646, 0.11574307829141617, -0.1403834968805313, -0.12520284950733185, 0.45504316687583923, -0.2793852984905243, 1.149355411529541, 0.31007617712020874, -0.26959478855133057, 0.894952118396759, -0.7284591197967529, 0.6242064237594604, -0.7895375490188599, 0.23527316749095917, 0.8092023134231567, -0.2857973873615265, 0.06802760064601898, 0.31616848707199097, 0.022128576412796974, -1.4677032232284546, 0.295083224773407, 0.01567392796278, 0.20313802361488342, 0.3248656690120697, -0.4268222153186798, 0.5121008157730103, -0.15228153765201569, -0.44772815704345703, -0.044306520372629166, -0.49467766284942627, 0.113773413002491, -0.3507314920425415, -0.5110823512077332, -0.6996207237243652, 0.5888410210609436, 0.2617340087890625, -0.18353471159934998, 1.6681969165802002], [-0.2582746744155884, -0.7097546458244324, 0.103506900370121, 0.9378563165664673, -0.31847047805786133, -0.11337723582983017, 0.6562244296073914, 0.7674683928489685, 0.2454311102628708, -0.3259599208831787, 0.4673249125480652, -0.2773655951023102, -0.5164182186126709, 0.7119613885879517, 0.7163178324699402, -0.4130725562572479, 0.4890223443508148, -0.17139722406864166, -0.14432434737682343, -0.21883511543273926, -0.09491685777902603, 0.4435765743255615, 0.3345922529697418, -0.12662097811698914, -0.782123863697052, -0.5797997117042542, -0.16319631040096283, -0.5302183628082275, 0.12432389706373215, -0.8367630243301392, 0.26684650778770447, -0.06777530163526535, 0.17536456882953644, -0.667153000831604, -0.042201243340969086, 0.35382869839668274, 0.38502275943756104, -1.2063086032867432, -0.9208674430847168, 0.293803334236145, 0.49872076511383057, -0.12398111820220947, 1.0123753547668457, -0.256227970123291, 0.6505253911018372, -1.420795202255249, 0.2361680418252945, 0.5483161807060242, 1.3553858995437622, 0.06508466601371765, -0.7666350603103638, -0.2615027129650116, 0.12733785808086395, -0.4590837359428406, 0.545731246471405, 0.29876238107681274, 0.02283947728574276, 0.10674824565649033, -0.5168575644493103, -0.006278654094785452, 0.7997318506240845, -0.5402175784111023, -0.18772375583648682, 0.33044660091400146]], "biases": [-0.40529513359069824, 0.07412725687026978, 0.48377227783203125, 0.7180691957473755, 0.03859518840909004, 0.558036744594574, -0.25696292519569397, 0.607425332069397, 0.1992650330066681, -0.00016396287537645549, -0.7030194997787476, -0.12947143614292145, -0.23047707974910736, -0.019258448854088783, 0.22380803525447845, 0.15369316935539246, 0.1394961178302765, 0.24640707671642303, 0.006978604942560196, -0.08877170830965042, 0.041999783366918564, 0.49475395679473877, -0.3024362623691559, 0.010095428675413132, 0.42106893658638, 0.22848384082317352, -0.2758634686470032, -0.4124930202960968, 0.30032917857170105, -0.18076682090759277, 0.5146489143371582, -0.030872557312250137, 0.004622803069651127, 0.642983078956604, 0.22495007514953613, 0.594558596611023, -0.15583740174770355, 0.026944948360323906, 0.30103328824043274, -0.18634852766990662, 0.3251935541629791, -0.04413774982094765, 0.10853936523199081, -0.01662377454340458, 0.01323697715997696, -0.4233063757419586, 0.012650012969970703, -0.1388234943151474, 0.780535101890564, -0.14242343604564667, -0.10301131010055542, 0.45814427733421326, 0.7678816318511963, 0.13990037143230438, 0.5259569883346558, 0.01835327036678791, -0.20441119372844696, 0.13699473440647125, -0.05200525373220444, -0.05121942609548569, 0.17302332818508148, -0.6262202858924866, -0.4525679051876068, 0.046026937663555145], "activation": "tanh"}, {"weights": [[-0.24019327759742737, -0.06814779341220856, 0.20446433126926422, -0.25447314977645874, 0.009301956743001938, -0.3924718201160431, -0.3115293085575104, -0.08758183568716049, 0.007549172267317772, -0.006793119478970766, 0.5405831336975098, -0.08157706260681152, -0.5169988870620728, -0.14881472289562225, 0.2911801040172577, -0.09530900418758392, -0.03790590912103653, 0.16347071528434753, -0.22494907677173615, -0.03573251888155937, -0.45513099431991577, 0.5264711380004883, -0.5249466896057129, 0.497956246137619, 0.1438872367143631, 0.24934935569763184, 0.5592330098152161, -0.3536989986896515, 0.004772149492055178, -0.06775439530611038, -0.34457120299339294, -0.33423471450805664, 0.5386609435081482, 0.575624406337738, 0.6682960987091064, -0.14664015173912048, 0.07737776637077332, 0.06335709989070892, -0.3421240746974945, 0.2797907888889313, -0.29214927554130554, 0.49884870648384094, -0.21572786569595337, -0.3244418799877167, 0.12258250266313553, -0.04226038604974747, 0.16219602525234222, -0.001526994863525033, -0.1351550966501236, 0.3214125335216522, -0.4483533501625061, -0.13462169468402863, 0.27647849917411804, -0.49462437629699707, 0.009851839393377304, 0.4463803768157959, -0.3584849536418915, 0.0538661889731884, 0.05253215879201889, 0.5280542969703674, -0.37014326453208923, 0.35175377130508423, 0.42779943346977234, -0.0343274287879467], [-0.04499707370996475, -0.4680003821849823, -0.12932169437408447, -0.19957618415355682, -0.03425607085227966, -0.32500654458999634, 0.1858317255973816, 0.23947063088417053, 0.2047896832227707, 0.09231191128492355, 0.44443225860595703, 0.011558860540390015, -0.33469676971435547, -0.5392339825630188, -0.2795398533344269, 0.18658727407455444, 0.24677172303199768, -0.13085275888442993, -0.2674136161804199, 0.28444531559944153, 0.11075688153505325, -0.2114291787147522, -0.006232851184904575, -0.1319812387228012, -0.37827304005622864, -0.5984795093536377, -0.22621874511241913, 0.0803881585597992, -0.3254050612449646, 0.030046457424759865, 0.09291237592697144, 0.06392829120159149, 0.27405139803886414, -0.07536163926124573, -0.02968764863908291, 0.40159472823143005, -0.21511249244213104, -0.0250890851020813, -0.33346888422966003, 0.0126497158780694, -0.06118308752775192, -0.1648915410041809, -0.19382847845554352, -0.010848421603441238, -0.11092546582221985, 0.5688595175743103, -0.3677150309085846, 0.04163110628724098, 0.6643933057785034, -0.18871620297431946, -0.310741126537323, 0.1871136575937271, 0.541928768157959, 0.13872067630290985, 0.2850189208984375, 0.04198778420686722, -0.14643973112106323, 0.2277081310749054, -0.2929174602031708, -0.1699686199426651, -0.08589686453342438, 0.30209988355636597, 0.17644603550434113, -0.35328444838523865], [-0.17208655178546906, -0.24127857387065887, -0.21310704946517944, 0.10255154222249985, 0.25444722175598145, 0.20319920778274536, -0.36194658279418945, 0.041118323802948, 0.255170077085495, -0.3578934967517853, -0.2320190966129303, -0.3685721158981323, -0.2616516649723053, 0.3031884729862213, -0.18847592175006866, -0.5820668935775757, 0.04732503741979599, 0.4469591975212097, 0.6951189041137695, -0.6326642036437988, 0.34069710969924927, 0.2660435140132904, -0.1563108116388321, 0.46265673637390137, -0.29632046818733215, -0.05915440246462822, -0.45747119188308716, 0.3303881883621216, -0.327422559261322, 0.0921831950545311, 0.4281911849975586, 0.017600543797016144, 0.41676872968673706, 0.25943174958229065, 0.7976629734039307, 0.2713397145271301, -0.014176799915730953, 0.16180211305618286, -0.16251510381698608, 0.32898110151290894, 0.05123814567923546, 0.33834129571914673, 0.27418187260627747, 0.5780794620513916, -0.45471009612083435, -0.14097239077091217, -0.2808034121990204, 0.568181574344635, -0.16860578954219818, 0.07679487019777298, 0.6087031364440918, -0.4227749705314636, 0.5335715413093567, -0.6318379640579224, 0.18723948299884796, 0.07550540566444397, -0.24765938520431519, -0.3074326813220978, 0.48373377323150635, -0.04020209610462189, 0.2585485577583313, -0.6945140361785889, 0.6230336427688599, 0.3559474050998688], [-0.04448727145791054, -0.19231534004211426, -0.16518248617649078, -0.18808680772781372, 0.24054430425167084, -0.010711758397519588, 0.15170739591121674, -0.4710996150970459, 0.14522431790828705, 0.1702403575181961, 0.15992335975170135, 0.004465111996978521, -0.02292528934776783, 0.5006674528121948, 0.8325135707855225, 0.03550785407423973, 0.22754928469657898, 0.1868406981229782, 0.13811543583869934, 0.0245230533182621, 0.31867873668670654, -0.10581054538488388, 0.2127668857574463, 0.013062609359622002, -0.10259498655796051, 0.02212430350482464, -0.06750479340553284, 0.2485484927892685, -0.09316031634807587, 0.12655727565288544, 0.21197627484798431, 0.05107201263308525, 0.06508889049291611, 0.07831356674432755, 0.14250536262989044, -0.051506634801626205, -0.08210133761167526, -0.2573743462562561, 0.05498447269201279, -0.0447588749229908, 0.14469246566295624, -0.050326116383075714, 0.2944641709327698, 0.24085864424705505, -0.01729622483253479, 0.3078048825263977, 0.12021010369062424, -0.3521188497543335, 0.22250202298164368, 0.0258498415350914, -0.12117834389209747, 0.04368019849061966, -0.120058074593544, -0.14042334258556366, 0.15768659114837646, -0.423389196395874, -0.4473802149295807, 0.00014598484267480671, -0.1043262854218483, -0.10844794660806656, 0.19168521463871002, 0.14917196333408356, -0.12487783282995224, -0.22911083698272705], [-0.3024996519088745, -0.05777892470359802, 0.03247055783867836, 0.07215116918087006, 0.3092215657234192, 0.1432676613330841, 0.4234517514705658, 0.17170658707618713, 0.3925483822822571, 0.02689841017127037, -0.35903894901275635, 0.13544423878192902, -0.06495440006256104, -0.1767062544822693, 0.28675875067710876, 0.5198566317558289, -0.693855345249176, 0.19006240367889404, 0.33123165369033813, -0.21080666780471802, 0.10601404309272766, 0.3955574333667755, -0.37584421038627625, 0.6270888447761536, 0.25469255447387695, 0.03641289100050926, 0.1442990005016327, -0.10799845308065414, 0.0960766151547432, -0.5249271988868713, 0.048810675740242004, -0.3000865578651428, -0.2001156359910965, -0.3257097899913788, -0.35349103808403015, 0.1007016971707344, 0.45647957921028137, -0.14928235113620758, 0.2026972472667694, -0.22475901246070862, -0.04810480400919914, 0.3097691535949707, -0.10736871510744095, 0.3895301818847656, -0.3765597343444824, 0.11476268619298935, -0.1776617020368576, 0.23194144666194916, 0.10766547173261642, 0.49441275000572205, 0.3089227080345154, 0.5283969640731812, -0.23756657540798187, -0.18247239291667938, -0.31063172221183777, 0.07387732714414597, -0.5178130269050598, -0.5211402177810669, -0.8097481727600098, 0.12816505134105682, -0.5043428540229797, -0.3002837896347046, -0.10452486574649811, -0.5838865041732788], [0.33641505241394043, -0.11864425241947174, 0.5104791522026062, -0.5668296217918396, -0.10570891201496124, -0.1618269383907318, 0.2167576402425766, -0.23122963309288025, -0.14275749027729034, 0.09126179665327072, 0.16444092988967896, -0.13759002089500427, -0.18288744986057281, 0.5765373706817627, 0.11393888294696808, 0.22245056927204132, 0.11431872099637985, -0.5138548612594604, 0.7988497018814087, -0.6837390065193176, 0.03237371891736984, -0.09091988950967789, 0.18849073350429535, 0.08885705471038818, 0.07664723694324493, 0.2667423188686371, -0.40691184997558594, -0.583778440952301, -0.5113211274147034, -0.8573092818260193, 0.6990510821342468, 0.6598275303840637, -0.08136716485023499, -0.3635297417640686, -0.35998788475990295, 0.03591836616396904, 0.2821991741657257, -0.5324782133102417, 0.412898987531662, -0.05253526195883751, -0.43995213508605957, -0.23961380124092102, -0.0728471428155899, -0.5505934953689575, 0.4463491141796112, -0.19219368696212769, -0.2689187824726105, -0.023888353258371353, 0.26871299743652344, 0.1422191858291626, -0.21589235961437225, -0.05035577341914177, -0.17606718838214874, -0.23287531733512878, -0.5378811955451965, 0.012591167353093624, 0.5578104853630066, 0.1063319742679596, -0.1102236807346344, -0.06902806460857391, 0.2938430905342102, 0.006873771548271179, -0.2242279052734375, 0.39433202147483826], [-0.5327441692352295, -0.18791405856609344, 0.6080801486968994, 0.30498483777046204, 0.3637506067752838, -0.05673746392130852, -0.06887950748205185, 0.5506571531295776, -0.05057961121201515, -0.605492353439331, 0.0712597668170929, 0.36653706431388855, -0.3281613290309906, 0.23612597584724426, -0.2981434464454651, 0.33831092715263367, 0.22626268863677979, 0.31263789534568787, -0.31202244758605957, -0.45580998063087463, -0.06243620812892914, 0.3885050415992737, 0.12362394481897354, -0.09988701343536377, -0.15581443905830383, 0.3301350772380829, 0.05194060131907463, 0.15129879117012024, 0.5919589996337891, -0.08885258436203003, -0.13431477546691895, 0.07681069523096085, -0.7230460047721863, -0.05506327748298645, -0.07820901274681091, -0.5600073933601379, -0.1711299568414688, -0.15431059896945953, -0.22922997176647186, 0.23816809058189392, 0.21323810517787933, 0.6961727738380432, 0.20093344151973724, -0.5358663201332092, -0.014567725360393524, 0.3204823136329651, 0.010961173102259636, -0.2895980477333069, 0.43849092721939087, 0.19892248511314392, 0.25909101963043213, 0.28692877292633057, 0.17534779012203217, -0.21373936533927917, -0.5397375226020813, 0.5803073048591614, -0.09318586438894272, 0.18005146086215973, 0.10508132725954056, -0.21645724773406982, 0.009717311710119247, 0.0791049674153328, -0.23656001687049866, -0.014594932086765766]], "biases": [-0.0018477262929081917, 0.14004771411418915, 0.12927182018756866, 0.0023617041297256947, 0.10382892191410065, -0.07731980830430984, 0.09324260801076889], "activation": "linear"}]} \ No newline at end of file diff --git a/Game/scripts/ai_ship_controller.gd b/Game/scripts/ai_ship_controller.gd index 981fba4d..26b16c87 100644 --- a/Game/scripts/ai_ship_controller.gd +++ b/Game/scripts/ai_ship_controller.gd @@ -17,13 +17,16 @@ extends ShipController # Uniform noise magnitude added to each action axis (0 = play at full skill). @export_range(0.0, 1.0) var action_noise: float = 0.0 -# Must mirror whatever the model was actually trained with (see -# ShipAIController's identical exports on the training side, curriculum -# stages 1-2 in TRAINING.md). A model trained grounded (mask on) never got a -# reward gradient on these axes, so its raw output there is untrained noise — -# leaving this true for such a model doesn't make it fly well, it just lets -# that noise reach the ship instead of being discarded like it was in -# training. Set false to match a grounded-trained model's actual behaviour. +# Only meaningful for a "continuous"-action_space model (see +# ShipActionCodec) — i.e. one exported before curriculum generation 4, such +# as Game/bots/promoted/reference-grounded.json. Must mirror whatever the +# model was actually trained with: a model trained grounded (mask on) never +# got a reward gradient on these axes, so its raw output there is untrained +# noise — leaving this true for such a model doesn't make it fly well, it +# just lets that noise reach the ship instead of being discarded like it was +# in training. Set false to match a grounded-trained model's actual +# behaviour. Generation-4-onward (multi_discrete) models train the full +# action space from the start, so these flags are ignored for them. @export var allow_vertical := true @export var allow_pitch_roll := true @@ -59,26 +62,18 @@ func get_action() -> ShipAction: func _decide() -> void: var obs := ShipObservations.build(_ship, _opponent, _ball, _attack_goal_position) var out := _policy.forward(obs) - # Output layout is the trainer's flattened action space (Box(7)), which - # gymnasium orders by SORTED key name — rotation xyz, thrust xyz, turbo - # (> 0 means on) — NOT ShipAction's thrust-first declaration order. - _action.rotation = Vector3( - _axis(out[0]) if allow_pitch_roll else 0.0, - _axis(out[1]), - _axis(out[2]) if allow_pitch_roll else 0.0 - ) - _action.thrust = Vector3( - _axis(out[3]), - _axis(out[4]) if allow_vertical else 0.0, - _axis(out[5]) - ) - _action.turbo = out[6] > 0.0 - - -func _axis(value: float) -> float: - if action_noise > 0.0: - value += randf_range(-action_noise, action_noise) - return clampf(value, -1.0, 1.0) + # See ShipActionCodec for the decode — the single source of truth shared + # with the training side, so this must never reimplement layout/ordering + # locally (see that file's header for why). + if _policy.action_space.get("type", "continuous") == "continuous": + _action = ShipActionCodec.from_continuous(out, action_noise) + if not allow_pitch_roll: + _action.rotation.x = 0.0 + _action.rotation.z = 0.0 + if not allow_vertical: + _action.thrust.y = 0.0 + else: + _action = ShipActionCodec.from_logits(out, action_noise) # Find ship/ball/opponent/goal once everything is spawned. ShipAction axes diff --git a/Game/scripts/policy_network.gd b/Game/scripts/policy_network.gd index 05be027e..b0fc41d7 100644 --- a/Game/scripts/policy_network.gd +++ b/Game/scripts/policy_network.gd @@ -14,9 +14,18 @@ extends RefCounted # {"weights": [[out x in floats]], "biases": [out floats], "activation": "tanh" | "linear"}, # ... # ] +# "action_space": {"type": "multi_discrete", "heads": [{"name","bins"}, ...]} // optional # } +# +# "action_space" is absent from every model exported before curriculum +# generation 4 (e.g. Game/bots/promoted/easy.json) — absence means +# {"type": "continuous"}, decoded via ShipActionCodec.from_continuous, the +# same flattened-Box(7)-mean-output path this class has always produced. +# This class itself never changes behaviour based on it; only the caller +# (AIShipController._decide) branches on action_space["type"]. var input_size: int = 0 +var action_space: Dictionary = {"type": "continuous"} var _layers: Array = [] @@ -32,6 +41,7 @@ static func load_from_file(path: String) -> PolicyNetwork: var net := PolicyNetwork.new() net.input_size = int(data.get("input_size", 0)) + net.action_space = data.get("action_space", {"type": "continuous"}) for layer in data["layers"]: # Flatten each layer's weights into a PackedFloat64Array for speed var out_size: int = layer["biases"].size() @@ -55,7 +65,14 @@ static func load_from_file(path: String) -> PolicyNetwork: func forward(observation: Array) -> Array: - var x := PackedFloat64Array(observation) + if observation.size() < input_size: + push_error("PolicyNetwork: observation has %d values, model expects %d" % [observation.size(), input_size]) + # Slice rather than trust the caller: ShipObservations.SIZE only ever + # grows (append-only), so an older/smaller model must still decode + # correctly against a newer, longer observation vector — the extra + # trailing values it never trained on are simply dropped here rather + # than corrupting the first layer's dot product by accident. + var x := PackedFloat64Array(observation.slice(0, input_size)) for layer in _layers: var in_size: int = layer["in_size"] var out_size: int = layer["out_size"] diff --git a/Game/scripts/ship.gd b/Game/scripts/ship.gd index 891f6528..e1310025 100644 --- a/Game/scripts/ship.gd +++ b/Game/scripts/ship.gd @@ -119,6 +119,16 @@ func _ready(): controller = child break + # Always on (moved here from ShipAIController.setup, which only enabled + # it for training-side ships): ShipObservations now reads own-contact + # state (see its "contact" section) for every ship, training or shipped, + # so the RigidBody3D contact list must exist unconditionally rather than + # only for whichever ship happened to be a training agent. Cheap — a + # short per-tick contact list from the physics engine, not a rendering + # cost like the headless skips just below. + contact_monitor = true + max_contacts_reported = 8 + _apply_team_color() _boundary = get_tree().get_first_node_in_group("arena_boundary") diff --git a/Game/scripts/ship_action_codec.gd b/Game/scripts/ship_action_codec.gd new file mode 100644 index 00000000..aa3cd9cf --- /dev/null +++ b/Game/scripts/ship_action_codec.gd @@ -0,0 +1,128 @@ +class_name ShipActionCodec +extends RefCounted + +# Single source of truth for the RL action layout — shared by training +# (ShipAIController.get_action_space/set_action) and in-game inference +# (AIShipController._decide via PolicyNetwork) so a trained policy's action +# output is decoded identically in both contexts. Mirrors ShipObservations' +# "do not fork this logic" role for observations; the train/inference seam +# broke once before over exactly this kind of divergence (commit 8c15c46). +# +# Curriculum generation 4 replaces the old continuous Gaussian action space +# (Box(7), see the "continuous" path below) with a per-axis MultiDiscrete +# space: PPO's Gaussian std reliably collapsed to ~0.13-0.15 within the first +# ~10% of every training run across 3 generations and never recovered, which +# made a *sustained* set-point (e.g. hovering, thrust.y ~= 0.408 given this +# ship's mass/thrust — see TRAINING.md) essentially unreachable: the +# collapsed distribution can brush the hover value but never hold it long +# enough to accumulate the reward signal that would move the mean. A +# discrete bin is a single, atomic, repeatable choice with non-zero +# probability under any softmax, which does not have that failure mode. +# +# HEADS order is deliberately gymnasium's *sorted* key order (verified: +# "rot_x" < "rot_y" < "rot_z" < "thrust_x" < "thrust_y" < "thrust_z" < +# "turbo") — godot_rl's ActionSpaceProcessor builds the Tuple action space +# from a gymnasium Dict, which sorts keys regardless of insertion order, so +# this order is what SB3/PPO actually samples/trains against and what +# set_action() receives keyed by. Do not reorder without re-verifying that +# sort order. +const HEADS := [ + {"name": "rot_x", "bins": [-1.0, -0.5, 0.0, 0.5, 1.0]}, + {"name": "rot_y", "bins": [-1.0, -0.5, 0.0, 0.5, 1.0]}, + {"name": "rot_z", "bins": [-1.0, -0.5, 0.0, 0.5, 1.0]}, + {"name": "thrust_x", "bins": [-1.0, -0.5, 0.0, 0.5, 1.0]}, + # Deliberately asymmetric: hovering this ship (mass 5.0, vertical_thrust + # 120, default gravity 9.8 m/s^2 — see ship.gd/ship.tscn) requires a + # sustained thrust.y ~= 0.408. Uniform-random selection over these 5 bins + # averages 0.34 — just below neutral buoyancy, so a fresh policy drifts + # gently through the volume instead of pinning to the floor (symmetric + # bins) or sticking to the ceiling (ceiling_pull_strength 11.5 > gravity + # 9.8, so the ceiling is easy to over-shoot into). This is the direct + # analogue of the RLGym/RLBot community fix for the same failure mode + # ("add more jump actions to the discrete action parser"). + {"name": "thrust_y", "bins": [-0.5, 0.0, 0.45, 0.75, 1.0]}, + {"name": "thrust_z", "bins": [-1.0, -0.5, 0.0, 0.5, 1.0]}, + {"name": "turbo", "bins": [0.0, 1.0]}, +] + + +static func action_space_dict() -> Dictionary: + var space := {} + for head in HEADS: + space[head["name"]] = {"size": head["bins"].size(), "action_type": "discrete"} + return space + + +# Training side: `action` is the Dictionary godot_rl's Sync node hands +# set_action() — one entry per HEADS key, each an int (or int-valued float) +# bin index in [0, bins.size()). +static func from_indices(action: Dictionary) -> ShipAction: + var result := ShipAction.new() + var values := {} + for head in HEADS: + var index: int = clampi(int(round(float(action[head["name"]]))), 0, head["bins"].size() - 1) + values[head["name"]] = head["bins"][index] + result.rotation = Vector3(values["rot_x"], values["rot_y"], values["rot_z"]) + result.thrust = Vector3(values["thrust_x"], values["thrust_y"], values["thrust_z"]) + result.turbo = values["turbo"] > 0.0 + return result + + +# In-game inference for a MultiDiscrete-trained export: `logits` is the raw +# policy_network.gd output — 32 floats (5+5+5+5+5+5+2), one contiguous slice +# per head in HEADS order (matches export_policy.py's action_net layer, +# which concatenates SB3's per-head categorical logits in that same order). +# argmax within each slice picks that head's bin, same as SB3's +# MultiCategoricalDistribution.mode() under deterministic inference. +static func from_logits(logits: Array, noise: float) -> ShipAction: + var result := ShipAction.new() + var values := {} + var offset := 0 + for head in HEADS: + var bins: Array = head["bins"] + var index := 0 + if noise > 0.0 and randf() < noise: + # eps-random-bin: the discrete analogue of continuous action_noise + # (see ai_ship_controller.gd) — degrades gracefully and keeps the + # same 0..1 monotonic difficulty semantics as the continuous path. + index = randi() % bins.size() + else: + var best_value: float = logits[offset] + for i in range(1, bins.size()): + if logits[offset + i] > best_value: + best_value = logits[offset + i] + index = i + values[head["name"]] = bins[index] + offset += bins.size() + result.rotation = Vector3(values["rot_x"], values["rot_y"], values["rot_z"]) + result.thrust = Vector3(values["thrust_x"], values["thrust_y"], values["thrust_z"]) + result.turbo = values["turbo"] > 0.0 + return result + + +# Legacy continuous decode — moved verbatim from ai_ship_controller.gd so +# every model exported before generation 4 (no "action_space" block in its +# JSON, e.g. Game/bots/promoted/easy.json) keeps behaving byte-identically. +# `out` is the trainer's flattened Box(7) output, gymnasium-sorted: rotation +# xyz, thrust xyz, turbo (> 0 means on) — NOT ShipAction's thrust-first +# declaration order. +static func from_continuous(out: Array, noise: float) -> ShipAction: + var result := ShipAction.new() + result.rotation = Vector3( + _continuous_axis(out[0], noise), + _continuous_axis(out[1], noise), + _continuous_axis(out[2], noise) + ) + result.thrust = Vector3( + _continuous_axis(out[3], noise), + _continuous_axis(out[4], noise), + _continuous_axis(out[5], noise) + ) + result.turbo = out[6] > 0.0 + return result + + +static func _continuous_axis(value: float, noise: float) -> float: + if noise > 0.0: + value += randf_range(-noise, noise) + return clampf(value, -1.0, 1.0) diff --git a/Game/scripts/ship_action_codec.gd.uid b/Game/scripts/ship_action_codec.gd.uid new file mode 100644 index 00000000..c53a6ba9 --- /dev/null +++ b/Game/scripts/ship_action_codec.gd.uid @@ -0,0 +1 @@ +uid://cvpbp3mj58ejd diff --git a/Game/scripts/ship_ai_controller.gd b/Game/scripts/ship_ai_controller.gd index fdc35a7d..283859b6 100644 --- a/Game/scripts/ship_ai_controller.gd +++ b/Game/scripts/ship_ai_controller.gd @@ -7,10 +7,11 @@ extends AIController3D # the trainer are written into an RLShipController, which the ship pulls like # any other controller. # -# Action space is ShipAction verbatim: 6 continuous axes (thrust xyz, -# rotation xyz, each -1..1) + binary turbo. ShipAction axes are ship-local -# (body frame), so they need no team mirroring — only observations do -# (see ShipObservations.canon). +# Action space/layout is owned by ShipActionCodec (get_action_space/ +# set_action just delegate to it) — see that file for the per-axis +# MultiDiscrete design and why. ShipAction axes are ship-local (body frame), +# so they need no team mirroring — only observations do (see +# ShipObservations.canon). # Reward shaping weights. Dense terms accrue per physics tick (60 sim-ticks # per sim-second); event terms fire once. Exported so tuning needs no code @@ -54,9 +55,13 @@ extends AIController3D # relative to the (then far weaker) ball-seeking shaping. @export var wall_contact_penalty := 0.0025 # Per-tick penalty for not being upright, scaled by tilt: 0 when flat, full -# value (-0.12/s) when inverted. A penalty rather than an upright bonus so a -# flat, idle ship farms nothing. -@export var tilt_penalty := 0.002 +# value when inverted. A penalty rather than an upright bonus so a flat, idle +# ship farms nothing. Lowered 4x for curriculum generation 4 (was 0.002, +# -0.12/s): a genuine aerial approach to a high ball requires pitching, and +# the old value quietly opposed the exact behaviour generation 4 is trying +# to teach. Not removed outright — an always-inverted bot still looks bad in +# a shipped game. +@export var tilt_penalty := 0.0005 # Per-tick bonus for own speed: 0 stationary, full value (+0.24/s) at # max_speed. Run07 lesson: after the kickoff flurry both ships parked next to # a cornered ball — with every other dense term near zero there, standing @@ -79,27 +84,15 @@ extends AIController3D # unaffected; the floor-lock curriculum stage turns it on. @export var airborne_penalty := 0.0 -# Locomotion curriculum: scales how much of the corresponding action axes -# actually reaches the ship, from 0.0 (fully discarded, grounded-only) to -# 1.0 (full effect) — this scales the *effect* of thrust.y/rotation.x/ -# rotation.z in set_action, not the action space's shape: the policy always -# outputs values for these axes (always contributing to PPO's entropy/log- -# prob), they're just attenuated here, so checkpoints stay resumable across -# ramp values. -# -# A hard 0/1 flip (the original bool mask) let PPO's action-distribution -# std collapse to ~0.13-0.15 within the first ~10% of steps, before the -# policy ever meaningfully explored the newly-unmasked axes — 3 independent -# 240M-step attempts at the all-or-nothing flip all landed at a stable -# ~28-32% win rate vs curric-s5-aggression (see TRAINING.md's generation 3 -# section). A gradual ramp across several short curriculum stages, each -# resuming from the previous ramp value's checkpoint, lets the policy adopt -# each axis incrementally instead of all at once. -@export_range(0.0, 1.0) var vertical_ramp := 1.0 -@export_range(0.0, 1.0) var pitch_roll_ramp := 1.0 +# Height above which a touch counts toward air_touch_fraction telemetry +# (see get_info) — not a reward term itself, see set_action/get_info's +# comments on why generation 4 deliberately does not add a standalone +# air-touch reward. +const AIR_TOUCH_HEIGHT := 5.0 # Contact normals with y above this are floor contact (exempt from the wall # penalty); below it they read as wall (sideways) or ceiling (downward). +# Mirrors ShipObservations.FLOOR_NORMAL_MIN_Y (see that file's comment). const FLOOR_NORMAL_MIN_Y := 0.7 # Longest possible ship-to-ball separation: the enclosure's interior diagonal. @@ -125,8 +118,32 @@ var attack_goal_position: Vector3 # (true on goal, false on timeout) rather than toggle, so no reset is needed. var goal_scored_this_episode := false +# Set only in the timeout branch (TrainingMode._physics_process), never on a +# goal — a goal is a genuine terminal (V(s)=0 is correct there); a timeout +# is an artificial episode boundary the value function should be bootstrapped +# through instead (see get_info). Same "always overwritten by both call +# sites, never cleared in reset()" pattern as goal_scored_this_episode above, +# for the same reason. +var truncated_this_episode := false +var terminal_obs: Array = [] + var _ticks_since_ball_touch := 1 << 30 # large so the first touch always pays +# Flight telemetry (see get_info) — leading indicators for whether the +# policy is actually using its vertical/pitch-roll authority, visible from +# the first rollout instead of only in a win-rate number measured a full +# training run later. Accumulated per-tick, reset() zeroes them each episode; +# get_info() reports the running fraction/mean so the *final* tick of an +# episode (the one VecMonitor's info_keywords captures) holds the whole +# episode's aggregate. +const AIRBORNE_ALTITUDE_THRESHOLD := 3.0 +var _telemetry_ticks := 0 +var _airborne_ticks := 0 +var _altitude_sum := 0.0 +var _thrust_y_sum := 0.0 +var _touches := 0 +var _air_touches := 0 + # Wire up references after the ship is spawned. `attack_goal` is the goal # this ship scores into (goal.team == opponent's team). @@ -138,10 +155,8 @@ func setup(p_ship: Ship, p_rl_controller: RLShipController, p_ball: RigidBody3D, attack_goal_position = p_attack_goal_position init(ship) - # Contact monitoring for the ball-touch reward (training-only cost; - # the shipped game leaves contact_monitor off). - ship.contact_monitor = true - ship.max_contacts_reported = 8 + # ship.contact_monitor is on unconditionally (see ship.gd) since + # ShipObservations now reads it for every ship, not just training agents. ship.body_entered.connect(_on_ship_body_entered) @@ -153,36 +168,48 @@ func get_reward() -> float: return reward -# Symmetric across both self-play agents: reports whether this episode ended -# in a goal at all, not which team scored — a clean "goal rate" signal -# distinct from rollout/ep_rew_mean, which mixes this with dense shaping -# (ball chasing/touching). See train.py's GoalRateCallback. +# Symmetric across both self-play agents. "goal_scored": whether this +# episode ended in a goal at all (not which team) — a clean "goal rate" +# signal distinct from rollout/ep_rew_mean, which mixes this with dense +# shaping (ball chasing/touching); see train.py's GoalRateCallback. +# "truncated"/"terminal_obs": only present on a timeout tick — remapped by +# cosmic_env.py into SB3's expected "TimeLimit.truncated"/ +# "terminal_observation" keys so PPO bootstraps V(s) through episode +# timeouts instead of treating every 30s draw as a true terminal state (a +# real, previously-unnoticed bug independent of the action-space work — see +# TRAINING.md). Flight telemetry fields are leading indicators for +# generation 4's core hypothesis (see train.py's FlightTelemetryCallback). func get_info() -> Dictionary: - return {"goal_scored": goal_scored_this_episode} + var info := {"goal_scored": goal_scored_this_episode} + if truncated_this_episode: + info["truncated"] = true + info["terminal_obs"] = terminal_obs + if _telemetry_ticks > 0: + info["airborne_fraction"] = float(_airborne_ticks) / _telemetry_ticks + info["mean_altitude"] = _altitude_sum / _telemetry_ticks + info["vertical_thrust_mean"] = _thrust_y_sum / _telemetry_ticks + if _touches > 0: + info["air_touch_fraction"] = float(_air_touches) / _touches + return info func get_action_space() -> Dictionary: - return { - "thrust": {"size": 3, "action_type": "continuous"}, - "rotation": {"size": 3, "action_type": "continuous"}, - "turbo": {"size": 2, "action_type": "discrete"}, - } + return ShipActionCodec.action_space_dict() func set_action(action) -> void: - var thrust: Array = action["thrust"] - var rot: Array = action["rotation"] - var thrust_y: float = thrust[1] * vertical_ramp - var pitch: float = rot[0] * pitch_roll_ramp - var roll: float = rot[2] * pitch_roll_ramp - rl_controller.action.thrust = Vector3(thrust[0], thrust_y, thrust[2]) - rl_controller.action.rotation = Vector3(pitch, rot[1], roll) - rl_controller.action.turbo = int(action["turbo"]) == 1 + rl_controller.action = ShipActionCodec.from_indices(action) func reset(): super() _ticks_since_ball_touch = 1 << 30 + _telemetry_ticks = 0 + _airborne_ticks = 0 + _altitude_sum = 0.0 + _thrust_y_sum = 0.0 + _touches = 0 + _air_touches = 0 func _physics_process(delta): @@ -238,19 +265,17 @@ func _physics_process(delta): var height := maxf(ship.global_position.y, 0.0) reward -= airborne_penalty * height / ArenaBoundary.INNER_HEIGHT + # Flight telemetry accumulation (see get_info) — not reward, just + # observation of what the policy is actually doing this tick. + _telemetry_ticks += 1 + _altitude_sum += ship.global_position.y + if ship.global_position.y > AIRBORNE_ALTITUDE_THRESHOLD: + _airborne_ticks += 1 + _thrust_y_sum += rl_controller.action.thrust.y + func _wall_or_ceiling_contact() -> bool: - var state := PhysicsServer3D.body_get_direct_state(ship.get_rid()) - if state == null: - return false - for i in state.get_contact_count(): - if not state.get_contact_collider_object(i) is ArenaBoundary: - continue - # Normal points from the surface into the ship: floor ≈ +Y (exempt), - # anything flatter or downward is a wall or the ceiling. - if state.get_contact_local_normal(i).y < FLOOR_NORMAL_MIN_Y: - return true - return false + return ShipObservations.contact_normal(ship) != Vector3.ZERO func _on_ship_body_entered(body: Node) -> void: @@ -264,3 +289,12 @@ func _on_ship_body_entered(body: Node) -> void: alignment = clampf(ball.linear_velocity.normalized().dot(to_goal.normalized()), 0.0, 1.0) reward += ball_touch_reward * lerpf(ball_touch_direction_floor, 1.0, alignment) _ticks_since_ball_touch = 0 + + # Telemetry only (see get_info's air_touch_fraction) — not a reward term. + # Generation 4 deliberately doesn't reward high touches directly (see + # TRAINING.md's "why no air-touch reward" note); this just measures + # whether the air-drill state setter is producing genuine aerial + # contests, so a future decision to add one is data-driven. + _touches += 1 + if ball.global_position.y > AIR_TOUCH_HEIGHT: + _air_touches += 1 diff --git a/Game/scripts/ship_observations.gd b/Game/scripts/ship_observations.gd index 31bb9f52..97306352 100644 --- a/Game/scripts/ship_observations.gd +++ b/Game/scripts/ship_observations.gd @@ -19,8 +19,20 @@ const POSITION_SCALE := Vector3(20.0, 10.0, 20.0) const BALL_SPEED_SCALE := 30.0 const GOAL_DISTANCE_SCALE := 40.0 -# Number of floats build() returns; the policy input size. -const SIZE := 31 +# Contact normals with y above this are floor contact; below it they read as +# wall (sideways) or ceiling (downward) — mirrors +# ShipAIController.FLOOR_NORMAL_MIN_Y (kept here too since ShipAIController's +# wall_contact_penalty and this observation feature must agree on what +# counts as "in contact" for the reward/observation to stay consistent). +const FLOOR_NORMAL_MIN_Y := 0.7 + +# Number of floats build() returns; the policy input size. APPEND-ONLY: new +# features go on the end and existing indices never move, so an old exported +# model (whose network was trained against a shorter SIZE) still decodes its +# first N inputs identically when SIZE grows — see PolicyNetwork.forward's +# input_size slice/guard. Do not retune an *existing* index without +# retraining every model in Game/bots/. +const SIZE := 35 # 180° rotation about Y for team 1; identity for team 0. A proper rotation @@ -62,6 +74,17 @@ static func build(ship: Ship, opponent: Ship, ball: RigidBody3D, attack_goal_pos _append(obs, canon(goal_rel, team) / POSITION_SCALE) obs.append(goal_rel.length() / GOAL_DISTANCE_SCALE) + # Own contact state (appended — see SIZE's append-only invariant). + # Added for generation 4: ShipAIController's wall_contact_penalty used to + # fire on a condition the observation vector couldn't see coming, + # leaving the value function to predict a reward with no supporting + # signal. Also gives the policy a direct "am I resting on a surface" + # signal it can use to push off (a real aerial mechanic), distinct from + # inferring it indirectly from position/up-vector. + var normal := contact_normal(ship) + _append(obs, canon(normal, team)) + obs.append(1.0 if normal != Vector3.ZERO else 0.0) + return obs @@ -69,3 +92,22 @@ static func _append(obs: Array, v: Vector3) -> void: obs.append(v.x) obs.append(v.y) obs.append(v.z) + + +# Aggregate wall/ceiling contact normal (zero if none, or if only floor +# contact — floor contact is excluded so "in_contact" means "touching +# something other than the ground it's expected to rest on", matching +# ShipAIController.wall_contact_penalty's own floor exemption). Requires +# ship.contact_monitor (see ship.gd's _ready — on unconditionally for every +# ship so training and in-game inference see identical observations). +static func contact_normal(ship: Ship) -> Vector3: + var state := PhysicsServer3D.body_get_direct_state(ship.get_rid()) + if state == null: + return Vector3.ZERO + for i in state.get_contact_count(): + if not state.get_contact_collider_object(i) is ArenaBoundary: + continue + var normal := state.get_contact_local_normal(i) + if normal.y < FLOOR_NORMAL_MIN_Y: + return normal + return Vector3.ZERO diff --git a/Game/scripts/training_mode.gd b/Game/scripts/training_mode.gd index 50297ded..d2b202f1 100644 --- a/Game/scripts/training_mode.gd +++ b/Game/scripts/training_mode.gd @@ -54,6 +54,16 @@ extends GameMode # trainee's near-goal resets are always finishing chances, not a coin flip # between attacking and defending an empty net. @export_range(0.0, 1.0) var attack_goal_bias := 0.5 +# Fourth episode-start branch (after kickoff/near-goal, before the fully- +# random fallback): ball spawned high, both ships spawned low and lateral — +# unsolvable without climbing. Default 0 (off) so ordinary runs are +# unaffected. Added for curriculum generation 4: the existing random branch +# already samples ship/ball Y across the full arena height, but that only +# randomizes the *initial* state — under gravity+drag a floor-pinned policy +# sinks back to the floor in ~1.5s, so the *stationary* state distribution +# stayed floor-pinned even though the initial one wasn't. See +# _place_air_drill. +@export_range(0.0, 1.0) var air_drill_chance := 0.0 # Placement bounds for randomized episode starts, derived from the standard # enclosure (ArenaBoundary). The inset keeps a randomly oriented ship (1x1x4 @@ -188,7 +198,7 @@ func _parse_eval_args() -> void: # of silently matching an unrelated inherited export. const TRAINING_MODE_OVERRIDES := [ "goal_reward", "draw_penalty", "kickoff_state_chance", - "ball_near_goal_chance", "attack_goal_bias", + "ball_near_goal_chance", "attack_goal_bias", "air_drill_chance", ] # ShipAIController @export names a curriculum run may override, read as # --ai_= to avoid colliding with the names above. @@ -196,7 +206,7 @@ const SHIP_AI_OVERRIDES := [ "ball_touch_reward", "ball_touch_cooldown_ticks", "ball_touch_direction_floor", "velocity_to_ball_weight", "ball_velocity_to_goal_weight", "ball_distance_penalty", "wall_contact_penalty", "tilt_penalty", "speed_reward_weight", "time_penalty", - "airborne_penalty", "vertical_ramp", "pitch_roll_ramp", + "airborne_penalty", ] @@ -243,11 +253,10 @@ func _ai_default(name: String) -> Variant: "ball_velocity_to_goal_weight": return 0.004 "ball_distance_penalty": return 0.002 "wall_contact_penalty": return 0.0025 - "tilt_penalty": return 0.002 + "tilt_penalty": return 0.0005 "speed_reward_weight": return 0.004 "time_penalty": return 0.001 "airborne_penalty": return 0.0 - "vertical_ramp", "pitch_roll_ramp": return 1.0 _: return null @@ -299,6 +308,14 @@ func _physics_process(_delta): agent.reward -= draw_penalty agent.done = true agent.goal_scored_this_episode = false + # Snapshot BEFORE _reset_episode() below, which moves the ship/ + # ball and would otherwise make this the post-reset state, not + # the terminal one PPO needs to bootstrap V(s) from (see + # ShipAIController.get_info / cosmic_env.py's truncation remap). + # A goal (_on_goal_scored) does NOT do this — a goal is a + # genuine terminal, V(s)=0 is correct there. + agent.truncated_this_episode = true + agent.terminal_obs = ShipObservations.build(agent.ship, agent.opponent, agent.ball, agent.attack_goal_position) _reset_episode() return @@ -330,6 +347,7 @@ func _on_goal_scored(conceding_team: int) -> void: agent.reward += goal_reward if agent.ship.team != conceding_team else -goal_reward agent.done = true agent.goal_scored_this_episode = true + agent.truncated_this_episode = false # genuine terminal, not a timeout _reset_episode() @@ -363,6 +381,8 @@ func _reset_episode() -> void: elif roll < kickoff_state_chance + ball_near_goal_chance: _place_ships_random() _place_ball_near_goal() + elif roll < kickoff_state_chance + ball_near_goal_chance + air_drill_chance: + _place_air_drill() else: _place_ships_random() _place_ball_random() @@ -373,6 +393,46 @@ func _place_ball_random() -> void: _place_body(ball, Transform3D(Basis.IDENTITY, _random_position()), velocity, Vector3.ZERO) +# Extra clearance for the air drill's ball placement specifically — well +# beyond SPAWN_INSET, and well beyond the ball's own radius. The ball (unlike +# _random_position) has no collision-avoidance resample, so this is the +# anti-exploit measure: the RLGym wall-bounce exploit ("hits the ball off a +# wall high up instead of doing a real aerial") needs a wall to bounce off, +# so simply not generating ball states anywhere near one removes the exploit +# from the training distribution entirely, rather than trying to price it +# out via reward shaping. +const AIR_DRILL_BALL_WALL_CLEARANCE := 5.0 + +# Air drill state (see air_drill_chance): ball spawned high, both ships +# spawned low and lateral, so the state is unsolvable without climbing. +func _place_air_drill() -> void: + var ball_half_x := ArenaBoundary.INNER_HALF_X - AIR_DRILL_BALL_WALL_CLEARANCE + var ball_half_z := ArenaBoundary.GOAL_LINE_Z - AIR_DRILL_BALL_WALL_CLEARANCE + var ball_position := Vector3( + randf_range(-ball_half_x, ball_half_x), + randf_range(ArenaBoundary.INNER_HEIGHT * 0.45, FIELD_MAX_Y), + randf_range(-ball_half_z, ball_half_z) + ) + var ball_velocity := _random_direction() * randf_range(0.0, MAX_RANDOM_BALL_SPEED * 0.5) + _place_body(ball, Transform3D(Basis.IDENTITY, ball_position), ball_velocity, Vector3.ZERO) + + for ship in ships: + if ship in _inert_ships: + continue + var lateral_offset := Vector3(randf_range(-1, 1), 0.0, randf_range(-1, 1)) + lateral_offset = lateral_offset.normalized() if lateral_offset.length_squared() > 0.001 else Vector3.FORWARD + lateral_offset *= randf_range(6.0, 14.0) + var ship_position := Vector3( + clampf(ball_position.x + lateral_offset.x, -FIELD_HALF_X, FIELD_HALF_X), + randf_range(FIELD_MIN_Y, 4.0), + clampf(ball_position.z + lateral_offset.z, -FIELD_HALF_Z, FIELD_HALF_Z) + ) + var orientation := Basis.from_euler(Vector3( + randf_range(-0.4, 0.4), randf_range(-PI, PI), randf_range(-0.4, 0.4) + )) + _place_body(ship, Transform3D(orientation, ship_position), Vector3.ZERO, Vector3.ZERO) + + # Attacking/defending drill states: ball close to a goal, moving toward it. # Which goal is picked is biased by attack_goal_bias (0.5 = uniform between # both, matching historical behaviour; 1.0 = always the goal team 0 attacks). diff --git a/TRAINING.md b/TRAINING.md index 50502567..c6782c47 100644 --- a/TRAINING.md +++ b/TRAINING.md @@ -161,12 +161,20 @@ the automated curriculum pipeline (`run_training.sh`) only ever writes new flat files there, never touching subdirectories. `Game/bots/promoted/.json` is the small, curated, hand-maintained set -actually referenced by the shipped game — currently just `easy.json` -(promoted 2026-07-24 from `curric-s6-unmask`, the strongest checkpoint at the -time). `match.tscn`/`spectate.tscn` point their `bot_model_path` exports here -directly, so a promoted file is never touched by training scripts, never -overwritten by a same-named future export, and never disturbed by pruning old -experiment files from the flat dump. +actually referenced by the shipped game — currently `easy.json` (promoted +2026-07-24 from `curric-s6-unmask`, the strongest checkpoint at the +time — note `curric-s6-unmask` was itself generation 1's *failed* unmask +stage, so `easy.json` is weaker than `reference-grounded.json` below; a +strong generation 4 result should promote a real replacement, plus +`medium.json`/`hard.json`) and `reference-grounded.json` (added for +generation 4 — a copy of generation 3's `curric-s5-aggression`, made before +the flat `Game/bots/` dump was scrapped for the redesign, kept as the +strongest grounded-era artifact and the fixed yardstick generations 1-3 were +all measured against; see "Generation 4"'s final report). `match.tscn`/ +`spectate.tscn` point their `bot_model_path` exports here directly, so a +promoted file is never touched by training scripts, never overwritten by a +same-named future export, and never disturbed by pruning old experiment +files from the flat dump. To promote a new bot into a tier: copy the chosen `Game/bots/.json` to `Game/bots/promoted/.json` (overwriting the old one), and note the @@ -185,13 +193,16 @@ movement. Each stage is a normal chained run — a new `--experiment` resumed via `--resume checkpoints//final.zip`, same as any other run — just with different curriculum flags. -`curriculum.py` has run through three generations so far. Generation 1 +`curriculum.py` has run through four generations so far. Generation 1 (below) ran stages 1-6 to completion/block and is archived; generation 2 started a fresh stage 1 seeded from generation 1's last clean pass instead of continuing to retry a stage that kept getting worse, but also failed 3 -attempts; generation 3 (the one `curriculum.py` actually runs today) -replaces generation 2's single all-or-nothing unmask stage with a gradual -ramp — see "Generation 3" below. +attempts; generation 3 replaced generation 2's single all-or-nothing unmask +stage with a gradual ramp, and also failed (worse, on its final attempt, +than either prior generation); generation 4 (the one `curriculum.py` actually +runs today) is a full redesign, not a further patch — see "Generation 4" +below, and "Generation 3" for why a fourth attempt at gating *when* the +policy could use full 3D controls was abandoned rather than retried again. ### Generation 1 (archived — see `curriculum_state_gen1.json`) @@ -319,28 +330,174 @@ stages, so the ramp is the sole studied variable. `curriculum_state_gen2.json`) rather than continuing to log against a stage list whose stage 0 no longer means what it used to. -**Open question, not yet resolved by data:** the ramp scales the action's -effect in Godot, which runs *after* PPO samples the action — PPO's own -std-collapse dynamics don't directly see the ramp, only the reward it -produces. It's possible this doesn't prevent the collapse, or even makes it -happen faster at low ramp values (weaker reward signal on those axes gives -less incentive to keep exploring them). Watch `train/std` per stage in -TensorBoard rather than assuming the ramp is working. If the final gated -stage still lands ~28-32%, that's evidence the plateau isn't an -exploration/collapse problem at all — worth revisiting reward shaping, or -trying `--opponent-mode frozen --opponent-model ` during the warmup -stages (implemented, never yet exercised in this project) to remove -self-play's moving-target instability while the policy first learns to use -the new axes. +**Open question, resolved 2026-08-04.** The gated `unmask` stage failed all +3 attempts: 29% → 30% → **24%** win rate vs `curric-s5-aggression` (the +third, worst by then) — landing in the same ~28-32% band the section above +flagged as "evidence the plateau isn't an exploration/collapse problem at +all." `train/std` collapsed from ~0.30 to ~0.13-0.15 within the first ~10% +of steps in every attempt of every generation regardless of hard-mask vs. +gradual-ramp mechanism, so gating *when* the axes were allowed to act never +addressed the actual cause. See "Generation 4" below for the redesign and +root-cause diagnosis this prompted, and `curriculum_state_gen3.json` for the +archived full log. + +### Generation 4 (current) — action space redesign, not a further ramp patch + +Three generations spent ~2 weeks trying different ways to gate *when* the +policy could use vertical thrust/pitch/roll on top of a continuous Gaussian +action space, and all three converged on the same failure: PPO's action +std collapsing within the first ~10% of steps and never recovering, +regardless of mechanism. Research into how self-play PPO bots that have +actually solved this class of problem (RLGym/RLBot's Necto/Nexto) approach +it turned up a structural difference — they don't gate control authority at +all; they train the full action space from step 1 using discrete/bucketed +actions, not a continuous Gaussian, plus reward/state-setter curriculum +instead of action masking. + +**Root cause, verified against this project's own physics** (not assumed): +flight in this game is a *sustained set-point*, not an impulse. Ship mass +5.0, `vertical_thrust` 120 (`ship.gd`/`ship.tscn`), default gravity 9.8 m/s² +→ hovering requires *holding* `thrust.y ≈ 0.408` continuously. A Gaussian +whose mean sits near 0 and whose σ has collapsed to ~0.13 samples +`thrust.y ∈ [-0.4, 0.4]` — it can brush the hover value but can never *hold* +it long enough to earn the reward gradient that would move the mean. That's +a fixed point; no ramp on the axis's downstream *effect* (which is applied +*after* PPO samples the action) moves it, exactly as the "open question" +above speculated might be the case. Independently, this redesign also found +and fixed a real, previously-unnoticed bug unrelated to the action space: +`godot_rl`'s `godot_env.py` never marks an episode timeout as a truncation +(it returns the same `done` array for both term and trunc — see its own +`# TODO update API to term, trunc`), so PPO was bootstrapping `V(s_T)=0` on +every 30s draw in every generation to date instead of correctly estimating +the value of the state it timed out in. + +**Action space**: switched to per-axis `MultiDiscrete` (7 heads, `nvec = +[5,5,5,5,5,5,2]`) instead of continuous `Box(7)` — see +`Game/scripts/ship_action_codec.gd`, the single source of truth for the +layout/decode shared by training and in-game inference. Not a single +lookup table (RLGym's approach for Rocket League's *coupled* car controls): +Cosmic Clash's 7 axes are near-independent thruster/torque channels, so a +curated combination table would throw away that factorization for no +benefit. `thrust_y`'s bins are deliberately asymmetric +(`-0.5, 0, 0.45, 0.75, 1.0`, vs. the symmetric `-1, -0.5, 0, 0.5, 1` on +every other axis) — a uniform-random policy over those 5 bins averages +0.34, just below the 0.408 hover point, so a fresh policy drifts gently +through the volume instead of pinning to the floor (symmetric bins) or +sticking to the ceiling (`ceiling_pull_strength` 11.5 > gravity 9.8). This +is the direct analogue of the RLGym/RLBot fix for the same failure mode +("add more jump actions to the discrete action parser"). `godot_rl`'s +`ActionSpaceProcessor` already emits `MultiDiscrete` with zero Python-side +changes when every action entry is `Discrete` — the only reason this +project's action space flattened to `Box(7)` before was that `turbo` +(binary) was mixed with continuous entries. + +**Backward compatibility**: every export before generation 4 (e.g. +`Game/bots/promoted/easy.json`) has no `"action_space"` field in its JSON; +absence means `{"type": "continuous"}` and decodes through the exact same +path as before (`ShipActionCodec.from_continuous`, moved verbatim out of +`ai_ship_controller.gd`). `PolicyNetwork.gd`'s forward pass itself never +changed — only the caller's decode branches on the model's declared type. +`export_policy.py`'s parity check is now index-level for a `MultiDiscrete` +model (argmax per head's logit slice, compared against SB3's own +`deterministic=True` chosen index) rather than comparing clipped floats, +since a head-order mistake would otherwise train and export cleanly and +only surface as silently wrong in-game behaviour. + +**No grounded stage.** Full action space live from step 1 — no successful +self-play RL bot in this problem class gates control authority, it's failed +9/9 attempts (3 generations × 3 attempts) here, and every prior generation's +checkpoints are a different, incompatible action/observation shape anyway +(nothing to resume from). 3 stages instead of a ramp: + +| Stage | Opponent | Timesteps | Gated | What it teaches | +|---|---|---|---|---| +| 1 — `bootstrap` | `inert` | 40M (~4h) | No | Empty-net finishing from a random policy — no moving target, full action space from the start. | +| 2 — `selfplay` | `self_play` | 160M (~16h) | Yes, vs stage 1 | Where essentially all the learning happens. | +| 3 — `gauntlet` | `frozen` = stage 2's own export | 120M (~12h) | Yes, vs stage 2 | A stationary opponent for a low-variance measurement, and a check that self-play didn't converge to a fixed point that only beats itself. | + +An "air drill" state-setter branch (`training_mode.gd`'s `air_drill_chance`, +new — ball spawned high, both ships spawned low and lateral, unsolvable +without climbing, kept clear of every wall so the RLGym-warned wall-bounce +exploit has no wall nearby to bounce off) runs at a constant rate across +*all* stages rather than being introduced late — gating *when* a skill gets +drilled would reproduce the exact "gate what the policy can do" pattern +that failed 3 generations running. + +**Observations**: `ShipObservations.SIZE` grew 31 → 35 (own contact normal ++ an `in_contact` flag, appended — never inserted, see that file's +append-only invariant) so the value function can actually see the condition +`wall_contact_penalty` fires on, instead of predicting a reward with no +supporting signal. + +**Reward shaping**: mostly unchanged — a farmability check on the existing +weights (`velocity_to_ball_weight`'s term telescopes to ~2.7 over a 20m +approach, well under `goal_reward`=80; not gameable) argues generation 2/3's +tuning was never the actual problem. Two changes: `airborne_penalty` is no +longer passed by any stage (previously ramped *up* in lockstep with the +axis generation 3 was trying to teach — directly adversarial to the goal of +genuine aerial play), and `tilt_penalty` dropped 4x (0.002 → 0.0005 default) +since an aerial approach to a high ball requires pitching. Deliberately +*not* added: a standalone air-touch reward — that's the exact exploit RLGym +warns about ("hits the ball off a wall high up instead of doing a real +aerial"); the air-drill state setter already makes aerial skill +instrumentally necessary to earn the existing ball-directed rewards. + +**Exploration**: `--reset-std` (meaningless under `MultiDiscrete` — no +`log_std`) is replaced by `--reset-logits ` (multiplies +`action_net`'s weights/bias, optionally scoped to specific heads via +`--reset-logits-heads`) for a deliberate post-diagnosis recovery, and more +importantly by `--entropy-floor` (`train.py`'s `EntropyFloorCallback`): a +*persistent* per-rollout controller nudging `ent_coef` to hold policy +entropy near a target that decays over the run, replacing the one-shot +`--reset-std` shock that reliably decayed away within ~10% of steps in +every prior generation with something that responds continuously instead of +once. `--ent-coef`'s default rose 0.0001 → 0.01 (tuned for `MultiDiscrete`'s +bounded ~10-nat entropy, not a Gaussian's unbounded differential entropy). +Per-head entropy (`train/entropy_head_`) replaces the old aggregate +`train/std` scalar — it identifies *which* axis is collapsing instead of +one number for all seven. + +**Validation before spending the full ~32h budget**: see the ladder below — +cheapest checks first (an offline action-space assertion, a headless Godot +boot, a 100k-step smoke run, export parity + an in-game round trip against +`easy.json`), then flight telemetry (`rollout/airborne_fraction`, +`mean_altitude`, `air_touch_fraction`, `vertical_thrust_mean` — leading +indicators visible from the first rollout instead of only in a win rate +measured a full run later), then a short controlled A/B (MultiDiscrete vs. +continuous, otherwise identical, ~20M steps each) before committing to the +full curriculum — every past generation bet a full day on an unfalsifiable +hypothesis, which is what made each failure expensive to diagnose. + +1. `training/test_action_space.py` — offline, seconds. Catches a head-order + mismatch, the single most likely silent killer (trains "fine" for 24h, + produces garbage — e.g. pitch commands driving strafe thrusters — with no + error). +2. `godot --headless --path Game res://scenes/training.tscn` with no + trainer listening, 30s — catches `class_name`/observation-size + regressions. +3. `.venv/bin/python train.py --experiment smoke --timesteps 100000 + --n-parallel 2` — confirms the `MultiDiscrete` handshake and new + callback metrics emit. +4. `export_policy.py` on the smoke checkpoint (mandatory index-level parity + check), then `evaluate.py .json ../Game/bots/promoted/easy.json + --episodes 4` — exercises the real GDScript decode path. +5. A short A/B: two 20M-step runs, identical except action space + (`MultiDiscrete` vs. the old continuous `Box(7)`), comparing + `rollout/airborne_fraction`. If discrete pulls meaningfully ahead, the + 32h curriculum is a justified bet; if both stay near zero, the + hypothesis above is wrong and reward/compute explanations move to the + front — cheaper than a 4th blind multi-day generation either way. All curriculum flags default to leaving Godot's own `@export` defaults alone (`train.py` only forwards a flag when you pass it), so ordinary runs are unaffected. Full flag list: `--opponent-mode {self_play,inert,frozen}`, `--opponent-model ` (for `frozen`), `--draw-penalty`, `--attack-goal-bias`, `--kickoff-chance`, `--near-goal-chance`, -`--vertical-ramp`, `--pitch-roll-ramp` (0.0-1.0 locomotion-unmask ramp), +`--air-drill-chance` (generation 4's state-setter aerial curriculum), `--velocity-to-ball-weight`, `--ball-distance-penalty`, `--ball-touch-reward`, -`--airborne-penalty`, `--ball-velocity-to-goal-weight`, `--goal-reward`. +`--airborne-penalty`, `--tilt-penalty`, `--ball-velocity-to-goal-weight`, +`--goal-reward`. (`--vertical-ramp`/`--pitch-roll-ramp` are gone — generation +4 has no locomotion mask/ramp to control.) ### Running it automatically @@ -348,17 +505,19 @@ are unaffected. Full flag list: `--opponent-mode {self_play,inert,frozen}`, pattern as `start_training.sh`) drives all stages end to end: for each stage it runs `run_training.sh` (pull, train, export, commit+push) with that stage's flags, then evaluates the resulting checkpoint against a reference -bot over 100 episodes — the fixed `rookie.json` baseline for a from-scratch -stage 1 (no `resume_from_experiment`/`reference_experiment` override on -`STAGES[0]`), the previous stage's promoted checkpoint by default for -stages 2+, or an explicit override in that stage's dict when it deliberately -skips a since-regressed branch (generation 1's stage 5) or seeds from a -fixed foundation checkpoint (generation 2's stage 1 — see above). +bot over 100 episodes — the previous stage's own passing export (stage 1 is +ungated, so this only applies to stages 2+). Once every stage passes, a +final (non-gating) report evaluates the result against both +`Game/bots/promoted/easy.json` (the shipped bot) and +`Game/bots/promoted/reference-grounded.json` (a copy of generation 3's +`curric-s5-aggression`, the strongest grounded-era artifact and the +yardstick generations 1-3 were all measured against) — those two numbers are +what actually answer "did generation 4 work?" ```bash cd training -./curriculum.sh # start/resume the curriculum -./curriculum.sh --seed-checkpoint checkpoints/run11/final.zip # override stage 1's resume source for this run +./curriculum.sh # start/resume the curriculum +./curriculum.sh --seed-checkpoint checkpoints/some/final.zip # override stage 1's resume source for this run ``` The gate is deliberately lenient: it blocks a stage only on a **clear @@ -374,13 +533,13 @@ result are logged to `curriculum_state.json` (committed alongside A stage gets up to 2 retries (3 attempts total) before the script stops and asks for a human look — it will not retry indefinitely or advance past a stage that keeps failing on its own. By default a retry resumes from that -stage's own previous attempt with a fresh `--reset-std`; a stage can instead -set `reset_retry_checkpoint: True` (generation 2's stage 1 does) to always -reset to its normal resume source instead — see the generation 1 → 2 -postmortem above for why blind same-checkpoint retries can make things -monotonically worse. Once you've looked at why a block happened (more -timesteps? a flag needs adjusting? the eval itself was misleading?), re-run -with `--force-retry` to try again or `--skip-to-next-stage` if you judge the +stage's own previous attempt (no `reset_retry_checkpoint` stage override is +set in generation 4 — nothing yet suggests a retry needs to reset to a +clean upstream checkpoint the way generation 3's single `unmask` stage did; +add one if a stage's retries turn out to be drifting rather than +converging). Once you've looked at why a block happened (more timesteps? a +flag needs adjusting? the eval itself was misleading?), re-run with +`--force-retry` to try again or `--skip-to-next-stage` if you judge the result good enough despite the gate. Running a stage by hand (e.g. to experiment with flags before trusting the diff --git a/training/cosmic_env.py b/training/cosmic_env.py index 43c41d5f..1889a3c7 100644 --- a/training/cosmic_env.py +++ b/training/cosmic_env.py @@ -10,6 +10,7 @@ learning policy: self-play by construction. import pathlib import subprocess +import numpy as np from godot_rl.core.godot_env import GodotEnv from godot_rl.wrappers.stable_baselines_wrapper import StableBaselinesGodotEnv @@ -72,8 +73,13 @@ class CosmicClashEnv(GodotEnv): class CosmicClashVecEnv(StableBaselinesGodotEnv): """SB3 VecEnv over N parallel CosmicClashEnv instances. - convert_action_space=True flattens the env's (Box(6), Discrete(2)) action - space into a single Box(7): thrust xyz, rotation xyz, turbo (>0 means on). + convert_action_space=True: godot_rl's ActionSpaceProcessor reports a + gym.spaces.MultiDiscrete when every per-axis action entry is Discrete + (see ShipActionCodec/ShipAIController.get_action_space) — nvec + [5,5,5,5,5,5,2] for rotation xyz, thrust xyz, turbo, in that + gymnasium-sorted key order. No conversion logic here needs to change for + that; this class's only functional addition is the truncation-info + remap below. """ def __init__(self, godot_bin: str, n_parallel: int = 1, seed: int = 0, port: int = GodotEnv.DEFAULT_PORT, **kwargs): @@ -90,3 +96,27 @@ class CosmicClashVecEnv(StableBaselinesGodotEnv): self.n_parallel = n_parallel self._check_valid_action_space() self.results = None + + def step(self, action): + """Remap ShipAIController.get_info()'s "truncated"/"terminal_obs" + into the keys SB3's on_policy_algorithm looks for + ("TimeLimit.truncated"/"terminal_observation") so PPO bootstraps + V(s) through an episode timeout instead of treating every 30s draw + as a true terminal state. + + Godot_rl's own godot_env.py never sets either key (it returns the + same `done` array for both term and trunc, "# TODO update API to + term, trunc") and StableBaselinesGodotEnv.step() only ever returns + that single collapsed `dones` array to SB3 — so without this, PPO + has no way to distinguish "episode ended because a goal was scored" + (a genuine terminal, V(s)=0 is correct) from "episode ended because + the 30s clock ran out" (an artificial boundary that should be + bootstrapped through), and was silently treating every draw as the + former in every curriculum generation to date. + """ + obs, rewards, dones, infos = super().step(action) + for info in infos: + if info.pop("truncated", False): + info["TimeLimit.truncated"] = True + info["terminal_observation"] = {"obs": np.array(info.pop("terminal_obs"), dtype=np.float32)} + return obs, rewards, dones, infos diff --git a/training/curriculum.py b/training/curriculum.py index 348d9380..fdb3e2b2 100644 --- a/training/curriculum.py +++ b/training/curriculum.py @@ -18,32 +18,49 @@ forever for the wrong reason. When a stage does fail MAX_RETRIES times in a row, the script stops and asks for a human look rather than retrying indefinitely or silently advancing past a bad stage. -This is generation 3 of the curriculum. Generation 1 (6 stages: score, -defend, no_draws, mechanics, aggression, unmask) ran 2026-07-21 through -2026-07-26 and is archived in curriculum_state_gen1.json — its final stage -("unmask", full 3D flight on top of the aggression retune) failed 3 straight -attempts, monotonically worsening (25% -> 20% -> 15% win rate vs -curric-s5-aggression) because every retry resumed the same drifting -checkpoint under identical flags instead of actually changing anything. -Generation 2 (archived in curriculum_state_gen2.json) started a fresh -single "unmask" stage seeded directly from curric-s5-aggression's own -checkpoint (FOUNDATION_EXPERIMENT below) with retuned reward weights — it -also failed 3 attempts, landing at a stable 32% / 28% / 31% win rate each -time, ruling out both "retune the reward weights" and "just give it more -time" as fixes. Generation 3 replaces the single all-or-nothing unmask -flip with a gradual ramp (4 stages: unmask-ramp25/50/75, then unmask at -full authority) — see TRAINING.md's "Generation 3" section for the full -postmortem and design. +This is generation 4 of the curriculum — a full redesign, not a patch. +Generations 1-3 (archived in curriculum_state_gen1.json/_gen2.json/_gen3.json) +all tried teaching full 3D flight by training grounded first and then +opening up vertical/pitch-roll authority (a hard 0/1 mask in gen 1/2, a +gradual float ramp in gen 3) on top of a continuous Gaussian action space. +All three failed: gen 1's hard mask went 25% -> 20% -> 15% win rate across 3 +attempts; gen 2's single-flip retune landed at a stable 32%/28%/31%; gen 3's +gradual ramp landed at 29%/30%/24% — actually the worst of the three by its +final attempt. Every attempt showed the same signature regardless of +mechanism: PPO's Gaussian action-distribution std collapsed from ~0.30 to +~0.13-0.15 within the first ~10% of steps and never recovered. The root +cause: hovering this ship (mass 5.0, vertical_thrust 120, default gravity +9.8 — see ship.gd) requires *holding* thrust.y ~= 0.408 continuously; a +collapsed near-zero-mean Gaussian can brush that value but never sustain it +long enough to earn the reward gradient that would move the mean. No amount +of gating *when* the axis is allowed to act fixes a problem in *how* the +policy represents a decision on it. + +Generation 4 (see TRAINING.md and Game/scripts/ship_action_codec.gd) +replaces the action space itself with per-axis MultiDiscrete bins instead of +a continuous Gaussian, trains the full action space from step 1 with no +grounded stage at all (no successful self-play RL bot in this problem class +gates control authority — see the RLGym/RLBot research cited in +TRAINING.md), and adds a state-setter "air drill" episode-start branch +(training_mode.gd's air_drill_chance) to force aerial practice instead of +relying on reward-driven exploration alone. All of generation 1-3's +checkpoint-lineage machinery (FOUNDATION_EXPERIMENT, locomotion-groundedness +tracking, resume/reference overrides for skipping a regressed branch) is +gone because there is nothing to resume from: every prior checkpoint is a +different, incompatible action/observation shape. The two strongest prior +artifacts are kept as fixed evaluation references instead (see +PROMOTED_EASY/PROMOTED_REFERENCE_GROUNDED below) — they remain playable +opponents forever via PolicyNetwork's format-versioned JSON even though +their own checkpoints and generation are gone. Every experiment name this script generates is timestamped (YYYYMMDD-HHMM-, applied once in run_stage_attempt) so runs stay unique across restarts/generations and sort chronologically in TensorBoard -and checkpoints/ — plain names like "curric-s1-score" from generation 1 -would otherwise collide with generation 2's own stage 1. +and checkpoints/. Usage: .venv/bin/python curriculum.py # run/resume the curriculum - .venv/bin/python curriculum.py --seed-checkpoint checkpoints/run11/final.zip + .venv/bin/python curriculum.py --seed-checkpoint checkpoints/some/final.zip .venv/bin/python curriculum.py --force-retry # after fixing something, retry the blocked stage .venv/bin/python curriculum.py --skip-to-next-stage # human judgment call: good enough, move on anyway @@ -61,20 +78,14 @@ from datetime import datetime TRAINING_DIR = pathlib.Path(__file__).resolve().parent STATE_PATH = TRAINING_DIR / "curriculum_state.json" EVAL_HISTORY_PATH = TRAINING_DIR / "eval_history.json" -ROOKIE_REFERENCE = TRAINING_DIR.parent / "Game" / "bots" / "rookie.json" -# Generation 1's last cleanly-passing checkpoint (see curriculum_state_gen1.json) -# — generations 2 and 3 both build on this directly instead of re-running -# stages 1-5. -FOUNDATION_EXPERIMENT = "curric-s5-aggression" - -# Groundedness (locomotion-mask state) for experiments that predate this -# generation's own log, so _grounded_for_experiment can still answer for -# them — see that function. -LEGACY_GROUNDED = { - "rookie": False, - FOUNDATION_EXPERIMENT: True, -} +# Fixed evaluation references — never touched by training scripts (see +# TRAINING.md's "Promoted bots" section) — kept forever as playable +# opponents via PolicyNetwork's format-versioned JSON even after their own +# checkpoints/generation are gone. The final report (not a gate) evaluates +# generation 4's result against both. +PROMOTED_EASY = TRAINING_DIR.parent / "Game" / "bots" / "promoted" / "easy.json" +PROMOTED_REFERENCE_GROUNDED = TRAINING_DIR.parent / "Game" / "bots" / "promoted" / "reference-grounded.json" MAX_RETRIES = 2 EVAL_EPISODES = 100 @@ -84,118 +95,108 @@ EVAL_EPISODES = 100 # module docstring) — advance rather than retry. REGRESSION_MARGIN = 0.15 -# Standing flags applied to every attempt, mirroring next_run.sh: reset-std -# reopens exploration every attempt (harmless on fresh starts — train.py -# only applies it on --resume), ent-coef keeps it from re-collapsing. -STANDING_ARGS = ["--reset-std", "0.3", "--ent-coef", "0.001"] +# Standing flags applied to every attempt. Generation 3's "--reset-std 0.3" +# (a one-shot shock, and meaningless anyway under MultiDiscrete — there is +# no log_std) is gone; EntropyFloorCallback (see train.py) is a continuous +# controller instead, which every generation's TensorBoard data argues is +# what was actually needed (a single reset at attempt start reliably decayed +# away within ~10% of steps, every time). --ent-coef raised an order of +# magnitude from generation 3's 0.001: that value was tuned for a Gaussian's +# unbounded differential entropy, not MultiDiscrete's bounded (~10-nat) +# entropy. +STANDING_ARGS = ["--ent-coef", "0.01", "--entropy-floor"] -# Ball-chasing/scoring reward flags shared by every unmask-ramp stage -# (generation 3 — see below): identical across all 4 stages so the ramp -# itself is the only studied variable. Lifted from generation 2's single -# "unmask" attempt (raised from stage 5's 0.05/0.006/0.5/0.02/60 defaults). -_UNMASK_RAMP_SHARED_FLAGS = [ - "--opponent-mode", "self_play", +# Reward-shaping flags shared by every stage so the studied variables (state +# mix, opponent mode) stay isolated — carried forward unchanged from +# generation 2/3, which the reward-farmability analysis in TRAINING.md +# confirmed were never the actual problem. draw_penalty and airborne_penalty +# are deliberately NOT overridden here (both default to 0.0 in +# training_mode.gd/ship_ai_controller.gd): generation 3's draw_penalty=5 and +# airborne_penalty ramping up in lockstep with the unmask ramp were both +# grounded-era, anti-flight pressures that have no place in a curriculum +# whose entire point is teaching flight. +_SHARED_REWARD_FLAGS = [ "--velocity-to-ball-weight", "0.08", "--ball-distance-penalty", "0.01", "--ball-touch-reward", "0.7", "--ball-velocity-to-goal-weight", "0.06", "--goal-reward", "80", - "--draw-penalty", "5", ] +# A stage dict may additionally set "abort_if": {"metric": "rollout/airborne_ +# fraction", "below": 0.05, "at_steps": N} to end that attempt early if a +# flight-telemetry metric (see train.py's FlightTelemetryCallback) hasn't +# cleared a bar by N *absolute* PPO timesteps (model.num_timesteps keeps +# accumulating across --resume, so N must account for whatever this stage +# inherits from its predecessor, not just this stage's own budget). +# Deliberately unset on every stage below for now — rung 5 of TRAINING.md's +# validation ladder (a short controlled A/B) should establish what a +# sensible threshold actually looks like before any stage bets a real 12h+ +# budget on a guessed one. STAGES = [ { - "name": "unmask-ramp25", - # Generation 3, step 1/4 of a gradual locomotion-unmask ramp — see - # TRAINING.md's "Generation 3" section for the full postmortem. - # Generation 2's single all-or-nothing "unmask" stage (flip - # vertical_ramp/pitch_roll_ramp 0 -> 1 in one step) failed 3 - # independent 240M-step attempts in a row, landing at a stable - # 32% / 28% / 31% win rate vs curric-s5-aggression each time — not - # noise (attempts 2-3 each gave the *same* checkpoint lineage - # another full 240M steps with zero improvement) and not fixable by - # more time. Every attempt shows train/std collapsing from ~0.30 to - # ~0.13-0.15 within the first ~10% of steps and never recovering — - # the policy locks the newly-opened axes back down before ever - # meaningfully exploring them. - # - # This stage instead scales vertical_ramp/pitch_roll_ramp to 25% - # authority. Ungated (see "gated" below and main()'s loop): this is - # a waypoint, not a measured transition — no eval runs, no - # regression gate applies, it always advances after training. - # airborne_penalty is off (0.0) here: at 25% authority the axis - # barely does anything yet, so there's nothing to discourage. + "name": "bootstrap", + # Stage 1/3: empty-net finishing practice from a random policy — no + # live opponent, so the full action space's first behaviour to + # emerge is "fly to ball, push it toward the net" without a moving + # target complicating credit assignment. Generation 1's own stage 1 + # (also inert-opponent, also empty-net) was the one stage across all + # 3 prior generations that unambiguously passed on its first + # attempt — reusing that shape here, just with the full action space + # live instead of yaw-only. "flags": [ - *_UNMASK_RAMP_SHARED_FLAGS, - "--vertical-ramp", "0.25", - "--pitch-roll-ramp", "0.25", - "--airborne-penalty", "0.0", + "--opponent-mode", "inert", + "--attack-goal-bias", "1.0", + "--kickoff-chance", "0.10", + "--near-goal-chance", "0.50", + "--air-drill-chance", "0.20", + *_SHARED_REWARD_FLAGS, ], - "gated": False, - "timesteps": 40_000_000, # ~4h at the standing n-parallel/speedup (20M took ~2h) - # Stage 0 MUST set this explicitly — resume_checkpoint()'s stage-0 - # branch returns None (train from scratch) without it. - "resume_from_experiment": FOUNDATION_EXPERIMENT, + "gated": False, # ungated waypoint: trains, checkpoints, always advances — no eval + "timesteps": 40_000_000, # ~4h at the standing n-parallel/speedup }, { - "name": "unmask-ramp50", - # Step 2/4: 50% authority. airborne_penalty at 1/3 of its final - # value — enough to start discouraging unproductive altitude, not - # enough to fight the still-partial vertical axis outright. - # resume_from_experiment deliberately omitted: chains from - # ramp25's pass via _resume_source_experiment's default - # (_passing_experiment_for_stage) — do not add an override here. + "name": "selfplay", + # Stage 2/3: this is where essentially all of the actual learning + # happens. Self-play (not frozen) as the main regime — it's what + # scales and what Necto/Nexto-class bots actually use; a frozen + # target this early would cap skill at "exploits one specific bot" + # instead of a moving, improving target. air_drill_chance stays on + # at a constant rate throughout (not introduced as a later stage) — + # gating *when* a skill is drilled reproduces the exact "gate what + # the policy is allowed to do" pattern that failed 3 generations in + # a row; only the state mix should vary between stages, never what + # the policy can act on. "flags": [ - *_UNMASK_RAMP_SHARED_FLAGS, - "--vertical-ramp", "0.5", - "--pitch-roll-ramp", "0.5", - "--airborne-penalty", "0.001", + "--opponent-mode", "self_play", + "--kickoff-chance", "0.15", + "--near-goal-chance", "0.25", + "--air-drill-chance", "0.25", + *_SHARED_REWARD_FLAGS, ], - "gated": False, - "timesteps": 40_000_000, + "gated": True, + "timesteps": 160_000_000, # ~16h }, { - "name": "unmask-ramp75", - # Step 3/4: 75% authority, airborne_penalty at 2/3 of its final - # value. Also chains automatically — no resume_from_experiment. + "name": "gauntlet", + # Stage 3/3: a stationary opponent (this stage's own predecessor's + # export) gives a low-variance measurement — important when the gate + # is a 100-episode sample with a lenient 15-point margin — and + # catches a self-play fixed point: a policy that only learned to + # beat itself will look fine in stage 2 and stall here. + # opponent_model_from_previous_stage resolves --opponent-model at + # run time to whatever stage 2's own passing export turns out to be + # (see run_stage_attempt) rather than a hardcoded name. "flags": [ - *_UNMASK_RAMP_SHARED_FLAGS, - "--vertical-ramp", "0.75", - "--pitch-roll-ramp", "0.75", - "--airborne-penalty", "0.002", + "--opponent-mode", "frozen", + "--kickoff-chance", "0.15", + "--near-goal-chance", "0.25", + "--air-drill-chance", "0.25", + *_SHARED_REWARD_FLAGS, ], - "gated": False, - "timesteps": 40_000_000, - }, - { - "name": "unmask", - # Step 4/4, the measured transition: full ramp (100% authority), - # airborne_penalty at its full value — behaviourally and eval-wise - # identical to generation 2's "unmask" stage's flags/config, so - # this stage's result is a direct, apples-to-apples comparison - # against the 3 failed all-or-nothing attempts (same reference, - # same opponent mode, same budget). Gated (default True): evaluated - # against FOUNDATION_EXPERIMENT exactly like every prior attempt. - # - # No resume_from_experiment here (deliberately, unlike generation - # 2's single-stage version) — this stage chains from ramp75's own - # checkpoint via the default resume path, not back to - # FOUNDATION_EXPERIMENT; only reference_experiment (the *eval* - # opponent) stays FOUNDATION_EXPERIMENT. - "flags": [ - *_UNMASK_RAMP_SHARED_FLAGS, - "--vertical-ramp", "1.0", - "--pitch-roll-ramp", "1.0", - "--airborne-penalty", "0.003", - ], - "grounded": False, - "timesteps": 240_000_000, # unchanged from the 3 failed attempts — same budget for a controlled comparison - "reference_experiment": FOUNDATION_EXPERIMENT, - # On retry, reset to the clean ramp75 checkpoint rather than - # compounding a failed full-ramp attempt's own drift — mirrors the - # generation 1 -> 2 postmortem (blind same-checkpoint retries only - # made things worse). - "reset_retry_checkpoint": True, + "gated": True, + "timesteps": 120_000_000, # ~12h + "opponent_model_from_previous_stage": True, }, ] @@ -227,50 +228,6 @@ def _logged_experiment_name(stage_index: int, attempt: int) -> str: raise RuntimeError(f"No logged experiment for stage {stage_index} attempt {attempt}") -def resume_checkpoint(stage_index: int, attempt: int, seed_checkpoint: str | None) -> str | None: - if attempt > 0 and not STAGES[stage_index].get("reset_retry_checkpoint"): - # Retry: keep training the same stage's own last attempt. - prev = _logged_experiment_name(stage_index, attempt - 1) - return str(TRAINING_DIR / "checkpoints" / prev / "final.zip") - if stage_index == 0 and seed_checkpoint: - return seed_checkpoint - if stage_index == 0 and not STAGES[0].get("resume_from_experiment"): - # Deliberately fresh by default: the curriculum exists because - # resuming self-play across a regime change (run10, run11) didn't - # work, so a from-scratch stage 1 starts from a random policy under - # its own regime unless --seed-checkpoint or resume_from_experiment - # says otherwise. - return None - # Either a later stage chaining off its predecessor, or - # reset_retry_checkpoint: this stage's own retries have been drifting - # rather than converging (see the "unmask" stage's comment) — resume - # from the stage's normal resume source instead of compounding the last - # failed attempt's drift. - prev_experiment = _resume_source_experiment(stage_index) - return str(TRAINING_DIR / "checkpoints" / prev_experiment / "final.zip") - - -def reference_bot(stage_index: int) -> str: - if stage_index == 0 and not STAGES[0].get("reference_experiment"): - return str(ROOKIE_REFERENCE) - prev_experiment = _reference_source_experiment(stage_index) - return str(TRAINING_DIR.parent / "Game" / "bots" / f"{prev_experiment}.json") - - -# A stage normally chains off "whatever passed at the previous index," but a -# stage can instead name an explicit resume_from_experiment/reference_experiment -# to skip a since-regressed branch, or (stage 0) to seed from a fixed -# foundation checkpoint instead of a from-scratch policy. -def _resume_source_experiment(stage_index: int) -> str: - override = STAGES[stage_index].get("resume_from_experiment") - return override if override else _passing_experiment_for_stage(stage_index - 1) - - -def _reference_source_experiment(stage_index: int) -> str: - override = STAGES[stage_index].get("reference_experiment") - return override if override else _passing_experiment_for_stage(stage_index - 1) - - def _passing_experiment_for_stage(stage_index: int) -> str: state = load_state() for entry in state["log"]: @@ -279,14 +236,31 @@ def _passing_experiment_for_stage(stage_index: int) -> str: raise RuntimeError(f"No passing attempt recorded for stage {stage_index} ({STAGES[stage_index]['name']})") -def _grounded_for_experiment(experiment: str) -> bool: - if experiment in LEGACY_GROUNDED: - return LEGACY_GROUNDED[experiment] - state = load_state() - for entry in state["log"]: - if entry["experiment"] == experiment: - return STAGES[entry["stage_index"]]["grounded"] - raise ValueError(f"Unknown experiment for groundedness lookup: {experiment}") +def resume_checkpoint(stage_index: int, attempt: int, seed_checkpoint: str | None) -> str | None: + if attempt > 0: + # Retry: keep training the same stage's own last attempt. No + # per-stage "reset to a clean upstream checkpoint" override in + # generation 4 (unlike generation 3's "unmask" stage) — nothing yet + # suggests a generation-4 retry needs that; add one if a stage's + # retries turn out to be drifting rather than converging. + prev = _logged_experiment_name(stage_index, attempt - 1) + return str(TRAINING_DIR / "checkpoints" / prev / "final.zip") + if stage_index == 0: + # Deliberately fresh unless --seed-checkpoint says otherwise: full + # action space live from step 1, nothing to inherit — every prior + # generation's checkpoints are a different, incompatible + # action/observation shape (see module docstring). + return seed_checkpoint + prev_experiment = _passing_experiment_for_stage(stage_index - 1) + return str(TRAINING_DIR / "checkpoints" / prev_experiment / "final.zip") + + +def reference_bot(stage_index: int) -> str: + """Only called for gated stages (stage 0 is ungated) — the previous + stage's own passing export, exactly like every prior generation's + default chaining.""" + prev_experiment = _passing_experiment_for_stage(stage_index - 1) + return str(TRAINING_DIR.parent / "Game" / "bots" / f"{prev_experiment}.json") def run_stage_attempt(stage_index: int, attempt: int, args) -> str: @@ -294,9 +268,6 @@ def run_stage_attempt(stage_index: int, attempt: int, args) -> str: # chronologically in TensorBoard/checkpoints — see module docstring. exp = f"{datetime.now().strftime('%Y%m%d-%H%M')}-{experiment_name(stage_index, attempt)}" resume = resume_checkpoint(stage_index, attempt, args.seed_checkpoint) - # A stage can override the run's timesteps budget (see "floor-lock", - # which deliberately runs much longer than the ~20M/~2h every stage so - # far has used); otherwise it falls back to curriculum.py's own --timesteps. timesteps = STAGES[stage_index].get("timesteps", args.timesteps) cmd = [ "./run_training.sh", exp, @@ -308,6 +279,17 @@ def run_stage_attempt(stage_index: int, attempt: int, args) -> str: ] if resume: cmd += ["--resume", resume] + if STAGES[stage_index].get("opponent_model_from_previous_stage"): + prev_experiment = _passing_experiment_for_stage(stage_index - 1) + opponent_model = TRAINING_DIR.parent / "Game" / "bots" / f"{prev_experiment}.json" + cmd += ["--opponent-model", str(opponent_model)] + abort_if = STAGES[stage_index].get("abort_if") + if abort_if: + cmd += [ + "--abort-metric", abort_if["metric"], + "--abort-below", str(abort_if["below"]), + "--abort-at-steps", str(abort_if["at_steps"]), + ] print(f"\n=== Stage {stage_index + 1}/{len(STAGES)} ({STAGES[stage_index]['name']}), " f"attempt {attempt + 1}/{MAX_RETRIES + 1}: {exp} ===") print(" ".join(cmd)) @@ -315,22 +297,9 @@ def run_stage_attempt(stage_index: int, attempt: int, args) -> str: return exp -def reference_grounded(stage_index: int) -> bool: - if stage_index == 0 and not STAGES[0].get("reference_experiment"): - # rookie.json predates the locomotion mask entirely — always full 3D. - return False - return _grounded_for_experiment(_reference_source_experiment(stage_index)) - - -def evaluate_attempt(experiment: str, reference: str, episodes: int, stage_index: int) -> dict: +def evaluate_attempt(experiment: str, reference: str, episodes: int) -> dict: candidate = TRAINING_DIR.parent / "Game" / "bots" / f"{experiment}.json" cmd = [".venv/bin/python", "evaluate.py", str(candidate), reference, "--episodes", str(episodes)] - # Must match how each side was actually trained — see ai_ship_controller.gd's - # allow_vertical/allow_pitch_roll and evaluate.py's --grounded-a/-b. - if STAGES[stage_index]["grounded"]: - cmd.append("--grounded-a") - if reference_grounded(stage_index): - cmd.append("--grounded-b") print(" ".join(cmd)) subprocess.run(cmd, cwd=TRAINING_DIR, check=True) history = json.loads(EVAL_HISTORY_PATH.read_text()) @@ -345,6 +314,33 @@ def decide(record: dict) -> str: return "pass" +def final_report(experiment: str) -> None: + """Not a gate — the two numbers that actually answer "did generation 4 + work?" (see TRAINING.md). promoted/easy.json is the shipped bot; + promoted/reference-grounded.json (a copy of generation 3's + curric-s5-aggression, made before the flat Game/bots/ dump was scrapped) + is the strongest grounded-era artifact and the yardstick generations 1-3 + were all measured against. reference-grounded.json was trained with the + locomotion mask on, so needs --grounded-b; easy.json was itself promoted + from a *failed* unmask stage (curric-s6-unmask) and is full 3D like + every generation-4 candidate, so needs no flag.""" + candidate = TRAINING_DIR.parent / "Game" / "bots" / f"{experiment}.json" + print("\n=== Curriculum complete — final report (informational, not a gate) ===") + for label, reference, extra_flags in [ + ("promoted/easy.json (shipped bot)", PROMOTED_EASY, []), + ("promoted/reference-grounded.json (strongest grounded-era bot)", PROMOTED_REFERENCE_GROUNDED, ["--grounded-b"]), + ]: + if not reference.exists(): + print(f" vs {label}: skipped, file not found") + continue + cmd = [".venv/bin/python", "evaluate.py", str(candidate), str(reference), "--episodes", str(EVAL_EPISODES), *extra_flags] + print(" ".join(cmd)) + subprocess.run(cmd, cwd=TRAINING_DIR, check=True) + record = json.loads(EVAL_HISTORY_PATH.read_text())[-1] + print(f" vs {label}: {record['wins_a']}-{record['wins_b']} ({record['draws']} draws), " + f"win rate {record['win_rate_a']:.0%}") + + def commit_progress(experiment: str) -> None: subprocess.run(["git", "add", "curriculum_state.json", "eval_history.json"], cwd=TRAINING_DIR, check=True) result = subprocess.run(["git", "diff", "--cached", "--quiet"], cwd=TRAINING_DIR) @@ -364,8 +360,7 @@ def main(): parser.add_argument("--speedup", type=int, default=16) parser.add_argument( "--seed-checkpoint", default=None, - help="Resume stage 1 from this checkpoint instead of its default resume source " - "(FOUNDATION_EXPERIMENT's checkpoint)", + help="Resume stage 1 from this checkpoint instead of training from scratch", ) parser.add_argument("--force-retry", action="store_true", help="Retry a blocked stage after human review") parser.add_argument("--skip-to-next-stage", action="store_true", help="Human judgment call: treat the blocked stage as good enough, advance anyway") @@ -413,14 +408,13 @@ def main(): last_experiment = experiment if not STAGES[stage_index].get("gated", True): - # Ungated ramp waypoint (see the unmask-ramp2X stages): trains, + # Ungated waypoint (see the bootstrap stage): trains, # checkpoints, and always advances — no eval, no regression - # gate, nothing to retry against. See TRAINING.md's - # "Generation 3" section. - print(f"{experiment}: ungated ramp waypoint — skipping eval, advancing unconditionally") + # gate, nothing to retry against. + print(f"{experiment}: ungated waypoint — skipping eval, advancing unconditionally") state["log"].append({ "stage_index": stage_index, "experiment": experiment, "attempt": attempt, - "decision": "pass", "note": "ungated ramp waypoint (no eval)", + "decision": "pass", "note": "ungated waypoint (no eval)", }) state["stage_index"] += 1 state["attempt"] = 0 @@ -430,7 +424,7 @@ def main(): continue reference = reference_bot(stage_index) - record = evaluate_attempt(experiment, reference, EVAL_EPISODES, stage_index) + record = evaluate_attempt(experiment, reference, EVAL_EPISODES) decision = decide(record) print(f"{experiment}: candidate {record['wins_a']}-{record['wins_b']} reference " @@ -469,6 +463,7 @@ def main(): # None only if the loop above never ran at all (e.g. re-invoking # after the curriculum was already "done") — nothing new to commit # in that case. + final_report(last_experiment) commit_progress(last_experiment) diff --git a/training/eval_history.json b/training/eval_history.json index 4677d8a2..7a46664e 100644 --- a/training/eval_history.json +++ b/training/eval_history.json @@ -178,5 +178,15 @@ "wins_b": 56, "draws": 14, "win_rate_a": 0.3 + }, + { + "timestamp": "2026-08-04T21:26:41+00:00", + "model_a": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/20260803-1829-curric-s4-unmask-retry2.json", + "model_b": "/Users/jcreek/Documents/repos/GitHub/CosmicClash/Game/bots/curric-s5-aggression.json", + "episodes": 100, + "wins_a": 24, + "wins_b": 56, + "draws": 20, + "win_rate_a": 0.24 } ] diff --git a/training/export_policy.py b/training/export_policy.py index 05f10f2b..df186a54 100644 --- a/training/export_policy.py +++ b/training/export_policy.py @@ -1,9 +1,15 @@ """Export a trained SB3 checkpoint to the JSON format PolicyNetwork.gd loads. -The exported file contains the deterministic policy MLP (obs -> action means); -the game clamps outputs to [-1, 1] and treats the last value as turbo (> 0). -A parity self-check compares the JSON forward pass against SB3's own -deterministic prediction before writing. +The exported file contains the deterministic policy MLP. For a MultiDiscrete +(curriculum generation 4+) model, the raw output is 32 per-head logits +decoded via ShipActionCodec.from_logits (argmax per head, mapped through +ACTION_HEADS' bin values below) and an "action_space" block is written to +the JSON so the game knows to decode it that way. For an older continuous +model, output is 7 action means, clamped to [-1, 1] and the last value +treated as turbo (> 0) — no "action_space" block, matching every export +before generation 4 (e.g. Game/bots/promoted/easy.json). A parity self-check +compares the JSON forward pass against SB3's own deterministic prediction +before writing, in either case. Example: .venv/bin/python export_policy.py checkpoints/smoke/final.zip ../Game/bots/rookie.json @@ -13,10 +19,28 @@ import argparse import json import pathlib +import gymnasium as gym import numpy as np import torch from stable_baselines3 import PPO +# MUST exactly match Game/scripts/ship_action_codec.gd's HEADS (name, order, +# and bin values) — this is what gets written into every generation-4 +# export's "action_space" block, and PolicyNetwork.gd/AIShipController never +# re-derive it, they just decode against whatever's in the file. Sizes are +# cross-checked against the live model's action_space.nvec below (a real +# assertion), but bin *values* have no automated cross-language check — +# treat any edit to either file as requiring the other. +ACTION_HEADS = [ + {"name": "rot_x", "bins": [-1.0, -0.5, 0.0, 0.5, 1.0]}, + {"name": "rot_y", "bins": [-1.0, -0.5, 0.0, 0.5, 1.0]}, + {"name": "rot_z", "bins": [-1.0, -0.5, 0.0, 0.5, 1.0]}, + {"name": "thrust_x", "bins": [-1.0, -0.5, 0.0, 0.5, 1.0]}, + {"name": "thrust_y", "bins": [-0.5, 0.0, 0.45, 0.75, 1.0]}, + {"name": "thrust_z", "bins": [-1.0, -0.5, 0.0, 0.5, 1.0]}, + {"name": "turbo", "bins": [0.0, 1.0]}, +] + def linear_to_layer(linear: torch.nn.Linear, activation: str) -> dict: return { @@ -66,20 +90,55 @@ def main(): policy = model.policy layers = extract_layers(policy) input_size = model.observation_space["obs"].shape[0] + is_multi_discrete = isinstance(model.action_space, gym.spaces.MultiDiscrete) - # Parity check: JSON forward pass must match SB3's deterministic action + output_data = {"input_size": int(input_size), "layers": layers} rng = np.random.default_rng(0) - for _ in range(16): - obs = rng.uniform(-1, 1, input_size).astype(np.float32) - expected, _ = model.predict({"obs": obs}, deterministic=True) - actual = np.clip(json_forward(layers, obs), -1.0, 1.0) - assert np.allclose(actual, expected, atol=1e-5), f"parity check failed: {actual} vs {expected}" + + if is_multi_discrete: + head_sizes = [len(head["bins"]) for head in ACTION_HEADS] + nvec = [int(n) for n in model.action_space.nvec] + assert nvec == head_sizes, ( + f"model action_space.nvec {nvec} doesn't match ACTION_HEADS sizes {head_sizes} — " + "update ACTION_HEADS to match ship_action_codec.gd's HEADS" + ) + output_data["action_space"] = {"type": "multi_discrete", "heads": ACTION_HEADS} + + # Index-level parity check: deterministic=True now returns one + # argmax index per head (not a float to clip), so compare argmax of + # the JSON forward pass's raw logits, sliced per head, against SB3's + # own chosen indices — a head-order mistake here would otherwise + # train/export cleanly and only surface as silently wrong in-game + # behaviour (e.g. pitch commands driving strafe thrusters). + offsets = [] + running = 0 + for size in head_sizes: + offsets.append((running, running + size)) + running += size + for _ in range(16): + obs = rng.uniform(-1, 1, input_size).astype(np.float32) + expected, _ = model.predict({"obs": obs}, deterministic=True) + logits = json_forward(layers, obs) + actual = np.array([int(np.argmax(logits[start:end])) for start, end in offsets]) + assert np.array_equal(actual, expected), f"parity check failed: {actual} vs {expected}" + else: + # Legacy continuous parity check, unchanged: JSON forward pass must + # match SB3's deterministic action mean. + for _ in range(16): + obs = rng.uniform(-1, 1, input_size).astype(np.float32) + expected, _ = model.predict({"obs": obs}, deterministic=True) + actual = np.clip(json_forward(layers, obs), -1.0, 1.0) + assert np.allclose(actual, expected, atol=1e-5), f"parity check failed: {actual} vs {expected}" output = pathlib.Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) with open(output, "w") as f: - json.dump({"input_size": int(input_size), "layers": layers}, f) - print(f"Exported {args.checkpoint} -> {output} (input size {input_size}, {len(layers)} layers, parity OK)") + json.dump(output_data, f) + action_space_label = "multi_discrete" if is_multi_discrete else "continuous" + print( + f"Exported {args.checkpoint} -> {output} (input size {input_size}, {len(layers)} layers, " + f"action_space={action_space_label}, parity OK)" + ) if __name__ == "__main__": diff --git a/training/requirements.txt b/training/requirements.txt index bd153067..59d07074 100644 --- a/training/requirements.txt +++ b/training/requirements.txt @@ -1,5 +1,19 @@ -godot-rl -stable-baselines3 -tensorboard +# Pinned as of curriculum generation 4: this codebase now depends on +# specific library internals (godot_rl's ActionSpaceProcessor discrete-only +# -> MultiDiscrete branch, stable_baselines3's MultiCategoricalDistribution +# logit layout — see Game/scripts/ship_action_codec.gd and train.py), not +# just documented public APIs. An unpinned reinstall (e.g. via +# setup_linux.sh on the remote training box) could silently resolve a newer +# version that changes that behaviour without any error, which would be a +# very expensive thing to discover partway through a 12h+ curriculum stage. +# torch/gymnasium are pinned too even though they're transitive deps of the +# two above, for the same reason (torch's log_std/action_net tensor +# shapes, gymnasium's Dict space key-sorting behaviour that +# ShipActionCodec's HEADS order relies on). +godot-rl==0.8.2 +stable-baselines3==2.4.0 +torch==2.13.0 +gymnasium==1.0.0 +tensorboard==2.21.0 # Optional, for --wandb logging: # wandb diff --git a/training/run_training.sh b/training/run_training.sh index ef0fe123..111e1f24 100755 --- a/training/run_training.sh +++ b/training/run_training.sh @@ -48,7 +48,12 @@ fi # Export for in-game use (parity-checked); models live in Game/bots/ .venv/bin/python export_policy.py "checkpoints/$EXP/final.zip" "../Game/bots/$EXP.json" -git add -A checkpoints logs eval_history.json "../Game/bots" +# Only final.zip, not the intermediate ppo_*_steps.zip checkpoints (.gitignore +# excludes them) — --resume only ever points at final.zip, so the "training +# never stranded on one machine" property is fully preserved at ~0.2MB/run +# instead of ~500MB/run (a single generation-3 experiment dir was 2401 files/ +# 506MB, of which final.zip was 221KB). +git add "checkpoints/$EXP/final.zip" logs eval_history.json "../Game/bots" if git diff --cached --quiet; then echo "Nothing new to commit" else diff --git a/training/test_action_space.py b/training/test_action_space.py new file mode 100644 index 00000000..88c5d3d4 --- /dev/null +++ b/training/test_action_space.py @@ -0,0 +1,105 @@ +"""Rung 0 of TRAINING.md's validation ladder: offline, no Godot, seconds to +run. Catches the single most likely silent killer in the generation-4 +action-space redesign — a head-order mismatch between the Python trainer and +Game/scripts/ship_action_codec.gd's HEADS. If they disagree, training still +runs happily for 24h+ (pitch commands driving strafe thrusters, say) and +only surfaces as inexplicably-bad behaviour, not an error. This can't +directly parse the GDScript file, but it locks the two real, checkable +invariants an order mismatch would actually depend on: that gymnasium's Dict +key-sorting produces the exact order ship_action_codec.gd's HEADS is written +in, and that godot_rl's ActionSpaceProcessor converts that into the +MultiDiscrete nvec the trainer expects. export_policy.py's own index-level +parity check plus a real in-game round trip (rung 3) are what catch anything +this can't. + +Usage: + .venv/bin/python test_action_space.py +""" + +import sys + +import gymnasium as gym +import numpy as np +from godot_rl.core.utils import ActionSpaceProcessor + +from export_policy import ACTION_HEADS + +# The order Game/scripts/ship_action_codec.gd's HEADS is written in — kept +# here as a literal, independent restatement (not derived from ACTION_HEADS) +# so this test can actually catch export_policy.py's own list being edited +# out of order too, not just catch nothing because both sides changed +# together. +EXPECTED_ORDER = ["rot_x", "rot_y", "rot_z", "thrust_x", "thrust_y", "thrust_z", "turbo"] + + +def check_action_heads_match_expected_order() -> None: + names = [head["name"] for head in ACTION_HEADS] + assert names == EXPECTED_ORDER, ( + f"export_policy.ACTION_HEADS order {names} != expected {EXPECTED_ORDER} — " + "this must match Game/scripts/ship_action_codec.gd's HEADS exactly" + ) + + +def check_gymnasium_sorts_to_expected_order() -> None: + # Build the Dict deliberately out of order (reversed) to prove it's + # gymnasium's sort doing the work here, not insertion order — this is + # exactly what godot_env.py does with the dict Godot sends over the wire + # (see godot_env.py's from_dict, which builds a spaces.Dict from + # ShipAIController.get_action_space()'s Dictionary). + sizes = {head["name"]: len(head["bins"]) for head in ACTION_HEADS} + reversed_dict = gym.spaces.Dict({name: gym.spaces.Discrete(sizes[name]) for name in reversed(EXPECTED_ORDER)}) + sorted_names = list(reversed_dict.keys()) + assert sorted_names == EXPECTED_ORDER, ( + f"gymnasium.spaces.Dict sorted {sorted_names}, expected {EXPECTED_ORDER} — " + "if this changed, every export from this generation onward would be silently " + "mis-ordered relative to ship_action_codec.gd" + ) + + +def check_action_space_processor_produces_expected_multi_discrete() -> None: + sizes = [len(head["bins"]) for head in ACTION_HEADS] + tuple_space = gym.spaces.Tuple([gym.spaces.Discrete(n) for n in sizes]) + processor = ActionSpaceProcessor(tuple_space, convert=True) + assert isinstance(processor.action_space, gym.spaces.MultiDiscrete), ( + f"expected MultiDiscrete, got {type(processor.action_space)} — the all-discrete branch " + "in godot_rl's ActionSpaceProcessor may have changed (see requirements.txt's pin note)" + ) + assert list(processor.action_space.nvec) == sizes, ( + f"MultiDiscrete nvec {list(processor.action_space.nvec)} != expected {sizes}" + ) + + +def check_round_trip_preserves_per_head_values() -> None: + # An integer action per env, one column per head in EXPECTED_ORDER — + # confirms to_original_dist splits a MultiDiscrete action back into the + # same per-head order it was built from (this is what set_action() on + # the Godot side receives, keyed by head name). + sizes = [len(head["bins"]) for head in ACTION_HEADS] + tuple_space = gym.spaces.Tuple([gym.spaces.Discrete(n) for n in sizes]) + processor = ActionSpaceProcessor(tuple_space, convert=True) + + n_envs = 3 + rng = np.random.default_rng(0) + action = np.stack([rng.integers(0, n, size=n_envs) for n in sizes], axis=1).astype(np.int64) + original = processor.to_original_dist(action) + assert len(original) == len(sizes) + for head_index, expected_column in enumerate(action.T): + np.testing.assert_array_equal(np.asarray(original[head_index]), expected_column) + + +def main() -> int: + checks = [ + check_action_heads_match_expected_order, + check_gymnasium_sorts_to_expected_order, + check_action_space_processor_produces_expected_multi_discrete, + check_round_trip_preserves_per_head_values, + ] + for check in checks: + check() + print(f"PASS: {check.__name__}") + print(f"\nAll {len(checks)} action-space checks passed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/training/train.py b/training/train.py index fc2eb1fd..4cb76f64 100644 --- a/training/train.py +++ b/training/train.py @@ -15,6 +15,7 @@ import argparse import os import pathlib +from gymnasium import spaces from stable_baselines3 import PPO from stable_baselines3.common.callbacks import BaseCallback, CheckpointCallback from stable_baselines3.common.utils import safe_mean @@ -25,6 +26,12 @@ from cosmic_env import CosmicClashVecEnv TRAINING_DIR = pathlib.Path(__file__).resolve().parent DEFAULT_GODOT_MACOS = "/Applications/Godot.app/Contents/MacOS/Godot" +# Must match Game/scripts/ship_action_codec.gd's HEADS order exactly (both +# are independently the gymnasium-sorted key order of the same 7 names) — +# training/test_action_space.py's rung-0 check asserts this. Used only for +# per-head entropy logging/reset-logits head selection below. +ACTION_HEAD_NAMES = ["rot_x", "rot_y", "rot_z", "thrust_x", "thrust_y", "thrust_z", "turbo"] + class GoalRateCallback(BaseCallback): """Logs rollout/goal_rate: the fraction of completed episodes in the @@ -54,6 +61,154 @@ class GoalRateCallback(BaseCallback): self.logger.record("rollout/goal_rate", safe_mean(rates)) +class FlightTelemetryCallback(BaseCallback): + """Logs rollout/{airborne_fraction,mean_altitude,air_touch_fraction, + vertical_thrust_mean} — leading indicators for curriculum generation 4's + core hypothesis (a discrete action space lets the policy actually hold a + sustained vertical set-point, e.g. hovering), visible from the very + first rollout instead of only in a win-rate number measured a full + 24h+ run later, which is what made every past generation's failure mode + expensive to diagnose. Requires VecMonitor(..., info_keywords=(..., + "airborne_fraction", "mean_altitude", "air_touch_fraction", + "vertical_thrust_mean")) — see ShipAIController.get_info.""" + + _KEYS = ("airborne_fraction", "mean_altitude", "air_touch_fraction", "vertical_thrust_mean") + + def _on_step(self) -> bool: + return True + + def _on_rollout_end(self) -> None: + if len(self.model.ep_info_buffer) == 0: + return + for key in self._KEYS: + values = [ep_info[key] for ep_info in self.model.ep_info_buffer if key in ep_info] + if values: + self.logger.record(f"rollout/{key}", safe_mean(values)) + + +class EntropyFloorCallback(BaseCallback): + """Replaces the old one-shot `--reset-std` shock (meaningless under + MultiDiscrete — there is no log_std) with a persistent controller. + Three curriculum generations' TensorBoard runs all show the same + signature: exploration (train/std, under the previous continuous + Gaussian) collapsing within the first ~10% of steps and never + recovering from a single reset applied at attempt start. A controller + that responds every rollout instead of once should not have that decay- + and-stay-collapsed failure mode. + + Reads mean policy entropy each rollout (recomputed from a fresh + minibatch via the same RolloutBuffer.get() plumbing PPO's own train() + uses, since _on_rollout_end fires before that iteration's train() call) + and nudges model.ent_coef multiplicatively toward a target that decays + linearly from target_start_frac to target_end_frac of the action + space's maximum possible entropy (sum of ln(n) over each MultiDiscrete + head) over the run. PPO reads self.ent_coef fresh inside train() each + update, so mutating it here from a callback takes effect on the very + next update with no subclassing needed. No-ops (does nothing) for a + non-MultiDiscrete action space, e.g. a continuous-action A/B run. + + Also logs train/entropy_head_ per action head — the direct + replacement for the old aggregate train/std scalar, and strictly more + useful: it identifies *which* axis is collapsing instead of one number + for all seven. + """ + + def __init__( + self, + total_timesteps: int, + target_start_frac: float = 0.55, + target_end_frac: float = 0.20, + adjust_rate: float = 1.02, + ent_coef_bounds: tuple[float, float] = (1e-4, 0.05), + ): + super().__init__() + self.total_timesteps = total_timesteps + self.target_start_frac = target_start_frac + self.target_end_frac = target_end_frac + self.adjust_rate = adjust_rate + self.ent_coef_bounds = ent_coef_bounds + self._is_multi_discrete = False + self._h_max = 0.0 + self._start_timesteps = 0 + + def _on_training_start(self) -> None: + import numpy as np + + self._is_multi_discrete = isinstance(self.model.action_space, spaces.MultiDiscrete) + if self._is_multi_discrete: + self._h_max = float(np.sum(np.log(self.model.action_space.nvec))) + # this call's own timesteps budget, not the resumed total — model. + # num_timesteps keeps accumulating across --resume calls, but + # total_timesteps below is this invocation's --timesteps. + self._start_timesteps = self.model.num_timesteps + + def _on_step(self) -> bool: + return True + + def _on_rollout_end(self) -> None: + if not self._is_multi_discrete: + return + import torch as th + + batch = next(self.model.rollout_buffer.get(batch_size=self.model.batch_size)) + with th.no_grad(): + distribution = self.model.policy.get_distribution(batch.observations) + per_head = getattr(distribution, "distribution", None) + if per_head is None: + return + + entropies = [dist.entropy().mean().item() for dist in per_head] + for name, entropy in zip(ACTION_HEAD_NAMES, entropies): + self.logger.record(f"train/entropy_head_{name}", entropy) + + mean_entropy = sum(entropies) + progress = min((self.model.num_timesteps - self._start_timesteps) / self.total_timesteps, 1.0) + target_frac = self.target_start_frac + (self.target_end_frac - self.target_start_frac) * progress + target = target_frac * self._h_max + if mean_entropy < target: + self.model.ent_coef = min(self.model.ent_coef * self.adjust_rate, self.ent_coef_bounds[1]) + else: + self.model.ent_coef = max(self.model.ent_coef / self.adjust_rate, self.ent_coef_bounds[0]) + self.logger.record("train/ent_coef_adaptive", self.model.ent_coef) + + +class AbortIfCallback(BaseCallback): + """Optional kill criterion (see curriculum.py's per-stage `abort_if`): + ends model.learn() early once `metric` (a rollout/* key logged by + FlightTelemetryCallback — must run earlier in the callback list so the + value exists by the time this checks it) is below `below` at or past + `at_steps`. Stops via SB3's own "_on_step returning False halts + training" contract rather than an exception, so the enclosing + try/finally in main() still runs and saves/exports/commits whatever + checkpoint exists — an aborted stage still leaves a usable, logged + artifact instead of either running a doomed stage to completion + unattended or leaving one stranded and uncommitted. + """ + + def __init__(self, metric: str, below: float, at_steps: int): + super().__init__() + self.metric = metric + self.below = below + self.at_steps = at_steps + self._checked = False + self._should_stop = False + + def _on_step(self) -> bool: + return not self._should_stop + + def _on_rollout_end(self) -> None: + if self._checked or self.model.num_timesteps < self.at_steps: + return + self._checked = True + value = self.logger.name_to_value.get(self.metric) + if value is not None and value < self.below: + print( + f"AbortIfCallback: {self.metric}={value:.4f} < {self.below} " + f"at {self.model.num_timesteps} steps — stopping early" + ) + self._should_stop = True + + def parse_args(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -75,18 +230,57 @@ def parse_args(): parser.add_argument("--port", type=int, default=11008, help="Base TCP port (one per instance)") parser.add_argument("--seed", type=int, default=0) parser.add_argument("--resume", default=None, help="Checkpoint .zip to resume from") - parser.add_argument("--ent-coef", type=float, default=0.0001, help="Entropy bonus coefficient (applied on resume too)") + parser.add_argument( + "--ent-coef", + type=float, + default=0.01, + help="Entropy bonus coefficient (applied on resume too). Raised from 0.0001 for curriculum " + "generation 4: that value was tuned for a continuous Gaussian's differential entropy " + "(unbounded, can go negative); MultiDiscrete entropy is bounded (~10 nats for this action " + "space) and needs an order of magnitude more coefficient to matter. See --entropy-floor.", + ) parser.add_argument("--n-steps", type=int, default=256, help="Rollout length per env between updates (applied on resume too)") parser.add_argument("--batch-size", type=int, default=256, help="PPO minibatch size (applied on resume too)") parser.add_argument( - "--reset-std", + "--reset-logits", type=float, default=None, - help="On resume, reset the policy action std to this value (recovers exploration after entropy collapse)", + help="On resume, multiply the policy's action_net weights/bias by this scale (e.g. 0.1), " + "pulling every head's softmax back toward uniform without discarding learned features — " + "the MultiDiscrete analogue of the old continuous --reset-std. Combine with " + "--reset-logits-heads to reset only specific heads.", + ) + parser.add_argument( + "--reset-logits-heads", + default=None, + help=f"Comma-separated subset of {ACTION_HEAD_NAMES} to apply --reset-logits to (default: all heads)", + ) + parser.add_argument( + "--entropy-floor", + action="store_true", + help="Enable EntropyFloorCallback: a persistent per-rollout controller nudging ent_coef to " + "hold policy entropy near a decaying target, replacing the one-shot --reset-std/" + "--reset-logits shock as the primary exploration mechanism (that flag remains for " + "resume-time recovery after a diagnosed collapse; this runs continuously).", + ) + parser.add_argument( + "--checkpoint-every", type=int, default=10_000_000, + help="Timesteps between checkpoints. Raised from 100_000 for curriculum generation 4: at the " + "old value a single 240M-step stage wrote ~2400 intermediate checkpoint files (only final.zip " + "is ever committed, see .gitignore/run_training.sh, but they still accumulate in the working " + "tree during the run).", ) - parser.add_argument("--checkpoint-every", type=int, default=100_000, help="Timesteps between checkpoints") parser.add_argument("--viz", action="store_true", help="Show game windows (debugging; slow)") parser.add_argument("--wandb", action="store_true", help="Also log to Weights & Biases") + parser.add_argument( + "--abort-metric", default=None, + help="Optional kill criterion (see curriculum.py's per-stage abort_if): a rollout/* metric name to watch", + ) + parser.add_argument("--abort-below", type=float, default=None, help="Stop early if --abort-metric drops below this") + parser.add_argument( + "--abort-at-steps", type=int, default=None, + help="Don't check --abort-metric until at least this many timesteps have elapsed", + ) curriculum = parser.add_argument_group( "curriculum", "Stage the training run — see TRAINING.md's Curriculum training section" @@ -112,12 +306,13 @@ def parse_args(): curriculum.add_argument("--kickoff-chance", type=float, default=None, help="Overrides kickoff_state_chance") curriculum.add_argument("--near-goal-chance", type=float, default=None, help="Overrides ball_near_goal_chance") curriculum.add_argument( - "--vertical-ramp", type=float, default=None, - help="0.0-1.0: fraction of vertical thrust that reaches the ship (locomotion-unmask ramp; default 1.0)", + "--air-drill-chance", type=float, default=None, + help="Overrides air_drill_chance: ball spawned high, both ships spawned low and lateral — " + "unsolvable without climbing (curriculum generation 4's state-setter aerial curriculum)", ) curriculum.add_argument( - "--pitch-roll-ramp", type=float, default=None, - help="0.0-1.0: fraction of pitch/roll rotation that reaches the ship (locomotion-unmask ramp; default 1.0)", + "--tilt-penalty", type=float, default=None, + help="Overrides ShipAIController.tilt_penalty (dense per-tick cost scaled by non-upright tilt)", ) curriculum.add_argument( "--velocity-to-ball-weight", type=float, default=None, @@ -158,8 +353,8 @@ def _curriculum_kwargs(args) -> dict: "attack_goal_bias": args.attack_goal_bias, "kickoff_state_chance": args.kickoff_chance, "ball_near_goal_chance": args.near_goal_chance, - "ai_vertical_ramp": args.vertical_ramp, - "ai_pitch_roll_ramp": args.pitch_roll_ramp, + "air_drill_chance": args.air_drill_chance, + "ai_tilt_penalty": args.tilt_penalty, "ai_velocity_to_ball_weight": args.velocity_to_ball_weight, "ai_ball_distance_penalty": args.ball_distance_penalty, "ai_ball_touch_reward": args.ball_touch_reward, @@ -192,7 +387,16 @@ def main(): speedup=args.speedup, **_curriculum_kwargs(args), ) - env = VecMonitor(env, info_keywords=("goal_scored",)) + env = VecMonitor( + env, + info_keywords=( + "goal_scored", + "airborne_fraction", + "mean_altitude", + "air_touch_fraction", + "vertical_thrust_mean", + ), + ) if args.resume: model = PPO.load( @@ -207,14 +411,22 @@ def main(): f"Resumed from {args.resume} at {model.num_timesteps} timesteps " f"(ent_coef={args.ent_coef}, n_steps={args.n_steps}, batch_size={args.batch_size})" ) - if args.reset_std is not None: - import math - + if args.reset_logits is not None: import torch + heads = args.reset_logits_heads.split(",") if args.reset_logits_heads else ACTION_HEAD_NAMES + nvec = list(model.action_space.nvec) + offset = 0 + offsets = {} + for name, size in zip(ACTION_HEAD_NAMES, nvec): + offsets[name] = (offset, offset + size) + offset += size with torch.no_grad(): - model.policy.log_std.fill_(math.log(args.reset_std)) - print(f"Reset policy action std to {args.reset_std}") + for name in heads: + start, end = offsets[name] + model.policy.action_net.weight[start:end].mul_(args.reset_logits) + model.policy.action_net.bias[start:end].mul_(args.reset_logits) + print(f"Reset action_net logits for heads {heads} by scale {args.reset_logits}") else: model = PPO( "MultiInputPolicy", @@ -232,12 +444,18 @@ def main(): save_path=str(checkpoint_dir), name_prefix="ppo", ) - goal_rate_callback = GoalRateCallback() + # Order matters for AbortIfCallback (must run after FlightTelemetryCallback + # so the rollout/* metric it watches has already been logged this round). + callbacks = [checkpoint_callback, GoalRateCallback(), FlightTelemetryCallback()] + if args.entropy_floor: + callbacks.append(EntropyFloorCallback(total_timesteps=args.timesteps)) + if args.abort_metric is not None and args.abort_below is not None and args.abort_at_steps is not None: + callbacks.append(AbortIfCallback(args.abort_metric, args.abort_below, args.abort_at_steps)) try: model.learn( args.timesteps, - callback=[checkpoint_callback, goal_rate_callback], + callback=callbacks, tb_log_name=args.experiment, reset_num_timesteps=not args.resume, )