From 8da0a9f8762e2874497a23392b48165c03d7a0b1 Mon Sep 17 00:00:00 2001
From: Josh Creek <8179928+jcreek@users.noreply.github.com>
Date: Sun, 22 Oct 2023 15:35:56 +0100
Subject: [PATCH] feat(*): Add rlgym-ppo training files

---
 rlgym training/.gitignore            |   4 ++
 rlgym training/requirements.txt      |  46 ++++++++++++
 rlgym training/rlgym-ppo-training.py | 103 +++++++++++++++++++++++++++
 3 files changed, 153 insertions(+)
 create mode 100644 rlgym training/.gitignore
 create mode 100644 rlgym training/requirements.txt
 create mode 100644 rlgym training/rlgym-ppo-training.py

diff --git a/rlgym training/.gitignore b/rlgym training/.gitignore
new file mode 100644
index 0000000..00efc92
--- /dev/null
+++ b/rlgym training/.gitignore	
@@ -0,0 +1,4 @@
+collision_meshes
+data
+wandb
+rlgym-training
diff --git a/rlgym training/requirements.txt b/rlgym training/requirements.txt
new file mode 100644
index 0000000..f74bc55
--- /dev/null
+++ b/rlgym training/requirements.txt	
@@ -0,0 +1,46 @@
+appdirs==1.4.4
+certifi==2022.12.7
+charset-normalizer==2.1.1
+click==8.1.7
+cloudpickle==3.0.0
+colorama==0.4.6
+comtypes==1.2.0
+docker-pycreds==0.4.0
+filelock==3.9.0
+fsspec==2023.4.0
+gitdb==4.0.11
+GitPython==3.1.40
+gym==0.26.2
+gym-notices==0.0.8
+idna==3.4
+importlib-metadata==6.8.0
+Jinja2==3.1.2
+MarkupSafe==2.1.2
+mpmath==1.3.0
+networkx==3.0
+numpy==1.26.1
+pathtools==0.1.2
+Pillow==9.3.0
+protobuf==4.24.4
+psutil==5.9.6
+pywin32==228
+pywinauto==0.6.8
+PyYAML==6.0.1
+requests==2.28.1
+rlgym==1.2.2
+rlgym-ppo==1.2.5
+rlgym-sim==1.2.5
+rlgym-tools==1.8.2
+RocketSim==1.2.0
+sentry-sdk==1.32.0
+setproctitle==1.3.3
+six==1.16.0
+smmap==5.0.1
+sympy==1.12
+torch==2.1.0+cu118
+torchaudio==2.1.0+cu118
+torchvision==0.16.0+cu118
+typing-extensions==4.8.0
+urllib3==1.26.13
+wandb==0.15.12
+zipp==3.17.0
diff --git a/rlgym training/rlgym-ppo-training.py b/rlgym training/rlgym-ppo-training.py
new file mode 100644
index 0000000..41e9edf
--- /dev/null
+++ b/rlgym training/rlgym-ppo-training.py	
@@ -0,0 +1,103 @@
+import numpy as np
+from rlgym_sim.utils.gamestates import GameState
+from rlgym_ppo.util import MetricsLogger
+
+
+class ExampleLogger(MetricsLogger):
+    def _collect_metrics(self, game_state: GameState) -> list:
+        return [game_state.players[0].car_data.linear_velocity,
+                game_state.players[0].car_data.rotation_mtx(),
+                game_state.orange_score]
+
+    def _report_metrics(self, collected_metrics, wandb_run, cumulative_timesteps):
+        avg_linvel = np.zeros(3)
+        for metric_array in collected_metrics:
+            p0_linear_velocity = metric_array[0]
+            avg_linvel += p0_linear_velocity
+        avg_linvel /= len(collected_metrics)
+        report = {"x_vel":avg_linvel[0],
+                  "y_vel":avg_linvel[1],
+                  "z_vel":avg_linvel[2],
+                  "Cumulative Timesteps":cumulative_timesteps}
+        wandb_run.log(report)
+
+
+def build_rocketsim_env():
+    import rlgym_sim
+    from rlgym_sim.utils.reward_functions import CombinedReward
+    from rlgym_sim.utils.reward_functions.common_rewards import VelocityPlayerToBallReward, VelocityBallToGoalReward, \
+        EventReward, SaveBoostReward
+    from rlgym_tools.extra_rewards.kickoff_reward import KickoffReward
+    from rlgym_sim.utils.obs_builders import DefaultObs
+    from rlgym_sim.utils.terminal_conditions.common_conditions import NoTouchTimeoutCondition, GoalScoredCondition
+    from rlgym_sim.utils import common_values
+    from rlgym_sim.utils.action_parsers import ContinuousAction
+
+    spawn_opponents = True
+    team_size = 1
+    game_tick_rate = 120
+    tick_skip = 8
+    timeout_seconds = 10
+    timeout_ticks = int(round(timeout_seconds * game_tick_rate / tick_skip))
+
+    action_parser = ContinuousAction()
+    terminal_conditions = [NoTouchTimeoutCondition(timeout_ticks), GoalScoredCondition()]
+
+    rewards_to_combine = (VelocityPlayerToBallReward(),
+                    VelocityBallToGoalReward(),
+                    EventReward(
+                        team_goal=1000.0,
+                        concede=-100.0,
+                        shot=10.0,
+                        save=60.0,
+                        demo=20.0,
+                    ),
+                    KickoffReward(),
+                    SaveBoostReward(),)
+    reward_weights = (1.0, 1.0, 1.0, 1.0, 1.0)
+
+    reward_fn = CombinedReward(reward_functions=rewards_to_combine,
+                               reward_weights=reward_weights)
+
+    obs_builder = DefaultObs(
+        pos_coef=np.asarray([1 / common_values.SIDE_WALL_X, 1 / common_values.BACK_NET_Y, 1 / common_values.CEILING_Z]),
+        ang_coef=1 / np.pi,
+        lin_vel_coef=1 / common_values.CAR_MAX_SPEED,
+        ang_vel_coef=1 / common_values.CAR_MAX_ANG_VEL)
+
+    env = rlgym_sim.make(tick_skip=tick_skip,
+                         team_size=team_size,
+                         spawn_opponents=spawn_opponents,
+                         terminal_conditions=terminal_conditions,
+                         reward_fn=reward_fn,
+                         obs_builder=obs_builder,
+                         action_parser=action_parser)
+
+    return env
+
+if __name__ == "__main__":
+    from rlgym_ppo import Learner
+    metrics_logger = ExampleLogger()
+
+    # 32 processes
+    n_proc = 32
+
+    # educated guess - could be slightly higher or lower
+    min_inference_size = max(1, int(round(n_proc * 0.9)))
+
+    learner = Learner(build_rocketsim_env,
+                      n_proc=n_proc,
+                      min_inference_size=min_inference_size,
+                      metrics_logger=metrics_logger,
+                      ppo_batch_size=50000,
+                      ts_per_iteration=50000,
+                      exp_buffer_size=150000,
+                      ppo_minibatch_size=50000,
+                      ppo_ent_coef=0.001,
+                      ppo_epochs=1,
+                      standardize_returns=True,
+                      standardize_obs=False,
+                      save_every_ts=100_000,
+                      timestep_limit=1_000_000_000,
+                      log_to_wandb=True)
+    learner.learn()
\ No newline at end of file