feat(*): Update bot to work in all game modes with reasonable rewards

bot/src/bot.py

@@ -11,6 +11,8 @@ from rlgym.utils.obs_builders import AdvancedObs
 from rlgym_compat import GameState
 from rlgym.utils.action_parsers import DiscreteAction
 
+from rlgym_tools.extra_obs.advanced_padder import AdvancedObsPadder
+
 
 class Sarpy(BaseAgent):
     def __init__(self, name, team, index):
@@ -18,7 +20,7 @@ class Sarpy(BaseAgent):
 
         # FIXME Hey, botmaker. Start here:
         # Swap the obs builder if you are using a different one, RLGym's AdvancedObs is also available
-        self.obs_builder = AdvancedObs()
+        self.obs_builder = AdvancedObsPadder(team_size=3)
         # Swap the action parser if you are using a different one, RLGym's Discrete and Continuous are also available
         self.act_parser = DiscreteAction()
         # Your neural network logic goes inside the Agent class, go take a look inside src/agent.py
@@ -59,33 +61,10 @@ class Sarpy(BaseAgent):
             # By default we treat every match as a 1v1 against a fixed opponent,
             # by doing this your bot can participate in 2v2 or 3v3 matches. Feel free to change this
             player = self.game_state.players[self.index]
-            teammates = [p for p in self.game_state.players if p.team_num == self.team]
-            opponents = [p for p in self.game_state.players if p.team_num != self.team]
-
-            if len(opponents) == 0:
-                # There's no opponent, we assume this model is 1v0
-                self.game_state.players = [player]
-            else:
-                # Sort by distance to ball
-                teammates.sort(
-                    key=lambda p: np.linalg.norm(
-                        self.game_state.ball.position - p.car_data.position
-                    )
-                )
-                opponents.sort(
-                    key=lambda p: np.linalg.norm(
-                        self.game_state.ball.position - p.car_data.position
-                    )
-                )
-
-                # Grab opponent in same "position" relative to it's teammates
-                opponent = opponents[min(teammates.index(player), len(opponents) - 1)]
-
-                self.game_state.players = [player, opponent]
 
             obs = self.obs_builder.build_obs(player, self.game_state, self.action)
             self.action = self.act_parser.parse_actions(
-                self.agent.act(obs), self.game_state
+                np.array(self.agent.act(obs)), self.game_state
             )[
                 0
             ]  # Dim is (N, 8)

rl-training.py

@@ -12,8 +12,14 @@ from rlgym.envs import Match
 from rlgym.utils.action_parsers import DiscreteAction
 from rlgym.utils.reward_functions.common_rewards import (
     VelocityBallToGoalReward,
+    LiuDistanceBallToGoalReward,
     BallYCoordinateReward,
     EventReward,
+    RewardIfClosestToBall,
+    LiuDistancePlayerToBallReward,
+    FaceBallReward,
+    TouchBallReward,
+    AlignBallGoal,
 )
 from rlgym.utils.obs_builders import AdvancedObs
 from rlgym.utils.state_setters import DefaultState
@@ -31,11 +37,8 @@ from rlgym.utils.reward_functions.common_rewards.ball_goal_rewards import (
 from rlgym.utils.reward_functions import CombinedReward
 
 from rlgym_tools.sb3_utils import SB3MultipleInstanceEnv
-
+from rlgym_tools.extra_rewards.kickoff_reward import KickoffReward
-
+from rlgym_tools.extra_obs.advanced_padder import AdvancedObsPadder
-# from observationBuilders.CustomObsBuilderBluePerspective import (
-#     CustomObsBuilderBluePerspective,
-# )
 
 if __name__ == "__main__":  # Required for multiprocessing
     # Set up the folders
@@ -53,23 +56,22 @@ if __name__ == "__main__":  # Required for multiprocessing
 
     fps = 120 / frame_skip
     gamma = np.exp(np.log(0.5) / (fps * half_life_seconds))  # Quick mafs
-    agents_per_match = 2
+    print(f"fps={fps}, gamma={gamma})")
-    num_instances = 10
-    target_steps = 100_000
-    steps = target_steps // (num_instances * agents_per_match)
-    batch_size = steps
 
     def exit_save(model):
         model.save("models/exit_save")
 
+    team_size = 3
+
     def get_match():  # Need to use a function so that each instance can call it and produce their own objects
         return Match(
-            team_size=1,
+            team_size=team_size,
             tick_skip=frame_skip,
             reward_function=CombinedReward(
                 (
                     VelocityPlayerToBallReward(),
                     VelocityBallToGoalReward(),
+                    LiuDistanceBallToGoalReward(),
                     EventReward(
                         team_goal=100.0,
                         concede=-100.0,
@@ -78,70 +80,86 @@ if __name__ == "__main__":  # Required for multiprocessing
                         demo=10.0,
                     ),
                     BallYCoordinateReward(),
+                    RewardIfClosestToBall(LiuDistancePlayerToBallReward()),
+                    LiuDistancePlayerToBallReward(),
+                    FaceBallReward(),
+                    TouchBallReward(),
+                    AlignBallGoal(),
+                    KickoffReward(),
                 ),
-                (0.1, 1.0, 1.0, 0.1),
+                (0.1, 1.0, 1.0, 1.0, 0.1, 0.2, 0.1, 0.2, 1.0, 0.1, 10.0),
             ),
-            # self_play=True,
+            spawn_opponents=True,
             terminal_conditions=[
-                TimeoutCondition(fps * 300),
+                TimeoutCondition(fps * 30),
-                NoTouchTimeoutCondition(fps * 20),
+                NoTouchTimeoutCondition(fps * 10),
                 GoalScoredCondition(),
             ],
-            obs_builder=AdvancedObs(),  # Not that advanced, good default
+            obs_builder=AdvancedObsPadder(
+                team_size=team_size
+            ),  # Not that advanced, good default
             state_setter=DefaultState(),  # Resets to kickoff position
             action_parser=DiscreteAction(),  # Discrete > Continuous don't @ me
         )
 
     # Generate the environment (the Rocket League game used by RL Gym)
     env = SB3MultipleInstanceEnv(
-        get_match, num_instances
+        get_match, 10
-    )  # Start x instances, waiting 60 seconds between each
+    )  # Start 10 instances, waiting 60 seconds between each
     env = VecCheckNan(env)  # Optional
     env = VecMonitor(env)  # Recommended, logs mean reward and ep_len to Tensorboard
     env = VecNormalize(
         env, norm_obs=False, gamma=gamma
     )  # Highly recommended, normalizes rewards
 
-    # Load the model that was last trained, or start a new one if the zip file doesn't exist
-    try:
-        model = PPO.load(
-            f"{models_dir}/exit_save.zip",
-            env,
-            device="auto",  # Need to set device again (if using a specific one)
-        )
-        print("Loaded exit_save.zip model")
-    except:
-        model = PPO(
-            MlpPolicy,
-            env,
-            learning_rate=5e-5,  # Around this is fairly common for PPO
-            n_steps=steps,  # Number of steps to perform before optimizing network
-            batch_size=batch_size,  # Batch size as high as possible within reason
-            n_epochs=1,  # PPO calls for multiple epochs
-            gamma=gamma,  # Gamma as calculated using half-life
-            ent_coef=0.01,  # From PPO Atari
-            vf_coef=1.0,  # From PPO Atari
-            tensorboard_log=log_dir,  # `tensorboard --logdir out/logs` in terminal to see graphs
-            verbose=3,  # Print out all the info as we're going
-            device="auto",  # Uses GPU if available
-        )
-        print("Created new model")
-
     # Save model every so often
     # Divide by num_envs (number of agents) because callback only increments every time all agents have taken a step
     # This saves to specified folder with a specified name
     callback = CheckpointCallback(
-        round(5_000_000 / env.num_envs), save_path=models_dir, name_prefix="rl_model"
+        round(1_000_000 / env.num_envs), save_path=models_dir, name_prefix="sarpy_model"
     )
 
-    atexit.register(exit_save, model)
-
     try:
         while True:
+            # Load the model that was last trained, or start a new one if the zip file doesn't exist
+            try:
+                # Now, if one wants to load a trained model from a checkpoint, use this function
+                # This will contain all the attributes of the original model
+                # Any attribute can be overwritten by using the custom_objects parameter,
+                # which includes n_envs (number of agents), which has to be overwritten to use a different amount
+                model = PPO.load(
+                    f"{models_dir}/exit_save.zip",
+                    env,
+                    custom_objects=dict(
+                        n_envs=env.num_envs, _last_obs=None
+                    ),  # Need this to change number of agents
+                    device="auto",  # Need to set device again (if using a specific one)
+                    force_reset=True,  # Make SB3 reset the env so it doesn't think we're continuing from last state
+                )
+                print("Loaded exit_save.zip model")
+            except:
+                model = PPO(
+                    MlpPolicy,
+                    env,
+                    n_epochs=32,  # PPO calls for multiple epochs
+                    learning_rate=1e-5,  # Around this is fairly common for PPO
+                    ent_coef=0.01,  # From PPO Atari
+                    vf_coef=1.0,  # From PPO Atari
+                    gamma=gamma,  # Gamma as calculated using half-life
+                    verbose=3,  # Print out all the info as we're going
+                    batch_size=4096,  # Batch size as high as possible within reason
+                    n_steps=4096,  # Number of steps to perform before optimizing network
+                    tensorboard_log="out/logs",  # `tensorboard --logdir out/logs` in terminal to see graphs
+                    device="auto",  # Uses GPU if available
+                )
+                print("Created new model")
+
+            atexit.register(exit_save, model)
+
             print("Learning...")
-            model.learn(25_000_000, callback=callback)
+            model.learn(100_000_000, callback=callback, reset_num_timesteps=False)
             model.save(f"{models_dir}/exit_save")
             print("Saved exit_save model")
-            model.save(f"mmr_models/{model.num_timesteps}")
+
     except Exception as e:
         print(e)