From 0115298e880771f816bfdc93f9c14cf04a413c13 Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Sun, 8 Oct 2023 22:27:50 +0100 Subject: [PATCH] feat(*): Update bot to work in all game modes with reasonable rewards --- bot/src/bot.py | 29 ++----------- rl-training.py | 114 ++++++++++++++++++++++++++++--------------------- 2 files changed, 70 insertions(+), 73 deletions(-) diff --git a/bot/src/bot.py b/bot/src/bot.py index df6510b..0ea2f63 100644 --- a/bot/src/bot.py +++ b/bot/src/bot.py @@ -11,6 +11,8 @@ from rlgym.utils.obs_builders import AdvancedObs from rlgym_compat import GameState from rlgym.utils.action_parsers import DiscreteAction +from rlgym_tools.extra_obs.advanced_padder import AdvancedObsPadder + class Sarpy(BaseAgent): def __init__(self, name, team, index): @@ -18,7 +20,7 @@ class Sarpy(BaseAgent): # FIXME Hey, botmaker. Start here: # Swap the obs builder if you are using a different one, RLGym's AdvancedObs is also available - self.obs_builder = AdvancedObs() + self.obs_builder = AdvancedObsPadder(team_size=3) # Swap the action parser if you are using a different one, RLGym's Discrete and Continuous are also available self.act_parser = DiscreteAction() # Your neural network logic goes inside the Agent class, go take a look inside src/agent.py @@ -59,33 +61,10 @@ class Sarpy(BaseAgent): # By default we treat every match as a 1v1 against a fixed opponent, # by doing this your bot can participate in 2v2 or 3v3 matches. Feel free to change this player = self.game_state.players[self.index] - teammates = [p for p in self.game_state.players if p.team_num == self.team] - opponents = [p for p in self.game_state.players if p.team_num != self.team] - - if len(opponents) == 0: - # There's no opponent, we assume this model is 1v0 - self.game_state.players = [player] - else: - # Sort by distance to ball - teammates.sort( - key=lambda p: np.linalg.norm( - self.game_state.ball.position - p.car_data.position - ) - ) - opponents.sort( - key=lambda p: np.linalg.norm( - self.game_state.ball.position - p.car_data.position - ) - ) - - # Grab opponent in same "position" relative to it's teammates - opponent = opponents[min(teammates.index(player), len(opponents) - 1)] - - self.game_state.players = [player, opponent] obs = self.obs_builder.build_obs(player, self.game_state, self.action) self.action = self.act_parser.parse_actions( - self.agent.act(obs), self.game_state + np.array(self.agent.act(obs)), self.game_state )[ 0 ] # Dim is (N, 8) diff --git a/rl-training.py b/rl-training.py index 41745af..4ad58c7 100644 --- a/rl-training.py +++ b/rl-training.py @@ -12,8 +12,14 @@ from rlgym.envs import Match from rlgym.utils.action_parsers import DiscreteAction from rlgym.utils.reward_functions.common_rewards import ( VelocityBallToGoalReward, + LiuDistanceBallToGoalReward, BallYCoordinateReward, EventReward, + RewardIfClosestToBall, + LiuDistancePlayerToBallReward, + FaceBallReward, + TouchBallReward, + AlignBallGoal, ) from rlgym.utils.obs_builders import AdvancedObs from rlgym.utils.state_setters import DefaultState @@ -31,11 +37,8 @@ from rlgym.utils.reward_functions.common_rewards.ball_goal_rewards import ( from rlgym.utils.reward_functions import CombinedReward from rlgym_tools.sb3_utils import SB3MultipleInstanceEnv - - -# from observationBuilders.CustomObsBuilderBluePerspective import ( -# CustomObsBuilderBluePerspective, -# ) +from rlgym_tools.extra_rewards.kickoff_reward import KickoffReward +from rlgym_tools.extra_obs.advanced_padder import AdvancedObsPadder if __name__ == "__main__": # Required for multiprocessing # Set up the folders @@ -53,23 +56,22 @@ if __name__ == "__main__": # Required for multiprocessing fps = 120 / frame_skip gamma = np.exp(np.log(0.5) / (fps * half_life_seconds)) # Quick mafs - agents_per_match = 2 - num_instances = 10 - target_steps = 100_000 - steps = target_steps // (num_instances * agents_per_match) - batch_size = steps + print(f"fps={fps}, gamma={gamma})") def exit_save(model): model.save("models/exit_save") + team_size = 3 + def get_match(): # Need to use a function so that each instance can call it and produce their own objects return Match( - team_size=1, + team_size=team_size, tick_skip=frame_skip, reward_function=CombinedReward( ( VelocityPlayerToBallReward(), VelocityBallToGoalReward(), + LiuDistanceBallToGoalReward(), EventReward( team_goal=100.0, concede=-100.0, @@ -78,70 +80,86 @@ if __name__ == "__main__": # Required for multiprocessing demo=10.0, ), BallYCoordinateReward(), + RewardIfClosestToBall(LiuDistancePlayerToBallReward()), + LiuDistancePlayerToBallReward(), + FaceBallReward(), + TouchBallReward(), + AlignBallGoal(), + KickoffReward(), ), - (0.1, 1.0, 1.0, 0.1), + (0.1, 1.0, 1.0, 1.0, 0.1, 0.2, 0.1, 0.2, 1.0, 0.1, 10.0), ), - # self_play=True, + spawn_opponents=True, terminal_conditions=[ - TimeoutCondition(fps * 300), - NoTouchTimeoutCondition(fps * 20), + TimeoutCondition(fps * 30), + NoTouchTimeoutCondition(fps * 10), GoalScoredCondition(), ], - obs_builder=AdvancedObs(), # Not that advanced, good default + obs_builder=AdvancedObsPadder( + team_size=team_size + ), # Not that advanced, good default state_setter=DefaultState(), # Resets to kickoff position action_parser=DiscreteAction(), # Discrete > Continuous don't @ me ) # Generate the environment (the Rocket League game used by RL Gym) env = SB3MultipleInstanceEnv( - get_match, num_instances - ) # Start x instances, waiting 60 seconds between each + get_match, 10 + ) # Start 10 instances, waiting 60 seconds between each env = VecCheckNan(env) # Optional env = VecMonitor(env) # Recommended, logs mean reward and ep_len to Tensorboard env = VecNormalize( env, norm_obs=False, gamma=gamma ) # Highly recommended, normalizes rewards - # Load the model that was last trained, or start a new one if the zip file doesn't exist - try: - model = PPO.load( - f"{models_dir}/exit_save.zip", - env, - device="auto", # Need to set device again (if using a specific one) - ) - print("Loaded exit_save.zip model") - except: - model = PPO( - MlpPolicy, - env, - learning_rate=5e-5, # Around this is fairly common for PPO - n_steps=steps, # Number of steps to perform before optimizing network - batch_size=batch_size, # Batch size as high as possible within reason - n_epochs=1, # PPO calls for multiple epochs - gamma=gamma, # Gamma as calculated using half-life - ent_coef=0.01, # From PPO Atari - vf_coef=1.0, # From PPO Atari - tensorboard_log=log_dir, # `tensorboard --logdir out/logs` in terminal to see graphs - verbose=3, # Print out all the info as we're going - device="auto", # Uses GPU if available - ) - print("Created new model") - # Save model every so often # Divide by num_envs (number of agents) because callback only increments every time all agents have taken a step # This saves to specified folder with a specified name callback = CheckpointCallback( - round(5_000_000 / env.num_envs), save_path=models_dir, name_prefix="rl_model" + round(1_000_000 / env.num_envs), save_path=models_dir, name_prefix="sarpy_model" ) - atexit.register(exit_save, model) - try: while True: + # Load the model that was last trained, or start a new one if the zip file doesn't exist + try: + # Now, if one wants to load a trained model from a checkpoint, use this function + # This will contain all the attributes of the original model + # Any attribute can be overwritten by using the custom_objects parameter, + # which includes n_envs (number of agents), which has to be overwritten to use a different amount + model = PPO.load( + f"{models_dir}/exit_save.zip", + env, + custom_objects=dict( + n_envs=env.num_envs, _last_obs=None + ), # Need this to change number of agents + device="auto", # Need to set device again (if using a specific one) + force_reset=True, # Make SB3 reset the env so it doesn't think we're continuing from last state + ) + print("Loaded exit_save.zip model") + except: + model = PPO( + MlpPolicy, + env, + n_epochs=32, # PPO calls for multiple epochs + learning_rate=1e-5, # Around this is fairly common for PPO + ent_coef=0.01, # From PPO Atari + vf_coef=1.0, # From PPO Atari + gamma=gamma, # Gamma as calculated using half-life + verbose=3, # Print out all the info as we're going + batch_size=4096, # Batch size as high as possible within reason + n_steps=4096, # Number of steps to perform before optimizing network + tensorboard_log="out/logs", # `tensorboard --logdir out/logs` in terminal to see graphs + device="auto", # Uses GPU if available + ) + print("Created new model") + + atexit.register(exit_save, model) + print("Learning...") - model.learn(25_000_000, callback=callback) + model.learn(100_000_000, callback=callback, reset_num_timesteps=False) model.save(f"{models_dir}/exit_save") print("Saved exit_save model") - model.save(f"mmr_models/{model.num_timesteps}") + except Exception as e: print(e)