diff --git a/bot/__init__.py b/bot/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bot/action/continuous_act.py b/bot/action/continuous_act.py new file mode 100644 index 0000000..d6c2259 --- /dev/null +++ b/bot/action/continuous_act.py @@ -0,0 +1,24 @@ +import numpy as np +from rlgym_compat import GameState + + +class ContinuousAction: + """ + Simple continuous action space. Operates in the range -1 to 1, even for the binary actions which are converted back to binary later. + This is for improved compatibility, stable baselines doesn't support tuple spaces right now. + """ + + def __init__(self): + pass + + def get_action_space(self): + raise NotImplementedError("We don't implement get_action_space to remove the gym dependency") + + def parse_actions(self, actions: np.ndarray, state: GameState) -> np.ndarray: + actions = actions.reshape((-1, 8)) + + actions[..., :5] = actions[..., :5].clip(-1, 1) + # The final 3 actions handle are jump, boost and handbrake. They are inherently discrete so we convert them to either 0 or 1. + actions[..., 5:] = actions[..., 5:] > 0 + + return actions \ No newline at end of file diff --git a/bot/action/default_act.py b/bot/action/default_act.py new file mode 100644 index 0000000..a4efde9 --- /dev/null +++ b/bot/action/default_act.py @@ -0,0 +1,30 @@ +import numpy as np +from rlgym_compat import GameState +from .continuous_act import ContinuousAction +from typing import Union, List + + +class DefaultAction(ContinuousAction): + """ + Continuous Action space, that also accepts a few other input formats for QoL reasons and to remain + compatible with older versions. + """ + + def __init__(self): + super().__init__() + + def get_action_space(self): + return super().get_action_space() + + def parse_actions(self, actions: Union[np.ndarray, List[np.ndarray], List[float]], state: GameState) -> np.ndarray: + + # allow other data types, this part should not be necessary but is nice to have in the default action parser. + if type(actions) != np.ndarray: + actions = np.asarray(actions) + + if len(actions.shape) == 1: + actions = actions.reshape((-1, 8)) + elif len(actions.shape) > 2: + raise ValueError('{} is not a valid action shape'.format(actions.shape)) + + return super().parse_actions(actions, state) diff --git a/bot/action/discrete_act.py b/bot/action/discrete_act.py new file mode 100644 index 0000000..b78a316 --- /dev/null +++ b/bot/action/discrete_act.py @@ -0,0 +1,24 @@ +import numpy as np +from rlgym_compat import GameState + + +class DiscreteAction: + """ + Simple discrete action space. All the analog actions have 3 bins by default: -1, 0 and 1. + """ + + def __init__(self, n_bins=3): + assert n_bins % 2 == 1, "n_bins must be an odd number" + self._n_bins = n_bins + + def get_action_space(self): + raise NotImplementedError("We don't implement get_action_space to remove the gym dependency") + + def parse_actions(self, actions: np.ndarray, state: GameState) -> np.ndarray: + actions = actions.reshape((-1, 8)).astype(dtype=np.float32) + + # map all binned actions from {0, 1, 2 .. n_bins - 1} to {-1 .. 1}. + actions[..., :5] = actions[..., :5] / (self._n_bins // 2) - 1 + + return actions + diff --git a/bot/agent.py b/bot/agent.py new file mode 100644 index 0000000..34c4f5f --- /dev/null +++ b/bot/agent.py @@ -0,0 +1,20 @@ +import os + + +class Agent: + def __init__(self): + # If you need to load your model from a file this is the time to do it + # You can do something like: + # + # self.actor = # your Model + # + # cur_dir = os.path.dirname(os.path.realpath(__file__)) + # with open(os.path.join(cur_dir, 'model.p'), 'rb') as file: + # model = pickle.load(file) + # self.actor.load_state_dict(model) + pass + + def act(self, state): + # Evaluate your model here + action = [1, 0, 0, 0, 0, 0, 0, 0] + return action diff --git a/bot/appearance.cfg b/bot/appearance.cfg new file mode 100644 index 0000000..45840e3 --- /dev/null +++ b/bot/appearance.cfg @@ -0,0 +1,53 @@ +# You don't have to manually edit this file! +# RLBotGUI has an appearance editor with a nice colorpicker, database of items and more! +# To open it up, simply click the (i) icon next to your bot's name and then click Edit Appearance + +[Bot Loadout] +team_color_id = 60 +custom_color_id = 0 +car_id = 23 +decal_id = 0 +wheels_id = 1565 +boost_id = 35 +antenna_id = 0 +hat_id = 0 +paint_finish_id = 1681 +custom_finish_id = 1681 +engine_audio_id = 0 +trails_id = 3220 +goal_explosion_id = 3018 + +[Bot Loadout Orange] +team_color_id = 3 +custom_color_id = 0 +car_id = 23 +decal_id = 0 +wheels_id = 1565 +boost_id = 35 +antenna_id = 0 +hat_id = 0 +paint_finish_id = 1681 +custom_finish_id = 1681 +engine_audio_id = 0 +trails_id = 3220 +goal_explosion_id = 3018 + +[Bot Paint Blue] +car_paint_id = 12 +decal_paint_id = 0 +wheels_paint_id = 7 +boost_paint_id = 7 +antenna_paint_id = 0 +hat_paint_id = 0 +trails_paint_id = 2 +goal_explosion_paint_id = 0 + +[Bot Paint Orange] +car_paint_id = 12 +decal_paint_id = 0 +wheels_paint_id = 14 +boost_paint_id = 14 +antenna_paint_id = 0 +hat_paint_id = 0 +trails_paint_id = 14 +goal_explosion_paint_id = 0 diff --git a/bot/bot.cfg b/bot/bot.cfg new file mode 100644 index 0000000..c9b59b4 --- /dev/null +++ b/bot/bot.cfg @@ -0,0 +1,31 @@ +[Locations] +# Path to loadout config. Can use relative path from here. +looks_config = ./appearance.cfg + +# Path to python file. Can use relative path from here. +python_file = ./bot.py +requirements_file = ./requirements.txt + +# Name of the bot in-game +name = RLGymExampleBot + +# The maximum number of ticks per second that your bot wishes to receive. +maximum_tick_rate_preference = 120 + +[Details] +# These values are optional but useful metadata for helper programs +# Name of the bot's creator/developer +developer = The RLBot community + +# Short description of the bot +description = This is a multi-line description + of the official rlgym example bot + +# Fun fact about the bot +fun_fact = + +# Link to github repository +github = https://github.com/RLGym/RLGymExampleBot + +# Programming language +language = rlgym diff --git a/bot/bot.py b/bot/bot.py new file mode 100644 index 0000000..9a3f30f --- /dev/null +++ b/bot/bot.py @@ -0,0 +1,95 @@ +from rlbot.agents.base_agent import BaseAgent, SimpleControllerState +from rlbot.utils.structures.game_data_struct import GameTickPacket + +import numpy as np + +from action.default_act import DefaultAction +from agent import Agent +from obs.default_obs import DefaultObs +from rlgym_compat import GameState + + +class RLGymExampleBot(BaseAgent): + def __init__(self, name, team, index): + super().__init__(name, team, index) + + # FIXME Hey, botmaker. Start here: + # Swap the obs builder if you are using a different one, RLGym's AdvancedObs is also available + self.obs_builder = DefaultObs() + # Swap the action parser if you are using a different one, RLGym's Discrete and Continuous are also available + self.act_parser = DefaultAction() + # Your neural network logic goes inside the Agent class, go take a look inside src/agent.py + self.agent = Agent() + # Adjust the tickskip if your agent was trained with a different value + self.tick_skip = 8 + + self.game_state: GameState = None + self.controls = None + self.action = None + self.update_action = True + self.ticks = 0 + self.prev_time = 0 + print('RLGymExampleBot Ready - Index:', index) + + def initialize_agent(self): + # Initialize the rlgym GameState object now that the game is active and the info is available + self.game_state = GameState(self.get_field_info()) + self.ticks = self.tick_skip # So we take an action the first tick + self.prev_time = 0 + self.controls = SimpleControllerState() + self.action = np.zeros(8) + self.update_action = True + + def get_output(self, packet: GameTickPacket) -> SimpleControllerState: + cur_time = packet.game_info.seconds_elapsed + delta = cur_time - self.prev_time + self.prev_time = cur_time + + ticks_elapsed = round(delta * 120) + self.ticks += ticks_elapsed + self.game_state.decode(packet, ticks_elapsed) + + if self.update_action: + self.update_action = False + + # FIXME Hey, botmaker. Verify that this is what you need for your agent + # By default we treat every match as a 1v1 against a fixed opponent, + # by doing this your bot can participate in 2v2 or 3v3 matches. Feel free to change this + player = self.game_state.players[self.index] + teammates = [p for p in self.game_state.players if p.team_num == self.team] + opponents = [p for p in self.game_state.players if p.team_num != self.team] + + if len(opponents) == 0: + # There's no opponent, we assume this model is 1v0 + self.game_state.players = [player] + else: + # Sort by distance to ball + teammates.sort(key=lambda p: np.linalg.norm(self.game_state.ball.position - p.car_data.position)) + opponents.sort(key=lambda p: np.linalg.norm(self.game_state.ball.position - p.car_data.position)) + + # Grab opponent in same "position" relative to it's teammates + opponent = opponents[min(teammates.index(player), len(opponents) - 1)] + + self.game_state.players = [player, opponent] + + obs = self.obs_builder.build_obs(player, self.game_state, self.action) + self.action = self.act_parser.parse_actions(self.agent.act(obs), self.game_state)[0] # Dim is (N, 8) + + if self.ticks >= self.tick_skip - 1: + self.update_controls(self.action) + + if self.ticks >= self.tick_skip: + self.ticks = 0 + self.update_action = True + + return self.controls + + def update_controls(self, action): + self.controls.throttle = action[0] + self.controls.steer = action[1] + self.controls.pitch = action[2] + self.controls.yaw = 0 if action[5] > 0 else action[3] + self.controls.roll = action[4] + self.controls.jump = action[5] > 0 + self.controls.boost = action[6] > 0 + self.controls.handbrake = action[7] > 0 diff --git a/bot/obs/advanced_obs.py b/bot/obs/advanced_obs.py new file mode 100644 index 0000000..47ab6b1 --- /dev/null +++ b/bot/obs/advanced_obs.py @@ -0,0 +1,83 @@ +import math +import numpy as np +from typing import Any, List +from rlgym_compat import common_values +from rlgym_compat import PlayerData, GameState, PhysicsObject + + +class AdvancedObs: + POS_STD = 2300 + ANG_STD = math.pi + + def __init__(self): + super().__init__() + + def reset(self, initial_state: GameState): + pass + + def build_obs(self, player: PlayerData, state: GameState, previous_action: np.ndarray) -> Any: + + if player.team_num == common_values.ORANGE_TEAM: + inverted = True + ball = state.inverted_ball + pads = state.inverted_boost_pads + else: + inverted = False + ball = state.ball + pads = state.boost_pads + + obs = [ball.position / self.POS_STD, + ball.linear_velocity / self.POS_STD, + ball.angular_velocity / self.ANG_STD, + previous_action, + pads] + + player_car = self._add_player_to_obs(obs, player, ball, inverted) + + allies = [] + enemies = [] + + for other in state.players: + if other.car_id == player.car_id: + continue + + if other.team_num == player.team_num: + team_obs = allies + else: + team_obs = enemies + + other_car = self._add_player_to_obs(team_obs, other, ball, inverted) + + # Extra info + team_obs.extend([ + (other_car.position - player_car.position) / self.POS_STD, + (other_car.linear_velocity - player_car.linear_velocity) / self.POS_STD + ]) + + obs.extend(allies) + obs.extend(enemies) + return np.concatenate(obs) + + def _add_player_to_obs(self, obs: List, player: PlayerData, ball: PhysicsObject, inverted: bool): + if inverted: + player_car = player.inverted_car_data + else: + player_car = player.car_data + + rel_pos = ball.position - player_car.position + rel_vel = ball.linear_velocity - player_car.linear_velocity + + obs.extend([ + rel_pos / self.POS_STD, + rel_vel / self.POS_STD, + player_car.position / self.POS_STD, + player_car.forward(), + player_car.up(), + player_car.linear_velocity / self.POS_STD, + player_car.angular_velocity / self.ANG_STD, + [player.boost_amount, + int(player.on_ground), + int(player.has_flip), + int(player.is_demoed)]]) + + return player_car diff --git a/bot/obs/default_obs.py b/bot/obs/default_obs.py new file mode 100644 index 0000000..dfb83f4 --- /dev/null +++ b/bot/obs/default_obs.py @@ -0,0 +1,78 @@ +import math +import numpy as np +from typing import Any, List +from rlgym_compat import common_values +from rlgym_compat import PlayerData, GameState + + +class DefaultObs: + def __init__(self, pos_coef=1/2300, ang_coef=1/math.pi, lin_vel_coef=1/2300, ang_vel_coef=1/math.pi): + """ + :param pos_coef: Position normalization coefficient + :param ang_coef: Rotation angle normalization coefficient + :param lin_vel_coef: Linear velocity normalization coefficient + :param ang_vel_coef: Angular velocity normalization coefficient + """ + super().__init__() + self.POS_COEF = pos_coef + self.ANG_COEF = ang_coef + self.LIN_VEL_COEF = lin_vel_coef + self.ANG_VEL_COEF = ang_vel_coef + + def reset(self, initial_state: GameState): + pass + + def build_obs(self, player: PlayerData, state: GameState, previous_action: np.ndarray) -> Any: + if player.team_num == common_values.ORANGE_TEAM: + inverted = True + ball = state.inverted_ball + pads = state.inverted_boost_pads + else: + inverted = False + ball = state.ball + pads = state.boost_pads + + obs = [ball.position * self.POS_COEF, + ball.linear_velocity * self.LIN_VEL_COEF, + ball.angular_velocity * self.ANG_VEL_COEF, + previous_action, + pads] + + self._add_player_to_obs(obs, player, inverted) + + allies = [] + enemies = [] + + for other in state.players: + if other.car_id == player.car_id: + continue + + if other.team_num == player.team_num: + team_obs = allies + else: + team_obs = enemies + + self._add_player_to_obs(team_obs, other, inverted) + + obs.extend(allies) + obs.extend(enemies) + return np.concatenate(obs) + + def _add_player_to_obs(self, obs: List, player: PlayerData, inverted: bool): + if inverted: + player_car = player.inverted_car_data + else: + player_car = player.car_data + + obs.extend([ + player_car.position * self.POS_COEF, + player_car.forward(), + player_car.up(), + player_car.linear_velocity * self.LIN_VEL_COEF, + player_car.angular_velocity * self.ANG_VEL_COEF, + [player.boost_amount, + int(player.on_ground), + int(player.has_flip), + int(player.is_demoed)]]) + + return player_car diff --git a/bot/requirements.txt b/bot/requirements.txt new file mode 100644 index 0000000..3a8310e --- /dev/null +++ b/bot/requirements.txt @@ -0,0 +1,10 @@ +# Include everything the framework requires +# You will automatically get updates for all versions starting with "1.". +rlbot==1.* +--find-links https://download.pytorch.org/whl/torch_stable.html +torch==2.0.1+cu117 +rlgym-compat>=1.1.0 +numpy + +# This will cause pip to auto-upgrade and stop scaring people with warning messages +pip