From cf4059a1d1d57be2b5348e9559aad077ea60d276 Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Sun, 22 Oct 2023 20:45:04 +0100 Subject: [PATCH] feat(*): Add bot files for rlbot --- rlgym training/bot/.gitignore | 112 ++++++++++++++++++ rlgym training/bot/.pylintrc | 2 + rlgym training/bot/LICENSE | 21 ++++ rlgym training/bot/README.md | 23 ++++ rlgym training/bot/RefreshEnv.cmd | 66 +++++++++++ rlgym training/bot/rlbot.cfg | 76 ++++++++++++ rlgym training/bot/run.py | 33 ++++++ rlgym training/bot/run_gui.py | 7 ++ rlgym training/bot/src/__init__.py | 0 .../bot/src/action/continuous_act.py | 24 ++++ rlgym training/bot/src/action/default_act.py | 30 +++++ rlgym training/bot/src/action/discrete_act.py | 24 ++++ rlgym training/bot/src/agent.py | 77 ++++++++++++ rlgym training/bot/src/appearance.cfg | 53 +++++++++ rlgym training/bot/src/bot.cfg | 30 +++++ rlgym training/bot/src/bot.py | 89 ++++++++++++++ rlgym training/bot/src/obs/advanced_obs.py | 83 +++++++++++++ rlgym training/bot/src/obs/default_obs.py | 78 ++++++++++++ rlgym training/bot/src/requirements.txt | 90 ++++++++++++++ 19 files changed, 918 insertions(+) create mode 100644 rlgym training/bot/.gitignore create mode 100644 rlgym training/bot/.pylintrc create mode 100644 rlgym training/bot/LICENSE create mode 100644 rlgym training/bot/README.md create mode 100644 rlgym training/bot/RefreshEnv.cmd create mode 100644 rlgym training/bot/rlbot.cfg create mode 100644 rlgym training/bot/run.py create mode 100644 rlgym training/bot/run_gui.py create mode 100644 rlgym training/bot/src/__init__.py create mode 100644 rlgym training/bot/src/action/continuous_act.py create mode 100644 rlgym training/bot/src/action/default_act.py create mode 100644 rlgym training/bot/src/action/discrete_act.py create mode 100644 rlgym training/bot/src/agent.py create mode 100644 rlgym training/bot/src/appearance.cfg create mode 100644 rlgym training/bot/src/bot.cfg create mode 100644 rlgym training/bot/src/bot.py create mode 100644 rlgym training/bot/src/obs/advanced_obs.py create mode 100644 rlgym training/bot/src/obs/default_obs.py create mode 100644 rlgym training/bot/src/requirements.txt diff --git a/rlgym training/bot/.gitignore b/rlgym training/bot/.gitignore new file mode 100644 index 0000000..122e90e --- /dev/null +++ b/rlgym training/bot/.gitignore @@ -0,0 +1,112 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# Intellij +*.iml +/.idea + +# Build output +/build + +# Gradle files +/.gradle + +# VSCode +.vscode/ +*.code-workspace diff --git a/rlgym training/bot/.pylintrc b/rlgym training/bot/.pylintrc new file mode 100644 index 0000000..9db66d4 --- /dev/null +++ b/rlgym training/bot/.pylintrc @@ -0,0 +1,2 @@ +[TYPECHECK] +generated-members=QuickChats.* diff --git a/rlgym training/bot/LICENSE b/rlgym training/bot/LICENSE new file mode 100644 index 0000000..b7ce65a --- /dev/null +++ b/rlgym training/bot/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 RLGym + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/rlgym training/bot/README.md b/rlgym training/bot/README.md new file mode 100644 index 0000000..ee8248b --- /dev/null +++ b/rlgym training/bot/README.md @@ -0,0 +1,23 @@ +# RLGymExampleBot +RLGym example bot for the RLBot framework, based on the official RLBotPythonExample + +## How to use this + +This bot runs the Actor class in `src/actor.py`, you're expected to replace that with the output of your model + +By default we use DefaultObs from RLGym, AdvancedObs is also available in this project. + +You can also provide your own custom ObservationBuilder by copying it over and replacing the `rlgym` imports with `rlgym_compat` (check `src/obs/` for some examples) + +## Changing the bot + +- Bot behavior is controlled by `src/bot.py` +- Bot appearance is controlled by `src/appearance.cfg` + +See https://github.com/RLBot/RLBotPythonExample/wiki for documentation and tutorials. + +## Running a match + +You can start a match by running `run.py`, the match config for it is in `rlbot.cfg` + +N.B You may need to run `pip3 install eel` in the src folder from a terminal to be able to debug the rlbot gui from here but DO NOT add it to the requirements.txt as the bot does not need it to run. diff --git a/rlgym training/bot/RefreshEnv.cmd b/rlgym training/bot/RefreshEnv.cmd new file mode 100644 index 0000000..567fd78 --- /dev/null +++ b/rlgym training/bot/RefreshEnv.cmd @@ -0,0 +1,66 @@ +@echo off +:: This file is taken from chocolatey: +:: https://github.com/chocolatey/choco/blob/master/src/chocolatey.resources/redirects/RefreshEnv.cmd +:: +:: RefreshEnv.cmd +:: +:: Batch file to read environment variables from registry and +:: set session variables to these values. +:: +:: With this batch file, there should be no need to reload command +:: environment every time you want environment changes to propagate + +::echo "RefreshEnv.cmd only works from cmd.exe, please install the Chocolatey Profile to take advantage of refreshenv from PowerShell" +echo | set /p dummy="Refreshing environment variables from registry for cmd.exe. Please wait..." + +goto main + +:: Set one environment variable from registry key +:SetFromReg + "%WinDir%\System32\Reg" QUERY "%~1" /v "%~2" > "%TEMP%\_envset.tmp" 2>NUL + for /f "usebackq skip=2 tokens=2,*" %%A IN ("%TEMP%\_envset.tmp") do ( + echo/set "%~3=%%B" + ) + goto :EOF + +:: Get a list of environment variables from registry +:GetRegEnv + "%WinDir%\System32\Reg" QUERY "%~1" > "%TEMP%\_envget.tmp" + for /f "usebackq skip=2" %%A IN ("%TEMP%\_envget.tmp") do ( + if /I not "%%~A"=="Path" ( + call :SetFromReg "%~1" "%%~A" "%%~A" + ) + ) + goto :EOF + +:main + echo/@echo off >"%TEMP%\_env.cmd" + + :: Slowly generating final file + call :GetRegEnv "HKLM\System\CurrentControlSet\Control\Session Manager\Environment" >> "%TEMP%\_env.cmd" + call :GetRegEnv "HKCU\Environment">>"%TEMP%\_env.cmd" >> "%TEMP%\_env.cmd" + + :: Special handling for PATH - mix both User and System + call :SetFromReg "HKLM\System\CurrentControlSet\Control\Session Manager\Environment" Path Path_HKLM >> "%TEMP%\_env.cmd" + call :SetFromReg "HKCU\Environment" Path Path_HKCU >> "%TEMP%\_env.cmd" + + :: Caution: do not insert space-chars before >> redirection sign + echo/set "Path=%%Path_HKLM%%;%%Path_HKCU%%" >> "%TEMP%\_env.cmd" + + :: Cleanup + del /f /q "%TEMP%\_envset.tmp" 2>nul + del /f /q "%TEMP%\_envget.tmp" 2>nul + + :: capture user / architecture + SET "OriginalUserName=%USERNAME%" + SET "OriginalArchitecture=%PROCESSOR_ARCHITECTURE%" + + :: Set these variables + call "%TEMP%\_env.cmd" + + :: reset user / architecture + SET "USERNAME=%OriginalUserName%" + SET "PROCESSOR_ARCHITECTURE=%OriginalArchitecture%" + + echo | set /p dummy="Finished." + echo . \ No newline at end of file diff --git a/rlgym training/bot/rlbot.cfg b/rlgym training/bot/rlbot.cfg new file mode 100644 index 0000000..ddfac8a --- /dev/null +++ b/rlgym training/bot/rlbot.cfg @@ -0,0 +1,76 @@ +[RLBot Configuration] +# Visit https://github.com/RLBot/RLBot/wiki/Config-File-Documentation to see what you can put here. + +[Team Configuration] +# Visit https://github.com/RLBot/RLBot/wiki/Config-File-Documentation to see what you can put here. + +[Match Configuration] +# Visit https://github.com/RLBot/RLBot/wiki/Config-File-Documentation to see what you can put here. +# Number of bots/players which will be spawned. We support up to max 64. +num_participants = 6 +game_mode = Soccer +game_map = Mannfield +enable_rendering = True +enable_state_setting = True + +[Mutator Configuration] +# Visit https://github.com/RLBot/RLBot/wiki/Config-File-Documentation to see what you can put here. + +[Participant Configuration] +# Put the name of your bot config file here. Only num_participants config files will be read! +# Everything needs a config, even players and default bots. We still set loadouts and names from config! +participant_config_0 = src/bot.cfg +participant_config_1 = src/bot.cfg +participant_config_2 = src/bot.cfg +participant_config_3 = src/bot.cfg +participant_config_4 = src/bot.cfg +participant_config_5 = src/bot.cfg +participant_config_6 = src/bot.cfg +participant_config_7 = src/bot.cfg +participant_config_8 = src/bot.cfg +participant_config_9 = src/bot.cfg + +# team 0 shoots on positive goal, team 1 shoots on negative goal +participant_team_0 = 0 +participant_team_1 = 1 +participant_team_2 = 0 +participant_team_3 = 1 +participant_team_4 = 0 +participant_team_5 = 1 +participant_team_6 = 0 +participant_team_7 = 1 +participant_team_8 = 0 +participant_team_9 = 1 + +# Accepted values are "human", "rlbot", "psyonix", and "party_member_bot" +# You can have up to 4 local players and they must be activated in game or it will crash. +# If no player is specified you will be spawned in as spectator! +# human - not controlled by the framework +# rlbot - controlled by the framework +# psyonix - default bots (skill level can be changed with participant_bot_skill +# party_member_bot - controlled by the framework but the game detects it as a human +participant_type_0 = rlbot +participant_type_1 = rlbot +participant_type_2 = rlbot +participant_type_3 = rlbot +participant_type_4 = rlbot +participant_type_5 = rlbot +participant_type_6 = rlbot +participant_type_7 = rlbot +participant_type_8 = rlbot +participant_type_9 = rlbot + + +# If participant is a bot and not RLBot controlled, this value will be used to set bot skill. +# 0.0 is Rookie, 0.5 is pro, 1.0 is all-star. You can set values in-between as well. +# Please leave a value here even if it isn't used :) +participant_bot_skill_0 = 1.0 +participant_bot_skill_1 = 1.0 +participant_bot_skill_2 = 1.0 +participant_bot_skill_3 = 1.0 +participant_bot_skill_4 = 1.0 +participant_bot_skill_5 = 1.0 +participant_bot_skill_6 = 1.0 +participant_bot_skill_7 = 1.0 +participant_bot_skill_8 = 1.0 +participant_bot_skill_9 = 1.0 diff --git a/rlgym training/bot/run.py b/rlgym training/bot/run.py new file mode 100644 index 0000000..23ea98a --- /dev/null +++ b/rlgym training/bot/run.py @@ -0,0 +1,33 @@ +import subprocess +import sys + +DEFAULT_LOGGER = 'rlbot' + +if __name__ == '__main__': + + try: + from rlbot.utils import public_utils, logging_utils + + logger = logging_utils.get_logger(DEFAULT_LOGGER) + if not public_utils.have_internet(): + logger.log(logging_utils.logging_level, + 'Skipping upgrade check for now since it looks like you have no internet') + elif public_utils.is_safe_to_upgrade(): + subprocess.call([sys.executable, "-m", "pip", "install", '-r', 'requirements.txt']) + subprocess.call([sys.executable, "-m", "pip", "install", 'rlbot', '--upgrade']) + + # https://stackoverflow.com/a/44401013 + rlbots = [module for module in sys.modules if module.startswith('rlbot')] + for rlbot_module in rlbots: + sys.modules.pop(rlbot_module) + + except ImportError: + subprocess.call([sys.executable, "-m", "pip", "install", '-r', 'requirements.txt', '--upgrade', '--upgrade-strategy=eager']) + + try: + from rlbot import runner + runner.main() + except Exception as e: + print("Encountered exception: ", e) + print("Press enter to close.") + input() diff --git a/rlgym training/bot/run_gui.py b/rlgym training/bot/run_gui.py new file mode 100644 index 0000000..a3bced8 --- /dev/null +++ b/rlgym training/bot/run_gui.py @@ -0,0 +1,7 @@ +from rlbot_gui import gui + +# This is a useful way to start up RLBotGUI directly from your bot project. You can use it to +# arrange a match with the settings you like, and if you have a good IDE like PyCharm, +# you can do breakpoint debugging on your bot. +if __name__ == '__main__': + gui.start() diff --git a/rlgym training/bot/src/__init__.py b/rlgym training/bot/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rlgym training/bot/src/action/continuous_act.py b/rlgym training/bot/src/action/continuous_act.py new file mode 100644 index 0000000..d6c2259 --- /dev/null +++ b/rlgym training/bot/src/action/continuous_act.py @@ -0,0 +1,24 @@ +import numpy as np +from rlgym_compat import GameState + + +class ContinuousAction: + """ + Simple continuous action space. Operates in the range -1 to 1, even for the binary actions which are converted back to binary later. + This is for improved compatibility, stable baselines doesn't support tuple spaces right now. + """ + + def __init__(self): + pass + + def get_action_space(self): + raise NotImplementedError("We don't implement get_action_space to remove the gym dependency") + + def parse_actions(self, actions: np.ndarray, state: GameState) -> np.ndarray: + actions = actions.reshape((-1, 8)) + + actions[..., :5] = actions[..., :5].clip(-1, 1) + # The final 3 actions handle are jump, boost and handbrake. They are inherently discrete so we convert them to either 0 or 1. + actions[..., 5:] = actions[..., 5:] > 0 + + return actions \ No newline at end of file diff --git a/rlgym training/bot/src/action/default_act.py b/rlgym training/bot/src/action/default_act.py new file mode 100644 index 0000000..a4efde9 --- /dev/null +++ b/rlgym training/bot/src/action/default_act.py @@ -0,0 +1,30 @@ +import numpy as np +from rlgym_compat import GameState +from .continuous_act import ContinuousAction +from typing import Union, List + + +class DefaultAction(ContinuousAction): + """ + Continuous Action space, that also accepts a few other input formats for QoL reasons and to remain + compatible with older versions. + """ + + def __init__(self): + super().__init__() + + def get_action_space(self): + return super().get_action_space() + + def parse_actions(self, actions: Union[np.ndarray, List[np.ndarray], List[float]], state: GameState) -> np.ndarray: + + # allow other data types, this part should not be necessary but is nice to have in the default action parser. + if type(actions) != np.ndarray: + actions = np.asarray(actions) + + if len(actions.shape) == 1: + actions = actions.reshape((-1, 8)) + elif len(actions.shape) > 2: + raise ValueError('{} is not a valid action shape'.format(actions.shape)) + + return super().parse_actions(actions, state) diff --git a/rlgym training/bot/src/action/discrete_act.py b/rlgym training/bot/src/action/discrete_act.py new file mode 100644 index 0000000..b78a316 --- /dev/null +++ b/rlgym training/bot/src/action/discrete_act.py @@ -0,0 +1,24 @@ +import numpy as np +from rlgym_compat import GameState + + +class DiscreteAction: + """ + Simple discrete action space. All the analog actions have 3 bins by default: -1, 0 and 1. + """ + + def __init__(self, n_bins=3): + assert n_bins % 2 == 1, "n_bins must be an odd number" + self._n_bins = n_bins + + def get_action_space(self): + raise NotImplementedError("We don't implement get_action_space to remove the gym dependency") + + def parse_actions(self, actions: np.ndarray, state: GameState) -> np.ndarray: + actions = actions.reshape((-1, 8)).astype(dtype=np.float32) + + # map all binned actions from {0, 1, 2 .. n_bins - 1} to {-1 .. 1}. + actions[..., :5] = actions[..., :5] / (self._n_bins // 2) - 1 + + return actions + diff --git a/rlgym training/bot/src/agent.py b/rlgym training/bot/src/agent.py new file mode 100644 index 0000000..285d7ec --- /dev/null +++ b/rlgym training/bot/src/agent.py @@ -0,0 +1,77 @@ +import os +import numpy as np +from rlgym_ppo import ppo +from rlgym_ppo.ppo import ContinuousPolicy +import torch + +model_zip = "exit_save.zip" + + +class Agent: + def __init__(self): + # Get the directory of agent.py + script_dir = os.path.dirname(os.path.realpath(__file__)) + + root_folder = os.path.join(script_dir, "../../checkpoints/") + checkpoint_load_folder = "" + + # Get a list of all subdirectories in the root folder + subdirectories = [ + f + for f in os.listdir(root_folder) + if os.path.isdir(os.path.join(root_folder, f)) + ] + + highest_checkpoint_folder = None + highest_checkpoint_number = -1 + + # Iterate through the subdirectories to find the highest "checkpoints-" directory + for subdir in subdirectories: + if subdir.startswith("checkpoints-") and subdir[12:].isdigit(): + checkpoint_number = int(subdir[12:]) + if checkpoint_number > highest_checkpoint_number: + highest_checkpoint_number = checkpoint_number + highest_checkpoint_folder = subdir + + if highest_checkpoint_folder: + checkpoint_load_folder = os.path.join( + root_folder, highest_checkpoint_folder + ) + + # Now, let's find the highest numbered folder within the highest checkpoint folder + highest_numbered_subfolder = None + highest_number = -1 + checkpoint_subdirectories = [ + f + for f in os.listdir(checkpoint_load_folder) + if os.path.isdir(os.path.join(checkpoint_load_folder, f)) + ] + + for subdir in checkpoint_subdirectories: + if subdir.isdigit(): + subdir_number = int(subdir) + if subdir_number > highest_number: + highest_number = subdir_number + highest_numbered_subfolder = subdir + + if highest_numbered_subfolder: + checkpoint_load_folder = os.path.join( + checkpoint_load_folder, highest_numbered_subfolder + ) + else: + print("No 'checkpoints-' directories found in the checkpoints folder.") + + input_shape = 109 # obs_space_size = np.prod(obs_space_size) + output_shape = 16 # 2*num_actions_in_rocket_league + layer_sizes = (256, 256, 256) # (256,256,256) by default + self.policy = ContinuousPolicy(input_shape, output_shape, layer_sizes, "cpu") + self.policy.load_state_dict( + torch.load(os.path.join(checkpoint_load_folder, "PPO_POLICY.pt")) + ) + # self.model = + + def act(self, obs): + action, logprob = self.policy.get_action(obs, deterministic=False) + action = self.actor.eval(obs) + # action, _ = self.model.predict(obs) + return action.tolist() diff --git a/rlgym training/bot/src/appearance.cfg b/rlgym training/bot/src/appearance.cfg new file mode 100644 index 0000000..45840e3 --- /dev/null +++ b/rlgym training/bot/src/appearance.cfg @@ -0,0 +1,53 @@ +# You don't have to manually edit this file! +# RLBotGUI has an appearance editor with a nice colorpicker, database of items and more! +# To open it up, simply click the (i) icon next to your bot's name and then click Edit Appearance + +[Bot Loadout] +team_color_id = 60 +custom_color_id = 0 +car_id = 23 +decal_id = 0 +wheels_id = 1565 +boost_id = 35 +antenna_id = 0 +hat_id = 0 +paint_finish_id = 1681 +custom_finish_id = 1681 +engine_audio_id = 0 +trails_id = 3220 +goal_explosion_id = 3018 + +[Bot Loadout Orange] +team_color_id = 3 +custom_color_id = 0 +car_id = 23 +decal_id = 0 +wheels_id = 1565 +boost_id = 35 +antenna_id = 0 +hat_id = 0 +paint_finish_id = 1681 +custom_finish_id = 1681 +engine_audio_id = 0 +trails_id = 3220 +goal_explosion_id = 3018 + +[Bot Paint Blue] +car_paint_id = 12 +decal_paint_id = 0 +wheels_paint_id = 7 +boost_paint_id = 7 +antenna_paint_id = 0 +hat_paint_id = 0 +trails_paint_id = 2 +goal_explosion_paint_id = 0 + +[Bot Paint Orange] +car_paint_id = 12 +decal_paint_id = 0 +wheels_paint_id = 14 +boost_paint_id = 14 +antenna_paint_id = 0 +hat_paint_id = 0 +trails_paint_id = 14 +goal_explosion_paint_id = 0 diff --git a/rlgym training/bot/src/bot.cfg b/rlgym training/bot/src/bot.cfg new file mode 100644 index 0000000..f90a594 --- /dev/null +++ b/rlgym training/bot/src/bot.cfg @@ -0,0 +1,30 @@ +[Locations] +# Path to loadout config. Can use relative path from here. +looks_config = ./appearance.cfg + +# Path to python file. Can use relative path from here. +python_file = ./bot.py +requirements_file = ./requirements.txt + +# Name of the bot in-game +name = Sarpy v2 + +# The maximum number of ticks per second that your bot wishes to receive. +maximum_tick_rate_preference = 120 + +[Details] +# These values are optional but useful metadata for helper programs +# Name of the bot's creator/developer +developer = Scruffy238 + +# Short description of the bot +description = This is a machine learning bot that uses the RLGym environment to train a neural network to play Rocket League. + +# Fun fact about the bot +fun_fact = Its name comes from SARPBC, the prequel to Rocket League. + +# Link to github repository +github = https://github.com/jcreek/Sarpy + +# Programming language +language = rlgym diff --git a/rlgym training/bot/src/bot.py b/rlgym training/bot/src/bot.py new file mode 100644 index 0000000..ba14ce2 --- /dev/null +++ b/rlgym training/bot/src/bot.py @@ -0,0 +1,89 @@ +from rlbot.agents.base_agent import BaseAgent, SimpleControllerState +from rlbot.utils.structures.game_data_struct import GameTickPacket + +import numpy as np + +from action.default_act import DefaultAction +from agent import Agent + +# from obs.default_obs import DefaultObs +from rlgym.utils.obs_builders import AdvancedObs +from rlgym_compat import GameState +from rlgym.utils.action_parsers import ContinuousAction + +from rlgym_tools.extra_obs.advanced_padder import AdvancedObsPadder + + +class Sarpy(BaseAgent): + def __init__(self, name, team, index): + super().__init__(name, team, index) + + # FIXME Hey, botmaker. Start here: + # Swap the obs builder if you are using a different one, RLGym's AdvancedObs is also available + self.obs_builder = AdvancedObsPadder(team_size=3) + # Swap the action parser if you are using a different one, RLGym's Discrete and Continuous are also available + self.act_parser = ContinuousAction() + # Your neural network logic goes inside the Agent class, go take a look inside src/agent.py + self.agent = Agent() + # Adjust the tickskip if your agent was trained with a different value + self.tick_skip = 8 + + self.game_state: GameState = None + self.controls = None + self.action = None + self.update_action = True + self.ticks = 0 + self.prev_time = 0 + print("Sarpy Ready - Index:", index) + + def initialize_agent(self): + # Initialize the rlgym GameState object now that the game is active and the info is available + self.game_state = GameState(self.get_field_info()) + self.ticks = self.tick_skip # So we take an action the first tick + self.prev_time = 0 + self.controls = SimpleControllerState() + self.action = np.zeros(8) + self.update_action = True + + def get_output(self, packet: GameTickPacket) -> SimpleControllerState: + cur_time = packet.game_info.seconds_elapsed + delta = cur_time - self.prev_time + self.prev_time = cur_time + + ticks_elapsed = round(delta * 120) + self.ticks += ticks_elapsed + self.game_state.decode(packet, ticks_elapsed) + + if self.update_action: + self.update_action = False + + # FIXME Hey, botmaker. Verify that this is what you need for your agent + # By default we treat every match as a 1v1 against a fixed opponent, + # by doing this your bot can participate in 2v2 or 3v3 matches. Feel free to change this + player = self.game_state.players[self.index] + + obs = self.obs_builder.build_obs(player, self.game_state, self.action) + self.action = self.act_parser.parse_actions( + np.array(self.agent.act(obs)), self.game_state + )[ + 0 + ] # Dim is (N, 8) + + if self.ticks >= self.tick_skip - 1: + self.update_controls(self.action) + + if self.ticks >= self.tick_skip: + self.ticks = 0 + self.update_action = True + + return self.controls + + def update_controls(self, action): + self.controls.throttle = action[0] + self.controls.steer = action[1] + self.controls.pitch = action[2] + self.controls.yaw = 0 if action[5] > 0 else action[3] + self.controls.roll = action[4] + self.controls.jump = action[5] > 0 + self.controls.boost = action[6] > 0 + self.controls.handbrake = action[7] > 0 diff --git a/rlgym training/bot/src/obs/advanced_obs.py b/rlgym training/bot/src/obs/advanced_obs.py new file mode 100644 index 0000000..47ab6b1 --- /dev/null +++ b/rlgym training/bot/src/obs/advanced_obs.py @@ -0,0 +1,83 @@ +import math +import numpy as np +from typing import Any, List +from rlgym_compat import common_values +from rlgym_compat import PlayerData, GameState, PhysicsObject + + +class AdvancedObs: + POS_STD = 2300 + ANG_STD = math.pi + + def __init__(self): + super().__init__() + + def reset(self, initial_state: GameState): + pass + + def build_obs(self, player: PlayerData, state: GameState, previous_action: np.ndarray) -> Any: + + if player.team_num == common_values.ORANGE_TEAM: + inverted = True + ball = state.inverted_ball + pads = state.inverted_boost_pads + else: + inverted = False + ball = state.ball + pads = state.boost_pads + + obs = [ball.position / self.POS_STD, + ball.linear_velocity / self.POS_STD, + ball.angular_velocity / self.ANG_STD, + previous_action, + pads] + + player_car = self._add_player_to_obs(obs, player, ball, inverted) + + allies = [] + enemies = [] + + for other in state.players: + if other.car_id == player.car_id: + continue + + if other.team_num == player.team_num: + team_obs = allies + else: + team_obs = enemies + + other_car = self._add_player_to_obs(team_obs, other, ball, inverted) + + # Extra info + team_obs.extend([ + (other_car.position - player_car.position) / self.POS_STD, + (other_car.linear_velocity - player_car.linear_velocity) / self.POS_STD + ]) + + obs.extend(allies) + obs.extend(enemies) + return np.concatenate(obs) + + def _add_player_to_obs(self, obs: List, player: PlayerData, ball: PhysicsObject, inverted: bool): + if inverted: + player_car = player.inverted_car_data + else: + player_car = player.car_data + + rel_pos = ball.position - player_car.position + rel_vel = ball.linear_velocity - player_car.linear_velocity + + obs.extend([ + rel_pos / self.POS_STD, + rel_vel / self.POS_STD, + player_car.position / self.POS_STD, + player_car.forward(), + player_car.up(), + player_car.linear_velocity / self.POS_STD, + player_car.angular_velocity / self.ANG_STD, + [player.boost_amount, + int(player.on_ground), + int(player.has_flip), + int(player.is_demoed)]]) + + return player_car diff --git a/rlgym training/bot/src/obs/default_obs.py b/rlgym training/bot/src/obs/default_obs.py new file mode 100644 index 0000000..dfb83f4 --- /dev/null +++ b/rlgym training/bot/src/obs/default_obs.py @@ -0,0 +1,78 @@ +import math +import numpy as np +from typing import Any, List +from rlgym_compat import common_values +from rlgym_compat import PlayerData, GameState + + +class DefaultObs: + def __init__(self, pos_coef=1/2300, ang_coef=1/math.pi, lin_vel_coef=1/2300, ang_vel_coef=1/math.pi): + """ + :param pos_coef: Position normalization coefficient + :param ang_coef: Rotation angle normalization coefficient + :param lin_vel_coef: Linear velocity normalization coefficient + :param ang_vel_coef: Angular velocity normalization coefficient + """ + super().__init__() + self.POS_COEF = pos_coef + self.ANG_COEF = ang_coef + self.LIN_VEL_COEF = lin_vel_coef + self.ANG_VEL_COEF = ang_vel_coef + + def reset(self, initial_state: GameState): + pass + + def build_obs(self, player: PlayerData, state: GameState, previous_action: np.ndarray) -> Any: + if player.team_num == common_values.ORANGE_TEAM: + inverted = True + ball = state.inverted_ball + pads = state.inverted_boost_pads + else: + inverted = False + ball = state.ball + pads = state.boost_pads + + obs = [ball.position * self.POS_COEF, + ball.linear_velocity * self.LIN_VEL_COEF, + ball.angular_velocity * self.ANG_VEL_COEF, + previous_action, + pads] + + self._add_player_to_obs(obs, player, inverted) + + allies = [] + enemies = [] + + for other in state.players: + if other.car_id == player.car_id: + continue + + if other.team_num == player.team_num: + team_obs = allies + else: + team_obs = enemies + + self._add_player_to_obs(team_obs, other, inverted) + + obs.extend(allies) + obs.extend(enemies) + return np.concatenate(obs) + + def _add_player_to_obs(self, obs: List, player: PlayerData, inverted: bool): + if inverted: + player_car = player.inverted_car_data + else: + player_car = player.car_data + + obs.extend([ + player_car.position * self.POS_COEF, + player_car.forward(), + player_car.up(), + player_car.linear_velocity * self.LIN_VEL_COEF, + player_car.angular_velocity * self.ANG_VEL_COEF, + [player.boost_amount, + int(player.on_ground), + int(player.has_flip), + int(player.is_demoed)]]) + + return player_car diff --git a/rlgym training/bot/src/requirements.txt b/rlgym training/bot/src/requirements.txt new file mode 100644 index 0000000..3489cd8 --- /dev/null +++ b/rlgym training/bot/src/requirements.txt @@ -0,0 +1,90 @@ +appdirs==1.4.4 +attrs==23.1.0 +bottle==0.12.25 +bottle-websocket==0.2.9 +certifi==2022.12.7 +cffi==1.16.0 +charset-normalizer==2.1.1 +click==8.1.7 +cloudpickle==3.0.0 +colorama==0.4.6 +comtypes==1.2.0 +dataclasses==0.6 +docker-pycreds==0.4.0 +docopt==0.6.2 +Eel==0.16.0 +exceptiongroup==1.1.3 +filelock==3.9.0 +flatbuffers==1.12 +fsspec==2023.4.0 +future==0.18.3 +gevent==23.9.1 +gevent-websocket==0.10.1 +gitdb==4.0.11 +GitPython==3.1.40 +greenlet==3.0.0 +gym==0.26.2 +gym-notices==0.0.8 +h11==0.14.0 +idna==3.4 +importlib-metadata==6.8.0 +inputs==0.5 +Jinja2==3.1.2 +llvmlite==0.38.1 +MarkupSafe==2.1.2 +mpmath==1.3.0 +networkx==3.0 +numba==0.55.1 +numpy==1.21.6 +outcome==1.3.0 +packaging==23.2 +pathtools==0.1.2 +Pillow==9.3.0 +protobuf==4.24.4 +psutil==5.9.6 +pycparser==2.21 +pyparsing==3.1.1 +PyQt5==5.15.10 +PyQt5-Qt5==5.15.2 +PyQt5-sip==12.13.0 +PySocks==1.7.1 +python-dotenv==1.0.0 +pywin32==228 +pywinauto==0.6.8 +PyYAML==6.0.1 +requests==2.28.1 +requirements-parser==0.5.0 +rlbot==1.67.7 +rlbot-gui==0.0.154 +rlgym==1.2.2 +rlgym-compat==1.1.0 +rlgym-ppo==1.2.5 +rlgym-sim==1.2.5 +rlgym-tools==1.8.2 +RLUtilities==0.0.13 +RocketSim==1.2.0 +scipy==1.11.3 +selenium==4.14.0 +sentry-sdk==1.32.0 +setproctitle==1.3.3 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.0 +sortedcontainers==2.4.0 +sympy==1.12 +torch==2.1.0+cu118 +torchaudio==2.1.0+cu118 +torchvision==0.16.0+cu118 +trio==0.22.2 +trio-websocket==0.11.1 +types-setuptools==68.2.0.0 +typing-extensions==4.8.0 +urllib3==1.26.13 +wandb==0.15.12 +webdriver-manager==4.0.1 +websockets==12.0 +whichcraft==0.6.1 +wsproto==1.2.0 +zipp==3.17.0 +zope.event==5.0 +zope.interface==6.1