mirror of
https://github.com/jcreek/Sarpy.git
synced 2026-07-12 18:53:44 +00:00
feat(*): Update bot to work in all game modes with reasonable rewards
This commit is contained in:
Josh Creek
+4
-25
@@ -11,6 +11,8 @@ from rlgym.utils.obs_builders import AdvancedObs
|
||||
from rlgym_compat import GameState
|
||||
from rlgym.utils.action_parsers import DiscreteAction
|
||||
|
||||
from rlgym_tools.extra_obs.advanced_padder import AdvancedObsPadder
|
||||
|
||||
|
||||
class Sarpy(BaseAgent):
|
||||
def __init__(self, name, team, index):
|
||||
@@ -18,7 +20,7 @@ class Sarpy(BaseAgent):
|
||||
|
||||
# FIXME Hey, botmaker. Start here:
|
||||
# Swap the obs builder if you are using a different one, RLGym's AdvancedObs is also available
|
||||
self.obs_builder = AdvancedObs()
|
||||
self.obs_builder = AdvancedObsPadder(team_size=3)
|
||||
# Swap the action parser if you are using a different one, RLGym's Discrete and Continuous are also available
|
||||
self.act_parser = DiscreteAction()
|
||||
# Your neural network logic goes inside the Agent class, go take a look inside src/agent.py
|
||||
@@ -59,33 +61,10 @@ class Sarpy(BaseAgent):
|
||||
# By default we treat every match as a 1v1 against a fixed opponent,
|
||||
# by doing this your bot can participate in 2v2 or 3v3 matches. Feel free to change this
|
||||
player = self.game_state.players[self.index]
|
||||
teammates = [p for p in self.game_state.players if p.team_num == self.team]
|
||||
opponents = [p for p in self.game_state.players if p.team_num != self.team]
|
||||
|
||||
if len(opponents) == 0:
|
||||
# There's no opponent, we assume this model is 1v0
|
||||
self.game_state.players = [player]
|
||||
else:
|
||||
# Sort by distance to ball
|
||||
teammates.sort(
|
||||
key=lambda p: np.linalg.norm(
|
||||
self.game_state.ball.position - p.car_data.position
|
||||
)
|
||||
)
|
||||
opponents.sort(
|
||||
key=lambda p: np.linalg.norm(
|
||||
self.game_state.ball.position - p.car_data.position
|
||||
)
|
||||
)
|
||||
|
||||
# Grab opponent in same "position" relative to it's teammates
|
||||
opponent = opponents[min(teammates.index(player), len(opponents) - 1)]
|
||||
|
||||
self.game_state.players = [player, opponent]
|
||||
|
||||
obs = self.obs_builder.build_obs(player, self.game_state, self.action)
|
||||
self.action = self.act_parser.parse_actions(
|
||||
self.agent.act(obs), self.game_state
|
||||
np.array(self.agent.act(obs)), self.game_state
|
||||
)[
|
||||
0
|
||||
] # Dim is (N, 8)
|
||||
|
||||
+66
-48
@@ -12,8 +12,14 @@ from rlgym.envs import Match
|
||||
from rlgym.utils.action_parsers import DiscreteAction
|
||||
from rlgym.utils.reward_functions.common_rewards import (
|
||||
VelocityBallToGoalReward,
|
||||
LiuDistanceBallToGoalReward,
|
||||
BallYCoordinateReward,
|
||||
EventReward,
|
||||
RewardIfClosestToBall,
|
||||
LiuDistancePlayerToBallReward,
|
||||
FaceBallReward,
|
||||
TouchBallReward,
|
||||
AlignBallGoal,
|
||||
)
|
||||
from rlgym.utils.obs_builders import AdvancedObs
|
||||
from rlgym.utils.state_setters import DefaultState
|
||||
@@ -31,11 +37,8 @@ from rlgym.utils.reward_functions.common_rewards.ball_goal_rewards import (
|
||||
from rlgym.utils.reward_functions import CombinedReward
|
||||
|
||||
from rlgym_tools.sb3_utils import SB3MultipleInstanceEnv
|
||||
|
||||
|
||||
# from observationBuilders.CustomObsBuilderBluePerspective import (
|
||||
# CustomObsBuilderBluePerspective,
|
||||
# )
|
||||
from rlgym_tools.extra_rewards.kickoff_reward import KickoffReward
|
||||
from rlgym_tools.extra_obs.advanced_padder import AdvancedObsPadder
|
||||
|
||||
if __name__ == "__main__": # Required for multiprocessing
|
||||
# Set up the folders
|
||||
@@ -53,23 +56,22 @@ if __name__ == "__main__": # Required for multiprocessing
|
||||
|
||||
fps = 120 / frame_skip
|
||||
gamma = np.exp(np.log(0.5) / (fps * half_life_seconds)) # Quick mafs
|
||||
agents_per_match = 2
|
||||
num_instances = 10
|
||||
target_steps = 100_000
|
||||
steps = target_steps // (num_instances * agents_per_match)
|
||||
batch_size = steps
|
||||
print(f"fps={fps}, gamma={gamma})")
|
||||
|
||||
def exit_save(model):
|
||||
model.save("models/exit_save")
|
||||
|
||||
team_size = 3
|
||||
|
||||
def get_match(): # Need to use a function so that each instance can call it and produce their own objects
|
||||
return Match(
|
||||
team_size=1,
|
||||
team_size=team_size,
|
||||
tick_skip=frame_skip,
|
||||
reward_function=CombinedReward(
|
||||
(
|
||||
VelocityPlayerToBallReward(),
|
||||
VelocityBallToGoalReward(),
|
||||
LiuDistanceBallToGoalReward(),
|
||||
EventReward(
|
||||
team_goal=100.0,
|
||||
concede=-100.0,
|
||||
@@ -78,70 +80,86 @@ if __name__ == "__main__": # Required for multiprocessing
|
||||
demo=10.0,
|
||||
),
|
||||
BallYCoordinateReward(),
|
||||
RewardIfClosestToBall(LiuDistancePlayerToBallReward()),
|
||||
LiuDistancePlayerToBallReward(),
|
||||
FaceBallReward(),
|
||||
TouchBallReward(),
|
||||
AlignBallGoal(),
|
||||
KickoffReward(),
|
||||
),
|
||||
(0.1, 1.0, 1.0, 0.1),
|
||||
(0.1, 1.0, 1.0, 1.0, 0.1, 0.2, 0.1, 0.2, 1.0, 0.1, 10.0),
|
||||
),
|
||||
# self_play=True,
|
||||
spawn_opponents=True,
|
||||
terminal_conditions=[
|
||||
TimeoutCondition(fps * 300),
|
||||
NoTouchTimeoutCondition(fps * 20),
|
||||
TimeoutCondition(fps * 30),
|
||||
NoTouchTimeoutCondition(fps * 10),
|
||||
GoalScoredCondition(),
|
||||
],
|
||||
obs_builder=AdvancedObs(), # Not that advanced, good default
|
||||
obs_builder=AdvancedObsPadder(
|
||||
team_size=team_size
|
||||
), # Not that advanced, good default
|
||||
state_setter=DefaultState(), # Resets to kickoff position
|
||||
action_parser=DiscreteAction(), # Discrete > Continuous don't @ me
|
||||
)
|
||||
|
||||
# Generate the environment (the Rocket League game used by RL Gym)
|
||||
env = SB3MultipleInstanceEnv(
|
||||
get_match, num_instances
|
||||
) # Start x instances, waiting 60 seconds between each
|
||||
get_match, 10
|
||||
) # Start 10 instances, waiting 60 seconds between each
|
||||
env = VecCheckNan(env) # Optional
|
||||
env = VecMonitor(env) # Recommended, logs mean reward and ep_len to Tensorboard
|
||||
env = VecNormalize(
|
||||
env, norm_obs=False, gamma=gamma
|
||||
) # Highly recommended, normalizes rewards
|
||||
|
||||
# Load the model that was last trained, or start a new one if the zip file doesn't exist
|
||||
try:
|
||||
model = PPO.load(
|
||||
f"{models_dir}/exit_save.zip",
|
||||
env,
|
||||
device="auto", # Need to set device again (if using a specific one)
|
||||
)
|
||||
print("Loaded exit_save.zip model")
|
||||
except:
|
||||
model = PPO(
|
||||
MlpPolicy,
|
||||
env,
|
||||
learning_rate=5e-5, # Around this is fairly common for PPO
|
||||
n_steps=steps, # Number of steps to perform before optimizing network
|
||||
batch_size=batch_size, # Batch size as high as possible within reason
|
||||
n_epochs=1, # PPO calls for multiple epochs
|
||||
gamma=gamma, # Gamma as calculated using half-life
|
||||
ent_coef=0.01, # From PPO Atari
|
||||
vf_coef=1.0, # From PPO Atari
|
||||
tensorboard_log=log_dir, # `tensorboard --logdir out/logs` in terminal to see graphs
|
||||
verbose=3, # Print out all the info as we're going
|
||||
device="auto", # Uses GPU if available
|
||||
)
|
||||
print("Created new model")
|
||||
|
||||
# Save model every so often
|
||||
# Divide by num_envs (number of agents) because callback only increments every time all agents have taken a step
|
||||
# This saves to specified folder with a specified name
|
||||
callback = CheckpointCallback(
|
||||
round(5_000_000 / env.num_envs), save_path=models_dir, name_prefix="rl_model"
|
||||
round(1_000_000 / env.num_envs), save_path=models_dir, name_prefix="sarpy_model"
|
||||
)
|
||||
|
||||
atexit.register(exit_save, model)
|
||||
|
||||
try:
|
||||
while True:
|
||||
# Load the model that was last trained, or start a new one if the zip file doesn't exist
|
||||
try:
|
||||
# Now, if one wants to load a trained model from a checkpoint, use this function
|
||||
# This will contain all the attributes of the original model
|
||||
# Any attribute can be overwritten by using the custom_objects parameter,
|
||||
# which includes n_envs (number of agents), which has to be overwritten to use a different amount
|
||||
model = PPO.load(
|
||||
f"{models_dir}/exit_save.zip",
|
||||
env,
|
||||
custom_objects=dict(
|
||||
n_envs=env.num_envs, _last_obs=None
|
||||
), # Need this to change number of agents
|
||||
device="auto", # Need to set device again (if using a specific one)
|
||||
force_reset=True, # Make SB3 reset the env so it doesn't think we're continuing from last state
|
||||
)
|
||||
print("Loaded exit_save.zip model")
|
||||
except:
|
||||
model = PPO(
|
||||
MlpPolicy,
|
||||
env,
|
||||
n_epochs=32, # PPO calls for multiple epochs
|
||||
learning_rate=1e-5, # Around this is fairly common for PPO
|
||||
ent_coef=0.01, # From PPO Atari
|
||||
vf_coef=1.0, # From PPO Atari
|
||||
gamma=gamma, # Gamma as calculated using half-life
|
||||
verbose=3, # Print out all the info as we're going
|
||||
batch_size=4096, # Batch size as high as possible within reason
|
||||
n_steps=4096, # Number of steps to perform before optimizing network
|
||||
tensorboard_log="out/logs", # `tensorboard --logdir out/logs` in terminal to see graphs
|
||||
device="auto", # Uses GPU if available
|
||||
)
|
||||
print("Created new model")
|
||||
|
||||
atexit.register(exit_save, model)
|
||||
|
||||
print("Learning...")
|
||||
model.learn(25_000_000, callback=callback)
|
||||
model.learn(100_000_000, callback=callback, reset_num_timesteps=False)
|
||||
model.save(f"{models_dir}/exit_save")
|
||||
print("Saved exit_save model")
|
||||
model.save(f"mmr_models/{model.num_timesteps}")
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
Reference in New Issue
Block a user