feat(*): Add self-play RL training pipeline with PPO trainer, in-game GDScript policy inference, and bot opponent support in Match mode

Game/addons/godot_rl_agents/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Edward Beeching
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE. 

Game/addons/godot_rl_agents/VENDORED.md

@@ -0,0 +1,3 @@
+Vendored from https://github.com/edbeeching/godot_rl_agents_plugin
+commit 998c357a0cd09b37f40a36d70c7867fc9f682338 (2026-06-13), MIT license (see LICENSE).
+The onnx/csharp files are unused (require the .NET Godot build); in-game inference uses scripts/policy_network.gd instead.

Game/addons/godot_rl_agents/controller/ai_controller_2d.gd

@@ -0,0 +1,136 @@
+extends Node2D
+class_name AIController2D
+
+enum ControlModes {
+	INHERIT_FROM_SYNC, ## Inherit setting from sync node
+	HUMAN, ## Test the environment manually
+	TRAINING, ## Train a model
+	ONNX_INFERENCE, ## Load a pretrained model using an .onnx file
+	RECORD_EXPERT_DEMOS ## Record observations and actions for expert demonstrations
+}
+@export var control_mode: ControlModes = ControlModes.INHERIT_FROM_SYNC
+## The path to a trained .onnx model file to use for inference (overrides the path set in sync node).
+@export var onnx_model_path := ""
+## Once the number of steps has passed, the flag 'needs_reset' will be set to 'true' for this instance.
+@export var reset_after := 1000
+
+@export_group("Record expert demos mode options")
+## Path where the demos will be saved. The file can later be used for imitation learning.
+@export var expert_demo_save_path: String
+## The action that erases the last recorded episode from the currently recorded data.
+@export var remove_last_episode_key: InputEvent
+## Action will be repeated for n frames. Will introduce control lag if larger than 1.
+## Can be used to ensure that action_repeat on inference and training matches
+## the recorded demonstrations.
+@export var action_repeat: int = 1
+
+@export_group("Multi-policy mode options")
+## Allows you to set certain agents to use different policies.
+## Changing has no effect with default SB3 training. Works with Rllib example.
+## Tutorial: https://github.com/edbeeching/godot_rl_agents/blob/main/docs/TRAINING_MULTIPLE_POLICIES.md
+@export var policy_name: String = "shared_policy"
+
+var onnx_model: ONNXModel
+
+var heuristic := "human"
+var done := false
+var reward := 0.0
+var n_steps := 0
+var needs_reset := false
+
+var _player: Node2D
+
+
+func _ready():
+	add_to_group("AGENT")
+
+
+func init(player: Node2D):
+	_player = player
+
+
+#region Methods that need implementing using the "extend script" option in Godot
+func get_obs() -> Dictionary:
+	assert(false, "the get_obs method is not implemented when extending from ai_controller")
+	return {"obs": []}
+
+
+func get_reward() -> float:
+	assert(false, "the get_reward method is not implemented when extending from ai_controller")
+	return 0.0
+
+
+func get_action_space() -> Dictionary:
+	assert(
+		false, "the get_action_space method is not implemented when extending from ai_controller"
+	)
+	return {
+		"example_actions_continous": {"size": 2, "action_type": "continuous"},
+		"example_actions_discrete": {"size": 2, "action_type": "discrete"},
+	}
+
+
+func set_action(action) -> void:
+	assert(false, "the set_action method is not implemented when extending from ai_controller")
+
+
+#endregion
+
+
+#region Methods that sometimes need implementing using the "extend script" option in Godot
+# Only needed if you are recording expert demos with this AIController
+func get_action() -> Array:
+	assert(
+		false,
+		"the get_action method is not implemented in extended AIController but demo_recorder is used"
+	)
+	return []
+
+
+# For providing additional info (e.g. `is_success` for SB3 training)
+func get_info() -> Dictionary:
+	return {}
+
+
+#endregion
+
+
+func _physics_process(delta):
+	n_steps += 1
+	if n_steps > reset_after:
+		needs_reset = true
+
+
+func get_obs_space():
+	# may need overriding if the obs space is complex
+	var obs = get_obs()
+	return {
+		"obs": {"size": [len(obs["obs"])], "space": "box"},
+	}
+
+
+func reset():
+	n_steps = 0
+	needs_reset = false
+
+
+func reset_if_done():
+	if done:
+		reset()
+
+
+func set_heuristic(h):
+	# sets the heuristic from "human" or "model" nothing to change here
+	heuristic = h
+
+
+func get_done():
+	return done
+
+
+func set_done_false():
+	done = false
+
+
+func zero_reward():
+	reward = 0.0

Game/addons/godot_rl_agents/controller/ai_controller_2d.gd.uid

@@ -0,0 +1 @@
+uid://dgfbh3y7s07g7

Game/addons/godot_rl_agents/controller/ai_controller_3d.gd

@@ -0,0 +1,136 @@
+extends Node3D
+class_name AIController3D
+
+enum ControlModes {
+	INHERIT_FROM_SYNC, ## Inherit setting from sync node
+	HUMAN, ## Test the environment manually
+	TRAINING, ## Train a model
+	ONNX_INFERENCE, ## Load a pretrained model using an .onnx file
+	RECORD_EXPERT_DEMOS ## Record observations and actions for expert demonstrations
+}
+@export var control_mode: ControlModes = ControlModes.INHERIT_FROM_SYNC
+## The path to a trained .onnx model file to use for inference (overrides the path set in sync node).
+@export var onnx_model_path := ""
+## Once the number of steps has passed, the flag 'needs_reset' will be set to 'true' for this instance.
+@export var reset_after := 1000
+
+@export_group("Record expert demos mode options")
+## Path where the demos will be saved. The file can later be used for imitation learning.
+@export var expert_demo_save_path: String
+## The action that erases the last recorded episode from the currently recorded data.
+@export var remove_last_episode_key: InputEvent
+## Action will be repeated for n frames. Will introduce control lag if larger than 1.
+## Can be used to ensure that action_repeat on inference and training matches
+## the recorded demonstrations.
+@export var action_repeat: int = 1
+
+@export_group("Multi-policy mode options")
+## Allows you to set certain agents to use different policies.
+## Changing has no effect with default SB3 training. Works with Rllib example.
+## Tutorial: https://github.com/edbeeching/godot_rl_agents/blob/main/docs/TRAINING_MULTIPLE_POLICIES.md
+@export var policy_name: String = "shared_policy"
+
+var onnx_model: ONNXModel
+
+var heuristic := "human"
+var done := false
+var reward := 0.0
+var n_steps := 0
+var needs_reset := false
+
+var _player: Node3D
+
+
+func _ready():
+	add_to_group("AGENT")
+
+
+func init(player: Node3D):
+	_player = player
+
+
+#region Methods that need implementing using the "extend script" option in Godot
+func get_obs() -> Dictionary:
+	assert(false, "the get_obs method is not implemented when extending from ai_controller")
+	return {"obs": []}
+
+
+func get_reward() -> float:
+	assert(false, "the get_reward method is not implemented when extending from ai_controller")
+	return 0.0
+
+
+func get_action_space() -> Dictionary:
+	assert(
+		false, "the get_action_space method is not implemented when extending from ai_controller"
+	)
+	return {
+		"example_actions_continous": {"size": 2, "action_type": "continuous"},
+		"example_actions_discrete": {"size": 2, "action_type": "discrete"},
+	}
+
+
+func set_action(action) -> void:
+	assert(false, "the set_action method is not implemented when extending from ai_controller")
+
+
+#endregion
+
+
+#region Methods that sometimes need implementing using the "extend script" option in Godot
+# Only needed if you are recording expert demos with this AIController
+func get_action() -> Array:
+	assert(
+		false,
+		"the get_action method is not implemented in extended AIController but demo_recorder is used"
+	)
+	return []
+
+
+# For providing additional info (e.g. `is_success` for SB3 training)
+func get_info() -> Dictionary:
+	return {}
+
+
+#endregion
+
+
+func _physics_process(delta):
+	n_steps += 1
+	if n_steps > reset_after:
+		needs_reset = true
+
+
+func get_obs_space():
+	# may need overriding if the obs space is complex
+	var obs = get_obs()
+	return {
+		"obs": {"size": [len(obs["obs"])], "space": "box"},
+	}
+
+
+func reset():
+	n_steps = 0
+	needs_reset = false
+
+
+func reset_if_done():
+	if done:
+		reset()
+
+
+func set_heuristic(h):
+	# sets the heuristic from "human" or "model" nothing to change here
+	heuristic = h
+
+
+func get_done():
+	return done
+
+
+func set_done_false():
+	done = false
+
+
+func zero_reward():
+	reward = 0.0

Game/addons/godot_rl_agents/controller/ai_controller_3d.gd.uid

@@ -0,0 +1 @@
+uid://qm8wpg7ccydk

Game/addons/godot_rl_agents/godot_rl_agents.gd

@@ -0,0 +1,16 @@
+@tool
+extends EditorPlugin
+
+
+func _enter_tree():
+	# Initialization of the plugin goes here.
+	# Add the new type with a name, a parent type, a script and an icon.
+	add_custom_type("Sync", "Node", preload("sync.gd"), preload("icon.png"))
+	#add_custom_type("RaycastSensor2D2", "Node", preload("raycast_sensor_2d.gd"), preload("icon.png"))
+
+
+func _exit_tree():
+	# Clean-up of the plugin goes here.
+	# Always remember to remove it from the engine when deactivated.
+	remove_custom_type("Sync")
+	#remove_custom_type("RaycastSensor2D2")

Game/addons/godot_rl_agents/godot_rl_agents.gd.uid

@@ -0,0 +1 @@
+uid://bdnkavfpf6ge6

Game/addons/godot_rl_agents/icon.png.import

@@ -0,0 +1,40 @@
+[remap]
+
+importer="texture"
+type="CompressedTexture2D"
+uid="uid://bxy3je5atsh68"
+path="res://.godot/imported/icon.png-45a871b53434e556222f5901d598ab34.ctex"
+metadata={
+"vram_texture": false
+}
+
+[deps]
+
+source_file="res://addons/godot_rl_agents/icon.png"
+dest_files=["res://.godot/imported/icon.png-45a871b53434e556222f5901d598ab34.ctex"]
+
+[params]
+
+compress/mode=0
+compress/high_quality=false
+compress/lossy_quality=0.7
+compress/uastc_level=0
+compress/rdo_quality_loss=0.0
+compress/hdr_compression=1
+compress/normal_map=0
+compress/channel_pack=0
+mipmaps/generate=false
+mipmaps/limit=-1
+roughness/mode=0
+roughness/src_normal=""
+process/channel_remap/red=0
+process/channel_remap/green=1
+process/channel_remap/blue=2
+process/channel_remap/alpha=3
+process/fix_alpha_border=true
+process/premult_alpha=false
+process/normal_map_invert_y=false
+process/hdr_as_srgb=false
+process/hdr_clamp_exposure=false
+process/size_limit=0
+detect_3d/compress_to=1

Game/addons/godot_rl_agents/onnx/csharp/ONNXInference.cs

@@ -0,0 +1,115 @@
+using Godot;
+using Microsoft.ML.OnnxRuntime;
+using Microsoft.ML.OnnxRuntime.Tensors;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace GodotONNX
+{
+	/// <include file='docs/ONNXInference.xml' path='docs/members[@name="ONNXInference"]/ONNXInference/*'/>
+	public partial class ONNXInference : GodotObject
+	{
+
+		private InferenceSession session;
+		/// <summary>
+		/// Path to the ONNX model. Use Initialize to change it. 
+		/// </summary>
+		private string modelPath;
+		private int batchSize;
+
+		private SessionOptions SessionOpt;
+
+		/// <summary>
+		/// init function
+		/// </summary>
+		/// <param name="Path"></param>
+		/// <param name="BatchSize"></param>
+		/// <returns>Returns the output size of the model</returns>
+		public int Initialize(string Path, int BatchSize)
+		{
+			modelPath = Path;
+			batchSize = BatchSize;
+			SessionOpt = SessionConfigurator.MakeConfiguredSessionOptions();
+			session = LoadModel(modelPath);
+			return session.OutputMetadata["output"].Dimensions[1];
+		}
+
+
+		/// <include file='docs/ONNXInference.xml' path='docs/members[@name="ONNXInference"]/Run/*'/>
+		public Godot.Collections.Dictionary<string, Godot.Collections.Array<float>> RunInference(Godot.Collections.Dictionary<string, Godot.Collections.Array<float>> obs, int state_ins)
+		{
+			//Current model: Any (Godot Rl Agents)
+			//Expects a tensor of shape [batch_size, input_size] type float for any output of the agents observation dictionary and a tensor of shape [batch_size] type float named state_ins
+
+			var modelInputsList = new List<NamedOnnxValue>
+			{
+				NamedOnnxValue.CreateFromTensor("state_ins", new DenseTensor<float>(new float[] { state_ins }, new int[] { batchSize }))
+			};
+			foreach (var key in obs.Keys)
+			{
+				var subObs = obs[key];
+				// Fill the input tensors for each key of the observation
+				// create span of observation from specific inputSize
+				var obsData = new float[subObs.Count]; //There's probably a better way to do this
+				for (int i = 0; i < subObs.Count; i++)
+				{
+					obsData[i] = subObs[i];
+				}
+				modelInputsList.Add(
+					NamedOnnxValue.CreateFromTensor(key, new DenseTensor<float>(obsData, new int[] { batchSize, subObs.Count }))
+				);
+			}
+
+			IReadOnlyCollection<string> outputNames = new List<string> { "output", "state_outs" }; //ONNX is sensible to these names, as well as the input names
+
+			IDisposableReadOnlyCollection<DisposableNamedOnnxValue> results;
+			//We do not use "using" here so we get a better exception explaination later
+			try
+			{
+				results = session.Run(modelInputsList, outputNames);
+			}
+			catch (OnnxRuntimeException e)
+			{
+				//This error usually means that the model is not compatible with the input, beacause of the input shape (size)
+				GD.Print("Error at inference: ", e);
+				return null;
+			}
+			//Can't convert IEnumerable<float> to Variant, so we have to convert it to an array or something
+			Godot.Collections.Dictionary<string, Godot.Collections.Array<float>> output = new Godot.Collections.Dictionary<string, Godot.Collections.Array<float>>();
+			DisposableNamedOnnxValue output1 = results.First();
+			DisposableNamedOnnxValue output2 = results.Last();
+			Godot.Collections.Array<float> output1Array = new Godot.Collections.Array<float>();
+			Godot.Collections.Array<float> output2Array = new Godot.Collections.Array<float>();
+
+			foreach (float f in output1.AsEnumerable<float>())
+			{
+				output1Array.Add(f);
+			}
+
+			foreach (float f in output2.AsEnumerable<float>())
+			{
+				output2Array.Add(f);
+			}
+
+			output.Add(output1.Name, output1Array);
+			output.Add(output2.Name, output2Array);
+
+			//Output is a dictionary of arrays, ex: { "output" : [0.1, 0.2, 0.3, 0.4, ...], "state_outs" : [0.5, ...]}
+			results.Dispose();
+			return output;
+		}
+		/// <include file='docs/ONNXInference.xml' path='docs/members[@name="ONNXInference"]/Load/*'/>
+		public InferenceSession LoadModel(string Path)
+		{
+			using Godot.FileAccess file = FileAccess.Open(Path, Godot.FileAccess.ModeFlags.Read);
+			byte[] model = file.GetBuffer((int)file.GetLength());
+			//file.Close(); file.Dispose(); //Close the file, then dispose the reference.
+			return new InferenceSession(model, SessionOpt); //Load the model
+		}
+		public void FreeDisposables()
+		{
+			session.Dispose();
+			SessionOpt.Dispose();
+		}
+	}
+}

Game/addons/godot_rl_agents/onnx/csharp/SessionConfigurator.cs

@@ -0,0 +1,131 @@
+using Godot;
+using Microsoft.ML.OnnxRuntime;
+
+namespace GodotONNX
+{
+	/// <include file='docs/SessionConfigurator.xml' path='docs/members[@name="SessionConfigurator"]/SessionConfigurator/*'/>
+
+	public static class SessionConfigurator
+	{
+		public enum ComputeName
+		{
+			CUDA,
+			ROCm,
+			DirectML,
+			CoreML,
+			CPU
+		}
+
+        /// <include file='docs/SessionConfigurator.xml' path='docs/members[@name="SessionConfigurator"]/GetSessionOptions/*'/>
+        public static SessionOptions MakeConfiguredSessionOptions()
+        {
+            SessionOptions sessionOptions = new();
+            SetOptions(sessionOptions);
+            return sessionOptions;
+        }
+
+        private static void SetOptions(SessionOptions sessionOptions)
+        {
+            sessionOptions.LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_WARNING;
+            ApplySystemSpecificOptions(sessionOptions);
+        }
+
+        /// <include file='docs/SessionConfigurator.xml' path='docs/members[@name="SessionConfigurator"]/SystemCheck/*'/>
+        static public void ApplySystemSpecificOptions(SessionOptions sessionOptions)
+        {
+            //Most code for this function is verbose only, the only reason it exists is to track
+            //implementation progress of the different compute APIs.
+
+            //December 2022: CUDA is not working. 
+
+            string OSName = OS.GetName(); //Get OS Name
+
+            //ComputeName ComputeAPI = ComputeCheck(); //Get Compute API
+            //                                         //TODO: Get CPU architecture
+
+            //Linux can use OpenVINO (C#) on x64 and ROCm on x86 (GDNative/C++)
+            //Windows can use OpenVINO (C#) on x64
+            //TODO: try TensorRT instead of CUDA
+            //TODO: Use OpenVINO for Intel Graphics
+
+            // Temporarily using CPU on all platforms to avoid errors detected with DML
+            ComputeName ComputeAPI = ComputeName.CPU;
+
+            //match OS and Compute API
+            GD.Print($"OS: {OSName} Compute API: {ComputeAPI}");
+
+            // CPU is set by default without appending necessary
+            // sessionOptions.AppendExecutionProvider_CPU(0);
+
+            /*
+            switch (OSName)
+            {
+                case "Windows": //Can use CUDA, DirectML
+                    if (ComputeAPI is ComputeName.CUDA)
+                    {
+                        //CUDA 
+                        //sessionOptions.AppendExecutionProvider_CUDA(0);
+                        //sessionOptions.AppendExecutionProvider_DML(0);
+                    }
+                    else if (ComputeAPI is ComputeName.DirectML)
+                    {
+                        //DirectML
+                        //sessionOptions.AppendExecutionProvider_DML(0);
+                    }
+                    break;
+                case "X11": //Can use CUDA, ROCm
+                    if (ComputeAPI is ComputeName.CUDA)
+                    {
+                        //CUDA
+                        //sessionOptions.AppendExecutionProvider_CUDA(0);
+                    }
+                    if (ComputeAPI is ComputeName.ROCm)
+                    {
+                        //ROCm, only works on x86 
+                        //Research indicates that this has to be compiled as a GDNative plugin
+                        //GD.Print("ROCm not supported yet, using CPU.");
+                        //sessionOptions.AppendExecutionProvider_CPU(0);
+                    }
+                    break;
+                case "macOS": //Can use CoreML
+                    if (ComputeAPI is ComputeName.CoreML)
+                    { //CoreML
+                      //TODO: Needs testing
+                        //sessionOptions.AppendExecutionProvider_CoreML(0);
+                        //CoreML on ARM64, out of the box, on x64 needs .tar file from GitHub
+                    }
+                    break;
+                default:
+                    GD.Print("OS not Supported.");
+                    break;
+            }
+            */
+        }
+
+
+        /// <include file='docs/SessionConfigurator.xml' path='docs/members[@name="SessionConfigurator"]/ComputeCheck/*'/>
+        public static ComputeName ComputeCheck()
+		{
+			string adapterName = Godot.RenderingServer.GetVideoAdapterName();
+			//string adapterVendor = Godot.RenderingServer.GetVideoAdapterVendor();
+			adapterName = adapterName.ToUpper(new System.Globalization.CultureInfo(""));
+			//TODO: GPU vendors for MacOS, what do they even use these days?
+		  
+			if (adapterName.Contains("INTEL"))
+			{
+				return ComputeName.DirectML;
+			}
+			if (adapterName.Contains("AMD") || adapterName.Contains("RADEON"))
+			{
+				return ComputeName.DirectML;
+			}
+			if (adapterName.Contains("NVIDIA"))
+			{
+				return ComputeName.CUDA;
+			}
+
+			GD.Print("Graphics Card not recognized."); //Should use CPU
+			return ComputeName.CPU;
+		}
+	}
+}

Game/addons/godot_rl_agents/onnx/csharp/docs/ONNXInference.xml

@@ -0,0 +1,31 @@
+<docs>
+	<members name="ONNXInference">
+		<ONNXInference>
+			<summary>
+       		The main <c>ONNXInference</c> Class that handles the inference process.
+        	</summary>
+		</ONNXInference>
+   		<Initialize>
+        	<summary>
+			Starts the inference process. 
+			</summary>
+			<param name="Path">Path to the ONNX model, expects a path inside resources.</param>
+			<param name="BatchSize">How many observations will the model recieve.</param>
+		</Initialize>
+		<Run>
+			<summary>
+			Runs the given input through the model and returns the output.
+			</summary>
+			<param name="obs">Dictionary containing all observations.</param>
+			<param name="state_ins">How many different agents are creating these observations.</param>
+			<returns>A Dictionary of arrays, containing instructions based on the observations.</returns>
+		</Run>
+		<Load>
+			<summary>
+			Loads the given model into the inference process, using the best Execution provider available.
+			</summary>
+			<param name="Path">Path to the ONNX model, expects a path inside resources.</param>
+			<returns>InferenceSession ready to run.</returns>
+		</Load>
+	</members>
+</docs>

Game/addons/godot_rl_agents/onnx/csharp/docs/SessionConfigurator.xml

@@ -0,0 +1,29 @@
+<docs>
+	<members name="SessionConfigurator">
+		<SessionConfigurator>
+			<summary>
+       		The main <c>SessionConfigurator</c> Class that handles the execution options and providers for the inference process.
+        	</summary>
+		</SessionConfigurator>
+  		<GetSessionOptions>
+    	    <summary>
+    	    Creates a SessionOptions with all available execution providers.
+    	    </summary>
+			<returns>SessionOptions with all available execution providers.</returns>
+    	</GetSessionOptions>
+    	<SystemCheck>
+        	<summary>
+        	Appends any execution provider available in the current system.  
+        	</summary>
+			<remarks>
+				This function is mainly verbose for tracking implementation progress of different compute APIs. 
+			</remarks>
+    	</SystemCheck>
+    	<ComputeCheck>
+        	<summary>
+        		Checks for available GPUs.
+        	</summary>
+			<returns>An integer identifier for each compute platform.</returns>
+    	</ComputeCheck>
+	</members>
+</docs>

Game/addons/godot_rl_agents/onnx/wrapper/ONNX_wrapper.gd

@@ -0,0 +1,51 @@
+extends Resource
+class_name ONNXModel
+var inferencer_script = load("res://addons/godot_rl_agents/onnx/csharp/ONNXInference.cs")
+
+var inferencer = null
+
+## How many action values the model outputs
+var action_output_size: int
+
+## Used to differentiate models
+## that only output continuous action mean (e.g. sb3, cleanrl export)
+## versus models that output mean and logstd (e.g. rllib export)
+var action_means_only: bool
+
+## Whether action_means_value has been set already for this model
+var action_means_only_set: bool
+
+# Must provide the path to the model and the batch size
+func _init(model_path, batch_size):
+	inferencer = inferencer_script.new()
+	action_output_size = inferencer.Initialize(model_path, batch_size)
+
+# This function is the one that will be called from the game,
+# requires the observations as an Dictionary and the state_ins as an int
+# returns a Dictionary containing the action the model takes.
+func run_inference(obs: Dictionary, state_ins: int) -> Dictionary:
+	if inferencer == null:
+		printerr("Inferencer not initialized")
+		return {}
+	return inferencer.RunInference(obs, state_ins)
+
+
+func _notification(what):
+	if what == NOTIFICATION_PREDELETE:
+		inferencer.FreeDisposables()
+		inferencer.free()
+
+# Check whether agent uses a continuous actions model with only action means or not
+func set_action_means_only(agent_action_space):
+	action_means_only_set = true
+	var continuous_only: bool = true
+	var continuous_actions: int
+	for action in agent_action_space:
+		if not agent_action_space[action]["action_type"] == "continuous":
+			continuous_only = false
+			break
+		else:
+			continuous_actions += agent_action_space[action]["size"]
+	if continuous_only:
+		if continuous_actions == action_output_size:
+			action_means_only = true

Game/addons/godot_rl_agents/onnx/wrapper/ONNX_wrapper.gd.uid

@@ -0,0 +1 @@
+uid://c35ckkxpe764s

Game/addons/godot_rl_agents/plugin.cfg

@@ -0,0 +1,7 @@
+[plugin]
+
+name="GodotRLAgents"
+description="Custom nodes for the godot rl agents toolkit	"
+author="Edward Beeching"
+version="0.1"
+script="godot_rl_agents.gd"

Game/addons/godot_rl_agents/rewards/ApproachNodeReward2D.gd

@@ -0,0 +1,30 @@
+extends RewardFunction2D
+class_name ApproachNodeReward2D
+
+## Calculates the reward for approaching node
+## a reward is only added when the agent reaches a new
+## best distance to the target object.
+
+## Best distance reward will be calculated for this object
+@export var target_node: Node2D
+
+## Scales the reward, 1.0 means the reward is equal to 
+## how much closer the agent is than the previous best.
+@export_range(0.0, 1.0, 0.0001, "or_greater") var reward_scale: float = 1.0
+
+var _best_distance
+
+
+func get_reward() -> float:
+	var reward := 0.0
+	var current_distance := global_position.distance_to(target_node.global_position)
+	if not _best_distance:
+		_best_distance = current_distance
+	if current_distance < _best_distance:
+		reward = (_best_distance - current_distance) * reward_scale
+		_best_distance = current_distance
+	return reward
+
+
+func reset():
+	_best_distance = null

Game/addons/godot_rl_agents/rewards/ApproachNodeReward2D.gd.uid

@@ -0,0 +1 @@
+uid://2jcows6svnje

Game/addons/godot_rl_agents/rewards/ApproachNodeReward3D.gd

@@ -0,0 +1,30 @@
+extends RewardFunction3D
+class_name ApproachNodeReward3D
+
+## Calculates the reward for approaching node
+## a reward is only added when the agent reaches a new
+## best distance to the target object.
+
+## Best distance reward will be calculated for this object
+@export var target_node: Node3D
+
+## Scales the reward, 1.0 means the reward is equal to 
+## how much closer the agent is than the previous best.
+@export_range(0.0, 1.0, 0.0001, "or_greater") var reward_scale: float = 1.0
+
+var _best_distance
+
+
+func get_reward() -> float:
+	var reward := 0.0
+	var current_distance := global_position.distance_to(target_node.global_position)
+	if not _best_distance:
+		_best_distance = current_distance
+	if current_distance < _best_distance:
+		reward = (_best_distance - current_distance) * reward_scale
+		_best_distance = current_distance
+	return reward
+
+
+func reset():
+	_best_distance = null

Game/addons/godot_rl_agents/rewards/ApproachNodeReward3D.gd.uid

@@ -0,0 +1 @@
+uid://bpgeoecqatvwi

Game/addons/godot_rl_agents/rewards/RewardFunction2D.gd

@@ -0,0 +1,10 @@
+extends Node2D
+class_name RewardFunction2D
+
+
+func get_reward():
+	return 0.0
+
+
+func reset():
+	return

Game/addons/godot_rl_agents/rewards/RewardFunction2D.gd.uid

@@ -0,0 +1 @@
+uid://52jl48u122l8

Game/addons/godot_rl_agents/rewards/RewardFunction3D.gd

@@ -0,0 +1,10 @@
+extends Node3D
+class_name RewardFunction3D
+
+
+func get_reward():
+	return 0.0
+
+
+func reset():
+	return

Game/addons/godot_rl_agents/rewards/RewardFunction3D.gd.uid

@@ -0,0 +1 @@
+uid://bwqq2ytgnepid

Game/addons/godot_rl_agents/sensors/sensors_2d/ExampleRaycastSensor2D.tscn

@@ -0,0 +1,48 @@
+[gd_scene load_steps=5 format=3 uid="uid://ddeq7mn1ealyc"]
+
+[ext_resource type="Script" path="res://addons/godot_rl_agents/sensors/sensors_2d/RaycastSensor2D.gd" id="1"]
+
+[sub_resource type="GDScript" id="2"]
+script/source = "extends Node2D
+
+
+
+func _physics_process(delta: float) -> void:
+    print(\"step start\")
+    
+"
+
+[sub_resource type="GDScript" id="1"]
+script/source = "extends RayCast2D
+
+var steps = 1
+
+func _physics_process(delta: float) -> void:
+    print(\"processing raycast\")
+    steps += 1
+    if steps % 2:
+        force_raycast_update()
+
+    print(is_colliding())
+"
+
+[sub_resource type="CircleShape2D" id="3"]
+
+[node name="ExampleRaycastSensor2D" type="Node2D"]
+script = SubResource("2")
+
+[node name="ExampleAgent" type="Node2D" parent="."]
+position = Vector2(573, 314)
+rotation = 0.286234
+
+[node name="RaycastSensor2D" type="Node2D" parent="ExampleAgent"]
+script = ExtResource("1")
+
+[node name="TestRayCast2D" type="RayCast2D" parent="."]
+script = SubResource("1")
+
+[node name="StaticBody2D" type="StaticBody2D" parent="."]
+position = Vector2(1, 52)
+
+[node name="CollisionShape2D" type="CollisionShape2D" parent="StaticBody2D"]
+shape = SubResource("3")

Game/addons/godot_rl_agents/sensors/sensors_2d/GridSensor2D.gd

@@ -0,0 +1,235 @@
+@tool
+extends ISensor2D
+class_name GridSensor2D
+
+@export var debug_view := false:
+	get:
+		return debug_view
+	set(value):
+		debug_view = value
+		_update()
+
+@export_flags_2d_physics var detection_mask := 0:
+	get:
+		return detection_mask
+	set(value):
+		detection_mask = value
+		_update()
+
+@export var collide_with_areas := false:
+	get:
+		return collide_with_areas
+	set(value):
+		collide_with_areas = value
+		_update()
+
+@export var collide_with_bodies := true:
+	get:
+		return collide_with_bodies
+	set(value):
+		collide_with_bodies = value
+		_update()
+
+@export_range(1, 200, 0.1) var cell_width := 20.0:
+	get:
+		return cell_width
+	set(value):
+		cell_width = value
+		_update()
+
+@export_range(1, 200, 0.1) var cell_height := 20.0:
+	get:
+		return cell_height
+	set(value):
+		cell_height = value
+		_update()
+
+@export_range(1, 21, 2, "or_greater") var grid_size_x := 3:
+	get:
+		return grid_size_x
+	set(value):
+		grid_size_x = value
+		_update()
+
+@export_range(1, 21, 2, "or_greater") var grid_size_y := 3:
+	get:
+		return grid_size_y
+	set(value):
+		grid_size_y = value
+		_update()
+
+var _obs_buffer: PackedFloat64Array
+var _rectangle_shape: RectangleShape2D
+var _collision_mapping: Dictionary
+var _n_layers_per_cell: int
+
+var _highlighted_cell_color: Color
+var _standard_cell_color: Color
+
+
+func get_observation():
+	return _obs_buffer
+
+
+func _update():
+	if Engine.is_editor_hint():
+		if is_node_ready():
+			_spawn_nodes()
+
+
+func _ready() -> void:
+	_set_colors()
+
+	if Engine.is_editor_hint():
+		if get_child_count() == 0:
+			_spawn_nodes()
+	else:
+		_spawn_nodes()
+
+
+func _set_colors() -> void:
+	_standard_cell_color = Color(100.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0)
+	_highlighted_cell_color = Color(255.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0)
+
+
+func _get_collision_mapping() -> Dictionary:
+	# defines which layer is mapped to which cell obs index
+	var total_bits = 0
+	var collision_mapping = {}
+	for i in 32:
+		var bit_mask = 2 ** i
+		if (detection_mask & bit_mask) > 0:
+			collision_mapping[i] = total_bits
+			total_bits += 1
+
+	return collision_mapping
+
+
+func _spawn_nodes():
+	for cell in get_children():
+		cell.name = "_%s" % cell.name  # Otherwise naming below will fail
+		cell.queue_free()
+
+	_collision_mapping = _get_collision_mapping()
+	#prints("collision_mapping", _collision_mapping, len(_collision_mapping))
+	# allocate memory for the observations
+	_n_layers_per_cell = len(_collision_mapping)
+	_obs_buffer = PackedFloat64Array()
+	_obs_buffer.resize(grid_size_x * grid_size_y * _n_layers_per_cell)
+	_obs_buffer.fill(0)
+	#prints(len(_obs_buffer), _obs_buffer )
+
+	_rectangle_shape = RectangleShape2D.new()
+	_rectangle_shape.set_size(Vector2(cell_width, cell_height))
+
+	var shift := Vector2(
+		-(grid_size_x / 2) * cell_width,
+		-(grid_size_y / 2) * cell_height,
+	)
+
+	for i in grid_size_x:
+		for j in grid_size_y:
+			var cell_position = Vector2(i * cell_width, j * cell_height) + shift
+			_create_cell(i, j, cell_position)
+
+
+func _create_cell(i: int, j: int, position: Vector2):
+	var cell := Area2D.new()
+	cell.position = position
+	cell.name = "GridCell %s %s" % [i, j]
+	cell.modulate = _standard_cell_color
+
+	if collide_with_areas:
+		cell.area_entered.connect(_on_cell_area_entered.bind(i, j))
+		cell.area_exited.connect(_on_cell_area_exited.bind(i, j))
+
+	if collide_with_bodies:
+		cell.body_entered.connect(_on_cell_body_entered.bind(i, j))
+		cell.body_exited.connect(_on_cell_body_exited.bind(i, j))
+
+	cell.collision_layer = 0
+	cell.collision_mask = detection_mask
+	cell.monitorable = true
+	add_child(cell)
+	cell.set_owner(get_tree().edited_scene_root)
+
+	var col_shape := CollisionShape2D.new()
+	col_shape.shape = _rectangle_shape
+	col_shape.name = "CollisionShape2D"
+	cell.add_child(col_shape)
+	col_shape.set_owner(get_tree().edited_scene_root)
+
+	if debug_view:
+		var quad = MeshInstance2D.new()
+		quad.name = "MeshInstance2D"
+		var quad_mesh = QuadMesh.new()
+
+		quad_mesh.set_size(Vector2(cell_width, cell_height))
+
+		quad.mesh = quad_mesh
+		cell.add_child(quad)
+		quad.set_owner(get_tree().edited_scene_root)
+
+
+func _update_obs(cell_i: int, cell_j: int, collision_layer: int, entered: bool):
+	for key in _collision_mapping:
+		var bit_mask = 2 ** key
+		if (collision_layer & bit_mask) > 0:
+			var collison_map_index = _collision_mapping[key]
+
+			var obs_index = (
+				(cell_i * grid_size_y * _n_layers_per_cell)
+				+ (cell_j * _n_layers_per_cell)
+				+ collison_map_index
+			)
+			#prints(obs_index, cell_i, cell_j)
+			if entered:
+				_obs_buffer[obs_index] += 1
+			else:
+				_obs_buffer[obs_index] -= 1
+
+
+func _toggle_cell(cell_i: int, cell_j: int):
+	var cell = get_node_or_null("GridCell %s %s" % [cell_i, cell_j])
+
+	if cell == null:
+		print("cell not found, returning")
+
+	var n_hits = 0
+	var start_index = (cell_i * grid_size_y * _n_layers_per_cell) + (cell_j * _n_layers_per_cell)
+	for i in _n_layers_per_cell:
+		n_hits += _obs_buffer[start_index + i]
+
+	if n_hits > 0:
+		cell.modulate = _highlighted_cell_color
+	else:
+		cell.modulate = _standard_cell_color
+
+
+func _on_cell_area_entered(area: Area2D, cell_i: int, cell_j: int):
+	#prints("_on_cell_area_entered", cell_i, cell_j)
+	_update_obs(cell_i, cell_j, area.collision_layer, true)
+	if debug_view:
+		_toggle_cell(cell_i, cell_j)
+	#print(_obs_buffer)
+
+
+func _on_cell_area_exited(area: Area2D, cell_i: int, cell_j: int):
+	#prints("_on_cell_area_exited", cell_i, cell_j)
+	_update_obs(cell_i, cell_j, area.collision_layer, false)
+	if debug_view:
+		_toggle_cell(cell_i, cell_j)
+
+
+func _on_cell_body_entered(body: Node2D, cell_i: int, cell_j: int):
+	#prints("_on_cell_body_entered", cell_i, cell_j)
+	_update_obs(cell_i, cell_j, body.collision_layer, true)
+	if debug_view:
+		_toggle_cell(cell_i, cell_j)
+
+
+func _on_cell_body_exited(body: Node2D, cell_i: int, cell_j: int):
+	#prints("_on_cell_body_exited", cell_i, cell_j)
+	_update_obs(cell_i, cell_j, body.collision_layer, false)
+	if debug_view:
+		_toggle_cell(cell_i, cell_j)

Game/addons/godot_rl_agents/sensors/sensors_2d/GridSensor2D.gd.uid

@@ -0,0 +1 @@
+uid://ctfloulhiht0v

Game/addons/godot_rl_agents/sensors/sensors_2d/ISensor2D.gd

@@ -0,0 +1,25 @@
+extends Node2D
+class_name ISensor2D
+
+var _obs: Array = []
+var _active := false
+
+
+func get_observation():
+	pass
+
+
+func activate():
+	_active = true
+
+
+func deactivate():
+	_active = false
+
+
+func _update_observation():
+	pass
+
+
+func reset():
+	pass

Game/addons/godot_rl_agents/sensors/sensors_2d/ISensor2D.gd.uid

@@ -0,0 +1 @@
+uid://dwhx5ltv86b4e

Game/addons/godot_rl_agents/sensors/sensors_2d/PositionSensor2D.gd

@@ -0,0 +1,65 @@
+extends ISensor2D
+class_name PositionSensor2D
+
+@export var objects_to_observe: Array[Node2D]
+
+## Whether to include relative x position in obs
+@export var include_x := true
+## Whether to include relative y position in obs
+@export var include_y := true
+
+## Max distance, values in obs will be normalized,
+## 0 will represent the closest distance possible, and 1 the farthest.
+## Do not use a much larger value than needed, as it would make the obs
+## very small after normalization.
+@export_range(0.01, 20_000) var max_distance := 1.0
+
+@export var use_separate_direction: bool = false
+
+@export var debug_lines: bool = true
+@export var debug_color: Color = Color.GREEN
+
+@onready var line: Line2D
+
+
+func _ready() -> void:
+	if debug_lines:
+		line = Line2D.new()
+		add_child(line)
+		line.width = 1
+		line.default_color = debug_color
+
+func get_observation():
+	var observations: Array[float]
+
+	if debug_lines:
+		line.clear_points()
+
+	for obj in objects_to_observe:
+		var relative_position := Vector2.ZERO
+
+		## If object has been removed, keep the zeroed position
+		if is_instance_valid(obj): relative_position = to_local(obj.global_position)
+
+		if debug_lines:
+			line.add_point(Vector2.ZERO)
+			line.add_point(relative_position)
+
+		var direction := Vector2.ZERO 
+		var distance := 0.0
+		if use_separate_direction:
+			direction = relative_position.normalized()
+			distance = min(relative_position.length() / max_distance, 1.0)
+			if include_x:
+				observations.append(direction.x)
+			if include_y:
+				observations.append(direction.y)
+			observations.append(distance)
+		else:
+			relative_position = relative_position.limit_length(max_distance) / max_distance
+			if include_x:
+				observations.append(relative_position.x)
+			if include_y:
+				observations.append(relative_position.y)
+
+	return observations

Game/addons/godot_rl_agents/sensors/sensors_2d/PositionSensor2D.gd.uid

@@ -0,0 +1 @@
+uid://dvny3m75wta1e

Game/addons/godot_rl_agents/sensors/sensors_2d/RGBCameraSensor2D.gd

@@ -0,0 +1,77 @@
+extends Node2D
+class_name RGBCameraSensor2D
+var camera_pixels = null
+
+@export var camera_zoom_factor := Vector2(0.1, 0.1)
+@onready var camera := $SubViewport/Camera
+@onready var preview_window := $Control
+@onready var camera_texture := $Control/CameraTexture as Sprite2D
+@onready var processed_texture := $Control/ProcessedTexture as Sprite2D
+@onready var sub_viewport := $SubViewport as SubViewport
+@onready var displayed_image: ImageTexture
+
+@export var render_image_resolution := Vector2i(36, 36)
+## Display size does not affect rendered or sent image resolution.
+## Scale is relative to either render image or downscale image resolution
+## depending on which mode is set.
+@export var displayed_image_scale_factor := Vector2i(8, 8)
+
+@export_group("Downscale image options")
+## Enable to downscale the rendered image before sending the obs.
+@export var downscale_image: bool = false
+## If downscale_image is true, will display the downscaled image instead of rendered image.
+@export var display_downscaled_image: bool = true
+## This is the resolution of the image that will be sent after downscaling
+@export var resized_image_resolution := Vector2i(36, 36)
+
+
+func _ready():
+	DisplayServer.register_additional_output(self)
+
+	camera.zoom = camera_zoom_factor
+
+	var preview_size: Vector2
+
+	sub_viewport.world_2d = get_tree().get_root().get_world_2d()
+	sub_viewport.size = render_image_resolution
+	camera_texture.scale = displayed_image_scale_factor
+
+	if downscale_image and display_downscaled_image:
+		camera_texture.visible = false
+		processed_texture.scale = displayed_image_scale_factor
+		preview_size = displayed_image_scale_factor * resized_image_resolution
+	else:
+		processed_texture.visible = false
+		preview_size = displayed_image_scale_factor * render_image_resolution
+
+	preview_window.size = preview_size
+
+
+func get_camera_pixel_encoding():
+	var image := camera_texture.get_texture().get_image() as Image
+
+	if downscale_image:
+		image.resize(
+			resized_image_resolution.x, resized_image_resolution.y, Image.INTERPOLATE_NEAREST
+		)
+		if display_downscaled_image:
+			if not processed_texture.texture:
+				displayed_image = ImageTexture.create_from_image(image)
+				processed_texture.texture = displayed_image
+			else:
+				displayed_image.update(image)
+
+	return image.get_data().hex_encode()
+
+
+func get_camera_shape() -> Array:
+	var size = resized_image_resolution if downscale_image else render_image_resolution
+
+	assert(
+		size.x >= 36 and size.y >= 36,
+		"Camera sensor sent image resolution must be 36x36 or larger."
+	)
+	if sub_viewport.transparent_bg:
+		return [4, size.y, size.x]
+	else:
+		return [3, size.y, size.x]

Game/addons/godot_rl_agents/sensors/sensors_2d/RGBCameraSensor2D.gd.uid

@@ -0,0 +1 @@
+uid://c8f8s63q3yyu1

Game/addons/godot_rl_agents/sensors/sensors_2d/RGBCameraSensor2D.tscn

@@ -0,0 +1,36 @@
+[gd_scene load_steps=3 format=3 uid="uid://bav1cl8uwc45c"]
+
+[ext_resource type="Script" path="res://addons/godot_rl_agents/sensors/sensors_2d/RGBCameraSensor2D.gd" id="1_txpo2"]
+
+[sub_resource type="ViewportTexture" id="ViewportTexture_jks1s"]
+viewport_path = NodePath("SubViewport")
+
+[node name="RGBCameraSensor2D" type="Node2D"]
+script = ExtResource("1_txpo2")
+displayed_image_scale_factor = Vector2(3, 3)
+
+[node name="RemoteTransform" type="RemoteTransform2D" parent="."]
+remote_path = NodePath("../SubViewport/Camera")
+
+[node name="SubViewport" type="SubViewport" parent="."]
+canvas_item_default_texture_filter = 0
+size = Vector2i(36, 36)
+render_target_update_mode = 4
+
+[node name="Camera" type="Camera2D" parent="SubViewport"]
+position_smoothing_speed = 2.0
+
+[node name="Control" type="Window" parent="."]
+canvas_item_default_texture_filter = 0
+title = "CameraSensor"
+position = Vector2i(20, 40)
+size = Vector2i(64, 64)
+theme_override_font_sizes/title_font_size = 12
+metadata/_edit_use_anchors_ = true
+
+[node name="CameraTexture" type="Sprite2D" parent="Control"]
+texture = SubResource("ViewportTexture_jks1s")
+centered = false
+
+[node name="ProcessedTexture" type="Sprite2D" parent="Control"]
+centered = false

Game/addons/godot_rl_agents/sensors/sensors_2d/RaycastSensor2D.gd

@@ -0,0 +1,123 @@
+@tool
+extends ISensor2D
+class_name RaycastSensor2D
+
+@export_flags_2d_physics var collision_mask := 1:
+	get:
+		return collision_mask
+	set(value):
+		collision_mask = value
+		_update()
+
+@export var collide_with_areas := false:
+	get:
+		return collide_with_areas
+	set(value):
+		collide_with_areas = value
+		_update()
+
+@export var collide_with_bodies := true:
+	get:
+		return collide_with_bodies
+	set(value):
+		collide_with_bodies = value
+		_update()
+
+@export var n_rays := 16.0:
+	get:
+		return n_rays
+	set(value):
+		n_rays = value
+		_update()
+
+@export_range(5, 3000, 5.0) var ray_length := 200:
+	get:
+		return ray_length
+	set(value):
+		ray_length = value
+		_update()
+@export_range(5, 360, 5.0) var cone_width := 360.0:
+	get:
+		return cone_width
+	set(value):
+		cone_width = value
+		_update()
+
+@export var debug_draw := false:
+	get:
+		return debug_draw
+	set(value):
+		debug_draw = value
+		_update()
+
+var _angles = []
+var rays := []
+
+
+func _update():
+	if Engine.is_editor_hint():
+		if is_node_ready():
+			_spawn_nodes()
+
+func _ready() -> void:
+	if Engine.is_editor_hint():
+		if get_child_count() == 0:
+			_spawn_nodes()
+	else:
+		_spawn_nodes()
+
+
+func _spawn_nodes():
+	for ray in get_children():
+		ray.queue_free()
+	rays = []
+
+	_angles = []
+	var step = cone_width / (n_rays)
+	var start = step / 2 - cone_width / 2
+
+	for i in n_rays:
+		var angle = start + i * step
+		var ray = RayCast2D.new()
+		ray.set_target_position(
+			Vector2(ray_length * cos(deg_to_rad(angle)), ray_length * sin(deg_to_rad(angle)))
+		)
+		if debug_draw:
+			ray.enabled = true
+		else:
+			ray.enabled = false
+		ray.collide_with_areas = collide_with_areas
+		ray.collide_with_bodies = collide_with_bodies
+		ray.collision_mask = collision_mask
+		add_child(ray)
+		ray.set_owner(get_tree().edited_scene_root)
+		ray.set_name("node_" + str(i))
+		rays.append(ray)
+
+		_angles.append(start + i * step)
+
+
+func get_observation() -> Array:
+	return self.calculate_raycasts()
+
+
+func calculate_raycasts() -> Array:
+	var result = []
+	for ray in rays:
+		if not debug_draw:
+			ray.enabled = true
+		ray.force_raycast_update()
+		var distance = _get_raycast_distance(ray)
+		result.append(distance)
+		if not debug_draw:
+			ray.enabled = false
+	return result
+
+
+func _get_raycast_distance(ray: RayCast2D) -> float:
+	if !ray.is_colliding():
+		return 0.0
+
+	var distance = (global_position - ray.get_collision_point()).length()
+	distance = clamp(distance, 0.0, ray_length)
+	return (ray_length - distance) / ray_length

Game/addons/godot_rl_agents/sensors/sensors_2d/RaycastSensor2D.gd.uid

@@ -0,0 +1 @@
+uid://c0eh0jjfthdgw

Game/addons/godot_rl_agents/sensors/sensors_2d/RaycastSensor2D.tscn

@@ -0,0 +1,7 @@
+[gd_scene load_steps=2 format=3 uid="uid://drvfihk5esgmv"]
+
+[ext_resource type="Script" path="res://addons/godot_rl_agents/sensors/sensors_2d/RaycastSensor2D.gd" id="1"]
+
+[node name="RaycastSensor2D" type="Node2D"]
+script = ExtResource("1")
+n_rays = 17.0

Game/addons/godot_rl_agents/sensors/sensors_3d/ExampleRaycastSensor3D.tscn

@@ -0,0 +1,6 @@
+[gd_scene format=3 uid="uid://biu787qh4woik"]
+
+[node name="ExampleRaycastSensor3D" type="Node3D"]
+
+[node name="Camera3D" type="Camera3D" parent="."]
+transform = Transform3D(1, 0, 0, 0, 1, 0, 0, 0, 1, 0.804183, 0, 2.70146)

Game/addons/godot_rl_agents/sensors/sensors_3d/GridSensor3D.gd

@@ -0,0 +1,258 @@
+@tool
+extends ISensor3D
+class_name GridSensor3D
+
+@export var debug_view := false:
+	get:
+		return debug_view
+	set(value):
+		debug_view = value
+		_update()
+
+@export_flags_3d_physics var detection_mask := 0:
+	get:
+		return detection_mask
+	set(value):
+		detection_mask = value
+		_update()
+
+@export var collide_with_areas := false:
+	get:
+		return collide_with_areas
+	set(value):
+		collide_with_areas = value
+		_update()
+
+@export var collide_with_bodies := false:
+	# NOTE! The sensor will not detect StaticBody3D, add an area to static bodies to detect them
+	get:
+		return collide_with_bodies
+	set(value):
+		collide_with_bodies = value
+		_update()
+
+@export_range(0.1, 2, 0.1) var cell_width := 1.0:
+	get:
+		return cell_width
+	set(value):
+		cell_width = value
+		_update()
+
+@export_range(0.1, 2, 0.1) var cell_height := 1.0:
+	get:
+		return cell_height
+	set(value):
+		cell_height = value
+		_update()
+
+@export_range(1, 21, 1, "or_greater") var grid_size_x := 3:
+	get:
+		return grid_size_x
+	set(value):
+		grid_size_x = value
+		_update()
+
+@export_range(1, 21, 1, "or_greater") var grid_size_z := 3:
+	get:
+		return grid_size_z
+	set(value):
+		grid_size_z = value
+		_update()
+
+var _obs_buffer: PackedFloat64Array
+var _box_shape: BoxShape3D
+var _collision_mapping: Dictionary
+var _n_layers_per_cell: int
+
+var _highlighted_box_material: StandardMaterial3D
+var _standard_box_material: StandardMaterial3D
+
+
+func get_observation():
+	return _obs_buffer
+
+
+func reset():
+	_obs_buffer.fill(0)
+
+
+func _update():
+	if Engine.is_editor_hint():
+		if is_node_ready():
+			_spawn_nodes()
+
+
+func _ready() -> void:
+	_make_materials()
+
+	if Engine.is_editor_hint():
+		if get_child_count() == 0:
+			_spawn_nodes()
+	else:
+		_spawn_nodes()
+
+
+func _make_materials() -> void:
+	if _highlighted_box_material != null and _standard_box_material != null:
+		return
+
+	_standard_box_material = StandardMaterial3D.new()
+	_standard_box_material.set_transparency(1)  # ALPHA
+	_standard_box_material.albedo_color = Color(
+		100.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0
+	)
+
+	_highlighted_box_material = StandardMaterial3D.new()
+	_highlighted_box_material.set_transparency(1)  # ALPHA
+	_highlighted_box_material.albedo_color = Color(
+		255.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0, 100.0 / 255.0
+	)
+
+
+func _get_collision_mapping() -> Dictionary:
+	# defines which layer is mapped to which cell obs index
+	var total_bits = 0
+	var collision_mapping = {}
+	for i in 32:
+		var bit_mask = 2 ** i
+		if (detection_mask & bit_mask) > 0:
+			collision_mapping[i] = total_bits
+			total_bits += 1
+
+	return collision_mapping
+
+
+func _spawn_nodes():
+	for cell in get_children():
+		cell.name = "_%s" % cell.name  # Otherwise naming below will fail
+		cell.queue_free()
+
+	_collision_mapping = _get_collision_mapping()
+	#prints("collision_mapping", _collision_mapping, len(_collision_mapping))
+	# allocate memory for the observations
+	_n_layers_per_cell = len(_collision_mapping)
+	_obs_buffer = PackedFloat64Array()
+	_obs_buffer.resize(grid_size_x * grid_size_z * _n_layers_per_cell)
+	_obs_buffer.fill(0)
+	#prints(len(_obs_buffer), _obs_buffer )
+
+	_box_shape = BoxShape3D.new()
+	_box_shape.set_size(Vector3(cell_width, cell_height, cell_width))
+
+	var shift := Vector3(
+		-(grid_size_x / 2) * cell_width,
+		0,
+		-(grid_size_z / 2) * cell_width,
+	)
+
+	for i in grid_size_x:
+		for j in grid_size_z:
+			var cell_position = Vector3(i * cell_width, 0.0, j * cell_width) + shift
+			_create_cell(i, j, cell_position)
+
+
+func _create_cell(i: int, j: int, position: Vector3):
+	var cell := Area3D.new()
+	cell.position = position
+	cell.name = "GridCell %s %s" % [i, j]
+
+	if collide_with_areas:
+		cell.area_entered.connect(_on_cell_area_entered.bind(i, j))
+		cell.area_exited.connect(_on_cell_area_exited.bind(i, j))
+
+	if collide_with_bodies:
+		cell.body_entered.connect(_on_cell_body_entered.bind(i, j))
+		cell.body_exited.connect(_on_cell_body_exited.bind(i, j))
+
+#		cell.body_shape_entered.connect(_on_cell_body_shape_entered.bind(i, j))
+#		cell.body_shape_exited.connect(_on_cell_body_shape_exited.bind(i, j))
+
+	cell.collision_layer = 0
+	cell.collision_mask = detection_mask
+	cell.monitorable = true
+	cell.input_ray_pickable = false
+	add_child(cell)
+	cell.set_owner(get_tree().edited_scene_root)
+
+	var col_shape := CollisionShape3D.new()
+	col_shape.shape = _box_shape
+	col_shape.name = "CollisionShape3D"
+	cell.add_child(col_shape)
+	col_shape.set_owner(get_tree().edited_scene_root)
+
+	if debug_view:
+		var box = MeshInstance3D.new()
+		box.name = "MeshInstance3D"
+		var box_mesh = BoxMesh.new()
+
+		box_mesh.set_size(Vector3(cell_width, cell_height, cell_width))
+		box_mesh.material = _standard_box_material
+
+		box.mesh = box_mesh
+		cell.add_child(box)
+		box.set_owner(get_tree().edited_scene_root)
+
+
+func _update_obs(cell_i: int, cell_j: int, collision_layer: int, entered: bool):
+	for key in _collision_mapping:
+		var bit_mask = 2 ** key
+		if (collision_layer & bit_mask) > 0:
+			var collison_map_index = _collision_mapping[key]
+
+			var obs_index = (
+				(cell_i * grid_size_z * _n_layers_per_cell)
+				+ (cell_j * _n_layers_per_cell)
+				+ collison_map_index
+			)
+			#prints(obs_index, cell_i, cell_j)
+			if entered:
+				_obs_buffer[obs_index] += 1
+			else:
+				_obs_buffer[obs_index] -= 1
+
+
+func _toggle_cell(cell_i: int, cell_j: int):
+	var cell = get_node_or_null("GridCell %s %s" % [cell_i, cell_j])
+
+	if cell == null:
+		print("cell not found, returning")
+
+	var n_hits = 0
+	var start_index = (cell_i * grid_size_z * _n_layers_per_cell) + (cell_j * _n_layers_per_cell)
+	for i in _n_layers_per_cell:
+		n_hits += _obs_buffer[start_index + i]
+
+	var cell_mesh = cell.get_node_or_null("MeshInstance3D")
+	if n_hits > 0:
+		cell_mesh.mesh.material = _highlighted_box_material
+	else:
+		cell_mesh.mesh.material = _standard_box_material
+
+
+func _on_cell_area_entered(area: Area3D, cell_i: int, cell_j: int):
+	#prints("_on_cell_area_entered", cell_i, cell_j)
+	_update_obs(cell_i, cell_j, area.collision_layer, true)
+	if debug_view:
+		_toggle_cell(cell_i, cell_j)
+	#print(_obs_buffer)
+
+
+func _on_cell_area_exited(area: Area3D, cell_i: int, cell_j: int):
+	#prints("_on_cell_area_exited", cell_i, cell_j)
+	_update_obs(cell_i, cell_j, area.collision_layer, false)
+	if debug_view:
+		_toggle_cell(cell_i, cell_j)
+
+
+func _on_cell_body_entered(body: Node3D, cell_i: int, cell_j: int):
+	#prints("_on_cell_body_entered", cell_i, cell_j)
+	_update_obs(cell_i, cell_j, body.collision_layer, true)
+	if debug_view:
+		_toggle_cell(cell_i, cell_j)
+
+
+func _on_cell_body_exited(body: Node3D, cell_i: int, cell_j: int):
+	#prints("_on_cell_body_exited", cell_i, cell_j)
+	_update_obs(cell_i, cell_j, body.collision_layer, false)
+	if debug_view:
+		_toggle_cell(cell_i, cell_j)

Game/addons/godot_rl_agents/sensors/sensors_3d/GridSensor3D.gd.uid

@@ -0,0 +1 @@
+uid://me8mehqmblq8

Game/addons/godot_rl_agents/sensors/sensors_3d/ISensor3D.gd

@@ -0,0 +1,25 @@
+extends Node3D
+class_name ISensor3D
+
+var _obs: Array = []
+var _active := false
+
+
+func get_observation():
+	pass
+
+
+func activate():
+	_active = true
+
+
+func deactivate():
+	_active = false
+
+
+func _update_observation():
+	pass
+
+
+func reset():
+	pass

Game/addons/godot_rl_agents/sensors/sensors_3d/ISensor3D.gd.uid

@@ -0,0 +1 @@
+uid://b6dvaob0xndoh

Game/addons/godot_rl_agents/sensors/sensors_3d/PositionSensor3D.gd

@@ -0,0 +1,79 @@
+extends ISensor3D
+class_name PositionSensor3D
+
+@export var objects_to_observe: Array[Node3D]
+
+## Whether to include relative x position in obs
+@export var include_x := true
+## Whether to include relative y position in obs
+@export var include_y := true
+## Whether to include relative z position in obs
+@export var include_z := true
+
+## Max distance, values in obs will be normalized,
+## 0 will represent the closest distance possible, and 1 the farthest.
+## Do not use a much larger value than needed, as it would make the obs
+## very small after normalization.
+@export_range(0.01, 2_500) var max_distance := 1.0
+
+@export var use_separate_direction: bool = false
+
+@export var debug_lines: bool = true
+@export var debug_color: Color = Color.GREEN
+
+@onready var mesh: ImmediateMesh
+
+
+func _ready() -> void:
+	if debug_lines:
+		var debug_mesh = MeshInstance3D.new()
+		add_child(debug_mesh)
+		var line_material := StandardMaterial3D.new()
+		line_material.albedo_color = debug_color
+		debug_mesh.material_override = line_material
+		debug_mesh.mesh = ImmediateMesh.new()
+		mesh = debug_mesh.mesh
+
+
+func get_observation():
+	var observations: Array[float]
+
+	if debug_lines:
+		mesh.clear_surfaces()
+		mesh.surface_begin(Mesh.PRIMITIVE_LINES)
+		mesh.surface_set_color(debug_color)
+
+	for obj in objects_to_observe:
+		var relative_position := Vector3.ZERO
+
+		## If object has been removed, keep the zeroed position
+		if is_instance_valid(obj): relative_position = to_local(obj.global_position)
+
+		if debug_lines:
+			mesh.surface_add_vertex(Vector3.ZERO)
+			mesh.surface_add_vertex(relative_position)
+
+		var direction := Vector3.ZERO 
+		var distance := 0.0
+		if use_separate_direction:
+			direction = relative_position.normalized()
+			distance = min(relative_position.length() / max_distance, 1.0)
+			if include_x:
+				observations.append(direction.x)
+			if include_y:
+				observations.append(direction.y)
+			if include_z:
+				observations.append(direction.z)
+			observations.append(distance)
+		else:
+			relative_position = relative_position.limit_length(max_distance) / max_distance
+			if include_x:
+				observations.append(relative_position.x)
+			if include_y:
+				observations.append(relative_position.y)
+			if include_z:
+				observations.append(relative_position.z)
+
+	if debug_lines:
+		mesh.surface_end()
+	return observations

Game/addons/godot_rl_agents/sensors/sensors_3d/PositionSensor3D.gd.uid

@@ -0,0 +1 @@
+uid://cew2a213sw1q

Game/addons/godot_rl_agents/sensors/sensors_3d/RGBCameraSensor3D.gd

@@ -0,0 +1,63 @@
+extends Node3D
+class_name RGBCameraSensor3D
+var camera_pixels = null
+
+@onready var camera_texture := $Control/CameraTexture as Sprite2D
+@onready var processed_texture := $Control/ProcessedTexture as Sprite2D
+@onready var sub_viewport := $SubViewport as SubViewport
+@onready var displayed_image: ImageTexture
+
+@export var render_image_resolution := Vector2i(36, 36)
+## Display size does not affect rendered or sent image resolution.
+## Scale is relative to either render image or downscale image resolution
+## depending on which mode is set.
+@export var displayed_image_scale_factor := Vector2i(8, 8)
+
+@export_group("Downscale image options")
+## Enable to downscale the rendered image before sending the obs.
+@export var downscale_image: bool = false
+## If downscale_image is true, will display the downscaled image instead of rendered image.
+@export var display_downscaled_image: bool = true
+## This is the resolution of the image that will be sent after downscaling
+@export var resized_image_resolution := Vector2i(36, 36)
+
+
+func _ready():
+	sub_viewport.size = render_image_resolution
+	camera_texture.scale = displayed_image_scale_factor
+
+	if downscale_image and display_downscaled_image:
+		camera_texture.visible = false
+		processed_texture.scale = displayed_image_scale_factor
+	else:
+		processed_texture.visible = false
+
+
+func get_camera_pixel_encoding():
+	var image := camera_texture.get_texture().get_image() as Image
+
+	if downscale_image:
+		image.resize(
+			resized_image_resolution.x, resized_image_resolution.y, Image.INTERPOLATE_NEAREST
+		)
+		if display_downscaled_image:
+			if not processed_texture.texture:
+				displayed_image = ImageTexture.create_from_image(image)
+				processed_texture.texture = displayed_image
+			else:
+				displayed_image.update(image)
+
+	return image.get_data().hex_encode()
+
+
+func get_camera_shape() -> Array:
+	var size = resized_image_resolution if downscale_image else render_image_resolution
+
+	assert(
+		size.x >= 36 and size.y >= 36,
+		"Camera sensor sent image resolution must be 36x36 or larger."
+	)
+	if sub_viewport.transparent_bg:
+		return [4, size.y, size.x]
+	else:
+		return [3, size.y, size.x]

Game/addons/godot_rl_agents/sensors/sensors_3d/RGBCameraSensor3D.gd.uid

@@ -0,0 +1 @@
+uid://6e38006xhqf0

Game/addons/godot_rl_agents/sensors/sensors_3d/RGBCameraSensor3D.tscn

@@ -0,0 +1,35 @@
+[gd_scene load_steps=3 format=3 uid="uid://baaywi3arsl2m"]
+
+[ext_resource type="Script" path="res://addons/godot_rl_agents/sensors/sensors_3d/RGBCameraSensor3D.gd" id="1"]
+
+[sub_resource type="ViewportTexture" id="ViewportTexture_y72s3"]
+viewport_path = NodePath("SubViewport")
+
+[node name="RGBCameraSensor3D" type="Node3D"]
+script = ExtResource("1")
+
+[node name="RemoteTransform" type="RemoteTransform3D" parent="."]
+remote_path = NodePath("../SubViewport/Camera")
+
+[node name="SubViewport" type="SubViewport" parent="."]
+size = Vector2i(36, 36)
+render_target_update_mode = 3
+
+[node name="Camera" type="Camera3D" parent="SubViewport"]
+near = 0.5
+
+[node name="Control" type="Control" parent="."]
+layout_mode = 3
+anchors_preset = 15
+anchor_right = 1.0
+anchor_bottom = 1.0
+grow_horizontal = 2
+grow_vertical = 2
+metadata/_edit_use_anchors_ = true
+
+[node name="CameraTexture" type="Sprite2D" parent="Control"]
+texture = SubResource("ViewportTexture_y72s3")
+centered = false
+
+[node name="ProcessedTexture" type="Sprite2D" parent="Control"]
+centered = false

Game/addons/godot_rl_agents/sensors/sensors_3d/RaycastSensor3D.gd

@@ -0,0 +1,197 @@
+@tool
+extends ISensor3D
+class_name RayCastSensor3D
+@export_flags_3d_physics var collision_mask = 1:
+	get:
+		return collision_mask
+	set(value):
+		collision_mask = value
+		_update()
+@export_flags_3d_physics var boolean_class_mask = 1:
+	get:
+		return boolean_class_mask
+	set(value):
+		boolean_class_mask = value
+		_update()
+
+@export var n_rays_width := 6.0:
+	get:
+		return n_rays_width
+	set(value):
+		n_rays_width = value
+		_update()
+
+@export var n_rays_height := 6.0:
+	get:
+		return n_rays_height
+	set(value):
+		n_rays_height = value
+		_update()
+
+@export var ray_length := 10.0:
+	get:
+		return ray_length
+	set(value):
+		ray_length = value
+		_update()
+
+@export var cone_width := 60.0:
+	get:
+		return cone_width
+	set(value):
+		cone_width = value
+		_update()
+
+@export var cone_height := 60.0:
+	get:
+		return cone_height
+	set(value):
+		cone_height = value
+		_update()
+
+@export var collide_with_areas := false:
+	get:
+		return collide_with_areas
+	set(value):
+		collide_with_areas = value
+		_update()
+
+@export var collide_with_bodies := true:
+	get:
+		return collide_with_bodies
+	set(value):
+		collide_with_bodies = value
+		_update()
+
+@export var class_sensor := false
+
+@export var debug_draw := false:
+	get:
+		return debug_draw
+	set(value):
+		debug_draw = value
+		_update()
+
+var rays := []
+var geo = null
+
+
+func _update():
+	if Engine.is_editor_hint():
+		if is_node_ready():
+			_spawn_nodes()
+
+
+func _ready() -> void:
+	if Engine.is_editor_hint():
+		if get_child_count() == 0:
+			_spawn_nodes()
+	else:
+		_spawn_nodes()
+
+
+func _spawn_nodes():
+	print("spawning nodes")
+	for ray in get_children():
+		ray.queue_free()
+	if geo:
+		geo.clear()
+	#$Lines.remove_points()
+	rays = []
+
+	var horizontal_step = cone_width / (n_rays_width)
+	var vertical_step = cone_height / (n_rays_height)
+
+	var horizontal_start = horizontal_step / 2 - cone_width / 2
+	var vertical_start = vertical_step / 2 - cone_height / 2
+
+	var points = []
+
+	for i in n_rays_width:
+		for j in n_rays_height:
+			var angle_w = horizontal_start + i * horizontal_step
+			var angle_h = vertical_start + j * vertical_step
+			#angle_h = 0.0
+			var ray = RayCast3D.new()
+			var cast_to = to_spherical_coords(ray_length, angle_w, angle_h)
+			ray.set_target_position(cast_to)
+
+			points.append(cast_to)
+
+			if debug_draw:
+				ray.enabled = true
+			else:
+				ray.enabled = false
+			ray.collide_with_bodies = collide_with_bodies
+			ray.collide_with_areas = collide_with_areas
+			ray.collision_mask = collision_mask
+			add_child(ray)
+			ray.set_owner(get_tree().edited_scene_root)
+			ray.set_name("node_" + str(i) + " " + str(j))
+			rays.append(ray)
+			ray.force_raycast_update()
+
+
+#    if Engine.editor_hint:
+#        _create_debug_lines(points)
+
+
+func _create_debug_lines(points):
+	if not geo:
+		geo = ImmediateMesh.new()
+		add_child(geo)
+
+	geo.clear()
+	geo.begin(Mesh.PRIMITIVE_LINES)
+	for point in points:
+		geo.set_color(Color.AQUA)
+		geo.add_vertex(Vector3.ZERO)
+		geo.add_vertex(point)
+	geo.end()
+
+
+func display():
+	if geo:
+		geo.display()
+
+
+func to_spherical_coords(r, inc, azimuth) -> Vector3:
+	return Vector3(
+		r * sin(deg_to_rad(inc)) * cos(deg_to_rad(azimuth)),
+		r * sin(deg_to_rad(azimuth)),
+		r * cos(deg_to_rad(inc)) * cos(deg_to_rad(azimuth))
+	)
+
+
+func get_observation() -> Array:
+	return self.calculate_raycasts()
+
+
+func calculate_raycasts() -> Array:
+	var result = []
+	for ray in rays:
+		if not debug_draw:
+			ray.set_enabled(true)
+		ray.force_raycast_update()
+		var distance = _get_raycast_distance(ray)
+
+		result.append(distance)
+		if class_sensor:
+			var hit_class: float = 0
+			if ray.get_collider():
+				var hit_collision_layer = ray.get_collider().collision_layer
+				hit_collision_layer = hit_collision_layer & collision_mask
+				hit_class = (hit_collision_layer & boolean_class_mask) > 0
+			result.append(float(hit_class))
+		if not debug_draw:
+			ray.set_enabled(false)
+	return result
+
+
+func _get_raycast_distance(ray: RayCast3D) -> float:
+	if !ray.is_colliding():
+		return 0.0
+
+	var distance = (global_transform.origin - ray.get_collision_point()).length()
+	distance = clamp(distance, 0.0, ray_length)
+	return (ray_length - distance) / ray_length

Game/addons/godot_rl_agents/sensors/sensors_3d/RaycastSensor3D.gd.uid

@@ -0,0 +1 @@
+uid://glop37kjpwci

Game/addons/godot_rl_agents/sensors/sensors_3d/RaycastSensor3D.tscn

@@ -0,0 +1,27 @@
+[gd_scene load_steps=2 format=3 uid="uid://b803cbh1fmy66"]
+
+[ext_resource type="Script" path="res://addons/godot_rl_agents/sensors/sensors_3d/RaycastSensor3D.gd" id="1"]
+
+[node name="RaycastSensor3D" type="Node3D"]
+script = ExtResource("1")
+n_rays_width = 4.0
+n_rays_height = 2.0
+ray_length = 11.0
+
+[node name="node_1 0" type="RayCast3D" parent="."]
+target_position = Vector3(-1.38686, -2.84701, 10.5343)
+
+[node name="node_1 1" type="RayCast3D" parent="."]
+target_position = Vector3(-1.38686, 2.84701, 10.5343)
+
+[node name="node_2 0" type="RayCast3D" parent="."]
+target_position = Vector3(1.38686, -2.84701, 10.5343)
+
+[node name="node_2 1" type="RayCast3D" parent="."]
+target_position = Vector3(1.38686, 2.84701, 10.5343)
+
+[node name="node_3 0" type="RayCast3D" parent="."]
+target_position = Vector3(4.06608, -2.84701, 9.81639)
+
+[node name="node_3 1" type="RayCast3D" parent="."]
+target_position = Vector3(4.06608, 2.84701, 9.81639)

Game/addons/godot_rl_agents/sync.gd

@@ -0,0 +1,621 @@
+extends Node
+class_name Sync
+
+# --fixed-fps 2000 --disable-render-loop
+
+enum ControlModes {
+	HUMAN, ## Test the environment manually
+	TRAINING, ## Train a model
+	ONNX_INFERENCE ## Load a pretrained model using an .onnx file
+}
+@export var control_mode: ControlModes = ControlModes.TRAINING
+## Action will be repeated for n frames (Godot physics steps).
+@export_range(1, 10, 1, "or_greater") var action_repeat := 8
+## Speeds up the physics in the environment to enable faster training.
+@export_range(0, 10, 0.1, "or_greater") var speed_up := 1.0
+## The path to a trained .onnx model file to use for inference (only needed for the 'Onnx Inference' control mode).
+@export var onnx_model_path := ""
+## Whether the inference will be deterministic (NOTE: Only applies to discrete actions in onnx inference mode)
+@export var deterministic_inference := true
+
+# Onnx model stored for each requested path
+var onnx_models: Dictionary
+
+@onready var start_time = Time.get_ticks_msec()
+
+const MAJOR_VERSION := "0"
+const MINOR_VERSION := "7"
+const DEFAULT_PORT := "11008"
+const DEFAULT_SEED := "1"
+var stream: StreamPeerTCP = null
+var connected = false
+var message_center
+var should_connect = true
+
+var all_agents: Array
+var agents_training: Array
+## Policy name of each agent, for use with multi-policy multi-agent RL cases
+var agents_training_policy_names: Array[String] = ["shared_policy"]
+var agents_inference: Array
+var agents_heuristic: Array
+
+## For recording expert demos
+var agent_demo_record: Node
+## File path for writing recorded trajectories
+var expert_demo_save_path: String
+## Stores recorded trajectories
+var demo_trajectories: Array
+## A trajectory includes obs: Array, acts: Array, terminal (set in Python env instead)
+var current_demo_trajectory: Array
+
+var need_to_send_obs = false
+var args = null
+var initialized = false
+var just_reset = false
+var onnx_model = null
+var n_action_steps = 0
+
+var _action_space_training: Array[Dictionary] = []
+var _action_space_inference: Array[Dictionary] = []
+var _obs_space_training: Array[Dictionary] = []
+
+
+# Called when the node enters the scene tree for the first time.
+func _ready():
+	await get_parent().ready
+	get_tree().set_pause(true)
+	_initialize()
+	await get_tree().create_timer(1.0).timeout
+	get_tree().set_pause(false)
+
+
+func _initialize():
+	_get_agents()
+	args = _get_args()
+	Engine.physics_ticks_per_second = _get_speedup() * 60  # Replace with function body.
+	Engine.time_scale = _get_speedup() * 1.0
+	prints(
+		"physics ticks",
+		Engine.physics_ticks_per_second,
+		Engine.time_scale,
+		_get_speedup(),
+		speed_up
+	)
+
+	_set_heuristic("human", all_agents)
+
+	_initialize_training_agents()
+	_initialize_inference_agents()
+	_initialize_demo_recording()
+
+	_set_seed()
+	_set_action_repeat()
+	initialized = true
+
+
+func _initialize_training_agents():
+	if agents_training.size() > 0:
+		_obs_space_training.resize(agents_training.size())
+		_action_space_training.resize(agents_training.size())
+		for agent_idx in range(0, agents_training.size()):
+			_obs_space_training[agent_idx] = agents_training[agent_idx].get_obs_space()
+			_action_space_training[agent_idx] = agents_training[agent_idx].get_action_space()
+		connected = connect_to_server()
+		if connected:
+			_set_heuristic("model", agents_training)
+			_handshake()
+			_send_env_info()
+		else:
+			push_warning(
+				"Couldn't connect to Python server, using human controls instead. ",
+				"Did you start the training server using e.g. `gdrl` from the console?"
+			)
+
+
+func _initialize_inference_agents():
+	if agents_inference.size() > 0:
+		if control_mode == ControlModes.ONNX_INFERENCE:
+			assert(
+				FileAccess.file_exists(onnx_model_path),
+				"Onnx Model Path set on Sync node does not exist: %s" % onnx_model_path
+			)
+			onnx_models[onnx_model_path] = ONNXModel.new(onnx_model_path, 1)
+
+		for agent in agents_inference:
+			var action_space = agent.get_action_space()
+			_action_space_inference.append(action_space)
+
+			var agent_onnx_model: ONNXModel
+			if agent.onnx_model_path.is_empty():
+				assert(
+					onnx_models.has(onnx_model_path),
+					(
+						"Node %s has no onnx model path set " % agent.get_path()
+						+ "and sync node's control mode is not set to OnnxInference. "
+						+ "Either add the path to the AIController, "
+						+ "or if you want to use the path set on sync node instead, "
+						+ "set control mode to OnnxInference."
+					)
+				)
+				prints(
+					"Info: AIController %s" % agent.get_path(),
+					"has no onnx model path set.",
+					"Using path set on the sync node instead."
+				)
+				agent_onnx_model = onnx_models[onnx_model_path]
+			else:
+				if not onnx_models.has(agent.onnx_model_path):
+					assert(
+						FileAccess.file_exists(agent.onnx_model_path),
+						(
+							"Onnx Model Path set on %s node does not exist: %s"
+							% [agent.get_path(), agent.onnx_model_path]
+						)
+					)
+					onnx_models[agent.onnx_model_path] = ONNXModel.new(agent.onnx_model_path, 1)
+				agent_onnx_model = onnx_models[agent.onnx_model_path]
+
+			agent.onnx_model = agent_onnx_model
+			if not agent_onnx_model.action_means_only_set:
+				agent_onnx_model.set_action_means_only(action_space)
+
+		_set_heuristic("model", agents_inference)
+
+
+func _initialize_demo_recording():
+	if agent_demo_record:
+		expert_demo_save_path = agent_demo_record.expert_demo_save_path
+		assert(
+			not expert_demo_save_path.is_empty(),
+			"Expert demo save path set in %s is empty." % agent_demo_record.get_path()
+		)
+
+		InputMap.add_action("RemoveLastDemoEpisode")
+		InputMap.action_add_event(
+			"RemoveLastDemoEpisode", agent_demo_record.remove_last_episode_key
+		)
+		current_demo_trajectory.resize(2)
+		current_demo_trajectory[0] = []
+		current_demo_trajectory[1] = []
+		agent_demo_record.heuristic = "demo_record"
+
+
+func _physics_process(_delta):
+	# two modes, human control, agent control
+	# pause tree, send obs, get actions, set actions, unpause tree
+
+	_demo_record_process()
+
+	if n_action_steps % action_repeat != 0:
+		n_action_steps += 1
+		return
+
+	n_action_steps += 1
+
+	_training_process()
+	_inference_process()
+	_heuristic_process()
+
+
+func _training_process():
+	if connected:
+		get_tree().set_pause(true)
+
+		var obs = _get_obs_from_agents(agents_training)
+		var info = _get_info_from_agents(agents_training)
+
+		if just_reset:
+			just_reset = false
+
+			var reply = {"type": "reset", "obs": obs, "info": info}
+			_send_dict_as_json_message(reply)
+			# this should go straight to getting the action and setting it checked the agent, no need to perform one phyics tick
+			get_tree().set_pause(false)
+			return
+
+		if need_to_send_obs:
+			need_to_send_obs = false
+			var reward = _get_reward_from_agents()
+			var done = _get_done_from_agents()
+			#_reset_agents_if_done() # this ensures the new observation is from the next env instance : NEEDS REFACTOR
+
+			var reply = {"type": "step", "obs": obs, "reward": reward, "done": done, "info": info}
+			_send_dict_as_json_message(reply)
+
+		var handled = handle_message()
+
+
+func _inference_process():
+	if agents_inference.size() > 0:
+		var obs: Array = _get_obs_from_agents(agents_inference)
+		var actions = []
+
+		for agent_id in range(0, agents_inference.size()):
+			var model: ONNXModel = agents_inference[agent_id].onnx_model
+			var action = model.run_inference(obs[agent_id], 1.0)
+			var action_dict = _extract_action_dict(
+				action["output"], _action_space_inference[agent_id], model.action_means_only
+			)
+			actions.append(action_dict)
+
+		_set_agent_actions(actions, agents_inference)
+		_reset_agents_if_done(agents_inference)
+		get_tree().set_pause(false)
+
+
+func _demo_record_process():
+	if not agent_demo_record:
+		return
+
+	if Input.is_action_just_pressed("RemoveLastDemoEpisode"):
+		print("[Sync script][Demo recorder] Removing last recorded episode.")
+		demo_trajectories.remove_at(demo_trajectories.size() - 1)
+		print("Remaining episode count: %d" % demo_trajectories.size())
+
+	if n_action_steps % agent_demo_record.action_repeat != 0:
+		return
+
+	var obs_dict: Dictionary = agent_demo_record.get_obs()
+
+	# Get the current obs from the agent
+	assert(
+		obs_dict.has("obs"),
+		"Demo recorder needs an 'obs' key in get_obs() returned dictionary to record obs from."
+	)
+	current_demo_trajectory[0].append(obs_dict.obs)
+
+	# Get the action applied for the current obs from the agent
+	agent_demo_record.set_action()
+	var acts = agent_demo_record.get_action()
+
+	var terminal = agent_demo_record.get_done()
+	# Record actions only for non-terminal states
+	if terminal:
+		agent_demo_record.set_done_false()
+	else:
+		current_demo_trajectory[1].append(acts)
+
+	if terminal:
+		#current_demo_trajectory[2].append(true)
+		demo_trajectories.append(current_demo_trajectory.duplicate(true))
+		print("[Sync script][Demo recorder] Recorded episode count: %d" % demo_trajectories.size())
+		current_demo_trajectory[0].clear()
+		current_demo_trajectory[1].clear()
+
+
+func _heuristic_process():
+	for agent in agents_heuristic:
+		_reset_agents_if_done(agents_heuristic)
+
+
+func _extract_action_dict(action_array: Array, action_space: Dictionary, action_means_only: bool):
+	var index = 0
+	var result = {}
+	for key in action_space.keys():
+		var size = action_space[key]["size"]
+		var action_type = action_space[key]["action_type"]
+
+		if action_type == "discrete":
+			var largest_logit: float = -INF  # Value of the largest logit for this action in the actions array
+			var largest_logit_idx: int  # Index of the largest logit for this action in the actions array
+			for logit_idx in range(0, size):
+				var logit_value = action_array[index + logit_idx]
+				if logit_value > largest_logit:
+					largest_logit = logit_value
+					largest_logit_idx = logit_idx
+			if deterministic_inference:
+				result[key] = largest_logit_idx  # Index of the largest logit is the discrete action value
+			else:
+				var exp_logit_sum: float  # Sum of exp of each logit
+				var exp_logits: Array[float]
+
+				for logit_idx in range(0, size):
+					# Normalize using the max logit to add stability in case a logit would be huge after exp
+					exp_logits.append(exp(action_array[index + logit_idx] - largest_logit))
+					exp_logit_sum += exp_logits[logit_idx]
+
+				# Choose a random number, will be used to select an action
+				var random_value = randf_range(0, exp_logit_sum)
+
+				# Select the first index at which the sum is larger than the random number
+				var sum: float
+				for exp_logit_idx in exp_logits.size():
+					sum += exp_logits[exp_logit_idx]
+					if sum > random_value:
+						result[key] = exp_logit_idx
+						break
+			index += size
+		elif action_type == "continuous":
+			# For continous actions, we only take the action mean values
+			result[key] = clamp_array(action_array.slice(index, index + size), -1.0, 1.0)
+			if action_means_only:
+				index += size  # model only outputs action means, so we move index by size
+			else:
+				index += size * 2  # model outputs logstd after action mean, we skip the logstd part
+
+		else:
+			assert(
+				false,
+				(
+					'Only "discrete" and "continuous" action types supported. Found: %s action type set.'
+					% action_type
+				)
+			)
+
+	return result
+
+
+## For AIControllers that inherit mode from sync, sets the correct mode.
+func _set_agent_mode(agent: Node):
+	var agent_inherits_mode: bool = agent.control_mode == agent.ControlModes.INHERIT_FROM_SYNC
+
+	if agent_inherits_mode:
+		match control_mode:
+			ControlModes.HUMAN:
+				agent.control_mode = agent.ControlModes.HUMAN
+			ControlModes.TRAINING:
+				agent.control_mode = agent.ControlModes.TRAINING
+			ControlModes.ONNX_INFERENCE:
+				agent.control_mode = agent.ControlModes.ONNX_INFERENCE
+
+
+func _get_agents():
+	all_agents = get_tree().get_nodes_in_group("AGENT")
+	for agent in all_agents:
+		_set_agent_mode(agent)
+
+		if agent.control_mode == agent.ControlModes.TRAINING:
+			agents_training.append(agent)
+		elif agent.control_mode == agent.ControlModes.ONNX_INFERENCE:
+			agents_inference.append(agent)
+		elif agent.control_mode == agent.ControlModes.HUMAN:
+			agents_heuristic.append(agent)
+		elif agent.control_mode == agent.ControlModes.RECORD_EXPERT_DEMOS:
+			assert(
+				not agent_demo_record,
+				"Currently only a single AIController can be used for recording expert demos."
+			)
+			agent_demo_record = agent
+
+	var training_agent_count = agents_training.size()
+	agents_training_policy_names.resize(training_agent_count)
+	for i in range(0, training_agent_count):
+		agents_training_policy_names[i] = agents_training[i].policy_name
+
+
+func _set_heuristic(heuristic, agents: Array):
+	for agent in agents:
+		agent.set_heuristic(heuristic)
+
+
+func _handshake():
+	print("performing handshake")
+
+	var json_dict = _get_dict_json_message()
+	assert(json_dict["type"] == "handshake")
+	var major_version = json_dict["major_version"]
+	var minor_version = json_dict["minor_version"]
+	if major_version != MAJOR_VERSION:
+		print("WARNING: major verison mismatch ", major_version, " ", MAJOR_VERSION)
+	if minor_version != MINOR_VERSION:
+		print("WARNING: minor verison mismatch ", minor_version, " ", MINOR_VERSION)
+
+	print("handshake complete")
+
+
+func _get_dict_json_message():
+	# returns a dictionary from of the most recent message
+	# this is not waiting
+	while stream.get_available_bytes() == 0:
+		stream.poll()
+		if stream.get_status() != 2:
+			print("server disconnected status, closing")
+			get_tree().quit()
+			return null
+
+		OS.delay_usec(10)
+
+	var message = stream.get_string()
+	var json_data = JSON.parse_string(message)
+
+	return json_data
+
+
+func _send_dict_as_json_message(dict):
+	stream.put_string(JSON.stringify(dict, "", false))
+
+
+func _send_env_info():
+	var json_dict = _get_dict_json_message()
+	assert(json_dict["type"] == "env_info")
+
+	var message = {
+		"type": "env_info",
+		"observation_space": _obs_space_training,
+		"action_space": _action_space_training,
+		"n_agents": len(agents_training),
+		"agent_policy_names": agents_training_policy_names
+	}
+	_send_dict_as_json_message(message)
+
+
+func connect_to_server():
+	print("Waiting for one second to allow server to start")
+	OS.delay_msec(1000)
+	print("trying to connect to server")
+	stream = StreamPeerTCP.new()
+
+	# "localhost" was not working on windows VM, had to use the IP
+	var ip = "127.0.0.1"
+	var port = _get_port()
+	var connect = stream.connect_to_host(ip, port)
+	stream.set_no_delay(true)  # TODO check if this improves performance or not
+	stream.poll()
+	# Fetch the status until it is either connected (2) or failed to connect (3)
+	while stream.get_status() < 2:
+		stream.poll()
+	return stream.get_status() == 2
+
+
+func _get_args():
+	print("getting command line arguments")
+	var arguments = {}
+	for argument in OS.get_cmdline_args():
+		print(argument)
+		if argument.find("=") > -1:
+			var key_value = argument.split("=")
+			arguments[key_value[0].lstrip("--")] = key_value[1]
+		else:
+			# Options without an argument will be present in the dictionary,
+			# with the value set to an empty string.
+			arguments[argument.lstrip("--")] = ""
+
+	return arguments
+
+
+func _get_speedup():
+	print(args)
+	return args.get("speedup", str(speed_up)).to_float()
+
+
+func _get_port():
+	return args.get("port", DEFAULT_PORT).to_int()
+
+
+func _set_seed():
+	var _seed = args.get("env_seed", DEFAULT_SEED).to_int()
+	seed(_seed)
+
+
+func _set_action_repeat():
+	action_repeat = args.get("action_repeat", str(action_repeat)).to_int()
+
+
+func disconnect_from_server():
+	stream.disconnect_from_host()
+
+
+func handle_message() -> bool:
+	# get json message: reset, step, close
+	var message = _get_dict_json_message()
+	if message["type"] == "close":
+		print("received close message, closing game")
+		get_tree().quit()
+		get_tree().set_pause(false)
+		return true
+
+	if message["type"] == "reset":
+		print("resetting all agents")
+		_reset_agents()
+		just_reset = true
+		get_tree().set_pause(false)
+		#print("resetting forcing draw")
+#        RenderingServer.force_draw()
+#        var obs = _get_obs_from_agents()
+#        print("obs ", obs)
+#        var reply = {
+#            "type": "reset",
+#            "obs": obs
+#        }
+#        _send_dict_as_json_message(reply)
+		return true
+
+	if message["type"] == "call":
+		var method = message["method"]
+		var returns = _call_method_on_agents(method)
+		var reply = {"type": "call", "returns": returns}
+		print("calling method from Python")
+		_send_dict_as_json_message(reply)
+		return handle_message()
+
+	if message["type"] == "action":
+		var action = message["action"]
+		_set_agent_actions(action, agents_training)
+		need_to_send_obs = true
+		get_tree().set_pause(false)
+		return true
+
+	print("message was not handled")
+	return false
+
+
+func _call_method_on_agents(method):
+	var returns = []
+	for agent in all_agents:
+		returns.append(agent.call(method))
+
+	return returns
+
+
+func _reset_agents_if_done(agents = all_agents):
+	for agent in agents:
+		if agent.get_done():
+			agent.set_done_false()
+
+
+func _reset_agents(agents = all_agents):
+	for agent in agents:
+		agent.needs_reset = true
+		#agent.reset()
+
+
+func _get_obs_from_agents(agents: Array = all_agents):
+	var obs = []
+	for agent in agents:
+		obs.append(agent.get_obs())
+	return obs
+
+
+func _get_reward_from_agents(agents: Array = agents_training):
+	var rewards = []
+	for agent in agents:
+		rewards.append(agent.get_reward())
+		agent.zero_reward()
+	return rewards
+
+
+func _get_info_from_agents(agents: Array = all_agents):
+	var info = []
+	for agent in agents:
+		info.append(agent.get_info())
+	return info
+
+
+func _get_done_from_agents(agents: Array = agents_training):
+	var dones = []
+	for agent in agents:
+		var done = agent.get_done()
+		if done:
+			agent.set_done_false()
+		dones.append(done)
+	return dones
+
+
+func _set_agent_actions(actions, agents: Array = all_agents):
+	for i in range(len(actions)):
+		agents[i].set_action(actions[i])
+
+
+func clamp_array(arr: Array, min: float, max: float):
+	var output: Array = []
+	for a in arr:
+		output.append(clamp(a, min, max))
+	return output
+
+
+## Save recorded export demos on window exit (Close game window instead of "Stop" button in Godot Editor)
+func _notification(what):
+	if demo_trajectories.size() == 0 or expert_demo_save_path.is_empty():
+		return
+
+	if what == NOTIFICATION_PREDELETE:
+		var json_string = JSON.stringify(demo_trajectories, "", false)
+		var file = FileAccess.open(expert_demo_save_path, FileAccess.WRITE)
+
+		if not file:
+			var error: Error = FileAccess.get_open_error()
+			assert(not error, "There was an error opening the file: %d" % error)
+
+		file.store_line(json_string)
+		var error = file.get_error()
+		assert(not error, "There was an error after trying to write to the file: %d" % error)

Game/addons/godot_rl_agents/sync.gd.uid

@@ -0,0 +1 @@
+uid://b4vk6sql5v1jl