organisation & splat changes

Assets/3_MarathonEnvs/Scripts/OriginalAgents/DeepMindHopperAgent.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using UnityEngine;
+using Unity.MLAgents;
+using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Sensors;
+public class DeepMindHopperAgent : MarathonAgent
+{
+
+    public List<float> RewardHackingVector;
+
+    public override void OnEpisodeBegin()
+    {
+        base.OnEpisodeBegin();
+
+        // set to true this to show monitor while training
+        //Monitor.SetActive(true);
+
+        StepRewardFunction = StepRewardHopper101;
+        TerminateFunction = TerminateOnNonFootHitTerrain;
+        ObservationsFunction = ObservationsDefault;
+
+        BodyParts["pelvis"] = GetComponentsInChildren<Rigidbody>().FirstOrDefault(x => x.name == "torso");
+        BodyParts["foot"] = GetComponentsInChildren<Rigidbody>().FirstOrDefault(x => x.name == "foot");
+        SetupBodyParts();
+    }
+
+    void ObservationsDefault(VectorSensor sensor)
+    {
+        if (ShowMonitor)
+        {
+        }
+
+        var pelvis = BodyParts["pelvis"];
+        Vector3 normalizedVelocity = this.GetNormalizedVelocity(pelvis.velocity);
+        sensor.AddObservation(normalizedVelocity);
+        sensor.AddObservation(pelvis.transform.forward); // gyroscope 
+        sensor.AddObservation(pelvis.transform.up);
+
+        sensor.AddObservation(SensorIsInTouch);
+        JointRotations.ForEach(x => sensor.AddObservation(x));
+        sensor.AddObservation(JointVelocity);
+        var foot = BodyParts["foot"];
+        Vector3 normalizedFootPosition = this.GetNormalizedPosition(foot.transform.position);
+        sensor.AddObservation(normalizedFootPosition.y);
+    }
+
+    float GetRewardOnEpisodeComplete()
+    {
+        return FocalPoint.transform.position.x;
+    }
+
+    void UpdateRewardHackingVector()
+    {
+        // float uprightBonus = GetForwardBonus("pelvis");
+        float uprightBonus = GetDirectionBonus("pelvis", Vector3.forward, 1f);
+        uprightBonus = Mathf.Clamp(uprightBonus, 0f, 1f);
+        float velocity = Mathf.Clamp(GetNormalizedVelocity("pelvis").x, 0f, 1f);
+        float position = Mathf.Clamp(GetNormalizedPosition("pelvis").x, 0f, 1f);
+        float effort = 1f - GetEffortNormalized();
+
+        if (RewardHackingVector?.Count == 0)
+            RewardHackingVector = Enumerable.Range(0, 6).Select(x => 0f).ToList();
+        RewardHackingVector[0] = velocity;
+        RewardHackingVector[1] = position;
+        RewardHackingVector[2] = effort;
+        RewardHackingVector[3] = uprightBonus;
+    }
+
+    float StepRewardHopper101()
+    {
+        UpdateRewardHackingVector();
+        float uprightBonus = GetDirectionBonus("pelvis", Vector3.forward, 1f);
+        uprightBonus = Mathf.Clamp(uprightBonus, 0f, 1f);
+        float velocity = Mathf.Clamp(GetNormalizedVelocity("pelvis").x, 0f, 1f);
+        // float position = Mathf.Clamp(GetNormalizedPosition("pelvis").x, 0f, 1f);
+        float effort = 1f - GetEffortNormalized();
+
+        uprightBonus *= 0.05f;
+        velocity *= 0.7f;
+        if (velocity >= .25f)
+            effort *= 0.25f;
+        else
+            effort *= velocity;
+
+        var reward = velocity
+                     + uprightBonus
+                     + effort;
+        if (ShowMonitor)
+        {
+            //var hist = new[] {reward, velocity, uprightBonus, effort};
+            //Monitor.Log("rewardHist", hist, displayType: Monitor.DisplayType.Independent);
+        }
+
+        return reward;
+    }
+}

Assets/3_MarathonEnvs/Scripts/OriginalAgents/DeepMindHopperAgent.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: c8e0e8a6cb2e04a0b89d72c85942082d
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/3_MarathonEnvs/Scripts/OriginalAgents/DeepMindWalkerAgent.cs

@@ -0,0 +1,82 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using UnityEngine;
+using Unity.MLAgents;
+using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Sensors;
+
+public class DeepMindWalkerAgent : MarathonAgent
+{
+    public override void OnEpisodeBegin()
+    {
+        base.OnEpisodeBegin();
+
+        // set to true this to show monitor while training
+        //Monitor.SetActive(true);
+
+        StepRewardFunction = StepRewardWalker106;
+        TerminateFunction = TerminateOnNonFootHitTerrain;
+        ObservationsFunction = ObservationsDefault;
+
+        BodyParts["pelvis"] = GetComponentsInChildren<Rigidbody>().FirstOrDefault(x => x.name == "torso");
+        BodyParts["left_thigh"] = GetComponentsInChildren<Rigidbody>().FirstOrDefault(x => x.name == "left_thigh");
+        BodyParts["right_thigh"] = GetComponentsInChildren<Rigidbody>().FirstOrDefault(x => x.name == "right_thigh");
+        BodyParts["right_foot"] = GetComponentsInChildren<Rigidbody>().FirstOrDefault(x => x.name == "right_foot");
+        BodyParts["left_foot"] = GetComponentsInChildren<Rigidbody>().FirstOrDefault(x => x.name == "left_foot");
+        SetupBodyParts();
+    }
+
+    void ObservationsDefault(VectorSensor sensor)
+    {
+        if (ShowMonitor)
+        {
+        }
+
+        var pelvis = BodyParts["pelvis"];
+        Vector3 normalizedVelocity = this.GetNormalizedVelocity(pelvis.velocity);
+        sensor.AddObservation(normalizedVelocity);
+        sensor.AddObservation(pelvis.transform.forward); // gyroscope 
+        sensor.AddObservation(pelvis.transform.up);
+
+        sensor.AddObservation(SensorIsInTouch);
+        JointRotations.ForEach(x => sensor.AddObservation(x));
+        sensor.AddObservation(JointVelocity);
+        sensor.AddObservation(new []{
+            this.GetNormalizedPosition(BodyParts["left_foot"].transform.position).y,
+            this.GetNormalizedPosition(BodyParts["right_foot"].transform.position).y
+        });
+    }
+
+    float StepRewardWalker106()
+    {
+        float heightPenality = 1f-GetHeightPenality(1.1f);
+        heightPenality = Mathf.Clamp(heightPenality, 0f, 1f);
+        float uprightBonus = GetDirectionBonus("pelvis", Vector3.forward, 1f);
+        uprightBonus = Mathf.Clamp(uprightBonus, 0f, 1f);
+        float velocity = Mathf.Clamp(GetNormalizedVelocity("pelvis").x, 0f, 1f);
+        float effort = 1f - GetEffortNormalized();
+
+        //if (ShowMonitor)
+        //{
+        //    var hist = new[] {velocity, uprightBonus, heightPenality, effort}.ToList();
+        //    Monitor.Log("rewardHist", hist.ToArray(), displayType: Monitor.DisplayType.Independent);
+        //}
+
+        heightPenality *= 0.05f;
+        uprightBonus *= 0.05f;
+        velocity *= 0.4f;
+        if (velocity >= .4f)
+            effort *= 0.4f;
+        else
+            effort *= velocity;
+
+        var reward = velocity
+                     + uprightBonus
+                     + heightPenality
+                     + effort;
+
+        return reward;
+    }
+}

Assets/3_MarathonEnvs/Scripts/OriginalAgents/DeepMindWalkerAgent.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 9077dc2f53ed9427e995a11ce8489a1b
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/3_MarathonEnvs/Scripts/OriginalAgents/MarathonManAgent.cs

@@ -0,0 +1,197 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+using Unity.MLAgents;
+using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Sensors;
+using System.Linq;
+using static BodyHelper002;
+
+public class MarathonManAgent : Agent, IOnTerrainCollision
+{
+	BodyManager002 _bodyManager;
+	bool _isDone;
+	bool _hasLazyInitialized;
+
+	public static BodyConfig BodyConfig = new BodyConfig
+	{
+		GetBodyPartGroup = (name) =>
+		{
+			name = name.ToLower();
+			if (name.Contains("mixamorig"))
+				return BodyPartGroup.None;
+
+			if (name.Contains("butt"))
+				return BodyPartGroup.Hips;
+			if (name.Contains("torso"))
+				return BodyPartGroup.Torso;
+			if (name.Contains("head"))
+				return BodyPartGroup.Head;
+			if (name.Contains("waist"))
+				return BodyPartGroup.Spine;
+
+			if (name.Contains("thigh"))
+				return BodyPartGroup.LegUpper;
+			if (name.Contains("shin"))
+				return BodyPartGroup.LegLower;
+			if (name.Contains("right_right_foot") || name.Contains("left_left_foot"))
+				return BodyPartGroup.Foot;
+			if (name.Contains("upper_arm"))
+				return BodyPartGroup.ArmUpper;
+			if (name.Contains("larm"))
+				return BodyPartGroup.ArmLower;
+			if (name.Contains("hand"))
+				return BodyPartGroup.Hand;
+
+			return BodyPartGroup.None;
+		},
+		GetMuscleGroup = (name) =>
+		{
+			name = name.ToLower();
+			if (name.Contains("mixamorig"))
+				return MuscleGroup.None;
+			if (name.Contains("butt"))
+				return MuscleGroup.Hips;
+			if (name.Contains("lower_waist")
+				|| name.Contains("abdomen_y"))
+				return MuscleGroup.Spine;
+			if (name.Contains("thigh")
+				|| name.Contains("hip"))
+				return MuscleGroup.LegUpper;
+			if (name.Contains("shin"))
+				return MuscleGroup.LegLower;
+			if (name.Contains("right_right_foot")
+				|| name.Contains("left_left_foot")
+				|| name.Contains("ankle_x"))
+				return MuscleGroup.Foot;
+			if (name.Contains("upper_arm"))
+				return MuscleGroup.ArmUpper;
+			if (name.Contains("larm"))
+				return MuscleGroup.ArmLower;
+			if (name.Contains("hand"))
+				return MuscleGroup.Hand;
+
+			return MuscleGroup.None;
+		},
+        GetRootBodyPart = () => BodyPartGroup.Hips,
+        GetRootMuscle = () => MuscleGroup.Hips
+    };
+
+
+
+	override public void CollectObservations(VectorSensor sensor)
+	{
+		if (!_hasLazyInitialized)
+		{
+			OnEpisodeBegin();
+		}
+
+		Vector3 normalizedVelocity = _bodyManager.GetNormalizedVelocity();
+        var pelvis = _bodyManager.GetFirstBodyPart(BodyPartGroup.Hips);
+        var shoulders = _bodyManager.GetFirstBodyPart(BodyPartGroup.Torso);
+
+        sensor.AddObservation(normalizedVelocity); 
+        sensor.AddObservation(pelvis.Rigidbody.transform.forward); // gyroscope 
+        sensor.AddObservation(pelvis.Rigidbody.transform.up);
+
+        sensor.AddObservation(shoulders.Rigidbody.transform.forward); // gyroscope 
+        sensor.AddObservation(shoulders.Rigidbody.transform.up);
+
+		sensor.AddObservation(_bodyManager.GetSensorIsInTouch());
+		foreach (var bodyPart in _bodyManager.BodyParts)
+		{
+			bodyPart.UpdateObservations();
+			sensor.AddObservation(bodyPart.ObsLocalPosition);
+			sensor.AddObservation(bodyPart.ObsRotation);
+			sensor.AddObservation(bodyPart.ObsRotationVelocity);
+			sensor.AddObservation(bodyPart.ObsVelocity);
+		}
+		sensor.AddObservation(_bodyManager.GetSensorObservations());
+
+		// _bodyManager.OnCollectObservationsHandleDebug(GetInfo());
+	}
+
+	public override void OnActionReceived(ActionBuffers actions)
+    {
+        float[] vectorAction = actions.ContinuousActions.Select(x=>x).ToArray();
+		
+		if (!_hasLazyInitialized)
+		{
+			return;
+		}
+		_isDone = false;
+		// apply actions to body
+		_bodyManager.OnAgentAction(vectorAction);
+
+		// manage reward
+        float velocity = Mathf.Clamp(_bodyManager.GetNormalizedVelocity().x, 0f, 1f);
+		var actionDifference = _bodyManager.GetActionDifference();
+		var actionsAbsolute = vectorAction.Select(x=>Mathf.Abs(x)).ToList();
+		var actionsAtLimit = actionsAbsolute.Select(x=> x>=1f ? 1f : 0f).ToList();
+		float actionaAtLimitCount = actionsAtLimit.Sum();
+        float notAtLimitBonus = 1f - (actionaAtLimitCount / (float) actionsAbsolute.Count);
+        float reducedPowerBonus = 1f - actionsAbsolute.Average();
+
+		// velocity *= 0.85f;
+		// reducedPowerBonus *=0f;
+		// notAtLimitBonus *=.1f;
+		// actionDifference *=.05f;
+        // var reward = velocity
+		// 				+ notAtLimitBonus
+		// 				+ reducedPowerBonus
+		// 				+ actionDifference;		
+        var pelvis = _bodyManager.GetFirstBodyPart(BodyPartGroup.Hips);
+		if (pelvis.Transform.position.y<0){
+			EndEpisode();
+		}
+
+        var reward = velocity;
+
+		AddReward(reward);
+		_bodyManager.SetDebugFrameReward(reward);
+	}
+
+	public override void OnEpisodeBegin()
+	{
+		if (!_hasLazyInitialized)
+		{
+			_bodyManager = GetComponent<BodyManager002>();
+			_bodyManager.BodyConfig = MarathonManAgent.BodyConfig;
+			_bodyManager.OnInitializeAgent();
+			_hasLazyInitialized = true;
+		}
+		_isDone = true;
+		_bodyManager.OnAgentReset();
+	}
+	public virtual void OnTerrainCollision(GameObject other, GameObject terrain)
+	{
+		// if (string.Compare(terrain.name, "Terrain", true) != 0)
+		if (terrain.GetComponent<Terrain>() == null)
+			return;
+		// if (!_styleAnimator.AnimationStepsReady)
+		// 	return;
+        // HACK - for when agent has not been initialized
+		if (_bodyManager == null)
+			return;
+		var bodyPart = _bodyManager.BodyParts.FirstOrDefault(x=>x.Transform.gameObject == other);
+		if (bodyPart == null)
+			return;
+		switch (bodyPart.Group)
+		{
+			case BodyHelper002.BodyPartGroup.None:
+			case BodyHelper002.BodyPartGroup.Foot:
+			// case BodyHelper002.BodyPartGroup.LegUpper:
+			case BodyHelper002.BodyPartGroup.LegLower:
+			case BodyHelper002.BodyPartGroup.Hand:
+			// case BodyHelper002.BodyPartGroup.ArmLower:
+			// case BodyHelper002.BodyPartGroup.ArmUpper:
+				break;
+			default:
+				// AddReward(-100f);
+				if (!_isDone){
+					EndEpisode();
+				}
+				break;
+		}
+	}
+}

Assets/3_MarathonEnvs/Scripts/OriginalAgents/MarathonManAgent.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 92cfc2ffe17d245ee822312e5451327c
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/3_MarathonEnvs/Scripts/OriginalAgents/MarathonTestBedDecision.cs

@@ -0,0 +1,51 @@
+using System.Collections.Generic;
+using System.Linq;
+using Unity.MLAgents;
+using UnityEngine;
+
+public class MarathonTestBedDecision //: Decision
+{
+    // Brain _brain;
+
+    // MarathonTestBedController _controller;
+
+
+    // // [Tooltip("Lock the top most element")]
+    // // /**< \brief Lock the top most element*/
+    // // public bool FreezeTop = false;
+    // // bool _lastFreezeTop;
+
+    // public override float[] Decide(
+    //     List<float> vectorObs,
+    //     List<Texture2D> visualObs,
+    //     float reward,
+    //     bool done,
+    //     List<float> memory)
+    // {
+    //     // lazy init
+    //     if (_controller == null)
+    //     {
+    //         _controller = FindObjectOfType<MarathonTestBedController>();
+    //         // _brain = GetComponent<Brain>();
+    //         // Actions = Enumerable.Repeat(0f, _brain.brainParameters.vectorActionSize[0]).ToArray();
+    //         // Actions = Enumerable.Repeat(0f, 100).ToArray();
+    //     }
+    //     if (_controller.ApplyRandomActions)
+    //     {
+    //         for (int i = 0; i < _controller.Actions.Length; i++)
+    //             _controller.Actions[i] = Random.value * 2 - 1;
+    //     }
+
+    //     return _controller.Actions;
+    // }
+
+    // public override List<float> MakeMemory(
+    //         List<float> vectorObs,
+    //         List<Texture2D> visualObs,
+    //         float reward,
+    //         bool done,
+    //         List<float> memory)
+    // {
+    //     return new List<float>();
+    // }
+}

Assets/3_MarathonEnvs/Scripts/OriginalAgents/MarathonTestBedDecision.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: fae7c31d3f13347d1bc0aab79eb5bf1f
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/3_MarathonEnvs/Scripts/OriginalAgents/OpenAIAntAgent.cs

@@ -0,0 +1,81 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using UnityEngine;
+using Unity.MLAgents;
+using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Sensors;
+
+public class OpenAIAntAgent : MarathonAgent
+{
+    public override void OnEpisodeBegin()
+    {
+        base.OnEpisodeBegin();
+
+        // set to true this to show monitor while training
+        //Monitor.SetActive(true);
+
+        StepRewardFunction = StepRewardAnt101;
+        TerminateFunction = TerminateAnt;
+        ObservationsFunction = ObservationsDefault;
+
+        BodyParts["pelvis"] = GetComponentsInChildren<Rigidbody>().FirstOrDefault(x => x.name == "torso_geom");
+        SetupBodyParts();
+    }
+
+    void ObservationsDefault(VectorSensor sensor)
+    {
+        if (ShowMonitor)
+        {
+        }
+
+        var pelvis = BodyParts["pelvis"];
+        Vector3 normalizedVelocity = GetNormalizedVelocity(pelvis.velocity);
+        sensor.AddObservation(normalizedVelocity);
+        sensor.AddObservation(pelvis.transform.forward); // gyroscope 
+        sensor.AddObservation(pelvis.transform.up);
+
+        sensor.AddObservation(SensorIsInTouch);
+        JointRotations.ForEach(x => sensor.AddObservation(x));
+        sensor.AddObservation(JointVelocity);
+        Vector3 normalizedFootPosition = this.GetNormalizedPosition(pelvis.transform.position);
+        sensor.AddObservation(normalizedFootPosition.y);
+
+    }
+
+    bool TerminateAnt()
+    {
+        var pelvis = BodyParts["pelvis"];
+		if (pelvis.transform.position.y<0){
+			return true;
+		}
+
+        var angle = GetForwardBonus("pelvis");
+        bool endOnAngle = (angle < .2f);
+        return endOnAngle;
+    }
+
+    float StepRewardAnt101()
+    {
+        float velocity = Mathf.Clamp(GetNormalizedVelocity("pelvis").x, 0f, 1f);
+        float effort = 1f - GetEffortNormalized();
+
+        velocity *= 0.7f;
+        if (velocity >= .3f)
+            effort *= 0.3f;
+        else
+            effort *= velocity;
+
+
+        var reward = velocity
+                     + effort;
+        //if (ShowMonitor)
+        //{
+        //    var hist = new[] {reward, velocity}.ToList();
+        //    Monitor.Log("rewardHist", hist.ToArray(), displayType: Monitor.DisplayType.Independent);
+        //}
+
+        return reward;
+    }
+}

Assets/3_MarathonEnvs/Scripts/OriginalAgents/OpenAIAntAgent.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: b072f3c2a55bd48149bbbd9701c7c6af
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/3_MarathonEnvs/Scripts/OriginalAgents/SparceMarathonManAgent.cs

@@ -0,0 +1,216 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+using Unity.MLAgents;
+using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Sensors;
+using System.Linq;
+using static BodyHelper002;
+using System;
+
+public class RollingAverage
+{
+	List<double> _window;
+	int _size;
+	int _count;
+	double _sum;
+	double _sumOfSquares;
+	public double Mean;
+	public double StandardDeviation;
+
+	public RollingAverage(int size)
+	{
+		_window = new List<double>(size);
+		_size = size;
+		_count = 0;
+		_sum = 0;
+		_sumOfSquares = 0;
+	}
+	public double Normalize(double val)
+	{
+		Add(val);
+		double normalized = val;
+		if (StandardDeviation != 0) 
+			normalized = (val - Mean) / StandardDeviation;
+		return normalized;
+	}
+	void Add (double val)
+	{
+		if (_count >= _size)
+		{
+			var removedVal = _window[0];
+			_window.RemoveAt(0);
+			_count--;
+			_sum -= removedVal;
+			_sumOfSquares -= removedVal * removedVal;
+		}
+		_window.Add(val);
+		_count++;
+		_sum += val;
+		_sumOfSquares += val * val;
+		// set Mean to Sum / Count, 
+		Mean = _sum / _count;
+		// set StandardDeviation to Math.Sqrt(SumOfSquares / Count - Mean * Mean).
+		StandardDeviation = Math.Sqrt(_sumOfSquares / _count - Mean * Mean);
+	}
+}
+
+public class SparceMarathonManAgent : Agent, IOnTerrainCollision
+{
+	BodyManager002 _bodyManager;
+	public float _heightReward;
+	public float _torsoUprightReward;
+	public float _torsoForwardReward;
+	public float _hipsUprightReward;
+	public float _hipsForwardReward;
+	public float _notAtLimitBonus;
+	public float _reducedPowerBonus;
+	public float _episodeMaxDistance;
+
+	static RollingAverage rollingAverage;
+
+	bool _isDone;
+	bool _hasLazyInitialized;
+
+	override public void CollectObservations(VectorSensor sensor)
+	{
+		if (!_hasLazyInitialized)
+		{
+			OnEpisodeBegin();
+		}
+
+		Vector3 normalizedVelocity = _bodyManager.GetNormalizedVelocity();
+        var pelvis = _bodyManager.GetFirstBodyPart(BodyPartGroup.Hips);
+        var shoulders = _bodyManager.GetFirstBodyPart(BodyPartGroup.Torso);
+
+        sensor.AddObservation(normalizedVelocity); 
+        sensor.AddObservation(pelvis.Rigidbody.transform.forward); // gyroscope 
+        sensor.AddObservation(pelvis.Rigidbody.transform.up);
+
+        sensor.AddObservation(shoulders.Rigidbody.transform.forward); // gyroscope 
+        sensor.AddObservation(shoulders.Rigidbody.transform.up);
+
+		sensor.AddObservation(_bodyManager.GetSensorIsInTouch());
+		foreach (var bodyPart in _bodyManager.BodyParts)
+		{
+			bodyPart.UpdateObservations();
+			sensor.AddObservation(bodyPart.ObsLocalPosition);
+			sensor.AddObservation(bodyPart.ObsRotation);
+			sensor.AddObservation(bodyPart.ObsRotationVelocity);
+			sensor.AddObservation(bodyPart.ObsVelocity);
+		}
+		sensor.AddObservation(_bodyManager.GetSensorObservations());
+
+		sensor.AddObservation(_notAtLimitBonus);
+		sensor.AddObservation(_reducedPowerBonus);
+		// _bodyManager.OnCollectObservationsHandleDebug(GetInfo());
+	}
+
+	public override void OnActionReceived(ActionBuffers actions)
+    {
+        float[] vectorAction = actions.ContinuousActions.Select(x=>x).ToArray();
+
+		if (!_hasLazyInitialized)
+		{
+			return;
+		}
+		_isDone = false;
+		// apply actions to body
+		_bodyManager.OnAgentAction(vectorAction);
+
+		// manage reward
+		var actionDifference = _bodyManager.GetActionDifference();
+		var actionsAbsolute = vectorAction.Select(x=>Mathf.Abs(x)).ToList();
+		var actionsAtLimit = actionsAbsolute.Select(x=> x>=1f ? 1f : 0f).ToList();
+		float actionaAtLimitCount = actionsAtLimit.Sum();
+        _notAtLimitBonus = 1f - (actionaAtLimitCount / (float) actionsAbsolute.Count);
+        _reducedPowerBonus = 1f - actionsAbsolute.Average();
+        _heightReward = _bodyManager.GetHeightNormalizedReward(1.2f);
+		_torsoUprightReward = _bodyManager.GetUprightNormalizedReward(BodyPartGroup.Torso);
+		_torsoForwardReward = _bodyManager.GetDirectionNormalizedReward(BodyPartGroup.Torso, Vector3.forward);
+		_hipsUprightReward = _bodyManager.GetUprightNormalizedReward(BodyPartGroup.Hips);
+		_hipsForwardReward = _bodyManager.GetDirectionNormalizedReward(BodyPartGroup.Hips, Vector3.forward);
+		_torsoUprightReward = Mathf.Clamp(_torsoUprightReward, 0f, 1f);
+		_torsoForwardReward = Mathf.Clamp(_torsoForwardReward, 0f, 1f);
+		_hipsUprightReward = Mathf.Clamp(_hipsUprightReward, 0f, 1f);
+		_hipsForwardReward = Mathf.Clamp(_hipsForwardReward, 0f, 1f);
+
+		var stepCount = StepCount > 0 ? StepCount : 1;
+		if ((stepCount >= MaxStep)
+                && (MaxStep > 0))
+        {
+            AddEpisodeEndReward();
+        }
+		else{
+			var pelvis = _bodyManager.GetFirstBodyPart(BodyPartGroup.Hips);
+			if (pelvis.Transform.position.y<0){
+	            AddEpisodeEndReward();
+				EndEpisode();
+			}
+		}
+	}
+
+	public override void OnEpisodeBegin()
+	{
+		if (!_hasLazyInitialized)
+		{
+			_bodyManager = GetComponent<BodyManager002>();
+			_bodyManager.BodyConfig = MarathonManAgent.BodyConfig;
+			_bodyManager.OnInitializeAgent();
+			_hasLazyInitialized = true;
+		}
+		_isDone = true;
+		_bodyManager.OnAgentReset();
+		_episodeMaxDistance = 0f;
+		if (rollingAverage == null)
+			rollingAverage = new RollingAverage(100);
+	}
+	public virtual void OnTerrainCollision(GameObject other, GameObject terrain)
+	{
+		// if (string.Compare(terrain.name, "Terrain", true) != 0)
+		if (terrain.GetComponent<Terrain>() == null)
+			return;
+		// if (!_styleAnimator.AnimationStepsReady)
+		// 	return;
+        // HACK - for when agent has not been initialized
+		if (_bodyManager == null)
+			return;
+		var bodyPart = _bodyManager.BodyParts.FirstOrDefault(x=>x.Transform.gameObject == other);
+		if (bodyPart == null)
+			return;
+		switch (bodyPart.Group)
+		{
+			case BodyHelper002.BodyPartGroup.Foot:
+				_episodeMaxDistance = _bodyManager.GetNormalizedPosition().x;
+				break;
+			case BodyHelper002.BodyPartGroup.None:
+			// case BodyHelper002.BodyPartGroup.LegUpper:
+			case BodyHelper002.BodyPartGroup.LegLower:
+			case BodyHelper002.BodyPartGroup.Hand:
+			// case BodyHelper002.BodyPartGroup.ArmLower:
+			// case BodyHelper002.BodyPartGroup.ArmUpper:
+				break;
+			default:
+				// AddReward(-100f);
+				if (!_isDone){
+					AddEpisodeEndReward();
+					EndEpisode();
+				}
+				break;
+		}
+	}
+
+	void AddEpisodeEndReward()
+	{
+		var reward = _episodeMaxDistance;
+
+		AddReward(reward);
+		_bodyManager.SetDebugFrameReward(reward);
+
+		// # normalized reward
+		// float normalizedReward = (float)rollingAverage.Normalize(reward);
+		// normalizedReward += (float)rollingAverage.Mean;
+		// AddReward(normalizedReward);
+		// _bodyManager.SetDebugFrameReward(normalizedReward);
+	}
+}

Assets/3_MarathonEnvs/Scripts/OriginalAgents/SparceMarathonManAgent.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 4ab14e49bdf8644ca9c2a67e85f3a11e
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/3_MarathonEnvs/Scripts/OriginalAgents/TerrainAntAgent.cs

@@ -0,0 +1,152 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using UnityEngine;
+using Unity.MLAgents;
+using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Sensors;
+
+public class TerrainAntAgent : MarathonAgent {
+
+    TerrainGenerator _terrainGenerator;
+    int _lastXPosInMeters;
+    int _stepCountAtLastMeter;
+    float _pain;
+    Vector3 _centerOfMass;
+
+    public override void OnEpisodeBegin()
+    {
+        base.OnEpisodeBegin();
+
+        BodyParts["pelvis"] = GetComponentsInChildren<Rigidbody>().FirstOrDefault(x => x.name == "torso_geom");
+
+        SetCenterOfMass();
+
+        if (_terrainGenerator == null)
+            _terrainGenerator = GetComponent<TerrainGenerator>();
+        _lastXPosInMeters = (int) BodyParts["pelvis"].transform.position.x;
+        _terrainGenerator.Reset();
+
+        // set to true this to show monitor while training
+        //Monitor.SetActive(true);
+
+        StepRewardFunction = StepRewardAnt101;
+        TerminateFunction = LocalTerminate;
+        ObservationsFunction = ObservationsDefault;
+        OnTerminateRewardValue = 0f;
+        // OnTerminateRewardValue = -100f;
+        _pain = 0f;
+
+        base.SetupBodyParts();
+        SetCenterOfMass();
+    }
+
+    bool LocalTerminate()
+    {
+        int newXPosInMeters = (int) BodyParts["pelvis"].transform.position.x;
+        if (newXPosInMeters > _lastXPosInMeters) {
+            _lastXPosInMeters = newXPosInMeters;
+            _stepCountAtLastMeter = this.StepCount;
+        }
+
+        SetCenterOfMass();
+        var xpos = _centerOfMass.x;
+        var terminate = false;
+		if (_terrainGenerator.IsPointOffEdge(BodyParts["pelvis"].transform.position)){
+            terminate = true;
+            AddReward(-1f);
+        }
+        if (this.StepCount-_stepCountAtLastMeter >= (200*5))
+            terminate = true;
+        else if (xpos < 4f && _pain > 1f)
+            terminate = true;
+        else if (xpos < 2f && _pain > 0f)
+            terminate = true;
+
+        return terminate;
+    }
+    public override void OnTerrainCollision(GameObject other, GameObject terrain) {
+        if (terrain.GetComponent<Terrain>() == null)
+            return;
+
+        switch (other.name.ToLowerInvariant().Trim())
+        {
+            case "pelvis": // dm_hopper
+                _pain += 5f;
+                NonFootHitTerrain = true;
+                break;
+            case "left_ankle_geom": // oai_ant
+            case "right_ankle_geom": // oai_ant
+            case "third_ankle_geom": // oai_ant
+            case "fourth_ankle_geom": // oai_ant
+                FootHitTerrain = true;
+                break;
+            default:
+                _pain += 5f;
+                NonFootHitTerrain = true;
+                break;
+        }
+    }
+  
+    void ObservationsDefault(VectorSensor sensor)
+    {
+        var pelvis = BodyParts["pelvis"];
+        Vector3 normalizedVelocity = GetNormalizedVelocity(pelvis.velocity);
+        sensor.AddObservation(normalizedVelocity);
+        sensor.AddObservation(pelvis.transform.forward); // gyroscope 
+        sensor.AddObservation(pelvis.transform.up);
+
+        sensor.AddObservation(SensorIsInTouch);
+        JointRotations.ForEach(x => sensor.AddObservation(x));
+        sensor.AddObservation(JointVelocity);
+        // Vector3 normalizedFootPosition = this.GetNormalizedPosition(pelvis.transform.position);
+        // sensor.AddObservation(normalizedFootPosition.y);
+
+        (List<float> distances, float fraction) = 
+            _terrainGenerator.GetDistances2d(
+                pelvis.transform.position, ShowMonitor);
+   
+        sensor.AddObservation(distances);
+        sensor.AddObservation(fraction);
+    }
+
+
+    void SetCenterOfMass()
+    {
+        _centerOfMass = Vector3.zero;
+        float c = 0f;
+        var bodyParts = this.gameObject.GetComponentsInChildren<Rigidbody>();
+ 
+        foreach (var part in bodyParts)
+        {
+            _centerOfMass += part.worldCenterOfMass * part.mass;
+            c += part.mass;
+        }
+        _centerOfMass /= c;
+    }
+
+    float StepRewardAnt101()
+    {
+        float velocity = Mathf.Clamp(GetNormalizedVelocity("pelvis").x, 0f, 1f);
+        float effort = 1f - GetEffortNormalized();
+
+        // velocity *= 0.7f;
+        // if (velocity >= .25f)
+        //     effort *= 0.25f;
+        // else
+        //     effort *= velocity;
+
+        // var reward = velocity
+        //              + effort;
+        // if (ShowMonitor)
+        // {
+        //     var hist = new[] {reward, velocity, effort};
+        //     Monitor.Log("rewardHist", hist, displayType: Monitor.DisplayType.Independent);
+        // }
+        _pain = 0f;
+        var reward = velocity;
+        return reward;
+        // return 0f;
+    }
+}

Assets/3_MarathonEnvs/Scripts/OriginalAgents/TerrainAntAgent.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 01dc2865764654ce99be0a7f7842997f
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/3_MarathonEnvs/Scripts/OriginalAgents/TerrainGenerator.cs

@@ -0,0 +1,229 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+using Unity.MLAgents;
+using System.Linq;
+using ManyWorlds;
+
+public class TerrainGenerator : MonoBehaviour
+{
+
+    Terrain terrain;
+	Agent _agent;
+	DecisionRequester _decisionRequester;
+	public int posXInTerrain;
+	public int posYInTerrain;
+	float[,] _heights;
+	float[,] _rowHeight;
+
+	public int heightIndex;
+	public float curHeight;
+	public float actionReward;
+
+	internal const float _minHeight = 0f;
+	internal const float _maxHeight = 10f;
+	internal const float _minSpawnHeight = 0f;//2f;
+	internal const float _maxSpawnHeight = 10f;//8f;
+	const float _midHeight = 5f;
+	float _mapScaleY;
+	float[,] _heightMap;
+	public List<float> debugLastHeights;
+	public List<float> debugLastNormHeights;
+	public float debugLastFraction;
+
+	PhysicsScene physicsScene;
+
+    // Start is called before the first frame update
+    void Start()
+    {
+        physicsScene = (GetComponentInParent<SpawnableEnv>().GetPhysicsScene());
+    }
+
+    // Update is called once per frame
+    void Update()
+    {
+        
+    }
+
+    public void Reset()
+    {
+		if (this.terrain == null)
+		{
+			var parent = gameObject.transform.parent;
+			terrain = parent.GetComponentInChildren<Terrain>();
+			var sharedTerrainData = terrain.terrainData;
+			terrain.terrainData = new TerrainData();
+			terrain.terrainData.heightmapResolution = sharedTerrainData.heightmapResolution;
+			terrain.terrainData.baseMapResolution = sharedTerrainData.baseMapResolution;
+			terrain.terrainData.SetDetailResolution(sharedTerrainData.detailResolution, sharedTerrainData.detailResolutionPerPatch);
+			terrain.terrainData.size = sharedTerrainData.size;
+			//terrain.terrainData.thickness = sharedTerrainData.thickness;
+			// terrain.terrainData.splatPrototypes = sharedTerrainData.splatPrototypes;
+			terrain.terrainData.terrainLayers = sharedTerrainData.terrainLayers;
+			var collider = terrain.GetComponent<TerrainCollider>();
+			collider.terrainData = terrain.terrainData;
+			_rowHeight = new float[terrain.terrainData.heightmapResolution,1];
+		}
+		if (this._agent == null)
+		{
+			_agent = GetComponent<Agent>();
+			_decisionRequester = GetComponent<DecisionRequester>();
+		}
+		//print($"HeightMap {this.terrain.terrainData.heightmapWidth}, {this.terrain.terrainData.heightmapHeight}. 
+		// Scale {this.terrain.terrainData.heightmapScale}. Resolution {this.terrain.terrainData.heightmapResolution}");
+		_mapScaleY = this.terrain.terrainData.heightmapScale.y;
+		// get the normalized position of this game object relative to the terrain
+        Vector3 tempCoord = (transform.position - terrain.gameObject.transform.position);
+        Vector3 coord;
+		tempCoord.x = Mathf.Clamp(tempCoord.x,0f, terrain.terrainData.size.x-0.000001f);
+		tempCoord.z = Mathf.Clamp(tempCoord.z,0f, terrain.terrainData.size.z-0.000001f);
+        coord.x = (tempCoord.x-1) / terrain.terrainData.size.x;
+        coord.y = tempCoord.y / terrain.terrainData.size.y;
+        coord.z = tempCoord.z / terrain.terrainData.size.z;
+        // get the position of the terrain heightmap where this game object is
+        posXInTerrain = (int) (coord.x * terrain.terrainData.heightmapResolution); 
+        posYInTerrain = (int) (coord.z * terrain.terrainData.heightmapResolution);
+        // we set an offset so that all the raising terrain is under this game object
+        int offset = 0 / 2;
+        // get the heights of the terrain under this game object
+        _heights = terrain.terrainData.GetHeights(posXInTerrain-offset,posYInTerrain-offset, 100,1);
+		curHeight = _midHeight;
+		heightIndex = posXInTerrain;
+		actionReward = 0f;
+        
+		ResetHeights();        
+    }
+	void ResetHeights()
+	{
+		if (_heightMap == null){
+			_heightMap = new float [terrain.terrainData.heightmapResolution, terrain.terrainData.heightmapResolution];
+        }
+		heightIndex = 0;
+		while(heightIndex <posXInTerrain)
+			SetNextHeight(0);
+
+		SetNextHeight(0);
+		SetNextHeight(0);
+		SetNextHeight(0);
+		SetNextHeight(0);
+		SetNextHeight(0);
+		SetNextHeight(0);
+		while(heightIndex < terrain.terrainData.heightmapResolution)
+		{
+			int action = Random.Range(0,21);
+			try
+			{
+				SetNextHeight(action);				
+			}
+			catch (System.Exception)
+			{
+				SetNextHeight(action);
+				throw;
+			}
+		}
+		this.terrain.terrainData.SetHeights(0, 0, _heightMap);
+
+	}
+	void SetNextHeight(int action)
+	{
+		float actionSize = 0f;
+		bool actionPos = (action-1) % 2 == 0;
+		if (action != 0)
+		{
+			actionSize = ((float)((action+1)/2)) * 0.1f;
+			curHeight += actionPos ? actionSize : -actionSize;
+			if (curHeight < _minSpawnHeight) {
+				curHeight = _minSpawnHeight;
+				actionSize = 0;
+			}
+			if (curHeight > _maxSpawnHeight)
+			{
+				curHeight = _maxSpawnHeight;
+				actionSize = 0;
+			}
+		}
+
+		float height = curHeight / _mapScaleY;
+		// var unit = terrain.terrainData.heightmapWidth / (int)_mapScaleY;
+		int unit = 1;
+		int startH = heightIndex * unit;
+        for (int h = startH; h < startH+unit; h++)
+        {
+            for (int w = 0; w < terrain.terrainData.heightmapResolution; w++){
+                _heightMap[w,h] = height;
+			}
+			height += 1/300f/_mapScaleY;
+        }
+		heightIndex++;
+	}
+
+	public List<float> GetDistances2d(IEnumerable<Vector3> points)
+	{
+		List<float> distances = points
+			.Select(x=> GetDistance2d(x))
+			.ToList();
+		return distances;
+	}
+	float GetDistance2d(Vector3 point)
+	{
+		int layerMask = ~(1 << 14);
+		RaycastHit hit;
+		if (!physicsScene.Raycast(point, Vector3.down, out hit,_maxHeight,layerMask))
+			return 1f;
+		float distance = hit.distance;
+		distance = Mathf.Clamp(distance, -1f, 1f);
+		return distance;
+	}
+
+	public bool IsPointOffEdge(Vector3 point)
+	{
+        Vector3 localPos = (point - terrain.gameObject.transform.position);
+		bool isOffEdge = false;
+		isOffEdge |= (localPos.z < 0f);
+		isOffEdge |= (localPos.z >= terrain.terrainData.size.z);
+		return isOffEdge;
+	}
+
+	public (List<float>, float) GetDistances2d(Vector3 pos, bool showDebug)
+	{
+		int layerMask = ~(1 << 14);
+        var xpos = pos.x;
+        xpos -= 2f;
+        float fraction = (xpos - (Mathf.Floor(xpos*5)/5)) * 5;
+        float ypos = pos.y;
+        List<Ray> rays = Enumerable.Range(0, 5*7).Select(x => new Ray(new Vector3(xpos+(x*.2f), TerrainGenerator._maxHeight, pos.z), Vector3.down)).ToList();
+		RaycastHit hit;
+		List<float> distances = rays.Select
+			( ray=> {
+				if (!physicsScene.Raycast(ray.origin, ray.direction, out hit,_maxHeight,layerMask))
+					return _maxHeight;
+				return ypos - (_maxHeight - hit.distance);
+			}).ToList();
+        if (Application.isEditor && showDebug)
+        {
+            var view = distances.Skip(10).Take(20).Select(x=>x).ToList();
+            //Monitor.Log("distances", view.ToArray());
+            var time = Time.deltaTime;
+			if (_decisionRequester?.DecisionPeriod > 1)
+				time *= _decisionRequester.DecisionPeriod;
+			for (int i = 0; i < rays.Count; i++)
+            {
+                var distance = distances[i];
+                var origin = new Vector3(rays[i].origin.x, ypos,0f);
+                var direction = distance > 0 ? Vector3.down : Vector3.up;
+                var color = distance > 0 ? Color.yellow : Color.red;
+                Debug.DrawRay(origin, direction*Mathf.Abs(distance), color, time, false);
+            }
+        }
+		List<float> normalizedDistances = distances
+			.Select(x => Mathf.Clamp(x, -10f, 10f))
+			.Select(x => x/10f)
+			.ToList();
+		;
+		debugLastNormHeights = normalizedDistances;
+		debugLastHeights = distances;
+		debugLastFraction = fraction;
+
+		return (normalizedDistances, fraction); 
+	}    
+}    

Assets/3_MarathonEnvs/Scripts/OriginalAgents/TerrainGenerator.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 4da94542694fa499d98d191423809c64
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/3_MarathonEnvs/Scripts/OriginalAgents/TerrainHopperAgent.cs

@@ -0,0 +1,167 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using UnityEngine;
+using Unity.MLAgents;
+using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Sensors;
+
+public class TerrainHopperAgent : MarathonAgent {
+
+    TerrainGenerator _terrainGenerator;
+    int _lastXPosInMeters;
+    int _stepCountAtLastMeter;
+    float _pain;
+    Vector3 _centerOfMass;
+
+    public override void OnEpisodeBegin()
+    {
+        base.OnEpisodeBegin();
+
+        BodyParts["pelvis"] = GetComponentsInChildren<Rigidbody>().FirstOrDefault(x=>x.name=="torso");
+        BodyParts["foot"] = GetComponentsInChildren<Rigidbody>().FirstOrDefault(x=>x.name=="foot");
+
+        SetCenterOfMass();
+
+        if (_terrainGenerator == null)
+            _terrainGenerator = GetComponent<TerrainGenerator>();
+        _lastXPosInMeters = (int) BodyParts["foot"].transform.position.x;
+        _terrainGenerator.Reset();
+        _stepCountAtLastMeter = 0;
+
+        // set to true this to show monitor while training
+        //Monitor.SetActive(true);
+
+        StepRewardFunction = StepRewardHopper101;
+        TerminateFunction = LocalTerminate;
+        ObservationsFunction = ObservationsDefault;
+        OnTerminateRewardValue = 0f;
+        _pain = 0f;
+
+        base.SetupBodyParts();
+        SetCenterOfMass();
+    }
+
+    bool LocalTerminate()
+    {
+        int newXPosInMeters = (int) BodyParts["foot"].transform.position.x;
+        if (newXPosInMeters > _lastXPosInMeters) {
+            _lastXPosInMeters = newXPosInMeters;
+            _stepCountAtLastMeter = this.StepCount;
+        }
+
+        SetCenterOfMass();
+        var xpos = _centerOfMass.x;
+        var terminate = false;
+        if (this.StepCount-_stepCountAtLastMeter >= (100*5))
+            terminate = true;
+        else if (xpos < 2f && _pain > 0f)
+            terminate = true;
+        else if (_pain > 1f)
+            terminate = true;
+
+        return terminate;
+    }
+    public override void OnTerrainCollision(GameObject other, GameObject terrain) {
+        if (terrain.GetComponent<Terrain>() == null)
+            return;
+
+        switch (other.name.ToLowerInvariant().Trim())
+        {
+            // case "torso": // dm_hopper
+            //     _pain += 5f;
+            //     NonFootHitTerrain = true;
+            //     break;
+            case "foot": // dm_hopper
+            case "calf": // dm_hopper
+                FootHitTerrain = true;
+                break;
+            default:
+            case "thigh": // dm_hopper
+            case "pelvis": // dm_hopper
+            case "torso": // dm_hopper
+                _pain += .5f;
+                NonFootHitTerrain = true;
+                break;
+        }
+    }
+  
+    void ObservationsDefault(VectorSensor sensor)
+    {
+        // var pelvis = BodyParts["pelvis"];
+        // sensor.AddObservation(pelvis.velocity);
+        // sensor.AddObservation(pelvis.transform.forward); // gyroscope 
+        // sensor.AddObservation(pelvis.transform.up);
+
+        // sensor.AddObservation(SensorIsInTouch);
+        // JointRotations.ForEach(x=>sensor.AddObservation(x));
+        // sensor.AddObservation(JointVelocity);
+        // var foot = BodyParts["foot"];
+        // sensor.AddObservation(foot.transform.position.y);
+
+        var pelvis = BodyParts["pelvis"];
+        Vector3 normalizedVelocity = this.GetNormalizedVelocity(pelvis.velocity);
+        sensor.AddObservation(normalizedVelocity);
+        sensor.AddObservation(pelvis.transform.forward); // gyroscope 
+        sensor.AddObservation(pelvis.transform.up);
+
+        sensor.AddObservation(SensorIsInTouch);
+        JointRotations.ForEach(x => sensor.AddObservation(x));
+        sensor.AddObservation(JointVelocity);
+        var foot = BodyParts["foot"];
+        Vector3 normalizedFootPosition = this.GetNormalizedPosition(foot.transform.position);
+        sensor.AddObservation(normalizedFootPosition.y);        
+
+        (List<float> distances, float fraction) = 
+            _terrainGenerator.GetDistances2d(
+                pelvis.transform.position, ShowMonitor);
+   
+        sensor.AddObservation(distances);
+        sensor.AddObservation(fraction);
+    }
+
+
+    void SetCenterOfMass()
+    {
+        _centerOfMass = Vector3.zero;
+        float c = 0f;
+        var bodyParts = this.gameObject.GetComponentsInChildren<Rigidbody>();
+ 
+        foreach (var part in bodyParts)
+        {
+            _centerOfMass += part.worldCenterOfMass * part.mass;
+            c += part.mass;
+        }
+        _centerOfMass /= c;
+    }
+
+    float StepRewardHopper101()
+    {
+        float uprightBonus = GetDirectionBonus("pelvis", Vector3.forward, 1f);
+        uprightBonus = Mathf.Clamp(uprightBonus, 0f, 1f);
+        float velocity = Mathf.Clamp(GetNormalizedVelocity("pelvis").x, 0f, 1f);
+        // float position = Mathf.Clamp(GetNormalizedPosition("pelvis").x, 0f, 1f);
+        float effort = 1f - GetEffortNormalized();
+
+        // uprightBonus *= 0.05f;
+        // velocity *= 0.7f;
+        // if (velocity >= .25f)
+        //     effort *= 0.25f;
+        // else
+        //     effort *= velocity;
+
+        // var reward = velocity
+        //              + uprightBonus
+        //              + effort;
+        // if (ShowMonitor)
+        // {
+        //     var hist = new[] {reward, velocity, uprightBonus, effort};
+        //     Monitor.Log("rewardHist", hist, displayType: Monitor.DisplayType.Independent);
+        // }
+        var reward = velocity;
+
+        _pain = 0f;
+        return reward;
+    }
+}

Assets/3_MarathonEnvs/Scripts/OriginalAgents/TerrainHopperAgent.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 152acbc2f18ab45f7a82449fec55784e
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/3_MarathonEnvs/Scripts/OriginalAgents/TerrainMarathonManAgent.cs

@@ -0,0 +1,183 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+using Unity.MLAgents;
+using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Sensors;
+using System.Linq;
+using static BodyHelper002;
+using ManyWorlds;
+
+public class TerrainMarathonManAgent : Agent, IOnTerrainCollision
+{
+	BodyManager002 _bodyManager;
+
+    TerrainGenerator _terrainGenerator;
+	SpawnableEnv _spawnableEnv;
+    int _stepCountAtLastMeter;
+    public int lastXPosInMeters;
+    public int maxXPosInMeters;
+	float _pain;
+
+	List<float> distances;
+	float fraction;
+	bool _hasLazyInitialized;
+
+	override public void CollectObservations(VectorSensor sensor)
+	{
+		if (!_hasLazyInitialized)
+		{
+			OnEpisodeBegin();
+		}
+
+		Vector3 normalizedVelocity = _bodyManager.GetNormalizedVelocity();
+        var pelvis = _bodyManager.GetFirstBodyPart(BodyPartGroup.Hips);
+        var shoulders = _bodyManager.GetFirstBodyPart(BodyPartGroup.Torso);
+
+        sensor.AddObservation(normalizedVelocity); 
+        sensor.AddObservation(pelvis.Rigidbody.transform.forward); // gyroscope 
+        sensor.AddObservation(pelvis.Rigidbody.transform.up);
+
+        sensor.AddObservation(shoulders.Rigidbody.transform.forward); // gyroscope 
+        sensor.AddObservation(shoulders.Rigidbody.transform.up);
+
+		sensor.AddObservation(_bodyManager.GetSensorIsInTouch());
+		foreach (var bodyPart in _bodyManager.BodyParts)
+		{
+			bodyPart.UpdateObservations();
+			sensor.AddObservation(bodyPart.ObsLocalPosition);
+			sensor.AddObservation(bodyPart.ObsRotation);
+			sensor.AddObservation(bodyPart.ObsRotationVelocity);
+			sensor.AddObservation(bodyPart.ObsVelocity);
+		}
+		sensor.AddObservation(_bodyManager.GetSensorObservations());
+        
+        (distances, fraction) = 
+            _terrainGenerator.GetDistances2d(
+                pelvis.Rigidbody.transform.position, _bodyManager.ShowMonitor);
+    
+        sensor.AddObservation(distances);
+        sensor.AddObservation(fraction);
+		// _bodyManager.OnCollectObservationsHandleDebug(GetInfo());
+	}
+
+	public override void OnActionReceived(ActionBuffers actions)
+    {
+        float[] vectorAction = actions.ContinuousActions.Select(x=>x).ToArray();
+
+		if (!_hasLazyInitialized)
+		{
+			return;
+		}
+		// apply actions to body
+		_bodyManager.OnAgentAction(vectorAction);
+
+		// manage reward
+        float velocity = Mathf.Clamp(_bodyManager.GetNormalizedVelocity().x, 0f, 1f);
+		var actionDifference = _bodyManager.GetActionDifference();
+		var actionsAbsolute = vectorAction.Select(x=>Mathf.Abs(x)).ToList();
+		var actionsAtLimit = actionsAbsolute.Select(x=> x>=1f ? 1f : 0f).ToList();
+		float actionaAtLimitCount = actionsAtLimit.Sum();
+        float notAtLimitBonus = 1f - (actionaAtLimitCount / (float) actionsAbsolute.Count);
+        float reducedPowerBonus = 1f - actionsAbsolute.Average();
+
+		// velocity *= 0.85f;
+		// reducedPowerBonus *=0f;
+		// notAtLimitBonus *=.1f;
+		// actionDifference *=.05f;
+        // var reward = velocity
+		// 				+ notAtLimitBonus
+		// 				+ reducedPowerBonus
+		// 				+ actionDifference;		
+        var reward = velocity;
+		AddReward(reward);
+		_bodyManager.SetDebugFrameReward(reward);
+
+        var pelvis = _bodyManager.GetFirstBodyPart(BodyPartGroup.Hips);
+		float xpos = 
+            _bodyManager.GetBodyParts(BodyPartGroup.Foot)
+            .Average(x=>x.Transform.position.x);
+		int newXPosInMeters = (int) xpos;
+        if (newXPosInMeters > lastXPosInMeters) {
+            lastXPosInMeters = newXPosInMeters;
+            _stepCountAtLastMeter = this.StepCount;
+        }
+		if (newXPosInMeters > maxXPosInMeters)
+			maxXPosInMeters = newXPosInMeters;
+		var terminate = false;
+		// bool isInBounds = _spawnableEnv.IsPointWithinBoundsInWorldSpace(pelvis.Transform.position);
+		// if (!isInBounds)
+        // if (pelvis.Rigidbody.transform.position.y < 0f)
+		if (_terrainGenerator.IsPointOffEdge(pelvis.Transform.position)){
+            terminate = true;
+            AddReward(-1f);
+		}
+        if (this.StepCount-_stepCountAtLastMeter >= (200*5))
+            terminate = true;
+		else if (xpos < 4f && _pain > 1f)
+            terminate = true;
+        else if (xpos < 2f && _pain > 0f)
+            terminate = true;
+		else if (_pain > 2f)
+            terminate = true;
+        if (terminate){
+			EndEpisode();
+		}
+        _pain = 0f;
+	}
+
+	public override void OnEpisodeBegin()
+	{
+		if (!_hasLazyInitialized)
+		{
+			_bodyManager = GetComponent<BodyManager002>();
+			_bodyManager.BodyConfig = MarathonManAgent.BodyConfig;
+			_bodyManager.OnInitializeAgent();
+			_hasLazyInitialized = true;
+		}
+
+		if (_bodyManager == null)
+			_bodyManager = GetComponent<BodyManager002>();
+		_bodyManager.OnAgentReset();
+        if (_terrainGenerator == null)
+            _terrainGenerator = GetComponent<TerrainGenerator>();
+		if (_spawnableEnv == null)
+			_spawnableEnv = GetComponentInParent<SpawnableEnv>();
+        _terrainGenerator.Reset();
+		lastXPosInMeters = (int)
+            _bodyManager.GetBodyParts(BodyPartGroup.Foot)
+            .Average(x=>x.Transform.position.x);
+        _pain = 0f;
+	}
+	public virtual void OnTerrainCollision(GameObject other, GameObject terrain)
+	{
+		// if (string.Compare(terrain.name, "Terrain", true) != 0)
+		if (terrain.GetComponent<Terrain>() == null)
+			return;
+		// if (!_styleAnimator.AnimationStepsReady)
+		// 	return;
+        // HACK - for when agent has not been initialized
+		if (_bodyManager == null)
+			return;
+		var bodyPart = _bodyManager.BodyParts.FirstOrDefault(x=>x.Transform.gameObject == other);
+		if (bodyPart == null)
+			return;
+		switch (bodyPart.Group)
+		{
+			case BodyHelper002.BodyPartGroup.None:
+			case BodyHelper002.BodyPartGroup.Foot:
+			case BodyHelper002.BodyPartGroup.LegLower:
+				break;
+			case BodyHelper002.BodyPartGroup.LegUpper:
+			case BodyHelper002.BodyPartGroup.Hand:
+			case BodyHelper002.BodyPartGroup.ArmLower:
+			case BodyHelper002.BodyPartGroup.ArmUpper:
+				_pain += .1f;
+				break;
+			default:
+				// AddReward(-100f);
+				_pain += 5f;
+				break;
+		}
+	}
+}

Assets/3_MarathonEnvs/Scripts/OriginalAgents/TerrainMarathonManAgent.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: cf9d05032146a423c9add69cc32051f0
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/3_MarathonEnvs/Scripts/OriginalAgents/TerrainWalkerAgent.cs

@@ -0,0 +1,159 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using UnityEngine;
+using Unity.MLAgents;
+using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Sensors;
+
+public class TerrainWalkerAgent : MarathonAgent {
+
+    TerrainGenerator _terrainGenerator;
+
+    int _lastXPosInMeters;
+    int _stepCountAtLastMeter;
+    float _pain;
+    Vector3 _centerOfMass;
+
+    public override void OnEpisodeBegin()
+    {
+        base.OnEpisodeBegin();
+
+        BodyParts["pelvis"] = GetComponentsInChildren<Rigidbody>().FirstOrDefault(x => x.name == "torso");
+        BodyParts["left_thigh"] = GetComponentsInChildren<Rigidbody>().FirstOrDefault(x => x.name == "left_thigh");
+        BodyParts["right_thigh"] = GetComponentsInChildren<Rigidbody>().FirstOrDefault(x => x.name == "right_thigh");
+        BodyParts["right_foot"] = GetComponentsInChildren<Rigidbody>().FirstOrDefault(x => x.name == "right_foot");
+        BodyParts["left_foot"] = GetComponentsInChildren<Rigidbody>().FirstOrDefault(x => x.name == "left_foot");
+
+        SetCenterOfMass();
+
+        if (_terrainGenerator == null)
+            _terrainGenerator = GetComponent<TerrainGenerator>();
+        _lastXPosInMeters = (int) BodyParts["pelvis"].transform.position.x;
+        _terrainGenerator.Reset();
+        _stepCountAtLastMeter = 0;
+
+        // set to true this to show monitor while training
+        //Monitor.SetActive(true);
+
+        StepRewardFunction = StepRewardWalker106;
+        TerminateFunction = LocalTerminate;
+        ObservationsFunction = ObservationsDefault;
+        OnTerminateRewardValue = 0f;
+        _pain = 0f;
+
+        base.SetupBodyParts();
+        SetCenterOfMass();
+    }
+
+    bool LocalTerminate()
+    {
+        int newXPosInMeters = (int) BodyParts["pelvis"].transform.position.x;
+        if (newXPosInMeters > _lastXPosInMeters) {
+            _lastXPosInMeters = newXPosInMeters;
+            _stepCountAtLastMeter = this.StepCount;
+        }
+
+        SetCenterOfMass();
+        var xpos = _centerOfMass.x;
+        var terminate = false;
+        if (this.StepCount-_stepCountAtLastMeter >= (100*5))
+            terminate = true;
+        else if (xpos < 4f && _pain > 1f)
+            terminate = true;
+        else if (xpos < 2f && _pain > 0f)
+            terminate = true;
+
+        return terminate;
+    }
+    public override void OnTerrainCollision(GameObject other, GameObject terrain) {
+        if (terrain.GetComponent<Terrain>() == null)
+            return;
+
+        switch (other.name.ToLowerInvariant().Trim())
+        {
+            case "pelvis": // dm_hopper
+                _pain += 5f;
+                NonFootHitTerrain = true;
+                break;
+            case "right_leg": // dm_walker
+            case "left_leg": // dm_walker
+            case "right_foot": // dm_walker
+            case "left_foot": // dm_walker
+                FootHitTerrain = true;
+                break;
+            default:
+                _pain += 5f;
+                NonFootHitTerrain = true;
+                break;
+        }
+    }
+  
+    void ObservationsDefault(VectorSensor sensor)
+    {
+        var pelvis = BodyParts["pelvis"];
+        Vector3 normalizedVelocity = this.GetNormalizedVelocity(pelvis.velocity);
+        sensor.AddObservation(normalizedVelocity);
+        sensor.AddObservation(pelvis.transform.forward); // gyroscope 
+        sensor.AddObservation(pelvis.transform.up);
+
+        sensor.AddObservation(SensorIsInTouch);
+        JointRotations.ForEach(x => sensor.AddObservation(x));
+        sensor.AddObservation(JointVelocity);
+        // sensor.AddObservation(new []{
+        //     this.GetNormalizedPosition(BodyParts["left_foot"].transform.position).y,
+        //     this.GetNormalizedPosition(BodyParts["right_foot"].transform.position).y
+        // });
+
+        (List<float> distances, float fraction) = 
+            _terrainGenerator.GetDistances2d(
+                pelvis.transform.position, ShowMonitor);
+   
+        sensor.AddObservation(distances);
+        sensor.AddObservation(fraction);
+    }
+
+
+    void SetCenterOfMass()
+    {
+        _centerOfMass = Vector3.zero;
+        float c = 0f;
+        var bodyParts = this.gameObject.GetComponentsInChildren<Rigidbody>();
+ 
+        foreach (var part in bodyParts)
+        {
+            _centerOfMass += part.worldCenterOfMass * part.mass;
+            c += part.mass;
+        }
+        _centerOfMass /= c;
+    }
+
+    float StepRewardWalker106()
+    {
+        float uprightBonus = GetDirectionBonus("pelvis", Vector3.forward, 1f);
+        uprightBonus = Mathf.Clamp(uprightBonus, 0f, 1f);
+        float velocity = Mathf.Clamp(GetNormalizedVelocity("pelvis").x, 0f, 1f);
+        float effort = 1f - GetEffortNormalized();
+
+        //if (ShowMonitor)
+        //{
+        //    var hist = new[] {velocity, uprightBonus, effort}.ToList();
+        //    Monitor.Log("rewardHist", hist.ToArray(), displayType: Monitor.DisplayType.Independent);
+        //}
+
+        // uprightBonus *= 0.1f;
+        // velocity *= 0.45f;
+        // if (velocity >= .45f)
+        //     effort *= 0.45f;
+        // else
+        //     effort *= velocity;
+
+        // var reward = velocity
+        //              + uprightBonus
+        //              + effort;
+        var reward = velocity;
+        _pain = 0f;
+        return reward;
+    }
+}

Assets/3_MarathonEnvs/Scripts/OriginalAgents/TerrainWalkerAgent.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 8c5a501017f084d2682727a59b227fe8
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: