Change directory to subfolder UnityProject

UnityProject/Assets/Scripts/CannyEdgeDetection.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: dc9f178cf7c14df4ab61b1f7dcbc58d5
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/Scripts/CannyEdgeDetection/CannyTest.cs

@@ -0,0 +1,178 @@
+//using UnityEngine;
+//using OpenCvSharp;
+//using System.Threading.Tasks;
+
+
+//public class CannyTest : MonoBehaviour
+//{
+
+//    // Video parameters
+//    public MeshRenderer CameraRenderer;
+//    public MeshRenderer ProcessedImageRenderer;
+//    public Camera textureCamera;
+
+//    // Video size
+//    private const int imWidth = 1280; //TODO: Set width and height based on agent observation size
+//    private const int imHeight = 720;
+//    private int imFrameRate;
+
+//    // OpenCVSharp parameters
+//    private Mat videoSourceImage;
+//    private Mat cannyImage;
+//    private Texture2D processedTexture;
+//    private Vec3b[] videoSourceImageData;
+//    private byte[] cannyImageData;
+
+//    // Frame rate parameter
+//    private int updateFrameCount = 0;
+//    private int textureCount = 0;
+//    private int displayCount = 0;
+
+//    void Start()
+//    {
+//        // assign the camera texture to the meshrenderer
+//        CameraRenderer.material.mainTexture = textureCamera.targetTexture;
+
+//        // initialize video / image with given size
+//        videoSourceImage = new Mat(imHeight, imWidth, MatType.CV_8UC3);
+//        videoSourceImageData = new Vec3b[imHeight * imWidth];
+//        cannyImage = new Mat(imHeight, imWidth, MatType.CV_8UC1);
+//        cannyImageData = new byte[imHeight * imWidth];
+
+//        // create processed video texture as Texture2D object
+//        processedTexture = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
+
+//        // assign the processedTexture to the meshrenderer for display
+//        ProcessedImageRenderer.material.mainTexture = processedTexture;
+//    }
+
+
+
+//    void Update()
+//    {
+
+//        updateFrameCount++;
+
+
+
+//        if (textureCamera.targetTexture.didUpdateThisFrame)
+//        {
+
+//            textureCount++;
+
+//            // convert texture of original video to OpenCVSharp Mat object
+//            TextureToMat();
+//            // update the opencv window of source video
+//            UpdateWindow(videoSourceImage);
+//            // create the canny edge image out of source image
+//            ProcessImage(videoSourceImage);
+//            // convert the OpenCVSharp Mat of canny image to Texture2D
+//            // the texture will be displayed automatically
+//            MatToTexture();
+
+//        }
+
+//    }
+//        else
+//        {
+//            Debug.Log("Can't find camera!");
+//        }
+
+
+//        // output frame rate information
+//        if (updateFrameCount % 30 == 0)
+//        {
+//            Debug.Log("Frame count: " + updateFrameCount + ", Texture count: " + textureCount + ", Display count: " + displayCount);
+//        }
+
+
+//    }
+
+
+//    // Convert Unity Texture2D object to OpenCVSharp Mat object
+//    void TextureToMat()
+//{
+//    // Color32 array : r, g, b, a
+//    Color32[] c = _webcamTexture.GetPixels32();
+
+//    // Parallel for loop
+//    // convert Color32 object to Vec3b object
+//    // Vec3b is the representation of pixel for Mat
+//    Parallel.For(0, imHeight, i =>
+//    {
+//        for (var j = 0; j < imWidth; j++)
+//        {
+//            var col = c[j + i * imWidth];
+//            var vec3 = new Vec3b
+//            {
+//                Item0 = col.b,
+//                Item1 = col.g,
+//                Item2 = col.r
+//            };
+//            // set pixel to an array
+//            videoSourceImageData[j + i * imWidth] = vec3;
+//        }
+//    });
+//    // assign the Vec3b array to Mat
+//    videoSourceImage.SetArray(0, 0, videoSourceImageData);
+//}
+
+
+
+//// Convert OpenCVSharp Mat object to Unity Texture2D object
+//void MatToTexture()
+//{
+//    // cannyImageData is byte array, because canny image is grayscale
+//    cannyImage.GetArray(0, 0, cannyImageData);
+//    // create Color32 array that can be assigned to Texture2D directly
+//    Color32[] c = new Color32[imHeight * imWidth];
+
+//    // parallel for loop
+//    Parallel.For(0, imHeight, i =>
+//    {
+//        for (var j = 0; j < imWidth; j++)
+//        {
+//            byte vec = cannyImageData[j + i * imWidth];
+//            var color32 = new Color32
+//            {
+//                r = vec,
+//                g = vec,
+//                b = vec,
+//                a = 0
+//            };
+//            c[j + i * imWidth] = color32;
+//        }
+//    });
+
+//    processedTexture.SetPixels32(c);
+//    // to update the texture, OpenGL manner
+//    processedTexture.Apply();
+//}
+
+
+
+//// Simple example of canny edge detect
+//void ProcessImage(Mat _image)
+//{
+//    Cv2.Flip(_image, _image, FlipMode.X);
+//    Cv2.Canny(_image, cannyImage, 100, 100);
+//}
+
+
+//// Display the original video in a opencv window
+//void UpdateWindow(Mat _image)
+//{
+//    Cv2.Flip(_image, _image, FlipMode.X);
+//    Cv2.ImShow("Copy video", _image);
+//    displayCount++;
+//}
+
+//// close the opencv window
+//public void OnDestroy()
+//{
+//    Cv2.DestroyAllWindows();
+
+//}
+
+
+//}

UnityProject/Assets/Scripts/CannyEdgeDetection/CannyTest.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 4d6afecccc2c5aa42a5c437ddf20a25c
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/Scripts/CannyEdgeDetection/ImageProcessor.cs

@@ -0,0 +1,192 @@
+///-----------------------------------------------------------------
+/// Namespace: <Cozmo>
+/// Class: <ImageProcessor>
+/// Description: <Converts a rendertexture to a opencv mat in order to use the canny algorithm.
+///               After processing the mat will be converted to a render texture back again.>
+/// Author: <Tobias Hassel> Date: <29.07.2019>
+/// Notes: <>
+///-----------------------------------------------------------------
+/// 
+
+using UnityEngine;
+using OpenCvSharp;
+using System.Threading.Tasks;
+
+namespace Cozmo
+{
+    public class ImageProcessor : MonoBehaviour
+    {
+        [Header("RenderTexture")]
+        [Tooltip("RenderTexture that will be passed to the LearningBrain.")]
+        public RenderTexture renderTextureCropped;
+
+        [Header("Debug Helper")]
+        [Tooltip("Reference to the MeshRenderer that will show the processed Image from Cozmo")]
+        public MeshRenderer processedImageRenderer;
+        [Tooltip("Reference to the MeshRenderer that will show the processed and cropped Image from Cozmo")]
+        public MeshRenderer processedImageRendererCropped;
+
+        /// <summary>
+        /// Center of Gravity in the cropped canny image
+        /// </summary>
+        public Point CenterOfGravity { get; private set; }
+
+        // OpenCVSharp parameters
+        private Mat cozmoImageMat;
+        private Mat cannyImage;
+        private Texture2D finalProcessedCozmoTexture;
+        private Vec3b[] cozmoImageData;
+        private byte[] cannyImageData;
+
+
+        private int imWidth = 320;          // Width of the camera image from the virtual cozmo
+        private int imHeight = 240;         // Height of the camera image from the virtual cozmo
+        private int croppedImHeight = 120;  // Height of the cropped camera image from the virtual cozmo 
+        private Camera textureCamera;       // Virtual Cozmo camera
+
+        private Texture2D originalCozmoTexture;
+
+
+        private void Start()
+        {
+            // Get reference to the cozmo camera
+            textureCamera = GetComponent<Camera>();
+
+            // Set image widths and heights based on the given RenderTextures
+            imWidth = textureCamera.targetTexture.width;
+            imHeight = textureCamera.targetTexture.height;
+
+            // assign the processed targetTexture to the renderer to display the image
+            processedImageRenderer.material.mainTexture = textureCamera.targetTexture;
+            processedImageRendererCropped.material.mainTexture = renderTextureCropped;
+
+            // initialize video / image with given size
+            cozmoImageMat = new Mat(imHeight, imWidth, MatType.CV_8UC3);
+            cozmoImageData = new Vec3b[imHeight * imWidth];
+            cannyImage = new Mat(imHeight, imWidth, MatType.CV_8UC1);
+            cannyImageData = new byte[croppedImHeight * imWidth];
+
+            originalCozmoTexture = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
+            finalProcessedCozmoTexture = new Texture2D(imWidth, croppedImHeight, TextureFormat.RGBA32, true, true);
+        }
+
+        ///// <summary>
+        ///// Gets called when a new image arrives from the camera this script lies on
+        ///// </summary>
+        ///// <param name="source"></param>
+        ///// <param name="destination"></param>
+        public void OnRenderImage(RenderTexture source, RenderTexture destination)
+        {
+            RenderTextureToTexture2D(source);
+            TextureToMat(originalCozmoTexture);
+            ProcessImage(cozmoImageMat);
+            cannyImage = CropImage(cannyImage);
+            FindCenterOfGravity(cannyImage);
+            MatToTexture(cannyImage);
+
+            Graphics.Blit(finalProcessedCozmoTexture, destination);
+            Graphics.Blit(finalProcessedCozmoTexture, renderTextureCropped);
+        }
+
+        // Crop image to just see the middle of the original image 
+        private Mat CropImage(Mat image)
+        {
+            //cut a fourth out of the top and bottom of the image 
+            OpenCvSharp.Rect rectCroped = new OpenCvSharp.Rect(0, image.Height / 4, image.Width, image.Height / 2);
+            Mat croppedImage = new Mat(image, rectCroped);
+            return croppedImage;
+        }
+
+        // Convert the rendertexture to a texture2d
+        private void RenderTextureToTexture2D(RenderTexture rTex)
+        {
+            RenderTexture.active = rTex;
+            originalCozmoTexture.ReadPixels(new UnityEngine.Rect(0, 0, rTex.width, rTex.height), 0, 0);
+            originalCozmoTexture.Apply();
+        }
+
+        // Convert Unity Texture2D object to OpenCVSharp Mat object
+        private void TextureToMat(Texture2D source)
+        {
+            // Color32 array : r, g, b, a
+            Color32[] c = source.GetPixels32();
+
+            // Parallel for loop
+            // convert Color32 object to Vec3b object
+            // Vec3b is the representation of pixel for Mat
+            Parallel.For(0, imHeight, i =>
+            {
+                for (var j = 0; j < imWidth; j++)
+                {
+                    var col = c[j + i * imWidth];
+                    var vec3 = new Vec3b
+                    {
+                        Item0 = col.b,
+                        Item1 = col.g,
+                        Item2 = col.r
+                    };
+                    // set pixel to an array
+                    cozmoImageData[j + i * imWidth] = vec3;
+                }
+            });
+
+            // assign the Vec3b array to Mat
+            cozmoImageMat.SetArray(0, 0, cozmoImageData);
+        }
+
+
+        // Simple example of canny edge detect
+        private void ProcessImage(Mat _image)
+        {
+            Cv2.Canny(_image, cannyImage, 100, 100);
+        }
+
+
+        // Convert OpenCVSharp Mat object to Unity Texture2D object
+        private void MatToTexture(Mat mat)
+        {
+            // cannyImageData is byte array, because canny image is binary
+            mat.GetArray(0, 0, cannyImageData);
+            // create Color32 array that can be assigned to Texture2D directly
+            Color32[] c = new Color32[croppedImHeight * imWidth];
+
+            // parallel for loop
+            Parallel.For(0, croppedImHeight, i =>
+            {
+                for (var j = 0; j < imWidth; j++)
+                {
+                    byte vec = cannyImageData[j + i * imWidth];
+                    var color32 = new Color32
+                    {
+                        r = vec,
+                        g = vec,
+                        b = vec,
+                        a = 0
+                    };
+                    c[j + i * imWidth] = color32;
+                }
+            });
+
+
+            finalProcessedCozmoTexture.SetPixels32(c);
+            // update texture
+            finalProcessedCozmoTexture.Apply();
+        }
+
+        // Find the Center of Gravity in the image
+        private void FindCenterOfGravity(Mat processedImage)
+        {
+            // find moments of the image
+            Moments m = new Moments(processedImage, true);
+            CenterOfGravity = new Point(m.M10 / m.M00, m.M01 / m.M00);
+
+#if UNITY_EDITOR
+            // show the image with a point mark at the centroid
+            Cv2.Circle(processedImage, CenterOfGravity, 5, new Scalar(128, 0, 0), -1);
+            Cv2.Flip(processedImage, processedImage, FlipMode.X);
+            Cv2.ImShow("Image with center", processedImage);
+#endif
+        }
+
+    }
+}

UnityProject/Assets/Scripts/CannyEdgeDetection/ImageProcessor.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: b940f59f459dc49479614595d02dcbbc
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/Scripts/EnvironmentGenerator.cs

@@ -0,0 +1,121 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+
+public class EnvironmentGenerator : MonoBehaviour
+{
+    [Tooltip("Width of the whole ground plane")]
+    public float width = 1;
+    [Tooltip("Height of the whole ground plane")]
+    public float height = 1;
+
+    [Tooltip("Amount of planes drawn horizontally in the whole plane area.")]
+    [Range(1, 20)]
+    public uint amountOfHorizontalElements = 5;
+    [Tooltip("Amount of planes drawn vertically in the whole plane area.")]
+    [Range(1, 20)]
+    public uint amountOfVerticalElements = 5;
+    [Tooltip("The material the ground uses")]
+    public Material groundMaterial;
+    [Tooltip("Only for testing.")]
+    public bool testDestroy = false;
+
+    public GameObject[,] CurrentEnvironment { get; private set; }
+
+    private void Start()
+    {
+        CreateNewEnvironment();
+    }
+
+    private void Update()
+    {
+        TestEnvironmentCreation();
+    }
+
+    /// <summary>
+    /// Only for testing, can be deleted if not needed anymore
+    /// </summary>
+    private void TestEnvironmentCreation()
+    {
+        if (testDestroy)
+        {
+            UpdateEnvironment();
+            testDestroy = !testDestroy;
+        }
+    }
+
+    /// <summary>
+    /// Creates a grid of planes 
+    /// </summary>
+    public void CreateNewEnvironment()
+    {
+        CurrentEnvironment = new GameObject[amountOfHorizontalElements, amountOfVerticalElements];
+
+        float widthSingleEleme = width / amountOfHorizontalElements; //width of a single quad
+        float heightSingleElem = height / amountOfVerticalElements; //height of a single quad
+
+        // calculate the left-upper-point of the whole environment plane to use it as a startpoint for creating the grid
+        Vector3 leftUpperPoint = transform.position + (Vector3.left * (width / 2) + (Vector3.forward * (height / 2)));
+
+        Vector3 currentStartPoint = leftUpperPoint + new Vector3(widthSingleEleme / 2, 0, -heightSingleElem / 2);
+
+        //First draw all horizontal planes than do the next line in the grid
+        for (int idxX = 0; idxX < amountOfVerticalElements; idxX++)
+        {
+            CreateHorizontalPlaneLine(currentStartPoint, widthSingleEleme, heightSingleElem, idxX);
+            currentStartPoint += Vector3.back * heightSingleElem;
+        }
+
+    }
+
+    // Create a horizontal line of Planes
+    private void CreateHorizontalPlaneLine(Vector3 startPoint, float widthPlane = 1, float heightPlane = 1, int idxX = 1)
+    {
+        Vector3 currentStartPoint = startPoint;
+
+        for (int idxY = 0; idxY < amountOfHorizontalElements; idxY++)
+        {
+            CreatePlane(widthPlane, heightPlane, idxX, currentStartPoint, idxY);
+            currentStartPoint += Vector3.right * widthPlane;
+        }
+    }
+
+    // Create a single Plane 
+    private void CreatePlane(float widthPlane, float heightPlane, int idxX, Vector3 currentStartPoint, int idxY)
+    {
+        GameObject plane = GameObject.CreatePrimitive(PrimitiveType.Quad);
+        plane.transform.localEulerAngles = new Vector3(90, 0, 0);
+        plane.transform.position = currentStartPoint;
+        plane.transform.localScale = new Vector3(widthPlane, heightPlane, 1);
+        plane.transform.SetParent(this.transform);
+        plane.GetComponent<Renderer>().material = groundMaterial;
+
+        CurrentEnvironment[idxX, idxY] = plane;
+    }
+
+    /// <summary>
+    /// Destroy all Planes and set the environment array back to null
+    /// </summary>
+    public void DeleteCurrentEnvironment()
+    {
+        foreach (Transform child in transform)
+        {
+            if (child != this)
+            {
+                Destroy(child.gameObject);
+            }
+        }
+
+        CurrentEnvironment = null;
+    }
+
+    /// <summary>
+    /// Deletes the current environment if there is any and creates a new one
+    /// </summary>
+    public void UpdateEnvironment()
+    {
+        DeleteCurrentEnvironment();
+        CreateNewEnvironment();
+    }
+}

UnityProject/Assets/Scripts/EnvironmentGenerator.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 6279e2499d1811045a857f890736f7ba
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/Scripts/LineGenerator.cs

@@ -0,0 +1,8 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+
+public class LineGenerator : MonoBehaviour
+{
+    
+}

UnityProject/Assets/Scripts/LineGenerator.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: d204f699bf0f397419a5f40c948ff063
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/Scripts/ML Cozmo.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 9c7eb7dc728d65c49b1b20f4c37fb8c7
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/Scripts/ML Cozmo/CozmoAcademy.cs

@@ -0,0 +1,36 @@
+///-----------------------------------------------------------------
+/// Namespace: <Cozmo>
+/// Class: <CozmoAcademy>
+/// Description: <The academy used for cozmo. Sets cozmo back to its starting position if the academy gets reset.>
+/// Author: <Tobias Hassel> Date: <29.07.2019>
+/// Notes: <>
+///-----------------------------------------------------------------
+/// 
+
+using MLAgents;
+using UnityEngine;
+
+namespace Cozmo
+{
+    public class CozmoAcademy : Academy
+    {
+        [Tooltip("Reference to the cozmo gameobject in the scene.")]
+        public GameObject cozmo;
+        [Tooltip("Reference to the transform which represents the starting position for cozmo.")]
+        public Transform startPoint;
+
+
+        public override void AcademyReset()
+        {
+            SetCozmoBackToStartingPosition();
+        }
+
+        // Resets Cozmos position and rotation to match its starting position
+        private void SetCozmoBackToStartingPosition()
+        {
+            cozmo.transform.position = startPoint.position;
+            cozmo.transform.rotation = startPoint.rotation;
+        }
+
+    }
+}

UnityProject/Assets/Scripts/ML Cozmo/CozmoAcademy.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 23300f50ba861924c93dc5008ffb90d7
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/Scripts/ML Cozmo/CozmoAgent.cs

@@ -0,0 +1,284 @@
+///-----------------------------------------------------------------
+/// Namespace: <Cozmo>
+/// Class: <CozmoAgent>
+/// Description: <The actual agent in the scene. Collects observations and executes the actions. 
+///               Also rewards the agent and sets an actionmask.>
+/// Author: <Tobias Hassel> Date: <29.07.2019>
+/// Notes: <>
+///-----------------------------------------------------------------
+/// 
+
+using MLAgents;
+using OpenCvSharp;
+using System;
+using System.Collections.Generic;
+using UnityEngine;
+
+namespace Cozmo
+{
+    public class CozmoAgent : Agent
+    {
+        // Possible Actions
+        private const int STOP = 0;
+        private const int FORWARD = 1;
+        private const int RIGHT = 2;
+        private const int LEFT = 3;
+
+        // Used to determine different areas in the image (near to the center, far away)
+        private const float NEAR_AREA_PERCENTAGE_OFFSET = 0.3f;
+
+        [Tooltip("The virtual Cozmo camera")]
+        public Camera renderCamera;
+        [Tooltip("Reference to the CozmoMovement script")]
+        public CozmoMovementController movementController;
+        [Tooltip("The time between decisions at inference")]
+        public float timeBetweenDecisionsAtInference;
+        [Tooltip("The amout of actions that should be remembered by the agent.")]
+        public int maxStoredMovementStates = 1;
+        [Tooltip("If activated the maxStoredMovementStates has no impact on the training.")]
+        public bool useOriginalActionMasking = true;
+
+        private Academy academy;                        // CozmoAcademy
+        private float timeSinceDecision;                // time since last decision
+        private ImageProcessor imageProcessor;          // reference to the ImageProcessor
+        private int nearAreaLimit = 0;                  // X coordinate limit for the near to the imagecenter area 
+        private int centerOfImageX = 0;                 // Middle of the image in x direction
+        private MovementState lastChosenMovement = MovementState.Stop;  // The last action/movement that was executed 
+        private Queue<MovementState> lastActions = new Queue<MovementState>(); // Queue to store the last chosen Actions
+
+        private double startTime = 0;
+
+        private void Start()
+        {
+            startTime = Time.time;
+            academy = FindObjectOfType(typeof(CozmoAcademy)) as CozmoAcademy;
+            imageProcessor = renderCamera.GetComponent<ImageProcessor>();
+            nearAreaLimit = (int)(renderCamera.targetTexture.width / 2 * NEAR_AREA_PERCENTAGE_OFFSET);
+            centerOfImageX = renderCamera.targetTexture.width / 2;
+        }
+
+
+        public void FixedUpdate()
+        {
+            WaitTimeInference();
+        }
+
+
+        public override void CollectObservations()
+        {
+            SetMask();
+        }
+
+        // Set ActionMask for training
+        // Needs to be called from CollectObservations()
+        private void SetMask()
+        {
+            if (useOriginalActionMasking)
+            {
+                // Stop is never allowed 
+                switch (lastChosenMovement)
+                {
+                    // Do not allow stop decision after a stop 
+                    case (MovementState.Stop):
+                        SetActionMask(STOP);
+                        break;
+                    // Do not allow stop after forward
+                    case (MovementState.Forward):
+                        SetActionMask(STOP);
+                        break;
+                    // Do not allow stop & left after right
+                    case (MovementState.Right):
+                        SetActionMask(STOP);
+                        SetActionMask(LEFT);
+                        break;
+                    // Do not allow stop & right after left
+                    case (MovementState.Left):
+                        SetActionMask(STOP);
+                        SetActionMask(RIGHT);
+                        break;
+                    default:
+                        throw new ArgumentException("Invalid MovementState.");
+                }
+            }
+            else
+            {
+                // Do not allow stop decision after a stop 
+                if (lastChosenMovement == MovementState.Stop)
+                {
+                    SetActionMask(STOP);
+                }
+
+                // Do not allow left decision if right was in the last actions
+                if (lastActions.Contains(MovementState.Right))
+                {
+                    SetActionMask(LEFT);
+                }
+
+                // Do not allow right decision if left was in the last actions
+                if (lastActions.Contains(MovementState.Left))
+                {
+                    SetActionMask(RIGHT);
+                }
+            }
+        }
+
+        // to be implemented by the developer
+        public override void AgentAction(float[] vectorAction, string textAction)
+        {
+            double elapsedTime = Time.time - startTime;
+            //Debug.Log("Elapsed time: " + elapsedTime);
+            startTime = Time.time;
+
+            int action = Mathf.FloorToInt(vectorAction[0]);
+            Point centerOfGravity = imageProcessor.CenterOfGravity;
+
+            AddReward(-0.01f);
+
+            switch (action)
+            {
+                case STOP:
+                    movementController.currentMovementState = MovementState.Stop;
+                    lastChosenMovement = MovementState.Stop;
+                    SetReward(-0.02f);
+                    break;
+                case FORWARD:
+                    movementController.currentMovementState = MovementState.Forward;
+                    lastChosenMovement = MovementState.Forward;
+                    SetReward(0.02f);
+                    break;
+                case RIGHT:
+                    movementController.currentMovementState = MovementState.Right;
+                    lastChosenMovement = MovementState.Right;
+                    SetReward(0.01f);
+                    break;
+                case LEFT:
+                    movementController.currentMovementState = MovementState.Left;
+                    lastChosenMovement = MovementState.Left;
+                    SetReward(0.01f);
+                    break;
+                default:
+                    //movement.Move(0);
+                    throw new ArgumentException("Invalid action value. Stop movement.");
+            }
+
+            if (!useOriginalActionMasking)
+                CollectLastMovementStates(lastChosenMovement);
+
+            // Render new image after movement in order to update the centerOfGravity
+            if (renderCamera != null)
+            {
+                renderCamera.Render();
+            }
+
+            RewardAgent();
+        }
+
+        // Set the reward for the agent based on how far away the center of gravity is from the center of the image
+        private void RewardAgent()
+        {
+            float centerOfGravityX = imageProcessor.CenterOfGravity.X;
+            float reward = 0;
+
+            // Center of gravity is far away from the center (left)
+            if (centerOfGravityX <= centerOfImageX - nearAreaLimit && centerOfGravityX >= 0)
+            {
+                float range = centerOfImageX - nearAreaLimit;
+                reward = -(1 - (centerOfGravityX / range));
+                // Clamp the reward to max -1 and divide it by 2
+                reward = Mathf.Clamp(reward, -1, 0) / 2;
+            }
+            // Center of gravity is near left of the center
+            else if ((centerOfGravityX <= centerOfImageX) && (centerOfGravityX >= (centerOfImageX - nearAreaLimit)))
+            {
+                float range = centerOfImageX - (centerOfImageX - nearAreaLimit);
+                float distanceToLeftFarBorder = centerOfGravityX - (centerOfImageX - nearAreaLimit);
+                reward = (distanceToLeftFarBorder / range);
+            }
+            // Center of gravity is far away from the center (right)
+            else if ((centerOfGravityX >= (centerOfImageX + nearAreaLimit)) && (centerOfGravityX <= renderCamera.targetTexture.width))
+            {
+                float range = renderCamera.targetTexture.width - (centerOfImageX + nearAreaLimit);
+                reward = -(((centerOfGravityX - (centerOfImageX + nearAreaLimit)) / range));
+                // Clamp the reward to max -1 in order to handle rewards if the center of gravity is outside of the image
+                reward = Mathf.Clamp(reward, -1, 0) / 2;
+            }
+            // Center of gravity is near right of the center
+            else if ((centerOfGravityX >= centerOfImageX) && (centerOfGravityX <= (centerOfImageX + nearAreaLimit)))
+            {
+                float range = (centerOfImageX + nearAreaLimit) - centerOfImageX;
+                float distanceToCenterOfImage = centerOfGravityX - centerOfImageX;
+                reward = (1 - distanceToCenterOfImage / range);
+            }
+            else
+            {
+                SetReward(-1);
+                AgentReset();
+                Debug.Log("Out of image range");
+            }
+
+            Debug.Log("Reward: " + reward);
+            SetReward(reward);
+        }
+
+        // Store the last movementStates in a Queue
+        private void CollectLastMovementStates(MovementState movementState)
+        {
+            // Check if Queue exists and values should be stored
+            if ((lastActions != null) && (maxStoredMovementStates > 0))
+            {
+                // maxStoredMovementStates is reached
+                if (lastActions.Count >= maxStoredMovementStates)
+                {
+                    // deque first value(s) when maxStoredMovementStates is reached
+                    for (int i = 0; i <= (lastActions.Count - maxStoredMovementStates); i++)
+                    {
+                        lastActions.Dequeue();
+                    }
+                }
+                // add last action to queue
+                lastActions.Enqueue(movementState);
+            }
+        }
+
+        // to be implemented by the developer
+        public override void AgentReset()
+        {
+            academy.AcademyReset();
+        }
+
+
+        private void OnTriggerEnter(Collider other)
+        {
+            if (other.transform.CompareTag("Goal"))
+            {
+                Done();
+            }
+        }
+
+        private void WaitTimeInference()
+        {
+            if (renderCamera != null)
+            {
+                renderCamera.Render();
+            }
+
+            if (!academy.GetIsInference())
+            {
+                RequestDecision();
+            }
+            else
+            {
+                if (timeSinceDecision >= timeBetweenDecisionsAtInference)
+                {
+                    timeSinceDecision = 0f;
+                    RequestDecision();
+                }
+                else
+                {
+                    timeSinceDecision += Time.fixedDeltaTime;
+                }
+            }
+        }
+
+    }
+}

UnityProject/Assets/Scripts/ML Cozmo/CozmoAgent.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 234fd6e10ef55ad4b82b3153890ab5d9
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/Scripts/Movemnet.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 8400d2753da4d924bbdb3537db7363ac
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/Scripts/Movemnet/CozmoMovement.cs

@@ -0,0 +1,120 @@
+///-----------------------------------------------------------------
+/// Namespace: <Cozmo>
+/// Class: <CozmoMovement>
+/// Description: <Defines the possible movement of the virtual cozmo.>
+/// Author: <Tobias Hassel> Date: <29.07.2019>
+/// Notes: <>
+///-----------------------------------------------------------------
+/// 
+
+
+using UnityEngine;
+
+namespace Cozmo
+{
+    public class CozmoMovement : MonoBehaviour
+    {
+        private const string MOVEMENT_AXIS_NAME = "Vertical";   // name of the movement input axis
+        private const string TURN_AXIS_NAME = "Horizontal";     // name of the rotation input axis
+
+        [Tooltip("Determines how fast the robot is moving forward and backwards.")]
+        public float m_Speed = 12f;
+        [Tooltip("Determines how fast the robot is turning in degrees per second.")]
+        public float m_TurnSpeed = 180f;
+
+        private Rigidbody cozmoRigidbody;           // Reference to cozmos rigidbody
+        private float m_MovementInputValue;         // current input value for movement
+        private float m_TurnInputValue;             // current input value for rotation
+
+
+        private void Awake()
+        {
+            cozmoRigidbody = GetComponent<Rigidbody>();
+        }
+
+
+        private void OnEnable()
+        {
+            // When the cozmo is turned on, make sure it's not kinematic.
+            cozmoRigidbody.isKinematic = false;
+
+            // reset movement and rotation values
+            m_MovementInputValue = 0f;
+            m_TurnInputValue = 0f;
+        }
+
+
+        private void OnDisable()
+        {
+            // When the cozmo is turned off, set it to kinematic so it stops moving.
+            cozmoRigidbody.isKinematic = true;
+        }
+
+
+        private void Update()
+        {
+            // Store the value of both input axes.
+            m_MovementInputValue = Input.GetAxis(MOVEMENT_AXIS_NAME);
+            m_TurnInputValue = Input.GetAxis(TURN_AXIS_NAME);
+        }
+
+
+        private void FixedUpdate()
+        {
+            // Adjust position and rotation 
+            Move();
+            Turn();
+        }
+
+        /// <summary>
+        /// Move the cozmo forward (0 < directionValue <= 1) or backwards (0 > directionvalue >= -1)
+        /// </summary>
+        /// <param name="directionValue"></param>
+        public void Move(float directionValue)
+        {
+            // clamp directionValue to make sure its between -1 and 1
+            Mathf.Clamp(directionValue, -1, 1);
+
+            Vector3 movement = directionValue * transform.forward * m_Speed * Time.deltaTime;
+
+            // Apply this movement to the rigidbody's position.
+            cozmoRigidbody.MovePosition(cozmoRigidbody.position + movement);
+        }
+
+        /// <summary>
+        /// Move cozmo based on the input value
+        /// </summary>
+        public void Move()
+        {
+            Move(m_MovementInputValue);
+        }
+
+
+        /// <summary>
+        /// Rotate cozmo based on the input value
+        /// </summary>
+        public void Turn()
+        {
+            Turn(m_TurnInputValue);
+        }
+
+        /// <summary>
+        /// Rotate cozmo 
+        /// </summary>
+        /// <param name="turnValue"></param>
+        public void Turn(float turnValue)
+        {
+            // clamp turnValue to make sure its between -1 and 1
+            Mathf.Clamp(turnValue, -1, 1);
+
+            float turn = turnValue * m_TurnSpeed * Time.deltaTime;
+
+            // Make this into a rotation in the y axis.
+            Quaternion turnRotation = Quaternion.Euler(0f, turn, 0f);
+
+            // Apply this rotation to the rigidbody's rotation.
+            cozmoRigidbody.MoveRotation(cozmoRigidbody.rotation * turnRotation);
+        }
+
+    }
+}

UnityProject/Assets/Scripts/Movemnet/CozmoMovement.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: d2056048a78a0cc4c97e7899ba1c0e31
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/Scripts/Movemnet/CozmoMovementController.cs

@@ -0,0 +1,55 @@
+///-----------------------------------------------------------------
+/// Namespace: <Cozmo>
+/// Class: <CozmoMovementController>
+/// Description: <Statemachine to control the movement of the virtual cozmo.>
+/// Author: <Tobias Hassel> Date: <29.07.2019>
+/// Notes: <>
+///-----------------------------------------------------------------
+/// 
+
+using System;
+using UnityEngine;
+
+namespace Cozmo
+{
+    // Different movementstates cozmo can use
+    public enum MovementState { Stop, Forward, Right, Left }
+
+    [RequireComponent(typeof(CozmoMovement))]
+    public class CozmoMovementController : MonoBehaviour
+    {
+        [Tooltip("Current MovementState of the Robot. Change this to make the robot move.")]
+        public MovementState currentMovementState = MovementState.Stop;
+
+        private CozmoMovement movement; // Movement script of the robot
+
+
+        void Start()
+        {
+            movement = GetComponent<CozmoMovement>();
+        }
+
+        private void FixedUpdate()
+        {
+            switch (currentMovementState)
+            {
+                case MovementState.Stop:
+                    movement.Move(0);
+                    break;
+                case MovementState.Forward:
+                    movement.Move(1);
+                    break;
+                case MovementState.Right:
+                    movement.Turn(1);
+                    break;
+                case MovementState.Left:
+                    movement.Turn(-1);
+                    break;
+                default:
+                    movement.Move(0);
+                    throw new ArgumentException("No real Movementstate was given. Default 'Stop' was chosen.");
+
+            }
+        }
+    }
+}

UnityProject/Assets/Scripts/Movemnet/CozmoMovementController.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: d3ecda30187828846b48edbee97aebd7
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/Scripts/Movemnet/CozmoMovementTester.cs

@@ -0,0 +1,126 @@
+///-----------------------------------------------------------------
+/// Namespace: <CozmoHelpers>
+/// Class: <CozmoMovementTester>
+/// Description: <Used to measure the time for rotating and moving the virtual cozmo to copy the 
+///               real cozmos movement behaviour.>
+/// Author: <Tobias Hassel> Date: <29.07.2019>
+/// Notes: <>
+///-----------------------------------------------------------------
+/// 
+
+using Cozmo;
+using System.Diagnostics;
+using UnityEngine;
+
+namespace CozmoHelpers
+{
+    public enum TestStates { MovementSpeed, RotationSpeed } // enum to determine test scenarios
+
+    [RequireComponent(typeof(CozmoMovement))]
+    public class CozmoMovementTester : MonoBehaviour
+    {
+
+        private const float DISTANCE_TO_MEASURE_SPEED = 1f; // Distance that is used to calculate cozmos movementspeed (in m)
+        private const float ROTATION_ANGLE = 90f;           // Angle that is used to calculate cozmos rotationspeed 
+
+        private CozmoMovement movement;             // Reference to the movement script of cozmo
+        private Vector3 startPositionCozmo;         // Original position of cozmo
+        private float lastY;                        // Last Y of cozmo
+        private Stopwatch movementWatch;            // Stopwatch to measure the time for the movement usecase
+        private Stopwatch rotationWatch;            // Stopwatch to measure the time for the rotation usecase  
+
+        [Tooltip("Choose which speed of the virtual cozmo should be tested.")]
+        public TestStates testState = TestStates.MovementSpeed;
+
+        [Tooltip("Start and stop testing from the inspector.")]
+        public bool isTesting = false;
+
+
+        void Start()
+        {
+            movement = GetComponent<CozmoMovement>();
+            movementWatch = new Stopwatch();
+            rotationWatch = new Stopwatch();
+
+
+            startPositionCozmo = movement.transform.position;   // Cache the starting position of Cozmo    
+            lastY = movement.transform.rotation.y;              // Cache the rotation Vector of Cozmo    
+        }
+
+        private void FixedUpdate()
+        {
+            if (isTesting)
+            {
+                if (testState == TestStates.MovementSpeed)
+                {
+                    TestMovementSpeed();
+                }
+                else if (testState == TestStates.RotationSpeed)
+                {
+                    TestRotationSpeed();
+                }
+            }
+        }
+
+        // Measure time for rotation 
+        private void TestRotationSpeed()
+        {
+
+            float currentY = movement.transform.rotation.eulerAngles.y;
+
+            // If distance between start and current position of cozmo reached the  DISTANCE_TO_MEASURE_SPEED stop the stopwatch
+            if (currentY >= lastY + ROTATION_ANGLE || currentY <= lastY - ROTATION_ANGLE)
+            {
+                if (rotationWatch.IsRunning)
+                {
+                    rotationWatch.Stop();
+                    UnityEngine.Debug.Log("Cozmo: " + gameObject.name +
+                        "\nElapsed time in milliseconds (Rotation): " + rotationWatch.ElapsedMilliseconds);
+                }
+
+                isTesting = false;
+                return;
+            }
+            else
+            {
+                // Start stopwatch to measure the time cozmo needs for a specific rotation
+                if (!rotationWatch.IsRunning)
+                {
+                    rotationWatch = Stopwatch.StartNew();
+                }
+
+                // Turn right
+                movement.Turn(1);
+            }
+        }
+
+        // Measure time for movement
+        private void TestMovementSpeed()
+        {
+            // If distance between start and current position of cozmo reached the  DISTANCE_TO_MEASURE_SPEED stop the stopwatch
+            if (Vector3.Distance(startPositionCozmo, movement.transform.position) >= DISTANCE_TO_MEASURE_SPEED)
+            {
+                if (movementWatch.IsRunning)
+                {
+                    movementWatch.Stop();
+                    UnityEngine.Debug.Log("Cozmo: " + gameObject.name +
+                        "/nElapsed time in milliseconds (Movement): " + movementWatch.ElapsedMilliseconds);
+                }
+
+                isTesting = false;
+                return;
+            }
+            else
+            {
+                // Start stopwatch to measure the time cozmo needs for a specific distance
+                if (!movementWatch.IsRunning)
+                {
+                    movementWatch = Stopwatch.StartNew();
+                }
+
+                // Move cozmo in forward direction
+                movement.Move(1);
+            }
+        }
+    }
+}

UnityProject/Assets/Scripts/Movemnet/CozmoMovementTester.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 4265d0764c65d26409f0f920e5293c82
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/Scripts/SceneHelper.cs

@@ -0,0 +1,35 @@
+///-----------------------------------------------------------------
+/// Namespace: <CozmoHelpers>
+/// Class: <SceneHelper>
+/// Description: <Used to deactivate specific objects that are only used in the sceneview not in the gameview>
+/// Author: <Tobias Hassel> Date: <29.07.2019>
+/// Notes: <>
+///-----------------------------------------------------------------
+/// 
+
+using System.Collections.Generic;
+using UnityEngine;
+
+namespace CozmoHelpers
+{
+    public class SceneHelper : MonoBehaviour
+    {
+        [Tooltip("All the objects in this list will be deactivated/activated when the game is running")]
+        public List<GameObject> toggleInPlayMode;
+
+
+        public void Awake()
+        {
+            ToggleObjectList();
+        }
+
+        // Deactivate/Activate all gameObjects referenced in the toggleInPlayMode list
+        private void ToggleObjectList()
+        {
+            foreach (GameObject go in toggleInPlayMode)
+            {
+                go.SetActive(!go.activeSelf);
+            }
+        }
+    }
+}

UnityProject/Assets/Scripts/SceneHelper.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: bc64682be4f86884ebe75488344a51b2
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: