Add opencvsharp.dll / Add Rendertexture observation / add Canny

Assets/Scripts/CannyEdgeDetection.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: dc9f178cf7c14df4ab61b1f7dcbc58d5
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Scripts/CannyEdgeDetection/CannyTest.cs

@@ -0,0 +1,178 @@
+//using UnityEngine;
+//using OpenCvSharp;
+//using System.Threading.Tasks;
+
+
+//public class CannyTest : MonoBehaviour
+//{
+
+//    // Video parameters
+//    public MeshRenderer CameraRenderer;
+//    public MeshRenderer ProcessedImageRenderer;
+//    public Camera textureCamera;
+
+//    // Video size
+//    private const int imWidth = 1280; //TODO: Set width and height based on agent observation size
+//    private const int imHeight = 720;
+//    private int imFrameRate;
+
+//    // OpenCVSharp parameters
+//    private Mat videoSourceImage;
+//    private Mat cannyImage;
+//    private Texture2D processedTexture;
+//    private Vec3b[] videoSourceImageData;
+//    private byte[] cannyImageData;
+
+//    // Frame rate parameter
+//    private int updateFrameCount = 0;
+//    private int textureCount = 0;
+//    private int displayCount = 0;
+
+//    void Start()
+//    {
+//        // assign the camera texture to the meshrenderer
+//        CameraRenderer.material.mainTexture = textureCamera.targetTexture;
+
+//        // initialize video / image with given size
+//        videoSourceImage = new Mat(imHeight, imWidth, MatType.CV_8UC3);
+//        videoSourceImageData = new Vec3b[imHeight * imWidth];
+//        cannyImage = new Mat(imHeight, imWidth, MatType.CV_8UC1);
+//        cannyImageData = new byte[imHeight * imWidth];
+
+//        // create processed video texture as Texture2D object
+//        processedTexture = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
+
+//        // assign the processedTexture to the meshrenderer for display
+//        ProcessedImageRenderer.material.mainTexture = processedTexture;
+//    }
+
+
+
+//    void Update()
+//    {
+
+//        updateFrameCount++;
+
+
+
+//        if (textureCamera.targetTexture.didUpdateThisFrame)
+//        {
+
+//            textureCount++;
+
+//            // convert texture of original video to OpenCVSharp Mat object
+//            TextureToMat();
+//            // update the opencv window of source video
+//            UpdateWindow(videoSourceImage);
+//            // create the canny edge image out of source image
+//            ProcessImage(videoSourceImage);
+//            // convert the OpenCVSharp Mat of canny image to Texture2D
+//            // the texture will be displayed automatically
+//            MatToTexture();
+
+//        }
+
+//    }
+//        else
+//        {
+//            Debug.Log("Can't find camera!");
+//        }
+
+
+//        // output frame rate information
+//        if (updateFrameCount % 30 == 0)
+//        {
+//            Debug.Log("Frame count: " + updateFrameCount + ", Texture count: " + textureCount + ", Display count: " + displayCount);
+//        }
+
+
+//    }
+
+
+//    // Convert Unity Texture2D object to OpenCVSharp Mat object
+//    void TextureToMat()
+//{
+//    // Color32 array : r, g, b, a
+//    Color32[] c = _webcamTexture.GetPixels32();
+
+//    // Parallel for loop
+//    // convert Color32 object to Vec3b object
+//    // Vec3b is the representation of pixel for Mat
+//    Parallel.For(0, imHeight, i =>
+//    {
+//        for (var j = 0; j < imWidth; j++)
+//        {
+//            var col = c[j + i * imWidth];
+//            var vec3 = new Vec3b
+//            {
+//                Item0 = col.b,
+//                Item1 = col.g,
+//                Item2 = col.r
+//            };
+//            // set pixel to an array
+//            videoSourceImageData[j + i * imWidth] = vec3;
+//        }
+//    });
+//    // assign the Vec3b array to Mat
+//    videoSourceImage.SetArray(0, 0, videoSourceImageData);
+//}
+
+
+
+//// Convert OpenCVSharp Mat object to Unity Texture2D object
+//void MatToTexture()
+//{
+//    // cannyImageData is byte array, because canny image is grayscale
+//    cannyImage.GetArray(0, 0, cannyImageData);
+//    // create Color32 array that can be assigned to Texture2D directly
+//    Color32[] c = new Color32[imHeight * imWidth];
+
+//    // parallel for loop
+//    Parallel.For(0, imHeight, i =>
+//    {
+//        for (var j = 0; j < imWidth; j++)
+//        {
+//            byte vec = cannyImageData[j + i * imWidth];
+//            var color32 = new Color32
+//            {
+//                r = vec,
+//                g = vec,
+//                b = vec,
+//                a = 0
+//            };
+//            c[j + i * imWidth] = color32;
+//        }
+//    });
+
+//    processedTexture.SetPixels32(c);
+//    // to update the texture, OpenGL manner
+//    processedTexture.Apply();
+//}
+
+
+
+//// Simple example of canny edge detect
+//void ProcessImage(Mat _image)
+//{
+//    Cv2.Flip(_image, _image, FlipMode.X);
+//    Cv2.Canny(_image, cannyImage, 100, 100);
+//}
+
+
+//// Display the original video in a opencv window
+//void UpdateWindow(Mat _image)
+//{
+//    Cv2.Flip(_image, _image, FlipMode.X);
+//    Cv2.ImShow("Copy video", _image);
+//    displayCount++;
+//}
+
+//// close the opencv window
+//public void OnDestroy()
+//{
+//    Cv2.DestroyAllWindows();
+
+//}
+
+
+//}

Assets/Scripts/CannyEdgeDetection/CannyTest.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 4d6afecccc2c5aa42a5c437ddf20a25c
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Scripts/CannyEdgeDetection/OnRenderImageTest.cs

@@ -0,0 +1,127 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+using OpenCvSharp;
+using System.Threading.Tasks;
+
+public class OnRenderImageTest : MonoBehaviour
+{
+    public MeshRenderer processedImageRenderer;
+
+    // OpenCVSharp parameters
+    private Mat videoSourceImage;
+    private Mat cannyImage;
+    private Texture2D processedTexture;
+    private Vec3b[] videoSourceImageData;
+    private byte[] cannyImageData;
+
+
+    private const int imWidth = 320; //TODO: Set width and height based on agent observation size
+    private const int imHeight = 240;
+
+    private Camera textureCamera;
+
+    private void Start()
+    {
+        textureCamera = GetComponent<Camera>();
+
+        //assign the processed targetTexture to the renderer to display the image
+        processedImageRenderer.material.mainTexture = textureCamera.targetTexture;
+
+        // initialize video / image with given size
+        videoSourceImage = new Mat(imHeight, imWidth, MatType.CV_8UC3);
+        videoSourceImageData = new Vec3b[imHeight * imWidth];
+        cannyImage = new Mat(imHeight, imWidth, MatType.CV_8UC1);
+        cannyImageData = new byte[imHeight * imWidth];
+    }
+
+
+    private void OnRenderImage(RenderTexture source, RenderTexture destination)
+    {
+        Texture2D tex = RenderTextureToTexture2D(source);
+        videoSourceImage = TextureToMat(tex);
+        cannyImage = ProcessImage(videoSourceImage);
+        processedTexture = MatToTexture(cannyImage);
+        Graphics.Blit(processedTexture, destination);
+    }
+
+    private Texture2D RenderTextureToTexture2D(RenderTexture rTex)
+    {
+        Texture2D tex = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
+        RenderTexture.active = rTex;
+        tex.ReadPixels(new UnityEngine.Rect(0, 0, rTex.width, rTex.height), 0, 0);
+        tex.Apply();
+        return tex;
+    }
+
+    // Convert Unity Texture2D object to OpenCVSharp Mat object
+    private Mat TextureToMat(Texture2D source)
+    {
+        // Color32 array : r, g, b, a
+        Color32[] c = source.GetPixels32();
+
+        // Parallel for loop
+        // convert Color32 object to Vec3b object
+        // Vec3b is the representation of pixel for Mat
+        Parallel.For(0, imHeight, i =>
+        {
+            for (var j = 0; j < imWidth; j++)
+            {
+                var col = c[j + i * imWidth];
+                var vec3 = new Vec3b
+                {
+                    Item0 = col.b,
+                    Item1 = col.g,
+                    Item2 = col.r
+                };
+                // set pixel to an array
+                videoSourceImageData[j + i * imWidth] = vec3;
+            }
+        });
+
+        // assign the Vec3b array to Mat
+        Mat tmpMat = new Mat(imHeight, imWidth, MatType.CV_8UC3);
+        tmpMat.SetArray(0, 0, videoSourceImageData);
+        return tmpMat;
+    }
+
+    // Simple example of canny edge detect
+    private Mat ProcessImage(Mat _image)
+    {
+        Mat cannyImg = new Mat();
+        Cv2.Canny(_image, cannyImg, 100, 100);
+        return cannyImg;
+    }
+
+    // Convert OpenCVSharp Mat object to Unity Texture2D object
+    private Texture2D MatToTexture(Mat mat)
+    {
+        // cannyImageData is byte array, because canny image is grayscale
+        mat.GetArray(0, 0, cannyImageData);
+        // create Color32 array that can be assigned to Texture2D directly
+        Color32[] c = new Color32[imHeight * imWidth];
+
+        // parallel for loop
+        Parallel.For(0, imHeight, i =>
+        {
+            for (var j = 0; j < imWidth; j++)
+            {
+                byte vec = cannyImageData[j + i * imWidth];
+                var color32 = new Color32
+                {
+                    r = vec,
+                    g = vec,
+                    b = vec,
+                    a = 0
+                };
+                c[j + i * imWidth] = color32;
+            }
+        });
+
+        Texture2D texture = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
+        texture.SetPixels32(c);
+        // to update the texture, OpenGL manner
+        texture.Apply();
+        return texture;
+    }
+}

Assets/Scripts/CannyEdgeDetection/OnRenderImageTest.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: b940f59f459dc49479614595d02dcbbc
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Scripts/ML Cozmo/CozmoAgent.cs

@@ -6,4 +6,39 @@ using UnityEngine;
 public class CozmoAgent : Agent
 {
 
+    public Camera renderCamera;
+    public Academy academy;
+    public float timeBetweenDecisionsAtInference;
+    private float timeSinceDecision;
+
+    public void FixedUpdate()
+    {
+        WaitTimeInference();
+    }
+
+    private void WaitTimeInference()
+    {
+        if (renderCamera != null)
+        {
+            renderCamera.Render();
+        }
+
+        if (!academy.GetIsInference())
+        {
+            RequestDecision();
+        }
+        else
+        {
+            if (timeSinceDecision >= timeBetweenDecisionsAtInference)
+            {
+                timeSinceDecision = 0f;
+                RequestDecision();
+            }
+            else
+            {
+                timeSinceDecision += Time.fixedDeltaTime;
+            }
+        }
+    }
+
 }

Assets/Scripts/Movemnet/CozmoMovementTester.cs

@@ -22,6 +22,10 @@ public class CozmoMovementTester : MonoBehaviour
     [Tooltip("Choose which speed of the virtual cozmo should be tested.")]
     public TestStates testState = TestStates.MovementSpeed;
 
+    [Tooltip("Start and stop testing")]
+    public bool isTesting = false;
+
+
     void Start()
     {
         movement = GetComponent<CozmoMovement>();
@@ -35,13 +39,16 @@ public class CozmoMovementTester : MonoBehaviour
 
     private void FixedUpdate()
     {
-        if (testState == TestStates.MovementSpeed)
+        if (isTesting)
         {
-            TestMovementSpeed();
-        }
-        else if (testState == TestStates.RotationSpeed)
-        {
-            TestRotationSpeed();
+            if (testState == TestStates.MovementSpeed)
+            {
+                TestMovementSpeed();
+            }
+            else if (testState == TestStates.RotationSpeed)
+            {
+                TestRotationSpeed();
+            }
         }
     }
 
@@ -60,6 +67,7 @@ public class CozmoMovementTester : MonoBehaviour
                     "\nElapsed time in milliseconds (Rotation): " + rotationWatch.ElapsedMilliseconds);
             }
 
+            isTesting = false;
             return;
         }
         else
@@ -87,6 +95,7 @@ public class CozmoMovementTester : MonoBehaviour
                     "/nElapsed time in milliseconds (Movement): " + movementWatch.ElapsedMilliseconds);
             }
 
+            isTesting = false;
             return;
         }
         else