Change directory to subfolder UnityProject

2019-08-24 23:01:28 +02:00
parent b7c8ed212a
commit fca6388ef2
3141 changed files with 0 additions and 0 deletions
@@ -0,0 +1,178 @@
+//using UnityEngine;
+//using OpenCvSharp;
+//using System.Threading.Tasks;
+
+
+//public class CannyTest : MonoBehaviour
+//{
+
+//    // Video parameters
+//    public MeshRenderer CameraRenderer;
+//    public MeshRenderer ProcessedImageRenderer;
+//    public Camera textureCamera;
+
+//    // Video size
+//    private const int imWidth = 1280; //TODO: Set width and height based on agent observation size
+//    private const int imHeight = 720;
+//    private int imFrameRate;
+
+//    // OpenCVSharp parameters
+//    private Mat videoSourceImage;
+//    private Mat cannyImage;
+//    private Texture2D processedTexture;
+//    private Vec3b[] videoSourceImageData;
+//    private byte[] cannyImageData;
+
+//    // Frame rate parameter
+//    private int updateFrameCount = 0;
+//    private int textureCount = 0;
+//    private int displayCount = 0;
+
+//    void Start()
+//    {
+//        // assign the camera texture to the meshrenderer
+//        CameraRenderer.material.mainTexture = textureCamera.targetTexture;
+
+//        // initialize video / image with given size
+//        videoSourceImage = new Mat(imHeight, imWidth, MatType.CV_8UC3);
+//        videoSourceImageData = new Vec3b[imHeight * imWidth];
+//        cannyImage = new Mat(imHeight, imWidth, MatType.CV_8UC1);
+//        cannyImageData = new byte[imHeight * imWidth];
+
+//        // create processed video texture as Texture2D object
+//        processedTexture = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
+
+//        // assign the processedTexture to the meshrenderer for display
+//        ProcessedImageRenderer.material.mainTexture = processedTexture;
+//    }
+
+
+
+//    void Update()
+//    {
+
+//        updateFrameCount++;
+
+
+
+//        if (textureCamera.targetTexture.didUpdateThisFrame)
+//        {
+
+//            textureCount++;
+
+//            // convert texture of original video to OpenCVSharp Mat object
+//            TextureToMat();
+//            // update the opencv window of source video
+//            UpdateWindow(videoSourceImage);
+//            // create the canny edge image out of source image
+//            ProcessImage(videoSourceImage);
+//            // convert the OpenCVSharp Mat of canny image to Texture2D
+//            // the texture will be displayed automatically
+//            MatToTexture();
+
+//        }
+
+//    }
+//        else
+//        {
+//            Debug.Log("Can't find camera!");
+//        }
+
+
+//        // output frame rate information
+//        if (updateFrameCount % 30 == 0)
+//        {
+//            Debug.Log("Frame count: " + updateFrameCount + ", Texture count: " + textureCount + ", Display count: " + displayCount);
+//        }
+
+
+//    }
+
+
+//    // Convert Unity Texture2D object to OpenCVSharp Mat object
+//    void TextureToMat()
+//{
+//    // Color32 array : r, g, b, a
+//    Color32[] c = _webcamTexture.GetPixels32();
+
+//    // Parallel for loop
+//    // convert Color32 object to Vec3b object
+//    // Vec3b is the representation of pixel for Mat
+//    Parallel.For(0, imHeight, i =>
+//    {
+//        for (var j = 0; j < imWidth; j++)
+//        {
+//            var col = c[j + i * imWidth];
+//            var vec3 = new Vec3b
+//            {
+//                Item0 = col.b,
+//                Item1 = col.g,
+//                Item2 = col.r
+//            };
+//            // set pixel to an array
+//            videoSourceImageData[j + i * imWidth] = vec3;
+//        }
+//    });
+//    // assign the Vec3b array to Mat
+//    videoSourceImage.SetArray(0, 0, videoSourceImageData);
+//}
+
+
+
+//// Convert OpenCVSharp Mat object to Unity Texture2D object
+//void MatToTexture()
+//{
+//    // cannyImageData is byte array, because canny image is grayscale
+//    cannyImage.GetArray(0, 0, cannyImageData);
+//    // create Color32 array that can be assigned to Texture2D directly
+//    Color32[] c = new Color32[imHeight * imWidth];
+
+//    // parallel for loop
+//    Parallel.For(0, imHeight, i =>
+//    {
+//        for (var j = 0; j < imWidth; j++)
+//        {
+//            byte vec = cannyImageData[j + i * imWidth];
+//            var color32 = new Color32
+//            {
+//                r = vec,
+//                g = vec,
+//                b = vec,
+//                a = 0
+//            };
+//            c[j + i * imWidth] = color32;
+//        }
+//    });
+
+//    processedTexture.SetPixels32(c);
+//    // to update the texture, OpenGL manner
+//    processedTexture.Apply();
+//}
+
+
+
+//// Simple example of canny edge detect
+//void ProcessImage(Mat _image)
+//{
+//    Cv2.Flip(_image, _image, FlipMode.X);
+//    Cv2.Canny(_image, cannyImage, 100, 100);
+//}
+
+
+//// Display the original video in a opencv window
+//void UpdateWindow(Mat _image)
+//{
+//    Cv2.Flip(_image, _image, FlipMode.X);
+//    Cv2.ImShow("Copy video", _image);
+//    displayCount++;
+//}
+
+//// close the opencv window
+//public void OnDestroy()
+//{
+//    Cv2.DestroyAllWindows();
+
+//}
+
+
+//}
@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 4d6afecccc2c5aa42a5c437ddf20a25c
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 
@@ -0,0 +1,192 @@
+///-----------------------------------------------------------------
+/// Namespace: <Cozmo>
+/// Class: <ImageProcessor>
+/// Description: <Converts a rendertexture to a opencv mat in order to use the canny algorithm.
+///               After processing the mat will be converted to a render texture back again.>
+/// Author: <Tobias Hassel> Date: <29.07.2019>
+/// Notes: <>
+///-----------------------------------------------------------------
+/// 
+
+using UnityEngine;
+using OpenCvSharp;
+using System.Threading.Tasks;
+
+namespace Cozmo
+{
+    public class ImageProcessor : MonoBehaviour
+    {
+        [Header("RenderTexture")]
+        [Tooltip("RenderTexture that will be passed to the LearningBrain.")]
+        public RenderTexture renderTextureCropped;
+
+        [Header("Debug Helper")]
+        [Tooltip("Reference to the MeshRenderer that will show the processed Image from Cozmo")]
+        public MeshRenderer processedImageRenderer;
+        [Tooltip("Reference to the MeshRenderer that will show the processed and cropped Image from Cozmo")]
+        public MeshRenderer processedImageRendererCropped;
+
+        /// <summary>
+        /// Center of Gravity in the cropped canny image
+        /// </summary>
+        public Point CenterOfGravity { get; private set; }
+
+        // OpenCVSharp parameters
+        private Mat cozmoImageMat;
+        private Mat cannyImage;
+        private Texture2D finalProcessedCozmoTexture;
+        private Vec3b[] cozmoImageData;
+        private byte[] cannyImageData;
+
+
+        private int imWidth = 320;          // Width of the camera image from the virtual cozmo
+        private int imHeight = 240;         // Height of the camera image from the virtual cozmo
+        private int croppedImHeight = 120;  // Height of the cropped camera image from the virtual cozmo 
+        private Camera textureCamera;       // Virtual Cozmo camera
+
+        private Texture2D originalCozmoTexture;
+
+
+        private void Start()
+        {
+            // Get reference to the cozmo camera
+            textureCamera = GetComponent<Camera>();
+
+            // Set image widths and heights based on the given RenderTextures
+            imWidth = textureCamera.targetTexture.width;
+            imHeight = textureCamera.targetTexture.height;
+
+            // assign the processed targetTexture to the renderer to display the image
+            processedImageRenderer.material.mainTexture = textureCamera.targetTexture;
+            processedImageRendererCropped.material.mainTexture = renderTextureCropped;
+
+            // initialize video / image with given size
+            cozmoImageMat = new Mat(imHeight, imWidth, MatType.CV_8UC3);
+            cozmoImageData = new Vec3b[imHeight * imWidth];
+            cannyImage = new Mat(imHeight, imWidth, MatType.CV_8UC1);
+            cannyImageData = new byte[croppedImHeight * imWidth];
+
+            originalCozmoTexture = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
+            finalProcessedCozmoTexture = new Texture2D(imWidth, croppedImHeight, TextureFormat.RGBA32, true, true);
+        }
+
+        ///// <summary>
+        ///// Gets called when a new image arrives from the camera this script lies on
+        ///// </summary>
+        ///// <param name="source"></param>
+        ///// <param name="destination"></param>
+        public void OnRenderImage(RenderTexture source, RenderTexture destination)
+        {
+            RenderTextureToTexture2D(source);
+            TextureToMat(originalCozmoTexture);
+            ProcessImage(cozmoImageMat);
+            cannyImage = CropImage(cannyImage);
+            FindCenterOfGravity(cannyImage);
+            MatToTexture(cannyImage);
+
+            Graphics.Blit(finalProcessedCozmoTexture, destination);
+            Graphics.Blit(finalProcessedCozmoTexture, renderTextureCropped);
+        }
+
+        // Crop image to just see the middle of the original image 
+        private Mat CropImage(Mat image)
+        {
+            //cut a fourth out of the top and bottom of the image 
+            OpenCvSharp.Rect rectCroped = new OpenCvSharp.Rect(0, image.Height / 4, image.Width, image.Height / 2);
+            Mat croppedImage = new Mat(image, rectCroped);
+            return croppedImage;
+        }
+
+        // Convert the rendertexture to a texture2d
+        private void RenderTextureToTexture2D(RenderTexture rTex)
+        {
+            RenderTexture.active = rTex;
+            originalCozmoTexture.ReadPixels(new UnityEngine.Rect(0, 0, rTex.width, rTex.height), 0, 0);
+            originalCozmoTexture.Apply();
+        }
+
+        // Convert Unity Texture2D object to OpenCVSharp Mat object
+        private void TextureToMat(Texture2D source)
+        {
+            // Color32 array : r, g, b, a
+            Color32[] c = source.GetPixels32();
+
+            // Parallel for loop
+            // convert Color32 object to Vec3b object
+            // Vec3b is the representation of pixel for Mat
+            Parallel.For(0, imHeight, i =>
+            {
+                for (var j = 0; j < imWidth; j++)
+                {
+                    var col = c[j + i * imWidth];
+                    var vec3 = new Vec3b
+                    {
+                        Item0 = col.b,
+                        Item1 = col.g,
+                        Item2 = col.r
+                    };
+                    // set pixel to an array
+                    cozmoImageData[j + i * imWidth] = vec3;
+                }
+            });
+
+            // assign the Vec3b array to Mat
+            cozmoImageMat.SetArray(0, 0, cozmoImageData);
+        }
+
+
+        // Simple example of canny edge detect
+        private void ProcessImage(Mat _image)
+        {
+            Cv2.Canny(_image, cannyImage, 100, 100);
+        }
+
+
+        // Convert OpenCVSharp Mat object to Unity Texture2D object
+        private void MatToTexture(Mat mat)
+        {
+            // cannyImageData is byte array, because canny image is binary
+            mat.GetArray(0, 0, cannyImageData);
+            // create Color32 array that can be assigned to Texture2D directly
+            Color32[] c = new Color32[croppedImHeight * imWidth];
+
+            // parallel for loop
+            Parallel.For(0, croppedImHeight, i =>
+            {
+                for (var j = 0; j < imWidth; j++)
+                {
+                    byte vec = cannyImageData[j + i * imWidth];
+                    var color32 = new Color32
+                    {
+                        r = vec,
+                        g = vec,
+                        b = vec,
+                        a = 0
+                    };
+                    c[j + i * imWidth] = color32;
+                }
+            });
+
+
+            finalProcessedCozmoTexture.SetPixels32(c);
+            // update texture
+            finalProcessedCozmoTexture.Apply();
+        }
+
+        // Find the Center of Gravity in the image
+        private void FindCenterOfGravity(Mat processedImage)
+        {
+            // find moments of the image
+            Moments m = new Moments(processedImage, true);
+            CenterOfGravity = new Point(m.M10 / m.M00, m.M01 / m.M00);
+
+#if UNITY_EDITOR
+            // show the image with a point mark at the centroid
+            Cv2.Circle(processedImage, CenterOfGravity, 5, new Scalar(128, 0, 0), -1);
+            Cv2.Flip(processedImage, processedImage, FlipMode.X);
+            Cv2.ImShow("Image with center", processedImage);
+#endif
+        }
+
+    }
+}
@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: b940f59f459dc49479614595d02dcbbc
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: