BachelorarbeitCozmo/Assets/Scripts/CannyEdgeDetection/ImageProcessor.cs

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using OpenCvSharp;
using System.Threading.Tasks;

public class ImageProcessor : MonoBehaviour
{
    [Header("RenderTexture")]
    [Tooltip("RenderTexture that will be passed to the LearningBrain.")]
    public RenderTexture renderTextureCropped;

    [Header("Debug Helper")]
    //[Tooltip("Reference to the MeshRenderer that will show the processed Image from Cozmo")]
    //public MeshRenderer processedImageRenderer;
    [Tooltip("Reference to the MeshRenderer that will show the processed and cropped Image from Cozmo")]
    public MeshRenderer processedImageRendererCropped;

    /// <summary>
    /// Center of Gravity in the cropped canny image
    /// </summary>
    public Point CenterOfGravity { get; private set; }

    // OpenCVSharp parameters
    private Mat videoSourceImage;
    private Mat cannyImage;
    private Texture2D processedTexture;
    private Vec3b[] videoSourceImageData;
    private byte[] cannyImageData;


    private int imWidth = 320;          // Width of the camera image from the virtual cozmo
    private int imHeight = 240;         // Height of the camera image from the virtual cozmo
    private int croppedImHeight = 120;  // Height of the cropped camera image from the virtual cozmo 
    private Camera textureCamera;       // Virtual Cozmo camera

    private void Start()
    {
        // Get reference to the cozmo camera
        textureCamera = GetComponent<Camera>();

        // Set image widths and heights based on the given RenderTextures
        imWidth = textureCamera.targetTexture.width;
        imHeight = textureCamera.targetTexture.height;
        croppedImHeight = renderTextureCropped.height;

        //assign the processed targetTexture to the renderer to display the image
        //processedImageRenderer.material.mainTexture = textureCamera.targetTexture;
        processedImageRendererCropped.material.mainTexture = renderTextureCropped;

        // initialize video / image with given size
        videoSourceImage = new Mat(imHeight, imWidth, MatType.CV_8UC3);
        videoSourceImageData = new Vec3b[imHeight * imWidth];
        cannyImage = new Mat(imHeight, imWidth, MatType.CV_8UC1);
        cannyImageData = new byte[croppedImHeight * imWidth];
    }

    /// <summary>
    /// Gets called when a new image arrives from the camera this script lies on
    /// </summary>
    /// <param name="source"></param>
    /// <param name="destination"></param>
    private void OnRenderImage(RenderTexture source, RenderTexture destination)
    {
        Texture2D tex = RenderTextureToTexture2D(source);
        videoSourceImage = TextureToMat(tex);
        cannyImage = ProcessImage(videoSourceImage);
        cannyImage = CropImage(cannyImage);
        CenterOfGravityTest(cannyImage);
        processedTexture = MatToTexture(cannyImage);
        //Graphics.Blit(processedTexture, destination);
        Graphics.Blit(processedTexture, renderTextureCropped);
    }

    // Crop image to just see the middle of the original image 
    private Mat CropImage(Mat image)
    {
        //cut a fourth out of the top and bottom of the image 
        OpenCvSharp.Rect rectCroped = new OpenCvSharp.Rect(0, image.Height / 4, image.Width, image.Height / 2);
        Mat croppedImage = new Mat(image, rectCroped);
        return croppedImage;
    }

    private Texture2D RenderTextureToTexture2D(RenderTexture rTex)
    {
        Texture2D tex = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
        RenderTexture.active = rTex;
        tex.ReadPixels(new UnityEngine.Rect(0, 0, rTex.width, rTex.height), 0, 0);
        tex.Apply();
        return tex;
    }

    // Convert Unity Texture2D object to OpenCVSharp Mat object
    private Mat TextureToMat(Texture2D source)
    {
        // Color32 array : r, g, b, a
        Color32[] c = source.GetPixels32();

        // Parallel for loop
        // convert Color32 object to Vec3b object
        // Vec3b is the representation of pixel for Mat
        Parallel.For(0, imHeight, i =>
        {
            for (var j = 0; j < imWidth; j++)
            {
                var col = c[j + i * imWidth];
                var vec3 = new Vec3b
                {
                    Item0 = col.b,
                    Item1 = col.g,
                    Item2 = col.r
                };
                // set pixel to an array
                videoSourceImageData[j + i * imWidth] = vec3;
            }
        });

        // assign the Vec3b array to Mat
        Mat tmpMat = new Mat(imHeight, imWidth, MatType.CV_8UC3);
        tmpMat.SetArray(0, 0, videoSourceImageData);
        return tmpMat;
    }

    // Simple example of canny edge detect
    private Mat ProcessImage(Mat _image)
    {
        Mat cannyImg = new Mat();
        Cv2.Canny(_image, cannyImg, 100, 100);
        return cannyImg;
    }

    // Convert OpenCVSharp Mat object to Unity Texture2D object
    private Texture2D MatToTexture(Mat mat)
    {
        // cannyImageData is byte array, because canny image is grayscale
        mat.GetArray(0, 0, cannyImageData);
        // create Color32 array that can be assigned to Texture2D directly
        Color32[] c = new Color32[croppedImHeight * imWidth];

        // parallel for loop
        Parallel.For(0, croppedImHeight, i =>
        {
            for (var j = 0; j < imWidth; j++)
            {
                byte vec = cannyImageData[j + i * imWidth];
                var color32 = new Color32
                {
                    r = vec,
                    g = vec,
                    b = vec,
                    a = 0
                };
                c[j + i * imWidth] = color32;
            }
        });

        Texture2D texture = new Texture2D(imWidth, croppedImHeight, TextureFormat.RGBA32, true, true);
        texture.SetPixels32(c);
        // to update the texture, OpenGL manner
        texture.Apply();
        return texture;
    }


    private void CenterOfGravityTest(Mat processedImage)
    {
        // find moments of the image
        Moments m = new Moments(processedImage, true);
        CenterOfGravity = new Point(m.M10 / m.M00, m.M01 / m.M00);

#if UNITY_EDITOR
        // show the image with a point mark at the centroid
        Cv2.Circle(processedImage, CenterOfGravity, 5, new Scalar(128, 0, 0), -1);
        Cv2.Flip(processedImage, processedImage, FlipMode.X);
        Cv2.ImShow("Image with center", processedImage);
#endif
    }

}