BachelorarbeitCozmo/Assets/Scripts/CannyEdgeDetection/ImageProcessor.cs

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using OpenCvSharp;
using System.Threading.Tasks;

public class ImageProcessor : MonoBehaviour
{
    [Header("RenderTexture")]
    [Tooltip("RenderTexture that will be passed to the LearningBrain.")]
    public RenderTexture renderTextureCropped;

    [Header("Debug Helper")]
    [Tooltip("Reference to the MeshRenderer that will show the processed Image from Cozmo")]
    public MeshRenderer processedImageRenderer;
    [Tooltip("Reference to the MeshRenderer that will show the processed and cropped Image from Cozmo")]
    public MeshRenderer processedImageRendererCropped;

    /// <summary>
    /// Center of Gravity in the cropped canny image
    /// </summary>
    public Point CenterOfGravity { get; private set; }

    // OpenCVSharp parameters
    private Mat cozmoImageMat;
    private Mat cannyImage;
    private Texture2D finalProcessedCozmoTexture;
    private Vec3b[] cozmoImageData;
    private byte[] cannyImageData;


    private int imWidth = 320;          // Width of the camera image from the virtual cozmo
    private int imHeight = 240;         // Height of the camera image from the virtual cozmo
    private int croppedImHeight = 120;  // Height of the cropped camera image from the virtual cozmo 
    private Camera textureCamera;       // Virtual Cozmo camera

    private Texture2D originalCozmoTexture;


    private void Start()
    {
        // Get reference to the cozmo camera
        textureCamera = GetComponent<Camera>();

        // Set image widths and heights based on the given RenderTextures
        imWidth = textureCamera.targetTexture.width;
        imHeight = textureCamera.targetTexture.height;

        // assign the processed targetTexture to the renderer to display the image
        processedImageRenderer.material.mainTexture = textureCamera.targetTexture;
        processedImageRendererCropped.material.mainTexture = renderTextureCropped;

        // initialize video / image with given size
        cozmoImageMat = new Mat(imHeight, imWidth, MatType.CV_8UC3);
        cozmoImageData = new Vec3b[imHeight * imWidth];
        cannyImage = new Mat(imHeight, imWidth, MatType.CV_8UC1);
        cannyImageData = new byte[croppedImHeight * imWidth];

        originalCozmoTexture = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
        finalProcessedCozmoTexture = new Texture2D(imWidth, croppedImHeight, TextureFormat.RGBA32, true, true);
    }

    ///// <summary>
    ///// Gets called when a new image arrives from the camera this script lies on
    ///// </summary>
    ///// <param name="source"></param>
    ///// <param name="destination"></param>
    public void OnRenderImage(RenderTexture source, RenderTexture destination)
    {
        RenderTextureToTexture2D(source);
        TextureToMat(originalCozmoTexture);
        ProcessImage(cozmoImageMat);
        cannyImage = CropImage(cannyImage);
        FindCenterOfGravity(cannyImage);
        MatToTexture(cannyImage);
        Graphics.Blit(finalProcessedCozmoTexture, destination);
        Graphics.Blit(finalProcessedCozmoTexture, renderTextureCropped);
    }

    // Crop image to just see the middle of the original image 
    private Mat CropImage(Mat image)
    {
        //cut a fourth out of the top and bottom of the image 
        OpenCvSharp.Rect rectCroped = new OpenCvSharp.Rect(0, image.Height / 4, image.Width, image.Height / 2);
        Mat croppedImage = new Mat(image, rectCroped);
        return croppedImage;
    }


    private void RenderTextureToTexture2D(RenderTexture rTex)
    {
        RenderTexture.active = rTex;
        originalCozmoTexture.ReadPixels(new UnityEngine.Rect(0, 0, rTex.width, rTex.height), 0, 0);
        originalCozmoTexture.Apply();
    }

    // Convert Unity Texture2D object to OpenCVSharp Mat object
    private void TextureToMat(Texture2D source)
    {
        // Color32 array : r, g, b, a
        Color32[] c = source.GetPixels32();

        // Parallel for loop
        // convert Color32 object to Vec3b object
        // Vec3b is the representation of pixel for Mat
        Parallel.For(0, imHeight, i =>
        {
            for (var j = 0; j < imWidth; j++)
            {
                var col = c[j + i * imWidth];
                var vec3 = new Vec3b
                {
                    Item0 = col.b,
                    Item1 = col.g,
                    Item2 = col.r
                };
                // set pixel to an array
                cozmoImageData[j + i * imWidth] = vec3;
            }
        });

        // assign the Vec3b array to Mat
        cozmoImageMat.SetArray(0, 0, cozmoImageData);
    }


    // Simple example of canny edge detect
    private void ProcessImage(Mat _image)
    {
        Cv2.Canny(_image, cannyImage, 100, 100);
    }


    // Convert OpenCVSharp Mat object to Unity Texture2D object
    private void MatToTexture(Mat mat)
    {
        // cannyImageData is byte array, because canny image is grayscale
        mat.GetArray(0, 0, cannyImageData);
        // create Color32 array that can be assigned to Texture2D directly
        Color32[] c = new Color32[croppedImHeight * imWidth];

        // parallel for loop
        Parallel.For(0, croppedImHeight, i =>
        {
            for (var j = 0; j < imWidth; j++)
            {
                byte vec = cannyImageData[j + i * imWidth];
                var color32 = new Color32
                {
                    r = vec,
                    g = vec,
                    b = vec,
                    a = 0
                };
                c[j + i * imWidth] = color32;
            }
        });


        finalProcessedCozmoTexture.SetPixels32(c);
        // update texture
        finalProcessedCozmoTexture.Apply();
    }

    // Find the Center of Gravity in the image
    private void FindCenterOfGravity(Mat processedImage)
    {
        // find moments of the image
        Moments m = new Moments(processedImage, true);
        CenterOfGravity = new Point(m.M10 / m.M00, m.M01 / m.M00);

#if UNITY_EDITOR
        // show the image with a point mark at the centroid
        Cv2.Circle(processedImage, CenterOfGravity, 5, new Scalar(128, 0, 0), -1);
        Cv2.Flip(processedImage, processedImage, FlipMode.X);
        Cv2.ImShow("Image with center", processedImage);
#endif
    }

}