BachelorarbeitCozmo/Assets/Scripts/CannyEdgeDetection/ImageProcessor.cs

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using OpenCvSharp;
using System.Threading.Tasks;

public class ImageProcessor : MonoBehaviour
{
    public MeshRenderer processedImageRenderer;
    public Point CenterOfGravity { get; private set; }

    // OpenCVSharp parameters
    private Mat videoSourceImage;
    private Mat cannyImage;
    private Texture2D processedTexture;
    private Vec3b[] videoSourceImageData;
    private byte[] cannyImageData;


    private const int imWidth = 320; //TODO: Set width and height based on agent observation size
    private const int imHeight = 240;

    private Camera textureCamera;

    private void Start()
    {
        textureCamera = GetComponent<Camera>();

        //assign the processed targetTexture to the renderer to display the image
        processedImageRenderer.material.mainTexture = textureCamera.targetTexture;

        // initialize video / image with given size
        videoSourceImage = new Mat(imHeight, imWidth, MatType.CV_8UC3);
        videoSourceImageData = new Vec3b[imHeight * imWidth];
        cannyImage = new Mat(imHeight, imWidth, MatType.CV_8UC1);
        cannyImageData = new byte[imHeight * imWidth];
    }


    private void OnRenderImage(RenderTexture source, RenderTexture destination)
    {
        Texture2D tex = RenderTextureToTexture2D(source);
        videoSourceImage = TextureToMat(tex);
        cannyImage = ProcessImage(videoSourceImage);
        CenterOfGravityTest(cannyImage);
        processedTexture = MatToTexture(cannyImage);
        Graphics.Blit(processedTexture, destination);
    }

    private Texture2D RenderTextureToTexture2D(RenderTexture rTex)
    {
        Texture2D tex = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
        RenderTexture.active = rTex;
        tex.ReadPixels(new UnityEngine.Rect(0, 0, rTex.width, rTex.height), 0, 0);
        tex.Apply();
        return tex;
    }

    // Convert Unity Texture2D object to OpenCVSharp Mat object
    private Mat TextureToMat(Texture2D source)
    {
        // Color32 array : r, g, b, a
        Color32[] c = source.GetPixels32();

        // Parallel for loop
        // convert Color32 object to Vec3b object
        // Vec3b is the representation of pixel for Mat
        Parallel.For(0, imHeight, i =>
        {
            for (var j = 0; j < imWidth; j++)
            {
                var col = c[j + i * imWidth];
                var vec3 = new Vec3b
                {
                    Item0 = col.b,
                    Item1 = col.g,
                    Item2 = col.r
                };
                // set pixel to an array
                videoSourceImageData[j + i * imWidth] = vec3;
            }
        });

        // assign the Vec3b array to Mat
        Mat tmpMat = new Mat(imHeight, imWidth, MatType.CV_8UC3);
        tmpMat.SetArray(0, 0, videoSourceImageData);
        return tmpMat;
    }

    // Simple example of canny edge detect
    private Mat ProcessImage(Mat _image)
    {
        Mat cannyImg = new Mat();
        Cv2.Canny(_image, cannyImg, 100, 100);
        return cannyImg;
    }

    // Convert OpenCVSharp Mat object to Unity Texture2D object
    private Texture2D MatToTexture(Mat mat)
    {
        // cannyImageData is byte array, because canny image is grayscale
        mat.GetArray(0, 0, cannyImageData);
        // create Color32 array that can be assigned to Texture2D directly
        Color32[] c = new Color32[imHeight * imWidth];

        // parallel for loop
        Parallel.For(0, imHeight, i =>
        {
            for (var j = 0; j < imWidth; j++)
            {
                byte vec = cannyImageData[j + i * imWidth];
                var color32 = new Color32
                {
                    r = vec,
                    g = vec,
                    b = vec,
                    a = 0
                };
                c[j + i * imWidth] = color32;
            }
        });

        Texture2D texture = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
        texture.SetPixels32(c);
        // to update the texture, OpenGL manner
        texture.Apply();
        return texture;
    }


    private void CenterOfGravityTest(Mat processedImage)
    {
        // find moments of the image
        Moments m = new Moments(processedImage, true);
        CenterOfGravity = new Point(m.M10 / m.M00, m.M01 / m.M00);

#if UNITY_EDITOR
        // show the image with a point mark at the centroid
        Cv2.Circle(processedImage, CenterOfGravity, 5, new Scalar(128, 0, 0), -1);
        Cv2.Flip(processedImage, processedImage, FlipMode.X);
        Cv2.ImShow("Image with center", processedImage);
#endif
    }

}