2019-07-29 16:29:21 +02:00
|
|
|
|
///-----------------------------------------------------------------
|
|
|
|
|
/// Namespace: <Cozmo>
|
|
|
|
|
/// Class: <ImageProcessor>
|
|
|
|
|
/// Description: <Converts a rendertexture to a opencv mat in order to use the canny algorithm.
|
|
|
|
|
/// After processing the mat will be converted to a render texture back again.>
|
|
|
|
|
/// Author: <Tobias Hassel> Date: <29.07.2019>
|
|
|
|
|
/// Notes: <>
|
|
|
|
|
///-----------------------------------------------------------------
|
|
|
|
|
///
|
|
|
|
|
|
2019-05-15 12:07:27 +02:00
|
|
|
|
using UnityEngine;
|
|
|
|
|
using OpenCvSharp;
|
|
|
|
|
using System.Threading.Tasks;
|
|
|
|
|
|
2019-07-29 16:29:21 +02:00
|
|
|
|
namespace Cozmo
|
2019-05-15 12:07:27 +02:00
|
|
|
|
{
|
2019-07-29 16:29:21 +02:00
|
|
|
|
public class ImageProcessor : MonoBehaviour
|
2019-05-15 12:07:27 +02:00
|
|
|
|
{
|
2019-07-29 16:29:21 +02:00
|
|
|
|
[Header("RenderTexture")]
|
|
|
|
|
[Tooltip("RenderTexture that will be passed to the LearningBrain.")]
|
|
|
|
|
public RenderTexture renderTextureCropped;
|
2019-05-15 12:07:27 +02:00
|
|
|
|
|
2019-07-29 16:29:21 +02:00
|
|
|
|
[Header("Debug Helper")]
|
|
|
|
|
[Tooltip("Reference to the MeshRenderer that will show the processed Image from Cozmo")]
|
|
|
|
|
public MeshRenderer processedImageRenderer;
|
|
|
|
|
[Tooltip("Reference to the MeshRenderer that will show the processed and cropped Image from Cozmo")]
|
|
|
|
|
public MeshRenderer processedImageRendererCropped;
|
2019-05-22 16:50:35 +02:00
|
|
|
|
|
2019-07-29 16:29:21 +02:00
|
|
|
|
/// <summary>
|
|
|
|
|
/// Center of Gravity in the cropped canny image
|
|
|
|
|
/// </summary>
|
|
|
|
|
public Point CenterOfGravity { get; private set; }
|
2019-05-15 12:07:27 +02:00
|
|
|
|
|
2019-07-29 16:29:21 +02:00
|
|
|
|
// OpenCVSharp parameters
|
|
|
|
|
private Mat cozmoImageMat;
|
|
|
|
|
private Mat cannyImage;
|
|
|
|
|
private Texture2D finalProcessedCozmoTexture;
|
|
|
|
|
private Vec3b[] cozmoImageData;
|
|
|
|
|
private byte[] cannyImageData;
|
2019-05-23 15:28:26 +02:00
|
|
|
|
|
2019-05-15 12:07:27 +02:00
|
|
|
|
|
2019-07-29 16:29:21 +02:00
|
|
|
|
private int imWidth = 320; // Width of the camera image from the virtual cozmo
|
|
|
|
|
private int imHeight = 240; // Height of the camera image from the virtual cozmo
|
|
|
|
|
private int croppedImHeight = 120; // Height of the cropped camera image from the virtual cozmo
|
|
|
|
|
private Camera textureCamera; // Virtual Cozmo camera
|
2019-05-22 16:50:35 +02:00
|
|
|
|
|
2019-07-29 16:29:21 +02:00
|
|
|
|
private Texture2D originalCozmoTexture;
|
2019-05-15 12:07:27 +02:00
|
|
|
|
|
2019-05-23 15:28:26 +02:00
|
|
|
|
|
2019-07-29 16:29:21 +02:00
|
|
|
|
private void Start()
|
|
|
|
|
{
|
|
|
|
|
// Get reference to the cozmo camera
|
|
|
|
|
textureCamera = GetComponent<Camera>();
|
|
|
|
|
|
|
|
|
|
// Set image widths and heights based on the given RenderTextures
|
|
|
|
|
imWidth = textureCamera.targetTexture.width;
|
|
|
|
|
imHeight = textureCamera.targetTexture.height;
|
|
|
|
|
|
|
|
|
|
// assign the processed targetTexture to the renderer to display the image
|
|
|
|
|
processedImageRenderer.material.mainTexture = textureCamera.targetTexture;
|
|
|
|
|
processedImageRendererCropped.material.mainTexture = renderTextureCropped;
|
|
|
|
|
|
|
|
|
|
// initialize video / image with given size
|
|
|
|
|
cozmoImageMat = new Mat(imHeight, imWidth, MatType.CV_8UC3);
|
|
|
|
|
cozmoImageData = new Vec3b[imHeight * imWidth];
|
|
|
|
|
cannyImage = new Mat(imHeight, imWidth, MatType.CV_8UC1);
|
|
|
|
|
cannyImageData = new byte[croppedImHeight * imWidth];
|
|
|
|
|
|
|
|
|
|
originalCozmoTexture = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
|
|
|
|
|
finalProcessedCozmoTexture = new Texture2D(imWidth, croppedImHeight, TextureFormat.RGBA32, true, true);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
///// <summary>
|
|
|
|
|
///// Gets called when a new image arrives from the camera this script lies on
|
|
|
|
|
///// </summary>
|
|
|
|
|
///// <param name="source"></param>
|
|
|
|
|
///// <param name="destination"></param>
|
|
|
|
|
public void OnRenderImage(RenderTexture source, RenderTexture destination)
|
|
|
|
|
{
|
|
|
|
|
RenderTextureToTexture2D(source);
|
|
|
|
|
TextureToMat(originalCozmoTexture);
|
|
|
|
|
ProcessImage(cozmoImageMat);
|
|
|
|
|
cannyImage = CropImage(cannyImage);
|
|
|
|
|
FindCenterOfGravity(cannyImage);
|
|
|
|
|
MatToTexture(cannyImage);
|
|
|
|
|
|
|
|
|
|
Graphics.Blit(finalProcessedCozmoTexture, destination);
|
|
|
|
|
Graphics.Blit(finalProcessedCozmoTexture, renderTextureCropped);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Crop image to just see the middle of the original image
|
|
|
|
|
private Mat CropImage(Mat image)
|
|
|
|
|
{
|
|
|
|
|
//cut a fourth out of the top and bottom of the image
|
|
|
|
|
OpenCvSharp.Rect rectCroped = new OpenCvSharp.Rect(0, image.Height / 4, image.Width, image.Height / 2);
|
|
|
|
|
Mat croppedImage = new Mat(image, rectCroped);
|
|
|
|
|
return croppedImage;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Convert the rendertexture to a texture2d
|
|
|
|
|
private void RenderTextureToTexture2D(RenderTexture rTex)
|
|
|
|
|
{
|
|
|
|
|
RenderTexture.active = rTex;
|
|
|
|
|
originalCozmoTexture.ReadPixels(new UnityEngine.Rect(0, 0, rTex.width, rTex.height), 0, 0);
|
|
|
|
|
originalCozmoTexture.Apply();
|
|
|
|
|
}
|
2019-05-15 12:07:27 +02:00
|
|
|
|
|
2019-07-29 16:29:21 +02:00
|
|
|
|
// Convert Unity Texture2D object to OpenCVSharp Mat object
|
|
|
|
|
private void TextureToMat(Texture2D source)
|
2019-05-15 12:07:27 +02:00
|
|
|
|
{
|
2019-07-29 16:29:21 +02:00
|
|
|
|
// Color32 array : r, g, b, a
|
|
|
|
|
Color32[] c = source.GetPixels32();
|
|
|
|
|
|
|
|
|
|
// Parallel for loop
|
|
|
|
|
// convert Color32 object to Vec3b object
|
|
|
|
|
// Vec3b is the representation of pixel for Mat
|
|
|
|
|
Parallel.For(0, imHeight, i =>
|
2019-05-15 12:07:27 +02:00
|
|
|
|
{
|
2019-07-29 16:29:21 +02:00
|
|
|
|
for (var j = 0; j < imWidth; j++)
|
2019-05-15 12:07:27 +02:00
|
|
|
|
{
|
2019-07-29 16:29:21 +02:00
|
|
|
|
var col = c[j + i * imWidth];
|
|
|
|
|
var vec3 = new Vec3b
|
|
|
|
|
{
|
|
|
|
|
Item0 = col.b,
|
|
|
|
|
Item1 = col.g,
|
|
|
|
|
Item2 = col.r
|
|
|
|
|
};
|
|
|
|
|
// set pixel to an array
|
|
|
|
|
cozmoImageData[j + i * imWidth] = vec3;
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// assign the Vec3b array to Mat
|
|
|
|
|
cozmoImageMat.SetArray(0, 0, cozmoImageData);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Simple example of canny edge detect
|
|
|
|
|
private void ProcessImage(Mat _image)
|
|
|
|
|
{
|
|
|
|
|
Cv2.Canny(_image, cannyImage, 100, 100);
|
|
|
|
|
}
|
2019-05-17 18:23:24 +02:00
|
|
|
|
|
2019-05-23 15:28:26 +02:00
|
|
|
|
|
2019-07-29 16:29:21 +02:00
|
|
|
|
// Convert OpenCVSharp Mat object to Unity Texture2D object
|
|
|
|
|
private void MatToTexture(Mat mat)
|
2019-05-23 15:28:26 +02:00
|
|
|
|
{
|
2019-08-17 14:45:12 +02:00
|
|
|
|
// cannyImageData is byte array, because canny image is binary
|
2019-07-29 16:29:21 +02:00
|
|
|
|
mat.GetArray(0, 0, cannyImageData);
|
|
|
|
|
// create Color32 array that can be assigned to Texture2D directly
|
|
|
|
|
Color32[] c = new Color32[croppedImHeight * imWidth];
|
|
|
|
|
|
|
|
|
|
// parallel for loop
|
|
|
|
|
Parallel.For(0, croppedImHeight, i =>
|
2019-05-23 15:28:26 +02:00
|
|
|
|
{
|
2019-07-29 16:29:21 +02:00
|
|
|
|
for (var j = 0; j < imWidth; j++)
|
2019-05-23 15:28:26 +02:00
|
|
|
|
{
|
2019-07-29 16:29:21 +02:00
|
|
|
|
byte vec = cannyImageData[j + i * imWidth];
|
|
|
|
|
var color32 = new Color32
|
|
|
|
|
{
|
|
|
|
|
r = vec,
|
|
|
|
|
g = vec,
|
|
|
|
|
b = vec,
|
|
|
|
|
a = 0
|
|
|
|
|
};
|
|
|
|
|
c[j + i * imWidth] = color32;
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
finalProcessedCozmoTexture.SetPixels32(c);
|
|
|
|
|
// update texture
|
|
|
|
|
finalProcessedCozmoTexture.Apply();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Find the Center of Gravity in the image
|
|
|
|
|
private void FindCenterOfGravity(Mat processedImage)
|
|
|
|
|
{
|
|
|
|
|
// find moments of the image
|
|
|
|
|
Moments m = new Moments(processedImage, true);
|
|
|
|
|
CenterOfGravity = new Point(m.M10 / m.M00, m.M01 / m.M00);
|
2019-05-17 18:23:24 +02:00
|
|
|
|
|
2019-05-17 19:12:07 +02:00
|
|
|
|
#if UNITY_EDITOR
|
2019-07-29 16:29:21 +02:00
|
|
|
|
// show the image with a point mark at the centroid
|
|
|
|
|
Cv2.Circle(processedImage, CenterOfGravity, 5, new Scalar(128, 0, 0), -1);
|
|
|
|
|
Cv2.Flip(processedImage, processedImage, FlipMode.X);
|
|
|
|
|
Cv2.ImShow("Image with center", processedImage);
|
2019-05-17 19:12:07 +02:00
|
|
|
|
#endif
|
2019-07-29 16:29:21 +02:00
|
|
|
|
}
|
2019-05-17 19:12:07 +02:00
|
|
|
|
|
2019-07-29 16:29:21 +02:00
|
|
|
|
}
|
2019-05-15 12:07:27 +02:00
|
|
|
|
}
|