Add commetns and refactoring
This commit is contained in:
Tobi
@@ -1,180 +1,192 @@
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
///-----------------------------------------------------------------
|
||||
/// Namespace: <Cozmo>
|
||||
/// Class: <ImageProcessor>
|
||||
/// Description: <Converts a rendertexture to a opencv mat in order to use the canny algorithm.
|
||||
/// After processing the mat will be converted to a render texture back again.>
|
||||
/// Author: <Tobias Hassel> Date: <29.07.2019>
|
||||
/// Notes: <>
|
||||
///-----------------------------------------------------------------
|
||||
///
|
||||
|
||||
using UnityEngine;
|
||||
using OpenCvSharp;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
public class ImageProcessor : MonoBehaviour
|
||||
namespace Cozmo
|
||||
{
|
||||
[Header("RenderTexture")]
|
||||
[Tooltip("RenderTexture that will be passed to the LearningBrain.")]
|
||||
public RenderTexture renderTextureCropped;
|
||||
|
||||
[Header("Debug Helper")]
|
||||
[Tooltip("Reference to the MeshRenderer that will show the processed Image from Cozmo")]
|
||||
public MeshRenderer processedImageRenderer;
|
||||
[Tooltip("Reference to the MeshRenderer that will show the processed and cropped Image from Cozmo")]
|
||||
public MeshRenderer processedImageRendererCropped;
|
||||
|
||||
/// <summary>
|
||||
/// Center of Gravity in the cropped canny image
|
||||
/// </summary>
|
||||
public Point CenterOfGravity { get; private set; }
|
||||
|
||||
// OpenCVSharp parameters
|
||||
private Mat cozmoImageMat;
|
||||
private Mat cannyImage;
|
||||
private Texture2D finalProcessedCozmoTexture;
|
||||
private Vec3b[] cozmoImageData;
|
||||
private byte[] cannyImageData;
|
||||
|
||||
|
||||
private int imWidth = 320; // Width of the camera image from the virtual cozmo
|
||||
private int imHeight = 240; // Height of the camera image from the virtual cozmo
|
||||
private int croppedImHeight = 120; // Height of the cropped camera image from the virtual cozmo
|
||||
private Camera textureCamera; // Virtual Cozmo camera
|
||||
|
||||
private Texture2D originalCozmoTexture;
|
||||
|
||||
|
||||
private void Start()
|
||||
public class ImageProcessor : MonoBehaviour
|
||||
{
|
||||
// Get reference to the cozmo camera
|
||||
textureCamera = GetComponent<Camera>();
|
||||
[Header("RenderTexture")]
|
||||
[Tooltip("RenderTexture that will be passed to the LearningBrain.")]
|
||||
public RenderTexture renderTextureCropped;
|
||||
|
||||
// Set image widths and heights based on the given RenderTextures
|
||||
imWidth = textureCamera.targetTexture.width;
|
||||
imHeight = textureCamera.targetTexture.height;
|
||||
[Header("Debug Helper")]
|
||||
[Tooltip("Reference to the MeshRenderer that will show the processed Image from Cozmo")]
|
||||
public MeshRenderer processedImageRenderer;
|
||||
[Tooltip("Reference to the MeshRenderer that will show the processed and cropped Image from Cozmo")]
|
||||
public MeshRenderer processedImageRendererCropped;
|
||||
|
||||
// assign the processed targetTexture to the renderer to display the image
|
||||
processedImageRenderer.material.mainTexture = textureCamera.targetTexture;
|
||||
processedImageRendererCropped.material.mainTexture = renderTextureCropped;
|
||||
/// <summary>
|
||||
/// Center of Gravity in the cropped canny image
|
||||
/// </summary>
|
||||
public Point CenterOfGravity { get; private set; }
|
||||
|
||||
// initialize video / image with given size
|
||||
cozmoImageMat = new Mat(imHeight, imWidth, MatType.CV_8UC3);
|
||||
cozmoImageData = new Vec3b[imHeight * imWidth];
|
||||
cannyImage = new Mat(imHeight, imWidth, MatType.CV_8UC1);
|
||||
cannyImageData = new byte[croppedImHeight * imWidth];
|
||||
|
||||
originalCozmoTexture = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
|
||||
finalProcessedCozmoTexture = new Texture2D(imWidth, croppedImHeight, TextureFormat.RGBA32, true, true);
|
||||
}
|
||||
|
||||
///// <summary>
|
||||
///// Gets called when a new image arrives from the camera this script lies on
|
||||
///// </summary>
|
||||
///// <param name="source"></param>
|
||||
///// <param name="destination"></param>
|
||||
public void OnRenderImage(RenderTexture source, RenderTexture destination)
|
||||
{
|
||||
RenderTextureToTexture2D(source);
|
||||
TextureToMat(originalCozmoTexture);
|
||||
ProcessImage(cozmoImageMat);
|
||||
cannyImage = CropImage(cannyImage);
|
||||
FindCenterOfGravity(cannyImage);
|
||||
MatToTexture(cannyImage);
|
||||
Graphics.Blit(finalProcessedCozmoTexture, destination);
|
||||
Graphics.Blit(finalProcessedCozmoTexture, renderTextureCropped);
|
||||
}
|
||||
|
||||
// Crop image to just see the middle of the original image
|
||||
private Mat CropImage(Mat image)
|
||||
{
|
||||
//cut a fourth out of the top and bottom of the image
|
||||
OpenCvSharp.Rect rectCroped = new OpenCvSharp.Rect(0, image.Height / 4, image.Width, image.Height / 2);
|
||||
Mat croppedImage = new Mat(image, rectCroped);
|
||||
return croppedImage;
|
||||
}
|
||||
// OpenCVSharp parameters
|
||||
private Mat cozmoImageMat;
|
||||
private Mat cannyImage;
|
||||
private Texture2D finalProcessedCozmoTexture;
|
||||
private Vec3b[] cozmoImageData;
|
||||
private byte[] cannyImageData;
|
||||
|
||||
|
||||
private void RenderTextureToTexture2D(RenderTexture rTex)
|
||||
{
|
||||
RenderTexture.active = rTex;
|
||||
originalCozmoTexture.ReadPixels(new UnityEngine.Rect(0, 0, rTex.width, rTex.height), 0, 0);
|
||||
originalCozmoTexture.Apply();
|
||||
}
|
||||
private int imWidth = 320; // Width of the camera image from the virtual cozmo
|
||||
private int imHeight = 240; // Height of the camera image from the virtual cozmo
|
||||
private int croppedImHeight = 120; // Height of the cropped camera image from the virtual cozmo
|
||||
private Camera textureCamera; // Virtual Cozmo camera
|
||||
|
||||
// Convert Unity Texture2D object to OpenCVSharp Mat object
|
||||
private void TextureToMat(Texture2D source)
|
||||
{
|
||||
// Color32 array : r, g, b, a
|
||||
Color32[] c = source.GetPixels32();
|
||||
private Texture2D originalCozmoTexture;
|
||||
|
||||
// Parallel for loop
|
||||
// convert Color32 object to Vec3b object
|
||||
// Vec3b is the representation of pixel for Mat
|
||||
Parallel.For(0, imHeight, i =>
|
||||
|
||||
private void Start()
|
||||
{
|
||||
for (var j = 0; j < imWidth; j++)
|
||||
{
|
||||
var col = c[j + i * imWidth];
|
||||
var vec3 = new Vec3b
|
||||
{
|
||||
Item0 = col.b,
|
||||
Item1 = col.g,
|
||||
Item2 = col.r
|
||||
};
|
||||
// set pixel to an array
|
||||
cozmoImageData[j + i * imWidth] = vec3;
|
||||
}
|
||||
});
|
||||
// Get reference to the cozmo camera
|
||||
textureCamera = GetComponent<Camera>();
|
||||
|
||||
// assign the Vec3b array to Mat
|
||||
cozmoImageMat.SetArray(0, 0, cozmoImageData);
|
||||
}
|
||||
// Set image widths and heights based on the given RenderTextures
|
||||
imWidth = textureCamera.targetTexture.width;
|
||||
imHeight = textureCamera.targetTexture.height;
|
||||
|
||||
// assign the processed targetTexture to the renderer to display the image
|
||||
processedImageRenderer.material.mainTexture = textureCamera.targetTexture;
|
||||
processedImageRendererCropped.material.mainTexture = renderTextureCropped;
|
||||
|
||||
// Simple example of canny edge detect
|
||||
private void ProcessImage(Mat _image)
|
||||
{
|
||||
Cv2.Canny(_image, cannyImage, 100, 100);
|
||||
}
|
||||
// initialize video / image with given size
|
||||
cozmoImageMat = new Mat(imHeight, imWidth, MatType.CV_8UC3);
|
||||
cozmoImageData = new Vec3b[imHeight * imWidth];
|
||||
cannyImage = new Mat(imHeight, imWidth, MatType.CV_8UC1);
|
||||
cannyImageData = new byte[croppedImHeight * imWidth];
|
||||
|
||||
originalCozmoTexture = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
|
||||
finalProcessedCozmoTexture = new Texture2D(imWidth, croppedImHeight, TextureFormat.RGBA32, true, true);
|
||||
}
|
||||
|
||||
// Convert OpenCVSharp Mat object to Unity Texture2D object
|
||||
private void MatToTexture(Mat mat)
|
||||
{
|
||||
// cannyImageData is byte array, because canny image is grayscale
|
||||
mat.GetArray(0, 0, cannyImageData);
|
||||
// create Color32 array that can be assigned to Texture2D directly
|
||||
Color32[] c = new Color32[croppedImHeight * imWidth];
|
||||
|
||||
// parallel for loop
|
||||
Parallel.For(0, croppedImHeight, i =>
|
||||
///// <summary>
|
||||
///// Gets called when a new image arrives from the camera this script lies on
|
||||
///// </summary>
|
||||
///// <param name="source"></param>
|
||||
///// <param name="destination"></param>
|
||||
public void OnRenderImage(RenderTexture source, RenderTexture destination)
|
||||
{
|
||||
for (var j = 0; j < imWidth; j++)
|
||||
RenderTextureToTexture2D(source);
|
||||
TextureToMat(originalCozmoTexture);
|
||||
ProcessImage(cozmoImageMat);
|
||||
cannyImage = CropImage(cannyImage);
|
||||
FindCenterOfGravity(cannyImage);
|
||||
MatToTexture(cannyImage);
|
||||
|
||||
Graphics.Blit(finalProcessedCozmoTexture, destination);
|
||||
Graphics.Blit(finalProcessedCozmoTexture, renderTextureCropped);
|
||||
}
|
||||
|
||||
// Crop image to just see the middle of the original image
|
||||
private Mat CropImage(Mat image)
|
||||
{
|
||||
//cut a fourth out of the top and bottom of the image
|
||||
OpenCvSharp.Rect rectCroped = new OpenCvSharp.Rect(0, image.Height / 4, image.Width, image.Height / 2);
|
||||
Mat croppedImage = new Mat(image, rectCroped);
|
||||
return croppedImage;
|
||||
}
|
||||
|
||||
// Convert the rendertexture to a texture2d
|
||||
private void RenderTextureToTexture2D(RenderTexture rTex)
|
||||
{
|
||||
RenderTexture.active = rTex;
|
||||
originalCozmoTexture.ReadPixels(new UnityEngine.Rect(0, 0, rTex.width, rTex.height), 0, 0);
|
||||
originalCozmoTexture.Apply();
|
||||
}
|
||||
|
||||
// Convert Unity Texture2D object to OpenCVSharp Mat object
|
||||
private void TextureToMat(Texture2D source)
|
||||
{
|
||||
// Color32 array : r, g, b, a
|
||||
Color32[] c = source.GetPixels32();
|
||||
|
||||
// Parallel for loop
|
||||
// convert Color32 object to Vec3b object
|
||||
// Vec3b is the representation of pixel for Mat
|
||||
Parallel.For(0, imHeight, i =>
|
||||
{
|
||||
byte vec = cannyImageData[j + i * imWidth];
|
||||
var color32 = new Color32
|
||||
for (var j = 0; j < imWidth; j++)
|
||||
{
|
||||
r = vec,
|
||||
g = vec,
|
||||
b = vec,
|
||||
a = 0
|
||||
};
|
||||
c[j + i * imWidth] = color32;
|
||||
}
|
||||
});
|
||||
var col = c[j + i * imWidth];
|
||||
var vec3 = new Vec3b
|
||||
{
|
||||
Item0 = col.b,
|
||||
Item1 = col.g,
|
||||
Item2 = col.r
|
||||
};
|
||||
// set pixel to an array
|
||||
cozmoImageData[j + i * imWidth] = vec3;
|
||||
}
|
||||
});
|
||||
|
||||
// assign the Vec3b array to Mat
|
||||
cozmoImageMat.SetArray(0, 0, cozmoImageData);
|
||||
}
|
||||
|
||||
|
||||
finalProcessedCozmoTexture.SetPixels32(c);
|
||||
// update texture
|
||||
finalProcessedCozmoTexture.Apply();
|
||||
}
|
||||
// Simple example of canny edge detect
|
||||
private void ProcessImage(Mat _image)
|
||||
{
|
||||
Cv2.Canny(_image, cannyImage, 100, 100);
|
||||
}
|
||||
|
||||
// Find the Center of Gravity in the image
|
||||
private void FindCenterOfGravity(Mat processedImage)
|
||||
{
|
||||
// find moments of the image
|
||||
Moments m = new Moments(processedImage, true);
|
||||
CenterOfGravity = new Point(m.M10 / m.M00, m.M01 / m.M00);
|
||||
|
||||
// Convert OpenCVSharp Mat object to Unity Texture2D object
|
||||
private void MatToTexture(Mat mat)
|
||||
{
|
||||
// cannyImageData is byte array, because canny image is grayscale
|
||||
mat.GetArray(0, 0, cannyImageData);
|
||||
// create Color32 array that can be assigned to Texture2D directly
|
||||
Color32[] c = new Color32[croppedImHeight * imWidth];
|
||||
|
||||
// parallel for loop
|
||||
Parallel.For(0, croppedImHeight, i =>
|
||||
{
|
||||
for (var j = 0; j < imWidth; j++)
|
||||
{
|
||||
byte vec = cannyImageData[j + i * imWidth];
|
||||
var color32 = new Color32
|
||||
{
|
||||
r = vec,
|
||||
g = vec,
|
||||
b = vec,
|
||||
a = 0
|
||||
};
|
||||
c[j + i * imWidth] = color32;
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
finalProcessedCozmoTexture.SetPixels32(c);
|
||||
// update texture
|
||||
finalProcessedCozmoTexture.Apply();
|
||||
}
|
||||
|
||||
// Find the Center of Gravity in the image
|
||||
private void FindCenterOfGravity(Mat processedImage)
|
||||
{
|
||||
// find moments of the image
|
||||
Moments m = new Moments(processedImage, true);
|
||||
CenterOfGravity = new Point(m.M10 / m.M00, m.M01 / m.M00);
|
||||
|
||||
#if UNITY_EDITOR
|
||||
// show the image with a point mark at the centroid
|
||||
Cv2.Circle(processedImage, CenterOfGravity, 5, new Scalar(128, 0, 0), -1);
|
||||
Cv2.Flip(processedImage, processedImage, FlipMode.X);
|
||||
Cv2.ImShow("Image with center", processedImage);
|
||||
// show the image with a point mark at the centroid
|
||||
Cv2.Circle(processedImage, CenterOfGravity, 5, new Scalar(128, 0, 0), -1);
|
||||
Cv2.Flip(processedImage, processedImage, FlipMode.X);
|
||||
Cv2.ImShow("Image with center", processedImage);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user