You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

ImageProcessor.cs 7.2KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192
  1. ///-----------------------------------------------------------------
  2. /// Namespace: <Cozmo>
  3. /// Class: <ImageProcessor>
  4. /// Description: <Converts a rendertexture to a opencv mat in order to use the canny algorithm.
  5. /// After processing the mat will be converted to a render texture back again.>
  6. /// Author: <Tobias Hassel> Date: <29.07.2019>
  7. /// Notes: <>
  8. ///-----------------------------------------------------------------
  9. ///
  10. using UnityEngine;
  11. using OpenCvSharp;
  12. using System.Threading.Tasks;
  13. namespace Cozmo
  14. {
  15. public class ImageProcessor : MonoBehaviour
  16. {
  17. [Header("RenderTexture")]
  18. [Tooltip("RenderTexture that will be passed to the LearningBrain.")]
  19. public RenderTexture renderTextureCropped;
  20. [Header("Debug Helper")]
  21. [Tooltip("Reference to the MeshRenderer that will show the processed Image from Cozmo")]
  22. public MeshRenderer processedImageRenderer;
  23. [Tooltip("Reference to the MeshRenderer that will show the processed and cropped Image from Cozmo")]
  24. public MeshRenderer processedImageRendererCropped;
  25. /// <summary>
  26. /// Center of Gravity in the cropped canny image
  27. /// </summary>
  28. public Point CenterOfGravity { get; private set; }
  29. // OpenCVSharp parameters
  30. private Mat cozmoImageMat;
  31. private Mat cannyImage;
  32. private Texture2D finalProcessedCozmoTexture;
  33. private Vec3b[] cozmoImageData;
  34. private byte[] cannyImageData;
  35. private int imWidth = 320; // Width of the camera image from the virtual cozmo
  36. private int imHeight = 240; // Height of the camera image from the virtual cozmo
  37. private int croppedImHeight = 120; // Height of the cropped camera image from the virtual cozmo
  38. private Camera textureCamera; // Virtual Cozmo camera
  39. private Texture2D originalCozmoTexture;
  40. private void Start()
  41. {
  42. // Get reference to the cozmo camera
  43. textureCamera = GetComponent<Camera>();
  44. // Set image widths and heights based on the given RenderTextures
  45. imWidth = textureCamera.targetTexture.width;
  46. imHeight = textureCamera.targetTexture.height;
  47. // assign the processed targetTexture to the renderer to display the image
  48. processedImageRenderer.material.mainTexture = textureCamera.targetTexture;
  49. processedImageRendererCropped.material.mainTexture = renderTextureCropped;
  50. // initialize video / image with given size
  51. cozmoImageMat = new Mat(imHeight, imWidth, MatType.CV_8UC3);
  52. cozmoImageData = new Vec3b[imHeight * imWidth];
  53. cannyImage = new Mat(imHeight, imWidth, MatType.CV_8UC1);
  54. cannyImageData = new byte[croppedImHeight * imWidth];
  55. originalCozmoTexture = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
  56. finalProcessedCozmoTexture = new Texture2D(imWidth, croppedImHeight, TextureFormat.RGBA32, true, true);
  57. }
  58. ///// <summary>
  59. ///// Gets called when a new image arrives from the camera this script lies on
  60. ///// </summary>
  61. ///// <param name="source"></param>
  62. ///// <param name="destination"></param>
  63. public void OnRenderImage(RenderTexture source, RenderTexture destination)
  64. {
  65. RenderTextureToTexture2D(source);
  66. TextureToMat(originalCozmoTexture);
  67. ProcessImage(cozmoImageMat);
  68. cannyImage = CropImage(cannyImage);
  69. FindCenterOfGravity(cannyImage);
  70. MatToTexture(cannyImage);
  71. Graphics.Blit(finalProcessedCozmoTexture, destination);
  72. Graphics.Blit(finalProcessedCozmoTexture, renderTextureCropped);
  73. }
  74. // Crop image to just see the middle of the original image
  75. private Mat CropImage(Mat image)
  76. {
  77. //cut a fourth out of the top and bottom of the image
  78. OpenCvSharp.Rect rectCroped = new OpenCvSharp.Rect(0, image.Height / 4, image.Width, image.Height / 2);
  79. Mat croppedImage = new Mat(image, rectCroped);
  80. return croppedImage;
  81. }
  82. // Convert the rendertexture to a texture2d
  83. private void RenderTextureToTexture2D(RenderTexture rTex)
  84. {
  85. RenderTexture.active = rTex;
  86. originalCozmoTexture.ReadPixels(new UnityEngine.Rect(0, 0, rTex.width, rTex.height), 0, 0);
  87. originalCozmoTexture.Apply();
  88. }
  89. // Convert Unity Texture2D object to OpenCVSharp Mat object
  90. private void TextureToMat(Texture2D source)
  91. {
  92. // Color32 array : r, g, b, a
  93. Color32[] c = source.GetPixels32();
  94. // Parallel for loop
  95. // convert Color32 object to Vec3b object
  96. // Vec3b is the representation of pixel for Mat
  97. Parallel.For(0, imHeight, i =>
  98. {
  99. for (var j = 0; j < imWidth; j++)
  100. {
  101. var col = c[j + i * imWidth];
  102. var vec3 = new Vec3b
  103. {
  104. Item0 = col.b,
  105. Item1 = col.g,
  106. Item2 = col.r
  107. };
  108. // set pixel to an array
  109. cozmoImageData[j + i * imWidth] = vec3;
  110. }
  111. });
  112. // assign the Vec3b array to Mat
  113. cozmoImageMat.SetArray(0, 0, cozmoImageData);
  114. }
  115. // Simple example of canny edge detect
  116. private void ProcessImage(Mat _image)
  117. {
  118. Cv2.Canny(_image, cannyImage, 100, 100);
  119. }
  120. // Convert OpenCVSharp Mat object to Unity Texture2D object
  121. private void MatToTexture(Mat mat)
  122. {
  123. // cannyImageData is byte array, because canny image is grayscale
  124. mat.GetArray(0, 0, cannyImageData);
  125. // create Color32 array that can be assigned to Texture2D directly
  126. Color32[] c = new Color32[croppedImHeight * imWidth];
  127. // parallel for loop
  128. Parallel.For(0, croppedImHeight, i =>
  129. {
  130. for (var j = 0; j < imWidth; j++)
  131. {
  132. byte vec = cannyImageData[j + i * imWidth];
  133. var color32 = new Color32
  134. {
  135. r = vec,
  136. g = vec,
  137. b = vec,
  138. a = 0
  139. };
  140. c[j + i * imWidth] = color32;
  141. }
  142. });
  143. finalProcessedCozmoTexture.SetPixels32(c);
  144. // update texture
  145. finalProcessedCozmoTexture.Apply();
  146. }
  147. // Find the Center of Gravity in the image
  148. private void FindCenterOfGravity(Mat processedImage)
  149. {
  150. // find moments of the image
  151. Moments m = new Moments(processedImage, true);
  152. CenterOfGravity = new Point(m.M10 / m.M00, m.M01 / m.M00);
  153. #if UNITY_EDITOR
  154. // show the image with a point mark at the centroid
  155. Cv2.Circle(processedImage, CenterOfGravity, 5, new Scalar(128, 0, 0), -1);
  156. Cv2.Flip(processedImage, processedImage, FlipMode.X);
  157. Cv2.ImShow("Image with center", processedImage);
  158. #endif
  159. }
  160. }
  161. }