Crop Image / Training now works / Got memory error while training

Assets/Scripts/CannyEdgeDetection/ImageProcessor.cs

@@ -6,7 +6,19 @@ using System.Threading.Tasks;
 
 public class ImageProcessor : MonoBehaviour
 {
-    public MeshRenderer processedImageRenderer;
+    [Header("RenderTexture")]
+    [Tooltip("RenderTexture that will be passed to the LearningBrain.")]
+    public RenderTexture renderTextureCropped;
+
+    [Header("Debug Helper")]
+    //[Tooltip("Reference to the MeshRenderer that will show the processed Image from Cozmo")]
+    //public MeshRenderer processedImageRenderer;
+    [Tooltip("Reference to the MeshRenderer that will show the processed and cropped Image from Cozmo")]
+    public MeshRenderer processedImageRendererCropped;
+
+    /// <summary>
+    /// Center of Gravity in the cropped canny image
+    /// </summary>
     public Point CenterOfGravity { get; private set; }
 
     // OpenCVSharp parameters
@@ -17,34 +29,56 @@ public class ImageProcessor : MonoBehaviour
     private byte[] cannyImageData;
 
 
-    private const int imWidth = 320; //TODO: Set width and height based on agent observation size
-    private const int imHeight = 240;
-
-    private Camera textureCamera;
+    private int imWidth = 320;          // Width of the camera image from the virtual cozmo
+    private int imHeight = 240;         // Height of the camera image from the virtual cozmo
+    private int croppedImHeight = 120;  // Height of the cropped camera image from the virtual cozmo 
+    private Camera textureCamera;       // Virtual Cozmo camera
 
     private void Start()
     {
+        // Get reference to the cozmo camera
         textureCamera = GetComponent<Camera>();
 
+        // Set image widths and heights based on the given RenderTextures
+        imWidth = textureCamera.targetTexture.width;
+        imHeight = textureCamera.targetTexture.height;
+        croppedImHeight = renderTextureCropped.height;
+
         //assign the processed targetTexture to the renderer to display the image
-        processedImageRenderer.material.mainTexture = textureCamera.targetTexture;
+        //processedImageRenderer.material.mainTexture = textureCamera.targetTexture;
+        processedImageRendererCropped.material.mainTexture = renderTextureCropped;
 
         // initialize video / image with given size
         videoSourceImage = new Mat(imHeight, imWidth, MatType.CV_8UC3);
         videoSourceImageData = new Vec3b[imHeight * imWidth];
         cannyImage = new Mat(imHeight, imWidth, MatType.CV_8UC1);
-        cannyImageData = new byte[imHeight * imWidth];
+        cannyImageData = new byte[croppedImHeight * imWidth];
     }
 
-
+    /// <summary>
+    /// Gets called when a new image arrives from the camera this script lies on
+    /// </summary>
+    /// <param name="source"></param>
+    /// <param name="destination"></param>
     private void OnRenderImage(RenderTexture source, RenderTexture destination)
     {
         Texture2D tex = RenderTextureToTexture2D(source);
         videoSourceImage = TextureToMat(tex);
         cannyImage = ProcessImage(videoSourceImage);
+        cannyImage = CropImage(cannyImage);
         CenterOfGravityTest(cannyImage);
         processedTexture = MatToTexture(cannyImage);
-        Graphics.Blit(processedTexture, destination);
+        //Graphics.Blit(processedTexture, destination);
+        Graphics.Blit(processedTexture, renderTextureCropped);
+    }
+
+    // Crop image to just see the middle of the original image 
+    private Mat CropImage(Mat image)
+    {
+        //cut a fourth out of the top and bottom of the image 
+        OpenCvSharp.Rect rectCroped = new OpenCvSharp.Rect(0, image.Height / 4, image.Width, image.Height / 2);
+        Mat croppedImage = new Mat(image, rectCroped);
+        return croppedImage;
     }
 
     private Texture2D RenderTextureToTexture2D(RenderTexture rTex)
@@ -101,10 +135,10 @@ public class ImageProcessor : MonoBehaviour
         // cannyImageData is byte array, because canny image is grayscale
         mat.GetArray(0, 0, cannyImageData);
         // create Color32 array that can be assigned to Texture2D directly
-        Color32[] c = new Color32[imHeight * imWidth];
+        Color32[] c = new Color32[croppedImHeight * imWidth];
 
         // parallel for loop
-        Parallel.For(0, imHeight, i =>
+        Parallel.For(0, croppedImHeight, i =>
         {
             for (var j = 0; j < imWidth; j++)
             {
@@ -120,7 +154,7 @@ public class ImageProcessor : MonoBehaviour
             }
         });
 
-        Texture2D texture = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
+        Texture2D texture = new Texture2D(imWidth, croppedImHeight, TextureFormat.RGBA32, true, true);
         texture.SetPixels32(c);
         // to update the texture, OpenGL manner
         texture.Apply();