hasselto66176/BachelorarbeitCozmo
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

New Reward function and MovementController

Browse Source
This commit is contained in:
Tobi
2019-05-23 17:28:37 +02:00
parent 411338a35d
commit 221ffb54dc
5 changed files with 156 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Assets/Scripts/CannyEdgeDetection/ImageProcessor.cs
+29 -166
View File
@@ -22,10 +22,10 @@ public class ImageProcessor : MonoBehaviour
public Point CenterOfGravity { get; private set; }
// OpenCVSharp parameters
private Mat videoSourceImage;
private Mat cozmoImageMat;
private Mat cannyImage;
private Texture2D processedTexture;
private Vec3b[] videoSourceImageData;
private Texture2D finalProcessedCozmoTexture;
private Vec3b[] cozmoImageData;
private byte[] cannyImageData;
@@ -34,8 +34,7 @@ public class ImageProcessor : MonoBehaviour
private int croppedImHeight = 120; // Height of the cropped camera image from the virtual cozmo
private Camera textureCamera; // Virtual Cozmo camera
private Texture2D firstTexture;
private Texture2D originalCozmoTexture;
private void Start()
@@ -46,21 +45,19 @@ public class ImageProcessor : MonoBehaviour
// Set image widths and heights based on the given RenderTextures
imWidth = textureCamera.targetTexture.width;
imHeight = textureCamera.targetTexture.height;
//croppedImHeight = renderTextureCropped.height;
//assign the processed targetTexture to the renderer to display the image
// assign the processed targetTexture to the renderer to display the image
processedImageRenderer.material.mainTexture = textureCamera.targetTexture;
processedImageRendererCropped.material.mainTexture = renderTextureCropped;
// initialize video / image with given size
videoSourceImage = new Mat(imHeight, imWidth, MatType.CV_8UC3);
videoSourceImageData = new Vec3b[imHeight * imWidth];
cozmoImageMat = new Mat(imHeight, imWidth, MatType.CV_8UC3);
cozmoImageData = new Vec3b[imHeight * imWidth];
cannyImage = new Mat(imHeight, imWidth, MatType.CV_8UC1);
cannyImageData = new byte[croppedImHeight * imWidth];
firstTexture = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
processedTexture = new Texture2D(imWidth, croppedImHeight, TextureFormat.RGBA32, true, true);
originalCozmoTexture = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
finalProcessedCozmoTexture = new Texture2D(imWidth, croppedImHeight, TextureFormat.RGBA32, true, true);
}
///// <summary>
@@ -68,63 +65,16 @@ public class ImageProcessor : MonoBehaviour
///// </summary>
///// <param name="source"></param>
///// <param name="destination"></param>
//private void OnRenderImage(RenderTexture source, RenderTexture destination)
//{
// Texture2D tex = RenderTextureToTexture2D(source);
// videoSourceImage = TextureToMat(tex);
// cannyImage = ProcessImage(videoSourceImage);
// cannyImage = CropImage(cannyImage);
// CenterOfGravityTest(cannyImage);
// processedTexture = MatToTexture(cannyImage);
// //processedTexture.Resize(80, 30);
// //processedTexture.Apply();
// //Graphics.Blit(processedTexture, destination);
// Graphics.Blit(processedTexture, renderTextureCropped);
//}
public void ProcessRenderTexture(RenderTexture source)
{
ProcessRenderTexture(source, source);
}
public void ProcessRenderTexture(RenderTexture source, RenderTexture target)
{
Texture2D tex = RenderTextureToTexture2D(source);
videoSourceImage = TextureToMat(tex);
cannyImage = ProcessImage(videoSourceImage);
cannyImage = CropImage(cannyImage);
CenterOfGravityTest(cannyImage);
processedTexture = MatToTexture(cannyImage);
//processedTexture.Resize(80, 30);
//processedTexture.Apply();
//Graphics.Blit(processedTexture, destination);
Graphics.Blit(processedTexture, target);
}
//public void ProcessRenderTexture_NEW(RenderTexture source, RenderTexture target)
//{
// RenderTextureToTexture2D_NEW(source);
// TextureToMat_NEW(firstTexture);
// ProcessImage_NEW(videoSourceImage);
// cannyImage = CropImage(cannyImage);
// CenterOfGravityTest(cannyImage);
// MatToTexture_NEW(cannyImage);
// //processedTexture.Resize(80, 30);
// //processedTexture.Apply();
// //Graphics.Blit(processedTexture, destination);
// Graphics.Blit(processedTexture, target);
//}
public void OnRenderImage(RenderTexture source, RenderTexture destination)
{
RenderTextureToTexture2D_NEW(source);
TextureToMat_NEW(firstTexture);
ProcessImage_NEW(videoSourceImage);
RenderTextureToTexture2D(source);
TextureToMat(originalCozmoTexture);
ProcessImage(cozmoImageMat);
cannyImage = CropImage(cannyImage);
CenterOfGravityTest(cannyImage);
MatToTexture_NEW(cannyImage);
Graphics.Blit(processedTexture, destination);
Graphics.Blit(processedTexture, renderTextureCropped);
FindCenterOfGravity(cannyImage);
MatToTexture(cannyImage);
Graphics.Blit(finalProcessedCozmoTexture, destination);
Graphics.Blit(finalProcessedCozmoTexture, renderTextureCropped);
}
// Crop image to just see the middle of the original image
@@ -136,33 +86,16 @@ public class ImageProcessor : MonoBehaviour
return croppedImage;
}
//// Crop image to just see the middle of the original image
//private Mat CropImage_NEW(Mat image)
//{
// //cut a fourth out of the top and bottom of the image
// OpenCvSharp.Rect rectCroped = new OpenCvSharp.Rect(0, image.Height / 4, image.Width, image.Height / 2);
// Mat croppedImage = new Mat(image, rectCroped);
// return croppedImage;
//}
private Texture2D RenderTextureToTexture2D(RenderTexture rTex)
{
Texture2D tex = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
RenderTexture.active = rTex;
tex.ReadPixels(new UnityEngine.Rect(0, 0, rTex.width, rTex.height), 0, 0);
tex.Apply();
return tex;
}
private void RenderTextureToTexture2D_NEW(RenderTexture rTex)
private void RenderTextureToTexture2D(RenderTexture rTex)
{
RenderTexture.active = rTex;
firstTexture.ReadPixels(new UnityEngine.Rect(0, 0, rTex.width, rTex.height), 0, 0);
firstTexture.Apply();
originalCozmoTexture.ReadPixels(new UnityEngine.Rect(0, 0, rTex.width, rTex.height), 0, 0);
originalCozmoTexture.Apply();
}
// Convert Unity Texture2D object to OpenCVSharp Mat object
private Mat TextureToMat(Texture2D source)
private void TextureToMat(Texture2D source)
{
// Color32 array : r, g, b, a
Color32[] c = source.GetPixels32();
@@ -182,94 +115,24 @@ public class ImageProcessor : MonoBehaviour
Item2 = col.r
};
// set pixel to an array
videoSourceImageData[j + i * imWidth] = vec3;
cozmoImageData[j + i * imWidth] = vec3;
}
});
// assign the Vec3b array to Mat
Mat tmpMat = new Mat(imHeight, imWidth, MatType.CV_8UC3);
tmpMat.SetArray(0, 0, videoSourceImageData);
return tmpMat;
cozmoImageMat.SetArray(0, 0, cozmoImageData);
}
// Convert Unity Texture2D object to OpenCVSharp Mat object
private void TextureToMat_NEW(Texture2D source)
{
// Color32 array : r, g, b, a
Color32[] c = source.GetPixels32();
// Parallel for loop
// convert Color32 object to Vec3b object
// Vec3b is the representation of pixel for Mat
Parallel.For(0, imHeight, i =>
{
for (var j = 0; j < imWidth; j++)
{
var col = c[j + i * imWidth];
var vec3 = new Vec3b
{
Item0 = col.b,
Item1 = col.g,
Item2 = col.r
};
// set pixel to an array
videoSourceImageData[j + i * imWidth] = vec3;
}
});
// assign the Vec3b array to Mat
videoSourceImage.SetArray(0, 0, videoSourceImageData);
}
// Simple example of canny edge detect
private Mat ProcessImage(Mat _image)
{
Mat cannyImg = new Mat();
Cv2.Canny(_image, cannyImg, 100, 100);
return cannyImg;
}
// Simple example of canny edge detect
private void ProcessImage_NEW(Mat _image)
private void ProcessImage(Mat _image)
{
Cv2.Canny(_image, cannyImage, 100, 100);
}
// Convert OpenCVSharp Mat object to Unity Texture2D object
private Texture2D MatToTexture(Mat mat)
{
// cannyImageData is byte array, because canny image is grayscale
mat.GetArray(0, 0, cannyImageData);
// create Color32 array that can be assigned to Texture2D directly
Color32[] c = new Color32[croppedImHeight * imWidth];
// parallel for loop
Parallel.For(0, croppedImHeight, i =>
{
for (var j = 0; j < imWidth; j++)
{
byte vec = cannyImageData[j + i * imWidth];
var color32 = new Color32
{
r = vec,
g = vec,
b = vec,
a = 0
};
c[j + i * imWidth] = color32;
}
});
Texture2D texture = new Texture2D(imWidth, croppedImHeight, TextureFormat.RGBA32, true, true);
texture.SetPixels32(c);
// to update the texture, OpenGL manner
texture.Apply();
return texture;
}
// Convert OpenCVSharp Mat object to Unity Texture2D object
private void MatToTexture_NEW(Mat mat)
private void MatToTexture(Mat mat)
{
// cannyImageData is byte array, because canny image is grayscale
mat.GetArray(0, 0, cannyImageData);
@@ -294,13 +157,13 @@ public class ImageProcessor : MonoBehaviour
});
processedTexture.SetPixels32(c);
// to update the texture, OpenGL manner
processedTexture.Apply();
finalProcessedCozmoTexture.SetPixels32(c);
// update texture
finalProcessedCozmoTexture.Apply();
}
private void CenterOfGravityTest(Mat processedImage)
// Find the Center of Gravity in the image
private void FindCenterOfGravity(Mat processedImage)
{
// find moments of the image
Moments m = new Moments(processedImage, true);