hasselto66176/BachelorarbeitCozmo
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix Memory error / Messy code

Browse Source
This commit is contained in:
Tobi
2019-05-23 15:28:26 +02:00
parent 8082fde44c
commit 411338a35d
8 changed files with 4468 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Assets/Brains/CozmoLearningBrain.asset
+2 -2
View File
@@ -17,8 +17,8 @@ MonoBehaviour:
numStackedVectorObservations: 1
vectorActionSize: 04000000
cameraResolutions:
- width: 320
height: 120
- width: 80
height: 30
blackAndWhite: 0
vectorActionDescriptions:
-
Assets/Scenes/CozmoTraining.unity
+72 -5
View File
@@ -974,6 +974,7 @@ MonoBehaviour:
m_EditorClassIdentifier:
renderTextureCropped: {fileID: 8400000, guid: 8bf12f4401860c043a77209428e97b29,
type: 2}
processedImageRenderer: {fileID: 1863707383}
processedImageRendererCropped: {fileID: 712179489}
--- !u!1 &918537782
GameObject:
@@ -1104,14 +1105,14 @@ MonoBehaviour:
- {fileID: 11400000, guid: 0d9d5ce547064a648b466ad7f208cf3d, type: 2}
maxSteps: 1000
trainingConfiguration:
width: 84
height: 84
width: 80
height: 30
qualityLevel: 1
timeScale: 10
targetFrameRate: -1
inferenceConfiguration:
width: 84
height: 84
width: 320
height: 120
qualityLevel: 5
timeScale: 1
targetFrameRate: 60
@@ -1684,6 +1685,72 @@ MeshFilter:
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1863707380}
m_Mesh: {fileID: 10210, guid: 0000000000000000e000000000000000, type: 0}
--- !u!1 &1957393249
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 1957393252}
- component: {fileID: 1957393251}
- component: {fileID: 1957393250}
m_Layer: 0
m_Name: EventSystem
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!114 &1957393250
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1957393249}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: 1077351063, guid: f70555f144d8491a825f0804e09c671c, type: 3}
m_Name:
m_EditorClassIdentifier:
m_HorizontalAxis: Horizontal
m_VerticalAxis: Vertical
m_SubmitButton: Submit
m_CancelButton: Cancel
m_InputActionsPerSecond: 10
m_RepeatDelay: 0.5
m_ForceModuleActive: 0
--- !u!114 &1957393251
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1957393249}
m_Enabled: 1
m_EditorHideFlags: 0
m_Script: {fileID: -619905303, guid: f70555f144d8491a825f0804e09c671c, type: 3}
m_Name:
m_EditorClassIdentifier:
m_FirstSelected: {fileID: 0}
m_sendNavigationEvents: 1
m_DragThreshold: 10
--- !u!4 &1957393252
Transform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1957393249}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_Children: []
m_Father: {fileID: 0}
m_RootOrder: 5
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!1 &2078653367
GameObject:
m_ObjectHideFlags: 0
@@ -1827,7 +1894,7 @@ MonoBehaviour:
agentCameras: []
agentRenderTextures:
- {fileID: 8400000, guid: 8bf12f4401860c043a77209428e97b29, type: 2}
maxStep: 1000
maxStep: 5000
resetOnDone: 1
onDemandDecision: 1
numberOfActionsBetweenDecisions: 1
Assets/Scripts/CannyEdgeDetection/ImageProcessor.cs
+148 -10
View File
@@ -11,8 +11,8 @@ public class ImageProcessor : MonoBehaviour
public RenderTexture renderTextureCropped;
[Header("Debug Helper")]
//[Tooltip("Reference to the MeshRenderer that will show the processed Image from Cozmo")]
//public MeshRenderer processedImageRenderer;
[Tooltip("Reference to the MeshRenderer that will show the processed Image from Cozmo")]
public MeshRenderer processedImageRenderer;
[Tooltip("Reference to the MeshRenderer that will show the processed and cropped Image from Cozmo")]
public MeshRenderer processedImageRendererCropped;
@@ -34,6 +34,10 @@ public class ImageProcessor : MonoBehaviour
private int croppedImHeight = 120; // Height of the cropped camera image from the virtual cozmo
private Camera textureCamera; // Virtual Cozmo camera
private Texture2D firstTexture;
private void Start()
{
// Get reference to the cozmo camera
@@ -42,10 +46,10 @@ public class ImageProcessor : MonoBehaviour
// Set image widths and heights based on the given RenderTextures
imWidth = textureCamera.targetTexture.width;
imHeight = textureCamera.targetTexture.height;
croppedImHeight = renderTextureCropped.height;
//croppedImHeight = renderTextureCropped.height;
//assign the processed targetTexture to the renderer to display the image
//processedImageRenderer.material.mainTexture = textureCamera.targetTexture;
processedImageRenderer.material.mainTexture = textureCamera.targetTexture;
processedImageRendererCropped.material.mainTexture = renderTextureCropped;
// initialize video / image with given size
@@ -53,14 +57,37 @@ public class ImageProcessor : MonoBehaviour
videoSourceImageData = new Vec3b[imHeight * imWidth];
cannyImage = new Mat(imHeight, imWidth, MatType.CV_8UC1);
cannyImageData = new byte[croppedImHeight * imWidth];
firstTexture = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
processedTexture = new Texture2D(imWidth, croppedImHeight, TextureFormat.RGBA32, true, true);
}
/// <summary>
/// Gets called when a new image arrives from the camera this script lies on
/// </summary>
/// <param name="source"></param>
/// <param name="destination"></param>
private void OnRenderImage(RenderTexture source, RenderTexture destination)
///// <summary>
///// Gets called when a new image arrives from the camera this script lies on
///// </summary>
///// <param name="source"></param>
///// <param name="destination"></param>
//private void OnRenderImage(RenderTexture source, RenderTexture destination)
//{
// Texture2D tex = RenderTextureToTexture2D(source);
// videoSourceImage = TextureToMat(tex);
// cannyImage = ProcessImage(videoSourceImage);
// cannyImage = CropImage(cannyImage);
// CenterOfGravityTest(cannyImage);
// processedTexture = MatToTexture(cannyImage);
// //processedTexture.Resize(80, 30);
// //processedTexture.Apply();
// //Graphics.Blit(processedTexture, destination);
// Graphics.Blit(processedTexture, renderTextureCropped);
//}
public void ProcessRenderTexture(RenderTexture source)
{
ProcessRenderTexture(source, source);
}
public void ProcessRenderTexture(RenderTexture source, RenderTexture target)
{
Texture2D tex = RenderTextureToTexture2D(source);
videoSourceImage = TextureToMat(tex);
@@ -68,7 +95,35 @@ public class ImageProcessor : MonoBehaviour
cannyImage = CropImage(cannyImage);
CenterOfGravityTest(cannyImage);
processedTexture = MatToTexture(cannyImage);
//processedTexture.Resize(80, 30);
//processedTexture.Apply();
//Graphics.Blit(processedTexture, destination);
Graphics.Blit(processedTexture, target);
}
//public void ProcessRenderTexture_NEW(RenderTexture source, RenderTexture target)
//{
// RenderTextureToTexture2D_NEW(source);
// TextureToMat_NEW(firstTexture);
// ProcessImage_NEW(videoSourceImage);
// cannyImage = CropImage(cannyImage);
// CenterOfGravityTest(cannyImage);
// MatToTexture_NEW(cannyImage);
// //processedTexture.Resize(80, 30);
// //processedTexture.Apply();
// //Graphics.Blit(processedTexture, destination);
// Graphics.Blit(processedTexture, target);
//}
public void OnRenderImage(RenderTexture source, RenderTexture destination)
{
RenderTextureToTexture2D_NEW(source);
TextureToMat_NEW(firstTexture);
ProcessImage_NEW(videoSourceImage);
cannyImage = CropImage(cannyImage);
CenterOfGravityTest(cannyImage);
MatToTexture_NEW(cannyImage);
Graphics.Blit(processedTexture, destination);
Graphics.Blit(processedTexture, renderTextureCropped);
}
@@ -81,6 +136,15 @@ public class ImageProcessor : MonoBehaviour
return croppedImage;
}
//// Crop image to just see the middle of the original image
//private Mat CropImage_NEW(Mat image)
//{
// //cut a fourth out of the top and bottom of the image
// OpenCvSharp.Rect rectCroped = new OpenCvSharp.Rect(0, image.Height / 4, image.Width, image.Height / 2);
// Mat croppedImage = new Mat(image, rectCroped);
// return croppedImage;
//}
private Texture2D RenderTextureToTexture2D(RenderTexture rTex)
{
Texture2D tex = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
@@ -90,6 +154,13 @@ public class ImageProcessor : MonoBehaviour
return tex;
}
private void RenderTextureToTexture2D_NEW(RenderTexture rTex)
{
RenderTexture.active = rTex;
firstTexture.ReadPixels(new UnityEngine.Rect(0, 0, rTex.width, rTex.height), 0, 0);
firstTexture.Apply();
}
// Convert Unity Texture2D object to OpenCVSharp Mat object
private Mat TextureToMat(Texture2D source)
{
@@ -121,6 +192,36 @@ public class ImageProcessor : MonoBehaviour
return tmpMat;
}
// Convert Unity Texture2D object to OpenCVSharp Mat object
private void TextureToMat_NEW(Texture2D source)
{
// Color32 array : r, g, b, a
Color32[] c = source.GetPixels32();
// Parallel for loop
// convert Color32 object to Vec3b object
// Vec3b is the representation of pixel for Mat
Parallel.For(0, imHeight, i =>
{
for (var j = 0; j < imWidth; j++)
{
var col = c[j + i * imWidth];
var vec3 = new Vec3b
{
Item0 = col.b,
Item1 = col.g,
Item2 = col.r
};
// set pixel to an array
videoSourceImageData[j + i * imWidth] = vec3;
}
});
// assign the Vec3b array to Mat
videoSourceImage.SetArray(0, 0, videoSourceImageData);
}
// Simple example of canny edge detect
private Mat ProcessImage(Mat _image)
{
@@ -129,6 +230,12 @@ public class ImageProcessor : MonoBehaviour
return cannyImg;
}
// Simple example of canny edge detect
private void ProcessImage_NEW(Mat _image)
{
Cv2.Canny(_image, cannyImage, 100, 100);
}
// Convert OpenCVSharp Mat object to Unity Texture2D object
private Texture2D MatToTexture(Mat mat)
{
@@ -161,6 +268,37 @@ public class ImageProcessor : MonoBehaviour
return texture;
}
// Convert OpenCVSharp Mat object to Unity Texture2D object
private void MatToTexture_NEW(Mat mat)
{
// cannyImageData is byte array, because canny image is grayscale
mat.GetArray(0, 0, cannyImageData);
// create Color32 array that can be assigned to Texture2D directly
Color32[] c = new Color32[croppedImHeight * imWidth];
// parallel for loop
Parallel.For(0, croppedImHeight, i =>
{
for (var j = 0; j < imWidth; j++)
{
byte vec = cannyImageData[j + i * imWidth];
var color32 = new Color32
{
r = vec,
g = vec,
b = vec,
a = 0
};
c[j + i * imWidth] = color32;
}
});
processedTexture.SetPixels32(c);
// to update the texture, OpenGL manner
processedTexture.Apply();
}
private void CenterOfGravityTest(Mat processedImage)
{
Assets/Scripts/ML Cozmo/CozmoAcademy.cs
+1
View File
@@ -12,6 +12,7 @@ public class CozmoAcademy : Academy
public override void AcademyReset()
{
cozmo.transform.position = startPoint.position;
cozmo.transform.rotation = startPoint.rotation;
}
//private void Update()
Assets/Scripts/ML Cozmo/CozmoAgent.cs
+5
View File
@@ -83,6 +83,7 @@ public class CozmoAgent : Agent
if (renderCamera != null)
{
renderCamera.Render();
//onRenderImageTest.ProcessRenderTexture_NEW(renderCamera.targetTexture, onRenderImageTest.renderTextureCropped);
}
@@ -102,6 +103,8 @@ public class CozmoAgent : Agent
SetReward(-1);
print("Reward: -1");
}
onRenderImageTest.enabled = false;
}
// to be implemented by the developer
@@ -129,6 +132,7 @@ public class CozmoAgent : Agent
if (!academy.GetIsInference())
{
//onRenderImageTest.ProcessRenderTexture_NEW(renderCamera.targetTexture, onRenderImageTest.renderTextureCropped);
RequestDecision();
}
else
@@ -136,6 +140,7 @@ public class CozmoAgent : Agent
if (timeSinceDecision >= timeBetweenDecisionsAtInference)
{
timeSinceDecision = 0f;
//onRenderImageTest.ProcessRenderTexture_NEW(renderCamera.targetTexture, onRenderImageTest.renderTextureCropped);
RequestDecision();
}
else
Assets/Textures/RenderTextures/CozmoCannyCroped.renderTexture
+2 -2
View File
@@ -12,8 +12,8 @@ RenderTexture:
Hash: 00000000000000000000000000000000
m_ForcedFallbackFormat: 4
m_DownscaleFallback: 0
m_Width: 320
m_Height: 120
m_Width: 80
m_Height: 30
m_AntiAliasing: 1
m_DepthFormat: 2
m_ColorFormat: 0