Browse Source

Crop Image / Training now works / Got memory error while training

Development
Tobi 5 years ago
parent
commit
8082fde44c

+ 1
- 1
Assets/Brains/CozmoLearningBrain.asset View File

vectorActionSize: 04000000 vectorActionSize: 04000000
cameraResolutions: cameraResolutions:
- width: 320 - width: 320
height: 240
height: 120
blackAndWhite: 0 blackAndWhite: 0
vectorActionDescriptions: vectorActionDescriptions:
- -

+ 104
- 10
Assets/Scenes/CozmoTraining.unity View File

- {fileID: 380448017} - {fileID: 380448017}
- {fileID: 2078653368} - {fileID: 2078653368}
- {fileID: 435986020} - {fileID: 435986020}
- {fileID: 712179487}
m_Father: {fileID: 0} m_Father: {fileID: 0}
m_RootOrder: 1 m_RootOrder: 1
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
m_Father: {fileID: 7570006596986120124} m_Father: {fileID: 7570006596986120124}
m_RootOrder: 2 m_RootOrder: 2
m_LocalEulerAnglesHint: {x: 35, y: 0, z: 0} m_LocalEulerAnglesHint: {x: 35, y: 0, z: 0}
--- !u!1 &712179486
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 712179487}
- component: {fileID: 712179490}
- component: {fileID: 712179489}
- component: {fileID: 712179488}
m_Layer: 0
m_Name: TextureDisplayCropped
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!4 &712179487
Transform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 712179486}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: 1.35, y: 1.5, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_Children: []
m_Father: {fileID: 99578049}
m_RootOrder: 6
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!64 &712179488
MeshCollider:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 712179486}
m_Material: {fileID: 0}
m_IsTrigger: 0
m_Enabled: 1
serializedVersion: 3
m_Convex: 0
m_CookingOptions: 14
m_Mesh: {fileID: 10210, guid: 0000000000000000e000000000000000, type: 0}
--- !u!23 &712179489
MeshRenderer:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 712179486}
m_Enabled: 1
m_CastShadows: 1
m_ReceiveShadows: 1
m_DynamicOccludee: 1
m_MotionVectors: 1
m_LightProbeUsage: 1
m_ReflectionProbeUsage: 1
m_RenderingLayerMask: 1
m_RendererPriority: 0
m_Materials:
- {fileID: 10303, guid: 0000000000000000f000000000000000, type: 0}
m_StaticBatchInfo:
firstSubMesh: 0
subMeshCount: 0
m_StaticBatchRoot: {fileID: 0}
m_ProbeAnchor: {fileID: 0}
m_LightProbeVolumeOverride: {fileID: 0}
m_ScaleInLightmap: 1
m_PreserveUVs: 0
m_IgnoreNormalsForChartDetection: 0
m_ImportantGI: 0
m_StitchLightmapSeams: 0
m_SelectedEditorRenderState: 3
m_MinimumChartSize: 4
m_AutoUVMaxDistance: 0.5
m_AutoUVMaxAngle: 89
m_LightmapParameters: {fileID: 0}
m_SortingLayerID: 0
m_SortingLayer: 0
m_SortingOrder: 0
--- !u!33 &712179490
MeshFilter:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 712179486}
m_Mesh: {fileID: 10210, guid: 0000000000000000e000000000000000, type: 0}
--- !u!1 &730369748 --- !u!1 &730369748
GameObject: GameObject:
m_ObjectHideFlags: 0 m_ObjectHideFlags: 0
m_Script: {fileID: 11500000, guid: b940f59f459dc49479614595d02dcbbc, type: 3} m_Script: {fileID: 11500000, guid: b940f59f459dc49479614595d02dcbbc, type: 3}
m_Name: m_Name:
m_EditorClassIdentifier: m_EditorClassIdentifier:
processedImageRenderer: {fileID: 1863707383}
renderTextureCropped: {fileID: 8400000, guid: 8bf12f4401860c043a77209428e97b29,
type: 2}
processedImageRendererCropped: {fileID: 712179489}
--- !u!1 &918537782 --- !u!1 &918537782
GameObject: GameObject:
m_ObjectHideFlags: 0 m_ObjectHideFlags: 0
m_Layer: 0 m_Layer: 0
m_Name: StartPoint m_Name: StartPoint
m_TagString: Untagged m_TagString: Untagged
m_Icon: {fileID: 3936346786652291628, guid: 0000000000000000d000000000000000, type: 0}
m_Icon: {fileID: 0}
m_NavMeshLayer: 0 m_NavMeshLayer: 0
m_StaticEditorFlags: 0 m_StaticEditorFlags: 0
m_IsActive: 1 m_IsActive: 1
type: 3} type: 3}
propertyPath: m_Icon propertyPath: m_Icon
value: value:
objectReference: {fileID: 5132851093641282708, guid: 0000000000000000d000000000000000,
type: 0}
objectReference: {fileID: 0}
- target: {fileID: 1857425275647696726, guid: 0f97dac5215d69a4795763340d82925d,
type: 3}
propertyPath: m_IsActive
value: 1
objectReference: {fileID: 0}
- target: {fileID: 7570006595835424293, guid: 0f97dac5215d69a4795763340d82925d, - target: {fileID: 7570006595835424293, guid: 0f97dac5215d69a4795763340d82925d,
type: 3} type: 3}
propertyPath: m_LocalPosition.x propertyPath: m_LocalPosition.x
propertyPath: m_LocalEulerAnglesHint.z propertyPath: m_LocalEulerAnglesHint.z
value: 0 value: 0
objectReference: {fileID: 0} objectReference: {fileID: 0}
- target: {fileID: 1857425275647696726, guid: 0f97dac5215d69a4795763340d82925d,
type: 3}
propertyPath: m_IsActive
value: 1
objectReference: {fileID: 0}
m_RemovedComponents: [] m_RemovedComponents: []
m_SourcePrefab: {fileID: 100100000, guid: 0f97dac5215d69a4795763340d82925d, type: 3} m_SourcePrefab: {fileID: 100100000, guid: 0f97dac5215d69a4795763340d82925d, type: 3}
--- !u!1 &7570006596986120123 stripped --- !u!1 &7570006596986120123 stripped
agentParameters: agentParameters:
agentCameras: [] agentCameras: []
agentRenderTextures: agentRenderTextures:
- {fileID: 8400000, guid: 196f45b12806aaa449adc2ee1ca3b091, type: 2}
- {fileID: 8400000, guid: 8bf12f4401860c043a77209428e97b29, type: 2}
maxStep: 1000 maxStep: 1000
resetOnDone: 1 resetOnDone: 1
onDemandDecision: 1 onDemandDecision: 1

+ 46
- 12
Assets/Scripts/CannyEdgeDetection/ImageProcessor.cs View File



public class ImageProcessor : MonoBehaviour public class ImageProcessor : MonoBehaviour
{ {
public MeshRenderer processedImageRenderer;
[Header("RenderTexture")]
[Tooltip("RenderTexture that will be passed to the LearningBrain.")]
public RenderTexture renderTextureCropped;

[Header("Debug Helper")]
//[Tooltip("Reference to the MeshRenderer that will show the processed Image from Cozmo")]
//public MeshRenderer processedImageRenderer;
[Tooltip("Reference to the MeshRenderer that will show the processed and cropped Image from Cozmo")]
public MeshRenderer processedImageRendererCropped;

/// <summary>
/// Center of Gravity in the cropped canny image
/// </summary>
public Point CenterOfGravity { get; private set; } public Point CenterOfGravity { get; private set; }


// OpenCVSharp parameters // OpenCVSharp parameters
private byte[] cannyImageData; private byte[] cannyImageData;




private const int imWidth = 320; //TODO: Set width and height based on agent observation size
private const int imHeight = 240;
private Camera textureCamera;
private int imWidth = 320; // Width of the camera image from the virtual cozmo
private int imHeight = 240; // Height of the camera image from the virtual cozmo
private int croppedImHeight = 120; // Height of the cropped camera image from the virtual cozmo
private Camera textureCamera; // Virtual Cozmo camera


private void Start() private void Start()
{ {
// Get reference to the cozmo camera
textureCamera = GetComponent<Camera>(); textureCamera = GetComponent<Camera>();


// Set image widths and heights based on the given RenderTextures
imWidth = textureCamera.targetTexture.width;
imHeight = textureCamera.targetTexture.height;
croppedImHeight = renderTextureCropped.height;

//assign the processed targetTexture to the renderer to display the image //assign the processed targetTexture to the renderer to display the image
processedImageRenderer.material.mainTexture = textureCamera.targetTexture;
//processedImageRenderer.material.mainTexture = textureCamera.targetTexture;
processedImageRendererCropped.material.mainTexture = renderTextureCropped;


// initialize video / image with given size // initialize video / image with given size
videoSourceImage = new Mat(imHeight, imWidth, MatType.CV_8UC3); videoSourceImage = new Mat(imHeight, imWidth, MatType.CV_8UC3);
videoSourceImageData = new Vec3b[imHeight * imWidth]; videoSourceImageData = new Vec3b[imHeight * imWidth];
cannyImage = new Mat(imHeight, imWidth, MatType.CV_8UC1); cannyImage = new Mat(imHeight, imWidth, MatType.CV_8UC1);
cannyImageData = new byte[imHeight * imWidth];
cannyImageData = new byte[croppedImHeight * imWidth];
} }



/// <summary>
/// Gets called when a new image arrives from the camera this script lies on
/// </summary>
/// <param name="source"></param>
/// <param name="destination"></param>
private void OnRenderImage(RenderTexture source, RenderTexture destination) private void OnRenderImage(RenderTexture source, RenderTexture destination)
{ {
Texture2D tex = RenderTextureToTexture2D(source); Texture2D tex = RenderTextureToTexture2D(source);
videoSourceImage = TextureToMat(tex); videoSourceImage = TextureToMat(tex);
cannyImage = ProcessImage(videoSourceImage); cannyImage = ProcessImage(videoSourceImage);
cannyImage = CropImage(cannyImage);
CenterOfGravityTest(cannyImage); CenterOfGravityTest(cannyImage);
processedTexture = MatToTexture(cannyImage); processedTexture = MatToTexture(cannyImage);
Graphics.Blit(processedTexture, destination);
//Graphics.Blit(processedTexture, destination);
Graphics.Blit(processedTexture, renderTextureCropped);
}

// Crop image to just see the middle of the original image
private Mat CropImage(Mat image)
{
//cut a fourth out of the top and bottom of the image
OpenCvSharp.Rect rectCroped = new OpenCvSharp.Rect(0, image.Height / 4, image.Width, image.Height / 2);
Mat croppedImage = new Mat(image, rectCroped);
return croppedImage;
} }


private Texture2D RenderTextureToTexture2D(RenderTexture rTex) private Texture2D RenderTextureToTexture2D(RenderTexture rTex)
// cannyImageData is byte array, because canny image is grayscale // cannyImageData is byte array, because canny image is grayscale
mat.GetArray(0, 0, cannyImageData); mat.GetArray(0, 0, cannyImageData);
// create Color32 array that can be assigned to Texture2D directly // create Color32 array that can be assigned to Texture2D directly
Color32[] c = new Color32[imHeight * imWidth];
Color32[] c = new Color32[croppedImHeight * imWidth];


// parallel for loop // parallel for loop
Parallel.For(0, imHeight, i =>
Parallel.For(0, croppedImHeight, i =>
{ {
for (var j = 0; j < imWidth; j++) for (var j = 0; j < imWidth; j++)
{ {
} }
}); });


Texture2D texture = new Texture2D(imWidth, imHeight, TextureFormat.RGBA32, true, true);
Texture2D texture = new Texture2D(imWidth, croppedImHeight, TextureFormat.RGBA32, true, true);
texture.SetPixels32(c); texture.SetPixels32(c);
// to update the texture, OpenGL manner // to update the texture, OpenGL manner
texture.Apply(); texture.Apply();

+ 2
- 2
Assets/Scripts/ML Cozmo/CozmoAgent.cs View File

} }
else if (centerOfGravity.X > renderCamera.targetTexture.width / 2 - farAreaLimit && centerOfGravity.X < renderCamera.targetTexture.width / 2 + farAreaLimit) else if (centerOfGravity.X > renderCamera.targetTexture.width / 2 - farAreaLimit && centerOfGravity.X < renderCamera.targetTexture.width / 2 + farAreaLimit)
{ {
SetReward(-0.7f);
print("Reward: -0.7");
SetReward(-0.3f);
print("Reward: -0.3");
} }
else else
{ {

+ 34
- 0
Assets/Textures/RenderTextures/CozmoCannyCroped.renderTexture View File

%YAML 1.1
%TAG !u! tag:unity3d.com,2011:
--- !u!84 &8400000
RenderTexture:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_Name: CozmoCannyCroped
m_ImageContentsHash:
serializedVersion: 2
Hash: 00000000000000000000000000000000
m_ForcedFallbackFormat: 4
m_DownscaleFallback: 0
m_Width: 320
m_Height: 120
m_AntiAliasing: 1
m_DepthFormat: 2
m_ColorFormat: 0
m_MipMap: 0
m_GenerateMips: 1
m_SRGB: 0
m_UseDynamicScale: 0
m_BindMS: 0
m_TextureSettings:
serializedVersion: 2
m_FilterMode: 1
m_Aniso: 0
m_MipBias: 0
m_WrapU: 1
m_WrapV: 1
m_WrapW: 1
m_Dimension: 2
m_VolumeDepth: 1

+ 8
- 0
Assets/Textures/RenderTextures/CozmoCannyCroped.renderTexture.meta View File

fileFormatVersion: 2
guid: 8bf12f4401860c043a77209428e97b29
NativeFormatImporter:
externalObjects: {}
mainObjectFileID: 8400000
userData:
assetBundleName:
assetBundleVariant:

+ 22
- 0
ConfigFiles/trainer_config.yaml View File

default:
trainer: ppo
batch_size: 1024
beta: 5.0e-3
buffer_size: 10240
epsilon: 0.2
gamma: 0.99
hidden_units: 128
lambd: 0.95
learning_rate: 3.0e-4
max_steps: 5.0e4
memory_size: 256
normalize: false
num_epoch: 3
num_layers: 2
time_horizon: 64
sequence_length: 64
summary_freq: 1000
use_recurrent: false
use_curiosity: false
curiosity_strength: 0.01
curiosity_enc_size: 128

+ 58020
- 0
Notebooks/UnitySDK.log
File diff suppressed because it is too large
View File


+ 7
- 15
Notebooks/getting-started.ipynb
File diff suppressed because it is too large
View File


Loading…
Cancel
Save