Browse Source

Add CenterOfGravity detection / Add reward function

Development
Tobi 5 years ago
parent
commit
a7dd5527bb

+ 2
- 2
Assets/Materials/GroundMaterial.mat View File

m_PrefabInstance: {fileID: 0} m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0} m_PrefabAsset: {fileID: 0}
m_Name: GroundMaterial m_Name: GroundMaterial
m_Shader: {fileID: 46, guid: 0000000000000000f000000000000000, type: 0}
m_Shader: {fileID: 10752, guid: 0000000000000000f000000000000000, type: 0}
m_ShaderKeywords: m_ShaderKeywords:
m_LightmapFlags: 4 m_LightmapFlags: 4
m_EnableInstancingVariants: 0 m_EnableInstancingVariants: 0
m_Scale: {x: 1, y: 1} m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0} m_Offset: {x: 0, y: 0}
- _MainTex: - _MainTex:
m_Texture: {fileID: 0}
m_Texture: {fileID: 2800000, guid: 199bbc3b06fa73149b6e1ad35ae6d936, type: 3}
m_Scale: {x: 1, y: 1} m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0} m_Offset: {x: 0, y: 0}
- _MetallicGlossMap: - _MetallicGlossMap:

+ 1030
- 164
Assets/Scenes/CozmoTraining.unity
File diff suppressed because it is too large
View File


Assets/Scripts/CannyEdgeDetection/OnRenderImageTest.cs → Assets/Scripts/CannyEdgeDetection/ImageProcessor.cs View File

using OpenCvSharp; using OpenCvSharp;
using System.Threading.Tasks; using System.Threading.Tasks;


public class OnRenderImageTest : MonoBehaviour
public class ImageProcessor : MonoBehaviour
{ {
public MeshRenderer processedImageRenderer; public MeshRenderer processedImageRenderer;
public Point CenterOfGravity { get; private set; }


// OpenCVSharp parameters // OpenCVSharp parameters
private Mat videoSourceImage; private Mat videoSourceImage;
Texture2D tex = RenderTextureToTexture2D(source); Texture2D tex = RenderTextureToTexture2D(source);
videoSourceImage = TextureToMat(tex); videoSourceImage = TextureToMat(tex);
cannyImage = ProcessImage(videoSourceImage); cannyImage = ProcessImage(videoSourceImage);
CenterOfGravityTest(cannyImage);
processedTexture = MatToTexture(cannyImage); processedTexture = MatToTexture(cannyImage);
Graphics.Blit(processedTexture, destination); Graphics.Blit(processedTexture, destination);
} }
texture.Apply(); texture.Apply();
return texture; return texture;
} }


private void CenterOfGravityTest(Mat processedImage)
{
// find moments of the image
Moments m = new Moments(processedImage, true);
CenterOfGravity = new Point(m.M10 / m.M00, m.M01 / m.M00);

// show the image with a point mark at the centroid
Cv2.Circle(processedImage, CenterOfGravity, 5, new Scalar(128, 0, 0), -1);
Cv2.Flip(processedImage, processedImage, FlipMode.X);
Cv2.ImShow("Image with center", processedImage);
//Cv2.WaitKey(0);
}
} }

Assets/Scripts/CannyEdgeDetection/OnRenderImageTest.cs.meta → Assets/Scripts/CannyEdgeDetection/ImageProcessor.cs.meta View File


+ 98
- 2
Assets/Scripts/ML Cozmo/CozmoAgent.cs View File

using MLAgents; using MLAgents;
using OpenCvSharp;
using System;
using System.Collections; using System.Collections;
using System.Collections.Generic; using System.Collections.Generic;
using UnityEngine; using UnityEngine;


public class CozmoAgent : Agent public class CozmoAgent : Agent
{ {
// Possible Actions
private const int STOP = 0;
private const int FORWARD = 1;
private const int RIGHT = 2;
private const int LEFT = 3;


// Used to determine different areas in the image (near to the center, far away)
private const float NEAR_AREA_PERCENTAGE_OFFSET = 0.2f;
private const float FAR_AREA_PERCENTAGE_OFFSET = 0.3f;

[Tooltip("The virtual Cozmo camera")]
public Camera renderCamera; public Camera renderCamera;
public Academy academy;
[Tooltip("Reference to the CozmoMovement script")]
public CozmoMovement movement;
public float timeBetweenDecisionsAtInference; public float timeBetweenDecisionsAtInference;
private float timeSinceDecision;

private Academy academy; // CozmoAcademy
private float timeSinceDecision; // time since last decision
private ImageProcessor onRenderImageTest; // reference to the ImageProcessor
private int nearAreaLimit = 0; // X coordinate limit for the near to the imagecenter area
private int farAreaLimit = 0; // X coordinate limit for the far away to the imagecenter area

// for testing
//private float[] floats = { 1.0f, 2.0f, 3.0f };

private void Start()
{
academy = FindObjectOfType(typeof(CozmoAcademy)) as CozmoAcademy;
onRenderImageTest = renderCamera.GetComponent<ImageProcessor>();
nearAreaLimit = (int)(renderCamera.targetTexture.width / 2 * NEAR_AREA_PERCENTAGE_OFFSET);
farAreaLimit = (int)(renderCamera.targetTexture.width / 2 * FAR_AREA_PERCENTAGE_OFFSET);
}


public void FixedUpdate() public void FixedUpdate()
{ {
WaitTimeInference(); WaitTimeInference();
// for testing
//AgentAction(floats, "ActionText");
} }



// to be implemented by the developer
public override void AgentAction(float[] vectorAction, string textAction)
{

int action = Mathf.FloorToInt(vectorAction[0]);
Point centerOfGravity = onRenderImageTest.CenterOfGravity;
Vector3 targetPos = transform.position;

AddReward(-0.01f);

switch (action)
{
case STOP:
movement.Move(0);
break;
case FORWARD:
movement.Move(1);
break;
case RIGHT:
movement.Turn(1);
break;
case LEFT:
movement.Turn(-1);
break;
default:
movement.Move(0);
throw new ArgumentException("Invalid action value. Stop movement.");
}

// Render new image after movement in order to update the centerOfGravity
if (renderCamera != null)
{
renderCamera.Render();
}


// If centerOfGravity lies near to the center of the image horizontally
if (centerOfGravity.X > renderCamera.targetTexture.width / 2 - nearAreaLimit && centerOfGravity.X < renderCamera.targetTexture.width / 2 + nearAreaLimit)
{
Done();
SetReward(1);
print("Reward: +1");
}
else if (centerOfGravity.X > renderCamera.targetTexture.width / 2 - farAreaLimit && centerOfGravity.X < renderCamera.targetTexture.width / 2 + farAreaLimit)
{
Done();
SetReward(-1);
print("Reward: -1");
}
else
{
Done();
SetReward(-2);
print("Reward: -2");
}
}

// to be implemented by the developer
public override void AgentReset()
{
academy.AcademyReset();
}


private void WaitTimeInference() private void WaitTimeInference()
{ {
if (renderCamera != null) if (renderCamera != null)

+ 23
- 0
Assets/Scripts/SceneHelper.cs View File

using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class SceneHelper : MonoBehaviour
{
[Tooltip("All the objects in this list will be deactivated/activated when the game is running")]
public List<GameObject> toggleInPlayMode;


public void Awake()
{
ToggleObjectList();
}

private void ToggleObjectList()
{
foreach (GameObject go in toggleInPlayMode)
{
go.SetActive(!go.activeSelf);
}
}
}

+ 11
- 0
Assets/Scripts/SceneHelper.cs.meta View File

fileFormatVersion: 2
guid: bc64682be4f86884ebe75488344a51b2
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

+ 8
- 0
Assets/Textures/Ground.meta View File

fileFormatVersion: 2
guid: 9019794d9e0b39645a29c012bfe3adcb
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

BIN
Assets/Textures/Ground/ersterTest.png View File


+ 88
- 0
Assets/Textures/Ground/ersterTest.png.meta View File

fileFormatVersion: 2
guid: 199bbc3b06fa73149b6e1ad35ae6d936
TextureImporter:
fileIDToRecycleName: {}
externalObjects: {}
serializedVersion: 9
mipmaps:
mipMapMode: 0
enableMipMap: 1
sRGBTexture: 1
linearTexture: 0
fadeOut: 0
borderMipMap: 0
mipMapsPreserveCoverage: 0
alphaTestReferenceValue: 0.5
mipMapFadeDistanceStart: 1
mipMapFadeDistanceEnd: 3
bumpmap:
convertToNormalMap: 0
externalNormalMap: 0
heightScale: 0.25
normalMapFilter: 0
isReadable: 0
streamingMipmaps: 0
streamingMipmapsPriority: 0
grayScaleToAlpha: 0
generateCubemap: 6
cubemapConvolution: 0
seamlessCubemap: 0
textureFormat: 1
maxTextureSize: 2048
textureSettings:
serializedVersion: 2
filterMode: -1
aniso: -1
mipBias: -100
wrapU: -1
wrapV: -1
wrapW: -1
nPOTScale: 1
lightmap: 0
compressionQuality: 50
spriteMode: 0
spriteExtrude: 1
spriteMeshType: 1
alignment: 0
spritePivot: {x: 0.5, y: 0.5}
spritePixelsToUnits: 100
spriteBorder: {x: 0, y: 0, z: 0, w: 0}
spriteGenerateFallbackPhysicsShape: 1
alphaUsage: 1
alphaIsTransparency: 0
spriteTessellationDetail: -1
textureType: 0
textureShape: 1
singleChannelComponent: 0
maxTextureSizeSet: 0
compressionQualitySet: 0
textureFormatSet: 0
platformSettings:
- serializedVersion: 2
buildTarget: DefaultTexturePlatform
maxTextureSize: 2048
resizeAlgorithm: 0
textureFormat: -1
textureCompression: 1
compressionQuality: 50
crunchedCompression: 0
allowsAlphaSplitting: 0
overridden: 0
androidETC2FallbackOverride: 0
spriteSheet:
serializedVersion: 2
sprites: []
outline: []
physicsShape: []
bones: []
spriteID:
vertices: []
indices:
edges: []
weights: []
spritePackingTag:
pSDRemoveMatte: 0
pSDShowRemoveMatteOption: 0
userData:
assetBundleName:
assetBundleVariant:

+ 2
- 2
Assets/Textures/Measurement/Materials/Schachbrett_10cm.mat View File

m_Scale: {x: 1, y: 1} m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0} m_Offset: {x: 0, y: 0}
- _MainTex: - _MainTex:
m_Texture: {fileID: 2800000, guid: cc90c9868898fb4499e99837f404fcdc, type: 3}
m_Scale: {x: 10, y: 10}
m_Texture: {fileID: 2800000, guid: 199bbc3b06fa73149b6e1ad35ae6d936, type: 3}
m_Scale: {x: 1, y: 1}
m_Offset: {x: 0, y: 0} m_Offset: {x: 0, y: 0}
- _MetallicGlossMap: - _MetallicGlossMap:
m_Texture: {fileID: 0} m_Texture: {fileID: 0}

Loading…
Cancel
Save