Add CenterOfGravity detection / Add reward function
This commit is contained in:
parent
7317135cd3
commit
a7dd5527bb
@ -8,7 +8,7 @@ Material:
|
||||
m_PrefabInstance: {fileID: 0}
|
||||
m_PrefabAsset: {fileID: 0}
|
||||
m_Name: GroundMaterial
|
||||
m_Shader: {fileID: 46, guid: 0000000000000000f000000000000000, type: 0}
|
||||
m_Shader: {fileID: 10752, guid: 0000000000000000f000000000000000, type: 0}
|
||||
m_ShaderKeywords:
|
||||
m_LightmapFlags: 4
|
||||
m_EnableInstancingVariants: 0
|
||||
@ -40,7 +40,7 @@ Material:
|
||||
m_Scale: {x: 1, y: 1}
|
||||
m_Offset: {x: 0, y: 0}
|
||||
- _MainTex:
|
||||
m_Texture: {fileID: 0}
|
||||
m_Texture: {fileID: 2800000, guid: 199bbc3b06fa73149b6e1ad35ae6d936, type: 3}
|
||||
m_Scale: {x: 1, y: 1}
|
||||
m_Offset: {x: 0, y: 0}
|
||||
- _MetallicGlossMap:
|
||||
|
File diff suppressed because it is too large
Load Diff
@ -4,9 +4,10 @@ using UnityEngine;
|
||||
using OpenCvSharp;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
public class OnRenderImageTest : MonoBehaviour
|
||||
public class ImageProcessor : MonoBehaviour
|
||||
{
|
||||
public MeshRenderer processedImageRenderer;
|
||||
public Point CenterOfGravity { get; private set; }
|
||||
|
||||
// OpenCVSharp parameters
|
||||
private Mat videoSourceImage;
|
||||
@ -41,6 +42,7 @@ public class OnRenderImageTest : MonoBehaviour
|
||||
Texture2D tex = RenderTextureToTexture2D(source);
|
||||
videoSourceImage = TextureToMat(tex);
|
||||
cannyImage = ProcessImage(videoSourceImage);
|
||||
CenterOfGravityTest(cannyImage);
|
||||
processedTexture = MatToTexture(cannyImage);
|
||||
Graphics.Blit(processedTexture, destination);
|
||||
}
|
||||
@ -124,4 +126,18 @@ public class OnRenderImageTest : MonoBehaviour
|
||||
texture.Apply();
|
||||
return texture;
|
||||
}
|
||||
|
||||
|
||||
private void CenterOfGravityTest(Mat processedImage)
|
||||
{
|
||||
// find moments of the image
|
||||
Moments m = new Moments(processedImage, true);
|
||||
CenterOfGravity = new Point(m.M10 / m.M00, m.M01 / m.M00);
|
||||
|
||||
// show the image with a point mark at the centroid
|
||||
Cv2.Circle(processedImage, CenterOfGravity, 5, new Scalar(128, 0, 0), -1);
|
||||
Cv2.Flip(processedImage, processedImage, FlipMode.X);
|
||||
Cv2.ImShow("Image with center", processedImage);
|
||||
//Cv2.WaitKey(0);
|
||||
}
|
||||
}
|
@ -1,21 +1,117 @@
|
||||
using MLAgents;
|
||||
using OpenCvSharp;
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using UnityEngine;
|
||||
|
||||
public class CozmoAgent : Agent
|
||||
{
|
||||
// Possible Actions
|
||||
private const int STOP = 0;
|
||||
private const int FORWARD = 1;
|
||||
private const int RIGHT = 2;
|
||||
private const int LEFT = 3;
|
||||
|
||||
// Used to determine different areas in the image (near to the center, far away)
|
||||
private const float NEAR_AREA_PERCENTAGE_OFFSET = 0.2f;
|
||||
private const float FAR_AREA_PERCENTAGE_OFFSET = 0.3f;
|
||||
|
||||
[Tooltip("The virtual Cozmo camera")]
|
||||
public Camera renderCamera;
|
||||
public Academy academy;
|
||||
[Tooltip("Reference to the CozmoMovement script")]
|
||||
public CozmoMovement movement;
|
||||
public float timeBetweenDecisionsAtInference;
|
||||
private float timeSinceDecision;
|
||||
|
||||
private Academy academy; // CozmoAcademy
|
||||
private float timeSinceDecision; // time since last decision
|
||||
private ImageProcessor onRenderImageTest; // reference to the ImageProcessor
|
||||
private int nearAreaLimit = 0; // X coordinate limit for the near to the imagecenter area
|
||||
private int farAreaLimit = 0; // X coordinate limit for the far away to the imagecenter area
|
||||
|
||||
// for testing
|
||||
//private float[] floats = { 1.0f, 2.0f, 3.0f };
|
||||
|
||||
private void Start()
|
||||
{
|
||||
academy = FindObjectOfType(typeof(CozmoAcademy)) as CozmoAcademy;
|
||||
onRenderImageTest = renderCamera.GetComponent<ImageProcessor>();
|
||||
nearAreaLimit = (int)(renderCamera.targetTexture.width / 2 * NEAR_AREA_PERCENTAGE_OFFSET);
|
||||
farAreaLimit = (int)(renderCamera.targetTexture.width / 2 * FAR_AREA_PERCENTAGE_OFFSET);
|
||||
}
|
||||
|
||||
public void FixedUpdate()
|
||||
{
|
||||
WaitTimeInference();
|
||||
// for testing
|
||||
//AgentAction(floats, "ActionText");
|
||||
}
|
||||
|
||||
|
||||
// to be implemented by the developer
|
||||
public override void AgentAction(float[] vectorAction, string textAction)
|
||||
{
|
||||
|
||||
int action = Mathf.FloorToInt(vectorAction[0]);
|
||||
Point centerOfGravity = onRenderImageTest.CenterOfGravity;
|
||||
Vector3 targetPos = transform.position;
|
||||
|
||||
AddReward(-0.01f);
|
||||
|
||||
switch (action)
|
||||
{
|
||||
case STOP:
|
||||
movement.Move(0);
|
||||
break;
|
||||
case FORWARD:
|
||||
movement.Move(1);
|
||||
break;
|
||||
case RIGHT:
|
||||
movement.Turn(1);
|
||||
break;
|
||||
case LEFT:
|
||||
movement.Turn(-1);
|
||||
break;
|
||||
default:
|
||||
movement.Move(0);
|
||||
throw new ArgumentException("Invalid action value. Stop movement.");
|
||||
}
|
||||
|
||||
// Render new image after movement in order to update the centerOfGravity
|
||||
if (renderCamera != null)
|
||||
{
|
||||
renderCamera.Render();
|
||||
}
|
||||
|
||||
|
||||
// If centerOfGravity lies near to the center of the image horizontally
|
||||
if (centerOfGravity.X > renderCamera.targetTexture.width / 2 - nearAreaLimit && centerOfGravity.X < renderCamera.targetTexture.width / 2 + nearAreaLimit)
|
||||
{
|
||||
Done();
|
||||
SetReward(1);
|
||||
print("Reward: +1");
|
||||
}
|
||||
else if (centerOfGravity.X > renderCamera.targetTexture.width / 2 - farAreaLimit && centerOfGravity.X < renderCamera.targetTexture.width / 2 + farAreaLimit)
|
||||
{
|
||||
Done();
|
||||
SetReward(-1);
|
||||
print("Reward: -1");
|
||||
}
|
||||
else
|
||||
{
|
||||
Done();
|
||||
SetReward(-2);
|
||||
print("Reward: -2");
|
||||
}
|
||||
}
|
||||
|
||||
// to be implemented by the developer
|
||||
public override void AgentReset()
|
||||
{
|
||||
academy.AcademyReset();
|
||||
}
|
||||
|
||||
|
||||
private void WaitTimeInference()
|
||||
{
|
||||
if (renderCamera != null)
|
||||
|
23
Assets/Scripts/SceneHelper.cs
Normal file
23
Assets/Scripts/SceneHelper.cs
Normal file
@ -0,0 +1,23 @@
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using UnityEngine;
|
||||
|
||||
public class SceneHelper : MonoBehaviour
|
||||
{
|
||||
[Tooltip("All the objects in this list will be deactivated/activated when the game is running")]
|
||||
public List<GameObject> toggleInPlayMode;
|
||||
|
||||
|
||||
public void Awake()
|
||||
{
|
||||
ToggleObjectList();
|
||||
}
|
||||
|
||||
private void ToggleObjectList()
|
||||
{
|
||||
foreach (GameObject go in toggleInPlayMode)
|
||||
{
|
||||
go.SetActive(!go.activeSelf);
|
||||
}
|
||||
}
|
||||
}
|
11
Assets/Scripts/SceneHelper.cs.meta
Normal file
11
Assets/Scripts/SceneHelper.cs.meta
Normal file
@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: bc64682be4f86884ebe75488344a51b2
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
8
Assets/Textures/Ground.meta
Normal file
8
Assets/Textures/Ground.meta
Normal file
@ -0,0 +1,8 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 9019794d9e0b39645a29c012bfe3adcb
|
||||
folderAsset: yes
|
||||
DefaultImporter:
|
||||
externalObjects: {}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
BIN
Assets/Textures/Ground/ersterTest.png
Normal file
BIN
Assets/Textures/Ground/ersterTest.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 459 KiB |
88
Assets/Textures/Ground/ersterTest.png.meta
Normal file
88
Assets/Textures/Ground/ersterTest.png.meta
Normal file
@ -0,0 +1,88 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 199bbc3b06fa73149b6e1ad35ae6d936
|
||||
TextureImporter:
|
||||
fileIDToRecycleName: {}
|
||||
externalObjects: {}
|
||||
serializedVersion: 9
|
||||
mipmaps:
|
||||
mipMapMode: 0
|
||||
enableMipMap: 1
|
||||
sRGBTexture: 1
|
||||
linearTexture: 0
|
||||
fadeOut: 0
|
||||
borderMipMap: 0
|
||||
mipMapsPreserveCoverage: 0
|
||||
alphaTestReferenceValue: 0.5
|
||||
mipMapFadeDistanceStart: 1
|
||||
mipMapFadeDistanceEnd: 3
|
||||
bumpmap:
|
||||
convertToNormalMap: 0
|
||||
externalNormalMap: 0
|
||||
heightScale: 0.25
|
||||
normalMapFilter: 0
|
||||
isReadable: 0
|
||||
streamingMipmaps: 0
|
||||
streamingMipmapsPriority: 0
|
||||
grayScaleToAlpha: 0
|
||||
generateCubemap: 6
|
||||
cubemapConvolution: 0
|
||||
seamlessCubemap: 0
|
||||
textureFormat: 1
|
||||
maxTextureSize: 2048
|
||||
textureSettings:
|
||||
serializedVersion: 2
|
||||
filterMode: -1
|
||||
aniso: -1
|
||||
mipBias: -100
|
||||
wrapU: -1
|
||||
wrapV: -1
|
||||
wrapW: -1
|
||||
nPOTScale: 1
|
||||
lightmap: 0
|
||||
compressionQuality: 50
|
||||
spriteMode: 0
|
||||
spriteExtrude: 1
|
||||
spriteMeshType: 1
|
||||
alignment: 0
|
||||
spritePivot: {x: 0.5, y: 0.5}
|
||||
spritePixelsToUnits: 100
|
||||
spriteBorder: {x: 0, y: 0, z: 0, w: 0}
|
||||
spriteGenerateFallbackPhysicsShape: 1
|
||||
alphaUsage: 1
|
||||
alphaIsTransparency: 0
|
||||
spriteTessellationDetail: -1
|
||||
textureType: 0
|
||||
textureShape: 1
|
||||
singleChannelComponent: 0
|
||||
maxTextureSizeSet: 0
|
||||
compressionQualitySet: 0
|
||||
textureFormatSet: 0
|
||||
platformSettings:
|
||||
- serializedVersion: 2
|
||||
buildTarget: DefaultTexturePlatform
|
||||
maxTextureSize: 2048
|
||||
resizeAlgorithm: 0
|
||||
textureFormat: -1
|
||||
textureCompression: 1
|
||||
compressionQuality: 50
|
||||
crunchedCompression: 0
|
||||
allowsAlphaSplitting: 0
|
||||
overridden: 0
|
||||
androidETC2FallbackOverride: 0
|
||||
spriteSheet:
|
||||
serializedVersion: 2
|
||||
sprites: []
|
||||
outline: []
|
||||
physicsShape: []
|
||||
bones: []
|
||||
spriteID:
|
||||
vertices: []
|
||||
indices:
|
||||
edges: []
|
||||
weights: []
|
||||
spritePackingTag:
|
||||
pSDRemoveMatte: 0
|
||||
pSDShowRemoveMatteOption: 0
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
@ -40,8 +40,8 @@ Material:
|
||||
m_Scale: {x: 1, y: 1}
|
||||
m_Offset: {x: 0, y: 0}
|
||||
- _MainTex:
|
||||
m_Texture: {fileID: 2800000, guid: cc90c9868898fb4499e99837f404fcdc, type: 3}
|
||||
m_Scale: {x: 10, y: 10}
|
||||
m_Texture: {fileID: 2800000, guid: 199bbc3b06fa73149b6e1ad35ae6d936, type: 3}
|
||||
m_Scale: {x: 1, y: 1}
|
||||
m_Offset: {x: 0, y: 0}
|
||||
- _MetallicGlossMap:
|
||||
m_Texture: {fileID: 0}
|
||||
|
Loading…
x
Reference in New Issue
Block a user