#if EXPERIMENTAL_KERNELS_ENABLED
/*
#pragma kernel Dense
#pragma kernel DenseTiled
#pragma kernel Dense10x16
#pragma kernel DenseTiled32x32
#pragma kernel DenseTiled64x64
#pragma kernel Dense64
#pragma kernel Relu
#pragma kernel Relu256xV
#pragma kernel Relu16x16
#pragma kernel ReluChannelsFirst16x2x16
#pragma kernel Relu_Cmod16_CNyx
#pragma kernel Relu_Nyxc
#pragma kernel Softmax
#pragma kernel Softmax256x2
#pragma kernel MaxPooling2D
#pragma kernel MaxPooling2D16x4x4
*/
/*
#pragma kernel Conv2D_Kernel3x3_32Channel
#pragma kernel Conv2D_Kernel3x3_1Channel
#pragma kernel Conv2D
//#pragma kernel Conv2DTiled16x16_Kernel3x3
#pragma kernel Conv2DTiled14x14_Kernel3x3
#pragma kernel Conv2DTiled13x13_Kernel3x3
//#pragma kernel Conv2DTiled12x12_Kernel3x3
#pragma kernel Fill

#pragma kernel Conv2D_Kernel3x3_Kmod16_Cmod4_KN
#pragma kernel Conv2D_Kernel3x3_Kmod16_Cmod4_KNyx
//#pragma kernel Conv2D_Kernel3x3_Cache_KCmod32_KNyx
//#pragma kernel Conv2D_Kernel3x3_Cache_KCmod64_KNyx
*/


// @TODO: BIAS and WEIGHTS have changed format 
// BIAS      (0,0,x,0) -> (0,0,0,x) --> (x)
// WEIGHTS   (y,0,x,0) -> (y,0,0,x) --> (y,x)
// DENSE_OUT (y,0,x,0) -> (y,0,0,x) --> (y,x)


//#pragma kernel Conv2D_Kmod16_Nmod8_KNY
//#pragma kernel Conv2D_Kernel3x3_64

#define BOUNDS_CHECKS 0

RWStructuredBuffer<int> Edata;

struct Tensor
{
	uint batch, height, width, channels;
	uint offset;
	uint dataLength;

	uint Index(uint b, uint h, uint w, uint ch)
	{
		uint index =
			b * height * width * channels +
			h * width * channels +
			w * channels +
			ch;
		return index + offset;
	}
	void Set(uint b, uint h, uint w, uint ch, float v, RWStructuredBuffer<float> data)
	{
		data[Index(b,h,w,ch)] = v;
	}
	void Set(int b, uint h, uint w, uint ch, float v, RWStructuredBuffer<float> data, int dataLength)
	{
		uint index = Index(b,h,w,ch);
		#if BOUNDS_CHECKS
		if (index < 0 || index >= dataLength)
		{
			InterlockedAdd(Edata[1], 1);
			return;
		}
		#endif

		data[Index(b,h,w,ch)] = v;
	}

	float Get(uint b, uint h, uint w, uint ch, StructuredBuffer<float> data)
	{
		return data[Index(b,h,w,ch)];
	}
	float Get(uint b, uint h, uint w, uint ch, StructuredBuffer<float> data, int dataLength)
	{
		int index = Index(b,h,w,ch);
		#if BOUNDS_CHECKS
		if (index < 0 || index >= dataLength)
		{
			InterlockedAdd(Edata[0], 1);
			return 0.0f;
		}
		#endif
		
		return data[Index(b,h,w,ch)];
	}
};

#define X ((Tensor)Xdecl)
int4 Xdecl[2];
StructuredBuffer<float> Xdata;

#define O ((Tensor)Odecl)
int4 Odecl[2];
RWStructuredBuffer<float> Odata;

#define W ((Tensor)Wdecl)
int4 Wdecl[2];

#define B ((Tensor)Bdecl)
int4 Bdecl[2];

#define K ((Tensor)Kdecl)
int4 Kdecl[2];

#define WBK ((Tensor)WBKdecl)
int4 WBKdecl[2];
StructuredBuffer<float> WBKdata;

uint _FilterSize;
uint _Border;
uint _Offset;
 
[numthreads(1,1,1)]
void Dense(uint3 groupID : SV_GroupID)
{
	uint b = groupID.y;
	uint x = groupID.x;
	float v = B.Get(0, 0, x, 0, WBKdata, WBK.dataLength);
	for (uint i = 0; i < X.width; ++i)
		v += X.Get(b, 0, i, 0, Xdata) * W.Get(0, i, x, 0, WBKdata, WBK.dataLength);

	O.Set(b, 0, x, 0, v, Odata, O.dataLength);
}

[numthreads(10,16,1)]
void Dense10x16(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint x = 10*groupID.x + groupThreadID.x;
	uint b = 16*groupID.y + groupThreadID.y;
	float v = B.Get(0, 0, x, 0, WBKdata, WBK.dataLength);

	for (uint i = 0; i < X.width;)
	{
		// can unroll up to 16 because numthreads.y=16
		v += X.Get(b, 0, i, 0, Xdata) * W.Get(0, i, x, 0, WBKdata, WBK.dataLength); ++i;
		v += X.Get(b, 0, i, 0, Xdata) * W.Get(0, i, x, 0, WBKdata, WBK.dataLength); ++i;
		v += X.Get(b, 0, i, 0, Xdata) * W.Get(0, i, x, 0, WBKdata, WBK.dataLength); ++i;
		v += X.Get(b, 0, i, 0, Xdata) * W.Get(0, i, x, 0, WBKdata, WBK.dataLength); ++i;

		v += X.Get(b, 0, i, 0, Xdata) * W.Get(0, i, x, 0, WBKdata, WBK.dataLength); ++i;
		v += X.Get(b, 0, i, 0, Xdata) * W.Get(0, i, x, 0, WBKdata, WBK.dataLength); ++i;
		v += X.Get(b, 0, i, 0, Xdata) * W.Get(0, i, x, 0, WBKdata, WBK.dataLength); ++i;
		v += X.Get(b, 0, i, 0, Xdata) * W.Get(0, i, x, 0, WBKdata, WBK.dataLength); ++i;
	}
	O.Set(b, 0, x, 0, v, Odata);
}


#undef THREAD_COUNT
#define THREAD_COUNT 64 // ATM support only 8x8

#undef BLOCK_WIDTH
#define BLOCK_WIDTH 8

#undef LOAD_WIDTH
#define LOAD_WIDTH THREAD_COUNT

#undef LOAD_DEPTH
#define LOAD_DEPTH BLOCK_WIDTH

groupshared float Conv_KcacheR[LOAD_DEPTH][LOAD_WIDTH];
groupshared float Conv_XcacheR[LOAD_DEPTH][LOAD_WIDTH];
[numthreads(THREAD_COUNT, 1, 1)]
void Conv2D_Kernel3x3_64(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ Conv_XcacheR
	#define K_ Conv_KcacheR

	uint id = groupThreadID.x;
	uint bx = groupID.x;
	uint by = groupID.y;

	uint bbx = id % BLOCK_WIDTH;
	uint bby = id / BLOCK_WIDTH;

	uint width = O.width;
	uint height = O.height;

	// ASSERT(LOAD_WIDTH == THREAD_COUNT)
	uint loadNYX = by*LOAD_WIDTH + id; // only works for 8x8
	uint loadX = loadNYX % width;
	uint loadNY = loadNYX / width;
	uint loadY = loadNY % height;
	uint loadN = loadNY / height;

	float v[BLOCK_WIDTH][BLOCK_WIDTH];
	for (uint yy = 0; yy < BLOCK_WIDTH; ++yy)
		for (uint xx = 0; xx < BLOCK_WIDTH; ++xx)
		{
			float bias = B.Get(0, 0, bx*LOAD_WIDTH + bbx*BLOCK_WIDTH + xx, 0, WBKdata, WBK.dataLength);
			v[yy][xx] = bias;
		}

	for (uint dy = 0; dy < 3; ++dy)
	{
		bool mask = true;

		if (loadY+dy < _Offset) mask = false;
		if (loadY+dy-_Offset >= X.height) mask = false;

		for (uint dx = 0; dx < 3; ++dx)
		{
			if (loadX+dx < _Offset) mask = false;
			if (loadX+dx-_Offset >= X.width) mask = false;

			for (uint m = 0; m < X.channels/LOAD_DEPTH; ++m)
			{
				for (uint q = 0; q < LOAD_DEPTH; ++q)
				{
					if (mask)
						X_[q][id] = X.Get(loadN, loadY+dy-_Offset, loadX+dx-_Offset, m*LOAD_DEPTH + q, Xdata);
					else
						X_[q][id] = 0;
					K_[q][id] = K.Get(dy, dx, m*LOAD_DEPTH + q, bx*LOAD_WIDTH + id, WBKdata, WBK.dataLength);
				}

				GroupMemoryBarrierWithGroupSync();

				for (uint yyy = 0; yyy < BLOCK_WIDTH; ++yyy)
					[unroll] for (uint xxx = 0; xxx < BLOCK_WIDTH; ++xxx) 
						[unroll] for (uint i = 0; i < LOAD_DEPTH; ++i)
						{
							v[yyy][xxx] += X_[i][bby*BLOCK_WIDTH + yyy] * K_[i][bbx*BLOCK_WIDTH + xxx];
						}

				GroupMemoryBarrierWithGroupSync();
			}
		}
	}

	for (uint yyy = 0; yyy < BLOCK_WIDTH; ++yyy)
		for (uint xxx = 0; xxx < BLOCK_WIDTH; ++xxx)
		{
			//O.Set(by*LOAD_WIDTH + bby*BLOCK_WIDTH + yyy, y, x, bx*LOAD_WIDTH + bbx*BLOCK_WIDTH + xxx, v[yyy][xxx], Odata);
			uint saveNYX = by*LOAD_WIDTH + bby*BLOCK_WIDTH + yyy;
			//uint saveNYX = by*LOAD_WIDTH + ((id>>3)<<3) + yyy;
			uint saveX = saveNYX % width;
			uint saveNY = saveNYX / width;
			uint saveY = saveNY % height;
			uint saveN = saveNY / height;

			uint saveK = bx*LOAD_WIDTH + bbx*BLOCK_WIDTH + xxx;
			O.Set(saveN, saveY, saveX, saveK, v[yyy][xxx], Odata);
		}

	#undef X_
	#undef K_
}


#undef THREAD_COUNT
#define THREAD_COUNT 64 // ATM support only 8x8

#undef BLOCK_WIDTH
#define BLOCK_WIDTH 8

#undef LOAD_WIDTH
#define LOAD_WIDTH THREAD_COUNT

#undef LOAD_DEPTH
#define LOAD_DEPTH BLOCK_WIDTH

#if 1

groupshared float DenseTiled_XcacheR[32][LOAD_WIDTH];
groupshared float DenseTiled_WcacheR[LOAD_DEPTH][LOAD_WIDTH];

[numthreads(THREAD_COUNT, 1, 1)]
void Dense64(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ DenseTiled_XcacheR
	#define W_ DenseTiled_WcacheR

	uint id = groupThreadID.x;
	uint bx = groupID.x;
	uint by = groupID.y;

	uint bbx = id % BLOCK_WIDTH;
	uint bby = id / BLOCK_WIDTH;

	float v[BLOCK_WIDTH][BLOCK_WIDTH];
	for (uint yy = 0; yy < BLOCK_WIDTH; ++yy)
		for (uint xx = 0; xx < BLOCK_WIDTH; ++xx)
		{
			float bias = B.Get(0, 0, bx*LOAD_WIDTH + bbx*BLOCK_WIDTH + xx, 0, WBKdata, WBK.dataLength);
			v[yy][xx] = bias;
		}

	for (uint m = 0; m < X.width/LOAD_DEPTH; ++m)
	{
		for (uint q = 0; q < LOAD_DEPTH; ++q)
		{
			X_[q][id] = X.Get(by*LOAD_WIDTH + id, 0, m*LOAD_DEPTH + q, 0, Xdata);
			W_[q][id] = W.Get(0, m*LOAD_DEPTH + q, bx*LOAD_WIDTH + id, 0, WBKdata, WBK.dataLength);
		}

		GroupMemoryBarrierWithGroupSync();

		for (uint yyy = 0; yyy < BLOCK_WIDTH; ++yyy)
		{
			X_[yyy][id] = X.Get(by*LOAD_WIDTH + id, 0, m*LOAD_DEPTH + yyy, 0, Xdata);
			[unroll] for (uint xxx = 0; xxx < BLOCK_WIDTH; ++xxx)
				[unroll] for (uint i = 0; i < LOAD_DEPTH; ++i)
				{
					v[yyy][xxx] += X_[i][bby*BLOCK_WIDTH + yyy] * W_[i][bbx*BLOCK_WIDTH + xxx];
				}
		}

		GroupMemoryBarrierWithGroupSync();
	}

	for (uint yyy = 0; yyy < BLOCK_WIDTH; ++yyy)
		for (uint xxx = 0; xxx < BLOCK_WIDTH; ++xxx)
			O.Set(by*LOAD_WIDTH + bby*BLOCK_WIDTH + yyy, 0, bx*LOAD_WIDTH + bbx*BLOCK_WIDTH + xxx, 0, v[yyy][xxx], Odata);

	#undef X_
	#undef W_
}

#elif 1
groupshared float DenseTiled_XcacheR[LOAD_DEPTH][LOAD_WIDTH];
groupshared float DenseTiled_WcacheR[LOAD_DEPTH][LOAD_WIDTH];

[numthreads(THREAD_COUNT, 1, 1)]
void Dense64(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ DenseTiled_XcacheR
	#define W_ DenseTiled_WcacheR

	uint id = groupThreadID.x;
	uint bx = groupID.x;
	uint by = groupID.y;

	uint bbx = id % BLOCK_WIDTH;
	uint bby = id / BLOCK_WIDTH;

	float v[BLOCK_WIDTH][BLOCK_WIDTH];
	for (uint yy = 0; yy < BLOCK_WIDTH; ++yy)
		for (uint xx = 0; xx < BLOCK_WIDTH; ++xx)
		{
			float bias = B.Get(0, 0, bx*LOAD_WIDTH + bbx*BLOCK_WIDTH + xx, 0, WBKdata, WBK.dataLength);
			v[yy][xx] = bias;
		}

	for (uint m = 0; m < X.width/LOAD_DEPTH; ++m)
	{
		for (uint q = 0; q < LOAD_DEPTH; ++q)
		{
			X_[q][id] = X.Get(by*LOAD_WIDTH + id, 0, m*LOAD_DEPTH + q, 0, Xdata);
			W_[q][id] = W.Get(0, m*LOAD_DEPTH + q, bx*LOAD_WIDTH + id, 0, WBKdata, WBK.dataLength);
		}

		GroupMemoryBarrierWithGroupSync();

		for (uint yyy = 0; yyy < BLOCK_WIDTH; ++yyy)
			[unroll] for (uint xxx = 0; xxx < BLOCK_WIDTH; ++xxx)
				[unroll] for (uint i = 0; i < LOAD_DEPTH; ++i)
				{
					//v[yyy][xxx] += X_[i][bby*BLOCK_WIDTH + yyy] * W_[i][bbx*BLOCK_WIDTH + xxx];
					v[yyy][xxx] = mad(X_[i][bby*BLOCK_WIDTH + yyy], W_[i][bbx*BLOCK_WIDTH + xxx], v[yyy][xxx]);
				}

		GroupMemoryBarrierWithGroupSync();
	}

	for (uint yyy = 0; yyy < BLOCK_WIDTH; ++yyy)
		for (uint xxx = 0; xxx < BLOCK_WIDTH; ++xxx)
			O.Set(by*LOAD_WIDTH + bby*BLOCK_WIDTH + yyy, 0, bx*LOAD_WIDTH + bbx*BLOCK_WIDTH + xxx, 0, v[yyy][xxx], Odata);

	#undef X_
	#undef W_
}

#elif 1

// unroll array to help some "naive" compilers to map to regs
// could be easier to lay out zigzagging patterns 
groupshared float DenseTiled_XcacheR[LOAD_DEPTH][LOAD_WIDTH];
groupshared float DenseTiled_WcacheR[LOAD_DEPTH][LOAD_WIDTH];

[numthreads(THREAD_COUNT, 1, 1)]
void Dense64(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ DenseTiled_XcacheR
	#define W_ DenseTiled_WcacheR

	uint id = groupThreadID.x;
	uint bx = groupID.x;
	uint by = groupID.y;

	uint bbx = id % BLOCK_WIDTH;
	uint bby = id / BLOCK_WIDTH;

	//float v[BLOCK_WIDTH][BLOCK_WIDTH];
	float
		v00, v01, v02, v03, v04, v05, v06, v07,
		v10, v11, v12, v13, v14, v15, v16, v17,
		v20, v21, v22, v23, v24, v25, v26, v27,
		v30, v31, v32, v33, v34, v35, v36, v37,
		v40, v41, v42, v43, v44, v45, v46, v47,
		v50, v51, v52, v53, v54, v55, v56, v57,
		v60, v61, v62, v63, v64, v65, v66, v67,
		v70, v71, v72, v73, v74, v75, v76, v77;

	float b0 = B.Get(0, 0, bx*LOAD_WIDTH + bbx*BLOCK_WIDTH + 0, 0, WBKdata, WBK.dataLength);
	float b1 = B.Get(0, 0, bx*LOAD_WIDTH + bbx*BLOCK_WIDTH + 1, 0, WBKdata, WBK.dataLength);
	float b2 = B.Get(0, 0, bx*LOAD_WIDTH + bbx*BLOCK_WIDTH + 2, 0, WBKdata, WBK.dataLength);
	float b3 = B.Get(0, 0, bx*LOAD_WIDTH + bbx*BLOCK_WIDTH + 3, 0, WBKdata, WBK.dataLength);
	float b4 = B.Get(0, 0, bx*LOAD_WIDTH + bbx*BLOCK_WIDTH + 4, 0, WBKdata, WBK.dataLength);
	float b5 = B.Get(0, 0, bx*LOAD_WIDTH + bbx*BLOCK_WIDTH + 5, 0, WBKdata, WBK.dataLength);
	float b6 = B.Get(0, 0, bx*LOAD_WIDTH + bbx*BLOCK_WIDTH + 6, 0, WBKdata, WBK.dataLength);
	float b7 = B.Get(0, 0, bx*LOAD_WIDTH + bbx*BLOCK_WIDTH + 7, 0, WBKdata, WBK.dataLength);

	#define L_(y, x) v##y##x = b##x
	L_(0,0); L_(0,1); L_(0,2); L_(0,3); L_(0,4); L_(0,5); L_(0,6); L_(0,7);
	L_(1,0); L_(1,1); L_(1,2); L_(1,3); L_(1,4); L_(1,5); L_(1,6); L_(1,7);
	L_(2,0); L_(2,1); L_(2,2); L_(2,3); L_(2,4); L_(2,5); L_(2,6); L_(2,7);
	L_(3,0); L_(3,1); L_(3,2); L_(3,3); L_(3,4); L_(3,5); L_(3,6); L_(3,7);
	L_(4,0); L_(4,1); L_(4,2); L_(4,3); L_(4,4); L_(4,5); L_(4,6); L_(4,7);
	L_(5,0); L_(5,1); L_(5,2); L_(5,3); L_(5,4); L_(5,5); L_(5,6); L_(5,7);
	L_(6,0); L_(6,1); L_(6,2); L_(6,3); L_(6,4); L_(6,5); L_(6,6); L_(6,7);
	L_(7,0); L_(7,1); L_(7,2); L_(7,3); L_(7,4); L_(7,5); L_(7,6); L_(7,7);	
	#undef L_

	for (uint m = 0; m < X.width/LOAD_DEPTH; ++m)
	{
		for (uint q = 0; q < LOAD_DEPTH; ++q)
		{
			X_[q][id] = X.Get(by*LOAD_WIDTH + id, 0, m*LOAD_DEPTH + q, 0, Xdata);
			W_[q][id] = W.Get(0, m*LOAD_DEPTH + q, bx*LOAD_WIDTH + id, 0, WBKdata, WBK.dataLength);
		}

		GroupMemoryBarrierWithGroupSync();

		[unroll] for (uint i = 0; i < LOAD_DEPTH; ++i)
		{
			//v[yyy][xxx] += X_[i][bby*BLOCK_WIDTH + yyy] * W_[i][bbx*BLOCK_WIDTH + xxx];
			#define XW_(y, x) v##y##x += X_[i][bby*BLOCK_WIDTH + ##y] * W_[i][bbx*BLOCK_WIDTH + ##x]
			XW_(0,0); XW_(0,1); XW_(0,2); XW_(0,3); XW_(0,4); XW_(0,5); XW_(0,6); XW_(0,7);
			XW_(1,0); XW_(1,1); XW_(1,2); XW_(1,3); XW_(1,4); XW_(1,5); XW_(1,6); XW_(1,7);
			XW_(2,0); XW_(2,1); XW_(2,2); XW_(2,3); XW_(2,4); XW_(2,5); XW_(2,6); XW_(2,7);
			XW_(3,0); XW_(3,1); XW_(3,2); XW_(3,3); XW_(3,4); XW_(3,5); XW_(3,6); XW_(3,7);
			XW_(4,0); XW_(4,1); XW_(4,2); XW_(4,3); XW_(4,4); XW_(4,5); XW_(4,6); XW_(4,7);
			XW_(5,0); XW_(5,1); XW_(5,2); XW_(5,3); XW_(5,4); XW_(5,5); XW_(5,6); XW_(5,7);
			XW_(6,0); XW_(6,1); XW_(6,2); XW_(6,3); XW_(6,4); XW_(6,5); XW_(6,6); XW_(6,7);
			XW_(7,0); XW_(7,1); XW_(7,2); XW_(7,3); XW_(7,4); XW_(7,5); XW_(7,6); XW_(7,7);		
			#undef XW_
		}

		GroupMemoryBarrierWithGroupSync();
	}

	#define S_(a, b) O.Set(by*LOAD_WIDTH + bby*BLOCK_WIDTH + ##a, 0, bx*LOAD_WIDTH + bbx*BLOCK_WIDTH + ##b, 0, v##a##b, Odata)
	S_(0,0); S_(0,1); S_(0,2); S_(0,3); S_(0,4); S_(0,5); S_(0,6); S_(0,7);
	S_(1,0); S_(1,1); S_(1,2); S_(1,3); S_(1,4); S_(1,5); S_(1,6); S_(1,7);
	S_(2,0); S_(2,1); S_(2,2); S_(2,3); S_(2,4); S_(2,5); S_(2,6); S_(2,7);
	S_(3,0); S_(3,1); S_(3,2); S_(3,3); S_(3,4); S_(3,5); S_(3,6); S_(3,7);
	S_(4,0); S_(4,1); S_(4,2); S_(4,3); S_(4,4); S_(4,5); S_(4,6); S_(4,7);
	S_(5,0); S_(5,1); S_(5,2); S_(5,3); S_(5,4); S_(5,5); S_(5,6); S_(5,7);
	S_(6,0); S_(6,1); S_(6,2); S_(6,3); S_(6,4); S_(6,5); S_(6,6); S_(6,7);
	S_(7,0); S_(7,1); S_(7,2); S_(7,3); S_(7,4); S_(7,5); S_(7,6); S_(7,7);
	#undef S_

	#undef X_
	#undef W_
}

#elif 1

groupshared float DenseTiled_XcacheR[2][LOAD_DEPTH][LOAD_WIDTH];
groupshared float DenseTiled_WcacheR[2][LOAD_DEPTH][LOAD_WIDTH];

[numthreads(THREAD_COUNT, 1, 1)]
void Dense64(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ DenseTiled_XcacheR
	#define W_ DenseTiled_WcacheR

	uint id = groupThreadID.x;
	uint bx = groupID.x;
	uint by = groupID.y;

	uint bbx = id % BLOCK_WIDTH;
	uint bby = id / BLOCK_WIDTH;

	float v[BLOCK_WIDTH][BLOCK_WIDTH];
	for (uint yy = 0; yy < BLOCK_WIDTH; ++yy)
		[unroll] for (uint xx = 0; xx < BLOCK_WIDTH; ++xx)
		{
			float bias = B.Get(0, 0, bx*LOAD_WIDTH + bbx*BLOCK_WIDTH + xx, 0, WBKdata, WBK.dataLength);
			v[yy][xx] = bias;
		}

	uint m = 0;
	for (uint q = 0; q < LOAD_DEPTH; ++q)
	{
		X_[0][q][id] = X.Get(by*LOAD_WIDTH + id, 0, m*LOAD_DEPTH + q, 0, Xdata);
		W_[0][q][id] = W.Get(0, m*LOAD_DEPTH + q, bx*LOAD_WIDTH + id, 0, WBKdata, WBK.dataLength);
	}
	GroupMemoryBarrierWithGroupSync();

	++m;

	for (; m < X.width/LOAD_DEPTH; ++m)
	{
		for (uint q = 0; q < LOAD_DEPTH; ++q)
		{
			X_[1][q][id] = X.Get(by*LOAD_WIDTH + id, 0, m*LOAD_DEPTH + q, 0, Xdata);
			W_[1][q][id] = W.Get(0, m*LOAD_DEPTH + q, bx*LOAD_WIDTH + id, 0, WBKdata, WBK.dataLength);
		}
		
		for (uint yyy = 0; yyy < BLOCK_WIDTH; ++yyy)
			[unroll] for (uint xxx = 0; xxx < BLOCK_WIDTH; ++xxx)
				[unroll]
				for (uint i = 0; i < LOAD_DEPTH; ++i)
				{
					v[yyy][xxx] += X_[0][i][bby*BLOCK_WIDTH + yyy] * W_[0][i][bbx*BLOCK_WIDTH + xxx];
			}

		++m;
		GroupMemoryBarrierWithGroupSync();

		if (m < X.width/LOAD_DEPTH)
		{
			for (uint q = 0; q < LOAD_DEPTH; ++q)
			{
				X_[0][q][id] = X.Get(by*LOAD_WIDTH + id, 0, m*LOAD_DEPTH + q, 0, Xdata);
				W_[0][q][id] = W.Get(0, m*LOAD_DEPTH + q, bx*LOAD_WIDTH + id, 0, WBKdata, WBK.dataLength);
			}
		}

		for (uint yyy = 0; yyy < BLOCK_WIDTH; ++yyy)
			[unroll] for (uint xxx = 0; xxx < BLOCK_WIDTH; ++xxx)
				[unroll]
				for (uint i = 0; i < LOAD_DEPTH; ++i)
				{
					v[yyy][xxx] += X_[1][i][bby*BLOCK_WIDTH + yyy] * W_[1][i][bbx*BLOCK_WIDTH + xxx];
			}
		GroupMemoryBarrierWithGroupSync();
	}

	for (uint yyy = 0; yyy < BLOCK_WIDTH; ++yyy)
		[unroll] for (uint xxx = 0; xxx < BLOCK_WIDTH; ++xxx)
			O.Set(by*LOAD_WIDTH + bby*BLOCK_WIDTH + yyy, 0, bx*LOAD_WIDTH + bbx*BLOCK_WIDTH + xxx, 0, v[yyy][xxx], Odata);

	#undef X_
	#undef W_
}

#else

groupshared float DenseTiled_XcacheR[LOAD_DEPTH][LOAD_WIDTH];
groupshared float DenseTiled_WcacheR[LOAD_DEPTH][LOAD_WIDTH];

[numthreads(THREAD_COUNT, 1, 1)]
void Dense64(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ DenseTiled_XcacheR
	#define W_ DenseTiled_WcacheR

	uint id = groupThreadID.x;
	uint bx = groupID.x;
	uint by = groupID.y;

	uint n = by * LOAD_WIDTH + id;
	uint x = bx * LOAD_WIDTH + id;

	float v[LOAD_WIDTH];
	float bias = B.Get(0, 0, x, 0, WBKdata, WBK.dataLength);
	[unroll] for (uint xx = 0; xx < LOAD_WIDTH; ++xx)
		v[xx] = bias;

	for (uint m = 0; m < X.width/LOAD_DEPTH; ++m)
	{
		float ww[LOAD_DEPTH];
		for (uint q = 0; q < LOAD_DEPTH; ++q)
		{
			X_[q][id] = X.Get(n, 0, m*LOAD_DEPTH + q, 0, Xdata);
			//W_[q][id] = W.Get(0, m*LOAD_DEPTH + q, x, 0, WBKdata, WBK.dataLength);
			ww[q] = W.Get(0, m*LOAD_DEPTH + q, x, 0, WBKdata, WBK.dataLength);
		}

		GroupMemoryBarrierWithGroupSync();

		for (uint w = 0; w < LOAD_WIDTH; ++w)
		{
			[unroll]
			for (uint i = 0; i < LOAD_DEPTH; ++i)
			{
				//v[w] += X_[i][w] * W_[i][id];
				v[w] += X_[i][w] * ww[i];
			}
		}

		GroupMemoryBarrierWithGroupSync();
	}

	[unroll] for ( xx = 0; xx < LOAD_WIDTH; ++xx)
		O.Set(by * LOAD_WIDTH + xx, 0, x, 0, v[xx], Odata);

	#undef X_
	#undef W_
}
#endif

#if 1
#undef TILE_WIDTH
#define TILE_WIDTH 16
groupshared float DenseTiled_Xcache64[16][TILE_WIDTH*TILE_WIDTH];
groupshared float DenseTiled_Wcache64[16][TILE_WIDTH*TILE_WIDTH];
[numthreads(TILE_WIDTH,TILE_WIDTH,1)]
void DenseTiled64x64(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ DenseTiled_Xcache64
	#define W_ DenseTiled_Wcache64

	uint tx = groupThreadID.x;
	uint ty = groupThreadID.y;
	uint x = groupID.x*TILE_WIDTH + tx;
	uint n = groupID.y*TILE_WIDTH + ty;

	float b0 = B.Get(0, 0, x*4+0, 0, WBKdata, WBK.dataLength);
	float b1 = B.Get(0, 0, x*4+1, 0, WBKdata, WBK.dataLength);
	float b2 = B.Get(0, 0, x*4+2, 0, WBKdata, WBK.dataLength);
	float b3 = B.Get(0, 0, x*4+3, 0, WBKdata, WBK.dataLength);
	
	float4 v0, v1, v2, v3;
	v0 = v1 = v2 = v3 = float4(b0, b1, b2, b3);

	for (uint m = 0; m < X.width/(TILE_WIDTH*4); ++m) 
	{
		for (uint yy = 0; yy < 4; ++yy)
			for (uint xx = 0; xx < 4; ++xx)
			{
				X_[yy*4+xx][ty*TILE_WIDTH+tx] = X.Get(n*4+yy, 0, (m*TILE_WIDTH + tx)*4+xx, 0, Xdata);
				W_[yy*4+xx][ty*TILE_WIDTH+tx] = W.Get(0, (m*TILE_WIDTH + ty)*4+yy, x*4+xx, 0, WBKdata, WBK.dataLength);
			}
		
		GroupMemoryBarrierWithGroupSync();

		//[unroll]
		for (uint i = 0; i < TILE_WIDTH; ++i)
		{
			[unroll]
			for (uint q = 0; q < 4; ++q)
			{
				float x0 = X_[0*4+q][ty*TILE_WIDTH+i];
				float x1 = X_[1*4+q][ty*TILE_WIDTH+i];
				float x2 = X_[2*4+q][ty*TILE_WIDTH+i];
				float x3 = X_[3*4+q][ty*TILE_WIDTH+i];
				
				float w0 = W_[q*4+0][i*TILE_WIDTH+tx];
				float w1 = W_[q*4+1][i*TILE_WIDTH+tx];
				float w2 = W_[q*4+2][i*TILE_WIDTH+tx];
				float w3 = W_[q*4+3][i*TILE_WIDTH+tx];

				v0.x = mad(x0, w0, v0.x); //--
				v1.x = mad(x1, w0, v1.x); 
				v2.x = mad(x2, w0, v2.x); 
				v3.x = mad(x3, w0, v3.x); 
				v0.y = mad(x0, w1, v0.y); //--
				v1.y = mad(x1, w1, v1.y);
				v2.y = mad(x2, w1, v2.y);
				v3.y = mad(x3, w1, v3.y);
				v0.z = mad(x0, w2, v0.z); //--
				v1.z = mad(x1, w2, v1.z); 
				v2.z = mad(x2, w2, v2.z); 
				v3.z = mad(x3, w2, v3.z); 
				v0.w = mad(x0, w3, v0.w); //--
				v1.w = mad(x1, w3, v1.w);
				v2.w = mad(x2, w3, v2.w);
				v3.w = mad(x3, w3, v3.w);
			}

			GroupMemoryBarrierWithGroupSync();
		}
	}

	O.Set(n*4+0, 0, x*4+0, 0, v0.x, Odata);
	O.Set(n*4+0, 0, x*4+1, 0, v0.y, Odata);
	O.Set(n*4+0, 0, x*4+2, 0, v0.z, Odata);
	O.Set(n*4+0, 0, x*4+3, 0, v0.w, Odata);
	
	O.Set(n*4+1, 0, x*4+0, 0, v1.x, Odata);
	O.Set(n*4+1, 0, x*4+1, 0, v1.y, Odata);
	O.Set(n*4+1, 0, x*4+2, 0, v1.z, Odata);
	O.Set(n*4+1, 0, x*4+3, 0, v1.w, Odata);
	 
	O.Set(n*4+2, 0, x*4+0, 0, v2.x, Odata);
	O.Set(n*4+2, 0, x*4+1, 0, v2.y, Odata);
	O.Set(n*4+2, 0, x*4+2, 0, v2.z, Odata);
	O.Set(n*4+2, 0, x*4+3, 0, v2.w, Odata);
	 
	O.Set(n*4+3, 0, x*4+0, 0, v3.x, Odata);
	O.Set(n*4+3, 0, x*4+1, 0, v3.y, Odata);
	O.Set(n*4+3, 0, x*4+2, 0, v3.z, Odata);
	O.Set(n*4+3, 0, x*4+3, 0, v3.w, Odata);
			  
	#undef X_
	#undef W_
}

#else

#define TILE_WIDTH 16
#define RTILE 4
groupshared float DenseTiled_Xcache64[RTILE*RTILE][TILE_WIDTH*TILE_WIDTH];
groupshared float DenseTiled_Wcache64[RTILE*RTILE][TILE_WIDTH*TILE_WIDTH];
[numthreads(TILE_WIDTH,TILE_WIDTH,1)]
void DenseTiled64x64(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ DenseTiled_Xcache64
	#define W_ DenseTiled_Wcache64

	uint tx = groupThreadID.x;
	uint ty = groupThreadID.y;
	uint x = groupID.x*TILE_WIDTH + tx;
	uint n = groupID.y*TILE_WIDTH + ty;

	float v[RTILE*RTILE];
	[unroll] for (uint xxxx = 0; xxxx < RTILE; ++xxxx)
	{
		float b = B.Get(0, 0, x*RTILE+xxxx, 0, WBKdata, WBK.dataLength);
		[unroll] for (uint yyyy = 0; yyyy < RTILE; ++yyyy)
			v[yyyy*RTILE+xxxx] = b;
	}

	for (uint m = 0; m < X.width/(TILE_WIDTH*RTILE); ++m) 
	{
		for (uint yy = 0; yy < RTILE; ++yy)
			[unroll] for (uint xx = 0; xx < RTILE; ++xx)
			{
				X_[yy*RTILE+xx][ty*TILE_WIDTH+tx] = X.Get(n*RTILE+yy, 0, (m*TILE_WIDTH + tx)*RTILE+xx, 0, Xdata);
				W_[yy*RTILE+xx][ty*TILE_WIDTH+tx] = W.Get(0, (m*TILE_WIDTH + ty)*RTILE+yy, x*RTILE+xx, 0, WBKdata, WBK.dataLength);
			}
		GroupMemoryBarrierWithGroupSync();

		for (uint ii = 0; ii < TILE_WIDTH; ++ii)
		{
			[unroll] for (uint yy = 0; yy < RTILE; ++yy)
				[unroll] for (uint xx = 0; xx < RTILE; ++xx)
					[unroll] for (uint i = 0; i < RTILE; ++i)
					{
						float x = X_[yy*RTILE+i][ty*TILE_WIDTH+ii];
						float w = W_[i*RTILE+xx][ii*TILE_WIDTH+tx];
						v[yy*RTILE+xx] = mad(x, w, v[yy*RTILE+xx]);
					}

			GroupMemoryBarrierWithGroupSync();
		}
	}

	[unroll] for (uint yy = 0; yy < RTILE; ++yy)
		[unroll] for (uint xx = 0; xx < RTILE; ++xx)
			O.Set(n*RTILE+yy, 0, x*RTILE+xx, 0, v[yy*RTILE+xx], Odata);
	  
	#undef X_
	#undef W_
}

#endif

#undef TILE_WIDTH
#define TILE_WIDTH 16 // 32 crashes on MacBookPro/AMD
groupshared float DenseTiled_Xcache32[4][TILE_WIDTH][TILE_WIDTH];
groupshared float DenseTiled_Wcache32[4][TILE_WIDTH][TILE_WIDTH];
[numthreads(TILE_WIDTH,TILE_WIDTH,1)]
void DenseTiled32x32(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ DenseTiled_Xcache32
	#define W_ DenseTiled_Wcache32

	uint tx = groupThreadID.x;
	uint ty = groupThreadID.y;
	uint x = groupID.x*TILE_WIDTH + tx;
	uint n = groupID.y*TILE_WIDTH + ty;

	float b0 = B.Get(0, 0, x*2+0, 0, WBKdata, WBK.dataLength);
	float b1 = B.Get(0, 0, x*2+1, 0, WBKdata, WBK.dataLength);
	float4 v = float4(b0, b1,
					  b0, b1);

	for (uint m = 0; m < X.width/(TILE_WIDTH*2);)
	{
		// @TODO: read in float2s
		float x0 = X.Get(n*2+0, 0, m*TILE_WIDTH*2 + tx*2+0, 0, Xdata);
		float x1 = X.Get(n*2+0, 0, m*TILE_WIDTH*2 + tx*2+1, 0, Xdata);
		float x2 = X.Get(n*2+1, 0, m*TILE_WIDTH*2 + tx*2+0, 0, Xdata);
		float x3 = X.Get(n*2+1, 0, m*TILE_WIDTH*2 + tx*2+1, 0, Xdata);

		float w0 = W.Get(0, m*TILE_WIDTH*2 + ty*2+0, x*2+0, 0, WBKdata, WBK.dataLength);		
		float w1 = W.Get(0, m*TILE_WIDTH*2 + ty*2+0, x*2+1, 0, WBKdata, WBK.dataLength);
		float w2 = W.Get(0, m*TILE_WIDTH*2 + ty*2+1, x*2+0, 0, WBKdata, WBK.dataLength);
		float w3 = W.Get(0, m*TILE_WIDTH*2 + ty*2+1, x*2+1, 0, WBKdata, WBK.dataLength);

		++m;

		X_[0][ty][tx] = x0;
		X_[1][ty][tx] = x1;
		X_[2][ty][tx] = x2;
		X_[3][ty][tx] = x3;

		W_[0][ty][tx] = w0;
		W_[1][ty][tx] = w1;
		W_[2][ty][tx] = w2;
		W_[3][ty][tx] = w3;

		GroupMemoryBarrierWithGroupSync();

		[unroll]
		for (uint i = 0; i < TILE_WIDTH; ++i)
		{
			float4 x = //X_[ty][i];
				float4(	X_[0][ty][i],
						X_[1][ty][i],
						X_[2][ty][i],
						X_[3][ty][i]);
			float4 w = //W_[i][tx];
				float4(	W_[0][i][tx],
						W_[1][i][tx],
						W_[2][i][tx],
						W_[3][i][tx]);
					
			v.x = mad(w.x, x.x, v.x);
			v.y = mad(w.y, x.x, v.y);
			v.z = mad(w.x, x.z, v.z);
			v.w = mad(w.y, x.z, v.w);

			v.x = mad(w.z, x.y, v.x);					  
			v.y = mad(w.w, x.y, v.y);
			v.z = mad(w.z, x.w, v.z);
			v.w = mad(w.w, x.w, v.w);

			//v.x += k.x*x.x + k.z*x.y;
			//v.y += k.y*x.x + k.w*x.y;
			//v.z += k.x*x.z + k.z*x.w;
			//v.w += k.y*x.z + k.w*x.w;
		}
		
		GroupMemoryBarrierWithGroupSync();
	}
	
	O.Set(n*2+0, 0, x*2+0, 0, v.x, Odata);
	O.Set(n*2+0, 0, x*2+1, 0, v.y, Odata);
	O.Set(n*2+1, 0, x*2+0, 0, v.z, Odata);
	O.Set(n*2+1, 0, x*2+1, 0, v.w, Odata);

	#undef X_
	#undef W_
}

// sligtly faster on AMD (56ms vs 62ms)
#undef TILE_WIDTH
#define TILE_WIDTH 16
//#define CACHE_ONLY_X
//#define TRANSPOSE_W
//#define TRANSPOSE_X
groupshared float DenseTiled_XcacheF[TILE_WIDTH][TILE_WIDTH];
#if !defined(CACHE_ONLY_X)
groupshared float DenseTiled_WcacheF[TILE_WIDTH][TILE_WIDTH];
#endif
[numthreads(TILE_WIDTH,TILE_WIDTH,1)]
void DenseTiled16x16_amd(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ DenseTiled_XcacheF
	#define W_ DenseTiled_WcacheF

	uint tx = groupThreadID.x;
	uint ty = groupThreadID.y;
	uint x = groupID.x*TILE_WIDTH + tx;
	uint b = groupID.y*TILE_WIDTH + ty;

	float v = B.Get(0, 0, x, 0, WBKdata, WBK.dataLength);

	for (uint m = 0; m < X.width/TILE_WIDTH; ++m)
	{
		#if defined(TRANSPOSE_X)
		X_[tx][ty] = X.Get(b, 0, m*TILE_WIDTH + tx, 0, Xdata);
		#else
		X_[ty][tx] = X.Get(b, 0, m*TILE_WIDTH + tx, 0, Xdata);
		#endif

		#if defined(CACHE_ONLY_X)
		float ww = WBKdata[wi];
		#else
		#if defined(TRANSPOSE_W)
		W_[tx][ty] = W.Get(0, m*TILE_WIDTH + ty, x, 0, WBKdata, WBK.dataLength);
		#else
		W_[ty][tx] = W.Get(0, m*TILE_WIDTH + ty, x, 0, WBKdata, WBK.dataLength);
		#endif
		#endif
		GroupMemoryBarrierWithGroupSync();

		//[unroll(groupthreads)]
		[unroll]
		for (uint i = 0; i < TILE_WIDTH; ++i)
		{
			#if defined(TRANSPOSE_X)
			float x = X_[i][ty];
			#else
			float x = X_[ty][i];
			#endif

			#if defined(CACHE_ONLY_X)
			//float w = ww;
			//if (i != TILE_WIDTH-1) { wi += W.width; ww = WBKdata[wi]; }
			float w = W.Get(0, m*TILE_WIDTH + i, x, 0, WBKdata, WBK.dataLength);
			#else
			#if defined(TRANSPOSE_W)
			float w = W_[tx][i];
			#else
			float w = W_[i][tx];
			#endif
			#endif

			v += x * w;
		}
	}
	
	O.Set(b, 0, x, 0, v, Odata);

	#undef X_
	#undef W_
}

#undef TILE_WIDTH
#define TILE_WIDTH 16
groupshared float DenseTiled_Xcache[TILE_WIDTH][TILE_WIDTH];
groupshared float DenseTiled_Wcache[TILE_WIDTH][TILE_WIDTH];
[numthreads(TILE_WIDTH,TILE_WIDTH,1)]
void DenseTiled(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ DenseTiled_Xcache
	#define W_ DenseTiled_Wcache

	uint tx = groupThreadID.x;
	uint ty = groupThreadID.y;
	uint x = groupID.x*TILE_WIDTH + tx;
	uint b = groupID.y*TILE_WIDTH + ty;

	bool mask = (x < O.width && b < O.batch);

	float v = B.Get(0, 0, x, 0, WBKdata, WBK.dataLength);

	for (uint m = 0; m < X.width/TILE_WIDTH; ++m)
	{
		if (mask)
		{
			X_[ty][tx] = X.Get(b, 0, m*TILE_WIDTH + tx, 0, Xdata);
			W_[ty][tx] = W.Get(0, m*TILE_WIDTH + ty, x, 0, WBKdata, WBK.dataLength);
		}
		else
		{
			X_[ty][tx] = 0;
			W_[ty][tx] = 0;
		}

		GroupMemoryBarrierWithGroupSync();

		[unroll]
		for (uint i = 0; i < TILE_WIDTH; ++i)
		{
			v += X_[ty][i] * W_[i][tx];
		}

		GroupMemoryBarrierWithGroupSync();
	}
	
	if (mask)
		O.Set(b, 0, x, 0, v, Odata);

	#undef X_
	#undef W_
}


groupshared float DenseTiled_XcacheP[TILE_WIDTH][TILE_WIDTH];
groupshared float DenseTiled_WcacheP[TILE_WIDTH][TILE_WIDTH];
// Prefetch - seems to be the same performance as DenseTiled16x16 without prefetch, has higher register pressure
[numthreads(TILE_WIDTH,TILE_WIDTH,1)]
void DenseTiledPrefetch16x16(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ DenseTiled_XcacheP
	#define W_ DenseTiled_WcacheP

	uint tx = groupThreadID.x;
	uint ty = groupThreadID.y;
	uint x = groupID.x*TILE_WIDTH + tx;
	uint b = groupID.y*TILE_WIDTH + ty;

	float v = B.Get(0, 0, x, 0, WBKdata, WBK.dataLength);

	float Xregs[TILE_WIDTH][TILE_WIDTH];
	float Wregs[TILE_WIDTH][TILE_WIDTH];
	for (uint m = 0; m < X.width/TILE_WIDTH; ++m)
	{
		Xregs[ty][tx] = X.Get(b, 0, m*TILE_WIDTH + tx, 0, Xdata);
		Wregs[ty][tx] = W.Get(0, m*TILE_WIDTH + ty, x, 0, WBKdata, WBK.dataLength);
		GroupMemoryBarrierWithGroupSync();
	}

	for (m = 0; m < X.width/TILE_WIDTH; ++m)
	{
		X_[ty][tx] = Xregs[ty][tx];
		W_[ty][tx] = Wregs[ty][tx];

		Xregs[ty][tx] = X.Get(b, 0, m*TILE_WIDTH + tx, 0, Xdata);
		Wregs[ty][tx] = W.Get(0, m*TILE_WIDTH + ty, x, 0, WBKdata, WBK.dataLength);

		for (uint i = 0; i < TILE_WIDTH;)
		{
			// can unroll up to 16 because TILE_WIDTH=16
			v += X_[ty][i] * W_[i][tx]; ++i;
			v += X_[ty][i] * W_[i][tx]; ++i;
			v += X_[ty][i] * W_[i][tx]; ++i;
			v += X_[ty][i] * W_[i][tx]; ++i;

			v += X_[ty][i] * W_[i][tx]; ++i;
			v += X_[ty][i] * W_[i][tx]; ++i;
			v += X_[ty][i] * W_[i][tx]; ++i;
			v += X_[ty][i] * W_[i][tx]; ++i;
		}

		GroupMemoryBarrierWithGroupSync();
	}

	O.Set(b, 0, x, 0, v, Odata);
	#undef X_
	#undef W_
}

[numthreads(1,1,1)]
void Relu(uint3 groupID : SV_GroupID)
{
	uint x = groupID.x;
	uint b = groupID.y;
	uint c = groupID.z;
	for (uint y = 0; y < X.height; ++y)
	{
		float v = X.Get(b, y, x, c, Xdata, X.dataLength);
		v = 0.5f * (v + abs(v));
		O.Set(b, y, x, c, v, Odata, O.dataLength);
	}
}

[numthreads(16,16,1)]
void Relu_Cmod16_CNyx(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint c = 16*groupID.x + groupThreadID.x;
	uint nyx = 16*groupID.y + groupThreadID.y;

	uint width = X.width;
	uint height = X.height;

	uint x = nyx % width;
	uint ny = nyx / width;
	uint y = ny % height;
	uint n = ny / height;

	float v = X.Get(n, y, x, c, Xdata, X.dataLength);
	v = 0.5f * (v + abs(v));
	O.Set(n, y, x, c, v, Odata, O.dataLength);
}

[numthreads(512,1,1)]
void Relu_Nyxc(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint nyxc = 512*groupID.x + groupThreadID.x;

	uint width = X.width;
	uint height = X.height;
	uint channels = X.channels;

	uint c = nyxc % channels;
	uint nyx = nyxc / channels;
	uint x = nyx % width;
	uint ny = nyx / width;
	uint y = ny % height;
	uint n = ny / height;

	float v = X.Get(n, y, x, c, Xdata, X.dataLength);
	v = 0.5f * (v + abs(v));
	O.Set(n, y, x, c, v, Odata, O.dataLength);
}

[numthreads(16,16,1)]
void Relu16x16(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint x = 16*groupID.x + groupThreadID.x;
	uint b = 16*groupID.y + groupThreadID.y;
	uint c = groupID.z;

	for (uint y = 0; y < X.height; ++y)
	{
		float v = X.Get(b, y, x, c, Xdata, X.dataLength);
		v = 0.5f * (v + abs(v));
		O.Set(b, y, x, c, v, Odata, O.dataLength);
	}
}

[numthreads(16,16,1)]
void Relu16x16_(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint x = 16*groupID.x + groupThreadID.x;
	uint b = 16*groupID.y + groupThreadID.y;

	for (uint y = 0; y < X.height; ++y)
	{
		for (uint c = 0; c < X.channels; ++c)
		{
			float v = X.Get(b, y, x, c, Xdata, X.dataLength);
			v = 0.5f * (v + abs(v));
			O.Set(b, y, x, c, v, Odata, O.dataLength);
		}
	}
}


// channels, width, batch
[numthreads(16,2,16)]
void ReluChannelsFirst16x2x16(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint c = 16*groupID.x + groupThreadID.x;
	uint x = 2*groupID.y + groupThreadID.y;
	uint b = 16*groupID.z + groupThreadID.z;

	for (uint y = 0; y < X.height; ++y)
	{
		float v = X.Get(b, y, x, c, Xdata, X.dataLength);
		v = 0.5f * (v + abs(v));
		O.Set(b, y, x, c, v, Odata, O.dataLength);
	}
}

[numthreads(256,1,1)]
void Relu256xV(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint x = 256*groupID.x + groupThreadID.x;
	uint b = groupID.y;
	uint c = groupID.z;

	for (uint y = 0; y < X.height; ++y)
	{
		float v = 0;
		for (uint b = 0; b < X.batch; )
		{
			v = X.Get(b, y, x, c, Xdata, X.dataLength);
			v = 0.5f * (v + abs(v));
			O.Set(b, y, x, c, v, Odata, O.dataLength);
			++b;

			v = X.Get(b, y, x, c, Xdata, X.dataLength);
			v = 0.5f * (v + abs(v));
			O.Set(b, y, x, c, v, Odata, O.dataLength);
			++b;

			v = X.Get(b, y, x, c, Xdata, X.dataLength);
			v = 0.5f * (v + abs(v));
			O.Set(b, y, x, c, v, Odata, O.dataLength);
			++b;

			v = X.Get(b, y, x, c, Xdata, X.dataLength);
			v = 0.5f * (v + abs(v));
			O.Set(b, y, x, c, v, Odata, O.dataLength);
			++b;
		}
	}
}


#define FLT_MAX 3.402823466e+38F

[numthreads(1,1,1)]
void Softmax(uint3 groupID : SV_GroupID)
{
	uint b = groupID.x;
	uint x = groupID.y;

	float maxV = -FLT_MAX;
	for (uint i = 0; i < X.width; ++i)
	{
		float v = X.Get(b, 0, i, 0, Xdata, X.dataLength);
		if (v > maxV)
			maxV = v;
	}

	float sum = 0.0f;
	for (i = 0; i < X.width; ++i)
	{
		float v = X.Get(b, 0, i, 0, Xdata, X.dataLength);
		sum += exp(v - maxV);
	}

	float v = X.Get(b, 0, x, 0, Xdata, X.dataLength);
	v = exp(v - maxV) / sum;
	O.Set(b, 0, x, 0, v, Odata, O.dataLength);
}

[numthreads(256,2,1)]
void Softmax256x2(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint b = 256*groupID.x + groupThreadID.x;
	uint x = 2*groupID.y + groupThreadID.y;

	float maxV = -FLT_MAX;
	for (uint i = 0; i < X.width; ++i)
	{
		float v = X.Get(b, 0, i, 0, Xdata, X.dataLength);
		if (v > maxV)
			maxV = v;
	}

	float sum = 0.0f;
	for (i = 0; i < X.width; ++i)
	{
		float v = X.Get(b, 0, i, 0, Xdata, X.dataLength);
		sum += exp(v - maxV);
	}

	float v = X.Get(b, 0, x, 0, Xdata, X.dataLength);
	v = exp(v - maxV) / sum;
	O.Set(b, 0, x, 0, v, Odata, O.dataLength);
}

[numthreads(1,1,1)]
void MaxPooling2D(uint3 groupID : SV_GroupID)
{
	uint c = groupID.x;
	uint x = groupID.y;
	uint y = groupID.z;

	for (uint b = 0; b < O.batch; ++b)
	{
		float v0 = X.Get(b, y*2,   x*2,   c, Xdata, X.dataLength);
		float v1 = X.Get(b, y*2+1, x*2,   c, Xdata, X.dataLength);
		float v2 = X.Get(b, y*2,   x*2+1, c, Xdata, X.dataLength);
		float v3 = X.Get(b, y*2+1, x*2+1, c, Xdata, X.dataLength);
		float v = max(v0, max(v1, max(v2, v3)));
		O.Set(b, y, x, c, v, Odata, O.dataLength);
	}
}

[numthreads(16,4,4)]
void MaxPooling2D16x4x4(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint c = 16*groupID.x + groupThreadID.x;
	uint x = 4*groupID.y + groupThreadID.y;
	uint y = 4*groupID.z + groupThreadID.z;

	for (uint b = 0; b < O.batch; ++b)
	{
		float v0 = X.Get(b, y*2,   x*2,   c, Xdata, X.dataLength);
		float v1 = X.Get(b, y*2+1, x*2,   c, Xdata, X.dataLength);
		float v2 = X.Get(b, y*2,   x*2+1, c, Xdata, X.dataLength);
		float v3 = X.Get(b, y*2+1, x*2+1, c, Xdata, X.dataLength);
		float v = max(v0, max(v1, max(v2, v3)));
		O.Set(b, y, x, c, v, Odata, O.dataLength);
	}
}

[numthreads(16,16,2)]
void Conv2D_Valid(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint k = 16*groupID.x + groupThreadID.x;
	uint n = 16*groupID.y + groupThreadID.y;
	uint y = 2*groupID.z + groupThreadID.z + _FilterSize;

	//for (int y = _FilterSize; y < X.height - _FilterSize; ++y)
	{
		for (uint x = _FilterSize; x < X.width - _FilterSize; ++x)
		{
			float v = B.Get(0, 0, k, 0, WBKdata, WBK.dataLength);
			for (int i = -(int)_FilterSize; i < (int)_FilterSize + 1; ++i)
			{
				for (int j = -(int)_FilterSize; j < (int)_FilterSize + 1; ++j)
				{
					for (uint c = 0; c < X.channels; ++c)
					{
						v += X.Get(n, y+j, x+i, c, Xdata, X.dataLength) * K.Get(_FilterSize+j, _FilterSize+i, c, k, WBKdata, WBK.dataLength);
					}
				}
			}
			O.Set(n, y-_FilterSize, x-_FilterSize, k, v, Odata, O.dataLength);
		}
	}
}

[numthreads(16,8,1)]
void Conv2D_Kmod16_Nmod8_KNY(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint k = 16*groupID.x + groupThreadID.x;
	uint n = 8*groupID.y + groupThreadID.y;
	uint y = 1*groupID.z + groupThreadID.z;

	//for (int y = _FilterSize; y < X.height - _FilterSize; ++y)
	{
		for (uint x = 0; x < X.width - _Border; ++x)
		{
			float v = B.Get(0, 0, k, 0, WBKdata, WBK.dataLength);
			for (uint j = 0; j < 2*_FilterSize+1; ++j)
			{
				if (y+j < _Offset) continue;
				if (y+j-_Offset >= X.height) continue;

				for (uint i = 0; i < 2*_FilterSize+1; ++i)
				{
					if (x+i < _Offset) continue;
					if (x+i-_Offset >= X.width) continue;

					for (uint c = 0; c < X.channels; ++c)
					{
						v += X.Get(n, y+j-_Offset, x+i-_Offset, c, Xdata, X.dataLength) * K.Get(j, i, c, k, WBKdata, WBK.dataLength);
					}
				}
			}
			O.Set(n, y, x, k, v, Odata, O.dataLength);
		}
	}
}

[numthreads(1,1,1)]
void Conv2D(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint k = 1*groupID.x + groupThreadID.x;
	uint n = 1*groupID.y + groupThreadID.y;
	uint y = 1*groupID.z + groupThreadID.z;

	//for (int y = _FilterSize; y < X.height - _FilterSize; ++y)
	{
		for (uint x = 0; x < X.width - _Border; ++x)
		{
			float v = B.Get(0, 0, k, 0, WBKdata, WBK.dataLength);
			for (uint j = 0; j < 2*_FilterSize+1; ++j)
			{
				if (y+j < _Offset) continue;
				if (y+j-_Offset >= X.height) continue;

				for (uint i = 0; i < 2*_FilterSize+1; ++i)
				{
					if (x+i < _Offset) continue;
					if (x+i-_Offset >= X.width) continue;

					for (uint c = 0; c < X.channels; ++c)
					{
						v += X.Get(n, y+j-_Offset, x+i-_Offset, c, Xdata, X.dataLength) * K.Get(j, i, c, k, WBKdata, WBK.dataLength);
					}
				}
			}
			O.Set(n, y, x, k, v, Odata, O.dataLength);
		}
	}
}

#if 0

#define MAX_TILE_WIDTH 16
#define KERNEL_COUNT 4
#define KERNEL_SIZE 3
#define KERNEL_RADIUS 1 //(KERNEL_SIZE-1)/2
groupshared float XCcache[MAX_TILE_WIDTH+KERNEL_SIZE-1][MAX_TILE_WIDTH+KERNEL_SIZE-1];
groupshared float Kcache[KERNEL_SIZE][KERNEL_SIZE][KERNEL_COUNT];

#undef TILE_WIDTH
#define TILE_WIDTH 13
[numthreads(TILE_WIDTH,TILE_WIDTH,KERNEL_COUNT)]
void Conv2DTiled14x14_Kernel3x3(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint tx = groupThreadID.x;
	uint ty = groupThreadID.y;
	uint tk = groupThreadID.z;
	uint gx = groupID.x;
	uint gy = groupID.y;
	uint gk = groupID.z;
	uint tileCornerX = gx*TILE_WIDTH;
	uint tileCornerY = gy*TILE_WIDTH;
	uint x = tileCornerX + tx;
	uint y = tileCornerY + ty;
	uint k = gk*KERNEL_COUNT + tk;
	uint idx = ty*TILE_WIDTH + tx;

	for (uint b = 0; b < X.batch; ++b)
	{
		float v = B.Get(0, 0, k, 0, WBKdata, WBK.dataLength);
		for (uint c = 0; c < X.channels; ++c)
		{
			if (tk == 0)
				XCcache[ty][tx] = X.Get(b, y, x, c, Xdata);
			else if (tk == 1 && idx < TILE_WIDTH * 2)
			{
				uint yy = idx / 2;
				uint xx = idx % 2 + TILE_WIDTH;
				XCcache[yy][xx] = X.Get(b, tileCornerY+yy, tileCornerX+xx, c, Xdata);
			}
			else if (tk == 2 && idx < (TILE_WIDTH + 2) * 2)
			{
				uint yy = idx / (TILE_WIDTH + 2) + TILE_WIDTH;
				uint xx = idx % (TILE_WIDTH + 2);
				XCcache[yy][xx] = X.Get(b, tileCornerY+yy, tileCornerX+xx, c, Xdata);
			}
			if (tk == 3)
			{
				uint kk = idx / (KERNEL_SIZE * KERNEL_SIZE);
				uint kyx = idx % (KERNEL_SIZE * KERNEL_SIZE);
				if (kk < KERNEL_COUNT)
				{
					uint yy = kyx / KERNEL_SIZE;
					uint xx = kyx % KERNEL_SIZE;
					Kcache[yy][xx][kk] = K.Get(yy, xx, c, gk*KERNEL_COUNT+kk, WBKdata, WBK.dataLength);
				}
			}
			GroupMemoryBarrierWithGroupSync();

			for (int i = 0; i < KERNEL_SIZE; ++i)
			{
				for (int j = 0; j < KERNEL_SIZE; ++j)
				{
					v += XCcache[ty+j][tx+i] * Kcache[j][i][tk];
				}
			}
		}
		O.Set(b, y, x, k, v, Odata, O.dataLength);
		GroupMemoryBarrierWithGroupSync();
	}
}

#undef TILE_WIDTH
#define TILE_WIDTH 12
[numthreads(TILE_WIDTH,TILE_WIDTH,KERNEL_COUNT)]
void Conv2DTiled13x13_Kernel3x3(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint tx = groupThreadID.x;
	uint ty = groupThreadID.y;
	uint tk = groupThreadID.z;
	uint gx = groupID.x;
	uint gy = groupID.y;
	uint gk = groupID.z;
	uint tileCornerX = gx*TILE_WIDTH;
	uint tileCornerY = gy*TILE_WIDTH;
	uint x = tileCornerX + tx;
	uint y = tileCornerY + ty;
	uint k = gk*KERNEL_COUNT + tk;
	uint idx = ty*TILE_WIDTH + tx;

	for (uint b = 0; b < X.batch; ++b)
	{
		float v = B.Get(0, 0, k, 0, WBKdata, WBK.dataLength);
		for (uint c = 0; c < X.channels; ++c)
		{
			if (tk == 0)
				XCcache[ty][tx] = X.Get(b, y, x, c, Xdata);
			else if (tk == 1 && idx < TILE_WIDTH * 2)
			{
				uint yy = idx / 2;
				uint xx = idx % 2 + TILE_WIDTH;
				XCcache[yy][xx] = X.Get(b, tileCornerY+yy, tileCornerX+xx, c, Xdata);
			}
			else if (tk == 2 && idx < (TILE_WIDTH + 2) * 2)
			{
				uint yy = idx / (TILE_WIDTH + 2) + TILE_WIDTH;
				uint xx = idx % (TILE_WIDTH + 2);
				XCcache[yy][xx] = X.Get(b, tileCornerY+yy, tileCornerX+xx, c, Xdata);
			}
			if (tk == 3)
			{
				uint kk = idx / (KERNEL_SIZE * KERNEL_SIZE);
				uint kyx = idx % (KERNEL_SIZE * KERNEL_SIZE);
				if (kk < KERNEL_COUNT)
				{
					uint yy = kyx / KERNEL_SIZE;
					uint xx = kyx % KERNEL_SIZE;
					Kcache[yy][xx][kk] = K.Get(yy, xx, c, gk*KERNEL_COUNT+kk, WBKdata, WBK.dataLength);
				}
			}
			GroupMemoryBarrierWithGroupSync();

			for (int i = 0; i < KERNEL_SIZE; ++i)
			{
				for (int j = 0; j < KERNEL_SIZE; ++j)
				{
					v += XCcache[ty+j][tx+i] * Kcache[j][i][tk];
				}
			}
		}
		O.Set(b, y, x, k, v, Odata, O.dataLength);
		GroupMemoryBarrierWithGroupSync();
	}
}

/*
#undef TILE_WIDTH
#define TILE_WIDTH 12
[numthreads(TILE_WIDTH,TILE_WIDTH,KERNEL_COUNT)]
void Conv2DTiled12x12_Kernel3x3(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint tx = groupThreadID.x;
	uint ty = groupThreadID.y;
	uint tk = groupThreadID.z;
	uint gx = groupID.x;
	uint gy = groupID.y;
	uint gk = groupID.z;
	uint tileCornerX = gx*TILE_WIDTH;
	uint tileCornerY = gy*TILE_WIDTH;
	uint x = tileCornerX + tx;
	uint y = tileCornerY + ty;
	uint k = gk*KERNEL_COUNT + tk;
	uint idx = ty*TILE_WIDTH + tx;

	for (uint b = 0; b < X.batch; ++b)
	{
		float v = B.Get(0, 0, k, 0, WBKdata, WBK.dataLength);
		for (uint c = 0; c < X.channels; ++c)
		{
			if (gk == 0)
				XCcache[ty][tx] = X.Get(b, y, x, c, Xdata);
			else if (gk == 1 && idx < TILE_WIDTH * 2)
			{
				uint yy = idx / 2;
				uint xx = idx % 2 + TILE_WIDTH;
				XCcache[yy][xx] = X.Get(b, tileCornerY+yy, tileCornerX+xx, c, Xdata);
			}
			else if (gk == 2 && idx < (TILE_WIDTH + 2) * 2)
			{
				uint yy = idx / (TILE_WIDTH + 2) + TILE_WIDTH;
				uint xx = idx % (TILE_WIDTH + 2);
				XCcache[yy][xx] = X.Get(b, tileCornerY+yy, tileCornerX+xx, c, Xdata);
			}
			else if (gk == 3 && ty < KERNEL_SIZE && tx < KERNEL_SIZE)
				Kcache[ty][tx][tk] = K.Get(ty, tx, c, k, WBKdata, WBK.dataLength);
			GroupMemoryBarrierWithGroupSync();

			for (int i = 0; i < KERNEL_SIZE; ++i)
			{
				for (int j = 0; j < KERNEL_SIZE; ++j)
				{
					v += XCcache[ty+j][tx+i] * Kcache[j][i][tk];
				}
			}
		}
		O.Set(b, y-KERNEL_RADIUS, x-KERNEL_RADIUS, k, v, Odata, O.dataLength);
		GroupMemoryBarrierWithGroupSync();
	}
}
*/

// %TODO: only supports up to 32 channels now
#undef KERNEL_COUNT
#undef CHANNEL_COUNT
#define KERNEL_COUNT 16
#define CHANNEL_COUNT 32
groupshared float K2cache[CHANNEL_COUNT][KERNEL_COUNT][9];
[numthreads(KERNEL_COUNT,CHANNEL_COUNT,1)]
void Conv2D_Kernel3x3_32Channel_Valid(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint tk = groupThreadID.x;
	uint k = KERNEL_COUNT*groupID.x + tk;
	uint n = CHANNEL_COUNT*groupID.y + groupThreadID.y;

	for (uint q = 0; q < 9; ++q)
	{
		uint tc = n % CHANNEL_COUNT;
		K2cache[tc][tk][q] = K.Get(q/3, q%3, tc, k, WBKdata, WBK.dataLength);
	}
	GroupMemoryBarrierWithGroupSync();

	for (uint y = 0; y < X.height - _FilterSize*2; ++y)
	{
		for (uint x = 0; x < X.width - _FilterSize*2; ++x)
		{
			float v = B.Get(0, 0, k, 0, WBKdata, WBK.dataLength);
			for (uint q = 0; q < 9; ++q)
				for (uint c = 0; c < CHANNEL_COUNT; c += 4)
				{
					//K.Get(q/3, q%3, c, k, WBKdata, WBK.dataLength);
					v += X.Get(n, y+q/3, x+q%3, c+0, Xdata, X.dataLength) * K2cache[c+0][tk][q];
					v += X.Get(n, y+q/3, x+q%3, c+1, Xdata, X.dataLength) * K2cache[c+1][tk][q];
					v += X.Get(n, y+q/3, x+q%3, c+2, Xdata, X.dataLength) * K2cache[c+2][tk][q];
					v += X.Get(n, y+q/3, x+q%3, c+3, Xdata, X.dataLength) * K2cache[c+3][tk][q];
				}
			O.Set(n, y, x, k, v, Odata, O.dataLength);
		}
	}
}

[numthreads(KERNEL_COUNT,CHANNEL_COUNT,1)]
void Conv2D_Kernel3x3_32Channel(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint tk = groupThreadID.x;
	uint k = KERNEL_COUNT*groupID.x + tk;
	uint n = CHANNEL_COUNT*groupID.y + groupThreadID.y;

	for (uint q = 0; q < 9; ++q)
	{
		uint tc = n % CHANNEL_COUNT;
		K2cache[tc][tk][q] = K.Get(q/3, q%3, tc, k, WBKdata, WBK.dataLength);
	}
	GroupMemoryBarrierWithGroupSync();

	for (uint y = 0; y < X.height - _Border; ++y)
	{
		for (uint x = 0; x < X.width - _Border; ++x)
		{
			float v = B.Get(0, 0, k, 0, WBKdata, WBK.dataLength);
			for (uint dy = 0; dy < 3; ++dy)
			{
				if (y+dy < _Offset) continue;
				if (y+dy-_Offset >= X.height) continue;
				for (uint dx = 0; dx < 3; ++dx)
				{
					if (x+dx < _Offset) continue;
					if (x+dx-_Offset >= X.width) continue;

					uint q = dy*3+dx;
					for (uint c = 0; c < CHANNEL_COUNT; c += 4)
					{
						//K.Get(q/3, q%3, c, k, WBKdata, WBK.dataLength);
						v += X.Get(n, y+dy-_Offset, x+dx-_Offset, c+0, Xdata, X.dataLength) * K2cache[c+0][tk][q];
						v += X.Get(n, y+dy-_Offset, x+dx-_Offset, c+1, Xdata, X.dataLength) * K2cache[c+1][tk][q];
						v += X.Get(n, y+dy-_Offset, x+dx-_Offset, c+2, Xdata, X.dataLength) * K2cache[c+2][tk][q];
						v += X.Get(n, y+dy-_Offset, x+dx-_Offset, c+3, Xdata, X.dataLength) * K2cache[c+3][tk][q];
					}
				}
			}
			O.Set(n, y, x, k, v, Odata, O.dataLength);
		}
	}
}

groupshared float X2cache[2][CHANNEL_COUNT][KERNEL_COUNT];
[numthreads(KERNEL_COUNT,CHANNEL_COUNT,1)]
void Conv2D_Kernel3x3_32Channel_(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint tk = groupThreadID.x;
	uint tn = groupThreadID.y;
	uint k = KERNEL_COUNT*groupID.x + tk;
	uint n = CHANNEL_COUNT*groupID.y + tn;

	for (uint q = 0; q < 9; ++q)
	{
		uint tc = n % CHANNEL_COUNT;
		K2cache[q][tc][tk] = K.Get(q/3, q%3, tc, k, WBKdata, WBK.dataLength);
	}
	//GroupMemoryBarrierWithGroupSync(); <-- unnecessary, we have one inside the loop

	for (uint y = 0; y < X.height - _FilterSize*2; ++y)
	{
		for (uint x = 0; x < X.width - _FilterSize*2; ++x)
		{
			float v = B.Get(0, 0, k, 0, WBKdata, WBK.dataLength);
			for (uint cBlock = 0; cBlock < CHANNEL_COUNT; cBlock += KERNEL_COUNT)
			{
				for (uint q = 0; q < 9; ++q)
				{
					uint tc = k % KERNEL_COUNT;
					X2cache[q%2][tn][tc] = X.Get(n, y+q/3, x+q%3, cBlock+tc, Xdata, X.dataLength);
					GroupMemoryBarrierWithGroupSync();

					for (tc = 0; tc < KERNEL_COUNT; ++tc)
						v += X2cache[q%2][tn][tc] * K2cache[q][cBlock+tc][tk];
				}
			}
			O.Set(n, y, x, k, v, Odata, O.dataLength);
		}
	}
}

// 16x8 => 0.101
// 32x4 => 0.114
//  8x8 => 0.131

#define PARAM_X 16
#define PARAM_Y 8
[numthreads(PARAM_X, PARAM_Y, 1)]
void Conv2D_Kernel3x3_Kmod16_Cmod4_KN(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint k = PARAM_X * groupID.x + groupThreadID.x;
	uint n = PARAM_Y * groupID.y + groupThreadID.y;

	for (uint y = 0; y < X.height - _Border; ++y)
	{
		for (uint x = 0; x < X.width - _Border; ++x)
		{
			float v = B.Get(0, 0, k, 0, WBKdata, WBK.dataLength);
			for (uint dy = 0; dy < 3; ++dy)
			{
				if (y+dy < _Offset) continue;
				if (y+dy-_Offset >= X.height) continue;
				for (uint dx = 0; dx < 3; ++dx)
				{
					if (x+dx < _Offset) continue;
					if (x+dx-_Offset >= X.width) continue;

					for (uint c = 0; c < X.channels; c += 4)
					{
						v += X.Get(n, y+dy-_Offset, x+dx-_Offset, c+0, Xdata, X.dataLength) * K.Get(dy, dx, c+0, k, WBKdata, WBK.dataLength);
						v += X.Get(n, y+dy-_Offset, x+dx-_Offset, c+1, Xdata, X.dataLength) * K.Get(dy, dx, c+1, k, WBKdata, WBK.dataLength);
						v += X.Get(n, y+dy-_Offset, x+dx-_Offset, c+2, Xdata, X.dataLength) * K.Get(dy, dx, c+2, k, WBKdata, WBK.dataLength);
						v += X.Get(n, y+dy-_Offset, x+dx-_Offset, c+3, Xdata, X.dataLength) * K.Get(dy, dx, c+3, k, WBKdata, WBK.dataLength);
					}
				}
			}
			O.Set(n, y, x, k, v, Odata, O.dataLength);
		}
	}
}
#undef PARAM_X
#undef PARAM_Y
#define PARAM_X 16
#define PARAM_Y 8

// 16x8 => 0.096
//  8x8 => 0.117
[numthreads(PARAM_X, PARAM_Y, 1)]
void Conv2D_Kernel3x3_Kmod16_Cmod4_KNyx(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint k = PARAM_X * groupID.x + groupThreadID.x;
	uint nyx = PARAM_Y * groupID.y + groupThreadID.y;

	uint width = X.width - _Border;
	uint height = X.height - _Border;

	uint x = nyx % width;
	uint ny = nyx / width;
	uint y = ny % height;
	uint n = ny / height;

	//for (uint y = 0; y < X.height - _Border; ++y)
	//{
	//	for (uint x = 0; x < X.width - _Border; ++x)
	//	{
			float v = B.Get(0, 0, k, 0, WBKdata, WBK.dataLength);
			for (uint dy = 0; dy < 3; ++dy)
			{
				if (y+dy < _Offset) continue;
				if (y+dy-_Offset >= X.height) continue;
				for (uint dx = 0; dx < 3; ++dx)
				{
					if (x+dx < _Offset) continue;
					if (x+dx-_Offset >= X.width) continue;

					for (uint c = 0; c < X.channels; c += 4)
					{
						v += X.Get(n, y+dy-_Offset, x+dx-_Offset, c+0, Xdata, X.dataLength) * K.Get(dy, dx, c+0, k, WBKdata, WBK.dataLength);
						v += X.Get(n, y+dy-_Offset, x+dx-_Offset, c+1, Xdata, X.dataLength) * K.Get(dy, dx, c+1, k, WBKdata, WBK.dataLength);
						v += X.Get(n, y+dy-_Offset, x+dx-_Offset, c+2, Xdata, X.dataLength) * K.Get(dy, dx, c+2, k, WBKdata, WBK.dataLength);
						v += X.Get(n, y+dy-_Offset, x+dx-_Offset, c+3, Xdata, X.dataLength) * K.Get(dy, dx, c+3, k, WBKdata, WBK.dataLength);
					}
				}
			}
			O.Set(n, y, x, k, v, Odata, O.dataLength);
	//	}
	//}
}

#undef CTILE
#define CTILE 16

#undef PARAM_X
#undef PARAM_Y
#define PARAM_X CTILE
#define PARAM_Y CTILE

#define TYPE float

groupshared TYPE Conv_XcacheT[CTILE][CTILE];
groupshared TYPE Conv_KcacheT[CTILE][CTILE];

[numthreads(PARAM_X, PARAM_Y, 1)]
void Conv2D_Kernel3x3_Cache_KCmod16_KNyx_(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ Conv_XcacheT
	#define K_ Conv_KcacheT

	uint gx = groupThreadID.x;
	uint gy = groupThreadID.y;

	uint k = PARAM_X * groupID.x + groupThreadID.x;
	uint nyx = PARAM_Y * groupID.y + groupThreadID.y;

	uint width = X.width - _Border;
	uint height = X.height - _Border;

	uint x = nyx % width;
	uint ny = nyx / width;
	uint y = ny % height;
	uint n = ny / height;

			//half v = B.Get(0, 0, k, 0, WBKdata, WBK.dataLength);
			TYPE v = WBKdata[k + B.offset];
			for (uint dy = 0; dy < 3; ++dy)
			{
				bool mask = true;

				if (y+dy < _Offset) mask = false;
				if (y+dy-_Offset >= X.height) mask = false;

				for (uint dx = 0; dx < 3; ++dx)
				{
					if (x+dx < _Offset) mask = false;
					if (x+dx-_Offset >= X.width) mask = false;

					int Xi = (( n             * X.height +
								y+dy-_Offset ) * X.width +
								x+dx-_Offset ) * X.channels +
								gx;

					int Ki = (( dy   * K.height +
								dx ) * K.width +
								/*m*CTILE +*/ gy ) * K.channels +
								k + K.offset;

					for (uint m = 0; m < X.channels/CTILE; ++m)
					{
						if (mask)
						{
							//X_[gy][gx] = X.Get(n, y+dy-_Offset, x+dx-_Offset, m*CTILE + gx, Xdata);
							X_[gy][gx] = Xdata[Xi + m*CTILE];
						}
						else
						{
							X_[gy][gx] = 0;
						}	
						//K_[gy][gx] = K.Get(dy, dx, m*CTILE + gy, k, WBKdata, WBK.dataLength);
						//K_[gy][gx] = WBKdata[((
						//	dy   * K.height +
						//	dx ) * K.width +
						//	m*CTILE + gy ) * K.channels +
						//	k + K.offset];
						//K_[gy][gx] = WBKdata[Ki + m*CTILE * K.channels];
						K_[gy][gx] = WBKdata[Ki + m*CTILE * K.channels];
						GroupMemoryBarrierWithGroupSync();

						for (uint i = 0; i < CTILE;)
						{
							/*
							// can unroll up to CTILE
							half4 x4 = ((half4[CTILE][CTILE/4])(X_))[gy][i];
							half4 k4 = ((half4[CTILE][CTILE/4])(K_))[gx][i];

							v += dot(x4, k4); ++i;
							v += dot(x4, k4); ++i;
							*/
							
							v += X_[gy][i] * K_[i][gx]; ++i;
							v += X_[gy][i] * K_[i][gx]; ++i;
							v += X_[gy][i] * K_[i][gx]; ++i;
							v += X_[gy][i] * K_[i][gx]; ++i;
							v += X_[gy][i] * K_[i][gx]; ++i;
							v += X_[gy][i] * K_[i][gx]; ++i;
							v += X_[gy][i] * K_[i][gx]; ++i;
							v += X_[gy][i] * K_[i][gx]; ++i;
								
						}
					}
				}
			}
			//O.Set(n, y, x, k, v, Odata, O.dataLength);
			Odata[((
				n   * O.height +
				y ) * O.width +
				x ) * O.channels +
				k] = v;

	#undef X_
	#undef K_
}

#undef CTILE
#define CTILE 16
groupshared float Conv_XcacheA[4][CTILE][CTILE];
groupshared float Conv_Kcache0[CTILE][CTILE];
groupshared float Conv_Kcache1[CTILE][CTILE];
groupshared float Conv_Kcache2[CTILE][CTILE];
groupshared float Conv_Kcache3[CTILE][CTILE];
[numthreads(CTILE, CTILE, 1)]
void Conv2D_Kernel3x3_Cache_KCmod32_KNyx____(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ Conv_XcacheA
	#define K_0 Conv_Kcache0
	#define K_1 Conv_Kcache1
	#define K_2 Conv_Kcache2
	#define K_3 Conv_Kcache3



	uint gx = groupThreadID.x;
	uint gy = groupThreadID.y;

	uint k = CTILE * groupID.x + groupThreadID.x;
	uint nyx = CTILE * groupID.y + groupThreadID.y;

	uint width = X.width - _Border;
	uint height = X.height - _Border;

	uint x = nyx % width;
	uint ny = nyx / width;
	uint y = ny % height;
	uint n = ny / height;
	
	float b0 = B.Get(0, 0, k*2+0, 0, WBKdata, WBK.dataLength);
	float b1 = B.Get(0, 0, k*2+1, 0, WBKdata, WBK.dataLength);
	float4 v = float4(b0, b1,
					  b0, b1);

	for (uint dy = 0; dy < 3; ++dy)
	{
		bool mask = true;

		if (y+dy < _Offset) mask = false;
		if (y+dy-_Offset >= X.height) mask = false;

		for (uint dx = 0; dx < 3; ++dx)
		{
			if (x+dx < _Offset) mask = false;
			if (x+dx-_Offset >= X.width) mask = false;

			for (uint m = 0; m < X.channels/(CTILE*2); ++m)
			{
				float x0 = 0;
				float x1 = 0;
				float x2 = 0;
				float x3 = 0;
				
				if (mask)
				{
					x0 = X.Get(n*2+0, y+dy-_Offset, x+dx-_Offset, (m*CTILE + gx)*2+0, Xdata);
					x1 = X.Get(n*2+0, y+dy-_Offset, x+dx-_Offset, (m*CTILE + gx)*2+1, Xdata);
					x2 = X.Get(n*2+1, y+dy-_Offset, x+dx-_Offset, (m*CTILE + gx)*2+0, Xdata);
					x3 = X.Get(n*2+1, y+dy-_Offset, x+dx-_Offset, (m*CTILE + gx)*2+1, Xdata);
				}

				float k0 = K.Get(dy, dx, (m*CTILE + gy)*2+0, k*2+0, WBKdata, WBK.dataLength);
				float k1 = K.Get(dy, dx, (m*CTILE + gy)*2+0, k*2+1, WBKdata, WBK.dataLength);
				float k2 = K.Get(dy, dx, (m*CTILE + gy)*2+1, k*2+0, WBKdata, WBK.dataLength);
				float k3 = K.Get(dy, dx, (m*CTILE + gy)*2+1, k*2+1, WBKdata, WBK.dataLength);

				//X_[gy][gx] = float4(x0, x1,
				//					x2, x3);
				//K_[gy][gx] = float4(k0, k1,
				//					k2, k3);
				X_[0][gy][gx] = x0;
				X_[1][gy][gx] = x1;
				X_[2][gy][gx] = x2;
				X_[3][gy][gx] = x3;

				K_0[gy][gx] = k0;
				K_1[gy][gx] = k1;
				K_2[gy][gx] = k2;
				K_3[gy][gx] = k3;

				GroupMemoryBarrierWithGroupSync();

				[unroll]
				for (uint i = 0; i < CTILE; ++i)
				{
					float4 x = //X_[gy][i];
						float4(	X_[0][gy][i],
								X_[1][gy][i],
								X_[2][gy][i],
								X_[3][gy][i]);
					//float4 k = //K_[i][gx];
					//	float4(	K_0[i][gx],
					//			K_1[i][gx],
					//			K_2[i][gx],
					//			K_3[i][gx]);
					k0 = K_0[i][gx];
					k1 = K_1[i][gx];
					k2 = K_2[i][gx];
					k3 = K_3[i][gx];
					
					v.x = mad(k0, x.x, v.x);
					v.x = mad(k2, x.y, v.x);
					
					v.y = mad(k1, x.x, v.y);
					v.y = mad(k2, x.y, v.y);
					
					v.z = mad(k0, x.z, v.z);
					v.z = mad(k2, x.w, v.z);
					
					v.w = mad(k1, x.z, v.w);
					v.w = mad(k3, x.w, v.w);

					//v.x += k.x*x.x + k.z*x.y;
					//v.y += k.y*x.x + k.w*x.y;
					//v.z += k.x*x.z + k.z*x.w;
					//v.w += k.y*x.z + k.w*x.w;
				}

				GroupMemoryBarrierWithGroupSync();
			}
		}
	}

	//Odata[nyx * O.channels + k] = v;
	
	/*Odata[((
		n   * O.height +
		y ) * O.width +
		x ) * O.channels +
		k] = v;
		*/

	O.Set(n*2+0, y, x, k*2+0, v.x, Odata);
	O.Set(n*2+0, y, x, k*2+1, v.y, Odata);
	O.Set(n*2+1, y, x, k*2+0, v.z, Odata);
	O.Set(n*2+1, y, x, k*2+1, v.w, Odata);
	
	#undef X_
	#undef K_
}


#undef CTILE
#define CTILE 16
groupshared float Conv_Xcache[4][CTILE][CTILE];
groupshared float Conv_Kcache[4][CTILE][CTILE];
[numthreads(CTILE, CTILE, 1)]
void Conv2D_Kernel3x3_Cache_KCmod32_KNyx(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ Conv_Xcache
	#define K_ Conv_Kcache

	uint gx = groupThreadID.x;
	uint gy = groupThreadID.y;

	uint k = CTILE * groupID.x + groupThreadID.x;
	uint nyx = CTILE * groupID.y + groupThreadID.y;

	uint width = X.width - _Border;
	uint height = X.height - _Border;

	uint x = nyx % width;
	uint ny = nyx / width;
	uint y = ny % height;
	uint n = ny / height;
	
	float b0 = B.Get(0, 0, k*2+0, 0, WBKdata, WBK.dataLength);
	float b1 = B.Get(0, 0, k*2+1, 0, WBKdata, WBK.dataLength);
	float4 v = float4(b0, b1,
					  b0, b1);

	for (uint dy = 0; dy < 3; ++dy)
	{
		bool mask = true;

		if (y+dy < _Offset) mask = false;
		if (y+dy-_Offset >= X.height) mask = false;

		for (uint dx = 0; dx < 3; ++dx)
		{
			if (x+dx < _Offset) mask = false;
			if (x+dx-_Offset >= X.width) mask = false;

			for (uint m = 0; m < X.channels/(CTILE*2); ++m)
			{
				float x0 = 0;
				float x1 = 0;
				float x2 = 0;
				float x3 = 0;
				
				if (mask)
				{
					x0 = X.Get(n*2+0, y+dy-_Offset, x+dx-_Offset, (m*CTILE + gx)*2+0, Xdata);
					x1 = X.Get(n*2+0, y+dy-_Offset, x+dx-_Offset, (m*CTILE + gx)*2+1, Xdata);
					x2 = X.Get(n*2+1, y+dy-_Offset, x+dx-_Offset, (m*CTILE + gx)*2+0, Xdata);
					x3 = X.Get(n*2+1, y+dy-_Offset, x+dx-_Offset, (m*CTILE + gx)*2+1, Xdata);
				}

				float k0 = K.Get(dy, dx, (m*CTILE + gy)*2+0, k*2+0, WBKdata, WBK.dataLength);
				float k1 = K.Get(dy, dx, (m*CTILE + gy)*2+0, k*2+1, WBKdata, WBK.dataLength);
				float k2 = K.Get(dy, dx, (m*CTILE + gy)*2+1, k*2+0, WBKdata, WBK.dataLength);
				float k3 = K.Get(dy, dx, (m*CTILE + gy)*2+1, k*2+1, WBKdata, WBK.dataLength);

				//X_[gy][gx] = float4(x0, x1,
				//					x2, x3);
				//K_[gy][gx] = float4(k0, k1,
				//					k2, k3);
				X_[0][gy][gx] = x0;
				X_[1][gy][gx] = x1;
				X_[2][gy][gx] = x2;
				X_[3][gy][gx] = x3;

				K_[0][gy][gx] = k0;
				K_[1][gy][gx] = k1;
				K_[2][gy][gx] = k2;
				K_[3][gy][gx] = k3;

				GroupMemoryBarrierWithGroupSync();

				[unroll]
				for (uint i = 0; i < CTILE; ++i)
				{
					float4 x = //X_[gy][i];
						float4(	X_[0][gy][i],
								X_[1][gy][i],
								X_[2][gy][i],
								X_[3][gy][i]);
					float4 k = //K_[i][gx];
						float4(	K_[0][i][gx],
								K_[1][i][gx],
								K_[2][i][gx],
								K_[3][i][gx]);
					
					v.x = mad(k.x, x.x, v.x);
					v.x = mad(k.z, x.y, v.x);
					
					v.y = mad(k.y, x.x, v.y);
					v.y = mad(k.w, x.y, v.y);
					
					v.z = mad(k.x, x.z, v.z);
					v.z = mad(k.z, x.w, v.z);
					
					v.w = mad(k.y, x.z, v.w);
					v.w = mad(k.w, x.w, v.w);

					//v.x += k.x*x.x + k.z*x.y;
					//v.y += k.y*x.x + k.w*x.y;
					//v.z += k.x*x.z + k.z*x.w;
					//v.w += k.y*x.z + k.w*x.w;
				}

				GroupMemoryBarrierWithGroupSync();
			}
		}
	}

	//Odata[nyx * O.channels + k] = v;
	
	/*Odata[((
		n   * O.height +
		y ) * O.width +
		x ) * O.channels +
		k] = v;
		*/

	O.Set(n*2+0, y, x, k*2+0, v.x, Odata);
	O.Set(n*2+0, y, x, k*2+1, v.y, Odata);
	O.Set(n*2+1, y, x, k*2+0, v.z, Odata);
	O.Set(n*2+1, y, x, k*2+1, v.w, Odata);
	
	#undef X_
	#undef K_
}

#if 0 // =====================================================================================================

#undef CTILE
#define CTILE 16
#define RTILE 4
groupshared float Conv_XcacheR[RTILE*RTILE][CTILE*CTILE];
groupshared float Conv_KcacheR[RTILE*RTILE][CTILE*CTILE];
[numthreads(CTILE, CTILE, 1)]
void Conv2D_Kernel3x3_Cache_KCmod64_KNyx(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ Conv_XcacheR
	#define K_ Conv_KcacheR

	uint gx = groupThreadID.x;
	uint gy = groupThreadID.y;

	uint k = CTILE * groupID.x + groupThreadID.x;
	uint nyx = CTILE * groupID.y + groupThreadID.y;

	uint width = X.width - _Border;
	uint height = X.height - _Border;

	uint x = nyx % width;
	uint ny = nyx / width;
	uint y = ny % height;
	uint n = ny / height;

	float v[RTILE][RTILE];
	for (uint xxxx = 0; xxxx < RTILE; ++xxxx)
	{
		float b = B.Get(0, 0, k*RTILE+xxxx, 0, WBKdata, WBK.dataLength);
		for (uint yyyy = 0; yyyy < RTILE; ++yyyy)
			v[yyyy][xxxx] = b;
	}

	for (uint dy = 0; dy < 3; ++dy)
	{
		bool mask = true;

		if (y+dy < _Offset) mask = false;
		if (y+dy-_Offset >= X.height) mask = false;

		for (uint dx = 0; dx < 3; ++dx)
		{
			if (x+dx < _Offset) mask = false;
			if (x+dx-_Offset >= X.width) mask = false;

			for (uint m = 0; m < X.channels/(CTILE*RTILE); ++m)
			{				
				for (uint yy = 0; yy < RTILE; ++yy)
					for (uint xx = 0; xx < RTILE; ++xx)
					{
						if (mask)
							X_[yy*RTILE+xx][gy*CTILE+gx] = X.Get(n*RTILE+yy, y+dy-_Offset, x+dx-_Offset, (m*CTILE + gx)*RTILE+xx, Xdata);
						else
							X_[yy*RTILE+xx][gy*CTILE+gx] = 0;
						K_[yy*RTILE+xx][gy*CTILE+gx] = K.Get(dy, dx, (m*CTILE + gy)*RTILE+yy, k*RTILE+xx, WBKdata, WBK.dataLength);
					}

				GroupMemoryBarrierWithGroupSync();

				for (uint ii = 0; ii < CTILE; ++ii)
				{
					float x[RTILE][RTILE];
					float k[RTILE][RTILE];

					[unroll]
					for (uint yy = 0; yy < RTILE; ++yy)
					{
						[unroll]
						for (uint xx = 0; xx < RTILE; ++xx)
						{
							x[yy][xx] = X_[yy*RTILE+xx][gy*CTILE+ii];
							k[yy][xx] = K_[yy*RTILE+xx][ii*CTILE+gx];
						}
					}


					[unroll]
					for (uint yyy = 0; yyy < RTILE; ++yyy)
					{
						[unroll]
						for (uint xxx = 0; xxx < RTILE; ++xxx)
						{
							[unroll]
							for (uint i = 0; i < RTILE; ++i)
							{
								v[yyy][xxx] = mad(x[yyy][i], k[i][xxx], v[yyy][xxx]);
							}
						}
					}
				}

				GroupMemoryBarrierWithGroupSync();
			}
		}
	}

	for (uint yy = 0; yy < RTILE; ++yy)
		for (uint xx = 0; xx < RTILE; ++xx)
			O.Set(n*RTILE+yy, y, x, k*RTILE+xx, v[yy][xx], Odata);
	
	#undef X_
	#undef K_
}

#elif 1 // =====================================================================================================

#undef CTILE
#define CTILE 16
groupshared float2 Conv_KcacheR[8][CTILE*CTILE];
groupshared float2 Conv_XcacheR[8][CTILE*CTILE];
[numthreads(CTILE, CTILE, 1)]
void Conv2D_Kernel3x3_Cache_KCmod64_KNyx(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ Conv_XcacheR
	#define K_ Conv_KcacheR

	uint gx = groupThreadID.x;
	uint gy = groupThreadID.y;

	uint k = CTILE * groupID.x + groupThreadID.x;
	uint nyx = CTILE * groupID.y + groupThreadID.y;

	uint width = X.width - _Border;
	uint height = X.height - _Border;

	uint x = nyx % width;
	uint ny = nyx / width;
	uint y = ny % height;
	uint n = ny / height;
	
	float b0 = B.Get(0, 0, k*4+0, 0, WBKdata, WBK.dataLength);
	float b1 = B.Get(0, 0, k*4+1, 0, WBKdata, WBK.dataLength);
	float b2 = B.Get(0, 0, k*4+2, 0, WBKdata, WBK.dataLength);
	float b3 = B.Get(0, 0, k*4+3, 0, WBKdata, WBK.dataLength);
	
	float4 v0, v1, v2, v3;
	v0 = v1 = v2 = v3 = float4(b0, b1, b2, b3);

	for (uint dy = 0; dy < 3; ++dy)
	{
		bool mask = true;

		if (y+dy < _Offset) mask = false;
		if (y+dy-_Offset >= X.height) mask = false;

		for (uint dx = 0; dx < 3; ++dx)
		{
			if (x+dx < _Offset) mask = false;
			if (x+dx-_Offset >= X.width) mask = false;

			for (uint m = 0; m < X.channels/(CTILE*4); ++m)
			{				
				for (uint yy = 0; yy < 4; ++yy)
					for (uint xx = 0; xx < 2; ++xx)
					{
						// 111ms
						if (mask)
						{
							X_[yy*2+xx][gy*CTILE+gx].x = X.Get(n*4+yy, y+dy-_Offset, x+dx-_Offset, (m*CTILE + gx)*4+xx*2+0, Xdata);
							X_[yy*2+xx][gy*CTILE+gx].y = X.Get(n*4+yy, y+dy-_Offset, x+dx-_Offset, (m*CTILE + gx)*4+xx*2+1, Xdata);
						}
						else
						{
							X_[yy*2+xx][gy*CTILE+gx].x = 0;
							X_[yy*2+xx][gy*CTILE+gx].y = 0;
						}

						K_[yy*2+xx][gy*CTILE+gx].x = K.Get(dy, dx, (m*CTILE + gy)*4+yy, k*4+xx*2+0, WBKdata, WBK.dataLength);
						K_[yy*2+xx][gy*CTILE+gx].y = K.Get(dy, dx, (m*CTILE + gy)*4+yy, k*4+xx*2+1, WBKdata, WBK.dataLength);
					}

				GroupMemoryBarrierWithGroupSync();

				for (uint i = 0; i < CTILE; ++i)
				{					
					#if 1 // ----------------------------------------------------------

					float2 x[8];
					float2 k[8];

					// 109ms
					// dcl_temps 29
					for (uint regs = 0; regs < 8; ++regs)
					{
						x[regs] = X_[regs][gy*CTILE+i];
						k[regs] = K_[regs][i*CTILE+gx];
					}

					for (uint q = 0; q < 4; ++q)
					{
						float
							k0 = k[q*2+0].x,
							k1 = k[q*2+0].y,
							k2 = k[q*2+1].x,
							k3 = k[q*2+1].y;
						float
							x0 = x[0+q/2].x,
							x1 = x[2+q/2].x,
							x2 = x[4+q/2].x,
							x3 = x[6+q/2].x;

						v0.x = mad(x0, k0, v0.x); //--
						v1.x = mad(x1, k0, v1.x); 
						v2.x = mad(x2, k0, v2.x); 
						v3.x = mad(x3, k0, v3.x); 
						v0.y = mad(x0, k1, v0.y); //--
						v1.y = mad(x1, k1, v1.y);
						v2.y = mad(x2, k1, v2.y);
						v3.y = mad(x3, k1, v3.y);
						v0.z = mad(x0, k2, v0.z); //--
						v1.z = mad(x1, k2, v1.z); 
						v2.z = mad(x2, k2, v2.z); 
						v3.z = mad(x3, k2, v3.z); 
						v0.w = mad(x0, k3, v0.w); //--
						v1.w = mad(x1, k3, v1.w);
						v2.w = mad(x2, k3, v2.w);
						v3.w = mad(x3, k3, v3.w);

						++q;

							k0 = k[q*2+0].x;
							k1 = k[q*2+0].y;
							k2 = k[q*2+1].x;
							k3 = k[q*2+1].y;

							x0 = x[0+q/2].y;
							x1 = x[2+q/2].y;
							x2 = x[4+q/2].y;
							x3 = x[6+q/2].y;

						v0.x = mad(x0, k0, v0.x); //--
						v1.x = mad(x1, k0, v1.x); 
						v2.x = mad(x2, k0, v2.x); 
						v3.x = mad(x3, k0, v3.x); 
						v0.y = mad(x0, k1, v0.y); //--
						v1.y = mad(x1, k1, v1.y);
						v2.y = mad(x2, k1, v2.y);
						v3.y = mad(x3, k1, v3.y);
						v0.z = mad(x0, k2, v0.z); //--
						v1.z = mad(x1, k2, v1.z); 
						v2.z = mad(x2, k2, v2.z); 
						v3.z = mad(x3, k2, v3.z); 
						v0.w = mad(x0, k3, v0.w); //--
						v1.w = mad(x1, k3, v1.w);
						v2.w = mad(x2, k3, v2.w);
						v3.w = mad(x3, k3, v3.w);
					}
				
					#endif  // ----------------------------------------------------------
				}

				GroupMemoryBarrierWithGroupSync();
			}
		}
	}
	
	#if 1 // ----------------------------------------------------------

	// 117ms
	O.Set(n*4+0, y, x, k*4+0, v0.x, Odata);
	O.Set(n*4+0, y, x, k*4+1, v0.y, Odata);
	O.Set(n*4+0, y, x, k*4+2, v0.z, Odata);
	O.Set(n*4+0, y, x, k*4+3, v0.w, Odata);
	
	O.Set(n*4+1, y, x, k*4+0, v1.x, Odata);
	O.Set(n*4+1, y, x, k*4+1, v1.y, Odata);
	O.Set(n*4+1, y, x, k*4+2, v1.z, Odata);
	O.Set(n*4+1, y, x, k*4+3, v1.w, Odata);
	
	O.Set(n*4+2, y, x, k*4+0, v2.x, Odata);
	O.Set(n*4+2, y, x, k*4+1, v2.y, Odata);
	O.Set(n*4+2, y, x, k*4+2, v2.z, Odata);
	O.Set(n*4+2, y, x, k*4+3, v2.w, Odata);
	
	O.Set(n*4+3, y, x, k*4+0, v3.x, Odata);
	O.Set(n*4+3, y, x, k*4+1, v3.y, Odata);
	O.Set(n*4+3, y, x, k*4+2, v3.z, Odata);
	O.Set(n*4+3, y, x, k*4+3, v3.w, Odata);

	#else // ----------------------------------------------------------
		
	// 118ms
	O.Set(n*4+0, y, x, k*4+0, v0.x, Odata);
	O.Set(n*4+1, y, x, k*4+0, v1.x, Odata);
	O.Set(n*4+2, y, x, k*4+0, v2.x, Odata);
	O.Set(n*4+3, y, x, k*4+0, v3.x, Odata);
	
	O.Set(n*4+0, y, x, k*4+1, v0.y, Odata);
	O.Set(n*4+1, y, x, k*4+1, v1.y, Odata);
	O.Set(n*4+2, y, x, k*4+1, v2.y, Odata);
	O.Set(n*4+3, y, x, k*4+1, v3.y, Odata);
	
	O.Set(n*4+0, y, x, k*4+2, v0.z, Odata);
	O.Set(n*4+1, y, x, k*4+2, v1.z, Odata);
	O.Set(n*4+2, y, x, k*4+2, v2.z, Odata);
	O.Set(n*4+3, y, x, k*4+2, v3.z, Odata);
	
	O.Set(n*4+0, y, x, k*4+3, v0.w, Odata);
	O.Set(n*4+1, y, x, k*4+3, v1.w, Odata);
	O.Set(n*4+2, y, x, k*4+3, v2.w, Odata);
	O.Set(n*4+3, y, x, k*4+3, v3.w, Odata);

	#endif // ----------------------------------------------------------

			  
	#undef X_
	#undef K_
}

#elif 1 // =====================================================================================================

#undef CTILE
#define CTILE 16
groupshared float Conv_KcacheR[16][CTILE*CTILE];
groupshared float Conv_XcacheR[16][CTILE*CTILE];
[numthreads(CTILE, CTILE, 1)]
void Conv2D_Kernel3x3_Cache_KCmod64_KNyx(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ Conv_XcacheR
	#define K_ Conv_KcacheR

	uint gx = groupThreadID.x;
	uint gy = groupThreadID.y;

	uint k = CTILE * groupID.x + groupThreadID.x;
	uint nyx = CTILE * groupID.y + groupThreadID.y;

	uint width = X.width - _Border;
	uint height = X.height - _Border;

	uint x = nyx % width;
	uint ny = nyx / width;
	uint y = ny % height;
	uint n = ny / height;
	
	float b0 = B.Get(0, 0, k*4+0, 0, WBKdata, WBK.dataLength);
	float b1 = B.Get(0, 0, k*4+1, 0, WBKdata, WBK.dataLength);
	float b2 = B.Get(0, 0, k*4+2, 0, WBKdata, WBK.dataLength);
	float b3 = B.Get(0, 0, k*4+3, 0, WBKdata, WBK.dataLength);
	
	float4 v0, v1, v2, v3;
	v0 = v1 = v2 = v3 = float4(b0, b1, b2, b3);

	for (uint dy = 0; dy < 3; ++dy)
	{
		bool mask = true;

		if (y+dy < _Offset) mask = false;
		if (y+dy-_Offset >= X.height) mask = false;

		for (uint dx = 0; dx < 3; ++dx)
		{
			if (x+dx < _Offset) mask = false;
			if (x+dx-_Offset >= X.width) mask = false;

			for (uint m = 0; m < X.channels/(CTILE*4); ++m)
			{				
				for (uint yy = 0; yy < 4; ++yy)
					for (uint xx = 0; xx < 4; ++xx)
					{
						#if 1  // ----------------------------------------------------------

						// 111ms
						if (mask)
							X_[yy*4+xx][gy*CTILE+gx] = X.Get(n*4+yy, y+dy-_Offset, x+dx-_Offset, (m*CTILE + gx)*4+xx, Xdata);
						else
							X_[yy*4+xx][gy*CTILE+gx] = 0;
						K_[yy*4+xx][gy*CTILE+gx] = K.Get(dy, dx, (m*CTILE + gy)*4+yy, k*4+xx, WBKdata, WBK.dataLength);

						#else  // ----------------------------------------------------------

						// 122ms
						if (mask)
							X_[yy*4+(gx%4)][gy*CTILE+xx*4+(gx/4)] = X.Get(n*4+yy, y+dy-_Offset, x+dx-_Offset, m*CTILE*4 + xx*CTILE + gx, Xdata);
						else
							X_[yy*4+(gx%4)][gy*CTILE+xx*4+(gx/4)] = 0;
						K_[yy*4+(k%4)][gy*CTILE+xx*4+(gx/4)] = K.Get(dy, dx, (m*CTILE + gy)*4+yy, CTILE*groupID.x*4 + xx*CTILE + gx, WBKdata, WBK.dataLength);

						#endif  // ----------------------------------------------------------
					}

				GroupMemoryBarrierWithGroupSync();

				for (uint i = 0; i < CTILE; ++i)
				{

					#if 0 // ----------------------------------------------------------

					float x[16];
					float k[16];

					k[0] = K_[0][i*CTILE+gx];
					x[0] = X_[0][gy*CTILE+i];
					x[4] = X_[4][gy*CTILE+i];
					x[8] = X_[8][gy*CTILE+i];
					x[12] = X_[12][gy*CTILE+i];

					for (uint q = 0; q < 3; ++q)
					{
						k[q*4+1] = K_[q*4+1][i*CTILE+gx];
						v0.x = mad(x[0*4+q], k[q*4+0], v0.x); //--
						v1.x = mad(x[1*4+q], k[q*4+0], v1.x);
						x[0*4+q+1] = X_[0*4+q+1][gy*CTILE+i];
						v2.x = mad(x[2*4+q], k[q*4+0], v2.x); 
						v3.x = mad(x[3*4+q], k[q*4+0], v3.x); 
						k[q*4+2] = K_[q*4+2][i*CTILE+gx];
						v0.y = mad(x[0*4+q], k[q*4+1], v0.y); //--
						v1.y = mad(x[1*4+q], k[q*4+1], v1.y);
						x[1*4+q+1] = X_[1*4+q+1][gy*CTILE+i];
						v2.y = mad(x[2*4+q], k[q*4+1], v2.y);
						v3.y = mad(x[3*4+q], k[q*4+1], v3.y);
						k[q*4+3] = K_[q*4+3][i*CTILE+gx];
						v0.z = mad(x[0*4+q], k[q*4+2], v0.z); //--
						v1.z = mad(x[1*4+q], k[q*4+2], v1.z); 
						x[2*4+q+1] = X_[2*4+q+1][gy*CTILE+i];
						v2.z = mad(x[2*4+q], k[q*4+2], v2.z); 
						v3.z = mad(x[3*4+q], k[q*4+2], v3.z); 
						k[q*4+4] = K_[q*4+4][i*CTILE+gx];
						v0.w = mad(x[0*4+q], k[q*4+3], v0.w); //--
						v1.w = mad(x[1*4+q], k[q*4+3], v1.w);
						x[3*4+q+1] = X_[3*4+q+1][gy*CTILE+i];
						v2.w = mad(x[2*4+q], k[q*4+3], v2.w);
						v3.w = mad(x[3*4+q], k[q*4+3], v3.w);
					}
					{
						k[q*4+1] = K_[q*4+1][i*CTILE+gx];
						v0.x = mad(x[0*4+q], k[q*4+0], v0.x); //--
						v1.x = mad(x[1*4+q], k[q*4+0], v1.x); 
						v2.x = mad(x[2*4+q], k[q*4+0], v2.x); 
						v3.x = mad(x[3*4+q], k[q*4+0], v3.x); 
						k[q*4+2] = K_[q*4+2][i*CTILE+gx];
						v0.y = mad(x[0*4+q], k[q*4+1], v0.y); //--
						v1.y = mad(x[1*4+q], k[q*4+1], v1.y);
						v2.y = mad(x[2*4+q], k[q*4+1], v2.y);
						v3.y = mad(x[3*4+q], k[q*4+1], v3.y);
						k[q*4+3] = K_[q*4+3][i*CTILE+gx];
						v0.z = mad(x[0*4+q], k[q*4+2], v0.z); //--
						v1.z = mad(x[1*4+q], k[q*4+2], v1.z); 
						v2.z = mad(x[2*4+q], k[q*4+2], v2.z); 
						v3.z = mad(x[3*4+q], k[q*4+2], v3.z); 
						v0.w = mad(x[0*4+q], k[q*4+3], v0.w); //--
						v1.w = mad(x[1*4+q], k[q*4+3], v1.w);
						v2.w = mad(x[2*4+q], k[q*4+3], v2.w);
						v3.w = mad(x[3*4+q], k[q*4+3], v3.w);
					}
		
					#elif 0 // ----------------------------------------------------------

					//float x[4];
					//float k[4];

					float k0 = K_[0*4+0][i*CTILE+gx];
					float x0 = X_[0*4+0][gy*CTILE+i];
					float x1 = X_[1*4+0][gy*CTILE+i];
					float x2 = X_[2*4+0][gy*CTILE+i];
					float x3 = X_[3*4+0][gy*CTILE+i];

					float k1, k2, k3;
					float x0p, x1p, x2p, x3p;

					uint q = 0;
					//for (uint q = 0; q < 4;)
					{
						//x[regs] = X_[regs][gy*CTILE+i];
						
							k1 = K_[q*4+1][i*CTILE+gx];
						v0.x = mad(x0, k0, v0.x); //--
						v1.x = mad(x1, k0, v1.x); 
							x0p = X_[0*4+q+1][gy*CTILE+i];
						v2.x = mad(x2, k0, v2.x); 
						v3.x = mad(x3, k0, v3.x);

							k2 = K_[q*4+2][i*CTILE+gx];
						v0.y = mad(x0, k1, v0.y); //--
						v1.y = mad(x1, k1, v1.y);
							x1p = X_[1*4+q+1][gy*CTILE+i];
						v2.y = mad(x2, k1, v2.y);
						v3.y = mad(x3, k1, v3.y);

							k3 = K_[q*4+3][i*CTILE+gx];
						v0.z = mad(x0, k2, v0.z); //--
						v1.z = mad(x1, k2, v1.z); 
							x2p = X_[2*4+q+1][gy*CTILE+i];
						v2.z = mad(x2, k2, v2.z); 
						v3.z = mad(x3, k2, v3.z);
						
							k0 = K_[q*4+4][i*CTILE+gx];
						v0.w = mad(x0, k3, v0.w); //--
						v1.w = mad(x1, k3, v1.w);
							x3p = X_[3*4+q+1][gy*CTILE+i];
						v2.w = mad(x2, k3, v2.w);
						v3.w = mad(x3, k3, v3.w);

						++q;

							k1 = K_[q*4+1][i*CTILE+gx];
						v0.x = mad(x0p, k0, v0.x); //--
						v1.x = mad(x1p, k0, v1.x); 
							x0 = X_[0*4+q+1][gy*CTILE+i];
						v2.x = mad(x2p, k0, v2.x); 
						v3.x = mad(x3p, k0, v3.x);
									 
							k2 = K_[q*4+2][i*CTILE+gx];
						v0.y = mad(x0p, k1, v0.y); //--
						v1.y = mad(x1p, k1, v1.y);
							x1 = X_[1*4+q+1][gy*CTILE+i];
						v2.y = mad(x2p, k1, v2.y);
						v3.y = mad(x3p, k1, v3.y);
									 
							k3 = K_[q*4+3][i*CTILE+gx];
						v0.z = mad(x0p, k2, v0.z); //--
						v1.z = mad(x1p, k2, v1.z); 
							x2 = X_[2*4+q+1][gy*CTILE+i];
						v2.z = mad(x2p, k2, v2.z); 
						v3.z = mad(x3p, k2, v3.z);
									 
							k0 = K_[q*4+4][i*CTILE+gx];
						v0.w = mad(x0p, k3, v0.w); //--
						v1.w = mad(x1p, k3, v1.w);
							x3 = X_[3*4+q+1][gy*CTILE+i];
						v2.w = mad(x2p, k3, v2.w);
						v3.w = mad(x3p, k3, v3.w);

						++q;

							k1 = K_[q*4+1][i*CTILE+gx];
						v0.x = mad(x0, k0, v0.x); //--
						v1.x = mad(x1, k0, v1.x); 
							x0p = X_[0*4+q+1][gy*CTILE+i];
						v2.x = mad(x2, k0, v2.x); 
						v3.x = mad(x3, k0, v3.x);

							k2 = K_[q*4+2][i*CTILE+gx];
						v0.y = mad(x0, k1, v0.y); //--
						v1.y = mad(x1, k1, v1.y);
							x1p = X_[1*4+q+1][gy*CTILE+i];
						v2.y = mad(x2, k1, v2.y);
						v3.y = mad(x3, k1, v3.y);

							k3 = K_[q*4+3][i*CTILE+gx];
						v0.z = mad(x0, k2, v0.z); //--
						v1.z = mad(x1, k2, v1.z); 
							x2p = X_[2*4+q+1][gy*CTILE+i];
						v2.z = mad(x2, k2, v2.z); 
						v3.z = mad(x3, k2, v3.z);
						
							k0 = K_[q*4+4][i*CTILE+gx];
						v0.w = mad(x0, k3, v0.w); //--
						v1.w = mad(x1, k3, v1.w);
							x3p = X_[3*4+q+1][gy*CTILE+i];
						v2.w = mad(x2, k3, v2.w);
						v3.w = mad(x3, k3, v3.w);

						++q;

							k1 = K_[q*4+1][i*CTILE+gx];
						v0.x = mad(x0p, k0, v0.x); //--
						v1.x = mad(x1p, k0, v1.x); 
							//x0p = X_[0*4+q][gy*CTILE+i];
						v2.x = mad(x2p, k0, v2.x); 
						v3.x = mad(x3p, k0, v3.x);
									 
							k2 = K_[q*4+2][i*CTILE+gx];
						v0.y = mad(x0p, k1, v0.y); //--
						v1.y = mad(x1p, k1, v1.y);
							//x1p = X_[1*4+q][gy*CTILE+i];
						v2.y = mad(x2p, k1, v2.y);
						v3.y = mad(x3p, k1, v3.y);
									 
							k3 = K_[q*4+3][i*CTILE+gx];
						v0.z = mad(x0p, k2, v0.z); //--
						v1.z = mad(x1p, k2, v1.z); 
							//x2p = X_[2*4+q][gy*CTILE+i];
						v2.z = mad(x2p, k2, v2.z); 
						v3.z = mad(x3p, k2, v3.z);
									 
							//k0 = K_[(q+1)*4][i*CTILE+gx];
						v0.w = mad(x0p, k3, v0.w); //--
						v1.w = mad(x1p, k3, v1.w);
							//x3p = X_[3*4+q][gy*CTILE+i];
						v2.w = mad(x2p, k3, v2.w);
						v3.w = mad(x3p, k3, v3.w);

						++q;
					}
			
					
					#elif 1 // ----------------------------------------------------------

					float x[16];
					float k[16];

					// 109ms
					// dcl_temps 29
					for (uint regs = 0; regs < 16; ++regs)
					{
						x[regs] = X_[regs][gy*CTILE+i];
						k[regs] = K_[regs][i*CTILE+gx];
					}

					for (uint q = 0; q < 4; ++q)
					{
						v0.x = mad(x[0*4+q], k[q*4+0], v0.x); //--
						v1.x = mad(x[1*4+q], k[q*4+0], v1.x); 
						v2.x = mad(x[2*4+q], k[q*4+0], v2.x); 
						v3.x = mad(x[3*4+q], k[q*4+0], v3.x); 
						v0.y = mad(x[0*4+q], k[q*4+1], v0.y); //--
						v1.y = mad(x[1*4+q], k[q*4+1], v1.y);
						v2.y = mad(x[2*4+q], k[q*4+1], v2.y);
						v3.y = mad(x[3*4+q], k[q*4+1], v3.y);
						v0.z = mad(x[0*4+q], k[q*4+2], v0.z); //--
						v1.z = mad(x[1*4+q], k[q*4+2], v1.z); 
						v2.z = mad(x[2*4+q], k[q*4+2], v2.z); 
						v3.z = mad(x[3*4+q], k[q*4+2], v3.z); 
						v0.w = mad(x[0*4+q], k[q*4+3], v0.w); //--
						v1.w = mad(x[1*4+q], k[q*4+3], v1.w);
						v2.w = mad(x[2*4+q], k[q*4+3], v2.w);
						v3.w = mad(x[3*4+q], k[q*4+3], v3.w);
					}
					
					#elif 1  // ----------------------------------------------------------

					// 111ms
					// dcl_temps 34
					[unroll]
					for (uint regs = 0; regs < 16; ++regs)
					{
						x[regs] = X_[regs][gy*CTILE+i];
						k[regs] = K_[regs][i*CTILE+gx];
					}
					v0.x = mad(x[0*4+0], k[0*4+0], v0.x); //--
					v1.x = mad(x[1*4+0], k[0*4+0], v1.x); 
					v2.x = mad(x[2*4+0], k[0*4+0], v2.x); 
					v3.x = mad(x[3*4+0], k[0*4+0], v3.x); 
					v0.y = mad(x[0*4+0], k[0*4+1], v0.y); //--
					v1.y = mad(x[1*4+0], k[0*4+1], v1.y);
					v2.y = mad(x[2*4+0], k[0*4+1], v2.y);
					v3.y = mad(x[3*4+0], k[0*4+1], v3.y);
					v0.z = mad(x[0*4+0], k[0*4+2], v0.z); //--
					v1.z = mad(x[1*4+0], k[0*4+2], v1.z); 
					v2.z = mad(x[2*4+0], k[0*4+2], v2.z); 
					v3.z = mad(x[3*4+0], k[0*4+2], v3.z); 
					v0.w = mad(x[0*4+0], k[0*4+3], v0.w); //--
					v1.w = mad(x[1*4+0], k[0*4+3], v1.w);
					v2.w = mad(x[2*4+0], k[0*4+3], v2.w);
					v3.w = mad(x[3*4+0], k[0*4+3], v3.w);
					
					v0.x = mad(x[0*4+1], k[1*4+0], v0.x); //--
					v1.x = mad(x[1*4+1], k[1*4+0], v1.x); 
					v2.x = mad(x[2*4+1], k[1*4+0], v2.x); 
					v3.x = mad(x[3*4+1], k[1*4+0], v3.x); 
					v0.y = mad(x[0*4+1], k[1*4+1], v0.y); //--
					v1.y = mad(x[1*4+1], k[1*4+1], v1.y);
					v2.y = mad(x[2*4+1], k[1*4+1], v2.y);
					v3.y = mad(x[3*4+1], k[1*4+1], v3.y);
					v0.z = mad(x[0*4+1], k[1*4+2], v0.z); //--
					v1.z = mad(x[1*4+1], k[1*4+2], v1.z); 
					v2.z = mad(x[2*4+1], k[1*4+2], v2.z); 
					v3.z = mad(x[3*4+1], k[1*4+2], v3.z); 
					v0.w = mad(x[0*4+1], k[1*4+3], v0.w); //--
					v1.w = mad(x[1*4+1], k[1*4+3], v1.w);
					v2.w = mad(x[2*4+1], k[1*4+3], v2.w);
					v3.w = mad(x[3*4+1], k[1*4+3], v3.w);
					
					v0.x = mad(x[0*4+2], k[2*4+0], v0.x); //--
					v1.x = mad(x[1*4+2], k[2*4+0], v1.x); 
					v2.x = mad(x[2*4+2], k[2*4+0], v2.x); 
					v3.x = mad(x[3*4+2], k[2*4+0], v3.x); 
					v0.y = mad(x[0*4+2], k[2*4+1], v0.y); //--
					v1.y = mad(x[1*4+2], k[2*4+1], v1.y);
					v2.y = mad(x[2*4+2], k[2*4+1], v2.y);
					v3.y = mad(x[3*4+2], k[2*4+1], v3.y);
					v0.z = mad(x[0*4+2], k[2*4+2], v0.z); //--
					v1.z = mad(x[1*4+2], k[2*4+2], v1.z); 
					v2.z = mad(x[2*4+2], k[2*4+2], v2.z); 
					v3.z = mad(x[3*4+2], k[2*4+2], v3.z); 
					v0.w = mad(x[0*4+2], k[2*4+3], v0.w); //--
					v1.w = mad(x[1*4+2], k[2*4+3], v1.w);
					v2.w = mad(x[2*4+2], k[2*4+3], v2.w);
					v3.w = mad(x[3*4+2], k[2*4+3], v3.w);
					
					v0.x = mad(x[0*4+3], k[3*4+0], v0.x); //--
					v1.x = mad(x[1*4+3], k[3*4+0], v1.x); 
					v2.x = mad(x[2*4+3], k[3*4+0], v2.x); 
					v3.x = mad(x[3*4+3], k[3*4+0], v3.x); 
					v0.y = mad(x[0*4+3], k[3*4+1], v0.y); //--
					v1.y = mad(x[1*4+3], k[3*4+1], v1.y);
					v2.y = mad(x[2*4+3], k[3*4+1], v2.y);
					v3.y = mad(x[3*4+3], k[3*4+1], v3.y);
					v0.z = mad(x[0*4+3], k[3*4+2], v0.z); //--
					v1.z = mad(x[1*4+3], k[3*4+2], v1.z); 
					v2.z = mad(x[2*4+3], k[3*4+2], v2.z); 
					v3.z = mad(x[3*4+3], k[3*4+2], v3.z); 
					v0.w = mad(x[0*4+3], k[3*4+3], v0.w); //--
					v1.w = mad(x[1*4+3], k[3*4+3], v1.w);
					v2.w = mad(x[2*4+3], k[3*4+3], v2.w);
					v3.w = mad(x[3*4+3], k[3*4+3], v3.w);
					
					#else  // ----------------------------------------------------------

					// 115 ms, reg dependencies
					// dcl_temps 32
					[unroll]
					for (uint regs = 0; regs < 16; ++regs)
					{
						x[regs] = X_[regs][gy*CTILE+i];
						k[regs] = K_[regs][i*CTILE+gx];
					}

					v0.x = mad(x[0*4+0], k[0*4+0], v0.x); //--
					v0.x = mad(x[0*4+1], k[1*4+0], v0.x);
					v0.x = mad(x[0*4+2], k[2*4+0], v0.x);
					v0.x = mad(x[0*4+3], k[3*4+0], v0.x);
					v0.y = mad(x[0*4+0], k[0*4+1], v0.y); //--
					v0.y = mad(x[0*4+1], k[1*4+1], v0.y);
					v0.y = mad(x[0*4+2], k[2*4+1], v0.y);
					v0.y = mad(x[0*4+3], k[3*4+1], v0.y);
					v0.z = mad(x[0*4+0], k[0*4+2], v0.z); //--
					v0.z = mad(x[0*4+1], k[1*4+2], v0.z);
					v0.z = mad(x[0*4+2], k[2*4+2], v0.z);
					v0.z = mad(x[0*4+3], k[3*4+2], v0.z);
					v0.w = mad(x[0*4+0], k[0*4+3], v0.w); //--
					v0.w = mad(x[0*4+1], k[1*4+3], v0.w);
					v0.w = mad(x[0*4+2], k[2*4+3], v0.w);
					v0.w = mad(x[0*4+3], k[3*4+3], v0.w);
					
					v1.x = mad(x[1*4+0], k[0*4+0], v1.x); //--
					v1.x = mad(x[1*4+1], k[1*4+0], v1.x);
					v1.x = mad(x[1*4+2], k[2*4+0], v1.x);
					v1.x = mad(x[1*4+3], k[3*4+0], v1.x); 
					v1.y = mad(x[1*4+0], k[0*4+1], v1.y); //--
					v1.y = mad(x[1*4+1], k[1*4+1], v1.y);
					v1.y = mad(x[1*4+2], k[2*4+1], v1.y);
					v1.y = mad(x[1*4+3], k[3*4+1], v1.y);
					v1.z = mad(x[1*4+0], k[0*4+2], v1.z); //--
					v1.z = mad(x[1*4+1], k[1*4+2], v1.z);
					v1.z = mad(x[1*4+2], k[2*4+2], v1.z);
					v1.z = mad(x[1*4+3], k[3*4+2], v1.z);
					v1.w = mad(x[1*4+0], k[0*4+3], v1.w); //--
					v1.w = mad(x[1*4+1], k[1*4+3], v1.w);
					v1.w = mad(x[1*4+2], k[2*4+3], v1.w);
					v1.w = mad(x[1*4+3], k[3*4+3], v1.w);
					
					v2.x = mad(x[2*4+0], k[0*4+0], v2.x); //--
					v2.x = mad(x[2*4+1], k[1*4+0], v2.x);
					v2.x = mad(x[2*4+2], k[2*4+0], v2.x);
					v2.x = mad(x[2*4+3], k[3*4+0], v2.x);
					v2.y = mad(x[2*4+0], k[0*4+1], v2.y); //--
					v2.y = mad(x[2*4+1], k[1*4+1], v2.y);
					v2.y = mad(x[2*4+2], k[2*4+1], v2.y);
					v2.y = mad(x[2*4+3], k[3*4+1], v2.y);
					v2.z = mad(x[2*4+0], k[0*4+2], v2.z); //--
					v2.z = mad(x[2*4+1], k[1*4+2], v2.z);
					v2.z = mad(x[2*4+2], k[2*4+2], v2.z);
					v2.z = mad(x[2*4+3], k[3*4+2], v2.z);
					v2.w = mad(x[2*4+0], k[0*4+3], v2.w); //--
					v2.w = mad(x[2*4+1], k[1*4+3], v2.w);
					v2.w = mad(x[2*4+2], k[2*4+3], v2.w);
					v2.w = mad(x[2*4+3], k[3*4+3], v2.w);
					
					v3.x = mad(x[3*4+0], k[0*4+0], v3.x); //--
					v3.x = mad(x[3*4+1], k[1*4+0], v3.x);
					v3.x = mad(x[3*4+2], k[2*4+0], v3.x);
					v3.x = mad(x[3*4+3], k[3*4+0], v3.x); 
					v3.y = mad(x[3*4+0], k[0*4+1], v3.y); //--
					v3.y = mad(x[3*4+1], k[1*4+1], v3.y);
					v3.y = mad(x[3*4+2], k[2*4+1], v3.y);
					v3.y = mad(x[3*4+3], k[3*4+1], v3.y);
					v3.z = mad(x[3*4+0], k[0*4+2], v3.z); //--
					v3.z = mad(x[3*4+1], k[1*4+2], v3.z);
					v3.z = mad(x[3*4+2], k[2*4+2], v3.z);
					v3.z = mad(x[3*4+3], k[3*4+2], v3.z);
					v3.w = mad(x[3*4+0], k[0*4+3], v3.w); //--
					v3.w = mad(x[3*4+1], k[1*4+3], v3.w);
					v3.w = mad(x[3*4+2], k[2*4+3], v3.w);
					v3.w = mad(x[3*4+3], k[3*4+3], v3.w);

					#endif  // ----------------------------------------------------------
				}

				GroupMemoryBarrierWithGroupSync();
			}
		}
	}
	
	#if 1 // ----------------------------------------------------------

	// 117ms
	O.Set(n*4+0, y, x, k*4+0, v0.x, Odata);
	O.Set(n*4+0, y, x, k*4+1, v0.y, Odata);
	O.Set(n*4+0, y, x, k*4+2, v0.z, Odata);
	O.Set(n*4+0, y, x, k*4+3, v0.w, Odata);
	
	O.Set(n*4+1, y, x, k*4+0, v1.x, Odata);
	O.Set(n*4+1, y, x, k*4+1, v1.y, Odata);
	O.Set(n*4+1, y, x, k*4+2, v1.z, Odata);
	O.Set(n*4+1, y, x, k*4+3, v1.w, Odata);
	
	O.Set(n*4+2, y, x, k*4+0, v2.x, Odata);
	O.Set(n*4+2, y, x, k*4+1, v2.y, Odata);
	O.Set(n*4+2, y, x, k*4+2, v2.z, Odata);
	O.Set(n*4+2, y, x, k*4+3, v2.w, Odata);
	
	O.Set(n*4+3, y, x, k*4+0, v3.x, Odata);
	O.Set(n*4+3, y, x, k*4+1, v3.y, Odata);
	O.Set(n*4+3, y, x, k*4+2, v3.z, Odata);
	O.Set(n*4+3, y, x, k*4+3, v3.w, Odata);

	#else // ----------------------------------------------------------
		
	// 118ms
	O.Set(n*4+0, y, x, k*4+0, v0.x, Odata);
	O.Set(n*4+1, y, x, k*4+0, v1.x, Odata);
	O.Set(n*4+2, y, x, k*4+0, v2.x, Odata);
	O.Set(n*4+3, y, x, k*4+0, v3.x, Odata);
	
	O.Set(n*4+0, y, x, k*4+1, v0.y, Odata);
	O.Set(n*4+1, y, x, k*4+1, v1.y, Odata);
	O.Set(n*4+2, y, x, k*4+1, v2.y, Odata);
	O.Set(n*4+3, y, x, k*4+1, v3.y, Odata);
	
	O.Set(n*4+0, y, x, k*4+2, v0.z, Odata);
	O.Set(n*4+1, y, x, k*4+2, v1.z, Odata);
	O.Set(n*4+2, y, x, k*4+2, v2.z, Odata);
	O.Set(n*4+3, y, x, k*4+2, v3.z, Odata);
	
	O.Set(n*4+0, y, x, k*4+3, v0.w, Odata);
	O.Set(n*4+1, y, x, k*4+3, v1.w, Odata);
	O.Set(n*4+2, y, x, k*4+3, v2.w, Odata);
	O.Set(n*4+3, y, x, k*4+3, v3.w, Odata);

	#endif // ----------------------------------------------------------

			  
	#undef X_
	#undef K_
}

#else // =====================================================================================================

#undef CTILE
#define CTILE 16
#define RTILE 4
groupshared float Conv_XcacheR[RTILE*RTILE][CTILE*CTILE];
groupshared float Conv_KcacheR[RTILE*RTILE][CTILE*CTILE];
[numthreads(CTILE, CTILE, 1)]
void Conv2D_Kernel3x3_Cache_KCmod64_KNyx(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ Conv_XcacheR
	#define K_ Conv_KcacheR

	uint gx = groupThreadID.x;
	uint gy = groupThreadID.y;

	uint k = CTILE * groupID.x + groupThreadID.x;
	uint nyx = CTILE * groupID.y + groupThreadID.y;

	uint width = X.width - _Border;
	uint height = X.height - _Border;

	uint x = nyx % width;
	uint ny = nyx / width;
	uint y = ny % height;
	uint n = ny / height;
	
	float v[RTILE*RTILE];
	for (uint xxxx = 0; xxxx < RTILE; ++xxxx)
	{
		float b = B.Get(0, 0, k*RTILE+xxxx, 0, WBKdata, WBK.dataLength);
		for (uint yyyy = 0; yyyy < RTILE; ++yyyy)
			v[yyyy*RTILE+xxxx] = b;
	}

	for (uint dy = 0; dy < 3; ++dy)
	{
		bool mask = true;

		if (y+dy < _Offset) mask = false;
		if (y+dy-_Offset >= X.height) mask = false;

		for (uint dx = 0; dx < 3; ++dx)
		{
			if (x+dx < _Offset) mask = false;
			if (x+dx-_Offset >= X.width) mask = false;

			for (uint m = 0; m < X.channels/(CTILE*RTILE); ++m)
			{				
			
				for (uint yy = 0; yy < RTILE; ++yy)
					for (uint xx = 0; xx < RTILE; ++xx)
					{
						if (mask)
							X_[yy*RTILE+xx][gy*CTILE+gx] = X.Get(n*RTILE+yy, y+dy-_Offset, x+dx-_Offset, (m*CTILE + gx)*RTILE+xx, Xdata);
						else
							X_[yy*RTILE+xx][gy*CTILE+gx] = 0;
						K_[yy*RTILE+xx][gy*CTILE+gx] = K.Get(dy, dx, (m*CTILE + gy)*RTILE+yy, k*RTILE+xx, WBKdata, WBK.dataLength);
					}

				GroupMemoryBarrierWithGroupSync();

				for (uint ii = 0; ii < CTILE; ++ii)
				{
					float x[RTILE*RTILE];
					float k[RTILE*RTILE];
					
					[unroll]
					for (uint iii = 0; iii < RTILE*RTILE; ++iii)
					{
						x[iii] = X_[iii][gy*CTILE+ii];
						k[iii] = K_[iii][ii*CTILE+gx];
					}

					[unroll]
					for (uint r = 0; r < RTILE*RTILE; ++r)
					{
						[unroll]
						for (uint i = 0; i < RTILE; ++i)
						{
							uint xxx = r % RTILE;
							v[r] = mad(x[r], k[i*RTILE+xxx], v[r]);

							//v[yyy][xxx] += x[yyy][i] * k[i][xxx];
						}
					}

				}

				GroupMemoryBarrierWithGroupSync();
			}
		}
	}

	for (uint yy = 0; yy < RTILE; ++yy)
		for (uint xx = 0; xx < RTILE; ++xx)
			O.Set(n*RTILE+yy, y, x, k*RTILE+xx, v[yy*RTILE+xx], Odata);
	
	#undef X_
	#undef K_
}
#endif

[numthreads(CTILE, CTILE, 1)]
void Conv2D_Kernel3x3_Cache_KCmod16_KNyx_TEMPLATE(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	#define X_ Conv_XcacheT
	#define K_ Conv_KcacheT

	uint gx = groupThreadID.x;
	uint gy = groupThreadID.y;

	uint k = CTILE * groupID.x + groupThreadID.x;
	uint nyx = CTILE * groupID.y + groupThreadID.y;

	uint width = X.width - _Border;
	uint height = X.height - _Border;

	uint x = nyx % width;
	uint ny = nyx / width;
	uint y = ny % height;
	uint n = ny / height;
	
	float v = B.Get(0, 0, k, 0, WBKdata, WBK.dataLength);
	for (uint dy = 0; dy < 3; ++dy)
	{
		bool mask = true;

		if (y+dy < _Offset) mask = false;
		if (y+dy-_Offset >= X.height) mask = false;

		for (uint dx = 0; dx < 3; ++dx)
		{
			if (x+dx < _Offset) mask = false;
			if (x+dx-_Offset >= X.width) mask = false;

			//for (uint m = 0; m < (9*128)/CTILE; ++m)
			for (uint m = 0; m < X.channels/CTILE; ++m)
			{
				if (mask)
					X_[gy][gx] = X.Get(n, y+dy-_Offset, x+dx-_Offset, m*CTILE + gx, Xdata);
				else
					X_[gy][gx] = 0;
				K_[gy][gx] = K.Get(dy, dx, m*CTILE + gy, k, WBKdata, WBK.dataLength);
				GroupMemoryBarrierWithGroupSync();

				[unroll]
				for (uint i = 0; i < CTILE; ++i)
				{
					float x = X_[gy][i];
					float k =.25;// K_[i][gx];
					v += x * k;
				}
			}
		}
	}

	//Odata[nyx * O.channels + k] = v;
	
	Odata[((
		n   * O.height +
		y ) * O.width +
		x ) * O.channels +
		k] = v;
	
	#undef X_
	#undef K_
}
// %TODO: only supports up to 51 kernels (51 = 16*16*2/(9kernel+1bias)) for now. Add a loop to handle more!
/*
groupshared float K1cache[KERNEL_SIZE][KERNEL_SIZE][32];
groupshared float B1cache[32];
[numthreads(16,16,2)]
void Conv2D_Kernel3x3_1Channel(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint k = 16*groupID.x + groupThreadID.x;
	uint n = 16*groupID.y + groupThreadID.y;
	uint y = 2*groupID.z + groupThreadID.z + _FilterSize;

	uint idx = 16*16*groupThreadID.z + 16*groupThreadID.y + groupThreadID.x;
	if (idx < 9 * K.channels)
	{
		uint kx = idx / K.channels;
		uint kk = idx % K.channels;
		K1cache[kx/3][kx%3][kk] = K.Get(kx/3, kx%3, 0, kk, WBKdata, WBK.dataLength);
	}
	else if (idx < 10 * K.channels)
	{
		uint kk = idx % K.channels;
		B1cache[kk] = B.Get(0, 0, kk, 0, WBKdata, WBK.dataLength);
	}
	GroupMemoryBarrierWithGroupSync();

	for (uint x = _FilterSize; x < X.width - _FilterSize; ++x)
	{			
		float v = B1cache[k];//B.Get(0, 0, k, 0, WBKdata, WBK.dataLength);
		for (int i = -_FilterSize; i < _FilterSize + 1; ++i)
		{
			for (int j = -_FilterSize; j < _FilterSize + 1; ++j)
			{
				v += X.Get(n, y+j, x+i, 0, Xdata, X.dataLength) * K1cache[_FilterSize+j][_FilterSize+i][k];
			}
		}
		O.Set(n, y-_FilterSize, x-_FilterSize, k, v, Odata, O.dataLength);
	}
}
*/

groupshared float K1cache[32][9];
[numthreads(32,16,1)]
void Conv2D_Kernel3x3_1Channel(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	uint tk = groupThreadID.x;
	uint k = 32*groupID.x + tk;
	uint n = 16*groupID.y + groupThreadID.y;

	//for (uint q = 0; q < 9; ++q)
	{
		uint q = n % 9;
		K1cache[tk][q] = K.Get(q/3, q%3, 0, k, WBKdata, WBK.dataLength);
	}
	GroupMemoryBarrierWithGroupSync();

	for (uint y = 0; y < X.height - _FilterSize*2; ++y)
	{
		for (uint x = 0; x < X.width - _FilterSize*2; ++x)
		{
			float v = B.Get(0, 0, k, 0, WBKdata, WBK.dataLength);
			//for (uint q = 0; q < 9; ++q)
			//	v += X.Get(n, y+q/3, x+q%3, 0, Xdata, X.dataLength) * K1cache[tk][q];
			v += X.Get(n, y+0, x+0, 0, Xdata, X.dataLength) * K1cache[tk][0];
			v += X.Get(n, y+0, x+1, 0, Xdata, X.dataLength) * K1cache[tk][1];
			v += X.Get(n, y+0, x+2, 0, Xdata, X.dataLength) * K1cache[tk][2];

			v += X.Get(n, y+1, x+0, 0, Xdata, X.dataLength) * K1cache[tk][3];
			v += X.Get(n, y+1, x+1, 0, Xdata, X.dataLength) * K1cache[tk][4];
			v += X.Get(n, y+1, x+2, 0, Xdata, X.dataLength) * K1cache[tk][5];

			v += X.Get(n, y+2, x+0, 0, Xdata, X.dataLength) * K1cache[tk][6];
			v += X.Get(n, y+2, x+1, 0, Xdata, X.dataLength) * K1cache[tk][7];
			v += X.Get(n, y+2, x+2, 0, Xdata, X.dataLength) * K1cache[tk][8];

			O.Set(n, y, x, k, v, Odata, O.dataLength);
		}
	}
}

float fillValue;

[numthreads(1,1,1)]
void Fill(uint3 groupID : SV_GroupID)
{
	uint b = groupID.x;
	uint h = groupID.y;
	uint w = groupID.z;
	for (uint ch = 0; ch < O.channels; ++ch)
		O.Set(b, h, w, ch+1, fillValue, Odata, O.dataLength);
}
#endif


/*
Cbufferconsts{
	uint n;
	uint dispatchDim_x;};
#define groupDim_x 512
groupshared float Accumulate_sharedMem[groupDim_x * channels];
[numthreads(groupDim_x, 1, 1)]
void Accumulate(uint tid: SV_GroupIndex, uint3 groupIdx: groupID)
{
	#define sharedMem Reduce_sharedMem
	unsigned int i = groupIdx.x * (groupDim_x * 2) + tid;
	unsigned int dispatchSize = (groupDim_x * 2) * dispatchDim_x;
	sharedMem[tid] = 0;
	do {
		sharedMem[tid] += g_idata[i] + g_idata[i+groupDim_x];
		i += dispatchSize;
	} while (i < n);
	GroupMemoryBarrierWithGroupSync();

	if (groupDim_x >= 256)
	{
		if (tid < 128) { sharedMem[tid] += sharedMem[tid + 128 * channels]; }
		GroupMemoryBarrierWithGroupSync();
	}

	if (groupDim_x >= 128)
	{
		if (tid <  64) { sharedMem[tid] += sharedMem[tid +  64]; }
		GroupMemoryBarrierWithGroupSync();
	}

	if (tid < 32)
	{
		if (groupDim_x >= 64) sharedMem[tid] += sharedMem[tid + 32* channels];
		if (groupDim_x >= 32) sharedMem[tid] += sharedMem[tid + 16* channels];
		if (groupDim_x >= 16) sharedMem[tid] += sharedMem[tid +  8* channels];
		if (groupDim_x >=  8) sharedMem[tid] += sharedMem[tid +  4* channels];
		if (groupDim_x >=  4) sharedMem[tid] += sharedMem[tid +  2* channels];
		if (groupDim_x >=  2) sharedMem[tid] += sharedMem[tid +  1* channels];
	}

	if (tid == 0) g_odata[groupIdx.x] = sharedMem[0];

	#undef sharedMem
}
*/
	/*
// Could do to reduce across NxN patch fitting within a group, HW <= HW / N
// Repeat, until HW == 1

// Alternatively reduce across Y axis, then X

#undef MAX_CHANNELS
#define MAX_CHANNELS 2048
groupshared float GlobalAvgPool2D_AccumulatorPerChannel[MAX_CHANNELS];
[numthreads(4,8,8)]
void GlobalAvgPool2D(uint3 dispatchThreadID : SV_DispatchThreadID, uint threadID : SV_ThreadID)
{
	// NOTE: dispatched over X (not O)
	DISPATCH_ARGS(X.channels, X.width, X.height);
	TENSOR_ARGS2(X, O);

	uint c = dispatchThreadID.x;
	uint x = dispatchThreadID.y;
	uint y = dispatchThreadID.z;

	if (c >= X.channels || c >= MAX_CHANNELS) return;
	if (x >= X.width) return;
	if (y >= X.height) return;

	// Accumulate
	for (uint n = 0; n < X.batch; ++n)
	{
		// Clear accumulator
		// @TODO: ThreadID
		//uint threadID = groupThreadID.x * 4 + groupThreadID.y * 8 + groupThreadID.z * 8;
		if (threadID < MAX_CHANNELS)
			GlobalAvgPool2D_AccumulatorPerChannel[threadID] = 0;
		GroupMemoryBarrierWithGroupSync();

		GlobalAvgPool2D_AccumulatorPerChannel[c] += X.Get(n, y, x, c);
		// @TODO: atomicAdd?

		GroupMemoryBarrierWithGroupSync();
		if (threadID < MAX_CHANNELS)
		{
			float v = GlobalAvgPool2D_AccumulatorPerChannel[threadID];
			O.Set(n, 0, 0, c, v / (X.width * X.height));
		}
	}
}*/


[numthreads(64,2,2)]
void Conv2D_Reg2x2(uint3 dispatchThreadID : SV_DispatchThreadID)
{
	DISPATCH_ARGS(K.kernelCount, O.width, O.height);
	TENSOR_SHARED2_ARGS4(X, K, B, WBK, O);

	uint k = dispatchThreadID.x;
	uint x = dispatchThreadID.y;
	uint y = dispatchThreadID.z;

	if (k >= K.channels) return;
	if (x*2 >= O.width) return;
	if (y*2 >= O.height) return;

	uint2 leftCorner = _Pad.xy;
	uint2 rightCorner = uint2(X.width, X.height) + _Pad.xy;
	for (uint n = 0; n < O.batch; ++n)
	{
		float4 acc = B.Get(k);
		for (uint dy = 0; dy < K.GetKernelHeight(); ++dy)
		{
			for (uint dx = 0; dx < K.GetKernelWidth(); ++dx)
			{
				uint2 pos0 = uint2(x*2+0, y*2+0) * _Stride.xy + uint2(dx, dy);
				uint2 pos1 = uint2(x*2+1, y*2+0) * _Stride.xy + uint2(dx, dy);
				uint2 pos2 = uint2(x*2+0, y*2+1) * _Stride.xy + uint2(dx, dy);
				uint2 pos3 = uint2(x*2+1, y*2+1) * _Stride.xy + uint2(dx, dy);

				for (uint c = 0; c < X.channels; ++c)
				{
					if (all(pos0 >= leftCorner) && all(pos0 < rightCorner))
						acc.x = fastfma(X.Get(n, pos0 - leftCorner, c), K.Get(dy, dx, c, k), acc.x);
					if (all(pos1 >= leftCorner) && all(pos1 < rightCorner))
						acc.y = fastfma(X.Get(n, pos1 - leftCorner, c), K.Get(dy, dx, c, k), acc.y);
					if (all(pos2 >= leftCorner) && all(pos2 < rightCorner))
						acc.z = fastfma(X.Get(n, pos2 - leftCorner, c), K.Get(dy, dx, c, k), acc.z);
					if (all(pos3 >= leftCorner) && all(pos3 < rightCorner))
						acc.w = fastfma(X.Get(n, pos3 - leftCorner, c), K.Get(dy, dx, c, k), acc.w);
				}
			}
		}

		O.Set(n, y*2+0, x*2+0, k, acc.x);
		O.Set(n, y*2+0, x*2+1, k, acc.y);
		O.Set(n, y*2+1, x*2+0, k, acc.z);
		O.Set(n, y*2+1, x*2+1, k, acc.w);
	}
}

#define SIZE 2
[numthreads(64, 2, 2)]
void Conv2D_Reg_Loop(uint3 dispatchThreadID : SV_DispatchThreadID)
{
	DISPATCH_ARGS(K.kernelCount, O.width, O.height);
	TENSOR_SHARED2_ARGS4(X, K, B, WBK, O);

	uint k = dispatchThreadID.x;
	uint x = dispatchThreadID.y;
	uint y = dispatchThreadID.z;

	if (k >= K.channels) return;
	if (x*SIZE >= O.width) return;
	if (y*SIZE >= O.height) return;

	uint2 leftCorner = _Pad.xy;
	uint2 rightCorner = uint2(X.width, X.height) + _Pad.xy;
	for (uint n = 0; n < O.batch; ++n)
	{
		float acc[SIZE*SIZE];
		[unroll]
		for (uint q = 0; q < SIZE*SIZE; ++q)
			acc[q] = B.Get(k);
		for (uint dy = 0; dy < K.GetKernelHeight(); ++dy)
		{
			for (uint dx = 0; dx < K.GetKernelWidth(); ++dx)
			{
				uint2 pos[SIZE*SIZE];
				[unroll]
				for (uint q = 0; q < SIZE*SIZE; ++q)
					pos[q] = uint2(x*SIZE+(q%SIZE), y*SIZE+(q/SIZE)) * _Stride.xy + uint2(dx, dy);

				// @TODO: investigate
				// WARNING: had to move both y check into the loop (as opposed to checking y in parent loop) - due to potential bug in Metal compiler

				for (uint c = 0; c < X.channels; ++c)
					[unroll]
					for (q = 0; q < SIZE*SIZE; ++q)
						if (all(pos[q] >= leftCorner) && all(pos[q] < rightCorner))
							acc[q] = fastfma(X.Get(n, pos[q] - leftCorner, c), K.Get(dy, dx, c, k), acc[q]);
			}
		}

		[unroll]
		for (q = 0; q < SIZE*SIZE; ++q)
			O.Set(n, y*SIZE+(q/SIZE), x*SIZE+(q%SIZE), k, acc[q]);
	}
}

NUMTHREADS((16,4,4), (8,4,4), (16,2,2))
//[numthreads(64, 1, 1)]
void Conv2D_safe(uint3 dispatchThreadID : SV_DispatchThreadID)
{
	DISPATCH_ARGS(K.kernelCount, O.width, O.height);
	TENSOR_SHARED2_ARGS4(X, K, B, WBK, O);

	uint k = dispatchThreadID.x;
	uint x = dispatchThreadID.y;
	uint y = dispatchThreadID.z;

	if (k >= K.channels) return;
	if (x >= O.width) return;
	if (y >= O.height) return;

	for (uint n = 0; n < O.batch; ++n)
	{
		float acc = B.Get(k);
		for (uint dy = 0; dy < K.GetKernelHeight(); ++dy)
		{
			for (uint dx = 0; dx < K.GetKernelWidth(); ++dx)
			{
				uint2 pos = uint2(x, y) * _Stride.xy + uint2(dx, dy);

				for (uint c = 0; c < X.channels; ++c)
					acc = fastfma(X.SafeGet(n, pos, c, _Pad.xy), K.Get(dy, dx, c, k), acc);
			}
		}

		O.Set(n, y, x, k, acc);
	}
}


#undef L1CACHESIZE
#define L1CACHESIZE 32
groupshared float Conv2D_L1Cached32_X[L1CACHESIZE];
[numthreads(L1CACHESIZE, 1, 1)]
void Conv2D_L1Cached32(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	DISPATCH_ARGS(K.kernelCount, O.width, O.height);
	TENSOR_SHARED2_ARGS4(X, K, B, WBK, O);

	#define X_ Conv2D_L1Cached32_X

	uint k = L1CACHESIZE * groupID.x + groupThreadID.x;
	uint x = groupID.y;
	uint y = groupID.z;

	if (x >= O.width) return;
	if (y >= O.height) return;

	for (uint n = 0; n < O.batch; ++n)
	{
		float acc = B.SafeGet(k);
		for (uint dy = 0; dy < K.GetKernelHeight(); ++dy)
		{
			for (uint dx = 0; dx < K.GetKernelWidth(); ++dx)
			{
				uint2 pos = uint2(x,y) * _Stride.xy + uint2(dx,dy);

				for (uint c = 0; c < X.channels; c += L1CACHESIZE)
				{
					// Cache X
					X_[groupThreadID.x] = X.SafeGet(n, pos, c + groupThreadID.x, _Pad.xy);
					GroupMemoryBarrierWithGroupSync();

					// X * K
					if (k < K.channels)
					{
						for (uint dc = 0; dc < L1CACHESIZE; ++dc)
							acc = fastfma(X_[dc], K.Get(dy, dx, c + dc, k), acc);
					}
					GroupMemoryBarrierWithGroupSync();
				}
			}
		}

		O.Set(n, y, x, k, acc);
	}

	#undef X_
}

#undef L1CACHESIZE
#define L1CACHESIZE 64
groupshared float Conv2D_L1Cached64_X[L1CACHESIZE];
[numthreads(L1CACHESIZE, 1, 1)]
void Conv2D_L1Cached64(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	DISPATCH_ARGS(K.kernelCount, O.width, O.height);
	TENSOR_SHARED2_ARGS4(X, K, B, WBK, O);

	#define X_ Conv2D_L1Cached64_X

	uint k = L1CACHESIZE * groupID.x + groupThreadID.x;
	uint x = groupID.y;
	uint y = groupID.z;

	if (x >= O.width) return;
	if (y >= O.height) return;

	for (uint n = 0; n < O.batch; ++n)
	{
		float acc = B.SafeGet(k);
		for (uint dy = 0; dy < K.GetKernelHeight(); ++dy)
		{
			for (uint dx = 0; dx < K.GetKernelWidth(); ++dx)
			{
				uint2 pos = uint2(x,y) * _Stride.xy + uint2(dx,dy);
				for (uint c = 0; c < X.channels; c += L1CACHESIZE)
				{
					// Cache X
					X_[groupThreadID.x] = X.SafeGet(n, pos, c + groupThreadID.x, _Pad.xy);
					GroupMemoryBarrierWithGroupSync();

					// X * K
					if (k < K.channels)
					{
						for (uint dc = 0; dc < L1CACHESIZE; ++dc)
							acc = fastfma(X_[dc], K.Get(dy, dx, c + dc, k), acc);
					}
					GroupMemoryBarrierWithGroupSync();
				}
			}
		}

		O.Set(n, y, x, k, acc);
	}

	#undef X_
}


#undef SIZE
#define SIZE 2
[numthreads(64, 2, 2)]
void Conv2D_Reg_Loop_safe(uint3 dispatchThreadID : SV_DispatchThreadID)
{
	DISPATCH_ARGS(K.kernelCount, O.width, O.height);
	TENSOR_SHARED2_ARGS4(X, K, B, WBK, O);

	uint k = dispatchThreadID.x;
	uint x = dispatchThreadID.y;
	uint y = dispatchThreadID.z;

	if (k >= K.channels) return;
	if (x*SIZE >= O.width) return;
	if (y*SIZE >= O.height) return;

	uint2 leftCorner = _Pad.xy;
	uint2 rightCorner = uint2(X.width, X.height) + _Pad.xy;
	for (uint n = 0; n < O.batch; ++n)
	{
		float acc[SIZE*SIZE];
		[unroll]
		for (uint q = 0; q < SIZE*SIZE; ++q)
			acc[q] = B.Get(k);
		for (uint dy = 0; dy < K.GetKernelHeight(); ++dy)
		{
			for (uint dx = 0; dx < K.GetKernelWidth(); ++dx)
			{
				uint2 pos[SIZE*SIZE];
				[unroll]
				for (uint q = 0; q < SIZE*SIZE; ++q)
					pos[q] = uint2(x*SIZE+(q%SIZE), y*SIZE+(q/SIZE)) * _Stride.xy + uint2(dx, dy);

				// @TODO: investigate
				// WARNING: had to move both y check into the loop (as opposed to checking y in parent loop) - due to potential bug in Metal compiler

				for (uint c = 0; c < X.channels; ++c)
					[unroll]
					for (q = 0; q < SIZE*SIZE; ++q)
						acc[q] = fastfma(X.SafeGet(n, pos[q], c, _Pad.xy), K.Get(dy, dx, c, k), acc[q]);
			}
		}

		[unroll]
		for (q = 0; q < SIZE*SIZE; ++q)
			O.Set(n, y*SIZE+(q/SIZE), x*SIZE+(q%SIZE), k, acc[q]);
	}
}


#undef L1CACHESIZE
#define L1CACHESIZE 64
#undef SIZE
#define SIZE 2
groupshared float Conv2D_L1Cached64_Reg_Loop2x2_X[SIZE*SIZE][L1CACHESIZE];
[numthreads(L1CACHESIZE, 1, 1)]
void Conv2D_L1Cached64_Reg_Loop2x2(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	DISPATCH_ARGS(K.kernelCount, O.width, O.height);
	TENSOR_SHARED2_ARGS4(X, K, B, WBK, O);

	#define X_ Conv2D_L1Cached64_Reg_Loop2x2_X

	uint k = L1CACHESIZE * groupID.x + groupThreadID.x;
	uint x = groupID.y;
	uint y = groupID.z;

	// need all threads to load channels, thus will do late check against kernel count
	if (x*SIZE >= O.width) return;
	if (y*SIZE >= O.height) return;

	for (uint n = 0; n < O.batch; ++n)
	{
		float acc[SIZE*SIZE];
		[unroll]
		for (uint q = 0; q < SIZE*SIZE; ++q)
			acc[q] = B.SafeGet(k);

		for (uint dy = 0; dy < K.GetKernelHeight(); ++dy)
		{
			for (uint dx = 0; dx < K.GetKernelWidth(); ++dx)
			{
				uint2 pos[SIZE*SIZE];
				[unroll]
				for (uint q = 0; q < SIZE*SIZE; ++q)
					pos[q] = uint2(x*SIZE+(q%SIZE), y*SIZE+(q/SIZE)) * _Stride.xy + uint2(dx, dy);

				for (uint c = 0; c < X.channels; c += L1CACHESIZE)
				{
					// Cache X
					uint dc = groupThreadID.x;
					[unroll]
					for (q = 0; q < SIZE*SIZE; ++q)
						X_[q][dc] = X.SafeGet(n, pos[q], c + dc, _Pad.xy);
					GroupMemoryBarrierWithGroupSync();

					// X * K
					if (k < K.channels) // need all threads to load channels, thus late check against kernel count
					{
						uint kIndex = K.Index(dy, dx, c, k);
						for (dc = 0; dc < L1CACHESIZE; ++dc)
						{
							for (q = 0; q < SIZE*SIZE; ++q)
								acc[q] = fastfma(X_[q][dc], K.data[kIndex], acc[q]); //K.Get(dy, dx, c + dc, k);
							kIndex += K.channels;
						}
					}
					GroupMemoryBarrierWithGroupSync();
				}
			}
		}

		if (k < K.channels) // need all threads to load channels, thus late check against kernel count
			[unroll]
			for (q = 0; q < SIZE*SIZE; ++q)
				O.Set(n, y*SIZE+(q/SIZE), x*SIZE+(q%SIZE), k, acc[q]);
	}

	#undef X_
}


#undef L1CACHESIZE
#define L1CACHESIZE 64
#undef SIZE
#define SIZE 4
groupshared float Conv2D_L1Cached64_Reg_Loop_X[SIZE*SIZE][L1CACHESIZE];
[numthreads(L1CACHESIZE, 1, 1)]
void Conv2D_L1Cached64_Reg_Loop(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	DISPATCH_ARGS(K.kernelCount, O.width, O.height);
	TENSOR_SHARED2_ARGS4(X, K, B, WBK, O);

	#define X_ Conv2D_L1Cached64_Reg_Loop_X

	uint k = L1CACHESIZE * groupID.x + groupThreadID.x;
	uint x = groupID.y;
	uint y = groupID.z;

	// need all threads to load channels, thus will do late check against kernel count
	if (x*SIZE >= O.width) return;
	if (y*SIZE >= O.height) return;

	for (uint n = 0; n < O.batch; ++n)
	{
		float acc[SIZE*SIZE];
		[unroll]
		for (uint q = 0; q < SIZE*SIZE; ++q)
			acc[q] = B.SafeGet(k);

		for (uint dy = 0; dy < K.GetKernelHeight(); ++dy)
		{
			for (uint dx = 0; dx < K.GetKernelWidth(); ++dx)
			{
				uint2 pos[SIZE*SIZE];
				[unroll]
				for (uint q = 0; q < SIZE*SIZE; ++q)
					pos[q] = uint2(x*SIZE+(q%SIZE), y*SIZE+(q/SIZE)) * _Stride.xy + uint2(dx, dy);

				for (uint c = 0; c < X.channels; c += L1CACHESIZE)
				{
					// Cache X
					uint dc = groupThreadID.x;
					[unroll]
					for (q = 0; q < SIZE*SIZE; ++q)
						X_[q][dc] = X.SafeGet(n, pos[q], c + dc, _Pad.xy);
					GroupMemoryBarrierWithGroupSync();

					// X * K
					if (k < K.channels) // need all threads to load channels, thus late check against kernel count
					{
						uint kIndex = K.Index(dy, dx, c, k);
						for (dc = 0; dc < L1CACHESIZE; ++dc)
						{
							for (q = 0; q < SIZE*SIZE; ++q)
								acc[q] = fastfma(X_[q][dc], K.data[kIndex], acc[q]);//K.Get(dy, dx, c + dc, k);
							kIndex += K.channels;
						}
					}
					GroupMemoryBarrierWithGroupSync();
				}
			}
		}

		if (k < K.channels) // need all threads to load channels, thus late check against kernel count
			[unroll]
			for (q = 0; q < SIZE*SIZE; ++q)
				O.Set(n, y*SIZE+(q/SIZE), x*SIZE+(q%SIZE), k, acc[q]);
	}

	#undef X_
}


#undef L1CACHESIZE
#define L1CACHESIZE 64
#define SIZE_W 4
#define SIZE_H 2
groupshared float Conv2D_L1Cached64_Reg_Loop_safe__X[SIZE_H*SIZE_W][L1CACHESIZE];
[numthreads(L1CACHESIZE, 1, 1)]
void Conv2D_L1Cached64_Reg_Loop_safe_(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	DISPATCH_ARGS(K.kernelCount, O.width, O.height);
	TENSOR_SHARED2_ARGS4(X, K, B, WBK, O);

	#define X_ Conv2D_L1Cached64_Reg_Loop_safe__X

	uint k = L1CACHESIZE * groupID.x + groupThreadID.x;
	uint x = groupID.y;
	uint y = groupID.z;

	// need all threads to load channels, thus will do late check against kernel count
	if (x*SIZE_W >= O.width) return;
	if (y*SIZE_H >= O.height) return;

	for (uint n = 0; n < O.batch; ++n)
	{
		float acc[SIZE_H*SIZE_W];
		[unroll]
		for (uint q = 0; q < SIZE_H*SIZE_W; ++q)
			acc[q] = B.SafeGet(k);

		for (uint dy = 0; dy < K.GetKernelHeight(); ++dy)
		{
			for (uint dx = 0; dx < K.GetKernelWidth(); ++dx)
			{
				uint2 pos[SIZE_H*SIZE_W];
				[unroll]
				for (uint q = 0; q < SIZE_H*SIZE_W; ++q)
					pos[q] = uint2(x*SIZE_W+(q%SIZE_W), y*SIZE_H+(q/SIZE_W)) * _Stride.xy + uint2(dx, dy);

				for (uint c = 0; c < X.channels; c += L1CACHESIZE)
				{
					// Cache X
					uint dc = groupThreadID.x;
					[unroll]
					for (q = 0; q < SIZE_H*SIZE_W; ++q)
						X_[q][dc] = X.SafeGet(n, pos[q], c + dc, _Pad.xy);
					GroupMemoryBarrierWithGroupSync();

					// X * K
					if (k < K.channels) // need all threads to load channels, thus late check against kernel count
					{
						uint kIndex = K.Index(dy, dx, c, k);
						for (dc = 0; dc < L1CACHESIZE; ++dc)
						{
							[unroll]
							for (q = 0; q < SIZE_H*SIZE_W; ++q)
								acc[q] = fastfma(X_[q][dc], K.data[kIndex], acc[q]);
							kIndex += K.channels;
						}
					}
					GroupMemoryBarrierWithGroupSync();
				}
			}
		}

		if (k < K.channels) // need all threads to load channels, thus late check against kernel count
			[unroll]
			for (q = 0; q < SIZE_H*SIZE_W; ++q)
			{
				uint ox = x*SIZE_W+(q%SIZE_W);
				uint oy = y*SIZE_H+(q/SIZE_W);
				if (ox < O.width && oy < O.height)
					O.Set(n, oy, ox, k, acc[q]);
			}
	}

	#undef X_
}
#undef SIZE_H
#undef SIZE_W


/*
#undef L1CACHESIZE
#define L1CACHESIZE 32
#define SIZE_W 4
#define SIZE_H 2
groupshared float Conv2D_L1Cached64_Reg_Loop_safe__X[SIZE_H*SIZE_W][L1CACHESIZE];
[numthreads(L1CACHESIZE, SIZE_W, SIZE_H)]
void Conv2D_L1Cached64_Reg_Loop_safe_(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	DISPATCH_ARGS(K.kernelCount, O.width, O.height);
	TENSOR_SHARED2_ARGS4(X, K, B, WBK, O);

	#define X_ Conv2D_L1Cached64_Reg_Loop_safe__X

	uint k = L1CACHESIZE * groupID.x + groupThreadID.x;
	uint x = SIZE_W * groupID.y + groupThreadID.y;
	uint y = SIZE_H * groupID.z + groupThreadID.z;

	// need all threads to load channels, thus will do late check against kernel count
	//if (x*SIZE_W >= O.width) return;
	//if (y*SIZE_H >= O.height) return;

	for (uint n = 0; n < O.batch; ++n)
	{
		float acc[SIZE_H*SIZE_W];
		[unroll]
		for (uint q = 0; q < SIZE_H*SIZE_W; ++q)
			acc[q] = B.SafeGet(k);

		for (uint dy = 0; dy < K.GetKernelHeight(); ++dy)
		{
			for (uint dx = 0; dx < K.GetKernelWidth(); ++dx)
			{
				//uint2 pos[SIZE_H*SIZE_W];
				//[unroll]
				//for (uint q = 0; q < SIZE_H*SIZE_W; ++q)
				//	pos[q] = uint2(x*SIZE_W+(q%SIZE_W), y*SIZE_H+(q/SIZE_W)) * _Stride.xy + uint2(dx, dy);

				for (uint c = 0; c < X.channels; c += L1CACHESIZE)
				{
					// Cache X
					uint dc = groupThreadID.x;
					uint gx = groupThreadID.y;
					uint gy = groupThreadID.z;
					//[unroll]
					//for (q = 0; q < SIZE_H*SIZE_W; ++q)
					//{
						uint2 pos = uint2(x*SIZE_W+gx, y*SIZE_H+gy) * _Stride.xy + uint2(dx, dy);
						X_[SIZE_W*gy+gx][dc] = X.SafeGet(n, pos, c + dc, _Pad.xy);
					//}
					GroupMemoryBarrierWithGroupSync();

					// X * K
					if (k < K.channels &&
						x*SIZE_W < O.width &&
						y*SIZE_H < O.height) // need all threads to load channels, thus late check against kernel count
					{
						uint kIndex = K.Index(dy, dx, c, k);
						for (dc = 0; dc < L1CACHESIZE; ++dc)
						{
							[unroll]
							for (q = 0; q < SIZE_H*SIZE_W; ++q)
								acc[q] += X_[q][dc] * K.data[kIndex];//K.Get(dy, dx, c + dc, k);
							kIndex += K.channels;
						}
					}
					GroupMemoryBarrierWithGroupSync();
				}
			}
		}

		if (k < K.channels) // need all threads to load channels, thus late check against kernel count
			[unroll]
			for (q = 0; q < SIZE_H*SIZE_W; ++q)
			{
				uint ox = x*SIZE_W+(q%SIZE_W);
				uint oy = y*SIZE_H+(q/SIZE_W);
				if (ox < O.width && oy < O.height)
					O.Set(n, oy, ox, k, acc[q]);
			}
	}

	#undef X_
}
#undef SIZE_H
#undef SIZE_W
*/

/*
#undef L1CACHESIZE
#define L1CACHESIZE 64
groupshared float Conv2D_RegCached_X[4][L1CACHESIZE];
[numthreads(L1CACHESIZE, 1, 1)]
void Conv2D_RegCached(uint3 dispatchThreadID : SV_DispatchThreadID, uint3 groupThreadID : SV_GroupThreadID)
{
	DISPATCH_ARGS(K.kernelCount, O.width, O.height);
	TENSOR_SHARED2_ARGS4(X, K, B, WBK, O);

	#define X_ Conv2D_RegCached_X

	uint k = dispatchThreadID.x;
	uint x = dispatchThreadID.y;
	uint y = dispatchThreadID.z;

	if (x*2 >= O.width) return;
	if (y*2 >= O.height) return;

	for (uint n = 0; n < O.batch; ++n)
	{
		float4 acc = B.SafeGet(k);
		for (uint dy = 0; dy < K.GetKernelHeight(); ++dy)
		{
			for (uint dx = 0; dx < K.GetKernelWidth(); ++dx)
			{
				uint2 pos0 = uint2(x*2+0,y*2+0) * _Stride + uint2(dx,dy);
				uint2 pos1 = uint2(x*2+1,y*2+0) * _Stride + uint2(dx,dy);
				uint2 pos2 = uint2(x*2+0,y*2+1) * _Stride + uint2(dx,dy);
				uint2 pos3 = uint2(x*2+1,y*2+1) * _Stride + uint2(dx,dy);

				// Cache X
				uint c_ = groupThreadID.x;
				if (c_ < X.channels)
				{
					X_[0][c_] = X.SafeGet(n, pos0, c_, _Pad.xy);
					X_[1][c_] = X.SafeGet(n, pos1, c_, _Pad.xy);
					X_[2][c_] = X.SafeGet(n, pos2, c_, _Pad.xy);
					X_[3][c_] = X.SafeGet(n, pos3, c_, _Pad.xy);
				}
				GroupMemoryBarrierWithGroupSync();

				// X * K
				if (k < K.channels)
					for (uint c = 0; c < X.channels; ++c)
					{
						acc.x += X_[0][c] * K.Get(dy, dx, c, k);
						acc.y += X_[1][c] * K.Get(dy, dx, c, k);
						acc.z += X_[2][c] * K.Get(dy, dx, c, k);
						acc.w += X_[3][c] * K.Get(dy, dx, c, k);
					}
				GroupMemoryBarrierWithGroupSync();
			}
		}

		O.Set(n, y*2+0, x*2+0, k, acc.x);
		O.Set(n, y*2+0, x*2+1, k, acc.y);
		O.Set(n, y*2+1, x*2+0, k, acc.z);
		O.Set(n, y*2+1, x*2+1, k, acc.w);
	}
}
*/

/*
[numthreads(16,4,4)]
void Conv2DTrans(uint3 dispatchThreadID : SV_DispatchThreadID)
{
	DISPATCH_ARGS(K.kernelCount, O.width, O.height);
	TENSOR_SHARED2_ARGS4(X, K, B, WBK, O);

	uint k = dispatchThreadID.x;
	uint x = dispatchThreadID.y;
	uint y = dispatchThreadID.z;

	if (k >= K.channels) return;
	if (x >= O.width) return;
	if (y >= O.height) return;

	uint2 strideMask = _Stride.xy - 1;

	for (uint n = 0; n < O.batch; ++n)
	{
		float acc = B.Get(k);
		for (uint dy = 0; dy < K.GetKernelHeight(); dy += _Stride.y)
		{
			for (uint dx = 0; dx < K.GetKernelWidth(); dx += _Stride.x)
			{
				uint dxShifted = dx + (x&strideMask.x);
				uint dyShifted = dy + (y&strideMask.y);

				uint xx = x + dxShifted;
				uint yy = y + dyShifted;

				uint oy = (yy - _Pad.y) / _Stride.y;
				uint ox = (xx - _Pad.x) / _Stride.x;

				bool mask = xx >= _Pad.x && yy >= _Pad.y && ox < X.width && oy < X.height;
				if (!mask) continue;

				// [unroll] - crashes metal compiler
				for (uint c = 0; c < X.channels; ++c)
				{
					acc += X.Get(n, oy, ox, c) * K.Get(	K.GetKernelHeight() - 1 - dyShifted,
														K.GetKernelWidth()  - 1 - dxShifted, c, k);
				}
			}
		}

		O.Set(n, y, x, k, acc);
	}
}
*/



#undef SIZE
#define SIZE 4
[numthreads(16, 4, 4)]
void Conv2DTrans_Reg_Loop_safe(uint3 dispatchThreadID : SV_DispatchThreadID)
{
	DISPATCH_ARGS(K.kernelCount, O.width, O.height);
	TENSOR_SHARED2_ARGS4(X, K, B, WBK, O);

	uint k = dispatchThreadID.x;
	uint x = dispatchThreadID.y;
	uint y = dispatchThreadID.z;

	if (k >= K.channels) return;
	if (x*SIZE >= O.width) return;
	if (y*SIZE >= O.height) return;

	uint2 strideMask = _Stride.xy - 1;

	uint2 pad = _Pad.xy / _Stride.xy;
	for (uint n = 0; n < O.batch; ++n)
	{
		float acc[SIZE*SIZE];
		[unroll]
		for (uint q = 0; q < SIZE*SIZE; ++q)
			acc[q] = B.Get(k);

		for (uint dy = 0; dy < K.GetKernelHeight(); dy += _Stride.y)
		{
			for (uint dx = 0; dx < K.GetKernelWidth(); dx += _Stride.x)
			{
				uint2 kernelPos[SIZE*SIZE];
				uint2 pos[SIZE*SIZE];

				[unroll]
				for (uint q = 0; q < SIZE*SIZE; ++q)
				{
					uint2 xy = uint2(x*SIZE+(q%SIZE), y*SIZE+(q/SIZE));
					kernelPos[q] = uint2(dx, dy) + (xy & strideMask);
					pos[q] = (xy + kernelPos[q]) / _Stride.xy;

					// transpose
					kernelPos[q] = uint2(K.GetKernelWidth(), K.GetKernelHeight()) - 1 - kernelPos[q];
				}

				for (uint c = 0; c < X.channels; ++c)
					[unroll]
					for (q = 0; q < SIZE*SIZE; ++q)
						acc[q] = fastfma(X.SafeGet(n, pos[q], c, pad.xy), K.Get(kernelPos[q].y, kernelPos[q].x, c, k), acc[q]);
						//acc[q] += X.SafeGet(n, pos[q], c, pad.xy) * K.Get(kernelPos[q].y, kernelPos[q].x, c, k);
			}
		}

		[unroll]
		for (q = 0; q < SIZE*SIZE; ++q)
			O.Set(n, y*SIZE+(q/SIZE), x*SIZE+(q%SIZE), k, acc[q]);
	}
}



#undef L1CACHESIZE
#define L1CACHESIZE 64
#define SIZE_W 4
#define SIZE_H 2
groupshared float Conv2DTrans_L1Cached64_Reg_Loop_safe__X[SIZE_H*SIZE_W][L1CACHESIZE];
[numthreads(L1CACHESIZE, 1, 1)]
void Conv2DTrans_L1Cached64_Reg_Loop_safe_(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	DISPATCH_ARGS(K.kernelCount, O.width, O.height);
	TENSOR_SHARED2_ARGS4(X, K, B, WBK, O);

	#define X_ Conv2DTrans_L1Cached64_Reg_Loop_safe__X

	uint k = L1CACHESIZE * groupID.x + groupThreadID.x;
	uint x = groupID.y;
	uint y = groupID.z;

	// need all threads to load channels, thus will do late check against kernel count
	if (x*SIZE_W >= O.width) return;
	if (y*SIZE_H >= O.height) return;

	uint2 strideMask = _Stride.xy - 1;
	uint2 pad = _Pad.xy / _Stride.xy;

	for (uint n = 0; n < O.batch; ++n)
	{
		float acc[SIZE_H*SIZE_W];
		[unroll]
		for (uint q = 0; q < SIZE_H*SIZE_W; ++q)
			acc[q] = B.SafeGet(k);

		for (uint dy = 0; dy < K.GetKernelHeight(); dy += _Stride.y)
		{
			for (uint dx = 0; dx < K.GetKernelWidth(); dx += _Stride.x)
			{
				uint2 kernelPos[SIZE_H*SIZE_W];
				uint2 pos[SIZE_H*SIZE_W];

				[unroll]
				for (uint q = 0; q < SIZE_H*SIZE_W; ++q)
				{
					uint2 xy = uint2(x*SIZE_W+(q%SIZE_W), y*SIZE_H+(q/SIZE_W));
					kernelPos[q] = uint2(dx, dy) + (xy & strideMask);
					pos[q] = (xy + kernelPos[q]) / _Stride.xy;

					// transpose
					kernelPos[q] = uint2(K.GetKernelWidth(), K.GetKernelHeight()) - 1 - kernelPos[q];
				}

				for (uint c = 0; c < X.channels; c += L1CACHESIZE)
				{
					// Cache X
					uint dc = groupThreadID.x;
					[unroll]
					for (q = 0; q < SIZE_H*SIZE_W; ++q)
						X_[q][dc] = X.SafeGet(n, pos[q], c + dc, pad.xy);
					GroupMemoryBarrierWithGroupSync();

					// X * K
					if (k < K.channels) // need all threads to load channels, thus late check against kernel count
					{
						for (dc = 0; dc < L1CACHESIZE; ++dc)
						{
							[unroll]
							for (q = 0; q < SIZE_H*SIZE_W; ++q)
								acc[q] = fastfma(X_[q][dc], K.Get(kernelPos[q].y, kernelPos[q].x, c + dc, k), acc[q]);
								//acc[q] += X_[q][dc] * K.Get(kernelPos[q].y, kernelPos[q].x, c + dc, k);
						}
					}
					GroupMemoryBarrierWithGroupSync();
				}
			}
		}

		if (k < K.channels) // need all threads to load channels, thus late check against kernel count
			[unroll]
			for (q = 0; q < SIZE_H*SIZE_W; ++q)
			{
				uint ox = x*SIZE_W+(q%SIZE_W);
				uint oy = y*SIZE_H+(q/SIZE_W);
				if (ox < O.width && oy < O.height)
					O.Set(n, oy, ox, k, acc[q]);
			}
	}

	#undef X_
}
#undef SIZE_H
#undef SIZE_W


/*
#undef L1CACHESIZE
#define L1CACHESIZE 64
groupshared float Conv2DTrans_L1Cached64_Reg_Loop_safe_X[L1CACHESIZE];
[numthreads(L1CACHESIZE, 1, 1)]
void Conv2DTrans_L1Cached64_Reg_Loop_safe(uint3 groupID : SV_GroupID, uint3 groupThreadID : SV_GroupThreadID)
{
	DISPATCH_ARGS(K.kernelCount, X.width, X.height);
	TENSOR_SHARED2_ARGS4(X, K, B, WBK, O);

	#define X_ Conv2DTrans_L1Cached64_Reg_Loop_safe_X

	uint k = L1CACHESIZE * groupID.x + groupThreadID.x;
	uint x = groupID.y;
	uint y = groupID.z;

	// need all threads to load channels, thus will do late check against kernel count
	if (x >= X.width) return;
	if (y >= X.height) return;

	uint2 pad = _Pad.xy / _Stride.xy;

	for (uint n = 0; n < O.batch; ++n)
	{
		for (uint sy = 0; sy < _Stride.y; ++sy)
		{
			for (uint sx = 0; sx < _Stride.x; ++sx)
			{
				float acc = B.SafeGet(k);

				for (uint dy = sy; dy < K.GetKernelHeight(); dy += _Stride.y)
				{
					for (uint dx = sx; dx < K.GetKernelWidth(); dx += _Stride.x)
					{
						uint2 pos = uint2(x, y) + uint2(sx + dx, sy + dy) / _Stride.xy;

						for (uint c = 0; c < X.channels; c += L1CACHESIZE)
						{
							// Cache X
							uint dc = groupThreadID.x;
							X_[dc] = X.SafeGet(n, pos, c + dc, pad);
							GroupMemoryBarrierWithGroupSync();

							// X * K
							if (k < K.channels) // need all threads to load channels, thus late check against kernel count
							{
								for (dc = 0; dc < L1CACHESIZE; ++dc)
								{
									acc = fastfma(	X_[dc],
													K.Get(	K.GetKernelHeight() - 1 - dy,
															K.GetKernelWidth()  - 1 - dx, c + dc, k),
											acc);
								}
							}
							GroupMemoryBarrierWithGroupSync();
						}
					}
				}

				uint oy = y * _Stride.y + sy;
				uint ox = x * _Stride.x + sx;
				if (oy < O.height && ox < O.width && k < K.channels)
					O.Set(n, oy, ox, k, acc);
			}
		}
	}

	#undef X_
}
*/
#endif


