comments

nnet_gpu/layers/BatchNormalization.py

@@ -3,6 +3,8 @@
 from . import seqinst
 from ..stream_handler import stream_maps
 
+## TODO: Convert operations to gpu kernel
+
 class BatchNormalization(Layer):
 	def __init__(self,momentum=0.9,epsilon=1e-10,name=None):
 		super().__init__()

nnet_gpu/layers/convolution/conv2d.py

@@ -95,6 +95,9 @@ def cal_outsize(self,sz,ksz,stride,pad,dilation=1):
 		return (sz + 2*pad - dksz)//stride + 1
 
 	def forward(self,inp,training=True):
+		"""
+		Simple, just do im2col and then dot product.
+		"""
 		inp=cp.ascontiguousarray(inp.transpose(0,3,1,2))
 		self.inp=inp
 		#inp[batches,channels,row,col]
@@ -116,7 +119,7 @@ def backprop(self,grads,layer=1):								#strides[batch,row,col,depth]
 		grads[batches,esz,esz,num_kernels],inp[batches,channels,row,col],kernels(channels,ksz,ksz,num_kernels),biases[1,num_kernels]
 		1.) For kernel gradient (self.d_ker):
 				Convolve the gradients as kernel over saved input with stride 1 and dilate the gradient with
-				current stride value and same current padding.
+				current stride value and current padding.
 				The channels are treated as batches and batches as channel so it gives the correct kernel gradient shape.
 
 		2.) For input gradient (self.d_inp):

nnet_gpu/layers/convolution/conv2dtranspose.py

@@ -29,6 +29,10 @@ def cal_outsize(self,sz,ksz,stride,pad,dilation=1):
 		return sz*stride
 
 	def forward(self,inp,training=True):
+		"""
+		Simply, reverse steps of conv2d.
+		Dot product, then col2im.
+		"""
 		self.inp=inp.transpose(0,3,1,2)
 		#inp[batches,channels,row,col]
 		self.batches,self.channels,self.row,self.col=self.inp.shape
@@ -47,6 +51,19 @@ def forward(self,inp,training=True):
 		return self.a_out				# a_out[batches,out_row,out_col,num_kernels]
 
 	def backprop(self,grads,layer=1):
+		"""
+		1.) For kernel gradient (self.d_ker):
+				Convolve the saved input as kernel over gradients with stride 1 and dilate the saved input with
+				current stride value and current padding.
+				The channels are treated as batches and batches as channel so it gives the correct kernel gradient shape.
+
+		2.) For input gradient (self.d_inp):
+				Convolution over gradients with self.kernels as kernel. Should give original input shape back.
+				All parameters stride,padding,dilation are same as current.
+
+		3.) For biases gradient :
+				It's just same as gradient. Just reshape and sum/mean it.
+		"""
 		if self.activation != echo:
 			grads*=self.activation(self.z_out,self.a_out,derivative=True)
 		self.d_ker.kernels=self.inp.transpose(0,2,3,1)	# t makes[batches,row,col,channels]
@@ -55,10 +72,9 @@ def backprop(self,grads,layer=1):
 			self.backp_stream.wait_event(self.grad_event)
 			self.d_c_w=self.d_ker.forward(grads.transpose(3,1,2,0))	#[channels,row,col,batches]
 		# self.d_c_w/=self.batches		#take mean change over batches
-		# Backprop for inp.	grads[batches,esz,esz,num_kernels]	self.flipped[num_kernels,kernel_size[0],kernel_size[1],channels]
 		if layer:
 			d_inputs=cp.ascontiguousarray(self.d_inp.forward(grads))
-			assert d_inputs.shape == (self.batches,self.row,self.col,self.channels),f"{(self.batches,self.row,self.col,self.channels)},{d_inputs.shape}"
+			# assert d_inputs.shape == (self.batches,self.row,self.col,self.channels),f"{(self.batches,self.row,self.col,self.channels)},{d_inputs.shape}"
 		else:
 			d_inputs=0
 		if self.bias_is_not_0:

nnet_gpu/layers/pooling/max_pool.py

@@ -3,6 +3,8 @@
 from .. import seqinst
 from ...stream_handler import stream_maps
 
+## TODO: Convert operations to gpu kernel
+
 class max_pool(Layer):
 	def __init__(self,input_shape=None,ksize=[2,2],stride=[2,2],name=None):
 		#inp[batches,row,col,channels], kernels[ksz,ksz], stride[row,col]