adding binary conv layer

code/complexity_considerations/binary_layer.py

@@ -0,0 +1,62 @@
+# Author: Mark McDonnell, mark.mcdonnell@unisa.edu.au
+import numpy as np
+
+from tensorflow.keras import backend as K
+from tensorflow.keras.layers import InputSpec, Conv2D
+#from tensorflow.keras import initializers
+
+
+class BinaryConv2D(Conv2D):
+    '''Binarized Convolution2D layer
+    References:
+    "BinaryNet: Training Deep Neural Networks with Weights and Activations Constrained to +1 or -1" [http://arxiv.org/abs/1602.02830]
+
+    adapated by Mark McDonnell from https://github.com/DingKe/nn_playground/blob/master/binarynet/binary_layers.py
+    '''
+
+    def __init__(self, filters, **kwargs):
+        super(BinaryConv2D, self).__init__(filters, **kwargs)
+
+    def build(self, input_shape):
+        if self.data_format == 'channels_first':
+            channel_axis = 1
+        else:
+            channel_axis = -1
+
+        input_dim = int(input_shape[channel_axis])
+
+        if input_dim is None:
+            raise ValueError('The channel dimension of the inputs '
+                             'should be defined. Found `None`.')
+
+        self.multiplier = np.sqrt(
+            2.0 / np.float(self.kernel_size[0]) / np.float(self.kernel_size[1]) / float(input_dim))
+
+        self.kernel = self.add_weight(shape=self.kernel_size + (input_dim, self.filters),
+                                      initializer=self.kernel_initializer,
+                                      name='kernel',
+                                      regularizer=self.kernel_regularizer,
+                                      constraint=self.kernel_constraint)
+
+        # Set input spec.
+        self.input_spec = InputSpec(ndim=4, axes={channel_axis: input_dim})
+        self.built = True
+
+    def call(self, inputs):
+
+        binary_kernel = self.kernel + K.stop_gradient(K.sign(self.kernel) - self.kernel)
+        binary_kernel = binary_kernel + K.stop_gradient(binary_kernel * self.multiplier - binary_kernel)
+
+        outputs = K.conv2d(inputs,
+                           binary_kernel,
+                           strides=self.strides,
+                           padding=self.padding,
+                           data_format=self.data_format,
+                           dilation_rate=self.dilation_rate)
+
+        return outputs
+
+    def get_config(self):
+        config = {'multiplier': self.multiplier}
+        base_config = super(BinaryConv2D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))

code/complexity_considerations/convert_to_1bit.py

@@ -1,10 +1,6 @@
 import numpy as np
-import code.config
-import code.focal_loss
-import dill
-from keras.models import load_model
-
 
+# TODO: it does not work with regular conv2d layer
 def convert_to_1bit_conv(model):
     ZeroOneWeightsDict = {}
     AllParamsDict = {}
@@ -17,13 +13,17 @@ def convert_to_1bit_conv(model):
             ww = layer.get_weights()
 
             # storage using 1 bit booleans
-            binary_weights = (0.5 * (np.sign(ww) + 1.0)).astype('bool')  # save weights as 0 or 1
+            #binary_weights = (0.5 * (np.sign(ww) + 1.0)).astype('bool')  # save weights as 0 or 1
+
+            #The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()
+
+            binary_weights = (0.5 * (np.sign(ww.all()) + 1.0)).astype('bool')  # save weights as 0 or 1
             ZeroOneWeightsDict[layer.name] = binary_weights
             AllParamsDict[layer.name] = binary_weights
             NumBinaryWeights += np.prod(ww[0].shape)
 
         elif 'bn' in layer.name:
-            # the saved model also nees floating point batch norm params
+            # the saved model also needs floating point batch norm params
             ww = layer.get_weights()
             AllParamsDict[layer.name] = ww
             cc = 0
@@ -42,22 +42,3 @@ def convert_to_1bit_conv(model):
     print('Num 32-bit weights (all batch norm parameters) = ', int(Num32bitWeights), '; weights memory = ', BNMemory
           , '  kB')
     print('Total memory = ', WeightsMemory + BNMemory, '  MB')
-
-
-if __name__ == '__main__':
-    model_path = '/home/javi/repos/DCASE2021-Task1/outputs/2020-12-23-16:09/best.h5'
-
-    n_classes = 10
-
-    if code.config.loss_type == 'focal_loss':
-        if type(code.config.fl_alpha) is not list:
-            alpha_list = [[code.config.fl_alpha] * n_classes]
-        else:
-            alpha_list = code.config.fl_alpha
-            # check_alpha_list(alpha_list, y.shape[1])
-
-    custom_object = {'categorical_focal_loss_fixed': dill.loads(
-        dill.dumps(code.focal_loss.categorical_focal_loss(gamma=code.config.fl_gamma, alpha=alpha_list)))}
-
-    model = load_model(model_path, custom_objects=custom_object)
-    convert_to_1bit_conv(model)

code/config.py

@@ -10,6 +10,7 @@
 n_filters: list = [32, 64, 128]
 pools_size: list = [(1, 10), (1, 5), (1, 5)]
 dropouts_rate: list = [0.3, 0.3, 0.3]
+binary_layer: bool = True
 
 ratio: int = 2
 pre_act: bool = False

code/get_model_size.py

@@ -1,4 +1,5 @@
 from complexity_considerations.model_size import get_keras_model_size
+from complexity_considerations.convert_to_1bit import convert_to_1bit_conv
 from keras.models import load_model
 from focal_loss import categorical_focal_loss
 import dill
@@ -27,3 +28,4 @@ def get_model_size(model_path):
     model = load_model(model_path, custom_objects=custom_object)
     print(model.summary())
     get_model_size(model)
+    convert_to_1bit_conv(model)

code/main.py

@@ -55,15 +55,17 @@
     model = res_conv_standard_post_csse(x.shape[1], x.shape[2], x.shape[3], y.shape[1],
                                         config.n_filters, config.pools_size, config.dropouts_rate, config.ratio,
                                         config.reshape_method, config.dense_layer,
-                                        pre_act=config.pre_act, shortcut=config.shortcut, verbose=config.verbose)
+                                        pre_act=config.pre_act, shortcut=config.shortcut, verbose=config.verbose,
+                                        binary_layer=config.binary_layer)
 
 else:
     model = res_conv_standard_post_csse_split_freqs(x.shape[1], x.shape[2], x.shape[3], y.shape[1],
                                                     config.n_filters, config.pools_size, config.dropouts_rate,
                                                     config.ratio,
                                                     config.reshape_method, config.dense_layer,
                                                     config.n_split_freqs, config.f_split_freqs,
-                                                    pre_act=config.pre_act, shortcut=config.shortcut, verbose=config.verbose)
+                                                    pre_act=config.pre_act, shortcut=config.shortcut,
+                                                    verbose=config.verbose, binary_layer=config.binary_layer)
 
 # checking focal loss if necessary
 if config.loss_type == 'focal_loss':

code/models.py

@@ -1,7 +1,7 @@
-import keras.layers
+import tensorflow.keras.layers
 from modules import network_module, freq_split
 
-from keras.models import Model
+from tensorflow.keras.models import Model
 
 
 __authors__ = "Javier Naranjo, Sergi Perez and Irene Martín"
@@ -17,37 +17,37 @@
 
 def res_conv_standard_post_csse(h, w, n_channels, n_classes,
                                 nfilters, pools_size, dropouts_rate, ratio, reshape_type, dense_layer,
-                                pre_act=False, shortcut='conv', verbose=False):
+                                pre_act=False, shortcut='conv', verbose=False, binary_layer=False):
     """
     Model
     """
 
-    ip = keras.layers.Input(shape=(h, w, n_channels))
+    ip = tensorflow.keras.layers.Input(shape=(h, w, n_channels))
 
     for i in range(0, len(nfilters)):
 
         if i == 0:
             x = network_module(ip, nfilters[i], ratio, pools_size[i], dropouts_rate[i], i,
-                               pre_act=pre_act, shortcut=shortcut)
+                               pre_act=pre_act, shortcut=shortcut, binary_layer=binary_layer)
 
         else:
             x = network_module(x, nfilters[i], ratio, pools_size[i], dropouts_rate[i], i,
-                               pre_act=pre_act, shortcut=shortcut)
+                               pre_act=pre_act, shortcut=shortcut, binary_layer=binary_layer)
 
     # Reshape
     if reshape_type == 'global_avg':
-        x = keras.layers.GlobalAveragePooling2D()(x)
+        x = tensorflow.keras.layers.GlobalAveragePooling2D()(x)
 
     elif reshape_type == 'flatten':
-        x = keras.layers.Flatten()(x)
+        x = tensorflow.keras.layers.Flatten()(x)
 
     elif reshape_type == 'global_max':
-        x = keras.layers.GlobalMaxPooling2D()(x)
+        x = tensorflow.keras.layers.GlobalMaxPooling2D()(x)
 
     if dense_layer is None:
-        x = keras.layers.Dense(n_classes)(x)
-        x = keras.layers.BatchNormalization()(x)
-        x = keras.layers.Activation('softmax')(x)
+        x = tensorflow.keras.layers.Dense(n_classes)(x)
+        x = tensorflow.keras.layers.BatchNormalization()(x)
+        x = tensorflow.keras.layers.Activation('softmax')(x)
 
     model = Model(ip, x)
 
@@ -60,39 +60,39 @@ def res_conv_standard_post_csse(h, w, n_channels, n_classes,
 def res_conv_standard_post_csse_split_freqs(h, w, n_channels, n_classes,
                                             nfilters, pools_size, dropouts_rate, ratio, reshape_type, dense_layer,
                                             n_split_freqs, f_split_freqs, pre_act=False, shortcut='conv',
-                                            verbose=False):
-    ip = keras.layers.Input(shape=(h, w, n_channels))
+                                            verbose=False, binary_layer=False):
+    ip = tensorflow.keras.layers.Input(shape=(h, w, n_channels))
 
     if n_split_freqs == 2:
 
-        splits = keras.layers.Lambda(freq_split, arguments={'n_split_freqs': n_split_freqs,
+        splits = tensorflow.keras.layers.Lambda(freq_split, arguments={'n_split_freqs': n_split_freqs,
                                                             'f_split_freqs': f_split_freqs})(ip)
 
         x1 = splits[0]
         x2 = splits[1]
 
         for i in range(0, len(nfilters)):
             x1 = network_module(x1, nfilters[i], ratio, pools_size[i], i, dropouts_rate[i],
-                                pre_act=pre_act, shortcut=shortcut)
+                                pre_act=pre_act, shortcut=shortcut, binary_layer=binary_layer)
             x2 = network_module(x2, nfilters[i], ratio, pools_size[i], i, dropouts_rate[i],
-                                pre_act=pre_act, shortcut=shortcut)
+                                pre_act=pre_act, shortcut=shortcut, binary_layer=binary_layer)
 
-        x = keras.layers.concatenate([x1, x2], axis=1)
+        x = tensorflow.keras.layers.concatenate([x1, x2], axis=1)
 
         # Reshape
         if reshape_type == 'global_avg':
-            x = keras.layers.GlobalAveragePooling2D()(x)
+            x = tensorflow.keras.layers.GlobalAveragePooling2D()(x)
 
         elif reshape_type == 'flatten':
-            x = keras.layers.Flatten()(x)
+            x = tensorflow.keras.layers.Flatten()(x)
 
         elif reshape_type == 'global_max':
-            x = keras.layers.GlobalMaxPooling2D()(x)
+            x = tensorflow.keras.layers.GlobalMaxPooling2D()(x)
 
         if dense_layer is None:
-            x = keras.layers.Dense(n_classes)(x)
-            x = keras.layers.BatchNormalization()(x)
-            x = keras.layers.Activation('softmax')(x)
+            x = tensorflow.keras.layers.Dense(n_classes)(x)
+            x = tensorflow.keras.layers.BatchNormalization()(x)
+            x = tensorflow.keras.layers.Activation('softmax')(x)
 
         model = Model(ip, x)
 
@@ -103,7 +103,7 @@ def res_conv_standard_post_csse_split_freqs(h, w, n_channels, n_classes,
 
     elif n_split_freqs == 3:
 
-        splits = keras.layers.Lambda(freq_split, arguments={'n_split_freqs': n_split_freqs,
+        splits = tensorflow.keras.layers.Lambda(freq_split, arguments={'n_split_freqs': n_split_freqs,
                                                             'f_split_freqs': f_split_freqs})(ip)
 
         x1 = splits[0]
@@ -112,26 +112,26 @@ def res_conv_standard_post_csse_split_freqs(h, w, n_channels, n_classes,
 
         for i in range(0, len(nfilters)):
             x1 = network_module(x1, nfilters[i], ratio, pools_size[i], i, dropouts_rate[i],
-                                pre_act=pre_act, shortcut=shortcut)
+                                pre_act=pre_act, shortcut=shortcut, binary_layer=binary_layer)
             x2 = network_module(x2, nfilters[i], ratio, pools_size[i], i, dropouts_rate[i],
-                                pre_act=pre_act, shortcut=shortcut)
+                                pre_act=pre_act, shortcut=shortcut, binary_layer=binary_layer)
             x3 = network_module(x3, nfilters[i], ratio, pools_size[i], i, dropouts_rate[i],
-                                pre_act=pre_act, shortcut=shortcut)
+                                pre_act=pre_act, shortcut=shortcut, binary_layer=binary_layer)
 
-        x = keras.layers.concatenate([x1, x2, x3], axis=1)
+        x = tensorflow.keras.layers.concatenate([x1, x2, x3], axis=1)
 
         # Reshape
         if reshape_type == 'global_avg':
-            x = keras.layers.GlobalAveragePooling2D()(x)
+            x = tensorflow.keras.layers.GlobalAveragePooling2D()(x)
         elif reshape_type == 'flatten':
-            x = keras.layers.Flatten()(x)
+            x = tensorflow.keras.layers.Flatten()(x)
         elif reshape_type == 'global_max':
-            x = keras.layers.GlobalMaxPooling2D()(x)
+            x = tensorflow.keras.layers.GlobalMaxPooling2D()(x)
 
         if dense_layer is None:
-            x = keras.layers.Dense(n_classes)(x)
-            x = keras.layers.BatchNormalization()(x)
-            x = keras.layers.Activation('softmax')(x)
+            x = tensorflow.keras.layers.Dense(n_classes)(x)
+            x = tensorflow.keras.layers.BatchNormalization()(x)
+            x = tensorflow.keras.layers.Activation('softmax')(x)
 
         model = Model(ip, x)