Merge pull request #6 from Machine-Listeners-Valencia/develop

Develop

.gitignore

@@ -1,4 +1,5 @@
 .idea
 data
 code/__pycache__/
-outputs
+outputs
+code/complexity_considerations/__pycache__/

code/complexity_considerations/binary_layer.py

@@ -0,0 +1,62 @@
+# Author: Mark McDonnell, mark.mcdonnell@unisa.edu.au
+import numpy as np
+
+from tensorflow.keras import backend as K
+from tensorflow.keras.layers import InputSpec, Conv2D
+#from tensorflow.keras import initializers
+
+
+class BinaryConv2D(Conv2D):
+    '''Binarized Convolution2D layer
+    References:
+    "BinaryNet: Training Deep Neural Networks with Weights and Activations Constrained to +1 or -1" [http://arxiv.org/abs/1602.02830]
+
+    adapated by Mark McDonnell from https://github.com/DingKe/nn_playground/blob/master/binarynet/binary_layers.py
+    '''
+
+    def __init__(self, filters, **kwargs):
+        super(BinaryConv2D, self).__init__(filters, **kwargs)
+
+    def build(self, input_shape):
+        if self.data_format == 'channels_first':
+            channel_axis = 1
+        else:
+            channel_axis = -1
+
+        input_dim = int(input_shape[channel_axis])
+
+        if input_dim is None:
+            raise ValueError('The channel dimension of the inputs '
+                             'should be defined. Found `None`.')
+
+        self.multiplier = np.sqrt(
+            2.0 / np.float(self.kernel_size[0]) / np.float(self.kernel_size[1]) / float(input_dim))
+
+        self.kernel = self.add_weight(shape=self.kernel_size + (input_dim, self.filters),
+                                      initializer=self.kernel_initializer,
+                                      name='kernel',
+                                      regularizer=self.kernel_regularizer,
+                                      constraint=self.kernel_constraint)
+
+        # Set input spec.
+        self.input_spec = InputSpec(ndim=4, axes={channel_axis: input_dim})
+        self.built = True
+
+    def call(self, inputs):
+
+        binary_kernel = self.kernel + K.stop_gradient(K.sign(self.kernel) - self.kernel)
+        binary_kernel = binary_kernel + K.stop_gradient(binary_kernel * self.multiplier - binary_kernel)
+
+        outputs = K.conv2d(inputs,
+                           binary_kernel,
+                           strides=self.strides,
+                           padding=self.padding,
+                           data_format=self.data_format,
+                           dilation_rate=self.dilation_rate)
+
+        return outputs
+
+    def get_config(self):
+        config = {'multiplier': self.multiplier}
+        base_config = super(BinaryConv2D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))

code/complexity_considerations/convert_to_1bit.py

@@ -0,0 +1,68 @@
+import numpy as np
+from scipy.io import savemat, loadmat
+
+
+# TODO: it does not work with regular conv2d layer
+def convert_to_1bit_conv(model, folder2store):
+    ZeroOneWeightsDict = {}
+    AllParamsDict = {}
+    NumBinaryWeights = 0.0
+    Num32bitWeights = 0.0
+    for layer in model.layers:
+        # print(layer.name)
+
+        if 'conv' in layer.name:
+            ww = layer.get_weights()
+
+            # storage using 1 bit booleans
+            binary_weights = (0.5 * (np.sign(ww) + 1.0)).astype('bool')  # save weights as 0 or 1
+
+            # The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()
+            # binary_weights = (0.5 * (np.sign(ww.all()) + 1.0)).astype('bool')  # save weights as 0 or 1
+            ZeroOneWeightsDict[layer.name] = binary_weights
+            AllParamsDict[layer.name] = binary_weights
+            NumBinaryWeights += np.prod(ww[0].shape)
+
+        elif 'bn' in layer.name:
+            # the saved model also needs floating point batch norm params
+            ww = layer.get_weights()
+            AllParamsDict[layer.name] = ww
+            cc = 0
+            for kk in ww:
+                # print(cc,layer.name,np.prod(kk.shape))
+                Num32bitWeights += np.prod(kk.shape)
+                cc = cc + 1
+
+    savemat(folder2store + 'FinalModel_01weights.mat', ZeroOneWeightsDict, do_compression=True, long_field_names=True)
+    savemat(folder2store + 'FinalModel_allparams.mat', AllParamsDict, do_compression=True, long_field_names=True)
+
+    WeightsMemory = NumBinaryWeights / 8 / 1024
+    BNMemory = 32.0 * Num32bitWeights / 8 / 1024
+    print('Num binary weights is less than 500kb: ', int(NumBinaryWeights), 'conv weights = conv weights memory of '
+          , WeightsMemory, '  kB')
+    print('Num 32-bit weights (all batch norm parameters) = ', int(Num32bitWeights), '; weights memory = ', BNMemory
+          , '  kB')
+    print('Total memory = ', WeightsMemory + BNMemory, '  MB')
+
+
+def set_to_1bit(model, folder2store):
+    AllParamsDict_loaded = loadmat(folder2store + 'FinalModel_allparams.mat')
+
+    conv_names = [m for m in list(AllParamsDict_loaded.keys()) if any(s in m for s in ['conv'])]
+    bn_names = [m for m in list(AllParamsDict_loaded.keys()) if any(s in m for s in ['bn'])]
+
+    c1 = 0
+    c2 = 0
+    for layer in model.layers:
+        if 'conv' in layer.name:
+            ww = AllParamsDict_loaded[conv_names[c1]].astype('float32') * 2.0 - 1.0
+            ww = ww * np.sqrt(2.0 / np.prod(ww[0].shape[0:3]))
+            layer.set_weights([ww[0]])
+            print('conv layer ', c1, ' has ', len(np.unique(ww)), ' unique weight values')
+            c1 = c1 + 1
+        elif 'bn' in layer.name:
+            ww = AllParamsDict_loaded[bn_names[c2]]
+            layer.set_weights(ww)
+            c2 = c2 + 1
+
+    return model

code/complexity_considerations/model_size.py

@@ -0,0 +1,137 @@
+import dcase_util
+import numpy
+
+
+def get_keras_model_size(keras_model, verbose=True, ui=None, excluded_layers=None):
+    """Calculate keras model size (non-zero parameters on disk)
+    Parameters
+    ----------
+    keras_model : keras.models.Model
+        keras model for the size calculation
+    verbose : bool
+        Print layer by layer information
+        Default value True
+    ui : dcase_util.ui.FancyLogger or dcase_util.ui.FancyPrinter
+        Print handler
+        Default value None
+    excluded_layers : list
+        List of layers to be excluded from the calculation
+        Default value [keras.layers.normalization.BatchNormalization, kapre.time_frequency.Melspectrogram]
+    Returns
+    -------
+    nothing
+    """
+
+    parameters_count = 0
+    parameters_count_nonzero = 0
+
+    parameters_bytes = 0
+    parameters_bytes_nonzero = 0
+
+    if verbose and ui is None:
+        # Initialize print handler
+        ui = dcase_util.ui.ui.FancyPrinter()
+
+    if excluded_layers is None:
+        # Populate excluded_layers list
+        excluded_layers = []
+        try:
+            import keras
+
+        except ImportError:
+            raise ImportError('Unable to import keras module. You can install it with `pip install keras`.')
+
+        excluded_layers.append(
+            keras.layers.normalization.BatchNormalization
+        )
+
+        # Include kapre layers only if kapre is installed
+        try:
+            import kapre
+            excluded_layers.append(
+                kapre.time_frequency.Melspectrogram
+            )
+        except ImportError:
+            pass
+
+    if verbose:
+        # Set up printing
+        ui.row_reset()
+        ui.row(
+            'Name', 'Param', 'NZ Param', 'Size', 'NZ Size',
+            widths=[30, 12, 12, 30, 30],
+            types=['str', 'int', 'int', 'str', 'str'],
+            separators=[True, False, True, False]
+        )
+        ui.row_sep()
+
+    for l in keras_model.layers:
+        # Loop layer by layer
+
+        current_parameters_count = 0
+        current_parameters_count_nonzero = 0
+        current_parameters_bytes = 0
+        current_parameters_bytes_nonzero = 0
+
+        weights = l.get_weights()
+        for w in weights:
+            current_parameters_count += numpy.prod(w.shape)
+            current_parameters_count_nonzero += numpy.count_nonzero(w.flatten())
+
+            if w.dtype in ['single', 'float32', 'int32', 'uint32']:
+                bytes = 32 / 8
+
+            elif w.dtype in ['float16', 'int16', 'uint16']:
+                bytes = 16 / 8
+
+            elif w.dtype in ['double', 'float64', 'int64', 'uint64']:
+                bytes = 64 / 8
+
+            elif w.dtype in ['int8', 'uint8']:
+                bytes = 8 / 8
+
+            else:
+                print('UNKNOWN TYPE', w.dtype)
+
+            current_parameters_bytes += numpy.prod(w.shape) * bytes
+            current_parameters_bytes_nonzero += numpy.count_nonzero(w.flatten()) * bytes
+
+        if l.__class__ not in excluded_layers:
+            parameters_count += current_parameters_count
+            parameters_count_nonzero += current_parameters_count_nonzero
+
+            parameters_bytes += current_parameters_bytes
+            parameters_bytes_nonzero += current_parameters_bytes_nonzero
+
+        if verbose:
+            ui.row(
+                l.name,
+                current_parameters_count,
+                current_parameters_count_nonzero,
+                dcase_util.utils.get_byte_string(current_parameters_bytes, show_bytes=False),
+                dcase_util.utils.get_byte_string(current_parameters_bytes_nonzero, show_bytes=False),
+            )
+
+    if verbose:
+        ui.row_sep()
+        ui.row(
+            'Total',
+            parameters_count,
+            parameters_count_nonzero,
+            dcase_util.utils.get_byte_string(parameters_bytes, show_bytes=True),
+            dcase_util.utils.get_byte_string(parameters_bytes_nonzero, show_bytes=True),
+        )
+        ui.line()
+
+    return {
+        'parameters': {
+            'all': {
+                'count': parameters_count,
+                'bytes': parameters_bytes
+            },
+            'non_zero': {
+                'count': parameters_count_nonzero,
+                'bytes': parameters_bytes_nonzero
+            }
+        }
+    }

code/config.py

@@ -1,15 +1,16 @@
 # data paths
+home_path = None
 data_path: str = '/repos/DCASE2021-Task1/data/gammatone_64/'
 code_path: str = '/repos/DCASE2021-Task1/code/'
-
-# audio representation hyperparameters
-# freq_bands = 64
+training_file: str = 'train_val_gammatone_mono_f1.h5'
+validation_file: str = 'train_val_gammatone_mono_f2.h5'
 
 # model parameters
 verbose: bool = True  # [True, False]
 n_filters: list = [32, 64, 128]
 pools_size: list = [(1, 10), (1, 5), (1, 5)]
 dropouts_rate: list = [0.3, 0.3, 0.3]
+binary_layer: bool = True
 
 ratio: int = 2
 pre_act: bool = False

code/data_augmentation.py

@@ -1,12 +1,12 @@
 import threading
 import numpy as np
-
+from tensorflow.keras.utils import Sequence
 
 __authors__ = "Javier Naranjo, Sergi Perez and Irene Martín"
 __copyright__ = "Machine Listeners Valencia"
 __credits__ = ["Machine Listeners Valencia"]
 __license__ = "MIT License"
-__version__ = "1.0.0"
+__version__ = "1.2.0"
 __maintainer__ = "Javier Naranjo"
 __email__ = "janal2@alumni.uv.es"
 __status__ = "Production"
@@ -41,20 +41,20 @@ def __next__(self):
             return self.it.__next__()
 
 
-class MixupGenerator():
-    def __init__(self, X_train, y_train, batch_size=32, alpha=0.2, shuffle=True):
-        self.X_train = X_train
+class MixupGenerator(Sequence):
+    def __init__(self, x_train, y_train, batch_size=32, alpha=0.2, shuffle=True):
+        self.X_train = x_train
         self.y_train = y_train
         self.batch_size = batch_size
         self.alpha = alpha
         self.shuffle = shuffle
-        self.sample_num = len(X_train)
+        self.sample_num = len(x_train)
         self.lock = threading.Lock()
 
     def __iter__(self):
         return self
 
-    @threadsafe_generator
+    #@threadsafe_generator
     def __call__(self):
         with self.lock:
             while True:

code/get_model_size.py

@@ -0,0 +1,27 @@
+from complexity_considerations.model_size import get_keras_model_size
+from complexity_considerations.convert_to_1bit import convert_to_1bit_conv, set_to_1bit
+import config
+from models import construct_model
+from load_data import load_h5s
+
+
+def get_model_size(model_path):
+    get_keras_model_size(model_path)
+
+
+if __name__ == '__main__':
+    x, y, val_x, val_y = load_h5s(config.home_path, config.data_path, config.validation_file, config.training_file)
+
+    model = construct_model(x, y)
+
+    folder2store = '/home/javi/repos/DCASE2021-Task1/outputs/2020-12-27-10:48/'
+    model_name = 'best.h5'
+
+    model.load_weights(folder2store + model_name)
+
+    print(model.summary())
+    get_model_size(model)
+    convert_to_1bit_conv(model, folder2store)
+    converted_model = set_to_1bit(model, folder2store)
+    #TODO: model size do not change
+    get_model_size(converted_model)