From f4054a597521fdc539befbc1f996eb047d9be97b Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Mon, 7 Oct 2019 16:47:18 +0100 Subject: [PATCH] update aggregator to the latest version --- niftynet/engine/windows_aggregator_grid.py | 185 +++++++------------ niftynet/engine/windows_aggregator_resize.py | 133 +++++-------- tests/windows_aggregator_grid_v2_test.py | 4 +- tests/windows_aggregator_resize_v2_test.py | 4 +- 4 files changed, 118 insertions(+), 208 deletions(-) diff --git a/niftynet/engine/windows_aggregator_grid.py b/niftynet/engine/windows_aggregator_grid.py index 344bbd9f..82b1a0bb 100755 --- a/niftynet/engine/windows_aggregator_grid.py +++ b/niftynet/engine/windows_aggregator_grid.py @@ -3,13 +3,14 @@ windows aggregator decode sampling grid coordinates and image id from batch data, forms image level output and write to hard drive. """ -from __future__ import absolute_import, print_function, division +from __future__ import absolute_import, division, print_function import os +from collections import OrderedDict import numpy as np +import pandas as pd -import tensorflow as tf # pylint: disable=too-many-nested-blocks # pylint: disable=too-many-branches import niftynet.io.misc_io as misc_io @@ -25,6 +26,7 @@ class GridSamplesAggregator(ImageWindowsAggregator): initialised as all zeros, and the values are replaced by image window data decoded from batch. """ + def __init__(self, image_reader, name='image', @@ -44,7 +46,7 @@ def __init__(self, self.fill_constant = fill_constant def decode_batch(self, window, location): - ''' + """ Function used to save multiple outputs listed in the window dictionary. For the fields that have the keyword 'window' in the dictionary key, it will be saved as image. The rest will be saved as @@ -52,126 +54,84 @@ def decode_batch(self, window, location): changed into the header by the user), the first column being the index of the window, followed by the list of output and the location array for each considered window + :param window: dictionary of output :param location: location of the input :return: - ''' + """ n_samples = location.shape[0] - location_init = np.copy(location) - init_ones = None - for i in window: - if 'window' in i: # all outputs to be created as images should + location_cropped = {} + for key in window: + if 'window' in key: # all outputs to be created as images should # contained the keyword "window" - init_ones = np.ones_like(window[i]) - window[i], _ = self.crop_batch(window[i], location_init, - self.window_border) - location_init = np.copy(location) - print(i, np.sum(window[i]), np.max(window[i])) - _, location = self.crop_batch(init_ones, location_init, - self.window_border) + window[key], location_cropped[key] = self.crop_batch( + window[key], location, self.window_border) + for batch_id in range(n_samples): - image_id, x_start, y_start, z_start, x_end, y_end, z_end = \ - location[batch_id, :] + image_id = location[batch_id, 0] if image_id != self.image_id: # image name changed: - # save current image and create an empty image + # save current result and create an empty result file self._save_current_image() self._save_current_csv() if self._is_stopping_signal(location[batch_id]): return False - self.image_out = {} - self.csv_out = {} - for i in window: - if 'window' in i: # check that we want to have an image - # and initialise accordingly - self.image_out[i] = self._initialise_empty_image( + + self.image_out, self.csv_out = {}, {} + for key in window: + if 'window' in key: + # to be saved as image + self.image_out[key] = self._initialise_empty_image( image_id=image_id, - n_channels=window[i].shape[-1], - dtype=window[i].dtype) - print("for output shape is ", self.image_out[i].shape) + n_channels=window[key].shape[-1], + dtype=window[key].dtype) else: - if not isinstance(window[i], (list, tuple, np.ndarray)): - self.csv_out[i] = self._initialise_empty_csv( - 1 + location_init[0, :].shape[-1]) - else: - window[i] = np.asarray(window[i]) - if n_samples > 1 and np.asarray(window[i]).ndim < 2: - window[i] = np.expand_dims(window[i], 1) - elif n_samples == 1 and np.asarray( - window[i]).shape[0] != n_samples: - window[i] = np.expand_dims(window[i], 0) - window_save = np.asarray(np.squeeze( - window[i][batch_id, ...])) - try: - assert window_save.ndim <= 2 - except (TypeError, AssertionError): - tf.logging.error( - "The output you are trying to " - "save as csv is more than " - "bidimensional. Did you want " - "to save an image instead? " - "Put the keyword window " - "in the output dictionary" - " in your application file") - if window_save.ndim < 2: - window_save = np.expand_dims(window_save, 0) - self.csv_out[i] = self._initialise_empty_csv( - n_channel=window_save.shape[-1] + location_init - [0, :].shape[-1]) - for i in window: - if 'window' in i: - self.image_out[i][ + # to be saved as csv file + n_elements = np.int64( + np.asarray(window[key]).size / n_samples) + table_header = [ + '{}_{}'.format(key, idx) + for idx in range(n_elements) + ] if n_elements > 1 else ['{}'.format(key)] + table_header += [ + 'coord_{}'.format(idx) + for idx in range(location.shape[-1]) + ] + self.csv_out[key] = self._initialise_empty_csv( + key_names=table_header) + + for key in window: + if 'window' in key: + x_start, y_start, z_start, x_end, y_end, z_end = \ + location_cropped[key][batch_id, 1:] + self.image_out[key][ x_start:x_end, y_start:y_end, z_start:z_end, ...] = \ - window[i][batch_id, ...] + window[key][batch_id, ...] else: - if isinstance(window[i], (list, tuple, np.ndarray)): - window[i] = np.asarray(window[i]) - if n_samples > 1 and window[i].ndim < 2: - window[i] = np.expand_dims(window[i], 1) - elif n_samples == 1 and window[i].shape[0] != n_samples: - window[i] = np.expand_dims(window[i], 0) - print(batch_id, "is batch_id ", window[i].shape) - window_save = np.squeeze(np.asarray( - window[i][batch_id, ...])) - try: - assert window_save.ndim <= 2 - except (TypeError, AssertionError): - tf.logging.error( - "The output you are trying to " - "save as csv is more than " - "bidimensional. Did you want " - "to save an image instead? " - "Put the keyword window " - "in the output dictionary" - " in your application file") - while window_save.ndim < 2: - window_save = np.expand_dims(window_save, 0) - window_save = np.asarray(window_save) - - window_loc = np.concatenate([ - window_save, np.tile( - location_init[batch_id, ...], - [window_save.shape[0], 1])], 1) - else: - window_loc = np.concatenate([ - np.reshape(window[i], [1, 1]), np.tile( - location_init[batch_id, ...], [1, 1])], 1) - self.csv_out[i] = np.concatenate([self.csv_out[i], - window_loc], 0) + window[key] = np.asarray(window[key]).reshape( + [n_samples, -1]) + window_save = window[key][batch_id:batch_id + 1, :] + window_loc = location[batch_id:batch_id + 1, :] + csv_row = np.concatenate([window_save, window_loc], 1) + csv_row = csv_row.ravel() + key_names = self.csv_out[key].columns + self.csv_out[key] = self.csv_out[key].append( + OrderedDict(zip(key_names, csv_row)), + ignore_index=True) return True def _initialise_empty_image(self, image_id, n_channels, dtype=np.float): - ''' + """ Initialise an empty image in which to populate the output :param image_id: image_id to be used in the reader :param n_channels: numbers of channels of the saved output (for multimodal output) :param dtype: datatype used for the saving :return: the initialised empty image - ''' + """ self.image_id = image_id spatial_shape = self.input_image[self.name].shape[:3] - output_image_shape = spatial_shape + (n_channels,) + output_image_shape = spatial_shape + (n_channels, ) empty_image = np.zeros(output_image_shape, dtype=dtype) for layer in self.reader.preprocessors: if isinstance(layer, PadLayer): @@ -182,26 +142,23 @@ def _initialise_empty_image(self, image_id, n_channels, dtype=np.float): return empty_image - def _initialise_empty_csv(self, n_channel): - ''' + def _initialise_empty_csv(self, key_names): + """ Initialise a csv output file with a first line of zeros + :param n_channel: number of saved fields :return: empty first line of the array to be saved as csv - ''' - return np.zeros([1, n_channel]) + """ + return pd.DataFrame(columns=key_names) def _save_current_image(self): - ''' + """ For all the outputs to be saved as images, go through the dictionary and save the resulting output after reversing the initial preprocessing :return: - ''' + """ if self.input_image is None: return - for i in self.image_out: - print(np.sum(self.image_out[i]), " is sum of image out %s before" - % i) - print("for output shape is now ", self.image_out[i].shape) for layer in reversed(self.reader.preprocessors): if isinstance(layer, PadLayer): for i in self.image_out: @@ -210,32 +167,26 @@ def _save_current_image(self): for i in self.image_out: self.image_out[i], _ = layer.inverse_op(self.image_out[i]) subject_name = self.reader.get_subject_id(self.image_id) - for i in self.image_out: - print(np.sum(self.image_out[i]), " is sum of image out %s after" - % i) for i in self.image_out: filename = "{}_{}_{}.nii.gz".format(i, subject_name, self.postfix) source_image_obj = self.input_image[self.name] - misc_io.save_data_array(self.output_path, - filename, - self.image_out[i], - source_image_obj, + misc_io.save_data_array(self.output_path, filename, + self.image_out[i], source_image_obj, self.output_interp_order) self.log_inferred(subject_name, filename) return def _save_current_csv(self): - ''' + """ For all output to be saved as csv, loop through the dictionary of output and create the csv :return: - ''' + """ if self.input_image is None: return subject_name = self.reader.get_subject_id(self.image_id) for i in self.csv_out: filename = "{}_{}_{}.csv".format(i, subject_name, self.postfix) - misc_io.save_csv_array(self.output_path, filename, self.csv_out[ - i][1:, :]) + misc_io.save_csv_array(self.output_path, filename, self.csv_out[i]) self.log_inferred(subject_name, filename) return diff --git a/niftynet/engine/windows_aggregator_resize.py b/niftynet/engine/windows_aggregator_resize.py index aa0c1668..7c072ab7 100755 --- a/niftynet/engine/windows_aggregator_resize.py +++ b/niftynet/engine/windows_aggregator_resize.py @@ -3,12 +3,14 @@ Windows aggregator resize each item in a batch output and save as an image. """ -from __future__ import absolute_import, print_function, division +from __future__ import absolute_import, division, print_function import os +from collections import OrderedDict import numpy as np -import tensorflow as tf +import pandas as pd + import niftynet.io.misc_io as misc_io from niftynet.engine.sampler_resize_v2 import zoom_3d from niftynet.engine.windows_aggregator_base import ImageWindowsAggregator @@ -23,6 +25,7 @@ class ResizeSamplesAggregator(ImageWindowsAggregator): window and save as a new image volume. Multiple output image can be proposed and csv output can be performed as well """ + def __init__(self, image_reader, name='image', @@ -54,81 +57,47 @@ def decode_batch(self, window, location): """ n_samples = location.shape[0] - location_init = np.copy(location) - test = None - for i in window: - if 'window' in i: - - test = np.ones_like(window[i]) - window[i], _ = self.crop_batch(window[i], location_init, - self.window_border) - location_init = np.copy(location) - print(i, np.sum(window[i]), np.max(window[i])) - _, location = self.crop_batch(test, location_init, self.window_border) - for batch_id in range(n_samples): if self._is_stopping_signal(location[batch_id]): return False self.image_id = location[batch_id, 0] - if self._is_stopping_signal(location[batch_id]): - return False - self.image_out = {} - self.csv_out = {} - - for i in window: - if 'window' in i: - - while window[i].ndim < 5: - window[i] = window[i][..., np.newaxis, :] - self.image_out[i] = window[i][batch_id, ...] + self.image_out, self.csv_out = {}, {} + for key in window: + if 'window' in key: + # saving image output + while window[key].ndim < 5: + window[key] = window[key][..., np.newaxis, :] + self.image_out[key] = window[key][batch_id, ...] else: - if not isinstance(window[i], (list, tuple, np.ndarray)): - window_loc = np.reshape(window[i], [1, 1]) - self.csv_out[i] = self._initialise_empty_csv(1) - else: - window[i] = np.asarray(window[i]) - if n_samples > 1 and window[i].ndim < 2: - window[i] = np.expand_dims(window[i], 1) - elif n_samples == 1 and window[i].shape[0] != n_samples: - window[i] = np.expand_dims(window[i], 0) - window_save = np.asarray(np.squeeze( - window[i][batch_id, ...])) - try: - assert window_save.ndim <= 2 - except (TypeError, AssertionError): - tf.logging.error( - "The output you are trying to " - "save as csv is more than " - "bidimensional. Did you want " - "to save an image instead? " - "Put the keyword window " - "in the output dictionary" - " in your application file") - while window_save.ndim < 2: - window_save = np.expand_dims(window_save, 0) - window_loc = window_save - self.csv_out[i] = self._initialise_empty_csv( - n_channel=window_save.shape[-1]) - - self.csv_out[i] = np.concatenate([self.csv_out[i], - window_loc], 0) - + # saving csv output + window[key] = np.asarray(window[key]).reshape( + [n_samples, -1]) + n_elements = window[key].shape[-1] + table_header = [ + '{}_{}'.format(key, idx) for idx in range(n_elements) + ] if n_elements > 1 else ['{}'.format(key)] + self.csv_out[key] = self._initialise_empty_csv( + key_names=table_header) + csv_row = window[key][batch_id:batch_id + 1, :].ravel() + self.csv_out[key] = self.csv_out[key].append( + OrderedDict(zip(table_header, csv_row)), + ignore_index=True) self._save_current_image() self._save_current_csv() return True def _initialise_image_shape(self, image_id, n_channels): - ''' + """ Return the shape of the empty image to be saved :param image_id: index to find the appropriate input image from the reader :param n_channels: number of channels of the image :return: shape of the empty image - ''' + """ self.image_id = image_id spatial_shape = self.input_image[self.name].shape[:3] - output_image_shape = spatial_shape + (1, n_channels,) + output_image_shape = spatial_shape + (1, n_channels) empty_image = np.zeros(output_image_shape, dtype=np.bool) for layer in self.reader.preprocessors: if isinstance(layer, PadLayer): @@ -136,19 +105,18 @@ def _initialise_image_shape(self, image_id, n_channels): return empty_image.shape def _save_current_image(self): - ''' + """ Loop through the dictionary of images output and resize and reverse the preprocessing prior to saving :return: - ''' + """ if self.input_image is None: return self.current_out = {} for i in self.image_out: resize_to_shape = self._initialise_image_shape( - image_id=self.image_id, - n_channels=self.image_out[i].shape[-1]) + image_id=self.image_id, n_channels=self.image_out[i].shape[-1]) window_shape = resize_to_shape current_out = self.image_out[i] while current_out.ndim < 5: @@ -156,19 +124,18 @@ def _save_current_image(self): if self.window_border and any([b > 0 for b in self.window_border]): np_border = self.window_border while len(np_border) < 5: - np_border = np_border + (0,) - np_border = [(b,) for b in np_border] + np_border = np_border + (0, ) + np_border = [(b, ) for b in np_border] current_out = np.pad(current_out, np_border, mode='edge') image_shape = current_out.shape zoom_ratio = \ [float(p) / float(d) for p, d in zip(window_shape, image_shape)] image_shape = list(image_shape[:3]) + [1, image_shape[-1]] - print(np.sum(self.image_out[i]), " is sum of image out %s before" - % i) current_out = np.reshape(current_out, image_shape) - current_out = zoom_3d(image=current_out, - ratio=zoom_ratio, - interp_order=self.output_interp_order) + current_out = zoom_3d( + image=current_out, + ratio=zoom_ratio, + interp_order=self.output_interp_order) self.current_out[i] = current_out for layer in reversed(self.reader.preprocessors): @@ -180,42 +147,34 @@ def _save_current_image(self): for i in self.image_out: self.image_out[i], _ = layer.inverse_op(self.image_out[i]) subject_name = self.reader.get_subject_id(self.image_id) - for i in self.image_out: - print(np.sum(self.image_out[i]), " is sum of image out %s after" - % i) for i in self.image_out: filename = "{}_{}_{}.nii.gz".format(i, subject_name, self.postfix) source_image_obj = self.input_image[self.name] - misc_io.save_data_array(self.output_path, - filename, - self.current_out[i], - source_image_obj, + misc_io.save_data_array(self.output_path, filename, + self.current_out[i], source_image_obj, self.output_interp_order) self.log_inferred(subject_name, filename) return def _save_current_csv(self): - ''' + """ Save all csv output present in the dictionary of csv_output. :return: - ''' + """ if self.input_image is None: return subject_name = self.reader.get_subject_id(self.image_id) for i in self.csv_out: filename = "{}_{}_{}.csv".format(i, subject_name, self.postfix) - misc_io.save_csv_array(self.output_path, - filename, - self.csv_out[i][1:, :]) + misc_io.save_csv_array(self.output_path, filename, self.csv_out[i]) self.log_inferred(subject_name, filename) return - def _initialise_empty_csv(self, n_channel): - ''' + def _initialise_empty_csv(self, key_names): + """ Initialise the array to be saved as csv as a line of zeros according to the number of elements to be saved :param n_channel: :return: - ''' - return np.zeros([1, n_channel]) - \ No newline at end of file + """ + return pd.DataFrame(columns=key_names) diff --git a/tests/windows_aggregator_grid_v2_test.py b/tests/windows_aggregator_grid_v2_test.py index e0e37649..cbbd8164 100755 --- a/tests/windows_aggregator_grid_v2_test.py +++ b/tests/windows_aggregator_grid_v2_test.py @@ -423,7 +423,7 @@ def test_init_3d_mo_bidimcsv(self): ) stats_pd = pd.read_csv(stats_filename) self.assertAllClose( - stats_pd.shape, [840, 11] + stats_pd.shape, [420, 14] ) sampler.close_all() @@ -590,7 +590,7 @@ def test_init_2d_mo_bidimcsv(self): ) stats_pd = pd.read_csv(stats_filename) self.assertAllClose( - stats_pd.shape, [20, 11] + stats_pd.shape, [10, 14] ) sampler.close_all() diff --git a/tests/windows_aggregator_resize_v2_test.py b/tests/windows_aggregator_resize_v2_test.py index c869009f..92b455ae 100755 --- a/tests/windows_aggregator_resize_v2_test.py +++ b/tests/windows_aggregator_resize_v2_test.py @@ -367,7 +367,7 @@ def test_init_3d_mo_bidimcsv(self): min_pd = pd.read_csv(sum_filename) self.assertAllClose(min_pd.shape, [1, 2]) stats_pd = pd.read_csv(stats_filename) - self.assertAllClose(stats_pd.shape, [2, 4]) + self.assertAllClose(stats_pd.shape, [1, 7]) sampler.close_all() def test_2d_init(self): @@ -548,7 +548,7 @@ def test_init_2d_mo_bidimcsv(self): min_pd = pd.read_csv(sum_filename) self.assertAllClose(min_pd.shape, [1, 2]) stats_pd = pd.read_csv(stats_filename) - self.assertAllClose(stats_pd.shape, [2, 4]) + self.assertAllClose(stats_pd.shape, [1, 7]) sampler.close_all() def test_25d_init(self):