Skip to content

Commit

Permalink
latest changes from upstream
Browse files Browse the repository at this point in the history
  • Loading branch information
@tomvars
tomvars committed Aug 8, 2018
2 parents 053dbad + 762b7fd commit fd57e74
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 224 deletions.
209 changes: 8 additions & 201 deletions demos/module_examples/ParallelCSVReaderDemo.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,9 @@
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"INFO:tensorflow:TensorFlow version 1.6.0\n",
"CRITICAL:tensorflow:Optional Python module cv2 not found, please install cv2 and retry if the application fails.\n",
"INFO:tensorflow:Available Image Loaders:\n",
"['nibabel', 'skimage', 'pillow', 'simpleitk', 'dummy'].\n",
"\u001b[1mINFO:niftynet:\u001b[0m Optional Python module yaml not found, please install yaml and retry if the application fails.\n",
"\u001b[1mINFO:niftynet:\u001b[0m Optional Python module yaml version None not found, please install yaml-None and retry if the application fails.\n"
]
}
],
"outputs": [],
"source": [
"import sys\n",
"niftynet_path = '/home/tom/phd/NiftyNet-Generator-PR/NiftyNet'\n",
Expand All @@ -37,19 +24,9 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Accessing: https://github.com/NifTK/NiftyNetModelZoo\n",
"mr_ct_regression_model_zoo_data: OK. \n",
"Already downloaded. Use the -r option to download again.\n"
]
}
],
"outputs": [],
"source": [
"from niftynet.utilities.download import download\n",
"download('mr_ct_regression_model_zoo_data')\n",
Expand All @@ -60,23 +37,9 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['image']\n",
"\u001b[1mINFO:niftynet:\u001b[0m \n",
"\n",
"Number of subjects 15, input section names: ['subject_id', 'CT']\n",
"-- using all subjects (without data partitioning).\n",
"\n",
"\u001b[1mINFO:niftynet:\u001b[0m Image reader: loading 15 subjects from sections ['CT'] as input [image]\n"
]
}
],
"outputs": [],
"source": [
"NetParam = namedtuple('NetParam', 'normalise_foreground_only foreground_type multimod_foreground_type histogram_ref_file norm_type cutoff normalisation whitening')\n",
"ActionParam = namedtuple('ActionParam', 'random_flipping_axes scaling_percentage rotation_angle rotation_angle_x rotation_angle_y rotation_angle_z do_elastic_deformation num_ctrl_points deformation_sigma proportion_to_deform')\n",
Expand Down Expand Up @@ -123,165 +86,9 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[2, 4, 8, 16]\n",
"\u001b[1mINFO:niftynet:\u001b[0m reading size of preprocessed images\n",
"\u001b[1mWARNING:niftynet:\u001b[0m queue_length should be larger than batch_size, defaulting to batch_size * 5.0 (500).\n",
"\u001b[1mINFO:niftynet:\u001b[0m Initiating dataset...\n",
"\u001b[1mINFO:niftynet:\u001b[0m self.from_generator: True\n",
"\u001b[1mINFO:niftynet:\u001b[0m Initiating dataset from generator...\n",
"Num Parallel Calls: 2\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 1 / 10\n",
"Time per batch: 25.9870502948761\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 2 / 10\n",
"Time per batch: 4.30904746055603\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 3 / 10\n",
"Time per batch: 4.451494932174683\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 4 / 10\n",
"Time per batch: 4.402526378631592\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 5 / 10\n",
"Time per batch: 4.456684112548828\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 6 / 10\n",
"Time per batch: 4.406505346298218\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 7 / 10\n",
"Time per batch: 4.4387054443359375\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 8 / 10\n",
"Time per batch: 4.466816663742065\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 9 / 10\n",
"Time per batch: 4.434375047683716\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 10 / 10\n",
"Time per batch: 4.4926910400390625\n",
"Mean batch time: 4.428760714001125\n",
"\u001b[1mINFO:niftynet:\u001b[0m reading size of preprocessed images\n",
"\u001b[1mWARNING:niftynet:\u001b[0m queue_length should be larger than batch_size, defaulting to batch_size * 5.0 (500).\n",
"\u001b[1mINFO:niftynet:\u001b[0m Initiating dataset...\n",
"\u001b[1mINFO:niftynet:\u001b[0m self.from_generator: True\n",
"\u001b[1mINFO:niftynet:\u001b[0m Initiating dataset from generator...\n",
"Num Parallel Calls: 4\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 1 / 10\n",
"Time per batch: 14.725233316421509\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 2 / 10\n",
"Time per batch: 2.5354156494140625\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 3 / 10\n",
"Time per batch: 2.524552822113037\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 4 / 10\n",
"Time per batch: 2.3708324432373047\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 5 / 10\n",
"Time per batch: 2.4221031665802\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 6 / 10\n",
"Time per batch: 2.48394513130188\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 7 / 10\n",
"Time per batch: 2.456294059753418\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 8 / 10\n",
"Time per batch: 2.394380807876587\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 9 / 10\n",
"Time per batch: 2.414968490600586\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 10 / 10\n",
"Time per batch: 2.4265353679656982\n",
"Mean batch time: 2.4476697709825306\n",
"\u001b[1mINFO:niftynet:\u001b[0m reading size of preprocessed images\n",
"\u001b[1mWARNING:niftynet:\u001b[0m queue_length should be larger than batch_size, defaulting to batch_size * 5.0 (500).\n",
"\u001b[1mINFO:niftynet:\u001b[0m Initiating dataset...\n",
"\u001b[1mINFO:niftynet:\u001b[0m self.from_generator: True\n",
"\u001b[1mINFO:niftynet:\u001b[0m Initiating dataset from generator...\n",
"Num Parallel Calls: 8\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 1 / 10\n",
"Time per batch: 7.562370777130127\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 2 / 10\n",
"Time per batch: 1.204786777496338\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 3 / 10\n",
"Time per batch: 1.3247928619384766\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 4 / 10\n",
"Time per batch: 1.256899118423462\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 5 / 10\n",
"Time per batch: 1.2707467079162598\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 6 / 10\n",
"Time per batch: 1.304060935974121\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 7 / 10\n",
"Time per batch: 1.2477483749389648\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 8 / 10\n",
"Time per batch: 1.3253648281097412\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 9 / 10\n",
"Time per batch: 1.3146610260009766\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 10 / 10\n",
"Time per batch: 1.2469687461853027\n",
"Mean batch time: 1.277336597442627\n",
"\u001b[1mINFO:niftynet:\u001b[0m reading size of preprocessed images\n",
"\u001b[1mWARNING:niftynet:\u001b[0m queue_length should be larger than batch_size, defaulting to batch_size * 5.0 (500).\n",
"\u001b[1mINFO:niftynet:\u001b[0m Initiating dataset...\n",
"\u001b[1mINFO:niftynet:\u001b[0m self.from_generator: True\n",
"\u001b[1mINFO:niftynet:\u001b[0m Initiating dataset from generator...\n",
"Num Parallel Calls: 16\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 1 / 10\n",
"Time per batch: 6.557478189468384\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 2 / 10\n",
"Time per batch: 1.01448655128479\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 3 / 10\n",
"Time per batch: 1.0860965251922607\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 4 / 10\n",
"Time per batch: 0.9710178375244141\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 5 / 10\n",
"Time per batch: 1.0350019931793213\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 6 / 10\n",
"Time per batch: 1.1063556671142578\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 7 / 10\n",
"Time per batch: 1.1326797008514404\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 8 / 10\n",
"Time per batch: 1.0397119522094727\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 9 / 10\n",
"Time per batch: 1.068472146987915\n",
"(100, 1, 100, 100, 100, 1, 1) (100, 1, 16, 1, 1, 1, 1)\n",
"Batch 10 / 10\n",
"Time per batch: 1.0207734107971191\n",
"Mean batch time: 1.0527328650156658\n"
]
}
],
"outputs": [],
"source": [
"import time\n",
"\n",
Expand Down
43 changes: 20 additions & 23 deletions niftynet/engine/image_window_dataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ class ImageWindowDataset(Layer):
"""

def __init__(self,
reader=None,
reader,
window_sizes=None,
batch_size=1,
windows_per_image=1,
Expand All @@ -50,13 +50,11 @@ def __init__(self,
name='image_dataset'):
Layer.__init__(self, name=name)

self.num_threads = 1

self.dataset = None
self.iterator = None
self.enqueuer = None
self.reader = reader

self.num_threads = 1
self.batch_size = batch_size
self.num_threads = num_threads
self.queue_length = int(max(queue_length, round(batch_size * 5.0)))
Expand All @@ -71,17 +69,14 @@ def __init__(self,
self.smaller_final_batch_mode = look_up_operations(
smaller_final_batch_mode.lower(), SMALLER_FINAL_BATCH_MODE)

self.n_subjects = 1
self.window = None
if reader is not None:
self.window = ImageWindow.from_data_reader_properties(
reader.input_sources,
reader.shapes,
reader.tf_dtypes,
window_sizes or (-1, -1, -1))
self.n_subjects = reader.num_subjects
self.window.n_samples = \
1 if self.from_generator else windows_per_image
self.reader = reader
self.window = ImageWindow.from_data_reader_properties(
reader.input_sources,
reader.shapes,
reader.tf_dtypes,
window_sizes or (-1, -1, -1))
self.window.n_samples = \
1 if self.from_generator else windows_per_image
# random seeds? (requires num_threads = 1)

@property
Expand Down Expand Up @@ -109,7 +104,7 @@ def tf_dtypes(self):
"""
returns a dictionary of sampler output tensorflow dtypes
"""
assert self.window, 'Unknown output shapes: self.window not initialised'
assert self.window, 'Unknown output dtypes: self.window not initialised'
return self.window.tf_dtypes

def layer_op(self, idx=None):
Expand All @@ -123,7 +118,7 @@ def layer_op(self, idx=None):
yield a dictionary
{
'image_name': a numpy array,
'image_name': a numpy array [h, w, d, chn],
'image_name_location': (image_id,
x_start, y_start, z_start,
x_end, y_end, z_end)
Expand All @@ -133,7 +128,7 @@ def layer_op(self, idx=None):
return a dictionary:
{
'image_name': a numpy array,
'image_name': a numpy array [n_samples, h, w, d, chn],
'image_name_location': [n_samples, 7]
}
Expand All @@ -157,6 +152,7 @@ def layer_op(self, idx=None):
assert self.window.n_samples == 1, \
'image_window_dataset.layer_op() requires: ' \
'windows_per_image should be 1.'

image_id, image_data, _ = self.reader(idx=idx)
for mod in list(image_data):
spatial_shape = image_data[mod].shape[:N_SPATIAL]
Expand Down Expand Up @@ -291,9 +287,10 @@ def _dataset_from_range(self):
:return: a `tf.data.Dataset`
"""
# dataset: a list of integers
dataset = tf.data.Dataset.range(self.n_subjects)
num_subjects = self.reader.num_subjects
dataset = tf.data.Dataset.range(num_subjects)
if self.shuffle:
dataset = dataset.shuffle(buffer_size=self.n_subjects, seed=None)
dataset = dataset.shuffle(buffer_size=num_subjects, seed=None)

# dataset: map each integer i to n windows sampled from subject i
def _tf_wrapper(idx):
Expand Down Expand Up @@ -366,7 +363,7 @@ def dummy_coordinates(cls, image_id, image_sizes, n_samples):
coords = [image_id] + starting_coordinates + image_spatial_shape
coords = np.tile(np.asarray(coords), [n_samples, 1])
return coords.astype(BUFFER_POSITION_NP_TYPE)

class ImageWindowDatasetCSV(ImageWindowDataset):
"""
Extending the default sampler to include csv data
Expand Down Expand Up @@ -397,7 +394,7 @@ def __init__(self,
epoch=epoch,
smaller_final_batch_mode=smaller_final_batch_mode,
name=name)

def layer_op(self, idx=None):
"""
Generating each image as a window.
Expand Down Expand Up @@ -454,7 +451,7 @@ def layer_op(self, idx=None):
image_data['label'] = label_data['label']
image_data['label_location'] = image_data['image_location']
return image_data

@property
def tf_shapes(self):
"""
Expand Down

0 comments on commit fd57e74

Please sign in to comment.