Merge branch 'dev' into update-paper-ref

.gitlab-ci.yml

@@ -197,6 +197,15 @@ quicktest:
     - QUICKTEST=True coverage run -a --source . -m unittest discover -s "tests" -p "*_test.py"
     - coverage report -m
 
+    # run global config tests
+    # These need to be run separately because NiftyNetGlobalConfig is a singleton, AND
+    # its operations pertain to a global configuration file (~/.niftynet/config.ini).
+    - GLOBAL_CONFIG_TEST_gcs=True python -m unittest tests.niftynet_global_config_test
+    - GLOBAL_CONFIG_TEST_necfc=True python -m unittest tests.niftynet_global_config_test
+    - GLOBAL_CONFIG_TEST_ecfl=True python -m unittest tests.niftynet_global_config_test
+    - GLOBAL_CONFIG_TEST_icfbu=True python -m unittest tests.niftynet_global_config_test
+    - GLOBAL_CONFIG_TEST_nenhc=True python -m unittest tests.niftynet_global_config_test
+
     - echo 'finished quick tests'
   tags:
     - gift-linux

README.md

@@ -66,6 +66,12 @@ The API reference is available on [Read the Docs][rtd-niftynet].
 
 [rtd-niftynet]: http://niftynet.rtfd.io/
 
+#### Customising NiftyNet
+
+Please see the [NiftyNet global settings][global-settings] section for customising NiftyNet's global settings.
+
+[global-settings]: ./config/README.md#global-niftynet-settings
+
 #### Contributing
 
 Please see the [contribution guidelines](./CONTRIBUTING.md).

config/README.md

@@ -1,3 +1,23 @@
+# Global NiftyNet settings
+
+The global NiftyNet configuration is read from `~/.niftynet/config.ini`.
+When NiftyNet is run, it will attempt to load this file for the global configuration.
+* If it does not exist, NiftyNet will create a default one.
+* If it exists but cannot be read for some reason (for instance incorrect formatting or wrong entries):
+   - NiftyNet will back it up with a timestamp (for instance `~/.niftynet/config-backup-2017-10-16-10-50-58-abc.ini` - `abc` being a random string) and,
+   - Create a default one.
+* Otherwise NiftyNet will read the global configuration from this file.
+
+Currently the minimal version of this file will look like the following:
+```ini
+[global]
+home = ~/niftynet
+```
+
+The `home` key specifies the root folder to be used by NiftyNet for storing and locating user data such as downloaded models, and new networks implemented by the user.
+This setting is configurable, and upon successfully loading this file NiftyNet will attempt to create the specified folder, if it does not already exist.
+
+
 # Configuration file
 
 To run a NiftyNet [application](../niftynet/application) or a customised

niftynet/utilities/niftynet_global_config.py

@@ -1,21 +1,111 @@
 # -*- coding: utf-8 -*-
 
-import os
-from os.path import expanduser
+from os.path import (expanduser, join, split, isdir, isfile, splitext)
+from os import (makedirs, rename)
+from random import choice
+from string import ascii_lowercase
+from time import strftime
+try:
+    from configparser import (ConfigParser, Error)
+except ImportError:
+    from ConfigParser import (ConfigParser, Error)
 from niftynet.utilities.decorators import singleton
 
 
 @singleton
 class NiftyNetGlobalConfig(object):
     """Global configuration settings"""
 
+    global_section = 'global'
+    home_key = 'home'
+
     def __init__(self):
         self._download_server_url = \
             'https://cmiclab.cs.ucl.ac.uk/CMIC/NiftyNetExampleServer'
-        self._config_home = os.path.join(expanduser("~"), '.niftynet')
+        self._config_home = join(expanduser('~'), '.niftynet')
+        self._config_file = join(self._config_home, 'config.ini')
+
+        config_opts = self.__load_or_create(self._config_file)
+
+        self._niftynet_home = expanduser(
+            config_opts[NiftyNetGlobalConfig.global_section][NiftyNetGlobalConfig.home_key])
+        if not isdir(self._niftynet_home):
+            makedirs(self._niftynet_home)
+
+    def __load_or_create(self, config_file):
+        """Load passed configuration file, if it exists; create a default
+        otherwise. If this method finds an incorrect config file, it
+        backs the file up with a human-readable timestamp suffix and
+        creates a default one.
+
+        :param config_file: no sanity checks are performed, as this
+        method is for internal use only
+        :type config_file: `os.path`
+        :returns: a dictionary of parsed configuration options
+        :rtype: `dict`
+        """
+        required_sections = [NiftyNetGlobalConfig.global_section]
+        required_keys = {
+            required_sections[0]: [NiftyNetGlobalConfig.home_key]
+        }
+        default_values = {
+            required_sections[0]: {
+                NiftyNetGlobalConfig.home_key: '~/niftynet'
+            }
+        }
+
+        backup = False
+        if isfile(config_file):
+            try:
+                config = ConfigParser()
+                config.read(config_file)
+
+                # check all required sections and keys present
+                for required_section in required_sections:
+                    if required_section not in config:
+                        backup = True
+                        break
+
+                    for required_key in required_keys[required_section]:
+                        if required_key not in config[required_section]:
+                            backup = True
+                            break
+
+                    if backup:
+                        break
+
+            except Error:
+                backup = True
+
+            if not backup:  # loaded file contains all required
+                            # config options: so return
+                return dict(config)
+
+        config_dir, config_filename = split(config_file)
+        if not isdir(config_dir):
+            makedirs(config_dir)
+
+        if backup:  # config file exists, but does not contain all required
+                    # config opts: so backup not to override
+            timestamp = strftime('%Y-%m-%d-%H-%M-%S')
+            random_str = ''.join(choice(ascii_lowercase) for _ in range(3))
+            backup_suffix = '-'.join(['backup', timestamp, random_str])
+
+            filename, extension = splitext(config_filename)
+            backup_filename = ''.join([filename, '-', backup_suffix, extension])
+            backup_file = join(config_dir, backup_filename)
+            rename(config_file, backup_file)
 
-        # ToDo: fetch NiftyNet home folder from a global configuration file
-        self._niftynet_home = os.path.join(expanduser("~"), 'niftynet')
+        # create a new default global config file
+        config = ConfigParser(default_values)
+        for required_section in required_sections:
+            for required_key in required_keys[required_section]:
+                config.add_section(required_section)
+                config[required_section][required_key] = \
+                    default_values[required_section][required_key]
+        with open(config_file, 'w') as new_config_file:
+            config.write(new_config_file)
+        return dict(config)
 
     def get_niftynet_home_folder(self):
         """Return the folder containing NiftyNet models and data"""
@@ -27,7 +117,7 @@ def get_niftynet_config_folder(self):
 
     def get_default_examples_folder(self):
         """Return the default folder containing NiftyNet examples"""
-        return os.path.join(self._niftynet_home, 'examples')
+        return join(self._niftynet_home, 'examples')
 
     def get_download_server_url(self):
         """Return the URL to the NiftyNet examples server"""

tests/niftynet_global_config_test.py

@@ -1,11 +1,113 @@
-from unittest import TestCase
+from unittest import (TestCase, skipUnless)
+from os.path import (expanduser, join, isdir, isfile)
+from os import (remove, makedirs, environ)
+from shutil import rmtree
+from glob import glob
 from niftynet.utilities.niftynet_global_config import NiftyNetGlobalConfig
 
 
 class NiftyNetGlobalConfigTest(TestCase):
 
+    @classmethod
+    def typify(cls, file_path):
+        """Append file type extension to passed file path."""
+        return '.'.join([file_path, cls.file_type])
+
+    @classmethod
+    def remove_path(cls, path):
+        """Remove passed item, whether it's a file or directory."""
+        if isdir(path):
+            rmtree(path)
+        elif isfile(path):
+            remove(path)
+
+    @classmethod
+    def setUpClass(cls):
+        cls.config_home = join(expanduser('~'), '.niftynet')
+        cls.file_type = 'ini'
+        cls.config_file = join(cls.config_home, cls.typify('config'))
+
+        cls.header = '[global]'
+        cls.default_config_opts = {
+            'home': '~/niftynet'
+        }
+
+    def setUp(self):
+        NiftyNetGlobalConfigTest.remove_path(NiftyNetGlobalConfigTest.config_home)
+        NiftyNetGlobalConfigTest.remove_path(
+            expanduser(NiftyNetGlobalConfigTest.default_config_opts['home'])
+        )
+
+    def tearDown(self):
+        self.setUp()
+
+    @skipUnless('GLOBAL_CONFIG_TEST_gcs' in environ,
+                'set GLOBAL_CONFIG_TEST_gcs to run')
     def test_global_config_singleton(self):
         global_config_1 = NiftyNetGlobalConfig()
         global_config_2 = NiftyNetGlobalConfig()
         self.assertEqual(global_config_1, global_config_2)
         self.assertTrue(global_config_1 is global_config_2)
+
+    @skipUnless('GLOBAL_CONFIG_TEST_necfc' in environ,
+                'set GLOBAL_CONFIG_TEST_necfc to run')
+    def test_non_existing_config_file_created(self):
+        self.assertFalse(isfile(NiftyNetGlobalConfigTest.config_file))
+        global_config = NiftyNetGlobalConfig()
+        self.assertTrue(isfile(NiftyNetGlobalConfigTest.config_file))
+        self.assertEqual(global_config.get_niftynet_config_folder(),
+                         NiftyNetGlobalConfigTest.config_home)
+
+    @skipUnless('GLOBAL_CONFIG_TEST_ecfl' in environ,
+                'set GLOBAL_CONFIG_TEST_ecfl to run')
+    def test_existing_config_file_loaded(self):
+        # create a config file with a custom NiftyNet home
+        makedirs(NiftyNetGlobalConfigTest.config_home)
+        custom_niftynet_home = '~/customniftynethome'
+        custom_niftynet_home_abs = expanduser(custom_niftynet_home)
+        config = ''.join(['home = ', custom_niftynet_home])
+        with open(NiftyNetGlobalConfigTest.config_file, 'w') as config_file:
+            config_file.write('\n'.join(
+                [NiftyNetGlobalConfigTest.header, config]))
+
+        global_config = NiftyNetGlobalConfig()
+        self.assertEqual(global_config.get_niftynet_home_folder(),
+                         custom_niftynet_home_abs)
+
+    @skipUnless('GLOBAL_CONFIG_TEST_icfbu' in environ,
+                'set GLOBAL_CONFIG_TEST_icfbu to run')
+    def test_incorrect_config_file_backed_up(self):
+        # create an incorrect config file at the correct location
+        makedirs(NiftyNetGlobalConfigTest.config_home)
+        incorrect_config = '\n'.join([NiftyNetGlobalConfigTest.header,
+                                      'invalid_home_tag = ~/niftynet'])
+        with open(NiftyNetGlobalConfigTest.config_file, 'w') as config_file:
+            config_file.write(incorrect_config)
+
+        # the following should back it up and replace it with default config
+        global_config = NiftyNetGlobalConfig()
+
+        self.assertTrue(isfile(NiftyNetGlobalConfigTest.config_file))
+        self.assertEqual(global_config.get_niftynet_config_folder(),
+                         NiftyNetGlobalConfigTest.config_home)
+
+        # check if incorrect file was backed up
+        found_files = glob(join(NiftyNetGlobalConfigTest.config_home,
+                                NiftyNetGlobalConfigTest.typify('config-backup-*')))
+        self.assertTrue(len(found_files) == 1)
+        with open(found_files[0], 'r') as backup_file:
+            self.assertEqual(backup_file.read(), incorrect_config)
+
+        # cleanup: remove backup file
+        NiftyNetGlobalConfigTest.remove_path(found_files[0])
+
+    @skipUnless('GLOBAL_CONFIG_TEST_nenhc' in environ,
+                'set GLOBAL_CONFIG_TEST_nenhc to run')
+    def test_non_existing_niftynet_home_created(self):
+        niftynet_home = expanduser(NiftyNetGlobalConfigTest.default_config_opts['home'])
+        NiftyNetGlobalConfigTest.remove_path(niftynet_home)
+        self.assertFalse(isdir(niftynet_home))
+
+        global_config = NiftyNetGlobalConfig()
+
+        self.assertTrue(isdir(niftynet_home))