Merge pull request nilearn#959 from aabadie/mem_check

[MRG] Memory check based on memory profiler

continuous_integration/install.sh

@@ -102,6 +102,8 @@ else
     exit 1
 fi
 
+pip install psutil memory_profiler
+
 if [[ "$COVERAGE" == "true" ]]; then
     pip install coverage coveralls
 fi

nilearn/_utils/testing.py

@@ -1,6 +1,5 @@
-"""Utilities for testing nilearn.
-"""
-# Author: Alexandre Abrahame, Philippe Gervais
+"""Utilities for testing nilearn."""
+# Author: Alexandre Abraham, Philippe Gervais
 # License: simplified BSD
 import contextlib
 import functools
@@ -10,6 +9,7 @@
 import sys
 import tempfile
 import warnings
+import gc
 
 import numpy as np
 import scipy.signal
@@ -60,6 +60,65 @@ def assert_warns(warning_class, func, *args, **kw):
         return output
 
 
+# we use memory_profiler library for memory consumption checks
+try:
+    from memory_profiler import memory_usage
+
+    def with_memory_profiler(func):
+        """A decorator to skip tests requiring memory_profiler."""
+        return func
+
+    def memory_used(func, *args, **kwargs):
+        """Compute memory usage when executing func."""
+        gc.collect()
+        mem_use = memory_usage((func, args, kwargs), interval=0.001)
+        return max(mem_use) - min(mem_use)
+
+except ImportError:
+    def with_memory_profiler(func):
+        """A decorator to skip tests requiring memory_profiler."""
+        def dummy_func():
+            import nose
+            raise nose.SkipTest('Test requires memory_profiler.')
+        return dummy_func
+
+    memory_usage = memory_used = None
+
+
+def assert_memory_less_than(memory_limit, tolerance,
+                            callable_obj, *args, **kwargs):
+    """Check memory consumption of a callable stays below a given limit.
+
+    Parameters
+    ----------
+    memory_limit : int
+        The expected memory limit in MiB.
+    tolerance: float
+        As memory_profiler results have some variability, this adds some
+        tolerance around memory_limit. Accepted values are in range [0.0, 1.0].
+    callable_obj: callable
+        The function to be called to check memory consumption.
+
+    """
+    mem_used = memory_used(callable_obj, *args, **kwargs)
+
+    if mem_used > memory_limit * (1 + tolerance):
+        raise ValueError("Memory consumption measured ({0:.2f} MiB) is "
+                         "greater than required memory limit ({1} MiB) within "
+                         "accepted tolerance ({2:.2f}%)."
+                         "".format(mem_used, memory_limit, tolerance * 100))
+
+    # We are confident in memory_profiler measures above 100MiB.
+    # We raise an error if the measure is below the limit of 50MiB to avoid
+    # false positive.
+    if mem_used < 50:
+        raise ValueError("Memory profiler measured an untrustable memory "
+                         "consumption ({0:.2f} MiB). The expected memory "
+                         "limit was {1:.2f} MiB. Try to bench with larger "
+                         "objects (at least 100MiB in memory).".
+                         format(mem_used, memory_limit))
+
+
 class MockRequest(object):
     def __init__(self, url):
         self.url = url

nilearn/tests/test_niimg_conversions.py

@@ -24,6 +24,8 @@
 from nilearn._utils.exceptions import DimensionError
 from nilearn._utils import testing, niimg_conversions
 from nilearn._utils.testing import assert_raises_regex
+from nilearn._utils.testing import with_memory_profiler
+from nilearn._utils.testing import assert_memory_less_than
 from nilearn._utils.niimg_conversions import _iter_check_niimg
 
 
@@ -341,6 +343,23 @@ def test_iter_check_niimgs():
                        _utils.check_niimg(img_2_4d).get_data())
 
 
+def _check_memory(list_img_3d):
+    # We intentionally add an offset of memory usage to avoid non trustable
+    # measures with memory_profiler.
+    mem_offset = b'a' * 100 * 1024 ** 2
+    list(_iter_check_niimg(list_img_3d))
+    return mem_offset
+
+
+@with_memory_profiler
+def test_iter_check_niimgs_memory():
+    # Verify that iterating over a list of images doesn't consume extra
+    # memory.
+    assert_memory_less_than(100, 0.1, _check_memory,
+                            [Nifti1Image(np.ones((100, 100, 200)), np.eye(4))
+                             for i in range(10)])
+
+
 def test_repr_niimgs():
     # Test with file path
     assert_equal(_utils._repr_niimgs("test"), "test")

nilearn/tests/test_testing.py

@@ -1,10 +1,37 @@
 import itertools
-
 import numpy as np
 
 from nose.tools import assert_equal, assert_raises
 
-from nilearn._utils.testing import generate_fake_fmri
+from nilearn._utils.testing import generate_fake_fmri, with_memory_profiler
+from nilearn._utils.testing import assert_memory_less_than, assert_raises_regex
+
+
+def create_object(size):
+    """Just create and return an object containing `size` bytes."""
+    mem_use = b'a' * size
+    return mem_use
+
+
+@with_memory_profiler
+def test_memory_usage():
+    # Valid measures
+    for mem in (500, 200, 100):
+        assert_memory_less_than(mem, 0.1, create_object, mem * 1024 ** 2)
+
+    # Ensure an exception is raised with too small objects as
+    # memory_profiler can return non trustable memory measure in this case.
+    assert_raises_regex(ValueError,
+                        "Memory profiler measured an untrustable memory",
+                        assert_memory_less_than, 50, 0.1,
+                        create_object, 25 * 1024 ** 2)
+
+    # Ensure ValueError is raised if memory used is above expected memory
+    # limit.
+    assert_raises_regex(ValueError,
+                        "Memory consumption measured",
+                        assert_memory_less_than, 50, 0.1,
+                        create_object, 100 * 1024 ** 2)
 
 
 def test_generate_fake_fmri():