Fixes for MongoDB 4.0 (IDSIA#858)

* Manually compute md5 hash for resources (pymongo 4.0 removed md5 support)

* "Fix" testcases for QueuedMongoObserver

- The testcases now pass through as before the update of pymongo, but the behavior of the QueuedMongoObserver is not correct!

* Remove unused import

* Add Python 3.10 test configurations

* Update pytest to support Python 3.10

* Remove _non_unicode_repr

- The default strings in Python 3 are unicode, they never print the "u''" prefix since Python 3.0 (see PEP 414)

* Skip pymongo tests for Python=3.10 due to mongomock

* Fix pytest version in tox configuration

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 fail_fast: true
 repos:
 -   repo: https://github.com/psf/black
-    rev: 21.5b1
+    rev: 22.1.0
     hooks:
     - id: black
       language_version: python3

azure-pipelines.yml

@@ -15,6 +15,10 @@ jobs:
         IMAGE_NAME: 'windows-latest'
         PYTHON_VERSION: '3.9'
         TOX_CMD: 'py39'
+      Windows py310:
+        IMAGE_NAME: 'windows-latest'
+        PYTHON_VERSION: '3.10'
+        TOX_CMD: 'py310'
       OSX py37:
         IMAGE_NAME: 'macOS-latest'
         PYTHON_VERSION: '3.7'
@@ -27,6 +31,10 @@ jobs:
         IMAGE_NAME: 'macOS-latest'
         PYTHON_VERSION: '3.9'
         TOX_CMD: 'py39'
+      OSX py310:
+        IMAGE_NAME: 'macOS-latest'
+        PYTHON_VERSION: '3.10'
+        TOX_CMD: 'py310'
       Linux py37:
         IMAGE_NAME: 'ubuntu-latest'
         PYTHON_VERSION: '3.7'
@@ -39,6 +47,10 @@ jobs:
         IMAGE_NAME: 'ubuntu-latest'
         PYTHON_VERSION: '3.9'
         TOX_CMD: 'py39'
+      Linux py310:
+        IMAGE_NAME: 'ubuntu-latest'
+        PYTHON_VERSION: '3.10'
+        TOX_CMD: 'py310'
       Linux numpy_117:
         IMAGE_NAME: 'ubuntu-latest'
         PYTHON_VERSION: '3.7'

dev-requirements.txt

@@ -1,4 +1,4 @@
-pytest==6.2.3 # tests/test_utils.py depends on that pytest version is exactly 6.2.3
+pytest==7.0.0 # tests/test_utils.py depends on that pytest version is exactly 7.0.0
 colorama
 docopt
 gitdb2
@@ -28,6 +28,6 @@ pymongo
 py-cpuinfo
 boto3
 moto
-google-compute-engine>
+google-compute-engine
 google-cloud-storage
 pre-commit

pyproject.toml

@@ -1,5 +1,5 @@
 [tool.black]
-target-version = ['py37', 'py38', 'py39']
+target-version = ['py37', 'py38', 'py39', 'py310']
 include = '\.pyi?$'
 exclude = '''
 (

sacred/commands.py

@@ -6,7 +6,6 @@
 import pprint
 import pydoc
 import re
-import sys
 from collections import namedtuple, OrderedDict
 
 from colorama import Fore, Style
@@ -54,27 +53,7 @@
 PathEntry = namedtuple("PathEntry", "key added modified typechanged doc")
 
 
-def _non_unicode_repr(objekt, context, maxlevels, level):
-    """
-    Used to override the pprint format method to get rid of unicode prefixes.
-
-    E.g.: 'John' instead of u'John'.
-    """
-    if sys.version_info[0] == 3 and sys.version_info[1] >= 8:
-        repr_string, isreadable, isrecursive = pprint._safe_repr(
-            objekt, context, maxlevels, level, sort_dicts=None
-        )
-    else:
-        repr_string, isreadable, isrecursive = pprint._safe_repr(
-            objekt, context, maxlevels, level
-        )
-    if repr_string.startswith('u"') or repr_string.startswith("u'"):
-        repr_string = repr_string[1:]
-    return repr_string, isreadable, isrecursive
-
-
 PRINTER = pprint.PrettyPrinter()
-PRINTER.format = _non_unicode_repr
 
 
 def print_config(_run):

sacred/observers/mongo.py

@@ -291,9 +291,9 @@ def resource_event(self, filename):
                     self.run_entry["resources"].append(resource)
                     self.save()
                 return
-        with open(filename, "rb") as f:
-            file_id = self.fs.put(f, filename=filename)
-        md5hash = self.fs.get(file_id).md5
+        # Pymongo 4.0: GridFS removed support for md5, we now have to compute
+        # it manually
+        md5hash = get_digest(filename)
         self.run_entry["resources"].append((filename, md5hash))
         self.save()
 

setup.py

@@ -39,7 +39,7 @@
     package_data={"sacred": [os.path.join("data", "*"), "py.typed"]},
     scripts=[],
     install_requires=Path("requirements.txt").read_text().splitlines(),
-    tests_require=["mock>=3.0, <5.0", "pytest==6.2.3"],
+    tests_require=["mock>=3.0, <5.0", "pytest==7.0.0"],
     classifiers=list(filter(None, classifiers.split("\n"))),
     description="Facilitates automated and reproducible experimental research",
     long_description=Path("README.rst").read_text(encoding="utf-8"),

tests/test_commands.py

@@ -21,7 +21,6 @@
     _format_entry,
     help_for_command,
     _iterate_marked,
-    _non_unicode_repr,
     _format_named_configs,
     _format_named_config,
 )
@@ -30,13 +29,6 @@
 import sacred.optional as opt
 
 
-def test_non_unicode_repr():
-    p = pprint.PrettyPrinter()
-    p.format = _non_unicode_repr
-    # make sure there is no u' in the representation
-    assert p.pformat("HelloWorld") == "'HelloWorld'"
-
-
 @pytest.fixture
 def cfg():
     return {

tests/test_observers/test_mongo_observer.py

@@ -1,10 +1,18 @@
 import datetime
 import os
+import sys
 from glob import glob
 
 import mock
 import pytest
 
+if sys.version_info >= (3, 10):
+    pytest.skip(
+        "Skip pymongo tests for Python 3.10 because mongomock doesn't "
+        "support Python 3.10",
+        allow_module_level=True,
+    )
+
 from sacred.metrics_logger import ScalarMetricLogEntry, linearize_metrics
 
 pymongo = pytest.importorskip("pymongo")

tests/test_observers/test_queue_mongo_observer.py

@@ -2,6 +2,14 @@
 from sacred.observers.queue import QueueObserver
 import mock
 import pytest
+import sys
+
+if sys.version_info >= (3, 10):
+    pytest.skip(
+        "Skip pymongo tests for Python 3.10 because mongomock doesn't "
+        "support Python 3.10",
+        allow_module_level=True,
+    )
 
 from sacred.metrics_logger import ScalarMetricLogEntry, linearize_metrics
 
@@ -185,7 +193,7 @@ def test_mongo_observer_resource_event(mongo_obs, sample_run):
     mongo_obs.join()
 
     db_run = mongo_obs.runs.find_one()
-    assert db_run["resources"] == [[filename, md5]]
+    assert db_run["resources"][0] == [filename, md5]
 
 
 @pytest.fixture

tests/test_utils.py

@@ -190,7 +190,7 @@ def test_module_is_in_cache():
 
 def test_get_package_version():
     package_version = get_package_version("pytest")
-    assert str(package_version) == "6.2.3"
+    assert str(package_version) == "7.0.0"
 
 
 def test_parse_version():
@@ -200,9 +200,9 @@ def test_parse_version():
 
 def test_get_package_version_comparison():
     package_version = get_package_version("pytest")
-    current_version = parse_version("6.2.3")
+    current_version = parse_version("7.0.0")
     old_version = parse_version("6.2.0")
-    new_version = parse_version("6.2.4")
+    new_version = parse_version("7.2.4")
     assert package_version == current_version
     assert not package_version < current_version
     assert not package_version > current_version

tox.ini

@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py37, py38, py39, setup, flake8, numpy-117, numpy-118, numpy-119, numpy-120, tensorflow-115, tensorflow-20, tensorflow-21, tensorflow-22, tensorflow-23, tensorflow-24
+envlist = py37, py38, py39, py310, setup, flake8, numpy-117, numpy-118, numpy-119, numpy-120, tensorflow-115, tensorflow-20, tensorflow-21, tensorflow-22, tensorflow-23, tensorflow-24
 
 [testenv]
 deps =
@@ -111,7 +111,7 @@ commands =
 [testenv:setup]
 basepython = python
 deps =
-    pytest==6.2.3
+    pytest==7.0.0
     mock
 commands =
     pytest {posargs}