refactored import file function #release-note

examples/calculator/tests.py

@@ -7,7 +7,7 @@
 from pulsar.apps import rpc, http
 from pulsar.apps.test import dont_run_with_thread
 
-from manage import server, Root, Calculator
+from examples.calculator.manage import server, Root, Calculator
 
 
 class TestRpcOnThread(unittest.TestCase):

examples/chat/tests.py

@@ -7,7 +7,7 @@
 from pulsar.apps.test import dont_run_with_thread
 from pulsar.utils.system import json
 
-from manage import server
+from examples.chat.manage import server
 
 
 class Message(ws.WS):

examples/djchat/test_app.py

@@ -1,6 +1,4 @@
 """Tests django chat application"""
-import sys
-import os
 import unittest
 import asyncio
 
@@ -10,11 +8,7 @@
 from pulsar.utils.string import gen_unique_id
 
 try:
-    from manage import server
-    djangopath = os.path.join(os.path.dirname(__file__))
-    if djangopath not in sys.path:
-        sys.path.append(djangopath)
-
+    from examples.djchat.manage import server
 except ImportError:
     server = None
 

examples/djchat/test_pulse.py

@@ -1,11 +1,11 @@
 """Tests the pulse Command."""
 import unittest
 
+from pulsar.apps import wsgi
 try:
     from pulsar.apps.pulse.management.commands.pulse import Command
 except ImportError:
     Command = None
-from pulsar.apps import wsgi
 
 
 @unittest.skipUnless(Command, 'Requires django')

examples/echo/tests.py

@@ -6,7 +6,7 @@
                     run_in_loop, get_event_loop)
 from pulsar.apps.test import dont_run_with_thread
 
-from .manage import server, Echo, EchoServerProtocol
+from examples.echo.manage import server, Echo, EchoServerProtocol
 
 
 class TestEchoServerThread(unittest.TestCase):

examples/echoudp/tests.py

@@ -4,7 +4,7 @@
 from pulsar import send, new_event_loop, get_application
 from pulsar.apps.test import dont_run_with_thread
 
-from .manage import server, Echo, EchoUdpServerProtocol
+from examples.echoudp.manage import server, Echo, EchoUdpServerProtocol
 
 
 class TestEchoUdpServerThread(unittest.TestCase):

examples/flaskapp/tests.py

@@ -6,7 +6,7 @@
 from pulsar.apps.http import HttpClient
 from pulsar.apps.test import dont_run_with_thread
 
-from .manage import server
+from examples.flaskapp.manage import server
 
 
 class TestFlaskThread(unittest.TestCase):

examples/flaskgreen/tests.py

@@ -6,7 +6,7 @@
 from pulsar.apps.http import HttpClient
 from pulsar.apps.test import dont_run_with_thread
 
-from manage import FlaskGreen, log_connection
+from examples.flaskgreen.manage import FlaskGreen, log_connection
 
 
 class TestFlaskGreenThread(unittest.TestCase):

examples/helloworld/tests.py

@@ -6,7 +6,7 @@
 from pulsar.apps.http import HttpClient
 from pulsar.apps.test import dont_run_with_thread
 
-from manage import server
+from examples.helloworld.manage import server
 
 
 class TestHelloWorldThread(unittest.TestCase):

examples/philosophers/tests.py

@@ -4,7 +4,7 @@
 from pulsar import send
 from pulsar.apps.test import test_timeout
 
-from .manage import DiningPhilosophers
+from examples.philosophers.manage import DiningPhilosophers
 
 
 class TestPhylosophers(unittest.TestCase):

examples/websocket/tests.py

@@ -7,7 +7,7 @@
 from pulsar.apps.http import HttpClient
 from pulsar.apps.test import dont_run_with_thread
 
-from .manage import server, frame_parser
+from examples.websocket.manage import server, frame_parser
 
 
 class Echo(WS):

pulsar/apps/test/__init__.py

@@ -357,6 +357,7 @@ class MyTestCase(unittest.TestCase):
 class TestVerbosity(TestOption):
     name = 'verbosity'
     flags = ['--verbosity']
+    validator = pulsar.validate_pos_int
     type = int
     default = 1
     desc = """Test verbosity, 0, 1, 2, 3"""

pulsar/apps/test/loader.py

@@ -8,14 +8,15 @@
 import re
 import sys
 import unittest
-from importlib import import_module
 
 import pulsar
+from pulsar.utils.importer import import_system_file
 
 from .result import TestRunner
 
 
-no_tags = ('tests', 'test')
+no_tags = set(('tests', 'test'))
+build_directories = set(('build', 'dist'))
 test_patterns = [re.compile(r'''test_(?P<name>.*).py'''),
                  re.compile(r'''(?P<name>.*)_test.py'''),
                  re.compile(r'''tests.py''')]
@@ -28,6 +29,12 @@ def issubclass_safe(cls, base_cls):
         return False
 
 
+def skip_module(mod_name):
+    return (mod_name.startswith('_') or
+            mod_name.startswith('.') or
+            mod_name in build_directories)
+
+
 class TestLoader:
     '''Classes used by the :class:`.TestSuite` to aggregate tests
     from a list of paths.
@@ -53,18 +60,15 @@ def tags(self, include=None, exclude=None):
         """Return a generator of tag information
         """
         for tag, file_path in self.test_files(include, exclude):
-            with self.import_module(file_path) as importer:
-                if importer.module:
-                    doc = importer.module.__doc__
-                    if doc:
-                        tag = '{0} - {1}'.format(tag, doc)
-                    yield tag
-
-    def test_modules(self, tags):
-        for tag, file_path in self.test_files(tags):
-            mod = self.import_module(file_path)
-            if mod:
-                yield mod
+            module = self.import_module(file_path).module()
+            if module:
+                doc = module.__doc__
+                if doc:
+                    tag = '{0} - {1}'.format(tag, doc)
+                yield tag
+
+    def import_module(self, file_name):
+        return ModuleImporter(file_name, self.logger)
 
     def test_files(self, tags=None, exclude_tags=None):
         """List of ``tag``, ``modules`` pairs.
@@ -110,18 +114,18 @@ def _check_tag(self, tag, import_tags, exclude_tags):
 
     def _test_files(self, include, exclude):
         if not self.modules:
-            yield from self.get_tests(self.root, include, exclude)
+            yield from self._get_tests(self.root, include, exclude)
         else:
             for name in self.modules:
                 absolute_path = os.path.join(self.root, name)
                 if os.path.isdir(absolute_path):
                     self.logger.debug('Loading from "%s"', name)
-                    yield from self.get_tests(absolute_path, include, exclude)
+                    yield from self._get_tests(absolute_path, include, exclude)
                 else:
                     raise ValueError('%s cannot be found in %s directory.'
                                      % (name, self.root))
 
-    def get_tests(self, path, include_tags, exclude_tags, tags=None):
+    def _get_tests(self, path, include_tags, exclude_tags, tags=None):
         """Collect python modules for testing.
 
             :parameter path: directory path where to search. Files starting
@@ -137,73 +141,62 @@ def get_tests(self, path, include_tags, exclude_tags, tags=None):
             tags = []
 
         for mod_name in os.listdir(path):
-            if mod_name.startswith('_') or mod_name.startswith('.'):
+            if skip_module(mod_name):
                 continue
             mod_path = os.path.join(path, mod_name)
             is_file = os.path.isfile(mod_path)
             if is_file:
-                tag = self.match(mod_name)
+                tag = self._match(mod_name)
                 if tag is None:  # does not match and is a file, skip.
                     continue
                 m_tags = tags + list(tag)
             else:
                 m_tags = tags if mod_name in no_tags else tags + [mod_name]
 
             tag = '.'.join(m_tags)
-            c = self._check_tag(tag, include_tags, exclude_tags)
-            if not c:
-                continue
+            if tag or is_file:
+                c = self._check_tag(tag, include_tags, exclude_tags)
+                if not c or (is_file and c < 2):
+                    continue
 
             if is_file:
                 yield tag, mod_path
             else:
-                yield from self.get_tests(mod_path, include_tags,
-                                          exclude_tags, m_tags)
-
-    def import_module(self, file_name):
-        return ModuleImporter(file_name, self.logger)
+                yield from self._get_tests(mod_path, include_tags,
+                                           exclude_tags, m_tags)
 
-    def match(self, name):
+    def _match(self, name):
         for pattern in test_patterns:
             p = pattern.search(name)
             if p:
                 return p.groups(0)
 
 
 class ModuleImporter:
-    module = None
-    add_to_path = False
 
     def __init__(self, file_name, logger):
         self.file_name = file_name
         self.logger = logger
 
     def __iter__(self):
-        with self:
-            if self.module:
-                for name in dir(self.module):
-                    obj = getattr(self.module, name)
-                    if issubclass_safe(obj, unittest.TestCase):
-                        yield obj
-
-    def __enter__(self):
-        path, name = os.path.split(self.file_name)
-        add_to_path = path not in sys.path
-        if add_to_path:
-            sys.path.insert(0, path)
-            self.add_to_path = add_to_path
-
-        try:
-            self.module = import_module(name[:-3])
-        except ImportError:
-            self.logger.error('Failed to import module %s. Skipping.',
-                              name, exc_info=True)
-            self.logger.debug('Full python path:\n%s', '\n'.join(sys.path))
-        except Exception:
-            self.logger.critical('Failed to import module %s. Skipping.',
-                                 name, exc_info=True)
-        return self
-
-    def __exit__(self, *args):
-        if self.add_to_path:
-            sys.path.pop(0)
+        module = self.module()
+        if module:
+            for name in dir(module):
+                obj = getattr(module, name)
+                if issubclass_safe(obj, unittest.TestCase):
+                    yield obj
+
+    def module(self):
+        if not hasattr(self, '_module'):
+            module = None
+            try:
+                module = import_system_file(self.file_name, False)
+            except ImportError:
+                self.logger.error('Failed to import module %s. Skipping.',
+                                  self.file_name, exc_info=True)
+                self.logger.debug('Full python path:\n%s', '\n'.join(sys.path))
+            except Exception:
+                self.logger.critical('Failed to import module %s. Skipping.',
+                                     self.file_name, exc_info=True)
+            self._module = module
+        return self._module

pulsar/apps/test/setup.py

@@ -71,7 +71,10 @@ def finalize_options(self):
                 setting = cfg.settings[name]
 
                 if setting.nargs in ('*', '+'):
-                    value = shlex.split(value)
+                    values = []
+                    for v in shlex.split(value):
+                        values.extend((c for c in v.split(',') if c))
+                    value = values
 
                 setattr(self, name, value)
                 params[name] = value

pulsar/utils/importer.py

@@ -1,8 +1,28 @@
 import os
-import sys
 import glob
 from importlib import import_module
 
+import importlib.util
+
+from .slugify import slugify
+
+try:
+    importlib.util.module_from_spec
+
+    def _import_system_file(filename):
+        module_name = slugify(filename, '_')
+        spec = importlib.util.spec_from_file_location(module_name, filename)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        return module
+
+except AttributeError:
+    from importlib.machinery import SourceFileLoader
+
+    def _import_system_file(filename):
+        module_name = slugify(filename, '_')
+        return SourceFileLoader(module_name, filename).load_module()
+
 
 def expand_star(mod_name):
     """Expand something like 'unuk.tasks.*' into a list of all the modules
@@ -78,31 +98,13 @@ def py_file(name):
         return name
 
 
-def import_system_file(mod, add_to_path=True):
-    if os.path.isfile(mod):
-        # it is a file in the system path
-        dir, name = os.path.split(mod)
-        name = py_file(name)
-        assert name
-        names = [name]
-        while dir and dir not in sys.path:
-            ndir, name = os.path.split(dir)
-            if dir == ndir:
-                dir = ''
-                break
-            dir = ndir
-            names.insert(0, name)
-        # the file was not in the system path
-        if not dir and add_to_path:
-            dir, name = os.path.split(mod)
-            if dir and dir != mod:
-                sys.path.append(dir)
-            mod_name = py_file(name)
+def import_system_file(mod, safe=True):
+    try:
+        if os.path.isfile(mod):
+            return _import_system_file(mod)
         else:
-            mod_name = '.'.join(names)
-        return import_module(mod_name)
-    else:
-        try:
             return import_module(mod)
-        except ImportError:
-            pass
+    except ImportError:
+        if not safe:
+            raise
+        pass

setup.cfg

@@ -11,4 +11,3 @@ exclude = __pycache__,dist,build,docs
 [pulsar_test]
 test_timeout = 10
 pid_file = test.pid
-test_modules = tests examples

tests/http/test_client.py

@@ -1,4 +1,4 @@
-import base
+from tests.http import base
 
 
 class TestHttpClient(base.TestHttpClient):