test compileall; only compile files in packages (not deps)

conda_build/post.py

@@ -22,7 +22,7 @@
 from conda_build import environ
 from conda_build import utils
 from conda_build import source
-from conda.compat import lchmod
+from conda.compat import lchmod, PY3
 from conda.misc import walk_prefix
 from conda.utils import md5_file
 
@@ -141,28 +141,45 @@ def rm_py_along_so():
                         os.unlink(join(root, name + ext))
 
 
-def compile_missing_pyc():
-    sp_dir = environ.get_sp_dir()
-    stdlib_dir = environ.get_stdlib_dir()
+def coerce_pycache_to_old_style(files, cwd):
+    """
+    For the sake of simplicity, remove the filename additions, like .cpython-35.pyc
 
-    need_compile = False
-    for root, dirs, files in os.walk(sp_dir):
-        for fn in files:
-            if fn.endswith('.py') and fn + 'c' not in files:
-                need_compile = True
-                break
-    if need_compile:
+    Since Conda allows only one Python install in a given prefix, there is no reason
+    for these additional suffixes.  Newer Python will find these pyc files without issue.
+    """
+    for f in files:
+        if not os.path.exists(f):
+            f = os.path.join(cwd, f)
+        if not os.path.isfile(f) or not f.endswith('py'):
+            continue
+        if '/' in f or '\\' in f:
+            folder = os.path.join(cwd, os.path.dirname(f), '__pycache__')
+        else:
+            folder = os.path.join(cwd, '__pycache__')
+        fname = os.path.join(folder, os.path.splitext(os.path.basename(f))[0] +
+                 '.cpython-{0}{1}.pyc'.format(sys.version_info.major,
+                                              sys.version_info.minor))
+        if os.path.isfile(fname):
+            os.rename(fname, f + 'c')
+    for root, folders, files in os.walk(cwd):
+        if root.endswith("__pycache__") and not files:
+            os.rmdir(root)
+
+
+def compile_missing_pyc(files, cwd=config.build_prefix, python_exe=config.build_python):
+    compile_files = [f for f in files if f.endswith('py') and f + 'c' not in files]
+    if compile_files:
         print('compiling .pyc files...')
-        call([config.build_python, '-Wi',
-                           join(stdlib_dir, 'compileall.py'),
-                           '-q', '-x', 'port_v3', sp_dir])
+        call([python_exe, '-Wi', '-m', 'py_compile'] + compile_files, cwd=cwd)
+        if PY3:
+            coerce_pycache_to_old_style(compile_files, cwd=cwd)
 
 
 def post_process(files, preserve_egg_dir=False):
     remove_easy_install_pth(files, preserve_egg_dir=preserve_egg_dir)
     rm_py_along_so()
-    if config.CONDA_PY < 30:
-        compile_missing_pyc()
+    compile_missing_pyc(files)
 
 
 def find_lib(link, path=None):

setup.cfg

@@ -1,7 +1,7 @@
 [flake8]
 max-line-length = 100
 ignore = E122,E123,E126,E127,E128,E731
-exclude = build,conda_build/_version.py
+exclude = build,conda_build/_version.py,tests/test-recipes/metadata/_compile-test/f2_bad.py
 
 [pytest]
 norecursedirs= tests/test-recipes .* *.egg* build dist

tests/test-recipes/metadata/_compile-test/f1.py

@@ -0,0 +1 @@
+print("file 1")

tests/test-recipes/metadata/_compile-test/f2_bad.py

@@ -0,0 +1 @@
+I am a bad file that should not pass compileall.

tests/test-recipes/metadata/_compile-test/f3.py

@@ -0,0 +1 @@
+print("File 3")

tests/test-recipes/metadata/_compile-test/meta.yaml

@@ -0,0 +1,20 @@
+# This recipe tests that all files are compiled with compileall.  Anything that can't be
+# compiled remains as a .py file, but this should not kill the build.
+
+package:
+  name: test_compileall
+  version: 1.0
+
+source:
+  path: .
+
+build:
+  script:
+    - cp *.py $PREFIX  # [unix]
+    - copy *.py %PREFIX%  # [win]
+
+requirements:
+  build:
+    - python
+  run:
+    - python

tests/test_build_recipes.py

@@ -573,6 +573,24 @@ def test_requirements_txt_for_run_reqs():
     subprocess.check_call(cmd.split())
 
 
+def test_compileall_compiles_all_good_files():
+    output_file = os.path.join(sys.prefix, "conda-bld", subdir,
+                               'test_compileall-1.0-py{0}{1}_0.tar.bz2'.format(
+                                    sys.version_info.major, sys.version_info.minor))
+    cmd = 'conda build --no-anaconda-upload {}'.format(os.path.join(metadata_dir,
+                                                                    "_compile-test"),
+                                                       env=os.environ.copy())
+    subprocess.check_call(cmd.split())
+    good_files = ['f1.py', 'f3.py']
+    bad_file = 'f2_bad.py'
+    for f in good_files:
+        assert package_has_file(output_file, f)
+        # look for the compiled file also
+        assert package_has_file(output_file, f + 'c')
+    assert package_has_file(output_file, bad_file)
+    assert not package_has_file(output_file, bad_file + 'c')
+
+
 def test_rendering_env_var():
     """
     This environment variable is provided for users to selectively change what they do

tests/test_post.py

@@ -0,0 +1,63 @@
+import os
+import shutil
+import sys
+
+from conda.compat import TemporaryDirectory, PY3
+import pytest
+
+from conda_build import post
+
+
+def test_compile_missing_pyc():
+    cwd = os.getcwd()
+    good_files = ['f1.py', 'f3.py']
+    bad_file = 'f2_bad.py'
+    with TemporaryDirectory() as tmp:
+        tmpdir = os.path.join(tmp, 'files')
+        shutil.copytree(os.path.join(os.path.dirname(__file__), 'test-recipes',
+                                     'metadata', '_compile-test'),
+                        tmpdir)
+        os.chdir(tmpdir)
+        try:
+            post.compile_missing_pyc(os.listdir(tmpdir), cwd=tmpdir, python_exe=sys.executable)
+            files = os.listdir(tmpdir)
+            for f in good_files:
+                assert f + 'c' in files
+            assert bad_file + 'c' not in files
+        except:
+            raise
+        finally:
+            os.chdir(cwd)
+
+
+@pytest.mark.skipif(not PY3, reason="test applies to py3 only")
+def test_coerce_pycache_to_old_style():
+    cwd = os.getcwd()
+    with TemporaryDirectory() as tmp:
+        os.makedirs(os.path.join(tmp, '__pycache__'))
+        os.makedirs(os.path.join(tmp, 'testdir', '__pycache__'))
+        with open(os.path.join(tmp, 'test.py'), 'w') as f:
+            f.write("\n")
+        with open(os.path.join(tmp, '__pycache__', 'test.cpython-{0}{1}.pyc'.format(
+                sys.version_info.major, sys.version_info.minor)), 'w') as f:
+            f.write("\n")
+        with open(os.path.join(tmp, 'testdir', 'test.py'), 'w') as f:
+            f.write("\n")
+        with open(os.path.join(tmp, 'testdir', '__pycache__', 'test.cpython-{0}{1}.pyc'.format(
+                sys.version_info.major, sys.version_info.minor)), 'w') as f:
+            f.write("\n")
+
+        os.chdir(tmp)
+        for root, dirs, files in os.walk(tmp):
+            fs = [os.path.join(root, _) for _ in files]
+            post.coerce_pycache_to_old_style(fs, cwd=tmp)
+        try:
+            assert os.path.isfile(os.path.join(tmp, 'test.pyc')), os.listdir(tmp)
+            assert os.path.isfile(os.path.join(tmp, 'testdir', 'test.pyc')), \
+                os.listdir(os.path.join(tmp, 'testdir'))
+            for root, dirs, files in os.walk(tmp):
+                assert '__pycache__' not in dirs
+        except:
+            raise
+        finally:
+            os.chdir(cwd)