From 3dcd2e507245ceda8fb0509a8bcc8fa6e6be5312 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sat, 30 Jul 2016 21:22:26 -0500 Subject: [PATCH 1/8] test compileall functionality. Rework to only compile files in package (not deps) --- conda_build/post.py | 21 +++++++------------ .../test-recipes/metadata/_compile-test/f1.py | 1 + .../metadata/_compile-test/f2_bad.py | 1 + .../test-recipes/metadata/_compile-test/f3.py | 1 + .../metadata/_compile-test/meta.yaml | 20 ++++++++++++++++++ tests/test_build_recipes.py | 16 ++++++++++++++ 6 files changed, 47 insertions(+), 13 deletions(-) create mode 100644 tests/test-recipes/metadata/_compile-test/f1.py create mode 100644 tests/test-recipes/metadata/_compile-test/f2_bad.py create mode 100644 tests/test-recipes/metadata/_compile-test/f3.py create mode 100644 tests/test-recipes/metadata/_compile-test/meta.yaml diff --git a/conda_build/post.py b/conda_build/post.py index 4c40acf5c7..5b2e1df98c 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -141,28 +141,23 @@ def rm_py_along_so(): os.unlink(join(root, name + ext)) -def compile_missing_pyc(): - sp_dir = environ.get_sp_dir() - stdlib_dir = environ.get_stdlib_dir() +def compile_missing_pyc(files): need_compile = False - for root, dirs, files in os.walk(sp_dir): - for fn in files: - if fn.endswith('.py') and fn + 'c' not in files: - need_compile = True - break + for fn in files: + if fn.endswith('.py') and fn + 'c' not in files: + need_compile = True + break if need_compile: print('compiling .pyc files...') - call([config.build_python, '-Wi', - join(stdlib_dir, 'compileall.py'), - '-q', '-x', 'port_v3', sp_dir]) + call([config.build_python, '-Wi', '-m', 'py_compile'] + files, + cwd=config.build_prefix) def post_process(files, preserve_egg_dir=False): remove_easy_install_pth(files, preserve_egg_dir=preserve_egg_dir) rm_py_along_so() - if config.CONDA_PY < 30: - compile_missing_pyc() + compile_missing_pyc(files) def find_lib(link, path=None): diff --git a/tests/test-recipes/metadata/_compile-test/f1.py b/tests/test-recipes/metadata/_compile-test/f1.py new file mode 100644 index 0000000000..ad3e0973bc --- /dev/null +++ b/tests/test-recipes/metadata/_compile-test/f1.py @@ -0,0 +1 @@ +print("file 1") diff --git a/tests/test-recipes/metadata/_compile-test/f2_bad.py b/tests/test-recipes/metadata/_compile-test/f2_bad.py new file mode 100644 index 0000000000..e56eb9712c --- /dev/null +++ b/tests/test-recipes/metadata/_compile-test/f2_bad.py @@ -0,0 +1 @@ +I am a bad file that should not pass compileall. diff --git a/tests/test-recipes/metadata/_compile-test/f3.py b/tests/test-recipes/metadata/_compile-test/f3.py new file mode 100644 index 0000000000..a64d6ff3c4 --- /dev/null +++ b/tests/test-recipes/metadata/_compile-test/f3.py @@ -0,0 +1 @@ +print("File 3") diff --git a/tests/test-recipes/metadata/_compile-test/meta.yaml b/tests/test-recipes/metadata/_compile-test/meta.yaml new file mode 100644 index 0000000000..8e0d03e446 --- /dev/null +++ b/tests/test-recipes/metadata/_compile-test/meta.yaml @@ -0,0 +1,20 @@ +# This recipe tests that all files are compiled with compileall. Anything that can't be +# compiled remains as a .py file, but this should not kill the build. + +package: + name: test_compileall + version: 1.0 + +source: + path: . + +build: + script: + - cp *.py $PREFIX # [unix] + - copy *.py %PREFIX% # [win] + +requirements: + build: + - python + run: + - python diff --git a/tests/test_build_recipes.py b/tests/test_build_recipes.py index a81f94e6e3..6f81a7f1b2 100644 --- a/tests/test_build_recipes.py +++ b/tests/test_build_recipes.py @@ -571,3 +571,19 @@ def test_requirements_txt_for_run_reqs(): cmd = 'conda build --no-anaconda-upload -c conda-forge {}'.format(os.path.join(metadata_dir, "_requirements_txt_run_reqs")) subprocess.check_call(cmd.split()) + + +def test_compileall_compiles_all_good_files(): + output_file = os.path.join(sys.prefix, "conda-bld", subdir, + 'test_compileall-1.0-py27_0.tar.bz2') + cmd = 'conda build --no-anaconda-upload {}'.format(os.path.join(metadata_dir, + "_compile-test")) + subprocess.check_call(cmd.split()) + good_files = ['f1.py', 'f3.py'] + bad_file = 'f2_bad.py' + for f in good_files: + assert package_has_file(output_file, f) + # look for the compiled file also + assert package_has_file(output_file, f + 'c') + assert package_has_file(output_file, bad_file) + assert not package_has_file(output_file, bad_file + 'c') From 8eeda6af5fe14f7864329c6e61bbbfd0e692b120 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sat, 30 Jul 2016 21:58:28 -0500 Subject: [PATCH 2/8] exclude known bad file from flake8 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 02ed3db507..8718875520 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [flake8] max-line-length = 100 ignore = E122,E123,E126,E127,E128,E731 -exclude = build,conda_build/_version.py +exclude = build,conda_build/_version.py,tests/test-recipes/metadata/_compile-test/f2_bad.py [pytest] norecursedirs= tests/test-recipes .* *.egg* build dist From 9e943e40d79c1bd5db0f682c2d652c13bdb7611a Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sat, 30 Jul 2016 22:44:47 -0500 Subject: [PATCH 3/8] fix compileall test for python3's different __pycache__ structure --- tests/test_build_recipes.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/test_build_recipes.py b/tests/test_build_recipes.py index 89c1b686f8..15a21f189b 100644 --- a/tests/test_build_recipes.py +++ b/tests/test_build_recipes.py @@ -575,7 +575,8 @@ def test_requirements_txt_for_run_reqs(): def test_compileall_compiles_all_good_files(): output_file = os.path.join(sys.prefix, "conda-bld", subdir, - 'test_compileall-1.0-py27_0.tar.bz2') + 'test_compileall-1.0-py{0}{1}_0.tar.bz2'.format( + sys.version_info.major, sys.version_info.minor)) cmd = 'conda build --no-anaconda-upload {}'.format(os.path.join(metadata_dir, "_compile-test")) subprocess.check_call(cmd.split()) @@ -584,9 +585,21 @@ def test_compileall_compiles_all_good_files(): for f in good_files: assert package_has_file(output_file, f) # look for the compiled file also - assert package_has_file(output_file, f + 'c') + if PY3: + assert package_has_file(output_file, os.path.join( + os.path.join('__pycache__', os.path.splitext(f)[0] + + '.cpython-{0}{1}.pyc'.format(sys.version_info.major, + sys.version_info.minor)))) + else: + assert package_has_file(output_file, f + 'c') assert package_has_file(output_file, bad_file) - assert not package_has_file(output_file, bad_file + 'c') + if PY3: + assert not package_has_file(output_file, os.path.join( + os.path.join('__pycache__', os.path.splitext(bad_file)[0] + + '.cpython-{0}{1}.pyc'.format(sys.version_info.major, + sys.version_info.minor)))) + else: + assert not package_has_file(output_file, bad_file + 'c') def test_rendering_env_var(): From d078aba810c998e4655773fdef95fe02088feb4a Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 31 Jul 2016 08:45:22 -0500 Subject: [PATCH 4/8] py3: move __pycache__ files up one; remove filename additions --- conda_build/post.py | 23 ++++++++++++++++++++++- tests/test_build_recipes.py | 16 ++-------------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/conda_build/post.py b/conda_build/post.py index 5b2e1df98c..9dc01b9288 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -22,7 +22,7 @@ from conda_build import environ from conda_build import utils from conda_build import source -from conda.compat import lchmod +from conda.compat import lchmod, PY3 from conda.misc import walk_prefix from conda.utils import md5_file @@ -141,6 +141,25 @@ def rm_py_along_so(): os.unlink(join(root, name + ext)) +def coerce_pycache_to_old_style(files): + """ + For the sake of simplicity, remove the filename additions, like .cpython-35.pyc + + Since Conda allows only one Python install in a given prefix, there is no reason + for these additional suffixes. Newer Python will find these pyc files without issue. + """ + for f in files: + if '/' in f: + folder = os.path.join(os.path.dirname(f), '__pycache__') + else: + folder = '__pycache__' + fname = os.path.join(folder, os.path.splitext(os.path.basename(f))[0] + + '.cpython-{0}{1}.pyc'.format(sys.version_info.major, + sys.version_info.minor)) + if os.path.isfile(fname): + os.rename(fname, f + 'c') + + def compile_missing_pyc(files): need_compile = False @@ -152,6 +171,8 @@ def compile_missing_pyc(files): print('compiling .pyc files...') call([config.build_python, '-Wi', '-m', 'py_compile'] + files, cwd=config.build_prefix) + if PY3: + coerce_pycache_to_old_style(files) def post_process(files, preserve_egg_dir=False): diff --git a/tests/test_build_recipes.py b/tests/test_build_recipes.py index 15a21f189b..28b38bb234 100644 --- a/tests/test_build_recipes.py +++ b/tests/test_build_recipes.py @@ -585,21 +585,9 @@ def test_compileall_compiles_all_good_files(): for f in good_files: assert package_has_file(output_file, f) # look for the compiled file also - if PY3: - assert package_has_file(output_file, os.path.join( - os.path.join('__pycache__', os.path.splitext(f)[0] + - '.cpython-{0}{1}.pyc'.format(sys.version_info.major, - sys.version_info.minor)))) - else: - assert package_has_file(output_file, f + 'c') + assert package_has_file(output_file, f + 'c') assert package_has_file(output_file, bad_file) - if PY3: - assert not package_has_file(output_file, os.path.join( - os.path.join('__pycache__', os.path.splitext(bad_file)[0] + - '.cpython-{0}{1}.pyc'.format(sys.version_info.major, - sys.version_info.minor)))) - else: - assert not package_has_file(output_file, bad_file + 'c') + assert not package_has_file(output_file, bad_file + 'c') def test_rendering_env_var(): From 8d7a862f09ece225f841923a0800b258db57c226 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 31 Jul 2016 11:31:58 -0500 Subject: [PATCH 5/8] fix __pycache__ coercion, add tests --- conda_build/post.py | 20 ++++++--------- tests/test_post.py | 62 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 12 deletions(-) create mode 100644 tests/test_post.py diff --git a/conda_build/post.py b/conda_build/post.py index 9dc01b9288..e779d11185 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -148,7 +148,7 @@ def coerce_pycache_to_old_style(files): Since Conda allows only one Python install in a given prefix, there is no reason for these additional suffixes. Newer Python will find these pyc files without issue. """ - for f in files: + for f in [f for f in files if os.path.isfile(f) and not f.endswith('pyc')]: if '/' in f: folder = os.path.join(os.path.dirname(f), '__pycache__') else: @@ -158,21 +158,17 @@ def coerce_pycache_to_old_style(files): sys.version_info.minor)) if os.path.isfile(fname): os.rename(fname, f + 'c') + if os.path.isdir(folder): + os.rmdir(folder) -def compile_missing_pyc(files): - - need_compile = False - for fn in files: - if fn.endswith('.py') and fn + 'c' not in files: - need_compile = True - break - if need_compile: +def compile_missing_pyc(files, cwd=config.build_prefix, python_exe=config.build_python): + compile_files = [f for f in files if f.endswith('py') and f + 'c' not in files] + if compile_files: print('compiling .pyc files...') - call([config.build_python, '-Wi', '-m', 'py_compile'] + files, - cwd=config.build_prefix) + call([python_exe, '-Wi', '-m', 'py_compile'] + compile_files, cwd=cwd) if PY3: - coerce_pycache_to_old_style(files) + coerce_pycache_to_old_style(compile_files) def post_process(files, preserve_egg_dir=False): diff --git a/tests/test_post.py b/tests/test_post.py new file mode 100644 index 0000000000..5fd760eda5 --- /dev/null +++ b/tests/test_post.py @@ -0,0 +1,62 @@ +import os +import shutil +import sys + +from conda.compat import TemporaryDirectory, PY3 +import pytest + +from conda_build import post + + +def test_compile_missing_pyc(): + cwd = os.getcwd() + good_files = ['f1.py', 'f3.py'] + bad_file = 'f2_bad.py' + with TemporaryDirectory() as tmp: + tmpdir = os.path.join(tmp, 'files') + shutil.copytree(os.path.join(os.path.dirname(__file__), 'test-recipes', + 'metadata', '_compile-test'), + tmpdir) + os.chdir(tmpdir) + try: + post.compile_missing_pyc(os.listdir(tmpdir), cwd=tmpdir, python_exe=sys.executable) + files = os.listdir(tmpdir) + for f in good_files: + assert f + 'c' in files + assert bad_file + 'c' not in files + except: + raise + finally: + os.chdir(cwd) + + +@pytest.mark.skipif(not PY3, reason="test applies to py3 only") +def test_coerce_pycache_to_old_style(): + cwd = os.getcwd() + with TemporaryDirectory() as tmp: + os.makedirs(os.path.join(tmp, '__pycache__')) + os.makedirs(os.path.join(tmp, 'testdir', '__pycache__')) + with open(os.path.join(tmp, 'test.py'), 'w') as f: + f.write("\n") + with open(os.path.join(tmp, '__pycache__', 'test.cpython-{0}{1}.pyc'.format( + sys.version_info.major, sys.version_info.minor)), 'w') as f: + f.write("\n") + with open(os.path.join(tmp, 'testdir', 'test.py'), 'w') as f: + f.write("\n") + with open(os.path.join(tmp, 'testdir', '__pycache__', 'test.cpython-{0}{1}.pyc'.format( + sys.version_info.major, sys.version_info.minor)), 'w') as f: + f.write("\n") + + os.chdir(tmp) + for root, dirs, files in os.walk(tmp): + fs = [os.path.join(root, f) for f in files] + post.coerce_pycache_to_old_style(fs) + try: + assert os.path.isfile(os.path.join(tmp, 'test.pyc')), os.listdir(tmp) + assert os.path.isfile(os.path.join(tmp, 'testdir', 'test.pyc')), os.listdir(os.path.join(tmp, 'testdir')) + for root, dirs, files in os.walk(tmp): + assert '__pycache__' not in dirs + except: + raise + finally: + os.chdir(cwd) From 7adb0477931b63e33e8a68207b4f8c844d983666 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 31 Jul 2016 11:39:54 -0500 Subject: [PATCH 6/8] flake8 --- tests/test_post.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_post.py b/tests/test_post.py index 5fd760eda5..1c0971d216 100644 --- a/tests/test_post.py +++ b/tests/test_post.py @@ -49,11 +49,12 @@ def test_coerce_pycache_to_old_style(): os.chdir(tmp) for root, dirs, files in os.walk(tmp): - fs = [os.path.join(root, f) for f in files] + fs = [os.path.join(root, _) for _ in files] post.coerce_pycache_to_old_style(fs) try: assert os.path.isfile(os.path.join(tmp, 'test.pyc')), os.listdir(tmp) - assert os.path.isfile(os.path.join(tmp, 'testdir', 'test.pyc')), os.listdir(os.path.join(tmp, 'testdir')) + assert os.path.isfile(os.path.join(tmp, 'testdir', 'test.pyc')), \ + os.listdir(os.path.join(tmp, 'testdir')) for root, dirs, files in os.walk(tmp): assert '__pycache__' not in dirs except: From a3420d4bc7d61e1e3d32e5862df00bf44056af56 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 31 Jul 2016 17:47:42 -0500 Subject: [PATCH 7/8] fix py3 tests for compileall --- conda_build/post.py | 19 ++++++++++++------- tests/test_build_recipes.py | 3 ++- tests/test_post.py | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/conda_build/post.py b/conda_build/post.py index e779d11185..e763ed1c68 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -141,25 +141,30 @@ def rm_py_along_so(): os.unlink(join(root, name + ext)) -def coerce_pycache_to_old_style(files): +def coerce_pycache_to_old_style(files, cwd): """ For the sake of simplicity, remove the filename additions, like .cpython-35.pyc Since Conda allows only one Python install in a given prefix, there is no reason for these additional suffixes. Newer Python will find these pyc files without issue. """ - for f in [f for f in files if os.path.isfile(f) and not f.endswith('pyc')]: + for f in files: + if not os.path.exists(f): + f = os.path.join(cwd, f) + if not os.path.isfile(f) or not f.endswith('py'): + continue if '/' in f: - folder = os.path.join(os.path.dirname(f), '__pycache__') + folder = os.path.join(cwd, os.path.dirname(f), '__pycache__') else: - folder = '__pycache__' + folder = os.path.join(cwd, '__pycache__') fname = os.path.join(folder, os.path.splitext(os.path.basename(f))[0] + '.cpython-{0}{1}.pyc'.format(sys.version_info.major, sys.version_info.minor)) if os.path.isfile(fname): os.rename(fname, f + 'c') - if os.path.isdir(folder): - os.rmdir(folder) + for root, folders, files in os.walk(cwd): + if root.endswith("__pycache__") and not files: + os.rmdir(root) def compile_missing_pyc(files, cwd=config.build_prefix, python_exe=config.build_python): @@ -168,7 +173,7 @@ def compile_missing_pyc(files, cwd=config.build_prefix, python_exe=config.build_ print('compiling .pyc files...') call([python_exe, '-Wi', '-m', 'py_compile'] + compile_files, cwd=cwd) if PY3: - coerce_pycache_to_old_style(compile_files) + coerce_pycache_to_old_style(compile_files, cwd=cwd) def post_process(files, preserve_egg_dir=False): diff --git a/tests/test_build_recipes.py b/tests/test_build_recipes.py index a8cd0f834e..83d840dd81 100644 --- a/tests/test_build_recipes.py +++ b/tests/test_build_recipes.py @@ -578,7 +578,8 @@ def test_compileall_compiles_all_good_files(): 'test_compileall-1.0-py{0}{1}_0.tar.bz2'.format( sys.version_info.major, sys.version_info.minor)) cmd = 'conda build --no-anaconda-upload {}'.format(os.path.join(metadata_dir, - "_compile-test")) + "_compile-test"), + env=os.environ.copy()) subprocess.check_call(cmd.split()) good_files = ['f1.py', 'f3.py'] bad_file = 'f2_bad.py' diff --git a/tests/test_post.py b/tests/test_post.py index 1c0971d216..9431b6047d 100644 --- a/tests/test_post.py +++ b/tests/test_post.py @@ -50,7 +50,7 @@ def test_coerce_pycache_to_old_style(): os.chdir(tmp) for root, dirs, files in os.walk(tmp): fs = [os.path.join(root, _) for _ in files] - post.coerce_pycache_to_old_style(fs) + post.coerce_pycache_to_old_style(fs, cwd=tmp) try: assert os.path.isfile(os.path.join(tmp, 'test.pyc')), os.listdir(tmp) assert os.path.isfile(os.path.join(tmp, 'testdir', 'test.pyc')), \ From a94f21fa5e89dc0317f48614a7630649cd828170 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 31 Jul 2016 19:01:34 -0500 Subject: [PATCH 8/8] fix windows py35 compileall --- conda_build/post.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/post.py b/conda_build/post.py index e763ed1c68..f7b61283fe 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -153,7 +153,7 @@ def coerce_pycache_to_old_style(files, cwd): f = os.path.join(cwd, f) if not os.path.isfile(f) or not f.endswith('py'): continue - if '/' in f: + if '/' in f or '\\' in f: folder = os.path.join(cwd, os.path.dirname(f), '__pycache__') else: folder = os.path.join(cwd, '__pycache__')