From 90ce9d98440ef4f81b92102c4560a07d774910d3 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 16 Dec 2020 16:21:49 +0000 Subject: [PATCH 01/59] Update dependencies and macOS 11 CI on Github --- .github/workflows/test.yml | 3 ++- .../resources/constraints-python27.txt | 8 ++++---- .../resources/constraints-python35.txt | 8 ++++---- .../resources/constraints-python36.txt | 11 +++++----- .../resources/constraints-python37.txt | 11 +++++----- cibuildwheel/resources/constraints.txt | 8 ++++---- .../resources/pinned_docker_images.cfg | 20 +++++++++---------- 7 files changed, 36 insertions(+), 33 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3ab6656cb..3422a7a16 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,7 +24,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-18.04, windows-latest, macos-latest] + # os: [ubuntu-latest, windows-latest, macos-11.0] + os: [macos-11.0] python_version: ['3.7'] timeout-minutes: 180 steps: diff --git a/cibuildwheel/resources/constraints-python27.txt b/cibuildwheel/resources/constraints-python27.txt index e387282d7..dd4a864be 100644 --- a/cibuildwheel/resources/constraints-python27.txt +++ b/cibuildwheel/resources/constraints-python27.txt @@ -10,17 +10,17 @@ contextlib2==0.6.0.post1 # via importlib-metadata, importlib-resources delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in distlib==0.3.1 # via virtualenv filelock==3.0.12 # via virtualenv -importlib-metadata==2.1.0 # via virtualenv +importlib-metadata==2.1.1 # via virtualenv importlib-resources==3.3.0 # via virtualenv pathlib2==2.3.5 # via importlib-metadata, importlib-resources, virtualenv scandir==1.10.0 # via pathlib2 singledispatch==3.4.0.3 # via importlib-resources six==1.15.0 # via pathlib2, virtualenv typing==3.7.4.3 # via importlib-resources -virtualenv==20.2.1 # via -r cibuildwheel/resources/constraints.in -wheel==0.35.1 # via -r cibuildwheel/resources/constraints.in, delocate +virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate zipp==1.2.0 # via importlib-metadata, importlib-resources # The following packages are considered to be unsafe in a requirements file: -pip==20.2.4 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 # via -r cibuildwheel/resources/constraints.in setuptools==44.1.1 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python35.txt b/cibuildwheel/resources/constraints-python35.txt index 4dc2ecd69..c6f210fa6 100644 --- a/cibuildwheel/resources/constraints-python35.txt +++ b/cibuildwheel/resources/constraints-python35.txt @@ -8,13 +8,13 @@ appdirs==1.4.4 # via virtualenv delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in distlib==0.3.1 # via virtualenv filelock==3.0.12 # via virtualenv -importlib-metadata==2.1.0 # via virtualenv +importlib-metadata==2.1.1 # via virtualenv importlib-resources==3.2.1 # via virtualenv six==1.15.0 # via virtualenv -virtualenv==20.2.1 # via -r cibuildwheel/resources/constraints.in -wheel==0.35.1 # via -r cibuildwheel/resources/constraints.in, delocate +virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate zipp==1.2.0 # via importlib-metadata, importlib-resources # The following packages are considered to be unsafe in a requirements file: -pip==20.2.4 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 # via -r cibuildwheel/resources/constraints.in setuptools==50.3.2 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python36.txt b/cibuildwheel/resources/constraints-python36.txt index 15bca5b6f..ea9e01091 100644 --- a/cibuildwheel/resources/constraints-python36.txt +++ b/cibuildwheel/resources/constraints-python36.txt @@ -8,13 +8,14 @@ appdirs==1.4.4 # via virtualenv delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in distlib==0.3.1 # via virtualenv filelock==3.0.12 # via virtualenv -importlib-metadata==3.1.0 # via virtualenv +importlib-metadata==3.3.0 # via virtualenv importlib-resources==3.3.0 # via virtualenv six==1.15.0 # via virtualenv -virtualenv==20.2.1 # via -r cibuildwheel/resources/constraints.in -wheel==0.35.1 # via -r cibuildwheel/resources/constraints.in, delocate +typing-extensions==3.7.4.3 # via importlib-metadata +virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate zipp==3.4.0 # via importlib-metadata, importlib-resources # The following packages are considered to be unsafe in a requirements file: -pip==20.2.4 # via -r cibuildwheel/resources/constraints.in -setuptools==50.3.2 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 # via -r cibuildwheel/resources/constraints.in +setuptools==51.0.0 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python37.txt b/cibuildwheel/resources/constraints-python37.txt index cf6d8693e..6e0f89f14 100644 --- a/cibuildwheel/resources/constraints-python37.txt +++ b/cibuildwheel/resources/constraints-python37.txt @@ -8,12 +8,13 @@ appdirs==1.4.4 # via virtualenv delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in distlib==0.3.1 # via virtualenv filelock==3.0.12 # via virtualenv -importlib-metadata==3.1.0 # via virtualenv +importlib-metadata==3.3.0 # via virtualenv six==1.15.0 # via virtualenv -virtualenv==20.2.1 # via -r cibuildwheel/resources/constraints.in -wheel==0.35.1 # via -r cibuildwheel/resources/constraints.in, delocate +typing-extensions==3.7.4.3 # via importlib-metadata +virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate zipp==3.4.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -pip==20.2.4 # via -r cibuildwheel/resources/constraints.in -setuptools==50.3.2 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 # via -r cibuildwheel/resources/constraints.in +setuptools==51.0.0 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints.txt b/cibuildwheel/resources/constraints.txt index 81a335e90..6d2f7d0b4 100644 --- a/cibuildwheel/resources/constraints.txt +++ b/cibuildwheel/resources/constraints.txt @@ -9,9 +9,9 @@ delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in distlib==0.3.1 # via virtualenv filelock==3.0.12 # via virtualenv six==1.15.0 # via virtualenv -virtualenv==20.2.1 # via -r cibuildwheel/resources/constraints.in -wheel==0.35.1 # via -r cibuildwheel/resources/constraints.in, delocate +virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate # The following packages are considered to be unsafe in a requirements file: -pip==20.2.4 # via -r cibuildwheel/resources/constraints.in -setuptools==50.3.2 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 # via -r cibuildwheel/resources/constraints.in +setuptools==51.0.0 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/pinned_docker_images.cfg b/cibuildwheel/resources/pinned_docker_images.cfg index 0646edf83..816ee2be9 100644 --- a/cibuildwheel/resources/pinned_docker_images.cfg +++ b/cibuildwheel/resources/pinned_docker_images.cfg @@ -1,22 +1,22 @@ [x86_64] -manylinux1 = quay.io/pypa/manylinux1_x86_64:2020-11-11-0f1f128 -manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2020-12-03-912b0de -manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2020-11-11-bc8ce45 +manylinux1 = quay.io/pypa/manylinux1_x86_64:2020-12-14-7638977 +manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2020-12-15-ba3529a +manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2020-12-15-6cc1890 [i686] -manylinux1 = quay.io/pypa/manylinux1_i686:2020-11-11-0f1f128 -manylinux2010 = quay.io/pypa/manylinux2010_i686:2020-12-03-912b0de -manylinux2014 = quay.io/pypa/manylinux2014_i686:2020-11-11-bc8ce45 +manylinux1 = quay.io/pypa/manylinux1_i686:2020-12-14-7638977 +manylinux2010 = quay.io/pypa/manylinux2010_i686:2020-12-15-ba3529a +manylinux2014 = quay.io/pypa/manylinux2014_i686:2020-12-15-6cc1890 [pypy_x86_64] -manylinux2010 = pypywheels/manylinux2010-pypy_x86_64:2020-11-21-a03b9e9 +manylinux2010 = pypywheels/manylinux2010-pypy_x86_64:2020-12-11-f1e0e80 [aarch64] -manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2020-11-11-bc8ce45 +manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2020-12-15-6cc1890 [ppc64le] -manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2020-11-11-bc8ce45 +manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2020-12-15-6cc1890 [s390x] -manylinux2014 = quay.io/pypa/manylinux2014_s390x:2020-11-11-bc8ce45 +manylinux2014 = quay.io/pypa/manylinux2014_s390x:2020-12-15-6cc1890 From 59801d4c987d24170a1c66193ac53e211d541ff0 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 16 Dec 2020 20:05:31 +0000 Subject: [PATCH 02/59] Disable macOS CPython 3.5 build for now --- cibuildwheel/macos.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 9ffca89dd..3d84cdf26 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -35,7 +35,8 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi python_configurations = [ # CPython PythonConfiguration(version='2.7', identifier='cp27-macosx_x86_64', url='https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg'), - PythonConfiguration(version='3.5', identifier='cp35-macosx_x86_64', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'), + # TODO: figure out what's going on in CPython 3.5 on macOS 11. or, remove it :) + # PythonConfiguration(version='3.5', identifier='cp35-macosx_x86_64', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'), PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'), PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.8/python-3.7.8-macosx10.9.pkg'), PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.4/python-3.8.4-macosx10.9.pkg'), From 609c0f39862fbb3e751dab254f912f89a87e566f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 16 Dec 2020 21:47:08 -0500 Subject: [PATCH 03/59] WIP: try new Python build --- cibuildwheel/macos.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 3d84cdf26..7da855c64 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -40,9 +40,11 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'), PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.8/python-3.7.8-macosx10.9.pkg'), PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.4/python-3.8.4-macosx10.9.pkg'), - PythonConfiguration(version='3.9', identifier='cp39-macosx_x86_64', url='https://www.python.org/ftp/python/3.9.0/python-3.9.0-macosx10.9.pkg'), + # TODO: Find some better way to select Universal2 vs. regular (note that this works on 10.9+, regardless of the name) + PythonConfiguration(version='3.9', identifier='cp39-macosx_x86_64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), # PyPy - PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'), + # TODO: may not support 11.0 yet + # PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'), PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2'), PythonConfiguration(version='3.7', identifier='pp37-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2'), ] From 7fb032d70f590b51412c751fc5532d91c74f737e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 18 Dec 2020 12:53:37 +0000 Subject: [PATCH 04/59] Try using _PYTHON_HOST_PLATFORM to create single-arch builds --- cibuildwheel/macos.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 7da855c64..67c4a11d9 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -42,6 +42,8 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.4/python-3.8.4-macosx10.9.pkg'), # TODO: Find some better way to select Universal2 vs. regular (note that this works on 10.9+, regardless of the name) PythonConfiguration(version='3.9', identifier='cp39-macosx_x86_64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), + PythonConfiguration(version='3.9', identifier='cp39-macosx_universal2', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), + PythonConfiguration(version='3.9', identifier='cp39-macosx_arm64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), # PyPy # TODO: may not support 11.0 yet # PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'), @@ -178,6 +180,14 @@ def setup_python(python_configuration: PythonConfiguration, # https://github.com/python/cpython/blob/a5ed2fe0eedefa1649aa93ee74a0bafc8e628a10/Lib/_osx_support.py#L260 env.setdefault('ARCHFLAGS', '-arch x86_64') + if python_configuration.version == '3.9': + if python_configuration.identifier.endswith('x86_64'): + env.setdefault('_PYTHON_HOST_PLATFORM', 'macosx-10.9-x86_64') + env.setdefault('ARCHFLAGS', '-arch x86_64') + elif python_configuration.identifier.endswith('arm64'): + env.setdefault('_PYTHON_HOST_PLATFORM', 'macosx-11.0-arm64') + env.setdefault('ARCHFLAGS', '-arch arm64') + log.step('Installing build tools...') call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate', *dependency_constraint_flags], env=env) From 9906cf71053e0249e389b9af9c8b17157b3e070d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 18 Dec 2020 14:16:28 +0000 Subject: [PATCH 05/59] Fix logger crash --- cibuildwheel/logger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index a8289e763..97fcecccf 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -23,6 +23,8 @@ 'win32': 'Windows 32bit', 'win_amd64': 'Windows 64bit', 'macosx_x86_64': 'macOS x86_64', + 'macosx_universal2': 'macOS Universal 2 - x86_64 and arm64', + 'macosx_arm64': 'macOS arm64 - Apple Silicon', } From 985ef4dfef1da1c807069e67441061cb0149c677 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 18 Dec 2020 11:32:28 -0500 Subject: [PATCH 06/59] wip: drop pypy for now --- cibuildwheel/macos.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 67c4a11d9..7140f720c 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -47,8 +47,8 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi # PyPy # TODO: may not support 11.0 yet # PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'), - PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2'), - PythonConfiguration(version='3.7', identifier='pp37-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2'), + # PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2'), + # PythonConfiguration(version='3.7', identifier='pp37-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2'), ] # skip builds as required From 31d0351ddf9235d2c8fcb47934e46a314cccc7d8 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 28 Dec 2020 00:57:53 +0000 Subject: [PATCH 07/59] Upload some sample wheels as artifacts --- .github/workflows/test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3422a7a16..12539e9cb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,6 +53,11 @@ jobs: run: | python bin/sample_build.py + - uses: actions/upload-artifact@v2 + with: + name: sample_wheels + path: wheelhouse + - name: Test cibuildwheel run: | python ./bin/run_tests.py From 2b800bbb3a5f51fbb288f7e55360aeba14a56a5e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 31 Dec 2020 16:40:10 +0000 Subject: [PATCH 08/59] Build some sample wheels --- .github/workflows/test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 12539e9cb..5276be33a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,6 +53,11 @@ jobs: run: | python bin/sample_build.py + - name: Get some sample wheels + run: | + python -m test.test_projects test.test_0_basic.basic_project sample_proj + cibuildwheel --output-dir wheelhouse sample_proj + - uses: actions/upload-artifact@v2 with: name: sample_wheels From 76d5f3d3e0a76256564fe8e30e05132e8fe6c848 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 31 Dec 2020 16:48:43 +0000 Subject: [PATCH 09/59] Call delocate with the correct archs --- cibuildwheel/__main__.py | 2 +- cibuildwheel/macos.py | 15 ++++++++++++++- docs/options.md | 5 +++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index e171ab20d..3040ccb67 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -118,7 +118,7 @@ def main() -> None: if platform == 'linux': repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}' elif platform == 'macos': - repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}' + repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}' else: repair_command_default = '' diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 2af1cc8d5..efa99a1ab 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -250,7 +250,20 @@ def build(options: BuildOptions) -> None: if options.repair_command: log.step('Repairing wheel...') - repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) + + if config.identifier.endswith('universal2'): + delocate_archs = 'x86_64,arm64' + elif config.identifier.endswith('arm64'): + delocate_archs = 'arm64' + else: + delocate_archs = 'x86_64' + + repair_command_prepared = prepare_command( + options.repair_command, + wheel=built_wheel, + dest_dir=repaired_wheel_dir, + delocate_archs=delocate_archs, + ) call(repair_command_prepared, env=env, shell=True) else: shutil.move(str(built_wheel), repaired_wheel_dir) diff --git a/docs/options.md b/docs/options.md index 73c25b3b5..ad976cd76 100644 --- a/docs/options.md +++ b/docs/options.md @@ -255,7 +255,7 @@ CIBW_BEFORE_BUILD: "{package}/bin/prepare_for_build.sh" Default: - on Linux: `'auditwheel repair -w {dest_dir} {wheel}'` -- on macOS: `'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}'` +- on macOS: `'delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}'` - on Windows: `''` A shell command to repair a built wheel by copying external library dependencies into the wheel tree and relinking them. @@ -264,7 +264,8 @@ The command is run on each built wheel (except for pure Python ones) before test The following placeholders must be used inside the command and will be replaced by `cibuildwheel`: - `{wheel}` for the absolute path to the built wheel -- `{dest_dir}` for the absolute path of the directory where to create the repaired wheel. +- `{dest_dir}` for the absolute path of the directory where to create the repaired wheel +- `{delocate_archs}` (macOS only) comma-separated list of architectures in the wheel. The command is run in a shell, so you can run multiple commands like `cmd1 && cmd2`. From 4d89e524eb9e0b5021aab0127358a93fa24079e3 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 1 Jan 2021 18:55:50 +0000 Subject: [PATCH 10/59] Add rename hack to make the universal wheel work on both archs --- cibuildwheel/macos.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index efa99a1ab..e26538d80 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -270,6 +270,16 @@ def build(options: BuildOptions) -> None: repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) + if repaired_wheel.stem.endswith('_universal2'): + # due to a bug in packaging/tags, this universal wheel isn't + # installable on arm64. so we rename it to add a 11_0 platform + # tag. + # See https://github.com/pypa/packaging/pull/380 and + # https://github.com/pypa/packaging/issues/379 + renamed_wheel = repaired_wheel.parent / f'{repaired_wheel.stem}.macosx_11_0_universal2.whl' + repaired_wheel.rename(renamed_wheel) + repaired_wheel = renamed_wheel + if options.test_command: log.step('Testing wheel...') # set up a virtual environment to install and test from, to make sure From 09fef56a46ba15b4e2fa688da4fdef3bea0cb979 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 1 Jan 2021 18:56:36 +0000 Subject: [PATCH 11/59] Make tests expect arm64 and universal2 --- test/utils.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/test/utils.py b/test/utils.py index 04f3e1476..2ddbc8130 100644 --- a/test/utils.py +++ b/test/utils.py @@ -10,10 +10,8 @@ import subprocess import sys from contextlib import contextmanager -from pathlib import Path from tempfile import mkdtemp -IS_WINDOWS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists() IS_WINDOWS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' @@ -99,6 +97,10 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, if platform == 'linux': python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux + if platform == 'macos': + # TODO: perhaps drop Python 3.5 across the board? + python_abi_tags.remove('cp35-cp35m') + wheels = [] for python_abi_tag in python_abi_tags: @@ -123,7 +125,16 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, platform_tags = ['win32'] elif platform == 'macos': - platform_tags = [f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64'] + if python_abi_tag == 'cp39-cp39': + platform_tags = [ + f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64', + f'macosx_{macosx_deployment_target.replace(".", "_")}_universal2.macosx_11_0_universal2', + 'macosx_11_0_arm64', + ] + else: + platform_tags = [ + f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64', + ] else: raise Exception('unsupported platform') From 7cab7125929b69208d582823312869ff0902151f Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 2 Jan 2021 19:25:09 +0000 Subject: [PATCH 12/59] Fix unit test --- unit_test/main_tests/main_options_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index c7e6bfddc..26e50a728 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -94,7 +94,7 @@ def get_default_repair_command(platform): if platform == 'linux': return 'auditwheel repair -w {dest_dir} {wheel}' elif platform == 'macos': - return 'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}' + return 'delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}' elif platform == 'windows': return '' else: From a1ac43e9ded2cc20608a59da3e471ed9ad8ef92f Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 10:23:36 +0000 Subject: [PATCH 13/59] Override buildable archs on macOS 11 --- cibuildwheel/macos.py | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 835d73a7f..8a70701ab 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -1,4 +1,5 @@ import os +import platform import shlex import shutil import subprocess @@ -7,7 +8,7 @@ import textwrap from os import PathLike from pathlib import Path -from typing import Dict, List, NamedTuple, Optional, Sequence, Union +from typing import Dict, List, NamedTuple, Optional, Sequence, Tuple, Union, cast from .environment import ParsedEnvironment from .logger import log @@ -26,6 +27,19 @@ def call(args: Union[str, Sequence[Union[str, PathLike]]], env: Optional[Dict[st return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) +def get_macos_version() -> Tuple[int, int]: + ''' + Returns the macOS major/minor version, as a tuple, e.g. (10, 15) or (11, 0) + + These tuples can be used in comparisons, e.g. + (10, 14) <= (11, 0) == True + (11, 2) <= (11, 0) != True + ''' + version_str, _, _ = platform.mac_ver() + version = tuple(map(int, version_str.split(".")[:2])) + return cast(Tuple[int, int], version) + + class PythonConfiguration(NamedTuple): version: str identifier: str @@ -36,22 +50,28 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi python_configurations = [ # CPython PythonConfiguration(version='2.7', identifier='cp27-macosx_x86_64', url='https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg'), - # TODO: figure out what's going on in CPython 3.5 on macOS 11. or, remove it :) - # PythonConfiguration(version='3.5', identifier='cp35-macosx_x86_64', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'), + PythonConfiguration(version='3.5', identifier='cp35-macosx_x86_64', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'), PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'), PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.9/python-3.7.9-macosx10.9.pkg'), PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.7/python-3.8.7-macosx10.9.pkg'), - # TODO: Find some better way to select Universal2 vs. regular (note that this works on 10.9+, regardless of the name) PythonConfiguration(version='3.9', identifier='cp39-macosx_x86_64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), PythonConfiguration(version='3.9', identifier='cp39-macosx_universal2', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), PythonConfiguration(version='3.9', identifier='cp39-macosx_arm64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), # PyPy # TODO: may not support 11.0 yet - # PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'), - # PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2'), - # PythonConfiguration(version='3.7', identifier='pp37-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2'), + PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'), + PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2'), + PythonConfiguration(version='3.7', identifier='pp37-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2'), ] + if get_macos_version() >= (11, 0): + # pypy doesn't work on macOS 11 yet + # See https://foss.heptapod.net/pypy/pypy/-/issues/3314 + python_configurations = [c for c in python_configurations if not c.identifier.startswith('pp')] + + # CPython 3.5 doesn't work on macOS 11 + python_configurations = [c for c in python_configurations if not c.identifier.startswith('cp35')] + # skip builds as required return [c for c in python_configurations if build_selector(c.identifier)] From 46eb28791f473ede2841132854473622e04fb902 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 13:50:21 +0000 Subject: [PATCH 14/59] Refactor for clarity --- cibuildwheel/linux.py | 2 +- cibuildwheel/logger.py | 2 +- cibuildwheel/macos.py | 2 +- cibuildwheel/windows.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 4118fd795..19f360646 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -251,7 +251,7 @@ def build(options: BuildOptions) -> None: docker.copy_out(container_output_dir, options.output_dir) log.step_end() except subprocess.CalledProcessError as error: - log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') + log.step_end_with_error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') troubleshoot(options.package_dir, error) exit(1) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index bfb81b3a0..4724b6051 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -111,7 +111,7 @@ def step_end(self, success: bool = True) -> None: self.step_start_time = None - def error(self, error: Union[BaseException, str]) -> None: + def step_end_with_error(self, error: Union[BaseException, str]) -> None: self.step_end(success=False) print() diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index f53b0acda..d43d29549 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -359,5 +359,5 @@ def build(options: BuildOptions) -> None: shutil.move(str(repaired_wheel), options.output_dir) log.build_end() except subprocess.CalledProcessError as error: - log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') + log.step_end_with_error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') exit(1) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 3e977e4b3..a156db3ae 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -330,5 +330,5 @@ def build(options: BuildOptions) -> None: shutil.move(str(repaired_wheel), options.output_dir) log.build_end() except subprocess.CalledProcessError as error: - log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') + log.step_end_with_error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') exit(1) From e405f3495efcde3edf1495bd7232d0ee53c89d9c Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 14:29:38 +0000 Subject: [PATCH 15/59] Print warnings when builds are deselected because of build OS --- cibuildwheel/logger.py | 12 ++++++++++++ cibuildwheel/macos.py | 26 +++++++++++++++++++++----- cibuildwheel/util.py | 14 ++++++++++++++ 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 4724b6051..a6c693f38 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -113,6 +113,18 @@ def step_end(self, success: bool = True) -> None: def step_end_with_error(self, error: Union[BaseException, str]) -> None: self.step_end(success=False) + self.error(error) + + def warning(self, message: str) -> None: + print() + + if self.fold_mode == 'github': + print(f'::warning::{message}') + else: + c = self.colors + print(f'{c.yellow}Warning{c.end} {message}') + + def error(self, error: Union[BaseException, str]) -> None: print() if self.fold_mode == 'github': diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index d43d29549..0dd5ebacd 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -13,7 +13,7 @@ from .logger import log from .util import (Architecture, BuildOptions, BuildSelector, NonPlatformWheelError, download, get_build_verbosity_extra_flags, get_pip_script, - install_certifi_script, prepare_command) + install_certifi_script, prepare_command, wrap_text) from .typing import PathOrStr @@ -64,16 +64,32 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi PythonConfiguration(version='3.7', identifier='pp37-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2'), ] + # skip builds as required + python_configurations = [c for c in python_configurations if build_selector(c.identifier)] + if get_macos_version() >= (11, 0): # pypy doesn't work on macOS 11 yet # See https://foss.heptapod.net/pypy/pypy/-/issues/3314 - python_configurations = [c for c in python_configurations if not c.identifier.startswith('pp')] + if any(c.identifier.startswith('pp') for c in python_configurations): + log.warning(wrap_text(''' + PyPy is currently unsupported when building on macOS 11. To build macOS PyPy wheels, + build on an older OS, such as macOS 10.15. To silence this warning, deselect PyPy by + adding "pp*-macosx*" to your CIBW_SKIP option. + ''')) + python_configurations = [c for c in python_configurations if not c.identifier.startswith('pp')] + + if any(c.identifier.startswith('cp35') for c in python_configurations): + # CPython 3.5 doesn't work on macOS 11 + log.warning(wrap_text(''' + CPython is unsupported when building on macOS 11. To build CPython 3.5 wheels, build + on an older OS, such as macOS 10.15. To silence this warning, deselect CPython 3.5 + by adding "cp35-macosx_x86_64" to your CIBW_SKIP option. + ''')) + python_configurations = [c for c in python_configurations if not c.identifier.startswith('cp35')] - # CPython 3.5 doesn't work on macOS 11 python_configurations = [c for c in python_configurations if not c.identifier.startswith('cp35')] - # skip builds as required - return [c for c in python_configurations if build_selector(c.identifier)] + return python_configurations SYMLINKS_DIR = Path('/tmp/cibw_bin') diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 960b9896d..3b4935ff5 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -229,3 +229,17 @@ def detect_ci_provider() -> Optional[CIProvider]: return CIProvider.other else: return None + + +def wrap_text(text: str, max_width: int = 80) -> str: + ''' + Formats text, suitable for printing arbitrary long passages to console. + ''' + # remove initial line indent + text = textwrap.dedent(text) + # remove leading/trailing whitespace + text = text.strip() + # remove consecutive whitespace + text = re.sub(r'\s+', ' ', text) + # wrap to max_width + return '\n'.join(textwrap.wrap(text, width=max_width)) From df60457001e50314ac0f4e4c7ae5cd7480578d8a Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 15:14:32 +0000 Subject: [PATCH 16/59] Use the ARCHS option to select universal2/arm64 builds --- cibuildwheel/__main__.py | 2 +- cibuildwheel/macos.py | 31 +++++++++++++++++-------------- cibuildwheel/util.py | 13 +++++++++++++ test/test_macos_archs.py | 25 +++++++++++++++++++++++++ test/utils.py | 34 +++++++++++++++++++++++++++------- 5 files changed, 83 insertions(+), 22 deletions(-) create mode 100644 test/test_macos_archs.py diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index d3ed34927..20ee2baa5 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -312,7 +312,7 @@ def print_build_identifiers( elif platform == 'windows': python_configurations = cibuildwheel.windows.get_python_configurations(build_selector) elif platform == 'macos': - python_configurations = cibuildwheel.macos.get_python_configurations(build_selector) + python_configurations = cibuildwheel.macos.get_python_configurations(build_selector, architectures) for config in python_configurations: print(config.identifier) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 0dd5ebacd..e12ea6489 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -46,7 +46,8 @@ class PythonConfiguration(NamedTuple): url: str -def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]: +def get_python_configurations(build_selector: BuildSelector, + architectures: List[Architecture]) -> List[PythonConfiguration]: python_configurations = [ # CPython PythonConfiguration(version='2.7', identifier='cp27-macosx_x86_64', url='https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg'), @@ -58,19 +59,22 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi PythonConfiguration(version='3.9', identifier='cp39-macosx_universal2', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), PythonConfiguration(version='3.9', identifier='cp39-macosx_arm64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), # PyPy - # TODO: may not support 11.0 yet PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'), PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2'), PythonConfiguration(version='3.7', identifier='pp37-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2'), ] - # skip builds as required + # filter out configs that don't match any of the selected architectures + python_configurations = [c for c in python_configurations + if any(c.identifier.endswith(a.value) for a in architectures)] + + # skip builds as required by BUILD/SKIP python_configurations = [c for c in python_configurations if build_selector(c.identifier)] if get_macos_version() >= (11, 0): - # pypy doesn't work on macOS 11 yet - # See https://foss.heptapod.net/pypy/pypy/-/issues/3314 if any(c.identifier.startswith('pp') for c in python_configurations): + # pypy doesn't work on macOS 11 yet + # See https://foss.heptapod.net/pypy/pypy/-/issues/3314 log.warning(wrap_text(''' PyPy is currently unsupported when building on macOS 11. To build macOS PyPy wheels, build on an older OS, such as macOS 10.15. To silence this warning, deselect PyPy by @@ -81,14 +85,12 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi if any(c.identifier.startswith('cp35') for c in python_configurations): # CPython 3.5 doesn't work on macOS 11 log.warning(wrap_text(''' - CPython is unsupported when building on macOS 11. To build CPython 3.5 wheels, build - on an older OS, such as macOS 10.15. To silence this warning, deselect CPython 3.5 - by adding "cp35-macosx_x86_64" to your CIBW_SKIP option. + CPython 3.5 is unsupported when building on macOS 11. To build CPython 3.5 wheels, + build on an older OS, such as macOS 10.15. To silence this warning, deselect CPython + 3.5 by adding "cp35-macosx_x86_64" to your CIBW_SKIP option. ''')) python_configurations = [c for c in python_configurations if not c.identifier.startswith('cp35')] - python_configurations = [c for c in python_configurations if not c.identifier.startswith('cp35')] - return python_configurations @@ -236,10 +238,11 @@ def setup_python(python_configuration: PythonConfiguration, def build(options: BuildOptions) -> None: - if options.architectures != [Architecture.x86_64]: + allowed_archs = {Architecture.x86_64, Architecture.universal2, Architecture.arm64} + if set(options.architectures) <= allowed_archs: raise ValueError(textwrap.dedent(f''' - Invalid archs option {options.architectures}. macOS only supports x86_64 for the moment. - If you want to set emulation architectures on Linux, use CIBW_ARCHS_LINUX instead. + Invalid archs option {options.architectures}. macOS only supports + these architectures: {', '.join(a.value for a in allowed_archs)}. ''')) temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel')) @@ -253,7 +256,7 @@ def build(options: BuildOptions) -> None: before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) call([before_all_prepared], shell=True, env=env) - python_configurations = get_python_configurations(options.build_selector) + python_configurations = get_python_configurations(options.build_selector, options.architectures) for config in python_configurations: log.build_start(config.identifier) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 3b4935ff5..3654b51e1 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -127,11 +127,17 @@ def __repr__(self) -> str: class Architecture(Enum): # mac/linux archs x86_64 = 'x86_64' + + # linux archs i686 = 'i686' aarch64 = 'aarch64' ppc64le = 'ppc64le' s390x = 's390x' + # mac archs + universal2 = 'universal2' + arm64 = 'arm64' + # windows archs x86 = 'x86' AMD64 = 'AMD64' @@ -150,11 +156,18 @@ def parse_config(config: str, platform: str) -> 'List[Architecture]': def auto_archs(platform: str) -> 'List[Architecture]': native_architecture = Architecture(platform_module.machine()) result = [native_architecture] + if platform == 'linux' and native_architecture == Architecture.x86_64: # x86_64 machines can run i686 docker containers result.append(Architecture.i686) + if platform == 'windows' and native_architecture == Architecture.AMD64: result.append(Architecture.x86) + + if platform == 'macos' and native_architecture == Architecture.arm64: + # arm64 can build and test both archs of a universal2 wheel. + result.append(Architecture.universal2) + return result diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py new file mode 100644 index 000000000..cfcaddebd --- /dev/null +++ b/test/test_macos_archs.py @@ -0,0 +1,25 @@ +import pytest + +from . import test_projects, utils + +basic_project = test_projects.new_c_project() + + +def test_cross_compiled_build(tmp_path): + if utils.platform != 'macos': + pytest.skip('this test is only relevant to macos') + + project_dir = tmp_path / 'project' + basic_project.generate(project_dir) + + actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ + 'CIBW_ARCHS': 'x86_64, universal2, arm64', + }) + + expected_wheels = ( + utils.expected_wheels('spam', '0.1.0', machine_arch='x86_64') + + utils.expected_wheels('spam', '0.1.0', machine_arch='arm64') + ) + assert set(actual_wheels) == set(expected_wheels) + +# TODO: add a TEST_COMMAND test when using cross-compiling diff --git a/test/utils.py b/test/utils.py index d8aa50eee..5be952d0b 100644 --- a/test/utils.py +++ b/test/utils.py @@ -100,9 +100,11 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, if platform == 'linux': python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux - if platform == 'macos': - # TODO: perhaps drop Python 3.5 across the board? + if platform == 'macos' and get_macos_version() >= (11, 0): + # CPython 3.5 doesn't work on macOS 11. python_abi_tags.remove('cp35-cp35m') + # pypy not supported on macOS 11. + python_abi_tags = [t for t in python_abi_tags if not t.startswith('pp')] wheels = [] @@ -129,11 +131,17 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, elif platform == 'macos': if python_abi_tag == 'cp39-cp39': - platform_tags = [ - f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64', - f'macosx_{macosx_deployment_target.replace(".", "_")}_universal2.macosx_11_0_universal2', - 'macosx_11_0_arm64', - ] + if machine_arch == 'x86_64': + platform_tags = [ + f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64', + ] + elif machine_arch == 'arm64': + platform_tags = [ + f'macosx_{macosx_deployment_target.replace(".", "_")}_universal2.macosx_11_0_universal2', + # macosx_deployment_target is ignored on arm64, because arm64 isn't supported on + # macOS earlier than 11.0 + 'macosx_11_0_arm64', + ] else: platform_tags = [ f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64', @@ -152,6 +160,18 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, return wheels +def get_macos_version(): + ''' + Returns the macOS major/minor version, as a tuple, e.g. (10, 15) or (11, 0) + + These tuples can be used in comparisons, e.g. + (10, 14) <= (11, 0) == True + (11, 2) <= (11, 0) != True + ''' + version_str, _, _ = pm.mac_ver() + return tuple(map(int, version_str.split(".")[:2])) + + platform = None if 'CIBW_PLATFORM' in os.environ: From 577fd5e8e34ba8a629a551f80dd16181c5b2f13e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 15:58:53 +0000 Subject: [PATCH 17/59] Implement testing for universal2/arm64 wheels --- cibuildwheel/macos.py | 139 +++++++++++++++++++++++++++++------------- 1 file changed, 95 insertions(+), 44 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index e12ea6489..6a06c53c8 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -7,7 +7,7 @@ import tempfile import textwrap from pathlib import Path -from typing import Dict, List, NamedTuple, Optional, Sequence, Tuple, cast +from typing import Any, Dict, List, NamedTuple, Optional, Sequence, Tuple, cast from .environment import ParsedEnvironment from .logger import log @@ -330,49 +330,100 @@ def build(options: BuildOptions) -> None: repaired_wheel.rename(renamed_wheel) repaired_wheel = renamed_wheel - if options.test_command: - log.step('Testing wheel...') - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) - venv_dir = Path(tempfile.mkdtemp()) - - # Use --no-download to ensure determinism by using seed libraries - # built into virtualenv - call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) - - virtualenv_env = env.copy() - virtualenv_env['PATH'] = os.pathsep.join([ - str(venv_dir / 'bin'), - virtualenv_env['PATH'], - ]) - - # check that we are using the Python from the virtual environment - call(['which', 'python'], env=virtualenv_env) - - if options.before_test: - before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir) - call(before_test_prepared, env=virtualenv_env, shell=True) - - # install the wheel - call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) - - # test the wheel - if options.test_requires: - call(['pip', 'install'] + options.test_requires, env=virtualenv_env) - - # run the tests from $HOME, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command( - options.test_command, - project=Path('.').resolve(), - package=options.package_dir.resolve() - ) - call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) - - # clean up - shutil.rmtree(venv_dir) + log.step_end() + + machine_arch = platform.machine() + testing_archs: List[str] = [] + + if machine_arch == 'x86_64': + if config.identifier.endswith('_arm64'): + log.warning(wrap_text(''' + While arm64 wheels can be built on x86_64, they cannot be tested. The + ability to test the arm64 wheels will be added in a future release of + cibuildwheel, once Apple Silicon CI runners are widely available. + ''')) + testing_archs = [] + elif config.identifier.endswith('_universal2'): + log.warning(wrap_text(''' + While universal2 wheels can be built on x86_64, the arm64 part of them + cannot currently be tested. The ability to test the arm64 part of a + universal2 wheel will be added in a future release of cibuildwheel, once + Apple Silicon CI runners are widely available. + ''')) + testing_archs = ['x86_64'] + else: + testing_archs = ['x86_64'] + elif machine_arch == 'arm64': + if config.identifier.endswith('_x86_64'): + # testing using rosetta2 emulation + testing_archs = ['x86_64'] + elif config.identifier.endswith('_universal2'): + # testing the x86_64 using rosetta2 emulation + testing_archs = ['arm64', 'x86_64'] + else: + testing_archs = ['arm64'] + + if options.test_command and len(testing_archs) > 0: + for testing_arch in testing_archs: + log.step('Testing wheel...' if testing_arch == machine_arch else f'Testing wheel on {testing_arch}...') + + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) + venv_dir = Path(tempfile.mkdtemp()) + + arch_prefix = [] + if testing_arch != machine_arch: + if machine_arch == 'arm64' and testing_arch == 'x86_64': + # rosetta2 will provide the emulation with just the arch prefix. + arch_prefix = ['arch', '-x86_64'] + else: + raise RuntimeError("don't know how to emulate {testing_arch} on {machine_arch}") + + # define a custom 'call' function that adds the arch prefix each time + def call_with_arch(args: Sequence[PathOrStr], **kwargs: Any) -> int: + if isinstance(args, str): + args = ' '.join(arch_prefix) + ' ' + args + else: + args = [*arch_prefix, *args] + return call(args, **kwargs) + + # Use --no-download to ensure determinism by using seed libraries + # built into virtualenv + call_with_arch(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) + + virtualenv_env = env.copy() + virtualenv_env['PATH'] = os.pathsep.join([ + str(venv_dir / 'bin'), + virtualenv_env['PATH'], + ]) + + # check that we are using the Python from the virtual environment + call_with_arch(['which', 'python'], env=virtualenv_env) + + if options.before_test: + before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir) + call_with_arch(before_test_prepared, env=virtualenv_env, shell=True) + + # install the wheel + call_with_arch(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) + + # test the wheel + if options.test_requires: + call_with_arch(['pip', 'install'] + options.test_requires, env=virtualenv_env) + + # run the tests from $HOME, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command( + options.test_command, + project=Path('.').resolve(), + package=options.package_dir.resolve() + ) + call_with_arch(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) + + # clean up + shutil.rmtree(venv_dir) # we're all done here; move it to output (overwrite existing) shutil.move(str(repaired_wheel), options.output_dir) From 914f74a0f03077c503eba0784faf4643adbfd52a Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 16:03:24 +0000 Subject: [PATCH 18/59] Interim fix for archs comparison This will change when #507 is merged --- cibuildwheel/macos.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 6a06c53c8..daac47358 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -239,9 +239,9 @@ def setup_python(python_configuration: PythonConfiguration, def build(options: BuildOptions) -> None: allowed_archs = {Architecture.x86_64, Architecture.universal2, Architecture.arm64} - if set(options.architectures) <= allowed_archs: + if any(a not in allowed_archs for a in options.architectures): raise ValueError(textwrap.dedent(f''' - Invalid archs option {options.architectures}. macOS only supports + Invalid archs option {[a.value for a in options.architectures]}. macOS only supports these architectures: {', '.join(a.value for a in allowed_archs)}. ''')) From afd08c11f2177d2ebe317ea282755c946fd81934 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 16:12:24 +0000 Subject: [PATCH 19/59] Maybe Azure secretly supports macOS 11.0? --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b144c2db5..db2a56cb3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,7 +10,7 @@ jobs: python ./bin/run_tests.py - job: macos_38 - pool: {vmImage: 'macOS-10.15'} + pool: {vmImage: 'macos-11.0'} steps: - task: UsePythonVersion@0 inputs: From 7a11143be8063ea33d0751c4266209f130159d16 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 16:14:07 +0000 Subject: [PATCH 20/59] Like this? --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index db2a56cb3..04dc3c945 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,7 +10,7 @@ jobs: python ./bin/run_tests.py - job: macos_38 - pool: {vmImage: 'macos-11.0'} + pool: {vmImage: 'macOS-11.0'} steps: - task: UsePythonVersion@0 inputs: From 16d96fa33d882c9ab0cd3d7e6e6990575c01c840 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 16:18:17 +0000 Subject: [PATCH 21/59] It does not. --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 04dc3c945..b144c2db5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,7 +10,7 @@ jobs: python ./bin/run_tests.py - job: macos_38 - pool: {vmImage: 'macOS-11.0'} + pool: {vmImage: 'macOS-10.15'} steps: - task: UsePythonVersion@0 inputs: From 65b530a5dfa1105d005f9d97b56a34217725416d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 17:01:08 +0000 Subject: [PATCH 22/59] On arm64, build native before cross-compile --- cibuildwheel/macos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index daac47358..dc5044d15 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -56,8 +56,8 @@ def get_python_configurations(build_selector: BuildSelector, PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.9/python-3.7.9-macosx10.9.pkg'), PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.7/python-3.8.7-macosx10.9.pkg'), PythonConfiguration(version='3.9', identifier='cp39-macosx_x86_64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), - PythonConfiguration(version='3.9', identifier='cp39-macosx_universal2', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), PythonConfiguration(version='3.9', identifier='cp39-macosx_arm64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), + PythonConfiguration(version='3.9', identifier='cp39-macosx_universal2', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), # PyPy PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'), PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2'), From 121715a56829cabd0fc61e933ea325f6e39b5c47 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 17:01:23 +0000 Subject: [PATCH 23/59] Skip cross-compile test on macOS <11 --- test/test_macos_archs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py index cfcaddebd..990d08e95 100644 --- a/test/test_macos_archs.py +++ b/test/test_macos_archs.py @@ -8,6 +8,8 @@ def test_cross_compiled_build(tmp_path): if utils.platform != 'macos': pytest.skip('this test is only relevant to macos') + if utils.get_macos_version() < (11, 0): + pytest.skip('this test only works on macOS 11 or greater') project_dir = tmp_path / 'project' basic_project.generate(project_dir) From bf272fe9a6ceca828ff1fdcd9ae81b03b02b2f6f Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 17:01:47 +0000 Subject: [PATCH 24/59] Use same interpreter for sample_build.py (for local testing of arm64) --- bin/sample_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/sample_build.py b/bin/sample_build.py index d2adb022e..c995defa8 100755 --- a/bin/sample_build.py +++ b/bin/sample_build.py @@ -22,4 +22,4 @@ options.project_python_path, project_dir ], check=True) - exit(subprocess.run(['cibuildwheel'], cwd=project_dir).returncode) + exit(subprocess.run([sys.executable, '-m', 'cibuildwheel'], cwd=project_dir).returncode) From a6ff55ed5b03cb60be2f45a271d66b86178e21a2 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 17:37:55 +0000 Subject: [PATCH 25/59] Update pip in the test virtualenv --- cibuildwheel/macos.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index dc5044d15..e246b19a6 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -401,6 +401,12 @@ def call_with_arch(args: Sequence[PathOrStr], **kwargs: Any) -> int: # check that we are using the Python from the virtual environment call_with_arch(['which', 'python'], env=virtualenv_env) + # ensure we have a recent enough pip in the virtualenv + # TODO(joerick): remove this before merge? virtualenv should be bundling a + # recent enough seed version. Waiting on this PR: + # https://github.com/pypa/virtualenv/pull/2036 + call_with_arch(['pip', 'install', 'pip>=20.3.3'], env=virtualenv_env) + if options.before_test: before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir) call_with_arch(before_test_prepared, env=virtualenv_env, shell=True) From e215446283aaec979abae665d9becdbcd00b2e08 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 17:40:15 +0000 Subject: [PATCH 26/59] Increase test timeout value --- bin/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run_tests.py b/bin/run_tests.py index b46bfbe6f..9e8e9075b 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -17,4 +17,4 @@ subprocess.check_call(unit_test_args) # run the integration tests - subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', '--timeout=1200', 'test']) + subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', '--timeout=2400', 'test']) From 0df5e1295f1512c5641412d8b67c7935e4f80755 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 18:56:07 +0000 Subject: [PATCH 27/59] Only print testing warnings if test command is set --- cibuildwheel/macos.py | 64 +++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index e246b19a6..5a1f72be5 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -332,38 +332,38 @@ def build(options: BuildOptions) -> None: log.step_end() - machine_arch = platform.machine() - testing_archs: List[str] = [] - - if machine_arch == 'x86_64': - if config.identifier.endswith('_arm64'): - log.warning(wrap_text(''' - While arm64 wheels can be built on x86_64, they cannot be tested. The - ability to test the arm64 wheels will be added in a future release of - cibuildwheel, once Apple Silicon CI runners are widely available. - ''')) - testing_archs = [] - elif config.identifier.endswith('_universal2'): - log.warning(wrap_text(''' - While universal2 wheels can be built on x86_64, the arm64 part of them - cannot currently be tested. The ability to test the arm64 part of a - universal2 wheel will be added in a future release of cibuildwheel, once - Apple Silicon CI runners are widely available. - ''')) - testing_archs = ['x86_64'] - else: - testing_archs = ['x86_64'] - elif machine_arch == 'arm64': - if config.identifier.endswith('_x86_64'): - # testing using rosetta2 emulation - testing_archs = ['x86_64'] - elif config.identifier.endswith('_universal2'): - # testing the x86_64 using rosetta2 emulation - testing_archs = ['arm64', 'x86_64'] - else: - testing_archs = ['arm64'] - - if options.test_command and len(testing_archs) > 0: + if options.test_command: + machine_arch = platform.machine() + testing_archs: List[str] = [] + + if machine_arch == 'x86_64': + if config.identifier.endswith('_arm64'): + log.warning(wrap_text(''' + While arm64 wheels can be built on x86_64, they cannot be tested. The + ability to test the arm64 wheels will be added in a future release of + cibuildwheel, once Apple Silicon CI runners are widely available. + ''')) + testing_archs = [] + elif config.identifier.endswith('_universal2'): + log.warning(wrap_text(''' + While universal2 wheels can be built on x86_64, the arm64 part of them + cannot currently be tested. The ability to test the arm64 part of a + universal2 wheel will be added in a future release of cibuildwheel, once + Apple Silicon CI runners are widely available. + ''')) + testing_archs = ['x86_64'] + else: + testing_archs = ['x86_64'] + elif machine_arch == 'arm64': + if config.identifier.endswith('_x86_64'): + # testing using rosetta2 emulation + testing_archs = ['x86_64'] + elif config.identifier.endswith('_universal2'): + # testing the x86_64 using rosetta2 emulation + testing_archs = ['arm64', 'x86_64'] + else: + testing_archs = ['arm64'] + for testing_arch in testing_archs: log.step('Testing wheel...' if testing_arch == machine_arch else f'Testing wheel on {testing_arch}...') From ffba89dc6441b3db8c3280ec809d042724b41f22 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 19:02:15 +0000 Subject: [PATCH 28/59] Improve output formatting Github annotations require all output to be on a single line, so let's make the log messages do that --- cibuildwheel/logger.py | 4 ++++ cibuildwheel/macos.py | 12 ++++++------ cibuildwheel/util.py | 8 +++----- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index a6c693f38..4dd6418a5 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -124,6 +124,8 @@ def warning(self, message: str) -> None: c = self.colors print(f'{c.yellow}Warning{c.end} {message}') + print() + def error(self, error: Union[BaseException, str]) -> None: print() @@ -133,6 +135,8 @@ def error(self, error: Union[BaseException, str]) -> None: c = self.colors print(f'{c.bright_red}Error{c.end} {error}') + print() + def _start_fold_group(self, name: str) -> None: self._end_fold_group() self.active_fold_group_name = name diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 5a1f72be5..613ea26c6 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -13,7 +13,7 @@ from .logger import log from .util import (Architecture, BuildOptions, BuildSelector, NonPlatformWheelError, download, get_build_verbosity_extra_flags, get_pip_script, - install_certifi_script, prepare_command, wrap_text) + install_certifi_script, prepare_command, unwrap) from .typing import PathOrStr @@ -75,7 +75,7 @@ def get_python_configurations(build_selector: BuildSelector, if any(c.identifier.startswith('pp') for c in python_configurations): # pypy doesn't work on macOS 11 yet # See https://foss.heptapod.net/pypy/pypy/-/issues/3314 - log.warning(wrap_text(''' + log.warning(unwrap(''' PyPy is currently unsupported when building on macOS 11. To build macOS PyPy wheels, build on an older OS, such as macOS 10.15. To silence this warning, deselect PyPy by adding "pp*-macosx*" to your CIBW_SKIP option. @@ -84,7 +84,7 @@ def get_python_configurations(build_selector: BuildSelector, if any(c.identifier.startswith('cp35') for c in python_configurations): # CPython 3.5 doesn't work on macOS 11 - log.warning(wrap_text(''' + log.warning(unwrap(''' CPython 3.5 is unsupported when building on macOS 11. To build CPython 3.5 wheels, build on an older OS, such as macOS 10.15. To silence this warning, deselect CPython 3.5 by adding "cp35-macosx_x86_64" to your CIBW_SKIP option. @@ -338,14 +338,14 @@ def build(options: BuildOptions) -> None: if machine_arch == 'x86_64': if config.identifier.endswith('_arm64'): - log.warning(wrap_text(''' + log.warning(unwrap(''' While arm64 wheels can be built on x86_64, they cannot be tested. The ability to test the arm64 wheels will be added in a future release of cibuildwheel, once Apple Silicon CI runners are widely available. ''')) testing_archs = [] elif config.identifier.endswith('_universal2'): - log.warning(wrap_text(''' + log.warning(unwrap(''' While universal2 wheels can be built on x86_64, the arm64 part of them cannot currently be tested. The ability to test the arm64 part of a universal2 wheel will be added in a future release of cibuildwheel, once @@ -363,7 +363,7 @@ def build(options: BuildOptions) -> None: testing_archs = ['arm64', 'x86_64'] else: testing_archs = ['arm64'] - + for testing_arch in testing_archs: log.step('Testing wheel...' if testing_arch == machine_arch else f'Testing wheel on {testing_arch}...') diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 3654b51e1..5f19431d6 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -244,15 +244,13 @@ def detect_ci_provider() -> Optional[CIProvider]: return None -def wrap_text(text: str, max_width: int = 80) -> str: +def unwrap(text: str) -> str: ''' - Formats text, suitable for printing arbitrary long passages to console. + Unwraps multi-line text to a single line ''' # remove initial line indent text = textwrap.dedent(text) # remove leading/trailing whitespace text = text.strip() # remove consecutive whitespace - text = re.sub(r'\s+', ' ', text) - # wrap to max_width - return '\n'.join(textwrap.wrap(text, width=max_width)) + return re.sub(r'\s+', ' ', text) From e29fab0e2b3e58c1431964546c04a93819b88a00 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 19:25:25 +0000 Subject: [PATCH 29/59] Add test for cross-compile testing behaviour --- test/test_macos_archs.py | 42 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py index 990d08e95..a09ab113f 100644 --- a/test/test_macos_archs.py +++ b/test/test_macos_archs.py @@ -1,4 +1,5 @@ import pytest +import platform from . import test_projects, utils @@ -15,13 +16,52 @@ def test_cross_compiled_build(tmp_path): basic_project.generate(project_dir) actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ + 'CIBW_BUILD': 'cp39-*', 'CIBW_ARCHS': 'x86_64, universal2, arm64', }) - expected_wheels = ( + all_macos_wheels = ( utils.expected_wheels('spam', '0.1.0', machine_arch='x86_64') + utils.expected_wheels('spam', '0.1.0', machine_arch='arm64') ) + + # only cpython 3.9 + expected_wheels = [w for w in all_macos_wheels if 'cp39' in w] assert set(actual_wheels) == set(expected_wheels) + +@pytest.mark.parametrize('build_universal2', [False, True]) +def test_cross_compiled_test(tmp_path, capfd, build_universal2): + if utils.platform != 'macos': + pytest.skip('this test is only relevant to macos') + if utils.get_macos_version() < (11, 0): + pytest.skip('this test only works on macOS 11 or greater') + + project_dir = tmp_path / 'project' + basic_project.generate(project_dir) + + actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ + 'CIBW_BUILD': 'cp39-*', + 'CIBW_TEST_COMMAND': '''python -c "import platform; print('running tests on ' + platform.machine())"''', + 'CIBW_ARCHS': 'universal2' if build_universal2 else 'x86_64 arm64', + }) + + captured = capfd.readouterr() + + if platform.machine() == 'x86_64': + # ensure that tests were run on only x86_64 + assert 'running tests on x86_64' in captured.out + assert 'running tests on arm64' not in captured.out + if build_universal2: + assert 'While universal2 wheels can be built on x86_64, the arm64 part of them cannot currently be tested' in captured.out + else: + assert 'While arm64 wheels can be built on x86_64, they cannot be tested' in captured.out + elif platform.machine() == 'arm64': + # ensure that tests were run on both x86_64 and arm64 + assert 'running tests on x86_64' in captured.out + assert 'running tests on arm64' in captured.out + + print(actual_wheels) + + # TODO: add a TEST_COMMAND test when using cross-compiling From e880aca1552c1a1e893de01b3279f3b8385d2220 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 6 Jan 2021 21:33:41 +0000 Subject: [PATCH 30/59] Tidy up --- test/test_macos_archs.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py index a09ab113f..2fd215354 100644 --- a/test/test_macos_archs.py +++ b/test/test_macos_archs.py @@ -5,6 +5,11 @@ basic_project = test_projects.new_c_project() +ALL_MACOS_WHEELS = ( + utils.expected_wheels('spam', '0.1.0', machine_arch='x86_64') + + utils.expected_wheels('spam', '0.1.0', machine_arch='arm64') +) + def test_cross_compiled_build(tmp_path): if utils.platform != 'macos': @@ -20,13 +25,7 @@ def test_cross_compiled_build(tmp_path): 'CIBW_ARCHS': 'x86_64, universal2, arm64', }) - all_macos_wheels = ( - utils.expected_wheels('spam', '0.1.0', machine_arch='x86_64') - + utils.expected_wheels('spam', '0.1.0', machine_arch='arm64') - ) - - # only cpython 3.9 - expected_wheels = [w for w in all_macos_wheels if 'cp39' in w] + expected_wheels = [w for w in ALL_MACOS_WHEELS if 'cp39' in w] assert set(actual_wheels) == set(expected_wheels) @@ -61,7 +60,9 @@ def test_cross_compiled_test(tmp_path, capfd, build_universal2): assert 'running tests on x86_64' in captured.out assert 'running tests on arm64' in captured.out - print(actual_wheels) + if build_universal2: + expected_wheels = [w for w in ALL_MACOS_WHEELS if 'cp39' in w and 'universal2' in w] + else: + expected_wheels = [w for w in ALL_MACOS_WHEELS if 'cp39' in w and 'universal2' not in w] - -# TODO: add a TEST_COMMAND test when using cross-compiling + assert set(actual_wheels) == set(expected_wheels) From 07681239e43cb64881989b5c91c43676d38e6b88 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 6 Jan 2021 21:42:34 +0000 Subject: [PATCH 31/59] Fix allowed archs check --- cibuildwheel/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index d9481d0fe..bd549a024 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -262,7 +262,7 @@ def detect_ci_provider() -> Optional[CIProvider]: ALLOWED_ARCHITECTURES = { 'linux': {Architecture.x86_64, Architecture.i686, Architecture.aarch64, Architecture.ppc64le, Architecture.s390x}, - 'macos': {Architecture.x86_64}, + 'macos': {Architecture.x86_64, Architecture.universal2, Architecture.arm64}, 'windows': {Architecture.AMD64, Architecture.x86}, } From 1606b797496d974763e6bfd952109bbdfada2f58 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 8 Jan 2021 14:28:39 +0000 Subject: [PATCH 32/59] Fix macOS Big Sur version checks --- cibuildwheel/macos.py | 5 ++++- test/test_macos_archs.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 655516689..bf25bb594 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -33,6 +33,7 @@ def get_macos_version() -> Tuple[int, int]: These tuples can be used in comparisons, e.g. (10, 14) <= (11, 0) == True + (10, 14) <= (10, 16) == True (11, 2) <= (11, 0) != True ''' version_str, _, _ = platform.mac_ver() @@ -71,7 +72,9 @@ def get_python_configurations(build_selector: BuildSelector, # skip builds as required by BUILD/SKIP python_configurations = [c for c in python_configurations if build_selector(c.identifier)] - if get_macos_version() >= (11, 0): + # When running on macOS 11 in x86_64 mode, the reported OS is the + # non-existent '10.16'. + if get_macos_version() >= (10, 16): if any(c.identifier.startswith('pp') for c in python_configurations): # pypy doesn't work on macOS 11 yet # See https://foss.heptapod.net/pypy/pypy/-/issues/3314 diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py index 2fd215354..5cec82c2f 100644 --- a/test/test_macos_archs.py +++ b/test/test_macos_archs.py @@ -14,7 +14,7 @@ def test_cross_compiled_build(tmp_path): if utils.platform != 'macos': pytest.skip('this test is only relevant to macos') - if utils.get_macos_version() < (11, 0): + if utils.get_macos_version() < (10, 16): pytest.skip('this test only works on macOS 11 or greater') project_dir = tmp_path / 'project' @@ -33,7 +33,7 @@ def test_cross_compiled_build(tmp_path): def test_cross_compiled_test(tmp_path, capfd, build_universal2): if utils.platform != 'macos': pytest.skip('this test is only relevant to macos') - if utils.get_macos_version() < (11, 0): + if utils.get_macos_version() < (10, 16): pytest.skip('this test only works on macOS 11 or greater') project_dir = tmp_path / 'project' From c4b5b167ca23cbdc4d5e88a6b0a835b3ba03194a Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 8 Jan 2021 14:29:07 +0000 Subject: [PATCH 33/59] Add clarifying comments --- cibuildwheel/macos.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index bf25bb594..80eed9fcb 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -228,9 +228,13 @@ def setup_python(python_configuration: PythonConfiguration, if python_configuration.version == '3.9': if python_configuration.identifier.endswith('x86_64'): + # even on the macos11.0 Python installer, on the x86_64 side it's + # compatible back to 10.9. env.setdefault('_PYTHON_HOST_PLATFORM', 'macosx-10.9-x86_64') env.setdefault('ARCHFLAGS', '-arch x86_64') elif python_configuration.identifier.endswith('arm64'): + # macOS 11 is the first OS with arm64 support, so the wheels + # have that as a minimum. env.setdefault('_PYTHON_HOST_PLATFORM', 'macosx-11.0-arm64') env.setdefault('ARCHFLAGS', '-arch arm64') From f43b3f50c5707be11d58f4f33dfb6e07392cffa2 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 8 Jan 2021 14:57:15 +0000 Subject: [PATCH 34/59] Fix warning lines counting as build identifiers in test --- test/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/utils.py b/test/utils.py index 5be952d0b..2c1025b4d 100644 --- a/test/utils.py +++ b/test/utils.py @@ -37,7 +37,10 @@ def cibuildwheel_get_build_identifiers(project_path, env=None): env=env, ) - return cmd_output.strip().split('\n') + lines = cmd_output.strip().split('\n') + + # ignore warning lines in output + return [l for l in lines if l != '' and not l.startswith('Warning ')] def cibuildwheel_run(project_path, package_dir='.', env=None, add_env=None, output_dir=None): From 8512e69733794caced2640bcb4193a7ce5dbb10d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 8 Jan 2021 14:59:16 +0000 Subject: [PATCH 35/59] Improve expected_wheels correctness --- test/utils.py | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/test/utils.py b/test/utils.py index 2c1025b4d..369e95a75 100644 --- a/test/utils.py +++ b/test/utils.py @@ -76,6 +76,19 @@ def cibuildwheel_run(project_path, package_dir='.', env=None, add_env=None, outp return wheels +def _get_arm64_macosx_deployment_target(macosx_deployment_target: str) -> str: + ''' + The first version of macOS that supports arm is 11.0. So the wheel tag + cannot contain an earlier deployment target, even if + MACOSX_DEPLOYMENT_TARGET sets it. + ''' + version_tuple = tuple(map(int, macosx_deployment_target.split('.'))) + if version_tuple <= (11, 0): + return '11.0' + else: + return macosx_deployment_target + + def expected_wheels(package_name, package_version, manylinux_versions=None, macosx_deployment_target='10.9', machine_arch=None): ''' @@ -103,12 +116,17 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, if platform == 'linux': python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux - if platform == 'macos' and get_macos_version() >= (11, 0): + if platform == 'macos' and get_macos_version() >= (10, 16): # CPython 3.5 doesn't work on macOS 11. + # 10.16 is sometimes reported as the macOS version on macOS 11. python_abi_tags.remove('cp35-cp35m') # pypy not supported on macOS 11. python_abi_tags = [t for t in python_abi_tags if not t.startswith('pp')] + if platform == 'macos' and machine_arch == 'arm64': + # currently, arm64 macs are only supported by cp39 + python_abi_tags = ['cp39-cp39'] + wheels = [] for python_abi_tag in python_abi_tags: @@ -133,18 +151,12 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, platform_tags = ['win32'] elif platform == 'macos': - if python_abi_tag == 'cp39-cp39': - if machine_arch == 'x86_64': - platform_tags = [ - f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64', - ] - elif machine_arch == 'arm64': - platform_tags = [ - f'macosx_{macosx_deployment_target.replace(".", "_")}_universal2.macosx_11_0_universal2', - # macosx_deployment_target is ignored on arm64, because arm64 isn't supported on - # macOS earlier than 11.0 - 'macosx_11_0_arm64', - ] + if python_abi_tag == 'cp39-cp39' and machine_arch == 'arm64': + arm64_macosx_deployment_target = _get_arm64_macosx_deployment_target(macosx_deployment_target) + platform_tags = [ + f'macosx_{macosx_deployment_target.replace(".", "_")}_universal2.macosx_{arm64_macosx_deployment_target.replace(".", "_")}_universal2', + f'macosx_{arm64_macosx_deployment_target.replace(".", "_")}_arm64', + ] else: platform_tags = [ f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64', From a99726f477744230c5fe1f6a18a0f3cc2d17a298 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 8 Jan 2021 15:18:37 +0000 Subject: [PATCH 36/59] Neaten comments --- cibuildwheel/macos.py | 4 ++-- test/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 233048448..7aed4c1ad 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -81,8 +81,8 @@ def get_python_configurations(build_selector: BuildSelector, # skip builds as required by BUILD/SKIP python_configurations = [c for c in python_configurations if build_selector(c.identifier)] - # When running on macOS 11 in x86_64 mode, the reported OS is the - # non-existent '10.16'. + # When running on macOS 11 and x86_64, the reported OS is '10.16', but + # there is no such OS - it really means macOS 11. if get_macos_version() >= (10, 16): if any(c.identifier.startswith('pp') for c in python_configurations): # pypy doesn't work on macOS 11 yet diff --git a/test/utils.py b/test/utils.py index 369e95a75..e9b48e19d 100644 --- a/test/utils.py +++ b/test/utils.py @@ -117,8 +117,8 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux if platform == 'macos' and get_macos_version() >= (10, 16): - # CPython 3.5 doesn't work on macOS 11. # 10.16 is sometimes reported as the macOS version on macOS 11. + # CPython 3.5 doesn't work on macOS 11. python_abi_tags.remove('cp35-cp35m') # pypy not supported on macOS 11. python_abi_tags = [t for t in python_abi_tags if not t.startswith('pp')] From dbb6d80a7270b0f2c358a94074399ee4ff6b529e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 8 Jan 2021 17:25:07 +0000 Subject: [PATCH 37/59] Route warnings and error messages to stderr, not stdout. --- cibuildwheel/logger.py | 16 ++++------------ test/utils.py | 5 +---- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index ee98feec0..60e1b3c21 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -116,26 +116,18 @@ def step_end_with_error(self, error: Union[BaseException, str]) -> None: self.error(error) def warning(self, message: str) -> None: - print() - if self.fold_mode == 'github': - print(f'::warning::{message}') + print(f'::warning::{message}\n', file=sys.stderr) else: c = self.colors - print(f'{c.yellow}Warning{c.end} {message}') - - print() + print(f'{c.yellow}Warning{c.end}: {message}\n', file=sys.stderr) def error(self, error: Union[BaseException, str]) -> None: - print() - if self.fold_mode == 'github': - print(f'::error::{error}') + print(f'::error::{error}\n', file=sys.stderr) else: c = self.colors - print(f'{c.bright_red}Error{c.end} {error}') - - print() + print(f'{c.bright_red}Error{c.end}: {error}\n', file=sys.stderr) def _start_fold_group(self, name: str) -> None: self._end_fold_group() diff --git a/test/utils.py b/test/utils.py index e9b48e19d..1539e5ca2 100644 --- a/test/utils.py +++ b/test/utils.py @@ -37,10 +37,7 @@ def cibuildwheel_get_build_identifiers(project_path, env=None): env=env, ) - lines = cmd_output.strip().split('\n') - - # ignore warning lines in output - return [l for l in lines if l != '' and not l.startswith('Warning ')] + return cmd_output.strip().split('\n') def cibuildwheel_run(project_path, package_dir='.', env=None, add_env=None, output_dir=None): From af8ab0af2bedd2a78eed6424d67bcf2d4d6b6586 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 9 Jan 2021 01:20:49 +0000 Subject: [PATCH 38/59] Update test to look in stderr for warnings --- test/test_macos_archs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py index 590eb5b97..7037a97ed 100644 --- a/test/test_macos_archs.py +++ b/test/test_macos_archs.py @@ -53,9 +53,9 @@ def test_cross_compiled_test(tmp_path, capfd, build_universal2): assert 'running tests on x86_64' in captured.out assert 'running tests on arm64' not in captured.out if build_universal2: - assert 'While universal2 wheels can be built on x86_64, the arm64 part of them cannot currently be tested' in captured.out + assert 'While universal2 wheels can be built on x86_64, the arm64 part of them cannot currently be tested' in captured.err else: - assert 'While arm64 wheels can be built on x86_64, they cannot be tested' in captured.out + assert 'While arm64 wheels can be built on x86_64, they cannot be tested' in captured.err elif platform.machine() == 'arm64': # ensure that tests were run on both x86_64 and arm64 assert 'running tests on x86_64' in captured.out From e83b7709296a7f5dc55c869aeea880da2a0e8573 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 9 Jan 2021 13:41:02 +0000 Subject: [PATCH 39/59] Add back other platforms to GHA CI --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 161206c44..f3de75af8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,8 +24,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - # os: [ubuntu-latest, windows-latest, macos-11.0] - os: [macos-11.0] + os: [ubuntu-latest, windows-latest, macos-11.0] python_version: ['3.7'] timeout-minutes: 180 steps: From 3f8fcb01ad604d871bcff7db73410c412bc1e800 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 23 Jan 2021 18:15:29 +0000 Subject: [PATCH 40/59] Fix unit tests --- unit_test/main_tests/conftest.py | 3 +++ unit_test/main_tests/main_platform_test.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/unit_test/main_tests/conftest.py b/unit_test/main_tests/conftest.py index 3ce025a0a..2a0f9bfda 100644 --- a/unit_test/main_tests/conftest.py +++ b/unit_test/main_tests/conftest.py @@ -73,6 +73,9 @@ def platform(request, monkeypatch): else: monkeypatch.setattr(platform_module, 'machine', lambda: 'x86_64') + if platform_value == 'macos': + monkeypatch.setattr(macos, 'get_macos_version', lambda: (11, 1)) + return platform_value diff --git a/unit_test/main_tests/main_platform_test.py b/unit_test/main_tests/main_platform_test.py index a06ffe685..f86de5409 100644 --- a/unit_test/main_tests/main_platform_test.py +++ b/unit_test/main_tests/main_platform_test.py @@ -141,4 +141,4 @@ def test_archs_platform_all(platform, intercepted_build_args, monkeypatch): elif platform == 'windows': assert build_options.architectures == {Architecture.x86, Architecture.AMD64} elif platform == 'macos': - assert build_options.architectures == {Architecture.x86_64} + assert build_options.architectures == {Architecture.x86_64, Architecture.arm64, Architecture.universal2} From bd45b1c9011bfdd32119ae39f4c4b7d8c3a1f3d2 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 24 Jan 2021 14:46:46 +0000 Subject: [PATCH 41/59] Remove pinned pip install, virtualenv should bundle this now --- cibuildwheel/macos.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 8635f8ea4..3315bd267 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -400,12 +400,6 @@ def call_with_arch(args: Sequence[PathOrStr], **kwargs: Any) -> int: # check that we are using the Python from the virtual environment call_with_arch(['which', 'python'], env=virtualenv_env) - # ensure we have a recent enough pip in the virtualenv - # TODO(joerick): remove this before merge? virtualenv should be bundling a - # recent enough seed version. Waiting on this PR: - # https://github.com/pypa/virtualenv/pull/2036 - call_with_arch(['pip', 'install', 'pip>=20.3.3'], env=virtualenv_env) - if options.before_test: before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir) call_with_arch(before_test_prepared, env=virtualenv_env, shell=True) From 00eed5aac56c1250fcdf4155b97d2a0e9a4c5dea Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 24 Jan 2021 14:50:47 +0000 Subject: [PATCH 42/59] Fix crashing test --- test/test_dependency_versions.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/test_dependency_versions.py b/test/test_dependency_versions.py index 1b415bb49..72c105973 100644 --- a/test/test_dependency_versions.py +++ b/test/test_dependency_versions.py @@ -55,6 +55,13 @@ def test_pinned_versions(tmp_path, python_version): if utils.IS_WINDOWS_RUNNING_ON_TRAVIS and python_version == '2.7': pytest.skip('Windows + Travis CI requires a workaround') + is_running_on_macos_11_or_later = ( + utils.platform == 'macos' and utils.get_macos_version() >= (10, 16) + ) + + if is_running_on_macos_11_or_later and python_version == '3.5': + pytest.skip('CPython 3.5 doesn\'t work on macOS Big Sur+') + project_dir = tmp_path / 'project' project_with_expected_version_checks.generate(project_dir) From 4dddd9e6481c96488f5640f9ed9d988054cfbc37 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 24 Jan 2021 15:08:46 +0000 Subject: [PATCH 43/59] Silence the pip version warnings --- cibuildwheel/linux.py | 1 + cibuildwheel/macos.py | 3 +++ cibuildwheel/windows.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index bde5e34f9..5d19dbd20 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -94,6 +94,7 @@ def build(options: BuildOptions) -> None: env = docker.get_environment() env['PATH'] = f'/opt/python/cp38-cp38/bin:{env["PATH"]}' + env['PIP_DISABLE_PIP_VERSION_CHECK'] = '1' env = options.environment.as_dictionary(env, executor=docker.environment_executor) before_all_prepared = prepare_command(options.before_all, project=container_project_path, package=container_package_dir) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 3315bd267..dd1a87430 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -195,6 +195,9 @@ def setup_python(python_configuration: PythonConfiguration, env.pop('__PYVENV_LAUNCHER__', None) env = environment.as_dictionary(prev_environment=env) + # we version pip ourselves, so we don't care about pip version checking + env['PIP_DISABLE_PIP_VERSION_CHECK'] = '1' + # check what version we're on call(['which', 'python'], env=env) call(['python', '--version'], env=env) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 429150ebf..45f0141ab 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -143,6 +143,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain str(installation_path / 'Scripts'), env['PATH'] ]) + env['PIP_DISABLE_PIP_VERSION_CHECK'] = '1' + # update env with results from CIBW_ENVIRONMENT env = environment.as_dictionary(prev_environment=env) From 381ea7dbad70f5767068fbc26485d7d0547c5e34 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 24 Jan 2021 15:48:22 +0000 Subject: [PATCH 44/59] Add docs for CIBW_ARCHS --- docs/options.md | 54 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/docs/options.md b/docs/options.md index 3a1c0cd8f..857df08fd 100644 --- a/docs/options.md +++ b/docs/options.md @@ -64,7 +64,7 @@ Default: `auto` For `linux` you need Docker running, on macOS or Linux. For `macos`, you need a Mac machine, and note that this script is going to automatically install MacPython on your system, so don't run on your development machine. For `windows`, you need to run in Windows, and `cibuildwheel` will install required versions of Python to `C:\cibw\python` using NuGet. -This option can also be set using the command-line option `--platform`. +This option can also be set using the [command-line option](#command-line) `--platform`. ### `CIBW_BUILD`, `CIBW_SKIP` {: #build-skip} @@ -155,24 +155,36 @@ CIBW_SKIP: pp* } -### `CIBW_ARCHS_LINUX` {: #archs} -> Build non-native architectures +### `CIBW_ARCHS` {: #archs} +> Change the architectures built on your machine by default. -A space-separated list of architectures to build. Use this in conjunction with -emulation, such as that provided by [docker/setup-qemu-action][setup-qemu-action] -or [tonistiigi/binfmt][binfmt], to build architectures other than those your -machine natively supports. +A space-separated list of architectures to build. -Options: `auto` `native` `all` `x86_64` `i686` `aarch64` `ppc64le` `s390x` +On macOS, this option can be used to cross-compile between `x86_64`, +`universal2` and `arm64` for Apple Silicon support. -Default: `auto`, meaning the native archs supported on the build machine. For -example, on an `x86_64` machine, `auto` expands to `x86_64` and `i686`. +On Linux, this option can be used to build non-native architectures under emulation, such as that +provided by [docker/setup-qemu-action][setup-qemu-action] +or [tonistiigi/binfmt][binfmt]. See [this guide](faq.md#automatic-updates) for more information. -`native` will only build on the exact architecture you currently are on; it will -not add `i686` for `x86_64`. +Options: +- Linux: `x86_64` `i686` `aarch64` `ppc64le` `s390x` +- macOS: `x86_64` `arm64` `universal2` +- Windows: `x86` `AMD64` +- `auto`: The default archs for your machine - see the table below. +- `native`: the native arch of the build machine - Matches [`platform.machine()`](https://docs.python.org/3/library/platform.html#platform.machine). +- `all` : expands to all the architectures supported on this OS. You may want + to use [CIBW_BUILD](#build-skip) with this option to target specific + architectures via build selectors. -`all` will expand to all known architectures; remember to use build selectors -to limit builds for each job; this list could grow in the future. +Default: `auto` + +| Runner | `native` | `auto` +|---|---|---|--- +| macOS / Intel | `x86_64` | `x86_64` +| macOS / Apple Silicon | `arm64` | `arm64`, `universal2` +| Linux / Intel | `x86_64` | `x86_64`, `i686` +| Windows / Intel | `AMD64` | `AMD64`, `x86` [setup-qemu-action]: https://github.com/docker/setup-qemu-action [binfmt]: https://hub.docker.com/r/tonistiigi/binfmt @@ -180,10 +192,20 @@ to limit builds for each job; this list could grow in the future. #### Examples ```yaml -# On an intel runner with qemu installed, build Intel and ARM wheels +# Build `universal2` and `arm64` wheels on an Intel runner. +# Note that the `arm64` wheel and the `arm64` part of the `universal2` wheel +# cannot be tested in this configuration. +CIBW_ARCHS_MACOS: "x86_64 universal2 arm64" + +# On an Linux Intel runner with qemu installed, build Intel and ARM wheels CIBW_ARCHS_LINUX: "auto aarch64" ``` +Platform-specific variants also available:
+`CIBW_ARCHS_MACOS` | `CIBW_ARCHS_WINDOWS` | `CIBW_ARCHS_LINUX` + +This option can also be set using the [command-line option](#command-line) `--archs`. + ## Build customization ### `CIBW_ENVIRONMENT` {: #environment} @@ -548,7 +570,7 @@ CIBW_BUILD_VERBOSITY: 1 ``` -## Command line options +## Command line options {: #command-line} ```text usage: cibuildwheel [-h] [--platform {auto,linux,macos,windows}] From 4b491ceb4a1f448594732116522462367e40a11e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 24 Jan 2021 16:07:59 +0000 Subject: [PATCH 45/59] Docs edits --- docs/options.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/options.md b/docs/options.md index 857df08fd..2058ee406 100644 --- a/docs/options.md +++ b/docs/options.md @@ -163,14 +163,14 @@ A space-separated list of architectures to build. On macOS, this option can be used to cross-compile between `x86_64`, `universal2` and `arm64` for Apple Silicon support. -On Linux, this option can be used to build non-native architectures under emulation, such as that -provided by [docker/setup-qemu-action][setup-qemu-action] -or [tonistiigi/binfmt][binfmt]. See [this guide](faq.md#automatic-updates) for more information. +On Linux, this option can be used to build non-native architectures under +emulation. See [this guide](faq.md#emulation) for more information. Options: + - Linux: `x86_64` `i686` `aarch64` `ppc64le` `s390x` - macOS: `x86_64` `arm64` `universal2` -- Windows: `x86` `AMD64` +- Windows: `AMD64` `x86` - `auto`: The default archs for your machine - see the table below. - `native`: the native arch of the build machine - Matches [`platform.machine()`](https://docs.python.org/3/library/platform.html#platform.machine). - `all` : expands to all the architectures supported on this OS. You may want @@ -180,11 +180,11 @@ Options: Default: `auto` | Runner | `native` | `auto` -|---|---|---|--- -| macOS / Intel | `x86_64` | `x86_64` -| macOS / Apple Silicon | `arm64` | `arm64`, `universal2` -| Linux / Intel | `x86_64` | `x86_64`, `i686` -| Windows / Intel | `AMD64` | `AMD64`, `x86` +|---|---|--- +| Linux / Intel | `x86_64` | `x86_64` `i686` +| Windows / Intel | `AMD64` | `AMD64` `x86` +| macOS / Intel | `x86_64` | `x86_64` +| macOS / Apple Silicon | `arm64` | `arm64` `universal2` [setup-qemu-action]: https://github.com/docker/setup-qemu-action [binfmt]: https://hub.docker.com/r/tonistiigi/binfmt @@ -193,8 +193,8 @@ Default: `auto` ```yaml # Build `universal2` and `arm64` wheels on an Intel runner. -# Note that the `arm64` wheel and the `arm64` part of the `universal2` wheel -# cannot be tested in this configuration. +# Note that the `arm64` wheel and the `arm64` part of the `universal2` +# wheel cannot be tested in this configuration. CIBW_ARCHS_MACOS: "x86_64 universal2 arm64" # On an Linux Intel runner with qemu installed, build Intel and ARM wheels From ef1baac8e24246f53d2a59a1964f10fc197aa478 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 24 Jan 2021 16:12:30 +0000 Subject: [PATCH 46/59] Indent subheads in TOC on all pages --- docs/extra.css | 4 ++++ docs/options.md | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/extra.css b/docs/extra.css index 0a4dad6b5..e7d035aea 100644 --- a/docs/extra.css +++ b/docs/extra.css @@ -83,5 +83,9 @@ h1, h2, h3, h4, h5, h6 { padding-top: 0.9em; } +.toctree-l3 { + border-left: 10px solid transparent; +} + /* import font awesome 4 for icons */ @import url(https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css); diff --git a/docs/options.md b/docs/options.md index 2058ee406..1b80c58f0 100644 --- a/docs/options.md +++ b/docs/options.md @@ -614,10 +614,6 @@ optional arguments: ```