[CI] Build Python wheels for MacOS (x86_64 and arm64) (#7621)

* Build Python wheels for OSX (x86_64 and arm64)

* Use Conda's libomp when running Python tests

* fix

* Add comment to explain CIBW_TARGET_OSX_ARM64

* Update release script

* Add comments in build_python_wheels.sh

* Document wheel pipeline

.github/workflows/main.yml

@@ -21,10 +21,7 @@ jobs:
         submodules: 'true'
     - name: Install system packages
       run: |
-        # Use libomp 11.1.0: https://github.com/dmlc/xgboost/issues/7039
-        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/679923b4eb48a8dc7ecc1f05d06063cd79b3fc00/Formula/libomp.rb -O $(find $(brew --repository) -name libomp.rb)
         brew install ninja libomp
-        brew pin libomp
     - name: Build gtest binary
       run: |
         mkdir build

.github/workflows/python_tests.yml

@@ -17,10 +17,7 @@ jobs:
     - name: Install osx system dependencies
       if: matrix.os == 'macos-10.15'
       run: |
-        # Use libomp 11.1.0: https://github.com/dmlc/xgboost/issues/7039
-        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/679923b4eb48a8dc7ecc1f05d06063cd79b3fc00/Formula/libomp.rb -O $(find $(brew --repository) -name libomp.rb)
         brew install ninja libomp
-        brew pin libomp
     - name: Install Ubuntu system dependencies
       if: matrix.os == 'ubuntu-latest'
       run: |
@@ -119,14 +116,16 @@ jobs:
         conda list
 
     - name: Build XGBoost on macos
+      shell: bash -l {0}
       run: |
-        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/679923b4eb48a8dc7ecc1f05d06063cd79b3fc00/Formula/libomp.rb -O $(find $(brew --repository) -name libomp.rb)
-        brew install ninja libomp
-        brew pin libomp
+        brew install ninja
 
         mkdir build
         cd build
-        cmake .. -GNinja -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON
+        # Set prefix, to use OpenMP library from Conda env
+        # See https://github.com/dmlc/xgboost/issues/7039#issuecomment-1025038228
+        # to learn why we don't use libomp from Homebrew.
+        cmake .. -GNinja -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DCMAKE_PREFIX_PATH=$CONDA_PREFIX
         ninja
 
     - name: Install Python package
@@ -141,24 +140,3 @@ jobs:
       shell: bash -l {0}
       run: |
         pytest -s -v ./tests/python
-
-    - name: Rename Python wheel
-      shell: bash -l {0}
-      run: |
-        TAG=macosx_10_15_x86_64.macosx_11_0_x86_64.macosx_12_0_x86_64
-        python tests/ci_build/rename_whl.py python-package/dist/*.whl ${{ github.sha }} ${TAG}
-
-    - name: Extract branch name
-      shell: bash
-      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-      id: extract_branch
-      if: github.ref == 'refs/heads/master' || contains(github.ref, 'refs/heads/release_')
-
-    - name: Upload Python wheel
-      shell: bash -l {0}
-      if: github.ref == 'refs/heads/master' || contains(github.ref, 'refs/heads/release_')
-      run: |
-        python -m awscli s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/${{ steps.extract_branch.outputs.branch }}/ --acl public-read
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_IAM_S3_UPLOADER }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_IAM_S3_UPLOADER }}

.github/workflows/python_wheels.yml

@@ -0,0 +1,57 @@
+name: XGBoost-Python-Wheels
+
+on: [push, pull_request]
+
+jobs:
+  python-wheels:
+    name: Build wheel for ${{ matrix.platform_id }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+        - os: macos-latest
+          python: 37
+          platform_id: macosx_x86_64
+          wheel_tag: macosx_10_15_x86_64.macosx_11_0_x86_64.macosx_12_0_x86_64
+        - os: macos-latest
+          python: 38
+          platform_id: macosx_arm64
+          wheel_tag: macosx_12_0_arm64
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: 'true'
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+    - name: Set env var for ARM64
+      shell: bash
+      run: echo "::set-output name=value::CIBW_TARGET_OSX_ARM64=1"
+      id: arm64_flag
+      if: matrix.platform_id == 'macosx_arm64'
+    - name: Build wheels
+      env:
+        CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }}
+        CIBW_ARCHS: all
+        CIBW_ENVIRONMENT: ${{ steps.arm64_flag.outputs.value }}
+        CIBW_TEST_SKIP: "*-macosx_arm64"
+        CIBW_BUILD_VERBOSITY: 3
+      run: bash tests/ci_build/build_python_wheels.sh
+
+    - name: Rename Python wheel
+      run: |
+        python tests/ci_build/rename_whl.py wheelhouse/*.whl ${{ github.sha }} ${{ matrix.wheel_tag }}
+    - name: Extract branch name
+      shell: bash
+      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+      id: extract_branch
+      if: github.ref == 'refs/heads/master' || contains(github.ref, 'refs/heads/release_')
+    - name: Upload Python wheel
+      if: github.ref == 'refs/heads/master' || contains(github.ref, 'refs/heads/release_')
+      run: |
+        python -m pip install awscli
+        python -m awscli s3 cp wheelhouse/*.whl s3://xgboost-nightly-builds/${{ steps.extract_branch.outputs.branch }}/ --acl public-read
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_IAM_S3_UPLOADER }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_IAM_S3_UPLOADER }}

dev/release-py-r.py

@@ -80,7 +80,8 @@ def download_py_packages(major: int, minor: int, commit_hash: str):
         "win_amd64",
         "manylinux2014_x86_64",
         "manylinux2014_aarch64",
-        "macosx_10_14_x86_64.macosx_10_15_x86_64.macosx_11_0_x86_64",
+        "macosx_10_15_x86_64.macosx_11_0_x86_64.macosx_12_0_x86_64",
+        "macosx_12_0_arm64"
     ]
 
     dir_URL = PREFIX + str(major) + "." + str(minor) + ".0" + "/"

doc/contrib/ci.rst

@@ -25,3 +25,15 @@ requests and every update to branches. A few tests however require manual activa
   details about noLD. This is a requirement for keeping XGBoost on CRAN (the R package index).
   To invoke this test suite for a particular pull request, simply add a review comment
   ``/gha run r-nold-test``. (Ordinary comment won't work. It needs to be a review comment.)
+
+GitHub Actions is also used to build Python wheels targeting MacOS Intel and Apple Silicon. See
+`.github/workflows/python_wheels.yml
+<https://github.com/dmlc/xgboost/tree/master/.github/workflows/python_wheels.yml>`_. The
+``python_wheels`` pipeline sets up environment variables prefixed ``CIBW_*`` to indicate the target
+OS and processor. The pipeline then invokes the script ``build_python_wheels.sh``, which in turns
+calls ``cibuildwheel`` to build the wheel. The ``cibuildwheel`` is a library that sets up a
+suitable Python environment for each OS and processor target. Since we don't have Apple Silion
+machine in GitHub Actions, cross-compilation is needed; ``cibuildwheel`` takes care of the complex
+task of cross-compiling a Python wheel. (Note that ``cibuildwheel`` will call
+``setup.py bdist_wheel``. Since XGBoost has a native library component, ``setup.py`` contains
+a glue code to call CMake and a C++ compiler to build the native library on the fly.)

python-package/setup.py

@@ -119,6 +119,13 @@ def build(
                 continue
             cmake_cmd.append('-D' + arg + '=' + value)
 
+        # Flag for cross-compiling for Apple Silicon
+        # We use environment variable because it's the only way to pass down custom flags
+        # through the cibuildwheel package, which otherwise calls `python setup.py bdist_wheel`
+        # command.
+        if 'CIBW_TARGET_OSX_ARM64' in os.environ:
+            cmake_cmd.append("-DCMAKE_OSX_ARCHITECTURES=arm64")
+
         self.logger.info('Run CMake command: %s', str(cmake_cmd))
         subprocess.check_call(cmake_cmd, cwd=build_dir)
 

tests/ci_build/build_python_wheels.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -e
+set -x
+
+# Bundle libomp 11.1.0 when targeting MacOS.
+# This is a workaround in order to prevent segfaults when running inside a Conda environment.
+# See https://github.com/dmlc/xgboost/issues/7039#issuecomment-1025125003 for more context.
+# The workaround is also used by the scikit-learn project.
+if [[ "$RUNNER_OS" == "macOS" ]]; then
+    # Make sure to use a libomp version binary compatible with the oldest
+    # supported version of the macos SDK as libomp will be vendored into the
+    # XGBoost wheels for MacOS.
+
+    if [[ "$CIBW_BUILD" == *-macosx_arm64 ]]; then
+        # arm64 builds must cross compile because CI is on x64
+        # cibuildwheel will take care of cross-compilation.
+        export PYTHON_CROSSENV=1
+        export MACOSX_DEPLOYMENT_TARGET=12.0
+        OPENMP_URL="https://anaconda.org/conda-forge/llvm-openmp/11.1.0/download/osx-arm64/llvm-openmp-11.1.0-hf3c4609_1.tar.bz2"
+    else
+        export MACOSX_DEPLOYMENT_TARGET=10.13
+        OPENMP_URL="https://anaconda.org/conda-forge/llvm-openmp/11.1.0/download/osx-64/llvm-openmp-11.1.0-hda6cdc1_1.tar.bz2"
+    fi
+
+    sudo conda create -n build $OPENMP_URL
+    PREFIX="/usr/local/miniconda/envs/build"
+
+    export CC=/usr/bin/clang
+    export CXX=/usr/bin/clang++
+    export CPPFLAGS="$CPPFLAGS -Xpreprocessor -fopenmp"
+    export CFLAGS="$CFLAGS -I$PREFIX/include"
+    export CXXFLAGS="$CXXFLAGS -I$PREFIX/include"
+    export LDFLAGS="$LDFLAGS -Wl,-rpath,$PREFIX/lib -L$PREFIX/lib -lomp"
+fi
+
+python -m pip install cibuildwheel
+python -m cibuildwheel python-package --output-dir wheelhouse

tests/ci_build/conda_env/macos_cpu_test.yml

@@ -10,6 +10,7 @@ dependencies:
 - pylint
 - numpy
 - scipy
+- llvm-openmp
 - scikit-learn
 - pandas
 - matplotlib