diff --git a/.gitignore b/.gitignore index 2390c7665c05..10d47923d8f2 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,3 @@ GPATH GRTAGS GSYMS GTAGS - -# Generated by build-support/bin/prepare-binary-deploy.sh for use by Travis-CI binary deploys. -/build-support/bin/native/s3-upload/ diff --git a/.travis.yml b/.travis.yml index 2db41ecaf48c..1c0edf67957c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,12 +45,14 @@ cache: matrix: include: - os: osx + # We request the oldest image we can for maximum compatibility. + osx_image: xcode6.4 language: generic env: - SHARD="OSX Native Engine Binary Builder" - - NATIVE_ENGINE_DEPLOY=1 + - PREPARE_DEPLOY=1 script: - - ./build-support/bin/native/prepare-binary-deploy.sh + - ./pants --version && ./build-support/bin/release.sh -n - os: linux dist: trusty @@ -61,30 +63,24 @@ matrix: - docker language: python python: "2.7.13" - addons: - apt: - packages: - - cmake env: - SHARD="Linux Native Engine Binary Builder" - - NATIVE_ENGINE_DEPLOY=1 - # Use the standard python-2.7 docker Debian Wheezy image for binary compatibility with old - # linux distros. - - DOCKER_IMAGE="python:2.7.13-wheezy" + - PREPARE_DEPLOY=1 before_install: - # Remove any Ubuntu binary cruft before building. + # Remove any host-side binary cruft before building. - git clean -fdx - - rm -rf ~/.cache/pants - - docker pull $DOCKER_IMAGE script: - # Map virtual mount paths 1:1 to simulate running in-place in travis. + - docker build --rm -t travis_ci + --build-arg TRAVIS_USER=$(id -un) + --build-arg TRAVIS_UID=$(id -u) + --build-arg TRAVIS_GROUP=$(id -gn) + --build-arg TRAVIS_GID=$(id -g) + build-support/docker/travis_ci/ - docker run --rm -t - -v "${TRAVIS_BUILD_DIR}:${TRAVIS_BUILD_DIR}" - -v "${HOME}:${HOME}" - -e "HOME=${HOME}" - "${DOCKER_IMAGE}" + -v ${TRAVIS_BUILD_DIR}:/workdir + travis_ci:latest sh -c " - cd $TRAVIS_BUILD_DIR && ./build-support/bin/native/prepare-binary-deploy.sh + ./pants --version && ./build-support/bin/release.sh -n " - os: linux @@ -141,7 +137,7 @@ matrix: env: - SHARD="Unit tests for pants and pants-plugins - shard 1" script: - - ./build-support/bin/ci.sh -x -efkmsrcnt -u 0/2 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmrcnt -u 0/2 "${SHARD}" - os: linux dist: trusty @@ -169,7 +165,7 @@ matrix: env: - SHARD="Unit tests for pants and pants-plugins - shard 2" script: - - ./build-support/bin/ci.sh -x -efkmsrcnt -u 1/2 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmrcnt -u 1/2 "${SHARD}" - os: linux dist: trusty @@ -197,7 +193,7 @@ matrix: env: - SHARD="Python contrib tests - shard 1" script: - - ./build-support/bin/ci.sh -x -efkmsrcjlpt -y 0/2 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmrcjlpt -y 0/2 "${SHARD}" - os: linux dist: trusty @@ -225,7 +221,7 @@ matrix: env: - SHARD="Python contrib tests - shard 2" script: - - ./build-support/bin/ci.sh -x -efkmsrcjlpt -y 1/2 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmrcjlpt -y 1/2 "${SHARD}" - os: linux dist: trusty @@ -253,7 +249,7 @@ matrix: env: - SHARD="Python integration tests for pants - shard 1" script: - - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 0/7 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmrjlpnt -i 0/7 "${SHARD}" - os: linux dist: trusty @@ -281,7 +277,7 @@ matrix: env: - SHARD="Python integration tests for pants - shard 2" script: - - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 1/7 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmrjlpnt -i 1/7 "${SHARD}" - os: linux dist: trusty @@ -309,7 +305,7 @@ matrix: env: - SHARD="Python integration tests for pants - shard 3" script: - - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 2/7 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmrjlpnt -i 2/7 "${SHARD}" - os: linux dist: trusty @@ -337,7 +333,7 @@ matrix: env: - SHARD="Python integration tests for pants - shard 4" script: - - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 3/7 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmrjlpnt -i 3/7 "${SHARD}" - os: linux dist: trusty @@ -365,7 +361,7 @@ matrix: env: - SHARD="Python integration tests for pants - shard 5" script: - - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 4/7 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmrjlpnt -i 4/7 "${SHARD}" - os: linux dist: trusty @@ -393,7 +389,7 @@ matrix: env: - SHARD="Python integration tests for pants - shard 6" script: - - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 5/7 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmrjlpnt -i 5/7 "${SHARD}" - os: linux dist: trusty @@ -421,7 +417,7 @@ matrix: env: - SHARD="Python integration tests for pants - shard 7" script: - - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 6/7 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmrjlpnt -i 6/7 "${SHARD}" # Rust tests - os: linux @@ -435,7 +431,7 @@ matrix: env: - SHARD="Rust tests" script: - - ./build-support/bin/ci.sh -abcfjklmnprstx + - ./build-support/bin/ci.sh -abcfjklmnprtx deploy: # See: https://docs.travis-ci.com/user/deployment/s3/ @@ -444,12 +440,12 @@ deploy: secret_access_key: secure: RQVzsNfZL8AgsXdjZ67j2tWs5Tjl/FKpmE1fyVgldMbua/xhW8dzdFrtOeWjTPX4/+sJZ4U7/tZectBtWejmrXUJiZQKJwJBnsyYxysENTWOV80BEYyoz2RPr8HSVbMZ1ZHtUafzO3OqV1x+Pvgpg8FUeUfsy3TGUk0JREO90Q0= bucket: binaries.pantsbuild.org - local_dir: build-support/bin/native/s3-upload - # Otherwise travis will stash build-support/bin/native/s3-upload and the deploy will fail. + local_dir: dist/deploy + # Otherwise travis will stash dist/deploy and the deploy will fail. skip_cleanup: true acl: public_read on: - condition: $NATIVE_ENGINE_DEPLOY = 1 + condition: $PREPARE_DEPLOY = 1 branch: master repo: pantsbuild/pants diff --git a/build-support/bin/check_native_engine_version.sh b/build-support/bin/check_native_engine_version.sh deleted file mode 100755 index c41e8e12d29d..000000000000 --- a/build-support/bin/check_native_engine_version.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -set -e - -REPO_ROOT="$(cd $(dirname "${BASH_SOURCE[0]}") && cd ../.. && pwd -P)" - -# Defines: -# + CACHE_ROOT: The pants cache root dir. -# + NATIVE_ENGINE_CACHE_DIR: The native engine binary root cache directory. -# + NATIVE_ENGINE_CACHE_TARGET_DIR: The directory containing all versions of the native engine for -# the current OS. -# + NATIVE_ENGINE_BINARY: The basename of the native engine binary for the current OS. -# + NATIVE_ENGINE_VERSION_RESOURCE: The path of the resource file containing the native engine -# version hash. -# Exposes: -# + calculate_current_hash: Calculates the current native engine version hash and echoes it to -# stdout. -# + bootstrap_native_code: Builds target-specific native engine binaries. -source "${REPO_ROOT}/build-support/bin/native/bootstrap.sh" - -readonly actual_native_engine_version="$(calculate_current_hash)" -readonly current_native_engine_version="$(cat ${NATIVE_ENGINE_VERSION_RESOURCE} | tr -d ' \n\r')" - - -if [ "${actual_native_engine_version}" != "${current_native_engine_version}" ]; then - die "failed verification: ${current_native_engine_version} != ${actual_native_engine_version}"; -else - echo "verified: ${current_native_engine_version}" -fi diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh index 4234646ac475..bea31f86dbaa 100755 --- a/build-support/bin/ci.sh +++ b/build-support/bin/ci.sh @@ -21,7 +21,6 @@ function usage() { echo " -k skip bootstrapped pants self compile check" echo " -m skip sanity checks of bootstrapped pants and repo BUILD" echo " files" - echo " -s skip self-distribution tests" echo " -r skip doc generation tests" echo " -j skip core jvm tests" echo " -l skip internal backends python tests" @@ -67,7 +66,6 @@ while getopts "hfxbkmsrjlpeu:ny:ci:at" opt; do b) skip_bootstrap="true" ;; k) bootstrap_compile_args=() ;; m) skip_sanity_checks="true" ;; - s) skip_distribution="true" ;; r) skip_docs="true" ;; j) skip_jvm="true" ;; l) skip_internal_backends="true" ;; @@ -169,14 +167,6 @@ if [[ "${skip_lint:-false}" == "false" ]]; then end_travis_section fi -if [[ "${skip_distribution:-false}" == "false" ]]; then - # N.B. Defer start_travis_section to those within release.sh, since we can't nest. - banner "Running pants distribution tests" - ( - ./build-support/bin/release.sh -n - ) || die "Failed to create pants distributions." -fi - if [[ "${skip_docs:-false}" == "false" ]]; then start_travis_section "DocGen" "Running site doc generation test" ./build-support/bin/publish_docs.sh || die "Failed to generate site docs." diff --git a/build-support/bin/native/bootstrap.sh b/build-support/bin/native/bootstrap.sh index 4f454d99355d..01d2c76b89fb 100644 --- a/build-support/bin/native/bootstrap.sh +++ b/build-support/bin/native/bootstrap.sh @@ -1,6 +1,10 @@ #!/usr/bin/env bash # Defines: +# + LIB_EXTENSION: The extension of native libraries. +# + KERNEL: The lower-cased name of the kernel as reported by uname. +# + OS_NAME: The name of the OS as seen by pants. +# + OS_ID: The ID of the current OS as seen by pants. # + CACHE_ROOT: The pants cache root dir. # + NATIVE_ENGINE_CACHE_DIR: The native engine binary root cache directory. # + NATIVE_ENGINE_CACHE_TARGET_DIR: The directory containing all versions of the native engine for @@ -16,24 +20,29 @@ REPO_ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd ../../.. && pwd -P) source ${REPO_ROOT}/build-support/common.sh -# Defines: -# + RUST_OSX_MIN_VERSION: The minimum minor version of OSX supported by Rust; eg 7 for OSX 10.7. -# + OSX_MAX_VERSION: The current latest OSX minor version; eg 12 for OSX Sierra 10.12. -# + LIB_EXTENSION: The extension of native libraries. -# + KERNEL: The lower-cased name of the kernel as reported by uname. -# + OS_NAME: The name of the OS as seen by pants. -# + OS_ID: The ID of the current OS as seen by pants. -# Exposes: -# + get_native_engine_version: Echoes the current native engine version. -# + get_rust_osx_versions: Produces the osx minor versions supported by Rust one per line. -# + get_rust_osx_ids: Produces the BinaryUtil osx os id paths supported by rust, one per line. -# + get_rust_os_ids: Produces the BinaryUtil os id paths supported by rust, one per line. -source ${REPO_ROOT}/build-support/bin/native/utils.sh +# TODO(John Sirois): Eliminate this replication of BinaryUtil logic internal to pants code when +# https://github.com/pantsbuild/pants/issues/4006 is complete. +readonly KERNEL=$(uname -s | tr '[:upper:]' '[:lower:]') +case "${KERNEL}" in + linux) + readonly LIB_EXTENSION=so + readonly OS_NAME=linux + readonly OS_ID=${OS_NAME}/$(uname -m) + ;; + darwin) + readonly LIB_EXTENSION=dylib + readonly OS_NAME=mac + readonly OS_ID=${OS_NAME}/$(sw_vers -productVersion | cut -d: -f2 | tr -d ' \t' | cut -d. -f1-2) + ;; + *) + die "Unknown kernel ${KERNEL}, cannot bootstrap pants native code!" + ;; +esac readonly NATIVE_ROOT="${REPO_ROOT}/src/rust/engine" readonly NATIVE_ENGINE_MODULE="native_engine" readonly NATIVE_ENGINE_BINARY="${NATIVE_ENGINE_MODULE}.so" -readonly NATIVE_ENGINE_VERSION_RESOURCE="${REPO_ROOT}/src/python/pants/engine/native_engine_version" +readonly NATIVE_ENGINE_RESOURCE="${REPO_ROOT}/src/python/pants/engine/${NATIVE_ENGINE_BINARY}" readonly CFFI_BOOTSTRAPPER="${REPO_ROOT}/build-support/native-engine/bootstrap_cffi.py" # N.B. Set $MODE to "debug" to generate a binary with debugging symbols. @@ -118,8 +127,7 @@ function _build_native_code() { } function bootstrap_native_code() { - # Bootstraps the native code and overwrites the native_engine_version to the resulting hash - # version if needed. + # Bootstraps the native code only if needed. local native_engine_version="$(calculate_current_hash)" local target_binary="${NATIVE_ENGINE_CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}" if [ ! -f "${target_binary}" ] @@ -139,10 +147,6 @@ function bootstrap_native_code() { mkdir -p "$(dirname ${target_binary})" cp "${native_binary}" "${target_binary}" - - # NB: The resource file emitted/over-written below is used by the `Native` class to default - # the native engine library version used by pants. More info can be read at the end of this - # document: src/python/pants/engine/README.md - echo ${native_engine_version} > ${NATIVE_ENGINE_VERSION_RESOURCE} fi + cp -p "${target_binary}" "${NATIVE_ENGINE_RESOURCE}" } diff --git a/build-support/bin/native/prepare-binary-deploy.sh b/build-support/bin/native/prepare-binary-deploy.sh deleted file mode 100755 index db9f86e7019f..000000000000 --- a/build-support/bin/native/prepare-binary-deploy.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash - -set -e - -REPO_ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd ../../.. && pwd -P) - -# Indirectly defines: -# + RUST_OSX_MIN_VERSION: The minimum minor version of OSX supported by Rust; eg 7 for OSX 10.7. -# + OSX_MAX_VERSION: The current latest OSX minor version; eg 12 for OSX Sierra 10.12. -# + LIB_EXTENSION: The extension of native libraries. -# + KERNEL: The lower-cased name of the kernel as reported by uname. -# + OS_NAME: The name of the OS as seen by pants. -# + OS_ID: The ID of the current OS as seen by pants. -# Indirectly exposes: -# + get_native_engine_version: Echoes the current native engine version. -# + get_rust_osx_versions: Produces the osx minor versions supported by Rust one per line. -# + get_rust_osx_ids: Produces the BinaryUtil osx os id paths supported by rust, one per line. -# + get_rust_os_ids: Produces the BinaryUtil os id paths supported by rust, one per line. -# Defines: -# + CACHE_ROOT: The pants cache root dir. -# + NATIVE_ENGINE_CACHE_DIR: The native engine binary root cache directory. -# + NATIVE_ENGINE_CACHE_TARGET_DIR: The directory containing all versions of the native engine for -# the current OS. -# + NATIVE_ENGINE_BINARY: The basename of the native engine binary for the current OS. -# + NATIVE_ENGINE_VERSION_RESOURCE: The path of the resource file containing the native engine -# version hash. -# Exposes: -# + calculate_current_hash: Calculates the current native engine version hash and echoes it to -# stdout. -# + bootstrap_native_code: Builds target-specific native engine binaries. -source "${REPO_ROOT}/build-support/bin/native/bootstrap.sh" - -readonly native_engine_version=$(get_native_engine_version) -readonly cached_bin_path="${NATIVE_ENGINE_CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}" - -readonly s3_upload_root="${REPO_ROOT}/build-support/bin/native/s3-upload" -readonly s3_native_engine_dir="${s3_upload_root}/bin/native-engine" - -function prepare_chroot() { - rm -rf "${s3_upload_root}" - mkdir -p "$(dirname ${s3_native_engine_dir})" - cp -vpr "${NATIVE_ENGINE_CACHE_DIR}" "${s3_native_engine_dir}" -} - -function prepare_osx_versions() { - for os_id in $(get_rust_osx_ids) - do - if [ "${OS_ID}" != "${os_id}" ] - then - local target="${s3_native_engine_dir}/${os_id}/${native_engine_version}/${NATIVE_ENGINE_BINARY}" - mkdir -p "$(dirname ${target})" - cp -vp "${cached_bin_path}" "${target}" - fi - done -} - -# Sanity check the locally built native engine binary exists in the first place. -ensure_file_exists "${cached_bin_path}" - -# Prepare a chroot for s3 deploy of the binary(ies). -prepare_chroot - -# Maybe add copies of the mac native engine binary for the other supported OSX versions. -if [ "${OS_NAME}" == "mac" ] -then - prepare_osx_versions -fi \ No newline at end of file diff --git a/build-support/bin/native/utils.sh b/build-support/bin/native/utils.sh deleted file mode 100644 index 805897699a82..000000000000 --- a/build-support/bin/native/utils.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash - -# Defines: -# + RUST_OSX_MIN_VERSION: The minimum minor version of OSX supported by Rust; eg 7 for OSX 10.7. -# + OSX_MAX_VERSION: The current latest OSX minor version; eg 12 for OSX Sierra 10.12. -# + LIB_EXTENSION: The extension of native libraries. -# + KERNEL: The lower-cased name of the kernel as reported by uname. -# + OS_NAME: The name of the OS as seen by pants. -# + OS_ID: The ID of the current OS as seen by pants. -# Exposes: -# + get_native_engine_version: Echoes the current native engine version. -# + get_rust_osx_versions: Produces the osx minor versions supported by Rust one per line. -# + get_rust_osx_ids: Produces the BinaryUtil osx os id paths supported by rust, one per line. -# + get_rust_os_ids: Produces the BinaryUtil os id paths supported by rust, one per line. - -REPO_ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd ../../.. && pwd -P) - -function get_native_engine_version() { - ${REPO_ROOT}/pants options --scope=GLOBAL --name=native-engine-version --output-format=json | \ - python -c 'import json, sys; print(json.load(sys.stdin)["native_engine_version"]["value"])' -} - -# Rust targets OSX 10.7+ as noted here: https://doc.rust-lang.org/book/getting-started.html#tier-1 -readonly RUST_OSX_MIN_VERSION=7 - -# Bump this when there is a new OSX released: -readonly OSX_MAX_VERSION=13 - -function get_rust_osx_versions() { - seq ${RUST_OSX_MIN_VERSION} ${OSX_MAX_VERSION} -} - -function get_rust_osx_ids() { - for rev in $(get_rust_osx_versions) - do - echo "mac/10.${rev}" - done -} - -function get_rust_os_ids() { - echo "linux/x86_64" - get_rust_osx_ids -} - -# TODO(John Sirois): Eliminate this replication of BinaryUtil logic internal to pants code when -# https://github.com/pantsbuild/pants/issues/4006 is complete. -readonly KERNEL=$(uname -s | tr '[:upper:]' '[:lower:]') -case "${KERNEL}" in - linux) - readonly LIB_EXTENSION=so - readonly OS_NAME=linux - readonly OS_ID=${OS_NAME}/$(uname -m) - ;; - darwin) - readonly LIB_EXTENSION=dylib - readonly OS_NAME=mac - readonly OS_ID=${OS_NAME}/$(sw_vers -productVersion | cut -d: -f2 | tr -d ' \t' | cut -d. -f1-2) - ;; - *) - die "Unknown kernel ${KERNEL}, cannot bootstrap pants native code!" - ;; -esac diff --git a/build-support/bin/pre-commit.sh b/build-support/bin/pre-commit.sh index ded37ff11dd1..daed8ed415da 100755 --- a/build-support/bin/pre-commit.sh +++ b/build-support/bin/pre-commit.sh @@ -9,8 +9,6 @@ then export GIT_HOOK=1 fi -# N.B. This check needs to happen first, before any inadvertent bootstrapping can take place. -echo -n "* Checking native_engine_version: " && ./build-support/bin/check_native_engine_version.sh || exit 1 echo "* Checking packages" && ./build-support/bin/check_packages.sh || exit 1 echo "* Checking headers" && ./build-support/bin/check_header.sh || exit 1 echo "* Checking for banned imports" && ./build-support/bin/check_banned_imports.sh || exit 1 diff --git a/build-support/bin/release.sh b/build-support/bin/release.sh index 268569aea4aa..84f5d9a5b664 100755 --- a/build-support/bin/release.sh +++ b/build-support/bin/release.sh @@ -9,8 +9,37 @@ PY=$(which python2.7) [[ -n "${PY}" ]] || die "You must have python2.7 installed and on the path to release." export PY +function run_local_pants() { + ${ROOT}/pants "$@" +} + +function local_version() { + run_local_pants --version 2>/dev/null +} + +readonly DEPLOY_DIR="${ROOT}/dist/deploy" +readonly DEPLOY_WHEELS_PATH="wheels/pantsbuild.pants/$(local_version)" +readonly DEPLOY_WHEEL_DIR="${DEPLOY_DIR}/${DEPLOY_WHEELS_PATH}" + source ${ROOT}/contrib/release_packages.sh +function find_pkg() { + local readonly pkg_name=$1 + find "${ROOT}/dist/${pkg_name}-$(local_version)/dist" \ + -type f \ + -name "${pkg_name}-$(local_version)-*.whl" +} + +function find_plat_name() { + # See: https://www.python.org/dev/peps/pep-0425/#id13 + "${PY}" << EOF +from __future__ import print_function +from distutils.util import get_platform + +print(get_platform().replace('-', '_').replace('.', '_')) +EOF +} + # # List of packages to be released # @@ -18,8 +47,9 @@ source ${ROOT}/contrib/release_packages.sh # # PKG_=( # "package.name" -# "build.target" +# "//a/build:target" # "pkg__install_test" +# "bdist_wheel flags" # NB: this entry is optional. # ) # function pkg__install_test() { # ... @@ -29,10 +59,11 @@ PKG_PANTS=( "pantsbuild.pants" "//src/python/pants:pants-packaged" "pkg_pants_install_test" + "--python-tag cp27 --plat-name=$(find_plat_name)" ) function pkg_pants_install_test() { PIP_ARGS="$@" - pip install ${PIP_ARGS} "${ROOT}/dist/pantsbuild.pants-$(local_version).tar.gz" || \ + pip install ${PIP_ARGS} "$(find_pkg pantsbuild.pants)" || \ die "pip install of pantsbuild.pants failed!" execute_packaged_pants_with_internal_backends list src:: || \ die "'pants list src::' failed in venv!" @@ -47,7 +78,7 @@ PKG_PANTS_TESTINFRA=( ) function pkg_pants_testinfra_install_test() { PIP_ARGS="$@" - pip install ${PIP_ARGS} "${ROOT}/dist/pantsbuild.pants.testinfra-$(local_version).tar.gz" && \ + pip install ${PIP_ARGS} "$(find_pkg pantsbuild.pants.testinfra)" && \ python -c "import pants_test" } @@ -61,10 +92,6 @@ RELEASE_PACKAGES=( # End of package declarations. # -function run_local_pants() { - ${ROOT}/pants "$@" -} - # When we do (dry-run) testing, we need to run the packaged pants. # It doesn't have internal backend plugins so when we execute it # at the repo build root, the root pants.ini will ask it to load @@ -74,7 +101,8 @@ function run_local_pants() { function execute_packaged_pants_with_internal_backends() { pip install --ignore-installed \ -r pants-plugins/3rdparty/python/requirements.txt &> /dev/null && \ - PANTS_PYTHON_REPOS_REPOS="['${ROOT}/dist']" pants \ + pants \ + --python-repos-repos="${DEPLOY_WHEEL_DIR}" \ --no-verify-config \ --pythonpath="['pants-plugins/src/python']" \ --backend-packages="[\ @@ -109,35 +137,33 @@ function pkg_install_test_func() { echo ${INSTALL_TEST_FUNC} } -function local_version() { - run_local_pants --version 2>/dev/null +function bdist_wheel_flags() { + PACKAGE=$1 + eval BDIST_WHEEL_FLAGS=\${$PACKAGE[3]} + echo ${BDIST_WHEEL_FLAGS} } function build_packages() { + rm -rf "${DEPLOY_WHEEL_DIR}" + mkdir -p "${DEPLOY_WHEEL_DIR}" + for PACKAGE in "${RELEASE_PACKAGES[@]}" do NAME=$(pkg_name $PACKAGE) BUILD_TARGET=$(pkg_build_target $PACKAGE) + BDIST_WHEEL_FLAGS=$(bdist_wheel_flags $PACKAGE) start_travis_section "${NAME}" "Building package ${NAME}-$(local_version) with target '${BUILD_TARGET}'" - run_local_pants setup-py --recursive ${BUILD_TARGET} || \ + run_local_pants setup-py \ + --run="bdist_wheel ${BDIST_WHEEL_FLAGS:---python-tag py27}" \ + ${BUILD_TARGET} || \ die "Failed to build package ${NAME}-$(local_version) with target '${BUILD_TARGET}'!" + wheel=$(find_pkg ${NAME}) + cp -p "${wheel}" "${DEPLOY_WHEEL_DIR}/$(basename ${wheel})" end_travis_section done } -function publish_packages() { - targets=() - for PACKAGE in "${RELEASE_PACKAGES[@]}" - do - targets+=($(pkg_build_target $PACKAGE)) - done - start_travis_section "Publishing" "Publishing packages" - run_local_pants setup-py --run="sdist upload --sign --identity=$(get_pgp_keyid)" \ - --recursive ${targets[@]} || die "Failed to publish packages!" - end_travis_section -} - function pre_install() { start_travis_section "SetupVenv" "Setting up virtualenv" VENV_DIR=$(mktemp -d -t pants.XXXXX) && \ @@ -198,7 +224,7 @@ function install_and_test_packages() { function dry_run_install() { build_packages && \ - install_and_test_packages --find-links=file://${ROOT}/dist + install_and_test_packages --find-links="${DEPLOY_WHEEL_DIR}" } ALLOWED_ORIGIN_URLS=( @@ -281,7 +307,7 @@ EOM ) die "${msg}" fi - ${PY} << EOF || die + "${PY}" << EOF || die from __future__ import print_function import os @@ -422,55 +448,42 @@ EOM fi } -# Indirectly defines: -# + RUST_OSX_MIN_VERSION: The minimum minor version of OSX supported by Rust; eg 7 for OSX 10.7. -# + OSX_MAX_VERSION: The current latest OSX minor version; eg 12 for OSX Sierra 10.12. -# + LIB_EXTENSION: The extension of native libraries. -# + KERNEL: The lower-cased name of the kernel as reported by uname. -# + OS_NAME: The name of the OS as seen by pants. -# + OS_ID: The ID of the current OS as seen by pants. -# Indirectly exposes: -# + get_native_engine_version: Echoes the current native engine version. -# + get_rust_osx_versions: Produces the osx minor versions supported by Rust one per line. -# + get_rust_osx_ids: Produces the BinaryUtil osx os id paths supported by rust, one per line. -# + get_rust_os_ids: Produces the BinaryUtil os id paths supported by rust, one per line. -# Defines: -# + CACHE_ROOT: The pants cache root dir. -# + NATIVE_ENGINE_CACHE_DIR: The native engine binary root cache directory. -# + NATIVE_ENGINE_CACHE_TARGET_DIR: The directory containing all versions of the native engine for -# the current OS. -# + NATIVE_ENGINE_BINARY: The basename of the native engine binary for the current OS. -# + NATIVE_ENGINE_VERSION_RESOURCE: The path of the resource file containing the native engine -# version hash. -# Exposes: -# + calculate_current_hash: Calculates the current native engine version hash and echoes it to -# stdout. -# + bootstrap_native_code: Builds target-specific native engine binaries. -source ${ROOT}/build-support/bin/native/bootstrap.sh - readonly BINARY_BASE_URL=https://binaries.pantsbuild.org -readonly NATIVE_ENGINE_BASE_URL=${BINARY_BASE_URL}/bin/native-engine -function check_native_engine() { - local readonly native_engine_version=${NATIVE_ENGINE_VERSION:-$(get_native_engine_version)} - banner "Checking for native engine release version ${native_engine_version}" +function list_prebuilt_wheels() { + wheel_listing="$(mktemp -t pants.wheels.XXXXX)" + curl -sSL "${BINARY_BASE_URL}/?prefix=${DEPLOY_WHEELS_PATH}" > "${wheel_listing}" + "${PY}" << EOF +from __future__ import print_function +import sys +import xml.etree.ElementTree as ET +root = ET.parse("${wheel_listing}") +ns = {'s3': 'http://s3.amazonaws.com/doc/2006-03-01/'} +for key in root.findall('s3:Contents/s3:Key', ns): + print(key.text) +EOF +} - local readonly headers=$(mktemp -t pants-release.XXXXXX) - local result=0 - for os_id in $(get_rust_os_ids) - do - local url=${NATIVE_ENGINE_BASE_URL}/${os_id}/${native_engine_version}/${NATIVE_ENGINE_BINARY} - echo -n " for ${os_id} -> ${url}... " - curl --progress-bar --fail --head ${url} &> ${headers} && echo OK || { - result=$(( ${result} + 1 )) && echo FAILURE && cat ${headers} && echo - } - done - rm -f ${headers} +function check_prebuilt_wheels() { + banner "Checking for prebuilt wheels for $(local_version)" - if (( ${result} != 0 )) - then - die "Failed to find ${result} releases of native engine version ${native_engine_version}" - fi + # TODO(John Sirois): FINISH ME + # 1. Use list_prebuilt_wheels to fetch all prebuilt wheels to a dir + # 2. Verify we have 1 or more wheels for each RELEASE_PACKAGES + exit 1 +} + +function publish_packages() { + rm -rf "${DEPLOY_WHEEL_DIR}" + mkdir -p "${DEPLOY_WHEEL_DIR}" + + start_travis_section "Publishing" "Publishing packages" + + check_prebuilt_wheels + + # TODO(John Sirois): FINISH ME: Setup a twine venv and for each wheel `twine upload --sign` + + end_travis_section } function usage() { @@ -490,7 +503,7 @@ function usage() { echo " and can be installed in an ephemeral virtualenv." echo " -l Lists all pantsbuild packages that this script releases." echo " -o Lists all pantsbuild package owners." - echo " -e Check that native engine binaries are deployed for this release." + echo " -e Check that wheels are prebuilt for this release." echo echo "All options (except for '-d') are mutually exclusive." @@ -509,7 +522,7 @@ while getopts "hdntloe" opt; do t) test_release="true" ;; l) list_packages && exit 0 ;; o) list_owners && exit 0 ;; - e) check_native_engine && exit 0 ;; + e) check_prebuilt_wheels && exit 0 ;; *) usage "Invalid option: -${OPTARG}" ;; esac done @@ -536,8 +549,8 @@ elif [[ "${test_release}" == "true" ]]; then else banner "Releasing packages to PyPi" && \ ( - check_origin && check_clean_branch && check_pgp && check_native_engine && check_owners && \ - dry_run_install && publish_packages && tag_release && publish_docs_if_master && \ + check_origin && check_clean_branch && check_pgp && check_owners && \ + publish_packages && tag_release && publish_docs_if_master && \ banner "Successfully released packages to PyPi" ) || die "Failed to release packages to PyPi." fi diff --git a/build-support/docker/travis_ci/Dockerfile b/build-support/docker/travis_ci/Dockerfile new file mode 100644 index 000000000000..aef13dcbbee1 --- /dev/null +++ b/build-support/docker/travis_ci/Dockerfile @@ -0,0 +1,37 @@ +# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +# Use the standard python-2.7 docker Debian Wheezy image for binary compatibility with old linux +# distros. +FROM python:2.7.13-wheezy + +# Ensure Pants runs under the 2.7.13 interpreter. +ENV PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython==2.7.13']" + +# Install various things Pants requires. +RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" > \ + /etc/apt/sources.list.d/java-8-debian.list && \ + echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" >> \ + /etc/apt/sources.list.d/java-8-debian.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886 && \ + echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ + echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + cmake \ + oracle-java8-installer + +# Setup a mount point for the travis ci workdir. +VOLUME /workdir + +# Setup a non-root user to execute the build under (avoids problems with npm install). +ARG TRAVIS_USER=travis_ci +ARG TRAVIS_UID=1000 +ARG TRAVIS_GROUP=root +ARG TRAVIS_GID=0 + +RUN groupadd --gid ${TRAVIS_GID} ${TRAVIS_GROUP} +RUN useradd -d /workdir -g ${TRAVIS_GROUP} --uid ${TRAVIS_UID} ${TRAVIS_USER} +USER ${TRAVIS_USER}:${TRAVIS_GROUP} + +WORKDIR /workdir diff --git a/src/python/pants/engine/.gitignore b/src/python/pants/engine/.gitignore new file mode 100644 index 000000000000..f5ae7ccc8435 --- /dev/null +++ b/src/python/pants/engine/.gitignore @@ -0,0 +1 @@ +/native_engine.so \ No newline at end of file diff --git a/src/python/pants/engine/BUILD b/src/python/pants/engine/BUILD index ad434ec8b76e..342e35ae0258 100644 --- a/src/python/pants/engine/BUILD +++ b/src/python/pants/engine/BUILD @@ -180,7 +180,7 @@ python_library( name='native', sources=['native.py'], dependencies=[ - ':native_engine_version', + ':native_engine_shared_library', '3rdparty/python:cffi', '3rdparty/python:setuptools', 'src/python/pants/binaries:binary_util', @@ -191,6 +191,6 @@ python_library( ) resources( - name='native_engine_version', - sources=['native_engine_version'] + name='native_engine_shared_library', + sources=['native_engine.so'] ) diff --git a/src/python/pants/engine/native.py b/src/python/pants/engine/native.py index 2536a2e7e944..9a3468d97960 100644 --- a/src/python/pants/engine/native.py +++ b/src/python/pants/engine/native.py @@ -12,12 +12,13 @@ import sysconfig import threading import traceback +from contextlib import closing import cffi +import pkg_resources import six -from pants.binaries.binary_util import BinaryUtil -from pants.util.dirutil import safe_mkdir, touch +from pants.util.dirutil import safe_mkdir, safe_mkdtemp, touch from pants.util.memo import memoized_property from pants.util.objects import datatype @@ -562,22 +563,12 @@ class Native(object): @staticmethod def create(options): """:param options: Any object that provides access to bootstrap option values.""" - binary_util = BinaryUtil.Factory.create() - return Native(binary_util, - options.native_engine_version, - options.native_engine_supportdir, - options.native_engine_visualize_to) + return Native(options.native_engine_visualize_to) - def __init__(self, binary_util, version, supportdir, visualize_to_dir): + def __init__(self, visualize_to_dir): """ - :param binary_util: The BinaryUtil subsystem instance for binary retrieval. - :param version: The binary version of the native engine. - :param supportdir: The supportdir for the native engine. :param visualize_to_dir: An existing directory (or None) to visualize executions to. """ - self._binary_util = binary_util - self._version = version - self._supportdir = supportdir self._visualize_to_dir = visualize_to_dir @property @@ -587,9 +578,12 @@ def visualize_to_dir(self): @memoized_property def binary(self): """Load and return the path to the native engine binary.""" - return self._binary_util.select_binary(self._supportdir, - self._version, - '{}.so'.format(NATIVE_ENGINE_MODULE)) + lib_name = '{}.so'.format(NATIVE_ENGINE_MODULE) + lib_path = os.path.join(safe_mkdtemp(), lib_name) + with closing(pkg_resources.resource_stream(__name__, lib_name)) as input_fp: + with open(lib_path, 'wb') as output_fp: + output_fp.write(input_fp.read()) + return lib_path @memoized_property def lib(self): diff --git a/src/python/pants/engine/native_engine_version b/src/python/pants/engine/native_engine_version deleted file mode 100644 index b6461610443a..000000000000 --- a/src/python/pants/engine/native_engine_version +++ /dev/null @@ -1 +0,0 @@ -647a1da5e0d1b5584e216c6ba52305cfd211f507 diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py index 148ff464bc1e..8d12040a2f0f 100644 --- a/src/python/pants/option/global_options.py +++ b/src/python/pants/option/global_options.py @@ -9,8 +9,6 @@ import os import sys -import pkg_resources - from pants.base.build_environment import (get_buildroot, get_default_pants_config_file, get_pants_cachedir, get_pants_configdir, pants_version) from pants.option.arg_splitter import GLOBAL_SCOPE @@ -135,12 +133,6 @@ def register_bootstrap_options(cls, register): help='Enables use of the v2 engine.') # These facilitate configuring the native engine. - register('--native-engine-version', advanced=True, - default=pkg_resources.resource_string('pants.engine', 'native_engine_version').strip(), - help='Native engine version.') - register('--native-engine-supportdir', advanced=True, default='bin/native-engine', - help='Find native engine binaries under this dir. Used as part of the path to ' - 'lookup the binary with --binary-util-baseurls and --pants-bootstrapdir.') register('--native-engine-visualize-to', advanced=True, default=None, type=dir_option, help='A directory to write execution and rule graphs to as `dot` files. The contents ' 'of the directory will be overwritten if any filenames collide.')