Load the native engine lib from a pkg_resource.

This switches pants to generating wheels instead of sdists and embeds
the proper platform-dependent native engine library as a resource within
the pantsbuild.pants wheel.

Fixes #4906

.gitignore

@@ -50,6 +50,3 @@ GPATH
 GRTAGS
 GSYMS
 GTAGS
-
-# Generated by build-support/bin/prepare-binary-deploy.sh for use by Travis-CI binary deploys.
-/build-support/bin/native/s3-upload/

.travis.yml

@@ -45,47 +45,40 @@ cache:
 matrix:
   include:
     - os: osx
+      # We request the oldest image we can for maximum compatibility.
+      osx_image: xcode6.4
       language: generic
       env:
         - SHARD="OSX Native Engine Binary Builder"
-        - NATIVE_ENGINE_DEPLOY=1
+        - PREPARE_DEPLOY=1
       script:
-        - ./build-support/bin/native/prepare-binary-deploy.sh
+        - ./pants --version && ./build-support/bin/release.sh -n
 
     - os: linux
       dist: trusty
       sudo: required
-      # Docker runs will write files as root, so avoid caching for this shard.
-      cache: false
       services:
         - docker
       language: python
       python: "2.7.13"
-      addons:
-        apt:
-          packages:
-            - cmake
       env:
         - SHARD="Linux Native Engine Binary Builder"
-        - NATIVE_ENGINE_DEPLOY=1
-        # Use the standard python-2.7 docker Debian Wheezy image for binary compatibility with old
-        # linux distros.
-        - DOCKER_IMAGE="python:2.7.13-wheezy"
+        - PREPARE_DEPLOY=1
       before_install:
-        # Remove any Ubuntu binary cruft before building.
+        # Remove any host-side binary cruft before building.
         - git clean -fdx
-        - rm -rf ~/.cache/pants
-        - docker pull $DOCKER_IMAGE
       script:
-        # Map virtual mount paths 1:1 to simulate running in-place in travis.
+        - docker build --rm -t travis_ci
+          --build-arg "TRAVIS_USER=$(id -un)"
+          --build-arg "TRAVIS_UID=$(id -u)"
+          --build-arg "TRAVIS_GROUP=$(id -gn)"
+          --build-arg "TRAVIS_GID=$(id -g)"
+            build-support/docker/travis_ci/
         - docker run --rm -t
-          -v "${TRAVIS_BUILD_DIR}:${TRAVIS_BUILD_DIR}"
-          -v "${HOME}:${HOME}"
-          -e "HOME=${HOME}"
-          "${DOCKER_IMAGE}"
-          sh -c "
-            cd $TRAVIS_BUILD_DIR && ./build-support/bin/native/prepare-binary-deploy.sh
-          "
+          -v "${HOME}:/travis/home"
+          -v "${TRAVIS_BUILD_DIR}:/travis/workdir"
+          travis_ci:latest
+          sh -c "./pants --version && ./build-support/bin/release.sh -n"
 
     - os: linux
       dist: trusty
@@ -141,7 +134,7 @@ matrix:
       env:
         - SHARD="Unit tests for pants and pants-plugins - shard 1"
       script:
-        - ./build-support/bin/ci.sh -x -efkmsrcnt -u 0/2 "${SHARD}"
+        - ./build-support/bin/ci.sh -x -efkmrcnt -u 0/2 "${SHARD}"
 
     - os: linux
       dist: trusty
@@ -169,7 +162,7 @@ matrix:
       env:
         - SHARD="Unit tests for pants and pants-plugins - shard 2"
       script:
-        - ./build-support/bin/ci.sh -x -efkmsrcnt -u 1/2 "${SHARD}"
+        - ./build-support/bin/ci.sh -x -efkmrcnt -u 1/2 "${SHARD}"
 
     - os: linux
       dist: trusty
@@ -197,7 +190,7 @@ matrix:
       env:
         - SHARD="Python contrib tests - shard 1"
       script:
-        - ./build-support/bin/ci.sh -x -efkmsrcjlpt -y 0/2 "${SHARD}"
+        - ./build-support/bin/ci.sh -x -efkmrcjlpt -y 0/2 "${SHARD}"
 
     - os: linux
       dist: trusty
@@ -225,7 +218,7 @@ matrix:
       env:
         - SHARD="Python contrib tests - shard 2"
       script:
-        - ./build-support/bin/ci.sh -x -efkmsrcjlpt -y 1/2 "${SHARD}"
+        - ./build-support/bin/ci.sh -x -efkmrcjlpt -y 1/2 "${SHARD}"
 
     - os: linux
       dist: trusty
@@ -253,7 +246,7 @@ matrix:
       env:
         - SHARD="Python integration tests for pants - shard 1"
       script:
-        - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 0/7 "${SHARD}"
+        - ./build-support/bin/ci.sh -x -efkmrjlpnt -i 0/7 "${SHARD}"
 
     - os: linux
       dist: trusty
@@ -281,7 +274,7 @@ matrix:
       env:
         - SHARD="Python integration tests for pants - shard 2"
       script:
-        - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 1/7 "${SHARD}"
+        - ./build-support/bin/ci.sh -x -efkmrjlpnt -i 1/7 "${SHARD}"
 
     - os: linux
       dist: trusty
@@ -309,7 +302,7 @@ matrix:
       env:
         - SHARD="Python integration tests for pants - shard 3"
       script:
-        - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 2/7 "${SHARD}"
+        - ./build-support/bin/ci.sh -x -efkmrjlpnt -i 2/7 "${SHARD}"
 
     - os: linux
       dist: trusty
@@ -337,7 +330,7 @@ matrix:
       env:
         - SHARD="Python integration tests for pants - shard 4"
       script:
-        - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 3/7 "${SHARD}"
+        - ./build-support/bin/ci.sh -x -efkmrjlpnt -i 3/7 "${SHARD}"
 
     - os: linux
       dist: trusty
@@ -365,7 +358,7 @@ matrix:
       env:
         - SHARD="Python integration tests for pants - shard 5"
       script:
-        - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 4/7 "${SHARD}"
+        - ./build-support/bin/ci.sh -x -efkmrjlpnt -i 4/7 "${SHARD}"
 
     - os: linux
       dist: trusty
@@ -393,7 +386,7 @@ matrix:
       env:
         - SHARD="Python integration tests for pants - shard 6"
       script:
-        - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 5/7 "${SHARD}"
+        - ./build-support/bin/ci.sh -x -efkmrjlpnt -i 5/7 "${SHARD}"
 
     - os: linux
       dist: trusty
@@ -421,7 +414,7 @@ matrix:
       env:
         - SHARD="Python integration tests for pants - shard 7"
       script:
-        - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 6/7 "${SHARD}"
+        - ./build-support/bin/ci.sh -x -efkmrjlpnt -i 6/7 "${SHARD}"
 
     # Rust tests
     - os: linux
@@ -435,7 +428,7 @@ matrix:
       env:
         - SHARD="Rust tests"
       script:
-        - ./build-support/bin/ci.sh -abcfjklmnprstx
+        - ./build-support/bin/ci.sh -abcfjklmnprtx
 
 deploy:
   # See: https://docs.travis-ci.com/user/deployment/s3/
@@ -444,12 +437,12 @@ deploy:
   secret_access_key:
     secure: RQVzsNfZL8AgsXdjZ67j2tWs5Tjl/FKpmE1fyVgldMbua/xhW8dzdFrtOeWjTPX4/+sJZ4U7/tZectBtWejmrXUJiZQKJwJBnsyYxysENTWOV80BEYyoz2RPr8HSVbMZ1ZHtUafzO3OqV1x+Pvgpg8FUeUfsy3TGUk0JREO90Q0=
   bucket: binaries.pantsbuild.org
-  local_dir: build-support/bin/native/s3-upload
-  # Otherwise travis will stash build-support/bin/native/s3-upload and the deploy will fail.
+  local_dir: dist/deploy
+  # Otherwise travis will stash dist/deploy and the deploy will fail.
   skip_cleanup: true
   acl: public_read
   on:
-    condition: $NATIVE_ENGINE_DEPLOY = 1
+    condition: $PREPARE_DEPLOY = 1
     branch: master
     repo: pantsbuild/pants
 

build-support/bin/ci.sh

@@ -21,7 +21,6 @@ function usage() {
   echo " -k           skip bootstrapped pants self compile check"
   echo " -m           skip sanity checks of bootstrapped pants and repo BUILD"
   echo "              files"
-  echo " -s           skip self-distribution tests"
   echo " -r           skip doc generation tests"
   echo " -j           skip core jvm tests"
   echo " -l           skip internal backends python tests"
@@ -67,7 +66,6 @@ while getopts "hfxbkmsrjlpeu:ny:ci:at" opt; do
     b) skip_bootstrap="true" ;;
     k) bootstrap_compile_args=() ;;
     m) skip_sanity_checks="true" ;;
-    s) skip_distribution="true" ;;
     r) skip_docs="true" ;;
     j) skip_jvm="true" ;;
     l) skip_internal_backends="true" ;;
@@ -169,14 +167,6 @@ if [[ "${skip_lint:-false}" == "false" ]]; then
   end_travis_section
 fi
 
-if [[ "${skip_distribution:-false}" == "false" ]]; then
-  # N.B. Defer start_travis_section to those within release.sh, since we can't nest.
-  banner "Running pants distribution tests"
-  (
-    ./build-support/bin/release.sh -n
-  ) || die "Failed to create pants distributions."
-fi
-
 if [[ "${skip_docs:-false}" == "false" ]]; then
   start_travis_section "DocGen" "Running site doc generation test"
   ./build-support/bin/publish_docs.sh || die "Failed to generate site docs."

build-support/bin/native/bootstrap.sh

@@ -1,6 +1,10 @@
 #!/usr/bin/env bash
 
 # Defines:
+# + LIB_EXTENSION: The extension of native libraries.
+# + KERNEL: The lower-cased name of the kernel as reported by uname.
+# + OS_NAME: The name of the OS as seen by pants.
+# + OS_ID: The ID of the current OS as seen by pants.
 # + CACHE_ROOT: The pants cache root dir.
 # + NATIVE_ENGINE_CACHE_DIR: The native engine binary root cache directory.
 # + NATIVE_ENGINE_CACHE_TARGET_DIR: The directory containing all versions of the native engine for
@@ -16,24 +20,29 @@
 REPO_ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd ../../.. && pwd -P)
 source ${REPO_ROOT}/build-support/common.sh
 
-# Defines:
-# + RUST_OSX_MIN_VERSION: The minimum minor version of OSX supported by Rust; eg 7 for OSX 10.7.
-# + OSX_MAX_VERSION: The current latest OSX minor version; eg 12 for OSX Sierra 10.12.
-# + LIB_EXTENSION: The extension of native libraries.
-# + KERNEL: The lower-cased name of the kernel as reported by uname.
-# + OS_NAME: The name of the OS as seen by pants.
-# + OS_ID: The ID of the current OS as seen by pants.
-# Exposes:
-# + get_native_engine_version: Echoes the current native engine version.
-# + get_rust_osx_versions: Produces the osx minor versions supported by Rust one per line.
-# + get_rust_osx_ids: Produces the BinaryUtil osx os id paths supported by rust, one per line.
-# + get_rust_os_ids: Produces the BinaryUtil os id paths supported by rust, one per line.
-source ${REPO_ROOT}/build-support/bin/native/utils.sh
+# TODO(John Sirois): Eliminate this replication of BinaryUtil logic internal to pants code when
+# https://github.com/pantsbuild/pants/issues/4006 is complete.
+readonly KERNEL=$(uname -s | tr '[:upper:]' '[:lower:]')
+case "${KERNEL}" in
+  linux)
+    readonly LIB_EXTENSION=so
+    readonly OS_NAME=linux
+    readonly OS_ID=${OS_NAME}/$(uname -m)
+    ;;
+  darwin)
+    readonly LIB_EXTENSION=dylib
+    readonly OS_NAME=mac
+    readonly OS_ID=${OS_NAME}/$(sw_vers -productVersion | cut -d: -f2 | tr -d ' \t' | cut -d. -f1-2)
+    ;;
+  *)
+    die "Unknown kernel ${KERNEL}, cannot bootstrap pants native code!"
+    ;;
+esac
 
 readonly NATIVE_ROOT="${REPO_ROOT}/src/rust/engine"
 readonly NATIVE_ENGINE_MODULE="native_engine"
 readonly NATIVE_ENGINE_BINARY="${NATIVE_ENGINE_MODULE}.so"
-readonly NATIVE_ENGINE_VERSION_RESOURCE="${REPO_ROOT}/src/python/pants/engine/native_engine_version"
+readonly NATIVE_ENGINE_RESOURCE="${REPO_ROOT}/src/python/pants/engine/${NATIVE_ENGINE_BINARY}"
 readonly CFFI_BOOTSTRAPPER="${REPO_ROOT}/build-support/native-engine/bootstrap_cffi.py"
 
 # N.B. Set $MODE to "debug" to generate a binary with debugging symbols.
@@ -121,8 +130,7 @@ function _build_native_code() {
 }
 
 function bootstrap_native_code() {
-  # Bootstraps the native code and overwrites the native_engine_version to the resulting hash
-  # version if needed.
+  # Bootstraps the native code only if needed.
   local native_engine_version="$(calculate_current_hash)"
   local target_binary="${NATIVE_ENGINE_CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}"
   if [ ! -f "${target_binary}" ]
@@ -142,10 +150,6 @@ function bootstrap_native_code() {
 
     mkdir -p "$(dirname ${target_binary})"
     cp "${native_binary}" "${target_binary}"
-
-    # NB: The resource file emitted/over-written below is used by the `Native` class to default
-    # the native engine library version used by pants. More info can be read at the end of this
-    # document:  src/python/pants/engine/README.md
-    echo ${native_engine_version} > ${NATIVE_ENGINE_VERSION_RESOURCE}
   fi
+  cp -p "${target_binary}" "${NATIVE_ENGINE_RESOURCE}"
 }