From 3740cc502faf87fa71ef68add168716743f5c9e1 Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Sat, 2 Dec 2017 11:56:14 -0700 Subject: [PATCH] Post suffixed-wheel release fixups (#5152) ### Problem In order to complete the `1.4.0.dev22` release, a few additional edits to the `release.sh` script were necessary. ### Solution 1. Update `find_pkg` to locate wheels by version. 2. Use a more portable implementation of case insensitive username matching. 3. URL-escape S3 keys (since `+` has special meaning in URLs). 4. Validate that if a whl is not cross-platform, we have it for exactly two platforms. --- build-support/bin/release.sh | 92 ++++++++++++++++++++++-------------- src/docs/release.md | 6 +-- 2 files changed, 58 insertions(+), 40 deletions(-) diff --git a/build-support/bin/release.sh b/build-support/bin/release.sh index b5f18e55cc7..27547ef8a11 100755 --- a/build-support/bin/release.sh +++ b/build-support/bin/release.sh @@ -34,8 +34,9 @@ source ${ROOT}/contrib/release_packages.sh function find_pkg() { local readonly pkg_name=$1 - local readonly search_dir=${2:-${ROOT}/dist/${pkg_name}-${PANTS_UNSTABLE_VERSION}/dist} - find "${search_dir}" -type f -name "${pkg_name}-${PANTS_UNSTABLE_VERSION}-*.whl" + local readonly version=$2 + local readonly search_dir=$3 + find "${search_dir}" -type f -name "${pkg_name}-${version}-*.whl" } function find_plat_name() { @@ -205,13 +206,14 @@ function build_pants_packages() { BDIST_WHEEL_FLAGS=$(bdist_wheel_flags $PACKAGE) start_travis_section "${NAME}" "Building package ${NAME}-${version} with target '${BUILD_TARGET}'" - run_local_pants setup-py \ - --run="sdist bdist_wheel ${BDIST_WHEEL_FLAGS:---python-tag py27}" \ - ${BUILD_TARGET} || \ - die "Failed to build package ${NAME}-${version} with target '${BUILD_TARGET}'!" - wheel=$(find_pkg ${NAME}) - cp -p "${wheel}" "${DEPLOY_PANTS_WHEEL_DIR}/${version}" - cp -p "${ROOT}/dist/${NAME}-${version}/dist/${NAME}-${version}.tar.gz" "${DEPLOY_PANTS_SDIST_DIR}/${version}" + ( + run_local_pants setup-py \ + --run="sdist bdist_wheel ${BDIST_WHEEL_FLAGS:---python-tag py27}" \ + ${BUILD_TARGET} && \ + wheel=$(find_pkg ${NAME} ${version} "${ROOT}/dist") && \ + cp -p "${wheel}" "${DEPLOY_PANTS_WHEEL_DIR}/${version}" && \ + cp -p "${ROOT}/dist/${NAME}-${version}/dist/${NAME}-${version}.tar.gz" "${DEPLOY_PANTS_SDIST_DIR}/${version}" + ) || die "Failed to build package ${NAME}-${version} with target '${BUILD_TARGET}'!" end_travis_section done pants_version_reset @@ -261,8 +263,6 @@ function install_and_test_packages() { --no-cache-dir ) - echo "PIP_ARGS: ${PIP_ARGS[@]}" - pre_install || die "Failed to setup virtualenv while testing ${NAME}-${VERSION}!" # Avoid caching plugin installs. @@ -473,20 +473,19 @@ function list_owners() { } function check_owner() { - username="$1" - package_name="$2" - - for owner in $(get_owners ${package_name}) - do - # NB: A case-insensitive comparison is done since pypi is case-insensitive wrt usernames. - # Note that the ^^ case operator requires bash 4. If you're on a Mac you may need to brew - # install bash, as the version that comes with MacOS is ancient. - if [[ "${username^^}" == "${owner^^}" ]] - then - return 0 - fi - done - return 1 + username=$(echo "$1" | tr '[:upper:]' '[:lower:]') + package_name="$2" + + for owner in $(get_owners ${package_name}) + do + # NB: A case-insensitive comparison is done since pypi is case-insensitive wrt usernames. + owner=$(echo "${owner}" | tr '[:upper:]' '[:lower:]') + if [[ "${username}" == "${owner}" ]] + then + return 0 + fi + done + return 1 } function check_owners() { @@ -542,6 +541,7 @@ function reversion_whls() { readonly BINARY_BASE_URL=https://binaries.pantsbuild.org function list_prebuilt_wheels() { + # List prebuilt wheels as tab-separated tuples of filename and URL-encoded name. wheel_listing="$(mktemp -t pants.wheels.XXXXX)" trap "rm -f ${wheel_listing}" RETURN @@ -550,11 +550,14 @@ function list_prebuilt_wheels() { "${PY}" << EOF from __future__ import print_function import sys +import urllib import xml.etree.ElementTree as ET root = ET.parse("${wheel_listing}") ns = {'s3': 'http://s3.amazonaws.com/doc/2006-03-01/'} for key in root.findall('s3:Contents/s3:Key', ns): - print(key.text) + # Because filenames may contain characters that have different meanings + # in URLs (namely '+'), # print the key both as url-encoded and as a file path. + print('{}\t{}'.format(key.text, urllib.quote_plus(key.text))) EOF done } @@ -566,12 +569,15 @@ function fetch_prebuilt_wheels() { ( cd "${to_dir}" list_prebuilt_wheels | { - while read path + while read path_tuple do - echo "${BINARY_BASE_URL}/${path}:" - local dest="${to_dir}/${path}" + local file_path=$(echo "$path_tuple" | awk -F'\t' '{print $1}') + local url_path=$(echo "$path_tuple" | awk -F'\t' '{print $2}') + echo "${BINARY_BASE_URL}/${url_path}:" + local dest="${to_dir}/${file_path}" mkdir -p "$(dirname "${dest}")" - curl --progress-bar -o "${dest}" "${BINARY_BASE_URL}/${path}" + curl --fail --progress-bar -o "${dest}" "${BINARY_BASE_URL}/${url_path}" \ + || die "Could not fetch ${dest}." done } ) @@ -593,8 +599,26 @@ function fetch_and_check_prebuilt_wheels() { for PACKAGE in "${RELEASE_PACKAGES[@]}" do NAME=$(pkg_name $PACKAGE) - packages=($(find_pkg "${NAME}" "${check_dir}")) - (( ${#packages[@]} > 0 )) || missing+=("${NAME}") + packages=($(find_pkg "${NAME}" "${PANTS_UNSTABLE_VERSION}" "${check_dir}")) + if [ ${#packages[@]} -eq 0 ]; then + missing+=("${NAME}") + continue + fi + + # Confirm that if the package is not cross platform that we have whls for two platforms. + local cross_platform="" + for package in "${packages[@]}" + do + if [[ "${package}" =~ "-none-any.whl" ]] + then + cross_platform="true" + fi + done + + if [ "${cross_platform}" != "true" ] && [ ${#packages[@]} -ne 2 ]; then + missing+=("${NAME} (expected whls for each platform: had only ${packages[@]})") + continue + fi # Here we re-name the linux platform specific wheels we build to masquerade as manylinux1 # compatible wheels. We take care to support this when we generate the wheels and pypi will @@ -678,10 +702,6 @@ function usage() { echo " -e Check that wheels are prebuilt for this release." echo echo "All options (except for '-d' and '-c') are mutually exclusive." - echo - echo "There is one environment variable that significantly affects behaviour: setting" - echo "SUFFIXED_VERSION to a non-empty value will cause all commands to operate on a" - echo "git HEAD-SHA suffixed version of pants." if (( $# > 0 )); then die "$@" diff --git a/src/docs/release.md b/src/docs/release.md index e3e41d02c95..70ed15571c1 100644 --- a/src/docs/release.md +++ b/src/docs/release.md @@ -72,9 +72,6 @@ script fail: username: password: EOF - - - The release script requires Bash 4. If you're on MacOS you may have to run `brew install bash`, - as the Bash that ships with MacOS is ancient. - Note that the release script expects your pantsbuild/pants git remote to be named `origin`. If you have another name for it, you should `git remote rename othername origin` before running @@ -163,7 +160,8 @@ is not required. 3. Publish to PyPi ------------------ -Now that we've smoke-tested this release, we can publish to PyPi: +Once the first two travis shards (the "binary builder" shards) have completed for your +release commit, you can publish to PyPi: :::bash $ ./build-support/bin/release.sh