Post suffixed-wheel release fixups (#5152)

### Problem

In order to complete the `1.4.0.dev22` release, a few additional edits to the `release.sh` script were necessary.

### Solution

1. Update `find_pkg` to locate wheels by version.
2. Use a more portable implementation of case insensitive username matching.
3. URL-escape S3 keys (since `+` has special meaning in URLs).
4. Validate that if a whl is not cross-platform, we have it for exactly two platforms.

build-support/bin/release.sh

@@ -34,8 +34,9 @@ source ${ROOT}/contrib/release_packages.sh
 
 function find_pkg() {
   local readonly pkg_name=$1
-  local readonly search_dir=${2:-${ROOT}/dist/${pkg_name}-${PANTS_UNSTABLE_VERSION}/dist}
-  find "${search_dir}" -type f -name "${pkg_name}-${PANTS_UNSTABLE_VERSION}-*.whl"
+  local readonly version=$2
+  local readonly search_dir=$3
+  find "${search_dir}" -type f -name "${pkg_name}-${version}-*.whl"
 }
 
 function find_plat_name() {
@@ -205,13 +206,14 @@ function build_pants_packages() {
     BDIST_WHEEL_FLAGS=$(bdist_wheel_flags $PACKAGE)
 
     start_travis_section "${NAME}" "Building package ${NAME}-${version} with target '${BUILD_TARGET}'"
-    run_local_pants setup-py \
-      --run="sdist bdist_wheel ${BDIST_WHEEL_FLAGS:---python-tag py27}" \
-        ${BUILD_TARGET} || \
-      die "Failed to build package ${NAME}-${version} with target '${BUILD_TARGET}'!"
-    wheel=$(find_pkg ${NAME})
-    cp -p "${wheel}" "${DEPLOY_PANTS_WHEEL_DIR}/${version}"
-    cp -p "${ROOT}/dist/${NAME}-${version}/dist/${NAME}-${version}.tar.gz" "${DEPLOY_PANTS_SDIST_DIR}/${version}"
+    (
+      run_local_pants setup-py \
+        --run="sdist bdist_wheel ${BDIST_WHEEL_FLAGS:---python-tag py27}" \
+          ${BUILD_TARGET} && \
+      wheel=$(find_pkg ${NAME} ${version} "${ROOT}/dist") && \
+      cp -p "${wheel}" "${DEPLOY_PANTS_WHEEL_DIR}/${version}" && \
+      cp -p "${ROOT}/dist/${NAME}-${version}/dist/${NAME}-${version}.tar.gz" "${DEPLOY_PANTS_SDIST_DIR}/${version}"
+    ) || die "Failed to build package ${NAME}-${version} with target '${BUILD_TARGET}'!"
     end_travis_section
   done
   pants_version_reset
@@ -261,8 +263,6 @@ function install_and_test_packages() {
     --no-cache-dir
   )
 
-  echo "PIP_ARGS: ${PIP_ARGS[@]}"
-
   pre_install || die "Failed to setup virtualenv while testing ${NAME}-${VERSION}!"
 
   # Avoid caching plugin installs.
@@ -473,20 +473,19 @@ function list_owners() {
 }
 
 function check_owner() {
-   username="$1"
-   package_name="$2"
-
-   for owner in $(get_owners ${package_name})
-   do
-     # NB: A case-insensitive comparison is done since pypi is case-insensitive wrt usernames.
-     # Note that the ^^ case operator requires bash 4.  If you're on a Mac you may need to brew
-     # install bash, as the version that comes with MacOS is ancient.
-     if [[ "${username^^}" == "${owner^^}" ]]
-     then
-       return 0
-     fi
-   done
-   return 1
+  username=$(echo "$1" | tr '[:upper:]' '[:lower:]')
+  package_name="$2"
+
+  for owner in $(get_owners ${package_name})
+  do
+    # NB: A case-insensitive comparison is done since pypi is case-insensitive wrt usernames.
+    owner=$(echo "${owner}" | tr '[:upper:]' '[:lower:]')
+    if [[ "${username}" == "${owner}" ]]
+    then
+      return 0
+    fi
+  done
+  return 1
 }
 
 function check_owners() {
@@ -542,6 +541,7 @@ function reversion_whls() {
 readonly BINARY_BASE_URL=https://binaries.pantsbuild.org
 
 function list_prebuilt_wheels() {
+  # List prebuilt wheels as tab-separated tuples of filename and URL-encoded name.
   wheel_listing="$(mktemp -t pants.wheels.XXXXX)"
   trap "rm -f ${wheel_listing}" RETURN
 
@@ -550,11 +550,14 @@ function list_prebuilt_wheels() {
     "${PY}" << EOF
 from __future__ import print_function
 import sys
+import urllib
 import xml.etree.ElementTree as ET
 root = ET.parse("${wheel_listing}")
 ns = {'s3': 'http://s3.amazonaws.com/doc/2006-03-01/'}
 for key in root.findall('s3:Contents/s3:Key', ns):
-  print(key.text)
+  # Because filenames may contain characters that have different meanings
+  # in URLs (namely '+'), # print the key both as url-encoded and as a file path.
+  print('{}\t{}'.format(key.text, urllib.quote_plus(key.text)))
 EOF
  done
 }
@@ -566,12 +569,15 @@ function fetch_prebuilt_wheels() {
   (
     cd "${to_dir}"
     list_prebuilt_wheels | {
-      while read path
+      while read path_tuple
       do
-        echo "${BINARY_BASE_URL}/${path}:"
-        local dest="${to_dir}/${path}"
+        local file_path=$(echo "$path_tuple" | awk -F'\t' '{print $1}')
+        local url_path=$(echo "$path_tuple" | awk -F'\t' '{print $2}')
+        echo "${BINARY_BASE_URL}/${url_path}:"
+        local dest="${to_dir}/${file_path}"
         mkdir -p "$(dirname "${dest}")"
-        curl --progress-bar -o "${dest}" "${BINARY_BASE_URL}/${path}"
+        curl --fail --progress-bar -o "${dest}" "${BINARY_BASE_URL}/${url_path}" \
+          || die "Could not fetch ${dest}."
       done
     }
   )
@@ -593,8 +599,26 @@ function fetch_and_check_prebuilt_wheels() {
   for PACKAGE in "${RELEASE_PACKAGES[@]}"
   do
     NAME=$(pkg_name $PACKAGE)
-    packages=($(find_pkg "${NAME}" "${check_dir}"))
-    (( ${#packages[@]} > 0 )) || missing+=("${NAME}")
+    packages=($(find_pkg "${NAME}" "${PANTS_UNSTABLE_VERSION}" "${check_dir}"))
+    if [ ${#packages[@]} -eq 0 ]; then
+      missing+=("${NAME}")
+      continue
+    fi
+
+    # Confirm that if the package is not cross platform that we have whls for two platforms.
+    local cross_platform=""
+    for package in "${packages[@]}"
+    do
+      if [[ "${package}" =~ "-none-any.whl" ]]
+      then
+        cross_platform="true"
+      fi
+    done
+
+    if [ "${cross_platform}" != "true" ] && [ ${#packages[@]} -ne 2 ]; then
+      missing+=("${NAME} (expected whls for each platform: had only ${packages[@]})")
+      continue
+    fi
 
     # Here we re-name the linux platform specific wheels we build to masquerade as manylinux1
     # compatible wheels. We take care to support this when we generate the wheels and pypi will
@@ -678,10 +702,6 @@ function usage() {
   echo " -e  Check that wheels are prebuilt for this release."
   echo
   echo "All options (except for '-d' and '-c') are mutually exclusive."
-  echo
-  echo "There is one environment variable that significantly affects behaviour: setting"
-  echo "SUFFIXED_VERSION to a non-empty value will cause all commands to operate on a"
-  echo "git HEAD-SHA suffixed version of pants."
 
   if (( $# > 0 )); then
     die "$@"

src/docs/release.md

@@ -72,9 +72,6 @@ script fail:
         username: <fill me in>
         password: <fill me in>
         EOF
-        
-  - The release script requires Bash 4.  If you're on MacOS you may have to run `brew install bash`,
-    as the Bash that ships with MacOS is ancient.
 
   - Note that the release script expects your pantsbuild/pants git remote to be named `origin`.
     If you have another name for it, you should `git remote rename othername origin` before running
@@ -163,7 +160,8 @@ is not required.
 3. Publish to PyPi
 ------------------
 
-Now that we've smoke-tested this release, we can publish to PyPi:
+Once the first two travis shards (the "binary builder" shards) have completed for your
+release commit, you can publish to PyPi:
 
     :::bash
     $ ./build-support/bin/release.sh