Rollup merge of #63050 - pietroalbini:vendor-awscli, r=Mark-Simulacrum

ci: download awscli from our mirror

This fixes multiple network issues we had when downloading awscli from PyPI on Azure Pipelines by vendoring awscli itself and its dependencies in our S3 bucket. Instructions on how to update the cache are present at the top of `src/ci/install-awscli.sh`.

r? @alexcrichton or @Mark-Simulacrum
fixes #62967

.azure-pipelines/steps/run.yml

@@ -138,43 +138,11 @@ steps:
 
 # Ensure the `aws` CLI is installed so we can deploy later on, cache docker
 # images, etc.
-- bash: |
-    set -e
-    # Temporary code to debug #62967.
-    debug_failed_connections() {
-        echo "trying to ping pypi.org"
-        ping pypi.org -c10 || true
-        echo "trying to ping google.com"
-        ping google.com -c10 || true
-        echo "trying to ping 8.8.8.8"
-        ping 8.8.8.8 -c10 || true
-        echo "trying to download pypi.org"
-        curl https://pypi.org || true
-        echo "trying to download from our S3 bucket"
-        curl https://rust-lang-ci2.s3.amazonaws.com || true
-        echo "trying to dig pypi.org"
-        dig pypi.org || true
-        echo "trying to dig files.pythonhosted.org"
-        dig files.pythonhosted.org || true
-        echo "trying to connect to pypi.org with openssl"
-        echo | openssl s_client -connect pypi.org:443 || true
-        echo "trying to connect to files.pythonhosted.org with openssl"
-        echo | openssl s_client -connect files.pythonhosted.org:443 || true
-    }
-    debug_failed_connections_and_fail() {
-        debug_failed_connections
-        return 1
-    }
-    source src/ci/shared.sh
-    sudo apt-get install -y python3-setuptools
-    debug_failed_connections
-    retry pip3 install -r src/ci/awscli-requirements.txt --upgrade --user || debug_failed_connections_and_fail
-    echo "##vso[task.prependpath]$HOME/.local/bin"
-  displayName: Install awscli (Linux)
-  condition: and(succeeded(), not(variables.SKIP_JOB), eq(variables['Agent.OS'], 'Linux'))
-- script: pip install -r src/ci/awscli-requirements.txt
-  displayName: Install awscli (non-Linux)
-  condition: and(succeeded(), not(variables.SKIP_JOB), ne(variables['Agent.OS'], 'Linux'))
+- bash: src/ci/install-awscli.sh
+  env:
+    AGENT_OS: $(Agent.OS)
+  condition: and(succeeded(), not(variables.SKIP_JOB))
+  displayName: Install awscli
 
 # Configure our CI_JOB_NAME variable which log analyzers can use for the main
 # step to see what's going on.

src/ci/install-awscli.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# This script downloads and installs awscli from the packages mirrored in our
+# own S3 bucket. This follows the recommendations at:
+#
+#    https://packaging.python.org/guides/index-mirrors-and-caches/#caching-with-pip
+#
+# To create a new mirrored copy you can run the command:
+#
+#    pip wheel awscli
+#
+# Before compressing please make sure all the wheels end with `-none-any.whl`.
+# If that's not the case you'll need to remove the non-cross-platform ones and
+# replace them with the .tar.gz downloaded from https://pypi.org. Also make
+# sure it's possible to call this script with both Python 2 and Python 3.
+
+set -euo pipefail
+IFS=$'\n\t'
+
+MIRROR="https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2019-07-27-awscli.tar"
+DEPS_DIR="/tmp/awscli-deps"
+
+pip="pip"
+pipflags=""
+if [[ "${AGENT_OS}" == "Linux" ]]; then
+    pip="pip3"
+    pipflags="--user"
+
+    sudo apt-get install -y python3-setuptools
+    echo "##vso[task.prependpath]$HOME/.local/bin"
+fi
+
+mkdir -p "${DEPS_DIR}"
+curl "${MIRROR}" | tar xf - -C "${DEPS_DIR}"
+"${pip}" install ${pipflags} --no-index "--find-links=${DEPS_DIR}" awscli
+rm -rf "${DEPS_DIR}"