Speed up downloads of hadoop distros

.travis.yml

@@ -25,6 +25,7 @@ matrix:
 cache:
   directories:
     - $HOME/.wheelhouse/
+    - $HOME/.cache/
 install:
   - pip install --upgrade pip
   - pip install tox

scripts/ci/setup_env.sh

@@ -50,7 +50,8 @@ if $ONLY_DOWNLOAD && $ONLY_EXTRACT; then
     exit 1
 fi
 
-mkdir -p $HADOOP_HOME
+mkdir -p ${HADOOP_HOME}
+mkdir -p ${CACHE}/{HADOOP_DISTRO}
 
 if [ $HADOOP_DISTRO = "cdh" ]; then
     URL="http://archive.cloudera.com/cdh5/cdh/5/hadoop-latest.tar.gz"
@@ -63,7 +64,7 @@ fi
 
 if ! $ONLY_EXTRACT; then
     echo "Downloading Hadoop from $URL to ${HADOOP_HOME}/hadoop.tar.gz"
-    curl -z ${HADOOP_HOME}/hadoop.tar.gz -o ${HADOOP_HOME}/hadoop.tar.gz -L $URL
+    curl -z ${CACHE}/${HADOOP_DISTRO}/hadoop.tar.gz -o ${CACHE}/${HADOOP_DISTRO}/hadoop.tar.gz -L $URL
 
     if [ $? != 0 ]; then
         echo "Failed to download Hadoop from $URL - abort" >&2
@@ -76,7 +77,7 @@ if $ONLY_DOWNLOAD; then
 fi
 
 echo "Extracting ${HADOOP_HOME}/hadoop.tar.gz into $HADOOP_HOME"
-tar zxf ${HADOOP_HOME}/hadoop.tar.gz --strip-components 1 -C $HADOOP_HOME
+tar zxf ${CACHE}/${HADOOP_DISTRO}/hadoop.tar.gz --strip-components 1 -C $HADOOP_HOME
 
 if [ $? != 0 ]; then
     echo "Failed to extract Hadoop from ${HADOOP_HOME}/hadoop.tar.gz to ${HADOOP_HOME} - abort" >&2

tox.ini

@@ -17,6 +17,7 @@ basepython =
   py34: python3.4
 setenv =
   COVERALLS_REPO_TOKEN=ic8IH7CrUrtweVbmY3VZQ7ncEGe1XJA5E
+  CACHE=$HOME/.cache/
   cdh: HADOOP_DISTRO=cdh
   cdh: HADOOP_HOME=/tmp/hadoop-cdh
   cdh: KRB5_CONFIG=/tmp/minikdc/work