Pull release components on tag (#1693) (#1832)

Pulls release versions of ova components during release
tags or a push to a releases branch. Uses gcloud credentials
to pull in latest tagged version of vic-machine-server.
Fixes #397.

(cherry picked from commit 914d606)

.drone.yml

@@ -79,7 +79,7 @@ pipeline:
 
   unified-ova-build:
     group: build
-    image: 'gcr.io/eminent-nation-87317/vic-product-build:2ea9bdfd'
+    image: 'gcr.io/eminent-nation-87317/vic-product-build:33e3b968'
     pull: true
     privileged: true
     environment:
@@ -91,6 +91,7 @@ pipeline:
       - harbor
       - vic_machine_server
       - vicengine
+      - gs_token_key
     volumes:
       - '/dev:/dev'
       - '/var/run/docker.sock:/var/run/docker.sock'

installer/build/build-ova.sh

@@ -82,7 +82,7 @@ url=$(gsutil ls -l "gs://vic-engine-builds" | grep -v TOTAL | grep vic_ | sort -
 setenv VICENGINE "$url"
 
 #set Harbor
-url=$(gsutil ls -l "gs://harbor-builds" | grep -v TOTAL | grep offline-installer | sort -k2 -r | (trap '' PIPE; head -n1) | xargs | cut -d ' ' -f 3 | sed 's/gs:\/\//https:\/\/storage.googleapis.com\//')
+url=$(gsutil ls -l "gs://harbor-builds" | grep -v TOTAL | grep offline-installer | grep -v offline-installer-latest | sort -k2 -r | (trap '' PIPE; head -n1) | xargs | cut -d ' ' -f 3 | sed 's/gs:\/\//https:\/\/storage.googleapis.com\//')
 setenv HARBOR "$url"
 
 export BUILD_DCHPHOTON_VERSION="1.13"

installer/build/container/Dockerfile

@@ -1,12 +1,14 @@
 FROM vmware/photon
 
 ENV GOVERSION=1.9.2
-ENV PATH=$PATH:/root/gsutil:/usr/local/go/bin
+ENV PATH=$PATH:/root/gsutil:/usr/local/go/bin:/usr/local/google-cloud-sdk/bin/
 
 RUN set -eux; \
     tdnf install -y make tar gzip python2 python-pip sed git \
     gawk docker gptfdisk e2fsprogs grub2 parted xz docker; \
-    curl -L'#' -k https://storage.googleapis.com/pub/gsutil.tar.gz | tar xzf - -C $HOME;  \
+    curl -L'#' -k https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-200.0.0-linux-x86_64.tar.gz  | tar xzf - -C /usr/local; \
+    mkdir -p /root/.gsutil/; \
+    /usr/local/google-cloud-sdk/install.sh --quiet; \
     curl -L'#' -k https://storage.googleapis.com/golang/go$GOVERSION.linux-amd64.tar.gz | tar xzf - -C /usr/local; \
     curl -o /usr/bin/jq -L'#' -k https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 && chmod +x /usr/bin/jq;
 

installer/scripts/ci-build.sh

@@ -20,35 +20,53 @@ cd installer
 INSTALLER_DIR=$(pwd)
 OPTIONS=""
 
-echo "BRANCH = $DRONE_BRANCH"
+echo -e "BRANCH = $DRONE_BRANCH\nEVENT = $DRONE_BUILD_EVENT\nTAG = $DRONE_TAG\n"
 
-# set options
+# set options based on drone args, if present
 if [ -n "${ADMIRAL}" ]; then
   OPTIONS="--admiral $ADMIRAL"
-elif [[ "$DRONE_BRANCH" == *"releases/"* ]]; then
-  admiral_release=$(curl -s https://hub.docker.com/v2/repositories/vmware/admiral/tags/\?page\=1\&page_size\=250 | jq '.results[] | .name'| cut -d "\"" -f2 | grep '^vic_v' | head -n 1 | cut -d'_' -f2)
-  OPTIONS="--admiral $admiral_release"
 fi
-
 if [ -n "${HARBOR}" ]; then
   OPTIONS="$OPTIONS --harbor $HARBOR"
-elif [[ "$DRONE_BRANCH" == *"releases/"* ]]; then
-  harbor_release=$(gsutil ls -l "gs://harbor-releases" | grep -v TOTAL | grep offline-installer | sort -k2 -r | (trap '' PIPE; head -n1) | xargs | cut -d ' ' -f 3 | sed 's/gs:\/\//https:\/\/storage.googleapis.com\//')
-  OPTIONS="$OPTIONS --harbor $harbor_release"
 fi
-
 if [ -n "${VICENGINE}" ]; then
   OPTIONS="$OPTIONS --vicengine $VICENGINE"
-elif [[ "$DRONE_BRANCH" == *"releases/"* ]]; then
-  vicengine_release=$(gsutil ls -l "gs://vic-engine-releases" | grep -v TOTAL | grep vic_ | sort -k2 -r | (trap '' PIPE; head -1) | xargs | cut -d ' ' -f 3 | sed 's/gs:\/\//https:\/\/storage.googleapis.com\//')
-  OPTIONS="$OPTIONS --vicengine $vicengine_release"
 fi
-
 if [ -n "${VIC_MACHINE_SERVER}" ]; then
   OPTIONS="$OPTIONS --vicmachineserver $VIC_MACHINE_SERVER"
-elif [[ "$DRONE_BRANCH" == *"releases/"* ]]; then
-  vicmachineserver_release="latest"
-  OPTIONS="$OPTIONS --vicmachineserver $vicmachineserver_release"
+fi
+
+# set release options if drone args not present
+if [[ ( "$DRONE_BUILD_EVENT" == "tag" && "$DRONE_TAG" != *"dev"* ) || "$DRONE_BRANCH" == *"releases/"* ]]; then
+  if [ -z "${ADMIRAL}" ]; then
+    admiral_release=$(curl -s https://hub.docker.com/v2/repositories/vmware/admiral/tags/\?page\=1\&page_size\=250 | jq '.results[] | .name'| cut -d "\"" -f2 | grep '^vic_v' | head -n 1 | cut -d'_' -f2)
+    OPTIONS="--admiral $admiral_release"
+  fi
+  if [ -z "${HARBOR}" ]; then
+    harbor_release=$(gsutil ls -l "gs://harbor-releases" | grep -v TOTAL | grep offline-installer | sort -k2 -r | (trap '' PIPE; head -n1) | xargs | cut -d ' ' -f 3 | sed 's/gs:\/\//https:\/\/storage.googleapis.com\//')
+    OPTIONS="$OPTIONS --harbor $harbor_release"
+  fi
+  if [ -z "${VICENGINE}" ]; then
+    vicengine_release=$(gsutil ls -l "gs://vic-engine-releases" | grep -v TOTAL | grep vic_ | sort -k2 -r | (trap '' PIPE; head -1) | xargs | cut -d ' ' -f 3 | sed 's/gs:\/\//https:\/\/storage.googleapis.com\//')
+    OPTIONS="$OPTIONS --vicengine $vicengine_release"
+  fi
+  if [ -z "${VIC_MACHINE_SERVER}" ]; then
+    # Listing container tags requires permissions
+    if [ -z "$(gcloud auth list --filter=status:ACTIVE --format='value(account)')" ]; then
+      if [ -z "${GS_TOKEN_KEY}" ]; then
+        echo "No google service account key found..."
+        exit 1
+      fi
+      echo "Attempting to login with google account service key"
+      KEY_FILE=".tmp.token"
+      echo "${GS_TOKEN_KEY}" > ${KEY_FILE}
+      gcloud auth activate-service-account --key-file ${KEY_FILE} || (echo "Login with service account key failed..." && exit 1)
+      rm -f ${KEY_FILE}
+    fi
+
+    vicmachineserver_release="$(gcloud container images list-tags gcr.io/eminent-nation-87317/vic-machine-server --filter='tags~.' | grep -v DIGEST | awk '{print $2}' | sed -rn 's/^(.*,)?(v([0-9]+\.){2}[0-9]+(-rc[0-9]+)?)(,.*)?$/\2/p' | head -n 1)"
+    OPTIONS="$OPTIONS --vicmachineserver $vicmachineserver_release"
+  fi
 fi
 
 echo "OPTIONS = $OPTIONS"