Cherry-pick a variety of changes into 1.4.1

.drone.yml

@@ -79,7 +79,7 @@ pipeline:
 
   unified-ova-build:
     group: build
-    image: 'gcr.io/eminent-nation-87317/vic-product-build:2ea9bdfd'
+    image: 'gcr.io/eminent-nation-87317/vic-product-build:33e3b968'
     pull: true
     privileged: true
     environment:
@@ -91,6 +91,7 @@ pipeline:
       - harbor
       - vic_machine_server
       - vicengine
+      - gs_token_key
     volumes:
       - '/dev:/dev'
       - '/var/run/docker.sock:/var/run/docker.sock'

installer/build/bootable/build-disks.sh

@@ -177,8 +177,8 @@ fi
 
 # These sizes are minimal for install, since partitions are resized to full disk space after firstboot.
 IMAGESIZES=(
-  "4GiB"
-  "1GiB"
+  "6GiB"
+  "2GiB"
 )
 IMAGES=(
   "vic-disk1"
@@ -207,7 +207,8 @@ elif [ "${ACTION}" == "export" ]; then
   log1 "export images to VMDKs"
   for i in "${!IMAGES[@]}"; do
     log2 "exporting ${IMAGES[$i]}.img to ${IMAGES[$i]}.vmdk"
-    DEV=$(losetup -l -O NAME,BACK-FILE -a | tail -n +2 | grep "${IMAGES[$i]}" | awk '{print $1}')
+    echo "export ${PACKAGE}/${IMAGES[$i]}"
+    DEV=$(losetup -l -O NAME,BACK-FILE -a | tail -n +2 | grep "${PACKAGE}/${IMAGES[$i]}" | awk '{print $1}')
     convert "${DEV}" "${IMAGEROOTS[$i]}" "${IMAGES[$i]}.img" "${IMAGES[$i]}.vmdk"
   done
 

installer/build/bootable/build-main.sh

@@ -137,7 +137,7 @@ function main {
     cd "${PACKAGE}"
     sed -i -e s~--version--~${BUILD_OVA_REVISION}~ vic-${BUILD_OVA_REVISION}.ovf
     log2 "rebuilding OVF manifest"
-    sha256sum --tag "vic-${BUILD_OVA_REVISION}.ovf" "vic-${BUILD_OVA_REVISION}.mf" *.vmdk | sed s/SHA256\ \(/SHA256\(/ > "vic-${BUILD_OVA_REVISION}.mf"
+    sha256sum --tag "vic-${BUILD_OVA_REVISION}.ovf" *.vmdk | sed s/SHA256\ \(/SHA256\(/ > "vic-${BUILD_OVA_REVISION}.mf"
     tar -cvf "${RESOURCE}/vic-${BUILD_OVA_REVISION}.ova" "vic-${BUILD_OVA_REVISION}.ovf" "vic-${BUILD_OVA_REVISION}.mf" *.vmdk
 
     OUTFILE=${RESOURCE}/vic-${BUILD_OVA_REVISION}.ova

installer/build/bootable/config/builder.ovf

@@ -246,7 +246,7 @@ EVALUATION LICENSE. If You are licensing the Software for evaluation purposes, Y
     <ProductSection ovf:class="registry" ovf:required="false">
       <Info>Registry Properties</Info>
       <Category>3. Registry Configuration</Category>
-      <Property ovf:key="port" ovf:qualifiers="MinValue(1),MaxValue(65535)" ovf:type="int" ovf:userConfigurable="true" ovf:value="443">
+      <Property ovf:key="registry_port" ovf:qualifiers="MinValue(1),MaxValue(65535)" ovf:type="int" ovf:userConfigurable="true" ovf:value="443">
         <Label>3.1. Registry Port</Label>
         <Description>Specifies the port on which registry will be published.</Description>
       </Property>
@@ -262,7 +262,7 @@ EVALUATION LICENSE. If You are licensing the Software for evaluation purposes, Y
     <ProductSection ovf:class="management_portal" ovf:required="false">
       <Info>Management Portal Properties</Info>
       <Category>4. Management Portal Configuration</Category>
-      <Property ovf:key="port" ovf:qualifiers="MinValue(1),MaxValue(65535)" ovf:type="int" ovf:userConfigurable="true" ovf:value="8282">
+      <Property ovf:key="management_portal_port" ovf:qualifiers="MinValue(1),MaxValue(65535)" ovf:type="int" ovf:userConfigurable="true" ovf:value="8282">
         <Label>4.1. Management Portal Port</Label>
         <Description>Specifies the port on which Management Portal will be published.</Description>
       </Property>

installer/build/build-ova.sh

@@ -82,7 +82,7 @@ url=$(gsutil ls -l "gs://vic-engine-builds" | grep -v TOTAL | grep vic_ | sort -
 setenv VICENGINE "$url"
 
 #set Harbor
-url=$(gsutil ls -l "gs://harbor-builds" | grep -v TOTAL | grep offline-installer | sort -k2 -r | (trap '' PIPE; head -n1) | xargs | cut -d ' ' -f 3 | sed 's/gs:\/\//https:\/\/storage.googleapis.com\//')
+url=$(gsutil ls -l "gs://harbor-builds" | grep -v TOTAL | grep offline-installer | grep -v offline-installer-latest | sort -k2 -r | (trap '' PIPE; head -n1) | xargs | cut -d ' ' -f 3 | sed 's/gs:\/\//https:\/\/storage.googleapis.com\//')
 setenv HARBOR "$url"
 
 export BUILD_DCHPHOTON_VERSION="1.13"

installer/build/container/Dockerfile

@@ -1,12 +1,14 @@
 FROM vmware/photon
 
 ENV GOVERSION=1.9.2
-ENV PATH=$PATH:/root/gsutil:/usr/local/go/bin
+ENV PATH=$PATH:/root/gsutil:/usr/local/go/bin:/usr/local/google-cloud-sdk/bin/
 
 RUN set -eux; \
     tdnf install -y make tar gzip python2 python-pip sed git \
     gawk docker gptfdisk e2fsprogs grub2 parted xz docker; \
-    curl -L'#' -k https://storage.googleapis.com/pub/gsutil.tar.gz | tar xzf - -C $HOME;  \
+    curl -L'#' -k https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-200.0.0-linux-x86_64.tar.gz  | tar xzf - -C /usr/local; \
+    mkdir -p /root/.gsutil/; \
+    /usr/local/google-cloud-sdk/install.sh --quiet; \
     curl -L'#' -k https://storage.googleapis.com/golang/go$GOVERSION.linux-amd64.tar.gz | tar xzf - -C /usr/local; \
     curl -o /usr/bin/jq -L'#' -k https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 && chmod +x /usr/bin/jq;
 

installer/build/scripts/harbor/configure_harbor.sh

@@ -115,9 +115,10 @@ configureHarborCfg ssl_cert $appliance_tls_cert
 configureHarborCfg ssl_cert_key $appliance_tls_key
 configureHarborCfg secretkey_path $data_dir
 
-# Set MySQL and Clair DB passwords on first boot
-configureHarborCfgOnce db_password "$(genPass)"
-configureHarborCfgOnce clair_db_password "$(genPass)"
+# Set Harbor DB and Clair DB passwords on first boot
+random_pwd=$(genPass)
+configureHarborCfgOnce db_password "$random_pwd"
+configureHarborCfgOnce clair_db_password "$random_pwd"
 
 setPortInYAML $harbor_compose_file "${REGISTRY_PORT}" "${NOTARY_PORT}"
 

installer/build/scripts/systemd/scripts/vic-appliance-environment.sh

@@ -23,8 +23,8 @@ APPLIANCE_TLS_CERT="$(ovfenv -k appliance.tls_cert | sed -E ':a;N;$!ba;s/\r{0,1}
 APPLIANCE_TLS_PRIVATE_KEY="$(ovfenv -k appliance.tls_cert_key | sed -E ':a;N;$!ba;s/\r{0,1}\n//g')"
 APPLIANCE_TLS_CA_CERT="$(ovfenv -k appliance.ca_cert | sed -E ':a;N;$!ba;s/\r{0,1}\n//g')"
 
-ADMIRAL_PORT="$(ovfenv -k management_portal.port)"
-REGISTRY_PORT="$(ovfenv -k registry.port)"
+ADMIRAL_PORT="$(ovfenv -k management_portal.management_portal_port)"
+REGISTRY_PORT="$(ovfenv -k registry.registry_port)"
 NOTARY_PORT="$(ovfenv -k registry.notary_port)"
 FILESERVER_PORT="$(ovfenv -k appliance.config_port)"
 HOSTNAME=""

installer/build/scripts/upgrade/upgrade.sh

@@ -214,7 +214,7 @@ function proceedWithUpgrade {
   fi
 
   log ""
-  log "Detected old appliance's version $ver as 1.2.0 or older."
+  log "Detected old appliance's version as $ver."
   log "Upgrade from this version is not a supported upgrade path."
   log "If the old appliance's version is not detected correctly, please contact VMware support."
   exit 1

installer/docs/BUILD.md

@@ -112,8 +112,8 @@ docker run -it --net=host -v $GOPATH/src/github.com/vmware/vic-product/installer
   --net:Network="VM Network" \
   --prop:appliance.root_pwd="password" \
   --prop:appliance.permit_root_login=True \
-  --prop:management_portal.port=8282 \
-  --prop:registry.port=443 \
+  --prop:management_portal.management_portal_port=8282 \
+  --prop:registry.registry_port=443 \
   /test-bin/$(ls -1t bin | grep "\.ova") \
   vi://$VC_USER:$VC_PASSWORD@$VC_IP/$VC_COMPUTE
 ```
@@ -155,22 +155,39 @@ Please note that you cannot trigger new CI builds manually, but have to promote
 
 Make sure `DRONE_SERVER` and `DRONE_TOKEN` environment variables are set before executing these commands.
 
-To promote existing successful CI build to staging...
+To promote existing successful CI build to staging (`vic-product-ova-builds` bucket):
 
-``
-$ drone deploy --param VICENGINE=<vic_engine_version> --param VIC_MACHINE_SERVER=<vic_machine_server> --param ADMIRAL=<admiral_tag> --param HARBOR=<harbor_version> vmware/vic-product <ci_build_number_to_promote> staging
-``
+```
+$ drone deploy --param VICENGINE=<vic_engine_version> \
+               --param VIC_MACHINE_SERVER=<vic_machine_server> \
+               --param ADMIRAL=<admiral_tag> \
+               --param HARBOR=<harbor_version> \
+               vmware/vic-product <ci_build_number_to_promote> staging
+```
 
-To promote existing successful CI build to release...
+To promote existing successful CI build to release (`vic-product-ova-releases` bucket):
 
-``
-$ drone deploy --param VICENGINE=<vic_engine_version> --param VIC_MACHINE_SERVER=<vic_machine_server> --param ADMIRAL=<admiral_tag> --param HARBOR=<harbor_version> vmware/vic-product <ci_build_number_to_promote> release
-``
+```
+$ drone deploy --param VICENGINE=<vic_engine_version> \
+               --param VIC_MACHINE_SERVER=<vic_machine_server> \
+               --param ADMIRAL=<admiral_tag> \
+               --param HARBOR=<harbor_version> \
+               vmware/vic-product <ci_build_number_to_promote> release
+```
+
+Example:
+
+```
+$ drone deploy --param VICENGINE=https://storage.googleapis.com/vic-engine-releases/vic_v1.4.0.tar.gz \
+               --param VIC_MACHINE_SERVER=latest \
+               --param ADMIRAL=v1.4.0 \
+               --param HARBOR=https://storage.googleapis.com/harbor-releases/harbor-offline-installer-v1.5.0.tgz \
+               vmware/vic-product <ci_build_number_to_promote> release
+
+```
 
 `vic_engine_version` and `harbor_version` can be specified as a URL or a file in `cwd`, eg. 'https://storage.googleapis.com/vic-engine-releases/vic_1.2.1.tar.gz'
 
 `admiral_tag` and `vic_machine_server` should be specified as docker image revision tag, eg. 'latest'
 
 `ci_build_number_to_promote` is the drone build number which will be promoted
-
-## Troubleshooting

installer/docs/RELEASE.md

@@ -3,12 +3,14 @@
 All examples in this document assume the user's fork is at `origin` and `vmware/vic-product` is at
 `upstream`.
 
+
 ## Branching
 
 When the team is ready to build a release candidate, create a branch off of master based on the
-release version number. A tag for ongoing development should also be created at the commit after
-where the release branch begins. If there have been commits to master since the intended branch
-point, first create a tag for the release candidate, then create the branch from that tag.
+release version number. 
+
+A tag for ongoing development should also be created at the commit **after**
+the start of the release branch. The tagging procedure is documented in [Tagging](#Tagging). 
 
 ```
 git remote update
@@ -22,8 +24,11 @@ git push upstream
 Configure branch protection on Github to have the same protection as the master branch.
 `Protect this branch` and `Require pull request reviews before merging` should be set.
 
-Development should continue on master. Commits that need to be pulled into the release should be
-cherry picked into the release branch after they are merged into master.
+
+## Cherry picking
+
+Commits that need to be pulled into the release should be cherry picked into the release branch
+after they are merged into master.
 
 ```
 git remote update
@@ -38,68 +43,93 @@ git push upstream
 ## Tagging
 
 On the master branch, tag the commit for the first release candidate. On the
-following commit, tag `dev` for ongoing development.  For example, if the
+following commit, tag `dev` for ongoing development. For example, if the
 current release is `v1.2.0`, the first release candidate will be `v1.2.0-rc1` and
 the tag for ongoing development will be `v1.3.0-dev`.
 
-When the team is ready to release, tag the commit in the release branch (`v1.2.0`) and push the tag
-to Github.
-
 ```
 git remote update
 git checkout upstream/releases/1.2.0
 git tag -a v1.2.0-rc1 aaaaaaa
 git push upstream v1.2.0-rc1
 ```
 
-Tag `dev` on the release branch after a release. For example, if `v1.2.0` was tagged on
-`/releases/1.2.0` and there is work for `v1.2.1`, on the following commit, tag `v1.2.1-dev`.
+If there is not yet a commit after the start of the release branch, create an empty commit after
+the commit for the release branch. This empty commit will be tagged for ongoing development on master.
+
+```
+# Create empty commit on master
+git remote update
+git checkout upstream/master
+git commit --allow-empty -m "v1.3.0-dev"
+git push upstream
+
+# Tag empty commit for ongoing development
+git remote update
+git checkout upstream/master
+git tag -a v1.3.0-dev bbbbbbb
+git pubsh upstream v1.3.0-dev
+```
+
+After the release candidate has passed QA and the team is ready to release, tag the commit in the
+release branch (`v1.2.0`) and push the tag to Github.
 
 ```
 git remote update
 git checkout upstream/releases/1.2.0
-git tag -a v1.2.1-dev bbbbbbb
-git push upstream v1.2.1-dev
+git tag -a v1.2.0 ccccccc
+git push upstream v1.2.0
 ```
 
+### Point releases
 
-## Building Release
+After a release, tag `dev` on the release branch for ongoing development.
+For example, if `v1.2.0` was tagged on `/releases/1.2.0` and there is work for `v1.2.1`, on the
+following commit, tag `v1.2.1-dev`.
 
 ```
-git fetch --all --tags --prune
-git checkout tags/v1.2.0
+git remote update
+git checkout upstream/releases/1.2.0
+git tag -a v1.2.1-dev ddddddd
+git push upstream v1.2.1-dev
 ```
 
-Follow instructions in [How to build VIC Product OVA](BUILD.md)
 
 ## Github Releases
 
 After pushing the tag to Github, go to https://github.com/vmware/vic-product/releases/new
 
 Select the appropriate tag
 
-Title for follows form `VIC Product <tag>` (`VIC Product v1.2.0-rc1`)
+Release title follows form `vSphere Integrated Containers Appliance <tag>` (e.g. `vSphere Integrated Containers Appliance v1.4.0-rc3`)
+
+Obtain artifact hashes from CI build output at the end of `unified-ova-build` step
 
-Description template for release candidates:
+Obtain version information from `/etc/vmware/version`
+
+Description template for release candidates and releases:
+
+```````
+### [Download OVA](https://storage.googleapis.com/vic-product-ova-releases/vic-v1.4.0-rc3-4824-d99cbdb4.ova)
+Filesize: 
+SHA256:  
+SHA1: 
+MD5: 
+
+### OVA will contain:
 ```
-OVA will contain:
-Admiral `v1.2.0-rc3`
-Harbor `harbor-offline-installer-v1.2.0-rc4.tgz`
-VIC Engine `vic_1.2.0-rc4.tar.gz`
+appliance=v1.4.0-rc3-4824-d99cbdb4
+harbor=harbor-offline-installer-v1.5.0-rc4.tgz
+engine=vic_v1.4.0-rc2.tar.gz
+admiral=vmware/admiral:vic_v1.4.0-rc4 45a773ffae33
+vic-machine-server=gcr.io/eminent-nation-87317/vic-machine-server:latest b3412e003674
 ```
+### [Changes from v1.3.1](https://github.com/vmware/vic-product/compare/v1.3.1...v1.4.0-rc3)
+
+```````
 
 If the release is a release candidate, mark `This is a pre-release`
 
-## OVA Releases
+## Building Releases
 
-Description template for release version of the OVA:
-```
-Admiral `v1.2.0`
-Harbor `harbor-offline-installer-v1.2.0.tgz`
-VIC Engine `vic_1.2.0.tar.gz`
-
-ef6b71d98bb6650240008b5281e97bf8592d5fd726833883718f471ed665fc5b  vic-v1.2.0-aaaaaaaa.ova
-85eabdf7e58fed8c09e4f3c45b2caa974ae89a16  vic-v1.2.0-aaaaaaaa.ova
-ebc669f7b4cebf7501cf141e3b6fa2e3  vic-v1.2.0-aaaaaaaa.ova
-4741.43 MB
-```
+Follow instructions in [How to build VIC Product OVA](BUILD.md)

installer/fileserver/server.go

@@ -142,7 +142,7 @@ func Init(conf *config) {
 
 	if ip, err := ip.FirstIPv4(ip.Eth0Interface); err == nil {
 		conf.serverHostname = getHostname(ovf, ip)
-		if port, ok := ovf.Properties["management_portal.port"]; ok {
+		if port, ok := ovf.Properties["management_portal.management_portal_port"]; ok {
 			conf.admiralPort = port
 		}
 	}

installer/fileserver/tasks.go

@@ -74,7 +74,7 @@ func registerWithPSC(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	admiralPort := ovf.Properties["management_portal.port"]
+	admiralPort := ovf.Properties["management_portal.management_portal_port"]
 
 	// Out of the box users
 	defCreateUsers, foundCreateUsers := ovf.Properties["default_users.create_def_users"]