From 45a6bd6269c465d4469daf44d77b9dbc3727ac2f Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Wed, 22 Apr 2020 15:14:42 +0000 Subject: [PATCH] Adds image automated build In order for the secret store CSI image to be automatically built and published to the staging registry (from which it will be promoted), the cloudbuild.yaml file has been added. The file was added in conformance with [1]. Adds the docker folder, which contains several items: - cloudbuild.yaml - BASEIMAGE: the base image to use when building an image for a certain os/arch (or os/arch/version for Windows). - BASEIMAGE_CORE: for Windows images, from which image to copy necessary files (DLLs). - Makefile - build.sh: script that can build, push, and create the manifest list. - Dockerfile: needed to build the Linux docker image. Can now accept the a BASEIMAGE arg, which can be useful when building multi-arch images. - windows.Dockerfile: needed to build the Windows docker image. Can now accept BASEIMAGE and BASEIMAGE_CORE args, which can be useful when building images for multiple Windows versions. The image building process will be triggered when changes to the files in the docker changes are made (for example, you bump the image version, so a new image is built). [1] https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md --- .gitignore | 1 + docker/BASEIMAGE | 4 + docker/BASEIMAGE_CORE | 3 + Dockerfile => docker/Dockerfile | 2 +- docker/Makefile | 41 +++++ docker/build.sh | 150 ++++++++++++++++++ docker/cloudbuild.yaml | 30 ++++ .../windows.Dockerfile | 2 +- 8 files changed, 231 insertions(+), 2 deletions(-) create mode 100644 docker/BASEIMAGE create mode 100644 docker/BASEIMAGE_CORE rename Dockerfile => docker/Dockerfile (78%) create mode 100644 docker/Makefile create mode 100755 docker/build.sh create mode 100644 docker/cloudbuild.yaml rename windows.Dockerfile => docker/windows.Dockerfile (83%) diff --git a/.gitignore b/.gitignore index 1d1f78520..776721bf0 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ cmd/secrets-store-csi-driver/secrets-store-csi-driver /output*/ /_output*/ /_output +/docker/_output # Emacs save files *~ diff --git a/docker/BASEIMAGE b/docker/BASEIMAGE new file mode 100644 index 000000000..0f7a131db --- /dev/null +++ b/docker/BASEIMAGE @@ -0,0 +1,4 @@ +linux/amd64=debian:9 +windows/amd64/1809=mcr.microsoft.com/windows/nanoserver:1809 +windows/amd64/1903=mcr.microsoft.com/windows/nanoserver:1903 +windows/amd64/1909=mcr.microsoft.com/windows/nanoserver:1909 diff --git a/docker/BASEIMAGE_CORE b/docker/BASEIMAGE_CORE new file mode 100644 index 000000000..e6b992bd4 --- /dev/null +++ b/docker/BASEIMAGE_CORE @@ -0,0 +1,3 @@ +windows/amd64/1809=mcr.microsoft.com/windows/servercore:1809 +windows/amd64/1903=mcr.microsoft.com/windows/servercore:1903 +windows/amd64/1909=mcr.microsoft.com/windows/servercore:1909 diff --git a/Dockerfile b/docker/Dockerfile similarity index 78% rename from Dockerfile rename to docker/Dockerfile index 9600ace75..1193e4c5b 100644 --- a/Dockerfile +++ b/docker/Dockerfile @@ -3,5 +3,5 @@ RUN apt-get update && apt-get install -y ca-certificates cifs-utils LABEL maintainers="ritazh" LABEL description="Secrets Store CSI Driver" -COPY ./_output/secrets-store-csi /secrets-store-csi +COPY _output/secrets-store-csi /secrets-store-csi ENTRYPOINT ["/secrets-store-csi"] diff --git a/docker/Makefile b/docker/Makefile new file mode 100644 index 000000000..62871f0f6 --- /dev/null +++ b/docker/Makefile @@ -0,0 +1,41 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +REGISTRY?=docker.io/deislabs +IMAGE_NAME=secrets-store-csi +IMAGE_VERSION?=v0.0.9 +IMAGE_TAG=$(REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION) +export + +LDFLAGS?='-X sigs.k8s.io/secrets-store-csi-driver/pkg/secrets-store.vendorVersion=$(IMAGE_VERSION) -extldflags "-static"' + +GO111MODULE ?= on +export GO111MODULE +DOCKER_CLI_EXPERIMENTAL = enabled +export GOPATH GOBIN GO111MODULE DOCKER_CLI_EXPERIMENTAL + +clean: + -rm -rf _output + +build-binaries: clean + CGO_ENABLED=0 GOOS=linux go build -a -ldflags ${LDFLAGS} -o _output/secrets-store-csi ../cmd/secrets-store-csi-driver + CGO_ENABLED=0 GOOS=windows go build -a -ldflags ${LDFLAGS} -o _output/secrets-store-csi.exe ../cmd/secrets-store-csi-driver + +build: build-binaries + bash -x ./build.sh build + +push: build + bash -x ./build.sh push + +.PHONY: clean build-binaries build push diff --git a/docker/build.sh b/docker/build.sh new file mode 100755 index 000000000..e6cc73090 --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +TASK=$1 + +pushd `dirname "$0"` +#trap 'popd' ERR + + +# Connecting to a Remote Docker requires certificates for authentication, which can be found +# at this path. By default, they can be found in the ${HOME} folder. We're expecting to find +# here ".docker-${os_version}" folders which contains the necessary certificates. +DOCKER_CERT_BASE_PATH="${DOCKER_CERT_BASE_PATH:-${HOME}}" + +# Returns list of all supported architectures from the BASEIMAGE file +listOsArchs() { + cut -d "=" -f 1 BASEIMAGE +} + +splitOsArch() { + os_arch=$1 + + if [[ $os_arch =~ .*/.*/.* ]]; then + # for Windows, we have to support both LTS and SAC channels, so we're building multiple Windows images. + # the format for this case is: OS/ARCH/OS_VERSION. + os_name=$(echo "$os_arch" | cut -d "/" -f 1) + arch=$(echo "$os_arch" | cut -d "/" -f 2) + os_version=$(echo "$os_arch" | cut -d "/" -f 3) + suffix="$os_name-$arch-$os_version" + + # currently, GCE does not have Hyper-V support, which means that the same node cannot be used to build + # multiple versions of Windows images. Which is why we have $REMOTE_DOCKER_URL_$os_version URLs configured. + # TODO(claudiub): once Hyper-V support has been added to GCE, revert this to just $REMOTE_DOCKER_URL. + remote_docker_url_name="REMOTE_DOCKER_URL_$os_version" + REMOTE_DOCKER_URL=$(eval echo "\${${remote_docker_url_name}:-}") + elif [[ $os_arch =~ .*/.* ]]; then + os_name=$(echo "$os_arch" | cut -d "/" -f 1) + arch=$(echo "$os_arch" | cut -d "/" -f 2) + suffix="$os_name-$arch" + else + echo "The BASEIMAGE file is not properly formatted. Expected entries to start with 'os/arch', found '${os_arch}' instead." + exit 1 + fi +} + +# Returns baseimage need to used in Dockerfile for any given architecture +getBaseImage() { + os_arch=$1 + file=${2:-BASEIMAGE} + grep "${os_arch}=" "${file}" | cut -d= -f2 +} + + +# This function will build test image for all the architectures +# mentioned in BASEIMAGE file. +build() { + os_archs=$(listOsArchs) + for os_arch in ${os_archs}; do + splitOsArch "${os_arch}" + if [[ "${os_name}" == "windows" && -z "${REMOTE_DOCKER_URL}" ]]; then + # If we have a Windows os_arch entry but no Remote Docker Daemon for it, + # we should skip it. + echo "Cannot build the image for ${os_arch}. REMOTE_DOCKER_URL_$os_version should be set, containing the URL to a Windows docker daemon." + continue + fi + + echo "Building image for OS/ARCH: ${os_arch}..." + + BASEIMAGE=$(getBaseImage "${os_arch}") + + if [[ "$os_name" = "linux" ]]; then + docker build --pull -t "${IMAGE_TAG}-${os_name}-${arch}" --build-arg BASEIMAGE="${BASEIMAGE}" . + elif [[ -n "${REMOTE_DOCKER_URL:-}" ]]; then + # NOTE(claudiub): We're using a remote Windows node to build the Windows Docker images. + # The node requires TLS authentication, and thus it is expected that the + # ca.pem, cert.pem, key.pem files can be found in the ${DOCKER_CERT_BASE_PATH}/.docker-${os_version} folder. + BASEIMAGE_CORE=$(getBaseImage "${os_arch}" "BASEIMAGE_CORE") + docker --tlsverify --tlscacert "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/ca.pem" \ + --tlscert "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/cert.pem" --tlskey "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/key.pem" \ + -H "${REMOTE_DOCKER_URL}" build --pull -t "${IMAGE_TAG}-${os_name}-${arch}-${os_version}" \ + --build-arg BASEIMAGE="${BASEIMAGE}" --build-arg BASEIMAGE_CORE="${BASEIMAGE_CORE}" -f windows.Dockerfile . + fi + done + popd +} + +docker_version_check() { + # The reason for this version check is even though "docker manifest" command is available in 18.03, it does + # not work properly in that version. So we insist on 18.06.0 or higher. + docker_version=$(docker version --format '{{.Client.Version}}' | cut -d"-" -f1) + if [[ ${docker_version} != 18.06.0 && ${docker_version} < 18.06.0 ]]; then + echo "Minimum docker version 18.06.0 is required for creating and pushing manifest images[found: ${docker_version}]" + exit 1 + fi +} + +# This function will push the docker images +push() { + docker_version_check + + os_archs=$(listOsArchs) + for os_arch in ${os_archs}; do + splitOsArch "${image}" "${os_arch}" + + if [[ "$os_name" = "linux" ]]; then + docker push "${IMAGE_TAG}-${os_name}-${arch}" + elif [[ -n "${REMOTE_DOCKER_URL:-}" ]]; then + # NOTE(claudiub): We're pushing the image we built on the remote Windows node. + docker --tlsverify --tlscacert "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/ca.pem" \ + --tlscert "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/cert.pem" --tlskey "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/key.pem" \ + -H "${REMOTE_DOCKER_URL}" push "${IMAGE_TAG}-${os_name}-${arch}-${os_version}" + else + echo "Cannot push the image for ${os_arch}. REMOTE_DOCKER_URL_${os_version} should be set, containing the URL to a Windows docker daemon." + # we should exclude this image from the manifest list as well, we couldn't build / push it. + os_archs=$(printf "%s\n" "$os_archs" | grep -v "$os_arch" || true) + fi + done + + # The manifest command is still experimental as of Docker 18.09.2 + export DOCKER_CLI_EXPERIMENTAL="enabled" + # Make os_archs list into image manifest. Eg: 'linux/amd64 windows/amd64/1809' to + # '${REGISTRY}/${IMAGE_NAME}:${IMAGE_VERSION}-linux-amd64 ${REGISTRY}/${IMAGE_NAME}:${IMAGE_VERSION}-windows-amd64-1809' + while IFS='' read -r line; do manifest+=("$line"); done < <(echo "$os_archs" | ${SED} "s~\/~-~g" | ${SED} -e "s~[^ ]*~$REGISTRY\/$IMAGE_NAME:$IMAGE_VERSION\-&~g") + docker manifest create --amend "${IMAGE_TAG}" "${manifest[@]}" + for os_arch in ${os_archs}; do + splitOsArch "${image}" "${os_arch}" + docker manifest annotate --os "${os_name}" --arch "${arch}" "${IMAGE_TAG}" "${IMAGE_TAG}-${suffix}" + done + docker manifest push --purge "${IMAGE_TAG}" +} + +shift +eval "${TASK}" diff --git a/docker/cloudbuild.yaml b/docker/cloudbuild.yaml new file mode 100644 index 000000000..baed7534b --- /dev/null +++ b/docker/cloudbuild.yaml @@ -0,0 +1,30 @@ +# See https://cloud.google.com/cloud-build/docs/build-config + +# this must be specified in seconds. If omitted, defaults to 600s (10 mins) +timeout: 900s +# this prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF, +# or any new substitutions added in the future. +options: + substitution_option: ALLOW_LOOSE +steps: + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20190906-745fed4' + entrypoint: make + dir: ./docker + env: + - DOCKER_CLI_EXPERIMENTAL=enabled + - TAG=$_GIT_TAG + - BASE_REF=$_PULL_BASE_REF + - REGISTRY=gcr.io/k8s-staging-csi-secrets-store + - DOCKER_CERT_BASE_PATH=/root + - REMOTE_DOCKER_URL_1809=tcp://img-promoter-1809.eastus.cloudapp.azure.com:2376 + - REMOTE_DOCKER_URL_1903=tcp://img-promoter-1903.eastus.cloudapp.azure.com:2376 + - REMOTE_DOCKER_URL_1909=tcp://img-promoter-1909.eastus.cloudapp.azure.com:2376 + args: + - manifest +substitutions: + # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and + # can be used as a substitution + _GIT_TAG: '12345' + # _PULL_BASE_REF will contain the ref that was pushed to to trigger this build - + # a branch like 'master' or 'release-0.2', or a tag like 'v0.2'. + _PULL_BASE_REF: 'master' diff --git a/windows.Dockerfile b/docker/windows.Dockerfile similarity index 83% rename from windows.Dockerfile rename to docker/windows.Dockerfile index b89ef2a76..43ef7e8f9 100644 --- a/windows.Dockerfile +++ b/docker/windows.Dockerfile @@ -3,7 +3,7 @@ FROM mcr.microsoft.com/windows/servercore:1809 as core FROM mcr.microsoft.com/windows/nanoserver:1809 LABEL description="Secrets Store CSI Driver" -COPY ./_output/secrets-store-csi.exe /secrets-store-csi.exe +COPY _output/secrets-store-csi.exe /secrets-store-csi.exe COPY --from=core /Windows/System32/netapi32.dll /Windows/System32/netapi32.dll USER ContainerAdministrator ENTRYPOINT ["/secrets-store-csi.exe"]