diff --git a/build/common.sh b/build/common.sh index be36ab39f6e4..548f05bd7853 100755 --- a/build/common.sh +++ b/build/common.sh @@ -1211,6 +1211,16 @@ function kube::release::gcs::copy_release_artifacts() { fi gsutil ls -lhr "${gcs_destination}" || return 1 + + if [[ -n "${KUBE_GCS_RELEASE_BUCKET_MIRROR:-}" ]]; then + local -r gcs_mirror="gs://${KUBE_GCS_RELEASE_BUCKET_MIRROR}/${KUBE_GCS_RELEASE_PREFIX}" + kube::log::status "Mirroring build to ${gcs_mirror}" + gsutil -q -m "${gcs_options[@]+${gcs_options[@]}}" rsync -d -r "${gcs_destination}" "${gcs_mirror}" || return 1 + if [[ ${KUBE_GCS_MAKE_PUBLIC} =~ ^[yY]$ ]]; then + kube::log::status "Marking all uploaded mirror objects public" + gsutil -q -m acl ch -R -g all:R "${gcs_mirror}" >/dev/null 2>&1 || return 1 + fi + fi } # Publish a new ci version, (latest,) but only if the release files actually @@ -1475,7 +1485,19 @@ function kube::release::gcs::verify_ci_ge() { # If new version is greater than the GCS version function kube::release::gcs::publish() { local -r publish_file="${1-}" - local -r publish_file_dst="gs://${KUBE_GCS_RELEASE_BUCKET}/${publish_file}" + + kube::release::gcs::publish_to_bucket "${KUBE_GCS_RELEASE_BUCKET}" "${publish_file}" || return 1 + + if [[ -n "${KUBE_GCS_RELEASE_BUCKET_MIRROR:-}" ]]; then + kube::release::gcs::publish_to_bucket "${KUBE_GCS_RELEASE_BUCKET_MIRROR}" "${publish_file}" || return 1 + fi +} + + +function kube::release::gcs::publish_to_bucket() { + local -r publish_bucket="${1}" + local -r publish_file="${2}" + local -r publish_file_dst="gs://${publish_bucket}/${publish_file}" mkdir -p "${RELEASE_STAGE}/upload" || return 1 echo "${KUBE_GCS_PUBLISH_VERSION}" > "${RELEASE_STAGE}/upload/latest" || return 1 @@ -1488,7 +1510,7 @@ function kube::release::gcs::publish() { gsutil acl ch -R -g all:R "${publish_file_dst}" >/dev/null 2>&1 || return 1 gsutil setmeta -h "Cache-Control:private, max-age=0" "${publish_file_dst}" >/dev/null 2>&1 || return 1 # If public, validate public link - local -r public_link="https://storage.googleapis.com/${KUBE_GCS_RELEASE_BUCKET}/${publish_file}" + local -r public_link="https://storage.googleapis.com/${publish_bucket}/${publish_file}" kube::log::status "Validating uploaded version file at ${public_link}" contents="$(curl -s "${public_link}")" else diff --git a/build/push-ci-build.sh b/build/push-ci-build.sh index 3b9c3136d03e..bf5c8e768921 100755 --- a/build/push-ci-build.sh +++ b/build/push-ci-build.sh @@ -41,6 +41,7 @@ KUBE_GCS_MAKE_PUBLIC='y' KUBE_GCS_UPLOAD_RELEASE='y' KUBE_GCS_DELETE_EXISTING='y' KUBE_GCS_RELEASE_BUCKET='kubernetes-release' +KUBE_GCS_RELEASE_BUCKET_MIRROR='kubernetes-release-dev' KUBE_GCS_RELEASE_PREFIX="ci/${LATEST}" KUBE_GCS_PUBLISH_VERSION="${LATEST}" diff --git a/cluster/common.sh b/cluster/common.sh index 2124e41915fb..4439e669796d 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -248,7 +248,7 @@ function detect-master-from-kubeconfig() { # Sets KUBE_VERSION variable to the proper version number (e.g. "v1.0.6", # "v1.2.0-alpha.1.881+376438b69c7612") or a version' publication of the form -# / (e.g. "release/stable",' "ci/latest-1"). +# / (e.g. "release/stable",' "ci/latest-1"). # # See the docs on getting builds for more information about version # publication. @@ -259,7 +259,12 @@ function detect-master-from-kubeconfig() { # KUBE_VERSION function set_binary_version() { if [[ "${1}" =~ "/" ]]; then - KUBE_VERSION=$(gsutil cat gs://kubernetes-release/${1}.txt) + IFS='/' read -a path <<< "${1}" + if [[ "${path[0]}" == "release" ]]; then + KUBE_VERSION=$(gsutil cat "gs://kubernetes-release/${1}.txt") + else + KUBE_VERSION=$(gsutil cat "gs://kubernetes-release-dev/${1}.txt") + fi else KUBE_VERSION=${1} fi @@ -287,8 +292,8 @@ function tars_from_version() { SERVER_BINARY_TAR_URL="https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz" SALT_TAR_URL="https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/kubernetes-salt.tar.gz" elif [[ ${KUBE_VERSION} =~ ${KUBE_CI_VERSION_REGEX} ]]; then - SERVER_BINARY_TAR_URL="https://storage.googleapis.com/kubernetes-release/ci/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz" - SALT_TAR_URL="https://storage.googleapis.com/kubernetes-release/ci/${KUBE_VERSION}/kubernetes-salt.tar.gz" + SERVER_BINARY_TAR_URL="https://storage.googleapis.com/kubernetes-release-dev/ci/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz" + SALT_TAR_URL="https://storage.googleapis.com/kubernetes-release-dev/ci/${KUBE_VERSION}/kubernetes-salt.tar.gz" else echo "Version doesn't match regexp" >&2 exit 1 diff --git a/cluster/gce/upgrade.sh b/cluster/gce/upgrade.sh index c66244cd1fb4..4a08abaa1ced 100755 --- a/cluster/gce/upgrade.sh +++ b/cluster/gce/upgrade.sh @@ -60,7 +60,7 @@ function usage() { release_stable=$(gsutil cat gs://kubernetes-release/release/stable.txt) release_latest=$(gsutil cat gs://kubernetes-release/release/latest.txt) - ci_latest=$(gsutil cat gs://kubernetes-release/ci/latest.txt) + ci_latest=$(gsutil cat gs://kubernetes-release-dev/ci/latest.txt) echo "Right now, versions are as follows:" echo " release/stable: ${0} ${release_stable}" diff --git a/docs/devel/getting-builds.md b/docs/devel/getting-builds.md index 1ad3e9be9062..6d3bff6a2cc1 100644 --- a/docs/devel/getting-builds.md +++ b/docs/devel/getting-builds.md @@ -30,9 +30,9 @@ Finally, you can just print the latest or stable version: You can also use the gsutil tool to explore the Google Cloud Storage release buckets. Here are some examples: ```sh -gsutil cat gs://kubernetes-release/ci/latest.txt # output the latest ci version number -gsutil cat gs://kubernetes-release/ci/latest-green.txt # output the latest ci version number that passed gce e2e -gsutil ls gs://kubernetes-release/ci/v0.20.0-29-g29a55cc/ # list the contents of a ci release +gsutil cat gs://kubernetes-release-dev/ci/latest.txt # output the latest ci version number +gsutil cat gs://kubernetes-release-dev/ci/latest-green.txt # output the latest ci version number that passed gce e2e +gsutil ls gs://kubernetes-release-dev/ci/v0.20.0-29-g29a55cc/ # list the contents of a ci release gsutil ls gs://kubernetes-release/release # list all official releases and rcs ``` diff --git a/hack/get-build.sh b/hack/get-build.sh index 8771a3a7c55d..baff5f4abb31 100755 --- a/hack/get-build.sh +++ b/hack/get-build.sh @@ -23,6 +23,7 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. source "${KUBE_ROOT}/cluster/common.sh" declare -r KUBE_RELEASE_BUCKET_URL="https://storage.googleapis.com/kubernetes-release" +declare -r KUBE_DEV_RELEASE_BUCKET_URL="https://storage.googleapis.com/kubernetes-release-dev" declare -r KUBE_TAR_NAME="kubernetes.tar.gz" usage() { @@ -74,7 +75,7 @@ else if [[ ${KUBE_VERSION} =~ ${KUBE_RELEASE_VERSION_REGEX} ]]; then curl --fail -o "kubernetes-${KUBE_VERSION}.tar.gz" "${KUBE_RELEASE_BUCKET_URL}/release/${KUBE_VERSION}/${KUBE_TAR_NAME}" elif [[ ${KUBE_VERSION} =~ ${KUBE_CI_VERSION_REGEX} ]]; then - curl --fail -o "kubernetes-${KUBE_VERSION}.tar.gz" "${KUBE_RELEASE_BUCKET_URL}/ci/${KUBE_VERSION}/${KUBE_TAR_NAME}" + curl --fail -o "kubernetes-${KUBE_VERSION}.tar.gz" "${KUBE_DEV_RELEASE_BUCKET_URL}/ci/${KUBE_VERSION}/${KUBE_TAR_NAME}" else echo "Version doesn't match regexp" >&2 exit 1 diff --git a/hack/jenkins/e2e-runner.sh b/hack/jenkins/e2e-runner.sh index 997ea60c0a5f..18dcd602d313 100755 --- a/hack/jenkins/e2e-runner.sh +++ b/hack/jenkins/e2e-runner.sh @@ -62,10 +62,15 @@ function fetch_published_version_tars() { clean_binaries local -r published_version="${1}" IFS='/' read -a varr <<< "${published_version}" - bucket="${varr[0]}" - build_version=$(gsutil cat gs://kubernetes-release/${published_version}.txt) + local -r path="${varr[0]}" + if [[ "${path}" == "release" ]]; then + local -r bucket="kubernetes-release" + else + local -r bucket="kubernetes-release-dev" + fi + build_version=$(gsutil cat "gs://${bucket}/${published_version}.txt") echo "Using published version $bucket/$build_version (from ${published_version})" - fetch_tars_from_gcs "${bucket}" "${build_version}" + fetch_tars_from_gcs "gs://${bucket}/${path}" "${build_version}" unpack_binaries # Set CLUSTER_API_VERSION for GKE CI export CLUSTER_API_VERSION=$(echo ${build_version} | cut -c 2-) @@ -79,13 +84,10 @@ function clean_binaries() { } function fetch_tars_from_gcs() { - local -r bucket="${1}" + local -r gspath="${1}" local -r build_version="${2}" - echo "Pulling binaries from GCS; using server version ${bucket}/${build_version}." - gsutil -mq cp \ - "gs://kubernetes-release/${bucket}/${build_version}/kubernetes.tar.gz" \ - "gs://kubernetes-release/${bucket}/${build_version}/kubernetes-test.tar.gz" \ - . + echo "Pulling binaries from GCS; using server version ${gspath}/${build_version}." + gsutil -mq cp "${gspath}/${build_version}/kubernetes.tar.gz" "${gspath}/${build_version}/kubernetes-test.tar.gz" . } function unpack_binaries() {