diff --git a/api/files/manifest.go b/api/files/manifest.go new file mode 100644 index 00000000..9bde7a89 --- /dev/null +++ b/api/files/manifest.go @@ -0,0 +1,63 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package files + +import ( + "fmt" + + "sigs.k8s.io/yaml" +) + +// Filestore holds information about a filestore (e.g. GCS or S3 bucket), +// to be written in a manifest file. +type Filestore struct { + // Base is the leading part of an artifact path, including the scheme. + // It is everything that is not the actual file name itself. + // e.g. "gs://prod-artifacts/myproject" + Base string `json:"base,omitempty"` + ServiceAccount string `json:"service-account,omitempty"` + Src bool `json:"src,omitempty"` +} + +// File holds information about a file artifact. File artifacts are copied from +// a source Filestore to N destination Filestores. +type File struct { + // Name is the relative path of the file, relative to the Filestore base + Name string `json:"name"` + // SHA256 holds the SHA256 hash of the specified file (hex encoded) + SHA256 string `json:"sha256,omitempty"` +} + +// Manifest stores the information in a manifest file (describing the +// desired state of a Docker Registry). +type Manifest struct { + // Filestores contains the source and destination (Src/Dest) filestores. + // Filestores are (for example) GCS or S3 buckets. + // It is possible that in the future, we support promoting to multiple + // filestores, in which case we would have more than just Src/Dest. + Filestores []Filestore `json:"filestores,omitempty"` + Files []File `json:"files,omitempty"` +} + +// ParseManifest parses a Manifest. +func ParseManifest(b []byte) (*Manifest, error) { + m := &Manifest{} + if err := yaml.Unmarshal(b, m); err != nil { + return nil, fmt.Errorf("error parsing manifest: %v", err) + } + return m, nil +} diff --git a/api/files/manifest_test.go b/api/files/manifest_test.go new file mode 100644 index 00000000..cd5e909b --- /dev/null +++ b/api/files/manifest_test.go @@ -0,0 +1,155 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package files_test + +import ( + "fmt" + "strings" + "testing" + + "k8s.io/release/pkg/api/files" +) + +func TestValidateFilestores(t *testing.T) { + tests := []struct { + filestores []files.Filestore + expectedError string + }{ + { + // Filestores are required + filestores: []files.Filestore{}, + expectedError: "filestore must be specified", + }, + { + // Filestores are required + filestores: nil, + expectedError: "filestore must be specified", + }, + { + filestores: []files.Filestore{ + {Src: true, Base: "gs://src"}, + }, + expectedError: "no destination filestores found", + }, + { + filestores: []files.Filestore{ + {Base: "gs://dest1"}, + }, + expectedError: "source filestore not found", + }, + { + filestores: []files.Filestore{ + {Src: true, Base: "gs://src1"}, + {Src: true, Base: "gs://src2"}, + }, + expectedError: "found multiple source filestores", + }, + { + filestores: []files.Filestore{ + {Src: true, Base: "gs://src"}, + {Base: "gs://dest1"}, + {Base: "gs://dest2"}, + }, + }, + { + filestores: []files.Filestore{ + {Src: true}, + {Base: "gs://dest"}, + }, + expectedError: "filestore did not have base set", + }, + { + filestores: []files.Filestore{ + {Src: true, Base: "gs://src"}, + {Base: "s3://dest"}, + }, + expectedError: "unsupported scheme in base", + }, + } + for _, test := range tests { + err := files.ValidateFilestores(test.filestores) + checkErrorMatchesExpected(t, err, test.expectedError) + } +} + +func TestValidateFiles(t *testing.T) { + oksha := "4f2f040fa2bfe9bea64911a2a756e8a1727a8bfd757c5e031631a6e699fcf246" + + tests := []struct { + files []files.File + expectedError string + }{ + { + // Files are required + files: []files.File{}, + expectedError: "file must be specified", + }, + { + // Files are required + files: nil, + expectedError: "file must be specified", + }, + { + files: []files.File{ + {Name: "foo", SHA256: oksha}, + }, + }, + { + files: []files.File{ + {SHA256: oksha}, + }, + expectedError: "name is required for file", + }, + { + files: []files.File{ + {Name: "foo", SHA256: "bad"}, + }, + expectedError: "sha256 was not valid (not hex)", + }, + { + files: []files.File{ + {Name: "foo"}, + }, + expectedError: "sha256 is required", + }, + { + files: []files.File{ + {Name: "foo", SHA256: "abcd"}, + }, + expectedError: "sha256 was not valid (bad length)", + }, + } + for _, test := range tests { + err := files.ValidateFiles(test.files) + checkErrorMatchesExpected(t, err, test.expectedError) + } +} + +func checkErrorMatchesExpected(t *testing.T, err error, expected string) { + if err != nil && expected == "" { + t.Errorf("unexpected error: %v", err) + } + if err != nil && expected != "" { + actual := fmt.Sprintf("%v", err) + if !strings.Contains(actual, expected) { + t.Errorf("error %q did not contain expected %q", err, expected) + } + } + if err == nil && expected != "" { + t.Errorf("expected error %q", expected) + } +} diff --git a/api/files/validation.go b/api/files/validation.go new file mode 100644 index 00000000..47ff4ab1 --- /dev/null +++ b/api/files/validation.go @@ -0,0 +1,110 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package files + +import ( + "encoding/hex" + "fmt" + "strings" + + "sigs.k8s.io/release-sdk/object" +) + +// Validate checks for semantic errors in the yaml fields (the structure of the +// yaml is checked during unmarshaling). +func (m *Manifest) Validate() error { + if err := ValidateFilestores(m.Filestores); err != nil { + return err + } + if err := ValidateFiles(m.Files); err != nil { + return err + } + return nil +} + +// ValidateFilestores validates the Filestores field of the manifest. +func ValidateFilestores(filestores []Filestore) error { + if len(filestores) == 0 { + return fmt.Errorf("at least one filestore must be specified") + } + + var source *Filestore + destinationCount := 0 + + for i := range filestores { + filestore := &filestores[i] + + if filestore.Base == "" { + return fmt.Errorf("filestore did not have base set") + } + + // Currently the only backend supported is GCS + if !strings.HasPrefix(filestore.Base, object.GcsPrefix) { + return fmt.Errorf( + "filestore has unsupported scheme in base %q", + filestore.Base) + } + + if filestore.Src { + if source != nil { + return fmt.Errorf("found multiple source filestores") + } + source = filestore + } else { + destinationCount++ + } + } + if source == nil { + return fmt.Errorf("source filestore not found") + } + + if destinationCount == 0 { + return fmt.Errorf("no destination filestores found") + } + + return nil +} + +// ValidateFiles validates the Files field of the manifest. +func ValidateFiles(files []File) error { + if len(files) == 0 { + return fmt.Errorf("at least one file must be specified") + } + + for i := range files { + f := &files[i] + + if f.Name == "" { + return fmt.Errorf("name is required for file") + } + + if f.SHA256 == "" { + return fmt.Errorf("sha256 is required for file") + } + + sha256, err := hex.DecodeString(f.SHA256) + if err != nil { + return fmt.Errorf("sha256 was not valid (not hex): %q", f.SHA256) + } + + if len(sha256) != 32 { + return fmt.Errorf("sha256 was not valid (bad length): %q", f.SHA256) + } + } + + return nil +} diff --git a/cmd/cip-mm/README.md b/cmd/cip-mm/README.md new file mode 100644 index 00000000..51ded7bd --- /dev/null +++ b/cmd/cip-mm/README.md @@ -0,0 +1,40 @@ +# cip-mm + +This tool **m**odifies promoter **m**anifests. For now it dumps some filtered +subset of a staging GCR and merges those contents back into a given promoter +manifest. + +## Examples + +- Add all images with a matching digest from staging repo + `gcr.io/k8s-staging-artifact-promoter` to a manifest, using the name and tags + already existing in the staging repo: + +``` +cip-mm \ + --base_dir=$HOME/go/src/github.com/kubernetes/k8s.io/k8s.gcr.io \ + --staging_repo=gcr.io/k8s-staging-artifact-promoter \ + --filter_digest=sha256:7594278deaf6eeaa35caedec81796d103e3c83a26d7beab091a5d25a9ba6aa16 +``` + +- Add a single image named "foo" and tagged "1.0" from staging repo + `gcr.io/k8s-staging-artifact-promoter` to a manifest: + +``` +cip-mm \ + --base_dir=$HOME/go/src/github.com/kubernetes/k8s.io/k8s.gcr.io \ + --staging_repo=gcr.io/k8s-staging-artifact-promoter \ + --filter_image=cip \ + --filter_tag=1.0 +``` + +- Add all images tagged `1.0` from staging repo + `gcr.io/k8s-staging-artifact-promoter` to a manifest: + +``` +cip-mm \ + --base_dir=$HOME/go/src/github.com/kubernetes/k8s.io/k8s.gcr.io \ + --staging_repo=gcr.io/k8s-staging-artifact-promoter \ + --filter_image=cip \ + --filter_tag=1.0 +``` diff --git a/cmd/cip-mm/main.go b/cmd/cip-mm/main.go new file mode 100644 index 00000000..8f384675 --- /dev/null +++ b/cmd/cip-mm/main.go @@ -0,0 +1,121 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + reg "sigs.k8s.io/k8s-container-image-promoter/legacy/dockerregistry" + "sigs.k8s.io/release-utils/log" +) + +var cmd = &cobra.Command{ + Short: "cip-mm → Container Image Promoter - Manifest Modificator", + Long: `cip-mm → Container Image Promoter - Manifest Modificator + +This tool **m**odifies promoter **m**anifests. For now it dumps some filtered +subset of a staging GCR and merges those contents back into a given promoter +manifest.`, + Use: "cip-mm", + SilenceUsage: true, + SilenceErrors: true, + PersistentPreRunE: initLogging, + RunE: func(cmd *cobra.Command, args []string) error { + return run() + }, +} + +type commandLineOptions struct { + baseDir string + stagingRepo string + filterImage string + filterDigest string + filterTag string + logLevel string +} + +var commandLineOpts = &commandLineOptions{} + +func init() { + cmd.PersistentFlags().StringVar( + &commandLineOpts.baseDir, + "base_dir", + "", + "the manifest directory to look at and modify", + ) + cmd.PersistentFlags().StringVar( + &commandLineOpts.stagingRepo, + "staging_repo", + "", + "the staging repo which we want to read from", + ) + cmd.PersistentFlags().StringVar( + &commandLineOpts.filterImage, + "filter_image", + "", + "filter staging repo by this image name", + ) + cmd.PersistentFlags().StringVar( + &commandLineOpts.filterDigest, + "filter_digest", + "", + "filter images by this digest", + ) + cmd.PersistentFlags().StringVar( + &commandLineOpts.filterTag, + "filter_tag", + "", + "filter images by this tag", + ) + cmd.PersistentFlags().StringVar( + &commandLineOpts.logLevel, + "log-level", + "info", + fmt.Sprintf("the logging verbosity, either %s", log.LevelNames()), + ) +} + +func main() { + if err := cmd.Execute(); err != nil { + logrus.Fatal(err) + } +} + +func initLogging(*cobra.Command, []string) error { + return log.SetupGlobalLogger(commandLineOpts.logLevel) +} + +func run() error { + opt := reg.GrowManifestOptions{} + if err := opt.Populate( + commandLineOpts.baseDir, commandLineOpts.stagingRepo, + commandLineOpts.filterImage, commandLineOpts.filterDigest, + commandLineOpts.filterTag); err != nil { + return err + } + + if err := opt.Validate(); err != nil { + return err + } + + ctx := context.Background() + return reg.GrowManifest(ctx, &opt) +} diff --git a/cmd/kpromo/Dockerfile b/cmd/kpromo/Dockerfile new file mode 100644 index 00000000..867591d4 --- /dev/null +++ b/cmd/kpromo/Dockerfile @@ -0,0 +1,51 @@ +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build the manager binary +ARG GO_VERSION +ARG OS_CODENAME +# TODO(codename): Consider parameterizing in Makefile based on codename +ARG DISTROLESS_IMAGE +FROM golang:${GO_VERSION}-${OS_CODENAME} as builder + +WORKDIR /go/src/k8s.io/release + +# Copy the sources +ENV package="./cmd/kpromo" +COPY ../../go.mod ../../go.sum ./ +COPY ../../pkg ./pkg/ +COPY ../../cmd/kpromo ${package}/ + +RUN go mod download + +# Build +ARG ARCH + +ENV CGO_ENABLED=0 +ENV GOOS=linux +ENV GOARCH=${ARCH} + +RUN go build -trimpath -ldflags '-s -w -buildid= -extldflags "-static"' \ + -o kpromo ${package} + +# Production image +FROM gcr.io/distroless/${DISTROLESS_IMAGE}:latest + +LABEL maintainers="Kubernetes Authors" +LABEL description="kpromo: The Kubernetes project artifact promoter" + +WORKDIR / +COPY --from=builder /go/src/k8s.io/release/kpromo . + +ENTRYPOINT ["/kpromo"] diff --git a/cmd/kpromo/Makefile b/cmd/kpromo/Makefile new file mode 100644 index 00000000..0732481f --- /dev/null +++ b/cmd/kpromo/Makefile @@ -0,0 +1,98 @@ +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# set default shell +SHELL=/bin/bash -o pipefail + +REGISTRY ?= gcr.io/k8s-staging-artifact-promoter +IMGNAME = kpromo +IMAGE_VERSION ?= v0.1.0-1 + +IMAGE = $(REGISTRY)/$(IMGNAME) + +TAG ?= $(shell git describe --tags --always --dirty) + +# Build args +GO_VERSION ?= 1.17 +OS_CODENAME ?= buster +DISTROLESS_IMAGE ?= static-debian10 + +# Configuration +CONFIG = $(OS_CODENAME) + +PLATFORMS ?= linux/amd64 + +HOST_GOOS ?= $(shell go env GOOS) +HOST_GOARCH ?= $(shell go env GOARCH) +GO_BUILD ?= go build + +BUILD_ARGS = --build-arg=GO_VERSION=$(GO_VERSION) \ + --build-arg=OS_CODENAME=$(OS_CODENAME) \ + --build-arg=DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) + +# Ensure support for 'docker buildx' and 'docker manifest' commands +export DOCKER_CLI_EXPERIMENTAL=enabled + +.PHONY: all build clean + +.PHONY: all +all: build + +.PHONY: build +build: + $(GO_BUILD) + +.PHONY: clean +clean: + rm kpromo + +# build with buildx +# https://github.com/docker/buildx/issues/59 +.PHONY: container +container: init-docker-buildx + echo "Building $(IMGNAME) for the following platforms: $(PLATFORMS)" + @for platform in $(PLATFORMS); do \ + echo "Starting build for $${platform} platform"; \ + docker buildx build \ + --load \ + --progress plain \ + --platform $${platform} \ + --tag $(IMAGE)-$${platform##*/}:$(IMAGE_VERSION) \ + --tag $(IMAGE)-$${platform##*/}:$(TAG) \ + --tag $(IMAGE)-$${platform##*/}:latest \ + $(BUILD_ARGS) \ + -f $(CURDIR)/Dockerfile \ + ../../.; \ + done + +.PHONY: push +push: container + echo "Pushing $(IMGNAME) tags" + @for platform in $(PLATFORMS); do \ + echo "Pushing tags for $${platform} platform"; \ + docker push $(IMAGE)-$${platform##*/}:$(IMAGE_VERSION); \ + docker push $(IMAGE)-$${platform##*/}:$(TAG); \ + docker push $(IMAGE)-$${platform##*/}:latest; \ + done + +.PHONY: manifest +manifest: push + docker manifest create --amend $(IMAGE):$(IMAGE_VERSION) $(IMAGE)-$(subst linux/,,$(firstword $(PLATFORMS))):$(IMAGE_VERSION) + @for platform in $(PLATFORMS); do docker manifest annotate --arch "$${platform##*/}" ${IMAGE}:${IMAGE_VERSION} ${IMAGE}-$${platform##*/}:${IMAGE_VERSION}; done + docker manifest push --purge $(IMAGE):$(IMAGE_VERSION) + +# enable buildx +.PHONY: init-docker-buildx +init-docker-buildx: + ./../../hack/init-buildx.sh diff --git a/cmd/kpromo/README.md b/cmd/kpromo/README.md new file mode 100644 index 00000000..3716585d --- /dev/null +++ b/cmd/kpromo/README.md @@ -0,0 +1,114 @@ +# kpromo - Artifact promoter + +kpromo is a tool responsible for artifact promotion. + +It has two operation modes: + +- `run` - Execute a file promotion (formerly "promobot-files") (image promotion coming soon) +- `manifest` - Generate/modify a file manifest to target for promotion (image support coming soon) + +Expectations: + +- `kpromo run` should only be run in auditable environments +- `kpromo manifest` should primarily be run by contributors + +- [File promotion](#file-promotion) + - [Running the file promoter](#running-the-file-promoter) + - [Generating a file promotion manifest](#generating-a-file-promotion-manifest) + - [Consumers](#consumers) + +## File promotion + +The file promoter copies files from source GCS buckets to one or more +destination buckets, by reading a `Manifest` file (in YAML). + +The `Manifest` lists the files and their hashes that should be copied from src to +dest. + +Example `Manifest` for files: + +```yaml +filestores: +- base: gs://staging/ + src: true +- base: gs://prod/subdir + service-account: foo@google-containers.iam.gserviceaccount.com +files: +- name: vegetables/artichoke + sha256: 2d4f26491e0e470236f73a0b8d6828db017eab988cd102fc19afe31f1f56aff7 +- name: vegetables/beetroot + sha256: 160b98e27ec99f77efe01e2996fa386f2b2aec552599f8bd861be0a857e7f29f +``` + +`filestores` is the equivalent of the container manifest `registries`, and lists +the buckets from which the promoter should read or write files. `files` is the +equivalent of `images`, and lists the files that should be promoted. + +`filestores` supports `service-account`, and it also supports relative paths - +note that the source files in the example above are in the root of the bucket, +but they are copied into a subdirectory of the target bucket. + +`files` is a list of files to be copied. The `name` is appended to the base of +the filestore, and then the files are copied. If the source file does not have +the matching sha256, it will not be copied. + +When errors are encountered building the list of files to be copied, no files +will be copied. When errors are encountered while copying files, we will still +attempt to copy remaining files, but the process will report the error. + +Currently only Google Cloud Storage (GCS) buckets supported, with a prefix of +`gs://` + +### Running the file promoter + +```console +$ kpromo run files --help + +Promote files from a staging object store to production + +Usage: + kpromo run files [flags] + +Flags: + --dry-run test run promotion without modifying any filestore (default true) + --files files path to the files manifest + --filestores filestores path to the filestores promoter manifest + -h, --help help for files + --use-service-account allow service account usage with gcloud calls + +Global Flags: + --log-level string the logging verbosity, either 'panic', 'fatal', 'error', 'warning', 'info', 'debug', 'trace' (default "info") +``` + +### Generating a file promotion manifest + +This tool will generate a manifest fragment for uploading a set of +files, located in the specified path. + +It takes a single argument `--src`, which is the base of the directory +tree; all files under that directory (recursively) are hashed and +output into the `files` section of a manifest. + +The manifest is written to stdout. + +```console +$ kpromo manifest files --help +Promote files from a staging object store to production + +Usage: + kpromo manifest files [flags] + +Flags: + -h, --help help for files + --prefix string only export files starting with the provided prefix + --src string the base directory to copy from + +Global Flags: + --log-level string the logging verbosity, either 'panic', 'fatal', 'error', 'warning', 'info', 'debug', 'trace' (default "info") +``` + +### Consumers + +- [`kOps`][kops-release-process] + +[kops-release-process]: https://kops.sigs.k8s.io/contributing/release-process/ diff --git a/cmd/kpromo/cloudbuild.yaml b/cmd/kpromo/cloudbuild.yaml new file mode 100644 index 00000000..f80972d5 --- /dev/null +++ b/cmd/kpromo/cloudbuild.yaml @@ -0,0 +1,55 @@ +# See https://git.k8s.io/test-infra/config/jobs/image-pushing/README.md for +# more details on image pushing process + +# this must be specified in seconds. If omitted, defaults to 600s (10 mins) +timeout: 1200s + +# this prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF, +# or any new substitutions added in the future. +options: + substitution_option: ALLOW_LOOSE + machineType: 'N1_HIGHCPU_8' + +steps: + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20201130-750d12f' + entrypoint: 'bash' + dir: ./cmd/kpromo + env: + - DOCKER_CLI_EXPERIMENTAL=enabled + - REGISTRY=gcr.io/$PROJECT_ID + - HOME=/root + - TAG=$_GIT_TAG + - PULL_BASE_REF=$_PULL_BASE_REF + - IMAGE_VERSION=$_IMAGE_VERSION + - GO_VERSION=$_GO_VERSION + - OS_CODENAME=$_OS_CODENAME + - DISTROLESS_IMAGE=$_DISTROLESS_IMAGE + args: + - '-c' + - | + gcloud auth configure-docker \ + && make manifest + +substitutions: + # _GIT_TAG will be filled with a git-based tag for the image, of the form + # vYYYYMMDD-hash, and can be used as a substitution + _GIT_TAG: '12345' + _PULL_BASE_REF: 'dev' + _IMAGE_VERSION: 'v0.0.0' + _GO_VERSION: '0.0.0' + _OS_CODENAME: 'codename' + _DISTROLESS_IMAGE: 'static-debian00' + +tags: +- 'kpromo' +- ${_GIT_TAG} +- ${_PULL_BASE_REF} +- ${_IMAGE_VERSION} +- ${_GO_VERSION} +- ${_OS_CODENAME} +- ${_DISTROLESS_IMAGE} + +images: + - 'gcr.io/$PROJECT_ID/kpromo-amd64:$_IMAGE_VERSION' + - 'gcr.io/$PROJECT_ID/kpromo-amd64:$_GIT_TAG' + - 'gcr.io/$PROJECT_ID/kpromo-amd64:latest' diff --git a/cmd/kpromo/cmd/manifest/files.go b/cmd/kpromo/cmd/manifest/files.go new file mode 100644 index 00000000..bcef90a0 --- /dev/null +++ b/cmd/kpromo/cmd/manifest/files.go @@ -0,0 +1,96 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manifest + +import ( + "context" + "os" + "path/filepath" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + + "k8s.io/release/pkg/promobot" + "sigs.k8s.io/yaml" +) + +// filesCmd represents the subcommand for `kpromo manifest files` +var filesCmd = &cobra.Command{ + Use: "files", + Short: "Promote files from a staging object store to production", + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, args []string) error { + return errors.Wrap(runFileManifest(filesOpts), "run `kpromo manifest files`") + }, +} + +var filesOpts = &promobot.GenerateManifestOptions{} + +func init() { + // TODO: Move this into a default options function in pkg/promobot + filesOpts.PopulateDefaults() + + filesCmd.PersistentFlags().StringVar( + &filesOpts.BaseDir, + "src", + filesOpts.BaseDir, + "the base directory to copy from", + ) + + filesCmd.PersistentFlags().StringVar( + &filesOpts.Prefix, + "prefix", + filesOpts.Prefix, + "only export files starting with the provided prefix", + ) + + // TODO: Consider moving this into a validation function + // nolint: errcheck + filesCmd.MarkPersistentFlagRequired("src") + + ManifestCmd.AddCommand(filesCmd) +} + +func runFileManifest(opts *promobot.GenerateManifestOptions) error { + ctx := context.Background() + + src, err := filepath.Abs(opts.BaseDir) + if err != nil { + return errors.Wrapf(err, "resolving %q to absolute path", src) + } + + opts.BaseDir = src + + manifest, err := promobot.GenerateManifest(ctx, *opts) + if err != nil { + return err + } + + manifestYAML, err := yaml.Marshal(manifest) + if err != nil { + return errors.Wrap(err, "serializing manifest") + } + + if _, err := os.Stdout.Write(manifestYAML); err != nil { + return err + } + + return nil +} + +// TODO: Validate options diff --git a/cmd/kpromo/cmd/manifest/manifest.go b/cmd/kpromo/cmd/manifest/manifest.go new file mode 100644 index 00000000..5032e335 --- /dev/null +++ b/cmd/kpromo/cmd/manifest/manifest.go @@ -0,0 +1,29 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manifest + +import ( + "github.com/spf13/cobra" +) + +// ManifestCmd is a kpromo subcommand which just holds further subcommands +var ManifestCmd = &cobra.Command{ + Use: "manifest", + Short: "Generate/modify a manifest for artifact promotion", + SilenceUsage: true, + SilenceErrors: true, +} diff --git a/cmd/kpromo/cmd/root.go b/cmd/kpromo/cmd/root.go new file mode 100644 index 00000000..8cdbf046 --- /dev/null +++ b/cmd/kpromo/cmd/root.go @@ -0,0 +1,79 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "k8s.io/release/cmd/kpromo/cmd/manifest" + "k8s.io/release/cmd/kpromo/cmd/run" + "sigs.k8s.io/release-utils/log" +) + +// rootCmd represents the base command when called without any subcommands +// TODO: Update command description +var rootCmd = &cobra.Command{ + Use: "kpromo", + Long: `kpromo - Kubernetes project artifact promoter + +kpromo is a tool responsible for artifact promotion. + +It has two operation modes: +- "run" - Execute a file promotion (formerly "promobot-files") (image promotion coming soon) +- "manifest" - Generate/modify a file manifest to target for promotion (image support coming soon) + +Expectations: +- "kpromo run" should only be run in auditable environments +- "kpromo manifest" should primarily be run by contributors + +Each subcommand should contain its own self-describing help output which +clarifies its purpose.`, + PersistentPreRunE: initLogging, +} + +type rootOptions struct { + logLevel string +} + +var rootOpts = &rootOptions{} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + if err := rootCmd.Execute(); err != nil { + logrus.Fatal(err) + } +} + +func init() { + rootCmd.PersistentFlags().StringVar( + &rootOpts.logLevel, + "log-level", + "info", + fmt.Sprintf("the logging verbosity, either %s", log.LevelNames()), + ) + + rootCmd.AddCommand(run.RunCmd) + rootCmd.AddCommand(manifest.ManifestCmd) +} + +func initLogging(*cobra.Command, []string) error { + return log.SetupGlobalLogger(rootOpts.logLevel) +} diff --git a/cmd/kpromo/cmd/run/files.go b/cmd/kpromo/cmd/run/files.go new file mode 100644 index 00000000..1f4bd6e2 --- /dev/null +++ b/cmd/kpromo/cmd/run/files.go @@ -0,0 +1,89 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package run + +import ( + "context" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + + "k8s.io/release/pkg/promobot" +) + +// filesCmd represents the subcommand for `kpromo run files` +var filesCmd = &cobra.Command{ + Use: "files", + Short: "Promote files from a staging object store to production", + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, args []string) error { + return errors.Wrap(runFilePromotion(filesOpts), "run `kpromo run files`") + }, +} + +var filesOpts = &promobot.PromoteFilesOptions{} + +func init() { + // TODO: Move this into a default options function in pkg/promobot + filesOpts.PopulateDefaults() + + filesCmd.PersistentFlags().StringVar( + &filesOpts.FilestoresPath, + "filestores", + filesOpts.FilestoresPath, + "path to the `filestores` promoter manifest", + ) + + filesCmd.PersistentFlags().StringVar( + &filesOpts.FilesPath, + "files", + filesOpts.FilesPath, + "path to the `files` manifest", + ) + + // TODO: Consider moving this to the root command + filesCmd.PersistentFlags().BoolVar( + &filesOpts.DryRun, + "dry-run", + filesOpts.DryRun, + "test run promotion without modifying any filestore", + ) + + filesCmd.PersistentFlags().BoolVar( + &filesOpts.UseServiceAccount, + "use-service-account", + filesOpts.UseServiceAccount, + "allow service account usage with gcloud calls", + ) + + // TODO: Consider moving this into a validation function + // nolint: errcheck + filesCmd.MarkPersistentFlagRequired("filestores") + // nolint: errcheck + filesCmd.MarkPersistentFlagRequired("files") + + RunCmd.AddCommand(filesCmd) +} + +func runFilePromotion(opts *promobot.PromoteFilesOptions) error { + ctx := context.Background() + + return promobot.RunPromoteFiles(ctx, *opts) +} + +// TODO: Validate options diff --git a/cmd/kpromo/cmd/run/run.go b/cmd/kpromo/cmd/run/run.go new file mode 100644 index 00000000..d6d15f96 --- /dev/null +++ b/cmd/kpromo/cmd/run/run.go @@ -0,0 +1,29 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package run + +import ( + "github.com/spf13/cobra" +) + +// RunCmd is a kpromo subcommand which just holds further subcommands +var RunCmd = &cobra.Command{ + Use: "run", + Short: "Run artifact promotion", + SilenceUsage: true, + SilenceErrors: true, +} diff --git a/cmd/kpromo/main.go b/cmd/kpromo/main.go new file mode 100644 index 00000000..b275a872 --- /dev/null +++ b/cmd/kpromo/main.go @@ -0,0 +1,25 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate + +package main + +import "k8s.io/release/cmd/kpromo/cmd" + +func main() { + cmd.Execute() +} diff --git a/cmd/kpromo/variants.yaml b/cmd/kpromo/variants.yaml new file mode 100644 index 00000000..20748285 --- /dev/null +++ b/cmd/kpromo/variants.yaml @@ -0,0 +1,6 @@ +variants: + default: + IMAGE_VERSION: 'v0.1.0-1' + GO_VERSION: '1.17' + OS_CODENAME: 'buster' + DISTROLESS_IMAGE: 'static-debian10' diff --git a/filepromoter/file.go b/filepromoter/file.go new file mode 100644 index 00000000..4ceb3ecd --- /dev/null +++ b/filepromoter/file.go @@ -0,0 +1,121 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filepromoter + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/sirupsen/logrus" + + api "k8s.io/release/pkg/api/files" + "sigs.k8s.io/release-utils/hash" +) + +// syncFileInfo tracks a file during the synchronization operation. +type syncFileInfo struct { + RelativePath string + AbsolutePath string + + // Some backends (GCS and S3) expose the MD5 of the content in metadata + // This can allow skipping unnecessary copies. + // Note: with multipart uploads or compression, the value is unobvious. + MD5 string + + Size int64 + + filestore syncFilestore +} + +// copyFileOp manages copying a single file. +type copyFileOp struct { + Source *syncFileInfo + Dest *syncFileInfo + + ManifestFile *api.File +} + +// Run implements SyncFileOp.Run +func (o *copyFileOp) Run(ctx context.Context) error { + // Download to our temp file + f, err := os.CreateTemp("", "promoter") + if err != nil { + return fmt.Errorf("error creating temp file: %v", err) + } + tempFilename := f.Name() + + defer func() { + if f != nil { + if err := f.Close(); err != nil { + logrus.Warnf( + "error closing temp file %q: %v", + tempFilename, err) + } + } + + if err := os.Remove(tempFilename); err != nil { + logrus.Warnf( + "unable to remove temp file %q: %v", + tempFilename, err) + } + }() + + in, err := o.Source.filestore.OpenReader(ctx, o.Source.RelativePath) + if err != nil { + return fmt.Errorf("error reading %q: %v", o.Source.AbsolutePath, err) + } + defer in.Close() + + if _, err := io.Copy(f, in); err != nil { + return fmt.Errorf( + "error downloading %s: %v", + o.Source.AbsolutePath, err) + } + // We close the file to be sure it is fully written + if err := f.Close(); err != nil { + return fmt.Errorf("error writing temp file %q: %v", tempFilename, err) + } + f = nil + + // Verify the source hash + sha256, err := hash.SHA256ForFile(tempFilename) + if err != nil { + return err + } + if sha256 != o.ManifestFile.SHA256 { + return fmt.Errorf( + "sha256 did not match for file %q: actual=%q expected=%q", + o.Source.AbsolutePath, sha256, o.ManifestFile.SHA256) + } + + // Upload to the destination + if err := o.Dest.filestore.UploadFile( + ctx, o.Dest.RelativePath, tempFilename); err != nil { + return err + } + + return nil +} + +// String is the pretty-printer for an operation, as used by dry-run. +func (o *copyFileOp) String() string { + return fmt.Sprintf( + "COPY %q to %q", + o.Source.AbsolutePath, o.Dest.AbsolutePath) +} diff --git a/filepromoter/filestore.go b/filepromoter/filestore.go new file mode 100644 index 00000000..ba5a9431 --- /dev/null +++ b/filepromoter/filestore.go @@ -0,0 +1,202 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filepromoter + +import ( + "context" + "fmt" + "io" + "net/url" + "strings" + + "cloud.google.com/go/storage" + "github.com/sirupsen/logrus" + "google.golang.org/api/option" + + api "k8s.io/release/pkg/api/files" + "sigs.k8s.io/release-sdk/object" +) + +// FilestorePromoter manages the promotion of files. +type FilestorePromoter struct { + Source *api.Filestore + Dest *api.Filestore + + Files []api.File + + // UseServiceAccount must be true, for service accounts to be used + // This gives some protection against a hostile manifest. + UseServiceAccount bool +} + +type syncFilestore interface { + // OpenReader opens an io.ReadCloser for the specified file + OpenReader(ctx context.Context, name string) (io.ReadCloser, error) + + // UploadFile uploads a local file to the specified destination + UploadFile(ctx context.Context, dest string, localFile string) error + + // ListFiles returns all the file artifacts in the filestore, recursively. + ListFiles(ctx context.Context) (map[string]*syncFileInfo, error) +} + +func openFilestore( + ctx context.Context, + filestore *api.Filestore, + useServiceAccount bool) (syncFilestore, error) { + u, err := url.Parse(filestore.Base) + if err != nil { + return nil, fmt.Errorf( + "error parsing filestore base %q: %v", + filestore.Base, err) + } + + if u.Scheme != "gs" { + return nil, fmt.Errorf( + "unrecognized scheme %q (supported schemes: %s)", + object.GcsPrefix, filestore.Base, + ) + } + + var opts []option.ClientOption + if useServiceAccount && filestore.ServiceAccount != "" { + ts := &gcloudTokenSource{ServiceAccount: filestore.ServiceAccount} + opts = append(opts, option.WithTokenSource(ts)) + } else { + opts = append(opts, option.WithoutAuthentication()) + } + + client, err := storage.NewClient(ctx, opts...) + if err != nil { + return nil, fmt.Errorf("error building GCS client: %v", err) + } + + prefix := strings.TrimPrefix(u.Path, "/") + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + bucket := u.Host + + s := &gcsSyncFilestore{ + filestore: filestore, + client: client, + bucket: bucket, + prefix: prefix, + } + return s, nil +} + +// computeNeededOperations determines the list of files that need to be copied +func (p *FilestorePromoter) computeNeededOperations( + source, dest map[string]*syncFileInfo, + destFilestore syncFilestore) ([]SyncFileOp, error) { + ops := make([]SyncFileOp, 0) + + for i := range p.Files { + f := &p.Files[i] + relativePath := f.Name + sourceFile := source[relativePath] + if sourceFile == nil { + // TODO: Should this be a warning? + absolutePath := joinFilepath(p.Source, relativePath) + return nil, fmt.Errorf( + "file %q not found in source (%q)", + relativePath, absolutePath) + } + + destFile := dest[relativePath] + if destFile == nil { + destFile = &syncFileInfo{} + destFile.RelativePath = sourceFile.RelativePath + destFile.AbsolutePath = joinFilepath( + p.Dest, + sourceFile.RelativePath) + destFile.filestore = destFilestore + ops = append(ops, ©FileOp{ + Source: sourceFile, + Dest: destFile, + ManifestFile: f, + }) + continue + } + + changed := false + if destFile.MD5 != sourceFile.MD5 { + logrus.Warnf("MD5 mismatch on source %q vs dest %q: %q vs %q", + sourceFile.AbsolutePath, + destFile.AbsolutePath, + sourceFile.MD5, + destFile.MD5) + changed = true + } + + if destFile.Size != sourceFile.Size { + logrus.Warnf("Size mismatch on source %q vs dest %q: %d vs %d", + sourceFile.AbsolutePath, + destFile.AbsolutePath, + sourceFile.Size, + destFile.Size) + changed = true + } + + if !changed { + logrus.Infof("metadata match for %q", destFile.AbsolutePath) + continue + } + ops = append(ops, ©FileOp{ + Source: sourceFile, + Dest: destFile, + ManifestFile: f, + }) + } + + return ops, nil +} + +func joinFilepath(filestore *api.Filestore, relativePath string) string { + s := strings.TrimSuffix(filestore.Base, "/") + s += "/" + s += strings.TrimPrefix(relativePath, "/") + return s +} + +// BuildOperations builds the required operations to sync from the +// Source Filestore to the Dest Filestore. +func (p *FilestorePromoter) BuildOperations( + ctx context.Context) ([]SyncFileOp, error) { + sourceFilestore, err := openFilestore(ctx, p.Source, p.UseServiceAccount) + if err != nil { + return nil, err + } + destFilestore, err := openFilestore(ctx, p.Dest, p.UseServiceAccount) + if err != nil { + return nil, err + } + + sourceFiles, err := sourceFilestore.ListFiles(ctx) + if err != nil { + return nil, err + } + + destFiles, err := destFilestore.ListFiles(ctx) + if err != nil { + return nil, err + } + + return p.computeNeededOperations(sourceFiles, destFiles, destFilestore) +} diff --git a/filepromoter/gcs.go b/filepromoter/gcs.go new file mode 100644 index 00000000..9330298d --- /dev/null +++ b/filepromoter/gcs.go @@ -0,0 +1,146 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filepromoter + +import ( + "context" + "encoding/hex" + "fmt" + "hash/crc32" + "io" + "os" + "strings" + + "cloud.google.com/go/storage" + "github.com/sirupsen/logrus" + "google.golang.org/api/iterator" + + api "k8s.io/release/pkg/api/files" + "sigs.k8s.io/release-sdk/object" +) + +type gcsSyncFilestore struct { + filestore *api.Filestore + client *storage.Client + bucket string + prefix string +} + +// OpenReader opens an io.ReadCloser for the specified file. +func (s *gcsSyncFilestore) OpenReader( + ctx context.Context, + name string) (io.ReadCloser, error) { + absolutePath := s.prefix + name + return s.client.Bucket(s.bucket).Object(absolutePath).NewReader(ctx) +} + +// UploadFile uploads a local file to the specified destination. +func (s *gcsSyncFilestore) UploadFile(ctx context.Context, dest, localFile string) error { + absolutePath := s.prefix + dest + + gcsURL := object.GcsPrefix + s.bucket + "/" + absolutePath + + in, err := os.Open(localFile) + if err != nil { + return fmt.Errorf("error opening %q: %v", localFile, err) + } + defer func() { + if err := in.Close(); err != nil { + logrus.Warnf("error closing %q: %v", localFile, err) + } + }() + + // Compute crc32 checksum for upload integrity + var fileCRC32C uint32 + { + hasher := crc32.New(crc32.MakeTable(crc32.Castagnoli)) + if _, err := io.Copy(hasher, in); err != nil { + return fmt.Errorf("error computing crc32 checksum: %v", err) + } + fileCRC32C = hasher.Sum32() + + if _, err := in.Seek(0, 0); err != nil { + return fmt.Errorf("error rewinding in file: %v", err) + } + } + + logrus.Infof("uploading to %s", gcsURL) + + w := s.client.Bucket(s.bucket).Object(absolutePath).NewWriter(ctx) + + w.CRC32C = fileCRC32C + w.SendCRC32C = true + + // Much bigger chunk size for faster uploading + w.ChunkSize = 128 * 1024 * 1024 + + if _, err := io.Copy(w, in); err != nil { + if err2 := w.Close(); err2 != nil { + logrus.Warnf("error closing upload stream: %v", err) + // TODO: Try to delete the possibly partially written file? + } + return fmt.Errorf("error uploading to %q: %v", gcsURL, err) + } + + if err := w.Close(); err != nil { + return fmt.Errorf("error uploading to %q: %v", gcsURL, err) + } + + return nil +} + +// ListFiles returns all the file artifacts in the filestore, recursively. +func (s *gcsSyncFilestore) ListFiles( + ctx context.Context) (map[string]*syncFileInfo, error) { + files := make(map[string]*syncFileInfo) + + q := &storage.Query{Prefix: s.prefix} + logrus.Infof("listing files in bucket %s with prefix %q", s.bucket, s.prefix) + it := s.client.Bucket(s.bucket).Objects(ctx, q) + for { + obj, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, fmt.Errorf( + "error listing objects in %q: %v", + s.filestore.Base, err) + } + name := obj.Name + if !strings.HasPrefix(name, s.prefix) { + return nil, fmt.Errorf( + "found object %q without prefix %q", + name, s.prefix) + } + + file := &syncFileInfo{} + file.AbsolutePath = object.GcsPrefix + s.bucket + "/" + obj.Name + file.RelativePath = strings.TrimPrefix(name, s.prefix) + if obj.MD5 == nil { + return nil, fmt.Errorf("MD5 not set on file %q", file.AbsolutePath) + } + + file.MD5 = hex.EncodeToString(obj.MD5) + file.Size = obj.Size + file.filestore = s + + files[file.RelativePath] = file + } + + return files, nil +} diff --git a/filepromoter/interfaces.go b/filepromoter/interfaces.go new file mode 100644 index 00000000..23d30a7f --- /dev/null +++ b/filepromoter/interfaces.go @@ -0,0 +1,24 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filepromoter + +import "context" + +// SyncFileOp defines a synchronization operation. +type SyncFileOp interface { + Run(ctx context.Context) error +} diff --git a/filepromoter/manifest.go b/filepromoter/manifest.go new file mode 100644 index 00000000..65b1a382 --- /dev/null +++ b/filepromoter/manifest.go @@ -0,0 +1,90 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filepromoter + +import ( + "context" + "fmt" + + "github.com/sirupsen/logrus" + + api "k8s.io/release/pkg/api/files" +) + +// ManifestPromoter promotes files as described in Manifest. +type ManifestPromoter struct { + Manifest *api.Manifest + + // UseServiceAccount must be true, for service accounts to be used + // This gives some protection against a hostile manifest. + UseServiceAccount bool +} + +// BuildOperations builds the required operations to sync from the +// Source Filestore to all Dest Filestores in the manifest. +func (p *ManifestPromoter) BuildOperations( + ctx context.Context) ([]SyncFileOp, error) { + source, err := getSourceFilestore(p.Manifest) + if err != nil { + return nil, err + } + + var operations []SyncFileOp + + for i := range p.Manifest.Filestores { + filestore := &p.Manifest.Filestores[i] + if filestore.Src { + continue + } + logrus.Infof("processing destination %q", filestore.Base) + fp := &FilestorePromoter{ + Source: source, + Dest: filestore, + Files: p.Manifest.Files, + UseServiceAccount: p.UseServiceAccount, + } + ops, err := fp.BuildOperations(ctx) + if err != nil { + return nil, fmt.Errorf( + "error building promotion operations for %q: %v", + filestore.Base, err) + } + operations = append(operations, ops...) + } + + return operations, nil +} + +// getSourceFilestore returns the Filestore with the source attribute +// It returns an error if the source filestore cannot be found, or if +// multiple filestores are marked as the source. +func getSourceFilestore(manifest *api.Manifest) (*api.Filestore, error) { + var source *api.Filestore + for i := range manifest.Filestores { + filestore := &manifest.Filestores[i] + if filestore.Src { + if source != nil { + return nil, fmt.Errorf("found multiple source filestores") + } + source = filestore + } + } + if source == nil { + return nil, fmt.Errorf("source filestore not found") + } + return source, nil +} diff --git a/filepromoter/token.go b/filepromoter/token.go new file mode 100644 index 00000000..ea3ac18d --- /dev/null +++ b/filepromoter/token.go @@ -0,0 +1,50 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filepromoter + +import ( + "sync" + + "github.com/sirupsen/logrus" + "golang.org/x/oauth2" + + "sigs.k8s.io/k8s-container-image-promoter/legacy/gcloud" +) + +// gcloudTokenSource implements oauth2.TokenSource. +type gcloudTokenSource struct { + mutex sync.Mutex + ServiceAccount string +} + +// Token implements TokenSource.Token. +func (s *gcloudTokenSource) Token() (*oauth2.Token, error) { + s.mutex.Lock() + defer s.mutex.Unlock() + + logrus.Infof("getting service-account-token for %q", s.ServiceAccount) + + token, err := gcloud.GetServiceAccountToken(s.ServiceAccount, true) + if err != nil { + logrus.Warnf("failed to get service-account-token for %q: %v", + s.ServiceAccount, err) + return nil, err + } + return &oauth2.Token{ + AccessToken: string(token), + }, nil +} diff --git a/promobot/hash.go b/promobot/hash.go new file mode 100644 index 00000000..b975daf2 --- /dev/null +++ b/promobot/hash.go @@ -0,0 +1,92 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package promobot + +import ( + "context" + "os" + "path/filepath" + "strings" + + "golang.org/x/xerrors" + + api "k8s.io/release/pkg/api/files" + "sigs.k8s.io/release-utils/hash" +) + +// GenerateManifestOptions holds the parameters for a hash-files operation. +type GenerateManifestOptions struct { + // BaseDir is the directory containing the files to hash + BaseDir string + + // Prefix exports only files matching the specified prefix. + // + // If we were instead to change BaseDir, we would also + // restrict the files, but the relative paths would also + // change. + Prefix string +} + +// PopulateDefaults sets the default values for GenerateManifestOptions. +func (o *GenerateManifestOptions) PopulateDefaults() { + // There are no fields with non-empty default values + // (but we still want to follow the PopulateDefaults pattern) +} + +// GenerateManifest generates a manifest containing the files in options.BaseDir +func GenerateManifest(ctx context.Context, options GenerateManifestOptions) (*api.Manifest, error) { + manifest := &api.Manifest{} + + if options.BaseDir == "" { + return nil, xerrors.New("must specify BaseDir") + } + + basedir := options.BaseDir + if !strings.HasSuffix(basedir, "/") { + basedir += "/" + } + + if err := filepath.Walk(basedir, func(p string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !strings.HasPrefix(p, basedir) { + return xerrors.Errorf("expected path %q to have prefix %q", p, basedir) + } + + if !strings.HasPrefix(p, filepath.Join(basedir, options.Prefix)) { + return nil + } + + if !info.IsDir() { + relativePath := strings.TrimPrefix(p, basedir) + sha256, err := hash.SHA256ForFile(p) + if err != nil { + return xerrors.Errorf("error hashing file %q: %w", p, err) + } + manifest.Files = append(manifest.Files, api.File{ + Name: relativePath, + SHA256: sha256, + }) + } + return nil + }); err != nil { + return nil, xerrors.Errorf("error walking path %q: %w", options.BaseDir, err) + } + + return manifest, nil +} diff --git a/promobot/hash_test.go b/promobot/hash_test.go new file mode 100644 index 00000000..612cca20 --- /dev/null +++ b/promobot/hash_test.go @@ -0,0 +1,76 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package promobot_test + +import ( + "context" + "os" + "testing" + + "k8s.io/release/pkg/promobot" + "k8s.io/utils/diff" + "sigs.k8s.io/yaml" +) + +func TestHash(t *testing.T) { + ctx := context.Background() + + var opt promobot.GenerateManifestOptions + opt.PopulateDefaults() + + opt.BaseDir = "testdata/files" + + manifest, err := promobot.GenerateManifest(ctx, opt) + if err != nil { + t.Fatalf("failed to generate manifest: %v", err) + } + + manifestYAML, err := yaml.Marshal(manifest) + if err != nil { + t.Fatalf("error serializing manifest: %v", err) + } + + AssertMatchesFile(t, string(manifestYAML), "testdata/files-manifest.yaml") +} + +// AssertMatchesFile verifies that the contents of p match actual. +// +// We break this out into a file because we also support the +// UPDATE_EXPECTED_OUTPUT magic env var. When that env var is +// set, we will write the actual output to the expected file, which +// is very handy when making bigger changes. The intention of these +// tests is to make the changes explicit, particularly in code +// review, not to force manual updates. +func AssertMatchesFile(t *testing.T, actual, p string) { + b, err := os.ReadFile(p) + if err != nil { + if os.Getenv("UPDATE_EXPECTED_OUTPUT") == "" { + t.Fatalf("error reading file %q: %v", p, err) + } + } + + expected := string(b) + + if actual != expected { + if os.Getenv("UPDATE_EXPECTED_OUTPUT") != "" { + if err := os.WriteFile(p, []byte(actual), 0o644); err != nil { + t.Fatalf("error writing file %q: %v", p, err) + } + } + t.Errorf("actual did not match expected; diff=%s", diff.StringDiff(actual, expected)) + } +} diff --git a/promobot/promotefiles.go b/promobot/promotefiles.go new file mode 100644 index 00000000..fe71554c --- /dev/null +++ b/promobot/promotefiles.go @@ -0,0 +1,222 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package promobot + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "sort" + + "github.com/sirupsen/logrus" + "golang.org/x/xerrors" + + api "k8s.io/release/pkg/api/files" + "k8s.io/release/pkg/filepromoter" +) + +// PromoteFilesOptions holds the flag-values for a file promotion +type PromoteFilesOptions struct { + // FilestoresPath is the path to the manifest file containing the filestores section + FilestoresPath string + + // FilesPath specifies a path to manifest files containing the files section. + FilesPath string + + // DryRun (if set) will not perform operations, but print them instead + DryRun bool + + // UseServiceAccount must be true, for service accounts to be used + // This gives some protection against a hostile manifest. + UseServiceAccount bool + + // Out is the destination for "normal" output (such as dry-run) + Out io.Writer +} + +// PopulateDefaults sets the default values for PromoteFilesOptions +func (o *PromoteFilesOptions) PopulateDefaults() { + o.DryRun = true + o.UseServiceAccount = false + o.Out = os.Stdout +} + +// RunPromoteFiles executes a file promotion command +func RunPromoteFiles(ctx context.Context, options PromoteFilesOptions) error { + manifest, err := ReadManifest(options) + if err != nil { + return err + } + + if options.DryRun { + fmt.Fprintf( + options.Out, + "********** START (DRY RUN) **********\n") + } else { + fmt.Fprintf( + options.Out, + "********** START **********\n") + } + + promoter := &filepromoter.ManifestPromoter{ + Manifest: manifest, + UseServiceAccount: options.UseServiceAccount, + } + + ops, err := promoter.BuildOperations(ctx) + if err != nil { + return fmt.Errorf( + "error building operations: %v", + err) + } + + // So that we can support future parallel execution, an error + // in one operation does not prevent us attempting the + // remaining operations + var errors []error + for _, op := range ops { + if _, err := fmt.Fprintf(options.Out, "%v\n", op); err != nil { + errors = append(errors, fmt.Errorf( + "error writing to output: %v", err)) + } + + if !options.DryRun { + if err := op.Run(ctx); err != nil { + logrus.Warnf("error copying file: %v", err) + errors = append(errors, err) + } + } + } + + if len(errors) != 0 { + fmt.Fprintf( + options.Out, + "********** FINISHED WITH ERRORS **********\n") + for _, err := range errors { + fmt.Fprintf(options.Out, "%v\n", err) + } + + return errors[0] + } + + if options.DryRun { + fmt.Fprintf( + options.Out, + "********** FINISHED (DRY RUN) **********\n") + } else { + fmt.Fprintf( + options.Out, + "********** FINISHED **********\n") + } + + return nil +} + +// ReadManifest reads a manifest. +func ReadManifest(options PromoteFilesOptions) (*api.Manifest, error) { + merged := &api.Manifest{} + + filestores, err := readFilestores(options.FilestoresPath) + if err != nil { + return nil, err + } + merged.Filestores = filestores + + files, err := readFiles(options.FilesPath) + if err != nil { + return nil, err + } + merged.Files = files + + // Validate the merged manifest + if err := merged.Validate(); err != nil { + return nil, fmt.Errorf("error validating merged manifest: %v", err) + } + + return merged, nil +} + +// readFilestores reads a filestores manifest +func readFilestores(p string) ([]api.Filestore, error) { + if p == "" { + return nil, fmt.Errorf("FilestoresPath is required") + } + + b, err := os.ReadFile(p) + if err != nil { + return nil, fmt.Errorf("error reading manifest %q: %v", p, err) + } + + manifest, err := api.ParseManifest(b) + if err != nil { + return nil, fmt.Errorf("error parsing manifest %q: %v", p, err) + } + + if len(manifest.Files) != 0 { + return nil, xerrors.Errorf( + "files should not be present in filestore manifest %q", + p) + } + + return manifest.Filestores, nil +} + +// readFiles reads and merges the file manifests from the file or directory filesPath +func readFiles(filesPath string) ([]api.File, error) { + // We first list and sort the paths, for a consistent ordering + var paths []string + err := filepath.Walk(filesPath, func(p string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + paths = append(paths, p) + return nil + }) + if err != nil { + return nil, xerrors.Errorf("error listing file manifests: %w", err) + } + + sort.Strings(paths) + + var files []api.File + for _, p := range paths { + b, err := os.ReadFile(p) + if err != nil { + return nil, xerrors.Errorf("error reading file %q: %w", p, err) + } + + manifest, err := api.ParseManifest(b) + if err != nil { + return nil, xerrors.Errorf("error parsing manifest %q: %v", p, err) + } + + if len(manifest.Filestores) != 0 { + return nil, xerrors.Errorf("filestores should not be present in manifest %q", p) + } + + files = append(files, manifest.Files...) + } + + return files, nil +} diff --git a/promobot/readmanifest_test.go b/promobot/readmanifest_test.go new file mode 100644 index 00000000..c9ed0ef3 --- /dev/null +++ b/promobot/readmanifest_test.go @@ -0,0 +1,63 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package promobot_test + +import ( + "testing" + + "k8s.io/release/pkg/promobot" + "sigs.k8s.io/yaml" +) + +func TestReadManifests(t *testing.T) { + grid := []struct { + Expected string + Options promobot.PromoteFilesOptions + }{ + { + Expected: "testdata/manifests/onefiles/expected.yaml", + Options: promobot.PromoteFilesOptions{ + FilestoresPath: "testdata/manifests/onefiles/filestores.yaml", + FilesPath: "testdata/manifests/onefiles/files.yaml", + }, + }, + { + Expected: "testdata/manifests/manyfiles/expected.yaml", + Options: promobot.PromoteFilesOptions{ + FilestoresPath: "testdata/manifests/manyfiles/filestores.yaml", + FilesPath: "testdata/manifests/manyfiles/files/", + }, + }, + } + + for _, g := range grid { + g := g // avoid closure go-tcha + t.Run(g.Expected, func(t *testing.T) { + manifest, err := promobot.ReadManifest(g.Options) + if err != nil { + t.Fatalf("failed to read manifest: %v", err) + } + + manifestYAML, err := yaml.Marshal(manifest) + if err != nil { + t.Fatalf("error serializing manifest: %v", err) + } + + AssertMatchesFile(t, string(manifestYAML), g.Expected) + }) + } +} diff --git a/promobot/testdata/files-manifest.yaml b/promobot/testdata/files-manifest.yaml new file mode 100644 index 00000000..a8b0c7c2 --- /dev/null +++ b/promobot/testdata/files-manifest.yaml @@ -0,0 +1,7 @@ +files: +- name: blue.png + sha256: 905fef7b0658ff5d266140d1cea1eb5b414393b4d0c7897b05beae78678395c3 +- name: green.png + sha256: 7e24ef9e8ed9454980182e787fc61dca44014571be346f3a5b341ce6c028e45d +- name: red.png + sha256: 5e6893c6c9ae8bf2a40b22b4274ca58d68c5614b476451a29859750bf434d6a8 diff --git a/promobot/testdata/files/blue.png b/promobot/testdata/files/blue.png new file mode 100644 index 00000000..62da4aa8 Binary files /dev/null and b/promobot/testdata/files/blue.png differ diff --git a/promobot/testdata/files/green.png b/promobot/testdata/files/green.png new file mode 100644 index 00000000..fc416df1 Binary files /dev/null and b/promobot/testdata/files/green.png differ diff --git a/promobot/testdata/files/red.png b/promobot/testdata/files/red.png new file mode 100644 index 00000000..42b3c515 Binary files /dev/null and b/promobot/testdata/files/red.png differ diff --git a/promobot/testdata/manifests/manyfiles/expected.yaml b/promobot/testdata/manifests/manyfiles/expected.yaml new file mode 100644 index 00000000..facf84ee --- /dev/null +++ b/promobot/testdata/manifests/manyfiles/expected.yaml @@ -0,0 +1,11 @@ +files: +- name: blue.png + sha256: 905fef7b0658ff5d266140d1cea1eb5b414393b4d0c7897b05beae78678395c3 +- name: green.png + sha256: 7e24ef9e8ed9454980182e787fc61dca44014571be346f3a5b341ce6c028e45d +- name: red.png + sha256: 5e6893c6c9ae8bf2a40b22b4274ca58d68c5614b476451a29859750bf434d6a8 +filestores: +- base: gs://src + src: true +- base: gs://dest diff --git a/promobot/testdata/manifests/manyfiles/files/blue.yaml b/promobot/testdata/manifests/manyfiles/files/blue.yaml new file mode 100644 index 00000000..fd9137f0 --- /dev/null +++ b/promobot/testdata/manifests/manyfiles/files/blue.yaml @@ -0,0 +1,4 @@ +files: +- name: blue.png + sha256: 905fef7b0658ff5d266140d1cea1eb5b414393b4d0c7897b05beae78678395c3 + diff --git a/promobot/testdata/manifests/manyfiles/files/green.yaml b/promobot/testdata/manifests/manyfiles/files/green.yaml new file mode 100644 index 00000000..533ba8b5 --- /dev/null +++ b/promobot/testdata/manifests/manyfiles/files/green.yaml @@ -0,0 +1,4 @@ +files: +- name: green.png + sha256: 7e24ef9e8ed9454980182e787fc61dca44014571be346f3a5b341ce6c028e45d + diff --git a/promobot/testdata/manifests/manyfiles/files/red.yaml b/promobot/testdata/manifests/manyfiles/files/red.yaml new file mode 100644 index 00000000..ffbf318a --- /dev/null +++ b/promobot/testdata/manifests/manyfiles/files/red.yaml @@ -0,0 +1,4 @@ +files: +- name: red.png + sha256: 5e6893c6c9ae8bf2a40b22b4274ca58d68c5614b476451a29859750bf434d6a8 + diff --git a/promobot/testdata/manifests/manyfiles/filestores.yaml b/promobot/testdata/manifests/manyfiles/filestores.yaml new file mode 100644 index 00000000..862cf909 --- /dev/null +++ b/promobot/testdata/manifests/manyfiles/filestores.yaml @@ -0,0 +1,4 @@ +filestores: +- base: gs://src + src: true +- base: gs://dest diff --git a/promobot/testdata/manifests/onefiles/expected.yaml b/promobot/testdata/manifests/onefiles/expected.yaml new file mode 100644 index 00000000..facf84ee --- /dev/null +++ b/promobot/testdata/manifests/onefiles/expected.yaml @@ -0,0 +1,11 @@ +files: +- name: blue.png + sha256: 905fef7b0658ff5d266140d1cea1eb5b414393b4d0c7897b05beae78678395c3 +- name: green.png + sha256: 7e24ef9e8ed9454980182e787fc61dca44014571be346f3a5b341ce6c028e45d +- name: red.png + sha256: 5e6893c6c9ae8bf2a40b22b4274ca58d68c5614b476451a29859750bf434d6a8 +filestores: +- base: gs://src + src: true +- base: gs://dest diff --git a/promobot/testdata/manifests/onefiles/files.yaml b/promobot/testdata/manifests/onefiles/files.yaml new file mode 100644 index 00000000..a8b0c7c2 --- /dev/null +++ b/promobot/testdata/manifests/onefiles/files.yaml @@ -0,0 +1,7 @@ +files: +- name: blue.png + sha256: 905fef7b0658ff5d266140d1cea1eb5b414393b4d0c7897b05beae78678395c3 +- name: green.png + sha256: 7e24ef9e8ed9454980182e787fc61dca44014571be346f3a5b341ce6c028e45d +- name: red.png + sha256: 5e6893c6c9ae8bf2a40b22b4274ca58d68c5614b476451a29859750bf434d6a8 diff --git a/promobot/testdata/manifests/onefiles/filestores.yaml b/promobot/testdata/manifests/onefiles/filestores.yaml new file mode 100644 index 00000000..862cf909 --- /dev/null +++ b/promobot/testdata/manifests/onefiles/filestores.yaml @@ -0,0 +1,4 @@ +filestores: +- base: gs://src + src: true +- base: gs://dest