Migrate kpromo/cip-mm tools/packages from k/release at 62b128c2

Signed-off-by: Stephen Augustus <foo@auggie.dev>

api/files/manifest.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package files
+
+import (
+	"fmt"
+
+	"sigs.k8s.io/yaml"
+)
+
+// Filestore holds information about a filestore (e.g. GCS or S3 bucket),
+// to be written in a manifest file.
+type Filestore struct {
+	// Base is the leading part of an artifact path, including the scheme.
+	// It is everything that is not the actual file name itself.
+	// e.g. "gs://prod-artifacts/myproject"
+	Base           string `json:"base,omitempty"`
+	ServiceAccount string `json:"service-account,omitempty"`
+	Src            bool   `json:"src,omitempty"`
+}
+
+// File holds information about a file artifact. File artifacts are copied from
+// a source Filestore to N destination Filestores.
+type File struct {
+	// Name is the relative path of the file, relative to the Filestore base
+	Name string `json:"name"`
+	// SHA256 holds the SHA256 hash of the specified file (hex encoded)
+	SHA256 string `json:"sha256,omitempty"`
+}
+
+// Manifest stores the information in a manifest file (describing the
+// desired state of a Docker Registry).
+type Manifest struct {
+	// Filestores contains the source and destination (Src/Dest) filestores.
+	// Filestores are (for example) GCS or S3 buckets.
+	// It is possible that in the future, we support promoting to multiple
+	// filestores, in which case we would have more than just Src/Dest.
+	Filestores []Filestore `json:"filestores,omitempty"`
+	Files      []File      `json:"files,omitempty"`
+}
+
+// ParseManifest parses a Manifest.
+func ParseManifest(b []byte) (*Manifest, error) {
+	m := &Manifest{}
+	if err := yaml.Unmarshal(b, m); err != nil {
+		return nil, fmt.Errorf("error parsing manifest: %v", err)
+	}
+	return m, nil
+}

api/files/manifest_test.go

@@ -0,0 +1,155 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package files_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"k8s.io/release/pkg/api/files"
+)
+
+func TestValidateFilestores(t *testing.T) {
+	tests := []struct {
+		filestores    []files.Filestore
+		expectedError string
+	}{
+		{
+			// Filestores are required
+			filestores:    []files.Filestore{},
+			expectedError: "filestore must be specified",
+		},
+		{
+			// Filestores are required
+			filestores:    nil,
+			expectedError: "filestore must be specified",
+		},
+		{
+			filestores: []files.Filestore{
+				{Src: true, Base: "gs://src"},
+			},
+			expectedError: "no destination filestores found",
+		},
+		{
+			filestores: []files.Filestore{
+				{Base: "gs://dest1"},
+			},
+			expectedError: "source filestore not found",
+		},
+		{
+			filestores: []files.Filestore{
+				{Src: true, Base: "gs://src1"},
+				{Src: true, Base: "gs://src2"},
+			},
+			expectedError: "found multiple source filestores",
+		},
+		{
+			filestores: []files.Filestore{
+				{Src: true, Base: "gs://src"},
+				{Base: "gs://dest1"},
+				{Base: "gs://dest2"},
+			},
+		},
+		{
+			filestores: []files.Filestore{
+				{Src: true},
+				{Base: "gs://dest"},
+			},
+			expectedError: "filestore did not have base set",
+		},
+		{
+			filestores: []files.Filestore{
+				{Src: true, Base: "gs://src"},
+				{Base: "s3://dest"},
+			},
+			expectedError: "unsupported scheme in base",
+		},
+	}
+	for _, test := range tests {
+		err := files.ValidateFilestores(test.filestores)
+		checkErrorMatchesExpected(t, err, test.expectedError)
+	}
+}
+
+func TestValidateFiles(t *testing.T) {
+	oksha := "4f2f040fa2bfe9bea64911a2a756e8a1727a8bfd757c5e031631a6e699fcf246"
+
+	tests := []struct {
+		files         []files.File
+		expectedError string
+	}{
+		{
+			// Files are required
+			files:         []files.File{},
+			expectedError: "file must be specified",
+		},
+		{
+			// Files are required
+			files:         nil,
+			expectedError: "file must be specified",
+		},
+		{
+			files: []files.File{
+				{Name: "foo", SHA256: oksha},
+			},
+		},
+		{
+			files: []files.File{
+				{SHA256: oksha},
+			},
+			expectedError: "name is required for file",
+		},
+		{
+			files: []files.File{
+				{Name: "foo", SHA256: "bad"},
+			},
+			expectedError: "sha256 was not valid (not hex)",
+		},
+		{
+			files: []files.File{
+				{Name: "foo"},
+			},
+			expectedError: "sha256 is required",
+		},
+		{
+			files: []files.File{
+				{Name: "foo", SHA256: "abcd"},
+			},
+			expectedError: "sha256 was not valid (bad length)",
+		},
+	}
+	for _, test := range tests {
+		err := files.ValidateFiles(test.files)
+		checkErrorMatchesExpected(t, err, test.expectedError)
+	}
+}
+
+func checkErrorMatchesExpected(t *testing.T, err error, expected string) {
+	if err != nil && expected == "" {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if err != nil && expected != "" {
+		actual := fmt.Sprintf("%v", err)
+		if !strings.Contains(actual, expected) {
+			t.Errorf("error %q did not contain expected %q", err, expected)
+		}
+	}
+	if err == nil && expected != "" {
+		t.Errorf("expected error %q", expected)
+	}
+}

api/files/validation.go

@@ -0,0 +1,110 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package files
+
+import (
+	"encoding/hex"
+	"fmt"
+	"strings"
+
+	"sigs.k8s.io/release-sdk/object"
+)
+
+// Validate checks for semantic errors in the yaml fields (the structure of the
+// yaml is checked during unmarshaling).
+func (m *Manifest) Validate() error {
+	if err := ValidateFilestores(m.Filestores); err != nil {
+		return err
+	}
+	if err := ValidateFiles(m.Files); err != nil {
+		return err
+	}
+	return nil
+}
+
+// ValidateFilestores validates the Filestores field of the manifest.
+func ValidateFilestores(filestores []Filestore) error {
+	if len(filestores) == 0 {
+		return fmt.Errorf("at least one filestore must be specified")
+	}
+
+	var source *Filestore
+	destinationCount := 0
+
+	for i := range filestores {
+		filestore := &filestores[i]
+
+		if filestore.Base == "" {
+			return fmt.Errorf("filestore did not have base set")
+		}
+
+		// Currently the only backend supported is GCS
+		if !strings.HasPrefix(filestore.Base, object.GcsPrefix) {
+			return fmt.Errorf(
+				"filestore has unsupported scheme in base %q",
+				filestore.Base)
+		}
+
+		if filestore.Src {
+			if source != nil {
+				return fmt.Errorf("found multiple source filestores")
+			}
+			source = filestore
+		} else {
+			destinationCount++
+		}
+	}
+	if source == nil {
+		return fmt.Errorf("source filestore not found")
+	}
+
+	if destinationCount == 0 {
+		return fmt.Errorf("no destination filestores found")
+	}
+
+	return nil
+}
+
+// ValidateFiles validates the Files field of the manifest.
+func ValidateFiles(files []File) error {
+	if len(files) == 0 {
+		return fmt.Errorf("at least one file must be specified")
+	}
+
+	for i := range files {
+		f := &files[i]
+
+		if f.Name == "" {
+			return fmt.Errorf("name is required for file")
+		}
+
+		if f.SHA256 == "" {
+			return fmt.Errorf("sha256 is required for file")
+		}
+
+		sha256, err := hex.DecodeString(f.SHA256)
+		if err != nil {
+			return fmt.Errorf("sha256 was not valid (not hex): %q", f.SHA256)
+		}
+
+		if len(sha256) != 32 {
+			return fmt.Errorf("sha256 was not valid (bad length): %q", f.SHA256)
+		}
+	}
+
+	return nil
+}

cmd/cip-mm/README.md

@@ -0,0 +1,40 @@
+# cip-mm
+
+This tool **m**odifies promoter **m**anifests. For now it dumps some filtered
+subset of a staging GCR and merges those contents back into a given promoter
+manifest.
+
+## Examples
+
+- Add all images with a matching digest from staging repo
+  `gcr.io/k8s-staging-artifact-promoter` to a manifest, using the name and tags
+  already existing in the staging repo:
+
+```
+cip-mm \
+  --base_dir=$HOME/go/src/github.com/kubernetes/k8s.io/k8s.gcr.io  \
+  --staging_repo=gcr.io/k8s-staging-artifact-promoter \
+  --filter_digest=sha256:7594278deaf6eeaa35caedec81796d103e3c83a26d7beab091a5d25a9ba6aa16
+```
+
+- Add a single image named "foo" and tagged "1.0" from staging repo
+  `gcr.io/k8s-staging-artifact-promoter` to a manifest:
+
+```
+cip-mm \
+  --base_dir=$HOME/go/src/github.com/kubernetes/k8s.io/k8s.gcr.io  \
+  --staging_repo=gcr.io/k8s-staging-artifact-promoter \
+  --filter_image=cip \
+  --filter_tag=1.0
+```
+
+- Add all images tagged `1.0` from staging repo
+  `gcr.io/k8s-staging-artifact-promoter` to a manifest:
+
+```
+cip-mm \
+  --base_dir=$HOME/go/src/github.com/kubernetes/k8s.io/k8s.gcr.io  \
+  --staging_repo=gcr.io/k8s-staging-artifact-promoter \
+  --filter_image=cip \
+  --filter_tag=1.0
+```