diff --git a/.github/workflows/gh-test.yml b/.github/workflows/gh-test.yml index f73710f65..7d247421c 100644 --- a/.github/workflows/gh-test.yml +++ b/.github/workflows/gh-test.yml @@ -8,7 +8,7 @@ on: types: [opened, reopened, synchronize] jobs: test-all: - name: Test GH + name: Test GH - Linux runs-on: ubuntu-latest steps: - name: Set up Go 1.x @@ -28,4 +28,36 @@ jobs: cd src/github.com/${{ github.repository }} # deploy local registry and run tests - ./hack/test-all-local-registry.sh \ No newline at end of file + ./hack/test-all-local-registry.sh + + test-all-windows: + name: Test GH - Windows + runs-on: windows-latest + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v1 + with: + go-version: "1.17" + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + with: + path: src/github.com/${{ github.repository }} + - name: Run Tests + run: | + bash -c ' + set -e -x + + export GOPATH=$(echo `pwd`) + export PATH="$GOPATH/bin:$PATH" + cd src/github.com/${{ github.repository }} + + export IMGPKG_E2E_IMAGE="ttl.sh/github-action-test-relocation-$GITHUB_RUN_ID" + export IMGPKG_E2E_RELOCATION_REPO="ttl.sh/github-action-imgpkg-test-$GITHUB_RUN_ID" + + go install github.com/sigstore/cosign/cmd/cosign@v0.5.0 + + alias cosign=cosign.exe + + export IMGPKG_BINARY_EXT=".exe" + ./hack/test-all.sh + ' \ No newline at end of file diff --git a/hack/build-binaries.sh b/hack/build-binaries.sh index 22c93901a..8c74af537 100755 --- a/hack/build-binaries.sh +++ b/hack/build-binaries.sh @@ -16,7 +16,7 @@ go mod tidy # there doesn't appear to be a simple way to disable the defaultDockerConfigProvider # Having defaultDockerConfigProvider enabled by default results in the imgpkg auth ordering not working correctly # Specifically, the docker config.json is loaded before cli flags (and maybe even IaaS metadata services) -git apply ./hack/patch-k8s-pkg-credentialprovider.patch +git apply --ignore-space-change --ignore-whitespace ./hack/patch-k8s-pkg-credentialprovider.patch # makes builds reproducible export CGO_ENABLED=0 diff --git a/hack/build.sh b/hack/build.sh index 7736aee18..f1f25f1e9 100755 --- a/hack/build.sh +++ b/hack/build.sh @@ -14,10 +14,10 @@ go mod tidy # there doesn't appear to be a simple way to disable the defaultDockerConfigProvider # Having defaultDockerConfigProvider enabled by default results in the imgpkg auth ordering not working correctly # Specifically, the docker config.json is loaded before cli flags (and maybe even IaaS metadata services) -git apply ./hack/patch-k8s-pkg-credentialprovider.patch +git apply --ignore-space-change --ignore-whitespace ./hack/patch-k8s-pkg-credentialprovider.patch # export GOOS=linux GOARCH=amd64 -go build -ldflags="$LDFLAGS" -trimpath -o imgpkg ./cmd/imgpkg/... +go build -ldflags="$LDFLAGS" -trimpath -o "imgpkg${IMGPKG_BINARY_EXT-}" ./cmd/imgpkg/... ./imgpkg version # compile tests, but do not run them: https://github.com/golang/go/issues/15513#issuecomment-839126426 diff --git a/hack/test-all.sh b/hack/test-all.sh index 9d24aacd6..e0bd8b51f 100755 --- a/hack/test-all.sh +++ b/hack/test-all.sh @@ -4,7 +4,7 @@ set -e -x -u ./hack/build.sh -export IMGPKG_BINARY="$PWD/imgpkg" +export IMGPKG_BINARY="$PWD/imgpkg${IMGPKG_BINARY_EXT-}" ./hack/test.sh ./hack/test-e2e.sh diff --git a/pkg/imgpkg/cmd/copy_repo_src_test.go b/pkg/imgpkg/cmd/copy_repo_src_test.go index 8d5dab696..db9de19b5 100644 --- a/pkg/imgpkg/cmd/copy_repo_src_test.go +++ b/pkg/imgpkg/cmd/copy_repo_src_test.go @@ -103,15 +103,15 @@ bundle: subject.BundleFlags.Bundle = fakeRegistry.ReferenceOnTestServer( bundleWithNested.BundleName + "@" + bundleWithNested.Digest) - tarDir := assets.CreateTempFolder("tar-copy") - imageTarPath := filepath.Join(tarDir, "bundle.tar") + bundleTarPath := filepath.Join(os.TempDir(), "bundle.tar") + defer os.Remove(bundleTarPath) - err = subject.CopyToTar(imageTarPath) + err = subject.CopyToTar(bundleTarPath) require.NoError(t, err) - assertTarballContainsOnlyDistributableLayers(imageTarPath, t) + assertTarballContainsOnlyDistributableLayers(bundleTarPath, t) - assertTarballLabelsOuterBundle(imageTarPath, subject.BundleFlags.Bundle, t) + assertTarballLabelsOuterBundle(bundleTarPath, subject.BundleFlags.Bundle, t) }) } @@ -203,8 +203,8 @@ images: subject.BundleFlags.Bundle = fakeRegistry.ReferenceOnTestServer(bundleWithNested.BundleName + "@" + bundleWithNested.Digest) - tarDir := assets.CreateTempFolder("tar-copy") - imageTarPath := filepath.Join(tarDir, "bundle.tar") + imageTarPath := filepath.Join(os.TempDir(), "bundle.tar") + defer os.Remove(imageTarPath) err := subject.CopyToTar(imageTarPath) require.NoError(t, err) diff --git a/pkg/imgpkg/cmd/push_test.go b/pkg/imgpkg/cmd/push_test.go index 2025e5ac6..bf0a571de 100644 --- a/pkg/imgpkg/cmd/push_test.go +++ b/pkg/imgpkg/cmd/push_test.go @@ -4,6 +4,7 @@ package cmd import ( + "fmt" "io/ioutil" "os" "path/filepath" @@ -56,7 +57,7 @@ func TestMultiImgpkgDirError(t *testing.T) { t.Fatalf("Expected validations to err, but did not") } - if !strings.Contains(err.Error(), "This directory contains multiple bundle definitions. Only a single instance of .imgpkg/images.yml can be provided") { + if !strings.Contains(err.Error(), fmt.Sprintf("This directory contains multiple bundle definitions. Only a single instance of .imgpkg%simages.yml can be provided", string(os.PathSeparator))) { t.Fatalf("Expected error to contain message about multiple .imgpkg dirs, got: %s", err) } } @@ -133,7 +134,7 @@ func TestBundleDirectoryErrors(t *testing.T) { { name: "no bundle", createBundleDir: false, - expectedError: "This directory is not a bundle. It is missing .imgpkg/images.yml", + expectedError: fmt.Sprintf("This directory is not a bundle. It is missing .imgpkg%simages.yml", string(os.PathSeparator)), }, } diff --git a/pkg/imgpkg/imagetar/tar_reader.go b/pkg/imgpkg/imagetar/tar_reader.go index b22f712c7..d5d30f76b 100644 --- a/pkg/imgpkg/imagetar/tar_reader.go +++ b/pkg/imgpkg/imagetar/tar_reader.go @@ -33,6 +33,7 @@ func (r TarReader) getIdsFromManifest(file tarFile) (*imagedesc.ImageRefDescript if err != nil { return nil, err } + defer manifestFile.Close() manifestBytes, err := ioutil.ReadAll(manifestFile) if err != nil { diff --git a/pkg/imgpkg/imagetar/tar_writer.go b/pkg/imgpkg/imagetar/tar_writer.go index 092555521..02804e761 100644 --- a/pkg/imgpkg/imagetar/tar_writer.go +++ b/pkg/imgpkg/imagetar/tar_writer.go @@ -49,7 +49,6 @@ func (w *TarWriter) Write() error { if err != nil { return err } - defer w.dst.Close() w.tf = tar.NewWriter(w.dst) diff --git a/pkg/imgpkg/registry/registry.go b/pkg/imgpkg/registry/registry.go index a56e08b53..31d5a91f8 100644 --- a/pkg/imgpkg/registry/registry.go +++ b/pkg/imgpkg/registry/registry.go @@ -11,6 +11,7 @@ import ( "net/http" "os" "regexp" + "runtime" "time" regname "github.com/google/go-containerregistry/pkg/name" @@ -250,8 +251,20 @@ func (r Registry) FirstImageExists(digests []string) (string, error) { } func newHTTPTransport(opts Opts) (*http.Transport, error) { - pool, err := x509.SystemCertPool() - if err != nil { + var pool *x509.CertPool + + // workaround for windows not returning system certs via x509.SystemCertPool() See: https://github.com/golang/go/issues/16736 + // instead windows lazily fetches ca certificates (over the network) as needed during cert verification time. + // to opt-into that tls.Config.RootCAs is set to nil on windows. + if runtime.GOOS != "windows" { + var err error + pool, err = x509.SystemCertPool() + if err != nil { + return nil, err + } + } + + if runtime.GOOS == "windows" && len(opts.CACertPaths) > 0 { pool = x509.NewCertPool() } diff --git a/test/e2e/copy_from_image_test.go b/test/e2e/copy_from_image_test.go index 39776aaaa..452b1a9c2 100644 --- a/test/e2e/copy_from_image_test.go +++ b/test/e2e/copy_from_image_test.go @@ -5,10 +5,8 @@ package e2e import ( "bytes" - "context" "fmt" "os" - "os/exec" "path/filepath" "strings" "testing" @@ -18,7 +16,6 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/types" - "github.com/k14s/imgpkg/pkg/imgpkg/util" "github.com/k14s/imgpkg/test/helpers" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -99,7 +96,7 @@ func TestCopyAnImageFromATarToARepoThatDoesNotContainNonDistributableLayersButTh t.Run("airgapped environment", func(t *testing.T) { env := helpers.BuildEnv(t) - airgappedRepo := startRegistryForAirgapTesting(t, env) + airgappedRepo, fakeRegistry := startRegistryForAirgapTesting(t, env) imgpkg := helpers.Imgpkg{T: t, L: helpers.Logger{}, ImgpkgPath: env.ImgpkgPath} @@ -116,7 +113,7 @@ func TestCopyAnImageFromATarToARepoThatDoesNotContainNonDistributableLayersButTh // copy to tar skipping NDL imgpkg.Run([]string{"copy", "-i", airgappedRepo, "--to-tar", tarFilePath}) - stopRegistryForAirgapTesting(t, env) + stopRegistryForAirgapTesting(fakeRegistry) _, err := imgpkg.RunWithOpts([]string{"copy", "--tar", tarFilePath, "--to-repo", repoToCopyName, "--include-non-distributable-layers"}, helpers.RunOpts{ AllowError: true, @@ -208,15 +205,20 @@ func TestCopyRepoToTarAndThenCopyFromTarToRepo(t *testing.T) { for _, mediaType := range []types.MediaType{types.OCIUncompressedRestrictedLayer, types.DockerForeignLayer} { t.Run(fmt.Sprintf("Without --include-non-distributable-layers flag and image contains a non-distributable layer should only copy distributable layers and print a warning message (Using MediaType %s)", mediaType), func(t *testing.T) { env := helpers.BuildEnv(t) - if mediaType == types.OCIUncompressedRestrictedLayer && strings.HasPrefix(env.RelocationRepo, "gcr.io") { - t.Skip("Skipping this test due gcr.io limitation.") + + if strings.HasPrefix(env.RelocationRepo, "index.docker.io") { + t.Skip("Skipping this test due index.docker.io limitation. See https://github.com/docker/hub-feedback/issues/2132") + } + + if mediaType == types.DockerForeignLayer && strings.HasPrefix(env.RelocationRepo, "ttl.sh") { + t.Skip("Skipping this test due ttl.sh limitation.") } - if mediaType == types.DockerForeignLayer && strings.HasPrefix(env.RelocationRepo, "index.docker.io") { - t.Skip("Skipping this test due dockerhub limitation.") + if mediaType == types.OCIUncompressedRestrictedLayer && strings.HasPrefix(env.RelocationRepo, "gcr.io") { + t.Skip("Skipping this test due gcr.io limitation.") } - airgappedRepo := startRegistryForAirgapTesting(t, env) + airgappedRepo, fakeRegistry := startRegistryForAirgapTesting(t, env) imgpkg := helpers.Imgpkg{T: t, L: helpers.Logger{}, ImgpkgPath: env.ImgpkgPath} defer env.Cleanup() @@ -235,7 +237,7 @@ func TestCopyRepoToTarAndThenCopyFromTarToRepo(t *testing.T) { imgpkg.Run([]string{"copy", "-i", airgappedRepo, "--to-tar", tarFilePath, "--include-non-distributable-layers"}) }) - stopRegistryForAirgapTesting(t, env) + stopRegistryForAirgapTesting(fakeRegistry) var stdOutWriter bytes.Buffer imgpkg.RunWithOpts([]string{"copy", "--tar", tarFilePath, "--to-repo", repoToCopyName}, helpers.RunOpts{ @@ -398,40 +400,16 @@ func TestCopyErrors(t *testing.T) { }) } -func stopRegistryForAirgapTesting(t *testing.T, env *helpers.Env) { - err := exec.Command("docker", "stop", "registry-for-airgapped-testing").Run() - require.NoError(t, err) - - env.AddCleanup(func() { - exec.Command("docker", "start", "registry-for-airgapped-testing").Run() - }) +func stopRegistryForAirgapTesting(fakeRegistry *helpers.FakeTestRegistryBuilder) { + fakeRegistry.CleanUp() } -func startRegistryForAirgapTesting(t *testing.T, env *helpers.Env) string { - dockerRunCmd := exec.Command("docker", "run", "-d", "-p", "5000", "--env", "REGISTRY_VALIDATION_MANIFESTS_URLS_ALLOW=- ^https?://", "--restart", "always", "--name", "registry-for-airgapped-testing", "registry:2") - _, err := dockerRunCmd.CombinedOutput() - require.NoError(t, err) +func startRegistryForAirgapTesting(t *testing.T, env *helpers.Env) (string, *helpers.FakeTestRegistryBuilder) { + fakeRegistry := helpers.NewFakeRegistry(t, env.Logger) env.AddCleanup(func() { - exec.Command("docker", "stop", "registry-for-airgapped-testing").Run() - exec.Command("docker", "rm", "registry-for-airgapped-testing").Run() - }) - - inspectCmd := exec.Command("docker", "inspect", `--format='{{(index (index .NetworkSettings.Ports "5000/tcp") 0).HostPort}}'`, "registry-for-airgapped-testing") - output, err := inspectCmd.CombinedOutput() - require.NoError(t, err) - - hostPort := strings.ReplaceAll(string(output), "'", "") - - registryURL := fmt.Sprintf("localhost:%s", strings.ReplaceAll(hostPort, "\n", "")) - registry, err := name.NewRegistry(registryURL) - require.NoError(t, err) - - err = util.Retry(func() error { - _, err = remote.Catalog(context.Background(), registry) - return err + fakeRegistry.CleanUp() }) - require.NoError(t, err) - return fmt.Sprintf("%s/repo/airgapped-image", registryURL) + return fakeRegistry.ReferenceOnTestServer("repo/airgapped-image"), fakeRegistry } diff --git a/test/e2e/deterministic_push_test.go b/test/e2e/deterministic_push_test.go index 727b65f76..b05caeb04 100644 --- a/test/e2e/deterministic_push_test.go +++ b/test/e2e/deterministic_push_test.go @@ -4,6 +4,7 @@ package e2e import ( + "runtime" "testing" "github.com/k14s/imgpkg/test/helpers" @@ -11,6 +12,10 @@ import ( ) func TestDeterministicPush(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Image pushed on windows results in a different sha due to backslashes used on filesystem. Skipping for now until fixed.") + } + env := helpers.BuildEnv(t) imgpkg := helpers.Imgpkg{T: t, L: helpers.Logger{}, ImgpkgPath: env.ImgpkgPath} defer env.Cleanup() @@ -21,7 +26,7 @@ func TestDeterministicPush(t *testing.T) { tag1Digest := helpers.ExtractDigest(t, out) // This expected digest should be the same regardless which OS imgpkg runs on - require.Equal(t, tag1Digest, "sha256:ceef30cbdce418efde0284f446df9cec9e535adcd6e1010dad30ddae1dc9367b", "Digest should match in all environments") + require.Equal(t, "sha256:ceef30cbdce418efde0284f446df9cec9e535adcd6e1010dad30ddae1dc9367b", tag1Digest, "Digest should match in all environments") out = imgpkg.Run([]string{"push", "--tty", "-i", env.Image + ":tag2", "-f", assetsPath}) tag2Digest := helpers.ExtractDigest(t, out) diff --git a/test/e2e/gcloud_expired_token_test.go b/test/e2e/gcloud_expired_token_test.go index 81ae82dae..1ff44d550 100644 --- a/test/e2e/gcloud_expired_token_test.go +++ b/test/e2e/gcloud_expired_token_test.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "testing" "github.com/k14s/imgpkg/test/helpers" @@ -16,6 +17,10 @@ import ( ) func TestCopyWithBundleLockInputToRepoDestinationUsingGCloudWithAnExpiredToken(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping test as docker image used requires linux") + } + env := helpers.BuildEnv(t) imgpkg := helpers.Imgpkg{T: t, L: helpers.Logger{}, ImgpkgPath: env.ImgpkgPath} defer env.Cleanup() diff --git a/test/e2e/push_test.go b/test/e2e/push_test.go index ce75b683c..82c0e54e3 100644 --- a/test/e2e/push_test.go +++ b/test/e2e/push_test.go @@ -8,6 +8,7 @@ import ( "io/fs" "os" "path/filepath" + "runtime" "testing" "github.com/k14s/imgpkg/test/helpers" @@ -42,6 +43,10 @@ images: } func TestPushFilesPermissions(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping test as this is a known issue: https://github.com/vmware-tanzu/carvel-imgpkg/issues/270") + } + env := helpers.BuildEnv(t) logger := helpers.Logger{} imgpkg := helpers.Imgpkg{T: t, L: helpers.Logger{}, ImgpkgPath: env.ImgpkgPath} @@ -63,12 +68,12 @@ func TestPushFilesPermissions(t *testing.T) { logger.Section("Check files permissions did not change", func() { info, err := os.Stat(filepath.Join(bundleDir, "exec_file.sh")) require.NoError(t, err) - assert.Equal(t, fs.FileMode(0700), info.Mode(), "have -rwx------ permissions") + assert.Equal(t, fs.FileMode(0700).String(), info.Mode().String(), "have -rwx------ permissions") info, err = os.Stat(filepath.Join(bundleDir, "read_only_config.yml")) require.NoError(t, err) - assert.Equal(t, fs.FileMode(0400), info.Mode(), "have -r-------- permissions") + assert.Equal(t, fs.FileMode(0400).String(), info.Mode().String(), "have -r-------- permissions") info, err = os.Stat(filepath.Join(bundleDir, "read_write_config.yml")) require.NoError(t, err) - assert.Equal(t, fs.FileMode(0600), info.Mode(), "have -rw------- permissions") + assert.Equal(t, fs.FileMode(0600).String(), info.Mode().String(), "have -rw------- permissions") }) } diff --git a/test/perf/perf_test.go b/test/perf/perf_test.go index 0ccbb8a5b..121fbe470 100644 --- a/test/perf/perf_test.go +++ b/test/perf/perf_test.go @@ -5,7 +5,6 @@ package perf import ( "fmt" - "os/exec" "strconv" "strings" "testing" @@ -79,23 +78,11 @@ func TestBenchmarkCopyingLargeBundleThatContainsImagesMostlyOnDockerHub(t *testi } func startRegistryForPerfTesting(t *testing.T, env *helpers.Env) string { - dockerRunCmd := exec.Command("docker", "run", "-d", "-p", "5000", "--env", "REGISTRY_VALIDATION_MANIFESTS_URLS_ALLOW=- ^https?://", "--restart", "always", "--name", "registry-for-perf-testing", "registry:2") - output, err := dockerRunCmd.CombinedOutput() - if err != nil { - t.Fatalf("output: %s, %s", output, err) - } + fakeRegistry := helpers.NewFakeRegistry(t, env.Logger) env.AddCleanup(func() { - exec.Command("docker", "stop", "registry-for-perf-testing").Run() - exec.Command("docker", "rm", "-v", "registry-for-perf-testing").Run() + fakeRegistry.CleanUp() }) - inspectCmd := exec.Command("docker", "inspect", `--format='{{(index (index .NetworkSettings.Ports "5000/tcp") 0).HostPort}}'`, "registry-for-perf-testing") - output, err = inspectCmd.CombinedOutput() - if err != nil { - t.Fatalf("output: %s, %s", output, err) - } - - hostPort := strings.ReplaceAll(string(output), "'", "") - return fmt.Sprintf("localhost:%s/repo/perf-image", strings.ReplaceAll(hostPort, "\n", "")) + return fakeRegistry.ReferenceOnTestServer("repo/perf-image") } diff --git a/vendor/github.com/vdemeester/k8s-pkg-credentialprovider/provider.go b/vendor/github.com/vdemeester/k8s-pkg-credentialprovider/provider.go index 8c9ad347b..f953bb471 100644 --- a/vendor/github.com/vdemeester/k8s-pkg-credentialprovider/provider.go +++ b/vendor/github.com/vdemeester/k8s-pkg-credentialprovider/provider.go @@ -70,7 +70,7 @@ type CachingDockerConfigProvider struct { // Enabled implements dockerConfigProvider func (d *defaultDockerConfigProvider) Enabled() bool { - return true + return false } // Provide implements dockerConfigProvider