Add support for the Authenticated Registries (vmware-tanzu#744)

To use registries that require authentication to host Tanzu CLI Plugins images users are expected to do the following:
1. Use `docker login <registry>` or `crane auth login <registry>` to authenticate with the registry
2. Specify environment variable `TANZU_CLI_AUTHENTICATED_REGISTRY=<registry>`. By specifying this environment variable, Tanzu CLI will use the default authentication mechanism instead of using Anonymous access to fetch images.

docs/dev/README.md

@@ -80,6 +80,7 @@ Below is the list of these variables:
 | -------------------- | ----------- | ----- |
 | `SQL_STATEMENTS_LOG_FILE` | Specifies a log file where SQL commands will be logged when _modifying_ the plugin inventory database.  This is done when publishing plugins using the `builder` plugin. | A file name with its path |
 | `TANZU_CLI_ADDITIONAL_PLUGIN_DISCOVERY_IMAGES_TEST_ONLY` | Specifies test plugin repositories to use as a supplement to the production Central Repository of plugins. Ignored if `TANZU_CLI_PRIVATE_PLUGIN_DISCOVERY_IMAGES` is set. | Comma-separated list of test plugin repository URIs| |
+| `TANZU_CLI_AUTHENTICATED_REGISTRY` | Specifies the list of registry hosts that requires authentication to pull images. Tanzu CLI will use default docker auth to communicate to these registries | Comma-separated list of registry host-names | |
 | `TANZU_CLI_ESSENTIALS_PLUGIN_GROUP_NAME` | Override the default name (`vmware-tanzucli/essentials`) of the Essential Plugins group.  Should not be needed. | Group name |
 | `TANZU_CLI_ESSENTIALS_PLUGIN_GROUP_VERSION` | Specify a fixed version to use for the Essential Plugins group instead of the latest.  Should not be needed. | Group version |
 | `TANZU_CLI_SKIP_CONTEXT_RECOMMENDED_PLUGIN_INSTALLATION` | Skips the auto-installation of the context recommended plugins

pkg/carvelhelpers/fetcher.go

@@ -4,11 +4,16 @@
 package carvelhelpers
 
 import (
+	"os"
+	"strings"
+
 	"github.com/pkg/errors"
 
 	ctlimg "github.com/vmware-tanzu/carvel-imgpkg/pkg/imgpkg/registry"
 
+	"github.com/vmware-tanzu/tanzu-cli/pkg/constants"
 	"github.com/vmware-tanzu/tanzu-cli/pkg/registry"
+	"github.com/vmware-tanzu/tanzu-cli/pkg/utils"
 )
 
 // GetFilesMapFromImage returns map of files metadata
@@ -32,8 +37,11 @@ func GetImageDigest(imageWithTag string) (string, string, error) {
 // newRegistry returns a new registry object by also taking
 // into account for any custom registry provided by the user
 func newRegistry(registryHost string) (registry.Registry, error) {
-	registryOpts := &ctlimg.Opts{
-		Anon: true,
+	registryOpts := &ctlimg.Opts{}
+
+	authenticatedRegistries := strings.Split(os.Getenv(constants.AuthenticatedRegistry), ",")
+	if !utils.ContainsRegistry(authenticatedRegistries, registryHost) {
+		registryOpts.Anon = true
 	}
 
 	regCertOptions, err := registry.GetRegistryCertOptions(registryHost)

pkg/constants/env_variables.go

@@ -79,4 +79,8 @@ const (
 	// SkipTAPScopesValidationOnTanzuContext skips the TAP scopes validation on the token acquired while creating "tanzu"
 	// context using tanzu login or tanzu context create command
 	SkipTAPScopesValidationOnTanzuContext = "TANZU_CLI_SKIP_TAP_SCOPES_VALIDATION_ON_TANZU_CONTEXT"
+
+	// AuthenticatedRegistry provides a comma separated list of registry hosts that requires authentication
+	// to pull images. Tanzu CLI will use default docker auth to communicate to these registries
+	AuthenticatedRegistry = "TANZU_CLI_AUTHENTICATED_REGISTRY"
 )

pkg/utils/url.go

@@ -51,3 +51,22 @@ func JoinURL(baseURL, relativeURL string) (string, error) {
 	// Return the joined URL as a string
 	return parsedBaseURL.String(), nil
 }
+
+// ContainsRegistry returns true if the specified registryHost is part of registries
+func ContainsRegistry(registries []string, registryHost string) bool {
+	cleanRegistryURL := func(u string) string {
+		u = strings.TrimSpace(u)
+		u = strings.TrimPrefix(u, "http://")
+		u = strings.TrimPrefix(u, "https://")
+		return strings.Split(u, "/")[0]
+	}
+	registryHost = cleanRegistryURL(registryHost)
+
+	for _, reg := range registries {
+		reg = cleanRegistryURL(reg)
+		if strings.EqualFold(reg, registryHost) {
+			return true
+		}
+	}
+	return false
+}

pkg/utils/url_test.go

@@ -630,3 +630,31 @@ func TestJoinUrl(t *testing.T) {
 		})
 	}
 }
+
+func TestContainsRegistry(t *testing.T) {
+	testCases := []struct {
+		input    string
+		expected bool
+	}{
+		{"google.com", true},
+		{"facebook.com", false},
+		{"xyz.com", false},
+		{"packages.xyz.com", true},
+		{"packages.example.com", false},
+		{"projects.example.com", true},
+		{"packages.repo.com", false},
+		{"registry.packages.repo.com", true},
+		{"registry.packages.repo.com/projects", true},
+		{"https://registry.packages.repo.com/projects", true},
+		{"registry.packages", false},
+	}
+
+	registries := []string{"https://google.com", "http://fb.com", "https://packages.xyz.com", "projects.example.com", "registry.packages.repo.com/projects/cli"}
+
+	for _, tc := range testCases {
+		got := ContainsRegistry(registries, tc.input)
+		if got != tc.expected {
+			t.Errorf("ContainsRegistry(%q, %q) = %t; want %t", registries, tc.input, got, tc.expected)
+		}
+	}
+}

test/e2e/airgapped/airgapped_suite_test.go

@@ -33,6 +33,7 @@ var (
 	e2eAirgappedCentralRepoWithAuth         string
 	e2eAirgappedCentralRepoWithAuthUsername string
 	e2eAirgappedCentralRepoWithAuthPassword string
+	e2eAirgappedCentralRepoWithAuthImage    string
 	pluginsSearchList                       []*framework.PluginInfo
 	pluginGroups                            []*framework.PluginGroup
 	tempDir                                 string
@@ -58,8 +59,9 @@ var _ = BeforeSuite(func() {
 	Expect(e2eAirgappedCentralRepoWithAuthUsername).NotTo(BeEmpty(), fmt.Sprintf("environment variable %s should set with airgapped central repository URL", framework.TanzuCliE2ETestAirgappedRepoWithAuthUsername))
 	e2eAirgappedCentralRepoWithAuthPassword = os.Getenv(framework.TanzuCliE2ETestAirgappedRepoWithAuthPassword)
 	Expect(e2eAirgappedCentralRepoWithAuthPassword).NotTo(BeEmpty(), fmt.Sprintf("environment variable %s should set with airgapped central repository URL", framework.TanzuCliE2ETestAirgappedRepoWithAuthPassword))
+	e2eAirgappedCentralRepoWithAuthImage = fmt.Sprintf("%s%s", e2eAirgappedCentralRepoWithAuth, filepath.Base(e2eTestLocalCentralRepoImage))
 
-	os.Setenv(framework.TanzuCliPluginDiscoverySignatureVerificationSkipList, fmt.Sprintf("%v,%v", e2eAirgappedCentralRepoImage, e2eTestLocalCentralRepoImage))
+	os.Setenv(framework.TanzuCliPluginDiscoverySignatureVerificationSkipList, fmt.Sprintf("%v,%v,%v", e2eAirgappedCentralRepoImage, e2eTestLocalCentralRepoImage, e2eAirgappedCentralRepoWithAuthImage))
 
 	e2eTestLocalCentralRepoPluginHost := os.Getenv(framework.TanzuCliE2ETestLocalCentralRepositoryHost)
 	Expect(e2eTestLocalCentralRepoPluginHost).NotTo(BeEmpty(), fmt.Sprintf("environment variable %s should set with local central repository host", framework.TanzuCliE2ETestLocalCentralRepositoryHost))

test/e2e/airgapped/airgapped_test.go

@@ -67,22 +67,20 @@ var _ = framework.CLICoreDescribe("[Tests:E2E][Feature:Airgapped-Plugin-Download
 		})
 	})
 	Context("Download plugin bundle, Upload plugin bundle and plugin lifecycle tests with plugin group 'vmware-tkg/default:v0.0.1'", func() {
+		originalHomeDir := framework.GetHomeDir()
+
 		// Test case: download plugin bundle for plugin-group vmware-tkg/default:v0.0.1
 		It("download plugin bundle with specific plugin-group vmware-tkg/default:v0.0.1", func() {
 			err := tf.PluginCmd.DownloadPluginBundle(e2eTestLocalCentralRepoImage, []string{"vmware-tkg/default:v0.0.1"}, []string{}, false, filepath.Join(tempDir, "plugin_bundle_vmware-tkg-default-v0.0.1.tar.gz"))
 			Expect(err).To(BeNil(), "should not get any error while downloading plugin bundle with specific group")
 		})
 
-		// Test case: upload plugin bundle downloaded using vmware-tkg/default:v0.0.1 plugin-group to the airgapped repository with authentication
-		It("upload plugin bundle that was downloaded using vmware-tkg/default:v0.0.1 plugin-group to the airgapped repository with authentication", func() {
-			curHomeDir := framework.GetHomeDir()
-			defer func() {
-				os.Setenv("HOME", curHomeDir)
-			}()
+		It("Authenticated Registry: Docker Login", func() {
 			// We are resetting the HOME environment variable for this specific tests as when we do docker login we need to have actually HOME variable set correctly
 			// otherwise the docker login fails with different errors on different systems (on macos we see keychain specific error)
-			// We are also using above defer function to revert the HOME environment variable after this test is ran
+			// We are reverting the HOME environment variable after this set of authenticated registry tests are run as part of "Authenticated Registry: Unset Environment Variable" test
 			os.Setenv("HOME", framework.OriginalHomeDir)
+			os.Setenv("TANZU_CLI_AUTHENTICATED_REGISTRY", e2eAirgappedCentralRepoWithAuth)
 
 			// Try uploading plugin bundle without docker login, it should fail
 			err := tf.PluginCmd.UploadPluginBundle(e2eAirgappedCentralRepoWithAuth, filepath.Join(tempDir, "plugin_bundle_vmware-tkg-default-v0.0.1.tar.gz"))
@@ -92,12 +90,57 @@ var _ = framework.CLICoreDescribe("[Tests:E2E][Feature:Airgapped-Plugin-Download
 			dockerloginCmd := fmt.Sprintf("docker login %s --username %s --password %s", e2eAirgappedCentralRepoWithAuth, e2eAirgappedCentralRepoWithAuthUsername, e2eAirgappedCentralRepoWithAuthPassword)
 			_, _, err = tf.Exec.Exec(dockerloginCmd)
 			Expect(err).To(BeNil())
+		})
 
+		// Test case: upload plugin bundle downloaded using vmware-tkg/default:v0.0.1 plugin-group to the airgapped repository with authentication
+		It("Authenticated Registry: upload plugin bundle that was downloaded using vmware-tkg/default:v0.0.1 plugin-group to the airgapped repository with authentication", func() {
 			// Try uploading plugin bundle after docker login, it should succeed
 			err = tf.PluginCmd.UploadPluginBundle(e2eAirgappedCentralRepoWithAuth, filepath.Join(tempDir, "plugin_bundle_vmware-tkg-default-v0.0.1.tar.gz"))
 			Expect(err).To(BeNil(), "should not get any error while uploading plugin bundle")
 		})
 
+		// Test case: validate that the updating the discovery source to point to new airgapped repository requiring authentication works
+		It("Authenticated Registry: update discovery source to point to new airgapped repository discovery image that requires authentication", func() {
+			err = framework.UpdatePluginDiscoverySource(tf, e2eAirgappedCentralRepoWithAuthImage)
+			Expect(err).To(BeNil(), "should not get any error for plugin source update for authenticated registry")
+		})
+
+		// Test case: Validate that the correct plugins and plugin group exists with `tanzu plugin search` and `tanzu plugin group search` output on authenticated registry
+		It("Authenticated Registry: validate the plugins from group 'vmware-tkg/default:v0.0.1' exists", func() {
+			// search plugin groups
+			pluginGroups, err = pluginlifecyclee2e.SearchAllPluginGroups(tf)
+			Expect(err).To(BeNil(), framework.NoErrorForPluginGroupSearch)
+			// check all expected plugin groups are available in the `plugin group search` output from the airgapped repository
+			expectedPluginGroups := []*framework.PluginGroup{{Group: "vmware-tkg/default", Latest: "v0.0.1", Description: "Desc for vmware-tkg/default:v0.0.1"}}
+			Expect(framework.IsAllPluginGroupsExists(pluginGroups, expectedPluginGroups)).Should(BeTrue(), "all required plugin groups for life cycle tests should exists in plugin group search output")
+
+			// search plugins and make sure correct number of plugins available
+			// check expected plugins are available in the `plugin search` output from the airgapped repository
+			expectedPlugins := pluginsForPGTKG001
+			expectedPlugins = append(expectedPlugins, essentialPlugins...) // Essential plugin will be always installed
+			pluginsSearchList, err = pluginlifecyclee2e.SearchAllPlugins(tf)
+			Expect(err).To(BeNil(), framework.NoErrorForPluginSearch)
+			Expect(len(pluginsSearchList)).To(Equal(len(expectedPlugins)))
+			Expect(framework.CheckAllPluginsExists(pluginsSearchList, expectedPlugins)).To(BeTrue())
+		})
+
+		// Test case: Validate that the plugins can be installed from the plugin-group for authenticated registry
+		It("Authenticated Registry: validate that plugins can be installed from group 'vmware-tkg/default:v0.0.1' for authenticated registry", func() {
+			// All plugins should get installed from the group
+			_, _, err := tf.PluginCmd.InstallPluginsFromGroup("", "vmware-tkg/default:v0.0.1")
+			Expect(err).To(BeNil())
+
+			// Verify all plugins got installed with `tanzu plugin list`
+			installedPlugins, err := tf.PluginCmd.ListInstalledPlugins()
+			Expect(err).To(BeNil())
+			Expect(framework.CheckAllPluginsExists(installedPlugins, pluginsForPGTKG001)).To(BeTrue())
+		})
+
+		It("Authenticated Registry: Unset Environment Variable", func() {
+			os.Setenv("HOME", originalHomeDir)
+			os.Unsetenv("TANZU_CLI_AUTHENTICATED_REGISTRY")
+		})
+
 		// Test case: upload plugin bundle downloaded using vmware-tkg/default:v0.0.1 plugin-group to the airgapped repository
 		It("upload plugin bundle that was downloaded using vmware-tkg/default:v0.0.1 plugin-group to the airgapped repository", func() {
 			err := tf.PluginCmd.UploadPluginBundle(e2eAirgappedCentralRepo, filepath.Join(tempDir, "plugin_bundle_vmware-tkg-default-v0.0.1.tar.gz"))