diff --git a/Makefile b/Makefile index 3f97dfeac..28d963737 100644 --- a/Makefile +++ b/Makefile @@ -73,6 +73,14 @@ ifndef TANZU_CLI_TMC_UNSTABLE_URL TANZU_CLI_TMC_UNSTABLE_URL = "" endif +ifndef TANZU_CLI_E2E_TEST_CENTRAL_REPO_URL +TANZU_CLI_E2E_TEST_CENTRAL_REPO_URL = gcr.io/eminent-nation-87317/tanzu-cli/test/v1/plugins/plugin-inventory:latest +endif + +## update PATH with tanzu binary path, and tanzu CLI tools binaries path +CURRENT_DIR = $(shell pwd) +$(eval export PATH=$(CURRENT_DIR)/bin:$(CURRENT_DIR)/hack/tools/bin:$(PATH)) + ## -------------------------------------- ## Help ## -------------------------------------- @@ -185,14 +193,25 @@ choco-package: ## Build a Chocolatey package test: fmt ## Run Tests ${GO} test `go list ./... | grep -v test/e2e` -timeout 60m -race -coverprofile coverage.txt ${GOTEST_VERBOSE} -.PHONY: e2e-cli-core -e2e-cli-core: ## Run CLI Core E2E Tests - $(eval export PATH=$(ROOT_DIR)/bin:$(ROOT_DIR)/hack/tools/bin:$(PATH)) +.PHONY: e2e-cli-core ## Execute all CLI Core E2E Tests +e2e-cli-core: e2e-cli-plugin-compatibility-test e2e-cli-tmc-test + ${GO} test `go list ./test/e2e/... | grep -v test/e2e/context/tmc | grep -v test/e2e/plugins_compatibility` -timeout 60m -race -coverprofile coverage.txt ${GOTEST_VERBOSE} + +.PHONY: e2e-cli-plugin-compatibility-test ## Execute CLI Core Plugin Compatibility E2E test cases +e2e-cli-plugin-compatibility-test: + $(eval export TANZU_CLI_PRE_RELEASE_REPO_IMAGE=$(TANZU_CLI_E2E_TEST_CENTRAL_REPO_URL)) + @if [ "${TANZU_CLI_E2E_TEST_CENTRAL_REPO_URL}" = "" ]; then \ + echo "***Skipping Plugin Compatibility test cases because environment variables TANZU_CLI_E2E_TEST_CENTRAL_REPO_URL is not set***" ; \ + else \ + ${GO} test ./test/e2e/plugins_compatibility -timeout 60m -race -coverprofile coverage.txt ${GOTEST_VERBOSE} ; \ + fi + +.PHONY: e2e-cli-tmc-test ## Execute CLI Core TMC Specific E2E test cases +e2e-cli-tmc-test: @if [ "${TANZU_API_TOKEN}" = "" ] && [ "$(TANZU_CLI_TMC_UNSTABLE_URL)" = "" ]; then \ echo "***Skipping TMC specific e2e tests cases because environment variables TANZU_API_TOKEN and TANZU_CLI_TMC_UNSTABLE_URL are not set***" ; \ - ${GO} test `go list ./test/e2e/... | grep -v test/e2e/context/tmc` -timeout 60m -race -coverprofile coverage.txt ${GOTEST_VERBOSE} ; \ else \ - ${GO} test ./test/e2e/... -timeout 60m -race -coverprofile coverage.txt ${GOTEST_VERBOSE} ; \ + ${GO} test ./test/e2e/context/tmc -timeout 60m -race -coverprofile coverage.txt ${GOTEST_VERBOSE} ; \ fi .PHONY: start-test-central-repo diff --git a/test/e2e/framework/config_lifecycle_operations.go b/test/e2e/framework/config_lifecycle_operations.go index 2117bb468..89c7f5396 100644 --- a/test/e2e/framework/config_lifecycle_operations.go +++ b/test/e2e/framework/config_lifecycle_operations.go @@ -22,14 +22,23 @@ const ( // ConfigLifecycleOps performs "tanzu config" command operations type ConfigLifecycleOps interface { + // ConfigSetFeatureFlag sets the tanzu config feature flag ConfigSetFeatureFlag(path, value string) error + // ConfigGetFeatureFlag gets the tanzu config feature flag ConfigGetFeatureFlag(path string) (string, error) + // ConfigUnsetFeature un-sets the tanzu config feature flag ConfigUnsetFeature(path string) error + // ConfigInit performs "tanzu config init" ConfigInit() error + // GetConfig gets the tanzu config GetConfig() (*configapi.ClientConfig, error) + // ConfigServerList returns the server list ConfigServerList() error + // ConfigServerDelete deletes given server from tanzu config ConfigServerDelete(serverName string) error + // DeleteCLIConfigurationFiles deletes cli configuration files DeleteCLIConfigurationFiles() error + // IsCLIConfigurationFilesExists checks the existence of cli configuration files IsCLIConfigurationFilesExists() bool } @@ -58,14 +67,14 @@ func (co *configOps) GetConfig() (*configapi.ClientConfig, error) { return cnf, nil } -// ConfigSetFeature sets the tanzu config feature flag +// ConfigSetFeatureFlag sets the given tanzu config feature flag func (co *configOps) ConfigSetFeatureFlag(path, value string) (err error) { confSetCmd := ConfigSet + path + " " + value _, _, err = co.Exec(confSetCmd) return err } -// ConfigSetFeature sets the tanzu config feature flag +// ConfigGetFeatureFlag gets the given tanzu config feature flag func (co *configOps) ConfigGetFeatureFlag(path string) (string, error) { cnf, err := co.GetConfig() if err != nil { diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index f01e72f99..585355391 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -29,6 +29,20 @@ const ( AddPluginSource = "tanzu plugin source add --name %s --type %s --uri %s" DeletePluginSource = "tanzu plugin source delete %s" ListPluginsCmdInJSON = "tanzu plugin list -o json" + SearchPluginsCmd = "tanzu plugin search" + InstallPLuginCmd = "tanzu plugin install %s" + UninstallPLuginCmd = "tanzu plugin uninstall %s" + JSONOutput = " -o json" + TestPluginsPrefix = "test-plugin-" + PluginSubCommand = "tanzu %s" + + // Central repository + CentralRepositoryFeatureFlag = "features.global.central-repository" + CentralRepositoryPreReleaseRepoImage = "TANZU_CLI_PRE_RELEASE_REPO_IMAGE" + TanzuCliE2ETestCentralRepositoryURL = "TANZU_CLI_E2E_TEST_CENTRAL_REPO_URL" + + // General constants + True = "true" // Context commands CreateContextWithEndPoint = "tanzu context create --endpoint %s --name %s" diff --git a/test/e2e/framework/output_handling.go b/test/e2e/framework/output_handling.go index bc4c95fa9..7663e2038 100644 --- a/test/e2e/framework/output_handling.go +++ b/test/e2e/framework/output_handling.go @@ -31,3 +31,8 @@ type ContextInfo struct { IsManagementCluster bool `json:"isManagementCluster"` } `json:"clusterOpts"` } + +type PluginCompatibilityConf struct { + PluginNames []string `json:"plugins"` + TestCentralRepoURL string `json:"test-central-repo-url"` +} diff --git a/test/e2e/framework/plugin_lifecycle_operations.go b/test/e2e/framework/plugin_lifecycle_operations.go index 7238373ff..fbeac441e 100644 --- a/test/e2e/framework/plugin_lifecycle_operations.go +++ b/test/e2e/framework/plugin_lifecycle_operations.go @@ -5,16 +5,27 @@ package framework import ( "fmt" + "strings" "encoding/json" "github.com/pkg/errors" + + "github.com/vmware-tanzu/tanzu-plugin-runtime/log" ) // PluginBasicOps helps to perform the plugin command operations type PluginBasicOps interface { // ListPlugins lists all plugins by running 'tanzu plugin list' command ListPlugins() ([]PluginListInfo, error) + // SearchPlugins searches all plugins for given filter (keyword|regex) by running 'tanzu plugin search' command + SearchPlugins(filter string) ([]PluginListInfo, error) + // InstallPlugin installs given plugin + InstallPlugin(pluginName string) error + // UninstallPlugin uninstalls given plugin + UninstallPlugin(pluginName string) error + // ExecuteSubCommand executes specific plugin sub-command + ExecuteSubCommand(pluginWithSubCommand string) (string, error) } // PluginSourceOps helps 'plugin source' commands @@ -74,3 +85,53 @@ func (po *pluginCmdOps) ListPlugins() ([]PluginListInfo, error) { } return list, nil } + +func (po *pluginCmdOps) SearchPlugins(filter string) ([]PluginListInfo, error) { + searchPluginCmdWithOptions := SearchPluginsCmd + if len(strings.TrimSpace(filter)) > 0 { + searchPluginCmdWithOptions = searchPluginCmdWithOptions + " " + strings.TrimSpace(filter) + } + searchPluginCmdWithOptions += JSONOutput + out, stdErr, err := po.cmdExe.Exec(searchPluginCmdWithOptions) + + if err != nil { + log.Errorf("error while executing plugin search command:'%s', error:'%s' stdErr:'%s'", searchPluginCmdWithOptions, err.Error(), stdErr.String()) + return nil, err + } + jsonStr := out.String() + var list []PluginListInfo + err = json.Unmarshal([]byte(jsonStr), &list) + if err != nil { + log.Errorf("failed to construct node from plugin search output:'%s' error:'%s' ", jsonStr, err.Error()) + return nil, errors.Wrapf(err, "failed to construct json node from search output:'%s'", jsonStr) + } + return list, nil +} + +func (po *pluginCmdOps) InstallPlugin(pluginName string) error { + installPluginCmd := fmt.Sprintf(InstallPLuginCmd, pluginName) + _, stdErr, err := po.cmdExe.Exec(installPluginCmd) + if err != nil { + log.Errorf("error while installing the plugin: %s, error: %s stdErr: %s", pluginName, err.Error(), stdErr.String()) + } + return err +} + +func (po *pluginCmdOps) UninstallPlugin(pluginName string) error { + uninstallPluginCmd := fmt.Sprintf(UninstallPLuginCmd, pluginName) + _, stdErr, err := po.cmdExe.Exec(uninstallPluginCmd) + if err != nil { + log.Errorf("error while uninstalling the plugin: %s, error: %s, stdErr: %s", pluginName, err.Error(), stdErr.String()) + } + return err +} + +func (po *pluginCmdOps) ExecuteSubCommand(pluginWithSubCommand string) (string, error) { + pluginCmdWithSubCommand := fmt.Sprintf(PluginSubCommand, pluginWithSubCommand) + stdOut, stdErr, err := po.cmdExe.Exec(pluginCmdWithSubCommand) + if err != nil { + log.Errorf("error while running the plugin command: %s, error: %s, stdErr: %s", pluginCmdWithSubCommand, err.Error(), stdErr.String()) + return stdOut.String(), errors.Wrap(err, stdErr.String()) + } + return stdOut.String(), nil +} diff --git a/test/e2e/plugins_compatibility/plugins_compatibility_helper_test.go b/test/e2e/plugins_compatibility/plugins_compatibility_helper_test.go new file mode 100644 index 000000000..e55b80d03 --- /dev/null +++ b/test/e2e/plugins_compatibility/plugins_compatibility_helper_test.go @@ -0,0 +1,26 @@ +// Copyright 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// plugincompatibility provides plugins compatibility E2E test cases +package plugincompatibility + +import ( + "strings" + + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/tanzu-cli/test/e2e/framework" +) + +// PluginsForCompatibilityTesting search for test-plugin-'s from the test central repository and returns all test-plugin-'s +func PluginsForCompatibilityTesting(tf *framework.Framework) []string { + list, err := tf.PluginCmd.SearchPlugins("") + Expect(err).To(BeNil(), "should not occur any error while searching for plugins") + testPlugins := make([]string, 0) + for _, plugin := range list { + if strings.HasPrefix(plugin.Name, framework.TestPluginsPrefix) { + testPlugins = append(testPlugins, plugin.Name) + } + } + return testPlugins +} diff --git a/test/e2e/plugins_compatibility/plugins_compatibility_suite_test.go b/test/e2e/plugins_compatibility/plugins_compatibility_suite_test.go new file mode 100644 index 000000000..a9a636707 --- /dev/null +++ b/test/e2e/plugins_compatibility/plugins_compatibility_suite_test.go @@ -0,0 +1,17 @@ +// Copyright 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// plugincompatibility provides plugins compatibility E2E test cases +package plugincompatibility_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestPluginsCompatibility(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "PluginsCompatibility Suite") +} diff --git a/test/e2e/plugins_compatibility/plugins_compatibility_test.go b/test/e2e/plugins_compatibility/plugins_compatibility_test.go new file mode 100644 index 000000000..5b728d587 --- /dev/null +++ b/test/e2e/plugins_compatibility/plugins_compatibility_test.go @@ -0,0 +1,79 @@ +// Copyright 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// plugincompatibility provides plugins compatibility E2E test cases +package plugincompatibility + +import ( + "fmt" + "os" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/tanzu-plugin-runtime/log" + + "github.com/vmware-tanzu/tanzu-cli/test/e2e/framework" +) + +// This test suite has test case for plugin compatibility use cases +// Goal of these test suites is to validate the plugins built with different Tanzu CLI Runtime Library can co-exists and operate at the same time. +// Below test suite, searches for test plugins (plugin name prefix with "test-plugin-") in the test central repository and +// installs all test plugins, executes basic commands on all installed test plugins, and finally uninstalls all test plugins. +// Each test plugin built using specific Tanzu CLI Runtime library versions. +var _ = framework.CLICoreDescribe("[Tests:E2E][Feature:Plugin-Compatibility]", func() { + var ( + tf *framework.Framework + plugins []string + ) + // In the BeforeSuite sets the "features.global.central-repository" flag + // and searches for the test-plugin-'s from the TANZU_CLI_E2E_TEST_CENTRAL_REPO_URL test central repository + BeforeSuite(func() { + tf = framework.NewFramework() + err := tf.Config.ConfigSetFeatureFlag(framework.CentralRepositoryFeatureFlag, framework.True) + Expect(err).To(BeNil()) + // get all plugins with name prefix "test-plugin-" + plugins = PluginsForCompatibilityTesting(tf) + Expect(len(plugins)).NotTo(BeZero(), fmt.Sprintf("there are no test-plugin-'s in test central repo:%s , make sure its valid test central repo with test-plugins", os.Getenv(framework.TanzuCliE2ETestCentralRepositoryURL))) + }) + Context("Install plugins for plugins compatibility", func() { + // Test case: install all test plugins + It("Install all test plugins", func() { + for _, plugin := range plugins { + log.Infof("Installing test plugin:%s", plugin) + err := tf.PluginCmd.InstallPlugin(plugin) + Expect(err).To(BeNil(), fmt.Sprintf("should not occur any error while installing the test plugin: %s", plugin)) + } + }) + }) + Context("Test installed compatibility test-plugins", func() { + // Test case: run basic commands on installed test plugins, to make sure works/co-exists with other plugins build with different runtime version + It("run basic commands on the installed test-plugins", func() { + for _, plugin := range plugins { + info, err := tf.PluginCmd.ExecuteSubCommand(plugin + " info") + Expect(err).To(BeNil(), "should not occur any error when plugin info command executed") + Expect(info).NotTo(BeNil(), "there should be some out for plugin info command executed") + } + }) + }) + Context("Test installed compatibility test-plugins with hello-world command", func() { + // Test case: run hello-world commands on installed test plugins, to make sure works/co-exists with other plugins build with different runtime version + It("run hello-world commands on the installed test-plugins", func() { + for _, plugin := range plugins { + output, err := tf.PluginCmd.ExecuteSubCommand(plugin + " hello-world") + Expect(err).To(BeNil(), "should not occur any error when plugin hello-world command executed") + Expect(output).To(ContainSubstring("the command hello-world executed successfully")) + } + }) + }) + Context("Uninstall all installed compatibility test-plugins", func() { + // Test case: uninstall all installed compatibility test-plugins + It("uninstall all test-plugins", func() { + for _, plugin := range plugins { + log.Infof("Uninstalling test plugin: %s", plugin) + err := tf.PluginCmd.UninstallPlugin(plugin) + Expect(err).To(BeNil(), fmt.Sprintf("should not occur any error while uninstalling the test plugin: %s", plugin)) + } + }) + }) +})