Skip to content

Commit

Permalink
test/e2e: Add clusterctl upgrade with ClusterClass test
Browse files Browse the repository at this point in the history 
Signed-off-by: Stefan Büringer buringerst@vmware.com
  • Loading branch information
@sbueringer
sbueringer committed Sep 20, 2022
1 parent fd11302 commit c8eda34
Show file tree
Hide file tree
Showing 17 changed files with 795 additions and 80 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ hack/tools/bin
# E2E test templates
test/e2e/data/infrastructure-docker/v1alpha3/cluster-template*.yaml
test/e2e/data/infrastructure-docker/v1alpha4/cluster-template*.yaml
test/e2e/data/infrastructure-docker/v1.2/cluster-template*.yaml
test/e2e/data/infrastructure-docker/v1beta1/cluster-template*.yaml

# E2e test extension deployment
Expand Down
6 changes: 5 additions & 1 deletion test/e2e/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ $(KUSTOMIZE_BIN): $(KUSTOMIZE) ## Build a local copy of kustomize
DOCKER_TEMPLATES := $(REPO_ROOT)/test/e2e/data/infrastructure-docker

.PHONY: cluster-templates
cluster-templates: $(KUSTOMIZE) cluster-templates-v1alpha3 cluster-templates-v1alpha4 cluster-templates-v1beta1 ## Generate cluster templates for all versions
cluster-templates: $(KUSTOMIZE) cluster-templates-v1alpha3 cluster-templates-v1alpha4 cluster-templates-v1.2 cluster-templates-v1beta1 ## Generate cluster templates for all versions

cluster-templates-v1alpha3: $(KUSTOMIZE) ## Generate cluster templates for v1alpha3
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha3/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1alpha3/cluster-template.yaml
Expand All @@ -84,6 +84,10 @@ cluster-templates-v1alpha4: $(KUSTOMIZE) ## Generate cluster templates for v1alp
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-kcp-scale-in --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-kcp-scale-in.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-ipv6 --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-ipv6.yaml

cluster-templates-v1.2: $(KUSTOMIZE) ## Generate cluster templates for v1.2
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.2/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.2/cluster-template.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.2/cluster-template-topology --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.2/cluster-template-topology.yaml

cluster-templates-v1beta1: $(KUSTOMIZE) ## Generate cluster templates for v1beta1
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1beta1/cluster-template.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template-md-remediation --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1beta1/cluster-template-md-remediation.yaml
Expand Down
114 changes: 73 additions & 41 deletions test/e2e/clusterctl_upgrade.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,18 @@ type ClusterctlUpgradeSpecInput struct {
// InitWithProvidersContract can be used to override the INIT_WITH_PROVIDERS_CONTRACT e2e config variable with a specific
// provider contract to use to initialise the secondary management cluster, e.g. `v1alpha3`
InitWithProvidersContract string
SkipCleanup bool
ControlPlaneWaiters clusterctl.ControlPlaneWaiters
PreInit func(managementClusterProxy framework.ClusterProxy)
PreUpgrade func(managementClusterProxy framework.ClusterProxy)
PostUpgrade func(managementClusterProxy framework.ClusterProxy)
MgmtFlavor string
WorkloadFlavor string
// InitWithKubernetesVersion can be used to override the INIT_WITH_KUBERNETES_VERSION e2e config variable with a specific
// Kubernetes version to use to create the secondary management cluster, e.g. `v1.25.0`
InitWithKubernetesVersion string
// UpgradeClusterctlVariables can be used to set additional variables for clusterctl upgrade.
UpgradeClusterctlVariables map[string]string
SkipCleanup bool
ControlPlaneWaiters clusterctl.ControlPlaneWaiters
PreInit func(managementClusterProxy framework.ClusterProxy)
PreUpgrade func(managementClusterProxy framework.ClusterProxy)
PostUpgrade func(managementClusterProxy framework.ClusterProxy)
MgmtFlavor string
WorkloadFlavor string
}

// ClusterctlUpgradeSpec implements a test that verifies clusterctl upgrade of a management cluster.
Expand Down Expand Up @@ -115,12 +120,15 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
testCancelWatches context.CancelFunc

managementClusterName string
clusterctlBinaryURL string
managementClusterNamespace *corev1.Namespace
managementClusterCancelWatches context.CancelFunc
managementClusterResources *clusterctl.ApplyClusterTemplateAndWaitResult
managementClusterProxy framework.ClusterProxy

initClusterctlBinaryURL string
initContract string
initKubernetesVersion string

workLoadClusterName string
)

Expand All @@ -130,6 +138,7 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName)
Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)

var clusterctlBinaryURLTemplate string
if input.InitWithBinary == "" {
Expect(input.E2EConfig.Variables).To(HaveKey(initWithBinaryVariableName), "Invalid argument. %s variable must be defined when calling %s spec", initWithBinaryVariableName, specName)
Expand All @@ -139,8 +148,27 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
clusterctlBinaryURLTemplate = input.InitWithBinary
}
clusterctlBinaryURLReplacer := strings.NewReplacer("{OS}", runtime.GOOS, "{ARCH}", runtime.GOARCH)
clusterctlBinaryURL = clusterctlBinaryURLReplacer.Replace(clusterctlBinaryURLTemplate)
Expect(input.E2EConfig.Variables).To(HaveKey(initWithKubernetesVersion))
initClusterctlBinaryURL = clusterctlBinaryURLReplacer.Replace(clusterctlBinaryURLTemplate)

// NOTE: by default we are considering all the providers, no matter of the contract.
// However, given that we want to test both v1alpha3 --> v1beta1 and v1alpha4 --> v1beta1, the INIT_WITH_PROVIDERS_CONTRACT
// variable can be used to select versions with a specific contract.
initContract = "*"
if input.E2EConfig.HasVariable(initWithProvidersContract) {
initContract = input.E2EConfig.GetVariable(initWithProvidersContract)
}
if input.InitWithProvidersContract != "" {
initContract = input.InitWithProvidersContract
}

if input.InitWithKubernetesVersion == "" {
Expect(input.E2EConfig.Variables).To(HaveKey(initWithKubernetesVersion), "Invalid argument. %s variable must be defined when calling %s spec", initWithKubernetesVersion, specName)
Expect(input.E2EConfig.Variables[initWithKubernetesVersion]).ToNot(BeEmpty(), "Invalid argument. %s variable can't be empty when calling %s spec", initWithKubernetesVersion, specName)
initKubernetesVersion = input.E2EConfig.GetVariable(initWithKubernetesVersion)
} else {
initKubernetesVersion = input.InitWithKubernetesVersion
}

Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)

Expand All @@ -164,7 +192,7 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
Flavor: input.MgmtFlavor,
Namespace: managementClusterNamespace.Name,
ClusterName: managementClusterName,
KubernetesVersion: input.E2EConfig.GetVariable(initWithKubernetesVersion),
KubernetesVersion: initKubernetesVersion,
ControlPlaneMachineCount: pointer.Int64Ptr(1),
WorkerMachineCount: pointer.Int64Ptr(1),
},
Expand Down Expand Up @@ -193,27 +221,15 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
managementClusterProxy = input.BootstrapClusterProxy.GetWorkloadCluster(ctx, cluster.Namespace, cluster.Name)

// Download the older clusterctl version to be used for setting up the management cluster to be upgraded

log.Logf("Downloading clusterctl binary from %s", clusterctlBinaryURL)
clusterctlBinaryPath := downloadToTmpFile(ctx, clusterctlBinaryURL)
log.Logf("Downloading clusterctl binary from %s", initClusterctlBinaryURL)
clusterctlBinaryPath := downloadToTmpFile(ctx, initClusterctlBinaryURL)
defer os.Remove(clusterctlBinaryPath) // clean up

err := os.Chmod(clusterctlBinaryPath, 0744) //nolint:gosec
Expect(err).ToNot(HaveOccurred(), "failed to chmod temporary file")

By("Initializing the workload cluster with older versions of providers")

// NOTE: by default we are considering all the providers, no matter of the contract.
// However, given that we want to test both v1alpha3 --> v1beta1 and v1alpha4 --> v1beta1, the INIT_WITH_PROVIDERS_CONTRACT
// variable can be used to select versions with a specific contract.
contract := "*"
if input.E2EConfig.HasVariable(initWithProvidersContract) {
contract = input.E2EConfig.GetVariable(initWithProvidersContract)
}
if input.InitWithProvidersContract != "" {
contract = input.InitWithProvidersContract
}

if input.PreInit != nil {
By("Running Pre-init steps against the management cluster")
input.PreInit(managementClusterProxy)
Expand All @@ -223,10 +239,10 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
ClusterctlBinaryPath: clusterctlBinaryPath, // use older version of clusterctl to init the management cluster
ClusterProxy: managementClusterProxy,
ClusterctlConfigPath: input.ClusterctlConfigPath,
CoreProvider: input.E2EConfig.GetProviderLatestVersionsByContract(contract, config.ClusterAPIProviderName)[0],
BootstrapProviders: input.E2EConfig.GetProviderLatestVersionsByContract(contract, config.KubeadmBootstrapProviderName),
ControlPlaneProviders: input.E2EConfig.GetProviderLatestVersionsByContract(contract, config.KubeadmControlPlaneProviderName),
InfrastructureProviders: input.E2EConfig.GetProviderLatestVersionsByContract(contract, input.E2EConfig.InfrastructureProviders()...),
CoreProvider: input.E2EConfig.GetProviderLatestVersionsByContract(initContract, config.ClusterAPIProviderName)[0],
BootstrapProviders: input.E2EConfig.GetProviderLatestVersionsByContract(initContract, config.KubeadmBootstrapProviderName),
ControlPlaneProviders: input.E2EConfig.GetProviderLatestVersionsByContract(initContract, config.KubeadmControlPlaneProviderName),
InfrastructureProviders: input.E2EConfig.GetProviderLatestVersionsByContract(initContract, input.E2EConfig.InfrastructureProviders()...),
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", cluster.Name),
}, input.E2EConfig.GetIntervals(specName, "wait-controllers")...)

Expand Down Expand Up @@ -313,6 +329,7 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
By("Upgrading providers to the latest version available")
clusterctl.UpgradeManagementClusterAndWait(ctx, clusterctl.UpgradeManagementClusterAndWaitInput{
ClusterctlConfigPath: input.ClusterctlConfigPath,
ClusterctlVariables: input.UpgradeClusterctlVariables,
ClusterProxy: managementClusterProxy,
Contract: clusterv1.GroupVersion.Version,
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", cluster.Name),
Expand Down Expand Up @@ -342,19 +359,34 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
// After upgrading we are sure the version is the latest version of the API,
// so it is possible to use the standard helpers

testMachineDeployments := framework.GetMachineDeploymentsByCluster(ctx, framework.GetMachineDeploymentsByClusterInput{
Lister: managementClusterProxy.GetClient(),
ClusterName: workLoadClusterName,
Namespace: testNamespace.Name,
})

framework.ScaleAndWaitMachineDeployment(ctx, framework.ScaleAndWaitMachineDeploymentInput{
ClusterProxy: managementClusterProxy,
Cluster: &clusterv1.Cluster{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace.Name}},
MachineDeployment: testMachineDeployments[0],
Replicas: 2,
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
workloadCluster := framework.GetClusterByName(ctx, framework.GetClusterByNameInput{
Getter: managementClusterProxy.GetClient(),
Namespace: testNamespace.Name,
Name: workLoadClusterName,
})
if workloadCluster.Spec.Topology != nil {
// Cluster is using ClusterClass, scale up via topology.
framework.ScaleAndWaitMachineDeploymentTopology(ctx, framework.ScaleAndWaitMachineDeploymentTopologyInput{
ClusterProxy: managementClusterProxy,
Cluster: workloadCluster,
Replicas: 2,
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
})
} else {
// Cluster is not using ClusterClass, scale up via MachineDeployment.
testMachineDeployments := framework.GetMachineDeploymentsByCluster(ctx, framework.GetMachineDeploymentsByClusterInput{
Lister: managementClusterProxy.GetClient(),
ClusterName: workLoadClusterName,
Namespace: testNamespace.Name,
})
framework.ScaleAndWaitMachineDeployment(ctx, framework.ScaleAndWaitMachineDeploymentInput{
ClusterProxy: managementClusterProxy,
Cluster: &clusterv1.Cluster{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace.Name}},
MachineDeployment: testMachineDeployments[0],
Replicas: 2,
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
})
}

By("THE UPGRADED MANAGEMENT CLUSTER WORKS!")

Expand Down
66 changes: 60 additions & 6 deletions test/e2e/clusterctl_upgrade_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,68 @@ import (
. "github.com/onsi/ginkgo/v2"
)

var _ = Describe("When testing clusterctl upgrades [clusterctl-Upgrade]", func() {
var _ = Describe("When testing clusterctl upgrades (v0.3=>current)", func() {
ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput {
return ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.25/clusterctl-{OS}-{ARCH}",
InitWithProvidersContract: "v1alpha3",
// CAPI v0.3.x does not work on Kubernetes >= v1.22.
InitWithKubernetesVersion: "v1.21.12",
// CAPI does not work with Kubernetes < v1.22 if ClusterClass is enabled, so we have to disable it.
UpgradeClusterctlVariables: map[string]string{
"CLUSTER_TOPOLOGY": "false",
},
}
})
})

var _ = Describe("When testing clusterctl upgrades (v0.4=>current)", func() {
ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput {
return ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.8/clusterctl-{OS}-{ARCH}",
InitWithProvidersContract: "v1alpha4",
InitWithKubernetesVersion: "v1.25.0",
}
})
})

var _ = Describe("When testing clusterctl upgrades (v1.2=>current)", func() {
ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput {
return ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.2/clusterctl-{OS}-{ARCH}",
InitWithProvidersContract: "v1beta1",
InitWithKubernetesVersion: "v1.25.0",
}
})
})

var _ = Describe("When testing clusterctl upgrades using ClusterClass (v1.2=>current)", func() {
ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput {
return ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.2/clusterctl-{OS}-{ARCH}",
InitWithProvidersContract: "v1beta1",
InitWithKubernetesVersion: "v1.25.0",
WorkloadFlavor: "topology",
}
})
})
Loading

0 comments on commit c8eda34

Please sign in to comment.