Skip to content

Commit

Permalink
test/e2e: Add clusterctl upgrade with ClusterClass test
Browse files Browse the repository at this point in the history
Signed-off-by: Stefan Büringer buringerst@vmware.com
  • Loading branch information
sbueringer committed Sep 20, 2022
1 parent fd11302 commit 2aa49b8
Show file tree
Hide file tree
Showing 14 changed files with 674 additions and 45 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ hack/tools/bin
# E2E test templates
test/e2e/data/infrastructure-docker/v1alpha3/cluster-template*.yaml
test/e2e/data/infrastructure-docker/v1alpha4/cluster-template*.yaml
test/e2e/data/infrastructure-docker/v1.2/cluster-template*.yaml
test/e2e/data/infrastructure-docker/v1beta1/cluster-template*.yaml

# E2e test extension deployment
Expand Down
6 changes: 5 additions & 1 deletion test/e2e/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ $(KUSTOMIZE_BIN): $(KUSTOMIZE) ## Build a local copy of kustomize
DOCKER_TEMPLATES := $(REPO_ROOT)/test/e2e/data/infrastructure-docker

.PHONY: cluster-templates
cluster-templates: $(KUSTOMIZE) cluster-templates-v1alpha3 cluster-templates-v1alpha4 cluster-templates-v1beta1 ## Generate cluster templates for all versions
cluster-templates: $(KUSTOMIZE) cluster-templates-v1alpha3 cluster-templates-v1alpha4 cluster-templates-v1.2 cluster-templates-v1beta1 ## Generate cluster templates for all versions

cluster-templates-v1alpha3: $(KUSTOMIZE) ## Generate cluster templates for v1alpha3
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha3/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1alpha3/cluster-template.yaml
Expand All @@ -84,6 +84,10 @@ cluster-templates-v1alpha4: $(KUSTOMIZE) ## Generate cluster templates for v1alp
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-kcp-scale-in --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-kcp-scale-in.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-ipv6 --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-ipv6.yaml

cluster-templates-v1.2: $(KUSTOMIZE) ## Generate cluster templates for v1.2
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.2/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.2/cluster-template.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.2/cluster-template-topology --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.2/cluster-template-topology.yaml

cluster-templates-v1beta1: $(KUSTOMIZE) ## Generate cluster templates for v1beta1
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1beta1/cluster-template.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template-md-remediation --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1beta1/cluster-template-md-remediation.yaml
Expand Down
97 changes: 63 additions & 34 deletions test/e2e/clusterctl_upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ type ClusterctlUpgradeSpecInput struct {
// InitWithProvidersContract can be used to override the INIT_WITH_PROVIDERS_CONTRACT e2e config variable with a specific
// provider contract to use to initialise the secondary management cluster, e.g. `v1alpha3`
InitWithProvidersContract string
// InitWithKubernetesVersion can be used to override the INIT_WITH_KUBERNETES_VERSION e2e config variable with a specific
// Kubernetes version to use to create the secondary management cluster, e.g. `v1.25.0`
InitWithKubernetesVersion string
SkipCleanup bool
ControlPlaneWaiters clusterctl.ControlPlaneWaiters
PreInit func(managementClusterProxy framework.ClusterProxy)
Expand Down Expand Up @@ -115,12 +118,15 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
testCancelWatches context.CancelFunc

managementClusterName string
clusterctlBinaryURL string
managementClusterNamespace *corev1.Namespace
managementClusterCancelWatches context.CancelFunc
managementClusterResources *clusterctl.ApplyClusterTemplateAndWaitResult
managementClusterProxy framework.ClusterProxy

initClusterctlBinaryURL string
initContract string
initKubernetesVersion string

workLoadClusterName string
)

Expand All @@ -130,6 +136,7 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName)
Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)

var clusterctlBinaryURLTemplate string
if input.InitWithBinary == "" {
Expect(input.E2EConfig.Variables).To(HaveKey(initWithBinaryVariableName), "Invalid argument. %s variable must be defined when calling %s spec", initWithBinaryVariableName, specName)
Expand All @@ -139,8 +146,27 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
clusterctlBinaryURLTemplate = input.InitWithBinary
}
clusterctlBinaryURLReplacer := strings.NewReplacer("{OS}", runtime.GOOS, "{ARCH}", runtime.GOARCH)
clusterctlBinaryURL = clusterctlBinaryURLReplacer.Replace(clusterctlBinaryURLTemplate)
Expect(input.E2EConfig.Variables).To(HaveKey(initWithKubernetesVersion))
initClusterctlBinaryURL = clusterctlBinaryURLReplacer.Replace(clusterctlBinaryURLTemplate)

// NOTE: by default we are considering all the providers, no matter of the contract.
// However, given that we want to test both v1alpha3 --> v1beta1 and v1alpha4 --> v1beta1, the INIT_WITH_PROVIDERS_CONTRACT
// variable can be used to select versions with a specific contract.
initContract = "*"
if input.E2EConfig.HasVariable(initWithProvidersContract) {
initContract = input.E2EConfig.GetVariable(initWithProvidersContract)
}
if input.InitWithProvidersContract != "" {
initContract = input.InitWithProvidersContract
}

if input.InitWithKubernetesVersion == "" {
Expect(input.E2EConfig.Variables).To(HaveKey(initWithKubernetesVersion), "Invalid argument. %s variable must be defined when calling %s spec", initWithKubernetesVersion, specName)
Expect(input.E2EConfig.Variables[initWithKubernetesVersion]).ToNot(BeEmpty(), "Invalid argument. %s variable can't be empty when calling %s spec", initWithKubernetesVersion, specName)
initKubernetesVersion = input.E2EConfig.GetVariable(initWithKubernetesVersion)
} else {
initKubernetesVersion = input.InitWithKubernetesVersion
}

Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)

Expand All @@ -164,7 +190,7 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
Flavor: input.MgmtFlavor,
Namespace: managementClusterNamespace.Name,
ClusterName: managementClusterName,
KubernetesVersion: input.E2EConfig.GetVariable(initWithKubernetesVersion),
KubernetesVersion: initKubernetesVersion,
ControlPlaneMachineCount: pointer.Int64Ptr(1),
WorkerMachineCount: pointer.Int64Ptr(1),
},
Expand Down Expand Up @@ -193,27 +219,15 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
managementClusterProxy = input.BootstrapClusterProxy.GetWorkloadCluster(ctx, cluster.Namespace, cluster.Name)

// Download the older clusterctl version to be used for setting up the management cluster to be upgraded

log.Logf("Downloading clusterctl binary from %s", clusterctlBinaryURL)
clusterctlBinaryPath := downloadToTmpFile(ctx, clusterctlBinaryURL)
log.Logf("Downloading clusterctl binary from %s", initClusterctlBinaryURL)
clusterctlBinaryPath := downloadToTmpFile(ctx, initClusterctlBinaryURL)
defer os.Remove(clusterctlBinaryPath) // clean up

err := os.Chmod(clusterctlBinaryPath, 0744) //nolint:gosec
Expect(err).ToNot(HaveOccurred(), "failed to chmod temporary file")

By("Initializing the workload cluster with older versions of providers")

// NOTE: by default we are considering all the providers, no matter of the contract.
// However, given that we want to test both v1alpha3 --> v1beta1 and v1alpha4 --> v1beta1, the INIT_WITH_PROVIDERS_CONTRACT
// variable can be used to select versions with a specific contract.
contract := "*"
if input.E2EConfig.HasVariable(initWithProvidersContract) {
contract = input.E2EConfig.GetVariable(initWithProvidersContract)
}
if input.InitWithProvidersContract != "" {
contract = input.InitWithProvidersContract
}

if input.PreInit != nil {
By("Running Pre-init steps against the management cluster")
input.PreInit(managementClusterProxy)
Expand All @@ -223,10 +237,10 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
ClusterctlBinaryPath: clusterctlBinaryPath, // use older version of clusterctl to init the management cluster
ClusterProxy: managementClusterProxy,
ClusterctlConfigPath: input.ClusterctlConfigPath,
CoreProvider: input.E2EConfig.GetProviderLatestVersionsByContract(contract, config.ClusterAPIProviderName)[0],
BootstrapProviders: input.E2EConfig.GetProviderLatestVersionsByContract(contract, config.KubeadmBootstrapProviderName),
ControlPlaneProviders: input.E2EConfig.GetProviderLatestVersionsByContract(contract, config.KubeadmControlPlaneProviderName),
InfrastructureProviders: input.E2EConfig.GetProviderLatestVersionsByContract(contract, input.E2EConfig.InfrastructureProviders()...),
CoreProvider: input.E2EConfig.GetProviderLatestVersionsByContract(initContract, config.ClusterAPIProviderName)[0],
BootstrapProviders: input.E2EConfig.GetProviderLatestVersionsByContract(initContract, config.KubeadmBootstrapProviderName),
ControlPlaneProviders: input.E2EConfig.GetProviderLatestVersionsByContract(initContract, config.KubeadmControlPlaneProviderName),
InfrastructureProviders: input.E2EConfig.GetProviderLatestVersionsByContract(initContract, input.E2EConfig.InfrastructureProviders()...),
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", cluster.Name),
}, input.E2EConfig.GetIntervals(specName, "wait-controllers")...)

Expand Down Expand Up @@ -342,19 +356,34 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
// After upgrading we are sure the version is the latest version of the API,
// so it is possible to use the standard helpers

testMachineDeployments := framework.GetMachineDeploymentsByCluster(ctx, framework.GetMachineDeploymentsByClusterInput{
Lister: managementClusterProxy.GetClient(),
ClusterName: workLoadClusterName,
Namespace: testNamespace.Name,
})

framework.ScaleAndWaitMachineDeployment(ctx, framework.ScaleAndWaitMachineDeploymentInput{
ClusterProxy: managementClusterProxy,
Cluster: &clusterv1.Cluster{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace.Name}},
MachineDeployment: testMachineDeployments[0],
Replicas: 2,
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
workloadCluster := framework.GetClusterByName(ctx, framework.GetClusterByNameInput{
Getter: managementClusterProxy.GetClient(),
Namespace: testNamespace.Name,
Name: workLoadClusterName,
})
if workloadCluster.Spec.Topology != nil {
// Cluster is using ClusterClass, scale up via topology.
framework.ScaleAndWaitMachineDeploymentTopology(ctx, framework.ScaleAndWaitMachineDeploymentTopologyInput{
ClusterProxy: managementClusterProxy,
Cluster: workloadCluster,
Replicas: 2,
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
})
} else {
// Cluster is not using ClusterClass, scale up via MachineDeployment.
testMachineDeployments := framework.GetMachineDeploymentsByCluster(ctx, framework.GetMachineDeploymentsByClusterInput{
Lister: managementClusterProxy.GetClient(),
ClusterName: workLoadClusterName,
Namespace: testNamespace.Name,
})
framework.ScaleAndWaitMachineDeployment(ctx, framework.ScaleAndWaitMachineDeploymentInput{
ClusterProxy: managementClusterProxy,
Cluster: &clusterv1.Cluster{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace.Name}},
MachineDeployment: testMachineDeployments[0],
Replicas: 2,
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
})
}

By("THE UPGRADED MANAGEMENT CLUSTER WORKS!")

Expand Down
16 changes: 16 additions & 0 deletions test/e2e/clusterctl_upgrade_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,19 @@ var _ = Describe("When testing clusterctl upgrades [clusterctl-Upgrade]", func()
}
})
})

var _ = Describe("When testing clusterctl upgrades using ClusterClass", func() {
ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput {
return ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.2/clusterctl-{OS}-{ARCH}",
InitWithProvidersContract: "v1beta1",
InitWithKubernetesVersion: "v1.25.0",
WorkloadFlavor: "topology",
}
})
})
20 changes: 11 additions & 9 deletions test/e2e/config/docker.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ providers:
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1alpha4/metadata.yaml"
- name: v1.2.0 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.0/core-components.yaml"
- name: v1.2.2 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.2/core-components.yaml"
type: "url"
contract: v1beta1
replacements:
Expand Down Expand Up @@ -87,8 +87,8 @@ providers:
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1alpha4/metadata.yaml"
- name: v1.2.0 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.0/bootstrap-components.yaml"
- name: v1.2.2 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.2/bootstrap-components.yaml"
type: "url"
contract: v1beta1
replacements:
Expand Down Expand Up @@ -125,8 +125,8 @@ providers:
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1alpha4/metadata.yaml"
- name: v1.2.0 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.0/control-plane-components.yaml"
- name: v1.2.2 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.2/control-plane-components.yaml"
type: "url"
contract: v1beta1
replacements:
Expand Down Expand Up @@ -165,16 +165,18 @@ providers:
files:
- sourcePath: "../data/shared/v1alpha4/metadata.yaml"
- sourcePath: "../data/infrastructure-docker/v1alpha4/cluster-template.yaml"
- name: v1.2.0 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.0/infrastructure-components-development.yaml"
- name: v1.2.2 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.2/infrastructure-components-development.yaml"
type: "url"
contract: v1beta1
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1beta1/metadata.yaml"
- sourcePath: "../data/infrastructure-docker/v1beta1/cluster-template.yaml"
- sourcePath: "../data/infrastructure-docker/v1.2/cluster-template.yaml"
- sourcePath: "../data/infrastructure-docker/v1.2/cluster-template-topology.yaml"
- sourcePath: "../data/infrastructure-docker/v1.2/clusterclass-quick-start.yaml"
- name: v1.3.99 # next; use manifest from source files
value: ../../../test/infrastructure/docker/config/default
replacements:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
---
# DockerCluster object referenced by the Cluster object
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerCluster
metadata:
name: '${CLUSTER_NAME}'
spec:
failureDomains:
fd1:
controlPlane: true
fd2:
controlPlane: true
fd3:
controlPlane: true
fd4:
controlPlane: false
fd5:
controlPlane: false
fd6:
controlPlane: false
fd7:
controlPlane: false
fd8:
controlPlane: false
---
# Cluster object with
# - Reference to the KubeadmControlPlane object
# - the label cni=${CLUSTER_NAME}-crs-0, so the cluster can be selected by the ClusterResourceSet.
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
name: '${CLUSTER_NAME}'
labels:
cni: "${CLUSTER_NAME}-crs-0"
spec:
clusterNetwork:
services:
cidrBlocks: ['${DOCKER_SERVICE_CIDRS}']
pods:
cidrBlocks: ['${DOCKER_POD_CIDRS}']
serviceDomain: '${DOCKER_SERVICE_DOMAIN}'
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerCluster
name: '${CLUSTER_NAME}'
controlPlaneRef:
kind: KubeadmControlPlane
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
name: "${CLUSTER_NAME}-control-plane"
---
# DockerMachineTemplate object referenced by the KubeadmControlPlane object
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachineTemplate
metadata:
name: "${CLUSTER_NAME}-control-plane"
spec:
template:
spec:
extraMounts:
- containerPath: "/var/run/docker.sock"
hostPath: "/var/run/docker.sock"
---
# KubeadmControlPlane referenced by the Cluster object with
# - the label kcp-adoption.step2, because it should be created in the second step of the kcp-adoption test.
kind: KubeadmControlPlane
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
metadata:
name: "${CLUSTER_NAME}-control-plane"
labels:
kcp-adoption.step2: ""
spec:
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
machineTemplate:
infrastructureRef:
kind: DockerMachineTemplate
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
name: "${CLUSTER_NAME}-control-plane"
kubeadmConfigSpec:
clusterConfiguration:
controllerManager:
extraArgs: {enable-hostpath-provisioner: 'true'}
apiServer:
# host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied.
certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal]
initConfiguration:
nodeRegistration:
criSocket: unix:///var/run/containerd/containerd.sock
kubeletExtraArgs:
eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
joinConfiguration:
nodeRegistration:
criSocket: unix:///var/run/containerd/containerd.sock
kubeletExtraArgs:
eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
version: "${KUBERNETES_VERSION}"
Loading

0 comments on commit 2aa49b8

Please sign in to comment.