diff --git a/tests/e2e/multiclustercidr/multiclustercidr_test.go b/tests/e2e/multiclustercidr/multiclustercidr_test.go index 09b95fabc70b..bd5b3e074460 100644 --- a/tests/e2e/multiclustercidr/multiclustercidr_test.go +++ b/tests/e2e/multiclustercidr/multiclustercidr_test.go @@ -106,7 +106,7 @@ var _ = Describe("Verify MultiClusterCIDR Configuration", Ordered, func() { It("Restart agent-0", func() { agents := []string{"agent-0"} - err := e2e.RestartClusterAgent(agents) + err := e2e.RestartCluster(agents) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) }) @@ -223,7 +223,7 @@ var _ = Describe("Verify MultiClusterCIDR Configuration", Ordered, func() { It("Delete and restart agent-0", func() { agents := []string{"agent-0"} - err := e2e.RestartClusterAgent(agents) + err := e2e.RestartCluster(agents) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) }) diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index 16325bac8367..fcd42975f2cb 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -427,10 +427,24 @@ func RestartCluster(nodeNames []string) error { return nil } -// RestartCluster restarts the k3s service on each node given -func RestartClusterAgent(nodeNames []string) error { +// StartCluster starts the k3s service on each node given +func StartCluster(nodeNames []string) error { + for _, nodeName := range nodeNames { + cmd := "sudo systemctl start k3s" + if strings.Contains(nodeName, "agent") { + cmd += "-agent" + } + if _, err := RunCmdOnNode(cmd, nodeName); err != nil { + return err + } + } + return nil +} + +// StopCluster starts the k3s service on each node given +func StopCluster(nodeNames []string) error { for _, nodeName := range nodeNames { - cmd := "sudo systemctl restart k3s-agent" + cmd := "sudo systemctl stop k3s*" if _, err := RunCmdOnNode(cmd, nodeName); err != nil { return err } diff --git a/tests/e2e/validatecluster/validatecluster_test.go b/tests/e2e/validatecluster/validatecluster_test.go index 6ec6fa9251be..955f6c700b70 100644 --- a/tests/e2e/validatecluster/validatecluster_test.go +++ b/tests/e2e/validatecluster/validatecluster_test.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "os" + "regexp" "strings" "testing" @@ -43,7 +44,7 @@ var ( var _ = ReportAfterEach(e2e.GenReport) var _ = Describe("Verify Create", Ordered, func() { - Context("Cluster :", func() { + Context("Cluster Starts up and deploys basic components", func() { It("Starts up with no issues", func() { var err error if *local { @@ -99,7 +100,6 @@ var _ = Describe("Verify Create", Ordered, func() { clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc", false) cmd := "curl -L --insecure http://" + clusterip + "/name.html" - fmt.Println(cmd) for _, nodeName := range serverNodeNames { Eventually(func(g Gomega) { res, err := e2e.RunCmdOnNode(cmd, nodeName) @@ -127,7 +127,7 @@ var _ = Describe("Verify Create", Ordered, func() { }, "240s", "5s").Should(Succeed()) cmd = "curl -L --insecure http://" + nodeExternalIP + ":" + nodeport + "/name.html" - fmt.Println(cmd) + Eventually(func(g Gomega) { res, err := e2e.RunCommand(cmd) g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res) @@ -210,55 +210,13 @@ var _ = Describe("Verify Create", Ordered, func() { Eventually(func(g Gomega) { cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec -i -t dnsutils -- nslookup kubernetes.default" + res, err := e2e.RunCommand(cmd) g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res) g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local")) }, "420s", "2s").Should(Succeed()) }) - It("Verifies Restart", func() { - _, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, *hardened) - Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed") - defer e2e.DeleteWorkload("daemonset.yaml", kubeConfigFile) - nodes, _ := e2e.ParseNodes(kubeConfigFile, false) - - Eventually(func(g Gomega) { - pods, _ := e2e.ParsePods(kubeConfigFile, false) - count := e2e.CountOfStringInSlice("test-daemonset", pods) - g.Expect(len(nodes)).Should((Equal(count)), "Daemonset pod count does not match node count") - podsRunning := 0 - for _, pod := range pods { - if strings.Contains(pod.Name, "test-daemonset") && pod.Status == "Running" && pod.Ready == "1/1" { - podsRunning++ - } - } - g.Expect(len(nodes)).Should((Equal(podsRunning)), "Daemonset running pods count does not match node count") - }, "620s", "5s").Should(Succeed()) - errRestart := e2e.RestartCluster(serverNodeNames) - Expect(errRestart).NotTo(HaveOccurred(), "Restart Nodes not happened correctly") - if len(agentNodeNames) > 0 { - errRestartAgent := e2e.RestartCluster(agentNodeNames) - Expect(errRestartAgent).NotTo(HaveOccurred(), "Restart Agent not happened correctly") - } - Eventually(func(g Gomega) { - nodes, err := e2e.ParseNodes(kubeConfigFile, false) - g.Expect(err).NotTo(HaveOccurred()) - for _, node := range nodes { - g.Expect(node.Status).Should(Equal("Ready")) - } - pods, _ := e2e.ParsePods(kubeConfigFile, false) - count := e2e.CountOfStringInSlice("test-daemonset", pods) - g.Expect(len(nodes)).Should((Equal(count)), "Daemonset pod count does not match node count") - podsRunningAr := 0 - for _, pod := range pods { - if strings.Contains(pod.Name, "test-daemonset") && pod.Status == "Running" && pod.Ready == "1/1" { - podsRunningAr++ - } - } - g.Expect(len(nodes)).Should((Equal(podsRunningAr)), "Daemonset pods are not running after the restart") - }, "620s", "5s").Should(Succeed()) - }) - It("Verifies Local Path Provisioner storage ", func() { res, err := e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed: "+res) @@ -314,6 +272,130 @@ var _ = Describe("Verify Create", Ordered, func() { }, "180s", "2s").Should(Succeed()) }) }) + + Context("Validate restart", func() { + It("Deletes daemonset", func() { + _, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, *hardened) + Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed") + defer e2e.DeleteWorkload("daemonset.yaml", kubeConfigFile) + nodes, _ := e2e.ParseNodes(kubeConfigFile, false) + + Eventually(func(g Gomega) { + pods, _ := e2e.ParsePods(kubeConfigFile, false) + count := e2e.CountOfStringInSlice("test-daemonset", pods) + g.Expect(len(nodes)).Should((Equal(count)), "Daemonset pod count does not match node count") + podsRunning := 0 + for _, pod := range pods { + if strings.Contains(pod.Name, "test-daemonset") && pod.Status == "Running" && pod.Ready == "1/1" { + podsRunning++ + } + } + g.Expect(len(nodes)).Should((Equal(podsRunning)), "Daemonset running pods count does not match node count") + }, "620s", "5s").Should(Succeed()) + }) + It("Restarts normally", func() { + errRestart := e2e.RestartCluster(append(serverNodeNames, agentNodeNames...)) + Expect(errRestart).NotTo(HaveOccurred(), "Restart Nodes not happened correctly") + + Eventually(func(g Gomega) { + nodes, err := e2e.ParseNodes(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + for _, node := range nodes { + g.Expect(node.Status).Should(Equal("Ready")) + } + pods, _ := e2e.ParsePods(kubeConfigFile, false) + count := e2e.CountOfStringInSlice("test-daemonset", pods) + g.Expect(len(nodes)).Should((Equal(count)), "Daemonset pod count does not match node count") + podsRunningAr := 0 + for _, pod := range pods { + if strings.Contains(pod.Name, "test-daemonset") && pod.Status == "Running" && pod.Ready == "1/1" { + podsRunningAr++ + } + } + g.Expect(len(nodes)).Should((Equal(podsRunningAr)), "Daemonset pods are not running after the restart") + }, "620s", "5s").Should(Succeed()) + }) + }) + + Context("Valdiate Certificate Rotation", func() { + It("Stops K3s and rotates certificates", func() { + errStop := e2e.StopCluster(serverNodeNames) + Expect(errStop).NotTo(HaveOccurred(), "Cluster could not be stoped successfully") + + for _, nodeName := range serverNodeNames { + cmd := "sudo k3s certificate rotate" + if _, err := e2e.RunCmdOnNode(cmd, nodeName); err != nil { + Expect(err).NotTo(HaveOccurred(), "Certificate could not be rotated successfully") + } + } + }) + + It("Start normally", func() { + // Since we stopped all the server, we have to start 2 at once to get it back up + // If we only start one at a time, the first will hang waiting for the second to be up + _, err := e2e.RunCmdOnNode("sudo systemctl --no-block start k3s", serverNodeNames[0]) + Expect(err).NotTo(HaveOccurred()) + err = e2e.StartCluster(serverNodeNames[1:]) + Expect(err).NotTo(HaveOccurred(), "Cluster could not be started successfully") + + Eventually(func(g Gomega) { + nodes, err := e2e.ParseNodes(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + for _, node := range nodes { + g.Expect(node.Status).Should(Equal("Ready")) + } + fmt.Println("help") + }, "620s", "5s").Should(Succeed()) + + Eventually(func(g Gomega) { + pods, err := e2e.ParsePods(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + for _, pod := range pods { + if strings.Contains(pod.Name, "helm-install") { + g.Expect(pod.Status).Should(Equal("Completed"), pod.Name) + } else { + g.Expect(pod.Status).Should(Equal("Running"), pod.Name) + } + } + }, "620s", "5s").Should(Succeed()) + }) + It("Validates certificates", func() { + const grepCert = "sudo ls -lt /var/lib/rancher/k3s/server/ | grep tls" + var expectResult = []string{"client-ca.crt", + "client-ca.key", + "client-ca.nochain.crt", + "dynamic-cert.json", "peer-ca.crt", + "peer-ca.key", "server-ca.crt", + "server-ca.key", "request-header-ca.crt", + "request-header-ca.key", "server-ca.crt", + "server-ca.key", "server-ca.nochain.crt", + "service.current.key", "service.key", + "apiserver-loopback-client__.crt", + "apiserver-loopback-client__.key", "", + } + + var finalResult string + var finalErr error + for _, nodeName := range serverNodeNames { + grCert, errGrep := e2e.RunCmdOnNode(grepCert, nodeName) + Expect(errGrep).NotTo(HaveOccurred(), "Certificate could not be created successfully") + re := regexp.MustCompile("tls-[0-9]+") + tls := re.FindAllString(grCert, -1)[0] + final := fmt.Sprintf("sudo diff -sr /var/lib/rancher/k3s/server/tls/ /var/lib/rancher/k3s/server/%s/"+ + "| grep -i identical | cut -f4 -d ' ' | xargs basename -a \n", tls) + finalResult, finalErr = e2e.RunCmdOnNode(final, nodeName) + Expect(finalErr).NotTo(HaveOccurred(), "Final Certification does not created successfully") + } + errRestartAgent := e2e.RestartCluster(agentNodeNames) + Expect(errRestartAgent).NotTo(HaveOccurred(), "Agent could not be restart successfully") + + finalCert := strings.Replace(finalResult, "\n", ",", -1) + finalCertArray := strings.Split(finalCert, ",") + Expect((finalCertArray)).Should((Equal(expectResult)), "Final certification does not match the expected results") + + }) + + }) }) var failed bool