diff --git a/Makefile b/Makefile index 8db4c7e2..9a33d483 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,10 @@ test-debug: envtest ginkgo test-update: envtest ginkgo KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" UPDATE_TESTCASES=true $(GINKGO) -r --fail-fast +.PHONY: test-tortoisectl +test-tortoisectl: envtest + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test -timeout 30s -run Test_TortoiseCtlStop ./cmd/tortoisectl/test/... --update + GINKGO ?= $(LOCALBIN)/ginkgo GINKGO_VERSION ?= v2.1.4 @@ -82,6 +86,10 @@ $(GINKGO): $(LOCALBIN) build: generate fmt vet ## Build manager binary. go build -o bin/manager main.go +.PHONY: build-tortoisectl +build-tortoisectl: + go build -o bin/tortoisectl cmd/tortoisectl/main.go + .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. go run ./main.go diff --git a/api/core/v1/pod_webhook.go b/api/core/v1/pod_webhook.go index 8b337af3..0affe213 100644 --- a/api/core/v1/pod_webhook.go +++ b/api/core/v1/pod_webhook.go @@ -118,7 +118,7 @@ func (h *PodWebhook) Default(ctx context.Context, obj runtime.Object) error { return nil } - h.podService.ModifyPodResource(pod, tortoise) + h.podService.ModifyPodSpecResource(&pod.Spec, tortoise) pod.Annotations[annotation.PodMutationAnnotation] = fmt.Sprintf("this pod is mutated by tortoise (%s)", tortoise.Name) return nil diff --git a/cmd/tortoisectl/commands/root.go b/cmd/tortoisectl/commands/root.go new file mode 100644 index 00000000..29d693f1 --- /dev/null +++ b/cmd/tortoisectl/commands/root.go @@ -0,0 +1,21 @@ +package commands + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{ + Use: "tortoisectl [COMMANDS]", + Short: "tortoisectl is a CLI for managing Tortoise", + Long: `tortoisectl is a CLI for managing Tortoise.`, +} + +func Execute() { + if err := rootCmd.Execute(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/cmd/tortoisectl/commands/stop.go b/cmd/tortoisectl/commands/stop.go new file mode 100644 index 00000000..0996533b --- /dev/null +++ b/cmd/tortoisectl/commands/stop.go @@ -0,0 +1,91 @@ +package commands + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/mercari/tortoise/pkg/deployment" + "github.com/mercari/tortoise/pkg/pod" + "github.com/mercari/tortoise/pkg/stoper" + "github.com/spf13/cobra" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/homedir" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var stopCmd = &cobra.Command{ + Use: "stop tortoise1 tortoise2...", + Short: "stop tortoise(s) safely", + Long: `stop is the command to turn off tortoise(s) safely.`, + RunE: func(cmd *cobra.Command, args []string) error { + // validation + if !stopAll { + if len(args) != 0 { + return fmt.Errorf("tortoise name shouldn't be specified because of --all flag") + } + } else { + if stopNamespace == "" { + return fmt.Errorf("namespace must be specified") + } + if len(args) == 0 { + return fmt.Errorf("tortoise name must be specified") + } + } + + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return fmt.Errorf("failed to build config: %v", err) + } + + client, err := client.New(config, client.Options{}) + if err != nil { + return fmt.Errorf("failed to create client: %v", err) + } + + deploymentService := deployment.New(client, "", "", nil) + podService, err := pod.New(map[string]int64{}, "", nil, nil) + if err != nil { + return fmt.Errorf("failed to create pod service: %v", err) + } + + stoperService := stoper.New(client, deploymentService, podService) + + err = stoperService.Stop(cmd.Context(), args, stopNamespace, stopAll, os.Stdout, stoper.NoLoweringResource) + if err != nil { + return fmt.Errorf("failed to stop tortoise(s): %v", err) + } + + return nil + }, +} + +var ( + // namespace to stop tortoise(s) in + stopNamespace string + // stop all tortoises in the specified namespace, or in all namespaces if no namespace is specified. + stopAll bool + // Stop tortoise without lowering resource requests. + // If this flag is specified and the current Deployment's resource request(s) is lower than the current Pods' request mutated by Tortoise, + // this CLI patches the deployment so that changing tortoise to Off won't result in lowering the resource request(s), damaging the service. + noLoweringResource bool + + // Path to KUBECONFIG + kubeconfig string +) + +func init() { + rootCmd.AddCommand(stopCmd) + + if home := homedir.HomeDir(); home != "" { + stopCmd.Flags().StringVar(&kubeconfig, "kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file") + } else { + stopCmd.Flags().StringVar(&kubeconfig, "kubeconfig", "", "absolute path to the kubeconfig file") + } + + stopCmd.Flags().StringVarP(&stopNamespace, "namespace", "n", "", "namespace to stop tortoise(s) in") + stopCmd.Flags().BoolVarP(&stopAll, "all", "A", false, "stop all tortoises in the specified namespace, or in all namespaces if no namespace is specified.") + stopCmd.Flags().BoolVar(&noLoweringResource, "no-lowering-resource", false, `Stop tortoise without lowering resource requests. + If this flag is specified and the current Deployment's resource request(s) is lower than the current Pods' request mutated by Tortoise, + this CLI patches the deployment so that changing tortoise to Off won't result in lowering the resource request(s), damaging the service.`) +} diff --git a/cmd/tortoisectl/main.go b/cmd/tortoisectl/main.go new file mode 100644 index 00000000..a03dd8ae --- /dev/null +++ b/cmd/tortoisectl/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/mercari/tortoise/cmd/tortoisectl/commands" + +func main() { + commands.Execute() +} diff --git a/cmd/tortoisectl/test/testdata/success/after/deployment.yaml b/cmd/tortoisectl/test/testdata/success/after/deployment.yaml new file mode 100644 index 00000000..90b761c3 --- /dev/null +++ b/cmd/tortoisectl/test/testdata/success/after/deployment.yaml @@ -0,0 +1,48 @@ +metadata: + name: mercari-app + namespace: test-0 +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: mercari + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + annotations: + kubectl.kubernetes.io/restartedAt: "2023-01-01T00:00:00Z" + creationTimestamp: null + labels: + app: mercari + spec: + containers: + - image: awesome-mercari-app-image + imagePullPolicy: Always + name: app + resources: + requests: + cpu: "10" + memory: 10Gi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + - image: awesome-istio-proxy-image + imagePullPolicy: Always + name: istio-proxy + resources: + requests: + cpu: "4" + memory: 4Gi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 +status: {} diff --git a/cmd/tortoisectl/test/testdata/success/after/tortoise.yaml b/cmd/tortoisectl/test/testdata/success/after/tortoise.yaml new file mode 100644 index 00000000..ab991fee --- /dev/null +++ b/cmd/tortoisectl/test/testdata/success/after/tortoise.yaml @@ -0,0 +1,143 @@ +metadata: + finalizers: + - tortoise.autoscaling.mercari.com/finalizer + name: mercari + namespace: test-0 +spec: + targetRefs: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app + updateMode: Auto +status: + autoscalingPolicy: + - containerName: app + policy: + cpu: Horizontal + memory: Vertical + - containerName: istio-proxy + policy: + cpu: Horizontal + memory: Vertical + conditions: + containerRecommendationFromVPA: + - containerName: app + maxRecommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + recommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + - containerName: istio-proxy + maxRecommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + recommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + containerResourceRequests: + - containerName: app + resource: + cpu: "6" + memory: 3Gi + - containerName: istio-proxy + resource: + cpu: "4" + memory: 3Gi + tortoiseConditions: + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + message: the current number of replicas is not bigger than the preferred max + replica number + reason: ScaledUpBasedOnPreferredMaxReplicas + status: "False" + type: ScaledUpBasedOnPreferredMaxReplicas + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + message: HPA target utilization is updated + reason: HPATargetUtilizationUpdated + status: "True" + type: HPATargetUtilizationUpdated + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + message: The recommendation is provided + status: "True" + type: VerticalRecommendationUpdated + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + status: "False" + type: FailedToReconcile + containerResourcePhases: + - containerName: app + resourcePhases: + cpu: + lastTransitionTime: null + phase: Working + memory: + lastTransitionTime: "2023-01-01T00:00:00Z" + phase: Working + - containerName: istio-proxy + resourcePhases: + cpu: + lastTransitionTime: null + phase: Working + memory: + lastTransitionTime: "2023-01-01T00:00:00Z" + phase: Working + recommendations: + horizontal: + maxReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-01-01T00:00:00Z" + value: 20 + minReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-01-01T00:00:00Z" + value: 5 + targetUtilizations: + - containerName: app + targetUtilization: + cpu: 50 + - containerName: istio-proxy + targetUtilization: + cpu: 75 + vertical: + containerResourceRecommendation: + - RecommendedResource: + cpu: "6" + memory: 3Gi + containerName: app + - RecommendedResource: + cpu: "4" + memory: 3Gi + containerName: istio-proxy + targets: + horizontalPodAutoscaler: tortoise-hpa-mercari + scaleTargetRef: + kind: "" + name: "" + verticalPodAutoscalers: + - name: tortoise-monitor-mercari + role: Monitor + tortoisePhase: Working diff --git a/cmd/tortoisectl/test/testdata/success/before/deployment.yaml b/cmd/tortoisectl/test/testdata/success/before/deployment.yaml new file mode 100644 index 00000000..96d10022 --- /dev/null +++ b/cmd/tortoisectl/test/testdata/success/before/deployment.yaml @@ -0,0 +1,30 @@ +metadata: + name: mercari-app + namespace: default +spec: + selector: + matchLabels: + app: mercari + strategy: {} + template: + metadata: + annotations: + kubectl.kubernetes.io/restartedAt: "2023-01-01T00:00:00Z" + creationTimestamp: null + labels: + app: mercari + spec: + containers: + - image: awesome-mercari-app-image + name: app + resources: + requests: + cpu: "10" + memory: 10Gi + - image: awesome-istio-proxy-image + name: istio-proxy + resources: + requests: + cpu: "4" + memory: 4Gi +status: {} diff --git a/cmd/tortoisectl/test/testdata/success/before/tortoise.yaml b/cmd/tortoisectl/test/testdata/success/before/tortoise.yaml new file mode 100644 index 00000000..794f5d48 --- /dev/null +++ b/cmd/tortoisectl/test/testdata/success/before/tortoise.yaml @@ -0,0 +1,143 @@ +metadata: + finalizers: + - tortoise.autoscaling.mercari.com/finalizer + name: mercari + namespace: default +spec: + updateMode: "Auto" + targetRefs: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app +status: + autoscalingPolicy: + - containerName: app + policy: + cpu: Horizontal + memory: Vertical + - containerName: istio-proxy + policy: + cpu: Horizontal + memory: Vertical + conditions: + containerRecommendationFromVPA: + - containerName: app + maxRecommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + recommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + - containerName: istio-proxy + maxRecommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + recommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + containerResourceRequests: + - containerName: app + resource: + cpu: "6" + memory: 3Gi + - containerName: istio-proxy + resource: + cpu: "4" + memory: 3Gi + tortoiseConditions: + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + message: the current number of replicas is not bigger than the preferred max + replica number + reason: ScaledUpBasedOnPreferredMaxReplicas + status: "False" + type: ScaledUpBasedOnPreferredMaxReplicas + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + message: HPA target utilization is updated + reason: HPATargetUtilizationUpdated + status: "True" + type: HPATargetUtilizationUpdated + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + message: The recommendation is provided + status: "True" + type: VerticalRecommendationUpdated + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + status: "False" + type: FailedToReconcile + containerResourcePhases: + - containerName: app + resourcePhases: + cpu: + lastTransitionTime: null + phase: Working + memory: + lastTransitionTime: "2023-01-01T00:00:00Z" + phase: Working + - containerName: istio-proxy + resourcePhases: + cpu: + lastTransitionTime: null + phase: Working + memory: + lastTransitionTime: "2023-01-01T00:00:00Z" + phase: Working + recommendations: + horizontal: + maxReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-01-01T00:00:00Z" + value: 20 + minReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-01-01T00:00:00Z" + value: 5 + targetUtilizations: + - containerName: app + targetUtilization: + cpu: 50 + - containerName: istio-proxy + targetUtilization: + cpu: 75 + vertical: + containerResourceRecommendation: + - RecommendedResource: + cpu: "6" + memory: 3Gi + containerName: app + - RecommendedResource: + cpu: "4" + memory: 3Gi + containerName: istio-proxy + targets: + horizontalPodAutoscaler: tortoise-hpa-mercari + scaleTargetRef: + kind: "" + name: "" + verticalPodAutoscalers: + - name: tortoise-monitor-mercari + role: Monitor + tortoisePhase: Working diff --git a/cmd/tortoisectl/test/tortoisectl_test.go b/cmd/tortoisectl/test/tortoisectl_test.go new file mode 100644 index 00000000..3eb4e005 --- /dev/null +++ b/cmd/tortoisectl/test/tortoisectl_test.go @@ -0,0 +1,290 @@ +package tortoisectl_test + +import ( + "bytes" + "context" + "errors" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/mercari/tortoise/api/v1beta3" + appv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/yaml" +) + +var ( + update = flag.Bool("update", false, "Whether to update current snapshots instead of testing") +) + +func TestMain(m *testing.M) { + flag.Parse() + + os.Exit(m.Run()) +} + +func buildTortoiseCtl(t *testing.T) { + t.Helper() + + execCommand(t, "go", "build", "-o", "./testdata/bin/tortoisectl", "../main.go") +} + +func prepareCluster(t *testing.T) (*envtest.Environment, *rest.Config) { + t.Helper() + + testEnv := &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + // cfg is defined in this file globally. + cfg, err := testEnv.Start() + if err != nil { + t.Fatalf("Failed to start test environment: %v", err) + } + + return testEnv, cfg +} + +func destryCluster(t *testing.T, testEnv *envtest.Environment) { + t.Helper() + + err := testEnv.Stop() + if err != nil { + t.Fatalf("Failed to destroy test environment: %v", err) + } +} + +func Test_TortoiseCtlStop(t *testing.T) { + // Build the latest binary. + buildTortoiseCtl(t) + testEnv, cfg := prepareCluster(t) + defer destryCluster(t, testEnv) + + // create the clientset + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + t.Fatalf("Failed to create clientset: %v", err) + } + + scheme := runtime.NewScheme() + err = v1beta3.AddToScheme(scheme) + if err != nil { + t.Fatalf("Failed to add scheme: %v", err) + } + kubeconfig := strings.Split(testEnv.ControlPlane.KubeCtl().Opts[0], "=")[1] + + tortoiseclient, err := client.New(cfg, client.Options{ + Scheme: scheme, + }) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + + tests := []struct { + name string + options []string + dir string + }{ + { + name: "stop tortoise successfully", + options: []string{}, + dir: "success", + }, + } + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + namespace := fmt.Sprintf("test-%d", i) + // Create a namespace + _, err := clientset.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create namespace: %v", err) + } + + deploymentYaml := fmt.Sprintf("./testdata/%s/before/deployment.yaml", tt.dir) + y, err := os.ReadFile(deploymentYaml) + if err != nil { + t.Fatalf("Failed to read deployment yaml: %v", err) + } + deploy := &appv1.Deployment{} + err = yaml.Unmarshal(y, deploy) + if err != nil { + t.Fatalf("Failed to unmarshal deployment yaml: %v", err) + } + + // Create a deployment + deploy.Namespace = namespace + _, err = clientset.AppsV1().Deployments(namespace).Create(context.Background(), deploy, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create deployment: %v", err) + } + + tortoiseYaml := fmt.Sprintf("./testdata/%s/before/tortoise.yaml", tt.dir) + y, err = os.ReadFile(tortoiseYaml) + if err != nil { + t.Fatalf("Failed to read tortoise yaml: %v", err) + } + tortoise := &v1beta3.Tortoise{} + err = yaml.Unmarshal(y, tortoise) + if err != nil { + t.Fatalf("Failed to unmarshal tortoise yaml: %v", err) + } + + // Create a tortoise + tortoise.Namespace = namespace + status := tortoise.DeepCopy().Status + err = tortoiseclient.Create(context.Background(), tortoise) + if err != nil { + t.Fatalf("Failed to create tortoise: %v", err) + } + + v := &v1beta3.Tortoise{} + err = tortoiseclient.Get(context.Background(), client.ObjectKey{Namespace: tortoise.Namespace, Name: tortoise.Name}, v) + if err != nil { + t.Fatalf("Failed to get tortoise: %v", err) + } + v.Status = status + err = tortoiseclient.Status().Update(context.Background(), v) + if err != nil { + t.Fatalf("Failed to update tortoise status: %v", err) + } + + out, _ := execCommand(t, "./testdata/bin/tortoisectl", "stop", "--kubeconfig", kubeconfig, "--namespace", namespace) + t.Log(out) + + deploy, err = clientset.AppsV1().Deployments(namespace).Get(context.Background(), deploy.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Failed to get deployment: %v", err) + } + + err = tortoiseclient.Get(context.Background(), client.ObjectKeyFromObject(tortoise), tortoise) + if err != nil { + t.Fatalf("Failed to get tortoise: %v", err) + } + + wantDeployment := fmt.Sprintf("./testdata/%s/after/deployment.yaml", tt.dir) + wantTortoiseYaml := fmt.Sprintf("./testdata/%s/after/tortoise.yaml", tt.dir) + if *update { + err = writeToFile(wantDeployment, deploy) + if err != nil { + t.Fatalf("Failed to write deployment yaml: %v", err) + } + + err = writeToFile(wantTortoiseYaml, tortoise) + if err != nil { + t.Fatalf("Failed to write tortoise yaml: %v", err) + } + + return + } + + y, err = os.ReadFile(wantDeployment) + if err != nil { + t.Fatalf("Failed to read deployment yaml: %v", err) + } + wantDeploy := &appv1.Deployment{} + err = yaml.Unmarshal(y, wantDeploy) + if err != nil { + t.Fatalf("Failed to decode deployment yaml: %v", err) + } + + diff := cmp.Diff(wantDeploy, deploy) + if diff != "" { + t.Fatalf("Deployment mismatch (-want +got):\n%s", diff) + } + + y, err = os.ReadFile(wantTortoiseYaml) + if err != nil { + t.Fatalf("Failed to read tortoise yaml: %v", err) + } + wantTortoise := &v1beta3.Tortoise{} + err = yaml.Unmarshal(y, wantTortoise) + if err != nil { + t.Fatalf("Failed to decode tortoise yaml: %v", err) + } + + diff = cmp.Diff(wantTortoise, tortoise) + if diff != "" { + t.Fatalf("Tortoise mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func writeToFile(path string, r any) error { + y, err := yaml.Marshal(r) + if err != nil { + return err + } + y, err = removeUnnecessaryFields(y) + if err != nil { + return err + } + err = os.WriteFile(path, y, 0644) + if err != nil { + return err + } + + return nil +} + +func removeUnnecessaryFields(rawdata []byte) ([]byte, error) { + data := make(map[string]interface{}) + err := yaml.Unmarshal(rawdata, &data) + if err != nil { + return nil, err + } + meta, ok := data["metadata"] + if !ok { + return nil, errors.New("no metadata") + } + typed, ok := meta.(map[string]any) + if !ok { + return nil, fmt.Errorf("metadata is unexpected type: %T", meta) + } + + delete(typed, "creationTimestamp") + delete(typed, "managedFields") + delete(typed, "resourceVersion") + delete(typed, "uid") + delete(typed, "generation") + + return yaml.Marshal(data) +} + +func execCommand(t *testing.T, s ...string) (string, string) { + t.Helper() + cmd := exec.Command(s[0], s[1:]...) + fmt.Println(cmd.String()) + + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + t.Logf("Running command: %s\n", cmd.String()) + err := cmd.Run() + if err != nil { + t.Errorf("Stdout: %s\n", stdout.String()) + t.Errorf("Stderr: %s\n", stderr.String()) + t.Fatalf("cmd.Run() failed: %s\n", err.Error()) + } + + return stdout.String(), stderr.String() +} diff --git a/go.mod b/go.mod index be38279f..b34fb85c 100644 --- a/go.mod +++ b/go.mod @@ -23,11 +23,17 @@ require ( sigs.k8s.io/yaml v1.3.0 ) +require ( + github.com/kyokomi/emoji/v2 v2.2.12 + github.com/spf13/cobra v1.7.0 + sigs.k8s.io/e2e-framework v0.3.0 +) + require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.10.1 // indirect + github.com/emicklei/go-restful/v3 v3.10.2 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect @@ -43,11 +49,13 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/imdario/mergo v0.3.6 // indirect + github.com/imdario/mergo v0.3.15 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -57,18 +65,19 @@ require ( github.com/prometheus/procfs v0.11.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.8.3 // indirect + github.com/vladimirvivien/gexe v0.2.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/net v0.20.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/term v0.16.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.4.0 // indirect golang.org/x/tools v0.9.1 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/apiextensions-apiserver v0.27.2 // indirect diff --git a/go.sum b/go.sum index 36149e0f..8aa5f79e 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -7,12 +9,13 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= -github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE= +github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= @@ -53,9 +56,12 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= +github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -71,10 +77,14 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kyokomi/emoji/v2 v2.2.12 h1:sSVA5nH9ebR3Zji1o31wu3yOwD1zKXQA2z0zUyeit60= +github.com/kyokomi/emoji/v2 v2.2.12/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -101,6 +111,9 @@ github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwa github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -114,6 +127,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/vladimirvivien/gexe v0.2.0 h1:nbdAQ6vbZ+ZNsolCgSVb9Fno60kzSuvtzVh6Ytqi/xY= +github.com/vladimirvivien/gexe v0.2.0/go.mod h1:LHQL00w/7gDUKIak24n801ABp8C+ni6eBht9vGVst8w= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -144,8 +159,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -161,16 +176,16 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.4.0 h1:Z81tqI5ddIoXDPvVQ7/7CC9TnLM7ubaFG2qXYd5BbYY= golang.org/x/time v0.4.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -191,8 +206,8 @@ google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -225,6 +240,8 @@ k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCf k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.15.1 h1:9UvgKD4ZJGcj24vefUFgZFP3xej/3igL9BsOUTb/+4c= sigs.k8s.io/controller-runtime v0.15.1/go.mod h1:7ngYvp1MLT+9GeZ+6lH3LOlcHkp/+tzA/fmHa4iq9kk= +sigs.k8s.io/e2e-framework v0.3.0 h1:eqQALBtPCth8+ulTs6lcPK7ytV5rZSSHJzQHZph4O7U= +sigs.k8s.io/e2e-framework v0.3.0/go.mod h1:C+ef37/D90Dc7Xq1jQnNbJYscrUGpxrWog9bx2KIa+c= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= diff --git a/pkg/pod/pod.go b/pkg/pod/pod.go index 07c409ec..43a9fff7 100644 --- a/pkg/pod/pod.go +++ b/pkg/pod/pod.go @@ -42,7 +42,13 @@ func New( }, nil } -func (s *Service) ModifyPodResource(pod *v1.Pod, t *v1beta3.Tortoise) { +type ModifyPodSpecResourceOption string + +var ( + NoScaleDown ModifyPodSpecResourceOption = "NoScaleDown" +) + +func (s *Service) ModifyPodSpecResource(podSpec *v1.PodSpec, t *v1beta3.Tortoise, opts ...ModifyPodSpecResourceOption) { if t.Spec.UpdateMode == v1beta3.UpdateModeOff || t.Status.TortoisePhase == "" || t.Status.TortoisePhase == v1beta3.TortoisePhaseInitializing || @@ -56,22 +62,26 @@ func (s *Service) ModifyPodResource(pod *v1.Pod, t *v1beta3.Tortoise) { newRequestsMap := map[containerNameAndResource]resource.Quantity{} // Update resource requests based on the tortoise.Status.Conditions.ContainerResourceRequests - for i, container := range pod.Spec.Containers { + for i, container := range podSpec.Containers { for k, oldReq := range container.Resources.Requests { newReq, ok := utils.GetRequestFromTortoise(t, container.Name, k) if !ok { // Unchange, just store the old value as a new value newReq = oldReq } + if containsOption(opts, NoScaleDown) && newReq.Cmp(oldReq) < 0 { + // If NoScaleDown option is specified, don't scale down the resource request. + newReq = oldReq + } oldRequestsMap[containerNameAndResource{containerName: container.Name, resourceName: k}] = oldReq newRequestsMap[containerNameAndResource{containerName: container.Name, resourceName: k}] = newReq - pod.Spec.Containers[i].Resources.Requests[k] = newReq + podSpec.Containers[i].Resources.Requests[k] = newReq requestChangeRatio[containerNameAndResource{containerName: container.Name, resourceName: k}] = float64(newReq.MilliValue()) / float64(oldReq.MilliValue()) } } // Update resource limits - for i, container := range pod.Spec.Containers { + for i, container := range podSpec.Containers { if container.Resources.Limits == nil { container.Resources.Limits = make(v1.ResourceList) } @@ -98,12 +108,12 @@ func (s *Service) ModifyPodResource(pod *v1.Pod, t *v1beta3.Tortoise) { if k == v1.ResourceCPU && newLim.Cmp(s.minimumCPULimit) < 0 { newLim = ptr.To(s.minimumCPULimit.DeepCopy()) } - pod.Spec.Containers[i].Resources.Limits[k] = *newLim + podSpec.Containers[i].Resources.Limits[k] = *newLim } } // Update GOMEMLIMIT and GOMAXPROCS - for i, container := range pod.Spec.Containers { + for i, container := range podSpec.Containers { for j, env := range container.Env { if env.Name == "GOMAXPROCS" { // e.g., If CPU is increased twice, GOMAXPROCS should be doubled. @@ -126,7 +136,7 @@ func (s *Service) ModifyPodResource(pod *v1.Pod, t *v1beta3.Tortoise) { newUncapedNum := float64(oldNum) * changeRatio // GOMAXPROCS should be an integer. newNum := int(math.Ceil(newUncapedNum)) - pod.Spec.Containers[i].Env[j].Value = strconv.Itoa(newNum) + podSpec.Containers[i].Env[j].Value = strconv.Itoa(newNum) } @@ -166,7 +176,7 @@ func (s *Service) ModifyPodResource(pod *v1.Pod, t *v1beta3.Tortoise) { } // See GOMEMLIMIT's format: https://pkg.go.dev/runtime#hdr-Environment_Variables newNum := int(float64(oldNum.Value()) * changeRatio) - pod.Spec.Containers[i].Env[j].Value = strconv.Itoa(newNum) + podSpec.Containers[i].Env[j].Value = strconv.Itoa(newNum) } } } @@ -216,3 +226,12 @@ type containerNameAndResource struct { containerName string resourceName v1.ResourceName } + +func containsOption(opts []ModifyPodSpecResourceOption, opt ModifyPodSpecResourceOption) bool { + for _, o := range opts { + if o == opt { + return true + } + } + return false +} diff --git a/pkg/pod/pod_test.go b/pkg/pod/pod_test.go index 63b0610e..20cab038 100644 --- a/pkg/pod/pod_test.go +++ b/pkg/pod/pod_test.go @@ -13,7 +13,7 @@ import ( "github.com/mercari/tortoise/pkg/features" ) -func TestService_ModifyPodResource(t *testing.T) { +func TestService_ModifyPodSpecResource(t *testing.T) { type fields struct { resourceLimitMultiplier map[string]int64 minimumCPULimit string @@ -22,6 +22,7 @@ func TestService_ModifyPodResource(t *testing.T) { type args struct { pod *v1.Pod tortoise *v1beta3.Tortoise + opts []ModifyPodSpecResourceOption } tests := []struct { name string @@ -415,6 +416,69 @@ func TestService_ModifyPodResource(t *testing.T) { }, }, }, + { + name: "Tortoise is Auto; NoScaleDown option", + args: args{ + opts: []ModifyPodSpecResourceOption{NoScaleDown}, + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container", + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("300m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + }, + }, + }, + }, + }, + tortoise: &v1beta3.Tortoise{ + Spec: v1beta3.TortoiseSpec{ + UpdateMode: v1beta3.UpdateModeAuto, + }, + Status: v1beta3.TortoiseStatus{ + TortoisePhase: v1beta3.TortoisePhaseWorking, + Conditions: v1beta3.Conditions{ + ContainerResourceRequests: []v1beta3.ContainerResourceRequests{ + { + ContainerName: "container", + Resource: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), // scale up + v1.ResourceMemory: resource.MustParse("1Mi"), // scale down + }, + }, + }, + }, + }, + }, + }, + want: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container", + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), // scale up + v1.ResourceMemory: resource.MustParse("100Mi"), // scale down is ignored + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("600m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + }, + }, + }, + }, + }, + }, { name: "Tortoise is Auto; hits resourceLimitMultiplier", fields: fields{ @@ -895,8 +959,8 @@ func TestService_ModifyPodResource(t *testing.T) { t.Fatalf("New() error = %v", err) } got := tt.args.pod.DeepCopy() - s.ModifyPodResource(got, tt.args.tortoise) - if d := cmp.Diff(got, tt.want); d != "" { + s.ModifyPodSpecResource(&got.Spec, tt.args.tortoise, tt.args.opts...) + if d := cmp.Diff(got.Spec, tt.want.Spec); d != "" { t.Errorf("ModifyPodResource() mismatch (-want +got):\n%s", d) } }) diff --git a/pkg/stoper/stoper.go b/pkg/stoper/stoper.go new file mode 100644 index 00000000..3eefe1d3 --- /dev/null +++ b/pkg/stoper/stoper.go @@ -0,0 +1,170 @@ +package stoper + +import ( + "context" + "errors" + "fmt" + "io" + "reflect" + "time" + + "github.com/kyokomi/emoji/v2" + "github.com/mercari/tortoise/api/v1beta3" + "github.com/mercari/tortoise/pkg/deployment" + "github.com/mercari/tortoise/pkg/pod" + v1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Stopr is the struct for stopping tortoise safely. +type Stopr struct { + c client.Client + + deploymentService *deployment.Service + podService *pod.Service +} + +func New(c client.Client, ds *deployment.Service, ps *pod.Service) *Stopr { + return &Stopr{ + c: c, + deploymentService: ds, + podService: ps, + } +} + +type StoprOption string + +var ( + NoLoweringResource StoprOption = "NoLoweringResource" +) + +func (s *Stopr) Stop(ctx context.Context, tortoiseNames []string, namespace string, all bool, writer io.Writer, opts ...StoprOption) error { + // It assumes the validation is already done in the CLI layer. + + targets := []types.NamespacedName{} + if all { + tortoises := &v1beta3.TortoiseList{} + opt := &client.ListOptions{} + if namespace != "" { + // stop all tortoises in the namespace + opt.Namespace = namespace + } + + if err := s.c.List(ctx, tortoises); err != nil { + return fmt.Errorf("failed to list tortoises: %w", err) + } + + for _, t := range tortoises.Items { + targets = append(targets, types.NamespacedName{Name: t.Name, Namespace: t.Namespace}) + } + } else { + for _, name := range tortoiseNames { + targets = append(targets, types.NamespacedName{Name: name, Namespace: namespace}) + } + } + + var finalerr error + for _, target := range targets { + writer.Write([]byte(fmt.Sprintf("%s stopping your tortoise %s...\n", emoji.Sprint(":turtle:"), &target))) + + // 1. Stop Tortoise. + tortoise, err := s.stopOne(ctx, target) + if err != nil { + if errors.Is(err, errTortoiseAlreadyStopped) { + writer.Write([]byte(fmt.Sprintf("%s this tortoise %s is already stopped...\n", emoji.Sprint(":sleeping:"), &target))) + continue + } + finalerr = errors.Join(finalerr, err) + writer.Write([]byte(fmt.Sprintf("%s failed to stop your tortoise %s.\nError: %v\n", emoji.Sprint(":face_with_spiral_eyes:"), &target, err))) + continue + } + writer.Write([]byte(fmt.Sprintf("%s stopped your tortoise %s\n", emoji.Sprint(":sleeping:"), &target))) + + // 2. Get the target deployment. + dp, err := s.deploymentService.GetDeploymentOnTortoise(ctx, tortoise) + if err != nil { + finalerr = errors.Join(finalerr, err) + writer.Write([]byte(fmt.Sprintf("%s failed to get deployment on your tortoise %s.\nError: %v\n", emoji.Sprint(":face_with_spiral_eyes:"), &target, err))) + continue + } + + // 3. [when NoLoweringResource is true] Patch the deployment to keep the resource requests high. + if containsOption(opts, NoLoweringResource) { + writer.Write([]byte(fmt.Sprintf("%s patching your deployment to keep the resource requests high...\n", emoji.Sprint(":hammer_and_wrench:")))) + updated, err := s.patchDeploymentToKeepResources(ctx, dp, tortoise) + if err != nil { + finalerr = errors.Join(finalerr, err) + writer.Write([]byte(fmt.Sprintf("%s failed to patch your deployment %s.\nError: %v\n", emoji.Sprint(":face_with_spiral_eyes:"), &target, err))) + continue + } + if updated { + // If the deployment is updated, we don't need to restart the deployment. + continue + } + } + + // 4. Restart the deployment to get back the original resource requests. + writer.Write([]byte(fmt.Sprintf("%s restarting your deployment to get back the original resource...\n", emoji.Sprint(":sleeping:")))) + if err := s.deploymentService.RolloutRestart(ctx, dp, tortoise, time.Now()); err != nil { + finalerr = errors.Join(finalerr, err) + writer.Write([]byte(fmt.Sprintf("%s failed to restart your deployment %s.\nError: %v\n", emoji.Sprint(":face_with_spiral_eyes:"), &target, err))) + continue + } + writer.Write([]byte(fmt.Sprintf("%s restarted your deployment and your Pods should get back the original resource soon.\n", emoji.Sprint(":muscle:")))) + } + + return finalerr +} + +func containsOption(opts []StoprOption, opt StoprOption) bool { + for _, o := range opts { + if o == opt { + return true + } + } + return false +} + +// patchDeploymentToKeepResources patches the deployment to keep the resource requests high. +// The first return value indicates whether the deployment is updated or not. +func (s *Stopr) patchDeploymentToKeepResources(ctx context.Context, dp *v1.Deployment, tortoise *v1beta3.Tortoise) (bool, error) { + originalDP := dp.DeepCopy() + + // Set to Auto because ModifyPodSpecResource doesn't change anything if it's set to Off. + tortoise.Spec.UpdateMode = v1beta3.UpdateModeAuto + s.podService.ModifyPodSpecResource(&dp.Spec.Template.Spec, tortoise, pod.NoScaleDown) + tortoise.Spec.UpdateMode = v1beta3.UpdateModeOff + // If not updated, early return + if reflect.DeepEqual(originalDP.Spec.Template.Spec.Containers, dp.Spec.Template.Spec.Containers) { + return false, nil + } + + if err := s.c.Update(ctx, dp); err != nil { + return false, fmt.Errorf("failed to update deployment: %w", err) + } + + return true, nil +} + +var errTortoiseAlreadyStopped = errors.New("tortoise is already stopped") + +// stopOne disables the tortoise. +func (s *Stopr) stopOne(ctx context.Context, target types.NamespacedName) (*v1beta3.Tortoise, error) { + t := &v1beta3.Tortoise{} + if err := s.c.Get(ctx, target, t); err != nil { + return nil, fmt.Errorf("failed to get tortoise: %w", err) + } + + if t.Spec.UpdateMode == v1beta3.UpdateModeOff { + return t, errTortoiseAlreadyStopped + } + + t.Spec.UpdateMode = v1beta3.UpdateModeOff + + if err := s.c.Update(ctx, t); err != nil { + return nil, fmt.Errorf("failed to update tortoise: %w", err) + } + + return t, nil +}