Run only upgrades e2e test for nightlies

Makefile

@@ -114,8 +114,6 @@ test/e2e/k3d-%:
 test/e2e/testrunner-eks:
 	KUBECONFIG=$(TMPDIR)/$(CLUSTER_NAME)-eks.kubeconfig.yaml bazel-bin/hack/bin/kubectl create -f hack/eks-storageclass.yaml
 	bazel test --stamp //e2e/upgrades/...  --action_env=KUBECONFIG=$(TMPDIR)/$(CLUSTER_NAME)-eks.kubeconfig.yaml
-	bazel test --stamp //e2e/create/...  --action_env=KUBECONFIG=$(TMPDIR)/$(CLUSTER_NAME)-eks.kubeconfig.yaml
-	bazel test --stamp //e2e/decommission/...  --action_env=KUBECONFIG=$(TMPDIR)/$(CLUSTER_NAME)-eks.kubeconfig.yaml
 
 # Use this target to run e2e tests with a eks cluster.
 # This target uses kubetest2 to start a eks k8s cluster and runs the e2e tests
@@ -143,9 +141,6 @@ test/e2e/testrunner-gke:
 	#bazel run --stamp --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 \
 	#	//manifests:install_operator.apply
 	bazel test --stamp //e2e/upgrades/...
-	bazel test --stamp //e2e/create/...
-	bazel test --stamp --test_arg=--pvc=true //e2e/pvcresize/...
-	bazel test --stamp //e2e/decommission/...
 
 # Use this target to run e2e tests with a gke cluster.
 # This target uses kubetest2 to start a gke k8s cluster and runs the e2e tests
@@ -171,8 +166,6 @@ test/e2e/gke:
 .PHONY: test/e2e/testrunner-openshift
 test/e2e/testrunner-openshift:
 	bazel test --stamp //e2e/upgrades/...  --action_env=KUBECONFIG=$(HOME)/openshift-$(CLUSTER_NAME)/auth/kubeconfig
-	bazel test --stamp //e2e/create/...  --action_env=KUBECONFIG=$(HOME)/openshift-$(CLUSTER_NAME)/auth/kubeconfig
-	bazel test --stamp //e2e/decommission/...  --action_env=KUBECONFIG=$(HOME)/openshift-$(CLUSTER_NAME)/auth/kubeconfig
 
 # Use this target to run e2e tests with a openshift cluster.
 # This target uses kubetest2 to start a openshift cluster and runs the e2e tests

hack/bin/BUILD.bazel

@@ -143,38 +143,16 @@ genrule(
 genrule(
     name = "fetch_kubetest2",
     srcs = select({
-        ":m1": ["@kubetest2_darwin//file"],
-        ":darwin": ["@kubetest2_darwin//file"],
-        ":k8": ["@kubetest2_linux//file"],
+        ":m1": ["@kubetest2_darwin//:files"],
+        ":darwin": ["@kubetest2_darwin//:files"],
+        ":k8": ["@kubetest2_linux//:files"],
     }),
-    outs = ["kubetest2"],
-    cmd = "cp $(SRCS) $@",
-    visibility = ["//visibility:public"],
-)
-
-# fetch_kubetest2_gke rules to fetch the binary for kubetest2_gke used on e2e tests that are saved on a google cloud bucket
-genrule(
-    name = "fetch_kubetest2_gke",
-    srcs = select({
-        ":m1": ["@kubetest2_gke_darwin//file"],
-        ":darwin": ["@kubetest2_gke_darwin//file"],
-        ":k8": ["@kubetest2_gke_linux//file"],
-    }),
-    outs = ["kubetest2-gke"],
-    cmd = "cp $(SRCS) $@",
-    visibility = ["//visibility:public"],
-)
-
-# fetch_kubetest2_exe`rules to fetch the binary for kubetest2_exe used on e2e tests that are saved on a google cloud bucket
-genrule(
-    name = "fetch_kubetest2_exe",
-    srcs = select({
-        ":m1": ["@kubetest2_exe_darwin//file"],
-        ":darwin": ["@kubetest2_exe_darwin//file"],
-        ":k8": ["@kubetest2_exe_linux//file"],
-    }),
-    outs = ["kubetest2-tester-exec"],
-    cmd = "cp $(SRCS) $@",
+    outs = [
+        "kubetest2",
+        "kubetest2-gke",
+        "kubetest2-tester-exec",
+    ],
+    cmd = "cp $(SRCS) $(@D)",
     visibility = ["//visibility:public"],
 )
 

hack/bin/deps.bzl

@@ -64,8 +64,6 @@ def install():
     install_k3d()
     install_kubetest2()
     install_kubetest2_aws()
-    install_kubetest2_exe()
-    install_kubetest2_gke()
     install_kustomize()
     install_oc()
     install_operator_sdk()
@@ -266,54 +264,31 @@ filegroup(
 
 ## Fetch kubetest2 binary used during e2e tests
 def install_kubetest2():
-    # install kubetest2 binary
-    http_file(
-       name = "kubetest2_darwin",
-       executable = 1,
-       sha256 = "5b20aadd05eca47dead180a7c8296d75e81c184aabf182d4a41ef96597db543d",
-       urls = ["https://storage.googleapis.com/crdb-bazel-artifacts/osx/kubetest2"],
-    )
+    url = "https://github.com/pseudomuto/kubetest2/releases/download/v0.1.0/kubetest2_0.1.0_{}_amd64.tar.gz"
+    content = """
+filegroup(
+    name = "files",
+    srcs = [
+        "kubetest2",
+        "kubetest2-gke",
+        "kubetest2-tester-exec",
+    ],
+    visibility = ["//visibility:public"],
+)
+"""
 
-    http_file(
+    http_archive(
         name = "kubetest2_linux",
-        executable = 1,
-        sha256 = "7f0b05654fa43ca1c607db297b5f3a775f65eea90355bb6b10137a7fffff5e1a",
-        urls = ["https://storage.googleapis.com/crdb-bazel-artifacts/linux/kubetest2"],
+        sha256 = "04319fb9cef1a5370548c47a0c65fa8d4f85a1b6d4417b318a93125d72590d1b",
+        urls = [url.format("linux")],
+        build_file_content = content,
     )
 
-## Fetch kubetest2-gke binary used during e2e tests
-def install_kubetest2_gke():
-    # install kubetest2-gke binary
-    # TODO osx support
-    http_file(
-       name = "kubetest2_gke_darwin",
-       executable = 1,
-       sha256 = "a1cbe02f61931dbe6c8d1662442f42cb538c81e4ec8cdd40f548f0e05cbd55a7",
-       urls = ["https://storage.googleapis.com/crdb-bazel-artifacts/osx/kubetest2-gke"],
-    )
-
-    http_file(
-        name = "kubetest2_gke_linux",
-        executable = 1,
-        sha256 = "9ac658234efc7f59968888662dd2d21908587789f6b812392ac5b6766b17c0b4",
-        urls = ["https://storage.googleapis.com/crdb-bazel-artifacts/linux/kubetest2-gke"],
-    )
-## Fetch kubetest2-tester-exe binary used during e2e tests
-def install_kubetest2_exe():
-    # install kubetest2-exe binary
-    # TODO osx support
-    http_file(
-       name = "kubetest2_exe_darwin",
-       executable = 1,
-       sha256 = "818690cb55590440e163b18dd139c8a8714df9480f869bafe19eb344047cf37c",
-       urls = ["https://storage.googleapis.com/crdb-bazel-artifacts/osx/kubetest2-tester-exec"],
-    )
-
-    http_file(
-        name = "kubetest2_exe_linux",
-        executable = 1,
-        sha256 = "4483f40f48b98e8a6aa41f58bfdf1f2787066a4e1ad1343e4281892aa1326736",
-        urls = ["https://storage.googleapis.com/crdb-bazel-artifacts/linux/kubetest2-tester-exec"],
+    http_archive(
+        name = "kubetest2_darwin",
+        sha256 = "972d639e9548f296c8470c3418630baa4176aabc2b2e5d47d62b9eb8ee2b91cf",
+        urls = [url.format("darwin")],
+        build_file_content = content,
     )
 
 ## Fetch operator-sdk used on generating csv

pkg/resource/BUILD.bazel

@@ -57,8 +57,8 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
-        "cluster_test.go",
         "certificate_test.go",
+        "cluster_test.go",
         "discovery_service_test.go",
         "pod_distruption_budget_test.go",
         "public_service_test.go",

pkg/testutil/env/BUILD.bazel

@@ -24,6 +24,7 @@ go_library(
         "//apis/v1alpha1:go_default_library",
         "//pkg/client/clientset/versioned:go_default_library",
         "//pkg/kube:go_default_library",
+        "@com_github_cenkalti_backoff//:go_default_library",
         "@com_github_cockroachdb_errors//:go_default_library",
         "@com_github_stretchr_testify//require:go_default_library",
         "@io_k8s_api//core/v1:go_default_library",

pkg/testutil/env/env.go

@@ -20,12 +20,14 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"time"
 
+	"github.com/cenkalti/backoff"
 	api "github.com/cockroachdb/cockroach-operator/apis/v1alpha1"
 	customClient "github.com/cockroachdb/cockroach-operator/pkg/client/clientset/versioned"
 	"github.com/cockroachdb/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	apiruntime "k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
@@ -40,15 +42,15 @@ import (
 // environments.
 var testBinaries = flag.String("binaries", "hack/bin", "")
 
-func NewEnv(builder apiruntime.SchemeBuilder) *Env {
+func NewEnv(builder runtime.SchemeBuilder) *Env {
 	flag.Parse()
 
 	// ensure hack/bin is added to the path and KUBEBUILDER_ASSETS
 	p := ExpandPath(*testBinaries)
 	os.Setenv("KUBEBUILDER_ASSETS", p)
 	PrependToPath(p)
 
-	scheme := apiruntime.NewScheme()
+	scheme := runtime.NewScheme()
 
 	if err := kscheme.AddToScheme(scheme); err != nil {
 		panic(err)
@@ -76,7 +78,7 @@ func NewEnv(builder apiruntime.SchemeBuilder) *Env {
 
 type Env struct {
 	envtest.Environment
-	Scheme *apiruntime.Scheme
+	Scheme *runtime.Scheme
 }
 
 func (env *Env) Start() *ActiveEnv {
@@ -129,7 +131,7 @@ func (k k8s) namespaceableResource(gvr schema.GroupVersionResource) dynamic.Name
 
 type ActiveEnv struct {
 	*k8s
-	scheme    *apiruntime.Scheme
+	scheme    *runtime.Scheme
 	resources []schema.GroupVersionResource
 }
 
@@ -139,8 +141,19 @@ func CreateActiveEnvForTest() *Env {
 }
 
 func loadResources(k *k8s) ([]schema.GroupVersionResource, error) {
-	lists, err := k.DiscoveryClient.ServerPreferredResources()
-	if err != nil {
+	b := backoff.NewExponentialBackOff()
+	b.MaxElapsedTime = 60 * time.Second
+	lists := make([]*metav1.APIResourceList, 0)
+
+	// GKE was having issues fetching api resources. Giving it some time
+	// (tested via time.Sleep(10*time.Second)) seemed to resolve this issue.
+	// Rather than a sleep call, we should retry until it's ready or ultimately
+	// fails.
+	if err := backoff.Retry(func() error {
+		var err error
+		lists, err = k.DiscoveryClient.ServerPreferredResources()
+		return err
+	}, b); err != nil {
 		return nil, errors.Wrap(err, "failed to fetch preferred server resource")
 	}