Merge pull request etcd-io#9599 from gyuho/test-parallel

*: parallelize integration tests with CPU parameters
jcalvert · Apr 20, 2018 · 353f938 · 353f938
2 parents 83f7f17 + afef4a5
commit 353f938
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 16 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -18,7 +18,9 @@ env:
   - TARGET=linux-amd64-build
   - TARGET=linux-amd64-unit
   - TARGET=linux-amd64-fmt
-  - TARGET=linux-amd64-integration
+  - TARGET=linux-amd64-integration-1-cpu
+  - TARGET=linux-amd64-integration-2-cpu
+  - TARGET=linux-amd64-integration-4-cpu
   - TARGET=linux-amd64-functional
   - TARGET=linux-386-build
   - TARGET=linux-386-unit
@@ -42,7 +44,11 @@ matrix:
   - go: tip
     env: TARGET=linux-amd64-fmt
   - go: tip
-    env: TARGET=linux-amd64-integration
+    env: TARGET=linux-amd64-integration-1-cpu
+  - go: tip
+    env: TARGET=linux-amd64-integration-2-cpu
+  - go: tip
+    env: TARGET=linux-amd64-integration-4-cpu
   - go: tip
     env: TARGET=linux-amd64-functional
   - go: tip
@@ -87,10 +93,20 @@ script:
           --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
           /bin/bash -c "GOARCH=amd64 PASSES='fmt bom dep' ./test"
         ;;
-      linux-amd64-integration)
+      linux-amd64-integration-1-cpu)
+        docker run --rm \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
+          /bin/bash -c "GOARCH=amd64 CPU=1 PASSES='integration' ./test"
+        ;;
+      linux-amd64-integration-2-cpu)
+        docker run --rm \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
+          /bin/bash -c "GOARCH=amd64 CPU=2 PASSES='integration' ./test"
+        ;;
+      linux-amd64-integration-4-cpu)
         docker run --rm \
           --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=amd64 PASSES='integration' ./test"
+          /bin/bash -c "GOARCH=amd64 CPU=4 PASSES='integration' ./test"
         ;;
       linux-amd64-functional)
         docker run --rm \

diff --git a/test b/test
@@ -101,6 +101,13 @@ if [ -z "$GOARCH" ]; then
 	GOARCH=$(go env GOARCH);
 fi
 
+# determine the number of CPUs to use for Go tests
+TEST_CPUS="1,2,4"
+if [ ! -z "${CPU}" ]; then
+	TEST_CPUS="${CPU}"
+fi
+echo "Running with" "${TEST_CPUS}"
+
 # determine whether target supports race detection
 if [ "$GOARCH" == "amd64" ]; then
 	RACE="--race"
@@ -134,7 +141,7 @@ function unit_pass {
 	else
 		USERTIMEOUT="${TIMEOUT}"
 	fi
-	go test ${GO_TEST_FLAG} -timeout "${USERTIMEOUT}"  "${COVER}" ${RACE} -cpu 1,2,4 ${RUN_ARG} "$@" "${TEST[@]}"
+	go test ${GO_TEST_FLAG} -timeout "${USERTIMEOUT}"  "${COVER}" ${RACE} -cpu "${TEST_CPUS}" ${RUN_ARG} "$@" "${TEST[@]}"
 }
 
 function integration_pass {
@@ -154,7 +161,7 @@ function integration_pass {
 	# if TESTCASE not set, PKG set, run all test cases in specified package
 	# if TESTCASE not set, PKG not set, run all tests in all integration and integration_extra packages
 	if [ -z "${TESTCASE}" ] && [ -z "${USERPKG}" ]; then
-		go test -timeout "${USERTIMEOUT}" -v -cpu 1,2,4 "$@" "${REPO_PATH}/integration"
+		go test -timeout "${USERTIMEOUT}" -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/integration"
 		integration_extra "$@"
 	else
 		if [ -z "${USERPKG}" ]; then
@@ -166,16 +173,16 @@ function integration_pass {
 		else
 			INTEGTESTPKG=("${TEST[@]}")
 		fi
-		go test -timeout "${USERTIMEOUT}" -v -cpu 1,2,4 "${RUN_ARG}"  "$@" "${INTEGTESTPKG[@]}"
+		go test -timeout "${USERTIMEOUT}" -v -cpu "${TEST_CPUS}" "${RUN_ARG}"  "$@" "${INTEGTESTPKG[@]}"
 	fi
 }
 
 function integration_extra {
-	go test -timeout 1m -v ${RACE} -cpu 1,2,4 "$@" "${REPO_PATH}/client/integration"
-	go test -timeout 25m -v ${RACE} -cpu 1,2,4 "$@" "${REPO_PATH}/clientv3/integration"
-	go test -timeout 1m -v -cpu 1,2,4 "$@" "${REPO_PATH}/contrib/raftexample"
+	go test -timeout 1m -v ${RACE} -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/client/integration"
+	go test -timeout 25m -v ${RACE} -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/clientv3/integration"
+	go test -timeout 1m -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/contrib/raftexample"
 	go test -timeout 5m -v ${RACE} -tags v2v3 "$@" "${REPO_PATH}/etcdserver/v2store"
-	go test -timeout 1m -v ${RACE} -cpu 1,2,4 -run=Example "$@" "${TEST[@]}"
+	go test -timeout 1m -v ${RACE} -cpu "${TEST_CPUS}" -run=Example "$@" "${TEST[@]}"
 }
 
 function functional_pass {
@@ -296,24 +303,24 @@ function e2e_pass {
 		USERTIMEOUT="${TIMEOUT}"
 	fi
 
-	go test -timeout "${USERTIMEOUT}" -v -cpu 1,2,4 "${RUN_ARG}"  "$@" "${REPO_PATH}/tests/e2e"
+	go test -timeout "${USERTIMEOUT}" -v -cpu "${TEST_CPUS}" "${RUN_ARG}"  "$@" "${REPO_PATH}/tests/e2e"
 }
 
 function integration_e2e_pass {
 	echo "Running integration and e2e tests..."
 
-	go test -timeout 20m -v -cpu 1,2,4 "$@" "${REPO_PATH}/tests/e2e" &
+	go test -timeout 20m -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/tests/e2e" &
 	e2epid="$!"
-	go test -timeout 20m -v -cpu 1,2,4 "$@" "${REPO_PATH}/integration" &
+	go test -timeout 20m -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/integration" &
 	intpid="$!"
 	wait $e2epid
 	wait $intpid
 	integration_extra "$@"
 }
 
 function grpcproxy_pass {
-	go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 "$@" "${REPO_PATH}/integration"
-	go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 "$@" "${REPO_PATH}/clientv3/integration"
+	go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/integration"
+	go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/clientv3/integration"
 	go test -timeout 20m -v -tags cluster_proxy "$@" "${REPO_PATH}/tests/e2e"
 }