From 67373bdd71bdeb6e41a994f952778fbbe0d5dc77 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Thu, 23 Jul 2020 15:08:08 -0500 Subject: [PATCH 1/2] Add unit tests running for Spark 3.0.1 --- jenkins/spark-nightly-build.sh | 4 ++++ jenkins/spark-premerge-build.sh | 3 +++ 2 files changed, 7 insertions(+) diff --git a/jenkins/spark-nightly-build.sh b/jenkins/spark-nightly-build.sh index 9c03e46b41c..33d8ba40ef8 100755 --- a/jenkins/spark-nightly-build.sh +++ b/jenkins/spark-nightly-build.sh @@ -20,6 +20,10 @@ set -ex . jenkins/version-def.sh mvn -U -B clean deploy $MVN_URM_MIRROR -Dmaven.repo.local=$WORKSPACE/.m2 +# Run unit tests against other spark versions +mvn -U -B -Pspark301tests test $MVN_URM_MIRROR -Dmaven.repo.local=$WORKSPACE/.m2 +# spark310 unit tests fail - https://github.com/NVIDIA/spark-rapids/issues/382 +#mvn -U -B -Pspark310tests test $MVN_URM_MIRROR -Dmaven.repo.local=$WORKSPACE/.m2 # Parse cudf and spark files from local mvn repo jenkins/printJarVersion.sh "CUDFVersion" "${WORKSPACE}/.m2/ai/rapids/cudf/${CUDF_VER}" "cudf-${CUDF_VER}" "-${CUDA_CLASSIFIER}.jar" diff --git a/jenkins/spark-premerge-build.sh b/jenkins/spark-premerge-build.sh index a9fffe658cd..818bcca4851 100755 --- a/jenkins/spark-premerge-build.sh +++ b/jenkins/spark-premerge-build.sh @@ -38,6 +38,9 @@ tar zxf $SPARK_HOME.tgz -C $ARTF_ROOT && \ rm -f $SPARK_HOME.tgz mvn -U -B $MVN_URM_MIRROR clean verify -Dpytest.TEST_TAGS='' +mvn -U -B $MVN_URM_MIRROR -Pspark301tests test -Dpytest.TEST_TAGS='' +# spark310 unit tests fail - https://github.com/NVIDIA/spark-rapids/issues/382 +#mvn -U -B $MVN_URM_MIRROR -Pspark310tests test -Dpytest.TEST_TAGS='' # The jacoco coverage should have been collected, but because of how the shade plugin # works and jacoco we need to clean some things up so jacoco will only report for the From 687568ab584aadc52c17efcc83f7fc12410a0d58 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Thu, 23 Jul 2020 15:27:40 -0500 Subject: [PATCH 2/2] dont' run python integration tests for other versions of Spark --- jenkins/spark-premerge-build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jenkins/spark-premerge-build.sh b/jenkins/spark-premerge-build.sh index 818bcca4851..64149f3051a 100755 --- a/jenkins/spark-premerge-build.sh +++ b/jenkins/spark-premerge-build.sh @@ -38,9 +38,10 @@ tar zxf $SPARK_HOME.tgz -C $ARTF_ROOT && \ rm -f $SPARK_HOME.tgz mvn -U -B $MVN_URM_MIRROR clean verify -Dpytest.TEST_TAGS='' -mvn -U -B $MVN_URM_MIRROR -Pspark301tests test -Dpytest.TEST_TAGS='' +# Run the unit tests for other Spark versions but dont run full python integration tests +env -u SPARK_HOME mvn -U -B $MVN_URM_MIRROR -Pspark301tests test -Dpytest.TEST_TAGS='' # spark310 unit tests fail - https://github.com/NVIDIA/spark-rapids/issues/382 -#mvn -U -B $MVN_URM_MIRROR -Pspark310tests test -Dpytest.TEST_TAGS='' +#env -u SPARK_HOME mvn -U -B $MVN_URM_MIRROR -Pspark310tests test -Dpytest.TEST_TAGS='' # The jacoco coverage should have been collected, but because of how the shade plugin # works and jacoco we need to clean some things up so jacoco will only report for the