diff --git a/integration_tests/src/main/python/map_test.py b/integration_tests/src/main/python/map_test.py index f71d5a52548..bf2a945d8e5 100644 --- a/integration_tests/src/main/python/map_test.py +++ b/integration_tests/src/main/python/map_test.py @@ -22,8 +22,6 @@ from pyspark.sql.types import * from pyspark.sql.types import IntegralType -# Mark all tests in current file as premerge_ci_1 in order to be run in first k8s pod for parallel build premerge job -pytestmark = pytest.mark.premerge_ci_1 basic_struct_gen = StructGen([ ['child' + str(ind), sub_gen] diff --git a/jenkins/spark-premerge-build.sh b/jenkins/spark-premerge-build.sh index 775f5385263..23213f40b9c 100755 --- a/jenkins/spark-premerge-build.sh +++ b/jenkins/spark-premerge-build.sh @@ -51,8 +51,9 @@ mvn_verify() { env -u SPARK_HOME LC_ALL="en_US.UTF-8" mvn $MVN_URM_MIRROR -Dbuildver=320 test -Drat.skip=true -Dmaven.javadoc.skip=true -Dskip -Dmaven.scalastyle.skip=true -Dcuda.version=$CUDA_CLASSIFIER -Dpytest.TEST_TAGS='' -pl '!tools' -DwildcardSuites=com.nvidia.spark.rapids.ConditionalsSuite,com.nvidia.spark.rapids.RegularExpressionSuite,com.nvidia.spark.rapids.RegularExpressionTranspilerSuite env -u SPARK_HOME mvn -U -B $MVN_URM_MIRROR -Dbuildver=321 clean install -Drat.skip=true -DskipTests -Dmaven.javadoc.skip=true -Dskip -Dmaven.scalastyle.skip=true -Dcuda.version=$CUDA_CLASSIFIER -pl aggregator -am env -u SPARK_HOME mvn -U -B $MVN_URM_MIRROR -Dbuildver=322 clean install -Drat.skip=true -DskipTests -Dmaven.javadoc.skip=true -Dskip -Dmaven.scalastyle.skip=true -Dcuda.version=$CUDA_CLASSIFIER -pl aggregator -am - env -u SPARK_HOME mvn -U -B $MVN_URM_MIRROR -Dbuildver=330 clean install -Drat.skip=true -DskipTests -Dmaven.javadoc.skip=true -Dskip -Dmaven.scalastyle.skip=true -Dcuda.version=$CUDA_CLASSIFIER -pl aggregator -am + env -u SPARK_HOME mvn -U -B $MVN_URM_MIRROR -Dbuildver=330 clean install -Drat.skip=true -Dmaven.javadoc.skip=true -Dskip -Dmaven.scalastyle.skip=true -Dcuda.version=$CUDA_CLASSIFIER -pl aggregator -am [[ $BUILD_MAINTENANCE_VERSION_SNAPSHOTS == "true" ]] && env -u SPARK_HOME mvn -U -B $MVN_URM_MIRROR -Dbuildver=331 clean install -Drat.skip=true -DskipTests -Dmaven.javadoc.skip=true -Dskip -Dmaven.scalastyle.skip=true -Dcuda.version=$CUDA_CLASSIFIER -pl aggregator -am + # TODO: move it to BUILD_MAINTENANCE_VERSION_SNAPSHOTS when we resolve all spark340 build issues [[ $BUILD_FEATURE_VERSION_SNAPSHOTS == "true" ]] && env -u SPARK_HOME mvn -U -B $MVN_URM_MIRROR -Dbuildver=340 clean install -Drat.skip=true -DskipTests -Dmaven.javadoc.skip=true -Dskip -Dmaven.scalastyle.skip=true -Dcuda.version=$CUDA_CLASSIFIER -pl aggregator -am # Here run Python integration tests tagged with 'premerge_ci_1' only, that would help balance test duration and memory @@ -135,16 +136,16 @@ nvidia-smi . jenkins/version-def.sh -# controls whether we build snapshots for the Spark maintenance versions like 3.1.4 and 3.2.2 +# controls whether we build snapshots for the Spark maintenance versions like 3.1.4 and 3.3.1 BUILD_MAINTENANCE_VERSION_SNAPSHOTS="false" # controls whether we build snapshots for the next Spark major or feature version like 3.4.0 or 4.0.0 BUILD_FEATURE_VERSION_SNAPSHOTS="false" PREMERGE_PROFILES="-PnoSnapshots,pre-merge" -if [[ ${PROJECT_VER} =~ ^22\.08\. ]]; then # enable snapshot builds for active development branch only +if [[ ${PROJECT_VER} =~ ^22\.10\. ]]; then # enable snapshot builds for active development branch only BUILD_MAINTENANCE_VERSION_SNAPSHOTS="true" BUILD_FEATURE_VERSION_SNAPSHOTS="false" PREMERGE_PROFILES="-Psnapshots,pre-merge" -elif [[ ${PROJECT_VER} =~ ^22\.10\. ]]; then +elif [[ ${PROJECT_VER} =~ ^22\.12\. ]]; then BUILD_MAINTENANCE_VERSION_SNAPSHOTS="true" BUILD_FEATURE_VERSION_SNAPSHOTS="true" PREMERGE_PROFILES="-Psnapshots,pre-merge" diff --git a/jenkins/version-def.sh b/jenkins/version-def.sh index d661bb04458..44eaa0df22e 100755 --- a/jenkins/version-def.sh +++ b/jenkins/version-def.sh @@ -49,7 +49,7 @@ echo "CUDF_VER: $CUDF_VER, CUDA_CLASSIFIER: $CUDA_CLASSIFIER, PROJECT_VER: $PROJ SPARK_VER: $SPARK_VER, SCALA_BINARY_VER: $SCALA_BINARY_VER" -SPARK_SHIM_VERSIONS_STR=${SPARK_SHIM_VERSIONS_STR:-"311 321cdh 312 313 314 320 321 322 330 331"} +SPARK_SHIM_VERSIONS_STR=${SPARK_SHIM_VERSIONS_STR:-"311 321cdh 312 313 320 321 322 330"} IFS=" " <<< $SPARK_SHIM_VERSIONS_STR read -r -a SPARK_SHIM_VERSIONS