Reduce time-consuming of pre-merge (#9887)

* add mark Signed-off-by: YanxuanLiu <yanxuanl@nvidia.com> * reset Signed-off-by: YanxuanLiu <yanxuanl@nvidia.com> * skip build only version Signed-off-by: YanxuanLiu <yanxuanl@nvidia.com> * restore build only steps Signed-off-by: YanxuanLiu <yanxuanl@nvidia.com> * remove nosnapshots Signed-off-by: YanxuanLiu <yanxuanl@nvidia.com> * remove test file Signed-off-by: YanxuanLiu <yanxuanl@nvidia.com> * add noSnapshots Signed-off-by: YanxuanLiu <yanxuanl@nvidia.com> * add buildver Signed-off-by: YanxuanLiu <yanxuanl@nvidia.com> * add pytest mark to delta lake Signed-off-by: YanxuanLiu <yanxuanl@nvidia.com> * comment py mark Signed-off-by: YanxuanLiu <yanxuanl@nvidia.com> * remove test file, add py test mark Signed-off-by: YanxuanLiu <yanxuanl@nvidia.com> * Update delta_lake_test.py * add pytest mark for sort Signed-off-by: YanxuanLiu <yanxuanl@nvidia.com> * add property for 2.13 Signed-off-by: YanxuanLiu <yanxuanl@nvidia.com> * add pytest mark to conditionals test Signed-off-by: YanxuanLiu <yanxuanl@nvidia.com> * move build only step out of loop Signed-off-by: YanxuanLiu <yanxuanl@nvidia.com> * move timezone tests to mvn_verify Signed-off-by: YanxuanLiu <yanxuanl@nvidia.com> --------- Signed-off-by: YanxuanLiu <yanxuanl@nvidia.com>
NVIDIA · Jan 4, 2024 · 3f60308 · 3f60308
1 parent ed1fa9f
commit 3f60308
Show file tree

Hide file tree

Showing 5 changed files with 26 additions and 12 deletions.
diff --git a/dist/pom.xml b/dist/pom.xml
@@ -216,6 +216,9 @@
         </profile>
         <profile>
             <id>pre-merge</id>
+            <properties>
+                <included_buildvers>${buildver}</included_buildvers>
+            </properties>
             <build>
                 <plugins>
                     <plugin>

diff --git a/integration_tests/src/main/python/conditionals_test.py b/integration_tests/src/main/python/conditionals_test.py
@@ -21,6 +21,9 @@
 from marks import datagen_overrides, allow_non_gpu
 import pyspark.sql.functions as f
 
+# mark this test as ci_1 for mvn verify sanity check in pre-merge CI
+pytestmark = pytest.mark.premerge_ci_1
+
 def mk_str_gen(pattern):
     return StringGen(pattern).with_special_case('').with_special_pattern('.{0,10}')
 

diff --git a/integration_tests/src/main/python/sort_test.py b/integration_tests/src/main/python/sort_test.py
@@ -22,6 +22,9 @@
 import pyspark.sql.functions as f
 from spark_session import is_before_spark_340
 
+# mark this test as ci_1 for mvn verify sanity check in pre-merge CI
+pytestmark = pytest.mark.premerge_ci_1
+
 # Many Spark versions have issues sorting decimals.
 # https://issues.apache.org/jira/browse/SPARK-40089
 _orderable_not_null_big_decimal_gen = DecimalGen(precision=20, scale=2, nullable=False)

diff --git a/jenkins/spark-premerge-build.sh b/jenkins/spark-premerge-build.sh
@@ -59,10 +59,12 @@ mvn_verify() {
               $MVN_CMD -B $MVN_URM_MIRROR -Dbuildver=$version test -rf tests $MVN_BUILD_ARGS -Dpytest.TEST_TAGS='' \
               -DwildcardSuites=org.apache.spark.sql.rapids.filecache.FileCacheIntegrationSuite
         # build only for other versions
-        elif [[ "${SPARK_SHIM_VERSIONS_NOSNAPSHOTS_TAIL[@]}" =~ "$version" ]]; then
-            $MVN_INSTALL_CMD -DskipTests -Dbuildver=$version
+        # elif [[ "${SPARK_SHIM_VERSIONS_NOSNAPSHOTS_TAIL[@]}" =~ "$version" ]]; then
+        #     $MVN_INSTALL_CMD -DskipTests -Dbuildver=$version
         fi
     done
+    # build base shim version for following test step with PREMERGE_PROFILES
+    $MVN_INSTALL_CMD -DskipTests -Dbuildver=$SPARK_BASE_SHIM_VERSION
 
     # enable UTF-8 for regular expression tests
     for version in "${SPARK_SHIM_VERSIONS_PREMERGE_UTF8[@]}"
@@ -95,6 +97,15 @@ mvn_verify() {
 
     # Triggering here until we change the jenkins file
     rapids_shuffle_smoke_test
+
+    # Test a portion of cases for non-UTC time zone because of limited GPU resources.
+    # Here testing: parquet scan, orc scan, csv scan, cast, TimeZoneAwareExpression, FromUTCTimestamp
+    # Nightly CIs will cover all the cases.
+    source "$(dirname "$0")"/test-timezones.sh
+    for tz in "${time_zones_test_cases[@]}"
+    do
+        TZ=$tz ./integration_tests/run_pyspark_from_build.sh -m tz_sensitive_test
+    done
 }
 
 rapids_shuffle_smoke_test() {
@@ -157,15 +168,6 @@ ci_2() {
     prepare_spark $SPARK_VER 2.12
     ./integration_tests/run_pyspark_from_build.sh
 
-    # Test a portion of cases for non-UTC time zone because of limited GPU resources.
-    # Here testing: parquet scan, orc scan, csv scan, cast, TimeZoneAwareExpression, FromUTCTimestamp
-    # Nightly CIs will cover all the cases.
-    source "$(dirname "$0")"/test-timezones.sh
-    for tz in "${time_zones_test_cases[@]}"
-    do
-        TZ=$tz ./integration_tests/run_pyspark_from_build.sh -m tz_sensitive_test
-    done
-
     # enable avro test separately
     INCLUDE_SPARK_AVRO_JAR=true TEST='avro_test.py' ./integration_tests/run_pyspark_from_build.sh
     # export 'LC_ALL' to set locale with UTF-8 so regular expressions are enabled
@@ -239,7 +241,7 @@ nvidia-smi
 
 . jenkins/version-def.sh
 
-PREMERGE_PROFILES="-PnoSnapshots,pre-merge"
+PREMERGE_PROFILES="-Ppre-merge"
 
 # If possible create '~/.m2' cache from pre-created m2 tarball to minimize the impact of unstable network connection.
 # Please refer to job 'update_premerge_m2_cache' on Blossom about building m2 tarball details.

diff --git a/scala2.13/dist/pom.xml b/scala2.13/dist/pom.xml
@@ -216,6 +216,9 @@
         </profile>
         <profile>
             <id>pre-merge</id>
+            <properties>
+                <included_buildvers>${buildver}</included_buildvers>
+            </properties>
             <build>
                 <plugins>
                     <plugin>