Skip redundant steps in nightly build [skip ci] (#9857)

* Skip redundant steps in nightly build script Signed-off-by: Peixin Li <pxli@nyu.edu> * set ono-dist to be deployed \for now Signed-off-by: Peixin Li <pxli@nyu.edu> * disable history expantion for DEPLOY_SUBMODULES ENV --------- Signed-off-by: Peixin Li <pxli@nyu.edu>
NVIDIA · Nov 28, 2023 · fd05f70 · fd05f70
1 parent a0c3c48
commit fd05f70
Showing 1 changed file with 9 additions and 4 deletions.
diff --git a/jenkins/spark-nightly-build.sh b/jenkins/spark-nightly-build.sh
@@ -102,6 +102,8 @@ function distWithReducedPom {
 
 # option to skip unit tests. Used in our CI to separate test runs in parallel stages
 SKIP_TESTS=${SKIP_TESTS:-"false"}
+set +H # turn off history expansion
+DEPLOY_SUBMODULES=${DEPLOY_SUBMODULES:-"!${DIST_PL}"} # TODO: deploy only required submodules to save time
 for buildver in "${SPARK_SHIM_VERSIONS[@]:1}"; do
     $MVN -U -B clean install $MVN_URM_MIRROR -Dmaven.repo.local=$M2DIR \
         -Dcuda.version=$DEFAULT_CUDA_CLASSIFIER \
@@ -117,10 +119,12 @@ for buildver in "${SPARK_SHIM_VERSIONS[@]:1}"; do
     fi
     distWithReducedPom "install"
     [[ $SKIP_DEPLOY != 'true' ]] && \
-        $MVN -B deploy -pl '!dist' $MVN_URM_MIRROR \
+        # this deploys selected submodules
+        $MVN -B deploy -pl $DEPLOY_SUBMODULES $MVN_URM_MIRROR \
             -Dmaven.repo.local=$M2DIR \
             -Dcuda.version=$DEFAULT_CUDA_CLASSIFIER \
             -DskipTests \
+            -Dmaven.scaladoc.skip -Dmaven.scalastyle.skip=true \
             -Dbuildver="${buildver}"
 done
 
@@ -161,10 +165,11 @@ distWithReducedPom "install"
 if [[ $SKIP_DEPLOY != 'true' ]]; then
     distWithReducedPom "deploy"
 
-    # this deploys submodules except dist that is unconditionally built with Spark 3.1.1
-    $MVN -B deploy -pl '!dist' \
+    # this deploys selected submodules that is unconditionally built with Spark 3.1.1
+    $MVN -B deploy -pl $DEPLOY_SUBMODULES \
         -Dbuildver=$SPARK_BASE_SHIM_VERSION \
-        -DskipTests=$SKIP_TESTS \
+        -DskipTests \
+        -Dmaven.scaladoc.skip -Dmaven.scalastyle.skip=true \
         $MVN_URM_MIRROR -Dmaven.repo.local=$M2DIR \
         -Dcuda.version=$DEFAULT_CUDA_CLASSIFIER
 fi