Support maven build options for databricks nightly build and build/bu…

…ildall script

Signed-off-by: Tim Liu <timl@nvidia.com>

build/buildall

@@ -47,8 +47,12 @@ function print_usage() {
   echo "        Build in parallel, N (4 by default) is passed via -P to xargs"
   echo "   --install"
   echo "        Intall the resulting jar instead of just building it"
+  echo "   -o=MVN_OPT, --option=MVN_OPT"
+  echo "        use this option to build project with maven. E.g., --option='-Dcudf.version=cuda11 -Pbuildver=311'"
 }
 
+export MVN_OPT=${MVN_OPT:-''}
+
 function bloopInstall() {
   BLOOP_DIR="${BLOOP_DIR:-$PWD/.bloop}"
   mkdir -p $BLOOP_DIR
@@ -60,7 +64,7 @@ function bloopInstall() {
       mkdir -p "$bloop_config_dir"
       rm -f "$bloop_config_dir"/*
 
-      mvn install ch.epfl.scala:maven-bloop_${BLOOP_SCALA_VERSION}:${BLOOP_VERSION}:bloopInstall -pl dist -am \
+      mvn $MVN_OPT install ch.epfl.scala:maven-bloop_${BLOOP_SCALA_VERSION}:${BLOOP_VERSION}:bloopInstall -pl dist -am \
         -Dbloop.configDirectory="$bloop_config_dir" \
         -DdownloadSources=true \
         -Dbuildver="$bv" \
@@ -234,7 +238,7 @@ function build_single_shim() {
   fi
 
   echo "#### REDIRECTING mvn output to $LOG_FILE ####"
-  mvn -U "$MVN_PHASE" \
+  mvn $MVN_OPT -U "$MVN_PHASE" \
       -DskipTests \
       -Dbuildver="$BUILD_VER" \
       -Drat.skip="$SKIP_CHECKS" \
@@ -270,7 +274,7 @@ time (
   # a negligible increase of the build time by ~2 seconds.
   joinShimBuildFrom="aggregator"
   echo "Resuming from $joinShimBuildFrom build only using $BASE_VER"
-  mvn $FINAL_OP -rf $joinShimBuildFrom $MODULE_OPT $MVN_PROFILE_OPT $INCLUDED_BUILDVERS_OPT \
+  mvn $MVN_OPT $FINAL_OP -rf $joinShimBuildFrom $MODULE_OPT $MVN_PROFILE_OPT $INCLUDED_BUILDVERS_OPT \
     -Dbuildver="$BASE_VER" \
     -DskipTests -Dskip -Dmaven.javadoc.skip
 )

jenkins/databricks/build.sh

@@ -20,9 +20,9 @@ set -ex
 SPARKSRCTGZ=$1
 # version of Apache Spark we are building against
 BASE_SPARK_VERSION=$2
-BUILD_PROFILES=$3
+MVN_OPT=$3
 BASE_SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS=$4
-BUILD_PROFILES=${BUILD_PROFILES:-'databricks312,!snapshot-shims'}
+MVN_OPT=${MVN_OPT:-''}
 BASE_SPARK_VERSION=${BASE_SPARK_VERSION:-'3.1.2'}
 BUILDVER=$(echo ${BASE_SPARK_VERSION} | sed 's/\.//g')db
 # the version of Spark used when we install the Databricks jars in .m2
@@ -34,7 +34,7 @@ SPARK_MAJOR_VERSION_STRING=spark_${SPARK_MAJOR_VERSION_NUM_STRING}
 
 echo "tgz is $SPARKSRCTGZ"
 echo "Base Spark version is $BASE_SPARK_VERSION"
-echo "build profiles $BUILD_PROFILES"
+echo "build profiles $MVN_OPT"
 echo "BASE_SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS is $BASE_SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS"
 
 sudo apt install -y maven rsync
@@ -442,7 +442,7 @@ mvn -B install:install-file \
    -Dversion=$SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS \
    -Dpackaging=jar
 
-mvn -B -Ddatabricks -Dbuildver=$BUILDVER clean package -DskipTests
+mvn -B -Ddatabricks -Dbuildver=$BUILDVER clean package -DskipTests $MVN_OPT
 
 cd /home/ubuntu
 tar -zcf spark-rapids-built.tgz spark-rapids