From 2de8700cbaca8bd249649a22b611d3c61187d36c Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Fri, 6 Nov 2020 15:21:11 -0600 Subject: [PATCH] Revert "Get cudf/spark dependency from the correct .m2 dir (#1062)" This reverts commit 0afbca6698ebb2c64090dd2812b3e3815b716ee0. --- build/build-info | 3 +-- build/dependency-info.sh | 8 ++------ jenkins/databricks/build.sh | 3 +-- jenkins/printJarVersion.sh | 1 - jenkins/spark-nightly-build.sh | 14 ++++++-------- pom.xml | 4 ++-- shims/spark300/pom.xml | 2 +- shims/spark300db/pom.xml | 2 +- shims/spark300emr/pom.xml | 2 +- shims/spark301/pom.xml | 2 +- shims/spark301emr/pom.xml | 2 +- shims/spark302/pom.xml | 2 +- shims/spark310/pom.xml | 2 +- 13 files changed, 19 insertions(+), 28 deletions(-) diff --git a/build/build-info b/build/build-info index 88cd8516d136..a905661920b6 100755 --- a/build/build-info +++ b/build/build-info @@ -1,7 +1,7 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,7 +19,6 @@ # This script generates the build info. # Arguments: # rapids4spark_version - The current version of spark plugin -set -e echo_build_properties() { echo version=$1 diff --git a/build/dependency-info.sh b/build/dependency-info.sh index e26b748ddc05..7633c79c20f3 100755 --- a/build/dependency-info.sh +++ b/build/dependency-info.sh @@ -23,18 +23,14 @@ # SPARK_VER - The version of spark # Parse cudf and spark dependency versions -set -e CUDF_VER=$1 CUDA_CLASSIFIER=$2 SERVER_ID=snapshots -# set defualt values for 'M2DIR' & 'WORKSPACE' so that shims can get the correct cudf/spark dependnecy -M2DIR=${M2DIR:-"$HOME/.m2/repository"} -WORKSPACE=${WORKSPACE:-"../.."} -${WORKSPACE}/jenkins/printJarVersion.sh "cudf_version" "${M2DIR}/ai/rapids/cudf/${CUDF_VER}" "cudf-${CUDF_VER}" "-${CUDA_CLASSIFIER}.jar" $SERVER_ID +${WORKSPACE}/jenkins/printJarVersion.sh "cudf_version" "${HOME}/.m2/repository/ai/rapids/cudf/${CUDF_VER}" "cudf-${CUDF_VER}" "-${CUDA_CLASSIFIER}.jar" $SERVER_ID SPARK_VER=$3 -SPARK_SQL_VER=`${WORKSPACE}/jenkins/printJarVersion.sh "spark_version" "${M2DIR}/org/apache/spark/spark-sql_2.12/${SPARK_VER}" "spark-sql_2.12-${SPARK_VER}" ".jar" $SERVER_ID` +SPARK_SQL_VER=`${WORKSPACE}/jenkins/printJarVersion.sh "spark_version" "${HOME}/.m2/repository/org/apache/spark/spark-sql_2.12/${SPARK_VER}" "spark-sql_2.12-${SPARK_VER}" ".jar" $SERVER_ID` # Split spark version from spark-sql_2.12 jar filename echo ${SPARK_SQL_VER/"-sql_2.12"/} diff --git a/jenkins/databricks/build.sh b/jenkins/databricks/build.sh index 6098a1432e9c..074976f402d7 100755 --- a/jenkins/databricks/build.sh +++ b/jenkins/databricks/build.sh @@ -44,8 +44,7 @@ tar -zxvf $SPARKSRCTGZ -C spark-rapids cd spark-rapids export WORKSPACE=`pwd` mvn -B '-Pdatabricks,!snapshot-shims' clean package -DskipTests || true -# export 'M2DIR' so that shims can get the correct cudf/spark dependnecy info -export M2DIR=/home/ubuntu/.m2/repository +M2DIR=/home/ubuntu/.m2/repository CUDF_JAR=${M2DIR}/ai/rapids/cudf/${CUDF_VERSION}/cudf-${CUDF_VERSION}-${CUDA_VERSION}.jar # pull normal Spark artifacts and ignore errors then install databricks jars, then build again diff --git a/jenkins/printJarVersion.sh b/jenkins/printJarVersion.sh index bc5099388684..fde53b4e54d4 100755 --- a/jenkins/printJarVersion.sh +++ b/jenkins/printJarVersion.sh @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -set -e function print_ver(){ TAG=$1 diff --git a/jenkins/spark-nightly-build.sh b/jenkins/spark-nightly-build.sh index 265075fb9082..6f675d090968 100755 --- a/jenkins/spark-nightly-build.sh +++ b/jenkins/spark-nightly-build.sh @@ -19,14 +19,12 @@ set -ex . jenkins/version-def.sh -export 'M2DIR' so that shims can get the correct cudf/spark dependnecy info -export M2DIR="$WORKSPACE/.m2" -mvn -U -B -Pinclude-databricks,snapshot-shims clean deploy $MVN_URM_MIRROR -Dmaven.repo.local=$M2DIR +mvn -U -B -Pinclude-databricks,snapshot-shims clean deploy $MVN_URM_MIRROR -Dmaven.repo.local=$WORKSPACE/.m2 # Run unit tests against other spark versions -mvn -U -B -Pspark301tests,snapshot-shims test $MVN_URM_MIRROR -Dmaven.repo.local=$M2DIR -mvn -U -B -Pspark302tests,snapshot-shims test $MVN_URM_MIRROR -Dmaven.repo.local=$M2DIR -mvn -U -B -Pspark310tests,snapshot-shims test $MVN_URM_MIRROR -Dmaven.repo.local=$M2DIR +mvn -U -B -Pspark301tests,snapshot-shims test $MVN_URM_MIRROR -Dmaven.repo.local=$WORKSPACE/.m2 +mvn -U -B -Pspark302tests,snapshot-shims test $MVN_URM_MIRROR -Dmaven.repo.local=$WORKSPACE/.m2 +mvn -U -B -Pspark310tests,snapshot-shims test $MVN_URM_MIRROR -Dmaven.repo.local=$WORKSPACE/.m2 # Parse cudf and spark files from local mvn repo -jenkins/printJarVersion.sh "CUDFVersion" "$M2DIR/ai/rapids/cudf/${CUDF_VER}" "cudf-${CUDF_VER}" "-${CUDA_CLASSIFIER}.jar" $SERVER_ID -jenkins/printJarVersion.sh "SPARKVersion" "$M2DIR/org/apache/spark/spark-core_2.12/${SPARK_VER}" "spark-core_2.12-${SPARK_VER}" ".jar" $SERVER_ID +jenkins/printJarVersion.sh "CUDFVersion" "${WORKSPACE}/.m2/ai/rapids/cudf/${CUDF_VER}" "cudf-${CUDF_VER}" "-${CUDA_CLASSIFIER}.jar" $SERVER_ID +jenkins/printJarVersion.sh "SPARKVersion" "${WORKSPACE}/.m2/org/apache/spark/spark-core_2.12/${SPARK_VER}" "spark-core_2.12-${SPARK_VER}" ".jar" $SERVER_ID diff --git a/pom.xml b/pom.xml index 8004b00199c3..a973c3c715f7 100644 --- a/pom.xml +++ b/pom.xml @@ -535,8 +535,8 @@ - - + + diff --git a/shims/spark300/pom.xml b/shims/spark300/pom.xml index 9fa09ec4c914..f74cdc05a203 100644 --- a/shims/spark300/pom.xml +++ b/shims/spark300/pom.xml @@ -44,7 +44,7 @@ - + diff --git a/shims/spark300db/pom.xml b/shims/spark300db/pom.xml index b3195235cdfb..341b18dc1382 100644 --- a/shims/spark300db/pom.xml +++ b/shims/spark300db/pom.xml @@ -44,7 +44,7 @@ - + diff --git a/shims/spark300emr/pom.xml b/shims/spark300emr/pom.xml index f71fa4c38502..fa3cc7ea2d1c 100644 --- a/shims/spark300emr/pom.xml +++ b/shims/spark300emr/pom.xml @@ -44,7 +44,7 @@ - + diff --git a/shims/spark301/pom.xml b/shims/spark301/pom.xml index 2a12177658fe..1cc032aaa8b0 100644 --- a/shims/spark301/pom.xml +++ b/shims/spark301/pom.xml @@ -44,7 +44,7 @@ - + diff --git a/shims/spark301emr/pom.xml b/shims/spark301emr/pom.xml index 07aca6fd589e..861a343adbda 100644 --- a/shims/spark301emr/pom.xml +++ b/shims/spark301emr/pom.xml @@ -44,7 +44,7 @@ - + diff --git a/shims/spark302/pom.xml b/shims/spark302/pom.xml index b83fda2fb2e6..13490ee73e04 100644 --- a/shims/spark302/pom.xml +++ b/shims/spark302/pom.xml @@ -44,7 +44,7 @@ - + diff --git a/shims/spark310/pom.xml b/shims/spark310/pom.xml index cc66c0d52614..a9ee5d66ba1b 100644 --- a/shims/spark310/pom.xml +++ b/shims/spark310/pom.xml @@ -44,7 +44,7 @@ - +