Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Revert "Get cudf/spark dependency from the correct .m2 dir" #1083

Merged
merged 1 commit into from
Nov 6, 2020
Merged

Revert "Get cudf/spark dependency from the correct .m2 dir" #1083

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions build/build-info
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/usr/bin/env bash

#
# Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved.
# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -19,7 +19,6 @@
# This script generates the build info.
# Arguments:
# rapids4spark_version - The current version of spark plugin
set -e

echo_build_properties() {
echo version=$1
Expand Down
8 changes: 2 additions & 6 deletions build/dependency-info.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,18 +23,14 @@
# SPARK_VER - The version of spark

# Parse cudf and spark dependency versions
set -e

CUDF_VER=$1
CUDA_CLASSIFIER=$2
SERVER_ID=snapshots
# set defualt values for 'M2DIR' & 'WORKSPACE' so that shims can get the correct cudf/spark dependnecy
M2DIR=${M2DIR:-"$HOME/.m2/repository"}
WORKSPACE=${WORKSPACE:-"../.."}
${WORKSPACE}/jenkins/printJarVersion.sh "cudf_version" "${M2DIR}/ai/rapids/cudf/${CUDF_VER}" "cudf-${CUDF_VER}" "-${CUDA_CLASSIFIER}.jar" $SERVER_ID
${WORKSPACE}/jenkins/printJarVersion.sh "cudf_version" "${HOME}/.m2/repository/ai/rapids/cudf/${CUDF_VER}" "cudf-${CUDF_VER}" "-${CUDA_CLASSIFIER}.jar" $SERVER_ID

SPARK_VER=$3
SPARK_SQL_VER=`${WORKSPACE}/jenkins/printJarVersion.sh "spark_version" "${M2DIR}/org/apache/spark/spark-sql_2.12/${SPARK_VER}" "spark-sql_2.12-${SPARK_VER}" ".jar" $SERVER_ID`
SPARK_SQL_VER=`${WORKSPACE}/jenkins/printJarVersion.sh "spark_version" "${HOME}/.m2/repository/org/apache/spark/spark-sql_2.12/${SPARK_VER}" "spark-sql_2.12-${SPARK_VER}" ".jar" $SERVER_ID`

# Split spark version from spark-sql_2.12 jar filename
echo ${SPARK_SQL_VER/"-sql_2.12"/}
3 changes: 1 addition & 2 deletions jenkins/databricks/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,7 @@ tar -zxvf $SPARKSRCTGZ -C spark-rapids
cd spark-rapids
export WORKSPACE=`pwd`
mvn -B '-Pdatabricks,!snapshot-shims' clean package -DskipTests || true
# export 'M2DIR' so that shims can get the correct cudf/spark dependnecy info
export M2DIR=/home/ubuntu/.m2/repository
M2DIR=/home/ubuntu/.m2/repository
CUDF_JAR=${M2DIR}/ai/rapids/cudf/${CUDF_VERSION}/cudf-${CUDF_VERSION}-${CUDA_VERSION}.jar

# pull normal Spark artifacts and ignore errors then install databricks jars, then build again
Expand Down
1 change: 0 additions & 1 deletion jenkins/printJarVersion.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e

function print_ver(){
TAG=$1
Expand Down
14 changes: 6 additions & 8 deletions jenkins/spark-nightly-build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,12 @@ set -ex

. jenkins/version-def.sh

export 'M2DIR' so that shims can get the correct cudf/spark dependnecy info
export M2DIR="$WORKSPACE/.m2"
mvn -U -B -Pinclude-databricks,snapshot-shims clean deploy $MVN_URM_MIRROR -Dmaven.repo.local=$M2DIR
mvn -U -B -Pinclude-databricks,snapshot-shims clean deploy $MVN_URM_MIRROR -Dmaven.repo.local=$WORKSPACE/.m2
# Run unit tests against other spark versions
mvn -U -B -Pspark301tests,snapshot-shims test $MVN_URM_MIRROR -Dmaven.repo.local=$M2DIR
mvn -U -B -Pspark302tests,snapshot-shims test $MVN_URM_MIRROR -Dmaven.repo.local=$M2DIR
mvn -U -B -Pspark310tests,snapshot-shims test $MVN_URM_MIRROR -Dmaven.repo.local=$M2DIR
mvn -U -B -Pspark301tests,snapshot-shims test $MVN_URM_MIRROR -Dmaven.repo.local=$WORKSPACE/.m2
mvn -U -B -Pspark302tests,snapshot-shims test $MVN_URM_MIRROR -Dmaven.repo.local=$WORKSPACE/.m2
mvn -U -B -Pspark310tests,snapshot-shims test $MVN_URM_MIRROR -Dmaven.repo.local=$WORKSPACE/.m2

# Parse cudf and spark files from local mvn repo
jenkins/printJarVersion.sh "CUDFVersion" "$M2DIR/ai/rapids/cudf/${CUDF_VER}" "cudf-${CUDF_VER}" "-${CUDA_CLASSIFIER}.jar" $SERVER_ID
jenkins/printJarVersion.sh "SPARKVersion" "$M2DIR/org/apache/spark/spark-core_2.12/${SPARK_VER}" "spark-core_2.12-${SPARK_VER}" ".jar" $SERVER_ID
jenkins/printJarVersion.sh "CUDFVersion" "${WORKSPACE}/.m2/ai/rapids/cudf/${CUDF_VER}" "cudf-${CUDF_VER}" "-${CUDA_CLASSIFIER}.jar" $SERVER_ID
jenkins/printJarVersion.sh "SPARKVersion" "${WORKSPACE}/.m2/org/apache/spark/spark-core_2.12/${SPARK_VER}" "spark-core_2.12-${SPARK_VER}" ".jar" $SERVER_ID
4 changes: 2 additions & 2 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -535,8 +535,8 @@
<target>
<mkdir dir="${project.build.directory}/extra-resources"/>
<mkdir dir="${project.build.directory}/tmp"/>
<exec executable="bash" failonerror="true" output="${project.build.directory}/extra-resources/rapids4spark-version-info.properties">
<arg value="${user.dir}/build/build-info"/>
<exec executable="bash" output="${project.build.directory}/extra-resources/rapids4spark-version-info.properties">
<arg value="${project.basedir}/../build/build-info"/>
<arg value="${project.version}"/>
<arg value="${cudf.version}"/>
</exec>
Expand Down
2 changes: 1 addition & 1 deletion shims/spark300/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
<configuration>
<target>
<mkdir dir="${project.build.directory}/extra-resources"/>
<exec executable="bash" failonerror="true" output="${project.build.directory}/extra-resources/spark-${spark300.version}-info.properties">
<exec executable="bash" output="${project.build.directory}/extra-resources/spark-${spark300.version}-info.properties">
<arg value="${user.dir}/build/dependency-info.sh"/>
<arg value="${cudf.version}"/>
<arg value="${cuda.version}"/>
Expand Down
2 changes: 1 addition & 1 deletion shims/spark300db/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
<configuration>
<target>
<mkdir dir="${project.build.directory}/extra-resources"/>
<exec executable="bash" failonerror="true" output="${project.build.directory}/extra-resources/spark-${spark30db.version}-info.properties">
<exec executable="bash" output="${project.build.directory}/extra-resources/spark-${spark30db.version}-info.properties">
<arg value="${user.dir}/build/dependency-info.sh"/>
<arg value="${cudf.version}"/>
<arg value="${cuda.version}"/>
Expand Down
2 changes: 1 addition & 1 deletion shims/spark300emr/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
<configuration>
<target>
<mkdir dir="${project.build.directory}/extra-resources"/>
<exec executable="bash" failonerror="true" output="${project.build.directory}/extra-resources/spark-${spark300emr.version}-info.properties">
<exec executable="bash" output="${project.build.directory}/extra-resources/spark-${spark300emr.version}-info.properties">
<arg value="${user.dir}/build/dependency-info.sh"/>
<arg value="${cudf.version}"/>
<arg value="${cuda.version}"/>
Expand Down
2 changes: 1 addition & 1 deletion shims/spark301/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
<configuration>
<target>
<mkdir dir="${project.build.directory}/extra-resources"/>
<exec executable="bash" failonerror="true" output="${project.build.directory}/extra-resources/spark-${spark301.version}-info.properties">
<exec executable="bash" output="${project.build.directory}/extra-resources/spark-${spark301.version}-info.properties">
<arg value="${user.dir}/build/dependency-info.sh"/>
<arg value="${cudf.version}"/>
<arg value="${cuda.version}"/>
Expand Down
2 changes: 1 addition & 1 deletion shims/spark301emr/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
<configuration>
<target>
<mkdir dir="${project.build.directory}/extra-resources"/>
<exec executable="bash" failonerror="true" output="${project.build.directory}/extra-resources/spark-${spark301emr.version}-info.properties">
<exec executable="bash" output="${project.build.directory}/extra-resources/spark-${spark301emr.version}-info.properties">
<arg value="${user.dir}/build/dependency-info.sh"/>
<arg value="${cudf.version}"/>
<arg value="${cuda.version}"/>
Expand Down
2 changes: 1 addition & 1 deletion shims/spark302/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
<configuration>
<target>
<mkdir dir="${project.build.directory}/extra-resources"/>
<exec executable="bash" failonerror="true" output="${project.build.directory}/extra-resources/spark-${spark302.version}-info.properties">
<exec executable="bash" output="${project.build.directory}/extra-resources/spark-${spark302.version}-info.properties">
<arg value="${user.dir}/build/dependency-info.sh"/>
<arg value="${cudf.version}"/>
<arg value="${cuda.version}"/>
Expand Down
2 changes: 1 addition & 1 deletion shims/spark310/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
<configuration>
<target>
<mkdir dir="${project.build.directory}/extra-resources"/>
<exec executable="bash" failonerror="true" output="${project.build.directory}/extra-resources/spark-${spark310.version}-info.properties">
<exec executable="bash" output="${project.build.directory}/extra-resources/spark-${spark310.version}-info.properties">
<arg value="${user.dir}/build/dependency-info.sh"/>
<arg value="${cudf.version}"/>
<arg value="${cuda.version}"/>
Expand Down