Skip to content

Commit

Permalink
cudf and spark version info in artifacts (#563)
Browse files Browse the repository at this point in the history 
* cudf and spark version info in artifacts

Add cudf and spark version info in rapids-plugin artifacts file:

rapids4spark-version-info.properties

Signed-off-by: Tim Liu <timl@nvidia.com>

* Add cuDF and spark version info in artifacts

Create the separate files in the jar files

Save relevant spark versions in all the shim jars

* Add cudf & spark version files

Create a separate file 'dependency-info.properties' in the jar to save cudf & spark version info

* Include shims SparkShimServiceProvider into META-INF

* SPARK_VER-info.properties files in the rapids jar

* Add prefix with spark for the version files

Co-authored-by: Tim Liu <timl@nvidia.com>
  • Loading branch information
@NvTimLiu
NvTimLiu and NvTimLiu authored Sep 1, 2020
1 parent 46225d7 commit abeb5bb
Show file tree
Hide file tree
Showing 8 changed files with 242 additions and 6 deletions.
36 changes: 36 additions & 0 deletions build/dependency-info.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#!/usr/bin/env bash

#
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# This script generates the dependency info.
# Arguments:
# CUDF_VER - The version of cudf
# CUDA_CLASSIFIER - The cuda runtime for cudf
# SPARK_VER - The version of spark

# Parse cudf and spark dependency versions

CUDF_VER=$1
CUDA_CLASSIFIER=$2
SERVER_ID=snapshots
${WORKSPACE}/jenkins/printJarVersion.sh "cudf_version" "${HOME}/.m2/repository/ai/rapids/cudf/${CUDF_VER}" "cudf-${CUDF_VER}" "-${CUDA_CLASSIFIER}.jar" $SERVER_ID

SPARK_VER=$3
SPARK_SQL_VER=`${WORKSPACE}/jenkins/printJarVersion.sh "spark_version" "${HOME}/.m2/repository/org/apache/spark/spark-sql_2.12/${SPARK_VER}" "spark-sql_2.12-${SPARK_VER}" ".jar" $SERVER_ID`

# Split spark version from spark-sql_2.12 jar filename
echo ${SPARK_SQL_VER/"-sql_2.12"/}
8 changes: 4 additions & 4 deletions jenkins/printJarVersion.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,16 @@ function print_ver(){
REPO=$2
VERSION=$3
SUFFIX=$4
SERVER_ID=$5

if [[ "$VERSION" == *"-SNAPSHOT" ]]; then
PREFIX=${VERSION%-SNAPSHOT}
TIMESTAMP=`grep -oP '(?<=timestamp>)[^<]+' < $REPO/maven-metadata-$SERVER_ID.xml`
BUILD_NUM=`grep -oP '(?<=buildNumber>)[^<]+' < $REPO/maven-metadata-$SERVER_ID.xml`
echo $TAG: $PREFIX-$TIMESTAMP-$BUILD_NUM$SUFFIX
echo $TAG=$PREFIX-$TIMESTAMP-$BUILD_NUM$SUFFIX
else
echo $TAG: $VERSION$SUFFIX
echo $TAG=$VERSION$SUFFIX
fi
}

. jenkins/version-def.sh
print_ver $1 $2 $3 $4
print_ver $1 $2 $3 $4 $5
4 changes: 2 additions & 2 deletions jenkins/spark-nightly-build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,5 @@ mvn -U -B -Pspark302tests,snapshot-shims test $MVN_URM_MIRROR -Dmaven.repo.local
mvn -U -B -Pspark310tests,snapshot-shims test $MVN_URM_MIRROR -Dmaven.repo.local=$WORKSPACE/.m2

# Parse cudf and spark files from local mvn repo
jenkins/printJarVersion.sh "CUDFVersion" "${WORKSPACE}/.m2/ai/rapids/cudf/${CUDF_VER}" "cudf-${CUDF_VER}" "-${CUDA_CLASSIFIER}.jar"
jenkins/printJarVersion.sh "SPARKVersion" "${WORKSPACE}/.m2/org/apache/spark/spark-core_2.12/${SPARK_VER}" "spark-core_2.12-${SPARK_VER}" ".jar"
jenkins/printJarVersion.sh "CUDFVersion" "${WORKSPACE}/.m2/ai/rapids/cudf/${CUDF_VER}" "cudf-${CUDF_VER}" "-${CUDA_CLASSIFIER}.jar" $SERVER_ID
jenkins/printJarVersion.sh "SPARKVersion" "${WORKSPACE}/.m2/org/apache/spark/spark-core_2.12/${SPARK_VER}" "spark-core_2.12-${SPARK_VER}" ".jar" $SERVER_ID
40 changes: 40 additions & 0 deletions shims/spark300/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,46 @@
<description>The RAPIDS SQL plugin for Apache Spark 3.0.0 Shim</description>
<version>0.2.0-SNAPSHOT</version>

<!-- Set 'spark.version' for the shims layer -->
<!-- Create a separate file 'SPARK_VER.properties' in the jar to save cudf & spark version info -->
<build>
<plugins>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>dependency</id>
<phase>generate-resources</phase>
<configuration>
<target>
<mkdir dir="${project.build.directory}/extra-resources"/>
<exec executable="bash" output="${project.build.directory}/extra-resources/spark-${spark300.version}-info.properties">
<arg value="${user.dir}/build/dependency-info.sh"/>
<arg value="${cudf.version}"/>
<arg value="${cuda.version}"/>
<arg value="${spark300.version}"/>
</exec>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>

<resources>
<resource>
<!-- Include the properties file to provide the build information. -->
<directory>${project.build.directory}/extra-resources</directory>
</resource>
<resource>
<directory>src/main/resources</directory>
</resource>
</resources>
</build>

<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
Expand Down
40 changes: 40 additions & 0 deletions shims/spark300db/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,46 @@
<description>The RAPIDS SQL plugin for Apache Spark 3.0.0 Databricks Shim</description>
<version>0.2.0-SNAPSHOT</version>

<!-- Set 'spark.version' for the shims layer -->
<!-- Create a separate file 'SPARK_VER.properties' in the jar to save cudf & spark version info -->
<build>
<plugins>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>dependency</id>
<phase>generate-resources</phase>
<configuration>
<target>
<mkdir dir="${project.build.directory}/extra-resources"/>
<exec executable="bash" output="${project.build.directory}/extra-resources/spark-${spark30db.version}-info.properties">
<arg value="${user.dir}/build/dependency-info.sh"/>
<arg value="${cudf.version}"/>
<arg value="${cuda.version}"/>
<arg value="${spark30db.version}"/>
</exec>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>

<resources>
<resource>
<!-- Include the properties file to provide the build information. -->
<directory>${project.build.directory}/extra-resources</directory>
</resource>
<resource>
<directory>src/main/resources</directory>
</resource>
</resources>
</build>

<properties>
<parquet.version>1.10.1</parquet.version>
<spark30db.version>3.0.0-databricks</spark30db.version>
Expand Down
40 changes: 40 additions & 0 deletions shims/spark301/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,46 @@
<description>The RAPIDS SQL plugin for Apache Spark 3.0.1 Shim</description>
<version>0.2.0-SNAPSHOT</version>

<!-- Set 'spark.version' for the shims layer -->
<!-- Create a separate file 'SPARK_VER.properties' in the jar to save cudf & spark version info -->
<build>
<plugins>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>dependency</id>
<phase>generate-resources</phase>
<configuration>
<target>
<mkdir dir="${project.build.directory}/extra-resources"/>
<exec executable="bash" output="${project.build.directory}/extra-resources/spark-${spark301.version}-info.properties">
<arg value="${user.dir}/build/dependency-info.sh"/>
<arg value="${cudf.version}"/>
<arg value="${cuda.version}"/>
<arg value="${spark301.version}"/>
</exec>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>

<resources>
<resource>
<!-- Include the properties file to provide the build information. -->
<directory>${project.build.directory}/extra-resources</directory>
</resource>
<resource>
<directory>src/main/resources</directory>
</resource>
</resources>
</build>

<dependencies>
<dependency>
<groupId>com.nvidia</groupId>
Expand Down
40 changes: 40 additions & 0 deletions shims/spark302/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,46 @@
<description>The RAPIDS SQL plugin for Apache Spark 3.0.2 Shim</description>
<version>0.2.0-SNAPSHOT</version>

<!-- Set 'spark.version' for the shims layer -->
<!-- Create a separate file 'SPARK_VER.properties' in the jar to save cudf & spark version info -->
<build>
<plugins>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>dependency</id>
<phase>generate-resources</phase>
<configuration>
<target>
<mkdir dir="${project.build.directory}/extra-resources"/>
<exec executable="bash" output="${project.build.directory}/extra-resources/spark-${spark302.version}-info.properties">
<arg value="${user.dir}/build/dependency-info.sh"/>
<arg value="${cudf.version}"/>
<arg value="${cuda.version}"/>
<arg value="${spark302.version}"/>
</exec>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>

<resources>
<resource>
<!-- Include the properties file to provide the build information. -->
<directory>${project.build.directory}/extra-resources</directory>
</resource>
<resource>
<directory>src/main/resources</directory>
</resource>
</resources>
</build>

<dependencies>
<dependency>
<groupId>com.nvidia</groupId>
Expand Down
40 changes: 40 additions & 0 deletions shims/spark310/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,46 @@
<description>The RAPIDS SQL plugin for Apache Spark 3.1.0 Shim</description>
<version>0.2.0-SNAPSHOT</version>

<!-- Set 'spark.version' for the shims layer -->
<!-- Create a separate file 'SPARK_VER.properties' in the jar to save cudf & spark version info -->
<build>
<plugins>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>dependency</id>
<phase>generate-resources</phase>
<configuration>
<target>
<mkdir dir="${project.build.directory}/extra-resources"/>
<exec executable="bash" output="${project.build.directory}/extra-resources/spark-${spark310.version}-info.properties">
<arg value="${user.dir}/build/dependency-info.sh"/>
<arg value="${cudf.version}"/>
<arg value="${cuda.version}"/>
<arg value="${spark310.version}"/>
</exec>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>

<resources>
<resource>
<!-- Include the properties file to provide the build information. -->
<directory>${project.build.directory}/extra-resources</directory>
</resource>
<resource>
<directory>src/main/resources</directory>
</resource>
</resources>
</build>

<dependencies>
<dependency>
<groupId>com.nvidia</groupId>
Expand Down

0 comments on commit abeb5bb

Please sign in to comment.