Remove support for databricks 7.0 runtime - shim spark300db (NVIDIA#1145

) * Remove databricks 3.0.0 shim layer Signed-off-by: Thomas Graves <tgraves@nvidia.com> * Update docs Signed-off-by: Thomas Graves <tgraves@nvidia.com>
sperlingxx · Nov 20, 2020 · aad21fe · aad21fe
1 parent 99d89e9
commit aad21fe
Show file tree

Hide file tree

Showing 22 changed files with 9 additions and 1,726 deletions.
diff --git a/docs/FAQ.md b/docs/FAQ.md
@@ -28,7 +28,6 @@ top of these changes and release updates as quickly as possible.
 
 The RAPIDS Accelerator for Apache Spark officially supports
 [Apache Spark](get-started/getting-started-on-prem.md),
-[Databricks Runtime 7.0](get-started/getting-started-databricks.md)
 [Databricks Runtime 7.3](get-started/getting-started-databricks.md)
 and [Google Cloud Dataproc](get-started/getting-started-gcp.md).
 Most distributions based off of Apache Spark 3.0.0 should work, but because the plugin replaces

diff --git a/docs/get-started/getting-started-databricks.md b/docs/get-started/getting-started-databricks.md
@@ -9,9 +9,9 @@ parent: Getting-Started
 This guide will run through how to set up the RAPIDS Accelerator for Apache Spark 3.0 on Databricks.  At the end of this guide, the reader will be able to run a sample Apache Spark application that runs on NVIDIA GPUs on Databricks.
 
 ## Prerequisites
-    * Apache Spark 3.0 running in DataBricks Runtime 7.0 ML with GPU or Runtime 7.3 ML with GPU
-    * AWS: 7.0 ML (includes Apache Spark 3.0.0, GPU, Scala 2.12) or 7.3 LTS ML (includes Apache Spark 3.0.1, GPU, Scala 2.12)
-    * Azure: 7.0 ML (GPU, Scala 2.12, Spark 3.0.0) or 7.3 LTS ML (GPU, Scala 2.12, Spark 3.0.1)
+    * Apache Spark 3.0 running in DataBricks Runtime 7.3 ML with GPU
+    * AWS:7.3 LTS ML (includes Apache Spark 3.0.1, GPU, Scala 2.12)
+    * Azure: 7.3 LTS ML (GPU, Scala 2.12, Spark 3.0.1)
 
 The number of GPUs per node dictates the number of Spark executors that can run in that node.
 

diff --git a/integration_tests/pom.xml b/integration_tests/pom.xml
@@ -33,9 +33,9 @@
     </properties>
     <profiles>
         <profile>
-            <id>spark300dbtests</id>
+            <id>spark301dbtests</id>
             <properties>
-                <spark.test.version>3.0.0-databricks</spark.test.version>
+                <spark.test.version>3.0.1-databricks</spark.test.version>
             </properties>
         </profile>
         <profile>

diff --git a/jenkins/Jenkinsfile-blossom.premerge b/jenkins/Jenkinsfile-blossom.premerge
@@ -223,7 +223,7 @@ pipeline {
                         step([$class                : 'JacocoPublisher',
                               execPattern           : '**/target/jacoco.exec',
                               classPattern          : 'target/jacoco_classes/',
-                              sourcePattern         : 'shuffle-plugin/src/main/scala/,udf-compiler/src/main/scala/,sql-plugin/src/main/java/,sql-plugin/src/main/scala/,shims/spark310/src/main/scala/,shims/spark300/src/main/scala/,shims/spark300db/src/main/scala/,shims/spark301/src/main/scala/,shims/spark302/src/main/scala/',
+                              sourcePattern         : 'shuffle-plugin/src/main/scala/,udf-compiler/src/main/scala/,sql-plugin/src/main/java/,sql-plugin/src/main/scala/,shims/spark310/src/main/scala/,shims/spark300/src/main/scala/,shims/spark301db/src/main/scala/,shims/spark301/src/main/scala/,shims/spark302/src/main/scala/',
                               sourceInclusionPattern: '**/*.java,**/*.scala'
                         ])
                     }

diff --git a/jenkins/Jenkinsfile.databricksnightly b/jenkins/Jenkinsfile.databricksnightly
diff --git a/jenkins/Jenkinsfile.databricksrelease b/jenkins/Jenkinsfile.databricksrelease
diff --git a/pom.xml b/pom.xml
@@ -125,7 +125,7 @@
             </build>
         </profile>
         <profile>
-            <id>databricks</id>
+            <id>databricks301</id>
             <properties>
 	        <rat.consoleOutput>true</rat.consoleOutput>
             </properties>

diff --git a/shims/aggregator/pom.xml b/shims/aggregator/pom.xml
@@ -33,17 +33,6 @@
     <version>0.3.0-SNAPSHOT</version>
 
    <profiles>
-        <profile>
-            <id>databricks</id>
-            <dependencies>
-                <dependency>
-                    <groupId>com.nvidia</groupId>
-                    <artifactId>rapids-4-spark-shims-spark300-databricks_${scala.binary.version}</artifactId>
-                    <version>${project.version}</version>
-                    <scope>compile</scope>
-                </dependency>
-            </dependencies>
-        </profile>
         <profile>
             <id>databricks301</id>
             <dependencies>
@@ -59,12 +48,6 @@
             <!-- use a separate profile to just pull databricks from maven repository without building it -->
             <id>include-databricks</id>
             <dependencies>
-                <dependency>
-                    <groupId>com.nvidia</groupId>
-                    <artifactId>rapids-4-spark-shims-spark300-databricks_${scala.binary.version}</artifactId>
-                    <version>${project.version}</version>
-                    <scope>compile</scope>
-                </dependency>
                 <dependency>
                     <groupId>com.nvidia</groupId>
                     <artifactId>rapids-4-spark-shims-spark301-databricks_${scala.binary.version}</artifactId>

diff --git a/shims/pom.xml b/shims/pom.xml
@@ -33,12 +33,6 @@
     <version>0.3.0-SNAPSHOT</version>
 
     <profiles>
-        <profile>
-            <id>databricks</id>
-            <modules>
-                <module>spark300db</module>
-            </modules>
-        </profile>
         <profile>
             <id>databricks301</id>
             <modules>