Rename Spark 3.1.0 shim to Spark 3.1.1 to match community (NVIDIA#1488)

* Rename spark310 to spark311

Signed-off-by: Thomas Graves <tgraves@nvidia.com>

* Fix typoes

Signed-off-by: Thomas Graves <tgraves@nvidia.com>

api_validation/README.md

@@ -20,7 +20,7 @@ cd api_validation
 // To run validation script on all version of Spark(3.0.0, 3.0.1 and 3.1.0-SNAPSHOT)
 sh auditAllVersions.sh
 
-// To run script on particular version we can use profile(spark300, spark301 and spark310)
+// To run script on particular version we can use profile(spark300, spark301 and spark311)
 mvn scala:run -P spark300
 ```
 

api_validation/auditAllVersions.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2020, NVIDIA CORPORATION.
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,4 +16,4 @@ set -ex
 
 mvn scala:run -P spark300
 mvn scala:run -P spark301
-mvn scala:run -P spark310
+mvn scala:run -P spark311

api_validation/pom.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright (c) 2020, NVIDIA CORPORATION.
+  Copyright (c) 2020-2021, NVIDIA CORPORATION.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -41,9 +41,9 @@
             </properties>
         </profile>
         <profile>
-            <id>spark310</id>
+            <id>spark311</id>
             <properties>
-                <spark.version>${spark310.version}</spark.version>
+                <spark.version>${spark311.version}</spark.version>
             </properties>
         </profile>
     </profiles>

api_validation/src/main/scala/com/nvidia/spark/rapids/api/ApiValidation.scala

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -69,7 +69,7 @@ object ApiValidation extends Logging {
     val gpuKeys = gpuExecs.keys
     var printNewline = false
 
-    val sparkToShimMap = Map("3.0.0" -> "spark300", "3.0.1" -> "spark301", "3.1.0" -> "spark310")
+    val sparkToShimMap = Map("3.0.0" -> "spark300", "3.0.1" -> "spark301", "3.1.1" -> "spark311")
     val sparkVersion = ShimLoader.getSparkShims.getSparkShimVersion.toString
     var shimVersion = sparkToShimMap(sparkVersion)
     // There is no separate implementation for Execs in spark-3.0.1.

docs/get-started/getting-started-on-prem.md

@@ -433,7 +433,7 @@ that matches your Spark version. Currently we support
    - Spark 3.0.0 (com.nvidia.spark.rapids.spark300.RapidsShuffleManager) 
    - Spark 3.0.1 (com.nvidia.spark.rapids.spark301.RapidsShuffleManager) 
    - Spark 3.0.2 (com.nvidia.spark.rapids.spark302.RapidsShuffleManager) 
-   - Spark 3.1.0 (com.nvidia.spark.rapids.spark310.RapidsShuffleManager)
+   - Spark 3.1.1 (com.nvidia.spark.rapids.spark311.RapidsShuffleManager)
 
 ```shell
 ...

integration_tests/pom.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright (c) 2020, NVIDIA CORPORATION.
+  Copyright (c) 2020-2021, NVIDIA CORPORATION.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -51,9 +51,9 @@
             </properties>
         </profile>
         <profile>
-            <id>spark310tests</id>
+            <id>spark311tests</id>
             <properties>
-                <spark.test.version>3.1.0-SNAPSHOT</spark.test.version>
+                <spark.test.version>3.1.1-SNAPSHOT</spark.test.version>
             </properties>
         </profile>
     </profiles>

jenkins/Jenkinsfile-blossom.premerge

@@ -1,6 +1,6 @@
 #!/usr/local/env groovy
 /*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -161,7 +161,7 @@ pipeline {
                         step([$class                : 'JacocoPublisher',
                               execPattern           : '**/target/jacoco.exec',
                               classPattern          : 'target/jacoco_classes/',
-                              sourcePattern         : 'shuffle-plugin/src/main/scala/,udf-compiler/src/main/scala/,sql-plugin/src/main/java/,sql-plugin/src/main/scala/,shims/spark310/src/main/scala/,shims/spark300/src/main/scala/,shims/spark301db/src/main/scala/,shims/spark301/src/main/scala/,shims/spark302/src/main/scala/',
+                              sourcePattern         : 'shuffle-plugin/src/main/scala/,udf-compiler/src/main/scala/,sql-plugin/src/main/java/,sql-plugin/src/main/scala/,shims/spark311/src/main/scala/,shims/spark300/src/main/scala/,shims/spark301db/src/main/scala/,shims/spark301/src/main/scala/,shims/spark302/src/main/scala/',
                               sourceInclusionPattern: '**/*.java,**/*.scala'
                         ])
                     }

jenkins/spark-nightly-build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,7 +25,7 @@ mvn -U -B -Pinclude-databricks,snapshot-shims clean deploy $MVN_URM_MIRROR -Dmav
 # Run unit tests against other spark versions
 mvn -U -B -Pspark301tests,snapshot-shims test $MVN_URM_MIRROR -Dmaven.repo.local=$M2DIR
 mvn -U -B -Pspark302tests,snapshot-shims test $MVN_URM_MIRROR -Dmaven.repo.local=$M2DIR
-mvn -U -B -Pspark310tests,snapshot-shims test $MVN_URM_MIRROR -Dmaven.repo.local=$M2DIR
+mvn -U -B -Pspark311tests,snapshot-shims test $MVN_URM_MIRROR -Dmaven.repo.local=$M2DIR
 
 # Parse cudf and spark files from local mvn repo
 jenkins/printJarVersion.sh "CUDFVersion" "$M2DIR/ai/rapids/cudf/${CUDF_VER}" "cudf-${CUDF_VER}" "-${CUDA_CLASSIFIER}.jar" $SERVER_ID

jenkins/spark-premerge-build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -40,7 +40,7 @@ tar zxf $SPARK_HOME.tgz -C $ARTF_ROOT && \
 mvn -U -B $MVN_URM_MIRROR '-P!snapshot-shims' clean verify -Dpytest.TEST_TAGS=''
 # Run the unit tests for other Spark versions but dont run full python integration tests
 env -u SPARK_HOME mvn -U -B $MVN_URM_MIRROR -Pspark301tests,snapshot-shims test -Dpytest.TEST_TAGS=''
-env -u SPARK_HOME mvn -U -B $MVN_URM_MIRROR -Pspark310tests,snapshot-shims test -Dpytest.TEST_TAGS=''
+env -u SPARK_HOME mvn -U -B $MVN_URM_MIRROR -Pspark311tests,snapshot-shims test -Dpytest.TEST_TAGS=''
 
 # The jacoco coverage should have been collected, but because of how the shade plugin
 # works and jacoco we need to clean some things up so jacoco will only report for the

jenkins/spark-tests.sh

@@ -64,8 +64,8 @@ PARQUET_PERF="$WORKSPACE/integration_tests/src/test/resources/parquet_perf"
 PARQUET_ACQ="$WORKSPACE/integration_tests/src/test/resources/parquet_acq"
 OUTPUT="$WORKSPACE/output"
 
-# spark.sql.cache.serializer conf is ignored for versions prior to 3.1.0
-SERIALIZER="--conf spark.sql.cache.serializer=com.nvidia.spark.rapids.shims.spark310.ParquetCachedBatchSerializer"
+# spark.sql.cache.serializer conf is ignored for versions prior to 3.1.1
+SERIALIZER="--conf spark.sql.cache.serializer=com.nvidia.spark.rapids.shims.spark311.ParquetCachedBatchSerializer"
 
 BASE_SPARK_SUBMIT_ARGS="$BASE_SPARK_SUBMIT_ARGS \
     --master spark://$HOSTNAME:7077 \

pom.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright (c) 2020, NVIDIA CORPORATION.
+  Copyright (c) 2020-2021, NVIDIA CORPORATION.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -135,7 +135,7 @@
             <id>spark301tests</id>
         </profile>
         <profile>
-            <id>spark310tests</id>
+            <id>spark311tests</id>
             <modules>
                 <module>tests-spark310+</module>
             </modules>
@@ -174,7 +174,7 @@
          please update the snapshot-shims profile as well so it is accurate -->
         <spark301.version>3.0.1</spark301.version>
         <spark302.version>3.0.2-SNAPSHOT</spark302.version>
-        <spark310.version>3.1.0-SNAPSHOT</spark310.version>
+        <spark311.version>3.1.1-SNAPSHOT</spark311.version>
         <mockito.version>3.6.0</mockito.version>
         <scala.plugin.version>4.3.0</scala.plugin.version>
     </properties>

shims/aggregator/pom.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright (c) 2020, NVIDIA CORPORATION.
+  Copyright (c) 2020-2021, NVIDIA CORPORATION.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -64,7 +64,7 @@
             <dependencies>
                 <dependency>
                     <groupId>com.nvidia</groupId>
-                    <artifactId>rapids-4-spark-shims-spark310_${scala.binary.version}</artifactId>
+                    <artifactId>rapids-4-spark-shims-spark311_${scala.binary.version}</artifactId>
                     <version>${project.version}</version>
                     <scope>compile</scope>
                 </dependency>

shims/pom.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright (c) 2020, NVIDIA CORPORATION.
+  Copyright (c) 2020-2021, NVIDIA CORPORATION.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -46,7 +46,7 @@
             </activation>
             <modules>
                 <module>spark302</module>
-                <module>spark310</module>
+                <module>spark311</module>
             </modules>
         </profile>
     </profiles>

shims/spark310/pom.xml → shims/spark311/pom.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright (c) 2020, NVIDIA CORPORATION.
+  Copyright (c) 2020-2021, NVIDIA CORPORATION.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -26,9 +26,9 @@
 	<relativePath>../pom.xml</relativePath>
     </parent>
     <groupId>com.nvidia</groupId>
-    <artifactId>rapids-4-spark-shims-spark310_2.12</artifactId>
-    <name>RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.1.0 Shim</name>
-    <description>The RAPIDS SQL plugin for Apache Spark 3.1.0 Shim</description>
+    <artifactId>rapids-4-spark-shims-spark311_2.12</artifactId>
+    <name>RAPIDS Accelerator for Apache Spark SQL Plugin Spark 3.1.1 Shim</name>
+    <description>The RAPIDS SQL plugin for Apache Spark 3.1.1 Shim</description>
     <version>0.4.0-SNAPSHOT</version>
 
     <!-- Set 'spark.version' for the shims layer -->
@@ -44,11 +44,11 @@
                         <configuration>
                             <target>
                                 <mkdir dir="${project.build.directory}/extra-resources"/>
-                                <exec executable="bash" failonerror="true" output="${project.build.directory}/extra-resources/spark-${spark310.version}-info.properties">
+                                <exec executable="bash" failonerror="true" output="${project.build.directory}/extra-resources/spark-${spark311.version}-info.properties">
                                     <arg value="${user.dir}/build/dependency-info.sh"/>
                                     <arg value="${cudf.version}"/>
                                     <arg value="${cuda.version}"/>
-                                    <arg value="${spark310.version}"/>
+                                    <arg value="${spark311.version}"/>
                                 </exec>
                             </target>
                         </configuration>
@@ -84,7 +84,7 @@
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-sql_${scala.binary.version}</artifactId>
-            <version>${spark310.version}</version>
+            <version>${spark311.version}</version>
 	    <scope>provided</scope>
         </dependency>
     </dependencies>

shims/spark311/src/main/resources/META-INF/services/com.nvidia.spark.rapids.SparkShimServiceProvider

@@ -0,0 +1 @@
+com.nvidia.spark.rapids.shims.spark311.SparkShimServiceProvider

shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuBroadcastHashJoinExec.scala → shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuBroadcastHashJoinExec.scala

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.nvidia.spark.rapids.shims.spark310
+package com.nvidia.spark.rapids.shims.spark311
 
 import com.nvidia.spark.rapids._
 import com.nvidia.spark.rapids.GpuMetricNames._

shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuBroadcastNestedLoopJoinExec.scala → shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuBroadcastNestedLoopJoinExec.scala

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.nvidia.spark.rapids.shims.spark310
+package com.nvidia.spark.rapids.shims.spark311
 
 import com.nvidia.spark.rapids.GpuBuildSide
 

shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuOrcScan.scala → shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuOrcScan.scala

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.nvidia.spark.rapids.shims.spark310
+package com.nvidia.spark.rapids.shims.spark311
 
 import com.nvidia.spark.rapids.{GpuOrcScanBase, RapidsConf}
 import org.apache.hadoop.conf.Configuration

shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuParquetScan.scala → shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuParquetScan.scala

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.nvidia.spark.rapids.shims.spark310
+package com.nvidia.spark.rapids.shims.spark311
 
 import com.nvidia.spark.rapids.{GpuParquetScanBase, RapidsConf}
 import org.apache.hadoop.conf.Configuration

shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuShuffleExchangeExec.scala → shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuShuffleExchangeExec.scala

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.nvidia.spark.rapids.shims.spark310
+package com.nvidia.spark.rapids.shims.spark311
 
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.plans.logical.Statistics

shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuShuffledHashJoinExec.scala → shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuShuffledHashJoinExec.scala

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.nvidia.spark.rapids.shims.spark310
+package com.nvidia.spark.rapids.shims.spark311
 
 import com.nvidia.spark.rapids._
 import com.nvidia.spark.rapids.GpuMetricNames._

shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuSortMergeJoinExec.scala → shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuSortMergeJoinExec.scala

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.nvidia.spark.rapids.shims.spark310
+package com.nvidia.spark.rapids.shims.spark311
 
 import com.nvidia.spark.rapids._
 

shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/ParquetCachedBatchSerializer.scala → shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/ParquetCachedBatchSerializer.scala

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.nvidia.spark.rapids.shims.spark310
+package com.nvidia.spark.rapids.shims.spark311
 
 import java.io.{InputStream, IOException}
 import java.lang.reflect.Method