diff --git a/api_validation/README.md b/api_validation/README.md
index e6b16a41e55..25bee6d650c 100644
--- a/api_validation/README.md
+++ b/api_validation/README.md
@@ -20,7 +20,7 @@ cd api_validation
// To run validation script on all version of Spark(3.0.0, 3.0.1 and 3.1.0-SNAPSHOT)
sh auditAllVersions.sh
-// To run script on particular version we can use profile(spark300, spark301 and spark310)
+// To run script on particular version we can use profile(spark300, spark301 and spark311)
mvn scala:run -P spark300
```
diff --git a/api_validation/auditAllVersions.sh b/api_validation/auditAllVersions.sh
index 33862ccff6e..ef75ab157c8 100644
--- a/api_validation/auditAllVersions.sh
+++ b/api_validation/auditAllVersions.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-# Copyright (c) 2020, NVIDIA CORPORATION.
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -16,4 +16,4 @@ set -ex
mvn scala:run -P spark300
mvn scala:run -P spark301
-mvn scala:run -P spark310
+mvn scala:run -P spark311
diff --git a/api_validation/pom.xml b/api_validation/pom.xml
index f0ff03cefde..15ae18becf6 100644
--- a/api_validation/pom.xml
+++ b/api_validation/pom.xml
@@ -1,6 +1,6 @@
3.0.1
3.0.2-SNAPSHOT
- 3.1.0-SNAPSHOT
+ 3.1.1-SNAPSHOT
3.6.0
4.3.0
diff --git a/shims/aggregator/pom.xml b/shims/aggregator/pom.xml
index d834fc2ad87..7ed3bcf0aa9 100644
--- a/shims/aggregator/pom.xml
+++ b/shims/aggregator/pom.xml
@@ -1,6 +1,6 @@
@@ -44,11 +44,11 @@
-
+
-
+
@@ -84,7 +84,7 @@
org.apache.spark
spark-sql_${scala.binary.version}
- ${spark310.version}
+ ${spark311.version}
provided
diff --git a/shims/spark311/src/main/resources/META-INF/services/com.nvidia.spark.rapids.SparkShimServiceProvider b/shims/spark311/src/main/resources/META-INF/services/com.nvidia.spark.rapids.SparkShimServiceProvider
new file mode 100644
index 00000000000..c15f4a8712c
--- /dev/null
+++ b/shims/spark311/src/main/resources/META-INF/services/com.nvidia.spark.rapids.SparkShimServiceProvider
@@ -0,0 +1 @@
+com.nvidia.spark.rapids.shims.spark311.SparkShimServiceProvider
diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuBroadcastHashJoinExec.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuBroadcastHashJoinExec.scala
similarity index 98%
rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuBroadcastHashJoinExec.scala
rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuBroadcastHashJoinExec.scala
index 6767f5f2849..1ac848db516 100644
--- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuBroadcastHashJoinExec.scala
+++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuBroadcastHashJoinExec.scala
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-package com.nvidia.spark.rapids.shims.spark310
+package com.nvidia.spark.rapids.shims.spark311
import com.nvidia.spark.rapids._
import com.nvidia.spark.rapids.GpuMetricNames._
diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuBroadcastNestedLoopJoinExec.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuBroadcastNestedLoopJoinExec.scala
similarity index 93%
rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuBroadcastNestedLoopJoinExec.scala
rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuBroadcastNestedLoopJoinExec.scala
index e41f6a999e1..d027cd41b61 100644
--- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuBroadcastNestedLoopJoinExec.scala
+++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuBroadcastNestedLoopJoinExec.scala
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-package com.nvidia.spark.rapids.shims.spark310
+package com.nvidia.spark.rapids.shims.spark311
import com.nvidia.spark.rapids.GpuBuildSide
diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuOrcScan.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuOrcScan.scala
similarity index 96%
rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuOrcScan.scala
rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuOrcScan.scala
index 1c65fcb4fee..9360bedb30f 100644
--- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuOrcScan.scala
+++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuOrcScan.scala
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-package com.nvidia.spark.rapids.shims.spark310
+package com.nvidia.spark.rapids.shims.spark311
import com.nvidia.spark.rapids.{GpuOrcScanBase, RapidsConf}
import org.apache.hadoop.conf.Configuration
diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuParquetScan.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuParquetScan.scala
similarity index 96%
rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuParquetScan.scala
rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuParquetScan.scala
index 2d03c067ef2..ae08dad6a05 100644
--- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuParquetScan.scala
+++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuParquetScan.scala
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-package com.nvidia.spark.rapids.shims.spark310
+package com.nvidia.spark.rapids.shims.spark311
import com.nvidia.spark.rapids.{GpuParquetScanBase, RapidsConf}
import org.apache.hadoop.conf.Configuration
diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuShuffleExchangeExec.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuShuffleExchangeExec.scala
similarity index 94%
rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuShuffleExchangeExec.scala
rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuShuffleExchangeExec.scala
index deb2e17b4c1..d2b0ca6ffba 100644
--- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuShuffleExchangeExec.scala
+++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuShuffleExchangeExec.scala
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-package com.nvidia.spark.rapids.shims.spark310
+package com.nvidia.spark.rapids.shims.spark311
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.plans.logical.Statistics
diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuShuffledHashJoinExec.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuShuffledHashJoinExec.scala
similarity index 98%
rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuShuffledHashJoinExec.scala
rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuShuffledHashJoinExec.scala
index 237f5094c35..63f4cf790d4 100644
--- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuShuffledHashJoinExec.scala
+++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuShuffledHashJoinExec.scala
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-package com.nvidia.spark.rapids.shims.spark310
+package com.nvidia.spark.rapids.shims.spark311
import com.nvidia.spark.rapids._
import com.nvidia.spark.rapids.GpuMetricNames._
diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuSortMergeJoinExec.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuSortMergeJoinExec.scala
similarity index 97%
rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuSortMergeJoinExec.scala
rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuSortMergeJoinExec.scala
index 523426235b7..8228b432f7c 100644
--- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuSortMergeJoinExec.scala
+++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuSortMergeJoinExec.scala
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-package com.nvidia.spark.rapids.shims.spark310
+package com.nvidia.spark.rapids.shims.spark311
import com.nvidia.spark.rapids._
diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/ParquetCachedBatchSerializer.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/ParquetCachedBatchSerializer.scala
similarity index 99%
rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/ParquetCachedBatchSerializer.scala
rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/ParquetCachedBatchSerializer.scala
index fc8e2aa9d98..380dc302bb3 100644
--- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/ParquetCachedBatchSerializer.scala
+++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/ParquetCachedBatchSerializer.scala
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-package com.nvidia.spark.rapids.shims.spark310
+package com.nvidia.spark.rapids.shims.spark311
import java.io.{InputStream, IOException}
import java.lang.reflect.Method
diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/Spark310Shims.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/Spark311Shims.scala
similarity index 97%
rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/Spark310Shims.scala
rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/Spark311Shims.scala
index 9203abd0da9..05796e8e41b 100644
--- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/Spark310Shims.scala
+++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/Spark311Shims.scala
@@ -14,11 +14,11 @@
* limitations under the License.
*/
-package com.nvidia.spark.rapids.shims.spark310
+package com.nvidia.spark.rapids.shims.spark311
import com.nvidia.spark.rapids._
import com.nvidia.spark.rapids.shims.spark301.Spark301Shims
-import com.nvidia.spark.rapids.spark310.RapidsShuffleManager
+import com.nvidia.spark.rapids.spark311.RapidsShuffleManager
import org.apache.spark.SparkEnv
import org.apache.spark.sql.SparkSession
@@ -38,11 +38,11 @@ import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BroadcastNes
import org.apache.spark.sql.internal.StaticSQLConf
import org.apache.spark.sql.rapids.{GpuFileSourceScanExec, GpuStringReplace, ShuffleManagerShimBase}
import org.apache.spark.sql.rapids.execution.{GpuBroadcastNestedLoopJoinExecBase, GpuShuffleExchangeExecBase}
-import org.apache.spark.sql.rapids.shims.spark310._
+import org.apache.spark.sql.rapids.shims.spark311._
import org.apache.spark.sql.types._
import org.apache.spark.storage.{BlockId, BlockManagerId}
-class Spark310Shims extends Spark301Shims {
+class Spark311Shims extends Spark301Shims {
override def getSparkShimVersion: ShimVersion = SparkShimServiceProvider.VERSION
@@ -93,7 +93,7 @@ class Spark310Shims extends Spark301Shims {
}
}
- def exprs310: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = Seq(
+ def exprs311: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = Seq(
GpuOverrides.expr[Cast](
"Convert a column of one type of data into another type",
new CastChecks(),
@@ -194,7 +194,7 @@ class Spark310Shims extends Spark301Shims {
).map(r => (r.getClassFor.asSubclass(classOf[Expression]), r)).toMap
override def getExprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = {
- super.exprs301 ++ exprs310
+ super.exprs301 ++ exprs311
}
override def getExecs: Map[Class[_ <: SparkPlan], ExecRule[_ <: SparkPlan]] = {
@@ -338,7 +338,7 @@ class Spark310Shims extends Spark301Shims {
val serName = plan.conf.getConf(StaticSQLConf.SPARK_CACHE_SERIALIZER)
val serClass = Class.forName(serName)
if (serClass == classOf[ParquetCachedBatchSerializer]) {
- org.apache.spark.sql.rapids.shims.spark310.GpuColumnarToRowTransitionExec(plan)
+ org.apache.spark.sql.rapids.shims.spark311.GpuColumnarToRowTransitionExec(plan)
} else {
GpuColumnarToRowExec(plan)
}
diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/SparkShimServiceProvider.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala
similarity index 71%
rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/SparkShimServiceProvider.scala
rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala
index a14695c1f7e..8ce22923520 100644
--- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/SparkShimServiceProvider.scala
+++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,13 +14,15 @@
* limitations under the License.
*/
-package com.nvidia.spark.rapids.shims.spark310
+package com.nvidia.spark.rapids.shims.spark311
import com.nvidia.spark.rapids.{SparkShims, SparkShimVersion}
object SparkShimServiceProvider {
- val VERSION = SparkShimVersion(3, 1, 0)
- val VERSIONNAMES = Seq(s"$VERSION", s"$VERSION-SNAPSHOT")
+ // keep 3.1.0 snapshot version for now until 3.1.1 is released
+ val VERSION310 = SparkShimVersion(3, 1, 0)
+ val VERSION = SparkShimVersion(3, 1, 1)
+ val VERSIONNAMES = Seq(s"$VERSION310-SNAPSHOT", s"$VERSION", s"$VERSION-SNAPSHOT")
}
class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider {
@@ -30,6 +32,6 @@ class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceP
}
def buildShim: SparkShims = {
- new Spark310Shims()
+ new Spark311Shims()
}
}
diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/spark310/RapidsShuffleManager.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/spark310/RapidsShuffleManager.scala
similarity index 84%
rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/spark310/RapidsShuffleManager.scala
rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/spark310/RapidsShuffleManager.scala
index a6c783c2e1f..018f5421b7e 100644
--- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/spark310/RapidsShuffleManager.scala
+++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/spark310/RapidsShuffleManager.scala
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,10 +14,10 @@
* limitations under the License.
*/
-package com.nvidia.spark.rapids.spark310
+package com.nvidia.spark.rapids.spark311
import org.apache.spark.SparkConf
-import org.apache.spark.sql.rapids.shims.spark310.RapidsShuffleInternalManager
+import org.apache.spark.sql.rapids.shims.spark311.RapidsShuffleInternalManager
/** A shuffle manager optimized for the RAPIDS Plugin for Apache Spark. */
sealed class RapidsShuffleManager(
diff --git a/shims/spark310/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/ParquetMaterializer.scala b/shims/spark311/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/ParquetMaterializer.scala
similarity index 100%
rename from shims/spark310/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/ParquetMaterializer.scala
rename to shims/spark311/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/ParquetMaterializer.scala
diff --git a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/GpuColumnarToRowTransitionExec.scala b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/GpuColumnarToRowTransitionExec.scala
similarity index 90%
rename from shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/GpuColumnarToRowTransitionExec.scala
rename to shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/GpuColumnarToRowTransitionExec.scala
index d282d30a0a1..e54d8c30793 100644
--- a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/GpuColumnarToRowTransitionExec.scala
+++ b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/GpuColumnarToRowTransitionExec.scala
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-package org.apache.spark.sql.rapids.shims.spark310
+package org.apache.spark.sql.rapids.shims.spark311
import com.nvidia.spark.rapids.GpuColumnarToRowExecParent
diff --git a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/GpuInMemoryTableScanExec.scala b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/GpuInMemoryTableScanExec.scala
similarity index 96%
rename from shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/GpuInMemoryTableScanExec.scala
rename to shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/GpuInMemoryTableScanExec.scala
index ea8ae8f68bc..ade9b30118a 100644
--- a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/GpuInMemoryTableScanExec.scala
+++ b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/GpuInMemoryTableScanExec.scala
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019-2020, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,10 +14,10 @@
* limitations under the License.
*/
-package org.apache.spark.sql.rapids.shims.spark310
+package org.apache.spark.sql.rapids.shims.spark311
import com.nvidia.spark.rapids.GpuExec
-import com.nvidia.spark.rapids.shims.spark310.ParquetCachedBatchSerializer
+import com.nvidia.spark.rapids.shims.spark311.ParquetCachedBatchSerializer
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.InternalRow
diff --git a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/GpuSchemaUtils.scala b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/GpuSchemaUtils.scala
similarity index 90%
rename from shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/GpuSchemaUtils.scala
rename to shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/GpuSchemaUtils.scala
index c71c77d6239..42e04cc5384 100644
--- a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/GpuSchemaUtils.scala
+++ b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/GpuSchemaUtils.scala
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-package org.apache.spark.sql.rapids.shims.spark310
+package org.apache.spark.sql.rapids.shims.spark311
import org.apache.spark.sql.catalyst.analysis.Resolver
import org.apache.spark.sql.types.StructType
diff --git a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/RapidsShuffleInternalManager.scala b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/RapidsShuffleInternalManager.scala
similarity index 93%
rename from shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/RapidsShuffleInternalManager.scala
rename to shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/RapidsShuffleInternalManager.scala
index 6ab55c757ec..72870b7eea6 100644
--- a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/RapidsShuffleInternalManager.scala
+++ b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/RapidsShuffleInternalManager.scala
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019-2020, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-package org.apache.spark.sql.rapids.shims.spark310
+package org.apache.spark.sql.rapids.shims.spark311
import org.apache.spark.{SparkConf, TaskContext}
import org.apache.spark.shuffle._
diff --git a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/ShuffleManagerShim.scala b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/ShuffleManagerShim.scala
similarity index 92%
rename from shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/ShuffleManagerShim.scala
rename to shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/ShuffleManagerShim.scala
index fc5603d9e6c..edc7a85410f 100644
--- a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/ShuffleManagerShim.scala
+++ b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/ShuffleManagerShim.scala
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.spark.sql.rapids.shims.spark310
+package org.apache.spark.sql.rapids.shims.spark311
import org.apache.spark.TaskContext
import org.apache.spark.shuffle.{ShuffleHandle, ShuffleManager, ShuffleReader, ShuffleReadMetricsReporter}
diff --git a/tests-spark310+/README.md b/tests-spark310+/README.md
index 02d7b66e942..d3601d5d057 100644
--- a/tests-spark310+/README.md
+++ b/tests-spark310+/README.md
@@ -14,9 +14,9 @@ Example:
As of writing of this document this module contains tests that are strongly tied to classes in
Spark-3.1.0-SNAPSHOT.
-These tests can be executed by choosing profile `spark310tests` like so,
+These tests can be executed by choosing profile `spark311tests` like so,
-`mvn -Pspark310tests -wildcardSuites=`
+`mvn -Pspark311tests -wildcardSuites=`
For a more comprehensive overview of tests in Rapids Accelerator please refer to the following
diff --git a/tests-spark310+/src/test/scala/com/nvidia/spark/rapids/Spark310ParquetWriterSuite.scala b/tests-spark310+/src/test/scala/com/nvidia/spark/rapids/Spark310ParquetWriterSuite.scala
index 30ab5c645a9..2796b40cd22 100644
--- a/tests-spark310+/src/test/scala/com/nvidia/spark/rapids/Spark310ParquetWriterSuite.scala
+++ b/tests-spark310+/src/test/scala/com/nvidia/spark/rapids/Spark310ParquetWriterSuite.scala
@@ -19,7 +19,7 @@ package com.nvidia.spark.rapids
import scala.collection.mutable
import ai.rapids.cudf.{ColumnVector, DType, Table, TableWriter}
-import com.nvidia.spark.rapids.shims.spark310.{ParquetCachedBatchSerializer, ParquetOutputFileFormat}
+import com.nvidia.spark.rapids.shims.spark311.{ParquetCachedBatchSerializer, ParquetOutputFileFormat}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapreduce.{RecordWriter, TaskAttemptContext}
import org.mockito.ArgumentMatchers._
diff --git a/tests/README.md b/tests/README.md
index 17e5addeb91..9d50c0ba603 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -56,7 +56,7 @@ default version runs against Spark 3.0.0, to run against other versions use one
profiles:
- `-Pspark301tests` (spark 3.0.1)
- `-Pspark302tests` (spark 3.0.2)
- - `-Pspark310tests` (spark 3.1.0)
+ - `-Pspark311tests` (spark 3.1.1)
Please refer to the [tests project POM](pom.xml) to see the list of test profiles supported.
Apache Spark specific configurations can be passed in by setting the `SPARK_CONF` environment
@@ -64,7 +64,7 @@ variable.
Examples:
- To run tests against Apache Spark 3.1.0,
- `mvn -P spark310tests test`
+ `mvn -P spark311tests test`
- To pass Apache Spark configs `--conf spark.dynamicAllocation.enabled=false --conf spark.task.cpus=1` do something like.
`SPARK_CONF="spark.dynamicAllocation.enabled=false,spark.task.cpus=1" mvn ...`
- To run test ParquetWriterSuite in package com.nvidia.spark.rapids, issue `mvn test -DwildcardSuites="com.nvidia.spark.rapids.ParquetWriterSuite"`
diff --git a/tests/pom.xml b/tests/pom.xml
index dfc87fc5da8..7c22c4045b3 100644
--- a/tests/pom.xml
+++ b/tests/pom.xml
@@ -1,6 +1,6 @@