From 6a445be64c058a561cc15a53b6bfadd6cf53bf6f Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Mon, 11 Jan 2021 13:19:19 -0600 Subject: [PATCH] Rename Spark 3.1.0 shim to Spark 3.1.1 to match community (#1488) * Rename spark310 to spark311 Signed-off-by: Thomas Graves * Fix typoes Signed-off-by: Thomas Graves --- api_validation/README.md | 2 +- api_validation/auditAllVersions.sh | 4 ++-- api_validation/pom.xml | 6 +++--- .../nvidia/spark/rapids/api/ApiValidation.scala | 4 ++-- docs/get-started/getting-started-on-prem.md | 2 +- integration_tests/pom.xml | 6 +++--- jenkins/Jenkinsfile-blossom.premerge | 4 ++-- jenkins/spark-nightly-build.sh | 4 ++-- jenkins/spark-premerge-build.sh | 4 ++-- jenkins/spark-tests.sh | 4 ++-- pom.xml | 6 +++--- shims/aggregator/pom.xml | 4 ++-- shims/pom.xml | 4 ++-- ...om.nvidia.spark.rapids.SparkShimServiceProvider | 1 - shims/{spark310 => spark311}/pom.xml | 14 +++++++------- ...om.nvidia.spark.rapids.SparkShimServiceProvider | 1 + .../shims/spark311}/GpuBroadcastHashJoinExec.scala | 4 ++-- .../spark311}/GpuBroadcastNestedLoopJoinExec.scala | 4 ++-- .../spark/rapids/shims/spark311}/GpuOrcScan.scala | 4 ++-- .../rapids/shims/spark311}/GpuParquetScan.scala | 4 ++-- .../shims/spark311}/GpuShuffleExchangeExec.scala | 4 ++-- .../shims/spark311}/GpuShuffledHashJoinExec.scala | 4 ++-- .../shims/spark311}/GpuSortMergeJoinExec.scala | 4 ++-- .../spark311}/ParquetCachedBatchSerializer.scala | 4 ++-- .../rapids/shims/spark311/Spark311Shims.scala} | 14 +++++++------- .../shims/spark311}/SparkShimServiceProvider.scala | 12 +++++++----- .../rapids/spark310/RapidsShuffleManager.scala | 6 +++--- .../parquet/rapids/ParquetMaterializer.scala | 0 .../spark311}/GpuColumnarToRowTransitionExec.scala | 4 ++-- .../shims/spark311}/GpuInMemoryTableScanExec.scala | 6 +++--- .../rapids/shims/spark311}/GpuSchemaUtils.scala | 4 ++-- .../spark311}/RapidsShuffleInternalManager.scala | 4 ++-- .../shims/spark311}/ShuffleManagerShim.scala | 4 ++-- tests-spark310+/README.md | 4 ++-- .../spark/rapids/Spark310ParquetWriterSuite.scala | 2 +- tests/README.md | 4 ++-- tests/pom.xml | 6 +++--- 37 files changed, 87 insertions(+), 85 deletions(-) delete mode 100644 shims/spark310/src/main/resources/META-INF/services/com.nvidia.spark.rapids.SparkShimServiceProvider rename shims/{spark310 => spark311}/pom.xml (90%) create mode 100644 shims/spark311/src/main/resources/META-INF/services/com.nvidia.spark.rapids.SparkShimServiceProvider rename shims/{spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310 => spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311}/GpuBroadcastHashJoinExec.scala (98%) rename shims/{spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310 => spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311}/GpuBroadcastNestedLoopJoinExec.scala (93%) rename shims/{spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310 => spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311}/GpuOrcScan.scala (96%) rename shims/{spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310 => spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311}/GpuParquetScan.scala (96%) rename shims/{spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310 => spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311}/GpuShuffleExchangeExec.scala (94%) rename shims/{spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310 => spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311}/GpuShuffledHashJoinExec.scala (98%) rename shims/{spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310 => spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311}/GpuSortMergeJoinExec.scala (97%) rename shims/{spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310 => spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311}/ParquetCachedBatchSerializer.scala (99%) rename shims/{spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/Spark310Shims.scala => spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/Spark311Shims.scala} (97%) rename shims/{spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310 => spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311}/SparkShimServiceProvider.scala (71%) rename shims/{spark310 => spark311}/src/main/scala/com/nvidia/spark/rapids/spark310/RapidsShuffleManager.scala (84%) rename shims/{spark310 => spark311}/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/ParquetMaterializer.scala (100%) rename shims/{spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310 => spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311}/GpuColumnarToRowTransitionExec.scala (90%) rename shims/{spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310 => spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311}/GpuInMemoryTableScanExec.scala (96%) rename shims/{spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310 => spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311}/GpuSchemaUtils.scala (90%) rename shims/{spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310 => spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311}/RapidsShuffleInternalManager.scala (93%) rename shims/{spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310 => spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311}/ShuffleManagerShim.scala (92%) diff --git a/api_validation/README.md b/api_validation/README.md index e6b16a41e55..25bee6d650c 100644 --- a/api_validation/README.md +++ b/api_validation/README.md @@ -20,7 +20,7 @@ cd api_validation // To run validation script on all version of Spark(3.0.0, 3.0.1 and 3.1.0-SNAPSHOT) sh auditAllVersions.sh -// To run script on particular version we can use profile(spark300, spark301 and spark310) +// To run script on particular version we can use profile(spark300, spark301 and spark311) mvn scala:run -P spark300 ``` diff --git a/api_validation/auditAllVersions.sh b/api_validation/auditAllVersions.sh index 33862ccff6e..ef75ab157c8 100644 --- a/api_validation/auditAllVersions.sh +++ b/api_validation/auditAllVersions.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2021, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,4 +16,4 @@ set -ex mvn scala:run -P spark300 mvn scala:run -P spark301 -mvn scala:run -P spark310 +mvn scala:run -P spark311 diff --git a/api_validation/pom.xml b/api_validation/pom.xml index f0ff03cefde..15ae18becf6 100644 --- a/api_validation/pom.xml +++ b/api_validation/pom.xml @@ -1,6 +1,6 @@ 3.0.1 3.0.2-SNAPSHOT - 3.1.0-SNAPSHOT + 3.1.1-SNAPSHOT 3.6.0 4.3.0 diff --git a/shims/aggregator/pom.xml b/shims/aggregator/pom.xml index d834fc2ad87..7ed3bcf0aa9 100644 --- a/shims/aggregator/pom.xml +++ b/shims/aggregator/pom.xml @@ -1,6 +1,6 @@ @@ -44,11 +44,11 @@ - + - + @@ -84,7 +84,7 @@ org.apache.spark spark-sql_${scala.binary.version} - ${spark310.version} + ${spark311.version} provided diff --git a/shims/spark311/src/main/resources/META-INF/services/com.nvidia.spark.rapids.SparkShimServiceProvider b/shims/spark311/src/main/resources/META-INF/services/com.nvidia.spark.rapids.SparkShimServiceProvider new file mode 100644 index 00000000000..c15f4a8712c --- /dev/null +++ b/shims/spark311/src/main/resources/META-INF/services/com.nvidia.spark.rapids.SparkShimServiceProvider @@ -0,0 +1 @@ +com.nvidia.spark.rapids.shims.spark311.SparkShimServiceProvider diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuBroadcastHashJoinExec.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuBroadcastHashJoinExec.scala similarity index 98% rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuBroadcastHashJoinExec.scala rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuBroadcastHashJoinExec.scala index 6767f5f2849..1ac848db516 100644 --- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuBroadcastHashJoinExec.scala +++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuBroadcastHashJoinExec.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.nvidia.spark.rapids.shims.spark310 +package com.nvidia.spark.rapids.shims.spark311 import com.nvidia.spark.rapids._ import com.nvidia.spark.rapids.GpuMetricNames._ diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuBroadcastNestedLoopJoinExec.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuBroadcastNestedLoopJoinExec.scala similarity index 93% rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuBroadcastNestedLoopJoinExec.scala rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuBroadcastNestedLoopJoinExec.scala index e41f6a999e1..d027cd41b61 100644 --- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuBroadcastNestedLoopJoinExec.scala +++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuBroadcastNestedLoopJoinExec.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.nvidia.spark.rapids.shims.spark310 +package com.nvidia.spark.rapids.shims.spark311 import com.nvidia.spark.rapids.GpuBuildSide diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuOrcScan.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuOrcScan.scala similarity index 96% rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuOrcScan.scala rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuOrcScan.scala index 1c65fcb4fee..9360bedb30f 100644 --- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuOrcScan.scala +++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuOrcScan.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.nvidia.spark.rapids.shims.spark310 +package com.nvidia.spark.rapids.shims.spark311 import com.nvidia.spark.rapids.{GpuOrcScanBase, RapidsConf} import org.apache.hadoop.conf.Configuration diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuParquetScan.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuParquetScan.scala similarity index 96% rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuParquetScan.scala rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuParquetScan.scala index 2d03c067ef2..ae08dad6a05 100644 --- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuParquetScan.scala +++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuParquetScan.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.nvidia.spark.rapids.shims.spark310 +package com.nvidia.spark.rapids.shims.spark311 import com.nvidia.spark.rapids.{GpuParquetScanBase, RapidsConf} import org.apache.hadoop.conf.Configuration diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuShuffleExchangeExec.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuShuffleExchangeExec.scala similarity index 94% rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuShuffleExchangeExec.scala rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuShuffleExchangeExec.scala index deb2e17b4c1..d2b0ca6ffba 100644 --- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuShuffleExchangeExec.scala +++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuShuffleExchangeExec.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.nvidia.spark.rapids.shims.spark310 +package com.nvidia.spark.rapids.shims.spark311 import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.plans.logical.Statistics diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuShuffledHashJoinExec.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuShuffledHashJoinExec.scala similarity index 98% rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuShuffledHashJoinExec.scala rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuShuffledHashJoinExec.scala index 237f5094c35..63f4cf790d4 100644 --- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuShuffledHashJoinExec.scala +++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuShuffledHashJoinExec.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.nvidia.spark.rapids.shims.spark310 +package com.nvidia.spark.rapids.shims.spark311 import com.nvidia.spark.rapids._ import com.nvidia.spark.rapids.GpuMetricNames._ diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuSortMergeJoinExec.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuSortMergeJoinExec.scala similarity index 97% rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuSortMergeJoinExec.scala rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuSortMergeJoinExec.scala index 523426235b7..8228b432f7c 100644 --- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/GpuSortMergeJoinExec.scala +++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/GpuSortMergeJoinExec.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.nvidia.spark.rapids.shims.spark310 +package com.nvidia.spark.rapids.shims.spark311 import com.nvidia.spark.rapids._ diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/ParquetCachedBatchSerializer.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/ParquetCachedBatchSerializer.scala similarity index 99% rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/ParquetCachedBatchSerializer.scala rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/ParquetCachedBatchSerializer.scala index fc8e2aa9d98..380dc302bb3 100644 --- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/ParquetCachedBatchSerializer.scala +++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/ParquetCachedBatchSerializer.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.nvidia.spark.rapids.shims.spark310 +package com.nvidia.spark.rapids.shims.spark311 import java.io.{InputStream, IOException} import java.lang.reflect.Method diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/Spark310Shims.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/Spark311Shims.scala similarity index 97% rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/Spark310Shims.scala rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/Spark311Shims.scala index 9203abd0da9..05796e8e41b 100644 --- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/Spark310Shims.scala +++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/Spark311Shims.scala @@ -14,11 +14,11 @@ * limitations under the License. */ -package com.nvidia.spark.rapids.shims.spark310 +package com.nvidia.spark.rapids.shims.spark311 import com.nvidia.spark.rapids._ import com.nvidia.spark.rapids.shims.spark301.Spark301Shims -import com.nvidia.spark.rapids.spark310.RapidsShuffleManager +import com.nvidia.spark.rapids.spark311.RapidsShuffleManager import org.apache.spark.SparkEnv import org.apache.spark.sql.SparkSession @@ -38,11 +38,11 @@ import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BroadcastNes import org.apache.spark.sql.internal.StaticSQLConf import org.apache.spark.sql.rapids.{GpuFileSourceScanExec, GpuStringReplace, ShuffleManagerShimBase} import org.apache.spark.sql.rapids.execution.{GpuBroadcastNestedLoopJoinExecBase, GpuShuffleExchangeExecBase} -import org.apache.spark.sql.rapids.shims.spark310._ +import org.apache.spark.sql.rapids.shims.spark311._ import org.apache.spark.sql.types._ import org.apache.spark.storage.{BlockId, BlockManagerId} -class Spark310Shims extends Spark301Shims { +class Spark311Shims extends Spark301Shims { override def getSparkShimVersion: ShimVersion = SparkShimServiceProvider.VERSION @@ -93,7 +93,7 @@ class Spark310Shims extends Spark301Shims { } } - def exprs310: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = Seq( + def exprs311: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = Seq( GpuOverrides.expr[Cast]( "Convert a column of one type of data into another type", new CastChecks(), @@ -194,7 +194,7 @@ class Spark310Shims extends Spark301Shims { ).map(r => (r.getClassFor.asSubclass(classOf[Expression]), r)).toMap override def getExprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = { - super.exprs301 ++ exprs310 + super.exprs301 ++ exprs311 } override def getExecs: Map[Class[_ <: SparkPlan], ExecRule[_ <: SparkPlan]] = { @@ -338,7 +338,7 @@ class Spark310Shims extends Spark301Shims { val serName = plan.conf.getConf(StaticSQLConf.SPARK_CACHE_SERIALIZER) val serClass = Class.forName(serName) if (serClass == classOf[ParquetCachedBatchSerializer]) { - org.apache.spark.sql.rapids.shims.spark310.GpuColumnarToRowTransitionExec(plan) + org.apache.spark.sql.rapids.shims.spark311.GpuColumnarToRowTransitionExec(plan) } else { GpuColumnarToRowExec(plan) } diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/SparkShimServiceProvider.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala similarity index 71% rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/SparkShimServiceProvider.scala rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala index a14695c1f7e..8ce22923520 100644 --- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/shims/spark310/SparkShimServiceProvider.scala +++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/SparkShimServiceProvider.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,15 @@ * limitations under the License. */ -package com.nvidia.spark.rapids.shims.spark310 +package com.nvidia.spark.rapids.shims.spark311 import com.nvidia.spark.rapids.{SparkShims, SparkShimVersion} object SparkShimServiceProvider { - val VERSION = SparkShimVersion(3, 1, 0) - val VERSIONNAMES = Seq(s"$VERSION", s"$VERSION-SNAPSHOT") + // keep 3.1.0 snapshot version for now until 3.1.1 is released + val VERSION310 = SparkShimVersion(3, 1, 0) + val VERSION = SparkShimVersion(3, 1, 1) + val VERSIONNAMES = Seq(s"$VERSION310-SNAPSHOT", s"$VERSION", s"$VERSION-SNAPSHOT") } class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceProvider { @@ -30,6 +32,6 @@ class SparkShimServiceProvider extends com.nvidia.spark.rapids.SparkShimServiceP } def buildShim: SparkShims = { - new Spark310Shims() + new Spark311Shims() } } diff --git a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/spark310/RapidsShuffleManager.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/spark310/RapidsShuffleManager.scala similarity index 84% rename from shims/spark310/src/main/scala/com/nvidia/spark/rapids/spark310/RapidsShuffleManager.scala rename to shims/spark311/src/main/scala/com/nvidia/spark/rapids/spark310/RapidsShuffleManager.scala index a6c783c2e1f..018f5421b7e 100644 --- a/shims/spark310/src/main/scala/com/nvidia/spark/rapids/spark310/RapidsShuffleManager.scala +++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/spark310/RapidsShuffleManager.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,10 @@ * limitations under the License. */ -package com.nvidia.spark.rapids.spark310 +package com.nvidia.spark.rapids.spark311 import org.apache.spark.SparkConf -import org.apache.spark.sql.rapids.shims.spark310.RapidsShuffleInternalManager +import org.apache.spark.sql.rapids.shims.spark311.RapidsShuffleInternalManager /** A shuffle manager optimized for the RAPIDS Plugin for Apache Spark. */ sealed class RapidsShuffleManager( diff --git a/shims/spark310/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/ParquetMaterializer.scala b/shims/spark311/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/ParquetMaterializer.scala similarity index 100% rename from shims/spark310/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/ParquetMaterializer.scala rename to shims/spark311/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/ParquetMaterializer.scala diff --git a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/GpuColumnarToRowTransitionExec.scala b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/GpuColumnarToRowTransitionExec.scala similarity index 90% rename from shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/GpuColumnarToRowTransitionExec.scala rename to shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/GpuColumnarToRowTransitionExec.scala index d282d30a0a1..e54d8c30793 100644 --- a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/GpuColumnarToRowTransitionExec.scala +++ b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/GpuColumnarToRowTransitionExec.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.apache.spark.sql.rapids.shims.spark310 +package org.apache.spark.sql.rapids.shims.spark311 import com.nvidia.spark.rapids.GpuColumnarToRowExecParent diff --git a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/GpuInMemoryTableScanExec.scala b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/GpuInMemoryTableScanExec.scala similarity index 96% rename from shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/GpuInMemoryTableScanExec.scala rename to shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/GpuInMemoryTableScanExec.scala index ea8ae8f68bc..ade9b30118a 100644 --- a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/GpuInMemoryTableScanExec.scala +++ b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/GpuInMemoryTableScanExec.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2020, NVIDIA CORPORATION. + * Copyright (c) 2019-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,10 @@ * limitations under the License. */ -package org.apache.spark.sql.rapids.shims.spark310 +package org.apache.spark.sql.rapids.shims.spark311 import com.nvidia.spark.rapids.GpuExec -import com.nvidia.spark.rapids.shims.spark310.ParquetCachedBatchSerializer +import com.nvidia.spark.rapids.shims.spark311.ParquetCachedBatchSerializer import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow diff --git a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/GpuSchemaUtils.scala b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/GpuSchemaUtils.scala similarity index 90% rename from shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/GpuSchemaUtils.scala rename to shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/GpuSchemaUtils.scala index c71c77d6239..42e04cc5384 100644 --- a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/GpuSchemaUtils.scala +++ b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/GpuSchemaUtils.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.apache.spark.sql.rapids.shims.spark310 +package org.apache.spark.sql.rapids.shims.spark311 import org.apache.spark.sql.catalyst.analysis.Resolver import org.apache.spark.sql.types.StructType diff --git a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/RapidsShuffleInternalManager.scala b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/RapidsShuffleInternalManager.scala similarity index 93% rename from shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/RapidsShuffleInternalManager.scala rename to shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/RapidsShuffleInternalManager.scala index 6ab55c757ec..72870b7eea6 100644 --- a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/RapidsShuffleInternalManager.scala +++ b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/RapidsShuffleInternalManager.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2020, NVIDIA CORPORATION. + * Copyright (c) 2019-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.apache.spark.sql.rapids.shims.spark310 +package org.apache.spark.sql.rapids.shims.spark311 import org.apache.spark.{SparkConf, TaskContext} import org.apache.spark.shuffle._ diff --git a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/ShuffleManagerShim.scala b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/ShuffleManagerShim.scala similarity index 92% rename from shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/ShuffleManagerShim.scala rename to shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/ShuffleManagerShim.scala index fc5603d9e6c..edc7a85410f 100644 --- a/shims/spark310/src/main/scala/org/apache/spark/sql/rapids/shims/spark310/ShuffleManagerShim.scala +++ b/shims/spark311/src/main/scala/org/apache/spark/sql/rapids/shims/spark311/ShuffleManagerShim.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.rapids.shims.spark310 +package org.apache.spark.sql.rapids.shims.spark311 import org.apache.spark.TaskContext import org.apache.spark.shuffle.{ShuffleHandle, ShuffleManager, ShuffleReader, ShuffleReadMetricsReporter} diff --git a/tests-spark310+/README.md b/tests-spark310+/README.md index 02d7b66e942..d3601d5d057 100644 --- a/tests-spark310+/README.md +++ b/tests-spark310+/README.md @@ -14,9 +14,9 @@ Example: As of writing of this document this module contains tests that are strongly tied to classes in Spark-3.1.0-SNAPSHOT. -These tests can be executed by choosing profile `spark310tests` like so, +These tests can be executed by choosing profile `spark311tests` like so, -`mvn -Pspark310tests -wildcardSuites=` +`mvn -Pspark311tests -wildcardSuites=` For a more comprehensive overview of tests in Rapids Accelerator please refer to the following diff --git a/tests-spark310+/src/test/scala/com/nvidia/spark/rapids/Spark310ParquetWriterSuite.scala b/tests-spark310+/src/test/scala/com/nvidia/spark/rapids/Spark310ParquetWriterSuite.scala index 30ab5c645a9..2796b40cd22 100644 --- a/tests-spark310+/src/test/scala/com/nvidia/spark/rapids/Spark310ParquetWriterSuite.scala +++ b/tests-spark310+/src/test/scala/com/nvidia/spark/rapids/Spark310ParquetWriterSuite.scala @@ -19,7 +19,7 @@ package com.nvidia.spark.rapids import scala.collection.mutable import ai.rapids.cudf.{ColumnVector, DType, Table, TableWriter} -import com.nvidia.spark.rapids.shims.spark310.{ParquetCachedBatchSerializer, ParquetOutputFileFormat} +import com.nvidia.spark.rapids.shims.spark311.{ParquetCachedBatchSerializer, ParquetOutputFileFormat} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.mapreduce.{RecordWriter, TaskAttemptContext} import org.mockito.ArgumentMatchers._ diff --git a/tests/README.md b/tests/README.md index 17e5addeb91..9d50c0ba603 100644 --- a/tests/README.md +++ b/tests/README.md @@ -56,7 +56,7 @@ default version runs against Spark 3.0.0, to run against other versions use one profiles: - `-Pspark301tests` (spark 3.0.1) - `-Pspark302tests` (spark 3.0.2) - - `-Pspark310tests` (spark 3.1.0) + - `-Pspark311tests` (spark 3.1.1) Please refer to the [tests project POM](pom.xml) to see the list of test profiles supported. Apache Spark specific configurations can be passed in by setting the `SPARK_CONF` environment @@ -64,7 +64,7 @@ variable. Examples: - To run tests against Apache Spark 3.1.0, - `mvn -P spark310tests test` + `mvn -P spark311tests test` - To pass Apache Spark configs `--conf spark.dynamicAllocation.enabled=false --conf spark.task.cpus=1` do something like. `SPARK_CONF="spark.dynamicAllocation.enabled=false,spark.task.cpus=1" mvn ...` - To run test ParquetWriterSuite in package com.nvidia.spark.rapids, issue `mvn test -DwildcardSuites="com.nvidia.spark.rapids.ParquetWriterSuite"` diff --git a/tests/pom.xml b/tests/pom.xml index dfc87fc5da8..7c22c4045b3 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -1,6 +1,6 @@