From 7782695a37147df60b2f753c7d549e0c3a26a28f Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Tue, 8 Mar 2022 23:01:16 -0800 Subject: [PATCH 1/4] fix db test failures Signed-off-by: Raza Jafri --- .../main/301db/scala/com/nvidia/spark/rapids/SparkShims.scala | 3 ++- .../312db/scala/com/nvidia/spark/rapids/shims/SparkShims.scala | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sql-plugin/src/main/301db/scala/com/nvidia/spark/rapids/SparkShims.scala b/sql-plugin/src/main/301db/scala/com/nvidia/spark/rapids/SparkShims.scala index 36b07462d3e..ba19ba9c850 100644 --- a/sql-plugin/src/main/301db/scala/com/nvidia/spark/rapids/SparkShims.scala +++ b/sql-plugin/src/main/301db/scala/com/nvidia/spark/rapids/SparkShims.scala @@ -17,8 +17,9 @@ package com.nvidia.spark.rapids.shims import com.nvidia.spark.rapids._ +import com.nvidia.spark.rapids.shims.spark301db.SparkShimServiceProvider object SparkShimImpl extends Spark30XdbShims with Spark30Xuntil33XShims { - override def getSparkShimVersion: ShimVersion = ShimLoader.getShimVersion + override def getSparkShimVersion: ShimVersion = SparkShimServiceProvider.VERSION } diff --git a/sql-plugin/src/main/312db/scala/com/nvidia/spark/rapids/shims/SparkShims.scala b/sql-plugin/src/main/312db/scala/com/nvidia/spark/rapids/shims/SparkShims.scala index fa544e24dff..a8921c0b629 100644 --- a/sql-plugin/src/main/312db/scala/com/nvidia/spark/rapids/shims/SparkShims.scala +++ b/sql-plugin/src/main/312db/scala/com/nvidia/spark/rapids/shims/SparkShims.scala @@ -17,6 +17,7 @@ package com.nvidia.spark.rapids.shims import com.nvidia.spark.rapids._ +import com.nvidia.spark.rapids.shims.spark312db.SparkShimServiceProvider import org.apache.parquet.schema.MessageType import org.apache.spark.sql.execution.datasources.DataSourceUtils @@ -24,7 +25,7 @@ import org.apache.spark.sql.execution.datasources.parquet.ParquetFilters object SparkShimImpl extends Spark31XdbShims with Spark30Xuntil33XShims { - override def getSparkShimVersion: ShimVersion = ShimLoader.getShimVersion + override def getSparkShimVersion: ShimVersion = SparkShimServiceProvider.VERSION override def getParquetFilters( schema: MessageType, From 500c2f5d99387083eab41e86ac56b1279f804f5c Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Wed, 9 Mar 2022 10:46:04 -0800 Subject: [PATCH 2/4] Revert "fix db test failures" This reverts commit 7782695a37147df60b2f753c7d549e0c3a26a28f. Signed-off-by: Raza Jafri --- .../main/301db/scala/com/nvidia/spark/rapids/SparkShims.scala | 3 +-- .../312db/scala/com/nvidia/spark/rapids/shims/SparkShims.scala | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/sql-plugin/src/main/301db/scala/com/nvidia/spark/rapids/SparkShims.scala b/sql-plugin/src/main/301db/scala/com/nvidia/spark/rapids/SparkShims.scala index ba19ba9c850..36b07462d3e 100644 --- a/sql-plugin/src/main/301db/scala/com/nvidia/spark/rapids/SparkShims.scala +++ b/sql-plugin/src/main/301db/scala/com/nvidia/spark/rapids/SparkShims.scala @@ -17,9 +17,8 @@ package com.nvidia.spark.rapids.shims import com.nvidia.spark.rapids._ -import com.nvidia.spark.rapids.shims.spark301db.SparkShimServiceProvider object SparkShimImpl extends Spark30XdbShims with Spark30Xuntil33XShims { - override def getSparkShimVersion: ShimVersion = SparkShimServiceProvider.VERSION + override def getSparkShimVersion: ShimVersion = ShimLoader.getShimVersion } diff --git a/sql-plugin/src/main/312db/scala/com/nvidia/spark/rapids/shims/SparkShims.scala b/sql-plugin/src/main/312db/scala/com/nvidia/spark/rapids/shims/SparkShims.scala index a8921c0b629..fa544e24dff 100644 --- a/sql-plugin/src/main/312db/scala/com/nvidia/spark/rapids/shims/SparkShims.scala +++ b/sql-plugin/src/main/312db/scala/com/nvidia/spark/rapids/shims/SparkShims.scala @@ -17,7 +17,6 @@ package com.nvidia.spark.rapids.shims import com.nvidia.spark.rapids._ -import com.nvidia.spark.rapids.shims.spark312db.SparkShimServiceProvider import org.apache.parquet.schema.MessageType import org.apache.spark.sql.execution.datasources.DataSourceUtils @@ -25,7 +24,7 @@ import org.apache.spark.sql.execution.datasources.parquet.ParquetFilters object SparkShimImpl extends Spark31XdbShims with Spark30Xuntil33XShims { - override def getSparkShimVersion: ShimVersion = SparkShimServiceProvider.VERSION + override def getSparkShimVersion: ShimVersion = ShimLoader.getShimVersion override def getParquetFilters( schema: MessageType, From 4b6794aa51d035e77668fa6afb9084f95d90720b Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Wed, 9 Mar 2022 11:05:21 -0800 Subject: [PATCH 3/4] db fix error Signed-off-by: Raza Jafri --- dist/unshimmed-common-from-spark301.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dist/unshimmed-common-from-spark301.txt b/dist/unshimmed-common-from-spark301.txt index bf0fdf06bd1..0db8779f5b7 100644 --- a/dist/unshimmed-common-from-spark301.txt +++ b/dist/unshimmed-common-from-spark301.txt @@ -19,6 +19,7 @@ com/nvidia/spark/rapids/ShimLoader* com/nvidia/spark/rapids/ShimVersion* com/nvidia/spark/rapids/SparkShimServiceProvider* com/nvidia/spark/rapids/SparkShimVersion* +com/nvidia/spark/rapids/DatabricksShimVersion* com/nvidia/spark/rapids/SparkShims* com/nvidia/spark/udf/Plugin* libjucx.so From 23abaea3c9f46c6d5e3a69cb931e4567784a1959 Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Wed, 9 Mar 2022 14:12:40 -0800 Subject: [PATCH 4/4] organized the file lexicographically Signed-off-by: Raza Jafri --- dist/unshimmed-common-from-spark301.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/unshimmed-common-from-spark301.txt b/dist/unshimmed-common-from-spark301.txt index 0db8779f5b7..c58ec7f83c9 100644 --- a/dist/unshimmed-common-from-spark301.txt +++ b/dist/unshimmed-common-from-spark301.txt @@ -6,6 +6,7 @@ com/nvidia/spark/ExclusiveModeGpuDiscoveryPlugin* com/nvidia/spark/RapidsUDF* com/nvidia/spark/SQLPlugin* com/nvidia/spark/rapids/ColumnarRdd* +com/nvidia/spark/rapids/DatabricksShimVersion* com/nvidia/spark/rapids/ExecutionPlanCaptureCallback* com/nvidia/spark/rapids/ExplainPlan* com/nvidia/spark/rapids/GpuKryoRegistrator* @@ -19,7 +20,6 @@ com/nvidia/spark/rapids/ShimLoader* com/nvidia/spark/rapids/ShimVersion* com/nvidia/spark/rapids/SparkShimServiceProvider* com/nvidia/spark/rapids/SparkShimVersion* -com/nvidia/spark/rapids/DatabricksShimVersion* com/nvidia/spark/rapids/SparkShims* com/nvidia/spark/udf/Plugin* libjucx.so