Fix

NVIDIA · Apr 30, 2024 · 7e1c587 · 7e1c587
1 parent c5872ca
commit 7e1c587
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 8 deletions.
diff --git a/...ke/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/Delta24xProvider.scala b/...ke/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/Delta24xProvider.scala
@@ -74,7 +74,9 @@ object Delta24xProvider extends DeltaIOProvider {
 
   override def getReadFileFormat(format: FileFormat): FileFormat = {
     val cpuFormat = format.asInstanceOf[DeltaParquetFileFormat]
-    GpuDelta24xParquetFileFormat(cpuFormat.metadata, cpuFormat.isSplittable)
+    GpuDelta24xParquetFileFormat(cpuFormat.metadata,
+      cpuFormat.isSplittable,
+      cpuFormat.disablePushDowns)
   }
 
   override def convertToGpu(

diff --git a/.../src/main/scala/com/nvidia/spark/rapids/delta/delta24x/GpuDelta24xParquetFileFormat.scala b/.../src/main/scala/com/nvidia/spark/rapids/delta/delta24x/GpuDelta24xParquetFileFormat.scala
@@ -56,7 +56,8 @@ case class GpuDelta24xParquetFileFormat(
       case ff: GpuDelta24xParquetFileFormat =>
         ff.columnMappingMode == columnMappingMode &&
           ff.referenceSchema == referenceSchema &&
-          ff.isSplittable == isSplittable
+          ff.isSplittable == isSplittable &&
+          ff.disableFilterPushdown == disableFilterPushdown
       case _ => false
     }
   }

diff --git a/...a-24x/src/main/scala/org/apache/spark/sql/delta/rapids/delta24x/GpuMergeIntoCommand.scala b/...a-24x/src/main/scala/org/apache/spark/sql/delta/rapids/delta24x/GpuMergeIntoCommand.scala
@@ -23,7 +23,6 @@ package org.apache.spark.sql.delta.rapids.delta24x
 
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize
 import com.nvidia.spark.rapids.delta._
-import com.nvidia.spark.rapids.delta.delta24x.GpuDelta24xParquetFileFormat
 import com.nvidia.spark.rapids.{BaseExprMeta, GpuOverrides, RapidsConf}
 import org.apache.spark.SparkContext
 import org.apache.spark.internal.Logging
@@ -1383,11 +1382,7 @@ object GpuMergeIntoCommand extends Logging {
       case l@LogicalRelation(
       hfsr@HadoopFsRelation(_, _, _, _, format: DeltaParquetFileFormat, _), _, _, _) =>
         // Disable splitting and filter pushdown in order to generate the row-indexes
-        val newFormat = GpuDelta24xParquetFileFormat(
-          format.metadata,
-          isSplittable = false,
-          disableFilterPushdown = true
-        )
+        val newFormat = format.copy(isSplittable = false, disablePushDowns = true)
         val newBaseRelation = hfsr.copy(
           fileFormat = newFormat)(hfsr.sparkSession)