diff --git a/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/Delta24xProvider.scala b/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/Delta24xProvider.scala index d3f952b856c2..eb4331317f92 100644 --- a/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/Delta24xProvider.scala +++ b/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/Delta24xProvider.scala @@ -74,7 +74,9 @@ object Delta24xProvider extends DeltaIOProvider { override def getReadFileFormat(format: FileFormat): FileFormat = { val cpuFormat = format.asInstanceOf[DeltaParquetFileFormat] - GpuDelta24xParquetFileFormat(cpuFormat.metadata, cpuFormat.isSplittable) + GpuDelta24xParquetFileFormat(cpuFormat.metadata, + cpuFormat.isSplittable, + cpuFormat.disablePushDowns) } override def convertToGpu( diff --git a/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/GpuDelta24xParquetFileFormat.scala b/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/GpuDelta24xParquetFileFormat.scala index fe21cbc45e09..19ca384797e8 100644 --- a/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/GpuDelta24xParquetFileFormat.scala +++ b/delta-lake/delta-24x/src/main/scala/com/nvidia/spark/rapids/delta/delta24x/GpuDelta24xParquetFileFormat.scala @@ -56,7 +56,8 @@ case class GpuDelta24xParquetFileFormat( case ff: GpuDelta24xParquetFileFormat => ff.columnMappingMode == columnMappingMode && ff.referenceSchema == referenceSchema && - ff.isSplittable == isSplittable + ff.isSplittable == isSplittable && + ff.disableFilterPushdown == disableFilterPushdown case _ => false } } diff --git a/delta-lake/delta-24x/src/main/scala/org/apache/spark/sql/delta/rapids/delta24x/GpuMergeIntoCommand.scala b/delta-lake/delta-24x/src/main/scala/org/apache/spark/sql/delta/rapids/delta24x/GpuMergeIntoCommand.scala index 74843cfac914..ae0adb6c5097 100644 --- a/delta-lake/delta-24x/src/main/scala/org/apache/spark/sql/delta/rapids/delta24x/GpuMergeIntoCommand.scala +++ b/delta-lake/delta-24x/src/main/scala/org/apache/spark/sql/delta/rapids/delta24x/GpuMergeIntoCommand.scala @@ -23,7 +23,6 @@ package org.apache.spark.sql.delta.rapids.delta24x import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.nvidia.spark.rapids.delta._ -import com.nvidia.spark.rapids.delta.delta24x.GpuDelta24xParquetFileFormat import com.nvidia.spark.rapids.{BaseExprMeta, GpuOverrides, RapidsConf} import org.apache.spark.SparkContext import org.apache.spark.internal.Logging @@ -1383,11 +1382,7 @@ object GpuMergeIntoCommand extends Logging { case l@LogicalRelation( hfsr@HadoopFsRelation(_, _, _, _, format: DeltaParquetFileFormat, _), _, _, _) => // Disable splitting and filter pushdown in order to generate the row-indexes - val newFormat = GpuDelta24xParquetFileFormat( - format.metadata, - isSplittable = false, - disableFilterPushdown = true - ) + val newFormat = format.copy(isSplittable = false, disablePushDowns = true) val newBaseRelation = hfsr.copy( fileFormat = newFormat)(hfsr.sparkSession)