Add GPU metrics to GpuFileSourceScanExec (#547)

* Add GPU metrics to GpuFileSourceScanExec Signed-off-by: Jason Lowe <jlowe@nvidia.com> * Extract GpuFileSourceScanExec from shims Signed-off-by: Jason Lowe <jlowe@nvidia.com> * Pass metrics via GPU file format rather than custom options map Signed-off-by: Jason Lowe <jlowe@nvidia.com> * Update code checking for DataSourceScanExec Signed-off-by: Jason Lowe <jlowe@nvidia.com> * Fix scaladoc warning and unused imports Signed-off-by: Jason Lowe <jlowe@nvidia.com> * Fix copyright Signed-off-by: Jason Lowe <jlowe@nvidia.com>
NVIDIA · Aug 14, 2020 · 20afca1 · 20afca1
1 parent b86fd32
commit 20afca1
Show file tree

Hide file tree

Showing 16 changed files with 728 additions and 628 deletions.
diff --git a/shims/spark300/src/main/scala/com/nvidia/spark/rapids/shims/spark300/Spark300Shims.scala b/shims/spark300/src/main/scala/com/nvidia/spark/rapids/shims/spark300/Spark300Shims.scala
@@ -35,7 +35,7 @@ import org.apache.spark.sql.execution.datasources.HadoopFsRelation
 import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec}
 import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BroadcastNestedLoopJoinExec, HashJoin, SortMergeJoinExec}
 import org.apache.spark.sql.execution.joins.ShuffledHashJoinExec
-import org.apache.spark.sql.rapids.{GpuTimeSub, ShuffleManagerShimBase}
+import org.apache.spark.sql.rapids.{GpuFileSourceScanExec, GpuTimeSub, ShuffleManagerShimBase}
 import org.apache.spark.sql.rapids.execution.{GpuBroadcastExchangeExecBase, GpuBroadcastNestedLoopJoinExecBase, GpuShuffleExchangeExecBase}
 import org.apache.spark.sql.rapids.shims.spark300._
 import org.apache.spark.sql.types._
@@ -148,6 +148,7 @@ class Spark300Shims extends SparkShims {
               wrapped.requiredSchema,
               wrapped.partitionFilters,
               wrapped.optionalBucketSet,
+              None,
               wrapped.dataFilters,
               wrapped.tableIdentifier)
           }

diff --git a/...300/src/main/scala/org/apache/spark/sql/rapids/shims/spark300/GpuFileSourceScanExec.scala b/...300/src/main/scala/org/apache/spark/sql/rapids/shims/spark300/GpuFileSourceScanExec.scala
diff --git a/.../spark300db/src/main/scala/com/nvidia/spark/rapids/shims/spark300db/Spark300dbShims.scala b/.../spark300db/src/main/scala/com/nvidia/spark/rapids/shims/spark300db/Spark300dbShims.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.datasources.HadoopFsRelation
 import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BroadcastNestedLoopJoinExec, HashJoin, SortMergeJoinExec}
 import org.apache.spark.sql.execution.joins.ShuffledHashJoinExec
-import org.apache.spark.sql.rapids.GpuTimeSub
+import org.apache.spark.sql.rapids.{GpuFileSourceScanExec, GpuTimeSub}
 import org.apache.spark.sql.rapids.execution.GpuBroadcastNestedLoopJoinExecBase
 import org.apache.spark.sql.rapids.shims.spark300db._
 import org.apache.spark.sql.types._
@@ -94,6 +94,8 @@ class Spark300dbShims extends Spark300Shims {
               wrapped.requiredSchema,
               wrapped.partitionFilters,
               wrapped.optionalBucketSet,
+              // TODO: Does Databricks have coalesced bucketing implemented?
+              None,
               wrapped.dataFilters,
               wrapped.tableIdentifier)
           }

diff --git a/...b/src/main/scala/org/apache/spark/sql/rapids/shims/spark300db/GpuFileSourceScanExec.scala b/...b/src/main/scala/org/apache/spark/sql/rapids/shims/spark300db/GpuFileSourceScanExec.scala