From fb5b4177b34dcddd01af32e5c90299bbbda21039 Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Mon, 11 Jan 2021 14:14:59 -0800 Subject: [PATCH 1/2] moved ParquetRecordMaterializer to shim package Signed-off-by: Raza Jafri --- .../rapids/shims/spark311/ParquetCachedBatchSerializer.scala | 2 +- .../rapids/{ => shims/spark311}/ParquetMaterializer.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename shims/spark311/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/{ => shims/spark311}/ParquetMaterializer.scala (99%) diff --git a/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/ParquetCachedBatchSerializer.scala b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/ParquetCachedBatchSerializer.scala index 380dc302bb3..eebec2ebdf9 100644 --- a/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/ParquetCachedBatchSerializer.scala +++ b/shims/spark311/src/main/scala/com/nvidia/spark/rapids/shims/spark311/ParquetCachedBatchSerializer.scala @@ -49,7 +49,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjectio import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, GenericArrayData, MapData} import org.apache.spark.sql.columnar.{CachedBatch, CachedBatchSerializer} import org.apache.spark.sql.execution.datasources.parquet.{ParquetReadSupport, ParquetToSparkSchemaConverter, ParquetWriteSupport, SparkToParquetSchemaConverter, VectorizedColumnReader} -import org.apache.spark.sql.execution.datasources.parquet.rapids.ParquetRecordMaterializer +import org.apache.spark.sql.execution.datasources.parquet.rapids.shims.spark311.ParquetRecordMaterializer import org.apache.spark.sql.execution.vectorized.{OffHeapColumnVector, WritableColumnVector} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy diff --git a/shims/spark311/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/ParquetMaterializer.scala b/shims/spark311/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/spark311/ParquetMaterializer.scala similarity index 99% rename from shims/spark311/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/ParquetMaterializer.scala rename to shims/spark311/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/spark311/ParquetMaterializer.scala index 516e5aa68db..5e826580dec 100644 --- a/shims/spark311/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/ParquetMaterializer.scala +++ b/shims/spark311/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/spark311/ParquetMaterializer.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.parquet.rapids +package org.apache.spark.sql.execution.datasources.parquet.rapids.shims.spark311 import java.time.ZoneId From e268940b14e51a6da35b678d0e9389f7ea3992ef Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Mon, 11 Jan 2021 16:24:04 -0800 Subject: [PATCH 2/2] addressed review comment Signed-off-by: Raza Jafri --- .../parquet/rapids/shims/spark311/ParquetMaterializer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shims/spark311/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/spark311/ParquetMaterializer.scala b/shims/spark311/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/spark311/ParquetMaterializer.scala index 5e826580dec..ebbf252b6fb 100644 --- a/shims/spark311/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/spark311/ParquetMaterializer.scala +++ b/shims/spark311/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/rapids/shims/spark311/ParquetMaterializer.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License.