From 8f9aa50f730a48c1697cbb4f4c9ba707963a37a1 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Sun, 3 Apr 2022 18:35:10 -0700 Subject: [PATCH] [SPARK-34863][SQL][FOLLOWUP] Disable `spark.sql.parquet.enableNestedColumnVectorizedReader` by default ### What changes were proposed in this pull request? This PR disables `spark.sql.parquet.enableNestedColumnVectorizedReader` by default. ### Why are the changes needed? In #34659 the config was turned mainly for testing reason. As the feature is new, we should turn it off by default. ### Does this PR introduce _any_ user-facing change? The config `spark.sql.parquet.enableNestedColumnVectorizedReader` is turned off by default now. ### How was this patch tested? Existing tests. Closes #36055 from sunchao/disable. Authored-by: Chao Sun Signed-off-by: Liang-Chi Hsieh (cherry picked from commit 1b08673a6d92e3e0fceb4a686a0ba77a87f1ebbc) Signed-off-by: Liang-Chi Hsieh --- .../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala | 2 +- .../src/test/resources/sql-tests/results/explain-aqe.sql.out | 3 +-- sql/core/src/test/resources/sql-tests/results/explain.sql.out | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 5bf59923787a7..56849b8718e19 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -1015,7 +1015,7 @@ object SQLConf { s"Requires ${PARQUET_VECTORIZED_READER_ENABLED.key} to be enabled.") .version("3.3.0") .booleanConf - .createWithDefault(true) + .createWithDefault(false) val PARQUET_RECORD_FILTER_ENABLED = buildConf("spark.sql.parquet.recordLevelFilter.enabled") .doc("If true, enables Parquet's native record-level filtering using the pushed down " + diff --git a/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out b/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out index f98fb1eb2a57a..f5e5b46d29ce6 100644 --- a/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out @@ -1125,8 +1125,7 @@ struct -- !query output == Physical Plan == *Filter v#x IN ([a],null) -+- *ColumnarToRow - +- FileScan parquet default.t[v#x] Batched: true, DataFilters: [v#x IN ([a],null)], Format: Parquet, Location [not included in comparison]/{warehouse_dir}/t], PartitionFilters: [], PushedFilters: [In(v, [[a],null])], ReadSchema: struct> ++- FileScan parquet default.t[v#x] Batched: false, DataFilters: [v#x IN ([a],null)], Format: Parquet, Location [not included in comparison]/{warehouse_dir}/t], PartitionFilters: [], PushedFilters: [In(v, [[a],null])], ReadSchema: struct> -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/explain.sql.out b/sql/core/src/test/resources/sql-tests/results/explain.sql.out index a563eda1e7b04..4e552d51a3953 100644 --- a/sql/core/src/test/resources/sql-tests/results/explain.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/explain.sql.out @@ -1067,8 +1067,7 @@ struct -- !query output == Physical Plan == *Filter v#x IN ([a],null) -+- *ColumnarToRow - +- FileScan parquet default.t[v#x] Batched: true, DataFilters: [v#x IN ([a],null)], Format: Parquet, Location [not included in comparison]/{warehouse_dir}/t], PartitionFilters: [], PushedFilters: [In(v, [[a],null])], ReadSchema: struct> ++- FileScan parquet default.t[v#x] Batched: false, DataFilters: [v#x IN ([a],null)], Format: Parquet, Location [not included in comparison]/{warehouse_dir}/t], PartitionFilters: [], PushedFilters: [In(v, [[a],null])], ReadSchema: struct> -- !query