From 57a3da400eb8d198baa228053368dfa24befd2a8 Mon Sep 17 00:00:00 2001 From: coastalwhite Date: Thu, 12 Sep 2024 15:04:33 +0200 Subject: [PATCH] revert: Automatically turn on Parquet prefiltered --- crates/polars-io/src/parquet/read/read_impl.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/polars-io/src/parquet/read/read_impl.rs b/crates/polars-io/src/parquet/read/read_impl.rs index 15be3f2a8e09..cefa05a40072 100644 --- a/crates/polars-io/src/parquet/read/read_impl.rs +++ b/crates/polars-io/src/parquet/read/read_impl.rs @@ -753,14 +753,16 @@ pub fn read_parquet( .map(Cow::Borrowed) .unwrap_or_else(|| Cow::Owned((0usize..reader_schema.len()).collect::>())); - if let ParallelStrategy::Auto = parallel { - if predicate.is_some_and(|predicate| { - predicate.live_variables().map_or(0, |v| v.len()) * n_row_groups + if let Some(predicate) = predicate { + if std::env::var("POLARS_PARQUET_AUTO_PREFILTERED").is_ok_and(|v| v == "1") + && predicate.live_variables().map_or(0, |v| v.len()) * n_row_groups >= POOL.current_num_threads() - }) { + { parallel = ParallelStrategy::Prefiltered; - } else if n_row_groups > materialized_projection.len() - || n_row_groups > POOL.current_num_threads() + } + } + if ParallelStrategy::Auto == parallel { + if n_row_groups > materialized_projection.len() || n_row_groups > POOL.current_num_threads() { parallel = ParallelStrategy::RowGroups; } else {