From f5f07823ad7c1ab9ae0553bff31f1b768d3ed886 Mon Sep 17 00:00:00 2001 From: nameexhaustion Date: Wed, 18 Oct 2023 09:28:16 +1100 Subject: [PATCH] c --- .../src/io/parquet/read/deserialize/nested_utils.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/polars-arrow/src/io/parquet/read/deserialize/nested_utils.rs b/crates/polars-arrow/src/io/parquet/read/deserialize/nested_utils.rs index ad53b72a7e6f..482d5117a7da 100644 --- a/crates/polars-arrow/src/io/parquet/read/deserialize/nested_utils.rs +++ b/crates/polars-arrow/src/io/parquet/read/deserialize/nested_utils.rs @@ -502,9 +502,6 @@ where if *remaining == 0 && items.is_empty() { return MaybeNext::None; } - if !items.is_empty() && items.front().unwrap().0.len() > chunk_size.unwrap_or(usize::MAX) { - return MaybeNext::Some(Ok(items.pop_front().unwrap())); - } match iter.next() { Err(e) => MaybeNext::Some(Err(e.into())), @@ -539,7 +536,8 @@ where Err(e) => return MaybeNext::Some(Err(e)), }; - // if possible, return the value immediately. + // this comparison is strictly greater to ensure the contents of the + // row are fully read. if !items.is_empty() && items.front().unwrap().0.len() > chunk_size.unwrap_or(usize::MAX) {