chore: Remove hive_parts from DSL source (#18694)

pola-rs · Sep 11, 2024 · fcf03de · fcf03de
1 parent d8acacf
commit fcf03de
Show file tree

Hide file tree

Showing 5 changed files with 3 additions and 14 deletions.
diff --git a/crates/polars-lazy/src/scan/ndjson.rs b/crates/polars-lazy/src/scan/ndjson.rs
@@ -157,7 +157,6 @@ impl LazyFileListReader for LazyJsonLineReader {
         Ok(LazyFrame::from(DslPlan::Scan {
             sources: Arc::new(Mutex::new(self.sources.to_dsl(false))),
             file_info: Arc::new(RwLock::new(None)),
-            hive_parts: None,
             predicate: None,
             file_options,
             scan_type,

diff --git a/crates/polars-plan/src/plans/builder_dsl.rs b/crates/polars-plan/src/plans/builder_dsl.rs
@@ -63,7 +63,6 @@ impl DslBuilder {
                 is_expanded: true,
             })),
             file_info: Arc::new(RwLock::new(Some(file_info))),
-            hive_parts: None,
             predicate: None,
             file_options,
             scan_type: FileScan::Anonymous {
@@ -107,7 +106,6 @@ impl DslBuilder {
         Ok(DslPlan::Scan {
             sources: Arc::new(Mutex::new(sources)),
             file_info: Arc::new(RwLock::new(None)),
-            hive_parts: None,
             predicate: None,
             file_options: options,
             scan_type: FileScan::Parquet {
@@ -139,7 +137,6 @@ impl DslBuilder {
         Ok(DslPlan::Scan {
             sources: Arc::new(Mutex::new(sources)),
             file_info: Arc::new(RwLock::new(None)),
-            hive_parts: None,
             file_options: FileScanOptions {
                 with_columns: None,
                 cache,
@@ -192,7 +189,6 @@ impl DslBuilder {
         Ok(DslPlan::Scan {
             sources: Arc::new(Mutex::new(sources)),
             file_info: Arc::new(RwLock::new(None)),
-            hive_parts: None,
             file_options: options,
             predicate: None,
             scan_type: FileScan::Csv {

diff --git a/crates/polars-plan/src/plans/conversion/dsl_to_ir.rs b/crates/polars-plan/src/plans/conversion/dsl_to_ir.rs
@@ -107,7 +107,6 @@ pub fn to_alp_impl(lp: DslPlan, ctxt: &mut DslConversionContext) -> PolarsResult
         DslPlan::Scan {
             sources,
             file_info,
-            hive_parts,
             predicate,
             mut file_options,
             mut scan_type,
@@ -188,9 +187,7 @@ pub fn to_alp_impl(lp: DslPlan, ctxt: &mut DslConversionContext) -> PolarsResult
                 }
             };
 
-            let hive_parts = if hive_parts.is_some() {
-                hive_parts
-            } else if file_options.hive_options.enabled.unwrap_or(false)
+            let hive_parts = if file_options.hive_options.enabled.unwrap_or(false)
                 && resolved_file_info.reader_schema.is_some()
             {
                 let paths = sources

diff --git a/crates/polars-plan/src/plans/conversion/mod.rs b/crates/polars-plan/src/plans/conversion/mod.rs
@@ -52,7 +52,7 @@ impl IR {
             IR::Scan {
                 sources,
                 file_info,
-                hive_parts,
+                hive_parts: _,
                 predicate,
                 scan_type,
                 output_schema: _,
@@ -63,7 +63,6 @@ impl IR {
                     is_expanded: true,
                 })),
                 file_info: Arc::new(RwLock::new(Some(file_info))),
-                hive_parts,
                 predicate: predicate.map(|e| e.to_expr(expr_arena)),
                 scan_type,
                 file_options: options,

diff --git a/crates/polars-plan/src/plans/mod.rs b/crates/polars-plan/src/plans/mod.rs
@@ -2,7 +2,6 @@ use std::fmt;
 use std::fmt::Debug;
 use std::sync::{Arc, Mutex, RwLock};
 
-use hive::HivePartitions;
 use polars_core::prelude::*;
 use recursive::recursive;
 
@@ -89,7 +88,6 @@ pub enum DslPlan {
         // are used as base of different queries in a loop. That way
         // the expensive schema resolving is cached.
         file_info: Arc<RwLock<Option<FileInfo>>>,
-        hive_parts: Option<Arc<Vec<HivePartitions>>>,
         predicate: Option<Expr>,
         file_options: FileScanOptions,
         scan_type: FileScan,
@@ -199,7 +197,7 @@ impl Clone for DslPlan {
             Self::PythonScan { options } => Self::PythonScan { options: options.clone() },
             Self::Filter { input, predicate } => Self::Filter { input: input.clone(), predicate: predicate.clone() },
             Self::Cache { input, id, cache_hits } => Self::Cache { input: input.clone(), id: id.clone(), cache_hits: cache_hits.clone() },
-            Self::Scan { sources, file_info, hive_parts, predicate, file_options, scan_type } => Self::Scan { sources: sources.clone(), file_info: file_info.clone(), hive_parts: hive_parts.clone(), predicate: predicate.clone(), file_options: file_options.clone(), scan_type: scan_type.clone() },
+            Self::Scan { sources, file_info, predicate, file_options, scan_type } => Self::Scan { sources: sources.clone(), file_info: file_info.clone(), predicate: predicate.clone(), file_options: file_options.clone(), scan_type: scan_type.clone() },
             Self::DataFrameScan { df, schema, output_schema, filter: selection } => Self::DataFrameScan { df: df.clone(), schema: schema.clone(), output_schema: output_schema.clone(), filter: selection.clone() },
             Self::Select { expr, input, options } => Self::Select { expr: expr.clone(), input: input.clone(), options: options.clone() },
             Self::GroupBy { input, keys, aggs,  apply, maintain_order, options } => Self::GroupBy { input: input.clone(), keys: keys.clone(), aggs: aggs.clone(), apply: apply.clone(), maintain_order: maintain_order.clone(), options: options.clone() },