From abe09a2ab1a2d37b65e90e50f302523a0406f63d Mon Sep 17 00:00:00 2001 From: Michael-J-Ward Date: Mon, 13 May 2024 15:32:42 -0500 Subject: [PATCH] remove PyDFField and related methods DFField was removed upstream. Ref: https://github.com/apache/datafusion/pull/9595 --- datafusion/__init__.py | 1 - datafusion/tests/test_imports.py | 3 +- src/common.rs | 2 - src/common/df_field.rs | 111 ------------------------------- src/expr.rs | 42 +----------- 5 files changed, 4 insertions(+), 155 deletions(-) delete mode 100644 src/common/df_field.rs diff --git a/datafusion/__init__.py b/datafusion/__init__.py index 76012e90..d0b823bb 100644 --- a/datafusion/__init__.py +++ b/datafusion/__init__.py @@ -37,7 +37,6 @@ ) from .common import ( - DFField, DFSchema, ) diff --git a/datafusion/tests/test_imports.py b/datafusion/tests/test_imports.py index 962891ac..2a8a3de8 100644 --- a/datafusion/tests/test_imports.py +++ b/datafusion/tests/test_imports.py @@ -27,7 +27,6 @@ ) from datafusion.common import ( - DFField, DFSchema, ) @@ -161,7 +160,7 @@ def test_class_module_is_datafusion(): assert klass.__module__ == "datafusion.expr" # schema - for klass in [DFField, DFSchema]: + for klass in [DFSchema]: assert klass.__module__ == "datafusion.common" diff --git a/src/common.rs b/src/common.rs index 45523173..682639ac 100644 --- a/src/common.rs +++ b/src/common.rs @@ -18,7 +18,6 @@ use pyo3::prelude::*; pub mod data_type; -pub mod df_field; pub mod df_schema; pub mod function; pub mod schema; @@ -26,7 +25,6 @@ pub mod schema; /// Initializes the `common` module to match the pattern of `datafusion-common` https://docs.rs/datafusion-common/18.0.0/datafusion_common/index.html pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_class::()?; - m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/common/df_field.rs b/src/common/df_field.rs deleted file mode 100644 index 68c05361..00000000 --- a/src/common/df_field.rs +++ /dev/null @@ -1,111 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use datafusion::arrow::datatypes::DataType; -use datafusion_common::{DFField, OwnedTableReference}; -use pyo3::prelude::*; - -use super::data_type::PyDataType; - -/// PyDFField wraps an arrow-datafusion `DFField` struct type -/// and also supplies convenience methods for interacting -/// with the `DFField` instance in the context of Python -#[pyclass(name = "DFField", module = "datafusion.common", subclass)] -#[derive(Debug, Clone)] -pub struct PyDFField { - pub field: DFField, -} - -impl From for DFField { - fn from(py_field: PyDFField) -> DFField { - py_field.field - } -} - -impl From for PyDFField { - fn from(field: DFField) -> PyDFField { - PyDFField { field } - } -} - -#[pymethods] -impl PyDFField { - #[new] - #[pyo3(signature = (qualifier=None, name="", data_type=DataType::Int64.into(), nullable=false))] - fn new(qualifier: Option, name: &str, data_type: PyDataType, nullable: bool) -> Self { - PyDFField { - field: DFField::new( - qualifier.map(OwnedTableReference::from), - name, - data_type.into(), - nullable, - ), - } - } - - // TODO: Need bindings for Array `Field` first - // #[staticmethod] - // #[pyo3(name = "from")] - // fn py_from(field: Field) -> Self {} - - // TODO: Need bindings for Array `Field` first - // #[staticmethod] - // #[pyo3(name = "from_qualified")] - // fn py_from_qualified(field: Field) -> Self {} - - #[pyo3(name = "name")] - fn py_name(&self) -> PyResult { - Ok(self.field.name().clone()) - } - - #[pyo3(name = "data_type")] - fn py_data_type(&self) -> PyResult { - Ok(self.field.data_type().clone().into()) - } - - #[pyo3(name = "is_nullable")] - fn py_is_nullable(&self) -> PyResult { - Ok(self.field.is_nullable()) - } - - #[pyo3(name = "qualified_name")] - fn py_qualified_name(&self) -> PyResult { - Ok(self.field.qualified_name()) - } - - // TODO: Need bindings for `Column` first - // #[pyo3(name = "qualified_column")] - // fn py_qualified_column(&self) -> PyResult {} - - // TODO: Need bindings for `Column` first - // #[pyo3(name = "unqualified_column")] - // fn py_unqualified_column(&self) -> PyResult {} - - #[pyo3(name = "qualifier")] - fn py_qualifier(&self) -> PyResult> { - Ok(self.field.qualifier().map(|q| format!("{}", q))) - } - - // TODO: Need bindings for Arrow `Field` first - // #[pyo3(name = "field")] - // fn py_field(&self) -> PyResult {} - - #[pyo3(name = "strip_qualifier")] - fn py_strip_qualifier(&self) -> PyResult { - Ok(self.field.clone().strip_qualifier().into()) - } -} diff --git a/src/expr.rs b/src/expr.rs index 83f5771f..4291df44 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -21,23 +21,19 @@ use std::convert::{From, Into}; use datafusion::arrow::datatypes::DataType; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::scalar::ScalarValue; -use datafusion_common::DFField; use datafusion_expr::{ col, expr::{AggregateFunction, InList, InSubquery, ScalarFunction, Sort, WindowFunction}, - lit, - utils::exprlist_to_fields, - Between, BinaryExpr, Case, Cast, Expr, GetFieldAccess, GetIndexedField, Like, LogicalPlan, - Operator, TryCast, + lit, Between, BinaryExpr, Case, Cast, Expr, GetFieldAccess, GetIndexedField, Like, Operator, + TryCast, }; use crate::common::data_type::{DataTypeMap, RexType}; -use crate::errors::{py_datafusion_err, py_runtime_err, py_type_err, DataFusionError}; +use crate::errors::{py_datafusion_err, py_runtime_err, py_type_err}; use crate::expr::aggregate_expr::PyAggregateFunction; use crate::expr::binary_expr::PyBinaryExpr; use crate::expr::column::PyColumn; use crate::expr::literal::PyLiteral; -use crate::sql::logical::PyLogicalPlan; use self::alias::PyAlias; use self::bool_expr::{ @@ -557,41 +553,9 @@ impl PyExpr { } }) } - - pub fn column_name(&self, plan: PyLogicalPlan) -> PyResult { - self._column_name(&plan.plan()).map_err(py_runtime_err) - } } impl PyExpr { - pub fn _column_name(&self, plan: &LogicalPlan) -> Result { - let field = Self::expr_to_field(&self.expr, plan)?; - Ok(field.qualified_column().flat_name()) - } - - /// Create a [DFField] representing an [Expr], given an input [LogicalPlan] to resolve against - pub fn expr_to_field( - expr: &Expr, - input_plan: &LogicalPlan, - ) -> Result { - match expr { - Expr::Sort(Sort { expr, .. }) => { - // DataFusion does not support create_name for sort expressions (since they never - // appear in projections) so we just delegate to the contained expression instead - Self::expr_to_field(expr, input_plan) - } - Expr::Wildcard { .. } => { - // Since * could be any of the valid column names just return the first one - Ok(input_plan.schema().field(0).clone()) - } - _ => { - let fields = - exprlist_to_fields(&[expr.clone()], input_plan).map_err(PyErr::from)?; - Ok(fields[0].clone()) - } - } - } - fn _types(expr: &Expr) -> PyResult { match expr { Expr::BinaryExpr(BinaryExpr {