Merge remote-tracking branch 'apache/main' into alamb/improve_descrip…

…tion

.github/actions/setup-builder/action.yaml

@@ -28,16 +28,18 @@ runs:
     - name: Install Build Dependencies
       shell: bash
       run: |
-        apt-get update
-        apt-get install -y protobuf-compiler
+        RETRY="ci/scripts/retry"
+        "${RETRY}" apt-get update
+        "${RETRY}" apt-get install -y protobuf-compiler
     - name: Setup Rust toolchain
       shell: bash
       # rustfmt is needed for the substrait build script
       run: |
+        RETRY="ci/scripts/retry"
         echo "Installing ${{ inputs.rust-version }}"
-        rustup toolchain install ${{ inputs.rust-version }}
-        rustup default ${{ inputs.rust-version }}
-        rustup component add rustfmt
+        "${RETRY}" rustup toolchain install ${{ inputs.rust-version }}
+        "${RETRY}" rustup default ${{ inputs.rust-version }}
+        "${RETRY}" rustup component add rustfmt
     - name: Configure rust runtime env
       uses: ./.github/actions/setup-rust-runtime
     - name: Fixup git permissions

ci/scripts/retry

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+x() {
+    echo "+ $*" >&2
+    "$@"
+}
+
+max_retry_time_seconds=$(( 3 * 60 ))
+retry_delay_seconds=10
+
+END=$(( $(date +%s) + ${max_retry_time_seconds} ))
+
+while (( $(date +%s) < $END )); do
+    x "$@" && exit 0
+    sleep "${retry_delay_seconds}"
+done
+
+echo "$0: retrying [$*] timed out" >&2
+exit 1

datafusion-examples/Cargo.toml

@@ -62,6 +62,7 @@ dashmap = { workspace = true }
 datafusion = { workspace = true, default-features = true, features = ["avro"] }
 datafusion-common = { workspace = true, default-features = true }
 datafusion-expr = { workspace = true }
+datafusion-functions-window-common = { workspace = true }
 datafusion-optimizer = { workspace = true, default-features = true }
 datafusion-physical-expr = { workspace = true, default-features = true }
 datafusion-proto = { workspace = true }

datafusion-examples/examples/advanced_udwf.rs

@@ -30,6 +30,7 @@ use datafusion_expr::function::WindowUDFFieldArgs;
 use datafusion_expr::{
     PartitionEvaluator, Signature, WindowFrame, WindowUDF, WindowUDFImpl,
 };
+use datafusion_functions_window_common::partition::PartitionEvaluatorArgs;
 
 /// This example shows how to use the full WindowUDFImpl API to implement a user
 /// defined window function. As in the `simple_udwf.rs` example, this struct implements
@@ -74,7 +75,10 @@ impl WindowUDFImpl for SmoothItUdf {
 
     /// Create a `PartitionEvaluator` to evaluate this function on a new
     /// partition.
-    fn partition_evaluator(&self) -> Result<Box<dyn PartitionEvaluator>> {
+    fn partition_evaluator(
+        &self,
+        _partition_evaluator_args: PartitionEvaluatorArgs,
+    ) -> Result<Box<dyn PartitionEvaluator>> {
         Ok(Box::new(MyPartitionEvaluator::new()))
     }
 

datafusion-examples/examples/simplify_udwf_expression.rs

@@ -27,6 +27,7 @@ use datafusion_expr::{
     expr::WindowFunction, simplify::SimplifyInfo, Expr, PartitionEvaluator, Signature,
     Volatility, WindowUDF, WindowUDFImpl,
 };
+use datafusion_functions_window_common::partition::PartitionEvaluatorArgs;
 
 /// This UDWF will show how to use the WindowUDFImpl::simplify() API
 #[derive(Debug, Clone)]
@@ -60,7 +61,10 @@ impl WindowUDFImpl for SimplifySmoothItUdf {
         &self.signature
     }
 
-    fn partition_evaluator(&self) -> Result<Box<dyn PartitionEvaluator>> {
+    fn partition_evaluator(
+        &self,
+        _partition_evaluator_args: PartitionEvaluatorArgs,
+    ) -> Result<Box<dyn PartitionEvaluator>> {
         todo!()
     }
 

datafusion/catalog/README.md

@@ -0,0 +1,26 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# DataFusion Catalog
+
+[DataFusion][df] is an extensible query execution framework, written in Rust, that uses Apache Arrow as its in-memory format.
+
+This crate is a submodule of DataFusion that provides catalog management functionality, including catalogs, schemas, and tables.
+
+[df]: https://crates.io/crates/datafusion

datafusion/common/src/dfschema.rs

@@ -406,33 +406,6 @@ impl DFSchema {
         }
     }
 
-    /// Check whether the column reference is ambiguous
-    pub fn check_ambiguous_name(
-        &self,
-        qualifier: Option<&TableReference>,
-        name: &str,
-    ) -> Result<()> {
-        let count = self
-            .iter()
-            .filter(|(field_q, f)| match (field_q, qualifier) {
-                (Some(q1), Some(q2)) => q1.resolved_eq(q2) && f.name() == name,
-                (None, None) => f.name() == name,
-                _ => false,
-            })
-            .take(2)
-            .count();
-        if count > 1 {
-            _schema_err!(SchemaError::AmbiguousReference {
-                field: Column {
-                    relation: None,
-                    name: name.to_string(),
-                },
-            })
-        } else {
-            Ok(())
-        }
-    }
-
     /// Find the qualified field with the given name
     pub fn qualified_field_with_name(
         &self,

datafusion/common/src/tree_node.rs

@@ -681,7 +681,7 @@ impl<T> Transformed<T> {
         }
     }
 
-    /// Create a `Transformed` with `transformed and [`TreeNodeRecursion::Continue`].
+    /// Create a `Transformed` with `transformed` and [`TreeNodeRecursion::Continue`].
     pub fn new_transformed(data: T, transformed: bool) -> Self {
         Self::new(data, transformed, TreeNodeRecursion::Continue)
     }

datafusion/core/Cargo.toml

@@ -67,8 +67,6 @@ math_expressions = ["datafusion-functions/math_expressions"]
 parquet = ["datafusion-common/parquet", "dep:parquet"]
 pyarrow = ["datafusion-common/pyarrow", "parquet"]
 regex_expressions = [
-    "datafusion-physical-expr/regex_expressions",
-    "datafusion-optimizer/regex_expressions",
     "datafusion-functions/regex_expressions",
 ]
 serde = ["arrow-schema/serde"]

datafusion/core/src/bin/print_functions_docs.rs

@@ -130,13 +130,14 @@ fn print_docs(
                 .find(|f| f.get_name() == name || f.get_aliases().contains(&name))
                 .unwrap();
 
-            let name = f.get_name();
             let aliases = f.get_aliases();
             let documentation = f.get_documentation();
 
             // if this name is an alias we need to display what it's an alias of
             if aliases.contains(&name) {
-                let _ = write!(docs, "_Alias of [{name}](#{name})._");
+                let fname = f.get_name();
+                let _ = writeln!(docs, r#"### `{name}`"#);
+                let _ = writeln!(docs, "_Alias of [{fname}](#{fname})._");
                 continue;
             }
 
@@ -183,10 +184,10 @@ fn print_docs(
 
             // next, aliases
             if !f.get_aliases().is_empty() {
-                let _ = write!(docs, "#### Aliases");
+                let _ = writeln!(docs, "#### Aliases");
 
                 for alias in f.get_aliases() {
-                    let _ = writeln!(docs, "- {alias}");
+                    let _ = writeln!(docs, "- {}", alias.replace("_", r#"\_"#));
                 }
             }
 

datafusion/core/src/dataframe/mod.rs

@@ -535,9 +535,26 @@ impl DataFrame {
         group_expr: Vec<Expr>,
         aggr_expr: Vec<Expr>,
     ) -> Result<DataFrame> {
+        let is_grouping_set = matches!(group_expr.as_slice(), [Expr::GroupingSet(_)]);
+        let aggr_expr_len = aggr_expr.len();
         let plan = LogicalPlanBuilder::from(self.plan)
             .aggregate(group_expr, aggr_expr)?
             .build()?;
+        let plan = if is_grouping_set {
+            let grouping_id_pos = plan.schema().fields().len() - 1 - aggr_expr_len;
+            // For grouping sets we do a project to not expose the internal grouping id
+            let exprs = plan
+                .schema()
+                .columns()
+                .into_iter()
+                .enumerate()
+                .filter(|(idx, _)| *idx != grouping_id_pos)
+                .map(|(_, column)| Expr::Column(column))
+                .collect::<Vec<_>>();
+            LogicalPlanBuilder::from(plan).project(exprs)?.build()?
+        } else {
+            plan
+        };
         Ok(DataFrame {
             session_state: self.session_state,
             plan,