Merge remote-tracking branch 'apache/main' into alamb/more_tree_node_…

…docs

.asf.yaml

@@ -26,7 +26,7 @@ notifications:
   pullrequests: github@arrow.apache.org
   jira_options: link label worklog
 github:
-  description: "Apache Arrow DataFusion SQL Query Engine"
+  description: "Apache DataFusion SQL Query Engine"
   homepage: https://arrow.apache.org/datafusion
   labels:
     - arrow

.github/dependabot.yml

@@ -28,6 +28,20 @@ updates:
       # arrow is bumped manually
       - dependency-name: "arrow*"
         update-types: ["version-update:semver-major"]
+  - package-ecosystem: cargo
+    directory: "datafusion-cli/"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+    target-branch: main
+    labels: [auto-dependencies]
+    ignore:
+      # arrow is bumped manually
+      - dependency-name: "arrow*"
+        update-types: ["version-update:semver-major"]
+      # datafusion is bumped manually
+      - dependency-name: "datafusion*"
+        update-types: ["version-update:semver-major"]
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:

.github/workflows/dependencies.yml

@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Dependencies
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+on:
+  push:
+    paths:
+      - "**/Cargo.toml"
+  pull_request:
+    paths:
+      - "**/Cargo.toml"
+  # manual trigger
+  # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
+  workflow_dispatch:
+
+jobs:
+  depcheck:
+    name: circular dependency check
+    runs-on: ubuntu-latest
+    container:
+      image: amd64/rust
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: stable
+      - name: Check dependencies
+        run: |
+          cd dev/depcheck
+          cargo run

.github/workflows/rust.yml

@@ -195,24 +195,7 @@ jobs:
       - name: Verify Working Directory Clean
         run: git diff --exit-code
 
-  depcheck:
-    name: circular dependency check
-    needs: [ linux-build-lib ]
-    runs-on: ubuntu-latest
-    container:
-      image: amd64/rust
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-      - name: Setup Rust toolchain
-        uses: ./.github/actions/setup-builder
-        with:
-          rust-version: stable
-      - name: Check dependencies
-        run: |
-          cd dev/depcheck
-          cargo run 
+
 
   # Run `cargo test doc` (test documentation examples)
   linux-test-doc:

Cargo.toml

@@ -51,7 +51,7 @@ license = "Apache-2.0"
 readme = "README.md"
 repository = "https://github.com/apache/arrow-datafusion"
 rust-version = "1.73"
-version = "37.0.0"
+version = "37.1.0"
 
 [workspace.dependencies]
 # We turn off default-features for some dependencies here so the workspaces which inherit them can
@@ -73,22 +73,22 @@ bytes = "1.4"
 chrono = { version = "0.4.34", default-features = false }
 ctor = "0.2.0"
 dashmap = "5.4.0"
-datafusion = { path = "datafusion/core", version = "37.0.0", default-features = false }
-datafusion-common = { path = "datafusion/common", version = "37.0.0", default-features = false }
-datafusion-common-runtime = { path = "datafusion/common-runtime", version = "37.0.0" }
-datafusion-execution = { path = "datafusion/execution", version = "37.0.0" }
-datafusion-expr = { path = "datafusion/expr", version = "37.0.0" }
-datafusion-functions = { path = "datafusion/functions", version = "37.0.0" }
-datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "37.0.0" }
-datafusion-functions-array = { path = "datafusion/functions-array", version = "37.0.0" }
-datafusion-optimizer = { path = "datafusion/optimizer", version = "37.0.0", default-features = false }
-datafusion-physical-expr = { path = "datafusion/physical-expr", version = "37.0.0", default-features = false }
-datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "37.0.0", default-features = false }
-datafusion-physical-plan = { path = "datafusion/physical-plan", version = "37.0.0" }
-datafusion-proto = { path = "datafusion/proto", version = "37.0.0" }
-datafusion-sql = { path = "datafusion/sql", version = "37.0.0" }
-datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "37.0.0" }
-datafusion-substrait = { path = "datafusion/substrait", version = "37.0.0" }
+datafusion = { path = "datafusion/core", version = "37.1.0", default-features = false }
+datafusion-common = { path = "datafusion/common", version = "37.1.0", default-features = false }
+datafusion-common-runtime = { path = "datafusion/common-runtime", version = "37.1.0" }
+datafusion-execution = { path = "datafusion/execution", version = "37.1.0" }
+datafusion-expr = { path = "datafusion/expr", version = "37.1.0" }
+datafusion-functions = { path = "datafusion/functions", version = "37.1.0" }
+datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "37.1.0" }
+datafusion-functions-array = { path = "datafusion/functions-array", version = "37.1.0" }
+datafusion-optimizer = { path = "datafusion/optimizer", version = "37.1.0", default-features = false }
+datafusion-physical-expr = { path = "datafusion/physical-expr", version = "37.1.0", default-features = false }
+datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "37.1.0", default-features = false }
+datafusion-physical-plan = { path = "datafusion/physical-plan", version = "37.1.0" }
+datafusion-proto = { path = "datafusion/proto", version = "37.1.0" }
+datafusion-sql = { path = "datafusion/sql", version = "37.1.0" }
+datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "37.1.0" }
+datafusion-substrait = { path = "datafusion/substrait", version = "37.1.0" }
 doc-comment = "0.3"
 env_logger = "0.11"
 futures = "0.3"
@@ -103,7 +103,7 @@ parquet = { version = "51.0.0", default-features = false, features = ["arrow", "
 rand = "0.8"
 rstest = "0.19.0"
 serde_json = "1"
-sqlparser = { version = "0.44.0", features = ["visitor"] }
+sqlparser = { version = "0.45.0", features = ["visitor"] }
 tempfile = "3"
 thiserror = "1.0.44"
 tokio = { version = "1.36", features = ["macros", "rt", "sync"] }
@@ -126,3 +126,7 @@ opt-level = 3
 overflow-checks = false
 panic = 'unwind'
 rpath = false
+
+[workspace.lints.clippy]
+# Detects large stack-allocated futures that may cause stack overflow crashes (see threshold in clippy.toml)
+large_futures = "warn"

README.md

@@ -17,7 +17,7 @@
   under the License.
 -->
 
-# DataFusion
+# Apache DataFusion
 
 [![Crates.io][crates-badge]][crates-url]
 [![Apache licensed][license-badge]][license-url]
@@ -40,7 +40,7 @@
 
 <img src="./docs/source/_static/images/2x_bgwhite_original.png" width="512" alt="logo"/>
 
-DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in
+Apache DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in
 [Rust](http://rustlang.org), using the [Apache Arrow](https://arrow.apache.org)
 in-memory format. [Python Bindings](https://github.com/apache/arrow-datafusion-python) are also available. DataFusion offers SQL and Dataframe APIs, excellent [performance](https://benchmark.clickhouse.com/), built-in support for CSV, Parquet, JSON, and Avro, extensive customization, and a great community.
 

benchmarks/Cargo.toml

@@ -6,7 +6,7 @@
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
-#   http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
@@ -26,6 +26,9 @@ repository = { workspace = true }
 license = { workspace = true }
 rust-version = { workspace = true }
 
+[lints]
+workspace = true
+
 [features]
 ci = []
 default = ["mimalloc"]

benchmarks/src/bin/parquet.rs

@@ -25,7 +25,7 @@ use structopt::StructOpt;
 static ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc;
 
 #[derive(Debug, Clone, StructOpt)]
-#[structopt(name = "Benchmarks", about = "Apache Arrow Rust Benchmarks.")]
+#[structopt(name = "Benchmarks", about = "Apache DataFusion Rust Benchmarks.")]
 enum ParquetBenchCmd {
     /// Benchmark sorting parquet files
     Sort(sort::RunOpt),

clippy.toml

@@ -6,3 +6,7 @@ disallowed-methods = [
 disallowed-types = [
     { path = "std::time::Instant", reason = "Use `datafusion_common::instant::Instant` instead for WASM compatibility" },
 ]
+
+# Lowering the threshold to help prevent stack overflows (default is 16384)
+# See: https://rust-lang.github.io/rust-clippy/master/index.html#/large_futures
+future-size-threshold = 10000