Merge remote-tracking branch 'apache/main' into alamb/update_copy_docs

.github/workflows/pr_benchmarks.yml

@@ -28,9 +28,10 @@ jobs:
           cd benchmarks
           mkdir data
           
-          # Setup the TPC-H data set with a scale factor of 10
+          # Setup the TPC-H data sets for scale factors 1 and 10
           ./bench.sh data tpch
-          
+          ./bench.sh data tpch10
+
       - name: Generate unique result names
         run: |
           echo "HEAD_LONG_SHA=$(git log -1 --format='%H')" >> "$GITHUB_ENV" 
@@ -44,6 +45,9 @@ jobs:
           cd benchmarks
 
           ./bench.sh run tpch
+          ./bench.sh run tpch_mem
+          ./bench.sh run tpch10
+          ./bench.sh run tpch_mem10
           
           # For some reason this step doesn't seem to propagate the env var down into the script
           if [ -d "results/HEAD" ]; then
@@ -64,6 +68,9 @@ jobs:
           cd benchmarks
 
           ./bench.sh run tpch
+          ./bench.sh run tpch_mem
+          ./bench.sh run tpch10
+          ./bench.sh run tpch_mem10
           
           echo ${{ github.event.issue.number }} > pr
           

Cargo.toml

@@ -49,7 +49,7 @@ license = "Apache-2.0"
 readme = "README.md"
 repository = "https://github.com/apache/arrow-datafusion"
 rust-version = "1.72"
-version = "36.0.0"
+version = "37.0.0"
 
 [workspace.dependencies]
 # We turn off default-features for some dependencies here so the workspaces which inherit them can
@@ -71,20 +71,20 @@ bytes = "1.4"
 chrono = { version = "0.4.34", default-features = false }
 ctor = "0.2.0"
 dashmap = "5.4.0"
-datafusion = { path = "datafusion/core", version = "36.0.0", default-features = false }
-datafusion-common = { path = "datafusion/common", version = "36.0.0", default-features = false }
-datafusion-common-runtime = { path = "datafusion/common-runtime", version = "36.0.0" }
-datafusion-execution = { path = "datafusion/execution", version = "36.0.0" }
-datafusion-expr = { path = "datafusion/expr", version = "36.0.0" }
-datafusion-functions = { path = "datafusion/functions", version = "36.0.0" }
-datafusion-functions-array = { path = "datafusion/functions-array", version = "36.0.0" }
-datafusion-optimizer = { path = "datafusion/optimizer", version = "36.0.0", default-features = false }
-datafusion-physical-expr = { path = "datafusion/physical-expr", version = "36.0.0", default-features = false }
-datafusion-physical-plan = { path = "datafusion/physical-plan", version = "36.0.0" }
-datafusion-proto = { path = "datafusion/proto", version = "36.0.0" }
-datafusion-sql = { path = "datafusion/sql", version = "36.0.0" }
-datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "36.0.0" }
-datafusion-substrait = { path = "datafusion/substrait", version = "36.0.0" }
+datafusion = { path = "datafusion/core", version = "37.0.0", default-features = false }
+datafusion-common = { path = "datafusion/common", version = "37.0.0", default-features = false }
+datafusion-common-runtime = { path = "datafusion/common-runtime", version = "37.0.0" }
+datafusion-execution = { path = "datafusion/execution", version = "37.0.0" }
+datafusion-expr = { path = "datafusion/expr", version = "37.0.0" }
+datafusion-functions = { path = "datafusion/functions", version = "37.0.0" }
+datafusion-functions-array = { path = "datafusion/functions-array", version = "37.0.0" }
+datafusion-optimizer = { path = "datafusion/optimizer", version = "37.0.0", default-features = false }
+datafusion-physical-expr = { path = "datafusion/physical-expr", version = "37.0.0", default-features = false }
+datafusion-physical-plan = { path = "datafusion/physical-plan", version = "37.0.0" }
+datafusion-proto = { path = "datafusion/proto", version = "37.0.0" }
+datafusion-sql = { path = "datafusion/sql", version = "37.0.0" }
+datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "37.0.0" }
+datafusion-substrait = { path = "datafusion/substrait", version = "37.0.0" }
 doc-comment = "0.3"
 env_logger = "0.11"
 futures = "0.3"

benchmarks/bench.sh

@@ -314,7 +314,7 @@ run_tpch() {
     fi
     TPCH_DIR="${DATA_DIR}/tpch_sf${SCALE_FACTOR}"
 
-    RESULTS_FILE="${RESULTS_DIR}/tpch.json"
+    RESULTS_FILE="${RESULTS_DIR}/tpch_sf${SCALE_FACTOR}.json"
     echo "RESULTS_FILE: ${RESULTS_FILE}"
     echo "Running tpch benchmark..."
     $CARGO_COMMAND --bin tpch -- benchmark datafusion --iterations 5 --path "${TPCH_DIR}" --format parquet -o ${RESULTS_FILE}
@@ -329,7 +329,7 @@ run_tpch_mem() {
     fi
     TPCH_DIR="${DATA_DIR}/tpch_sf${SCALE_FACTOR}"
 
-    RESULTS_FILE="${RESULTS_DIR}/tpch_mem.json"
+    RESULTS_FILE="${RESULTS_DIR}/tpch_mem_sf${SCALE_FACTOR}.json"
     echo "RESULTS_FILE: ${RESULTS_FILE}"
     echo "Running tpch_mem benchmark..."
     # -m means in memory