[REVIEW] Abstract Syntax Tree evaluator

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## New Features
 
 - PR #5974 Use libcudf instead of cupy for `arange` or column creation from a scalar.
+- PR #5494 Add Abstract Syntax Tree (AST) evaluator.
 
 ## Improvements
 
@@ -39,7 +40,6 @@
 - PR #5364 Validate array interface during buffer construction
 - PR #5418 Add support for `DataFrame.info`
 - PR #5425 Add Python `Groupby.rolling()`
-- PR #5359 Add duration types
 - PR #5434 Add nvtext function generate_character_grams
 - PR #5442 Add support for `cudf.isclose`
 - PR #5444 Remove usage of deprecated RMM APIs and headers.
@@ -189,6 +189,7 @@
 - PR #5720 Replace owning raw pointers with std::unique_ptr
 - PR #5702 Add inherited methods to python docs and other docs fixes
 - PR #5733 Add support for `size` property in `DataFrame`/ `Series` / `Index`/ `MultiIndex`
+- PR #5735 Force timestamp creation only with duration
 - PR #5743 Reduce number of test cases in concatenate benchmark
 - PR #5748 Disable `tolist` API in `Series` & `Index` and add `tolist` dispatch in `dask-cudf`
 - PR #5744 Reduce number of test cases in reduction benchmark

conda/recipes/libcudf/meta.yaml

@@ -1,4 +1,4 @@
-# Copyright (c) 2018, NVIDIA CORPORATION.
+# Copyright (c) 2018-2020, NVIDIA CORPORATION.
 
 {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
 {% set minor_version =  version.split('.')[0] + '.' + version.split('.')[1] %}
@@ -46,6 +46,8 @@ test:
     - test -f $PREFIX/lib/libcudf.so
     - test -f $PREFIX/lib/libcudftestutil.a
     - test -f $PREFIX/include/cudf/aggregation.hpp
+    - test -f $PREFIX/include/cudf/ast/linearizer.hpp
+    - test -f $PREFIX/include/cudf/ast/operators.hpp
     - test -f $PREFIX/include/cudf/binaryop.hpp
     - test -f $PREFIX/include/cudf/column/column_factories.hpp
     - test -f $PREFIX/include/cudf/column/column.hpp

cpp/CMakeLists.txt

@@ -614,6 +614,8 @@ add_library(cudf
             src/aggregation/aggregation.cpp
             src/aggregation/aggregation.cu
             src/aggregation/result_cache.cpp
+            src/ast/ast.cu
+            src/ast/linearizer.cpp
 )
 
 # Override RPATH for cudf

cpp/benchmarks/CMakeLists.txt

@@ -30,7 +30,7 @@ include_directories("${CUB_INCLUDE_DIR}"
 if(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES)
     include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}")
 endif(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES)
-                  
+
 include_directories("${CMAKE_BINARY_DIR}/include"
                     "${CMAKE_SOURCE_DIR}/include"
                     "${CMAKE_SOURCE_DIR}"
@@ -174,23 +174,23 @@ set(HASHING_BENCH_SRC
 ConfigureBench(HASHING_BENCH "${HASHING_BENCH_SRC}")
 
 ###################################################################################################
-# - merge benchmark -----------------------------------------------------------------------------
+# - merge benchmark -------------------------------------------------------------------------------
 
 set(MERGE_BENCH_SRC
   "${CMAKE_CURRENT_SOURCE_DIR}/merge/merge_benchmark.cpp")
 
 ConfigureBench(MERGE_BENCH "${MERGE_BENCH_SRC}")
 
 ###################################################################################################
-# - null_mask benchmark -----------------------------------------------------------------------------
+# - null_mask benchmark ---------------------------------------------------------------------------
 
 set(NULLMASK_BENCH_SRC
   "${CMAKE_CURRENT_SOURCE_DIR}/null_mask/set_null_mask_benchmark.cpp")
 
 ConfigureBench(NULLMASK_BENCH "${NULLMASK_BENCH_SRC}")
 
 ###################################################################################################
-# - parquet writer chunks benchmark -----------------------------------------------------------------------------
+# - parquet writer chunks benchmark ---------------------------------------------------------------
 
 set(PARQUET_WRITER_CHUNKS_BENCH_SRC
   "${CMAKE_CURRENT_SOURCE_DIR}/io/parquet/parquet_writer_chunks_benchmark.cpp")
@@ -222,29 +222,45 @@ set(CSV_READER_BENCH_SRC
 ConfigureBench(CSV_READER_BENCH "${CSV_READER_BENCH_SRC}")
 
 ###################################################################################################
-# - parquet writer benchmark -----------------------------------------------------------------------------
+# - parquet writer benchmark ----------------------------------------------------------------------
 
 set(PARQUET_WRITER_BENCH_SRC
   "${CMAKE_CURRENT_SOURCE_DIR}/io/parquet/parquet_writer_benchmark.cpp")
 
 ConfigureBench(PARQUET_WRITER_BENCH "${PARQUET_WRITER_BENCH_SRC}")
 
 ###################################################################################################
-# - orc writer benchmark -----------------------------------------------------------------------------
+# - orc writer benchmark --------------------------------------------------------------------------
 
 set(ORC_WRITER_BENCH_SRC
   "${CMAKE_CURRENT_SOURCE_DIR}/io/orc/orc_writer_benchmark.cpp")
 
 ConfigureBench(ORC_WRITER_BENCH "${ORC_WRITER_BENCH_SRC}")
 
 ###################################################################################################
-# - csv writer benchmark -----------------------------------------------------------------------------
+# - csv writer benchmark --------------------------------------------------------------------------
 
 set(CSV_WRITER_BENCH_SRC
   "${CMAKE_CURRENT_SOURCE_DIR}/io/csv/csv_writer_benchmark.cpp")
 
 ConfigureBench(CSV_WRITER_BENCH "${CSV_WRITER_BENCH_SRC}")
 
+###################################################################################################
+# - ast benchmark ---------------------------------------------------------------------------------
+
+set(AST_BENCH_SRC
+  "${CMAKE_CURRENT_SOURCE_DIR}/ast/ast_benchmark.cu")
+
+ConfigureBench(AST_BENCH "${AST_BENCH_SRC}")
+
+###################################################################################################
+# - binaryop benchmark ----------------------------------------------------------------------------
+
+set(BINARYOP_BENCH_SRC
+  "${CMAKE_CURRENT_SOURCE_DIR}/binaryop/binaryop_benchmark.cu")
+
+ConfigureBench(BINARYOP_BENCH "${BINARYOP_BENCH_SRC}")
+
 ###################################################################################################
 # - subword tokenizer benchmark -------------------------------------------------------------------
 

cpp/benchmarks/ast/ast_benchmark.cu

@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <benchmark/benchmark.h>
+
+#include <thrust/iterator/counting_iterator.h>
+
+#include <cudf/ast/ast.cuh>
+#include <cudf/column/column_factories.hpp>
+#include <cudf/table/table.hpp>
+#include <cudf/table/table_view.hpp>
+#include <cudf/types.hpp>
+#include <cudf/utilities/error.hpp>
+#include <tests/utilities/column_wrapper.hpp>
+
+#include <fixture/benchmark_fixture.hpp>
+#include <synchronization/synchronization.hpp>
+
+#include <algorithm>
+#include <iostream>
+#include <list>
+#include <numeric>
+#include <vector>
+
+enum class TreeType {
+  IMBALANCED_LEFT  // All operator expressions have a left child operator expression and a right
+                   // child column reference
+};
+
+template <typename key_type, TreeType tree_type, bool reuse_columns>
+class AST : public cudf::benchmark {
+};
+
+template <typename key_type, TreeType tree_type, bool reuse_columns>
+static void BM_ast_transform(benchmark::State& state)
+{
+  const cudf::size_type table_size{(cudf::size_type)state.range(0)};
+  const cudf::size_type tree_levels = (cudf::size_type)state.range(1);
+
+  // Create table data
+  auto n_cols          = reuse_columns ? 1 : tree_levels + 1;
+  auto column_wrappers = std::vector<cudf::test::fixed_width_column_wrapper<key_type>>(n_cols);
+  auto columns         = std::vector<cudf::column_view>(n_cols);
+
+  auto data_iterator = thrust::make_counting_iterator(0);
+  std::generate_n(column_wrappers.begin(), n_cols, [=]() {
+    return cudf::test::fixed_width_column_wrapper<key_type>(data_iterator,
+                                                            data_iterator + table_size);
+  });
+  std::transform(
+    column_wrappers.begin(), column_wrappers.end(), columns.begin(), [](auto const& col) {
+      return static_cast<cudf::column_view>(col);
+    });
+
+  cudf::table_view table{columns};
+
+  // Create column references
+  auto column_refs = std::vector<cudf::ast::column_reference>();
+  std::transform(thrust::make_counting_iterator(0),
+                 thrust::make_counting_iterator(n_cols),
+                 std::back_inserter(column_refs),
+                 [](auto const& column_id) {
+                   return cudf::ast::column_reference(reuse_columns ? 0 : column_id);
+                 });
+
+  // Create expression trees
+
+  // Note that a std::list is required here because of its guarantees against reference invalidation
+  // when items are added or removed. References to items in a std::vector are not safe if the
+  // vector must re-allocate.
+  auto expressions = std::list<cudf::ast::expression>();
+
+  // Construct tree that chains additions like (((a + b) + c) + d)
+  auto const op = cudf::ast::ast_operator::ADD;
+  if (reuse_columns) {
+    expressions.push_back(cudf::ast::expression(op, column_refs.at(0), column_refs.at(0)));
+    for (cudf::size_type i = 0; i < tree_levels - 1; i++) {
+      expressions.push_back(cudf::ast::expression(op, expressions.back(), column_refs.at(0)));
+    }
+  } else {
+    expressions.push_back(cudf::ast::expression(op, column_refs.at(0), column_refs.at(1)));
+    std::transform(std::next(column_refs.cbegin(), 2),
+                   column_refs.cend(),
+                   std::back_inserter(expressions),
+                   [&](auto const& column_ref) {
+                     return cudf::ast::expression(op, expressions.back(), column_ref);
+                   });
+  }
+
+  auto const& expression_tree_root = expressions.back();
+
+  // Execute benchmark
+  for (auto _ : state) {
+    cuda_event_timer raii(state, true);  // flush_l2_cache = true, stream = 0
+    cudf::ast::compute_column(table, expression_tree_root);
+  }
+
+  // Use the number of bytes read from global memory
+  state.SetBytesProcessed(static_cast<int64_t>(state.iterations()) * state.range(0) *
+                          (tree_levels + 1) * sizeof(key_type));
+}
+
+#define AST_TRANSFORM_BENCHMARK_DEFINE(name, key_type, tree_type, reuse_columns) \
+  BENCHMARK_TEMPLATE_DEFINE_F(AST, name, key_type, tree_type, reuse_columns)     \
+  (::benchmark::State & st) { BM_ast_transform<key_type, tree_type, reuse_columns>(st); }
+
+AST_TRANSFORM_BENCHMARK_DEFINE(ast_int32_imbalanced_unique,
+                               int32_t,
+                               TreeType::IMBALANCED_LEFT,
+                               false);
+AST_TRANSFORM_BENCHMARK_DEFINE(ast_int32_imbalanced_reuse,
+                               int32_t,
+                               TreeType::IMBALANCED_LEFT,
+                               true);
+AST_TRANSFORM_BENCHMARK_DEFINE(ast_double_imbalanced_unique,
+                               double,
+                               TreeType::IMBALANCED_LEFT,
+                               false);
+
+static void CustomRanges(benchmark::internal::Benchmark* b)
+{
+  auto row_counts       = std::vector<cudf::size_type>{100'000, 1'000'000, 10'000'000, 100'000'000};
+  auto operation_counts = std::vector<cudf::size_type>{1, 2, 5, 10};
+  for (auto const& row_count : row_counts) {
+    for (auto const& operation_count : operation_counts) { b->Args({row_count, operation_count}); }
+  }
+}
+
+BENCHMARK_REGISTER_F(AST, ast_int32_imbalanced_unique)
+  ->Apply(CustomRanges)
+  ->Unit(benchmark::kMillisecond)
+  ->UseManualTime();
+
+BENCHMARK_REGISTER_F(AST, ast_int32_imbalanced_reuse)
+  ->Apply(CustomRanges)
+  ->Unit(benchmark::kMillisecond)
+  ->UseManualTime();
+
+BENCHMARK_REGISTER_F(AST, ast_double_imbalanced_unique)
+  ->Apply(CustomRanges)
+  ->Unit(benchmark::kMillisecond)
+  ->UseManualTime();