[WIP] Serialize/deserialize with libcudf pack/unpack

CHANGELOG.md

@@ -8,6 +8,7 @@
 - PR #5288 Drop `auto_pickle` decorator #5288
 - PR #5231 Type `Buffer` as `uint8`
 - PR #5308 Coerce frames to `Buffer`s in deserialization
+- PR #5309 Handle host frames in serialization
 
 ## Bug Fixes
 
@@ -27,6 +28,7 @@
 - PR #4789 Disallow timestamp sum and diffs via binary ops
 - PR #4815 Add JNI total memory allocated API
 - PR #4906 Add Java bindings for interleave_columns
+- PR #4941 Add `cudf::pack` and `cudf::unpack`
 - PR #4900 Add `get_element` to obtain scalar from a column given an index
 - PR #4938 Add Java bindings for strip
 - PR #4923 Add Java and JNI bindings for string split
@@ -49,6 +51,7 @@
 - PR #5192 Add support for `cudf.to_datetime`
 - PR #5203 Add Java bindings for is_integer and is_float
 - PR #5205 Add ci test for libcudf, libnvstrings headers existence check in meta.yml
+- PR #5293 Add Java bindings for replace_with_backrefs
 
 ## Improvements
 
@@ -167,6 +170,7 @@
 - PR #4998 Clean up dispatch of aggregation methods in result_cache
 - PR #5000 Performance improvements in `isin` and dask_cudf backend
 - PR #5002 Fix Column.__reduce__ to accept `null_count`
+- PR #5025 Serialize/deserialize with libcudf pack/unpack
 - PR #5006 Add Java bindings for strip, lstrip and rstrip
 - PR #5047 Add Cython binding for libcudf++ CSV reader
 - PR #5027 Move nvstrings standalone docs pages to libcudf doxygen pages
@@ -214,6 +218,10 @@
 - PR #5233 Remove experimental namespace used during libcudf++ refactor
 - PR #5213 Documentation enhancements to `cudf` python APIs
 - PR #5251 Fix more mispellings in cpp comments and strings
+- PR #5254 Deprecate nvstrings, nvcategory and nvtext
+- PR #5270 Add support to check for "NaT" and "None" strings while typecasting to `datetime64`
+- PR #5298 Remove unused native deps from java library
+- PR #5216 Make documentation uniform for params
 - PR #5261 Add short git commit to conda package name
 
 ## Bug Fixes
@@ -341,6 +349,7 @@
 - PR #5224 Add new headers from 5198 to libcudf/meta.yaml
 - PR #5228 Fix datetime64 scalar dtype handling for unsupported time units
 - PR #5256 ORC reader: fix loading individual timestamp columns
+- PR #5285 Fix DEBUG compilation failure due to `fixed_point.hpp`
 
 
 # cuDF 0.13.0 (31 Mar 2020)

cpp/CMakeLists.txt

@@ -507,6 +507,7 @@ add_library(cudf
             src/copying/slice.cpp
             src/copying/split.cpp
             src/copying/contiguous_split.cu
+            src/copying/pack.cpp
             src/copying/copy_range.cu
             src/copying/get_element.cu
             src/filling/fill.cu

cpp/include/cudf/binaryop.hpp

@@ -86,7 +86,7 @@ enum class binary_operator : int32_t {
  * @param lhs         The left operand scalar
  * @param rhs         The right operand column
  * @param output_type The desired data type of the output column
- * @param mr          Memory resource for allocating output column
+ * @param mr          Device memory resource used to allocate the returned column's device memory
  * @return            Output column of `output_type` type containing the result of
  *                    the binary operation
  * @throw cudf::logic_error if @p output_type dtype isn't fixed-width
@@ -111,7 +111,7 @@ std::unique_ptr<column> binary_operation(
  * @param lhs         The left operand column
  * @param rhs         The right operand scalar
  * @param output_type The desired data type of the output column
- * @param mr          Memory resource for allocating output column
+ * @param mr          Device memory resource used to allocate the returned column's device memory
  * @return            Output column of `output_type` type containing the result of
  *                    the binary operation
  * @throw cudf::logic_error if @p output_type dtype isn't fixed-width
@@ -134,7 +134,7 @@ std::unique_ptr<column> binary_operation(
  * @param lhs         The left operand column
  * @param rhs         The right operand column
  * @param output_type The desired data type of the output column
- * @param mr          Memory resource for allocating output column
+ * @param mr          Device memory resource used to allocate the returned column's device memory
  * @return            Output column of `output_type` type containing the result of
  *                    the binary operation
  * @throw cudf::logic_error if @p lhs and @p rhs are different sizes
@@ -162,7 +162,7 @@ std::unique_ptr<column> binary_operation(
  * @param output_type The desired data type of the output column. It is assumed
  *                    that output_type is compatible with the output data type
  *                    of the function in the PTX code
- * @param mr          Memory resource for allocating output column
+ * @param mr          Device memory resource used to allocate the returned column's device memory
  * @return            Output column of `output_type` type containing the result of
  *                    the binary operation
  * @throw cudf::logic_error if @p lhs and @p rhs are different sizes

cpp/include/cudf/column/column.hpp

@@ -58,8 +58,8 @@ class column {
    * and copies.
    *
    * @param other The `column` to copy
-   * @param stream The stream on which to execute all allocations and copies
-   * @param mr The resource to use for all allocations
+   * @param stream CUDA stream used for device memory operations.
+   * @param mr Device memory resource to use for all device memory allocations
    */
   column(column const& other,
          cudaStream_t stream,
@@ -113,9 +113,8 @@ class column {
    * This accounts for the `column_view`'s offset.
    *
    * @param view The view to copy
-   * @param stream The stream on which all allocations and copies will be
-   * executed
-   * @param mr The resource to use for all allocations
+   * @param stream CUDA stream used for device memory operations.
+   * @param mr Device memory resource to use for all device memory allocations
    */
   explicit column(column_view view,
                   cudaStream_t stream                 = 0,

cpp/include/cudf/column/column_device_view.cuh

@@ -373,8 +373,7 @@ class alignas(16) column_device_view : public detail::column_device_view_base {
    * A `column_device_view` should be passed by value into GPU kernels.
    *
    * @param source_view The `column_view` to make usable in device code
-   * @param stream optional, stream on which the memory for children will be
-   * allocated
+   * @param stream CUDA stream used for device memory operations for children columns.
    * @return A `unique_ptr` to a `column_device_view` that makes the data from
    *`source_view` available in device memory.
    */
@@ -465,8 +464,7 @@ class alignas(16) mutable_column_device_view : public detail::column_device_view
    * A `mutable_column_device_view` should be passed by value into GPU kernels.
    *
    * @param source_view The `column_view` to make usable in device code
-   * @param stream optional, stream on which the memory for children will be
-   * allocated
+   * @param stream CUDA stream used for device memory operations for children columns.
    * @return A `unique_ptr` to a `mutable_column_device_view` that makes the
    * data from `source_view` available in device memory.
    */

cpp/include/cudf/column/column_factories.hpp

@@ -50,10 +50,8 @@ std::unique_ptr<column> make_empty_column(data_type type);
  * @param[in] size The number of elements in the column
  * @param[in] state Optional, controls allocation/initialization of the
  * column's null mask. By default, no null mask is allocated.
- * @param[in] stream Optional stream on which to issue all memory allocation and
- * device kernels
- * @param[in] mr Optional resource to use for device memory
- * allocation of the column's `data` and `null_mask`.
+ * @param[in] stream CUDA stream used for device memory operations and kernel launches.
+ * @param[in] mr Device memory resource used to allocate the returned column's device memory
  */
 std::unique_ptr<column> make_numeric_column(
   data_type type,
@@ -76,10 +74,8 @@ std::unique_ptr<column> make_numeric_column(
  * @param[in] size The number of elements in the column
  * @param[in] null_mask Null mask to use for this column.
  * @param[in] null_count Optional number of nulls in the null_mask.
- * @param[in] stream Optional stream on which to issue all memory allocation and
- * device kernels
- * @param[in] mr Optional resource to use for device memory
- * allocation of the column's `data` and `null_mask`.
+ * @param[in] stream CUDA stream used for device memory operations and kernel launches.
+ * @param[in] mr Device memory resource used to allocate the returned column's device memory
  */
 template <typename B>
 std::unique_ptr<column> make_numeric_column(
@@ -112,10 +108,8 @@ std::unique_ptr<column> make_numeric_column(
  * @param[in] size The number of elements in the column
  * @param[in] state Optional, controls allocation/initialization of the
  * column's null mask. By default, no null mask is allocated.
- * @param[in] stream Optional stream on which to issue all memory allocation and
- * device kernels
- * @param[in] mr Optional resource to use for device memory
- * allocation of the column's `data` and `null_mask`.
+ * @param[in] stream CUDA stream used for device memory operations and kernel launches.
+ * @param[in] mr Device memory resource used to allocate the returned column's device memory
  */
 std::unique_ptr<column> make_timestamp_column(
   data_type type,
@@ -138,10 +132,8 @@ std::unique_ptr<column> make_timestamp_column(
  * @param[in] size The number of elements in the column
  * @param[in] null_mask Null mask to use for this column.
  * @param[in] null_count Optional number of nulls in the null_mask.
- * @param[in] stream Optional stream on which to issue all memory allocation and
- * device kernels
- * @param[in] mr Optional resource to use for device memory
- * allocation of the column's `data` and `null_mask`.
+ * @param[in] stream CUDA stream used for device memory operations and kernel launches.
+ * @param[in] mr Device memory resource used to allocate the returned column's device memory
  */
 template <typename B>
 std::unique_ptr<column> make_timestamp_column(
@@ -174,10 +166,8 @@ std::unique_ptr<column> make_timestamp_column(
  * @param[in] size The number of elements in the column
  * @param[in] state Optional, controls allocation/initialization of the
  * column's null mask. By default, no null mask is allocated.
- * @param[in] stream Optional stream on which to issue all memory allocation and device
- * kernels
- * @param[in] mr Optional resource to use for device memory
- * allocation of the column's `data` and `null_mask`.
+ * @param[in] stream CUDA stream used for device memory operations and kernel launches.
+ * @param[in] mr Device memory resource used to allocate the returned column's device memory
  */
 std::unique_ptr<column> make_fixed_width_column(
   data_type type,
@@ -200,10 +190,8 @@ std::unique_ptr<column> make_fixed_width_column(
  * @param[in] size The number of elements in the column
  * @param[in] null_mask Null mask to use for this column.
  * @param[in] null_count Optional number of nulls in the null_mask.
- * @param[in] stream Optional stream on which to issue all memory allocation and device
- * kernels
- * @param[in] mr Optional resource to use for device memory
- * allocation of the column's `data` and `null_mask`.
+ * @param[in] stream CUDA stream used for device memory operations and kernel launches.
+ * @param[in] mr Device memory resource used to allocate the returned column's device memory
  */
 template <typename B>
 std::unique_ptr<column> make_fixed_width_column(
@@ -239,10 +227,9 @@ std::unique_ptr<column> make_fixed_width_column(
  * @param strings The vector of pointer/size pairs.
  *                Each pointer must be a device memory address or `nullptr`
  * (indicating a null string). The size must be the number of bytes.
- * @param stream Optional stream for use with all memory allocation
- *               and device kernels
- * @param mr Optional resource to use for device memory
- *           allocation of the column's `null_mask` and children.
+ * @param stream CUDA stream used for device memory operations and kernel launches.
+ * @param mr Device memory resource used for allocation of the column's `null_mask` and children
+ * columns' device memory.
  */
 std::unique_ptr<column> make_strings_column(
   const rmm::device_vector<thrust::pair<const char*, size_type>>& strings,
@@ -271,10 +258,9 @@ std::unique_ptr<column> make_strings_column(
  * bytes.
  * @param null_placeholder string_view indicating null string in given list of
  * string_views.
- * @param stream Optional stream for use with all memory allocation
- *               and device kernels
- * @param mr Optional resource to use for device memory
- *           allocation of the column's `null_mask` and children.
+ * @param stream CUDA stream used for device memory operations and kernel launches.
+ * @param mr Device memory resource used for allocation of the column's `null_mask` and children
+ * columns' device memory.
  */
 std::unique_ptr<column> make_strings_column(
   const rmm::device_vector<string_view>& string_views,
@@ -310,10 +296,9 @@ std::unique_ptr<column> make_strings_column(
  * @param null_count The number of null string entries. If equal to
  * `UNKNOWN_NULL_COUNT`, the null count will be computed dynamically on the
  * first invocation of `column::null_count()`
- * @param stream Optional stream for use with all memory allocation
- *               and device kernels
- * @param mr Optional resource to use for device memory
- *           allocation of the column's `null_mask` and children.
+ * @param stream CUDA stream used for device memory operations and kernel launches.
+ * @param mr Device memory resource used for allocation of the column's `null_mask` and children
+ * columns' device memory.
  */
 std::unique_ptr<column> make_strings_column(
   const rmm::device_vector<char>& strings,
@@ -351,10 +336,9 @@ std::unique_ptr<column> make_strings_column(
  * @param null_count The number of null string entries. If equal to
  * `UNKNOWN_NULL_COUNT`, the null count will be computed dynamically on the
  * first invocation of `column::null_count()`
- * @param stream Optional stream for use with all memory allocation
- *               and device kernels
- * @param mr Optional resource to use for device memory
- *           allocation of the column's `null_mask` and children.
+ * @param stream CUDA stream used for device memory operations and kernel launches.
+ * @param mr Device memory resource used for allocation of the column's `null_mask` and children
+ * columns' device memory.
  */
 std::unique_ptr<column> make_strings_column(
   const std::vector<char>& strings,
@@ -380,10 +364,9 @@ std::unique_ptr<column> make_strings_column(
  * @param null_count The number of null string entries.
  * @param null_mask The bits specifying the null strings in device memory.
  *                  Arrow format for nulls is used for interpeting this bitmask.
- * @param stream Optional stream for use with all memory allocation
- *               and device kernels
- * @param mr Optional resource to use for device memory
- *           allocation of the column's `null_mask` and children.
+ * @param stream CUDA stream used for device memory operations and kernel launches.
+ * @param mr Device memory resource used for allocation of the column's `null_mask` and children
+ * columns' device memory.
  */
 std::unique_ptr<column> make_strings_column(
   size_type num_strings,
@@ -404,9 +387,8 @@ std::unique_ptr<column> make_strings_column(
  *
  * @param s The scalar to use for values in the column.
  * @param size The number of rows for the output column.
- * @param stream Optional stream for use with all memory allocation
- *               and device kernels
- * @param mr Optional resource to use for device memory.
+ * @param stream CUDA stream used for device memory operations and kernel launches.
+ * @param mr Device memory resource used to allocate the returned column's device memory.
  */
 std::unique_ptr<column> make_column_from_scalar(
   scalar const& s,

cpp/include/cudf/concatenate.hpp

@@ -35,8 +35,7 @@ namespace cudf {
  * Returns empty `device_buffer` if the column is not nullable
  *
  * @param views Vector of column views whose bitmask will to be concatenated
- * @param mr Optional, the memory resource that will be used for allocating
- * the device memory for the new device_buffer
+ * @param mr Device memory resource used for allocating the new device_buffer
  * @return rmm::device_buffer A `device_buffer` containing the bitmasks of all
  * the column views in the views vector
  */
@@ -52,7 +51,7 @@ rmm::device_buffer concatenate_masks(
  *
  * @param columns_to_concat The column views to be concatenated into a single
  * column
- * @param mr Optional The resource to use for all allocations
+ * @param mr Device memory resource used to allocate the returned column's device memory.
  * @return Unique pointer to a single table having all the rows from the
  * elements of `columns_to_concat` respectively in the same order.
  */
@@ -83,7 +82,7 @@ std::unique_ptr<column> concatenate(
  *
  * @param tables_to_concat The table views to be concatenated into a single
  * table
- * @param mr Optional The resource to use for all allocations
+ * @param mr Device memory resource used to allocate the returned table's device memory.
  * @return Unique pointer to a single table having all the rows from the
  * elements of `tables_to_concat` respectively in the same order.
  */