Consolidate C++ conda recipes and add libcudf-tests package (NVID…

…IA#10326)

This PR includes the following changes:

- Adds a `libcudf-tests` package to the `libcudf` recipe
  - This is a prerequisite for removing "Project Flash" from our build/CI scripts
  - The `libcudf-tests` package was added as an additional output to the existing `libcudf` recipe (which was renamed to `libcudf-split`)
- Consolidates remaining C++ recipes into `libcudf-split` recipe
  - This gets rid of a lot of duplicate code between the recipes and reduces the number of times we have to call `conda build` in our CI scripts

The result of consolidating the C++ recipes into a single `libcudf-split` recipe is that one top-level build occurs, which builds all of the C++ components (in `conda/recipes/libcudf/build.sh`) but does _not_ install them. The installation for each package occurs in the corresponding `conda/recipes/libcudf/install_libcudf.sh`, `conda/recipes/libcudf/install_libcudf_kafka.sh`, etc. scripts.

There are some implications to these changes. Namely that **any top-level `host` requirement which includes a `run_exports` value (i.e. shared libraries) must now be manually specified in the `run` dependencies of the corresponding `outputs` package.** To help keep things DRY, dependency version specs can be specified in `conda/recipes/librmm/conda_build_config.yaml`. The exception here is the version spec used for `cudatoolkit` since that comes from an environment variable in the CI process.

Authors:
  - AJ Schmidt (https://github.com/ajschmidt8)

Approvers:
  - Vyas Ramasubramani (https://github.com/vyasr)
  - Jordan Jacobelli (https://github.com/Ethyling)
  - https://github.com/jakirkham
  - Jason Lowe (https://github.com/jlowe)

URL: rapidsai/cudf#10326

ci/checks/headers_test.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2020, NVIDIA CORPORATION.
+# Copyright (c) 2020-2022, NVIDIA CORPORATION.
 #####################################
 # conda existence test for headers  #
 #####################################
@@ -10,7 +10,7 @@ DIRNAMES="cudf cudf_test"
 
 # existence tests for lib${LIBNAME}
 for DIRNAME in ${DIRNAMES[@]}; do
-    HEADERS=`cd cpp && find include/${DIRNAME}/ -type f \( -iname "*.h" -o  -iname "*.hpp" \) -printf "    - test -f \\\$PREFIX/%p\n" | sort`
+    HEADERS=`cd cpp && find include/${DIRNAME}/ -type f \( -iname "*.h" -o  -iname "*.hpp" \) -printf "        - test -f \\\$PREFIX/%p\n" | sort`
     META_TESTS=`grep -E "test -f .*/include/${DIRNAME}/.*\.h(pp)?" conda/recipes/lib${LIBNAME}/meta.yaml | sort`
     HEADER_DIFF=`diff <(echo "$HEADERS") <(echo "$META_TESTS")`
     LIB_RETVAL=$?

ci/cpu/build.sh

@@ -6,7 +6,10 @@
 set -e
 
 # Set path and build parallel level
-export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
+# FIXME: PATH variable shouldn't be necessary.
+# This should be removed once we either stop using the `remote-docker-plugin`
+# or the following issue is addressed: https://github.com/gpuopenanalytics/remote-docker-plugin/issues/47
+export PATH=/usr/local/gcc9/bin:/opt/conda/bin:/usr/local/cuda/bin:$PATH
 export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
 
 # Set home to the job's workspace
@@ -31,10 +34,6 @@ if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then
   export VERSION_SUFFIX=`date +%y%m%d`
 fi
 
-export CMAKE_CUDA_COMPILER_LAUNCHER="sccache"
-export CMAKE_CXX_COMPILER_LAUNCHER="sccache"
-export CMAKE_C_COMPILER_LAUNCHER="sccache"
-
 ################################################################################
 # SETUP - Check environment
 ################################################################################
@@ -94,16 +93,6 @@ if [ "$BUILD_LIBCUDF" == '1' ]; then
       cp "$LIBCUDF_BUILD_DIR/ninja_log.html" "$WORKSPACE/build-metrics/BuildMetrics.html"
       cp "$LIBCUDF_BUILD_DIR/ninja.log" "$WORKSPACE/build-metrics/ninja.log"
   fi
-
-  gpuci_logger "Build conda pkg for libcudf_kafka"
-  gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcudf_kafka $CONDA_BUILD_ARGS
-  mkdir -p ${CONDA_BLD_DIR}/libcudf_kafka/work
-  cp -r ${CONDA_BLD_DIR}/work/* ${CONDA_BLD_DIR}/libcudf_kafka/work
-
-  gpuci_logger "Building libcudf examples"
-  gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcudf_example $CONDA_BUILD_ARGS
-  mkdir -p ${CONDA_BLD_DIR}/libcudf_example/work
-  cp -r ${CONDA_BLD_DIR}/work/* ${CONDA_BLD_DIR}/libcudf_example/work
 fi
 
 if [ "$BUILD_CUDF" == '1' ]; then

ci/cpu/prebuild.sh

@@ -1,13 +1,12 @@
 #!/usr/bin/env bash
 
-# Copyright (c) 2020, NVIDIA CORPORATION.
+# Copyright (c) 2020-2022, NVIDIA CORPORATION.
 set -e
 
 #Always upload cudf packages
 export UPLOAD_CUDF=1
 export UPLOAD_LIBCUDF=1
 export UPLOAD_CUDF_KAFKA=1
-export UPLOAD_LIBCUDF_KAFKA=1
 
 if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
     #If project flash is not activate, always build both

ci/cpu/upload.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-#
+# Copyright (c) 2018-2022, NVIDIA CORPORATION.
 # Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh
 
 set -e
@@ -29,45 +29,31 @@ fi
 
 gpuci_logger "Starting conda uploads"
 if [[ "$BUILD_LIBCUDF" == "1" && "$UPLOAD_LIBCUDF" == "1" ]]; then
-  export LIBCUDF_FILE=$(conda build --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/libcudf --output)
-  test -e ${LIBCUDF_FILE}
-  echo "Upload libcudf"
-  echo ${LIBCUDF_FILE}
-  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBCUDF_FILE} --no-progress
+  export LIBCUDF_FILES=$(conda build --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/libcudf --output)
+  LIBCUDF_FILES=$(echo "$LIBCUDF_FILES" | sed 's/.*libcudf-example.*//') # skip libcudf-example pkg upload
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing --no-progress $LIBCUDF_FILES
 fi
 
 if [[ "$BUILD_CUDF" == "1" && "$UPLOAD_CUDF" == "1" ]]; then
   export CUDF_FILE=$(conda build --croot "${CONDA_BLD_DIR}" conda/recipes/cudf --python=$PYTHON --output)
   test -e ${CUDF_FILE}
-  echo "Upload cudf"
-  echo ${CUDF_FILE}
+  echo "Upload cudf: ${CUDF_FILE}"
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUDF_FILE} --no-progress
 
   export DASK_CUDF_FILE=$(conda build --croot "${CONDA_BLD_DIR}" conda/recipes/dask-cudf --python=$PYTHON --output)
   test -e ${DASK_CUDF_FILE}
-  echo "Upload dask-cudf"
-  echo ${DASK_CUDF_FILE}
+  echo "Upload dask-cudf: ${DASK_CUDF_FILE}"
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${DASK_CUDF_FILE} --no-progress
 
   export CUSTREAMZ_FILE=$(conda build --croot "${CONDA_BLD_DIR}" conda/recipes/custreamz --python=$PYTHON --output)
   test -e ${CUSTREAMZ_FILE}
-  echo "Upload custreamz"
-  echo ${CUSTREAMZ_FILE}
+  echo "Upload custreamz: ${CUSTREAMZ_FILE}"
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUSTREAMZ_FILE} --no-progress
 fi
 
-if [[ "$BUILD_LIBCUDF" == "1" && "$UPLOAD_LIBCUDF_KAFKA" == "1" ]]; then
-  export LIBCUDF_KAFKA_FILE=$(conda build --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/libcudf_kafka --output)
-  test -e ${LIBCUDF_KAFKA_FILE}
-  echo "Upload libcudf_kafka"
-  echo ${LIBCUDF_KAFKA_FILE}
-  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBCUDF_KAFKA_FILE} --no-progress
-fi
-
 if [[ "$BUILD_CUDF" == "1" && "$UPLOAD_CUDF_KAFKA" == "1" ]]; then
   export CUDF_KAFKA_FILE=$(conda build --croot "${CONDA_BLD_DIR}" conda/recipes/cudf_kafka --python=$PYTHON --output)
   test -e ${CUDF_KAFKA_FILE}
-  echo "Upload cudf_kafka"
-  echo ${CUDF_KAFKA_FILE}
+  echo "Upload cudf_kafka: ${CUDF_KAFKA_FILE}"
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUDF_KAFKA_FILE} --no-progress
 fi

ci/gpu/build.sh

@@ -36,10 +36,6 @@ export INSTALL_DASK_MAIN=1
 # ucx-py version
 export UCX_PY_VERSION='0.26.*'
 
-export CMAKE_CUDA_COMPILER_LAUNCHER="sccache"
-export CMAKE_CXX_COMPILER_LAUNCHER="sccache"
-export CMAKE_C_COMPILER_LAUNCHER="sccache"
-
 ################################################################################
 # TRAP - Setup trap for removing jitify cache
 ################################################################################
@@ -166,8 +162,6 @@ if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
     fi
 else
     #Project Flash
-    export LIB_BUILD_DIR="$WORKSPACE/ci/artifacts/cudf/cpu/libcudf_work/cpp/build"
-    export LD_LIBRARY_PATH="$LIB_BUILD_DIR:$CONDA_PREFIX/lib:$LD_LIBRARY_PATH"
 
     if hasArg --skip-tests; then
         gpuci_logger "Skipping Tests"
@@ -177,17 +171,16 @@ else
     gpuci_logger "Check GPU usage"
     nvidia-smi
 
-    set -x
-    cd $LIB_BUILD_DIR
+    gpuci_mamba_retry install -y -c ${CONDA_ARTIFACT_PATH} libcudf libcudf_kafka libcudf-tests
 
     gpuci_logger "GoogleTests"
-
-    for gt in gtests/* ; do
-        test_name=$(basename ${gt})
+    # Run libcudf and libcudf_kafka gtests from libcudf-tests package
+    for gt in "$CONDA_PREFIX/bin/gtests/libcudf"*/* ; do
         echo "Running GoogleTest $test_name"
         ${gt} --gtest_output=xml:"$WORKSPACE/test-results/"
     done
 
+    export LIB_BUILD_DIR="$WORKSPACE/ci/artifacts/cudf/cpu/libcudf_work/cpp/build"
     # Copy libcudf build time results
     echo "Checking for build time log $LIB_BUILD_DIR/ninja_log.xml"
     if [[ -f "$LIB_BUILD_DIR/ninja_log.xml" ]]; then
@@ -204,7 +197,7 @@ else
             export GTEST_CUDF_RMM_MODE=cuda
             COMPUTE_SANITIZER_CMD="compute-sanitizer --tool memcheck"
             mkdir -p "$WORKSPACE/test-results/"
-            for gt in gtests/*; do
+            for gt in "$CONDA_PREFIX/bin/gtests/libcudf"*/* ; do
                 test_name=$(basename ${gt})
                 if [[ "$test_name" == "ERROR_TEST" ]]; then
                   continue
@@ -217,16 +210,6 @@ else
         fi
     fi
 
-    CUDF_CONDA_FILE=`find ${CONDA_ARTIFACT_PATH} -name "libcudf-*.tar.bz2"`
-    CUDF_CONDA_FILE=`basename "$CUDF_CONDA_FILE" .tar.bz2` #get filename without extension
-    CUDF_CONDA_FILE=${CUDF_CONDA_FILE//-/=} #convert to conda install
-    KAFKA_CONDA_FILE=`find ${CONDA_ARTIFACT_PATH} -name "libcudf_kafka-*.tar.bz2"`
-    KAFKA_CONDA_FILE=`basename "$KAFKA_CONDA_FILE" .tar.bz2` #get filename without extension
-    KAFKA_CONDA_FILE=${KAFKA_CONDA_FILE//-/=} #convert to conda install
-
-    gpuci_logger "Installing $CUDF_CONDA_FILE & $KAFKA_CONDA_FILE"
-    gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} "$CUDF_CONDA_FILE" "$KAFKA_CONDA_FILE"
-
     install_dask
 
     gpuci_logger "Build python libs from source"

ci/gpu/java.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2018-2020, NVIDIA CORPORATION.
+# Copyright (c) 2018-2022, NVIDIA CORPORATION.
 ##############################################
 # cuDF GPU build and test script for CI      #
 ##############################################
@@ -122,19 +122,8 @@ function install_dask {
 # INSTALL - Install libcudf artifacts
 ################################################################################
 
-export LIB_BUILD_DIR="$WORKSPACE/ci/artifacts/cudf/cpu/libcudf_work/cpp/build"
-export CUDF_ROOT=${LIB_BUILD_DIR}
-export LD_LIBRARY_PATH="$LIB_BUILD_DIR:$CONDA_PREFIX/lib:$LD_LIBRARY_PATH"
-
-CUDF_CONDA_FILE=`find ${CONDA_ARTIFACT_PATH} -name "libcudf-*.tar.bz2"`
-CUDF_CONDA_FILE=`basename "$CUDF_CONDA_FILE" .tar.bz2` #get filename without extension
-CUDF_CONDA_FILE=${CUDF_CONDA_FILE//-/=} #convert to conda install
-KAFKA_CONDA_FILE=`find ${CONDA_ARTIFACT_PATH} -name "libcudf_kafka-*.tar.bz2"`
-KAFKA_CONDA_FILE=`basename "$KAFKA_CONDA_FILE" .tar.bz2` #get filename without extension
-KAFKA_CONDA_FILE=${KAFKA_CONDA_FILE//-/=} #convert to conda install
-
-gpuci_logger "Installing $CUDF_CONDA_FILE & $KAFKA_CONDA_FILE"
-gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} "$CUDF_CONDA_FILE" "$KAFKA_CONDA_FILE"
+gpuci_logger "Installing libcudf & libcudf_kafka"
+gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} libcudf libcudf_kafka
 
 install_dask
 

conda/recipes/libcudf/build.sh

@@ -1,8 +1,5 @@
-# Copyright (c) 2018-2019, NVIDIA CORPORATION.
+#!/bin/bash
+# Copyright (c) 2018-2022, NVIDIA CORPORATION.
 
-if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
-    # This assumes the script is executed from the root of the repo directory
-    ./build.sh -v libcudf --allgpuarch --cmake-args=\"-DCMAKE_INSTALL_LIBDIR=lib\"
-else
-    ./build.sh -v libcudf tests --allgpuarch --build_metrics --incl_cache_stats --cmake-args=\"-DCMAKE_INSTALL_LIBDIR=lib\"
-fi
+export cudf_ROOT="$(realpath ./cpp/build)"
+./build.sh -n -v libcudf libcudf_kafka benchmarks tests --cmake-args=\"-DCMAKE_INSTALL_LIBDIR=lib\"

conda/recipes/libcudf/conda_build_config.yaml

@@ -0,0 +1,14 @@
+cmake_version:
+  - ">=3.20.1"
+
+gtest_version:
+  - "=1.10.0"
+
+arrow_cpp_version:
+  - ">=6.0.1,<6.0.2.0a0"
+
+dlpack_version:
+  - ">=0.5,<0.6.0a0"
+
+librdkafka_version:
+  - ">=1.7.0,<1.8.0a0"

conda/recipes/libcudf/install_libcudf.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# Copyright (c) 2018-2022, NVIDIA CORPORATION.
+
+cmake --install cpp/build

conda/recipes/libcudf/install_libcudf_example.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# Copyright (c) 2018-2022, NVIDIA CORPORATION.
+
+./cpp/examples/build.sh

conda/recipes/libcudf/install_libcudf_kafka.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# Copyright (c) 2018-2022, NVIDIA CORPORATION.
+
+cmake --install cpp/libcudf_kafka/build

conda/recipes/libcudf/install_libcudf_tests.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# Copyright (c) 2018-2022, NVIDIA CORPORATION.
+
+cmake --install cpp/build --component testing
+cmake --install cpp/libcudf_kafka/build --component testing