Add option to build faiss and treelite shared libs, inherit common de…

…pendencies from raft (rapidsai#4256) Depends on rapidsai/raft#345, and CI will be blocked until `raft` is removed from `libcumlprims` install export set. Authors: - Paul Taylor (https://github.com/trxcllnt) - Divye Gala (https://github.com/divyegala) - Ashwin Srinath (https://github.com/shwina) - Corey J. Nolet (https://github.com/cjnolet) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: rapidsai#4256
vimarsh6739 · Jan 23, 2022 · da07840 · da07840
1 parent f63fc06
commit da07840
Show file tree

Hide file tree

Showing 12 changed files with 169 additions and 247 deletions.
diff --git a/build.sh b/build.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.
+# Copyright (c) 2019-2022, NVIDIA CORPORATION.
 
 # cuml build script
 
@@ -23,30 +23,32 @@ VALIDFLAGS="-v -g -n --allgpuarch --singlegpu --nolibcumltest --nvtx --show_depr
 VALIDARGS="${VALIDTARGETS} ${VALIDFLAGS}"
 HELP="$0 [<target> ...] [<flag> ...]
  where <target> is:
-   clean            - remove all existing build artifacts and configuration (start over)
-   libcuml          - build the cuml C++ code only. Also builds the C-wrapper library
-                      around the C++ code.
-   cuml             - build the cuml Python package
-   cpp-mgtests      - build libcuml mnmg tests. Builds MPI communicator, adding MPI as dependency.
-   prims            - build the ml-prims tests
-   bench            - build the libcuml C++ benchmark
-   prims-bench      - build the ml-prims C++ benchmark
-   cppdocs          - build the C++ API doxygen documentation
-   pydocs           - build the general and Python API documentation
+   clean             - remove all existing build artifacts and configuration (start over)
+   libcuml           - build the cuml C++ code only. Also builds the C-wrapper library
+                       around the C++ code.
+   cuml              - build the cuml Python package
+   cpp-mgtests       - build libcuml mnmg tests. Builds MPI communicator, adding MPI as dependency.
+   prims             - build the ml-prims tests
+   bench             - build the libcuml C++ benchmark
+   prims-bench       - build the ml-prims C++ benchmark
+   cppdocs           - build the C++ API doxygen documentation
+   pydocs            - build the general and Python API documentation
  and <flag> is:
-   -v               - verbose build mode
-   -g               - build for debug
-   -n               - no install step
-   -h               - print this text
-   --allgpuarch     - build for all supported GPU architectures
-   --singlegpu      - Build libcuml and cuml without multigpu components
-   --nolibcumltest  - disable building libcuml C++ tests for a faster build
-   --nvtx           - Enable nvtx for profiling support
-   --show_depr_warn - show cmake deprecation warnings
-   --codecov        - Enable code coverage support by compiling with Cython linetracing
-                      and profiling enabled (WARNING: Impacts performance)
-   --ccache         - Use ccache to cache previous compilations
-   --nocloneraft    - CMake will clone RAFT even if it is in the environment, use this flag to disable that behavior
+   -v                - verbose build mode
+   -g                - build for debug
+   -n                - no install step
+   -h                - print this text
+   --allgpuarch      - build for all supported GPU architectures
+   --singlegpu       - Build libcuml and cuml without multigpu components
+   --nolibcumltest   - disable building libcuml C++ tests for a faster build
+   --nvtx            - Enable nvtx for profiling support
+   --show_depr_warn  - show cmake deprecation warnings
+   --codecov         - Enable code coverage support by compiling with Cython linetracing
+                       and profiling enabled (WARNING: Impacts performance)
+   --ccache          - Use ccache to cache previous compilations
+   --nocloneraft     - CMake will clone RAFT even if it is in the environment, use this flag to disable that behavior
+   --static-faiss    - Force CMake to use the FAISS static libs, cloning and building them if necessary
+   --static-treelite - Force CMake to use the Treelite static libs, cloning and building them if necessary
 
  default action (no args) is to build and install 'libcuml', 'cuml', and 'prims' targets only for the detected GPU arch
 
@@ -77,6 +79,7 @@ BUILD_CUML_STD_COMMS=ON
 BUILD_CUML_TESTS=ON
 BUILD_CUML_MG_TESTS=OFF
 BUILD_STATIC_FAISS=OFF
+BUILD_STATIC_TREELITE=OFF
 CMAKE_LOG_LEVEL=WARNING
 DISABLE_FORCE_CLONE_RAFT=OFF
 
@@ -194,6 +197,12 @@ while true; do
         --nocloneraft )
             DISABLE_FORCE_CLONE_RAFT=ON
             ;;
+        --static-faiss )
+            BUILD_STATIC_FAISS=ON
+            ;;
+        --static-treelite )
+            BUILD_STATIC_TREELITE=ON
+            ;;
         --)
             shift
             break
@@ -245,6 +254,8 @@ if completeBuild || hasArg libcuml || hasArg prims || hasArg bench || hasArg pri
           -DBUILD_CUML_TESTS=${BUILD_CUML_TESTS} \
           -DBUILD_CUML_MPI_COMMS=${BUILD_CUML_MG_TESTS} \
           -DBUILD_CUML_MG_TESTS=${BUILD_CUML_MG_TESTS} \
+          -DCUML_USE_FAISS_STATIC=${BUILD_STATIC_FAISS} \
+          -DCUML_USE_TREELITE_STATIC=${BUILD_STATIC_TREELITE} \
           -DDISABLE_FORCE_CLONE_RAFT=${DISABLE_FORCE_CLONE_RAFT} \
           -DNVTX=${NVTX} \
           -DUSE_CCACHE=${CCACHE} \

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -57,12 +57,13 @@ option(CUDA_ENABLE_KERNEL_INFO "Enable kernel resource usage info" OFF)
 option(CUDA_ENABLE_LINE_INFO "Enable lineinfo in nvcc" OFF)
 option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" ON)
 option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON)
-option(DISABLE_FORCE_CLONE_RAFT "By default, CPM will clone RAFT even if it's already in the environment. Set to disable that behavior." OFF)
 option(DISABLE_OPENMP "Disable OpenMP" OFF)
 option(ENABLE_CUMLPRIMS_MG "Enable algorithms that use libcumlprims_mg" ON)
 option(NVTX "Enable nvtx markers" OFF)
 option(SINGLEGPU "Disable all mnmg components and comms libraries" OFF)
 option(USE_CCACHE "Cache build artifacts with ccache" OFF)
+option(CUML_USE_FAISS_STATIC "Build and statically link the FAISS library for nearest neighbors search on GPU" OFF)
+option(CUML_USE_TREELITE_STATIC "Build and statically link the treelite library" OFF)
 
 set(CUML_CPP_ALGORITHMS "ALL" CACHE STRING "Experimental: Choose which algorithms are built into libcuml++.so. Only 'FIL' and 'ALL' are supported right now.")
 set_property(CACHE CUML_CPP_ALGORITHMS PROPERTY STRINGS "ALL" "FIL")
@@ -153,12 +154,12 @@ if(NOT BUILD_CUML_CPP_LIBRARY)
 endif()
 
 if(CUML_CPP_ALGORITHMS STREQUAL "ALL")
-  set(LINK_FAISS ON)
+  set(CUML_USE_RAFT_NN ON)
 elseif(CUML_CPP_ALGORITHMS STREQUAL "FIL")
   set(SINGLEGPU ON)
   set(BUILD_CUML_C_LIBRARY OFF)
   set(BUILD_CUML_EXAMPLES OFF)
-  set(LINK_FAISS OFF)
+  set(CUML_USE_RAFT_NN OFF)
 endif()
 
 # SingleGPU build disables cumlprims_mg and comms components
@@ -189,16 +190,9 @@ endif()
 # add third party dependencies using CPM
 rapids_cpm_init()
 
-include(cmake/thirdparty/get_thrust.cmake)
-include(cmake/thirdparty/get_rmm.cmake)
-
-if(LINK_FAISS)
-  include(cmake/thirdparty/get_faiss.cmake)
-endif()
-
+include(cmake/thirdparty/get_raft.cmake)
 include(cmake/thirdparty/get_treelite.cmake)
 include(cmake/thirdparty/get_gputreeshap.cmake)
-include(cmake/thirdparty/get_raft.cmake)
 
 if(NOT SINGLEGPU)
   include(cmake/thirdparty/get_nccl.cmake)
@@ -213,7 +207,7 @@ if(BUILD_CUML_TESTS OR BUILD_PRIMS_TESTS)
   include(cmake/thirdparty/get_gtest.cmake)
 endif()
 
-if(BUILD_CUML_BENCH)
+if(BUILD_CUML_BENCH OR BUILD_CUML_PRIMS_BENCH)
   include(cmake/thirdparty/get_gbench.cmake)
 endif()
 
@@ -388,22 +382,13 @@ if(BUILD_CUML_CPP_LIBRARY)
 
   target_link_libraries(${CUML_CPP_TARGET}
     PUBLIC
-      rmm::rmm
-      cuml::Thrust
       raft::raft
-      raft::raft_nn
-      raft::raft_distance
+      raft::nn
+      raft::distance
     PRIVATE
-      CUDA::cublas
       CUDA::cufft
-      CUDA::curand
-      CUDA::cusolver
-      CUDA::cudart
-      CUDA::cusparse
+      ${TREELITE_LIBS}
       GPUTreeShap::GPUTreeShap
-      $<$<BOOL:${LINK_FAISS}>:FAISS::FAISS>
-      $<IF:$<BOOL:${Treelite_ADDED}>,treelite::treelite_static,treelite::treelite>
-      $<IF:$<BOOL:${Treelite_ADDED}>,treelite::treelite_runtime_static,treelite::treelite_runtime>
       $<$<BOOL:${OpenMP_FOUND}>:OpenMP::OpenMP_CXX>
       $<$<OR:$<BOOL:${BUILD_CUML_STD_COMMS}>,$<BOOL:${BUILD_CUML_MPI_COMMS}>>:NCCL::NCCL>
       $<$<BOOL:${BUILD_CUML_MPI_COMMS}>:${MPI_CXX_LIBRARIES}>
@@ -448,8 +433,6 @@ if(BUILD_CUML_C_LIBRARY)
   target_link_libraries(${CUML_C_TARGET}
     PUBLIC
       ${CUML_CPP_TARGET}
-    PRIVATE
-      FAISS::FAISS
   )
 
   # ensure CUDA symbols aren't relocated to the middle of the debug build binaries
@@ -508,14 +491,34 @@ functions that share compatible APIs with other RAPIDS projects.
 
 ]=])
 
-set(code_string
+set(code_string )
+
+if (TARGET treelite::treelite)
+    string(APPEND code_string
 [=[
-thrust_create_target(cuml::Thrust FROM_OPTIONS)
+if (TARGET treelite::treelite AND (NOT TARGET treelite))
+    add_library(treelite ALIAS treelite::treelite)
+endif()
+if (TARGET treelite::treelite_runtime AND (NOT TARGET treelite_runtime))
+    add_library(treelite_runtime ALIAS treelite::treelite_runtime)
+endif()
+]=])
+else()
+    string(APPEND code_string
+[=[
+if (TARGET treelite::treelite_static AND (NOT TARGET treelite_static))
+    add_library(treelite_static ALIAS treelite::treelite_static)
+endif()
+if (TARGET treelite::treelite_runtime_static AND (NOT TARGET treelite_runtime_static))
+    add_library(treelite_runtime_static ALIAS treelite::treelite_runtime_static)
+endif()
 ]=])
 
- rapids_export(INSTALL cuml
+endif()
+
+rapids_export(INSTALL cuml
     EXPORT_SET cuml-exports
-    GLOBAL_TARGETS cuml
+    GLOBAL_TARGETS ${CUML_C_TARGET} ${CUML_CPP_TARGET}
     NAMESPACE cuml::
     DOCUMENTATION doc_string
     FINAL_CODE_BLOCK code_string
@@ -526,7 +529,7 @@ thrust_create_target(cuml::Thrust FROM_OPTIONS)
 
 rapids_export(BUILD cuml
     EXPORT_SET cuml-exports
-    GLOBAL_TARGETS cuml
+    GLOBAL_TARGETS ${CUML_C_TARGET} ${CUML_CPP_TARGET}
     NAMESPACE cuml::
     DOCUMENTATION doc_string
     FINAL_CODE_BLOCK code_string

diff --git a/cpp/bench/CMakeLists.txt b/cpp/bench/CMakeLists.txt
@@ -1,5 +1,5 @@
 #=============================================================================
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.
+# Copyright (c) 2019-2022, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -44,11 +44,10 @@ if(BUILD_CUML_BENCH)
     PUBLIC
       cuml::${CUML_CPP_TARGET}
       benchmark::benchmark
+      ${TREELITE_LIBS}
       raft::raft
-      raft::raft_nn
-      raft::raft_distance
-      $<IF:$<BOOL:${Treelite_ADDED}>,treelite::treelite_static,treelite::treelite>
-      $<IF:$<BOOL:${Treelite_ADDED}>,treelite::treelite_runtime_static,treelite::treelite_runtime>
+      raft::nn
+      raft::distance
   )
 
   target_include_directories(${CUML_CPP_BENCH_TARGET}
@@ -85,14 +84,12 @@ if(BUILD_CUML_PRIMS_BENCH)
 
   target_link_libraries(${PRIMS_BENCH_TARGET}
     PUBLIC
-      cuml
-      CUDA::cublas
+      cuml::${CUML_CPP_TARGET}
       benchmark::benchmark
+      ${TREELITE_LIBS}
       raft::raft
-      raft::raft_nn
-      raft::raft_distance
-      $<IF:$<BOOL:${Treelite_ADDED}>,treelite::treelite_static,treelite::treelite>
-      $<IF:$<BOOL:${Treelite_ADDED}>,treelite::treelite_runtime_static,treelite::treelite_runtime>
+      raft::nn
+      raft::distance
   )
 
   target_include_directories(${PRIMS_BENCH_TARGET}

diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake
diff --git a/cpp/cmake/thirdparty/get_gtest.cmake b/cpp/cmake/thirdparty/get_gtest.cmake
@@ -1,5 +1,5 @@
 #=============================================================================
-# Copyright (c) 2021, NVIDIA CORPORATION.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.