Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added optional flag for building Arrow with S3 filesystem support #8531

Merged
merged 5 commits into from
Jun 16, 2021
Merged

Added optional flag for building Arrow with S3 filesystem support #8531

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1d69192
Added optional flag for building Arrow with S3 filesystem support
jdye64 Jun 16, 2021
986ef73
move variable location to be more inline with existing variables
jdye64 Jun 16, 2021
1c8ad97
Build Arrow S3 tests conditionally
jdye64 Jun 16, 2021
f96ee81
Fixing macro syntax issue
jdye64 Jun 16, 2021
20b2a4f
checkstyle
jdye64 Jun 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cpp/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ option(BUILD_BENCHMARKS "Configure CMake to build (google) benchmarks" OFF)
option(BUILD_SHARED_LIBS "Build cuDF shared libraries" ON)
option(JITIFY_USE_CACHE "Use a file cache for JIT compiled kernels" ON)
option(CUDF_USE_ARROW_STATIC "Build and statically link Arrow libraries" OFF)
option(CUDF_ENABLE_ARROW_S3 "Build/Enable AWS S3 Arrow filesystem support" ON)
option(PER_THREAD_DEFAULT_STREAM "Build with per-thread default stream" OFF)
option(DISABLE_DEPRECATION_WARNING "Disable warnings generated from deprecated declarations." OFF)
# Option to enable line info in CUDA device compilation to allow introspection when profiling / memchecking
Expand All @@ -57,6 +58,7 @@ message(VERBOSE "CUDF: Configure CMake to build (google) benchmarks: ${BUILD_BEN
message(VERBOSE "CUDF: Build cuDF shared libraries: ${BUILD_SHARED_LIBS}")
message(VERBOSE "CUDF: Use a file cache for JIT compiled kernels: ${JITIFY_USE_CACHE}")
message(VERBOSE "CUDF: Build and statically link Arrow libraries: ${CUDF_USE_ARROW_STATIC}")
message(VERBOSE "CUDF: Build and enable S3 filesystem support for Arrow: ${CUDF_ENABLE_ARROW_S3}")
message(VERBOSE "CUDF: Build with per-thread default stream: ${PER_THREAD_DEFAULT_STREAM}")
message(VERBOSE "CUDF: Disable warnings generated from deprecated declarations: ${DISABLE_DEPRECATION_WARNING}")
message(VERBOSE "CUDF: Enable the -lineinfo option for nvcc (useful for cuda-memcheck / profiler: ${CUDA_ENABLE_LINEINFO}")
Expand Down
3 changes: 3 additions & 0 deletions cpp/cmake/cudf-build-config.cmake.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ else()
if(NOT DEFINED CUDF_USE_ARROW_STATIC)
set(CUDF_USE_ARROW_STATIC OFF)
endif()
if (NOT DEFINED CUDF_ENABLE_ARROW_S3)
set(CUDF_ENABLE_ARROW_S3 OFF)
endif()
include(@CUDF_SOURCE_DIR@/cmake/thirdparty/CUDF_GetArrow.cmake)
endif()

Expand Down
11 changes: 8 additions & 3 deletions cpp/cmake/thirdparty/CUDF_GetArrow.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,11 @@
# limitations under the License.
#=============================================================================

function(find_and_configure_arrow VERSION BUILD_STATIC)
function(find_and_configure_arrow VERSION BUILD_STATIC ENABLE_S3)

set(ARROW_BUILD_SHARED ON)
set(ARROW_BUILD_STATIC OFF)
set(ARROW_BUILD_S3 OFF)
jdye64 marked this conversation as resolved.
Show resolved Hide resolved
set(CPMAddOrFindPackage CPMFindPackage)

if(NOT ARROW_ARMV8_ARCH)
Expand All @@ -35,6 +36,10 @@ function(find_and_configure_arrow VERSION BUILD_STATIC)
set(CPMAddOrFindPackage CPMAddPackage)
endif()

if(ENABLE_S3)
set(ARROW_BUILD_S3 ON)
endif()

cmake_language(CALL ${CPMAddOrFindPackage}
NAME Arrow
VERSION ${VERSION}
Expand All @@ -50,7 +55,7 @@ function(find_and_configure_arrow VERSION BUILD_STATIC)
"ARROW_WITH_BACKTRACE ON"
"ARROW_CXXFLAGS -w"
"ARROW_JEMALLOC OFF"
"ARROW_S3 ON"
"ARROW_S3 ${ARROW_BUILD_S3}"
# Arrow modifies CMake's GLOBAL RULE_LAUNCH_COMPILE unless this is off
"ARROW_USE_CCACHE OFF"
"ARROW_ARMV8_ARCH ${ARROW_ARMV8_ARCH}"
Expand Down Expand Up @@ -124,4 +129,4 @@ endfunction()

set(CUDF_VERSION_Arrow 1.0.1)

find_and_configure_arrow(${CUDF_VERSION_Arrow} ${CUDF_USE_ARROW_STATIC})
find_and_configure_arrow(${CUDF_VERSION_Arrow} ${CUDF_USE_ARROW_STATIC} ${CUDF_ENABLE_ARROW_S3})
6 changes: 5 additions & 1 deletion cpp/tests/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,11 @@ ConfigureTest(CSV_TEST io/csv_test.cpp)
ConfigureTest(ORC_TEST io/orc_test.cpp)
ConfigureTest(PARQUET_TEST io/parquet_test.cpp)
ConfigureTest(JSON_TEST io/json_test.cpp)
ConfigureTest(ARROW_IO_SOURCE_TEST io/arrow_io_source_test.cpp)

# ARROW_IO_TEST requires Arrow being built with S3 support
if(CUDF_ENABLE_ARROW_S3)
ConfigureTest(ARROW_IO_SOURCE_TEST io/arrow_io_source_test.cpp)
endif()

###################################################################################################
# - sort tests ------------------------------------------------------------------------------------
Expand Down
2 changes: 1 addition & 1 deletion java/ci/build-in-docker.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ export PATH=/usr/local/cmake-3.19.0-Linux-x86_64/bin:$PATH
rm -rf "$WORKSPACE/cpp/build"
mkdir -p "$WORKSPACE/cpp/build"
cd "$WORKSPACE/cpp/build"
cmake .. -DUSE_NVTX=$ENABLE_NVTX -DCUDF_USE_ARROW_STATIC=ON -DBUILD_TESTS=$SKIP_CPP_TESTS -DPER_THREAD_DEFAULT_STREAM=$ENABLE_PTDS -DRMM_LOGGING_LEVEL=$RMM_LOGGING_LEVEL
cmake .. -DUSE_NVTX=$ENABLE_NVTX -DCUDF_USE_ARROW_STATIC=ON -DCUDF_ENABLE_ARROW_S3=OFF -DBUILD_TESTS=$SKIP_CPP_TESTS -DPER_THREAD_DEFAULT_STREAM=$ENABLE_PTDS -DRMM_LOGGING_LEVEL=$RMM_LOGGING_LEVEL

make -j$PARALLEL_LEVEL
make install DESTDIR=$INSTALL_PREFIX
Expand Down