From ad6b5130905add6e597b7ff6ea8e9dbc4320efed Mon Sep 17 00:00:00 2001 From: David Wendt Date: Wed, 22 Nov 2023 12:02:10 -0500 Subject: [PATCH 1/4] Move creation of env.yaml outside local repo directory --- ci/build_docs.sh | 6 ++++-- ci/check_style.sh | 6 ++++-- ci/test_cpp_common.sh | 7 +++++-- ci/test_java.sh | 7 +++++-- ci/test_notebooks.sh | 7 +++++-- ci/test_python_common.sh | 7 +++++-- 6 files changed, 28 insertions(+), 12 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index c831311357c..7f2ced2e7a5 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -6,12 +6,14 @@ set -euo pipefail rapids-logger "Create test conda environment" . /opt/conda/etc/profile.d/conda.sh +ENV_YAML_DIR="$(mktemp -d)" + rapids-dependency-file-generator \ --output conda \ --file_key docs \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee ${ENV_YAML_DIR}/env.yaml -rapids-mamba-retry env create --force -f env.yaml -n docs +rapids-mamba-retry env create --force -f ${ENV_YAML_DIR}/env.yaml -n docs conda activate docs rapids-print-env diff --git a/ci/check_style.sh b/ci/check_style.sh index 9eed0029893..c38f5226a81 100755 --- a/ci/check_style.sh +++ b/ci/check_style.sh @@ -6,12 +6,14 @@ set -euo pipefail rapids-logger "Create checks conda environment" . /opt/conda/etc/profile.d/conda.sh +ENV_YAML_DIR="$(mktemp -d)" + rapids-dependency-file-generator \ --output conda \ --file_key checks \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee ${ENV_YAML_DIR}/env.yaml -rapids-mamba-retry env create --force -f env.yaml -n checks +rapids-mamba-retry env create --force -f ${ENV_YAML_DIR}/env.yaml -n checks conda activate checks FORMAT_FILE_URL=https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-24.02/cmake-format-rapids-cmake.json diff --git a/ci/test_cpp_common.sh b/ci/test_cpp_common.sh index c7c095dc4df..6b7061f0150 100644 --- a/ci/test_cpp_common.sh +++ b/ci/test_cpp_common.sh @@ -6,12 +6,15 @@ set -euo pipefail . /opt/conda/etc/profile.d/conda.sh rapids-logger "Generate C++ testing dependencies" + +ENV_YAML_DIR="$(mktemp -d)" + rapids-dependency-file-generator \ --output conda \ --file_key test_cpp \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee ${ENV_YAML_DIR}/env.yaml -rapids-mamba-retry env create --force -f env.yaml -n test +rapids-mamba-retry env create --force -f ${ENV_YAML_DIR}/env.yaml -n test # Temporarily allow unbound variables for conda activation. set +u diff --git a/ci/test_java.sh b/ci/test_java.sh index e4df62501cc..458314bb73b 100755 --- a/ci/test_java.sh +++ b/ci/test_java.sh @@ -6,12 +6,15 @@ set -euo pipefail . /opt/conda/etc/profile.d/conda.sh rapids-logger "Generate Java testing dependencies" + +ENV_YAML_DIR="$(mktemp -d)" + rapids-dependency-file-generator \ --output conda \ --file_key test_java \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee ${ENV_YAML_DIR}/env.yaml -rapids-mamba-retry env create --force -f env.yaml -n test +rapids-mamba-retry env create --force -f ${ENV_YAML_DIR}/env.yaml -n test export CMAKE_GENERATOR=Ninja diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index 348428fdef1..d61283edc8b 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -6,12 +6,15 @@ set -euo pipefail . /opt/conda/etc/profile.d/conda.sh rapids-logger "Generate notebook testing dependencies" + +ENV_YAML_DIR="$(mktemp -d)" + rapids-dependency-file-generator \ --output conda \ --file_key test_notebooks \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee ${ENV_YAML_DIR}/env.yaml -rapids-mamba-retry env create --force -f env.yaml -n test +rapids-mamba-retry env create --force -f ${ENV_YAML_DIR}/env.yaml -n test # Temporarily allow unbound variables for conda activation. set +u diff --git a/ci/test_python_common.sh b/ci/test_python_common.sh index 0e922c105dd..5de41316625 100755 --- a/ci/test_python_common.sh +++ b/ci/test_python_common.sh @@ -8,12 +8,15 @@ set -euo pipefail . /opt/conda/etc/profile.d/conda.sh rapids-logger "Generate Python testing dependencies" + +ENV_YAML_DIR="$(mktemp -d)" + rapids-dependency-file-generator \ --output conda \ --file_key test_python \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee ${ENV_YAML_DIR}/env.yaml -rapids-mamba-retry env create --force -f env.yaml -n test +rapids-mamba-retry env create --force -f ${ENV_YAML_DIR}/env.yaml -n test # Temporarily allow unbound variables for conda activation. set +u From 84e7babcde569ab5ba2a13a0f63a8403de2b1d53 Mon Sep 17 00:00:00 2001 From: David Wendt Date: Tue, 28 Nov 2023 19:51:09 -0500 Subject: [PATCH 2/4] replace PWD with temp-dir for test-results --- ci/test_cpp_common.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/test_cpp_common.sh b/ci/test_cpp_common.sh index 6b7061f0150..819ec0feeb0 100644 --- a/ci/test_cpp_common.sh +++ b/ci/test_cpp_common.sh @@ -22,7 +22,8 @@ conda activate test set -u CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ +RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} +RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" rapids-print-env From 1b9f4fb19a8a70fd0cbfd2f4025266ecab96e307 Mon Sep 17 00:00:00 2001 From: David Wendt Date: Fri, 1 Dec 2023 13:33:48 -0500 Subject: [PATCH 3/4] remove PWD from test_python_common.sh --- ci/test_python_common.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/test_python_common.sh b/ci/test_python_common.sh index 5de41316625..024323d0a95 100755 --- a/ci/test_python_common.sh +++ b/ci/test_python_common.sh @@ -27,8 +27,9 @@ rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) -RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} -RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} +RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} +RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ +RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${RESULTS_DIR}/coverage-results"} mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" rapids-print-env From 3a88e955a39d75040aaad359c5a935e6f31b09fc Mon Sep 17 00:00:00 2001 From: David Wendt Date: Thu, 7 Dec 2023 10:24:11 -0500 Subject: [PATCH 4/4] add double-quotes around dir var --- ci/build_docs.sh | 4 ++-- ci/check_style.sh | 4 ++-- ci/test_cpp_common.sh | 4 ++-- ci/test_java.sh | 4 ++-- ci/test_notebooks.sh | 4 ++-- ci/test_python_common.sh | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 7f2ced2e7a5..67d611340c5 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -11,9 +11,9 @@ ENV_YAML_DIR="$(mktemp -d)" rapids-dependency-file-generator \ --output conda \ --file_key docs \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee ${ENV_YAML_DIR}/env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" -rapids-mamba-retry env create --force -f ${ENV_YAML_DIR}/env.yaml -n docs +rapids-mamba-retry env create --force -f "${ENV_YAML_DIR}/env.yaml" -n docs conda activate docs rapids-print-env diff --git a/ci/check_style.sh b/ci/check_style.sh index c38f5226a81..088860d38df 100755 --- a/ci/check_style.sh +++ b/ci/check_style.sh @@ -11,9 +11,9 @@ ENV_YAML_DIR="$(mktemp -d)" rapids-dependency-file-generator \ --output conda \ --file_key checks \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee ${ENV_YAML_DIR}/env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" -rapids-mamba-retry env create --force -f ${ENV_YAML_DIR}/env.yaml -n checks +rapids-mamba-retry env create --force -f "${ENV_YAML_DIR}/env.yaml" -n checks conda activate checks FORMAT_FILE_URL=https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-24.02/cmake-format-rapids-cmake.json diff --git a/ci/test_cpp_common.sh b/ci/test_cpp_common.sh index 819ec0feeb0..37aba751a09 100644 --- a/ci/test_cpp_common.sh +++ b/ci/test_cpp_common.sh @@ -12,9 +12,9 @@ ENV_YAML_DIR="$(mktemp -d)" rapids-dependency-file-generator \ --output conda \ --file_key test_cpp \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee ${ENV_YAML_DIR}/env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" -rapids-mamba-retry env create --force -f ${ENV_YAML_DIR}/env.yaml -n test +rapids-mamba-retry env create --force -f "${ENV_YAML_DIR}/env.yaml" -n test # Temporarily allow unbound variables for conda activation. set +u diff --git a/ci/test_java.sh b/ci/test_java.sh index 458314bb73b..0863795162d 100755 --- a/ci/test_java.sh +++ b/ci/test_java.sh @@ -12,9 +12,9 @@ ENV_YAML_DIR="$(mktemp -d)" rapids-dependency-file-generator \ --output conda \ --file_key test_java \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee ${ENV_YAML_DIR}/env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" -rapids-mamba-retry env create --force -f ${ENV_YAML_DIR}/env.yaml -n test +rapids-mamba-retry env create --force -f "${ENV_YAML_DIR}/env.yaml" -n test export CMAKE_GENERATOR=Ninja diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index d61283edc8b..b746a18aed1 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -12,9 +12,9 @@ ENV_YAML_DIR="$(mktemp -d)" rapids-dependency-file-generator \ --output conda \ --file_key test_notebooks \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee ${ENV_YAML_DIR}/env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" -rapids-mamba-retry env create --force -f ${ENV_YAML_DIR}/env.yaml -n test +rapids-mamba-retry env create --force -f "${ENV_YAML_DIR}/env.yaml" -n test # Temporarily allow unbound variables for conda activation. set +u diff --git a/ci/test_python_common.sh b/ci/test_python_common.sh index 024323d0a95..1c330d47ac6 100755 --- a/ci/test_python_common.sh +++ b/ci/test_python_common.sh @@ -14,9 +14,9 @@ ENV_YAML_DIR="$(mktemp -d)" rapids-dependency-file-generator \ --output conda \ --file_key test_python \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee ${ENV_YAML_DIR}/env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" -rapids-mamba-retry env create --force -f ${ENV_YAML_DIR}/env.yaml -n test +rapids-mamba-retry env create --force -f "${ENV_YAML_DIR}/env.yaml" -n test # Temporarily allow unbound variables for conda activation. set +u