Skip to content

Commit

Permalink
[internal] Increase engine build parallelism, and include arch in fin…
Browse files Browse the repository at this point in the history
…gerprint. (#12201)

In two commits:
* Include the `arch` in the engine fingerprint to rebuild when it changes (i.e. on M1 with rosetta)
* Build both Python extension modules in parallel
  • Loading branch information
stuhood authored Jun 14, 2021
1 parent 982de2a commit 1ea924e
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 15 deletions.
25 changes: 11 additions & 14 deletions build-support/bin/rust/bootstrap_code.sh
Original file line number Diff line number Diff line change
Expand Up @@ -37,15 +37,14 @@ readonly NATIVE_ENGINE_RESOURCE_PYO3="${REPO_ROOT}/src/python/pants/engine/inter
readonly NATIVE_ENGINE_RESOURCE_METADATA="${NATIVE_ENGINE_RESOURCE}.metadata"

function _build_native_code() {
# NB: See Cargo.toml with regard to the `extension-module` feature.
"${REPO_ROOT}/cargo" build --features=extension-module ${MODE_FLAG} -p engine || die
echo "${NATIVE_ROOT}/target/${MODE}/libengine.${LIB_EXTENSION}"
}

function _build_native_code_pyo3() {
# NB: See engine_pyo3/Cargo.toml with regard to the `extension-module` feature.
"${REPO_ROOT}/cargo" build --features=engine_pyo3/extension-module ${MODE_FLAG} -p engine_pyo3 || die
echo "${NATIVE_ROOT}/target/${MODE}/libengine_pyo3.${LIB_EXTENSION}"
banner "Building native engine..."
# NB: See Cargo.toml with regard to the `extension-module` features.
"${REPO_ROOT}/cargo" build \
--features=extension-module \
--features=engine_pyo3/extension-module \
${MODE_FLAG} \
-p engine \
-p engine_pyo3 || die
}

function bootstrap_native_code() {
Expand All @@ -68,14 +67,12 @@ function bootstrap_native_code() {
engine_version_in_metadata="$(sed -n 's/^engine_version: //p' "${NATIVE_ENGINE_RESOURCE_METADATA}")"
fi
if [[ ! -f "${NATIVE_ENGINE_RESOURCE}" || ! -f "${NATIVE_ENGINE_RESOURCE_PYO3}" || "${engine_version_calculated}" != "${engine_version_in_metadata}" ]]; then
banner "Building native engine..."
banner "Building native_engine.so"
local -r native_binary="$(_build_native_code)" || die
banner "Building native_engine_pyo3.so"
local -r native_binary_pyo3="$(_build_native_code_pyo3)" || die
_build_native_code || die

# If bootstrapping the native engine fails, don't attempt to run pants
# afterwards.
local -r native_binary="${NATIVE_ROOT}/target/${MODE}/libengine.${LIB_EXTENSION}"
local -r native_binary_pyo3="${NATIVE_ROOT}/target/${MODE}/libengine_pyo3.${LIB_EXTENSION}"
if [[ ! -f "${native_binary}" ]]; then
die "Failed to build native engine, file missing at ${native_binary}."
fi
Expand Down
2 changes: 1 addition & 1 deletion build-support/bin/rust/calculate_engine_hash.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ function calculate_current_hash() {
(
echo "${MODE_FLAG}"
echo "${RUST_TOOLCHAIN_CONTENTS}"
uname
uname -mps
python --version 2>&1
git ls-files --cached --others --exclude-standard \
"${NATIVE_ROOT}" \
Expand Down

0 comments on commit 1ea924e

Please sign in to comment.