Get rid of engine_cffi crate, and inline into the engine crate.

[ci skip-jvm-tests]

build-support/bin/native/bootstrap_code.sh

@@ -43,11 +43,12 @@ function _build_native_code() {
     # Change into $REPO_ROOT in a subshell. This is necessary so that we're in a child of a
     # directory containing the rust-toolchain file, so that rustup knows which version of rust we
     # should be using.
+    # NB: See Cargo.toml with regard to the `extension-module` feature.
     cd "${REPO_ROOT}"
-    "${REPO_ROOT}/build-support/bin/native/cargo" build ${MODE_FLAG} \
-      --manifest-path "${NATIVE_ROOT}/Cargo.toml" -p engine_cffi
+    "${REPO_ROOT}/build-support/bin/native/cargo" build --features=extension-module ${MODE_FLAG} \
+      --manifest-path "${NATIVE_ROOT}/Cargo.toml" -p engine
   ) || die
-  echo "${NATIVE_ROOT}/target/${MODE}/libengine_cffi.${LIB_EXTENSION}"
+  echo "${NATIVE_ROOT}/target/${MODE}/libengine.${LIB_EXTENSION}"
 }
 
 function bootstrap_native_code() {

build-support/bin/native/cargo.sh

@@ -7,6 +7,8 @@ set -e
 REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../../.. && pwd -P)
 
 export PY=${PY:-python3}
+# Consumed by the cpython crate.
+export PYTHON_SYS_EXECUTABLE="${PY}"
 
 # Exports:
 # + CARGO_HOME: The CARGO_HOME of the Pants-controlled rust toolchain.
@@ -15,8 +17,17 @@ export PY=${PY:-python3}
 # shellcheck source=build-support/bin/native/bootstrap_rust.sh
 source "${REPO_ROOT}/build-support/bin/native/bootstrap_rust.sh"
 
+# Exposes:
+# + activate_pants_venv: Activate a virtualenv for pants requirements, creating it if needed.
+#
+# This is necessary for any `cpython`-dependent crates, which need a python interpeter on the PATH.
+# shellcheck source=build-support/pants_venv
+source "${REPO_ROOT}/build-support/pants_venv"
+
 bootstrap_rust >&2
 
+activate_pants_venv
+
 download_binary="${REPO_ROOT}/build-support/bin/download_binary.sh"
 
 # The following is needed by grpcio-sys and we have no better way to hook its build.rs than this;

build-support/pants_venv

@@ -58,6 +58,13 @@ MESSAGE
 }
 
 function activate_pants_venv() {
+  # Allow for multiple calls to `activate_pants_venv`.
+  if [ -n "$VENV_ACTIVATED_FOR_PANTS" ]; then
+    return
+  else
+    export VENV_ACTIVATED_FOR_PANTS="true"
+  fi
+
   fingerprint=""
   for req in "${REQUIREMENTS[@]}"; do
     fingerprint="${fingerprint}$(fingerprint_data < "${req}")"

src/rust/engine/.gitignore

@@ -3,8 +3,6 @@
 /*/**/Cargo.lock
 !fs/fs_util/Cargo.lock
 !process_executor/Cargo.lock
-engine_cffi/logs/
-engine_cffi/src/cffi/
 # These modules contain purely generated code.
 process_execution/bazel_protos/src/gen/
 process_execution/bazel_protos/src/gen_for_tower/

src/rust/engine/Cargo.toml

@@ -22,7 +22,6 @@ members = [
   "async_semaphore",
   "boxfuture",
   "concrete_time",
-  "engine_cffi",
   "fs",
   "fs/brfs",
   "fs/fs_util",
@@ -60,7 +59,6 @@ default-members = [
   "async_semaphore",
   "boxfuture",
   "concrete_time",
-  "engine_cffi",
   "fs",
   "fs/fs_util",
   "fs/store",
@@ -84,6 +82,18 @@ default-members = [
   "workunit_store"
 ]
 
+[lib]
+crate-type = ["cdylib"]
+
+[features]
+# NB: To actually load this crate from python, the `extension-module` feature must be enabled. But
+# unfortunately, enabling `extension-module` causes tests linked against `cpython` to fail. We
+# define a feature to enable that, but we do not enable it by default: someone building this module
+# in order to extract `libengine.so` should pass `cargo build .. --features=extension-module`.
+#  see https://github.com/PyO3/pyo3/issues/340
+extension-module = ["cpython/extension-module"]
+default = []
+
 [dependencies]
 async_semaphore = { path = "async_semaphore" }
 async-trait = "0.1"
@@ -114,6 +124,7 @@ rule_graph = { path = "rule_graph" }
 sharded_lmdb = { path = "sharded_lmdb" }
 smallvec = "0.6"
 store = { path = "fs/store" }
+tar_api = { path = "tar_api" }
 task_executor = { path = "task_executor" }
 tempfile = "3"
 time = "0.1.40"

src/rust/engine/engine_cffi/src/lib.rs → src/rust/engine/src/externs/interface.rs

@@ -30,8 +30,6 @@
   clippy::transmute_ptr_to_ptr,
   clippy::zero_ptr
 )]
-// There are some large async-await types generated here. Nothing to worry about?
-#![type_length_limit = "2066838"]
 
 ///
 /// This crate is a wrapper around the engine crate which exposes a python module via cpython.
@@ -45,7 +43,7 @@ use cpython::{
   PyResult as CPyResult, PyTuple, PyType, Python, PythonObject,
 };
 
-use engine::{
+use crate::{
   externs, nodes, Core, ExecutionRequest, ExecutionTermination, Failure, Function, Intrinsics,
   Params, Rule, Scheduler, Session, Tasks, Types, Value,
 };
@@ -71,9 +69,6 @@ use tempfile::TempDir;
 use tokio;
 use workunit_store::{Workunit, WorkunitState};
 
-#[cfg(test)]
-mod tests;
-
 py_exception!(native_engine, PollTimeout);
 
 py_module_initializer!(native_engine, |py, m| {
@@ -1112,7 +1107,7 @@ fn rule_subgraph_visualize(
   })
 }
 
-fn generate_panic_string(payload: &(dyn Any + Send)) -> String {
+pub(crate) fn generate_panic_string(payload: &(dyn Any + Send)) -> String {
   match payload
     .downcast_ref::<String>()
     .cloned()

src/rust/engine/engine_cffi/src/tests.rs → src/rust/engine/src/externs/interface_tests.rs

@@ -1,4 +1,5 @@
-use crate::generate_panic_string;
+use crate::externs::interface::generate_panic_string;
+
 use std::any::Any;
 
 #[test]

src/rust/engine/src/externs.rs → src/rust/engine/src/externs/mod.rs

@@ -8,6 +8,10 @@
   clippy::zero_ptr
 )]
 
+mod interface;
+#[cfg(test)]
+mod interface_tests;
+
 use std::collections::BTreeMap;
 use std::fmt;
 use std::sync::atomic;

src/rust/engine/src/lib.rs

@@ -32,10 +32,10 @@
 
 mod context;
 mod core;
-pub mod externs;
+mod externs;
 mod interning;
 mod intrinsics;
-pub mod nodes;
+mod nodes;
 mod scheduler;
 mod selectors;
 mod tasks;