Use venv for Python dependencies (macOS) (#22036)

Create a series of scripts to set up and manage a Python virtual
environment that can be used to build (and test) Drake. Remove Python
packages from macOS setup. Add dependencies to the virtual environment
as needed. Adjust install test wrapper to use the virtual environment.
Adjust macOS wheel builder to set up its own virtual environment only
for additional wheel-only packages.

Co-authored-by: Jeremy Nimmer <jeremy.nimmer@tri.global>

cmake/WORKSPACE.in

@@ -10,6 +10,7 @@ python_repository(
     name = "python",
     linux_interpreter_path = "@Python_EXECUTABLE@",
     macos_interpreter_path = "@Python_EXECUTABLE@",
+    requirements_flavor = "build",
 )
 
 # Custom repository rules injected by CMake.

setup/mac/binary_distribution/install_prereqs.sh

@@ -6,13 +6,18 @@
 set -euxo pipefail
 
 with_update=1
+with_python_dependencies=1
 
 while [ "${1:-}" != "" ]; do
   case "$1" in
     # Do NOT call brew update during execution of this script.
     --without-update)
       with_update=0
       ;;
+    # Do NOT install Python (pip) dependencies.
+    --without-python-dependencies)
+      with_python_dependencies=0
+      ;;
     *)
       echo 'Invalid command line argument' >&2
       exit 5
@@ -67,4 +72,6 @@ if ! command -v pip3.12 &>/dev/null; then
   exit 2
 fi
 
-pip3.12 install --break-system-packages -r "${BASH_SOURCE%/*}/requirements.txt"
+if [[ "${with_python_dependencies}" -eq 1 ]]; then
+  pip3.12 install --break-system-packages -r "${BASH_SOURCE%/*}/requirements.txt"
+fi

setup/mac/binary_distribution/requirements.txt

@@ -1,3 +1,10 @@
+# PyPI packages to install via pip for binary Drake distributions.
+
+# WARNING for Drake Developers: if you change this file, then you must
+# regenerate the "locked" requirements files via:
+#
+#     tools/workspace/python/venv_upgrade
+
 ipywidgets
 matplotlib
 notebook

setup/mac/install_prereqs.sh

@@ -5,7 +5,7 @@
 
 set -euxo pipefail
 
-binary_distribution_args=()
+binary_distribution_args=(--without-python-dependencies)
 source_distribution_args=()
 
 while [ "${1:-}" != "" ]; do

setup/mac/source_distribution/install_prereqs.sh

@@ -48,14 +48,3 @@ brew bundle --file="${BASH_SOURCE%/*}/Brewfile" --no-lock
 if [[ "${with_test_only}" -eq 1 ]]; then
   brew bundle --file="${BASH_SOURCE%/*}/Brewfile-test-only" --no-lock
 fi
-
-if ! command -v pip3.12 &>/dev/null; then
-  echo 'ERROR: pip3.12 is NOT installed. The post-install step for the python@3.12 formula may have failed.' >&2
-  exit 2
-fi
-
-pip3.12 install --break-system-packages -r "${BASH_SOURCE%/*}/requirements.txt"
-
-if [[ "${with_test_only}" -eq 1 ]]; then
-  pip3.12 install --break-system-packages -r "${BASH_SOURCE%/*}/requirements-test-only.txt"
-fi

setup/mac/source_distribution/requirements-build.in

@@ -0,0 +1,21 @@
+# PyPI packages to make available for Drake source builds.
+
+# WARNING for Drake Developers: if you change this file, then you must
+# regenerate the "locked" requirements files via:
+#
+#     tools/workspace/python/venv_upgrade
+
+-r ../binary_distribution/requirements.txt
+
+# Packages that we are happy to take the most recent version any time we update.
+# The versions of these packages are "locked" in the requirements-build.txt file
+# and can change any time it updates. These are in addition to anything listed
+# in the binary_distribution requirements.
+
+# (None)
+
+# The following are constrained or pinned version of packages. Typically we
+# shouldn't need constraints, so anything listed here must be accompanied by
+# a comment explaining the need for the constraint.
+
+# (None)