diff --git a/cmake/WORKSPACE.in b/cmake/WORKSPACE.in index 7c98f5f9288a..04df10d0beb2 100644 --- a/cmake/WORKSPACE.in +++ b/cmake/WORKSPACE.in @@ -10,6 +10,7 @@ python_repository( name = "python", linux_interpreter_path = "@Python_EXECUTABLE@", macos_interpreter_path = "@Python_EXECUTABLE@", + requirements_flavor = "build", ) # Custom repository rules injected by CMake. diff --git a/setup/mac/binary_distribution/install_prereqs.sh b/setup/mac/binary_distribution/install_prereqs.sh index 0da4fe6ac1cb..d01e0eb5dace 100755 --- a/setup/mac/binary_distribution/install_prereqs.sh +++ b/setup/mac/binary_distribution/install_prereqs.sh @@ -6,6 +6,7 @@ set -euxo pipefail with_update=1 +with_python_dependencies=1 while [ "${1:-}" != "" ]; do case "$1" in @@ -13,6 +14,10 @@ while [ "${1:-}" != "" ]; do --without-update) with_update=0 ;; + # Do NOT install Python (pip) dependencies. + --without-python-dependencies) + with_python_dependencies=0 + ;; *) echo 'Invalid command line argument' >&2 exit 5 @@ -67,4 +72,6 @@ if ! command -v pip3.12 &>/dev/null; then exit 2 fi -pip3.12 install --break-system-packages -r "${BASH_SOURCE%/*}/requirements.txt" +if [[ "${with_python_dependencies}" -eq 1 ]]; then + pip3.12 install --break-system-packages -r "${BASH_SOURCE%/*}/requirements.txt" +fi diff --git a/setup/mac/binary_distribution/requirements.txt b/setup/mac/binary_distribution/requirements.txt index c81696a72f0a..acd8f84b3543 100644 --- a/setup/mac/binary_distribution/requirements.txt +++ b/setup/mac/binary_distribution/requirements.txt @@ -1,3 +1,10 @@ +# PyPI packages to install via pip for binary Drake distributions. + +# WARNING for Drake Developers: if you change this file, then you must +# regenerate the "locked" requirements files via: +# +# tools/workspace/python/venv_upgrade + ipywidgets matplotlib notebook diff --git a/setup/mac/install_prereqs.sh b/setup/mac/install_prereqs.sh index 68c553b200d5..8827b92d12e1 100755 --- a/setup/mac/install_prereqs.sh +++ b/setup/mac/install_prereqs.sh @@ -5,7 +5,7 @@ set -euxo pipefail -binary_distribution_args=() +binary_distribution_args=(--without-python-dependencies) source_distribution_args=() while [ "${1:-}" != "" ]; do diff --git a/setup/mac/source_distribution/install_prereqs.sh b/setup/mac/source_distribution/install_prereqs.sh index b91f4845c40b..dfc8d410861b 100755 --- a/setup/mac/source_distribution/install_prereqs.sh +++ b/setup/mac/source_distribution/install_prereqs.sh @@ -48,14 +48,3 @@ brew bundle --file="${BASH_SOURCE%/*}/Brewfile" --no-lock if [[ "${with_test_only}" -eq 1 ]]; then brew bundle --file="${BASH_SOURCE%/*}/Brewfile-test-only" --no-lock fi - -if ! command -v pip3.12 &>/dev/null; then - echo 'ERROR: pip3.12 is NOT installed. The post-install step for the python@3.12 formula may have failed.' >&2 - exit 2 -fi - -pip3.12 install --break-system-packages -r "${BASH_SOURCE%/*}/requirements.txt" - -if [[ "${with_test_only}" -eq 1 ]]; then - pip3.12 install --break-system-packages -r "${BASH_SOURCE%/*}/requirements-test-only.txt" -fi diff --git a/setup/mac/source_distribution/requirements-build.in b/setup/mac/source_distribution/requirements-build.in new file mode 100644 index 000000000000..03cb65294f3f --- /dev/null +++ b/setup/mac/source_distribution/requirements-build.in @@ -0,0 +1,21 @@ +# PyPI packages to make available for Drake source builds. + +# WARNING for Drake Developers: if you change this file, then you must +# regenerate the "locked" requirements files via: +# +# tools/workspace/python/venv_upgrade + +-r ../binary_distribution/requirements.txt + +# Packages that we are happy to take the most recent version any time we update. +# The versions of these packages are "locked" in the requirements-build.txt file +# and can change any time it updates. These are in addition to anything listed +# in the binary_distribution requirements. + +# (None) + +# The following are constrained or pinned version of packages. Typically we +# shouldn't need constraints, so anything listed here must be accompanied by +# a comment explaining the need for the constraint. + +# (None) diff --git a/setup/mac/source_distribution/requirements-build.txt b/setup/mac/source_distribution/requirements-build.txt new file mode 100644 index 000000000000..08f8e50eea2c --- /dev/null +++ b/setup/mac/source_distribution/requirements-build.txt @@ -0,0 +1,319 @@ +# This file is generated by drake/tools/workspace/python/venv_upgrade; do not edit. +# +anyio==4.6.0 + # via + # httpx + # jupyter-server +appnope==0.1.4 + # via ipykernel +argon2-cffi==23.1.0 + # via jupyter-server +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +arrow==1.3.0 + # via isoduration +asttokens==2.4.1 + # via stack-data +async-lru==2.0.4 + # via jupyterlab +attrs==24.2.0 + # via + # jsonschema + # referencing +babel==2.16.0 + # via jupyterlab-server +beautifulsoup4==4.12.3 + # via nbconvert +bleach==6.1.0 + # via nbconvert +certifi==2024.8.30 + # via + # httpcore + # httpx + # requests +cffi==1.17.1 + # via argon2-cffi-bindings +charset-normalizer==3.4.0 + # via requests +comm==0.2.2 + # via + # ipykernel + # ipywidgets +contourpy==1.3.0 + # via matplotlib +cycler==0.12.1 + # via matplotlib +debugpy==1.8.6 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +executing==2.1.0 + # via stack-data +fastjsonschema==2.20.0 + # via nbformat +fonttools==4.54.1 + # via matplotlib +fqdn==1.5.1 + # via jsonschema +h11==0.14.0 + # via httpcore +httpcore==1.0.6 + # via httpx +httpx==0.27.2 + # via jupyterlab +idna==3.10 + # via + # anyio + # httpx + # jsonschema + # requests +ipykernel==6.29.5 + # via jupyterlab +ipython==8.28.0 + # via + # ipykernel + # ipywidgets +ipywidgets==8.1.5 + # via -r ../binary_distribution/requirements.txt +isoduration==20.11.0 + # via jsonschema +jedi==0.19.1 + # via ipython +jinja2==3.1.4 + # via + # jupyter-server + # jupyterlab + # jupyterlab-server + # nbconvert +json5==0.9.25 + # via jupyterlab-server +jsonpointer==3.0.0 + # via jsonschema +jsonschema[format-nongpl]==4.23.0 + # via + # jupyter-events + # jupyterlab-server + # nbformat +jsonschema-specifications==2024.10.1 + # via jsonschema +jupyter-client==8.6.3 + # via + # ipykernel + # jupyter-server + # nbclient +jupyter-core==5.7.2 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # nbclient + # nbconvert + # nbformat +jupyter-events==0.10.0 + # via jupyter-server +jupyter-lsp==2.2.5 + # via jupyterlab +jupyter-server==2.14.2 + # via + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # notebook + # notebook-shim +jupyter-server-terminals==0.5.3 + # via jupyter-server +jupyterlab==4.2.5 + # via notebook +jupyterlab-pygments==0.3.0 + # via nbconvert +jupyterlab-server==2.27.3 + # via + # jupyterlab + # notebook +jupyterlab-widgets==3.0.13 + # via ipywidgets +kiwisolver==1.4.7 + # via matplotlib +markupsafe==3.0.1 + # via + # jinja2 + # nbconvert +matplotlib==3.9.2 + # via -r ../binary_distribution/requirements.txt +matplotlib-inline==0.1.7 + # via + # ipykernel + # ipython +mistune==3.0.2 + # via nbconvert +nbclient==0.10.0 + # via nbconvert +nbconvert==7.16.4 + # via jupyter-server +nbformat==5.10.4 + # via + # jupyter-server + # nbclient + # nbconvert +nest-asyncio==1.6.0 + # via ipykernel +notebook==7.2.2 + # via -r ../binary_distribution/requirements.txt +notebook-shim==0.2.4 + # via + # jupyterlab + # notebook +numpy==2.1.2 + # via + # contourpy + # matplotlib +overrides==7.7.0 + # via jupyter-server +packaging==24.1 + # via + # ipykernel + # jupyter-server + # jupyterlab + # jupyterlab-server + # matplotlib + # nbconvert +pandocfilters==1.5.1 + # via nbconvert +parso==0.8.4 + # via jedi +pexpect==4.9.0 + # via ipython +pillow==10.4.0 + # via + # -r ../binary_distribution/requirements.txt + # matplotlib +platformdirs==4.3.6 + # via jupyter-core +prometheus-client==0.21.0 + # via jupyter-server +prompt-toolkit==3.0.48 + # via ipython +psutil==6.0.0 + # via ipykernel +ptyprocess==0.7.0 + # via + # pexpect + # terminado +pure-eval==0.2.3 + # via stack-data +pycparser==2.22 + # via cffi +pydot==3.0.2 + # via -r ../binary_distribution/requirements.txt +pygments==2.18.0 + # via + # ipython + # nbconvert +pyparsing==3.1.4 + # via + # matplotlib + # pydot +python-dateutil==2.9.0.post0 + # via + # arrow + # jupyter-client + # matplotlib +python-json-logger==2.0.7 + # via jupyter-events +pyyaml==6.0.2 + # via + # -r ../binary_distribution/requirements.txt + # jupyter-events +pyzmq==26.2.0 + # via + # ipykernel + # jupyter-client + # jupyter-server +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications + # jupyter-events +requests==2.32.3 + # via jupyterlab-server +rfc3339-validator==0.1.4 + # via + # jsonschema + # jupyter-events +rfc3986-validator==0.1.1 + # via + # jsonschema + # jupyter-events +rpds-py==0.20.0 + # via + # jsonschema + # referencing +send2trash==1.8.3 + # via jupyter-server +six==1.16.0 + # via + # asttokens + # bleach + # python-dateutil + # rfc3339-validator +sniffio==1.3.1 + # via + # anyio + # httpx +soupsieve==2.6 + # via beautifulsoup4 +stack-data==0.6.3 + # via ipython +terminado==0.18.1 + # via + # jupyter-server + # jupyter-server-terminals +tinycss2==1.3.0 + # via nbconvert +tornado==6.4.1 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # notebook + # terminado +traitlets==5.14.3 + # via + # comm + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # jupyter-events + # jupyter-server + # jupyterlab + # matplotlib-inline + # nbclient + # nbconvert + # nbformat +types-python-dateutil==2.9.0.20241003 + # via arrow +uri-template==1.3.0 + # via jsonschema +urllib3==2.2.3 + # via requests +wcwidth==0.2.13 + # via prompt-toolkit +webcolors==24.8.0 + # via jsonschema +webencodings==0.5.1 + # via + # bleach + # tinycss2 +websocket-client==1.8.0 + # via jupyter-server +widgetsnbextension==4.0.13 + # via ipywidgets + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/setup/mac/source_distribution/requirements-test-only.txt b/setup/mac/source_distribution/requirements-test-only.txt deleted file mode 100644 index d93dccfb4520..000000000000 --- a/setup/mac/source_distribution/requirements-test-only.txt +++ /dev/null @@ -1,4 +0,0 @@ -flask -six -u-msgpack-python -websockets diff --git a/setup/mac/source_distribution/requirements-test.in b/setup/mac/source_distribution/requirements-test.in new file mode 100644 index 000000000000..05bf08dedc5a --- /dev/null +++ b/setup/mac/source_distribution/requirements-test.in @@ -0,0 +1,24 @@ +# PyPI packages to make available for Drake source builds with tests. + +# WARNING for Drake Developers: if you change this file, then you must +# regenerate the "locked" requirements files via: +# +# tools/workspace/python/venv_upgrade + +-r requirements-build.in + +# Packages that we are happy to take the most recent version any time we update. +# The versions of these packages are "locked" in the requirements-test.txt file +# and can change any time it updates. These are in addition to anything listed +# in the build requirements. + +flask +six +u-msgpack-python +websockets + +# The following are constrained or pinned version of packages. Typically we +# shouldn't need constraints, so anything listed here must be accompanied by +# a comment explaining the need for the constraint. + +# (None) diff --git a/setup/mac/source_distribution/requirements-test.txt b/setup/mac/source_distribution/requirements-test.txt new file mode 100644 index 000000000000..7b92dcb8b3c1 --- /dev/null +++ b/setup/mac/source_distribution/requirements-test.txt @@ -0,0 +1,336 @@ +# This file is generated by drake/tools/workspace/python/venv_upgrade; do not edit. +# +anyio==4.6.0 + # via + # httpx + # jupyter-server +appnope==0.1.4 + # via ipykernel +argon2-cffi==23.1.0 + # via jupyter-server +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +arrow==1.3.0 + # via isoduration +asttokens==2.4.1 + # via stack-data +async-lru==2.0.4 + # via jupyterlab +attrs==24.2.0 + # via + # jsonschema + # referencing +babel==2.16.0 + # via jupyterlab-server +beautifulsoup4==4.12.3 + # via nbconvert +bleach==6.1.0 + # via nbconvert +blinker==1.8.2 + # via flask +certifi==2024.8.30 + # via + # httpcore + # httpx + # requests +cffi==1.17.1 + # via argon2-cffi-bindings +charset-normalizer==3.4.0 + # via requests +click==8.1.7 + # via flask +comm==0.2.2 + # via + # ipykernel + # ipywidgets +contourpy==1.3.0 + # via matplotlib +cycler==0.12.1 + # via matplotlib +debugpy==1.8.6 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +executing==2.1.0 + # via stack-data +fastjsonschema==2.20.0 + # via nbformat +flask==3.0.3 + # via -r requirements-test.in +fonttools==4.54.1 + # via matplotlib +fqdn==1.5.1 + # via jsonschema +h11==0.14.0 + # via httpcore +httpcore==1.0.6 + # via httpx +httpx==0.27.2 + # via jupyterlab +idna==3.10 + # via + # anyio + # httpx + # jsonschema + # requests +ipykernel==6.29.5 + # via jupyterlab +ipython==8.28.0 + # via + # ipykernel + # ipywidgets +ipywidgets==8.1.5 + # via -r ../binary_distribution/requirements.txt +isoduration==20.11.0 + # via jsonschema +itsdangerous==2.2.0 + # via flask +jedi==0.19.1 + # via ipython +jinja2==3.1.4 + # via + # flask + # jupyter-server + # jupyterlab + # jupyterlab-server + # nbconvert +json5==0.9.25 + # via jupyterlab-server +jsonpointer==3.0.0 + # via jsonschema +jsonschema[format-nongpl]==4.23.0 + # via + # jupyter-events + # jupyterlab-server + # nbformat +jsonschema-specifications==2024.10.1 + # via jsonschema +jupyter-client==8.6.3 + # via + # ipykernel + # jupyter-server + # nbclient +jupyter-core==5.7.2 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # nbclient + # nbconvert + # nbformat +jupyter-events==0.10.0 + # via jupyter-server +jupyter-lsp==2.2.5 + # via jupyterlab +jupyter-server==2.14.2 + # via + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # notebook + # notebook-shim +jupyter-server-terminals==0.5.3 + # via jupyter-server +jupyterlab==4.2.5 + # via notebook +jupyterlab-pygments==0.3.0 + # via nbconvert +jupyterlab-server==2.27.3 + # via + # jupyterlab + # notebook +jupyterlab-widgets==3.0.13 + # via ipywidgets +kiwisolver==1.4.7 + # via matplotlib +markupsafe==3.0.1 + # via + # jinja2 + # nbconvert + # werkzeug +matplotlib==3.9.2 + # via -r ../binary_distribution/requirements.txt +matplotlib-inline==0.1.7 + # via + # ipykernel + # ipython +mistune==3.0.2 + # via nbconvert +nbclient==0.10.0 + # via nbconvert +nbconvert==7.16.4 + # via jupyter-server +nbformat==5.10.4 + # via + # jupyter-server + # nbclient + # nbconvert +nest-asyncio==1.6.0 + # via ipykernel +notebook==7.2.2 + # via -r ../binary_distribution/requirements.txt +notebook-shim==0.2.4 + # via + # jupyterlab + # notebook +numpy==2.1.2 + # via + # contourpy + # matplotlib +overrides==7.7.0 + # via jupyter-server +packaging==24.1 + # via + # ipykernel + # jupyter-server + # jupyterlab + # jupyterlab-server + # matplotlib + # nbconvert +pandocfilters==1.5.1 + # via nbconvert +parso==0.8.4 + # via jedi +pexpect==4.9.0 + # via ipython +pillow==10.4.0 + # via + # -r ../binary_distribution/requirements.txt + # matplotlib +platformdirs==4.3.6 + # via jupyter-core +prometheus-client==0.21.0 + # via jupyter-server +prompt-toolkit==3.0.48 + # via ipython +psutil==6.0.0 + # via ipykernel +ptyprocess==0.7.0 + # via + # pexpect + # terminado +pure-eval==0.2.3 + # via stack-data +pycparser==2.22 + # via cffi +pydot==3.0.2 + # via -r ../binary_distribution/requirements.txt +pygments==2.18.0 + # via + # ipython + # nbconvert +pyparsing==3.1.4 + # via + # matplotlib + # pydot +python-dateutil==2.9.0.post0 + # via + # arrow + # jupyter-client + # matplotlib +python-json-logger==2.0.7 + # via jupyter-events +pyyaml==6.0.2 + # via + # -r ../binary_distribution/requirements.txt + # jupyter-events +pyzmq==26.2.0 + # via + # ipykernel + # jupyter-client + # jupyter-server +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications + # jupyter-events +requests==2.32.3 + # via jupyterlab-server +rfc3339-validator==0.1.4 + # via + # jsonschema + # jupyter-events +rfc3986-validator==0.1.1 + # via + # jsonschema + # jupyter-events +rpds-py==0.20.0 + # via + # jsonschema + # referencing +send2trash==1.8.3 + # via jupyter-server +six==1.16.0 + # via + # -r requirements-test.in + # asttokens + # bleach + # python-dateutil + # rfc3339-validator +sniffio==1.3.1 + # via + # anyio + # httpx +soupsieve==2.6 + # via beautifulsoup4 +stack-data==0.6.3 + # via ipython +terminado==0.18.1 + # via + # jupyter-server + # jupyter-server-terminals +tinycss2==1.3.0 + # via nbconvert +tornado==6.4.1 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # notebook + # terminado +traitlets==5.14.3 + # via + # comm + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # jupyter-events + # jupyter-server + # jupyterlab + # matplotlib-inline + # nbclient + # nbconvert + # nbformat +types-python-dateutil==2.9.0.20241003 + # via arrow +u-msgpack-python==2.8.0 + # via -r requirements-test.in +uri-template==1.3.0 + # via jsonschema +urllib3==2.2.3 + # via requests +wcwidth==0.2.13 + # via prompt-toolkit +webcolors==24.8.0 + # via jsonschema +webencodings==0.5.1 + # via + # bleach + # tinycss2 +websocket-client==1.8.0 + # via jupyter-server +websockets==13.1 + # via -r requirements-test.in +werkzeug==3.0.4 + # via flask +widgetsnbextension==4.0.13 + # via ipywidgets + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/setup/mac/source_distribution/requirements.txt b/setup/mac/source_distribution/requirements.txt deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/install/BUILD.bazel b/tools/install/BUILD.bazel index bf1220e9ee28..913ec149ddba 100644 --- a/tools/install/BUILD.bazel +++ b/tools/install/BUILD.bazel @@ -26,8 +26,14 @@ drake_py_library( name = "install_test_helper", testonly = 1, srcs = ["install_test_helper.py"], - data = ["//:install"], + data = [ + "//:install", + "@python//:venv_bin", + ], imports = ["."], + deps = [ + "@rules_python//python/runfiles", + ], ) exports_files( diff --git a/tools/install/install_test.py b/tools/install/install_test.py index 8c7008231c75..15743c8df4f1 100644 --- a/tools/install/install_test.py +++ b/tools/install/install_test.py @@ -2,9 +2,12 @@ import functools import os import re +import subprocess import sys import unittest +from python import runfiles + import install_test_helper @@ -26,6 +29,15 @@ def test_basic_paths(self): self.assertSetEqual(set(['bin', 'include', 'lib', 'share']), content) def _run_one_command(self, test_command): + # On macOS, we must run the install tests against venv's interpreter, + # so that necessary libraries are available. On Ubuntu, the libraries + # are installed system-wide (without a venv). + if sys.platform == "darwin": + manifest = runfiles.Create() + python = manifest.Rlocation("python/bin/python3") + else: + python = install_test_helper.get_python_executable() + # Our launched processes should be independent, not inherit their # runfiles from the install_test.py runner. env = dict(os.environ) @@ -35,8 +47,9 @@ def _run_one_command(self, test_command): # Execute the test_command. print("+ {}".format(test_command), file=sys.stderr) - install_test_helper.check_call( - [os.path.join(os.getcwd(), test_command)], + assert test_command.endswith(".py") + subprocess.check_call( + [python, os.path.join(os.getcwd(), test_command)], env=env) diff --git a/tools/jupyter/BUILD.bazel b/tools/jupyter/BUILD.bazel index c04f639d61db..2b4d42782c82 100644 --- a/tools/jupyter/BUILD.bazel +++ b/tools/jupyter/BUILD.bazel @@ -32,6 +32,10 @@ drake_jupyter_py_binary( drake_py_unittest( name = "jupyter_notebook_test", + data = ["@python//:venv_bin"], + deps = [ + "@rules_python//python/runfiles", + ], ) # Add example failing notebook. diff --git a/tools/jupyter/test/jupyter_notebook_test.py b/tools/jupyter/test/jupyter_notebook_test.py index 96c616b12036..d02ca274aa8f 100644 --- a/tools/jupyter/test/jupyter_notebook_test.py +++ b/tools/jupyter/test/jupyter_notebook_test.py @@ -1,9 +1,18 @@ import subprocess +import sys import unittest +from python import runfiles + class TestJupyterNotebook(unittest.TestCase): def test_help(self): """Ensures that `jupyter notebook` is installed (#12042).""" - status = subprocess.call(args=["jupyter", "notebook", "--help"]) + manifest = runfiles.Create() + if sys.platform == "darwin": + jupyter = manifest.Rlocation("python/bin/jupyter") + else: + jupyter = "jupyter" + + status = subprocess.call(args=[jupyter, "notebook", "--help"]) self.assertEqual(status, 0) diff --git a/tools/skylark/py.bzl b/tools/skylark/py.bzl index c83815d23a0f..7e5d37fc79d5 100644 --- a/tools/skylark/py.bzl +++ b/tools/skylark/py.bzl @@ -7,8 +7,30 @@ load( _py_test = "py_test", ) -py_binary = _py_binary +# All of Drake's Python code should depend on our requirements.txt pins, so we +# add it as a data dependency to every python rule. If this particular build +# doesn't use a requirements.txt, then the file will be empty (and thus inert). -py_library = _py_library +def _add_requirements(data): + return (data or []) + ["@python//:requirements.txt"] -py_test = _py_test +def py_binary(name, *, data = None, **kwargs): + _py_binary( + name = name, + data = _add_requirements(data), + **kwargs + ) + +def py_library(name, *, data = None, **kwargs): + _py_library( + name = name, + data = _add_requirements(data), + **kwargs + ) + +def py_test(name, *, data = None, **kwargs): + _py_test( + name = name, + data = _add_requirements(data), + **kwargs + ) diff --git a/tools/wheel/macos/build-wheel.sh b/tools/wheel/macos/build-wheel.sh index c15ade5af3ef..e3e65355d0f5 100755 --- a/tools/wheel/macos/build-wheel.sh +++ b/tools/wheel/macos/build-wheel.sh @@ -39,32 +39,6 @@ if [ -e "/opt/drake" ]; then exit 1 fi -# ----------------------------------------------------------------------------- -# Set up a Python virtual environment with the latest setuptools. -# ----------------------------------------------------------------------------- - -# TODO(matthew.woehlke) If/when we fix the Drake build to get its (PyPI) -# dependencies some other way, note that, as of writing, building the actual -# wheel has additional dependencies; this logic may need to lower rather than -# being deleted outright. See (and consider partly reverting) the commit that -# added this comment. - -readonly pyvenv_root="/opt/drake-wheel-build/$python/python" - -# NOTE: Xcode ships python3, make sure to use the one from brew. -"$python_executable" -m venv "$pyvenv_root" - -# We also need pythonX.Y-config, which isn't created as of writing (see also -# https://github.com/pypa/virtualenv/issues/169). Don't fail if it already -# exists, though, e.g. if the bug has been fixed. -ln -s "$python_prefix/bin/$python-config" \ - "$pyvenv_root/bin/$python-config" || true # Allowed to already exist. - -. "$pyvenv_root/bin/activate" - -pip install -r "$git_root/setup/mac/binary_distribution/requirements.txt" -pip install -r "$git_root/setup/mac/source_distribution/requirements.txt" - # ----------------------------------------------------------------------------- # Build and "install" Drake. # ----------------------------------------------------------------------------- @@ -90,7 +64,7 @@ EOF cmake "$git_root" \ -DDRAKE_VERSION_OVERRIDE="${DRAKE_VERSION}" \ -DCMAKE_INSTALL_PREFIX="/opt/drake-dist/$python" \ - -DPython_EXECUTABLE="$pyvenv_root/bin/$python" + -DPython_EXECUTABLE="$python_executable" make install # Build wheel tools. @@ -105,6 +79,23 @@ ln -s "$(bazel info bazel-bin)" "$build_root"/bazel-bin # later. find "$build_root" -type d -print0 | xargs -0 chmod u+w +# ----------------------------------------------------------------------------- +# Set up a Python virtual environment. +# ----------------------------------------------------------------------------- + +readonly pyvenv_root="/opt/drake-wheel-build/$python/python" + +# NOTE: Xcode ships python3, make sure to use the one from brew. +"$python_executable" -m venv "$pyvenv_root" + +# We also need pythonX.Y-config, which isn't created as of writing (see also +# https://github.com/pypa/virtualenv/issues/169). Don't fail if it already +# exists, though, e.g. if the bug has been fixed. +ln -s "$python_prefix/bin/$python-config" \ + "$pyvenv_root/bin/$python-config" || true # Allowed to already exist. + +. "$pyvenv_root/bin/activate" + # ----------------------------------------------------------------------------- # Install tools to build the wheel. # ----------------------------------------------------------------------------- diff --git a/tools/workspace/python/package.BUILD.bazel b/tools/workspace/python/package.BUILD.bazel index e85a1ed49d46..ef242811b8dc 100644 --- a/tools/workspace/python/package.BUILD.bazel +++ b/tools/workspace/python/package.BUILD.bazel @@ -42,3 +42,14 @@ cc_library( visibility = ["//visibility:public"], deps = [":python_headers"], ) + +exports_files(["requirements.txt"]) + +filegroup( + name = "venv_bin", + visibility = ["//visibility:public"], + data = [ + "bin", + "requirements.txt", + ], +) diff --git a/tools/workspace/python/repository.bzl b/tools/workspace/python/repository.bzl index 6ac86e728aa7..4c7bbb51a9aa 100644 --- a/tools/workspace/python/repository.bzl +++ b/tools/workspace/python/repository.bzl @@ -1,15 +1,29 @@ """ -Finds local system Python headers and libraries using python-config and makes -them available to be used as a C/C++ dependency. +This rule configures Python for use by Drake, in two parts: +(a) Finds local system Python headers and libraries using python-config and + makes them available to be used as a C/C++ dependency. +(b) macOS only: creates (or syncs) a virtual environment for dependencies. -There are two available targets: +For part (a) there are two available targets: -(1) "@foo//:python" for the typical case where we are creating loadable dynamic -libraries to be used as Python modules. +(1) "@python//:python" for the typical case where we are creating loadable +dynamic libraries to be used as Python modules. -(2) "@foo//:python_direct_link" for the unusual case of linking the CPython +(2) "@python//:python_direct_link" for the unusual case of linking the CPython interpreter as a library into an executable with a C++ main() function. +For part (b) the environment is used in all python rules and tests by default, +but if a test needs to shell out to a venv binary, the `@python//:venv_bin` +can be used to put the binaries' path into runfiles. + +If the {macos,linux}_interpreter_path being used only mentions the python major +version (i.e., it is "/path/to/python3" not "/path/to/python3.##") and if the +interpreter is changed to a different minor version without any change to the +path, then you must run `bazel sync --configure` to re-run this repository rule +in order to make bazel aware of the new minor version. This hazard cannot occur +on any of Drake's supported platforms with our default values, but if you are +trying something out of the ordinary, be aware. + Arguments: name: A unique name for this rule. linux_interpreter_path: (Optional) Interpreter path for the Python runtime, @@ -17,6 +31,7 @@ Arguments: macos_interpreter_path: (Optional) Interpreter path for the Python runtime, when running on macOS. The format substitution "{homebrew_prefix}" is available for use in this string. + requirements_flavor: (Optional) Which choice of requirements.txt to use. """ load( @@ -106,6 +121,41 @@ def _get_linkopts(repo_ctx, python_config): linkopts.insert(i, "-Wl,-rpath," + path) return linkopts +def _prepare_venv(repo_ctx, python): + # Only macOS uses a venv at the moment. + os_name = repo_ctx.os.name # "linux" or "mac os x" + if os_name != "mac os x": + execute_or_fail(repo_ctx, ["mkdir", "bin"]) + repo_ctx.file("requirements.txt", content = "") + return python + + # Choose which requirements to use. + requirements = repo_ctx.path(Label( + "@drake//setup:{}/source_distribution/requirements-{}.txt".format( + "mac", + repo_ctx.attr.requirements_flavor, + ), + )).realpath + repo_ctx.symlink(requirements, "requirements.txt") + repo_ctx.watch(requirements) + + # Run pip-sync to ensure the venv content matches the requirements.txt; it + # will (un)install any packages as necessary, or even create the venv when + # it doesn't exist at all yet. + sync_label = Label("@drake//tools/workspace/python:venv_sync") + sync = repo_ctx.path(sync_label).realpath + repo_ctx.report_progress("Running venv_sync") + execute_or_fail(repo_ctx, [ + sync, + "--python", + python, + "--repository", + repo_ctx.path("").realpath, + ]) + repo_ctx.watch(sync) + + return repo_ctx.path("bin/python3") + def _impl(repo_ctx): # Add the BUILD file. repo_ctx.symlink( @@ -133,6 +183,9 @@ def _impl(repo_ctx): if repo_ctx.os.name == "mac os x": linkopts_module.insert(0, "-undefined dynamic_lookup") + # Set up (or sync) the venv. + bin_path = _prepare_venv(repo_ctx, python) + version_content = """ # DO NOT EDIT: generated by python_repository() # WARNING: Avoid using this macro in any repository rules which require @@ -147,7 +200,7 @@ PYTHON_INCLUDES = {includes} PYTHON_LINKOPTS_EMBEDDED = {linkopts_embedded} PYTHON_LINKOPTS_MODULE = {linkopts_module} """.format( - bin_path = python, + bin_path = bin_path, extension_suffix = extension_suffix, version = version, site_packages_relpath = site_packages_relpath, @@ -170,11 +223,14 @@ interpreter_path_attrs = { # CMakeLists.txt and doc/_pages/installation.md. default = "{homebrew_prefix}/bin/python3.12", ), + "requirements_flavor": attr.string( + default = "test", + values = ["build", "test"], + ), } python_repository = repository_rule( _impl, attrs = interpreter_path_attrs, - local = True, configure = True, ) diff --git a/tools/workspace/python/venv_expunge b/tools/workspace/python/venv_expunge new file mode 100755 index 000000000000..3fe6d0ac7618 --- /dev/null +++ b/tools/workspace/python/venv_expunge @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Drake script to remove the Python venv. + +set -eux -o pipefail + +# Chdir to the Drake root. +cd "$(dirname $0)/../../.." +readonly venv_root="$(bazel info output_base).venv" + +# Remove the venv. +rm -r "$venv_root" diff --git a/tools/workspace/python/venv_sync b/tools/workspace/python/venv_sync new file mode 100755 index 000000000000..84ade8d3f76e --- /dev/null +++ b/tools/workspace/python/venv_sync @@ -0,0 +1,76 @@ +#!/bin/bash +# +# Drake script to set up the Python venv. +# Uses https://github.com/jazzband/pip-tools under the hood for setup. +# +# Users must NOT run this manually. + +set -eux -o pipefail + +# Process command line arguments. +python= +repository= +while [ "${1:-}" != "" ]; do + case "$1" in + --python) + # The python interpreter to use. + readonly python="$2" + shift + ;; + --repository) + # The bazel repository rule root to interface with. We'll use the + # requirements.txt found there, and write back the `bin` symlink. + readonly repository="$2" + shift + ;; + *) + echo 'Invalid command line argument' >&2 + exit 5 + esac + shift +done +if [[ -z "${python}" ]]; then + echo "error: --python is required" + exit 1 +fi +if [[ -z "${repository}" ]]; then + echo "error: --repository is required" + exit 1 +fi + +# Place the venv(s) in a sibling directory to the output base. That should be a +# suitable disk location for build artifacts, but without polluting the actual +# output base that Bazel owns. +bazel_output_base=$(cd "${repository}/../.." && pwd) +venv_base="${bazel_output_base}.venv" +mkdir -p "${venv_base}" + +# Install pip-tools into a virtual environment. As suggested by the pip-tools +# docs, we segregate the pip-tools apps from the environment they are managing, +# so that changes to the managed environment cannot break pip-tools. +readonly venv_jazzband="${venv_base}/venv.jazzband" +if [ ! -d "${venv_jazzband}" ]; then + "${python}" -m venv "${venv_jazzband}" +fi +# TODO(jeremy.nimmer) Ideally, we also would pin all of the dependencies of +# pip-tools here, but it's not obvious to me how to do that in a way which is +# easy to upgrade/re-pin over time. +"${venv_jazzband}/bin/pip" install -U pip-tools==7.4.1 + +# Prepare the venv that will hold Drake's requirements. +readonly venv_drake="${venv_base}/venv.drake" +if [ ! -d "${venv_drake}" ]; then + "${python}" -m venv "${venv_drake}" +fi +# Any environment managed by pip-tools requires a sufficiently new version of +# pip as compared to the Ubuntu 22.04 default. +"${venv_drake}/bin/pip" install -U pip==24.2 + +# Run the pip-sync tool. +"${venv_jazzband}/bin/pip-sync" \ + --verbose \ + --python-executable="${venv_drake}/bin/python3" \ + "${repository}/requirements.txt" + +# Symlink our venv bin path for the repository.bzl to use. +ln -nsf "${venv_drake}/bin" "${repository}/bin" diff --git a/tools/workspace/python/venv_upgrade b/tools/workspace/python/venv_upgrade new file mode 100755 index 000000000000..7f97df218962 --- /dev/null +++ b/tools/workspace/python/venv_upgrade @@ -0,0 +1,47 @@ +#!/bin/bash +# +# Drake script to upgrade our requirements lockfile. +# - Users should make edits to requirements.in, and then +# - run this script to compile requirements.in to requirements.txt. + +set -eux -o pipefail + +generate_requirements() { + in_name="$1.in" + out_name="$1.txt" + + # Remove any existing lockfile to avoid it being used as a baseline. + rm -f "$out_name" requirements.tmp + + # Compile requirements.in to its locked form of requirements.txt. + "$venv_root"/venv.jazzband/bin/pip-compile \ + --verbose \ + --output-file requirements.tmp \ + "$in_name" + + # Edit the comment atop the file that explains how to upgrade. + echo '# This file is generated by' \ + 'drake/tools/workspace/python/venv_upgrade;' \ + 'do not edit.' > "$out_name" + tail -n+6 requirements.tmp >> "$out_name" + + # Remove the temporary file. + rm requirements.tmp +} + +# Determine what platform we are on. +if [[ "$(uname)" == "Darwin" ]]; then + readonly platform=mac +else + readonly platform=ubuntu +fi + +# Chdir to the Drake root. +cd "$(dirname $0)/../../.." +readonly venv_root="$(bazel info output_base).venv" + +# Chdir to where the requirements files are located. +cd "setup/$platform/source_distribution" + +generate_requirements requirements-build +generate_requirements requirements-test