Skip to content

Commit

Permalink
Build: decouple install_build_tools from PythonEnvironment
Browse files Browse the repository at this point in the history 
The method `install_build_tools` is not tied to the Python environment, and it's
more related to the build's director, instead.

I'm moving it to the `BuildDirector` class also because it's needed there when
using `build.commands` as the first step to execute to install the `build.tools`
specified by the user to build their docs.
  • Loading branch information
@humitos
humitos committed May 23, 2022
1 parent 5365087 commit 2286206
Show file tree
Hide file tree
Showing 4 changed files with 141 additions and 137 deletions.
2 changes: 1 addition & 1 deletion readthedocs/config/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -793,7 +793,7 @@ def validate_build_config_with_tools(self):

commands = []
with self.catch_validation_error("build.commands"):
commands = self.pop_config("build.commands")
commands = self.pop_config("build.commands", default=[])
validate_list(commands)

if not tools:
Expand Down
138 changes: 137 additions & 1 deletion readthedocs/doc_builder/director.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import os
import shutil
import tarfile
from collections import defaultdict

import structlog
Expand All @@ -13,6 +14,7 @@
from readthedocs.projects.exceptions import RepositoryError
from readthedocs.projects.models import Feature
from readthedocs.projects.signals import after_build, before_build, before_vcs
from readthedocs.storage import build_tools_storage

log = structlog.get_logger(__name__)

Expand Down Expand Up @@ -157,7 +159,7 @@ def setup_environment(self):

# Install all ``build.tools`` specified by the user
if self.data.config.using_build_tools:
self.language_environment.install_build_tools()
self.install_build_tools()

self.run_build_job("pre_create_environment")
self.create_environment()
Expand Down Expand Up @@ -354,6 +356,140 @@ def run_build_commands(self):
# ignore=shutil.ignore_patterns(*self.ignore_patterns),
)

def install_build_tools(self):
"""
Install all ``build.tools`` defined by the user in the config file.
It uses ``asdf`` behind the scenes to manage all the tools and versions
of them. These tools/versions are stored in the Cloud cache and are
downloaded on each build (~50 - ~100Mb).
If the requested tool/version is not present in the cache, it's
installed via ``asdf`` on the fly.
"""
if settings.RTD_DOCKER_COMPOSE:
# Create a symlink for ``root`` user to use the same ``.asdf``
# installation as the ``docs`` user. Required for local building
# since everything is run as ``root`` when using Local Development
# instance
cmd = [
"ln",
"-s",
os.path.join(settings.RTD_DOCKER_WORKDIR, ".asdf"),
"/root/.asdf",
]
self.build_environment.run(
*cmd,
record=False,
)

for tool, version in self.data.config.build.tools.items():
full_version = version.full_version # e.g. 3.9 -> 3.9.7

# TODO: generate the correct path for the Python version
# see https://github.com/readthedocs/readthedocs.org/pull/8447#issuecomment-911562267
# tool_path = f'{self.config.build.os}/{tool}/2021-08-30/{full_version}.tar.gz'
tool_path = f"{self.data.config.build.os}-{tool}-{full_version}.tar.gz"
tool_version_cached = build_tools_storage.exists(tool_path)
if tool_version_cached:
remote_fd = build_tools_storage.open(tool_path, mode="rb")
with tarfile.open(fileobj=remote_fd) as tar:
# Extract it on the shared path between host and Docker container
extract_path = os.path.join(self.data.project.doc_path, "tools")
tar.extractall(extract_path)

# Move the extracted content to the ``asdf`` installation
cmd = [
"mv",
f"{extract_path}/{full_version}",
os.path.join(
settings.RTD_DOCKER_WORKDIR,
f".asdf/installs/{tool}/{full_version}",
),
]
self.build_environment.run(
*cmd,
record=False,
)
else:
log.debug(
"Cached version for tool not found.",
os=self.data.config.build.os,
tool=tool,
full_version=full_version,
tool_path=tool_path,
)
# If the tool version selected is not available from the
# cache we compile it at build time
cmd = [
# TODO: make ``PYTHON_CONFIGURE_OPTS="--enable-shared"``
# environment variable to work here. Note that
# ``self.build_environment.run`` does not support passing
# environment for a particular command:
# https://github.com/readthedocs/readthedocs.org/blob/9d2d1a2/readthedocs/doc_builder/environments.py#L430-L431
"asdf",
"install",
tool,
full_version,
]
self.build_environment.run(
*cmd,
)

# Make the tool version chosen by the user the default one
cmd = [
"asdf",
"global",
tool,
full_version,
]
self.build_environment.run(
*cmd,
)

# Recreate shims for this tool to make the new version
# installed available
# https://asdf-vm.com/learn-more/faq.html#newly-installed-exectable-not-running
cmd = [
"asdf",
"reshim",
tool,
]
self.build_environment.run(
*cmd,
record=False,
)

if all(
[
tool == "python",
# Do not install them if the tool version was cached
# because these dependencies are already installed when
# created with our script and uploaded to the cache's
# bucket
not tool_version_cached,
# Do not install them on conda/mamba since they are not
# needed because the environment is managed by conda/mamba
# itself
self.data.config.python_interpreter not in ("conda", "mamba"),
]
):
# Install our own requirements if the version is compiled
cmd = [
"python",
"-mpip",
"install",
"-U",
"virtualenv",
# We cap setuptools to avoid breakage of projects
# relying on setup.py invokations,
# see https://github.com/readthedocs/readthedocs.org/issues/8659
"setuptools<58.3.0",
]
self.build_environment.run(
*cmd,
)

# Helpers
#
# TODO: move somewhere or change names to make them private or something to
Expand Down
135 changes: 0 additions & 135 deletions readthedocs/doc_builder/python_environments.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,16 @@
import copy
import itertools
import os
import tarfile

import structlog
import yaml
from django.conf import settings

from readthedocs.config import PIP, SETUPTOOLS, ParseError
from readthedocs.config import parse as parse_yaml
from readthedocs.config.models import PythonInstall, PythonInstallRequirements
from readthedocs.doc_builder.config import load_yaml_config
from readthedocs.doc_builder.loader import get_builder_class
from readthedocs.projects.models import Feature
from readthedocs.storage import build_tools_storage

log = structlog.get_logger(__name__)

Expand All @@ -40,138 +37,6 @@ def __init__(self, version, build_env, config=None):
version_slug=self.version.slug,
)

def install_build_tools(self):
"""
Install all ``build.tools`` defined by the user in the config file.
It uses ``asdf`` behind the scenes to manage all the tools and versions
of them. These tools/versions are stored in the Cloud cache and are
downloaded on each build (~50 - ~100Mb).
If the requested tool/version is not present in the cache, it's
installed via ``asdf`` on the fly.
"""
if settings.RTD_DOCKER_COMPOSE:
# Create a symlink for ``root`` user to use the same ``.asdf``
# installation as the ``docs`` user. Required for local building
# since everything is run as ``root`` when using Local Development
# instance
cmd = [
'ln',
'-s',
os.path.join(settings.RTD_DOCKER_WORKDIR, '.asdf'),
'/root/.asdf',
]
self.build_env.run(
*cmd,
record=False,
)

for tool, version in self.config.build.tools.items():
full_version = version.full_version # e.g. 3.9 -> 3.9.7

# TODO: generate the correct path for the Python version
# see https://github.com/readthedocs/readthedocs.org/pull/8447#issuecomment-911562267
# tool_path = f'{self.config.build.os}/{tool}/2021-08-30/{full_version}.tar.gz'
tool_path = f'{self.config.build.os}-{tool}-{full_version}.tar.gz'
tool_version_cached = build_tools_storage.exists(tool_path)
if tool_version_cached:
remote_fd = build_tools_storage.open(tool_path, mode='rb')
with tarfile.open(fileobj=remote_fd) as tar:
# Extract it on the shared path between host and Docker container
extract_path = os.path.join(self.project.doc_path, 'tools')
tar.extractall(extract_path)

# Move the extracted content to the ``asdf`` installation
cmd = [
'mv',
f'{extract_path}/{full_version}',
os.path.join(
settings.RTD_DOCKER_WORKDIR,
f'.asdf/installs/{tool}/{full_version}',
),
]
self.build_env.run(
*cmd,
record=False,
)
else:
log.debug(
'Cached version for tool not found.',
os=self.config.build.os,
tool=tool,
full_version=full_version,
tool_path=tool_path,
)
# If the tool version selected is not available from the
# cache we compile it at build time
cmd = [
# TODO: make ``PYTHON_CONFIGURE_OPTS="--enable-shared"``
# environment variable to work here. Note that
# ``self.build_env.run`` does not support passing
# environment for a particular command:
# https://github.com/readthedocs/readthedocs.org/blob/9d2d1a2/readthedocs/doc_builder/environments.py#L430-L431
'asdf',
'install',
tool,
full_version,
]
self.build_env.run(
*cmd,
)

# Make the tool version chosen by the user the default one
cmd = [
'asdf',
'global',
tool,
full_version,
]
self.build_env.run(
*cmd,
)

# Recreate shims for this tool to make the new version
# installed available
# https://asdf-vm.com/learn-more/faq.html#newly-installed-exectable-not-running
cmd = [
'asdf',
'reshim',
tool,
]
self.build_env.run(
*cmd,
record=False,
)

if all([
tool == 'python',
# Do not install them if the tool version was cached
# because these dependencies are already installed when
# created with our script and uploaded to the cache's
# bucket
not tool_version_cached,
# Do not install them on conda/mamba since they are not
# needed because the environment is managed by conda/mamba
# itself
self.config.python_interpreter not in ('conda', 'mamba'),
]):
# Install our own requirements if the version is compiled
cmd = [
'python',
'-mpip',
'install',
'-U',
'virtualenv',
# We cap setuptools to avoid breakage of projects
# relying on setup.py invokations,
# see https://github.com/readthedocs/readthedocs.org/issues/8659
'setuptools<58.3.0',
]
self.build_env.run(
*cmd,
)

def install_requirements(self):
"""Install all requirements from the config object."""
for install in self.config.python.install:
Expand Down
3 changes: 3 additions & 0 deletions readthedocs/projects/tasks/builds.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -604,6 +604,9 @@ def execute(self):
try:
# NOTE: check if the build uses `build.commands` and only run those
if self.data.config.build.commands:
self.update_build(state=BUILD_STATE_INSTALLING)
self.data.build_director.install_build_tools()

self.update_build(state=BUILD_STATE_BUILDING)
self.data.build_director.run_build_commands()

Expand Down

0 comments on commit 2286206

Please sign in to comment.