Docs: mention how to use inv docker.compilebuildtool (#10554)

* Docs: mention how to use `inv docker.compilebuildtool` Mention how to pre-compile `build.tools` when working on the development instance. * Invoke: refactor Docker settings/constants This commit allows us to import these settings without having any Python dependency. This way, this value can be used from `common/dockerfiles/tasks.py` to compile the tools in an easy way. * Update docs/dev/install.rst Co-authored-by: Eric Holscher <25510+ericholscher@users.noreply.github.com> * Rename constant file with a better name * Update common/ * Module name fix --------- Co-authored-by: Eric Holscher <25510+ericholscher@users.noreply.github.com>
readthedocs · Jul 25, 2023 · a514752 · a514752
1 parent 01db229
commit a514752
Show file tree

Hide file tree

Showing 5 changed files with 83 additions and 62 deletions.
diff --git a/common b/common
diff --git a/docs/dev/install.rst b/docs/dev/install.rst
@@ -104,6 +104,12 @@ Check that everything works
 #. Go to the "Read the Docs" project, under section :guilabel:`Build a version`, click on the :guilabel:`Build version` button selecting "latest",
    and wait until it finishes (this can take several minutes).
 
+.. warning::
+
+   Read the Docs will compile the Python/Node.js/Rust/Go version on-the-fly each time when building the documentation.
+   To speed things up, you can pre-compile and cache all these versions by using ``inv docker.compilebuildtool`` command.
+   *We strongly recommend to pre-compile these versions if you want to build documentation on your development instance.*
+
 #. Click on the "View docs" button to browse the documentation, and verify that it shows the Read the Docs documentation page.
 
 
@@ -177,6 +183,10 @@ save some work while typing docker compose commands. This section explains these
 ``inv docker.buildassets``
     Build all the assets and "deploy" them to the storage.
 
+``inv docker.compilebuildtool``
+    Pre-compile and cache tools that can be specified in ``build.tools`` to speed up builds.
+    It requires ``inv docker.up`` running in another terminal to be able to upload the pre-compiled version to the cache.
+
 Adding a new Python dependency
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

diff --git a/readthedocs/builds/constants_docker.py b/readthedocs/builds/constants_docker.py
@@ -0,0 +1,64 @@
+"""
+Define constants here to allow import them without any external dependency.
+
+There are situations where we want to have access to these values without Django installed
+(e.g. common/dockerfiles/tasks.py)
+
+Note these constants where previously defined as Django settings in ``readthedocs/settings/base.py``.
+"""
+
+DOCKER_DEFAULT_IMAGE = "readthedocs/build"
+
+# Adding a new tool/version to this setting requires:
+#
+# - a mapping between the expected version in the config file, to the full
+# version installed via asdf (found via ``asdf list all <tool>``)
+#
+# - running the script ``./scripts/compile_version_upload.sh`` in
+# development and production environments to compile and cache the new
+# tool/version
+#
+# Note that when updating this options, you should also update the file:
+# readthedocs/rtd_tests/fixtures/spec/v2/schema.json
+RTD_DOCKER_BUILD_SETTINGS = {
+    # Mapping of build.os options to docker image.
+    "os": {
+        "ubuntu-20.04": f"{DOCKER_DEFAULT_IMAGE}:ubuntu-20.04",
+        "ubuntu-22.04": f"{DOCKER_DEFAULT_IMAGE}:ubuntu-22.04",
+    },
+    # Mapping of build.tools options to specific versions.
+    "tools": {
+        "python": {
+            "2.7": "2.7.18",
+            "3.6": "3.6.15",
+            "3.7": "3.7.17",
+            "3.8": "3.8.17",
+            "3.9": "3.9.17",
+            "3.10": "3.10.12",
+            "3.11": "3.11.4",
+            # Always point to the latest stable release.
+            "3": "3.11.4",
+            "miniconda3-4.7": "miniconda3-4.7.12",
+            "mambaforge-4.10": "mambaforge-4.10.3-10",
+        },
+        "nodejs": {
+            "14": "14.20.1",
+            "16": "16.18.1",
+            "18": "18.16.1",  # LTS
+            "19": "19.0.1",
+            "20": "20.3.1",
+        },
+        "rust": {
+            "1.55": "1.55.0",
+            "1.61": "1.61.0",
+            "1.64": "1.64.0",
+            "1.70": "1.70.0",
+        },
+        "golang": {
+            "1.17": "1.17.13",
+            "1.18": "1.18.10",
+            "1.19": "1.19.10",
+            "1.20": "1.20.5",
+        },
+    },
+}
diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py
@@ -10,6 +10,7 @@
 
 from django.conf import settings
 
+from readthedocs.builds import constants_docker
 from readthedocs.config.utils import list_to_dict, to_dict
 from readthedocs.core.utils.filesystem import safe_open
 from readthedocs.projects.constants import GENERIC
@@ -353,7 +354,7 @@ def get_valid_python_versions_for_image(self, build_image):
         """
         if build_image not in settings.DOCKER_IMAGE_SETTINGS:
             build_image = '{}:{}'.format(
-                settings.DOCKER_DEFAULT_IMAGE,
+                constants_docker.DOCKER_DEFAULT_IMAGE,
                 self.default_build_image,
             )
         return settings.DOCKER_IMAGE_SETTINGS[build_image]['python']['supported_versions']
@@ -375,7 +376,7 @@ def get_default_python_version_for_image(self, build_image, python_version):
         """
         if build_image not in settings.DOCKER_IMAGE_SETTINGS:
             build_image = '{}:{}'.format(
-                settings.DOCKER_DEFAULT_IMAGE,
+                constants_docker.DOCKER_DEFAULT_IMAGE,
                 self.default_build_image,
             )
         return (
@@ -488,7 +489,7 @@ def validate_build(self):
             if ':' not in build['image']:
                 # Prepend proper image name to user's image name
                 build['image'] = '{}:{}'.format(
-                    settings.DOCKER_DEFAULT_IMAGE,
+                    constants_docker.DOCKER_DEFAULT_IMAGE,
                     build['image'],
                 )
         # Update docker default settings from image name
@@ -873,7 +874,7 @@ def validate_old_build_config(self):
         with self.catch_validation_error('build.image'):
             image = self.pop_config('build.image', self.default_build_image)
             build['image'] = '{}:{}'.format(
-                settings.DOCKER_DEFAULT_IMAGE,
+                constants_docker.DOCKER_DEFAULT_IMAGE,
                 validate_choice(
                     image,
                     self.valid_build_images,

diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py
@@ -12,7 +12,7 @@
 from readthedocs.core.logs import shared_processors
 from corsheaders.defaults import default_headers
 from readthedocs.core.settings import Settings
-
+from readthedocs.builds import constants_docker
 
 try:
     import readthedocsext  # noqa
@@ -558,10 +558,9 @@ def TEMPLATES(self):
 
     RTD_DOCKER_COMPOSE = False
 
-    DOCKER_DEFAULT_IMAGE = 'readthedocs/build'
     DOCKER_VERSION = 'auto'
     DOCKER_DEFAULT_VERSION = 'latest'
-    DOCKER_IMAGE = '{}:{}'.format(DOCKER_DEFAULT_IMAGE, DOCKER_DEFAULT_VERSION)
+    DOCKER_IMAGE = '{}:{}'.format(constants_docker.DOCKER_DEFAULT_IMAGE, DOCKER_DEFAULT_VERSION)
     DOCKER_IMAGE_SETTINGS = {
         # A large number of users still have this pinned in their config file.
         # We must have documented it at some point.
@@ -620,60 +619,7 @@ def TEMPLATES(self):
     })
     # Additional binds for the build container
     RTD_DOCKER_ADDITIONAL_BINDS = {}
-
-    # Adding a new tool/version to this setting requires:
-    #
-    # - a mapping between the expected version in the config file, to the full
-    # version installed via asdf (found via ``asdf list all <tool>``)
-    #
-    # - running the script ``./scripts/compile_version_upload.sh`` in
-    # development and production environments to compile and cache the new
-    # tool/version
-    #
-    # Note that when updating this options, you should also update the file:
-    # readthedocs/rtd_tests/fixtures/spec/v2/schema.json
-    RTD_DOCKER_BUILD_SETTINGS = {
-        # Mapping of build.os options to docker image.
-        'os': {
-            'ubuntu-20.04': f'{DOCKER_DEFAULT_IMAGE}:ubuntu-20.04',
-            'ubuntu-22.04': f'{DOCKER_DEFAULT_IMAGE}:ubuntu-22.04',
-        },
-        # Mapping of build.tools options to specific versions.
-        'tools': {
-            'python': {
-                '2.7': '2.7.18',
-                '3.6': '3.6.15',
-                '3.7': '3.7.17',
-                '3.8': '3.8.17',
-                '3.9': '3.9.17',
-                '3.10': '3.10.12',
-                '3.11': '3.11.4',
-                'miniconda3-4.7': 'miniconda3-4.7.12',
-                'mambaforge-4.10': 'mambaforge-4.10.3-10',
-            },
-            'nodejs': {
-                '14': '14.20.1',
-                '16': '16.18.1',
-                '18': '18.16.1',  # LTS
-                '19': '19.0.1',
-                '20': '20.3.1',
-            },
-            'rust': {
-                '1.55': '1.55.0',
-                '1.61': '1.61.0',
-                '1.64': '1.64.0',
-                '1.70': '1.70.0',
-            },
-            'golang': {
-                '1.17': '1.17.13',
-                '1.18': '1.18.10',
-                '1.19': '1.19.10',
-                '1.20': '1.20.5',
-            },
-        },
-    }
-    # Always point to the latest stable release.
-    RTD_DOCKER_BUILD_SETTINGS['tools']['python']['3'] = RTD_DOCKER_BUILD_SETTINGS['tools']['python']['3.11']
+    RTD_DOCKER_BUILD_SETTINGS = constants_docker.RTD_DOCKER_BUILD_SETTINGS
     # This is used for the image used to clone the users repo,
     # since we can't read their config file image choice before cloning
     RTD_DOCKER_CLONE_IMAGE = RTD_DOCKER_BUILD_SETTINGS["os"]["ubuntu-22.04"]