[Experiment] Add PreprocessedExtension

setuptools/command/build_ext.py

@@ -12,7 +12,7 @@
 from distutils import log
 
 from setuptools.errors import BaseError
-from setuptools.extension import Extension, Library
+from setuptools.extension import Extension, Library, PreprocessedExtension
 
 try:
     # Attempt to use Cython for building extensions, if available
@@ -184,30 +184,41 @@ def finalize_options(self):
         self.shlibs = [ext for ext in self.extensions if isinstance(ext, Library)]
         if self.shlibs:
             self.setup_shlib_compiler()
+
         for ext in self.extensions:
-            ext._full_name = self.get_ext_fullname(ext.name)
-        for ext in self.extensions:
+            ext.__dict__.update(self._update_ext_info(ext))
             fullname = ext._full_name
             self.ext_map[fullname] = ext
 
             # distutils 3.1 will also ask for module names
             # XXX what to do with conflicts?
             self.ext_map[fullname.split('.')[-1]] = ext
 
-            ltd = self.shlibs and self.links_to_dynamic(ext) or False
-            ns = ltd and use_stubs and not isinstance(ext, Library)
-            ext._links_to_dynamic = ltd
-            ext._needs_stub = ns
-            filename = ext._file_name = self.get_ext_filename(fullname)
-            libdir = os.path.dirname(os.path.join(self.build_lib, filename))
-            if ltd and libdir not in ext.library_dirs:
-                ext.library_dirs.append(libdir)
-            if ltd and use_stubs and os.curdir not in ext.runtime_library_dirs:
-                ext.runtime_library_dirs.append(os.curdir)
-
         if self.editable_mode:
             self.inplace = True
 
+    def _update_ext_info(self, ext: Extension) -> dict:
+        """Setuptools needs to add extra attributes to build extension objects"""
+        full_name = self.get_ext_fullname(ext.name)
+        file_name = self.get_ext_filename(full_name)
+        ltd = self.shlibs and self.links_to_dynamic(ext) or False
+        ns = ltd and use_stubs and not isinstance(ext, Library)
+        libdir = os.path.dirname(os.path.join(self.build_lib, file_name))
+
+        updates = {
+            "_file_name": file_name,
+            "_full_name": full_name,
+            "_links_to_dynamic": ltd,
+            "_needs_stub": ns,
+        }
+
+        if ltd and libdir not in ext.library_dirs:
+            updates["library_dirs"] = [*ext.library_dirs, libdir]
+        if ltd and use_stubs and os.curdir not in ext.runtime_library_dirs:
+            updates["runtime_library_dirs"] = [*ext.runtime_library_dirs, os.curdir]
+
+        return updates
+
     def setup_shlib_compiler(self):
         compiler = self.shlib_compiler = new_compiler(
             compiler=self.compiler, dry_run=self.dry_run, force=self.force
@@ -246,20 +257,44 @@ def build_extension(self, ext):
         try:
             if isinstance(ext, Library):
                 self.compiler = self.shlib_compiler
-            _build_ext.build_extension(self, ext)
+            _build_ext.build_extension(self, self._preprocess(ext))
             if ext._needs_stub:
                 build_lib = self.get_finalized_command('build_py').build_lib
                 self.write_stub(build_lib, ext)
         finally:
             self.compiler = _compiler
 
+    def build_extensions(self):
+        self._create_shared_files()
+        super().build_extensions()
+
+    def _create_shared_files(self):
+        for ext in self.extensions:
+            if isinstance(ext, PreprocessedExtension):
+                log.debug(f"creating shared files for {ext._full_name!r}")
+                ext.create_shared_files(self)
+
+    def _preprocess(self, ext: Extension) -> Extension:
+        if not isinstance(ext, PreprocessedExtension):
+            return ext
+
+        log.debug(f"preprocessing extension {ext._full_name!r}")
+        target = ext.preprocess(self)  # may be missing build info
+        updates = self._update_ext_info(target)
+        target.__dict__.update(updates)  # update build info
+        ext.__dict__.update(updates)  # ... for consistency ...
+        return target
+
     def links_to_dynamic(self, ext):
         """Return true if 'ext' links to a dynamic lib in the same package"""
+        if not (ext.libraries and self.shlibs):
+            return False  # avoid calculations if not necessary
+
         # XXX this should check to ensure the lib is actually being built
         # XXX as dynamic, and not just using a locally-found version or a
         # XXX static-compiled version
-        libnames = dict.fromkeys([lib._full_name for lib in self.shlibs])
-        pkg = '.'.join(ext._full_name.split('.')[:-1] + [''])
+        libnames = {self.get_ext_fullname(lib.name) for lib in self.shlibs}
+        pkg = '.'.join(self.get_ext_fullname(ext.name).split('.')[:-1] + [''])
         return any(pkg + libname in libnames for libname in ext.libraries)
 
     def get_source_files(self) -> List[str]:

setuptools/extension.py

@@ -1,12 +1,20 @@
+from __future__ import annotations
+
 import re
 import functools
+from typing import TYPE_CHECKING
+
 import distutils.core
 import distutils.errors
 import distutils.extension
 
 from .monkey import get_unpatched
 
 
+if TYPE_CHECKING:  # pragma: no cover
+    from .command.build_ext import build_ext as BuildExt
+
+
 def _have_cython():
     """
     Return True if Cython can be imported.
@@ -144,5 +152,53 @@ def _convert_pyx_sources_to_lang(self):
         self.sources = list(map(sub, self.sources))
 
 
+class PreprocessedExtension(Extension):
+    """
+    Similar to ``Extension``, but allows creating temporary files needed for the build,
+    before the final compilation.
+    Particularly useful when "transpiling" other languages to C.
+
+    .. note::
+       It is important to add to the original ``source``/``depends`` attributes
+       all files that are necessary for pre-processing so that ``setuptools``
+       automatically adds them to the ``sdist``.
+       Temporary files generated during pre-processing on the other hand
+       should be part of the ``source``/``depends`` attributes of the
+       extension object returned by the ``preprocess`` method.
+    """
+
+    def create_shared_files(self, build_ext: BuildExt) -> None:
+        """*(Optional abstract method)*
+        Generate files that work as build-time dependencies for other extension
+        modules (e.g. headers).
+
+        .. important::
+           ``setuptools`` will call ``create_static_lib`` sequentially
+           (even during a parallel build).
+        """
+
+    def preprocess(self, build_ext: BuildExt) -> Extension:  # pragma: no cover
+        """*(Required abstract method)*
+        The returned ``Extension`` object will be used instead of the original
+        ``PreprocessedExtension`` object when ``build_ext.build_extension`` runs
+        (so that ``sources`` and ``dependencies`` can be augmented/replaced with
+        temporary pre-processed files).
+        Note that 2 objects **SHOULD** be consistent with each other.
+
+        If necessary, a temporary directory **name** can be accessed via
+        ``build_ext.build_temp`` (this directory might not have been created yet).
+        You can also access other attributes like ``build_ext.parallel``,
+        ``build_ext.inplace`` or ``build_ext.editable_mode``.
+
+        .. important::
+           For each extension module ``preprocess`` is called just before
+           compiling (in a single step).
+           If your extension module needs to produce files that are necessary
+           for the compilation of another extension module, please use
+           ``create_shared_files``.
+        """
+        raise NotImplementedError  # needs to be implemented by the relevant plugin.
+
+
 class Library(Extension):
     """Just like a regular Extension, but built as a library instead"""

setuptools/tests/test_build_ext.py

@@ -3,12 +3,14 @@
 import distutils.command.build_ext as orig
 from distutils.sysconfig import get_config_var
 from importlib.util import cache_from_source as _compiled_file_name
+from importlib.machinery import EXTENSION_SUFFIXES
+from pathlib import Path
 
 from jaraco import path
 
 from setuptools.command.build_ext import build_ext, get_abi3_suffix
 from setuptools.dist import Distribution
-from setuptools.extension import Extension
+from setuptools.extension import Extension, Library
 from setuptools.errors import CompileError
 
 from . import environment
@@ -225,8 +227,84 @@ def test_non_optional(self, tmpdir_cwd):
             cmd.run()
 
 
-def test_build_ext_config_handling(tmpdir_cwd):
-    files = {
+class TestLinksToDynamic:
+    """Regression tests for ``links_to_dynamic``."""
+
+    @pytest.mark.parametrize("libraries", [[], ["glob", "tar"]])
+    def test_links_to_dynamic_no_shlib(self, libraries):
+        ext = Extension("name", ["source.c"], libraries=libraries)
+        dist = Distribution({"ext_modules": [ext]})
+        cmd = build_ext(dist)
+        cmd.finalize_options()
+        assert cmd.links_to_dynamic(ext) is False
+
+    @pytest.mark.parametrize("shlib", [[], ["glob", "tar"]])
+    def test_links_to_dynamic_no_libraries(self, shlib):
+        ext = Extension("name", ["source.c"])
+        shlibs = [Library(name, [f"{name}.c"]) for name in shlib]
+        dist = Distribution({"ext_modules": [ext, *shlibs]})
+        cmd = build_ext(dist)
+        cmd.finalize_options()
+        assert cmd.links_to_dynamic(ext) is False
+
+    def test_links_to_dynamic_local_libraries(self):
+        dist = Distribution()
+        local_libs = ["glob", "tar"]
+
+        ext = Extension("name", ["source.c"], libraries=local_libs)
+        shlibs = [Library(name, [f"{name}.c"]) for name in local_libs]
+        dist = Distribution({"ext_modules": [ext, *shlibs]})
+        cmd = build_ext(dist)
+        cmd.finalize_options()
+        assert cmd.links_to_dynamic(ext) is True
+
+
+class TestExtensionAttrPatches:
+    """
+    Regression tests for the extra information ``setuptools`` adds to
+    the extension objects when ``build_ext`` runs.
+    """
+
+    @pytest.mark.parametrize(
+        "ext_package, ext_name, expected_full_name",
+        [
+            (None, "helloworld", "helloworld"),
+            (None, "hello.world", "hello.world"),
+            ("hello", "world", "hello.world"),
+            ("hello", "wor.ld", "hello.wor.ld"),
+        ],
+    )
+    def test_names(self, ext_package, ext_name, expected_full_name):
+        ext = Extension(ext_name, [f"{ext_name}.c"])
+        dist = Distribution({"ext_package": ext_package, "ext_modules": [ext]})
+        cmd = build_ext(dist)
+        cmd.ensure_finalized()
+        assert ext._full_name == expected_full_name
+        file_name = ext._file_name.replace(os.sep, "/")
+        assert expected_full_name.replace(".", "/") in file_name
+
+    @pytest.fixture(params=[True, False], ids=["use_stubs", "dont_use_stubs"])
+    def use_stubs(self, monkeypatch, request):
+        monkeypatch.setattr("setuptools.command.build_ext.use_stubs", request.param)
+        yield request.param
+
+    def test_links_to_dynamic(self, use_stubs):
+        local_libs = ["glob", "tar"]
+        ext = Extension("name", ["source.c"], libraries=local_libs)
+        shlibs = [Library(name, [f"{name}.c"]) for name in local_libs]
+
+        dist = Distribution({"ext_modules": [ext, *shlibs]})
+        cmd = build_ext(dist)
+        cmd.ensure_finalized()
+        assert ext._links_to_dynamic is True
+        assert ext._needs_stub is use_stubs
+        assert cmd.build_lib in ext.library_dirs
+        if use_stubs:
+            assert os.curdir in ext.runtime_library_dirs
+
+
+class TestBuildExamples:
+    SIMPLE = {
         'setup.py': DALS(
             """
             from setuptools import Extension, setup
@@ -287,9 +365,83 @@ def test_build_ext_config_handling(tmpdir_cwd):
             """
         ),
     }
-    path.build(files)
-    code, output = environment.run_setup_py(
-        cmd=['build'],
-        data_stream=(0, 2),
+
+    PREPROCESSED = {
+        **SIMPLE,
+        "foo.c": "#include <xxx-generate-compilation-error-if-not-preprocessed-xxx>\n",
+        "foo.template": SIMPLE["foo.c"],
+        "setup.py": DALS(
+            """
+            import os, shutil
+            from setuptools import setup
+            from setuptools.extension import Extension, PreprocessedExtension
+
+            class MyExtension(PreprocessedExtension):
+                def preprocess(self, build_ext):
+                    os.makedirs(build_ext.build_temp, exist_ok=False)
+                    temp_file = os.path.join(build_ext.build_temp, "foo.c")
+                    shutil.copy("foo.template", temp_file)  # simulate preprocessing
+                    assert os.path.exists(temp_file)
+                    return Extension(self.name, [temp_file])
+
+            setup(ext_modules=[MyExtension('foo', ['foo.c'])])
+            """
+        ),
+    }
+
+    PREPROCESSED_SHARED_FILES = {
+        **SIMPLE,
+        "foo.c": "#include <xxx-generate-compilation-error-if-not-preprocessed-xxx>\n",
+        "fooc.template": SIMPLE["foo.c"],
+        "fooh.template": "#define HELLO_WORLD 1\n",
+        "bar.c": '#include "foo.h"\n' + SIMPLE["foo.c"].replace("foo", "bar"),
+        "setup.py": DALS(
+            """
+            import os, shutil
+            from setuptools import setup
+            from setuptools.extension import Extension, PreprocessedExtension
+
+            class FooExtension(PreprocessedExtension):
+                shared_files_created = False  # test attribute
+
+                def create_shared_files(self, build_ext):
+                    os.makedirs(build_ext.build_temp, exist_ok=False)
+                    fooh_file = os.path.join(build_ext.build_temp, "foo.h")
+                    shutil.copy("fooh.template", fooh_file)
+                    assert os.path.exists(fooh_file)
+                    self.shared_files_created = True
+
+                def preprocess(self, build_ext):
+                    fooc_file = os.path.join(build_ext.build_temp, "foo.c")
+                    shutil.copy("fooc.template", fooc_file)  # simulate preprocessing
+                    assert os.path.exists(fooc_file)
+                    return Extension(self.name, [fooc_file])
+
+            foo = FooExtension('foo', ['foo.c'])
+
+            class BarExtension(PreprocessedExtension):
+                def preprocess(self, build_ext):
+                    assert foo.shared_files_created is True
+                    self.include_dirs.append(build_ext.build_temp)
+                    return self
+
+            bar = BarExtension('bar', ['bar.c'], depends=['foo.h'])
+
+            setup(ext_modules=[bar, foo])
+            """
+        ),
+    }
+
+    @pytest.mark.parametrize(
+        "example", ["SIMPLE", "PREPROCESSED", "PREPROCESSED_SHARED_FILES"]
     )
-    assert code == 0, '\nSTDOUT:\n%s\nSTDERR:\n%s' % output
+    def test_build_ext_config_handling(self, tmpdir_cwd, example):
+        path.build(getattr(self, example))
+        code, output = environment.run_setup_py(
+            cmd=['build'],
+            data_stream=(0, 2),
+        )
+        assert code == 0, '\nSTDOUT:\n%s\nSTDERR:\n%s' % output
+
+        genfiles = Path("foo_build").rglob("foo.*")
+        assert any(str(f).endswith(s) for f in genfiles for s in EXTENSION_SUFFIXES)