Merge pull request #158 from pfmoore/zipapp

Distribute a zipapp version of pip

noxfile.py

@@ -28,6 +28,8 @@ def check(session):
 
     # Get rid of provided-by-nox pip
     session.run("python", "-m", "pip", "uninstall", "pip", "--yes")
+    # Run the pip.pyz file
+    session.run("python", "scripts/check_zipapp.py", str(public / "pip.pyz"), "--version")
     # Run the get-pip.py file
     session.run("python", str(location))
     # Ensure that pip is installed
@@ -38,7 +40,7 @@ def check(session):
 @nox.session
 def generate(session):
     """Update the scripts, to the latest versions."""
-    session.install("packaging", "requests", "cachecontrol[filecache]", "rich")
+    session.install("packaging", "requests", "cachecontrol[filecache]", "rich", "pkg_metadata")
 
     session.run("python", "scripts/generate.py")
 

scripts/check_zipapp.py

@@ -0,0 +1,21 @@
+import subprocess
+import sys
+
+# This code needs to support all versions of Python we test against
+proc = subprocess.Popen(
+    [sys.executable] + sys.argv[1:],
+    stdout=subprocess.PIPE,
+    stderr=subprocess.PIPE,
+)
+proc.wait()
+
+out = proc.stdout.read()
+err = proc.stderr.read()
+
+if proc.returncode == 0:
+    print(out)
+elif b"does not support python" in err:
+    print(err)
+else:
+    print(err)
+    raise SystemExit("Failed")

scripts/generate.py

@@ -1,20 +1,23 @@
 """Update all the get-pip.py scripts."""
 
+import io
 import itertools
 import operator
 import re
+import shutil
 from base64 import b85encode
 from functools import lru_cache
 from io import BytesIO
 from pathlib import Path
 from typing import Dict, Iterable, List, TextIO, Tuple
-from zipfile import ZipFile
+from zipfile import ZipFile, ZipInfo
 
 import requests
 from cachecontrol import CacheControl
 from cachecontrol.caches.file_cache import FileCache
 from packaging.specifiers import SpecifierSet
 from packaging.version import Version
+from pkg_metadata import bytes_to_json
 from rich.console import Console
 
 SCRIPT_CONSTRAINTS = {
@@ -115,8 +118,8 @@ def get_ordered_templates() -> List[Tuple[Version, Path]]:
     fallback = None
     ordered_templates = []
     for template in all_templates:
-        # `moved.py` isn't one of the templates to be used here.
-        if template.name == "moved.py":
+        # `moved.py` and `zipapp_main.py` aren't templates to be used here.
+        if template.name in ("moved.py", "zipapp_main.py"):
             continue
         if template.name == "default.py":
             fallback = template
@@ -269,6 +272,63 @@ def generate_moved(destination: str, *, location: str, console: Console):
         f.write(rendered_template)
 
 
+def generate_zipapp(pip_version: Version, *, console: Console, pip_versions: Dict[Version, Tuple[str, str]]) -> None:
+    wheel_url, wheel_hash = pip_versions[pip_version]
+    console.log(f"  Downloading [green]{Path(wheel_url).name}")
+    original_wheel = download_wheel(wheel_url, wheel_hash)
+
+    zipapp_name = f"public/pip-{pip_version}.pyz"
+
+    console.log(f"  Creating [green]{zipapp_name}")
+    with open(zipapp_name, "wb") as f:
+        # Write shebang at the start of the file
+        f.write(b"#!/usr/bin/env python\n")
+
+        # Write the remainder of the zipapp as a zipfile
+        with ZipFile(f, mode="w") as dest:
+            console.log("  Copying pip from original wheel to zipapp")
+
+            # Version check - 0 means "don't check"
+            major = 0
+            minor = 0
+            with ZipFile(io.BytesIO(original_wheel)) as src:
+                for info in src.infolist():
+                    # Ignore all content apart from the "pip" subdirectory
+                    if info.filename.startswith("pip/"):
+                        data = src.read(info)
+                        dest.writestr(info, data)
+                    elif info.filename.endswith(".dist-info/METADATA"):
+                        data = bytes_to_json(src.read(info))
+                        if "requires_python" in data:
+                            py_req = data["requires_python"]
+                            py_req = py_req.replace(" ", "")
+                            m = re.match(r"^>=(\d+)\.(\d+)$", py_req)
+                            if m:
+                                major, minor = map(int, m.groups())
+                                console.log(f"  Zipapp requires Python {py_req}")
+                            else:
+                                console.log(f"  Python requirement {py_req} too complex - check skipped")
+
+            # Write the main script
+            # Use a ZipInfo object to ensure reproducibility - otherwise the current time
+            # is embedded in the file. We also set the create_system to 0 (DOS), as otherwise
+            # it defaults to a value that depends on the OS we're running on.
+            main_info = ZipInfo()
+            main_info.filename = "__main__.py"
+            main_info.create_system = 0
+
+            # Note that we explicitly do *not* try to match the newline format
+            # of the source here, as we're writing the content into the zipapp
+            # and we want a reproducible value, i.e., always use the same
+            # newline format.
+            template = Path("templates") / "zipapp_main.py"
+            zipapp_main = template.read_text(encoding="utf-8").format(major=major, minor=minor)
+            dest.writestr(main_info, zipapp_main)
+
+    # Make the unversioned pip.pyz
+    shutil.copyfile(zipapp_name, "public/pip.pyz")
+
+
 def main() -> None:
     console = Console()
     with console.status("Fetching pip versions..."):
@@ -290,6 +350,9 @@ def main() -> None:
                 status.update(f"Working on [magenta]{legacy}")
                 generate_moved(legacy, console=console, location=current)
 
+    with console.status("Generating zipapp...") as status:
+        generate_zipapp(max(pip_versions), console=console, pip_versions=pip_versions)
+
 
 if __name__ == "__main__":
     main()

templates/zipapp_main.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+import sys
+
+# /!\ This version compatibility check section must be Python 2 compatible. /!\
+PYTHON_REQUIRES = ({major}, {minor})
+
+if PYTHON_REQUIRES != (0, 0):
+    def version_str(version):  # type: ignore
+        return ".".join(str(v) for v in version)
+
+    if sys.version_info[:2] < PYTHON_REQUIRES:
+        raise SystemExit(
+            "This version of pip does not support python " +
+            version_str(sys.version_info[:2]) +
+            " (requires >= " +
+            version_str(PYTHON_REQUIRES) +
+            ")."
+        )
+# /!\ Version check done. We can use Python 3 syntax now. /!\
+
+import os
+import runpy
+
+lib = os.path.dirname(__file__)
+sys.path.insert(0, lib)
+runpy.run_module("pip", run_name="__main__")