Ensure setuptools version when running setup.py. (#4753)

Previously, even though python `Interpreter` instances were properly
setup with the required `setuptools` (and `wheel`) requirement extra,
this was not used to execute `setup.py` with a custom list of `--run`
commands.

Fixes #4752

src/python/pants/backend/python/tasks2/setup_py.py

@@ -51,6 +51,19 @@ def __init__(self, source_dir, setup_command, **kw):
     self.__setup_command = setup_command.split()
     super(SetupPyRunner, self).__init__(source_dir, **kw)
 
+  def mixins(self):
+    mixins = super(SetupPyRunner, self).mixins().copy()
+    for (key, version) in self._interpreter.extras:
+      if key == 'setuptools':
+        mixins['setuptools'] = 'setuptools=={}'.format(version)
+        break
+    else:
+      # We know Pants sets up python interpreters with wheel and setuptools via the `PythonSetup`
+      # subsystem; so this should never happen
+      raise AssertionError("Expected interpreter {} to have the extra 'setuptools'"
+                           .format(self._interpreter))
+    return mixins
+
   def _setup_command(self):
     return self.__setup_command
 
@@ -456,9 +469,6 @@ def write_target_source(target, src):
           chroot.copy(os.path.join(target.target_base, src, '__init__.py'),
                       os.path.join(self.SOURCE_ROOT, src, '__init__.py'))
 
-    def write_codegen_source(relpath, abspath):
-      chroot.copy(abspath, os.path.join(self.SOURCE_ROOT, relpath))
-
     def write_target(target):
       for rel_source in target.sources_relative_to_buildroot():
         abs_source_path = os.path.join(get_buildroot(), rel_source)
@@ -473,6 +483,9 @@ def write_target(target):
       elif self.is_resources_target(dependency):
         write_target(dependency)
 
+  def _setup_boilerplate(self):
+    return SETUP_BOILERPLATE
+
   def write_setup(self, root_target, reduced_dependencies, chroot):
     """Write the setup.py of a target.
 
@@ -537,7 +550,7 @@ def convert(input):
     # >>>
     #
     # For more information, see http://bugs.python.org/issue13943
-    chroot.write(SETUP_BOILERPLATE.format(
+    chroot.write(self._setup_boilerplate().format(
       setup_dict=pprint.pformat(convert(setup_keywords), indent=4),
       setup_target=repr(root_target)
     ), 'setup.py')

tests/python/pants_test/backend/python/tasks2/test_setup_py.py

@@ -11,33 +11,125 @@
 from textwrap import dedent
 
 from mock import Mock
+from pex.package import Package
 from twitter.common.collections import OrderedSet
 from twitter.common.dirutil.chroot import Chroot
 
+from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.tasks2.setup_py import SetupPy
 from pants.base.exceptions import TaskError
 from pants.build_graph.build_file_aliases import BuildFileAliases
 from pants.build_graph.prep_command import PrepCommand
 from pants.build_graph.resources import Resources
 from pants.build_graph.target import Target
 from pants.fs.archive import TGZ
-from pants.util.contextutil import temporary_dir, temporary_file
+from pants.util.contextutil import environment_as, temporary_dir, temporary_file
 from pants.util.dirutil import safe_mkdir
 from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase
 from pants_test.subsystem.subsystem_util import init_subsystem
 
 
-class TestSetupPy(PythonTaskTestBase):
+class SetupPyTestBase(PythonTaskTestBase):
   @classmethod
   def task_type(cls):
     return SetupPy
 
   def setUp(self):
-    super(TestSetupPy, self).setUp()
-    distdir = os.path.join(self.build_root, 'dist')
-    self.set_options(pants_distdir=distdir)
+    super(SetupPyTestBase, self).setUp()
+    self.distdir = os.path.join(self.build_root, 'dist')
+    self.set_options(pants_distdir=self.distdir)
     init_subsystem(Target.Arguments)
 
+  @contextmanager
+  def run_execute(self, target, recursive=False):
+    self.set_options(recursive=recursive)
+    context = self.context(target_roots=[target])
+    setup_py = self.create_task(context)
+    setup_py.execute()
+    yield context.products.get_data(SetupPy.PYTHON_DISTS_PRODUCT)
+
+
+class TestSetupPyInterpreter(SetupPyTestBase):
+  class PythonPathInspectableSetupPy(SetupPy):
+    def _setup_boilerplate(self):
+      return dedent("""
+        # DO NOT EDIT THIS FILE -- AUTOGENERATED BY PANTS
+        # Target: {setup_target}
+
+        from setuptools import setup
+
+        from foo.commands.print_sys_path import PrintSysPath
+
+        setup(
+          cmdclass={{'print_sys_path': PrintSysPath}},
+          **{setup_dict}
+        )
+        """)
+
+  @classmethod
+  def task_type(cls):
+    return cls.PythonPathInspectableSetupPy
+
+  def test_setuptools_version(self):
+    self.create_file('src/python/foo/__init__.py')
+    self.create_python_library(
+      relpath='src/python/foo/commands',
+      name='commands',
+      source_contents_map={
+        'print_sys_path.py': dedent("""
+          import os
+          import sys
+          from setuptools import Command
+
+
+          class PrintSysPath(Command):
+            user_options = []
+
+            def initialize_options(self):
+              pass
+
+            def finalize_options(self):
+              pass
+
+            def run(self):
+              with open(os.path.join(os.path.dirname(__file__), 'sys_path.txt'), 'w') as fp:
+                fp.write(os.linesep.join(sys.path))
+          """)
+      },
+    )
+    foo = self.create_python_library(
+      relpath='src/python/foo',
+      name='foo',
+      dependencies=[
+        'src/python/foo/commands',
+      ],
+      provides=dedent("""
+      setup_py(
+        name='foo',
+        version='0.0.0',
+      )
+      """)
+    )
+    self.set_options(run='print_sys_path')
+
+    # Make sure setup.py can see our custom distutils Command 'print_sys_path'.
+    sdist_srcdir = os.path.join(self.distdir, 'foo-0.0.0', 'src')
+    with environment_as(PYTHONPATH=sdist_srcdir):
+      with self.run_execute(foo):
+        with open(os.path.join(sdist_srcdir, 'foo', 'commands', 'sys_path.txt')) as fp:
+          # The 1st element of the sys.path should be our custom SetupPyRunner Installer's
+          # setuptools mixin, which should match the setuptools version specified by the
+          # PythonSetup subsystem.
+          package = Package.from_href(fp.readline().strip())
+          self.assertEqual('setuptools', package.name)
+          init_subsystem(PythonSetup)
+          self.assertEqual(PythonSetup.global_instance().setuptools_version, package.raw_version)
+
+
+class TestSetupPy(SetupPyTestBase):
+
+  def setUp(self):
+    super(TestSetupPy, self).setUp()
     self.dependency_calculator = SetupPy.DependencyCalculator(self.build_graph)
 
   @property
@@ -80,14 +172,6 @@ def test_reduced_dependencies_1(self):
     self.assert_requirements(target_map['bar'], {'baz==0.0.0'})
     self.assert_requirements(target_map['baz'], set())
 
-  @contextmanager
-  def run_execute(self, target, recursive=False):
-    self.set_options(recursive=recursive)
-    context = self.context(target_roots=[target])
-    setup_py = self.create_task(context)
-    setup_py.execute()
-    yield context.products.get_data(SetupPy.PYTHON_DISTS_PRODUCT)
-
   def test_execution_reduced_dependencies_1(self):
     dep_map = OrderedDict(foo=['bar'], bar=['baz'], baz=[])
     target_map = self.create_dependencies(dep_map)