rewrite BinaryTool a little and remove native toolchain subsystem

src/python/pants/backend/codegen/protobuf/subsystem/protoc.py

@@ -5,12 +5,14 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
+import os
+
 from pants.binaries.binary_tool import NativeTool
 
 
 class Protoc(NativeTool):
   options_scope = 'protoc'
-  support_dir = 'bin/protobuf'
+  support_subdir = 'protobuf'
   default_version = '2.4.1'
 
   deprecated_option_scope = 'gen.protoc'

src/python/pants/backend/native/subsystems/BUILD

@@ -0,0 +1,8 @@
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library(
+  dependencies=[
+    'src/python/pants/binaries:binary_util',
+  ],
+)

src/python/pants/backend/native/subsystems/ld.py

@@ -0,0 +1,14 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.binaries.binary_tool import NativeTool
+
+
+class LD(NativeTool):
+  options_scope = 'ld'
+  # TODO: figure out how to express the "mutual" version for linux and osx ld
+  default_version = '???'

src/python/pants/backend/native/subsystems/llvm.py

@@ -0,0 +1,14 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.binaries.binary_tool import NativeTool
+
+
+class LLVM(NativeTool):
+  options_scope = 'llvm'
+  default_version = '5.0.1'
+  archive_type = 'tgz'

src/python/pants/backend/python/BUILD

@@ -28,6 +28,7 @@ python_library(
     ':python_chroot',
     ':python_requirement',
     ':python_requirements',
+    ':sandboxed_interpreter',
     ':sdist_builder',
     ':thrift_builder',
   ]
@@ -87,7 +88,6 @@ python_library(
   ]
 )
 
-
 python_library(
   name = 'python_artifact',
   sources = ['python_artifact.py'],
@@ -125,6 +125,11 @@ python_library(
   sources = ['python_requirements.py'],
 )
 
+python_library(
+  name = 'sandboxed_interpreter',
+  sources = ['sandboxed_interpreter.py'],
+)
+
 python_library(
   name = 'sdist_builder',
   sources = ['sdist_builder.py'],

src/python/pants/backend/python/sandboxed_interpreter.py

@@ -0,0 +1,81 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import os
+
+# from pex.executor import Executor
+from pex.interpreter import PythonInterpreter
+
+from pants.util.memo import memoized_method
+
+
+# INC_DIR_INPUT = b"""
+# import sys
+# from distutils import sysconfig
+
+# sys.stdout.write(sysconfig.get_python_inc())
+# """
+
+
+class SandboxedInterpreter(PythonInterpreter):
+
+  class ToolchainLocationError(Exception):
+    def __init__(self, dir_path):
+      msg = "path '{}' does not exist or is not a directory".format(dir_path)
+      super(ToolchainLocationError, self).__init__(msg)
+
+  class BaseInterpreterError(Exception): pass
+
+  # using another PythonInterpreter to populate the superclass constructor args
+  def __init__(self, llvm_base_dir, base_interp):
+
+    if not os.path.isdir(llvm_base_dir):
+      raise ToolchainLocationError(llvm_base_dir)
+    if not isinstance(base_interp, PythonInterpreter):
+      raise BaseInterpreterError(
+        "invalid PythonInterpreter: '{}'".format(repr(base_interp)))
+
+    self._llvm_base_dir = llvm_base_dir
+
+    # this feels a little hacky -- what if pex's PythonInterpreter later needs
+    # another constructor arg that's not just a property of the class?
+    super(SandboxedInterpreter, self).__init__(
+      base_interp.binary, base_interp.identity, extras=base_interp.extras)
+
+  # made into an instance method here (unlike PythonInterpreter superclass) to
+  # use instance property self._llvm_base_dir
+  @memoized_method
+  def sanitized_environment(self):
+    sanitized_env = super(SandboxedInterpreter, self).sanitized_environment()
+
+    # use our compiler at the front of the path
+    # TODO: when we provide ld, remove the previous PATH entries
+    sanitized_env['PATH'] = ':'.join([
+      os.path.join(self._llvm_base_dir, 'bin'),
+      os.environ.get('PATH'),
+    ])
+
+    # llvm_include = os.path.join(self._llvm_base_dir, 'include')
+    # python_inc_stdout, _ = Executor.execute([self.binary], env=sanitized_env, stdin_payload=INC_DIR_INPUT)
+    # sanitized_env['CPATH'] = '{}:{}'.format(llvm_include, python_inc_stdout)
+
+    # TODO: we may not need this. if removed, (probably) remove the 'lib/' dir
+    # from the llvm packaging script too!
+    # sanitized_env['LD_LIBRARY_PATH'] = os.path.join(self._llvm_base_dir, 'lib')
+
+    # TODO: see Lib/distutils/sysconfig.py and Lib/_osx_support.py in CPython.
+    # this line tells distutils to only compile for 64-bit archs -- if not, it
+    # will attempt to build a fat binary for 32- and 64-bit archs, which makes
+    # clang invoke "lipo", an osx command which does not appear to be open
+    # source.
+    sanitized_env['ARCHFLAGS'] = '-arch x86_64'
+
+    env_vars_to_scrub = ['CC', 'CXX']
+    for env_var in env_vars_to_scrub:
+      sanitized_env.pop(env_var, None)
+
+    return sanitized_env

src/python/pants/backend/python/tasks/BUILD

@@ -6,6 +6,9 @@ python_library(
     '3rdparty/python:pex',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     '3rdparty/python/twitter/commons:twitter.common.dirutil',
+    'src/python/pants/backend/native',
+    # FIXME: why does this still work?
+    # 'src/python/pants/backend/python:sandboxed_interpreter',
     'src/python/pants/backend/python:python_requirement',
     'src/python/pants/backend/python:python_requirements',
     'src/python/pants/backend/python:interpreter_cache',

src/python/pants/backend/python/tasks/build_local_python_distributions.py

@@ -13,7 +13,8 @@
 
 from pex.interpreter import PythonInterpreter
 
-from pants.backend.python.subsystems.python_native_toolchain import PythonNativeToolchain, SandboxedInterpreter
+from pants.backend.native.subsystems.llvm import LLVM
+from pants.backend.python.sandboxed_interpreter import SandboxedInterpreter
 from pants.backend.python.tasks.pex_build_util import is_local_python_dist
 from pants.backend.python.tasks.setup_py import SetupPyRunner
 from pants.base.build_environment import get_buildroot
@@ -25,17 +26,6 @@
 from pants.util.memo import memoized_property
 
 
-PANTSSETUP_IMPORT_BOILERPLATE = """
-# DO NOT EDIT THIS FILE -- AUTOGENERATED BY PANTS
-# Target: {setup_target}
-
-from distutils.core import Extension
-
-def find_external_modules():
-  return [Extension(str('native'), [{native_sources_joined}])]
-"""
-
-
 class BuildLocalPythonDistributions(Task):
   """Create python distributions (.whl) from python_dist targets."""
 
@@ -56,24 +46,19 @@ def implementation_version(cls):
 
   @classmethod
   def subsystem_dependencies(cls):
-    return super(BuildLocalPythonDistributions, cls).subsystem_dependencies() + (PythonNativeToolchain.Factory.scoped(cls),)
-
-  @classmethod
-  def register_options(cls, register):
-    super(BuildLocalPythonDistributions, cls).register_options(register)
-    register('--an-option', default='wow')
+    return super(BuildLocalPythonDistributions, cls).subsystem_dependencies() + (
+      LLVM.scoped(cls),
+    )
 
   @memoized_property
-  def python_native_toolchain(self):
-    return PythonNativeToolchain.Factory.scoped_instance(self).create()
+  def llvm_base_dir(self):
+    return LLVM.scoped_instance(self).select()
 
   @property
   def cache_target_dirs(self):
     return True
 
   def execute(self):
-    self.context.log.debug('an_option: {}'.format(self.context.options.for_scope(self.options_scope).an_option))
-
     dist_targets = self.context.targets(is_local_python_dist)
     built_dists = set()
 
@@ -117,7 +102,7 @@ def _create_dist(self, dist_tgt, dist_target_dir):
     self.context.log.info('dist_target_dir: {}'.format(dist_target_dir))
     interpreter = self.context.products.get_data(PythonInterpreter)
     sandboxed_interpreter = SandboxedInterpreter(
-      self.python_native_toolchain.llvm_toolchain_dir(), interpreter)
+      self.llvm_base_dir, interpreter)
     self._copy_sources(dist_tgt, dist_target_dir)
     # Build a whl using SetupPyRunner and return its absolute path.
     setup_runner = SetupPyRunner(dist_target_dir, 'bdist_wheel', interpreter=sandboxed_interpreter)

src/python/pants/backend/python/tasks/resolve_requirements_task_base.py

@@ -64,7 +64,7 @@ def resolve_requirements(self, req_libs, local_dist_targets=None):
             BuildLocalPythonDistributions.PYTHON_DISTS)
 
           if built_dists is not None:
-            synthetic_address = ':'.join(2 * [binary_tgt.invalidation_hash()])
+            synthetic_address = ':'.join(2 * [target_set_id])
 
             local_dist_req_lib = build_req_lib_provided_by_setup_file(
               self.context.build_graph,