combine Clang, GCC, Binutils, and XCodeCLITools to form the NativeToolchain subsystem and consume it in BuildLocalPythonDistributions for native code

examples/src/python/example/python_distribution/hello/fasthello/BUILD

@@ -7,7 +7,8 @@
 python_dist(
   name='fasthello',
   sources=[
-    'super_greet.c',
+    'c_greet.c',
+    'cpp_greet.cpp',
     'hello_package/hello.py',
     'hello_package/__init__.py',
     'setup.py'

examples/src/python/example/python_distribution/hello/fasthello/c_greet.c

@@ -0,0 +1,14 @@
+#include <Python.h>
+
+static PyObject * c_greet(PyObject *self, PyObject *args) {
+  return Py_BuildValue("s", "Hello from C!");
+}
+
+static PyMethodDef Methods[] = {
+  {"c_greet", c_greet, METH_VARARGS, "A greeting in the C language."},
+  {NULL, NULL, 0, NULL}
+};
+
+PyMODINIT_FUNC initc_greet(void) {
+  (void) Py_InitModule("c_greet", Methods);
+}

examples/src/python/example/python_distribution/hello/fasthello/cpp_greet.cpp

@@ -0,0 +1,14 @@
+#include <Python.h>
+
+static PyObject * cpp_greet(PyObject *self, PyObject *args) {
+  return Py_BuildValue("s", "Hello from C++!");
+}
+
+static PyMethodDef Methods[] = {
+  {"cpp_greet", cpp_greet, METH_VARARGS, "A greeting in the C++ language."},
+  {NULL, NULL, 0, NULL}
+};
+
+PyMODINIT_FUNC initcpp_greet(void) {
+  (void) Py_InitModule("cpp_greet", Methods);
+}

examples/src/python/example/python_distribution/hello/fasthello/hello_package/hello.py

@@ -5,8 +5,11 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
-import super_greet
+import c_greet
+import cpp_greet
 
 def hello():
-  print(super_greet.super_greet())
-  return super_greet.super_greet()
+  return '\n'.join([
+    c_greet.c_greet(),
+    cpp_greet.cpp_greet(),
+  ])

examples/src/python/example/python_distribution/hello/fasthello/main.py

@@ -10,4 +10,4 @@
 
 
 if __name__ == '__main__':
-  hello.hello()
+  print(hello.hello())

examples/src/python/example/python_distribution/hello/fasthello/setup.py

@@ -9,11 +9,12 @@
 from distutils.core import Extension
 
 
-c_module = Extension(str('super_greet'), sources=[str('super_greet.c')])
+c_module = Extension(str('c_greet'), sources=[str('c_greet.c')])
+cpp_module = Extension(str('cpp_greet'), sources=[str('cpp_greet.cpp')])
 
 setup(
   name='fasthello',
   version='1.0.0',
-  ext_modules=[c_module],
+  ext_modules=[c_module, cpp_module],
   packages=find_packages(),
 )

examples/tests/python/example/python_distribution/hello/test_fasthello/test_fasthello.py

@@ -11,4 +11,7 @@
 
 # Example of writing a test that depends on a python_dist target.
 def test_fasthello():
-  assert hello.hello() == "Super hello"
+  assert hello.hello() == '\n'.join([
+    'Hello from C!',
+    'Hello from C++!',
+  ])

src/python/pants/backend/native/subsystems/BUILD

@@ -3,6 +3,13 @@
 
 python_library(
   dependencies=[
+    'src/python/pants/backend/native/subsystems/platform_specific/darwin',
+    'src/python/pants/backend/native/subsystems/platform_specific/linux',
     'src/python/pants/binaries:binary_util',
+    'src/python/pants/subsystem',
+    'src/python/pants/util:contextutil',
+    'src/python/pants/util:memo',
+    'src/python/pants/util:osutil',
+    'src/python/pants/util:process_handler',
   ],
 )

src/python/pants/backend/native/subsystems/clang.py

@@ -0,0 +1,19 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import os
+
+from pants.binaries.binary_tool import ExecutablePathProvider, NativeTool
+
+
+class Clang(NativeTool, ExecutablePathProvider):
+  options_scope = 'clang'
+  default_version = '6.0.0'
+  archive_type = 'tgz'
+
+  def path_entries(self):
+    return [os.path.join(self.select(), 'bin')]

src/python/pants/backend/native/subsystems/llvm.py → src/python/pants/backend/native/subsystems/gcc.py

@@ -5,10 +5,15 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
-from pants.binaries.binary_tool import NativeTool
+import os
 
+from pants.binaries.binary_tool import ExecutablePathProvider, NativeTool
 
-class LLVM(NativeTool):
-  options_scope = 'llvm'
-  default_version = '5.0.1'
+
+class GCC(NativeTool, ExecutablePathProvider):
+  options_scope = 'gcc'
+  default_version = '7.3.0'
   archive_type = 'tgz'
+
+  def path_entries(self):
+    return [os.path.join(self.select(), 'bin')]

src/python/pants/backend/native/subsystems/native_toolchain.py

@@ -0,0 +1,101 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.backend.native.subsystems.clang import Clang
+from pants.backend.native.subsystems.gcc import GCC
+from pants.backend.native.subsystems.platform_specific.darwin.xcode_cli_tools import XCodeCLITools
+from pants.backend.native.subsystems.platform_specific.linux.binutils import Binutils
+from pants.binaries.binary_tool import ExecutablePathProvider
+from pants.subsystem.subsystem import Subsystem
+from pants.util.memo import memoized_method
+from pants.util.osutil import get_os_name, normalize_os_name
+
+
+class NativeToolchain(Subsystem, ExecutablePathProvider):
+  """Abstraction over platform-specific tools to compile and link native code.
+
+  This "native toolchain" subsystem is an abstraction that exposes directories
+  containing executables to compile and link "native" code (for now, C and C++
+  are supported). Consumers of this subsystem can add these directories to their
+  PATH to invoke subprocesses which use these tools.
+
+  This abstraction is necessary for two reasons. First, because there are
+  multiple binaries involved in compilation and linking, which often invoke
+  other binaries that must also be available on the PATH. Second, because unlike
+  other binary tools in Pants, we can't provide the same package built for both
+  OSX and Linux, because there is no open-source linker for OSX with a
+  compatible license.
+
+  So when this subsystem is consumed, Pants will download and unpack archives
+  (if necessary) which together provide an appropriate "native toolchain" for
+  the host platform. On OSX, Pants will also find and provide path entries for
+  the XCode command-line tools, or error out with installation instructions if
+  the XCode tools could not be found.
+  """
+
+  options_scope = 'native-toolchain'
+
+  # This is a list of subsystems which implement `ExecutablePathProvider` and
+  # can be provided for all supported platforms.
+  _CROSS_PLATFORM_SUBSYSTEMS = [Clang, GCC]
+
+  # This is a map of {<platform> -> [<subsystem_cls>, ...]}; the key is the
+  # normalized OS name, and the value is a list of subsystem class objects that
+  # implement `ExecutablePathProvider`. The native toolchain subsystem will
+  # declare dependencies only on the subsystems for the platform Pants is
+  # executing on.
+  _PLATFORM_SPECIFIC_SUBSYSTEMS = {
+    'darwin': [XCodeCLITools],
+    'linux': [Binutils],
+  }
+
+  class UnsupportedPlatformError(Exception):
+    """Thrown if the native toolchain is invoked on an unrecognized platform.
+
+    Note that the native toolchain should work on all of Pants's supported
+    platforms."""
+
+  @classmethod
+  @memoized_method
+  def _get_platform_specific_subsystems(cls):
+    """Return the subsystems used by the native toolchain for this platform."""
+    os_name = get_os_name()
+    normed_os_name = normalize_os_name(os_name)
+
+    subsystems_for_host = cls._PLATFORM_SPECIFIC_SUBSYSTEMS.get(normed_os_name, None)
+
+    if subsystems_for_host is None:
+      raise cls.UnsupportedPlatformError(
+        "Pants doesn't support building native code on this platform "
+        "(uname: '{}').".format(os_name))
+
+    # NB: path entries for cross-platform subsystems currently take precedence
+    # over platform-specific ones -- this could be made configurable.
+    all_subsystems_for_toolchain = cls._CROSS_PLATFORM_SUBSYSTEMS + subsystems_for_host
+
+    return all_subsystems_for_toolchain
+
+  @classmethod
+  def subsystem_dependencies(cls):
+    prev = super(NativeToolchain, cls).subsystem_dependencies()
+    cur_platform_subsystems = cls._get_platform_specific_subsystems()
+    return prev + tuple(sub.scoped(cls) for sub in cur_platform_subsystems)
+
+  @memoized_method
+  def _subsystem_instances(self):
+    cur_platform_subsystems = self._get_platform_specific_subsystems()
+    return [sub.scoped_instance(self) for sub in cur_platform_subsystems]
+
+  # TODO(cosmicexplorer): We might want to run a very small test suite verifying
+  # the toolchain can compile and link native code before returning, especially
+  # since we don't provide the tools on OSX.
+  def path_entries(self):
+    combined_path_entries = []
+    for subsystem in self._subsystem_instances():
+      combined_path_entries.extend(subsystem.path_entries())
+
+    return combined_path_entries

src/python/pants/backend/native/subsystems/platform_specific/darwin/BUILD

@@ -0,0 +1,11 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library(
+  dependencies=[
+    'src/python/pants/binaries:binary_util',
+    'src/python/pants/subsystem',
+    'src/python/pants/util:dirutil',
+  ],
+)

src/python/pants/backend/native/subsystems/platform_specific/darwin/xcode_cli_tools.py

@@ -0,0 +1,44 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import os
+
+from pants.binaries.binary_tool import ExecutablePathProvider
+from pants.subsystem.subsystem import Subsystem
+from pants.util.dirutil import is_executable
+
+
+class XCodeCLITools(Subsystem, ExecutablePathProvider):
+  """Subsystem to detect and provide the XCode command line developer tools.
+
+  This subsystem exists to give a useful error message if the tools aren't
+  installed, and because the install location may not be on the PATH when Pants
+  is invoked."""
+
+  options_scope = 'xcode-cli-tools'
+
+  # TODO(cosmicexplorer): make this an option?
+  _INSTALL_LOCATION = '/usr/bin'
+
+  _REQUIRED_TOOLS = frozenset(['cc', 'c++', 'ld', 'lipo'])
+
+  class XCodeToolsUnavailable(Exception):
+    """Thrown if the XCode CLI tools could not be located."""
+
+  def _check_executables_exist(self):
+    for filename in self._REQUIRED_TOOLS:
+      executable_path = os.path.join(self._INSTALL_LOCATION, filename)
+      if not is_executable(executable_path):
+        raise self.XCodeToolsUnavailable(
+          "'{}' is not an executable file, but it is required to build "
+          "native code on this platform. You may need to install the XCode "
+          "command line developer tools."
+          .format(executable_path))
+
+  def path_entries(self):
+    self._check_executables_exist()
+    return [self._INSTALL_LOCATION]

src/python/pants/backend/native/subsystems/platform_specific/linux/BUILD

@@ -0,0 +1,8 @@
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library(
+  dependencies=[
+    'src/python/pants/binaries:binary_util',
+  ],
+)

src/python/pants/backend/native/subsystems/platform_specific/linux/binutils.py

@@ -0,0 +1,19 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import os
+
+from pants.binaries.binary_tool import ExecutablePathProvider, NativeTool
+
+
+class Binutils(NativeTool, ExecutablePathProvider):
+  options_scope = 'binutils'
+  default_version = '2.30'
+  archive_type = 'tgz'
+
+  def path_entries(self):
+    return [os.path.join(self.select(), 'bin')]

src/python/pants/backend/python/tasks/BUILD

@@ -6,6 +6,7 @@ python_library(
     '3rdparty/python:pex',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     '3rdparty/python/twitter/commons:twitter.common.dirutil',
+    'src/python/pants/backend/native/subsystems',
     'src/python/pants/backend/python:python_requirement',
     'src/python/pants/backend/python:python_requirements',
     'src/python/pants/backend/python:interpreter_cache',