diff --git a/src/python/pants/backend/codegen/protobuf/subsystem/protoc.py b/src/python/pants/backend/codegen/protobuf/subsystem/protoc.py index e5298eab5d2..261c6f7e127 100644 --- a/src/python/pants/backend/codegen/protobuf/subsystem/protoc.py +++ b/src/python/pants/backend/codegen/protobuf/subsystem/protoc.py @@ -5,12 +5,14 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) +import os + from pants.binaries.binary_tool import NativeTool class Protoc(NativeTool): options_scope = 'protoc' - support_dir = 'bin/protobuf' + support_subdir = 'protobuf' default_version = '2.4.1' deprecated_option_scope = 'gen.protoc' diff --git a/src/python/pants/backend/native/__init__.py b/src/python/pants/backend/native/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/native/subsystems/BUILD b/src/python/pants/backend/native/subsystems/BUILD new file mode 100644 index 00000000000..8db0b45aa03 --- /dev/null +++ b/src/python/pants/backend/native/subsystems/BUILD @@ -0,0 +1,8 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_library( + dependencies=[ + 'src/python/pants/binaries:binary_util', + ], +) diff --git a/src/python/pants/backend/native/subsystems/__init__.py b/src/python/pants/backend/native/subsystems/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/native/subsystems/ld.py b/src/python/pants/backend/native/subsystems/ld.py new file mode 100644 index 00000000000..a8536d1c077 --- /dev/null +++ b/src/python/pants/backend/native/subsystems/ld.py @@ -0,0 +1,14 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.binaries.binary_tool import NativeTool + + +class LD(NativeTool): + options_scope = 'ld' + # TODO: figure out how to express the "mutual" version for linux and osx ld + default_version = '???' diff --git a/src/python/pants/backend/native/subsystems/llvm.py b/src/python/pants/backend/native/subsystems/llvm.py new file mode 100644 index 00000000000..9c97eed26ec --- /dev/null +++ b/src/python/pants/backend/native/subsystems/llvm.py @@ -0,0 +1,14 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.binaries.binary_tool import NativeTool + + +class LLVM(NativeTool): + options_scope = 'llvm' + default_version = '5.0.1' + archive_type = 'tgz' diff --git a/src/python/pants/backend/python/BUILD b/src/python/pants/backend/python/BUILD index 79bbbb292e5..b879bbd6666 100644 --- a/src/python/pants/backend/python/BUILD +++ b/src/python/pants/backend/python/BUILD @@ -28,6 +28,7 @@ python_library( ':python_chroot', ':python_requirement', ':python_requirements', + ':sandboxed_interpreter', ':sdist_builder', ':thrift_builder', ] @@ -87,7 +88,6 @@ python_library( ] ) - python_library( name = 'python_artifact', sources = ['python_artifact.py'], @@ -125,6 +125,11 @@ python_library( sources = ['python_requirements.py'], ) +python_library( + name = 'sandboxed_interpreter', + sources = ['sandboxed_interpreter.py'], +) + python_library( name = 'sdist_builder', sources = ['sdist_builder.py'], diff --git a/src/python/pants/backend/python/sandboxed_interpreter.py b/src/python/pants/backend/python/sandboxed_interpreter.py new file mode 100644 index 00000000000..82bdc5b4eee --- /dev/null +++ b/src/python/pants/backend/python/sandboxed_interpreter.py @@ -0,0 +1,81 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import os + +# from pex.executor import Executor +from pex.interpreter import PythonInterpreter + +from pants.util.memo import memoized_method + + +# INC_DIR_INPUT = b""" +# import sys +# from distutils import sysconfig + +# sys.stdout.write(sysconfig.get_python_inc()) +# """ + + +class SandboxedInterpreter(PythonInterpreter): + + class ToolchainLocationError(Exception): + def __init__(self, dir_path): + msg = "path '{}' does not exist or is not a directory".format(dir_path) + super(ToolchainLocationError, self).__init__(msg) + + class BaseInterpreterError(Exception): pass + + # using another PythonInterpreter to populate the superclass constructor args + def __init__(self, llvm_base_dir, base_interp): + + if not os.path.isdir(llvm_base_dir): + raise ToolchainLocationError(llvm_base_dir) + if not isinstance(base_interp, PythonInterpreter): + raise BaseInterpreterError( + "invalid PythonInterpreter: '{}'".format(repr(base_interp))) + + self._llvm_base_dir = llvm_base_dir + + # this feels a little hacky -- what if pex's PythonInterpreter later needs + # another constructor arg that's not just a property of the class? + super(SandboxedInterpreter, self).__init__( + base_interp.binary, base_interp.identity, extras=base_interp.extras) + + # made into an instance method here (unlike PythonInterpreter superclass) to + # use instance property self._llvm_base_dir + @memoized_method + def sanitized_environment(self): + sanitized_env = super(SandboxedInterpreter, self).sanitized_environment() + + # use our compiler at the front of the path + # TODO: when we provide ld, remove the previous PATH entries + sanitized_env['PATH'] = ':'.join([ + os.path.join(self._llvm_base_dir, 'bin'), + os.environ.get('PATH'), + ]) + + # llvm_include = os.path.join(self._llvm_base_dir, 'include') + # python_inc_stdout, _ = Executor.execute([self.binary], env=sanitized_env, stdin_payload=INC_DIR_INPUT) + # sanitized_env['CPATH'] = '{}:{}'.format(llvm_include, python_inc_stdout) + + # TODO: we may not need this. if removed, (probably) remove the 'lib/' dir + # from the llvm packaging script too! + # sanitized_env['LD_LIBRARY_PATH'] = os.path.join(self._llvm_base_dir, 'lib') + + # TODO: see Lib/distutils/sysconfig.py and Lib/_osx_support.py in CPython. + # this line tells distutils to only compile for 64-bit archs -- if not, it + # will attempt to build a fat binary for 32- and 64-bit archs, which makes + # clang invoke "lipo", an osx command which does not appear to be open + # source. + sanitized_env['ARCHFLAGS'] = '-arch x86_64' + + env_vars_to_scrub = ['CC', 'CXX'] + for env_var in env_vars_to_scrub: + sanitized_env.pop(env_var, None) + + return sanitized_env diff --git a/src/python/pants/backend/python/subsystems/python_native_toolchain.py b/src/python/pants/backend/python/subsystems/python_native_toolchain.py deleted file mode 100644 index c8bc445348a..00000000000 --- a/src/python/pants/backend/python/subsystems/python_native_toolchain.py +++ /dev/null @@ -1,135 +0,0 @@ -# coding=utf-8 -# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import (absolute_import, division, generators, nested_scopes, print_function, - unicode_literals, with_statement) - -import os -from contextlib import contextmanager - -from pex.executor import Executor -from pex.interpreter import PythonInterpreter - -from pants.binaries.binary_util import BinaryUtil -from pants.fs.archive import TGZ -from pants.subsystem.subsystem import Subsystem -from pants.util.contextutil import temporary_dir, environment_as -from pants.util.memo import memoized_method - - -INC_DIR_INPUT = b""" -import sys -from distutils import sysconfig - -sys.stdout.write(sysconfig.get_python_inc()) -""" - - -class SandboxedInterpreter(PythonInterpreter): - - class ToolchainLocationError(Exception): - def __init__(self, dir_path): - msg = "path '{}' does not exist or is not a directory".format(dir_path) - super(ToolchainLocationError, self).__init__(msg) - - class BaseInterpreterError(Exception): pass - - def __init__(self, llvm_toolchain_dir, base_interp): - - if not os.path.isdir(llvm_toolchain_dir): - raise ToolchainLocationError(llvm_toolchain_dir) - if not isinstance(base_interp, PythonInterpreter): - raise BaseInterpreterError("invalid PythonInterpreter: '{}'".format(repr(base_interp))) - - self._llvm_toolchain_dir = llvm_toolchain_dir - - super(SandboxedInterpreter, self).__init__( - base_interp.binary, base_interp.identity, extras=base_interp.extras) - - # made into an instance method here (unlike parent class) to use - # self._llvm_toolchain_dir - @memoized_method - def sanitized_environment(self): - pre_sanitized_env = super(SandboxedInterpreter, self).sanitized_environment() - pre_sanitized_env['PATH'] = ':'.join([ - os.path.join(self._llvm_toolchain_dir, 'bin'), - '/usr/bin', - ]) - - llvm_include = os.path.join(self._llvm_toolchain_dir, 'include') - python_inc_stdout, _ = Executor.execute([self.binary], env=pre_sanitized_env, stdin_payload=INC_DIR_INPUT) - pre_sanitized_env['CPATH'] = '{}:{}'.format(llvm_include, python_inc_stdout) - - # TODO: we may not need this. if removed, (probably) remove the 'lib/' dir - # from the llvm packaging script too! - # pre_sanitized_env['LD_LIBRARY_PATH'] = os.path.join(self._llvm_toolchain_dir, 'lib') - # TODO: see Lib/distutils/sysconfig.py and Lib/_osx_support.py in CPython. - # this line tells distutils to only compile for 64-bit archs -- if not, it - # will attempt to build a fat binary for 32- and 64-bit archs, which makes - # clang invoke "lipo", an osx command which does not appear to be open - # source. - pre_sanitized_env['ARCHFLAGS'] = '-arch x86_64' - for env_var in ['CC', 'CXX']: - pre_sanitized_env.pop(env_var, None) - return pre_sanitized_env - - -class PythonNativeToolchain(object): - """Represents a self-boostrapping set of binaries and libraries used to - compile native code in for python dists.""" - - class InvalidToolRequest(Exception): - - def __init__(self, rel_path_requested): - msg = "relative path '{}' does not exist in the python native toolchain".format(rel_path_requested) - super(InvalidToolRequest, self).__init__(msg) - - class Factory(Subsystem): - options_scope = 'python-native-toolchain' - - @classmethod - def subsystem_dependencies(cls): - return super(PythonNativeToolchain.Factory, cls).subsystem_dependencies() + (BinaryUtil.Factory,) - - @classmethod - def register_options(cls, register): - super(PythonNativeToolchain.Factory, cls).register_options(register) - register('--supportdir', advanced=True, - help='Find the go distributions under this dir. Used as part ' - 'of the path to lookup the distribution with ' - '--binary-util-baseurls and --pants-bootstrapdir', - default='bin/python-native-toolchain') - register('--llvm-version', advanced=True, - help='LLVM version used to compile python native extensions. ' - 'Used as part of the path to lookup the distribution ' - 'with --binary-util-baseurls and --pants-bootstrapdir', - default='5.0.1') - - # NB: create() is an instance method to allow the user to choose global or - # scoped -- It's not unreasonable to imagine different stacks for different - # python versions. Get an instance of this with - # PythonNativeToolchain.Factory.scoped_instance(self).create()! - def create(self): - binary_util = BinaryUtil.Factory.create() - options = self.get_options() - return PythonNativeToolchain(binary_util=binary_util, - relpath=options.supportdir, - llvm_version=options.llvm_version) - - def __init__(self, binary_util, relpath, llvm_version): - self._binary_util = binary_util - self._relpath = relpath - self._llvm_version = llvm_version - - @memoized_method - def llvm_toolchain_dir(self): - llvm_archive_path = self._binary_util.select_binary( - self._relpath, self._llvm_version, 'llvm-tools.tar.gz') - distribution_workdir = os.path.dirname(llvm_archive_path) - outdir = os.path.join(distribution_workdir, 'unpacked') - if not os.path.exists(outdir): - with temporary_dir(root_dir=distribution_workdir) as tmp_dist: - TGZ.extract(llvm_archive_path, tmp_dist) - os.rename(tmp_dist, outdir) - return outdir diff --git a/src/python/pants/backend/python/tasks/BUILD b/src/python/pants/backend/python/tasks/BUILD index ae2ced4ea59..0158bb6718b 100644 --- a/src/python/pants/backend/python/tasks/BUILD +++ b/src/python/pants/backend/python/tasks/BUILD @@ -6,6 +6,9 @@ python_library( '3rdparty/python:pex', '3rdparty/python/twitter/commons:twitter.common.collections', '3rdparty/python/twitter/commons:twitter.common.dirutil', + 'src/python/pants/backend/native', + # FIXME: why does this still work? + # 'src/python/pants/backend/python:sandboxed_interpreter', 'src/python/pants/backend/python:python_requirement', 'src/python/pants/backend/python:python_requirements', 'src/python/pants/backend/python:interpreter_cache', diff --git a/src/python/pants/backend/python/tasks/build_local_python_distributions.py b/src/python/pants/backend/python/tasks/build_local_python_distributions.py index ac03cfa359a..8004768e631 100644 --- a/src/python/pants/backend/python/tasks/build_local_python_distributions.py +++ b/src/python/pants/backend/python/tasks/build_local_python_distributions.py @@ -13,7 +13,8 @@ from pex.interpreter import PythonInterpreter -from pants.backend.python.subsystems.python_native_toolchain import PythonNativeToolchain, SandboxedInterpreter +from pants.backend.native.subsystems.llvm import LLVM +from pants.backend.python.sandboxed_interpreter import SandboxedInterpreter from pants.backend.python.tasks.pex_build_util import is_local_python_dist from pants.backend.python.tasks.setup_py import SetupPyRunner from pants.base.build_environment import get_buildroot @@ -25,17 +26,6 @@ from pants.util.memo import memoized_property -PANTSSETUP_IMPORT_BOILERPLATE = """ -# DO NOT EDIT THIS FILE -- AUTOGENERATED BY PANTS -# Target: {setup_target} - -from distutils.core import Extension - -def find_external_modules(): - return [Extension(str('native'), [{native_sources_joined}])] -""" - - class BuildLocalPythonDistributions(Task): """Create python distributions (.whl) from python_dist targets.""" @@ -56,24 +46,19 @@ def implementation_version(cls): @classmethod def subsystem_dependencies(cls): - return super(BuildLocalPythonDistributions, cls).subsystem_dependencies() + (PythonNativeToolchain.Factory.scoped(cls),) - - @classmethod - def register_options(cls, register): - super(BuildLocalPythonDistributions, cls).register_options(register) - register('--an-option', default='wow') + return super(BuildLocalPythonDistributions, cls).subsystem_dependencies() + ( + LLVM.scoped(cls), + ) @memoized_property - def python_native_toolchain(self): - return PythonNativeToolchain.Factory.scoped_instance(self).create() + def llvm_base_dir(self): + return LLVM.scoped_instance(self).select() @property def cache_target_dirs(self): return True def execute(self): - self.context.log.debug('an_option: {}'.format(self.context.options.for_scope(self.options_scope).an_option)) - dist_targets = self.context.targets(is_local_python_dist) built_dists = set() @@ -117,7 +102,7 @@ def _create_dist(self, dist_tgt, dist_target_dir): self.context.log.info('dist_target_dir: {}'.format(dist_target_dir)) interpreter = self.context.products.get_data(PythonInterpreter) sandboxed_interpreter = SandboxedInterpreter( - self.python_native_toolchain.llvm_toolchain_dir(), interpreter) + self.llvm_base_dir, interpreter) self._copy_sources(dist_tgt, dist_target_dir) # Build a whl using SetupPyRunner and return its absolute path. setup_runner = SetupPyRunner(dist_target_dir, 'bdist_wheel', interpreter=sandboxed_interpreter) diff --git a/src/python/pants/backend/python/tasks/resolve_requirements_task_base.py b/src/python/pants/backend/python/tasks/resolve_requirements_task_base.py index 6ffddac6325..25af6baeb6c 100644 --- a/src/python/pants/backend/python/tasks/resolve_requirements_task_base.py +++ b/src/python/pants/backend/python/tasks/resolve_requirements_task_base.py @@ -64,7 +64,7 @@ def resolve_requirements(self, req_libs, local_dist_targets=None): BuildLocalPythonDistributions.PYTHON_DISTS) if built_dists is not None: - synthetic_address = ':'.join(2 * [binary_tgt.invalidation_hash()]) + synthetic_address = ':'.join(2 * [target_set_id]) local_dist_req_lib = build_req_lib_provided_by_setup_file( self.context.build_graph, diff --git a/src/python/pants/binaries/BUILD b/src/python/pants/binaries/BUILD index b884d20d8fb..d899fe02a3d 100644 --- a/src/python/pants/binaries/BUILD +++ b/src/python/pants/binaries/BUILD @@ -3,12 +3,13 @@ python_library( name='binary_util', - sources=['binary_tool.py', 'binary_tool_mixin.py', 'binary_util.py'], + sources=['binary_tool.py', 'binary_util.py'], dependencies=[ '3rdparty/python/twitter/commons:twitter.common.collections', 'src/python/pants/base:build_environment', 'src/python/pants/base:deprecated', 'src/python/pants/base:exceptions', + 'src/python/pants/fs', 'src/python/pants/net', 'src/python/pants/option', 'src/python/pants/subsystem', diff --git a/src/python/pants/binaries/binary_tool.py b/src/python/pants/binaries/binary_tool.py index a09980b8db9..453d16e0e95 100644 --- a/src/python/pants/binaries/binary_tool.py +++ b/src/python/pants/binaries/binary_tool.py @@ -5,23 +5,25 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) +import os + from pants.binaries.binary_util import BinaryUtil from pants.subsystem.subsystem import Subsystem -from pants.util.memo import memoized_method +from pants.util.memo import memoized_method, memoized_property class BinaryToolBase(Subsystem): """Base class for subsytems that configure binary tools. - Typically, a specific subclass is created via create_binary_tool_subsystem_cls() below. - That subclass can be further subclassed, manually, e.g., to add any extra options. + Subclasses can be further subclassed, manually, e.g., to add any extra options. :API: public """ # Subclasses must set these to appropriate values for the tool they define. # They must also set options_scope to the tool name as understood by BinaryUtil. - support_dir = None + support_subdir = None platform_dependent = None + archive_type = None default_version = None # Subclasses may set these to effect migration from an old --version option to this one. @@ -32,6 +34,12 @@ class BinaryToolBase(Subsystem): # Subclasses may set this to provide extra register() kwargs for the --version option. extra_version_option_kwargs = None + @classmethod + def subsystem_dependencies(cls): + return super(BinaryToolBase, cls).subsystem_dependencies() + ( + BinaryUtil.Factory, + ) + @classmethod def register_options(cls, register): super(BinaryToolBase, cls).register_options(register) @@ -70,10 +78,33 @@ def select(self, context=None): version = old_opts.get(self.replaces_name) return self._select_for_version(version) + @memoized_property + def _binary_util(self): + return BinaryUtil.Factory.create() + + # can override this and call super() to compose + @classmethod + def support_dir_paths(cls): + return [] + + @classmethod + def _support_dir(cls): + paths = cls.support_dir_paths() + if len(paths) == 0: + # TODO: raise subclassed exception here! + raise Exception('???') + subdir = cls.support_subdir or cls.options_scope + paths.append(subdir) + return os.path.join(*paths) + @memoized_method def _select_for_version(self, version): - return BinaryUtil.Factory.create().select( - self.support_dir, version, self.options_scope, self.platform_dependent) + return self._binary_util.select( + self._support_dir(), + version, + self.options_scope, + self.platform_dependent, + self.archive_type) class NativeTool(BinaryToolBase): @@ -83,6 +114,10 @@ class NativeTool(BinaryToolBase): """ platform_dependent = True + @classmethod + def support_dir_paths(cls): + return super(NativeTool, cls).support_dir_paths() + ['bin'] + class Script(BinaryToolBase): """A base class for platform-independent scripts. @@ -90,3 +125,7 @@ class Script(BinaryToolBase): :API: public """ platform_dependent = False + + @classmethod + def support_dir_paths(cls): + return super(Script, cls).support_dir_paths() + ['scripts'] diff --git a/src/python/pants/binaries/binary_util.py b/src/python/pants/binaries/binary_util.py index 6a21fa0e96f..4e871e2059c 100644 --- a/src/python/pants/binaries/binary_util.py +++ b/src/python/pants/binaries/binary_util.py @@ -14,6 +14,7 @@ from pants.base.build_environment import get_buildroot from pants.base.exceptions import TaskError +from pants.fs.archive import archive_extensions, archiver from pants.net.http.fetcher import Fetcher from pants.subsystem.subsystem import Subsystem from pants.util.contextutil import temporary_file @@ -129,11 +130,29 @@ def __init__(self, baseurls, timeout_secs, bootstrapdir, path_by_id=None): if path_by_id: self._path_by_id.update((tuple(k), tuple(v)) for k, v in path_by_id.items()) - def select(self, supportdir, version, name, platform_dependent): + def select(self, supportdir, version, name, platform_dependent, archive_type): + if archive_type is None: + full_name = name + else: + # TODO: throw a subclassed exception here if archive type doesn't exist! + arch_ext = archive_extensions[archive_type] + full_name = '{}.{}'.format(name, arch_ext) + + logger.debug("full_name: '{}'".format(full_name)) + if platform_dependent: - return self._select_binary(supportdir, version, name) + downloaded_file = self._select_binary(supportdir, version, full_name) else: - return self._select_script(supportdir, version, name) + downloaded_file = self._select_script(supportdir, version, full_name) + + if archive_type is None: + return downloaded_file + + selected_archiver = archiver(archive_type) + # use filename without extension as the directory name + unpacked_dirname, _ = os.path.splitext(downloaded_file) + selected_archiver.extract(downloaded_file, unpacked_dirname) + return unpacked_dirname def select_binary(self, supportdir, version, name): return self._select_binary(supportdir, version, name)