diff --git a/src/python/pants/backend/native/config/environment.py b/src/python/pants/backend/native/config/environment.py index fa2ce2364d67..0cc6ed62793d 100644 --- a/src/python/pants/backend/native/config/environment.py +++ b/src/python/pants/backend/native/config/environment.py @@ -13,10 +13,7 @@ class UnsupportedPlatformError(Exception): - """Thrown if the native toolchain is invoked on an unrecognized platform. - - Note that the native toolchain should work on all of Pants's supported - platforms.""" + """Thrown if pants is running on an unrecognized platform.""" class Platform(datatype(['normalized_os_name'])): @@ -48,11 +45,14 @@ class Executable(object): @abstractproperty def path_entries(self): - """???""" + """A list of directory paths containing this executable, to be used in a subprocess's PATH. + + This may be multiple directories, e.g. if the main executable program invokes any subprocesses. + """ @abstractproperty def exe_filename(self): - """???""" + """The "entry point" -- which file to invoke when PATH is set to `path_entries()`.""" class Linker(datatype([ diff --git a/src/python/pants/backend/native/register.py b/src/python/pants/backend/native/register.py index 06fce71f3174..71a6288c3d61 100644 --- a/src/python/pants/backend/native/register.py +++ b/src/python/pants/backend/native/register.py @@ -11,9 +11,8 @@ from pants.backend.native.subsystems.binaries.llvm import create_llvm_rules from pants.backend.native.subsystems.native_toolchain import create_native_toolchain_rules from pants.backend.native.subsystems.xcode_cli_tools import create_xcode_cli_tools_rules -from pants.backend.native.targets.c_library import CLibrary -from pants.backend.native.targets.cpp_library import CppLibrary from pants.backend.native.targets.native_artifact import NativeArtifact +from pants.backend.native.targets.native_library import CLibrary, CppLibrary from pants.backend.native.tasks.c_compile import CCompile from pants.backend.native.tasks.cpp_compile import CppCompile from pants.backend.native.tasks.link_shared_libraries import LinkSharedLibraries diff --git a/src/python/pants/backend/native/subsystems/binaries/llvm.py b/src/python/pants/backend/native/subsystems/binaries/llvm.py index f50dbcce019e..8cfe7e1cb911 100644 --- a/src/python/pants/backend/native/subsystems/binaries/llvm.py +++ b/src/python/pants/backend/native/subsystems/binaries/llvm.py @@ -24,7 +24,7 @@ class LLVMReleaseUrlGenerator(BinaryToolUrlGenerator): # TODO(cosmicexplorer): Give a more useful error message than KeyError if the host platform was # not recognized (and make it easy for other BinaryTool subclasses to do this as well). _SYSTEM_ID = { - 'darwin': 'apple-darwin', + 'mac': 'apple-darwin', 'linux': 'linux-gnu-ubuntu-16.04', } diff --git a/src/python/pants/backend/native/subsystems/native_compile_settings.py b/src/python/pants/backend/native/subsystems/native_compile_settings.py new file mode 100644 index 000000000000..6a1caae5d0eb --- /dev/null +++ b/src/python/pants/backend/native/subsystems/native_compile_settings.py @@ -0,0 +1,54 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.subsystem.subsystem import Subsystem + + +class NativeCompileSettings(Subsystem): + """Any settings relevant to a compiler invocation.""" + + default_header_file_extensions = None + default_source_file_extensions = None + + @classmethod + def register_options(cls, register): + super(NativeCompileSettings, cls).register_options(register) + + # TODO: have some more formal method of mirroring options between a target and a subsystem? + register('--strict-deps', type=bool, default=True, fingerprint=True, advanced=True, + help='The default for the "strict_deps" argument for targets of this language.') + register('--fatal-warnings', type=bool, default=True, fingerprint=True, advanced=True, + help='The default for the "fatal_warnings" argument for targets of this language.') + + # TODO: make a list of file extension option type? + register('--header-file-extensions', type=list, default=cls.default_header_file_extensions, + fingerprint=True, advanced=True, + help='The allowed extensions for header files, as a list of strings.') + register('--source-file-extensions', type=list, default=cls.default_source_file_extensions, + fingerprint=True, advanced=True, + help='The allowed extensions for source files, as a list of strings.') + + def get_subsystem_target_mirrored_field_value(self, field_name, target): + """Get the attribute `field_name` from `target` if set, else from this subsystem's options.""" + tgt_setting = getattr(target, field_name) + if tgt_setting is None: + return getattr(self.get_options(), field_name) + return tgt_setting + + +class CCompileSettings(NativeCompileSettings): + options_scope = 'c-compile-settings' + + default_header_file_extensions = ['.h'] + default_source_file_extensions = ['.c'] + + +class CppCompileSettings(NativeCompileSettings): + options_scope = 'cpp-compile-settings' + + default_header_file_extensions = ['.h', '.hpp', '.tpp'] + default_source_file_extensions = ['.cpp', '.cxx', '.cc'] diff --git a/src/python/pants/backend/native/subsystems/native_toolchain.py b/src/python/pants/backend/native/subsystems/native_toolchain.py index ce491434b56a..a6c015bb5458 100644 --- a/src/python/pants/backend/native/subsystems/native_toolchain.py +++ b/src/python/pants/backend/native/subsystems/native_toolchain.py @@ -19,31 +19,17 @@ class NativeToolchain(Subsystem): """Abstraction over platform-specific tools to compile and link native code. - This "native toolchain" subsystem is an abstraction that exposes directories - containing executables to compile and link "native" code (for now, C and C++ - are supported). Consumers of this subsystem can add these directories to their - PATH to invoke subprocesses which use these tools. - - This abstraction is necessary for two reasons. First, because there are - multiple binaries involved in compilation and linking, which often invoke - other binaries that must also be available on the PATH. Second, because unlike - other binary tools in Pants, we can't provide the same package built for both - OSX and Linux, because there is no open-source linker for OSX with a - compatible license. - - So when this subsystem is consumed, Pants will download and unpack archives - (if necessary) which together provide an appropriate "native toolchain" for - the host platform. On OSX, Pants will also find and provide path entries for - the XCode command-line tools, or error out with installation instructions if - the XCode tools could not be found. + When this subsystem is consumed, Pants will download and unpack archives (if necessary) which + together provide an appropriate "native toolchain" for the host platform: a compiler and linker, + usually. This subsystem exposes the toolchain through `@rule`s, which tasks then request during + setup or execution (synchronously, for now). + + NB: Currently, on OSX, Pants will find and invoke the XCode command-line tools, or error out with + installation instructions if the XCode tools could not be found. """ options_scope = 'native-toolchain' - # This is a list of subsystems which implement `ExecutablePathProvider` and - # can be provided for all supported platforms. - _CROSS_PLATFORM_SUBSYSTEMS = [LLVM] - @classmethod def subsystem_dependencies(cls): return super(NativeToolchain, cls).subsystem_dependencies() + ( @@ -72,7 +58,7 @@ def _xcode_cli_tools(self): @rule(Linker, [Select(Platform), Select(NativeToolchain)]) def select_linker(platform, native_toolchain): - # TODO: make it possible to yield Get with a non-static + # TODO(#5933): make it possible to yield Get with a non-static # subject type and use `platform.resolve_platform_specific()`, something like: # linker = platform.resolve_platform_specific({ # 'darwin': lambda: Get(Linker, XCodeCLITools, native_toolchain._xcode_cli_tools), diff --git a/src/python/pants/backend/native/subsystems/xcode_cli_tools.py b/src/python/pants/backend/native/subsystems/xcode_cli_tools.py index 2a41246834ab..f84802067856 100644 --- a/src/python/pants/backend/native/subsystems/xcode_cli_tools.py +++ b/src/python/pants/backend/native/subsystems/xcode_cli_tools.py @@ -54,7 +54,7 @@ def _check_executables_exist(self): def path_entries(self): self._check_executables_exist() - return [self._INSTALL_LOCATION] + return [self._install_location] def linker(self, platform): return Linker( diff --git a/src/python/pants/backend/native/targets/c_library.py b/src/python/pants/backend/native/targets/c_library.py deleted file mode 100644 index 8e919316cb27..000000000000 --- a/src/python/pants/backend/native/targets/c_library.py +++ /dev/null @@ -1,21 +0,0 @@ -# coding=utf-8 -# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import (absolute_import, division, generators, nested_scopes, print_function, - unicode_literals, with_statement) - -from pants.backend.native.targets.native_library import NativeLibrary - - -class CLibrary(NativeLibrary): - """???""" - - default_sources_globs = [ - '*.h', - '*.c', - ] - - @classmethod - def alias(cls): - return 'ctypes_compatible_c_library' diff --git a/src/python/pants/backend/native/targets/cpp_library.py b/src/python/pants/backend/native/targets/cpp_library.py deleted file mode 100644 index f14998eff5cf..000000000000 --- a/src/python/pants/backend/native/targets/cpp_library.py +++ /dev/null @@ -1,22 +0,0 @@ -# coding=utf-8 -# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import (absolute_import, division, generators, nested_scopes, print_function, - unicode_literals, with_statement) - -from pants.backend.native.targets.native_library import NativeLibrary - - -class CppLibrary(NativeLibrary): - """???""" - - default_sources_globs = [ - '*.h', - '*.hpp', - '*.cpp', - ] - - @classmethod - def alias(cls): - return 'ctypes_compatible_cpp_library' diff --git a/src/python/pants/backend/native/targets/native_artifact.py b/src/python/pants/backend/native/targets/native_artifact.py index 5e679ebecdad..2b452958b9f7 100644 --- a/src/python/pants/backend/native/targets/native_artifact.py +++ b/src/python/pants/backend/native/targets/native_artifact.py @@ -12,15 +12,15 @@ class NativeArtifact(datatype(['lib_name']), PayloadField): - """???""" + """A BUILD file object declaring a target can be exported to other languages with a native ABI.""" - # TODO: why do we need this to be a method and not e.g. a field? + # TODO: This should probably be made into an @classproperty (see PR #5901). @classmethod def alias(cls): return 'native_artifact' - def as_filename(self, platform): - # TODO: check that the name conforms to some format (e.g. no dots?) + def as_shared_lib(self, platform): + # TODO: check that the name conforms to some format in the constructor (e.g. no dots?). return platform.resolve_platform_specific({ 'darwin': lambda: 'lib{}.dylib'.format(self.lib_name), 'linux': lambda: 'lib{}.so'.format(self.lib_name), diff --git a/src/python/pants/backend/native/targets/native_library.py b/src/python/pants/backend/native/targets/native_library.py index c93583751946..597c0681dacb 100644 --- a/src/python/pants/backend/native/targets/native_library.py +++ b/src/python/pants/backend/native/targets/native_library.py @@ -5,8 +5,6 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) -import logging - from pants.backend.native.targets.native_artifact import NativeArtifact from pants.base.exceptions import TargetDefinitionException from pants.base.payload import Payload @@ -14,38 +12,31 @@ from pants.build_graph.target import Target -logger = logging.getLogger(__name__) - - class NativeLibrary(Target): - """???""" + """A class wrapping targets containing sources for C-family languages and related code.""" @classmethod - def provides_native_artifact(cls, target): - return isinstance(target, cls) and bool(target.provides) + def produces_ctypes_dylib(cls, target): + return isinstance(target, cls) and bool(target.ctypes_dylib) - def __init__(self, address, payload=None, sources=None, provides=None, + def __init__(self, address, payload=None, sources=None, ctypes_dylib=None, strict_deps=None, fatal_warnings=None, **kwargs): - logger.debug("address: {}".format(address)) - logger.debug("sources: {}".format(sources)) if not payload: payload = Payload() sources_field = self.create_sources_field(sources, address.spec_path, key_arg='sources') payload.add_fields({ 'sources': sources_field, - 'provides': provides, + 'ctypes_dylib': ctypes_dylib, 'strict_deps': PrimitiveField(strict_deps), 'fatal_warnings': PrimitiveField(fatal_warnings), }) - logger.debug("sources_field.sources: {}".format(sources_field.sources)) - - if provides and not isinstance(provides, NativeArtifact): + if ctypes_dylib and not isinstance(ctypes_dylib, NativeArtifact): raise TargetDefinitionException( "Target must provide a valid pants '{}' object. Received an object with type '{}' " "and value: {}." - .format(NativeArtifact.alias(), type(provides).__name__, provides)) + .format(NativeArtifact.alias(), type(ctypes_dylib).__name__, ctypes_dylib)) super(NativeLibrary, self).__init__(address=address, payload=payload, **kwargs) @@ -58,5 +49,30 @@ def fatal_warnings(self): return self.payload.fatal_warnings @property - def provides(self): - return self.payload.provides + def ctypes_dylib(self): + return self.payload.ctypes_dylib + + +class CLibrary(NativeLibrary): + + default_sources_globs = [ + '*.h', + '*.c', + ] + + @classmethod + def alias(cls): + return 'ctypes_compatible_c_library' + + +class CppLibrary(NativeLibrary): + + default_sources_globs = [ + '*.h', + '*.hpp', + '*.cpp', + ] + + @classmethod + def alias(cls): + return 'ctypes_compatible_cpp_library' diff --git a/src/python/pants/backend/native/tasks/BUILD b/src/python/pants/backend/native/tasks/BUILD index a245b7c4aea8..d6981c226f02 100644 --- a/src/python/pants/backend/native/tasks/BUILD +++ b/src/python/pants/backend/native/tasks/BUILD @@ -12,6 +12,7 @@ python_library( 'src/python/pants/base:workunit', 'src/python/pants/build_graph', 'src/python/pants/task', + 'src/python/pants/util:collections', 'src/python/pants/util:contextutil', 'src/python/pants/util:memo', 'src/python/pants/util:objects', diff --git a/src/python/pants/backend/native/tasks/c_compile.py b/src/python/pants/backend/native/tasks/c_compile.py index 1a1ebbf26168..6a44b4ff017c 100644 --- a/src/python/pants/backend/native/tasks/c_compile.py +++ b/src/python/pants/backend/native/tasks/c_compile.py @@ -8,82 +8,60 @@ import os from pants.backend.native.config.environment import CCompiler -from pants.backend.native.targets.c_library import CLibrary -from pants.backend.native.tasks.native_compile import NativeCompile, ObjectFiles +from pants.backend.native.subsystems.native_compile_settings import CCompileSettings +from pants.backend.native.subsystems.native_toolchain import NativeToolchain +from pants.backend.native.targets.native_library import CLibrary +from pants.backend.native.tasks.native_compile import NativeCompile from pants.base.exceptions import TaskError from pants.base.workunit import WorkUnit, WorkUnitLabel from pants.util.contextutil import get_joined_path from pants.util.memo import memoized_property -from pants.util.objects import SubclassesOf, datatype +from pants.util.objects import SubclassesOf from pants.util.process_handler import subprocess -class CCompileRequest(datatype([ - 'c_compiler', - 'include_dirs', - 'sources', - 'fatal_warnings', - 'output_dir', -])): pass - - class CCompile(NativeCompile): - default_header_file_extensions = ['.h'] - default_source_file_extensions = ['.c'] + # Compile only C library targets. + source_target_constraint = SubclassesOf(CLibrary) @classmethod def implementation_version(cls): return super(CCompile, cls).implementation_version() + [('CCompile', 0)] - class CCompileError(TaskError): - """???""" + class CCompileError(TaskError): pass - # Compile only C library targets. - source_target_constraint = SubclassesOf(CLibrary) + @classmethod + def subsystem_dependencies(cls): + return super(CCompile, cls).subsystem_dependencies() + ( + CCompileSettings.scoped(cls), + NativeToolchain.scoped(cls), + ) @memoized_property - def c_compiler(self): + def _toolchain(self): + return NativeToolchain.scoped_instance(self) + + def get_compile_settings(self): + return CCompileSettings.scoped_instance(self) + + def get_compiler(self): return self._request_single(CCompiler, self._toolchain) - # FIXME: note somewhere that this means source file names within a target must be unique (even if - # the files are in different subdirectories) -- check this at the target level!!! - def collect_cached_objects(self, versioned_target): - return ObjectFiles(versioned_target.results_dir, os.listdir(versioned_target.results_dir)) - - def compile(self, versioned_target): - compile_request = self._make_compile_request(versioned_target) - return self._execute_compile_request(compile_request) - - def _make_compile_request(self, vt): - include_dirs = self.include_dirs_for_target(vt.target) - self.context.log.debug("include_dirs: {}".format(include_dirs)) - sources_by_type = self.get_sources_headers_for_target(vt.target) - fatal_warnings = self.get_task_target_field_value('fatal_warnings', vt.target) - return CCompileRequest( - c_compiler=self.c_compiler, - include_dirs=include_dirs, - sources=sources_by_type.sources, - fatal_warnings=fatal_warnings, - output_dir=vt.results_dir) - - def _execute_compile_request(self, compile_request): + def compile(self, compile_request): sources = compile_request.sources output_dir = compile_request.output_dir if len(sources) == 0: - # FIXME: do we need this log message? Should we still have it for intentionally header-only + # TODO: do we need this log message? Should we still have it for intentionally header-only # libraries (that might be a confusing message to see)? self.context.log.debug("no sources in request {}, skipping".format(compile_request)) - return ObjectFiles(output_dir, []) + return - - # TODO: add -fPIC, but only to object files used for shared libs (how do we determine that?) -- - # alternatively, only allow using native code to build shared libs. - c_compiler = compile_request.c_compiler + c_compiler = compile_request.compiler err_flags = ['-Werror'] if compile_request.fatal_warnings else [] - # We are executing in the results_dir, so get absolute paths for everything. - # TODO: -fPIC all the time??? + # We are going to execute in `output_dir`, so get absolute paths for everything. + # TODO: If we need to produce static libs, don't add -fPIC! (could use Variants -- see #5788). cmd = [c_compiler.exe_filename] + err_flags + ['-c', '-fPIC'] + [ '-I{}'.format(os.path.abspath(inc_dir)) for inc_dir in compile_request.include_dirs ] + [os.path.abspath(src) for src in sources] @@ -109,8 +87,3 @@ def _execute_compile_request(self, compile_request): raise self.CCompileError( "Error compiling C sources with command {} for request {}. Exit code was: {}." .format(cmd, compile_request, rc)) - - # NB: We take everything produced in the output directory without verifying its correctness. - ret = ObjectFiles(output_dir, os.listdir(output_dir)) - self.context.log.debug("ret: {}".format(ret)) - return ret diff --git a/src/python/pants/backend/native/tasks/cpp_compile.py b/src/python/pants/backend/native/tasks/cpp_compile.py index 389e53f357cd..2758b3e88265 100644 --- a/src/python/pants/backend/native/tasks/cpp_compile.py +++ b/src/python/pants/backend/native/tasks/cpp_compile.py @@ -8,75 +8,58 @@ import os from pants.backend.native.config.environment import CppCompiler -from pants.backend.native.targets.cpp_library import CppLibrary -from pants.backend.native.tasks.native_compile import NativeCompile, ObjectFiles +from pants.backend.native.subsystems.native_compile_settings import CppCompileSettings +from pants.backend.native.subsystems.native_toolchain import NativeToolchain +from pants.backend.native.targets.native_library import CppLibrary +from pants.backend.native.tasks.native_compile import NativeCompile from pants.base.exceptions import TaskError from pants.base.workunit import WorkUnit, WorkUnitLabel from pants.util.contextutil import get_joined_path from pants.util.memo import memoized_property -from pants.util.objects import SubclassesOf, datatype +from pants.util.objects import SubclassesOf from pants.util.process_handler import subprocess -class CppCompileRequest(datatype([ - 'cpp_compiler', - 'include_dirs', - 'sources', - 'fatal_warnings', - 'output_dir', -])): pass - - class CppCompile(NativeCompile): - default_header_file_extensions = ['.h', '.hpp', '.tpp'] - default_source_file_extensions = ['.cpp', '.cxx', '.cc'] + # Compile only C++ library targets. + source_target_constraint = SubclassesOf(CppLibrary) @classmethod def implementation_version(cls): return super(CppCompile, cls).implementation_version() + [('CppCompile', 0)] - class CppCompileError(TaskError): - """???""" + class CppCompileError(TaskError): pass - source_target_constraint = SubclassesOf(CppLibrary) + @classmethod + def subsystem_dependencies(cls): + return super(CppCompile, cls).subsystem_dependencies() + ( + CppCompileSettings.scoped(cls), + NativeToolchain.scoped(cls), + ) @memoized_property - def cpp_compiler(self): + def _toolchain(self): + return NativeToolchain.scoped_instance(self) + + def get_compile_settings(self): + return CppCompileSettings.scoped_instance(self) + + def get_compiler(self): return self._request_single(CppCompiler, self._toolchain) - # FIXME: note somewhere that this means source file names within a target must be unique -- check - # this at the target level!!! - def collect_cached_objects(self, versioned_target): - return ObjectFiles(versioned_target.results_dir, os.listdir(versioned_target.results_dir)) - - def compile(self, versioned_target): - compile_request = self._make_compile_request(versioned_target) - return self._execute_compile_request(compile_request) - - def _make_compile_request(self, vt): - include_dirs = self.include_dirs_for_target(vt.target) - sources_by_type = self.get_sources_headers_for_target(vt.target) - fatal_warnings = self.get_task_target_field_value('fatal_warnings', vt.target) - return CppCompileRequest( - cpp_compiler=self.cpp_compiler, - include_dirs=include_dirs, - sources=sources_by_type.sources, - fatal_warnings=fatal_warnings, - output_dir=vt.results_dir) - - def _execute_compile_request(self, compile_request): + def compile(self, compile_request): sources = compile_request.sources output_dir = compile_request.output_dir if len(sources) == 0: self.context.log.debug("no sources for request {}, skipping".format(compile_request)) - return ObjectFiles(output_dir, []) + return - cpp_compiler = compile_request.cpp_compiler + cpp_compiler = compile_request.compiler err_flags = ['-Werror'] if compile_request.fatal_warnings else [] - # We are executing in the results_dir, so get absolute paths for everything. - # TODO: -fPIC all the time??? + # We are going to execute in `output_dir`, so get absolute paths for everything. + # TODO: If we need to produce static libs, don't add -fPIC! (could use Variants -- see #5788). cmd = [cpp_compiler.exe_filename] + err_flags + ['-c', '-fPIC'] + [ '-I{}'.format(os.path.abspath(inc_dir)) for inc_dir in compile_request.include_dirs ] + [os.path.abspath(src) for src in sources] @@ -102,6 +85,3 @@ def _execute_compile_request(self, compile_request): raise self.CppCompileError( "Error compiling C++ sources with command {} for request {}. Exit code was: {}." .format(cmd, compile_request, rc)) - - # NB: We take everything produced in the output directory without verifying its correctness. - return ObjectFiles(output_dir, os.listdir(output_dir)) diff --git a/src/python/pants/backend/native/tasks/link_shared_libraries.py b/src/python/pants/backend/native/tasks/link_shared_libraries.py index e61e9d010615..94652fdaf24d 100644 --- a/src/python/pants/backend/native/tasks/link_shared_libraries.py +++ b/src/python/pants/backend/native/tasks/link_shared_libraries.py @@ -8,11 +8,13 @@ import os from pants.backend.native.config.environment import Linker +from pants.backend.native.subsystems.native_toolchain import NativeToolchain from pants.backend.native.targets.native_library import NativeLibrary -from pants.backend.native.tasks.native_compile import ObjectFiles +from pants.backend.native.tasks.native_compile import NativeTargetDependencies, ObjectFiles from pants.backend.native.tasks.native_task import NativeTask from pants.base.exceptions import TaskError from pants.base.workunit import WorkUnit, WorkUnitLabel +from pants.util.collections import assert_single_element from pants.util.contextutil import get_joined_path from pants.util.memo import memoized_property from pants.util.objects import datatype @@ -38,6 +40,7 @@ def product_types(cls): @classmethod def prepare(cls, options, round_manager): + round_manager.require(NativeTargetDependencies) round_manager.require(ObjectFiles) @property @@ -48,64 +51,87 @@ def cache_target_dirs(self): def implementation_version(cls): return super(LinkSharedLibraries, cls).implementation_version() + [('LinkSharedLibraries', 0)] - class LinkSharedLibrariesError(TaskError): - """???""" + class LinkSharedLibrariesError(TaskError): pass + + @classmethod + def subsystem_dependencies(cls): + return super(LinkSharedLibraries, cls).subsystem_dependencies() + (NativeToolchain.scoped(cls),) + + @memoized_property + def _toolchain(self): + return NativeToolchain.scoped_instance(self) @memoized_property def linker(self): return self._request_single(Linker, self._toolchain) + def _retrieve_single_product_at_target_base(self, product_mapping, target): + self.context.log.debug("product_mapping: {}".format(product_mapping)) + self.context.log.debug("target: {}".format(target)) + product = product_mapping.get(target) + single_base_dir = assert_single_element(product.keys()) + single_product = assert_single_element(product[single_base_dir]) + return single_product + def execute(self): - targets_providing_artifacts = self.context.targets(NativeLibrary.provides_native_artifact) + targets_providing_artifacts = self.context.targets(NativeLibrary.produces_ctypes_dylib) + native_target_deps_product = self.context.products.get(NativeTargetDependencies) compiled_objects_product = self.context.products.get(ObjectFiles) shared_libs_product = self.context.products.get(SharedLibrary) + all_shared_libs_by_name = {} + with self.invalidated(targets_providing_artifacts, invalidate_dependents=True) as invalidation_check: for vt in invalidation_check.all_vts: if vt.valid: shared_library = self._retrieve_shared_lib_from_cache(vt) else: - link_request = self._make_link_request(vt, compiled_objects_product) + link_request = self._make_link_request( + vt, compiled_objects_product, native_target_deps_product) shared_library = self._execute_link_request(link_request) - # FIXME: de-dup libs by name? just disallow it i think + same_name_shared_lib = all_shared_libs_by_name.get(shared_library.name, None) + if same_name_shared_lib: + # TODO: test this branch! + raise self.LinkSharedLibrariesError( + "The name '{name}' was used for two shared libraries: {prev} and {cur}." + .format(name=shared_library.name, + prev=same_name_shared_lib, + cur=shared_library)) + else: + all_shared_libs_by_name[shared_library.name] = shared_library + shared_libs_product.add(vt.target, vt.target.target_base).append(shared_library) def _retrieve_shared_lib_from_cache(self, vt): - native_artifact = vt.target.provides - path_to_cached_lib = os.path.join(vt.results_dir, native_artifact.as_filename(self.linker.platform)) - # TODO: check if path exists!! + native_artifact = vt.target.ctypes_dylib + path_to_cached_lib = os.path.join( + vt.results_dir, native_artifact.as_shared_lib(self.linker.platform)) + if not os.path.isfile(path_to_cached_lib): + raise self.LinkSharedLibrariesError("The shared library at {} does not exist!" + .format(path_to_cached_lib)) return SharedLibrary(name=native_artifact.lib_name, path=path_to_cached_lib) - def _make_link_request(self, vt, compiled_objects_product): - # FIXME: should coordinate to ensure we get the same deps for link and compile (could put that - # in the ObjectFiles type tbh) - deps = self.native_deps(vt.target) - + def _make_link_request(self, vt, compiled_objects_product, native_target_deps_product): self.context.log.debug("link target: {}".format(vt.target)) + deps = self._retrieve_single_product_at_target_base(native_target_deps_product, vt.target) + all_compiled_object_files = [] for dep_tgt in deps: self.context.log.debug("dep_tgt: {}".format(dep_tgt)) - product_mapping = compiled_objects_product.get(dep_tgt) - base_dirs = product_mapping.keys() - assert(len(base_dirs) == 1) - single_base_dir = base_dirs[0] - object_files_list = product_mapping[single_base_dir] - assert(len(object_files_list) == 1) - single_product = object_files_list[0] - self.context.log.debug("single_product: {}".format(single_product)) - object_files_for_target = single_product.file_paths() - self.context.log.debug("object_files_for_target: {}".format(object_files_for_target)) - # TODO: dedup object file paths? can we assume they are already deduped? - all_compiled_object_files.extend(object_files_for_target) + object_files = self._retrieve_single_product_at_target_base(compiled_objects_product, dep_tgt) + self.context.log.debug("object_files: {}".format(object_files)) + object_file_paths = object_files.file_paths() + self.context.log.debug("object_file_paths: {}".format(object_file_paths)) + all_compiled_object_files.extend(object_file_paths) return LinkSharedLibraryRequest( linker=self.linker, object_files=all_compiled_object_files, - native_artifact=vt.target.provides, + native_artifact=vt.target.ctypes_dylib, output_dir=vt.results_dir) _SHARED_CMDLINE_ARGS = { @@ -120,16 +146,13 @@ def _execute_link_request(self, link_request): object_files = link_request.object_files if len(object_files) == 0: - # TODO: there's a legitimate reason to have no object files, but we don't support that yet (we - # need to expand LinkSharedLibraryRequest) - raise self.LinkSharedLibrariesError( - "no object files were provided in request {} -- this is not yet supported" - .format(link_request)) + raise self.LinkSharedLibrariesError("No object files were provided in request {}!" + .format(link_request)) linker = link_request.linker native_artifact = link_request.native_artifact output_dir = link_request.output_dir - resulting_shared_lib_path = os.path.join(output_dir, native_artifact.as_filename(linker.platform)) + resulting_shared_lib_path = os.path.join(output_dir, native_artifact.as_shared_lib(linker.platform)) # We are executing in the results_dir, so get absolute paths for everything. cmd = ([linker.exe_filename] + self._get_shared_lib_cmdline_args() + diff --git a/src/python/pants/backend/native/tasks/native_compile.py b/src/python/pants/backend/native/tasks/native_compile.py index 85c107145e13..467efe4cdb2a 100644 --- a/src/python/pants/backend/native/tasks/native_compile.py +++ b/src/python/pants/backend/native/tasks/native_compile.py @@ -6,21 +6,26 @@ unicode_literals, with_statement) import os -import re from abc import abstractmethod from collections import defaultdict +from pants.backend.native.config.environment import Executable from pants.backend.native.targets.native_library import NativeLibrary -# FIXME: when i deleted toolchain_task.py, it kept using the .pyc file. this shouldn't happen (the -# import should have failed)!!! from pants.backend.native.tasks.native_task import NativeTask from pants.base.exceptions import TaskError +from pants.build_graph.dependency_context import DependencyContext from pants.util.memo import memoized_method, memoized_property from pants.util.objects import SubclassesOf, datatype -class NativeSourcesByType(datatype(['rel_root', 'headers', 'sources'])): - """???""" +class NativeCompileRequest(datatype([ + ('compiler', SubclassesOf(Executable)), + # TODO: add type checking for Collection.of()! + 'include_dirs', + 'sources', + ('fatal_warnings', bool), + 'output_dir', +])): pass # TODO: verify that filenames are valid fileNAMES and not deeper paths? does this matter? @@ -30,84 +35,131 @@ def file_paths(self): return [os.path.join(self.root_dir, fname) for fname in self.filenames] +# FIXME: this is a temporary hack -- we could introduce something like a "NativeRequirement" with +# dependencies, header, object file, library name (more?) instead of using multiple products. +class NativeTargetDependencies(datatype(['native_deps'])): pass + + class NativeCompile(NativeTask): @classmethod def product_types(cls): - return [ObjectFiles] + return [ObjectFiles, NativeTargetDependencies] @property def cache_target_dirs(self): return True - # FIXME: add NB: to note how you have to override this or whatever - default_header_file_extensions = None - default_source_file_extensions = None - - @classmethod - def register_options(cls, register): - super(NativeCompile, cls).register_options(register) + @abstractmethod + def get_compile_settings(self): + """An instance of `NativeCompileSettings` which is used in `NativeCompile`. - register('--fatal-warnings', type=bool, default=True, fingerprint=True, advanced=True, - help='???/The default for the "fatal_warnings" argument for targets of this language.') + :return: :class:`pants.backend.native.subsystems.native_compile_settings.NativeCompileSettings` + """ - # TODO(cosmicexplorer): make a list of file extension option type? - register('--header-file-extensions', type=list, default=cls.default_header_file_extensions, - fingerprint=True, advanced=True, - help='???/the allowed file extensions, as a list of strings (file extensions)') - register('--source-file-extensions', type=list, default=cls.default_source_file_extensions, - fingerprint=True, advanced=True, - help='???/the allowed file extensions, as a list of strings (file extensions)') + @memoized_property + def _compile_settings(self): + return self.get_compile_settings() @classmethod def implementation_version(cls): return super(NativeCompile, cls).implementation_version() + [('NativeCompile', 0)] + @memoized_property + def _header_file_extensions(self): + return self._compile_settings.get_options().header_file_extensions + + @memoized_property + def _source_file_extensions(self): + return self._compile_settings.get_options().source_file_extensions + class NativeCompileError(TaskError): - """???""" + """Raised for errors in this class's logic. + + Subclasses are advised to create their own exception class. + """ - # NB: these are not provided by NativeTask, but are a convention. - # TODO: mention that you gotta override at least the source target constraint (???) + # `NativeCompile` will use the `source_target_constraint` to determine what targets have "sources" + # to compile, and the `dependent_target_constraint` to determine which dependent targets to + # operate on for `strict_deps` calculation. + # NB: `source_target_constraint` must be overridden. source_target_constraint = None dependent_target_constraint = SubclassesOf(NativeLibrary) + def native_deps(self, target): + return self.strict_deps_for_target( + target, predicate=self.dependent_target_constraint.satisfied_by) + + def strict_deps_for_target(self, target, predicate=None): + """Get the dependencies of `target` filtered by `predicate`, accounting for 'strict_deps'. + + If 'strict_deps' is on, instead of using the transitive closure of dependencies, targets will + only be able to see their immediate dependencies declared in the BUILD file. The 'strict_deps' + setting is obtained from the result of `get_compile_settings()`. + + NB: This includes the current target in the result. + """ + if self._compile_settings.get_subsystem_target_mirrored_field_value('strict_deps', target): + strict_deps = target.strict_dependencies(DependencyContext()) + if predicate: + filtered_deps = filter(predicate, strict_deps) + else: + filtered_deps = strict_deps + deps = [target] + filtered_deps + else: + deps = self.context.build_graph.transitive_subgraph_of_addresses( + [target.address], predicate=predicate) + + return deps + + @staticmethod + def _add_product_at_target_base(product_mapping, target, value): + product_mapping.add(target, target.target_base).append(value) + def execute(self): object_files_product = self.context.products.get(ObjectFiles) + native_deps_product = self.context.products.get(NativeTargetDependencies) source_targets = self.context.targets(self.source_target_constraint.satisfied_by) with self.invalidated(source_targets, invalidate_dependents=True) as invalidation_check: - for vt in invalidation_check.all_vts: - if vt.valid: - object_files = self.collect_cached_objects(vt) - else: - object_files = self.compile(vt) + for vt in invalidation_check.invalid_vts: + deps = self.native_deps(vt.target) + self._add_product_at_target_base(native_deps_product, vt.target, deps) + compile_request = self._make_compile_request(vt, deps) + self.context.log.debug("compile_request: {}".format(compile_request)) + self.compile(compile_request) - object_files_product.add(vt.target, vt.target.target_base).append(object_files) + for vt in invalidation_check.all_vts: + object_files = self.collect_cached_objects(vt) + self._add_product_at_target_base(object_files_product, vt.target, object_files) + # This may be calculated many times for a target, so we memoize it. @memoized_method - def include_dirs_for_target(self, target): - deps = self.native_deps(target) - sources_fields = [dep_tgt.sources_relative_to_target_base() for dep_tgt in deps] - return [src_field.rel_root for src_field in sources_fields] + def _include_dirs_for_target(self, target): + return target.sources_relative_to_target_base().rel_root - @memoized_property - def _header_exts(self): - return self.get_options().header_file_extensions + class NativeSourcesByType(datatype(['rel_root', 'headers', 'sources'])): pass - @memoized_property - def _source_exts(self): - return self.get_options().source_file_extensions - - @memoized_method def get_sources_headers_for_target(self, target): - header_extensions = self._header_exts - source_extensions = self._source_exts + """Split a target's sources into header and source files. + + This method will use the result of `get_compile_settings()` to get the extensions belonging to + header and source files, and then it will group the sources by those extensions. + + :return: :class:`NativeCompile.NativeSourcesByType` + :raises: :class:`NativeCompile.NativeCompileError` if there is an error processing the sources. + """ + header_extensions = self._header_file_extensions + source_extensions = self._source_file_extensions header_files = [] source_files = [] - # Relative to source root so the exception message makes sense. + # Get source paths relative to the target base so the exception message with the target and + # paths makes sense. target_relative_sources = target.sources_relative_to_target_base() rel_root = target_relative_sources.rel_root + + # Group the sources by extension. Check whether a file has an extension using `endswith()`. for src in target_relative_sources: found_file_ext = None for h_ext in header_extensions: @@ -123,7 +175,7 @@ def get_sources_headers_for_target(self, target): found_file_ext = s_ext continue if not found_file_ext: - dashed_options_scope = re.sub(r'\.', '-', self.options_scope) + # TODO: test this error! raise self.NativeCompileError( "Source file '{source_file}' for target '{target}' " "does not have any of this task's known file extensions. " @@ -132,16 +184,13 @@ def get_sources_headers_for_target(self, target): "--{processed_scope}-source-file-extensions: (value was: {source_exts})" .format(source_file=src, target=target.address.spec, - processed_scope=dashed_options_scope, + processed_scope=self.get_options_scope_equivalent_flag_component(), header_exts=header_extensions, source_exts=source_extensions)) - self.context.log.debug("header_files: {}".format(header_files)) - self.context.log.debug("source_files: {}".format(source_files)) - # Unique file names are required because we just dump object files into a single directory, and # the compiler will silently just produce a single object file if provided non-unique filenames. - # TODO(cosmicexplorer): add some shading to file names so we can remove this check. + # TODO: add some shading to file names so we can remove this check. seen_filenames = defaultdict(list) for src in source_files: seen_filenames[os.path.basename(src)].append(src) @@ -157,16 +206,43 @@ def get_sources_headers_for_target(self, target): headers_for_compile = [os.path.join(rel_root, h) for h in header_files] sources_for_compile = [os.path.join(rel_root, s) for s in source_files] - self.context.log.debug("target: {}, target.target_base: {}, source root: {}, rel_path: {}, headers_for_compile: {}, sources_for_compile: {}" - .format(target, target.target_base, target._sources_field.source_root, target._sources_field.rel_path, headers_for_compile, sources_for_compile)) - return NativeSourcesByType(rel_root, headers_for_compile, sources_for_compile) - # TODO: document how these two are supposed to both produce an ObjectFiles (getting from the cache - # vs getting from a compile). + return self.NativeSourcesByType(rel_root, headers_for_compile, sources_for_compile) + @abstractmethod - def collect_cached_objects(self, versioned_target): - """???/use vt.results_dir!""" + def get_compiler(self): + """An instance of `Executable` which can be invoked to compile files. + + :return: :class:`pants.backend.native.config.environment.Executable` + """ + + @memoized_property + def _compiler(self): + return self.get_compiler() + + def _make_compile_request(self, versioned_target, dependencies): + target = versioned_target.target + include_dirs = [self._include_dirs_for_target(dep_tgt) for dep_tgt in dependencies] + sources_by_type = self.get_sources_headers_for_target(target) + return NativeCompileRequest( + compiler=self._compiler, + include_dirs=include_dirs, + sources=sources_by_type.sources, + fatal_warnings=self._compile_settings.get_subsystem_target_mirrored_field_value( + 'fatal_warnings', target), + output_dir=versioned_target.results_dir) @abstractmethod - def compile(self, versioned_target): - """???""" + def compile(self, compile_request): + """Perform the process of compilation, writing object files to the request's 'output_dir'. + + NB: This method must arrange the output files so that `collect_cached_objects()` can collect all + of the results (or vice versa)! + """ + + def collect_cached_objects(self, versioned_target): + """Scan `versioned_target`'s results directory and return the output files from that directory. + + :return: :class:`ObjectFiles` + """ + return ObjectFiles(versioned_target.results_dir, os.listdir(versioned_target.results_dir)) diff --git a/src/python/pants/backend/native/tasks/native_task.py b/src/python/pants/backend/native/tasks/native_task.py index f21a93e2521c..fb12e2b7b10d 100644 --- a/src/python/pants/backend/native/tasks/native_task.py +++ b/src/python/pants/backend/native/tasks/native_task.py @@ -5,65 +5,13 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) -from pants.backend.native.subsystems.native_toolchain import NativeToolchain -from pants.backend.native.targets.native_library import NativeLibrary -from pants.build_graph.dependency_context import DependencyContext from pants.task.task import Task -from pants.util.memo import memoized_property -from pants.util.objects import SubclassesOf class NativeTask(Task): - native_target_constraint = SubclassesOf(NativeLibrary) - - @classmethod - def subsystem_dependencies(cls): - return super(NativeTask, cls).subsystem_dependencies() + (NativeToolchain.scoped(cls),) - - @classmethod - def register_options(cls, register): - super(NativeTask, cls).register_options(register) - - register('--strict-deps', type=bool, default=True, fingerprint=True, advanced=True, - help='???/The default for the "strict_deps" argument for targets of this language.') - - @memoized_property - def _toolchain(self): - return NativeToolchain.scoped_instance(self) - + # FIXME(#5869): delete this when we can request Subsystems from options in tasks! def _request_single(self, product, subject): - # FIXME(cosmicexplorer): This is not supposed to be exposed to Tasks yet -- see #4769 to track - # the status of exposing v2 products in v1 tasks. + # NB: This is not supposed to be exposed to Tasks yet -- see #4769 to track the status of + # exposing v2 products in v1 tasks. return self.context._scheduler.product_request(product, [subject])[0] - - def get_task_target_field_value(self, field_name, target): - """???/for fields with the same name on targets and tasks""" - tgt_setting = getattr(target, field_name) - if tgt_setting is None: - return getattr(self.get_options(), field_name) - return tgt_setting - - def native_deps(self, target): - return self.strict_deps_for_target(target, predicate=self.native_target_constraint.satisfied_by) - - def strict_deps_for_target(self, target, predicate=None): - """???/figure out strict deps and stuff - - This includes the current target in the result. - """ - # TODO: note that the target's gotta have a strict_deps prop - if self.get_task_target_field_value('strict_deps', target): - # FIXME: does this include the current target? it almost definitely should (should it - # though???) this actually isn't clear - strict_deps = target.strict_dependencies(DependencyContext()) - if predicate: - filtered_deps = filter(predicate, strict_deps) - else: - filtered_deps = strict_deps - deps = [target] + filtered_deps - else: - deps = self.context.build_graph.transitive_subgraph_of_addresses([target.address], - predicate=predicate) - - return deps diff --git a/src/python/pants/backend/python/subsystems/python_native_code.py b/src/python/pants/backend/python/subsystems/python_native_code.py new file mode 100644 index 000000000000..4bf9978b1ab5 --- /dev/null +++ b/src/python/pants/backend/python/subsystems/python_native_code.py @@ -0,0 +1,123 @@ +# coding=utf-8 +# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from collections import defaultdict + +from pants.backend.native.subsystems.native_toolchain import NativeToolchain +from pants.backend.native.targets.native_library import NativeLibrary +from pants.backend.python.subsystems.python_setup import PythonSetup +from pants.backend.python.targets.python_binary import PythonBinary +from pants.backend.python.targets.python_distribution import PythonDistribution +from pants.base.exceptions import IncompatiblePlatformsError +from pants.subsystem.subsystem import Subsystem +from pants.util.memo import memoized_property +from pants.util.objects import Exactly + + +class PythonNativeCode(Subsystem): + """A subsystem which exposes components of the native backend to the python backend.""" + + options_scope = 'python-native-code' + + default_native_source_extensions = ['.c', '.cpp', '.cc'] + + @classmethod + def register_options(cls, register): + super(PythonNativeCode, cls).register_options(register) + + register('--native-source-extensions', type=list, default=cls.default_native_source_extensions, + fingerprint=True, advanced=True, + help='The extensions recognized for native source files in `python_dist()` sources.') + + @classmethod + def subsystem_dependencies(cls): + return super(PythonNativeCode, cls).subsystem_dependencies() + ( + NativeToolchain.scoped(cls), + PythonSetup.scoped(cls), + ) + + @memoized_property + def _native_source_extensions(self): + return self.get_options().native_source_extensions + + @memoized_property + def native_toolchain(self): + return NativeToolchain.scoped_instance(self) + + @memoized_property + def _python_setup(self): + return PythonSetup.scoped_instance(self) + + def pydist_has_native_sources(self, target): + return target.has_sources(extension=tuple(self._native_source_extensions)) + + def native_target_has_native_sources(self, target): + return target.has_sources() + + @memoized_property + def _native_target_matchers(self): + return { + Exactly(PythonDistribution): lambda tgt: self.pydist_has_native_sources, + Exactly(NativeLibrary): lambda tgt: self.native_target_has_native_sources, + } + + def _any_targets_have_native_sources(self, targets): + for tgt in targets: + for type_constraint, target_predicate in self._native_target_matchers.items(): + if type_constraint.satisfied_by(tgt) and target_predicate(tgt): + return True + return False + + def get_targets_by_declared_platform(self, targets): + """ + Aggregates a dict that maps a platform string to a list of targets that specify the platform. + If no targets have platforms arguments, return a dict containing platforms inherited from + the PythonSetup object. + + :param tgts: a list of :class:`Target` objects. + :returns: a dict mapping a platform string to a list of targets that specify the platform. + """ + targets_by_platforms = defaultdict(list) + + for tgt in targets: + for platform in tgt.platforms: + targets_by_platforms[platform].append(tgt) + + if not targets_by_platforms: + for platform in self._python_setup.platforms: + targets_by_platforms[platform] = ['(No target) Platform inherited from either the ' + '--platforms option or a pants.ini file.'] + return targets_by_platforms + + _PYTHON_PLATFORM_TARGETS_CONSTRAINT = Exactly(PythonBinary, PythonDistribution) + + def check_build_for_current_platform_only(self, targets): + """ + Performs a check of whether the current target closure has native sources and if so, ensures + that Pants is only targeting the current platform. + + :param tgts: a list of :class:`Target` objects. + :return: a boolean value indicating whether the current target closure has native sources. + :raises: :class:`pants.base.exceptions.IncompatiblePlatformsError` + """ + if not self._any_targets_have_native_sources(targets): + return False + + targets_with_platforms = filter(self._PYTHON_PLATFORM_TARGETS_CONSTRAINT.satisfied_by, targets) + platforms_with_sources = self.get_targets_by_declared_platform(targets_with_platforms) + platform_names = platforms_with_sources.keys() + + # There will always be at least 1 platform, because we checked that they have native sources. + assert(len(platform_names) >= 1) + if platform_names == ['current']: + return True + + raise IncompatiblePlatformsError( + 'The target set contains one or more targets that depend on ' + 'native code. Please ensure that the platform arguments in all relevant targets and build ' + 'options are compatible with the current platform. Found targets for platforms: {}' + .format(str(platforms_with_sources))) diff --git a/src/python/pants/backend/python/targets/python_distribution.py b/src/python/pants/backend/python/targets/python_distribution.py index e723f6d6b731..0263a49318f7 100644 --- a/src/python/pants/backend/python/targets/python_distribution.py +++ b/src/python/pants/backend/python/targets/python_distribution.py @@ -19,8 +19,6 @@ class PythonDistribution(Target): default_sources_globs = '*.py' - native_source_extensions = ['.c', '.cpp', '.cc'] - @classmethod def alias(cls): return 'python_dist' diff --git a/src/python/pants/backend/python/tasks/BUILD b/src/python/pants/backend/python/tasks/BUILD index 21c815c1f618..76e396fff5eb 100644 --- a/src/python/pants/backend/python/tasks/BUILD +++ b/src/python/pants/backend/python/tasks/BUILD @@ -3,23 +3,20 @@ python_library( dependencies=[ + '3rdparty/python:pex', '3rdparty/python/twitter/commons:twitter.common.collections', '3rdparty/python/twitter/commons:twitter.common.dirutil', - '3rdparty/python:pex', 'src/python/pants/backend/native/config', 'src/python/pants/backend/native/subsystems', 'src/python/pants/backend/native/targets', 'src/python/pants/backend/native/tasks', + 'src/python/pants/backend/python:interpreter_cache', + 'src/python/pants/backend/python:pex_util', 'src/python/pants/backend/python:python_requirement', 'src/python/pants/backend/python:python_requirements', - 'src/python/pants/backend/python:interpreter_cache', 'src/python/pants/backend/python/subsystems', 'src/python/pants/backend/python/targets', 'src/python/pants/backend/python/tasks/coverage:plugin', - 'src/python/pants/backend/python:interpreter_cache', - 'src/python/pants/backend/python:pex_util', - 'src/python/pants/backend/python:python_requirement', - 'src/python/pants/backend/python:python_requirements', 'src/python/pants/base:build_environment', 'src/python/pants/base:exceptions', 'src/python/pants/base:fingerprint_strategy', diff --git a/src/python/pants/backend/python/tasks/build_local_python_distributions.py b/src/python/pants/backend/python/tasks/build_local_python_distributions.py index 14f307791302..b6d6760be2de 100644 --- a/src/python/pants/backend/python/tasks/build_local_python_distributions.py +++ b/src/python/pants/backend/python/tasks/build_local_python_distributions.py @@ -12,10 +12,10 @@ from pex.interpreter import PythonInterpreter -from pants.backend.native.subsystems.native_toolchain import NativeToolchain from pants.backend.native.targets.native_library import NativeLibrary from pants.backend.native.tasks.link_shared_libraries import SharedLibrary from pants.backend.python.python_requirement import PythonRequirement +from pants.backend.python.subsystems.python_native_code import PythonNativeCode from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary from pants.backend.python.tasks.pex_build_util import is_local_python_dist from pants.backend.python.tasks.setup_py import (SetupPyExecutionEnvironment, SetupPyNativeTools, @@ -53,19 +53,26 @@ def implementation_version(cls): @classmethod def subsystem_dependencies(cls): - return super(BuildLocalPythonDistributions, cls).subsystem_dependencies() + (NativeToolchain.scoped(cls),) + return super(BuildLocalPythonDistributions, cls).subsystem_dependencies() + ( + PythonNativeCode.scoped(cls), + ) + @memoized_property + def _python_native_code_settings(self): + return PythonNativeCode.scoped_instance(self) + + # FIXME(#5869): delete this and get Subsystems from options, when that is possible. def _request_single(self, product, subject): - # FIXME(#4769): This is not supposed to be exposed to Tasks yet -- see #4769 to track the status - # of exposing v2 products in v1 tasks. + # NB: This is not supposed to be exposed to Tasks yet -- see #4769 to track the status of + # exposing v2 products in v1 tasks. return self.context._scheduler.product_request(product, [subject])[0] @memoized_property def _setup_py_native_tools(self): - native_toolchain = NativeToolchain.scoped_instance(self) + native_toolchain = self._python_native_code_settings.native_toolchain return self._request_single(SetupPyNativeTools, native_toolchain) - # TODO: This should probably be made into a class property, when that is made. + # TODO: This should probably be made into an @classproperty (see PR #5901). @property def cache_target_dirs(self): return True @@ -86,8 +93,10 @@ def _get_setup_requires_to_resolve(self, dist_target): return reqs_to_resolve - # TODO: document the existence of these directories! + # NB: these are all the immediate subdirectories of the target's results directory. + # This contains any modules from a setup_requires(). setup_requires_site_subdir = 'setup_requires_site' + # This will contain the sources used to build the python_dist(). dist_subdir = 'python_dist_subdir' @classmethod @@ -126,13 +135,12 @@ def _get_native_artifact_deps(self, target): native_artifact_targets = [] if target.dependencies: for dep_tgt in target.dependencies: - if not NativeLibrary.provides_native_artifact(dep_tgt): + if not NativeLibrary.produces_ctypes_dylib(dep_tgt): raise TargetDefinitionException( target, "Target '{}' is invalid: the only dependencies allowed in python_dist() targets " - "are {}() targets with a provides= kwarg." - # FIXME: make this error message work! - .format(dep_tgt.address.spec, '???')) + "are C or C++ targets with a ctypes_dylib= kwarg." + .format(dep_tgt.address.spec)) native_artifact_targets.append(dep_tgt) return native_artifact_targets @@ -151,8 +159,6 @@ def _copy_sources(self, dist_tgt, dist_target_dir): def _add_artifacts(self, dist_target_dir, shared_libs_product, native_artifact_targets): all_shared_libs = [] - # FIXME: dedup names of native artifacts? should that happen in the LinkSharedLibraries step? - # (yes it should) for tgt in native_artifact_targets: product_mapping = shared_libs_product.get(tgt) base_dir = assert_single_element(product_mapping.keys()) @@ -184,9 +190,9 @@ def _prepare_and_create_dist(self, interpreter, shared_libs_product, versioned_t is_platform_specific = False native_tools = None - if dist_target.has_native_sources: + if self._python_native_code_settings.pydist_has_native_sources(dist_target): # We add the native tools if we need to compile code belonging to this python_dist() target. - # TODO: check that the native toolchain isn't loaded without native code in a test! + # TODO: test this branch somehow! native_tools = self._setup_py_native_tools # Native code in this python_dist() target requires marking the dist as platform-specific. is_platform_specific = True diff --git a/src/python/pants/backend/python/tasks/pex_build_util.py b/src/python/pants/backend/python/tasks/pex_build_util.py index 6fe4c7ef616a..0f9204970a2e 100644 --- a/src/python/pants/backend/python/tasks/pex_build_util.py +++ b/src/python/pants/backend/python/tasks/pex_build_util.py @@ -6,13 +6,11 @@ unicode_literals, with_statement) import os -from collections import defaultdict from pex.fetcher import Fetcher from pex.resolver import resolve from twitter.common.collections import OrderedSet -from pants.backend.native.targets.native_library import NativeLibrary from pants.backend.python.pex_util import get_local_platform from pants.backend.python.subsystems.python_setup import PythonSetup from pants.backend.python.targets.python_binary import PythonBinary @@ -21,7 +19,7 @@ from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary from pants.backend.python.targets.python_tests import PythonTests from pants.base.build_environment import get_buildroot -from pants.base.exceptions import IncompatiblePlatformsError, TaskError +from pants.base.exceptions import TaskError from pants.build_graph.files import Files from pants.python.python_repos import PythonRepos @@ -49,73 +47,6 @@ def has_python_requirements(tgt): return isinstance(tgt, PythonRequirementLibrary) -def is_python_binary(tgt): - return isinstance(tgt, PythonBinary) - - -def tgt_closure_has_native_sources(tgts): - """Determine if any target in the current target closure has native (c or cpp) sources.""" - pydist_targets = filter(is_local_python_dist, tgts) - has_pydist_native_sources = any(tgt.has_native_sources for tgt in pydist_targets) - native_targets = filter(lambda t: isinstance(t, NativeLibrary), tgts) - has_native_library_sources = any(tgt.has_sources() for tgt in native_targets) - return has_pydist_native_sources or has_native_library_sources - - -def tgt_closure_platforms(tgts): - """ - Aggregates a dict that maps a platform string to a list of targets that specify the platform. - If no targets have platforms arguments, return a dict containing platforms inherited from - the PythonSetup object. - - :param tgts: a list of :class:`Target` objects. - :returns: a dict mapping a platform string to a list of targets that specify the platform. - """ - tgts_by_platforms = defaultdict(list) - - for tgt in tgts: - for platform in tgt.platforms: - tgts_by_platforms[platform].append(tgt) - - if not tgts_by_platforms: - for platform in PythonSetup.global_instance().platforms: - tgts_by_platforms[platform] = ['(No target) Platform inherited from either the ' - '--platforms option or a pants.ini file.'] - return tgts_by_platforms - - -def build_for_current_platform_only_check(tgts): - """ - Performs a check of whether the current target closure has native sources and if so, ensures that - Pants is only targeting the current platform. - - :param tgts: a list of :class:`Target` objects. - :return: a boolean value indicating whether the current target closure has native sources. - """ - # FIXME: This should really be checking the platforms in the closure of - # `PythonRequirementLibrary`s, not the targets themselves. 3rdparty libraries and local - # `python_dist()`s are the only way platform-specific code can be inserted into a - # pex. `SetupPyRunner` ensures that the artifacts built locally are marked with 'current' if they - # contain any native code. - if tgt_closure_has_native_sources(tgts): - def predicate(x): - return is_python_binary(x) or is_local_python_dist(x) - platforms_with_sources = tgt_closure_platforms(filter(predicate, tgts)) - platform_names = platforms_with_sources.keys() - - # There will always be at least 1 platform, because we checked that they have native sources. - if platform_names == ['current']: - return True - - raise IncompatiblePlatformsError( - 'The target set contains one or more targets that depend on ' - 'native code. Please ensure that the platform arguments in all relevant targets and build ' - 'options are compatible with the current platform. Found targets for platforms: {}' - .format(str(platforms_with_sources))) - - return False - - def _create_source_dumper(builder, tgt): if type(tgt) == Files: # Loose `Files` as opposed to `Resources` or `PythonTarget`s have no (implied) package structure diff --git a/src/python/pants/backend/python/tasks/python_binary_create.py b/src/python/pants/backend/python/tasks/python_binary_create.py index 85cc176bd6f3..3d19f565d514 100644 --- a/src/python/pants/backend/python/tasks/python_binary_create.py +++ b/src/python/pants/backend/python/tasks/python_binary_create.py @@ -11,10 +11,10 @@ from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo +from pants.backend.python.subsystems.python_native_code import PythonNativeCode from pants.backend.python.targets.python_binary import PythonBinary from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary -from pants.backend.python.tasks.pex_build_util import (build_for_current_platform_only_check, - dump_requirement_libs, dump_sources, +from pants.backend.python.tasks.pex_build_util import (dump_requirement_libs, dump_sources, has_python_requirements, has_python_sources, has_resources, is_python_target) from pants.base.build_environment import get_buildroot @@ -24,11 +24,20 @@ from pants.util.contextutil import temporary_dir from pants.util.dirutil import safe_mkdir_for from pants.util.fileutil import atomic_copy +from pants.util.memo import memoized_property class PythonBinaryCreate(Task): """Create an executable .pex file.""" + @classmethod + def subsystem_dependencies(cls): + return super(PythonBinaryCreate, cls).subsystem_dependencies() + (PythonNativeCode.scoped(cls),) + + @memoized_property + def _python_native_code_settings(self): + return PythonNativeCode.scoped_instance(self) + @classmethod def product_types(cls): return ['pex_archives', 'deployable_archives'] @@ -131,7 +140,7 @@ def _create_binary(self, binary_tgt, results_dir): dump_sources(builder, tgt, self.context.log) # We need to ensure that we are resolving for only the current platform if we are # including local python dist targets that have native extensions. - build_for_current_platform_only_check(self.context.targets()) + self._python_native_code_settings.check_build_for_current_platform_only(self.context.targets()) dump_requirement_libs(builder, interpreter, req_tgts, self.context.log, platforms=binary_tgt.platforms) diff --git a/src/python/pants/backend/python/tasks/resolve_requirements_task_base.py b/src/python/pants/backend/python/tasks/resolve_requirements_task_base.py index a52cb47ddd77..f3ad3da46c68 100644 --- a/src/python/pants/backend/python/tasks/resolve_requirements_task_base.py +++ b/src/python/pants/backend/python/tasks/resolve_requirements_task_base.py @@ -13,13 +13,14 @@ from pex.pex_builder import PEXBuilder from pants.backend.python.python_requirement import PythonRequirement +from pants.backend.python.subsystems.python_native_code import PythonNativeCode from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary -from pants.backend.python.tasks.pex_build_util import (build_for_current_platform_only_check, - dump_requirement_libs, dump_requirements) +from pants.backend.python.tasks.pex_build_util import dump_requirement_libs, dump_requirements from pants.base.hash_utils import hash_all from pants.invalidation.cache_manager import VersionedTargetSet from pants.task.task import Task from pants.util.dirutil import safe_concurrent_creation +from pants.util.memo import memoized_property class ResolveRequirementsTaskBase(Task): @@ -30,6 +31,16 @@ class ResolveRequirementsTaskBase(Task): for running the relevant python code. """ + @classmethod + def subsystem_dependencies(cls): + return super(ResolveRequirementsTaskBase, cls).subsystem_dependencies() + ( + PythonNativeCode.scoped(cls), + ) + + @memoized_property + def _python_native_code_settings(self): + return PythonNativeCode.scoped_instance(self) + @classmethod def prepare(cls, options, round_manager): round_manager.require_data(PythonInterpreter) @@ -55,7 +66,10 @@ def resolve_requirements(self, interpreter, req_libs): # We need to ensure that we are resolving for only the current platform if we are # including local python dist targets that have native extensions. tgts = self.context.targets() - maybe_platforms = ['current'] if build_for_current_platform_only_check(tgts) else None + if self._python_native_code_settings.check_build_for_current_platform_only(tgts): + maybe_platforms = ['current'] + else: + maybe_platforms = None path = os.path.realpath(os.path.join(self.workdir, str(interpreter.identity), target_set_id)) # Note that we check for the existence of the directory, instead of for invalid_vts, diff --git a/src/python/pants/backend/python/tasks/setup_py.py b/src/python/pants/backend/python/tasks/setup_py.py index d5760072b6d8..7b617adf57f8 100644 --- a/src/python/pants/backend/python/tasks/setup_py.py +++ b/src/python/pants/backend/python/tasks/setup_py.py @@ -102,7 +102,8 @@ class SetupPyNativeTools(datatype([ ])): """The native tools needed for a setup.py invocation. -This class exists so that ???""" + This class exists because `SetupPyExecutionEnvironment` is created manually, one per target. + """ @rule(SetupPyNativeTools, [Select(CCompiler), Select(CppCompiler), Select(Linker)]) @@ -110,7 +111,6 @@ def get_setup_py_native_tools(c_compiler, cpp_compiler, linker): yield SetupPyNativeTools(c_compiler=c_compiler, cpp_compiler=cpp_compiler, linker=linker) -# TODO: It would be pretty useful to have an Optional TypeConstraint. class SetupRequiresSiteDir(datatype(['site_dir'])): pass @@ -142,7 +142,7 @@ def ensure_setup_requires_site_dir(reqs_to_resolve, interpreter, site_dir, class SetupPyExecutionEnvironment(datatype([ - # TODO: It would be pretty useful to have an Optional TypeConstraint. + # TODO: It might be pretty useful to have an Optional TypeConstraint. 'setup_requires_site_dir', # If None, don't execute in the toolchain environment. 'setup_py_native_tools', @@ -166,7 +166,14 @@ def as_environment(self): native_tools.cpp_compiler.path_entries + native_tools.linker.path_entries ) - ret['PATH'] = get_joined_path(all_path_entries) + # FIXME(#5662): It seems that crti.o is provided by glibc, which we don't provide yet, so this + # lets Travis pass for now. + ret['PATH'] = native_tools.linker.platform.resolve_platform_specific({ + 'darwin': lambda: get_joined_path(all_path_entries), + # Append our tools after the ones already on the PATH -- this is shameful and should be + # removed when glibc is introduced. + 'linux': lambda: get_joined_path(all_path_entries, os.environ.copy()), + }) return ret diff --git a/src/python/pants/option/optionable.py b/src/python/pants/option/optionable.py index 6bfdc6001d12..c5f5cde6fd5d 100644 --- a/src/python/pants/option/optionable.py +++ b/src/python/pants/option/optionable.py @@ -57,6 +57,16 @@ def known_scope_infos(cls): """ yield cls.get_scope_info() + @classmethod + def get_options_scope_equivalent_flag_component(cls): + """Return a string representing this optionable's scope as it would be in a command line flag. + + This method can be used to generate error messages with flags e.g. to fix some error with a + pants command. These flags will then be as specific as possible, including e.g. all dependent + subsystem scopes. + """ + return re.sub(r'\.', '-', cls.options_scope) + @classmethod def get_description(cls): # First line of docstring. diff --git a/src/python/pants/option/scope.py b/src/python/pants/option/scope.py index 386e262fdfd8..3440d1570b64 100644 --- a/src/python/pants/option/scope.py +++ b/src/python/pants/option/scope.py @@ -12,6 +12,7 @@ GLOBAL_SCOPE_CONFIG_SECTION = 'GLOBAL' +# FIXME: convert this to a datatype? class ScopeInfo(namedtuple('_ScopeInfo', ['scope', 'category', 'optionable_cls'])): """Information about a scope.""" diff --git a/src/python/pants/util/collections.py b/src/python/pants/util/collections.py index 55f44a991323..f825c45ba16a 100644 --- a/src/python/pants/util/collections.py +++ b/src/python/pants/util/collections.py @@ -22,7 +22,7 @@ def recursively_update(d, d2): def assert_single_element(iterable): - """???""" + """Convert `iterable` to a list, assert that the list has one element, then return the element.""" result = list(iterable) assert(len(result) == 1) return result[0] diff --git a/src/python/pants/util/dirutil.py b/src/python/pants/util/dirutil.py index 1fcf25822ba8..e3dc6fb7b655 100644 --- a/src/python/pants/util/dirutil.py +++ b/src/python/pants/util/dirutil.py @@ -503,7 +503,7 @@ def split_basename_and_dirname(path): def narrow_relative_paths(cur_root_dir, new_root_subdir, rel_paths): - """???""" + """If `cur_root_dir` contains `new_root_subdir`, relativize `rel_paths` to `new_root_subdir`.""" for rel_file in rel_paths: file_abs = os.path.join(cur_root_dir, rel_file) yield os.path.relpath(file_abs, new_root_subdir) diff --git a/src/python/pants/util/objects.py b/src/python/pants/util/objects.py index a9a721989643..50a0a9aba1d9 100644 --- a/src/python/pants/util/objects.py +++ b/src/python/pants/util/objects.py @@ -109,17 +109,15 @@ def _replace(_self, **kwds): raise ValueError('Got unexpected field names: %r' % kwds.keys()) return result - # TODO: would we want to expose a self.as_tuple() method so we can tuple assign? + # TODO: would we want to expose a self.as_tuple() method (which just calls __getnewargs__) so we + # can tuple assign? E.g.: # class A(datatype(['field'])): pass # x = A(field='asdf') # field_value, = x.as_tuple() # print(field_value) # => 'asdf' - def as_tuple(self): - return tuple(self._super_iter()) - def __getnewargs__(self): '''Return self as a plain tuple. Used by copy and pickle.''' - return self.as_tuple() + return tuple(self._super_iter()) def __repr__(self): args_formatted = [] @@ -313,12 +311,14 @@ def satisfied_by_type(self, obj_type): class Collection(object): """Constructs classes representing collections of objects of a particular type.""" + # TODO: could we check that the input is iterable in the ctor? @classmethod @memoized def of(cls, *element_types): union = '|'.join(element_type.__name__ for element_type in element_types) type_name = b'{}.of({})'.format(cls.__name__, union) + # TODO: could we allow type checking in the datatype() invocation here? supertypes = (cls, datatype(['dependencies'], superclass_name='Collection')) properties = {'element_types': element_types} collection_of_type = type(type_name, supertypes, properties) diff --git a/testprojects/src/python/python_distribution/ctypes/BUILD b/testprojects/src/python/python_distribution/ctypes/BUILD index a182a66b04c5..306448365557 100644 --- a/testprojects/src/python/python_distribution/ctypes/BUILD +++ b/testprojects/src/python/python_distribution/ctypes/BUILD @@ -4,13 +4,13 @@ ctypes_compatible_c_library( name='c_library', sources=['some_math.h', 'some_math.c', 'src-subdir/add_three.h', 'src-subdir/add_three.c'], - provides=native_artifact(lib_name='asdf-c'), + ctypes_dylib=native_artifact(lib_name='asdf-c'), ) ctypes_compatible_cpp_library( name='cpp_library', sources=['some_more_math.hpp', 'some_more_math.cpp'], - provides=native_artifact(lib_name='asdf-cpp'), + ctypes_dylib=native_artifact(lib_name='asdf-cpp'), ) python_dist( diff --git a/testprojects/src/python/python_distribution/ctypes/ctypes_python_pkg/ctypes_wrapper.py b/testprojects/src/python/python_distribution/ctypes/ctypes_python_pkg/ctypes_wrapper.py index 7a64bb31cee7..c8b12a5ffd08 100644 --- a/testprojects/src/python/python_distribution/ctypes/ctypes_python_pkg/ctypes_wrapper.py +++ b/testprojects/src/python/python_distribution/ctypes/ctypes_python_pkg/ctypes_wrapper.py @@ -10,7 +10,9 @@ def get_generated_shared_lib(lib_name): + # These are the same filenames as in setup.py. filename = 'lib{}.so'.format(lib_name) + # The data files are in the root directory, but we are in ctypes_python_pkg/. rel_path = os.path.join(os.path.dirname(__file__), '..', filename) return os.path.normpath(rel_path) diff --git a/testprojects/src/python/python_distribution/ctypes/setup.py b/testprojects/src/python/python_distribution/ctypes/setup.py index 057c7e3efe67..87a09cb8fc49 100644 --- a/testprojects/src/python/python_distribution/ctypes/setup.py +++ b/testprojects/src/python/python_distribution/ctypes/setup.py @@ -12,5 +12,6 @@ name='ctypes_test', version='0.0.1', packages=find_packages(), + # Declare two files at the top-level directory (denoted by ''). data_files=[('', ['libasdf-c.so', 'libasdf-cpp.so'])], ) diff --git a/tests/python/pants_test/backend/native/subsystems/BUILD b/tests/python/pants_test/backend/native/subsystems/BUILD deleted file mode 100644 index e87e712b8cac..000000000000 --- a/tests/python/pants_test/backend/native/subsystems/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -# coding=utf-8 -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -python_tests( - dependencies=[ - 'src/python/pants/backend/native/subsystems', - 'src/python/pants/util:contextutil', - 'src/python/pants/util:process_handler', - 'tests/python/pants_test:base_test', - 'tests/python/pants_test/subsystem:subsystem_utils', - ], -) diff --git a/tests/python/pants_test/backend/native/subsystems/__init__.py b/tests/python/pants_test/backend/native/subsystems/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tests/python/pants_test/backend/native/subsystems/test_native_toolchain.py b/tests/python/pants_test/backend/native/subsystems/test_native_toolchain.py deleted file mode 100644 index 75b176a2b3d9..000000000000 --- a/tests/python/pants_test/backend/native/subsystems/test_native_toolchain.py +++ /dev/null @@ -1,69 +0,0 @@ -# coding=utf-8 -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import (absolute_import, division, generators, nested_scopes, print_function, - unicode_literals, with_statement) - -import unittest - -from pants.backend.native.subsystems.native_toolchain import NativeToolchain -from pants.util.contextutil import environment_as, get_joined_path -from pants.util.process_handler import subprocess -from pants_test.base_test import BaseTest -from pants_test.subsystem.subsystem_util import global_subsystem_instance - - -# TODO(cosmicexplorer): Can we have some form of this run in an OSX shard on -# Travis? -# FIXME(cosmicexplorer): We need to test gcc as well, but the gcc driver can't -# find the right include directories for system headers in Travis. We need to -# use the clang driver to find library paths, then use those when invoking gcc. -@unittest.skip('Skipped because this entire backend is about to get redone, see #5780.') -class TestNativeToolchain(BaseTest): - - def setUp(self): - super(TestNativeToolchain, self).setUp() - self.toolchain = global_subsystem_instance(NativeToolchain) - - def _invoke_capturing_output(self, cmd, cwd=None): - if cwd is None: - cwd = self.build_root - - toolchain_dirs = self.toolchain.path_entries() - isolated_toolchain_path = get_joined_path(toolchain_dirs) - try: - with environment_as(PATH=isolated_toolchain_path): - return subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - raise Exception( - "Command failed while invoking the native toolchain " - "with code '{code}', cwd='{cwd}', cmd='{cmd}'. Combined stdout and stderr:\n{out}" - .format(code=e.returncode, cwd=cwd, cmd=' '.join(cmd), out=e.output), - e) - - def test_hello_c(self): - self.create_file('hello.c', contents=""" -#include "stdio.h" - -int main() { - printf("%s\\n", "hello, world!"); -} -""") - - self._invoke_capturing_output(['clang', 'hello.c', '-o', 'hello_clang']) - c_output = self._invoke_capturing_output(['./hello_clang']) - self.assertEqual(c_output, 'hello, world!\n') - - def test_hello_cpp(self): - self.create_file('hello.cpp', contents=""" -#include - -int main() { - std::cout << "hello, world!" << std::endl; -} -""") - - self._invoke_capturing_output(['clang++', 'hello.cpp', '-o', 'hello_clang++']) - cpp_output = self._invoke_capturing_output(['./hello_clang++']) - self.assertEqual(cpp_output, 'hello, world!\n') diff --git a/tests/python/pants_test/backend/python/tasks/BUILD b/tests/python/pants_test/backend/python/tasks/BUILD index 4b45a87bd803..473a839a61e7 100644 --- a/tests/python/pants_test/backend/python/tasks/BUILD +++ b/tests/python/pants_test/backend/python/tasks/BUILD @@ -69,11 +69,13 @@ python_tests( python_tests( name='ctypes_integration', - sources=['test_ctypes_integration.py'], + sources=['test_ctypes_integration.py', 'test_python_distribution_integration.py'], dependencies=[ + 'src/python/pants/base:build_environment', 'src/python/pants/util:contextutil', 'src/python/pants/util:process_handler', 'tests/python/pants_test:int-test', + 'tests/python/pants_test/backend/python/tasks:python_task_test_base', ], tags={'integration'}, timeout=2400, diff --git a/tests/python/pants_test/backend/python/tasks/python_task_test_base.py b/tests/python/pants_test/backend/python/tasks/python_task_test_base.py index af63c876b073..2f0fde288779 100644 --- a/tests/python/pants_test/backend/python/tasks/python_task_test_base.py +++ b/tests/python/pants_test/backend/python/tasks/python_task_test_base.py @@ -6,6 +6,7 @@ unicode_literals, with_statement) import os +import sysconfig from textwrap import dedent from pants.backend.python.register import build_file_aliases as register_python @@ -14,6 +15,28 @@ from pants_test.task_test_base import TaskTestBase +def normalize_platform_tag(platform_tag): + return platform_tag.replace('-', '_') + + +def name_and_platform(whl): + # The wheel filename is of the format + # {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl + # See https://www.python.org/dev/peps/pep-0425/. + # We don't care about the python or abi versions (they depend on what we're currently + # running on), we just want to make sure we have all the platforms we expect. + parts = os.path.splitext(whl)[0].split('-') + dist = parts[0] + version = parts[1] + platform_tag = parts[-1] + return dist, version, normalize_platform_tag(platform_tag) + + +def check_wheel_platform_matches_host(wheel_dist): + _, _, wheel_platform = name_and_platform(wheel_dist) + return wheel_platform == normalize_platform_tag(sysconfig.get_platform()) + + class PythonTaskTestBase(InterpreterCacheTestMixin, TaskTestBase): """ :API: public diff --git a/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py b/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py index 1a8ed236a186..76d68bdcf31d 100644 --- a/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py +++ b/tests/python/pants_test/backend/python/tasks/test_build_local_python_distributions.py @@ -12,7 +12,9 @@ from pants.backend.python.targets.python_distribution import PythonDistribution from pants.backend.python.tasks.build_local_python_distributions import \ BuildLocalPythonDistributions -from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase +from pants_test.backend.python.tasks.python_task_test_base import (PythonTaskTestBase, + check_wheel_platform_matches_host, + name_and_platform) from pants_test.engine.scheduler_test_base import SchedulerTestBase @@ -21,30 +23,79 @@ class TestBuildLocalPythonDistributions(PythonTaskTestBase, SchedulerTestBase): def task_type(cls): return BuildLocalPythonDistributions - def setUp(self): - super(TestBuildLocalPythonDistributions, self).setUp() - - # Setup simple python_dist target - sources = ['foo.py', 'bar.py', '__init__.py', 'setup.py'] - self.filemap = { - 'src/python/dist/__init__.py': '', - 'src/python/dist/foo.py': 'print("foo")', - 'src/python/dist/bar.py': 'print("bar")', - 'src/python/dist/setup.py': dedent(""" + _dist_specs = { + 'src/python/dist:universal_dist': { + 'key': 'universal', + 'sources': ['foo.py', 'bar.py', '__init__.py', 'setup.py'], + 'filemap': { + 'src/python/dist/__init__.py': '', + 'src/python/dist/foo.py': 'print("foo")', + 'src/python/dist/bar.py': 'print("bar")', + 'src/python/dist/setup.py': dedent(""" from setuptools import setup, find_packages setup( - name='my_dist', + name='universal_dist', version='0.0.0', packages=find_packages() ) """) - } - for rel_path, content in self.filemap.items(): - self.create_file(rel_path, content) + } + }, + 'src/python/plat_specific_dist:plat_specific_dist': { + 'key': 'platform_specific', + 'sources': ['__init__.py', 'setup.py', 'native_source.c'], + 'filemap': { + 'src/python/plat_specific_dist/__init__.py': '', + 'src/python/plat_specific_dist/setup.py': dedent(""" + from distutils.core import Extension + from setuptools import setup, find_packages + setup( + name='platform_specific_dist', + version='0.0.0', + packages=find_packages(), + extensions=[Extension('native_source', sources=['native_source.c'])] + ) + """), + 'src/python/plat_specific_dist/native_source.c': dedent(""" + #include + + static PyObject * native_source(PyObject *self, PyObject *args) { + return Py_BuildValue("s", "Hello from C!"); + } + + static PyMethodDef Methods[] = { + {"native_source", native_source, METH_VARARGS, ""}, + {NULL, NULL, 0, NULL} + }; + + PyMODINIT_FUNC initnative_source(void) { + (void) Py_InitModule("native_source", Methods); + } + """), + } + }, + } + + def setUp(self): + super(TestBuildLocalPythonDistributions, self).setUp() + + self.target_dict = {} - self.python_dist_tgt = self.make_target(spec='src/python/dist:my_dist', - target_type=PythonDistribution, - sources=sources) + # Create a python_dist() target from each specification and insert it into `self.target_dict`. + for target_spec, file_spec in self._dist_specs.items(): + filemap = file_spec['filemap'] + for rel_path, content in filemap.items(): + self.create_file(rel_path, content) + + sources = file_spec['sources'] + python_dist_tgt = self.make_target(spec=target_spec, + target_type=PythonDistribution, + sources=sources) + key = file_spec['key'] + self.target_dict[key] = python_dist_tgt + + def _all_dist_targets(self): + return self.target_dict.values() def _scheduling_context(self, **kwargs): rules = ( @@ -54,17 +105,47 @@ def _scheduling_context(self, **kwargs): scheduler = self.mk_scheduler(rules=rules) return self.context(scheduler=scheduler, **kwargs) - def test_python_create_distributions(self): + def _retrieve_single_product_at_target_base(self, product_mapping, target): + product = product_mapping.get(target) + base_dirs = product.keys() + self.assertEqual(1, len(base_dirs)) + single_base_dir = base_dirs[0] + all_products = product[single_base_dir] + self.assertEqual(1, len(all_products)) + single_product = all_products[0] + return single_product + + def _create_distribution_synthetic_target(self, python_dist_target): context = self._scheduling_context( - target_roots=[self.python_dist_tgt], + target_roots=[python_dist_target], for_task_types=[BuildLocalPythonDistributions]) - self.assertEquals([self.python_dist_tgt], context.build_graph.targets()) + self.assertEquals(set(self._all_dist_targets()), set(context.build_graph.targets())) python_create_distributions_task = self.create_task(context) python_create_distributions_task.execute() - synthetic_tgts = set(context.build_graph.targets()) - {self.python_dist_tgt} + synthetic_tgts = set(context.build_graph.targets()) - set(self._all_dist_targets()) self.assertEquals(1, len(synthetic_tgts)) synthetic_target = next(iter(synthetic_tgts)) - self.assertEquals(['my_dist==0.0.0'], + + return context, synthetic_target + + def test_python_create_universal_distribution(self): + universal_dist = self.target_dict['universal'] + context, synthetic_target = self._create_distribution_synthetic_target(universal_dist) + self.assertEquals(['universal_dist==0.0.0'], + [str(x.requirement) for x in synthetic_target.requirements.value]) + + local_wheel_products = context.products.get('local_wheels') + local_wheel = self._retrieve_single_product_at_target_base(local_wheel_products, universal_dist) + _, _, wheel_platform = name_and_platform(local_wheel) + self.assertEqual('any', wheel_platform) + + def test_python_create_platform_specific_distribution(self): + platform_specific_dist = self.target_dict['platform_specific'] + context, synthetic_target = self._create_distribution_synthetic_target(platform_specific_dist) + self.assertEquals(['platform_specific_dist==0.0.0'], [str(x.requirement) for x in synthetic_target.requirements.value]) - # def test_only_native_deps_allowed(self): + local_wheel_products = context.products.get('local_wheels') + local_wheel = self._retrieve_single_product_at_target_base( + local_wheel_products, platform_specific_dist) + self.assertTrue(check_wheel_platform_matches_host(local_wheel)) diff --git a/tests/python/pants_test/backend/python/tasks/test_ctypes_integration.py b/tests/python/pants_test/backend/python/tasks/test_ctypes_integration.py index fcad16ffbbca..0e41be89761e 100644 --- a/tests/python/pants_test/backend/python/tasks/test_ctypes_integration.py +++ b/tests/python/pants_test/backend/python/tasks/test_ctypes_integration.py @@ -7,33 +7,17 @@ import glob import os -import sysconfig from zipfile import ZipFile +from pants.backend.native.config.environment import Platform from pants.option.scope import GLOBAL_SCOPE_CONFIG_SECTION from pants.util.contextutil import temporary_dir from pants.util.dirutil import is_executable from pants.util.process_handler import subprocess +from pants_test.backend.python.tasks.python_task_test_base import name_and_platform from pants_test.pants_run_integration_test import PantsRunIntegrationTest -def normalize_platform_tag(platform_tag): - return platform_tag.replace('-', '_') - - -def name_and_platform(whl): - # The wheel filename is of the format - # {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl - # See https://www.python.org/dev/peps/pep-0425/. - # We don't care about the python or abi versions (they depend on what we're currently - # running on), we just want to make sure we have all the platforms we expect. - parts = os.path.splitext(whl)[0].split('-') - dist = parts[0] - version = parts[1] - platform_tag = parts[-1] - return dist, version, normalize_platform_tag(platform_tag) - - def invoke_pex_for_output(pex_file_to_run): return subprocess.check_output([pex_file_to_run], stderr=subprocess.STDOUT) @@ -68,8 +52,12 @@ def test_binary(self): self.assertEqual(len(globbed_wheel), 1) wheel_dist = globbed_wheel[0] - wheel_uname, wheel_name, wheel_platform = name_and_platform(wheel_dist) - self.assertEqual(wheel_platform, normalize_platform_tag(sysconfig.get_platform())) + _, _, wheel_platform = name_and_platform(wheel_dist) + contains_current_platform = Platform.create().resolve_platform_specific({ + 'darwin': lambda: wheel_platform.startswith('macosx'), + 'linux': lambda: wheel_platform.startswith('linux'), + }) + self.assertTrue(contains_current_platform) # Verify that the wheel contains our shared libraries. wheel_files = ZipFile(wheel_dist).namelist() diff --git a/tests/python/pants_test/backend/python/tasks/test_python_distribution_integration.py b/tests/python/pants_test/backend/python/tasks/test_python_distribution_integration.py index e1e04e8f4b27..9f18394a4a59 100644 --- a/tests/python/pants_test/backend/python/tasks/test_python_distribution_integration.py +++ b/tests/python/pants_test/backend/python/tasks/test_python_distribution_integration.py @@ -7,8 +7,8 @@ import glob import os -import sys +from pants.backend.native.config.environment import Platform from pants.base.build_environment import get_buildroot from pants.util.contextutil import environment_as, temporary_dir from pants.util.process_handler import subprocess @@ -141,10 +141,10 @@ def test_pants_resolves_local_dists_for_current_platform_only(self): self._assert_native_greeting(output) def test_pants_tests_local_dists_for_current_platform_only(self): - if 'linux' in sys.platform: - platform_string = 'linux-x86_64' - else: - platform_string = 'macosx-10.12-x86_64' + platform_string = Platform.create().resolve_platform_specific({ + 'darwin': lambda: 'macosx-10.12-x86_64', + 'linux': lambda: 'linux-x86_64', + }) # Use a platform-specific string for testing because the test goal # requires the coverage package and the pex resolver raises an Untranslatable error when # attempting to translate the coverage sdist for incompatible platforms. diff --git a/tests/python/pants_test/build_graph/test_target.py b/tests/python/pants_test/build_graph/test_target.py index 863b24c97ecd..457cd1d2cc9d 100644 --- a/tests/python/pants_test/build_graph/test_target.py +++ b/tests/python/pants_test/build_graph/test_target.py @@ -294,7 +294,7 @@ def _generate_strict_dependencies(self): def test_strict_dependencies(self): self._generate_strict_dependencies() dep_context = mock.Mock() - dep_context.compiler_plugin_types = () + dep_context.types_with_closure = () dep_context.codegen_types = () dep_context.alias_types = (Target,) dep_context.target_closure_kwargs = {'include_scopes': Scopes.JVM_COMPILE_SCOPES}