diff --git a/src/python/pants/engine/scheduler.py b/src/python/pants/engine/scheduler.py index 4d354ebb386e..51819d43c61e 100644 --- a/src/python/pants/engine/scheduler.py +++ b/src/python/pants/engine/scheduler.py @@ -418,6 +418,9 @@ def __init__(self, scheduler, session): self._session = session self._run_count = 0 + def assert_ruleset_valid(self): + self._scheduler._assert_ruleset_valid() + def graph_len(self): return self._scheduler.graph_len() diff --git a/tests/python/pants_test/engine/BUILD b/tests/python/pants_test/engine/BUILD index d2072a8296c2..97749cff201d 100644 --- a/tests/python/pants_test/engine/BUILD +++ b/tests/python/pants_test/engine/BUILD @@ -76,18 +76,6 @@ python_tests( ] ) -python_tests( - name='graph', - sources=['test_graph.py'], - coverage=['pants.engine.graph'], - dependencies=[ - '3rdparty/python:future', - 'src/python/pants/build_graph', - 'src/python/pants/engine:scheduler', - 'tests/python/pants_test/engine/examples:planners', - ] -) - python_tests( name='isolated_process', sources=['test_isolated_process.py'], @@ -135,7 +123,6 @@ python_tests( 'src/python/pants/engine:nodes', 'src/python/pants/engine:scheduler', 'src/python/pants/util:contextutil', - 'tests/python/pants_test/engine/examples:planners', 'tests/python/pants_test/engine/examples:scheduler_inputs', 'tests/python/pants_test/engine:util', ] @@ -201,7 +188,6 @@ python_tests( 'src/python/pants/engine:selectors', 'src/python/pants/util:objects', 'tests/python/pants_test/engine/examples:parsers', - 'tests/python/pants_test/engine/examples:planners', 'tests/python/pants_test/subsystem:subsystem_utils', ] ) @@ -216,7 +202,6 @@ python_tests( 'src/python/pants/base:cmd_line_spec_parser', 'src/python/pants/build_graph', 'src/python/pants/engine:scheduler', - 'tests/python/pants_test/engine/examples:planners', 'tests/python/pants_test/engine/examples:scheduler_inputs', ] ) diff --git a/tests/python/pants_test/engine/examples/BUILD b/tests/python/pants_test/engine/examples/BUILD index 9f764a1840f0..353de5ece9a0 100644 --- a/tests/python/pants_test/engine/examples/BUILD +++ b/tests/python/pants_test/engine/examples/BUILD @@ -19,27 +19,6 @@ python_library( ] ) -python_library( - name='planners', - sources=['planners.py'], - dependencies=[ - '3rdparty/python:future', - ':parsers', - ':sources', - 'src/python/pants/base:exceptions', - 'src/python/pants/base:project_tree', - 'src/python/pants/build_graph', - 'src/python/pants/engine:build_files', - 'src/python/pants/engine:fs', - 'src/python/pants/engine:mapper', - 'src/python/pants/engine:nodes', - 'src/python/pants/engine:parser', - 'src/python/pants/engine:scheduler', - 'src/python/pants/engine:selectors', - 'src/python/pants/engine:struct', - ] -) - python_library( name='sources', sources=['sources.py'], @@ -52,38 +31,6 @@ python_library( ] ) -python_library( - name='visualizer', - sources=['visualizer.py'], - dependencies=[ - ':planners', - 'src/python/pants/base:cmd_line_spec_parser', - 'src/python/pants/binaries', - 'src/python/pants/build_graph', - 'src/python/pants/engine:scheduler', - 'src/python/pants/util:contextutil', - 'src/python/pants/util:desktop', - 'src/python/pants/util:process_handler', - 'tests/python/pants_test/engine:util', - ] -) - -python_binary( - name='viz', - entry_point='pants_test.engine.examples.visualizer:main_addresses', - dependencies=[ - ':visualizer' - ] -) - -python_binary( - name='viz-fs', - entry_point='pants_test.engine.examples.visualizer:main_filespecs', - dependencies=[ - ':visualizer' - ] -) - resources( name='fs_test', # Note that this test data dir is bundled into a tarfile, to preserve symlink structure diff --git a/tests/python/pants_test/engine/examples/planners.py b/tests/python/pants_test/engine/examples/planners.py deleted file mode 100644 index ead850b953df..000000000000 --- a/tests/python/pants_test/engine/examples/planners.py +++ /dev/null @@ -1,496 +0,0 @@ -# coding=utf-8 -# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -import functools -import re -from abc import abstractmethod -from builtins import str -from os import sep as os_sep -from os.path import join as os_path_join - -from pants.base.exceptions import TaskError -from pants.base.file_system_project_tree import FileSystemProjectTree -from pants.base.project_tree import Dir -from pants.build_graph.address import Address -from pants.engine.addressable import addressable_list -from pants.engine.build_files import create_graph_rules -from pants.engine.fs import DirectoryDigest, FilesContent, PathGlobs, Snapshot, create_fs_rules -from pants.engine.mapper import AddressFamily, AddressMapper -from pants.engine.parser import SymbolTable -from pants.engine.rules import AggregationRule, SingletonRule, TaskRule, rule -from pants.engine.scheduler import Scheduler -from pants.engine.selectors import Get, Select, SelectVariant -from pants.engine.struct import HasProducts, Struct, StructWithDeps, Variants -from pants.option.global_options import DEFAULT_EXECUTION_OPTIONS -from pants.util.meta import AbstractClass -from pants.util.objects import SubclassesOf, datatype -from pants_test.engine.examples.parsers import JsonParser -from pants_test.engine.examples.sources import Sources - - -def printing_func(func): - @functools.wraps(func) - def wrapper(*inputs): - product = func(*inputs) - return_val = product if product else '<<>>'.format(func.__name__) - print('{} executed for {}, returned: {}'.format(func.__name__, inputs, return_val)) - return return_val - return wrapper - - -class Target(Struct, HasProducts): - """A placeholder for the most-numerous Struct subclass. - - This particular implementation holds a collection of other Structs in a `configurations` field. - """ - - def __init__(self, name=None, configurations=None, **kwargs): - """ - :param string name: The name of this target which forms its address in its namespace. - :param list configurations: The configurations that apply to this target in various contexts. - """ - super(Target, self).__init__(name=name, **kwargs) - - self.configurations = configurations - - @property - def products(self): - return self.configurations - - @addressable_list(SubclassesOf(Struct)) - def configurations(self): - """The configurations that apply to this target in various contexts. - - :rtype list of :class:`pants.engine.configuration.Struct` - """ - - -class JavaSources(Sources, StructWithDeps): - extensions = ('.java',) - - -class ScalaSources(Sources, StructWithDeps): - extensions = ('.scala',) - - -class PythonSources(Sources, StructWithDeps): - extensions = ('.py',) - - -class ThriftSources(Sources, StructWithDeps): - extensions = ('.thrift',) - - -class ResourceSources(Sources): - extensions = tuple() - - -class ScalaInferredDepsSources(Sources): - """A Sources subclass which can be converted to ScalaSources via dep inference.""" - extensions = ('.scala',) - - -class JVMPackageName(datatype(['name'])): - """A typedef to represent a fully qualified JVM package name.""" - pass - - -class SourceRoots(datatype(['srcroots'])): - """Placeholder for the SourceRoot subsystem.""" - - -@printing_func -@rule(Address, [Select(JVMPackageName), Select(Snapshot)]) -def select_package_address(jvm_package_name, snapshot): - """Return the Address from the given AddressFamilies which provides the given package.""" - address_families = yield [Get(AddressFamily, Dir, ds) for ds in snapshot.dir_stats] - addresses = [address for address_family in address_families - for address in address_family.addressables.keys()] - if len(addresses) == 0: - raise ValueError('No targets existed in {} to provide {}'.format( - address_families, jvm_package_name)) - elif len(addresses) > 1: - raise ValueError('Multiple targets might be able to provide {}:\n {}'.format( - jvm_package_name, '\n '.join(str(a) for a in addresses))) - yield addresses[0].to_address() - - -@printing_func -@rule(PathGlobs, [Select(JVMPackageName), Select(SourceRoots)]) -def calculate_package_search_path(jvm_package_name, source_roots): - """Return PathGlobs to match directories where the given JVMPackageName might exist.""" - rel_package_dir = jvm_package_name.name.replace('.', os_sep) - specs = [os_path_join(srcroot, rel_package_dir) for srcroot in source_roots.srcroots] - return PathGlobs(include=specs) - - -@printing_func -@rule(ScalaSources, [Select(ScalaInferredDepsSources)]) -def reify_scala_sources(sources): - """Given a ScalaInferredDepsSources object, create ScalaSources.""" - snapshot = yield Get(Snapshot, PathGlobs, sources.path_globs) - source_files_content = yield Get(FilesContent, DirectoryDigest, snapshot.directory_digest) - packages = set() - import_re = re.compile(r'^import ([^;]*);?$') - for filecontent in source_files_content.dependencies: - for line in filecontent.content.splitlines(): - match = import_re.search(line) - if match: - packages.add(match.group(1).rsplit('.', 1)[0]) - - dependency_addresses = yield [Get(Address, JVMPackageName(p)) for p in packages] - - kwargs = sources._asdict() - kwargs['dependencies'] = list(set(dependency_addresses)) - yield ScalaSources(**kwargs) - - -def aggregate_sources(*sources): - """Chooses one "source" of Sources: fails if more than one has a non-empty set of files.""" - raise NotImplemented("TODO: Sources") - - -def aggregate_classpaths(*classpaths): - """Merges classpaths (...by concatenating their creator strings, in this example case).""" - raise NotImplemented("TODO: Classpath") - - -class Requirement(Struct): - """A setuptools requirement.""" - - def __init__(self, req, repo=None, **kwargs): - """ - :param string req: A setuptools compatible requirement specifier; eg: `pantsbuild.pants>0.0.42`. - :param string repo: An optional custom find-links repo URL. - """ - super(Requirement, self).__init__(req=req, repo=repo, **kwargs) - - -class Classpath(Struct): - """Placeholder product.""" - - def __init__(self, creator, **kwargs): - super(Classpath, self).__init__(creator=creator, **kwargs) - - -class ManagedResolve(Struct): - """A frozen ivy resolve that when combined with a ManagedJar can produce a Jar.""" - - def __init__(self, revs, **kwargs): - """ - :param dict revs: A dict of artifact org#name to version. - """ - super(ManagedResolve, self).__init__(revs=revs, **kwargs) - - def __repr__(self): - return "ManagedResolve({})".format(self.revs) - - -class Jar(Struct): - """A java jar.""" - - def __init__(self, org=None, name=None, rev=None, **kwargs): - """ - :param string org: The Maven ``groupId`` of this dependency. - :param string name: The Maven ``artifactId`` of this dependency; also serves as the name portion - of the address of this jar if defined at the top level of a BUILD file. - :param string rev: The Maven ``version`` of this dependency. - """ - super(Jar, self).__init__(org=org, name=name, rev=rev, **kwargs) - - -class ManagedJar(Struct): - """A java jar template, which can be merged with a ManagedResolve to determine a concrete version.""" - - def __init__(self, org, name, **kwargs): - """ - :param string org: The Maven ``groupId`` of this dependency. - :param string name: The Maven ``artifactId`` of this dependency; also serves as the name portion - of the address of this jar if defined at the top level of a BUILD file. - """ - super(ManagedJar, self).__init__(org=org, name=name, **kwargs) - - -@printing_func -@rule(Jar, [Select(ManagedJar), SelectVariant(ManagedResolve, 'resolve')]) -def select_rev(managed_jar, managed_resolve): - (org, name) = (managed_jar.org, managed_jar.name) - rev = managed_resolve.revs.get('{}#{}'.format(org, name), None) - if not rev: - raise TaskError('{} does not have a managed version in {}.'.format(managed_jar, managed_resolve)) - return Jar(org=managed_jar.org, name=managed_jar.name, rev=rev) - - -@printing_func -@rule(Classpath, [Select(Jar)]) -def ivy_resolve(jars): - return Classpath(creator='ivy_resolve') - - -@printing_func -@rule(Classpath, [Select(ResourceSources)]) -def isolate_resources(resources): - """Copies resources into a private directory, and provides them as a Classpath entry.""" - return Classpath(creator='isolate_resources') - - -class ThriftConfiguration(StructWithDeps): - pass - - -class ApacheThriftConfiguration(ThriftConfiguration): - def __init__(self, rev=None, strict=True, **kwargs): - """ - :param string rev: The version of the apache thrift compiler to use. - :param bool strict: `False` to turn strict compiler warnings off (not recommended). - """ - super(ApacheThriftConfiguration, self).__init__(rev=rev, strict=strict, **kwargs) - - -class ApacheThriftJavaConfiguration(ApacheThriftConfiguration): - pass - - -class ApacheThriftPythonConfiguration(ApacheThriftConfiguration): - pass - - -class ApacheThriftError(TaskError): - pass - - -@rule(JavaSources, [Select(ThriftSources), SelectVariant(ApacheThriftJavaConfiguration, 'thrift')]) -def gen_apache_java_thrift(sources, config): - return gen_apache_thrift(sources, config) - - -@rule(PythonSources, [Select(ThriftSources), SelectVariant(ApacheThriftPythonConfiguration, 'thrift')]) -def gen_apache_python_thrift(sources, config): - return gen_apache_thrift(sources, config) - - -@printing_func -def gen_apache_thrift(sources, config): - if config.rev == 'fail': - raise ApacheThriftError('Failed to generate via apache thrift for ' - 'sources: {}, config: {}'.format(sources, config)) - if isinstance(config, ApacheThriftJavaConfiguration): - return JavaSources(files=['Fake.java'], dependencies=config.dependencies) - elif isinstance(config, ApacheThriftPythonConfiguration): - return PythonSources(files=['fake.py'], dependencies=config.dependencies) - - -class BuildPropertiesConfiguration(Struct): - pass - - -@printing_func -@rule(Classpath, [Select(BuildPropertiesConfiguration)]) -def write_name_file(name): - """Write a file containing the name of this target in the CWD.""" - return Classpath(creator='write_name_file') - - -class Scrooge(datatype(['tool_address'])): - """Placeholder for a Scrooge subsystem.""" - - -class ScroogeConfiguration(ThriftConfiguration): - def __init__(self, rev=None, strict=True, **kwargs): - """ - :param string rev: The version of the scrooge compiler to use. - :param bool strict: `False` to turn strict compiler warnings off (not recommended). - """ - super(ScroogeConfiguration, self).__init__(rev=rev, strict=strict, **kwargs) - - -class ScroogeScalaConfiguration(ScroogeConfiguration): - pass - - -class ScroogeJavaConfiguration(ScroogeConfiguration): - pass - - -@rule(ScalaSources, - [Select(ThriftSources), - SelectVariant(ScroogeScalaConfiguration, 'thrift')]) -def gen_scrooge_scala_thrift(sources, config): - scrooge_classpath = yield Get(Classpath, Address, Scrooge.tool_address) - yield gen_scrooge_thrift(sources, config, scrooge_classpath) - - -@rule(JavaSources, - [Select(ThriftSources), - SelectVariant(ScroogeJavaConfiguration, 'thrift')]) -def gen_scrooge_java_thrift(sources, config): - scrooge_classpath = yield Get(Classpath, Address, Scrooge.tool_address) - yield gen_scrooge_thrift(sources, config, scrooge_classpath) - - -@printing_func -def gen_scrooge_thrift(sources, config, scrooge_classpath): - if isinstance(config, ScroogeJavaConfiguration): - return JavaSources(files=['Fake.java'], dependencies=config.dependencies) - elif isinstance(config, ScroogeScalaConfiguration): - return ScalaSources(files=['Fake.scala'], dependencies=config.dependencies) - - -@printing_func -@rule(Classpath, [Select(JavaSources)]) -def javac(sources): - classpath = yield [(Get(Classpath, Address, d) if type(d) is Address else Get(Classpath, Jar, d)) - for d in sources.dependencies] - print('compiling {} with {}'.format(sources, classpath)) - yield Classpath(creator='javac') - - -@printing_func -@rule(Classpath, [Select(ScalaSources)]) -def scalac(sources): - classpath = yield [(Get(Classpath, Address, d) if type(d) is Address else Get(Classpath, Jar, d)) - for d in sources.dependencies] - print('compiling {} with {}'.format(sources, classpath)) - yield Classpath(creator='scalac') - - -class Goal(AbstractClass): - """A synthetic aggregate product produced by a goal, which is its own task.""" - - def __init__(self, *args): - if all(arg is None for arg in args): - msg = '\n '.join(p.__name__ for p in self.products()) - raise TaskError('Unable to produce any of the products for goal `{}`:\n {}'.format( - self.name(), msg)) - - @classmethod - @abstractmethod - def name(cls): - """Returns the name of the Goal.""" - - @classmethod - def rule(cls): - """Returns a Rule for this Goal, used to install the Goal. - - A Goal is it's own synthetic output product, and its constructor acts as its task function. It - selects each of its products as optional, but fails synchronously if none of them are available. - """ - return TaskRule(cls, [Select(p, optional=True) for p in cls.products()], cls) - - @classmethod - @abstractmethod - def products(cls): - """Returns the products that this Goal requests.""" - - def __eq__(self, other): - return type(self) == type(other) - - def __ne__(self, other): - return not (self == other) - - def __hash__(self): - return hash(type(self)) - - def __str__(self): - return '{}()'.format(type(self).__name__) - - def __repr__(self): - return str(self) - - -class GenGoal(Goal): - """A goal that requests all known types of sources.""" - - @classmethod - def name(cls): - return 'gen' - - @classmethod - def products(cls): - return [JavaSources, PythonSources, ResourceSources, ScalaSources] - - -class ExampleTable(SymbolTable): - @classmethod - def table(cls): - return {'apache_thrift_java_configuration': ApacheThriftJavaConfiguration, - 'apache_thrift_python_configuration': ApacheThriftPythonConfiguration, - 'jar': Jar, - 'managed_jar': ManagedJar, - 'managed_resolve': ManagedResolve, - 'requirement': Requirement, - 'scrooge_java_configuration': ScroogeJavaConfiguration, - 'scrooge_scala_configuration': ScroogeScalaConfiguration, - 'java': JavaSources, - 'python': PythonSources, - 'resources': ResourceSources, - 'scala': ScalaSources, - 'thrift': ThriftSources, - 'target': Target, - 'variants': Variants, - 'build_properties': BuildPropertiesConfiguration, - 'inferred_scala': ScalaInferredDepsSources} - - -def setup_json_scheduler(build_root, native): - """Return a build graph and scheduler configured for BLD.json files under the given build root. - - :rtype :class:`pants.engine.scheduler.SchedulerSession` - """ - - symbol_table = ExampleTable() - - # Register "literal" subjects required for these rules. - address_mapper = AddressMapper(build_patterns=('BLD.json',), - parser=JsonParser(symbol_table)) - - work_dir = os_path_join(build_root, '.pants.d') - project_tree = FileSystemProjectTree(build_root) - - rules = [ - # Codegen. These rules implement codegen by providing multiple ways to generate - # language-specific Sources, and then choosing one value to use at runtime. The runtime - # choice of Sources instance is implemented as an AggregationRule. - GenGoal.rule(), - gen_apache_java_thrift, - gen_apache_python_thrift, - gen_scrooge_scala_thrift, - gen_scrooge_java_thrift, - SingletonRule(Scrooge, Scrooge(Address.parse('src/scala/scrooge'))), - AggregationRule(JavaSources, aggregate_sources), - AggregationRule(ScalaSources, aggregate_sources), - AggregationRule(PythonSources, aggregate_sources), - ] + [ - # scala dependency inference - reify_scala_sources, - select_package_address, - calculate_package_search_path, - SingletonRule(SourceRoots, SourceRoots(('src/java','src/scala'))), - ] + [ - # Remote dependency resolution - ivy_resolve, - select_rev, - ] + [ - # Classpath providers, and an AggregationRule to merge them. - isolate_resources, - write_name_file, - javac, - scalac, - AggregationRule(Classpath, aggregate_classpaths), - ] + ( - create_graph_rules(address_mapper, symbol_table) - ) + ( - create_fs_rules() - ) - - scheduler = Scheduler(native, - project_tree, - work_dir, - rules, - DEFAULT_EXECUTION_OPTIONS, - None, - None) - return scheduler.new_session() diff --git a/tests/python/pants_test/engine/examples/visualizer.py b/tests/python/pants_test/engine/examples/visualizer.py deleted file mode 100644 index 3d271f6334d7..000000000000 --- a/tests/python/pants_test/engine/examples/visualizer.py +++ /dev/null @@ -1,83 +0,0 @@ -# coding=utf-8 -# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -import os -import sys -from textwrap import dedent - -from pants.base.cmd_line_spec_parser import CmdLineSpecParser -from pants.engine.fs import PathGlobs -from pants.util import desktop -from pants.util.contextutil import temporary_file_path -from pants.util.process_handler import subprocess -from pants_test.engine.examples.planners import setup_json_scheduler -from pants_test.engine.util import init_native - - -# TODO: These aren't tests themselves, so they should be under examples/ or testprojects/? -def visualize_execution_graph(scheduler): - with temporary_file_path(cleanup=False, suffix='.dot') as dot_file: - scheduler.visualize_graph_to_file(dot_file) - print('dot file saved to: {}'.format(dot_file)) - - with temporary_file_path(cleanup=False, suffix='.svg') as image_file: - subprocess.check_call('dot -Tsvg -o{} {}'.format(image_file, dot_file), shell=True) - print('svg file saved to: {}'.format(image_file)) - desktop.ui_open(image_file) - - -def visualize_build_request(build_root, goals, subjects): - native = init_native() - scheduler = setup_json_scheduler(build_root, native) - - execution_request = scheduler.build_request(goals, subjects) - # NB: Calls `schedule` independently of `execute`, in order to render a graph before validating it. - scheduler.schedule(execution_request) - visualize_execution_graph(scheduler) - - -def pop_build_root_and_goals(description, args): - def usage(error_message): - print(error_message, file=sys.stderr) - print(dedent(""" - {} - """.format(sys.argv[0])), file=sys.stderr) - sys.exit(1) - - if len(args) < 2: - usage('Must supply at least the build root path and one goal: {}'.format(description)) - - build_root = args.pop(0) - - if not os.path.isdir(build_root): - usage('First argument must be a valid build root, {} is not a directory.'.format(build_root)) - build_root = os.path.realpath(build_root) - - def is_goal(arg): return os.path.sep not in arg - - goals = [arg for arg in args if is_goal(arg)] - if not goals: - usage('Must supply at least one goal.') - - return build_root, goals, [arg for arg in args if not is_goal(arg)] - - -def main_addresses(): - build_root, goals, args = pop_build_root_and_goals( - '[build root path] [goal]+ [address spec]*', sys.argv[1:]) - - cmd_line_spec_parser = CmdLineSpecParser(build_root) - spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in args] - visualize_build_request(build_root, goals, spec_roots) - - -def main_filespecs(): - build_root, goals, args = pop_build_root_and_goals( - '[build root path] [filespecs]*', sys.argv[1:]) - - # Create PathGlobs for each arg relative to the buildroot. - path_globs = PathGlobs(include=args) - visualize_build_request(build_root, goals, path_globs) diff --git a/tests/python/pants_test/engine/test_engine.py b/tests/python/pants_test/engine/test_engine.py index 73b4fd33e232..538192d38de4 100644 --- a/tests/python/pants_test/engine/test_engine.py +++ b/tests/python/pants_test/engine/test_engine.py @@ -4,50 +4,15 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import os import unittest from builtins import object, str from textwrap import dedent -from pants.build_graph.address import Address -from pants.engine.nodes import Return from pants.engine.rules import RootRule, TaskRule, rule from pants.engine.selectors import Get, Select -from pants.util.contextutil import temporary_dir from pants.util.objects import datatype -from pants_test.engine.examples.planners import Classpath, setup_json_scheduler from pants_test.engine.scheduler_test_base import SchedulerTestBase -from pants_test.engine.util import (assert_equal_with_printing, init_native, - remove_locations_from_traceback) - - -class EngineExamplesTest(unittest.TestCase): - - _native = init_native() - - def setUp(self): - build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') - self.scheduler = setup_json_scheduler(build_root, self._native) - - self.java = Address.parse('src/java/simple') - - def request(self, products, *addresses): - return self.scheduler.execution_request(products, addresses) - - def test_serial_execution_simple(self): - request = self.request([Classpath], self.java) - result = self.scheduler.execute(request) - with temporary_dir() as tempdir: - self.scheduler.visualize_graph_to_file(os.path.join(tempdir, 'run.0.dot')) - self.assertEqual(Return(Classpath(creator='javac')), result.root_products[0][1]) - self.assertIsNone(result.error) - - def test_product_request_return(self): - count = 0 - for computed_product in self.scheduler.product_request(Classpath, [self.java]): - self.assertIsInstance(computed_product, Classpath) - count += 1 - self.assertGreater(count, 0) +from pants_test.engine.util import assert_equal_with_printing, remove_locations_from_traceback class A(object): diff --git a/tests/python/pants_test/engine/test_graph.py b/tests/python/pants_test/engine/test_graph.py deleted file mode 100644 index fe4230618a1c..000000000000 --- a/tests/python/pants_test/engine/test_graph.py +++ /dev/null @@ -1,161 +0,0 @@ -# coding=utf-8 -# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -import functools -import unittest -from builtins import range - -from pants.engine.nodes import Return - - -_WAITING = 'TODO: Waiting' - - -@unittest.skip('Skipped to expedite landing #3821; see: #4027.') -class GraphTest(unittest.TestCase): - - def setUp(self): - super(GraphTest, self).setUp() - self.pg = 'TODO: These tests need to be ported to native tests.' - - @classmethod - def _mk_chain(cls, graph, sequence, states=[_WAITING, Return]): - """Create a chain of dependencies (e.g. 'A'->'B'->'C'->'D') in the graph from a sequence.""" - for state in states: - dest = None - for src in reversed(sequence): - if state is _WAITING: - graph.add_dependencies(src, [dest] if dest else []) - else: - graph.complete_node(src, state([dest])) - dest = src - return sequence - - def test_disallow_completed_state_change(self): - self.pg.complete_node('A', Return('done!')) - with self.assertRaises('TODO: CompletedNodeException: These tests should be ported to native tests.'): - self.pg.add_dependencies('A', ['B']) - - def test_disallow_completing_with_incomplete_deps(self): - self.pg.add_dependencies('A', ['B']) - self.pg.add_dependencies('B', ['C']) - with self.assertRaises('TODO: IncompleteDependencyException: These tests should be ported to native tests.'): - self.pg.complete_node('A', Return('done!')) - - def test_dependency_edges(self): - self.pg.add_dependencies('A', ['B', 'C']) - self.assertEquals({'B', 'C'}, set(self.pg.dependencies_of('A'))) - self.assertEquals({'A'}, set(self.pg.dependents_of('B'))) - self.assertEquals({'A'}, set(self.pg.dependents_of('C'))) - - def test_cycle_simple(self): - self.pg.add_dependencies('A', ['B']) - self.pg.add_dependencies('B', ['A']) - # NB: Order matters: the second insertion is the one tracked as a cycle. - self.assertEquals({'B'}, set(self.pg.dependencies_of('A'))) - self.assertEquals(set(), set(self.pg.dependencies_of('B'))) - self.assertEquals(set(), set(self.pg.cyclic_dependencies_of('A'))) - self.assertEquals({'A'}, set(self.pg.cyclic_dependencies_of('B'))) - - def test_cycle_indirect(self): - self.pg.add_dependencies('A', ['B']) - self.pg.add_dependencies('B', ['C']) - self.pg.add_dependencies('C', ['A']) - - self.assertEquals({'B'}, set(self.pg.dependencies_of('A'))) - self.assertEquals({'C'}, set(self.pg.dependencies_of('B'))) - self.assertEquals(set(), set(self.pg.dependencies_of('C'))) - self.assertEquals(set(), set(self.pg.cyclic_dependencies_of('A'))) - self.assertEquals(set(), set(self.pg.cyclic_dependencies_of('B'))) - self.assertEquals({'A'}, set(self.pg.cyclic_dependencies_of('C'))) - - def test_cycle_long(self): - # Creating a long chain is allowed. - nodes = list(range(0, 100)) - self._mk_chain(self.pg, nodes, states=(_WAITING,)) - walked_nodes = [node for node, _ in self.pg.walk([nodes[0]])] - self.assertEquals(nodes, walked_nodes) - - # Closing the chain is not. - begin, end = nodes[0], nodes[-1] - self.pg.add_dependencies(end, [begin]) - self.assertEquals(set(), set(self.pg.dependencies_of(end))) - self.assertEquals({begin}, set(self.pg.cyclic_dependencies_of(end))) - - def test_walk(self): - nodes = list('ABCDEF') - self._mk_chain(self.pg, nodes) - walked_nodes = list((node for node, _ in self.pg.walk(nodes[0]))) - self.assertEquals(nodes, walked_nodes) - - def test_invalidate_all(self): - chain_list = list('ABCDEFGHIJKLMNOPQRSTUVWXYZ') - invalidators = ( - self.pg.invalidate, - functools.partial(self.pg.invalidate, lambda node, _: node == 'Z') - ) - - for invalidator in invalidators: - self._mk_chain(self.pg, chain_list) - - self.assertTrue(self.pg.completed_nodes()) - self.assertTrue(self.pg.dependents()) - self.assertTrue(self.pg.dependencies()) - self.assertTrue(self.pg.cyclic_dependencies()) - - invalidator() - - self.assertFalse(self.pg._nodes) - - def test_invalidate_partial(self): - comparison_pg = 'TODO: These tests need to be ported to native tests.' - chain_a = list('ABCDEF') - chain_b = list('GHIJKL') - - # Add two dependency chains to the primary graph. - self._mk_chain(self.pg, chain_a) - self._mk_chain(self.pg, chain_b) - - # Add only the dependency chain we won't invalidate to the comparison graph. - self._mk_chain(comparison_pg, chain_b) - - # Invalidate one of the chains in the primary graph from the right-most node. - self.pg.invalidate(lambda node, _: node == chain_a[-1]) - - # Ensure the final structure of the primary graph matches the comparison graph. - pg_structure = {n: e.structure() for n, e in self.pg._nodes.items()} - comparison_structure = {n: e.structure() for n, e in comparison_pg._nodes.items()} - self.assertEquals(pg_structure, comparison_structure) - - def test_invalidate_count(self): - self._mk_chain(self.pg, list('ABCDEFGHIJKLMNOPQRSTUVWXYZ')) - invalidated_count = self.pg.invalidate(lambda node, _: node == 'I') - self.assertEquals(invalidated_count, 9) - - def test_invalidate_partial_identity_check(self): - # Create a graph with a chain from A..Z. - chain = self._mk_chain(self.pg, list('ABCDEFGHIJKLMNOPQRSTUVWXYZ')) - self.assertTrue(list(self.pg.completed_nodes())) - - # Track the pre-invaliation nodes (from A..Q). - index_of_q = chain.index('Q') - before_nodes = [node for node, _ in self.pg.completed_nodes() if node in chain[:index_of_q + 1]] - self.assertTrue(before_nodes) - - # Invalidate all nodes under Q. - self.pg.invalidate(lambda node, _: node == chain[index_of_q]) - self.assertTrue(list(self.pg.completed_nodes())) - - # Check that the root node and all fs nodes were removed via a identity checks. - for node, entry in self.pg._nodes.items(): - self.assertFalse(node in before_nodes, 'node:\n{}\nwasnt properly removed'.format(node)) - - for associated in (entry.dependencies, entry.dependents, entry.cyclic_dependencies): - for associated_entry in associated: - self.assertFalse( - associated_entry.node in before_nodes, - 'node:\n{}\nis still associated with:\n{}\nin {}'.format(node, associated_entry.node, entry) - ) diff --git a/tests/python/pants_test/engine/test_rules.py b/tests/python/pants_test/engine/test_rules.py index 04e211120b5b..0478d1b98bd9 100644 --- a/tests/python/pants_test/engine/test_rules.py +++ b/tests/python/pants_test/engine/test_rules.py @@ -15,17 +15,9 @@ from pants.engine.selectors import Get, Select from pants.util.objects import Exactly from pants_test.engine.examples.parsers import JsonParser -from pants_test.engine.examples.planners import Goal from pants_test.engine.util import TargetTable, assert_equal_with_printing, create_scheduler -class AGoal(Goal): - - @classmethod - def products(cls): - return [A] - - class A(object): def __repr__(self): diff --git a/tests/python/pants_test/engine/test_scheduler.py b/tests/python/pants_test/engine/test_scheduler.py index 9a96a92aecb0..8b29666bfabb 100644 --- a/tests/python/pants_test/engine/test_scheduler.py +++ b/tests/python/pants_test/engine/test_scheduler.py @@ -4,242 +4,16 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import os import unittest -from builtins import object, str +from builtins import object from textwrap import dedent -from pants.base.cmd_line_spec_parser import CmdLineSpecParser -from pants.base.specs import Specs -from pants.build_graph.address import Address -from pants.engine.addressable import BuildFileAddresses -from pants.engine.nodes import Return, Throw from pants.engine.rules import RootRule, TaskRule -from pants.engine.selectors import Select, SelectVariant -from pants.util.contextutil import temporary_dir -from pants_test.engine.examples.planners import (ApacheThriftJavaConfiguration, Classpath, GenGoal, - Jar, ThriftSources, setup_json_scheduler) -from pants_test.engine.util import (assert_equal_with_printing, create_scheduler, init_native, +from pants.engine.selectors import Select +from pants_test.engine.util import (assert_equal_with_printing, create_scheduler, remove_locations_from_traceback) -walk = "TODO: Should port tests that attempt to inspect graph internals to the native code." - - -class SchedulerTest(unittest.TestCase): - - _native = init_native() - - def setUp(self): - build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') - self.spec_parser = CmdLineSpecParser(build_root) - self.scheduler = setup_json_scheduler(build_root, self._native) - - self.guava = Address.parse('3rdparty/jvm:guava') - self.thrift = Address.parse('src/thrift/codegen/simple') - self.java = Address.parse('src/java/codegen/simple') - self.java_simple = Address.parse('src/java/simple') - self.java_multi = Address.parse('src/java/multiple_classpath_entries') - self.no_variant_thrift = Address.parse('src/java/codegen/selector:conflict') - self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured') - self.resources = Address.parse('src/resources/simple') - self.consumes_resources = Address.parse('src/java/consumes_resources') - self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty') - self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') - self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common') - self.managed_resolve_latest = Address.parse('3rdparty/jvm/managed:latest-hadoop') - self.inferred_deps = Address.parse('src/scala/inferred_deps') - - def tearDown(self): - super(SchedulerTest, self).tearDown() - # Without eagerly dropping this reference, each instance created for a test method - # will live until all tests in this class have completed: can confirm by editing - # the `scheduler_destroy` call in `src/python/pants/engine/native.py`. - self.scheduler = None - - def parse_specs(self, *specs): - return Specs(tuple(self.spec_parser.parse_spec(spec) for spec in specs)) - - def assert_select_for_subjects(self, walk, selector, subjects, variants=None): - raise ValueError(walk) - - def build(self, execution_request): - """Execute the given request and return roots as a list of ((subject, product), value) tuples.""" - result = self.scheduler.execute(execution_request) - self.assertIsNone(result.error) - return result.root_products - - def request(self, products, *subjects): - return self.scheduler.execution_request(products, subjects) - - def assert_root(self, root, subject, return_value): - """Asserts that the given root has the given result.""" - self.assertEquals(subject, root[0][0]) - self.assertEquals(Return(return_value), root[1]) - - def assert_root_failed(self, root, subject, msg_str): - """Asserts that the root was a Throw result containing the given msg string.""" - self.assertEquals(subject, root[0][0]) - self.assertEquals(Throw, type(root[1])) - self.assertIn(msg_str, str(root[1].exc)) - - def test_compile_only_3rdparty(self): - build_request = self.request([Classpath], self.guava) - root, = self.build(build_request) - self.assert_root(root, self.guava, Classpath(creator='ivy_resolve')) - - @unittest.skip('Skipped to expedite landing #3821; see: #4027.') - def test_compile_only_3rdparty_internal(self): - build_request = self.request([Classpath], '3rdparty/jvm:guava') - root, = self.build(build_request) - - # Expect a SelectNode for each of the Jar/Classpath. - self.assert_select_for_subjects(walk, Select(Jar), [self.guava]) - self.assert_select_for_subjects(walk, Select(Classpath), [self.guava]) - - @unittest.skip('Skipped to expedite landing #3821; see: #4020.') - def test_gen(self): - build_request = self.request([GenGoal], self.thrift) - root, = self.build(build_request) - - # Root: expect the synthetic GenGoal product. - self.assert_root(root, self.thrift, GenGoal("non-empty input to satisfy the Goal constructor")) - - variants = {'thrift': 'apache_java'} - # Expect ThriftSources to have been selected. - self.assert_select_for_subjects(walk, Select(ThriftSources), [self.thrift], variants=variants) - # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants. - self.assert_select_for_subjects(walk, SelectVariant(ApacheThriftJavaConfiguration, - variant_key='thrift'), - [self.thrift], - variants=variants) - - @unittest.skip('Skipped to expedite landing #3821; see: #4020.') - def test_codegen_simple(self): - build_request = self.request([Classpath], self.java) - root, = self.build(build_request) - - # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants. - subjects = [self.guava, self.java, self.thrift] - variant_subjects = [ - Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2', type_alias='jar'), - Jar(org='commons-lang', name='commons-lang', rev='2.5', type_alias='jar'), - Address.parse('src/thrift:slf4j-api')] - - # Root: expect a DependenciesNode depending on a SelectNode with compilation via javac. - self.assert_root(root, self.java, Classpath(creator='javac')) - - # Confirm that exactly the expected subjects got Classpaths. - self.assert_select_for_subjects(walk, Select(Classpath), subjects) - self.assert_select_for_subjects(walk, Select(Classpath), variant_subjects, - variants={'thrift': 'apache_java'}) - - def test_consumes_resources(self): - build_request = self.request([Classpath], self.consumes_resources) - root, = self.build(build_request) - self.assert_root(root, self.consumes_resources, Classpath(creator='javac')) - - @unittest.skip('Skipped to expedite landing #3821; see: #4027.') - def test_consumes_resources_internal(self): - build_request = self.request([Classpath], self.consumes_resources) - root, = self.build(build_request) - - # Confirm a classpath for the resources target and other subjects. We know that they are - # reachable from the root (since it was involved in this walk). - subjects = [self.resources, - self.consumes_resources, - self.guava] - self.assert_select_for_subjects(walk, Select(Classpath), subjects) - - @unittest.skip('Skipped to expedite landing #3821; see: #4020.') - def test_managed_resolve(self): - """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" - build_request = self.request([Classpath], self.consumes_managed_thirdparty) - root, = self.build(build_request) - - # Validate the root. - self.assert_root(root, self.consumes_managed_thirdparty, Classpath(creator='javac')) - - # Confirm that we produced classpaths for the managed jars. - managed_jars = [self.managed_guava, self.managed_hadoop] - self.assert_select_for_subjects(walk, Select(Classpath), [self.consumes_managed_thirdparty]) - self.assert_select_for_subjects(walk, Select(Classpath), managed_jars, - variants={'resolve': 'latest-hadoop'}) - - # Confirm that the produced jars had the appropriate versions. - self.assertEquals({Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'), - Jar('com.google.guava', 'guava', '18.0')}, - {ret.value for node, ret in walk - if node.product == Jar}) - - def test_dependency_inference(self): - """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" - build_request = self.request([Classpath], self.inferred_deps) - root, = self.build(build_request) - self.assert_root(root, self.inferred_deps, Classpath(creator='scalac')) - - @unittest.skip('Skipped to expedite landing #3821; see: #4027.') - def test_dependency_inference_internal(self): - """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" - build_request = self.request([Classpath], self.inferred_deps) - root, = self.build(build_request) - - # Confirm that we requested a classpath for the root and inferred targets. - self.assert_select_for_subjects(walk, Select(Classpath), [self.inferred_deps, self.java_simple]) - - def test_multiple_classpath_entries(self): - """Multiple Classpath products for a single subject currently cause a failure.""" - build_request = self.request([Classpath], self.java_multi) - root, = self.build(build_request) - - # Validate that the root failed. - self.assert_root_failed(root, self.java_multi, "Conflicting values produced for") - - def test_descendant_specs(self): - """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory.""" - specs = self.parse_specs('3rdparty/jvm::') - build_request = self.scheduler.execution_request([BuildFileAddresses], [specs]) - ((subject, _), root), = self.build(build_request) - - # Validate the root. - self.assertEqual(specs, subject) - self.assertEqual(BuildFileAddresses, type(root.value)) - - # Confirm that a few expected addresses are in the list. - self.assertIn(self.guava, root.value.dependencies) - self.assertIn(self.managed_guava, root.value.dependencies) - self.assertIn(self.managed_resolve_latest, root.value.dependencies) - - def test_sibling_specs(self): - """Test that sibling Addresses are parsed in the 3rdparty/jvm directory.""" - specs = self.parse_specs('3rdparty/jvm:') - build_request = self.scheduler.execution_request([BuildFileAddresses], [specs]) - ((subject, _), root), = self.build(build_request) - - # Validate the root. - self.assertEqual(specs, subject) - self.assertEqual(BuildFileAddresses, type(root.value)) - - # Confirm that an expected address is in the list. - self.assertIn(self.guava, root.value.dependencies) - # And that a subdirectory address is not. - self.assertNotIn(self.managed_guava, root.value.dependencies) - - def test_scheduler_visualize(self): - specs = self.parse_specs('3rdparty/jvm::') - build_request = self.request([BuildFileAddresses], specs) - self.build(build_request) - - with temporary_dir() as td: - output_path = os.path.join(td, 'output.dot') - self.scheduler.visualize_graph_to_file(output_path) - with open(output_path, 'rb') as fh: - graphviz_output = fh.read().strip() - - self.assertIn('digraph', graphviz_output) - self.assertIn(' -> ', graphviz_output) - - class A(object): pass