fix python_dist() caching

src/python/pants/backend/python/tasks/pex_build_util.py

@@ -166,24 +166,29 @@ def _resolve_multi(interpreter, requirements, platforms, find_links):
   return distributions
 
 
-def inject_synthetic_dist_requirements(build_graph, local_built_dists, synthetic_address, binary_tgt=None):
-  """Inject a synthetic requirements library from a local wheel.
+def inject_synthetic_dist_requirements(build_graph, local_built_dists, synthetic_address, in_tgts=None):
+  """Inject a synthetic requirements library from a locally-built wheel.
 
   :param build_graph: The build graph needed for injecting synthetic targets.
   :param local_built_dists: A list of paths to locally built wheels to package into
   requirements libraries.
   :param synthetic_address: A generative address for addressing synthetic targets.
-  :param binary_tgt: An optional parameter to be passed only when called by the `python_binary_create`
-  task. This is needed to ensure that only python_dist targets in a binary target's closure are included
-  in the binary for the case where a user specifies mulitple binary targets in a single invocation of
-  `./pants binary`.
-  :return: a :class: `PythonRequirementLibrary` containing a requirements that maps to a locally-built wheels.
+  :param in_tgts: If not None, an iterable of :class:`PythonDistribution`
+  targets. This is needed by the :class:`PythonBinaryCreate` task to ensure that
+  only :class:`PythonDistribution` targets in a binary target's closure are
+  included in the binary for the case where a user specifies multiple binary
+  targets in a single invocation of `./pants binary`.
+  :return: a :class:`PythonRequirementLibrary` containing a requirements that maps to locally-built wheels.
   """
-  def should_create_req(bin_tgt, loc):
-    if not bin_tgt:
+  tgt_ids = None
+  if in_tgts is not None:
+    tgt_ids = [tgt.id for tgt in in_tgts]
+
+  def within_allowed_targets(whl_loc):
+    if not tgt_ids:
       return True
-    # Ensure that a target is in a binary target's closure. See docstring for more detail.
-    return any([tgt.id in loc for tgt in bin_tgt.closure()])
+    # Ensure that a target is in a specified set. See docstring for more detail.
+    return any([tid in whl_loc for tid in tgt_ids])
 
   def python_requirement_from_wheel(path):
     base = os.path.basename(path)
@@ -195,7 +200,7 @@ def python_requirement_from_wheel(path):
   local_whl_reqs = [
     python_requirement_from_wheel(whl_location)
     for whl_location in local_built_dists
-    if should_create_req(binary_tgt, whl_location)
+    if within_allowed_targets(whl_location)
   ]
 
   if not local_whl_reqs:

src/python/pants/backend/python/tasks/python_binary_create.py

@@ -134,10 +134,10 @@ def _create_binary(self, binary_tgt, results_dir):
       # Handle locally-built python distribution dependencies.
       built_dists = self.context.products.get_data(BuildLocalPythonDistributions.PYTHON_DISTS)
       if built_dists:
-        req_tgts = inject_synthetic_dist_requirements(self.context.build_graph,
-                                                      built_dists,
-                                                      ':'.join(2 * [binary_tgt.invalidation_hash()]),
-                                                      binary_tgt) + req_tgts
+        synthetic_address = ':'.join(2 * [binary_tgt.invalidation_hash()])
+        locally_built_dist_libs = inject_synthetic_dist_requirements(
+          self.context.build_graph, built_dists, synthetic_address, binary_tgt.closure())
+        req_tgts = locally_built_dist_libs + req_tgts
 
       dump_requirements(builder, interpreter, req_tgts, self.context.log, binary_tgt.platforms)
 

src/python/pants/backend/python/tasks/resolve_requirements_task_base.py

@@ -62,13 +62,21 @@ def resolve_requirements(self, req_libs, local_dist_targets=None):
           # Handle locally-built python distribution dependencies.
           built_dists = self.context.products.get_data(BuildLocalPythonDistributions.PYTHON_DISTS)
           if built_dists:
-            req_libs = inject_synthetic_dist_requirements(self.context.build_graph,
-                                                          built_dists,
-                                                          ':'.join(2 * [target_set_id])) + req_libs
-          self._build_requirements_pex(interpreter, safe_path, req_libs)
+            synthetic_address = ':'.join(2 * [target_set_id])
+            locally_built_dist_libs = inject_synthetic_dist_requirements(
+              self.context.build_graph, built_dists, synthetic_address)
+            req_libs = locally_built_dist_libs + req_libs
+
+          self._build_requirements_pex(interpreter, safe_path, req_libs, built_dists)
+
     return PEX(path, interpreter=interpreter)
 
-  def _build_requirements_pex(self, interpreter, path, req_libs):
+  def _build_requirements_pex(self, interpreter, path, req_libs, built_dists):
     builder = PEXBuilder(path=path, interpreter=interpreter, copy=True)
     dump_requirements(builder, interpreter, req_libs, self.context.log)
+
+    if built_dists:
+      for dist in built_dists:
+        builder.add_dist_location(dist)
+
     builder.freeze()

tests/python/pants_test/backend/python/tasks/test_python_distribution_integration.py

@@ -6,6 +6,7 @@
                         unicode_literals, with_statement)
 
 import os
+import re
 
 from pants.base.build_environment import get_buildroot
 from pants.util.process_handler import subprocess
@@ -57,6 +58,22 @@ def test_with_install_requires(self):
     self.assertIn('United States', output)
     os.remove(pex)
 
+  @staticmethod
+  def rewrite_hello_c_source(orig_content):
+    return re.sub('"Super hello"', '"Super hello!"', orig_content)
+
+  def test_invalidation(self):
+    run_command=['run', '{}:main_with_no_conflict'.format(self.fasthello_install_requires)]
+    unmodified_pants_run = self.run_pants(command=run_command)
+    self.assert_success(unmodified_pants_run)
+    self.assertIn('Super hello\n', unmodified_pants_run.stdout_data)
+
+    c_source_file = '{}/super_greet.c'.format(self.fasthello_install_requires)
+    with self.rewrite_file_content(c_source_file, self.rewrite_hello_c_source):
+      modified_pants_run = self.run_pants(command=run_command)
+      self.assert_success(modified_pants_run)
+      self.assertIn('Super hello!\n', modified_pants_run.stdout_data)
+
   def test_with_conflicting_transitive_deps(self):
     command=['run', '{}:main_with_conflicting_dep'.format(self.fasthello_install_requires)]
     pants_run = self.run_pants(command=command)

tests/python/pants_test/pants_run_integration_test.py

@@ -423,6 +423,27 @@ def temporary_file_content(self, path, content):
     finally:
       os.unlink(path)
 
+  @contextmanager
+  def rewrite_file_content(self, path, transform_content_fun):
+    """Temporarily rewrite the contents of a file for the purpose of an integration test."""
+    base_name = os.path.basename(path)
+    path = os.path.realpath(path)
+    assert path.startswith(
+      os.path.realpath(get_buildroot())), 'cannot write paths outside of the buildroot!'
+    assert os.path.isfile(path), "path '{}' does not exist or is not a file!".format(path)
+    file_content = None
+    with open(path, 'r') as fh:
+      file_content = fh.read()
+    new_content = transform_content_fun(file_content)
+    with temporary_dir() as containing_dir:
+      tmp_src_filename = os.path.join(containing_dir, base_name)
+      try:
+        os.rename(path, tmp_src_filename)
+        with self.temporary_file_content(path, new_content):
+          yield
+      finally:
+        os.rename(tmp_src_filename, path)
+
   @contextmanager
   def mock_buildroot(self):
     """Construct a mock buildroot and return a helper object for interacting with it."""