From ee3df93c4a6a6a9354186378a5c7bdd52e43b3c2 Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <dwagnerhall@twitter.com>
Date: Thu, 21 Jun 2018 16:43:34 +0100
Subject: [PATCH 01/10] Targets always have a Snapshot

Unless they're somehow specially generated, their Snapshot will be
popualted during parsing in parallel in the engine.

If they weren't generated during parsing, the scheduler arg will be used
to synchronously capture one on demand.
---
 .../contrib/scrooge/tasks/test_scrooge_gen.py |  6 +-
 .../codegen/antlr/java/antlr_java_gen.py      |  2 +
 .../codegen/antlr/python/antlr_py_gen.py      |  2 +
 src/python/pants/build_graph/BUILD            |  1 +
 src/python/pants/build_graph/target.py        |  8 ++
 src/python/pants/engine/BUILD                 |  1 -
 src/python/pants/engine/legacy/graph.py       |  8 +-
 src/python/pants/source/BUILD                 | 10 ++
 src/python/pants/source/payload_fields.py     | 16 +++-
 src/python/pants/source/wrapped_globs.py      | 31 ++++---
 src/python/pants/task/simple_codegen_task.py  | 93 +++++++++++++------
 .../codegen/antlr/java/test_antlr_java_gen.py |  5 +-
 tests/python/pants_test/source/BUILD          |  1 +
 .../pants_test/source/test_payload_fields.py  | 24 +++--
 .../pants_test/source/test_wrapped_globs.py   | 37 ++++++--
 .../task/test_simple_codegen_task.py          | 11 ++-
 16 files changed, 181 insertions(+), 75 deletions(-)

diff --git a/contrib/scrooge/tests/python/pants_test/contrib/scrooge/tasks/test_scrooge_gen.py b/contrib/scrooge/tests/python/pants_test/contrib/scrooge/tasks/test_scrooge_gen.py
index 1197836c58f..4e0702a94c5 100644
--- a/contrib/scrooge/tests/python/pants_test/contrib/scrooge/tasks/test_scrooge_gen.py
+++ b/contrib/scrooge/tests/python/pants_test/contrib/scrooge/tasks/test_scrooge_gen.py
@@ -15,7 +15,6 @@
 from pants.base.exceptions import TargetDefinitionException
 from pants.build_graph.build_file_aliases import BuildFileAliases
 from pants.goal.context import Context
-from pants.source.wrapped_globs import EagerFilesetWithSpec
 from pants_test.jvm.nailgun_task_test_base import NailgunTaskTestBase
 from twitter.common.collections import OrderedSet
 
@@ -139,10 +138,7 @@ def _test_help(self, language, library_type, compiler_args, sources):
       self.assertEquals(call_kwargs['fatal_warnings'], False)
 
       sources = call_kwargs['sources']
-      if isinstance(sources, EagerFilesetWithSpec):
-        self.assertEquals(sources.files, [])
-      else:
-        self.assertEquals(sources, [])
+      self.assertEquals(sources.files, ())
 
     finally:
       Context.add_new_target = saved_add_new_target
diff --git a/src/python/pants/backend/codegen/antlr/java/antlr_java_gen.py b/src/python/pants/backend/codegen/antlr/java/antlr_java_gen.py
index a7b3bc51007..c0199d193e8 100644
--- a/src/python/pants/backend/codegen/antlr/java/antlr_java_gen.py
+++ b/src/python/pants/backend/codegen/antlr/java/antlr_java_gen.py
@@ -39,6 +39,8 @@ class AntlrJavaGen(SimpleCodegenTask, NailgunTask):
   """Generate .java source code from ANTLR grammar files."""
   gentarget_type = JavaAntlrLibrary
 
+  sources_globs = ('**/*.java',)
+
   class AmbiguousPackageError(TaskError):
     """Raised when a java package cannot be unambiguously determined for a JavaAntlrLibrary."""
 
diff --git a/src/python/pants/backend/codegen/antlr/python/antlr_py_gen.py b/src/python/pants/backend/codegen/antlr/python/antlr_py_gen.py
index ce6a7a22b9d..0ffbd409536 100644
--- a/src/python/pants/backend/codegen/antlr/python/antlr_py_gen.py
+++ b/src/python/pants/backend/codegen/antlr/python/antlr_py_gen.py
@@ -33,6 +33,8 @@ class AntlrPyGen(SimpleCodegenTask, NailgunTask):
   """Generate Python source code from ANTLR grammar files."""
   gentarget_type = PythonAntlrLibrary
 
+  sources_globs = ('**/*.py',)
+
   @classmethod
   def register_options(cls, register):
     super(AntlrPyGen, cls).register_options(register)
diff --git a/src/python/pants/build_graph/BUILD b/src/python/pants/build_graph/BUILD
index f4dd17ee70d..735fe19f6de 100644
--- a/src/python/pants/build_graph/BUILD
+++ b/src/python/pants/build_graph/BUILD
@@ -21,6 +21,7 @@ python_library(
     'src/python/pants/base:validation',
     'src/python/pants/option',
     'src/python/pants/source',
+    'src/python/pants/source:payload_fields',
     'src/python/pants/subsystem',
     'src/python/pants/util:dirutil',
     'src/python/pants/util:memo',
diff --git a/src/python/pants/build_graph/target.py b/src/python/pants/build_graph/target.py
index 526f8f73f93..153045b3028 100644
--- a/src/python/pants/build_graph/target.py
+++ b/src/python/pants/build_graph/target.py
@@ -549,6 +549,14 @@ def sources_count(self):
     """Returns the count of source files owned by the target."""
     return len(self._sources_field.sources.files)
 
+  def sources_snapshot(self, scheduler=None):
+    """
+    Get a Snapshot of the sources attribute of this target.
+
+    This API is experimental, and is subject to change.
+    """
+    return self._sources_field.snapshot(scheduler=scheduler)
+
   @property
   def derived_from(self):
     """Returns the target this target was derived from.
diff --git a/src/python/pants/engine/BUILD b/src/python/pants/engine/BUILD
index 00dc0eb4bf9..08ad5123e97 100644
--- a/src/python/pants/engine/BUILD
+++ b/src/python/pants/engine/BUILD
@@ -139,7 +139,6 @@ python_library(
     '3rdparty/python/twitter/commons:twitter.common.collections',
     ':selectors',
     'src/python/pants/base:specs',
-    'src/python/pants/build_graph',
     'src/python/pants/util:objects',
   ]
 )
diff --git a/src/python/pants/engine/legacy/graph.py b/src/python/pants/engine/legacy/graph.py
index ae3f0b631ef..375a23356e7 100644
--- a/src/python/pants/engine/legacy/graph.py
+++ b/src/python/pants/engine/legacy/graph.py
@@ -26,7 +26,6 @@
 from pants.engine.selectors import Get, Select
 from pants.option.global_options import GlobMatchErrorBehavior
 from pants.source.wrapped_globs import EagerFilesetWithSpec, FilesetRelPathWrapper
-from pants.util.dirutil import fast_relpath
 from pants.util.objects import Collection, datatype
 
 
@@ -364,9 +363,6 @@ def hydrate_target(target_adaptor):
 
 
 def _eager_fileset_with_spec(spec_path, filespec, snapshot, include_dirs=False):
-  fds = snapshot.path_stats if include_dirs else snapshot.files
-  files = tuple(fast_relpath(fd.path, spec_path) for fd in fds)
-
   rel_include_globs = filespec['globs']
 
   relpath_adjusted_filespec = FilesetRelPathWrapper.to_filespec(rel_include_globs, spec_path)
@@ -376,8 +372,8 @@ def _eager_fileset_with_spec(spec_path, filespec, snapshot, include_dirs=False):
 
   return EagerFilesetWithSpec(spec_path,
                               relpath_adjusted_filespec,
-                              files=files,
-                              files_hash=snapshot.directory_digest.fingerprint)
+                              snapshot,
+                              include_dirs=include_dirs)
 
 
 @rule(HydratedField, [Select(SourcesField), Select(GlobMatchErrorBehavior)])
diff --git a/src/python/pants/source/BUILD b/src/python/pants/source/BUILD
index 44e94df6adc..0b921fab32f 100644
--- a/src/python/pants/source/BUILD
+++ b/src/python/pants/source/BUILD
@@ -2,6 +2,7 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 python_library(
+  sources = globs('*.py', exclude=[globs('payload_fields.py')]),
   dependencies=[
     '3rdparty/python:six',
     '3rdparty/python/twitter/commons:twitter.common.dirutil',
@@ -14,3 +15,12 @@ python_library(
     'src/python/pants/util:memo',
   ]
 )
+
+python_library(
+  name = 'payload_fields',
+  sources = ['payload_fields.py'],
+  dependencies = [
+    ':source',
+    'src/python/pants/engine:fs',
+  ],
+)
diff --git a/src/python/pants/source/payload_fields.py b/src/python/pants/source/payload_fields.py
index 0109efd3cb7..ec4bd40004f 100644
--- a/src/python/pants/source/payload_fields.py
+++ b/src/python/pants/source/payload_fields.py
@@ -8,9 +8,10 @@
 from hashlib import sha1
 
 from pants.base.payload_field import PayloadField
+from pants.engine.fs import PathGlobs, Snapshot
 from pants.source.filespec import matches_filespec
 from pants.source.source_root import SourceRootConfig
-from pants.source.wrapped_globs import FilesetWithSpec
+from pants.source.wrapped_globs import EagerFilesetWithSpec, FilesetWithSpec
 from pants.util.memo import memoized_property
 
 
@@ -64,6 +65,19 @@ def address(self):
     """Returns the address this sources field refers to (used by some derived classes)"""
     return self._ref_address
 
+  def snapshot(self, scheduler=None):
+    """
+    Returns a Snapshot containing the sources, relative to the build root.
+
+    This API is experimental, and subject to change.
+    """
+    if isinstance(self._sources, EagerFilesetWithSpec):
+      snapshot = self._sources.snapshot
+      if snapshot is not None:
+        return snapshot
+    input_pathglobs = PathGlobs(tuple(self.relative_to_buildroot()))
+    return scheduler.product_request(Snapshot, [input_pathglobs])[0]
+
   def relative_to_buildroot(self):
     """All sources joined with their relative paths."""
     return list(self.sources.paths_from_buildroot_iter())
diff --git a/src/python/pants/source/wrapped_globs.py b/src/python/pants/source/wrapped_globs.py
index 4e83631c5c0..851bd3623ca 100644
--- a/src/python/pants/source/wrapped_globs.py
+++ b/src/python/pants/source/wrapped_globs.py
@@ -13,6 +13,7 @@
 from twitter.common.dirutil.fileset import Fileset
 
 from pants.base.build_environment import get_buildroot
+from pants.engine.fs import EMPTY_SNAPSHOT
 from pants.util.dirutil import fast_relpath, fast_relpath_optional
 from pants.util.memo import memoized_property
 from pants.util.meta import AbstractClass
@@ -28,7 +29,7 @@ def _no_content(path):
   @staticmethod
   def empty(rel_root):
     """Creates an empty FilesetWithSpec object for the given rel_root."""
-    return EagerFilesetWithSpec(rel_root, {'globs': []}, tuple(), '<empty>')
+    return EagerFilesetWithSpec(rel_root, {'globs': []}, EMPTY_SNAPSHOT)
 
   @abstractmethod
   def matches(self, path_from_buildroot):
@@ -80,32 +81,40 @@ def paths_from_buildroot_iter(self):
 
 
 class EagerFilesetWithSpec(FilesetWithSpec):
-  def __init__(self, rel_root, filespec, files, files_hash):
+
+  def __init__(self, rel_root, filespec, snapshot, include_dirs=False):
     """
     :param rel_root: The root for the given filespec, relative to the buildroot.
     :param filespec: A filespec as generated by `FilesetRelPathWrapper`, which represents
       what globs or file list it came from. Must be relative to buildroot.
-    :param files: A list of matched files, with declared order and duplicates preserved.
-    :param files_hash: A string fingerprint for all files in the fileset.
+    :param snapshot: A Snapshot of the files, rooted at the buildroot.
     """
     super(EagerFilesetWithSpec, self).__init__(rel_root, filespec)
-    self._files = files
-    self._files_hash = files_hash
+    self._include_dirs = include_dirs
+    self._snapshot = snapshot
 
-  @property
+  @memoized_property
   def files(self):
-    return self._files
+    fds = self._snapshot.path_stats if self._include_dirs else self._snapshot.files
+    return tuple(fast_relpath(fd.path, self.rel_root) for fd in fds)
 
   @property
   def files_hash(self):
-    return self._files_hash
+    return self._snapshot.directory_digest.fingerprint
+
+  @property
+  def snapshot(self):
+    return self._snapshot
 
   def __repr__(self):
-    return 'EagerFilesetWithSpec(rel_root={!r}, files={!r})'.format(self.rel_root, self.files)
+    return 'EagerFilesetWithSpec(rel_root={!r}, snapshot={!r})'.format(
+      self.rel_root,
+      self._snapshot,
+    )
 
   def matches(self, path_from_buildroot):
     path_relative_to_rel_root = fast_relpath_optional(path_from_buildroot, self.rel_root)
-    return path_relative_to_rel_root is not None and path_relative_to_rel_root in self._files
+    return path_relative_to_rel_root is not None and path_relative_to_rel_root in self.files
 
 
 class LazyFilesetWithSpec(FilesetWithSpec):
diff --git a/src/python/pants/task/simple_codegen_task.py b/src/python/pants/task/simple_codegen_task.py
index 72e7497cb4d..0ac7143776f 100644
--- a/src/python/pants/task/simple_codegen_task.py
+++ b/src/python/pants/task/simple_codegen_task.py
@@ -13,10 +13,12 @@
 from twitter.common.collections import OrderedSet
 
 from pants.base.build_environment import get_buildroot
+from pants.base.deprecated import deprecated_conditional
 from pants.base.exceptions import TaskError
 from pants.base.workunit import WorkUnitLabel
 from pants.build_graph.address import Address
 from pants.build_graph.address_lookup_error import AddressLookupError
+from pants.engine.fs import PathGlobs, PathGlobsAndRoot
 from pants.source.wrapped_globs import EagerFilesetWithSpec, FilesetRelPathWrapper
 from pants.task.task import Task
 from pants.util.dirutil import fast_relpath, safe_delete, safe_walk
@@ -34,6 +36,15 @@ class SimpleCodegenTask(Task):
   # E.g., JavaThriftLibrary. If not provided, the subclass must implement is_gentarget.
   gentarget_type = None
 
+  # Subclasses may override to provide a list of glob patterns matching the generated sources,
+  # relative to the target's workdir.
+  # These must be a tuple of strings, e.g. ('**/*.java',).
+  # If this is not set, the deprecated find_sources will be used.
+  sources_globs = None
+
+  # Tuple of glob patterns to exclude from the above matches.
+  sources_exclude_globs = ()
+
   def __init__(self, context, workdir):
     """
     Add pass-thru Task Constructor for public API visibility.
@@ -210,17 +221,27 @@ def execute(self):
 
       with self.context.new_workunit(name='execute', labels=[WorkUnitLabel.MULTITOOL]):
         for vt in invalidation_check.all_vts:
+
+          sources = None
+          must_recapture = False
+
           # Build the target and handle duplicate sources.
           if not vt.valid:
             if self._do_validate_sources_present(vt.target):
               self.execute_codegen(vt.target, vt.results_dir)
-              self._handle_duplicate_sources(vt.target, vt.results_dir)
+              sources = self._capture_sources(vt.target, vt.results_dir)
+              must_recapture = self._handle_duplicate_sources(vt.target, vt.results_dir, sources)
             vt.update()
 
+          # _handle_duplicate_sources may delete files from the filesystem, so we need to
+          # re-capture the sources.
+          if sources is None or must_recapture:
+            sources = self._capture_sources(vt.target, vt.results_dir)
+
           self._inject_synthetic_target(
             vt.target,
             vt.results_dir,
-            vt.cache_key,
+            sources,
           )
         self._mark_transitive_invalidation_hashes_dirty(
           vt.target.address for vt in invalidation_check.all_vts
@@ -246,35 +267,42 @@ def synthetic_target_dir(self, target, target_workdir):
     """
     return target_workdir
 
-  def _create_sources_with_fingerprint(self, target_workdir, fingerprint, files):
-    """Create an EagerFilesetWithSpec to pass to the sources argument for synthetic target injection.
-
-    We are creating and passing an EagerFilesetWithSpec to the synthetic target injection in the
-    hopes that it will save the time of having to refingerprint the sources.
+  def _capture_sources(self, target, target_workdir):
+    if self.sources_globs is None:
+      files = list(self.find_sources(target, target_workdir))
+    else:
+      files = self.sources_globs
 
-    :param target_workdir: The directory containing the generated code for the target.
-    :param fingerprint: the fingerprint of the VersionedTarget with which the EagerFilesetWithSpec
-           will be created.
-    :param files: a list of exact paths to generated sources.
-    """
     results_dir_relpath = os.path.relpath(target_workdir, get_buildroot())
-    filespec = FilesetRelPathWrapper.to_filespec(
-      [os.path.join(results_dir_relpath, file) for file in files])
-    return EagerFilesetWithSpec(results_dir_relpath, filespec=filespec,
-      files=files, files_hash='{}.{}'.format(fingerprint.id, fingerprint.hash))
+    buildroot_relative_globs = tuple(os.path.join(results_dir_relpath, file) for file in files)
+    buildroot_relative_excludes = tuple(
+      os.path.join(results_dir_relpath, file)
+        for file in self.sources_exclude_globs
+    )
+
+    snapshot = self.context._scheduler.capture_snapshots((
+      PathGlobsAndRoot(
+        PathGlobs(buildroot_relative_globs, buildroot_relative_excludes),
+        str(get_buildroot()),
+      ),
+    ))[0]
+
+    return EagerFilesetWithSpec(
+      results_dir_relpath,
+      FilesetRelPathWrapper.to_filespec(buildroot_relative_globs),
+      snapshot,
+    )
 
   def _inject_synthetic_target(
     self,
     target,
     target_workdir,
-    fingerprint,
+    sources,
   ):
     """Create, inject, and return a synthetic target for the given target and workdir.
 
     :param target: The target to inject a synthetic target for.
     :param target_workdir: The work directory containing the generated code for the target.
-    :param fingerprint: The fingerprint to create the synthetic target
-           with to avoid re-fingerprinting.
     """
 
     synthetic_target_type = self.synthetic_target_type(target)
@@ -301,10 +329,6 @@ def _inject_synthetic_target(
 
       copied_attributes['exports'] = sorted(union)
 
-    sources = list(self.find_sources(target, target_workdir))
-    if fingerprint:
-      sources = self._create_sources_with_fingerprint(target_workdir, fingerprint, sources)
-
     synthetic_target = self.context.add_new_target(
       address=self._get_synthetic_address(target, target_workdir),
       target_type=synthetic_target_type,
@@ -372,6 +396,13 @@ def find_sources(self, target, target_workdir):
     :return: A set of filepaths relative to the target_workdir.
     :rtype: OrderedSet
     """
+    deprecated_conditional(
+      lambda: True,
+      '1.10.0.dev0',
+      'SimpleCodegenTask.find_sources is deprecated. Subclasses should instead specify '
+      'sources_globs and sources_exclude_globs. '
+      'Class to update: {}'.format(self.__class__.__name__)
+    )
     return OrderedSet(self._find_sources_in_workdir(target_workdir))
 
   def _find_sources_in_workdir(self, target_workdir):
@@ -381,12 +412,14 @@ def _find_sources_in_workdir(self, target_workdir):
       for name in files:
         yield os.path.join(rel_root, name)
 
-  def _handle_duplicate_sources(self, target, target_workdir):
+  def _handle_duplicate_sources(self, target, target_workdir, sources):
     """Handles duplicate sources generated by the given gen target by either failure or deletion.
 
     This method should be called after all dependencies have been injected into the graph, but
     before injecting the synthetic version of this target.
 
+    Returns a boolean indicating whether it modified the underlying filesystem.
+
     NB(gm): Some code generators may re-generate code that their dependent libraries generate.
     This results in targets claiming to generate sources that they really don't, so we try to
     filter out sources that were actually generated by dependencies of the target. This causes
@@ -394,16 +427,14 @@ def _handle_duplicate_sources(self, target, target_workdir):
     default, this behavior is disabled, and duplication in generated sources will raise a
     TaskError. This is controlled by the --allow-dups flag.
     """
-    # Compute the raw sources owned by this target.
-    by_target = self.find_sources(target, target_workdir)
 
     # Walk dependency gentargets and record any sources owned by those targets that are also
     # owned by this target.
     duplicates_by_target = OrderedDict()
     def record_duplicates(dep):
       if dep == target or not self.is_gentarget(dep.concrete_derived_from):
-        return
-      duped_sources = [s for s in dep.sources_relative_to_source_root() if s in by_target and
+        return False
+      duped_sources = [s for s in dep.sources_relative_to_source_root() if s in sources.files and
                        not self.ignore_dup(target, dep, s)]
       if duped_sources:
         duplicates_by_target[dep] = duped_sources
@@ -411,7 +442,7 @@ def record_duplicates(dep):
 
     # If there were no dupes, we're done.
     if not duplicates_by_target:
-      return
+      return False
 
     # If there were duplicates warn or error.
     messages = ['{target} generated sources that had already been generated by dependencies.'
@@ -425,11 +456,15 @@ def record_duplicates(dep):
     else:
       raise self.DuplicateSourceError(message)
 
+    did_modify = False
+
     # Finally, remove duplicates from the workdir. This prevents us from having to worry
     # about them during future incremental compiles.
     for dep, duped_sources in duplicates_by_target.items():
       for duped_source in duped_sources:
         safe_delete(os.path.join(target_workdir, duped_source))
+        did_modify = True
+    return did_modify
 
   class DuplicateSourceError(TaskError):
     """A target generated the same code that was generated by one of its dependencies.
diff --git a/tests/python/pants_test/backend/codegen/antlr/java/test_antlr_java_gen.py b/tests/python/pants_test/backend/codegen/antlr/java/test_antlr_java_gen.py
index f18350a96f4..f53242c11d2 100644
--- a/tests/python/pants_test/backend/codegen/antlr/java/test_antlr_java_gen.py
+++ b/tests/python/pants_test/backend/codegen/antlr/java/test_antlr_java_gen.py
@@ -16,7 +16,6 @@
 from pants.backend.codegen.antlr.java.java_antlr_library import JavaAntlrLibrary
 from pants.base.exceptions import TaskError
 from pants.build_graph.build_file_aliases import BuildFileAliases
-from pants.invalidation.build_invalidator import CacheKey
 from pants.util.dirutil import safe_mkdtemp
 from pants_test.jvm.nailgun_task_test_base import NailgunTaskTestBase
 
@@ -78,8 +77,8 @@ def execute_antlr_test(self, expected_package, target_workdir_fun=None):
 
     # Generate code, then create a synthetic target.
     task.execute_codegen(target, target_workdir)
-    fingerprint = CacheKey("test", target.invalidation_hash())
-    syn_target = task._inject_synthetic_target(target, target_workdir, fingerprint)
+    sources = task._capture_sources(target, target_workdir)
+    syn_target = task._inject_synthetic_target(target, target_workdir, sources)
 
     actual_sources = [s for s in Fileset.rglobs('*.java', root=target_workdir)]
     expected_sources = syn_target.sources_relative_to_source_root()
diff --git a/tests/python/pants_test/source/BUILD b/tests/python/pants_test/source/BUILD
index f37a6e712a6..6dc75ec1780 100644
--- a/tests/python/pants_test/source/BUILD
+++ b/tests/python/pants_test/source/BUILD
@@ -14,6 +14,7 @@ python_tests(
   sources = ['test_payload_fields.py'],
   dependencies = [
     'src/python/pants/source',
+    'src/python/pants/source:payload_fields',
     'tests/python/pants_test:test_base',
     'tests/python/pants_test/subsystem:subsystem_utils',
   ]
diff --git a/tests/python/pants_test/source/test_payload_fields.py b/tests/python/pants_test/source/test_payload_fields.py
index a4870c395b3..0e66a998c40 100644
--- a/tests/python/pants_test/source/test_payload_fields.py
+++ b/tests/python/pants_test/source/test_payload_fields.py
@@ -5,8 +5,10 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
+from pants.base.project_tree import File
+from pants.engine.fs import DirectoryDigest, Path, Snapshot
 from pants.source.payload_fields import SourcesField
-from pants.source.wrapped_globs import EagerFilesetWithSpec, Globs, LazyFilesetWithSpec
+from pants.source.wrapped_globs import Globs, LazyFilesetWithSpec
 from pants_test.test_base import TestBase
 
 
@@ -69,16 +71,22 @@ def test_passes_lazy_fileset_with_spec_through(self):
     self.assertEqual(['foo/a.txt'], list(sf.source_paths))
 
   def test_passes_eager_fileset_with_spec_through(self):
-    self.create_file('foo/a.txt', 'a_contents')
+    self.create_file('foo/foo/a.txt', 'a_contents')
+
+    fileset = self.sources_for(['foo/a.txt'], 'foo')
 
-    fileset = EagerFilesetWithSpec(rel_root='foo',
-                                   # Glob spec is relative to build root
-                                   filespec={'globs': ['foo/foo/a.txt']},
-                                   # files are relative to `rel_root`
-                                   files=['foo/a.txt'],
-                                   files_hash={'foo/a.txt': b'12345'})
     sf = SourcesField(sources=fileset)
 
     self.assertIs(fileset, sf.sources)
     self.assertEqual(['foo/a.txt'], list(sf.source_paths))
     self.assertEqual(['foo/foo/a.txt'], list(sf.relative_to_buildroot()))
+
+    digest = str('56001a7e48555f156420099a99da60a7a83acc90853046709341bf9f00a6f944')
+    want_snapshot = Snapshot(
+      DirectoryDigest(digest, 77),
+      (Path('foo/foo/a.txt', stat=File('foo/foo/a.txt')),)
+    )
+
+    # We explicitly pass a None scheduler because we expect no scheduler lookups to be required
+    # in order to get a Snapshot.
+    self.assertEqual(sf.snapshot(scheduler=None), want_snapshot)
diff --git a/tests/python/pants_test/source/test_wrapped_globs.py b/tests/python/pants_test/source/test_wrapped_globs.py
index e604421bd66..a6edde54564 100644
--- a/tests/python/pants_test/source/test_wrapped_globs.py
+++ b/tests/python/pants_test/source/test_wrapped_globs.py
@@ -12,7 +12,9 @@
 from pants.base.payload import Payload
 from pants.build_graph.address_lookup_error import AddressLookupError
 from pants.build_graph.build_file_aliases import BuildFileAliases
+from pants.build_graph.files import Files
 from pants.build_graph.target import Target
+from pants.engine.fs import EMPTY_SNAPSHOT
 from pants.source.wrapped_globs import EagerFilesetWithSpec, Globs, LazyFilesetWithSpec, RGlobs
 from pants_test.test_base import TestBase
 
@@ -224,13 +226,17 @@ def test_rglob_follows_symlinked_dirs_by_default(self):
 
 class FilesetWithSpecTest(TestBase):
 
+  @classmethod
+  def alias_groups(cls):
+    return BuildFileAliases(targets={'files': Files})
+
   def test_lazy_fileset_with_spec_fails_if_filespec_not_prefixed_by_relroot(self):
     with self.assertRaises(ValueError):
       LazyFilesetWithSpec('foo', {'globs':['notfoo/a.txt']}, lambda: ['foo/a.txt'])
 
   def test_eager_fileset_with_spec_fails_if_filespec_not_prefixed_by_relroot(self):
     with self.assertRaises(ValueError):
-      EagerFilesetWithSpec('foo', {'globs':['notfoo/a.txt']}, files=['files'], files_hash='deadbeef')
+      EagerFilesetWithSpec('foo', {'globs':['notfoo/a.txt']}, EMPTY_SNAPSHOT)
 
   def test_lazy_fileset_with_spec_fails_if_exclude_filespec_not_prefixed_by_relroot(self):
     with self.assertRaises(ValueError):
@@ -242,10 +248,29 @@ def test_eager_fileset_with_spec_fails_if_exclude_filespec_not_prefixed_by_relro
     with self.assertRaises(ValueError):
       EagerFilesetWithSpec('foo',
                            {'globs': [], 'exclude': [{'globs': ['notfoo/a.txt']}]},
-                           files=['files'],
-                           files_hash='deadbeef')
+                           EMPTY_SNAPSHOT)
 
   def test_iter_relative_paths(self):
-    efws = EagerFilesetWithSpec('test_root', {'globs': []}, files=['a', 'b', 'c'], files_hash='deadbeef')
-    result = list(efws.paths_from_buildroot_iter())
-    self.assertEquals(result, ['test_root/a', 'test_root/b', 'test_root/c'])
+    self.create_files('test_root', ['a', 'b', 'c'])
+    efws = self.sources_for(['a', 'b', 'c'], 'test_root')
+    self.assertEquals(
+      efws.files_hash,
+      str('cb11a7f0b5a1e22b93c36783608ba531ea831c2f68a5c9f9498417b211bcfea4'),
+    )
+    self.assertEquals(
+      list(efws.paths_from_buildroot_iter()),
+      ['test_root/a', 'test_root/b', 'test_root/c'],
+    )
+
+  def test_source_snapshot(self):
+    self.create_file('package/dir/foo')
+    self.add_to_build_file('package/dir', 'files(name = "target", sources = ["foo"])')
+    target = self.target('package/dir:target')
+    snapshot = target.sources_snapshot(scheduler=self.scheduler)
+    snapshot_paths = tuple(file.path for file in snapshot.path_stats)
+    self.assertEqual(
+      ('package/dir/foo',),
+      snapshot_paths
+    )
+    self.assertEqual(target.sources_relative_to_target_base().files, ('foo',))
+    self.assertEqual(target.sources_relative_to_buildroot(), ['package/dir/foo'])
diff --git a/tests/python/pants_test/task/test_simple_codegen_task.py b/tests/python/pants_test/task/test_simple_codegen_task.py
index e53c20af589..26b6e93b439 100644
--- a/tests/python/pants_test/task/test_simple_codegen_task.py
+++ b/tests/python/pants_test/task/test_simple_codegen_task.py
@@ -12,7 +12,6 @@
 from pants.build_graph.build_file_aliases import BuildFileAliases
 from pants.build_graph.register import build_file_aliases as register_core
 from pants.build_graph.target import Target
-from pants.invalidation.build_invalidator import CacheKey
 from pants.task.simple_codegen_task import SimpleCodegenTask
 from pants.util.dirutil import safe_mkdtemp
 from pants_test.task_test_base import TaskTestBase, ensure_cached
@@ -210,9 +209,11 @@ def execute():
       for target in targets:
         target_workdir = target_workdirs[target]
         task.execute_codegen(target, target_workdir)
-        task._handle_duplicate_sources(target, target_workdir)
-        fingerprint = CacheKey("test", target.invalidation_hash())
-        syn_targets.append(task._inject_synthetic_target(target, target_workdir, fingerprint))
+        sources = task._capture_sources(target, target_workdir)
+        task._handle_duplicate_sources(target, target_workdir, sources)
+        # _handle_duplicate_sources may delete files from the filesystem, so we need to re-capture.
+        sources = task._capture_sources(target, target_workdir)
+        syn_targets.append(task._inject_synthetic_target(target, target_workdir, sources))
 
     if should_fail:
       # If we're expected to fail, validate the resulting message.
@@ -241,7 +242,7 @@ def test_duplicated_code_generation_pass(self):
     parent, good, bad = self._do_test_duplication(targets, allow_dups=True, should_fail=False)
     # Confirm that the duped sources were removed.
     for source in bad.sources_relative_to_source_root():
-      self.assertNotIn(source, parent.sources_relative_to_source_root())
+      self.assertNotIn(source, tuple(parent.sources_relative_to_source_root()))
 
   def test_duplicated_code_generation_nodupes(self):
     # Without the duplicated target, either mode is fine.

From ad85f4901a202ffc2c97102add974fed0798bead Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <dwagnerhall@twitter.com>
Date: Thu, 21 Jun 2018 16:46:10 +0100
Subject: [PATCH 02/10] Cloc uses pre-captured Snapshots

---
 .../pants/backend/graph_info/tasks/cloc.py    | 32 ++++++++-----------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/src/python/pants/backend/graph_info/tasks/cloc.py b/src/python/pants/backend/graph_info/tasks/cloc.py
index 0d98a3ecdce..ce2620837dd 100644
--- a/src/python/pants/backend/graph_info/tasks/cloc.py
+++ b/src/python/pants/backend/graph_info/tasks/cloc.py
@@ -9,7 +9,7 @@
 
 from pants.backend.graph_info.subsystems.cloc_binary import ClocBinary
 from pants.base.workunit import WorkUnitLabel
-from pants.engine.fs import FilesContent, PathGlobs, PathGlobsAndRoot, Snapshot
+from pants.engine.fs import FilesContent, PathGlobs, PathGlobsAndRoot
 from pants.engine.isolated_process import ExecuteProcessRequest
 from pants.task.console_task import ConsoleTask
 from pants.util.contextutil import temporary_dir
@@ -35,16 +35,18 @@ def console_output(self, targets):
     if not self.get_options().transitive:
       targets = self.context.target_roots
 
+    input_snapshots = tuple(
+      target.sources_snapshot(scheduler=self.context._scheduler) for target in targets
+    )
+    input_files = set(f.path for snapshot in input_snapshots for f in snapshot.files)
+
     # TODO: Work out a nice library-like utility for writing an argfile, as this will be common.
     with temporary_dir() as tmpdir:
       list_file = os.path.join(tmpdir, 'input_files_list')
-      input_files = set()
       with open(list_file, 'w') as list_file_out:
-        for target in targets:
-          for source in target.sources_relative_to_buildroot():
-            input_files.add(source)
-            list_file_out.write(source)
-            list_file_out.write(b'\n')
+        for input_file in sorted(input_files):
+          list_file_out.write(input_file)
+          list_file_out.write(b'\n')
       list_file_snapshot = self.context._scheduler.capture_snapshots((
         PathGlobsAndRoot(
           PathGlobs(('input_files_list',)),
@@ -54,17 +56,11 @@ def console_output(self, targets):
 
     cloc_path, cloc_snapshot = ClocBinary.global_instance().hackily_snapshot(self.context)
 
-    # TODO: This should use an input file snapshot which should be provided on the Target object,
-    # rather than hackily re-snapshotting each of the input files.
-    # See https://github.com/pantsbuild/pants/issues/5762
-    input_pathglobs = PathGlobs(tuple(input_files))
-    input_snapshot = self.context._scheduler.product_request(Snapshot, [input_pathglobs])[0]
-
-    directory_digest = self.context._scheduler.merge_directories((
-      cloc_snapshot.directory_digest,
-      input_snapshot.directory_digest,
-      list_file_snapshot.directory_digest,
-    ))
+    directory_digest = self.context._scheduler.merge_directories(tuple(s.directory_digest for s in
+      input_snapshots + (
+      cloc_snapshot,
+      list_file_snapshot,
+    )))
 
     cmd = (
       '/usr/bin/perl',

From f6bf98aaa7f8c1cbdce1fedb8c40e71d7a2b706b Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <dwagnerhall@twitter.com>
Date: Tue, 26 Jun 2018 17:40:58 +0100
Subject: [PATCH 03/10] Use synthetic target dir

---
 src/python/pants/task/simple_codegen_task.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/python/pants/task/simple_codegen_task.py b/src/python/pants/task/simple_codegen_task.py
index 0ac7143776f..ec6a585dcb0 100644
--- a/src/python/pants/task/simple_codegen_task.py
+++ b/src/python/pants/task/simple_codegen_task.py
@@ -225,18 +225,20 @@ def execute(self):
           sources = None
           must_recapture = False
 
+          synthetic_target_dir = self.synthetic_target_dir(vt.target, vt.results_dir)
+
           # Build the target and handle duplicate sources.
           if not vt.valid:
             if self._do_validate_sources_present(vt.target):
               self.execute_codegen(vt.target, vt.results_dir)
-              sources = self._capture_sources(vt.target, vt.results_dir)
+              sources = self._capture_sources(vt.target, synthetic_target_dir)
               must_recapture = self._handle_duplicate_sources(vt.target, vt.results_dir, sources)
             vt.update()
 
           # _handle_duplicate_sources may delete files from the filesystem, so we need to
           # re-capture the sources.
           if sources is None or must_recapture:
-            sources = self._capture_sources(vt.target, vt.results_dir)
+            sources = self._capture_sources(vt.target, synthetic_target_dir)
 
           self._inject_synthetic_target(
             vt.target,
@@ -267,13 +269,13 @@ def synthetic_target_dir(self, target, target_workdir):
     """
     return target_workdir
 
-  def _capture_sources(self, target, target_workdir):
+  def _capture_sources(self, target, synthetic_target_dir):
     if self.sources_globs is None:
-      files = list(self.find_sources(target, target_workdir))
+      files = list(self.find_sources(target, synthetic_target_dir))
     else:
       files = self.sources_globs
 
-    results_dir_relpath = os.path.relpath(target_workdir, get_buildroot())
+    results_dir_relpath = os.path.relpath(synthetic_target_dir, get_buildroot())
     buildroot_relative_globs = tuple(os.path.join(results_dir_relpath, file) for file in files)
     buildroot_relative_excludes = tuple(
       os.path.join(results_dir_relpath, file)

From 5d9ac89d7f50e69700e0c3921721cbfba32dc48e Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <dwagnerhall@twitter.com>
Date: Tue, 26 Jun 2018 18:32:57 +0100
Subject: [PATCH 04/10] Review comments

---
 src/python/pants/build_graph/target.py       |  3 +-
 src/python/pants/task/simple_codegen_task.py | 71 +++++++++++---------
 2 files changed, 41 insertions(+), 33 deletions(-)

diff --git a/src/python/pants/build_graph/target.py b/src/python/pants/build_graph/target.py
index 153045b3028..93d39300ef7 100644
--- a/src/python/pants/build_graph/target.py
+++ b/src/python/pants/build_graph/target.py
@@ -898,8 +898,7 @@ def create_sources_field(self, sources, sources_rel_path, key_arg=None):
         ('Passing collections as the value of the sources argument to create_sources_field is '
          'deprecated, and now takes a slow path. Instead, class {} should have its sources '
          'argument populated by the engine, either by using the standard parsing pipeline, or by '
-         'requesting a SourcesField product.'
-         'the v2 engine.').format(self.__class__.__name__)
+         'requesting a SourcesField product from the v2 engine.').format(self.__class__.__name__)
       )
       sources = Files.create_fileset_with_spec(sources_rel_path, *sources)
     elif not isinstance(sources, FilesetWithSpec):
diff --git a/src/python/pants/task/simple_codegen_task.py b/src/python/pants/task/simple_codegen_task.py
index ec6a585dcb0..ca3362aed52 100644
--- a/src/python/pants/task/simple_codegen_task.py
+++ b/src/python/pants/task/simple_codegen_task.py
@@ -220,30 +220,33 @@ def execute(self):
                           fingerprint_strategy=self.get_fingerprint_strategy()) as invalidation_check:
 
       with self.context.new_workunit(name='execute', labels=[WorkUnitLabel.MULTITOOL]):
+        vts_to_sources = OrderedDict()
         for vt in invalidation_check.all_vts:
-
-          sources = None
-          must_recapture = False
-
           synthetic_target_dir = self.synthetic_target_dir(vt.target, vt.results_dir)
 
+          key = (vt, synthetic_target_dir)
+          vts_to_sources[key] = None
+
           # Build the target and handle duplicate sources.
           if not vt.valid:
             if self._do_validate_sources_present(vt.target):
               self.execute_codegen(vt.target, vt.results_dir)
-              sources = self._capture_sources(vt.target, synthetic_target_dir)
-              must_recapture = self._handle_duplicate_sources(vt.target, vt.results_dir, sources)
+              sources = self._capture_sources(((vt.target, synthetic_target_dir),))[0]
+              # _handle_duplicate_sources may delete files from the filesystem, so we need to
+              # re-capture the sources.
+              if not self._handle_duplicate_sources(vt.target, vt.results_dir, sources):
+                vts_to_sources[key] = sources
             vt.update()
 
-          # _handle_duplicate_sources may delete files from the filesystem, so we need to
-          # re-capture the sources.
-          if sources is None or must_recapture:
-            sources = self._capture_sources(vt.target, synthetic_target_dir)
-
+        vts_to_capture = tuple(key for key, sources in vts_to_sources.items() if sources is None)
+        filesets = self._capture_sources(vts_to_capture)
+        for key, fileset in zip(vts_to_capture, filesets):
+          vts_to_sources[key] = fileset
+        for (vt, synthetic_target_dir), fileset in vts_to_sources.items():
           self._inject_synthetic_target(
             vt.target,
             vt.results_dir,
-            sources,
+            fileset,
           )
         self._mark_transitive_invalidation_hashes_dirty(
           vt.target.address for vt in invalidation_check.all_vts
@@ -269,31 +272,37 @@ def synthetic_target_dir(self, target, target_workdir):
     """
     return target_workdir
 
-  def _capture_sources(self, target, synthetic_target_dir):
-    if self.sources_globs is None:
-      files = list(self.find_sources(target, synthetic_target_dir))
-    else:
-      files = self.sources_globs
+  # Accepts tuple of tuples of (target, synthetic_target_dir)
+  # Returns tuple of EagerFilesetWithSpecs in matching order.
+  def _capture_sources(self, targets_and_dirs):
+    to_capture = []
 
-    results_dir_relpath = os.path.relpath(synthetic_target_dir, get_buildroot())
-    buildroot_relative_globs = tuple(os.path.join(results_dir_relpath, file) for file in files)
-    buildroot_relative_excludes = tuple(
-      os.path.join(results_dir_relpath, file)
-        for file in self.sources_exclude_globs
-    )
+    for (target, synthetic_target_dir) in targets_and_dirs:
+      if self.sources_globs is None:
+        files = list(self.find_sources(target, synthetic_target_dir))
+      else:
+        files = self.sources_globs
+
+      results_dir_relpath = os.path.relpath(synthetic_target_dir, get_buildroot())
+      buildroot_relative_globs = tuple(os.path.join(results_dir_relpath, file) for file in files)
+      buildroot_relative_excludes = tuple(
+        os.path.join(results_dir_relpath, file)
+          for file in self.sources_exclude_globs
+      )
+      to_capture.append(
+        PathGlobsAndRoot(
+          PathGlobs(buildroot_relative_globs, buildroot_relative_excludes),
+          str(get_buildroot()),
+        )
+      )
 
-    snapshot = self.context._scheduler.capture_snapshots((
-      PathGlobsAndRoot(
-        PathGlobs(buildroot_relative_globs, buildroot_relative_excludes),
-        str(get_buildroot()),
-      ),
-    ))[0]
+    snapshots = self.context._scheduler.capture_snapshots(tuple(to_capture))
 
-    return EagerFilesetWithSpec(
+    return tuple(EagerFilesetWithSpec(
       results_dir_relpath,
       FilesetRelPathWrapper.to_filespec(buildroot_relative_globs),
       snapshot,
-    )
+    ) for snapshot in snapshots)
 
   def _inject_synthetic_target(
     self,

From bc44379f5c9f5abbfb9c761d2d513bdec28820d6 Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <dwagnerhall@twitter.com>
Date: Tue, 26 Jun 2018 21:34:58 +0100
Subject: [PATCH 05/10] Fix up tests

---
 .../backend/codegen/antlr/java/test_antlr_java_gen.py         | 2 +-
 tests/python/pants_test/task/test_simple_codegen_task.py      | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/python/pants_test/backend/codegen/antlr/java/test_antlr_java_gen.py b/tests/python/pants_test/backend/codegen/antlr/java/test_antlr_java_gen.py
index f53242c11d2..fbb4f32cf1e 100644
--- a/tests/python/pants_test/backend/codegen/antlr/java/test_antlr_java_gen.py
+++ b/tests/python/pants_test/backend/codegen/antlr/java/test_antlr_java_gen.py
@@ -77,7 +77,7 @@ def execute_antlr_test(self, expected_package, target_workdir_fun=None):
 
     # Generate code, then create a synthetic target.
     task.execute_codegen(target, target_workdir)
-    sources = task._capture_sources(target, target_workdir)
+    sources = task._capture_sources(((target, target_workdir),))[0]
     syn_target = task._inject_synthetic_target(target, target_workdir, sources)
 
     actual_sources = [s for s in Fileset.rglobs('*.java', root=target_workdir)]
diff --git a/tests/python/pants_test/task/test_simple_codegen_task.py b/tests/python/pants_test/task/test_simple_codegen_task.py
index 26b6e93b439..784b43d1af2 100644
--- a/tests/python/pants_test/task/test_simple_codegen_task.py
+++ b/tests/python/pants_test/task/test_simple_codegen_task.py
@@ -209,10 +209,10 @@ def execute():
       for target in targets:
         target_workdir = target_workdirs[target]
         task.execute_codegen(target, target_workdir)
-        sources = task._capture_sources(target, target_workdir)
+        sources = task._capture_sources(((target, target_workdir),))[0]
         task._handle_duplicate_sources(target, target_workdir, sources)
         # _handle_duplicate_sources may delete files from the filesystem, so we need to re-capture.
-        sources = task._capture_sources(target, target_workdir)
+        sources = task._capture_sources(((target, target_workdir),))[0]
         syn_targets.append(task._inject_synthetic_target(target, target_workdir, sources))
 
     if should_fail:

From ed26f5d3416c3edc5bfa97b806d4c54bb364040f Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <dwagnerhall@twitter.com>
Date: Wed, 27 Jun 2018 11:23:58 +0100
Subject: [PATCH 06/10] Use loop variable, rather than most recent value

---
 src/python/pants/task/simple_codegen_task.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/python/pants/task/simple_codegen_task.py b/src/python/pants/task/simple_codegen_task.py
index ca3362aed52..81c42a3f09a 100644
--- a/src/python/pants/task/simple_codegen_task.py
+++ b/src/python/pants/task/simple_codegen_task.py
@@ -276,8 +276,10 @@ def synthetic_target_dir(self, target, target_workdir):
   # Returns tuple of EagerFilesetWithSpecs in matching order.
   def _capture_sources(self, targets_and_dirs):
     to_capture = []
+    results_dirs = []
+    filespecs = []
 
-    for (target, synthetic_target_dir) in targets_and_dirs:
+    for target, synthetic_target_dir in targets_and_dirs:
       if self.sources_globs is None:
         files = list(self.find_sources(target, synthetic_target_dir))
       else:
@@ -295,14 +297,16 @@ def _capture_sources(self, targets_and_dirs):
           str(get_buildroot()),
         )
       )
+      results_dirs.append(results_dir_relpath)
+      filespecs.append(FilesetRelPathWrapper.to_filespec(buildroot_relative_globs))
 
     snapshots = self.context._scheduler.capture_snapshots(tuple(to_capture))
 
     return tuple(EagerFilesetWithSpec(
       results_dir_relpath,
-      FilesetRelPathWrapper.to_filespec(buildroot_relative_globs),
+      filespec,
       snapshot,
-    ) for snapshot in snapshots)
+    ) for (results_dir_relpath, filespec, snapshot) in zip(results_dirs, filespecs, snapshots))
 
   def _inject_synthetic_target(
     self,

From 23dfc97256a19671714309eea0b1cb499fc6c984 Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <dwagnerhall@twitter.com>
Date: Wed, 27 Jun 2018 11:26:06 +0100
Subject: [PATCH 07/10] Simplify

---
 src/python/pants/task/simple_codegen_task.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/python/pants/task/simple_codegen_task.py b/src/python/pants/task/simple_codegen_task.py
index 81c42a3f09a..29e970492da 100644
--- a/src/python/pants/task/simple_codegen_task.py
+++ b/src/python/pants/task/simple_codegen_task.py
@@ -231,7 +231,7 @@ def execute(self):
           if not vt.valid:
             if self._do_validate_sources_present(vt.target):
               self.execute_codegen(vt.target, vt.results_dir)
-              sources = self._capture_sources(((vt.target, synthetic_target_dir),))[0]
+              sources = self._capture_sources((key,))[0]
               # _handle_duplicate_sources may delete files from the filesystem, so we need to
               # re-capture the sources.
               if not self._handle_duplicate_sources(vt.target, vt.results_dir, sources):
@@ -245,7 +245,7 @@ def execute(self):
         for (vt, synthetic_target_dir), fileset in vts_to_sources.items():
           self._inject_synthetic_target(
             vt.target,
-            vt.results_dir,
+            synthetic_target_dir,
             fileset,
           )
         self._mark_transitive_invalidation_hashes_dirty(
@@ -321,7 +321,6 @@ def _inject_synthetic_target(
     """
 
     synthetic_target_type = self.synthetic_target_type(target)
-    target_workdir = self.synthetic_target_dir(target, target_workdir)
     synthetic_extra_dependencies = self.synthetic_target_extra_dependencies(target, target_workdir)
 
     copied_attributes = {}

From 541222b97706b43ee618e554902915fee723631a Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <dwagnerhall@twitter.com>
Date: Wed, 27 Jun 2018 12:01:05 +0100
Subject: [PATCH 08/10] Improve deprecation message

---
 src/python/pants/task/simple_codegen_task.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/python/pants/task/simple_codegen_task.py b/src/python/pants/task/simple_codegen_task.py
index 29e970492da..d46eb99863d 100644
--- a/src/python/pants/task/simple_codegen_task.py
+++ b/src/python/pants/task/simple_codegen_task.py
@@ -415,7 +415,7 @@ def find_sources(self, target, target_workdir):
       '1.10.0.dev0',
       'SimpleCodegenTask.find_sources is deprecated. Subclasses should instead specify '
       'sources_globs and sources_exclude_globs. '
-      'Class to update: {}'.format(self.__class__.__name__)
+      'Class to update: {}. find_sources'.format(self.__class__.__name__)
     )
     return OrderedSet(self._find_sources_in_workdir(target_workdir))
 

From 82ce7b872fa798a3ebebc82ae19ab63238d466af Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <dwagnerhall@twitter.com>
Date: Wed, 27 Jun 2018 12:08:24 +0100
Subject: [PATCH 09/10] Add sources_globs to all SimpleCodegenTasks

---
 contrib/avro/src/python/pants/contrib/avro/tasks/avro_gen.py   | 2 ++
 .../go/src/python/pants/contrib/go/tasks/go_protobuf_gen.py    | 2 ++
 contrib/go/src/python/pants/contrib/go/tasks/go_thrift_gen.py  | 2 ++
 .../jax_ws/src/python/pants/contrib/jax_ws/tasks/jax_ws_gen.py | 2 ++
 .../src/python/pants/contrib/scrooge/tasks/scrooge_gen.py      | 2 ++
 .../src/python/pants/contrib/thrifty/java_thrifty_gen.py       | 2 ++
 src/python/pants/backend/codegen/jaxb/jaxb_gen.py              | 2 ++
 src/python/pants/backend/codegen/protobuf/java/protobuf_gen.py | 2 ++
 src/python/pants/backend/codegen/ragel/java/ragel_gen.py       | 3 +++
 .../backend/codegen/thrift/java/apache_thrift_java_gen.py      | 2 ++
 .../backend/codegen/thrift/python/apache_thrift_py_gen.py      | 2 ++
 src/python/pants/backend/codegen/wire/java/wire_gen.py         | 2 ++
 12 files changed, 25 insertions(+)

diff --git a/contrib/avro/src/python/pants/contrib/avro/tasks/avro_gen.py b/contrib/avro/src/python/pants/contrib/avro/tasks/avro_gen.py
index 8e46f17e6ae..551089f5fa9 100644
--- a/contrib/avro/src/python/pants/contrib/avro/tasks/avro_gen.py
+++ b/contrib/avro/src/python/pants/contrib/avro/tasks/avro_gen.py
@@ -23,6 +23,8 @@
 class AvroJavaGenTask(SimpleCodegenTask, NailgunTask):
   """Compile Avro schema, protocol, and IDL files to Java code."""
 
+  sources_globs = ('**/*',)
+
   @classmethod
   def register_options(cls, register):
     super(AvroJavaGenTask, cls).register_options(register)
diff --git a/contrib/go/src/python/pants/contrib/go/tasks/go_protobuf_gen.py b/contrib/go/src/python/pants/contrib/go/tasks/go_protobuf_gen.py
index 5aec29b9f94..cdec0d063f2 100644
--- a/contrib/go/src/python/pants/contrib/go/tasks/go_protobuf_gen.py
+++ b/contrib/go/src/python/pants/contrib/go/tasks/go_protobuf_gen.py
@@ -25,6 +25,8 @@
 
 class GoProtobufGen(SimpleCodegenTask):
 
+  sources_globs = ('**/*',)
+
   _NAMESPACE_PARSER = re.compile(r'^\s*option\s+go_package\s*=\s*"([^\s]+)"\s*;', re.MULTILINE)
   _PACKAGE_PARSER = re.compile(r'^\s*package\s+([^\s]+)\s*;', re.MULTILINE)
 
diff --git a/contrib/go/src/python/pants/contrib/go/tasks/go_thrift_gen.py b/contrib/go/src/python/pants/contrib/go/tasks/go_thrift_gen.py
index 1d7c498b9ed..a65486efc00 100644
--- a/contrib/go/src/python/pants/contrib/go/tasks/go_thrift_gen.py
+++ b/contrib/go/src/python/pants/contrib/go/tasks/go_thrift_gen.py
@@ -25,6 +25,8 @@
 
 class GoThriftGen(SimpleCodegenTask):
 
+  sources_globs = ('**/*',)
+
   @classmethod
   def register_options(cls, register):
     super(GoThriftGen, cls).register_options(register)
diff --git a/contrib/jax_ws/src/python/pants/contrib/jax_ws/tasks/jax_ws_gen.py b/contrib/jax_ws/src/python/pants/contrib/jax_ws/tasks/jax_ws_gen.py
index bc945eeb81e..f21dc736843 100644
--- a/contrib/jax_ws/src/python/pants/contrib/jax_ws/tasks/jax_ws_gen.py
+++ b/contrib/jax_ws/src/python/pants/contrib/jax_ws/tasks/jax_ws_gen.py
@@ -25,6 +25,8 @@
 class JaxWsGen(SimpleCodegenTask, NailgunTask):
   """Generates Java files from wsdl files using the JAX-WS compiler."""
 
+  sources_globs = ('**/*',)
+
   @classmethod
   def register_options(cls, register):
     super(JaxWsGen, cls).register_options(register)
diff --git a/contrib/scrooge/src/python/pants/contrib/scrooge/tasks/scrooge_gen.py b/contrib/scrooge/src/python/pants/contrib/scrooge/tasks/scrooge_gen.py
index 16acab4948c..4d104fa9f9d 100644
--- a/contrib/scrooge/src/python/pants/contrib/scrooge/tasks/scrooge_gen.py
+++ b/contrib/scrooge/src/python/pants/contrib/scrooge/tasks/scrooge_gen.py
@@ -38,6 +38,8 @@ class ScroogeGen(SimpleCodegenTask, NailgunTask):
     'compiler_args'
   ])
 
+  sources_globs = ('**/*',)
+
   @classmethod
   def register_options(cls, register):
     super(ScroogeGen, cls).register_options(register)
diff --git a/contrib/thrifty/src/python/pants/contrib/thrifty/java_thrifty_gen.py b/contrib/thrifty/src/python/pants/contrib/thrifty/java_thrifty_gen.py
index bd082976a1c..023525d7311 100644
--- a/contrib/thrifty/src/python/pants/contrib/thrifty/java_thrifty_gen.py
+++ b/contrib/thrifty/src/python/pants/contrib/thrifty/java_thrifty_gen.py
@@ -23,6 +23,8 @@ class JavaThriftyGen(NailgunTaskBase, SimpleCodegenTask):
 
   gentarget_type = JavaThriftyLibrary
 
+  sources_globs = ('**/*',)
+
   @classmethod
   def register_options(cls, register):
     super(JavaThriftyGen, cls).register_options(register)
diff --git a/src/python/pants/backend/codegen/jaxb/jaxb_gen.py b/src/python/pants/backend/codegen/jaxb/jaxb_gen.py
index 9f5ddec0add..1ae5ece2346 100644
--- a/src/python/pants/backend/codegen/jaxb/jaxb_gen.py
+++ b/src/python/pants/backend/codegen/jaxb/jaxb_gen.py
@@ -23,6 +23,8 @@ class JaxbGen(SimpleCodegenTask, NailgunTask):
   _XJC_MAIN = 'com.sun.tools.xjc.Driver'
   _XML_BIND_VERSION = '2.3.0'
 
+  sources_globs = ('**/*',)
+
   @classmethod
   def register_options(cls, register):
     super(JaxbGen, cls).register_options(register)
diff --git a/src/python/pants/backend/codegen/protobuf/java/protobuf_gen.py b/src/python/pants/backend/codegen/protobuf/java/protobuf_gen.py
index ececd471d19..3e4e09aa40e 100644
--- a/src/python/pants/backend/codegen/protobuf/java/protobuf_gen.py
+++ b/src/python/pants/backend/codegen/protobuf/java/protobuf_gen.py
@@ -27,6 +27,8 @@
 
 class ProtobufGen(SimpleCodegenTask):
 
+  sources_globs = ('**/*',)
+
   @classmethod
   def subsystem_dependencies(cls):
     return super(ProtobufGen, cls).subsystem_dependencies() + (Protoc.scoped(cls),)
diff --git a/src/python/pants/backend/codegen/ragel/java/ragel_gen.py b/src/python/pants/backend/codegen/ragel/java/ragel_gen.py
index 6f406c74573..928a63184bc 100644
--- a/src/python/pants/backend/codegen/ragel/java/ragel_gen.py
+++ b/src/python/pants/backend/codegen/ragel/java/ragel_gen.py
@@ -20,6 +20,9 @@
 
 
 class RagelGen(SimpleCodegenTask):
+
+  sources_globs = ('**/*',)
+
   @classmethod
   def subsystem_dependencies(cls):
     return super(RagelGen, cls).subsystem_dependencies() + (Ragel.scoped(cls),)
diff --git a/src/python/pants/backend/codegen/thrift/java/apache_thrift_java_gen.py b/src/python/pants/backend/codegen/thrift/java/apache_thrift_java_gen.py
index bdceb572480..9e1dce3edd2 100644
--- a/src/python/pants/backend/codegen/thrift/java/apache_thrift_java_gen.py
+++ b/src/python/pants/backend/codegen/thrift/java/apache_thrift_java_gen.py
@@ -25,6 +25,8 @@ class ApacheThriftJavaGen(ApacheThriftGenBase):
 
   _COMPILER = 'thrift'
 
+  sources_globs = ('**/*',)
+
   @classmethod
   def subsystem_dependencies(cls):
     return super(ApacheThriftJavaGen, cls).subsystem_dependencies() + (ThriftDefaults,)
diff --git a/src/python/pants/backend/codegen/thrift/python/apache_thrift_py_gen.py b/src/python/pants/backend/codegen/thrift/python/apache_thrift_py_gen.py
index 2cffc43ea66..54670637666 100644
--- a/src/python/pants/backend/codegen/thrift/python/apache_thrift_py_gen.py
+++ b/src/python/pants/backend/codegen/thrift/python/apache_thrift_py_gen.py
@@ -21,6 +21,8 @@ class ApacheThriftPyGen(ApacheThriftGenBase):
     'new_style': None
   }
 
+  sources_globs = ('**/*',)
+
   def synthetic_target_type(self, target):
     return PythonLibrary
 
diff --git a/src/python/pants/backend/codegen/wire/java/wire_gen.py b/src/python/pants/backend/codegen/wire/java/wire_gen.py
index 1341676cb84..a6f32aea3ec 100644
--- a/src/python/pants/backend/codegen/wire/java/wire_gen.py
+++ b/src/python/pants/backend/codegen/wire/java/wire_gen.py
@@ -25,6 +25,8 @@
 
 class WireGen(NailgunTaskBase, SimpleCodegenTask):
 
+  sources_globs = ('**/*',)
+
   @classmethod
   def register_options(cls, register):
     super(WireGen, cls).register_options(register)

From c31b5698f3e3369a5dc64523d45061206f313ddb Mon Sep 17 00:00:00 2001
From: Daniel Wagner-Hall <dwagnerhall@twitter.com>
Date: Wed, 27 Jun 2018 12:09:08 +0100
Subject: [PATCH 10/10] Remove dead code

---
 .../backend/codegen/wire/java/wire_gen.py     | 27 -------------------
 1 file changed, 27 deletions(-)

diff --git a/src/python/pants/backend/codegen/wire/java/wire_gen.py b/src/python/pants/backend/codegen/wire/java/wire_gen.py
index a6f32aea3ec..019552e740c 100644
--- a/src/python/pants/backend/codegen/wire/java/wire_gen.py
+++ b/src/python/pants/backend/codegen/wire/java/wire_gen.py
@@ -116,30 +116,3 @@ def execute_codegen(self, target, target_workdir):
                             workunit_labels=[WorkUnitLabel.TOOL])
       if result != 0:
         raise TaskError('Wire compiler exited non-zero ({0})'.format(result))
-
-  class WireCompilerVersionError(TaskError):
-    """Indicates the wire compiler version could not be determined."""
-
-  def _calculate_proto_paths(self, target):
-    """Computes the set of paths that wire uses to lookup imported protos.
-
-    The protos under these paths are not compiled, but they are required to compile the protos that
-    imported.
-    :param target: the JavaWireLibrary target to compile.
-    :return: an ordered set of directories to pass along to wire.
-    """
-    proto_paths = OrderedSet()
-    proto_paths.add(os.path.join(get_buildroot(), self.context.source_roots.find(target).path))
-
-    def collect_proto_paths(dep):
-      if not dep.has_sources():
-        return
-      for source in dep.sources_relative_to_buildroot():
-        if source.endswith('.proto'):
-          root = self.context.source_roots.find_by_path(source)
-          if root:
-            proto_paths.add(os.path.join(get_buildroot(), root.path))
-
-    collect_proto_paths(target)
-    target.walk(collect_proto_paths)
-    return proto_paths