Targets always have a Snapshot

contrib/avro/src/python/pants/contrib/avro/tasks/avro_gen.py

@@ -23,6 +23,8 @@
 class AvroJavaGenTask(SimpleCodegenTask, NailgunTask):
   """Compile Avro schema, protocol, and IDL files to Java code."""
 
+  sources_globs = ('**/*',)
+
   @classmethod
   def register_options(cls, register):
     super(AvroJavaGenTask, cls).register_options(register)

contrib/go/src/python/pants/contrib/go/tasks/go_protobuf_gen.py

@@ -25,6 +25,8 @@
 
 class GoProtobufGen(SimpleCodegenTask):
 
+  sources_globs = ('**/*',)
+
   _NAMESPACE_PARSER = re.compile(r'^\s*option\s+go_package\s*=\s*"([^\s]+)"\s*;', re.MULTILINE)
   _PACKAGE_PARSER = re.compile(r'^\s*package\s+([^\s]+)\s*;', re.MULTILINE)
 

contrib/go/src/python/pants/contrib/go/tasks/go_thrift_gen.py

@@ -25,6 +25,8 @@
 
 class GoThriftGen(SimpleCodegenTask):
 
+  sources_globs = ('**/*',)
+
   @classmethod
   def register_options(cls, register):
     super(GoThriftGen, cls).register_options(register)

contrib/jax_ws/src/python/pants/contrib/jax_ws/tasks/jax_ws_gen.py

@@ -25,6 +25,8 @@
 class JaxWsGen(SimpleCodegenTask, NailgunTask):
   """Generates Java files from wsdl files using the JAX-WS compiler."""
 
+  sources_globs = ('**/*',)
+
   @classmethod
   def register_options(cls, register):
     super(JaxWsGen, cls).register_options(register)

contrib/scrooge/src/python/pants/contrib/scrooge/tasks/scrooge_gen.py

@@ -38,6 +38,8 @@ class ScroogeGen(SimpleCodegenTask, NailgunTask):
     'compiler_args'
   ])
 
+  sources_globs = ('**/*',)
+
   @classmethod
   def register_options(cls, register):
     super(ScroogeGen, cls).register_options(register)

contrib/scrooge/tests/python/pants_test/contrib/scrooge/tasks/test_scrooge_gen.py

@@ -15,7 +15,6 @@
 from pants.base.exceptions import TargetDefinitionException
 from pants.build_graph.build_file_aliases import BuildFileAliases
 from pants.goal.context import Context
-from pants.source.wrapped_globs import EagerFilesetWithSpec
 from pants_test.jvm.nailgun_task_test_base import NailgunTaskTestBase
 from twitter.common.collections import OrderedSet
 
@@ -139,10 +138,7 @@ def _test_help(self, language, library_type, compiler_args, sources):
       self.assertEquals(call_kwargs['fatal_warnings'], False)
 
       sources = call_kwargs['sources']
-      if isinstance(sources, EagerFilesetWithSpec):
-        self.assertEquals(sources.files, [])
-      else:
-        self.assertEquals(sources, [])
+      self.assertEquals(sources.files, ())
 
     finally:
       Context.add_new_target = saved_add_new_target

contrib/thrifty/src/python/pants/contrib/thrifty/java_thrifty_gen.py

@@ -23,6 +23,8 @@ class JavaThriftyGen(NailgunTaskBase, SimpleCodegenTask):
 
   gentarget_type = JavaThriftyLibrary
 
+  sources_globs = ('**/*',)
+
   @classmethod
   def register_options(cls, register):
     super(JavaThriftyGen, cls).register_options(register)

src/python/pants/backend/codegen/antlr/java/antlr_java_gen.py

@@ -39,6 +39,8 @@ class AntlrJavaGen(SimpleCodegenTask, NailgunTask):
   """Generate .java source code from ANTLR grammar files."""
   gentarget_type = JavaAntlrLibrary
 
+  sources_globs = ('**/*.java',)
+
   class AmbiguousPackageError(TaskError):
     """Raised when a java package cannot be unambiguously determined for a JavaAntlrLibrary."""
 

src/python/pants/backend/codegen/antlr/python/antlr_py_gen.py

@@ -33,6 +33,8 @@ class AntlrPyGen(SimpleCodegenTask, NailgunTask):
   """Generate Python source code from ANTLR grammar files."""
   gentarget_type = PythonAntlrLibrary
 
+  sources_globs = ('**/*.py',)
+
   @classmethod
   def register_options(cls, register):
     super(AntlrPyGen, cls).register_options(register)

src/python/pants/backend/codegen/jaxb/jaxb_gen.py

@@ -23,6 +23,8 @@ class JaxbGen(SimpleCodegenTask, NailgunTask):
   _XJC_MAIN = 'com.sun.tools.xjc.Driver'
   _XML_BIND_VERSION = '2.3.0'
 
+  sources_globs = ('**/*',)
+
   @classmethod
   def register_options(cls, register):
     super(JaxbGen, cls).register_options(register)

src/python/pants/backend/codegen/protobuf/java/protobuf_gen.py

@@ -27,6 +27,8 @@
 
 class ProtobufGen(SimpleCodegenTask):
 
+  sources_globs = ('**/*',)
+
   @classmethod
   def subsystem_dependencies(cls):
     return super(ProtobufGen, cls).subsystem_dependencies() + (Protoc.scoped(cls),)

src/python/pants/backend/codegen/ragel/java/ragel_gen.py

@@ -20,6 +20,9 @@
 
 
 class RagelGen(SimpleCodegenTask):
+
+  sources_globs = ('**/*',)
+
   @classmethod
   def subsystem_dependencies(cls):
     return super(RagelGen, cls).subsystem_dependencies() + (Ragel.scoped(cls),)

src/python/pants/backend/codegen/thrift/java/apache_thrift_java_gen.py

@@ -25,6 +25,8 @@ class ApacheThriftJavaGen(ApacheThriftGenBase):
 
   _COMPILER = 'thrift'
 
+  sources_globs = ('**/*',)
+
   @classmethod
   def subsystem_dependencies(cls):
     return super(ApacheThriftJavaGen, cls).subsystem_dependencies() + (ThriftDefaults,)

src/python/pants/backend/codegen/thrift/python/apache_thrift_py_gen.py

@@ -21,6 +21,8 @@ class ApacheThriftPyGen(ApacheThriftGenBase):
     'new_style': None
   }
 
+  sources_globs = ('**/*',)
+
   def synthetic_target_type(self, target):
     return PythonLibrary
 

src/python/pants/backend/codegen/wire/java/wire_gen.py

@@ -25,6 +25,8 @@
 
 class WireGen(NailgunTaskBase, SimpleCodegenTask):
 
+  sources_globs = ('**/*',)
+
   @classmethod
   def register_options(cls, register):
     super(WireGen, cls).register_options(register)
@@ -114,30 +116,3 @@ def execute_codegen(self, target, target_workdir):
                             workunit_labels=[WorkUnitLabel.TOOL])
       if result != 0:
         raise TaskError('Wire compiler exited non-zero ({0})'.format(result))
-
-  class WireCompilerVersionError(TaskError):
-    """Indicates the wire compiler version could not be determined."""
-
-  def _calculate_proto_paths(self, target):
-    """Computes the set of paths that wire uses to lookup imported protos.
-
-    The protos under these paths are not compiled, but they are required to compile the protos that
-    imported.
-    :param target: the JavaWireLibrary target to compile.
-    :return: an ordered set of directories to pass along to wire.
-    """
-    proto_paths = OrderedSet()
-    proto_paths.add(os.path.join(get_buildroot(), self.context.source_roots.find(target).path))
-
-    def collect_proto_paths(dep):
-      if not dep.has_sources():
-        return
-      for source in dep.sources_relative_to_buildroot():
-        if source.endswith('.proto'):
-          root = self.context.source_roots.find_by_path(source)
-          if root:
-            proto_paths.add(os.path.join(get_buildroot(), root.path))
-
-    collect_proto_paths(target)
-    target.walk(collect_proto_paths)
-    return proto_paths

src/python/pants/backend/graph_info/tasks/cloc.py

@@ -9,7 +9,7 @@
 
 from pants.backend.graph_info.subsystems.cloc_binary import ClocBinary
 from pants.base.workunit import WorkUnitLabel
-from pants.engine.fs import FilesContent, PathGlobs, PathGlobsAndRoot, Snapshot
+from pants.engine.fs import FilesContent, PathGlobs, PathGlobsAndRoot
 from pants.engine.isolated_process import ExecuteProcessRequest
 from pants.task.console_task import ConsoleTask
 from pants.util.contextutil import temporary_dir
@@ -35,16 +35,18 @@ def console_output(self, targets):
     if not self.get_options().transitive:
       targets = self.context.target_roots
 
+    input_snapshots = tuple(
+      target.sources_snapshot(scheduler=self.context._scheduler) for target in targets
+    )
+    input_files = set(f.path for snapshot in input_snapshots for f in snapshot.files)
+
     # TODO: Work out a nice library-like utility for writing an argfile, as this will be common.
     with temporary_dir() as tmpdir:
       list_file = os.path.join(tmpdir, 'input_files_list')
-      input_files = set()
       with open(list_file, 'w') as list_file_out:
-        for target in targets:
-          for source in target.sources_relative_to_buildroot():
-            input_files.add(source)
-            list_file_out.write(source)
-            list_file_out.write(b'\n')
+        for input_file in sorted(input_files):
+          list_file_out.write(input_file)
+          list_file_out.write(b'\n')
       list_file_snapshot = self.context._scheduler.capture_snapshots((
         PathGlobsAndRoot(
           PathGlobs(('input_files_list',)),
@@ -54,17 +56,11 @@ def console_output(self, targets):
 
     cloc_path, cloc_snapshot = ClocBinary.global_instance().hackily_snapshot(self.context)
 
-    # TODO: This should use an input file snapshot which should be provided on the Target object,
-    # rather than hackily re-snapshotting each of the input files.
-    # See https://github.com/pantsbuild/pants/issues/5762
-    input_pathglobs = PathGlobs(tuple(input_files))
-    input_snapshot = self.context._scheduler.product_request(Snapshot, [input_pathglobs])[0]
-
-    directory_digest = self.context._scheduler.merge_directories((
-      cloc_snapshot.directory_digest,
-      input_snapshot.directory_digest,
-      list_file_snapshot.directory_digest,
-    ))
+    directory_digest = self.context._scheduler.merge_directories(tuple(s.directory_digest for s in
+      input_snapshots + (
+      cloc_snapshot,
+      list_file_snapshot,
+    )))
 
     cmd = (
       '/usr/bin/perl',

src/python/pants/build_graph/BUILD

@@ -21,6 +21,7 @@ python_library(
     'src/python/pants/base:validation',
     'src/python/pants/option',
     'src/python/pants/source',
+    'src/python/pants/source:payload_fields',
     'src/python/pants/subsystem',
     'src/python/pants/util:dirutil',
     'src/python/pants/util:memo',

src/python/pants/build_graph/target.py

@@ -549,6 +549,14 @@ def sources_count(self):
     """Returns the count of source files owned by the target."""
     return len(self._sources_field.sources.files)
 
+  def sources_snapshot(self, scheduler=None):
+    """
+    Get a Snapshot of the sources attribute of this target.
+
+    This API is experimental, and is subject to change.
+    """
+    return self._sources_field.snapshot(scheduler=scheduler)
+
   @property
   def derived_from(self):
     """Returns the target this target was derived from.
@@ -890,8 +898,7 @@ def create_sources_field(self, sources, sources_rel_path, key_arg=None):
         ('Passing collections as the value of the sources argument to create_sources_field is '
          'deprecated, and now takes a slow path. Instead, class {} should have its sources '
          'argument populated by the engine, either by using the standard parsing pipeline, or by '
-         'requesting a SourcesField product.'
-         'the v2 engine.').format(self.__class__.__name__)
+         'requesting a SourcesField product from the v2 engine.').format(self.__class__.__name__)
       )
       sources = Files.create_fileset_with_spec(sources_rel_path, *sources)
     elif not isinstance(sources, FilesetWithSpec):

src/python/pants/engine/BUILD

@@ -139,7 +139,6 @@ python_library(
     '3rdparty/python/twitter/commons:twitter.common.collections',
     ':selectors',
     'src/python/pants/base:specs',
-    'src/python/pants/build_graph',
     'src/python/pants/util:objects',
   ]
 )

src/python/pants/engine/legacy/graph.py

@@ -26,7 +26,6 @@
 from pants.engine.selectors import Get, Select
 from pants.option.global_options import GlobMatchErrorBehavior
 from pants.source.wrapped_globs import EagerFilesetWithSpec, FilesetRelPathWrapper
-from pants.util.dirutil import fast_relpath
 from pants.util.objects import Collection, datatype
 
 
@@ -364,9 +363,6 @@ def hydrate_target(target_adaptor):
 
 
 def _eager_fileset_with_spec(spec_path, filespec, snapshot, include_dirs=False):
-  fds = snapshot.path_stats if include_dirs else snapshot.files
-  files = tuple(fast_relpath(fd.path, spec_path) for fd in fds)
-
   rel_include_globs = filespec['globs']
 
   relpath_adjusted_filespec = FilesetRelPathWrapper.to_filespec(rel_include_globs, spec_path)
@@ -376,8 +372,8 @@ def _eager_fileset_with_spec(spec_path, filespec, snapshot, include_dirs=False):
 
   return EagerFilesetWithSpec(spec_path,
                               relpath_adjusted_filespec,
-                              files=files,
-                              files_hash=snapshot.directory_digest.fingerprint)
+                              snapshot,
+                              include_dirs=include_dirs)
 
 
 @rule(HydratedField, [Select(SourcesField), Select(GlobMatchErrorBehavior)])

src/python/pants/source/BUILD

@@ -2,6 +2,7 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 python_library(
+  sources = globs('*.py', exclude=[globs('payload_fields.py')]),
   dependencies=[
     '3rdparty/python:six',
     '3rdparty/python/twitter/commons:twitter.common.dirutil',
@@ -14,3 +15,12 @@ python_library(
     'src/python/pants/util:memo',
   ]
 )
+
+python_library(
+  name = 'payload_fields',
+  sources = ['payload_fields.py'],
+  dependencies = [
+    ':source',
+    'src/python/pants/engine:fs',
+  ],
+)

src/python/pants/source/payload_fields.py

@@ -8,9 +8,10 @@
 from hashlib import sha1
 
 from pants.base.payload_field import PayloadField
+from pants.engine.fs import PathGlobs, Snapshot
 from pants.source.filespec import matches_filespec
 from pants.source.source_root import SourceRootConfig
-from pants.source.wrapped_globs import FilesetWithSpec
+from pants.source.wrapped_globs import EagerFilesetWithSpec, FilesetWithSpec
 from pants.util.memo import memoized_property
 
 
@@ -64,6 +65,19 @@ def address(self):
     """Returns the address this sources field refers to (used by some derived classes)"""
     return self._ref_address
 
+  def snapshot(self, scheduler=None):
+    """
+    Returns a Snapshot containing the sources, relative to the build root.
+
+    This API is experimental, and subject to change.
+    """
+    if isinstance(self._sources, EagerFilesetWithSpec):
+      snapshot = self._sources.snapshot
+      if snapshot is not None:
+        return snapshot
+    input_pathglobs = PathGlobs(tuple(self.relative_to_buildroot()))
+    return scheduler.product_request(Snapshot, [input_pathglobs])[0]
+
   def relative_to_buildroot(self):
     """All sources joined with their relative paths."""
     return list(self.sources.paths_from_buildroot_iter())