[engine] Support implicit sources in v2 engine

src/python/pants/bin/engine_initializer.py

@@ -18,8 +18,10 @@
 from pants.engine.legacy.change_calculator import EngineChangeCalculator
 from pants.engine.legacy.graph import HydratedTargets, LegacyBuildGraph, create_legacy_graph_tasks
 from pants.engine.legacy.parser import LegacyPythonCallbacksParser
-from pants.engine.legacy.structs import (JvmAppAdaptor, PythonTargetAdaptor, RemoteSourcesAdaptor,
-                                         TargetAdaptor)
+from pants.engine.legacy.structs import (JavaLibraryAdaptor, JunitTestsAdaptor, JvmAppAdaptor,
+                                         PythonLibraryAdaptor, PythonTargetAdaptor,
+                                         PythonTestsAdaptor, RemoteSourcesAdaptor,
+                                         ScalaLibraryAdaptor, TargetAdaptor)
 from pants.engine.mapper import AddressMapper
 from pants.engine.parser import SymbolTable
 from pants.engine.scheduler import LocalScheduler
@@ -50,10 +52,18 @@ def table(cls):
     # API until after https://github.com/pantsbuild/pants/issues/3560 has been completed.
     # These should likely move onto Target subclasses as the engine gets deeper into beta
     # territory.
+    for alias in ['java_library', 'java_agent', 'javac_plugin']:
+      aliases[alias] = JavaLibraryAdaptor
+    for alias in ['scala_library', 'scalac_plugin']:
+      aliases[alias] = ScalaLibraryAdaptor
+    for alias in ['python_library', 'pants_plugin']:
+      aliases[alias] = PythonLibraryAdaptor
+
+    aliases['junit_tests'] = JunitTestsAdaptor
     aliases['jvm_app'] = JvmAppAdaptor
+    aliases['python_tests'] = PythonTestsAdaptor
+    aliases['python_binary'] = PythonTargetAdaptor
     aliases['remote_sources'] = RemoteSourcesAdaptor
-    for alias in ('python_library', 'python_tests', 'python_binary'):
-      aliases[alias] = PythonTargetAdaptor
 
     return aliases
 

src/python/pants/engine/legacy/BUILD

@@ -40,6 +40,7 @@ python_library(
     'src/python/pants/engine:struct',
     'src/python/pants/source',
     'src/python/pants/util:contextutil',
+    'src/python/pants/util:memo',
     'src/python/pants/util:meta',
     'src/python/pants/util:objects',
   ],

src/python/pants/engine/legacy/structs.py

@@ -16,6 +16,7 @@
 from pants.engine.struct import Struct, StructWithDeps
 from pants.source import wrapped_globs
 from pants.util.contextutil import exception_logging
+from pants.util.memo import memoized_method
 from pants.util.meta import AbstractClass
 from pants.util.objects import datatype
 
@@ -29,27 +30,41 @@ class TargetAdaptor(StructWithDeps):
   Extends StructWithDeps to add a `dependencies` field marked Addressable.
   """
 
-  @property
-  def has_concrete_sources(self):
-    """Returns true if this target has non-deferred sources.
+  @memoized_method
+  def get_sources(self):
+    """Returns target's non-deferred sources if exists or the default sources if defined.
 
     NB: once ivy is implemented in the engine, we can fetch sources natively here, and/or
     refactor how deferred sources are implemented.
       see: https://github.com/pantsbuild/pants/issues/2997
     """
     sources = getattr(self, 'sources', None)
-    return sources is not None
+    if not sources:
+      if self.default_sources_globs:
+        return Globs(*self.default_sources_globs,
+                     spec_path=self.address.spec_path,
+                     exclude=self.default_sources_exclude_globs or [])
+    return sources
 
   @property
   def field_adaptors(self):
     """Returns a tuple of Fields for captured fields which need additional treatment."""
     with exception_logging(logger, 'Exception in `field_adaptors` property'):
-      if not self.has_concrete_sources:
+      sources = self.get_sources()
+      if not sources:
         return tuple()
-      base_globs = BaseGlobs.from_sources_field(self.sources, self.address.spec_path)
+      base_globs = BaseGlobs.from_sources_field(sources, self.address.spec_path)
       path_globs, excluded_path_globs = base_globs.to_path_globs(self.address.spec_path)
       return (SourcesField(self.address, 'sources', base_globs.filespecs, path_globs, excluded_path_globs),)
 
+  @property
+  def default_sources_globs(self):
+    return None
+
+  @property
+  def default_sources_exclude_globs(self):
+    return None
+
 
 class Field(object):
   """A marker for Target(Adaptor) fields for which the engine might perform extra construction."""
@@ -88,6 +103,38 @@ def __str__(self):
     return 'SourcesField(address={}, arg={}, filespecs={!r})'.format(self.address, self.arg, self.filespecs)
 
 
+class JavaLibraryAdaptor(TargetAdaptor):
+
+  @property
+  def default_sources_globs(self):
+    return ('*.java',)
+
+  @property
+  def default_sources_exclude_globs(self):
+    return JunitTestsAdaptor.java_test_globs
+
+
+class ScalaLibraryAdaptor(TargetAdaptor):
+
+  @property
+  def default_sources_globs(self):
+    return ('*.scala',)
+
+  @property
+  def default_sources_exclude_globs(self):
+    return JunitTestsAdaptor.scala_test_globs
+
+
+class JunitTestsAdaptor(TargetAdaptor):
+
+  java_test_globs = ('*Test.java',)
+  scala_test_globs = ('*Test.scala', '*Spec.scala')
+
+  @property
+  def default_sources_globs(self):
+    return self.java_test_globs + self.scala_test_globs
+
+
 class BundlesField(datatype('BundlesField', ['address', 'bundles', 'filespecs_list', 'path_globs_list', 'excluded_path_globs_list']), Field):
   """Represents the `bundles` argument, each of which has a PathGlobs to represent its `fileset`."""
 
@@ -183,6 +230,26 @@ def field_adaptors(self):
       return field_adaptors + (sources_field,)
 
 
+class PythonLibraryAdaptor(PythonTargetAdaptor):
+
+  @property
+  def default_sources_globs(self):
+    return ('*.py',)
+
+  @property
+  def default_sources_exclude_globs(self):
+    return PythonTestsAdaptor.python_test_globs
+
+
+class PythonTestsAdaptor(PythonTargetAdaptor):
+
+  python_test_globs = ('test_*.py', '*_test.py')
+
+  @property
+  def default_sources_globs(self):
+    return self.python_test_globs
+
+
 class BaseGlobs(Locatable, AbstractClass):
   """An adaptor class to allow BUILD file parsing from ContextAwareObjectFactories."""
 

testprojects/tests/python/pants/file_sets/BUILD

@@ -68,3 +68,13 @@ python_library(
                 ),
   dependencies=[]
 )
+
+python_library(
+  name='implicit_sources',
+  dependencies=[]
+)
+
+python_tests(
+  name='test_with_implicit_sources',
+  dependencies=[]
+)

tests/python/pants_test/engine/legacy/test_filemap_integration.py

@@ -14,16 +14,22 @@
 class FilemapIntegrationTest(PantsRunIntegrationTest):
   PATH_PREFIX = 'testprojects/tests/python/pants/file_sets/'
   TEST_EXCLUDE_FILES = {
-    'a.py', 'aa.py', 'aaa.py', 'ab.py', 'aabb.py', 'dir1/a.py', 'dir1/aa.py', 'dir1/aaa.py',
+    'a.py', 'aa.py', 'aaa.py', 'ab.py', 'aabb.py', 'test_a.py',
+    'dir1/a.py', 'dir1/aa.py', 'dir1/aaa.py',
     'dir1/ab.py', 'dir1/aabb.py', 'dir1/dirdir1/a.py', 'dir1/dirdir1/aa.py', 'dir1/dirdir1/ab.py'
   }
 
   def setUp(self):
     super(FilemapIntegrationTest, self).setUp()
+
     project_tree = FileSystemProjectTree(os.path.abspath(self.PATH_PREFIX), ['BUILD', '.*'])
     scan_set = set()
+
+    def should_ignore(file):
+      return file.endswith('.pyc')
+
     for root, dirs, files in project_tree.walk(''):
-      scan_set.update({os.path.join(root, f) for f in files})
+      scan_set.update({os.path.join(root, f) for f in files if not should_ignore(f)})
 
     self.assertEquals(scan_set, self.TEST_EXCLUDE_FILES)
 
@@ -97,3 +103,11 @@ def test_exclude_composite(self):
     self.assertEquals(self.TEST_EXCLUDE_FILES -
                       {'a.py', 'aaa.py', 'dir1/a.py', 'dir1/dirdir1/a.py'},
                       test_out)
+
+  def test_implicit_sources(self):
+    test_out = self._extract_exclude_output('implicit_sources')
+    self.assertEquals({'a.py', 'aa.py', 'aaa.py', 'aabb.py', 'ab.py'},
+                      test_out)
+
+    test_out = self._extract_exclude_output('test_with_implicit_sources')
+    self.assertEquals({'test_a.py'}, test_out)

tests/python/pants_test/engine/legacy/test_graph.py

@@ -94,6 +94,20 @@ def test_sources_ordering(self):
       self.assertEquals(['p', 'a', 'n', 't', 's', 'b', 'u', 'i', 'l', 'd'],
                         sources)
 
+  def test_implicit_sources(self):
+    expected_sources = {
+      'testprojects/tests/python/pants/file_sets:implicit_sources':
+        ['a.py', 'aa.py', 'aaa.py', 'aabb.py', 'ab.py'],
+      'testprojects/tests/python/pants/file_sets:test_with_implicit_sources':
+        ['test_a.py']
+    }
+
+    for spec, exp_sources in expected_sources.items():
+      with self.open_scheduler([spec]) as (graph, _, _):
+        target = graph.get_target(Address.parse(spec))
+        sources = sorted([os.path.basename(s) for s in target.sources_relative_to_buildroot()])
+        self.assertEquals(exp_sources, sources)
+
   def test_target_macro_override(self):
     """Tests that we can "wrap" an existing target type with additional functionality.