warn or error on matching source globs

src/python/pants/bin/BUILD

@@ -27,6 +27,7 @@ python_library(
     'src/python/pants/engine:native',
     'src/python/pants/engine:parser',
     'src/python/pants/engine:scheduler',
+    'src/python/pants/engine:rules',
     'src/python/pants/goal',
     'src/python/pants/goal:context',
     'src/python/pants/goal:run_tracker',

src/python/pants/bin/goal_runner.py

@@ -23,6 +23,7 @@
 from pants.init.subprocess import Subprocess
 from pants.init.target_roots_calculator import TargetRootsCalculator
 from pants.java.nailgun_executor import NailgunProcessGroup
+from pants.option.global_options import GlobMatchErrorBehavior
 from pants.option.ranked_value import RankedValue
 from pants.reporting.reporting import Reporting
 from pants.scm.subsystems.changed import Changed
@@ -106,6 +107,8 @@ def _init_graph(self,
         pants_ignore_patterns,
         workdir,
         self._global_options.build_file_imports,
+        glob_match_error_behavior=GlobMatchErrorBehavior.create(
+          self._global_options.glob_expansion_failure),
         native=native,
         build_file_aliases=self._build_config.registered_aliases(),
         rules=self._build_config.rules(),

src/python/pants/engine/BUILD

@@ -50,6 +50,7 @@ python_library(
     ':rules',
     ':selectors',
     'src/python/pants/base:project_tree',
+    'src/python/pants/option',
     'src/python/pants/source',
     'src/python/pants/util:meta',
     'src/python/pants/util:objects',

src/python/pants/engine/fs.py

@@ -9,6 +9,7 @@
 
 from pants.base.project_tree import Dir, File
 from pants.engine.rules import RootRule
+from pants.option.global_options import GlobMatchErrorBehavior
 from pants.util.objects import Collection, datatype
 
 
@@ -29,24 +30,47 @@ class Path(datatype(['path', 'stat'])):
   """
 
 
-class PathGlobs(datatype(['include', 'exclude'])):
+class PathGlobs(datatype([
+    'include',
+    'exclude',
+    ('glob_match_error_behavior', GlobMatchErrorBehavior),
+])):
   """A wrapper around sets of filespecs to include and exclude.
 
   The syntax supported is roughly git's glob syntax.
+
+  NB: this object is interpreted from within Snapshot::lift_path_globs() -- that method will need to
+  be aware of any changes to this object's definition.
   """
 
+  def __new__(cls, include, exclude, glob_match_error_behavior=None):
+    return super(PathGlobs, cls).__new__(
+      cls,
+      tuple(include),
+      tuple(exclude),
+      GlobMatchErrorBehavior.create(glob_match_error_behavior))
+
+  def with_match_error_behavior(self, glob_match_error_behavior):
+    return PathGlobs(
+      include=self.include,
+      exclude=self.exclude,
+      glob_match_error_behavior=glob_match_error_behavior)
+
   @staticmethod
-  def create(relative_to, include, exclude=tuple()):
+  def create(relative_to, include, exclude=tuple(), glob_match_error_behavior=None):
     """Given various file patterns create a PathGlobs object (without using filesystem operations).
 
     :param relative_to: The path that all patterns are relative to (which will itself be relative
       to the buildroot).
-    :param included: A list of filespecs to include.
-    :param excluded: A list of filespecs to exclude.
+    :param include: A list of filespecs to include.
+    :param exclude: A list of filespecs to exclude.
+    :param glob_match_error_behavior: The value to pass to GlobMatchErrorBehavior.create()
     :rtype: :class:`PathGlobs`
     """
-    return PathGlobs(tuple(join(relative_to, f) for f in include),
-                     tuple(join(relative_to, f) for f in exclude))
+    encoded_reldir = relative_to.decode('utf-8')
+    return PathGlobs(include=tuple(join(encoded_reldir, f.decode('utf-8')) for f in include),
+                     exclude=tuple(join(encoded_reldir, f.decode('utf-8')) for f in exclude),
+                     glob_match_error_behavior=glob_match_error_behavior)
 
 
 class PathGlobsAndRoot(datatype([('path_globs', PathGlobs), ('root', str)])):

src/python/pants/engine/legacy/BUILD

@@ -61,6 +61,7 @@ python_library(
     'src/python/pants/engine:mapper',
     'src/python/pants/engine:parser',
     'src/python/pants/engine:selectors',
+    'src/python/pants/option',
     'src/python/pants/source',
     'src/python/pants/util:dirutil',
     'src/python/pants/util:objects',

src/python/pants/engine/legacy/graph.py

@@ -24,6 +24,7 @@
 from pants.engine.legacy.structs import BundleAdaptor, BundlesField, SourcesField, TargetAdaptor
 from pants.engine.rules import TaskRule, rule
 from pants.engine.selectors import Get, Select
+from pants.option.global_options import GlobMatchErrorBehavior
 from pants.source.wrapped_globs import EagerFilesetWithSpec, FilesetRelPathWrapper
 from pants.util.dirutil import fast_relpath
 from pants.util.objects import Collection, datatype
@@ -364,7 +365,9 @@ def _eager_fileset_with_spec(spec_path, filespec, snapshot, include_dirs=False):
   fds = snapshot.path_stats if include_dirs else snapshot.files
   files = tuple(fast_relpath(fd.path, spec_path) for fd in fds)
 
-  relpath_adjusted_filespec = FilesetRelPathWrapper.to_filespec(filespec['globs'], spec_path)
+  rel_include_globs = filespec['globs']
+
+  relpath_adjusted_filespec = FilesetRelPathWrapper.to_filespec(rel_include_globs, spec_path)
   if filespec.has_key('exclude'):
     relpath_adjusted_filespec['exclude'] = [FilesetRelPathWrapper.to_filespec(e['globs'], spec_path)
                                             for e in filespec['exclude']]
@@ -375,33 +378,43 @@ def _eager_fileset_with_spec(spec_path, filespec, snapshot, include_dirs=False):
                               files_hash=snapshot.directory_digest.fingerprint)
 
 
-@rule(HydratedField, [Select(SourcesField)])
-def hydrate_sources(sources_field):
-  """Given a SourcesField, request a Snapshot for its path_globs and create an EagerFilesetWithSpec."""
-
-  snapshot = yield Get(Snapshot, PathGlobs, sources_field.path_globs)
-  fileset_with_spec = _eager_fileset_with_spec(sources_field.address.spec_path,
-                                               sources_field.filespecs,
-                                               snapshot)
+@rule(HydratedField, [Select(SourcesField), Select(GlobMatchErrorBehavior)])
+def hydrate_sources(sources_field, glob_match_error_behavior):
+  """Given a SourcesField, request a Snapshot for its path_globs and create an EagerFilesetWithSpec.
+  """
+  # TODO(#5864): merge the target's selection of --glob-expansion-failure (which doesn't exist yet)
+  # with the global default!
+  path_globs = sources_field.path_globs.with_match_error_behavior(glob_match_error_behavior)
+  snapshot = yield Get(Snapshot, PathGlobs, path_globs)
+  fileset_with_spec = _eager_fileset_with_spec(
+    sources_field.address.spec_path,
+    sources_field.filespecs,
+    snapshot)
   yield HydratedField(sources_field.arg, fileset_with_spec)
 
 
-@rule(HydratedField, [Select(BundlesField)])
-def hydrate_bundles(bundles_field):
+@rule(HydratedField, [Select(BundlesField), Select(GlobMatchErrorBehavior)])
+def hydrate_bundles(bundles_field, glob_match_error_behavior):
   """Given a BundlesField, request Snapshots for each of its filesets and create BundleAdaptors."""
-  snapshot_list = yield [Get(Snapshot, PathGlobs, pg) for pg in bundles_field.path_globs_list]
+  path_globs_with_match_errors = [
+    pg.with_match_error_behavior(glob_match_error_behavior)
+    for pg in bundles_field.path_globs_list
+  ]
+  snapshot_list = yield [Get(Snapshot, PathGlobs, pg) for pg in path_globs_with_match_errors]
+
+  spec_path = bundles_field.address.spec_path
 
   bundles = []
   zipped = zip(bundles_field.bundles,
                bundles_field.filespecs_list,
                snapshot_list)
   for bundle, filespecs, snapshot in zipped:
-    spec_path = bundles_field.address.spec_path
+    rel_spec_path = getattr(bundle, 'rel_path', spec_path)
     kwargs = bundle.kwargs()
     # NB: We `include_dirs=True` because bundle filesets frequently specify directories in order
     # to trigger a (deprecated) default inclusion of their recursive contents. See the related
     # deprecation in `pants.backend.jvm.tasks.bundle_create`.
-    kwargs['fileset'] = _eager_fileset_with_spec(getattr(bundle, 'rel_path', spec_path),
+    kwargs['fileset'] = _eager_fileset_with_spec(rel_spec_path,
                                                  filespecs,
                                                  snapshot,
                                                  include_dirs=True)

src/python/pants/engine/legacy/structs.py

@@ -55,7 +55,7 @@ def field_adaptors(self):
         return tuple()
       base_globs = BaseGlobs.from_sources_field(sources, self.address.spec_path)
       path_globs = base_globs.to_path_globs(self.address.spec_path)
-      return (SourcesField(self.address, 'sources', base_globs.filespecs, path_globs),)
+      return (SourcesField(self.address, 'sources', base_globs.filespecs, base_globs, path_globs),)
 
   @property
   def default_sources_globs(self):
@@ -70,7 +70,7 @@ class Field(object):
   """A marker for Target(Adaptor) fields for which the engine might perform extra construction."""
 
 
-class SourcesField(datatype(['address', 'arg', 'filespecs', 'path_globs']), Field):
+class SourcesField(datatype(['address', 'arg', 'filespecs', 'base_globs', 'path_globs']), Field):
   """Represents the `sources` argument for a particular Target.
 
   Sources are currently eagerly computed in-engine in order to provide the `BuildGraph`
@@ -92,7 +92,8 @@ def __repr__(self):
     return str(self)
 
   def __str__(self):
-    return 'SourcesField(address={}, arg={}, filespecs={!r})'.format(self.address, self.arg, self.filespecs)
+    return '{}(address={}, input_globs={}, arg={}, filespecs={!r})'.format(
+      type(self).__name__, self.address, self.base_globs, self.arg, self.filespecs)
 
 
 class JavaLibraryAdaptor(TargetAdaptor):
@@ -173,6 +174,9 @@ def _construct_bundles_field(self):
       rel_root = getattr(bundle, 'rel_path', self.address.spec_path)
 
       base_globs = BaseGlobs.from_sources_field(bundle.fileset, rel_root)
+      # TODO: we want to have this field set from the global option --glob-expansion-failure, or
+      # something set on the target. Should we move --glob-expansion-failure to be a bootstrap
+      # option? See #5864.
       path_globs = base_globs.to_path_globs(rel_root)
 
       filespecs_list.append(base_globs.filespecs)
@@ -205,6 +209,7 @@ def field_adaptors(self):
       sources_field = SourcesField(self.address,
                                    'resources',
                                    base_globs.filespecs,
+                                   base_globs,
                                    path_globs)
       return field_adaptors + (sources_field,)
 
@@ -288,6 +293,8 @@ def legacy_globs_class(self):
     """The corresponding `wrapped_globs` class for this BaseGlobs."""
 
   def __init__(self, *patterns, **kwargs):
+    self._patterns = patterns
+    self._kwargs = kwargs
     raw_spec_path = kwargs.pop('spec_path')
     self._file_globs = self.legacy_globs_class.to_filespec(patterns).get('globs', [])
     raw_exclude = kwargs.pop('exclude', [])
@@ -324,14 +331,36 @@ def _exclude_filespecs(self):
       return []
 
   def to_path_globs(self, relpath):
-    """Return two PathGlobs representing the included and excluded Files for these patterns."""
+    """Return a PathGlobs representing the included and excluded Files for these patterns."""
     return PathGlobs.create(relpath, self._file_globs, self._excluded_file_globs)
 
+  def _gen_init_args_str(self):
+    all_arg_strs = []
+    positional_args = ', '.join([repr(p) for p in self._patterns])
+    if positional_args:
+      all_arg_strs.append(positional_args)
+    keyword_args = ', '.join([
+      '{}={}'.format(k, repr(v)) for k, v in self._kwargs.items()
+    ])
+    if keyword_args:
+      all_arg_strs.append(keyword_args)
+
+    return ', '.join(all_arg_strs)
+
+  def __repr__(self):
+    return '{}({})'.format(type(self).__name__, self._gen_init_args_str())
+
+  def __str__(self):
+    return '{}({})'.format(self.path_globs_kwarg, self._gen_init_args_str())
+
 
 class Files(BaseGlobs):
   path_globs_kwarg = 'files'
   legacy_globs_class = wrapped_globs.Globs
 
+  def __str__(self):
+    return '[{}]'.format(', '.join(repr(p) for p in self._patterns))
+
 
 class Globs(BaseGlobs):
   path_globs_kwarg = 'globs'

src/python/pants/init/engine_initializer.py

@@ -24,8 +24,10 @@
 from pants.engine.mapper import AddressMapper
 from pants.engine.native import Native
 from pants.engine.parser import SymbolTable
+from pants.engine.rules import SingletonRule
 from pants.engine.scheduler import Scheduler
 from pants.init.options_initializer import OptionsInitializer
+from pants.option.global_options import GlobMatchErrorBehavior
 from pants.option.options_bootstrapper import OptionsBootstrapper
 from pants.scm.change_calculator import EngineChangeCalculator
 from pants.util.objects import datatype
@@ -131,6 +133,7 @@ def setup_legacy_graph(pants_ignore_patterns,
                          build_root=None,
                          native=None,
                          build_file_aliases=None,
+                         glob_match_error_behavior=None,
                          rules=None,
                          build_ignore_patterns=None,
                          exclude_target_regexps=None,
@@ -151,6 +154,9 @@ def setup_legacy_graph(pants_ignore_patterns,
     :param Native native: An instance of the native-engine subsystem.
     :param build_file_aliases: BuildFileAliases to register.
     :type build_file_aliases: :class:`pants.build_graph.build_file_aliases.BuildFileAliases`
+    :param glob_match_error_behavior: How to behave if a glob specified for a target's sources or
+                                      bundles does not expand to anything.
+    :type glob_match_error_behavior: :class:`pants.option.global_options.GlobMatchErrorBehavior`
     :param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD
                                        files, usually taken from the '--build-ignore' global option.
     :param list exclude_target_regexps: A list of regular expressions for excluding targets.
@@ -194,6 +200,8 @@ def setup_legacy_graph(pants_ignore_patterns,
       create_fs_rules() +
       create_process_rules() +
       create_graph_rules(address_mapper, symbol_table) +
+      [SingletonRule(GlobMatchErrorBehavior,
+                     GlobMatchErrorBehavior.create(glob_match_error_behavior))] +
       rules
     )
 

src/python/pants/option/BUILD

@@ -13,6 +13,7 @@ python_library(
     'src/python/pants/util:eval',
     'src/python/pants/util:memo',
     'src/python/pants/util:meta',
+    'src/python/pants/util:objects',
     'src/python/pants/util:strutil',
   ]
 )

src/python/pants/option/global_options.py

@@ -16,6 +16,50 @@
 from pants.option.optionable import Optionable
 from pants.option.scope import ScopeInfo
 from pants.subsystem.subsystem_client_mixin import SubsystemClientMixin
+from pants.util.memo import memoized_method
+from pants.util.objects import datatype
+
+
+class GlobMatchErrorBehavior(datatype(['failure_behavior'])):
+  """Describe the action to perform when matching globs in BUILD files to source files.
+
+  NB: this object is interpreted from within Snapshot::lift_path_globs() -- that method will need to
+  be aware of any changes to this object's definition.
+  """
+
+  IGNORE = 'ignore'
+  WARN = 'warn'
+  ERROR = 'error'
+
+  allowed_values = [IGNORE, WARN, ERROR]
+
+  default_value = IGNORE
+
+  default_option_value = WARN
+
+  # FIXME(cosmicexplorer): add helpers in pants.util.memo for class properties and memoized class
+  # properties!
+  @classmethod
+  @memoized_method
+  def _singletons(cls):
+    return { behavior: cls(behavior) for behavior in cls.allowed_values }
+
+  @classmethod
+  def create(cls, value=None):
+    if isinstance(value, cls):
+      return value
+    if not value:
+      value = cls.default_value
+    return cls._singletons()[value]
+
+  def __new__(cls, *args, **kwargs):
+    this_object = super(GlobMatchErrorBehavior, cls).__new__(cls, *args, **kwargs)
+
+    if this_object.failure_behavior not in cls.allowed_values:
+      raise cls.make_type_error("Value {!r} for failure_behavior must be one of: {!r}."
+                                .format(this_object.failure_behavior, cls.allowed_values))
+
+    return this_object
 
 
 class GlobalOptionsRegistrar(SubsystemClientMixin, Optionable):
@@ -244,3 +288,11 @@ def register_options(cls, register):
     register('--lock', advanced=True, type=bool, default=True,
              help='Use a global lock to exclude other versions of pants from running during '
                   'critical operations.')
+    # TODO: Make a custom type abstract class (or something) to automate the production of an option
+    # with specific allowed values from a datatype (ideally using singletons for the allowed
+    # values).
+    register('--glob-expansion-failure', type=str,
+             choices=GlobMatchErrorBehavior.allowed_values,
+             default=GlobMatchErrorBehavior.default_option_value,
+             help="Raise an exception if any targets declaring source files "
+                  "fail to match any glob provided in the 'sources' argument.")

src/python/pants/source/wrapped_globs.py

@@ -163,6 +163,7 @@ def create_fileset_with_spec(cls, rel_path, *patterns, **kwargs):
     :param rel_path: The relative path to create a FilesetWithSpec for.
     :param patterns: glob patterns to apply.
     :param exclude: A list of {,r,z}globs objects, strings, or lists of strings to exclude.
+                    NB: this argument is contained within **kwargs!
     """
     for pattern in patterns:
       if not isinstance(pattern, string_types):