Support loading plugins via pkg_resources entry points

This provides an alternative way to find the methods `build_file_aliases` and `register_goals` that
are called as part of extension loading, specifically that for off-the-shelf plugins installed as
distrubutions, these methods are found using pkg_resources' entry points configuration.

Source backends are useful for quick and easy loading of custom tasks or aliases from sources that
live in a project. This change is not entended to replace source backends, but rather to make use
of richer metadata avilable on packaged distributions used for deploying off-the-shelf plugins.

Distributions (egg/sdist/etc) have version information and can define their entry points, which allows
cleaner separation of layout and implementation from core pants. Version requirements can be checked at
load and plugins can assert other plugins are loaded before them (since order matters eg "replace=True").

Adding such a plugin when using virtualenv pants would consist of adding it to requirements.txt used to
setup the virtualenv (or simply pip installing if using global site-packages), then adding it to
pants.ini's `plugins` list under the section `backends`.

For deployments running pants from a pex, the plugins could either be built into the distributed pex or
plugins could still be installed to a virualenv using a requirments.txt if that env were somehow then
added to the python path when invoking the pex.

This change does not include support for resolving, fetching or in any way installing plugins at
runtime -- requested plugins are loaded if they are found or an error is raised if not.  If or how how
to support loading plugins that are not previously installed is left for a separate discussion.

Testing Done:
https://travis-ci.org/pantsbuild/pants/builds/43419387
Added tests with mocked out distributions to test_extension_loader.py.
Demo repo: https://github.com/foursquare/pants-plugins-demo

Bugs closed: 843

Reviewed at https://rbcommons.com/s/twitter/r/1429/

src/python/pants/base/extension_loader.py

@@ -5,22 +5,92 @@
 from __future__ import (nested_scopes, generators, division, absolute_import, with_statement,
                         print_function, unicode_literals)
 
+from pkg_resources import working_set, Requirement
+
 from twitter.common.collections import OrderedSet
 
 from pants.base.build_configuration import BuildConfiguration
 from pants.base.exceptions import BackendConfigurationError
 
 
-def load_build_configuration_from_source(additional_backends=None):
+class PluginLoadingError(Exception): pass
+class PluginNotFound(PluginLoadingError): pass
+class PluginLoadOrderError(PluginLoadingError): pass
+
+
+def load_plugins_and_backends(plugins=None, backends=None):
+  """Load named plugins and source backends
+
+  :param list<str> plugins: Plugins to load (see `load_plugins`).
+  :param list<str> backends: Source backends to load (see `load_build_configuration_from_source`).
+  """
+  build_configuration = BuildConfiguration()
+  load_plugins(build_configuration, plugins or [])
+  load_build_configuration_from_source(build_configuration, additional_backends=backends or [])
+  return build_configuration
+
+
+def load_plugins(build_configuration, plugins, load_from=None):
+  """Load named plugins from the current working_set into the supplied build_configuration
+
+  "Loading" a plugin here refers to calling registration methods -- it is assumed each plugin
+  is already on the path and an error will be thrown if it is not. Plugins should define their
+  entrypoints in the `pantsbuild.plugin` group when configuring their distribution.
+
+  Like source backends, the `build_file_aliases` method and `register_goals` methods are called if
+  those entry points are defined.
+
+  * Plugins are loaded in the order they are provided. *
+
+  This is important as loading can add, remove or replace exiting tasks installed by other plugins.
+
+  If a plugin needs to assert that another plugin is registered before it, it can define an
+  entrypoint "load_after" which can return a list of plugins which must have been loaded before it
+  can be loaded. This does not change the order or what plugins are loaded in any way -- it is
+  purely an assertion to guard against misconfiguration.
+
+  :param BuildConfiguration build_configuration: The BuildConfiguration (for adding aliases).
+  :param list<str> plugins: A list of plugin names optionally with versions, in requirement format.
+                            eg ['widgetpublish', 'widgetgen==1.2'].
+  :param WorkingSet load_from: A pkg_resources.WorkingSet to use instead of global (for testing).
+  """
+  load_from = load_from or working_set
+  loaded = {}
+  for plugin in plugins:
+    req = Requirement.parse(plugin)
+    dist = load_from.find(req)
+
+    if not dist:
+      raise PluginNotFound('Could not find plugin: {}'.format(req))
+
+    entries = dist.get_entry_map().get('pantsbuild.plugin', {})
+
+    if 'load_after' in entries:
+      deps = entries['load_after'].load()()
+      for dep_name in deps:
+        dep = Requirement.parse(dep_name)
+        if dep.key not in loaded:
+          raise PluginLoadOrderError('Plugin {0} must be loaded after {1}'.format(plugin, dep))
+
+    if 'build_file_aliases' in entries:
+      aliases = entries['build_file_aliases'].load()()
+      build_configuration.register_aliases(aliases)
+
+    if 'register_goals' in entries:
+      entries['register_goals'].load()()
+
+    loaded[dist.as_requirement().key] = dist
+
+
+def load_build_configuration_from_source(build_configuration, additional_backends=None):
   """Installs pants backend packages to provide targets and helper functions to BUILD files and
   goals to the cli.
 
+  :param BuildConfiguration build_configuration: The BuildConfiguration (for adding aliases).
   :param additional_backends: An optional list of additional packages to load backends from.
-  :returns: a new :class:``pants.base.build_configuration.BuildConfiguration``.
   :raises: :class:``pants.base.exceptions.BuildConfigurationError`` if there is a problem loading
     the build configuration.
   """
-  build_configuration = BuildConfiguration()
   backend_packages = ['pants.backend.authentication',
                       'pants.backend.core',
                       'pants.backend.python',
@@ -32,8 +102,6 @@ def load_build_configuration_from_source(additional_backends=None):
   for backend_package in OrderedSet(backend_packages + (additional_backends or [])):
     load_backend(build_configuration, backend_package)
 
-  return build_configuration
-
 
 def load_backend(build_configuration, backend_package):
   """Installs the given backend package into the build configuration.

src/python/pants/bin/pants_exe.py

@@ -17,7 +17,7 @@
 from pants.base.build_file_parser import BuildFileParser
 from pants.base.build_graph import BuildGraph
 from pants.base.config import Config
-from pants.base.extension_loader import load_build_configuration_from_source
+from pants.base.extension_loader import load_plugins_and_backends
 from pants.base.workunit import WorkUnit
 from pants.commands.command import Command
 from pants.goal.initialize_reporting import initial_reporting
@@ -136,8 +136,10 @@ def _run():
   else:
     run_tracker.log(Report.INFO, '(To run a reporting server: ./pants goal server)')
 
-  backend_packages = config.getlist('backends', 'packages')
-  build_configuration = load_build_configuration_from_source(additional_backends=backend_packages)
+  backend_packages = config.getlist('backends', 'packages', [])
+  plugins = config.getlist('backends', 'plugins', [])
+
+  build_configuration = load_plugins_and_backends(plugins, backend_packages)
   build_file_parser = BuildFileParser(build_configuration=build_configuration,
                                       root_dir=root_dir,
                                       run_tracker=run_tracker)

tests/python/pants_test/base/test_extension_loader.py

@@ -14,16 +14,34 @@
 
 from pants.base.build_configuration import BuildConfiguration
 from pants.base.build_file_aliases import BuildFileAliases
-from pants.base.extension_loader import load_backend
+from pants.base.extension_loader import load_backend, load_plugins, PluginNotFound, PluginLoadOrderError
 from pants.base.exceptions import BuildConfigurationError
 from pants.base.target import Target
 from pants.goal.task_registrar import TaskRegistrar
 from pants.goal.goal import Goal
+from pkg_resources import yield_lines, working_set, Distribution, WorkingSet, EmptyProvider, VersionConflict
+
+
+class MockMetadata(EmptyProvider):
+  def __init__(self, metadata):
+    self.metadata = metadata
+
+  def has_metadata(self, name):
+    return name in self.metadata
+
+  def get_metadata(self, name):
+    return self.metadata[name]
+
+  def get_metadata_lines(self, name):
+    return yield_lines(self.get_metadata(name))
 
 
 class LoaderTest(unittest.TestCase):
   def setUp(self):
     self.build_configuration = BuildConfiguration()
+    self.working_set = WorkingSet()
+    for entry in working_set.entries:
+      self.working_set.add_entry(entry)
 
   def tearDown(self):
     Goal.clear()
@@ -105,3 +123,111 @@ def test_load_invalid_module(self):
     with self.create_register(module_name='register2') as backend_package:
       with self.assertRaises(BuildConfigurationError):
         load_backend(self.build_configuration, backend_package)
+
+  def test_load_missing_plugin(self):
+    with self.assertRaises(PluginNotFound):
+      self.load_plugins(['Foobar'])
+
+
+  def get_mock_plugin(self, name, version, reg=None, alias=None, after=None):
+    """Make a fake Distribution (optionally with entry points)
+
+    Note the entry points do not actually point to code in the returned distribution --
+    the distribution does not even have a location and does not contain any code, just metadata.
+
+    A module is synthesized on the fly and installed into sys.modules under a random name.
+    If optional entry point callables are provided, those are added as methods to the module and
+    their name (foo/bar/baz in fake module) is added as the requested entry point to the mocked
+    metadata added to the returned dist.
+
+    :param str name: project_name for distribution (see pkg_resources)
+    :param str version: version for distribution (see pkg_resources)
+    :param callable reg: Optional callable for goal registration entry point
+    :param callable alias: Optional callable for build_file_aliases entry point
+    :param callable after: Optional callable for load_after list entry point
+    """
+
+    plugin_pkg = b'demoplugin{0}'.format(uuid.uuid4().hex)
+    pkg = types.ModuleType(plugin_pkg)
+    sys.modules[plugin_pkg] = pkg
+    module_name = b'{0}.{1}'.format(plugin_pkg, 'demo')
+    plugin = types.ModuleType(module_name)
+    setattr(pkg, 'demo', plugin)
+    sys.modules[module_name] = plugin
+
+    metadata = {}
+    entry_lines = []
+
+    if reg is not None:
+      setattr(plugin, 'foo', reg)
+      entry_lines.append('register_goals = {}:foo\n'.format(module_name))
+
+    if alias is not None:
+      setattr(plugin, 'bar', alias)
+      entry_lines.append('build_file_aliases = {}:bar\n'.format(module_name))
+
+    if after is not None:
+      setattr(plugin, 'baz', after)
+      entry_lines.append('load_after = {}:baz\n'.format(module_name))
+
+    if entry_lines:
+      entry_data = '[pantsbuild.plugin]\n{}\n'.format('\n'.join(entry_lines))
+      metadata = {'entry_points.txt': entry_data}
+
+    return Distribution(project_name=name, version=version, metadata=MockMetadata(metadata))
+
+  def load_plugins(self, plugins):
+    load_plugins(self.build_configuration, plugins, load_from=self.working_set)
+
+  def test_plugin_load_and_order(self):
+    d1 = self.get_mock_plugin('demo1', '0.0.1', after=lambda: ['demo2'])
+    d2 = self.get_mock_plugin('demo2', '0.0.3')
+    self.working_set.add(d1)
+
+    # Attempting to load 'demo1' then 'demo2' should fail as 'demo1' requires 'after'=['demo2'].
+    with self.assertRaises(PluginLoadOrderError):
+      self.load_plugins(['demo1', 'demo2'])
+
+    # Attempting to load 'demo2' first should fail as it is not (yet) installed.
+    with self.assertRaises(PluginNotFound):
+      self.load_plugins(['demo2', 'demo1'])
+
+    # Installing demo2 and then loading in correct order should work though.
+    self.working_set.add(d2)
+    self.load_plugins(['demo2>=0.0.2', 'demo1'])
+
+    # But asking for a bad (not installed) version fails.
+    with self.assertRaises(VersionConflict):
+      self.load_plugins(['demo2>=0.0.5'])
+
+  def test_plugin_installs_goal(self):
+    def reg_goal():
+      Goal.by_name('plugindemo').install(TaskRegistrar('foo', lambda: 1))
+    self.working_set.add(self.get_mock_plugin('regdemo', '0.0.1', reg=reg_goal))
+
+    # Start without the custom goal.
+    self.assertEqual(0, len(Goal.by_name('plugindemo').ordered_task_names()))
+
+    # Load plugin which registers custom goal.
+    self.load_plugins(['regdemo'])
+
+    # Now the custom goal exists.
+    self.assertEqual(1, len(Goal.by_name('plugindemo').ordered_task_names()))
+    self.assertEqual('foo', Goal.by_name('plugindemo').ordered_task_names()[0])
+
+  def test_plugin_installs_alias(self):
+    def reg_alias():
+      return BuildFileAliases.create(targets={'pluginalias': Target}, objects={'FROMPLUGIN': 100})
+    self.working_set.add(self.get_mock_plugin('aliasdemo', '0.0.1', alias=reg_alias))
+
+    # Start with no aliases.
+    self.assert_empty_aliases()
+
+    # Now load the plugin which defines aliases.
+    self.load_plugins(['aliasdemo'])
+
+    # Aliases now exist.
+    registered_aliases = self.build_configuration.registered_aliases()
+    self.assertEqual(Target, registered_aliases.targets['pluginalias'])
+    self.assertEqual(100, registered_aliases.objects['FROMPLUGIN'])
+

tests/python/pants_test/jvm/jvm_tool_task_test_base.py

@@ -10,7 +10,7 @@
 import shutil
 
 from pants.backend.jvm.tasks.bootstrap_jvm_tools import BootstrapJvmTools
-from pants.base.extension_loader import load_build_configuration_from_source
+from pants.base.extension_loader import load_plugins_and_backends
 from pants.util.dirutil import safe_mkdir, safe_mkdtemp, safe_walk
 from pants_test.task_test_base import TaskTestBase
 
@@ -20,7 +20,7 @@ class JvmToolTaskTestBase(TaskTestBase):
 
   @property
   def alias_groups(self):
-    return load_build_configuration_from_source().registered_aliases()
+    return load_plugins_and_backends().registered_aliases()
 
   def setUp(self):
     real_config = self.config()

tests/python/pants_test/test_utf8_header.py

@@ -13,7 +13,7 @@
 from pants.base.build_file_parser import BuildFileParser
 from pants.base.build_graph import BuildGraph
 from pants.base.config import Config
-from pants.base.extension_loader import load_build_configuration_from_source
+from pants.base.extension_loader import load_plugins_and_backends
 
 
 class Utf8HeaderTest(unittest.TestCase):
@@ -25,7 +25,7 @@ def test_file_have_coding_utf8(self):
 
     config = Config.load()
     backend_packages = config.getlist('backends', 'packages')
-    build_configuration = load_build_configuration_from_source(backend_packages)
+    build_configuration = load_plugins_and_backends(backends=backend_packages)
     build_file_parser = BuildFileParser(root_dir=get_buildroot(),
                                         build_configuration=build_configuration)
     address_mapper = BuildFileAddressMapper(build_file_parser)