Allow in-repo scalac plugins to have in-repo deps.

scalac plugins are not read from the regular classpath,
so this requires special handling.

javac plugins are read from the regular classpath,
so deps work naturally there.

Note that we don't support external plugins with deps,
as there's no easy way to resolve just those deps
by the time we're figuring out plugins.  However no-dep
plugins seem to be the norm - e.g., SBT doesn't appear to
support plugin deps at all.  To have external plugins
with deps, you presumably need to publish the plugin
as a "fat jar".

Also adds examples to demonstrate javac/scalac plugin deps.

examples/src/java/org/pantsbuild/example/javac/plugin/BUILD

@@ -4,11 +4,16 @@
 javac_plugin(
   name = 'simple_javac_plugin',
   sources = ['SimpleJavacPlugin.java'],
-  dependencies = [],
+  dependencies = [':simple_javac_plugin_helper'],
   classname = 'org.pantsbuild.example.javac.plugin.SimpleJavacPlugin',
   scope='compile',
 )
 
+java_library(
+  name = 'simple_javac_plugin_helper',
+  sources = ['SimpleJavacPluginHelper.java'],
+)
+
 # The plugin will only run on this target if told to via options.
 java_library(
   name = 'global',

examples/src/java/org/pantsbuild/example/javac/plugin/SimpleJavacPlugin.java

@@ -16,7 +16,7 @@ public class SimpleJavacPlugin implements Plugin {
 
   @Override
   public String getName() {
-    return "simple_javac_plugin";
+    return SimpleJavacPluginHelper.getName();
   }
 
   @Override

examples/src/java/org/pantsbuild/example/javac/plugin/SimpleJavacPluginHelper.java

@@ -0,0 +1,14 @@
+// Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+package org.pantsbuild.example.javac.plugin;
+
+
+/**
+ * A trivial helper class, to test that plugin dependencies are handled correctly.
+ */
+public class SimpleJavacPluginHelper {
+  public static String getName() {
+    return "simple_javac_plugin";
+  }
+}

examples/src/scala/org/pantsbuild/example/scalac/plugin/BUILD

@@ -6,10 +6,18 @@ scalac_plugin(
   plugin='simple_scalac_plugin',
   classname='org.pantsbuild.example.scalac.plugin.SimpleScalacPlugin',
   sources=['SimpleScalacPlugin.scala'],
-  dependencies=[':other_simple_scalac_plugin'],
+  dependencies=[
+    ':other_simple_scalac_plugin',
+    ':simple_scalac_plugin_helper'
+  ],
   scope='compile',
 )
 
+scala_library(
+  name = 'simple_scalac_plugin_helper',
+  sources = ['SimpleScalacPluginHelper.scala'],
+)
+
 scalac_plugin(
   name='other_simple_scalac_plugin',
   plugin='other_simple_scalac_plugin',

examples/src/scala/org/pantsbuild/example/scalac/plugin/README.md

@@ -114,4 +114,8 @@ compiling its own `scalac_plugin` target, or any of that target's dependencies.
 To use a plugin on its own code, you must publish it and consume the published plugin
 via `scala-plugin-dep`.
 
+Note also that scalac itself does not support plugins with dependencies in classpath elements
+other than the one the plugin class itself was loaded from.  Therefore if you have a plugin with
+dependencies you must publish it as a "fat jar" and consume that jar. You cannot depend on it
+in-repo.
 

examples/src/scala/org/pantsbuild/example/scalac/plugin/SimpleScalacPlugin.scala

@@ -6,15 +6,16 @@ package org.pantsbuild.example.scalac.plugin
 import tools.nsc.{Global, Phase}
 import tools.nsc.plugins.{Plugin, PluginComponent}
 
+
 /**
  * A very simple plugin that just logs its existence in the pipeline.
  *
  * @param global The compiler instance this plugin is installed in.
  */
 class SimpleScalacPlugin(val global: Global) extends Plugin {
   // Required Plugin boilerplate.
-  val name = "simple_scalac_plugin"
-  val description = "Logs a greeting."
+  val name = SimpleScalacPluginHelper.name
+  val description = "Logs a greeting"
   val components = List[PluginComponent](Component)
 
   var pluginOpts: List[String] = Nil

examples/src/scala/org/pantsbuild/example/scalac/plugin/SimpleScalacPluginHelper.scala

@@ -0,0 +1,12 @@
+// Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+package org.pantsbuild.example.scalac.plugin
+
+
+/**
+ * A trivial helper class, to test that plugin dependencies are handled correctly.
+ */
+object SimpleScalacPluginHelper {
+  val name = "simple_scalac_plugin"
+}

src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py

@@ -318,6 +318,10 @@ def extra_compile_time_classpath_elements(self):
     """
     return []
 
+  def scalac_plugin_classpath_elements(self):
+    """Classpath entries containing scalac plugins."""
+    return []
+
   def write_extra_resources(self, compile_context):
     """Writes any extra, out-of-band resources for a target to its classes directory.
 
@@ -523,7 +527,7 @@ def do_compile(self,
       safe_mkdir(cc.classes_dir)
       self.validate_analysis(cc.analysis_file)
 
-    # Get the classpath generated by upstream JVM tasks and our own prepare_compile().
+    # Get the classpath generated by upstream JVM tasks and our own preparation.
     classpath_products = self.context.products.get_data('runtime_classpath')
 
     extra_compile_time_classpath = self._compute_extra_classpath(

src/python/pants/backend/jvm/tasks/jvm_compile/zinc/zinc_compile.py

@@ -22,6 +22,7 @@
 from pants.backend.jvm.targets.javac_plugin import JavacPlugin
 from pants.backend.jvm.targets.jvm_target import JvmTarget
 from pants.backend.jvm.targets.scalac_plugin import ScalacPlugin
+from pants.backend.jvm.tasks.classpath_util import ClasspathUtil
 from pants.backend.jvm.tasks.jvm_compile.analysis_tools import AnalysisTools
 from pants.backend.jvm.tasks.jvm_compile.jvm_compile import JvmCompile
 from pants.backend.jvm.tasks.jvm_compile.zinc.zinc_analysis import ZincAnalysis
@@ -315,15 +316,17 @@ def compile(self, args, classpath, sources, classes_output_dir, upstream_analysi
     # Search for scalac plugins on the entire classpath, which will allow use of
     # in-repo plugins for scalac (which works naturally for javac).
     # Note that:
-    # - At this point the classpath will already have the extra_compile_time_classpath_elements()
-    #   appended to it, so those will also get searched here.
+    # - We also search in the extra scalac plugin dependencies, if specified.
     # - In scala 2.11 and up, the plugin's classpath element can be a dir, but for 2.10 it must be
     #   a jar.  So in-repo plugins will only work with 2.10 if --use-classpath-jars is true.
     # - We exclude our own classes_output_dir, because if we're a plugin ourselves, then our
     #   classes_output_dir doesn't have scalac-plugin.xml yet, and we don't want that fact to get
     #   memoized (which in practice will only happen if this plugin uses some other plugin, thus
     #   triggering the plugin search mechanism, which does the memoizing).
-    scalac_plugin_search_classpath = set(classpath) - {classes_output_dir}
+    scalac_plugin_search_classpath = (
+      (set(classpath) | set(self.scalac_plugin_classpath_elements())) -
+      {classes_output_dir}
+    )
     zinc_args.extend(self._scalac_plugin_args(scalac_plugin_map, scalac_plugin_search_classpath))
     if upstream_analysis:
       zinc_args.extend(['-analysis-map',
@@ -406,22 +409,37 @@ def _javac_plugin_args(cls, javac_plugin_map):
       ret.append('-C-Xplugin:{} {}'.format(plugin, ' '.join(args)))
     return ret
 
-  @classmethod
-  def _scalac_plugin_args(cls, scalac_plugin_map, classpath):
+  def _scalac_plugin_args(self, scalac_plugin_map, classpath):
     if not scalac_plugin_map:
       return []
 
-    plugin_jar_map = cls._find_scalac_plugins(scalac_plugin_map.keys(), classpath)
+    plugin_jar_map = self._find_scalac_plugins(scalac_plugin_map.keys(), classpath)
     ret = []
-    for name, jar in plugin_jar_map.items():
-      ret.append('-S-Xplugin:{}'.format(jar))
+    for name, cp_entries in plugin_jar_map.items():
+      # Note that the first element in cp_entries is the one containing the plugin's metadata,
+      # meaning that this is the plugin that will be loaded, even if there happen to be other
+      # plugins in the list of entries (e.g., because this plugin depends on another plugin).
+      ret.append('-S-Xplugin:{}'.format(':'.join(cp_entries)))
       for arg in scalac_plugin_map[name]:
         ret.append('-S-P:{}:{}'.format(name, arg))
     return ret
 
-  @classmethod
-  def _find_scalac_plugins(cls, scalac_plugins, classpath):
-    """Returns a map from plugin name to plugin jar/dir."""
+  def _find_scalac_plugins(self, scalac_plugins, classpath):
+    """Returns a map from plugin name to list of plugin classpath entries.
+
+    The first entry in each list is the classpath entry containing the plugin metadata.
+    The rest are the internal transitive deps of the plugin.
+
+    This allows us to have in-repo plugins with dependencies (unlike javac, scalac doesn't load
+    plugins or their deps from the regular classpath, so we have to provide these entries
+    separately, in the -Xplugin: flag).
+
+    Note that we don't currently support external plugins with dependencies, as we can't know which
+    external classpath elements are required, and we'd have to put the entire external classpath
+    on each -Xplugin: flag, which seems excessive.
+    Instead, external plugins should be published as "fat jars" (which appears to be the norm,
+    since SBT doesn't support plugins with dependencies).
+    """
     # Allow multiple flags and also comma-separated values in a single flag.
     plugin_names = set([p for val in scalac_plugins for p in val.split(',')])
     if not plugin_names:
@@ -430,17 +448,26 @@ def _find_scalac_plugins(cls, scalac_plugins, classpath):
     active_plugins = {}
     buildroot = get_buildroot()
 
+    def expand():
+      plugin_target_closure = self._plugin_targets('scalac').get(name, [])
+      return ClasspathUtil.internal_classpath(plugin_target_closure,
+                                              self.context.products.get_data('runtime_classpath'),
+                                              self._confs)
+    cp_product = self.context.products.get_data('runtime_classpath')
     for classpath_element in classpath:
-      name = cls._maybe_get_plugin_name(classpath_element)
+      name = self._maybe_get_plugin_name(classpath_element)
       if name in plugin_names:
+        plugin_target_closure = self._plugin_targets('scalac').get(name, [])
         # It's important to use relative paths, as the compiler flags get embedded in the zinc
         # analysis file, and we port those between systems via the artifact cache.
-        rel_classpath_element = os.path.relpath(classpath_element, buildroot)
+        rel_classpath_elements = [
+          os.path.relpath(cpe, buildroot) for cpe in
+          ClasspathUtil.internal_classpath(plugin_target_closure, cp_product, self._confs)]
         # Some classpath elements may be repeated, so we allow for that here.
-        if active_plugins.get(name, rel_classpath_element) != rel_classpath_element:
+        if active_plugins.get(name, rel_classpath_elements) != rel_classpath_elements:
           raise TaskError('Plugin {} defined in {} and in {}'.format(name, active_plugins[name],
                                                                      classpath_element))
-        active_plugins[name] = rel_classpath_element
+        active_plugins[name] = rel_classpath_elements
         if len(active_plugins) == len(plugin_names):
           # We've found all the plugins, so return now to spare us from processing
           # of the rest of the classpath for no reason.
@@ -511,8 +538,15 @@ def product_types(cls):
     return ['runtime_classpath', 'classes_by_source', 'product_deps_by_src', 'zinc_args']
 
   def extra_compile_time_classpath_elements(self):
-    """Classpath entries containing plugins."""
-    return self.tool_classpath('javac-plugin-dep') + self.tool_classpath('scalac-plugin-dep')
+    # javac plugins are loaded from the regular class entries containing javac plugins,
+    # so we can provide them here.
+    # Note that, unlike javac, scalac plugins are not loaded from the regular classpath,
+    # so we don't provide them here.
+    return self.tool_classpath('javac-plugin-dep')
+
+  def scalac_plugin_classpath_elements(self):
+    """Classpath entries containing scalac plugins."""
+    return self.tool_classpath('scalac-plugin-dep')
 
   def select(self, target):
     # Require that targets are marked for JVM compilation, to differentiate from