Fix the kythe bootclasspath. (#4527)

We must load the Java 9 versions of com.sun.tools.javac
and javax.tools embedded in the Kythe jars, and not the
runtime's versions.

A recent Kythe change exposed this problem. Unclear why it
didn't bite us before.

contrib/kythe/src/python/pants/contrib/kythe/tasks/extract_java.py

@@ -7,7 +7,6 @@
 
 import os
 
-from pants.backend.jvm.subsystems.jvm import JVM
 from pants.backend.jvm.subsystems.shader import Shader
 from pants.backend.jvm.tasks.jvm_tool_task_mixin import JvmToolTaskMixin
 from pants.base.exceptions import TaskError
@@ -26,10 +25,6 @@ def implementation_version(cls):
     # Bump this version to invalidate all past artifacts generated by this task.
     return super(ExtractJava, cls).implementation_version() + [('KytheExtract', 6), ]
 
-  @classmethod
-  def subsystem_dependencies(cls):
-    return super(ExtractJava, cls).subsystem_dependencies() + (JVM.scoped(cls),)
-
   @classmethod
   def product_types(cls):
     # TODO: Support indexpack files?
@@ -58,19 +53,23 @@ def execute(self):
     targets_to_zinc_args = self.context.products.get_data('zinc_args')
 
     with self.invalidated(indexable_targets, invalidate_dependents=True) as invalidation_check:
-      cp = self.tool_classpath('kythe-extractor')
+      extractor_cp = self.tool_classpath('kythe-extractor')
       for vt in invalidation_check.invalid_vts:
         self.context.log.info('Kythe extracting from {}\n'.format(vt.target.address.spec))
         javac_args = self._get_javac_args_from_zinc_args(targets_to_zinc_args[vt.target])
-        jvm_options = list(JVM.scoped_instance(self).get_jvm_options())
+        # Kythe jars embed a copy of Java 9's com.sun.tools.javac and javax.tools, for use on JDK8.
+        # We must put these jars on the bootclasspath, ahead of any others, to ensure that we load
+        # the Java 9 versions, and not the runtime's versions.
+        jvm_options = ['-Xbootclasspath/p:{}'.format(':'.join(extractor_cp))]
+        jvm_options.extend(self.get_options().jvm_options)
         jvm_options.extend([
           '-DKYTHE_CORPUS={}'.format(vt.target.address.spec),
           '-DKYTHE_ROOT_DIRECTORY={}'.format(vt.target.target_base),
           '-DKYTHE_OUTPUT_DIRECTORY={}'.format(vt.results_dir)
         ])
 
         result = self.dist.execute_java(
-          classpath=cp, main=self._KYTHE_EXTRACTOR_MAIN,
+          classpath=extractor_cp, main=self._KYTHE_EXTRACTOR_MAIN,
           jvm_options=jvm_options, args=javac_args, workunit_name='kythe-extract')
         if result != 0:
           raise TaskError('java {main} ... exited non-zero ({result})'.format(

contrib/kythe/src/python/pants/contrib/kythe/tasks/index_java.py

@@ -50,18 +50,24 @@ def entries_file(_vt):
 
     with self.invalidated(indexable_targets, invalidate_dependents=True) as invalidation_check:
       kindex_files = self.context.products.get_data('kindex_files')
-      cp = self.tool_classpath('kythe-indexer')
       vts_to_index = (invalidation_check.all_vts if self.get_options().force
                       else invalidation_check.invalid_vts)
 
+      indexer_cp = self.tool_classpath('kythe-indexer')
+      # Kythe jars embed a copy of Java 9's com.sun.tools.javac and javax.tools, for use on JDK8.
+      # We must put these jars on the bootclasspath, ahead of any others, to ensure that we load
+      # the Java 9 versions, and not the runtime's versions.
+      jvm_options = ['-Xbootclasspath/p:{}'.format(':'.join(indexer_cp))]
+      jvm_options.extend(self.get_options().jvm_options)
+
       for vt in vts_to_index:
         self.context.log.info('Kythe indexing {}'.format(vt.target.address.spec))
         kindex_file = kindex_files.get(vt.target)
         if not kindex_file:
           raise TaskError('No .kindex file found for {}'.format(vt.target.address.spec))
         args = [kindex_file, '--out', entries_file(vt)]
-        result = self.runjava(classpath=cp, main=self._KYTHE_INDEXER_MAIN,
-                              jvm_options=self.get_options().jvm_options,
+        result = self.runjava(classpath=indexer_cp, main=self._KYTHE_INDEXER_MAIN,
+                              jvm_options=jvm_options,
                               args=args, workunit_name='kythe-index',
                               workunit_labels=[WorkUnitLabel.COMPILER])
         if result != 0: