[Hermetic zinc compile] Memoize scalac classpath snapshots (pantsbuil…

…d#6491)

* [Hermetic zinc compile] Memoize scalac classpath snapshots

* apply comments from @illicitonion in PR

* apply comments by @stuhood from PR

* add missing parameter

* replace old method call to new one

* fix dumb error and prevent _memoized_scalac_classpath from returning None

src/python/pants/backend/jvm/subsystems/scala_platform.py

@@ -9,10 +9,15 @@
 from pants.backend.jvm.subsystems.jvm_tool_mixin import JvmToolMixin
 from pants.backend.jvm.subsystems.zinc_language_mixin import ZincLanguageMixin
 from pants.backend.jvm.targets.jar_library import JarLibrary
+from pants.backend.jvm.tasks.classpath_entry import ClasspathEntry
+from pants.base.build_environment import get_buildroot
 from pants.build_graph.address import Address
 from pants.build_graph.injectables_mixin import InjectablesMixin
+from pants.engine.fs import PathGlobs, PathGlobsAndRoot
 from pants.java.jar.jar_dependency import JarDependency
 from pants.subsystem.subsystem import Subsystem
+from pants.util.dirutil import fast_relpath
+from pants.util.memo import memoized_method
 
 
 # full_version - the full scala version to use.
@@ -132,17 +137,28 @@ def register_custom_tool(key):
     register_custom_tool('scala-repl')
     register_custom_tool('scalastyle')
 
-  def _tool_classpath(self, tool, products):
+  def _tool_classpath(self, tool, products, scheduler):
     """Return the proper classpath based on products and scala version."""
-    return self.tool_classpath_from_products(products,
-                                             self.versioned_tool_name(tool, self.version),
-                                             scope=self.options_scope)
-
-  def compiler_classpath(self, products):
-    return self._tool_classpath('scalac', products)
-
-  def style_classpath(self, products):
-    return self._tool_classpath('scalastyle', products)
+    classpath = self.tool_classpath_from_products(products,
+                                                  self.versioned_tool_name(tool, self.version),
+                                                  scope=self.options_scope)
+    classpath = tuple(fast_relpath(c, get_buildroot()) for c in classpath)
+
+    return self._memoized_scalac_classpath(classpath, scheduler)
+
+  @memoized_method
+  def _memoized_scalac_classpath(self, scala_path, scheduler):
+    snapshots = scheduler.capture_snapshots(tuple(PathGlobsAndRoot(PathGlobs([path]), get_buildroot()) for path in scala_path))
+    return [ClasspathEntry(path, snapshot) for path, snapshot in list(zip(scala_path, snapshots))]
+
+  def compiler_classpath_entries(self, products, scheduler):
+    """Returns classpath entries for the scalac tool."""
+    return self._tool_classpath('scalac', products, scheduler)
+
+  def style_classpath(self, products, scheduler):
+    """Returns classpath as paths for scalastyle."""
+    classpath_entries = self._tool_classpath('scalastyle', products, scheduler)
+    return [classpath_entry.path for classpath_entry in classpath_entries]
 
   @property
   def version(self):

src/python/pants/backend/jvm/tasks/jvm_compile/zinc/zinc_compile.py

@@ -30,7 +30,7 @@
 from pants.base.exceptions import TaskError
 from pants.base.hash_utils import hash_file
 from pants.base.workunit import WorkUnitLabel
-from pants.engine.fs import DirectoryToMaterialize, PathGlobs, PathGlobsAndRoot
+from pants.engine.fs import DirectoryToMaterialize
 from pants.engine.isolated_process import ExecuteProcessRequest
 from pants.java.distribution.distribution import DistributionLocator
 from pants.util.contextutil import open_zip
@@ -261,8 +261,10 @@ def javac_classpath(self):
     # the JDK it was invoked with.
     return Java.global_javac_classpath(self.context.products)
 
-  def scalac_classpath(self):
-    return ScalaPlatform.global_instance().compiler_classpath(self.context.products)
+  def scalac_classpath_entries(self):
+    """Returns classpath entries for the scalac classpath."""
+    return ScalaPlatform.global_instance().compiler_classpath_entries(
+      self.context.products, self.context._scheduler)
 
   def write_extra_resources(self, compile_context):
     """Override write_extra_resources to produce plugin and annotation processor files."""
@@ -299,16 +301,18 @@ def relative_to_exec_root(path):
       # TODO: Support workdirs not nested under buildroot by path-rewriting.
       return fast_relpath(path, get_buildroot())
 
-    scala_path = self.scalac_classpath()
     classes_dir = ctx.classes_dir
     analysis_cache = ctx.analysis_file
 
-    scala_path = tuple(relative_to_exec_root(c) for c in scala_path)
     analysis_cache = relative_to_exec_root(analysis_cache)
     classes_dir = relative_to_exec_root(classes_dir)
     # TODO: Have these produced correctly, rather than having to relativize them here
     relative_classpath = tuple(relative_to_exec_root(c) for c in absolute_classpath)
 
+    # list of classpath entries
+    scalac_classpath_entries = self.scalac_classpath_entries()
+    scala_path = [classpath_entry.path for classpath_entry in scalac_classpath_entries]
+
     zinc_args = []
     zinc_args.extend([
       '-log-level', self.get_options().level,
@@ -411,15 +415,9 @@ def relative_to_exec_root(path):
               "execution".format(dep)
             )
 
-      if scala_path:
-        # TODO: ScalaPlatform._tool_classpath should capture this and memoize it.
-        # See https://github.com/pantsbuild/pants/issues/6435
-        snapshots.append(
-          self.context._scheduler.capture_snapshots((PathGlobsAndRoot(
-            PathGlobs(scala_path),
-            get_buildroot(),
-          ),))[0]
-        )
+      snapshots.extend(
+        classpath_entry.directory_digest for classpath_entry in scalac_classpath_entries
+      )
 
       merged_input_digest = self.context._scheduler.merge_directories(
         tuple(s.directory_digest for s in (snapshots)) + directory_digests

src/python/pants/backend/jvm/tasks/scaladoc_gen.py

@@ -48,7 +48,8 @@ def create_scaladoc_command(self, classpath, gendir, *targets):
       return None
 
     scala_platform = ScalaPlatform.global_instance()
-    tool_classpath = scala_platform.compiler_classpath(self.context.products)
+    tool_classpath = [cp_entry.path for cp_entry in scala_platform.compiler_classpath_entries(
+      self.context.products, self.context._scheduler)]
 
     args = ['-usejavacp',
             '-classpath', ':'.join(classpath),

src/python/pants/backend/jvm/tasks/scalastyle.py

@@ -145,7 +145,7 @@ def call(srcs):
           def to_java_boolean(x):
             return str(x).lower()
 
-          cp = ScalaPlatform.global_instance().style_classpath(self.context.products)
+          cp = ScalaPlatform.global_instance().style_classpath(self.context.products, self.context._scheduler)
           scalastyle_args = [
             '-c', scalastyle_config,
             '-v', to_java_boolean(scalastyle_verbose),