Move ivy/coursier link farms under versioned task directories (#6686)

### Problem

As described in #6679, the ivy and coursier link farms are not versioned along with the tasks that consume them. This had the effect of breaking backwards compatibility when those links converted from being symlinks to hardlinks in #6246.

### Solution

Move the "task versioned workdir" logic from `CacheManager` to `Task`, to allow it to be used in more places, and then use it for both `IvyTaskMixin` and `CoursierResolve`.

### Result

Rather than having a "global" link farm at `.pants.d/{ivy,coursier}`, the installed instances of tasks extending `IvyTaskMixin` or `CoursierResolve` will each have their own independent, versioned link farms.

Fixes #6679.

src/python/pants/backend/jvm/ivy_utils.py

@@ -41,7 +41,7 @@ class IvyResolutionStep(object):
   # It also specifies the abstract methods that define the components of resolution steps.
 
   def __init__(self, confs, hash_name, pinned_artifacts, soft_excludes, ivy_resolution_cache_dir,
-               ivy_repository_cache_dir, global_ivy_workdir):
+               ivy_repository_cache_dir, ivy_workdir):
     """
     :param confs: A tuple of string ivy confs to resolve for.
     :param hash_name: A unique string name for this resolve.
@@ -50,7 +50,7 @@ def __init__(self, confs, hash_name, pinned_artifacts, soft_excludes, ivy_resolu
                           fact.
     :param ivy_repository_cache_dir: The cache directory used by Ivy for repository cache data.
     :param ivy_resolution_cache_dir: The cache directory used by Ivy for resolution cache data.
-    :param global_ivy_workdir: The workdir that all ivy outputs live in.
+    :param ivy_workdir: A task-specific workdir that all ivy outputs live in.
     """
 
     self.confs = confs
@@ -60,7 +60,7 @@ def __init__(self, confs, hash_name, pinned_artifacts, soft_excludes, ivy_resolu
 
     self.ivy_repository_cache_dir = ivy_repository_cache_dir
     self.ivy_resolution_cache_dir = ivy_resolution_cache_dir
-    self.global_ivy_workdir = global_ivy_workdir
+    self.ivy_workdir = ivy_workdir
 
     self.workdir_reports_by_conf = {c: self.resolve_report_path(c) for c in confs}
 
@@ -83,7 +83,7 @@ def exec_and_load(self, executor, extra_args, targets, jvm_options, workunit_nam
 
   @property
   def workdir(self):
-    return os.path.join(self.global_ivy_workdir, self.hash_name)
+    return os.path.join(self.ivy_workdir, self.hash_name)
 
   @property
   def hardlink_classpath_filename(self):
@@ -99,7 +99,7 @@ def frozen_resolve_file(self):
 
   @property
   def hardlink_dir(self):
-    return os.path.join(self.global_ivy_workdir, 'jars')
+    return os.path.join(self.ivy_workdir, 'jars')
 
   @abstractmethod
   def ivy_xml_path(self):
@@ -174,7 +174,7 @@ def exec_and_load(self, executor, extra_args, targets, jvm_options, workunit_nam
 
     if not result.all_linked_artifacts_exist():
       raise IvyResolveMappingError(
-        'Some artifacts were not linked to {} for {}'.format(self.global_ivy_workdir,
+        'Some artifacts were not linked to {} for {}'.format(self.ivy_workdir,
                                                              result))
     return result
 
@@ -240,7 +240,7 @@ def exec_and_load(self, executor, extra_args, targets, jvm_options,
 
     if not result.all_linked_artifacts_exist():
       raise IvyResolveMappingError(
-        'Some artifacts were not linked to {} for {}'.format(self.global_ivy_workdir,
+        'Some artifacts were not linked to {} for {}'.format(self.ivy_workdir,
                                                              result))
 
     frozen_resolutions_by_conf = result.get_frozen_resolutions_by_conf(targets)

src/python/pants/backend/jvm/tasks/coursier_resolve.py

@@ -143,11 +143,10 @@ def resolve(self, targets, compile_classpath, sources, javadoc):
         if not invalidation_check.all_vts:
           continue
 
-        pants_workdir = self.get_options().pants_workdir
         resolve_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
 
-        vt_set_results_dir = self._prepare_vts_results_dir(pants_workdir, resolve_vts)
-        pants_jar_base_dir = self._prepare_workdir(pants_workdir)
+        vt_set_results_dir = self._prepare_vts_results_dir(resolve_vts)
+        pants_jar_base_dir = self._prepare_workdir()
         coursier_cache_dir = CoursierSubsystem.global_instance().get_options().cache_dir
 
         # If a report is requested, do not proceed with loading validated result.
@@ -165,7 +164,7 @@ def resolve(self, targets, compile_classpath, sources, javadoc):
                                                                                artifact_set,
                                                                                manager)
 
-        results = self._get_result_from_coursier(jars_to_resolve, global_excludes, pinned_coords, pants_workdir,
+        results = self._get_result_from_coursier(jars_to_resolve, global_excludes, pinned_coords,
                                                  coursier_cache_dir, sources, javadoc)
 
         for conf, result_list in results.items():
@@ -184,24 +183,21 @@ def _override_classifiers_for_conf(self, conf):
     else:
       return None
 
-  def _prepare_vts_results_dir(self, pants_workdir, vts):
+  def _prepare_vts_results_dir(self, vts):
     """
     Given a `VergetTargetSet`, prepare its results dir.
     """
-    vt_set_results_dir = os.path.join(pants_workdir, 'coursier', 'workdir', vts.cache_key.hash)
+    vt_set_results_dir = os.path.join(self.versioned_workdir, 'results', vts.cache_key.hash)
     safe_mkdir(vt_set_results_dir)
     return vt_set_results_dir
 
-  def _prepare_workdir(self, pants_workdir):
-    """
-    Given pants workdir, prepare the location in pants workdir to store all the hardlinks
-    to coursier cache dir.
-    """
-    pants_jar_base_dir = os.path.join(pants_workdir, 'coursier', 'cache')
+  def _prepare_workdir(self):
+    """Prepare the location in our task workdir to store all the hardlinks to coursier cache dir."""
+    pants_jar_base_dir = os.path.join(self.versioned_workdir, 'cache')
     safe_mkdir(pants_jar_base_dir)
     return pants_jar_base_dir
 
-  def _get_result_from_coursier(self, jars_to_resolve, global_excludes, pinned_coords, pants_workdir,
+  def _get_result_from_coursier(self, jars_to_resolve, global_excludes, pinned_coords,
                                 coursier_cache_path, sources, javadoc):
     """
     Calling coursier and return the result per invocation.
@@ -212,7 +208,6 @@ def _get_result_from_coursier(self, jars_to_resolve, global_excludes, pinned_coo
     :param jars_to_resolve: List of `JarDependency`s to resolve
     :param global_excludes: List of `M2Coordinate`s to exclude globally
     :param pinned_coords: List of `M2Coordinate`s that need to be pinned.
-    :param pants_workdir: Pants' workdir
     :param coursier_cache_path: path to where coursier cache is stored.
 
     :return: The aggregation of results by conf from coursier. Each coursier call could return
@@ -264,7 +259,7 @@ def _get_result_from_coursier(self, jars_to_resolve, global_excludes, pinned_coo
                    '--cache', coursier_cache_path
                    ] + repo_args + artifact_types_arg + advanced_options
 
-    coursier_work_temp_dir = os.path.join(pants_workdir, 'tmp')
+    coursier_work_temp_dir = os.path.join(self.versioned_workdir, 'tmp')
     safe_mkdir(coursier_work_temp_dir)
 
     results_by_conf = self._get_default_conf_results(common_args, coursier_jar, global_excludes, jars_to_resolve,
@@ -682,7 +677,7 @@ def register_options(cls, register):
 
   @classmethod
   def implementation_version(cls):
-    return super(CoursierResolve, cls).implementation_version() + [('CoursierResolve', 1)]
+    return super(CoursierResolve, cls).implementation_version() + [('CoursierResolve', 2)]
 
   def execute(self):
     """Resolves the specified confs for the configured targets and returns an iterator over

src/python/pants/backend/jvm/tasks/ivy_task_mixin.py

@@ -101,7 +101,7 @@ def register_options(cls, register):
 
   @classmethod
   def implementation_version(cls):
-    return super(IvyTaskMixin, cls).implementation_version() + [('IvyTaskMixin', 4)]
+    return super(IvyTaskMixin, cls).implementation_version() + [('IvyTaskMixin', 5)]
 
   @memoized_property
   def ivy_repository_cache_dir(self):
@@ -230,8 +230,9 @@ def _ivy_resolve(self,
       resolve_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
 
       resolve_hash_name = resolve_vts.cache_key.hash
-      global_ivy_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir,
-                                        'ivy')
+      # NB: This used to be a global directory, but is now specific to each task that includes
+      # this mixin.
+      ivy_workdir = os.path.join(self.versioned_workdir, 'ivy')
       targets = resolve_vts.targets
 
       fetch = IvyFetchStep(confs,
@@ -240,14 +241,14 @@ def _ivy_resolve(self,
                            self.get_options().soft_excludes,
                            self.ivy_resolution_cache_dir,
                            self.ivy_repository_cache_dir,
-                           global_ivy_workdir)
+                           ivy_workdir)
       resolve = IvyResolveStep(confs,
                                resolve_hash_name,
                                pinned_artifacts,
                                self.get_options().soft_excludes,
                                self.ivy_resolution_cache_dir,
                                self.ivy_repository_cache_dir,
-                               global_ivy_workdir)
+                               ivy_workdir)
 
       return self._perform_resolution(fetch, resolve, executor, extra_args, invalidation_check,
                                       resolve_vts, targets, workunit_name)

src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py

@@ -8,6 +8,7 @@
 import json
 import logging
 import os
+import re
 
 from six import text_type
 
@@ -56,18 +57,20 @@ def stdout_contents(wu):
     return f.read().rstrip()
 
 
-def _create_desandboxify_fn(possible_beginning_paths):
+def _create_desandboxify_fn(possible_path_patterns):
   # Takes a collection of possible canonical prefixes, and returns a function that
   # if it finds a matching prefix, strips the path prior to the prefix and returns it
   # if it doesn't it returns the original path
   # TODO remove this after https://github.com/scalameta/scalameta/issues/1791 is released
+  regexes = [re.compile(p) for p in possible_path_patterns]
   def desandboxify(path):
     if not path:
       return path
-    for p in possible_beginning_paths:
-      if p in path:
-        new_path = path[path.index(p):]
-        return new_path
+    for r in regexes:
+      match = r.search(path)
+      print('>>> matched {} with {} against {}'.format(match, r.pattern, path))
+      if match:
+        return match.group(0)
     return path
   return desandboxify
 
@@ -857,10 +860,9 @@ def _collect_metai_classpath(self, metacp_result, relative_input_paths):
     #    TODO remove this after https://github.com/scalameta/scalameta/issues/1791 is released
     desandboxify = _create_desandboxify_fn(
       [
-        os.path.join(relative_workdir, 'ivy', 'jars'),
-        os.path.join(relative_workdir, 'compile', 'rsc'),
-        os.path.join(relative_workdir, '.jdk'),
-        '.jdk'
+        os.path.join(relative_workdir, 'resolve', 'ivy', '[^/]*', 'ivy', 'jars', '.*'),
+        os.path.join(relative_workdir, 'compile', 'rsc', '.*'),
+        os.path.join(relative_workdir, '\.jdk', '.*'),
       ]
       )
 

src/python/pants/invalidation/cache_manager.py

@@ -272,14 +272,13 @@ def __init__(self,
                fingerprint_strategy=None,
                invalidation_report=None,
                task_name=None,
-               task_version=None,
+               task_version_slug=None,
                artifact_write_callback=lambda _: None):
     """
     :API: public
     """
     self._cache_key_generator = cache_key_generator
     self._task_name = task_name or 'UNKNOWN'
-    self._task_version = task_version or 'Unknown_0'
     self._invalidate_dependents = invalidate_dependents
     self._invalidator = build_invalidator
     self._fingerprint_strategy = fingerprint_strategy
@@ -288,9 +287,7 @@ def __init__(self,
 
     # Create the task-versioned prefix of the results dir, and a stable symlink to it
     # (useful when debugging).
-    task_version_sha = sha1(self._task_version.encode('utf-8')).hexdigest()[:12]
-    self._results_dir_prefix = os.path.join(results_dir_root,
-                                            task_version_sha)
+    self._results_dir_prefix = os.path.join(results_dir_root, task_version_slug)
     safe_mkdir(self._results_dir_prefix)
     stable_prefix = os.path.join(results_dir_root, self._STABLE_DIR_NAME)
     safe_delete(stable_prefix)

src/python/pants/task/task.py

@@ -74,6 +74,11 @@ def implementation_version(cls):
   def implementation_version_str(cls):
     return '.'.join(['_'.join(map(str, x)) for x in cls.implementation_version()])
 
+  @classmethod
+  @memoized_method
+  def implementation_version_slug(cls):
+    return sha1(cls.implementation_version_str().encode('utf-8')).hexdigest()[:12]
+
   @classmethod
   def stable_name(cls):
     """The stable name of this task type.
@@ -247,6 +252,19 @@ def workdir(self):
     safe_mkdir(self._workdir)
     return self._workdir
 
+  @memoized_property
+  def versioned_workdir(self):
+    """The Task.workdir suffixed with a fingerprint of the Task implementation version.
+
+    When choosing whether to store values directly in `self.workdir` or below it in
+    the directory returned by this property, you should generally prefer this value.
+
+    :API: public
+    """
+    versioned_workdir = os.path.join(self.workdir, self.implementation_version_slug())
+    safe_mkdir(versioned_workdir)
+    return versioned_workdir
+
   def _options_fingerprint(self, scope):
     options_hasher = sha1()
     options_hasher.update(scope.encode('utf-8'))
@@ -468,7 +486,7 @@ def _do_invalidation_check(self,
                                              fingerprint_strategy=fingerprint_strategy,
                                              invalidation_report=self.context.invalidation_report,
                                              task_name=self._task_name,
-                                             task_version=self.implementation_version_str(),
+                                             task_version_slug=self.implementation_version_slug(),
                                              artifact_write_callback=self.maybe_write_artifact)
 
     # If this Task's execution has been forced, invalidate all our target fingerprints.

testprojects/src/java/org/pantsbuild/testproject/deployexcludes/BUILD

@@ -1,17 +1,22 @@
 # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-jvm_binary(
-  main='org.pantsbuild.testproject.deployexcludes.DeployExcludesMain',
-  dependencies=[
-    ':lib',
-  ],
-  deploy_excludes=[
-    exclude(org='com.google.guava', name='guava'),
-  ],
-)
+def jvm_excludes_binary(name, excludes=True):
+  deploy_excludes = [exclude(org='com.google.guava', name='guava')] if excludes else []
+  jvm_binary(
+    name=name,
+    main='org.pantsbuild.testproject.deployexcludes.DeployExcludesMain',
+    dependencies=[
+      ':lib',
+    ],
+    deploy_excludes=deploy_excludes,
+  )
+
+jvm_excludes_binary('deployexcludes', excludes=True)
+jvm_excludes_binary('nodeployexcludes', excludes=False)
 
-java_library(name='lib',
+java_library(
+  name='lib',
   sources=['DeployExcludesMain.java'],
   dependencies=[
     '3rdparty:guava',

tests/python/pants_test/backend/jvm/tasks/BUILD

@@ -80,6 +80,7 @@ python_tests(
     'src/python/pants/util:dirutil',
     'tests/python/pants_test:int-test',
   ],
+  timeout=180,
   tags = {'integration'},
 )
 
@@ -319,6 +320,7 @@ python_tests(
     'src/python/pants/util:contextutil',
     'tests/python/pants_test:int-test',
   ],
+  timeout = 120,
   tags = {'integration'},
 )
 

tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile.py

@@ -115,7 +115,7 @@ def test_metacp_job_scheduled_for_jar_library(self):
 
   def test_desandbox_fn(self):
     # TODO remove this after https://github.com/scalameta/scalameta/issues/1791 is released
-    desandbox = _create_desandboxify_fn(['.pants.d/cool/beans', '.pants.d/c/r/c'])
+    desandbox = _create_desandboxify_fn(['.pants.d/cool/beans.*', '.pants.d/c/r/c/.*'])
     self.assertEqual(desandbox('/some/path/.pants.d/cool/beans'), '.pants.d/cool/beans')
     self.assertEqual(desandbox('/some/path/.pants.d/c/r/c/beans'), '.pants.d/c/r/c/beans')
     self.assertEqual(desandbox(