Added support for the ivy cache being under a symlink'ed dir

Addresses issue from discussion in https://groups.google.com/forum/#!topic/pants-devel/dhnRICX-I3g

Testing Done:
CI running at: https://travis-ci.org/pantsbuild/pants/builds/60273133

Besides adding the unit test, used the repro case from https://rbcommons.com/s/twitter/r/2085/ to compile a target after moving the cache to a symlink:

```
mkdir ~/realcachedir
ln -s ~/realcachedir ~/symlinkcachedir
```

In `pants.ini` added:

```
[ivy]
cache_dir: /Users/Zundel/symlinkcachedir
```

Then compiled something that required ivy resolution.

```
./pants compile examples/src/java/org/pantsbuild/example/antlr3
```

Bugs closed: 1458

Reviewed at https://rbcommons.com/s/twitter/r/2129/

src/python/pants/backend/jvm/ivy_utils.py

@@ -181,43 +181,74 @@ def cachepath(path):
       with safe_open(path, 'r') as cp:
         yield (path.strip() for path in cp.read().split(os.pathsep) if path.strip())
 
-  @staticmethod
-  def symlink_cachepath(ivy_cache_dir, inpath, symlink_dir, outpath, existing_symlink_map):
+  @classmethod
+  def _find_new_symlinks(cls, existing_symlink_path, updated_symlink_path):
+    """Find the difference between the existing and updated symlink path.
+
+    :param existing_symlink_path: map from path : symlink
+    :param updated_symlink_path: map from path : symlink after new resolve
+    :return: the portion of updated_symlink_path that is not found in existing_symlink_path.
+    """
+    diff_map = OrderedDict()
+    for key, value in updated_symlink_path.iteritems():
+      if not key in existing_symlink_path:
+        diff_map[key] = value
+    return diff_map
+
+  @classmethod
+  def symlink_cachepath(cls, ivy_cache_dir, inpath, symlink_dir, outpath, existing_symlink_map):
     """Symlinks all paths listed in inpath that are under ivy_cache_dir into symlink_dir.
 
     If there is an existing symlink for a file under inpath, it is used rather than creating
     a new symlink. Preserves all other paths. Writes the resulting paths to outpath.
     Returns a map of path -> symlink to that path.
     """
     safe_mkdir(symlink_dir)
+    # The ivy_cache_dir might itself be a symlink. In this case, ivy may return paths that
+    # reference the realpath of the .jar file after it is resolved in the cache dir. To handle
+    # this case, add both the symlink'ed path and the realpath to the jar to the symlink map.
+    real_ivy_cache_dir = os.path.realpath(ivy_cache_dir)
+    updated_symlink_map = OrderedDict()
     with safe_open(inpath, 'r') as infile:
-      paths = filter(None, infile.read().strip().split(os.pathsep))
-    new_paths = []
+      inpaths = filter(None, infile.read().strip().split(os.pathsep))
+      paths = OrderedSet()
+      for path in inpaths:
+        paths.add(path)
+        realpath = os.path.realpath(path)
+        if path != realpath:
+          paths.add(realpath)
+          if realpath.startswith(real_ivy_cache_dir):
+            paths.add(os.path.join(ivy_cache_dir, realpath[len(real_ivy_cache_dir)+1:]))
+
     for path in paths:
-      if not path.startswith(ivy_cache_dir):
-        new_paths.append(path)
+      if path.startswith(ivy_cache_dir):
+        updated_symlink_map[path] = os.path.join(symlink_dir, os.path.relpath(path, ivy_cache_dir))
+      elif path.startswith(real_ivy_cache_dir):
+        updated_symlink_map[path] = os.path.join(symlink_dir, os.path.relpath(path, real_ivy_cache_dir))
+      else:
+        # This path is outside the cache. We won't symlink it.
+        updated_symlink_map[path] = path
+
+    # Create symlinks for paths in the ivy cache dir that we haven't seen before.
+    new_symlinks = cls._find_new_symlinks(existing_symlink_map, updated_symlink_map)
+
+    for path, symlink in new_symlinks.iteritems():
+      if path == symlink:
+        # Skip paths that aren't going to be symlinked.
         continue
-      if path in existing_symlink_map:
-        new_paths.append(existing_symlink_map[path])
-        continue
-      symlink = os.path.join(symlink_dir, os.path.relpath(path, ivy_cache_dir))
-      try:
-        os.makedirs(os.path.dirname(symlink))
-      except OSError as e:
-        if e.errno != errno.EEXIST:
-          raise
-      # Note: The try blocks cannot be combined. It may be that the dir exists but the link doesn't.
+      safe_mkdir(os.path.dirname(symlink))
       try:
         os.symlink(path, symlink)
       except OSError as e:
         # We don't delete and recreate the symlink, as this may break concurrently executing code.
         if e.errno != errno.EEXIST:
           raise
-      new_paths.append(symlink)
+
+    # (re)create the classpath with all of the paths
     with safe_open(outpath, 'w') as outfile:
-      outfile.write(':'.join(new_paths))
-    symlink_map = dict(zip(paths, new_paths))
-    return symlink_map
+      outfile.write(':'.join(OrderedSet(updated_symlink_map.values())))
+
+    return dict(updated_symlink_map)
 
   @staticmethod
   def identify(targets):

src/python/pants/backend/jvm/tasks/ivy_resolve.py

@@ -146,7 +146,7 @@ def execute(self):
         # Add the artifacts from each dependency module.
         artifact_paths = []
         for artifact in ivy_info.get_artifacts_for_jar_library(target, memo=ivy_jar_memo):
-          artifact_paths.append(symlink_map[artifact.path])
+          artifact_paths.append(symlink_map[os.path.realpath(artifact.path)])
         compile_classpath.add_for_target(target, [(conf, entry) for entry in artifact_paths])
 
     if self._report:

tests/python/pants_test/backend/jvm/tasks/BUILD

@@ -7,6 +7,7 @@ target(
     ':bootstrap_jvm_tools',
     ':checkstyle',
     ':ivy_imports',
+    ':ivy_utils',
     ':junit_run',
     ':scalastyle',
     ':unpack_jars',
@@ -61,10 +62,25 @@ python_tests(
     'src/python/pants/backend/jvm/targets:jvm',
     'src/python/pants/backend/jvm/targets:scala',
     'src/python/pants/backend/jvm/tasks:ivy_resolve',
+    'src/python/pants/util:contextutil',
     'tests/python/pants_test/jvm:nailgun_task_test_base',
   ]
 )
 
+python_tests(
+  name = 'ivy_utils',
+  sources = ['test_ivy_utils.py'],
+  dependencies = [
+    '3rdparty/python:mock',
+    'src/python/pants/backend/core:plugin',
+    'src/python/pants/backend/jvm:plugin',
+    'src/python/pants/backend/jvm:ivy_utils',
+    'src/python/pants/util:contextutil',
+    'tests/python/pants_test:base_test',
+    'tests/python/pants_test/base:context_utils',
+  ]
+)
+
 python_tests(
   name = 'junit_run',
   sources = ['test_junit_run.py'],

tests/python/pants_test/backend/jvm/tasks/test_ivy_resolve.py

@@ -18,6 +18,7 @@
 from pants.backend.jvm.targets.java_library import JavaLibrary
 from pants.backend.jvm.targets.scala_library import ScalaLibrary
 from pants.backend.jvm.tasks.ivy_resolve import IvyResolve
+from pants.util.contextutil import temporary_dir
 from pants_test.jvm.jvm_tool_task_test_base import JvmToolTaskTestBase
 
 
@@ -161,3 +162,23 @@ def test_resolve_no_deps(self):
     # Resolve a library with no deps, and confirm that the empty product is created.
     target = self.make_target('//:a', ScalaLibrary)
     self.assertTrue(self.resolve([target]))
+
+  def test_resolve_symkinked_cache(self):
+    """Test to make sure resolve works when --ivy-cache-dir is a symlinked path.
+
+    When ivy returns the path to a resolved jar file, it might be the realpath to the jar file,
+    not the symlink'ed path we are expecting for --ivy-cache-dir.  Make sure that resolve correctly
+    recognizes these as belonging in the cache dir and lookups for either the symlinked cache
+    dir or the realpath to the cache dir are recognized.
+    """
+    with temporary_dir() as realcachedir:
+      with temporary_dir() as symlinkdir:
+        symlink_cache_dir=os.path.join(symlinkdir, 'symlinkedcache')
+        os.symlink(realcachedir, symlink_cache_dir)
+        self.set_options_for_scope('ivy', cache_dir=symlink_cache_dir)
+
+        dep = JarDependency('commons-lang', 'commons-lang', '2.5')
+        jar_lib = self.make_target('//:a', JarLibrary, jars=[dep])
+        # Confirm that the deps were added to the appropriate targets.
+        compile_classpath = self.resolve([jar_lib])
+        self.assertEquals(1, len(compile_classpath.get_for_target(jar_lib)))

tests/python/pants_test/tasks/test_ivy_utils.py → tests/python/pants_test/backend/jvm/tasks/test_ivy_utils.py

@@ -5,7 +5,7 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
-import logging
+import os
 import xml.etree.ElementTree as ET
 from textwrap import dedent
 
@@ -15,7 +15,7 @@
 from pants.backend.jvm.ivy_utils import IvyModuleRef, IvyUtils
 from pants.backend.jvm.register import build_file_aliases as register_jvm
 from pants.backend.jvm.targets.exclude import Exclude
-from pants.util.contextutil import temporary_file_path
+from pants.util.contextutil import temporary_dir, temporary_file_path
 from pants_test.base_test import BaseTest
 
 
@@ -229,3 +229,72 @@ def find_single(self, elem, xpath):
 
   def assert_attributes(self, elem, **kwargs):
     self.assertEqual(dict(**kwargs), dict(elem.attrib))
+
+  def test_find_new_symlinks(self):
+    map1 = { 'foo' : 'bar'}
+    map2 = { }
+    diff_map = IvyUtils._find_new_symlinks(map1, map2)
+    self.assertEquals({}, diff_map)
+    diff_map = IvyUtils._find_new_symlinks(map2, map1)
+    self.assertEquals({'foo' : 'bar'}, diff_map)
+
+  def test_symlink_cachepath(self):
+    self.maxDiff = None
+    with temporary_dir() as mock_cache_dir:
+      with temporary_dir() as symlink_dir:
+        with temporary_dir() as classpath_dir:
+          input_path = os.path.join(classpath_dir, 'inpath')
+          output_path = os.path.join(classpath_dir, 'classpath')
+          existing_symlink_map = {}
+          foo_path = os.path.join(mock_cache_dir, 'foo.jar')
+          with open(foo_path, 'w') as foo:
+            foo.write("test jar contents")
+
+          with open(input_path, 'w') as inpath:
+            inpath.write(foo_path)
+          result_map = IvyUtils.symlink_cachepath(mock_cache_dir, input_path, symlink_dir,
+                                                  output_path, existing_symlink_map)
+          symlink_foo_path = os.path.join(symlink_dir, 'foo.jar')
+          self.assertEquals(
+            {
+              foo_path : symlink_foo_path,
+              os.path.realpath(foo_path) : symlink_foo_path
+            },
+            result_map)
+          with open(output_path, 'r') as outpath:
+            self.assertEquals(symlink_foo_path, outpath.readline())
+          self.assertTrue(os.path.islink(symlink_foo_path))
+          self.assertTrue(os.path.exists(symlink_foo_path))
+
+          # Now add an additional path to the existing map
+          bar_path = os.path.join(mock_cache_dir, 'bar.jar')
+          with open(bar_path, 'w') as bar:
+            bar.write("test jar contents2")
+          with open(input_path, 'w') as inpath:
+            inpath.write(os.pathsep.join([foo_path, bar_path]))
+          existing_symlink_map = result_map
+          result_map = IvyUtils.symlink_cachepath(mock_cache_dir, input_path, symlink_dir,
+                                                  output_path, existing_symlink_map)
+          symlink_bar_path = os.path.join(symlink_dir, 'bar.jar')
+          self.assertEquals(
+            {
+              foo_path : symlink_foo_path,
+              os.path.realpath(foo_path) : symlink_foo_path,
+              bar_path : symlink_bar_path,
+              os.path.realpath(bar_path) : symlink_bar_path,
+            },
+            result_map)
+          with open(output_path, 'r') as outpath:
+            self.assertEquals(symlink_foo_path + os.pathsep + symlink_bar_path, outpath.readline())
+          self.assertTrue(os.path.islink(symlink_foo_path))
+          self.assertTrue(os.path.exists(symlink_foo_path))
+          self.assertTrue(os.path.islink(symlink_bar_path))
+          self.assertTrue(os.path.exists(symlink_bar_path))
+
+          # Reverse the ordering and make sure order is preserved in the output path
+          with open(input_path, 'w') as inpath:
+            inpath.write(os.pathsep.join([bar_path, foo_path]))
+          IvyUtils.symlink_cachepath(mock_cache_dir, input_path, symlink_dir,
+                                                  output_path, result_map)
+          with open(output_path, 'r') as outpath:
+            self.assertEquals(symlink_bar_path + os.pathsep + symlink_foo_path, outpath.readline())

tests/python/pants_test/tasks/BUILD

@@ -30,7 +30,6 @@ target(
     ':execution_graph',
     ':filter',
     ':group_task',
-    ':ivy_utils',
     ':jar_create',
     ':jar_publish',
     ':jar_task',
@@ -308,20 +307,6 @@ python_tests(
   ],
 )
 
-python_tests(
-  name = 'ivy_utils',
-  sources = ['test_ivy_utils.py'],
-  dependencies = [
-    '3rdparty/python:mock',
-    'src/python/pants/backend/core:plugin',
-    'src/python/pants/backend/jvm:plugin',
-    'src/python/pants/backend/jvm:ivy_utils',
-    'src/python/pants/util:contextutil',
-    'tests/python/pants_test:base_test',
-    'tests/python/pants_test/base:context_utils',
-  ]
-)
-
 python_tests(
   name = 'jar_create',
   sources = ['test_jar_create.py'],