Merge 09790ec into ad7d09d

src/python/pants/backend/jvm/ivy_utils.py

@@ -43,6 +43,14 @@ def add_module(self, module):
     for caller in module.callers:
       self.deps_by_caller[caller].add(module.ref)
 
+  def get_jars_for_ivy_module(self, jar):
+    ref = IvyModuleRef(jar.org, jar.name, jar.rev)
+    deps = OrderedSet()
+    for dep in self.deps_by_caller.get(ref, []):
+      deps.add(dep)
+      deps.update(self.get_jars_for_ivy_module(dep))
+    return deps
+
 
 class IvyUtils(object):
   IVY_TEMPLATE_PACKAGE_NAME = __name__

src/python/pants/backend/jvm/tasks/depmap.py

@@ -9,9 +9,12 @@
 import json
 import os
 
+from twitter.common.collections import OrderedSet
+
 from pants.backend.core.tasks.console_task import ConsoleTask
 from pants.backend.core.targets.dependencies import Dependencies
 from pants.backend.core.targets.resources import Resources
+from pants.backend.jvm.ivy_utils import IvyUtils
 from pants.backend.jvm.targets.jar_dependency import JarDependency
 from pants.backend.jvm.targets.scala_library import ScalaLibrary
 from pants.base.build_environment import get_buildroot
@@ -45,13 +48,6 @@ def _jar_id(jar):
     else:
       return '{0}:{1}'.format(jar.org, jar.name)
 
-  @staticmethod
-  def _address(address):
-    """
-    :type address: pants.base.address.SyntheticAddress
-    """
-    return '{0}:{1}'.format(address.spec_path, address.target_name)
-
   @classmethod
   def setup_parser(cls, option_group, args, mkflag):
     super(Depmap, cls).setup_parser(option_group, args, mkflag)
@@ -99,6 +95,13 @@ def setup_parser(cls, option_group, args, mkflag):
                             dest="depmap_is_formatted",
                             default=True,
                             help='Causes project-info output to be a single line of JSON')
+    option_group.add_option(mkflag("single-module-per-codegen"),
+                            mkflag("single-module-per-codegen", negate=True),
+                            action="store_false",
+                            dest="single_codegen_module",
+                            default=True,
+                            help='Creates one module for all code generated sources per'
+                                ' codegen task')
 
   def __init__(self, *args, **kwargs):
     super(Depmap, self).__init__(*args, **kwargs)
@@ -119,6 +122,10 @@ def __init__(self, *args, **kwargs):
     self.separator = self.context.options.depmap_separator
     self.project_info = self.context.options.depmap_is_project_info
     self.format = self.context.options.depmap_is_formatted
+    self._ivy_utils = IvyUtils(config=self.context.config,
+                               options=self.context.options,
+                               log=self.context.log)
+    self._single_codegen_module = self.context.options.single_codegen_module
 
   def console_output(self, targets):
     if len(self.context.target_roots) == 0:
@@ -157,6 +164,14 @@ def _dep_id(self, dependency):
     else:
       return "%(org)s%(sep)s%(name)s" % params, True
 
+  def _address(self, target):
+    """
+    :type Target: pants.base.Target
+    """
+    if target.is_codegen and self._single_codegen_module:
+      return '{0}:{1}'.format(target.address.spec_path, os.path.basename(target.address.spec_path))
+    return '{0}:{1}'.format(target.address.spec_path, target.address.target_name)
+
   def _output_dependency_tree(self, target):
     def output_dep(dep, indent):
       return "%s%s" % (indent * "  ", dep)
@@ -244,6 +259,7 @@ def output_deps(outputted, dep, parent=None):
   def project_info_output(self, targets):
     targets_map = {}
     resource_target_map = {}
+    ivy_info = self._ivy_utils.parse_xml_report(targets, 'default')
 
     def process_target(current_target):
       """
@@ -253,52 +269,76 @@ def get_target_type(target):
         if target.is_test:
           return Depmap.SourceRootTypes.TEST
         else:
-          if isinstance(target, Resources) and target in resource_target_map and resource_target_map[target].is_test:
+          if (isinstance(target, Resources) and
+              target in resource_target_map and
+              resource_target_map[target].is_test):
             return Depmap.SourceRootTypes.TEST_RESOURCE
           elif isinstance(target, Resources):
             return Depmap.SourceRootTypes.RESOURCE
           else:
             return Depmap.SourceRootTypes.SOURCE
 
-      info = {
-        'targets': [],
-        'libraries': [],
-        'roots': [],
-        'target_type': get_target_type(current_target)
-      }
+      def get_transitive_jars(jar_lib):
+        transitive_jars = OrderedSet()
+        if ivy_info:
+          for jar in jar_lib.jar_dependencies:
+            transitive_jars.update(ivy_info.get_jars_for_ivy_module(jar))
+        return transitive_jars
 
+      info = {}
+      if self._address(current_target) in targets_map:
+        info = targets_map[self._address(current_target)]
+      else:
+        info = {
+          'targets': set(),
+          'libraries': set(),
+          'roots': [],
+          'target_type': get_target_type(current_target)
+        }
+
+      target_libraries = set()
+      if current_target.is_jar_library:
+        target_libraries = get_transitive_jars(current_target)
       for dep in current_target.dependencies:
-        if dep.is_java or dep.is_jar_library or dep.is_jvm or dep.is_scala or dep.is_scalac_plugin:
-          info['targets'].append(self._address(dep.address))
+        info['targets'].add(self._address(dep))
         if dep.is_jar_library:
           for jar in dep.jar_dependencies:
-            info['libraries'].append(self._jar_id(jar))
+            target_libraries.add(jar)
+          # Add all the jars pulled in by this jar_library
+          target_libraries.update(get_transitive_jars(dep))
         if isinstance(dep, Resources):
-          info['targets'].append(self._address(dep.address))
           resource_target_map[dep] = current_target
 
-      java_sources_targets = list(current_target.java_sources) if isinstance(current_target, ScalaLibrary) else list()
+      java_sources_targets = list(current_target.java_sources) if isinstance(current_target,
+                                                                             ScalaLibrary) else []
       """
       :type java_sources_targets:list[pants.base.target.Target]
       """
-
       roots = set(itertools.chain(
         *[self._source_roots_for_target(t) for t in java_sources_targets + [current_target]]
       ))
 
-      info['roots'] = map(lambda (source_root, package_prefix): {
+      info['roots'].extend(map(lambda (source_root, package_prefix): {
         'source_root': source_root,
         'package_prefix': package_prefix
-      }, roots)
-      targets_map[self._address(current_target.address)] = info
+      }, roots))
+
+      info['libraries'].update([self._jar_id(lib) for lib in target_libraries])
+      targets_map[self._address(current_target)] = info
 
     for target in targets:
       process_target(target)
 
+    for target_add in targets_map.keys():
+      info = targets_map[target_add]
+      info['libraries'] = list(info['libraries'])
+      info['targets'] = list(info['targets'])
+
     graph_info = {
       'targets': targets_map,
       'libraries': self._resolve_jars_info()
     }
+
     if self.format:
       return json.dumps(graph_info, indent=4, separators=(',', ': ')).splitlines()
     else:
@@ -323,4 +363,3 @@ def root_package_prefix(source_file):
       source = os.path.dirname(source_file)
       return os.path.join(get_buildroot(), target.target_base, source), source.replace(os.sep, '.')
     return map(root_package_prefix, target.sources_relative_to_source_root())
-

tests/python/pants_test/tasks/BUILD

@@ -74,6 +74,7 @@ python_test_suite(
 python_test_suite(
   name = 'integration',
   dependencies = [
+    ':depmap_integration',
     ':eclipse_integration',
     ':ensime_integration',
     ':idea_integration',
@@ -277,6 +278,16 @@ python_tests(
   ]
 )
 
+python_tests(
+  name = 'depmap_integration',
+  sources = ['test_depmap_integration.py'],
+  dependencies = [
+    'src/python/pants/ivy:ivy',
+    'src/python/pants/util:contextutil',
+    'tests/python/pants_test:int-test',
+  ],
+)
+
 python_tests(
   name = 'detect_duplicates',
   sources = ['test_detect_duplicates.py'],

tests/python/pants_test/tasks/test_depmap_integration.py

@@ -0,0 +1,65 @@
+# coding=utf-8
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (nested_scopes, generators, division, absolute_import, with_statement,
+                        print_function, unicode_literals)
+
+import json
+import os
+
+from pants.util.contextutil import temporary_dir
+
+from pants.ivy.bootstrapper import Bootstrapper
+from pants.base.build_environment import get_buildroot
+from pants_test.pants_run_integration_test import PantsRunIntegrationTest
+
+
+class DepmapIntegrationTest(PantsRunIntegrationTest):
+
+  def _assert_run_success(self, pants_run):
+    self.assertEquals(pants_run.returncode, self.PANTS_SUCCESS_CODE,
+                      'goal depmap expected success, got {0}\n'
+                      'got stderr:\n{1}\n'
+                      'got stdout:\n{2}\n'.format(pants_run.returncode,
+                                                  pants_run.stderr_data,
+                                                  pants_run.stdout_data))
+
+  def test_depmap_with_resolve(self):
+    with temporary_dir(root_dir=self.workdir_root()) as workdir:
+      depmap_out_file = '{workdir}/depmap_out.txt'.format(workdir=workdir)
+      test_target = 'examples/tests/java/com/pants/examples/usethrift:usethrift'
+      pants_run = self.run_pants_with_workdir(
+        ['goal', 'resolve', 'depmap', test_target, '--depmap-project-info',
+         '--depmap-output-file={out_file}'.format(out_file=depmap_out_file)], workdir)
+      self._assert_run_success(pants_run)
+      self.assertTrue(os.path.exists(depmap_out_file),
+                      msg='Could not find depmap output file in {out_file}'
+                           .format(out_file=depmap_out_file))
+      with open(depmap_out_file) as json_file:
+        json_data = json.load(json_file)
+        targets = json_data['targets']
+        libraries = json_data['libraries']
+        # check single code gen module is listed in the target
+        codegen_target = os.path.join(os.path.relpath(workdir,get_buildroot()),
+                                      'gen/thrift/combined/gen-java:gen-java')
+        self.assertTrue(codegen_target in targets)
+        # check if transitively pulled in jar exists as dependency
+        self.assertTrue('org.hamcrest:hamcrest-core:1.3' in targets[test_target]['libraries'])
+        #check correct library path.
+        ivy_cache_dir = Bootstrapper.instance().ivy_cache_dir
+        self.assertEquals(libraries['commons-lang:commons-lang:2.5'],
+                          [os.path.join(ivy_cache_dir,
+                                        'commons-lang/commons-lang/jars/commons-lang-2.5.jar')])
+
+  def test_depmap_without_resolve(self):
+    with temporary_dir(root_dir=self.workdir_root()) as workdir:
+      depmap_out_file = '{workdir}/depmap_out.txt'.format(workdir=workdir)
+      pants_run = self.run_pants_with_workdir(
+        ['goal', 'depmap', 'testprojects/src/java/com/pants/testproject/unicode/main',
+         '--depmap-project-info',
+         '--depmap-output-file={out_file}'.format(out_file=depmap_out_file)], workdir)
+      self._assert_run_success(pants_run)
+      self.assertTrue(os.path.exists(depmap_out_file),
+                      msg='Could not find depmap output file {out_file}'
+                           .format(out_file=depmap_out_file))