Merge branch 'master' of https://github.com/pantsbuild/pants into jsi…

…rois/logging/early_setup

BUILD.tools

@@ -15,7 +15,7 @@ jar_library(name = 'xalan',
             jars = [jar(org = 'xalan', name = 'xalan', rev = '2.7.1')])
 
 jar_library(name = 'jmake',
-            jars = [jar(org = 'org.pantsbuild', name = 'jmake', rev = '1.3.8-9')])
+            jars = [jar(org = 'org.pantsbuild', name = 'jmake', rev = '1.3.8-10')])
 
 jar_library(name = 'java-compiler',
             jars = [jar(org = 'com.twitter.common.tools', name = 'java-compiler', rev = '0.0.12')])
@@ -75,7 +75,7 @@ jar_library(name = 'benchmark-java-allocation-instrumenter-2.1',
 
 jar_library(name = 'zinc',
             jars = [
-              jar(org = 'com.typesafe.zinc', name = 'zinc', rev = '0.3.2-M1')
+              jar(org = 'com.typesafe.zinc', name = 'zinc', rev = '0.3.7')
                 .exclude(org = 'com.martiansoftware', name = 'nailgun-server')
                 .exclude(org = 'org.ensime', name = 'ensime-sbt-cmd')
             ])
@@ -149,7 +149,7 @@ jar_library(name = 'scrooge-linter',
 
 jar_library(name = 'jar-tool',
             jars = [
-              jar(org = 'com.twitter.common', name = 'jar-tool', rev = '0.1.7')
+              jar(org = 'com.twitter.common', name = 'jar-tool', rev = '0.1.9')
             ])
 
 jar_library(name = 'wire-runtime',

examples/src/java/com/pants/examples/hello/main/BUILD

@@ -9,7 +9,7 @@ jvm_app(name='main',
     ':main-bin'
   ],
   bundles = [
-    bundle(relative_to='config').add(globs('config/*'))
+    bundle(relative_to='config', fileset=globs('config/*'))
   ]
 )
 

examples/src/protobuf/com/pants/examples/imports/BUILD

@@ -5,12 +5,5 @@ java_protobuf_library(name='imports',
   sources=['test_imports.proto',],
   imports=[
     '3rdparty:protobuf-test-import',
-    ':jars',
   ],
 )
-
-jar_library(name='jars',
-  dependencies=[
-    'examples/src/protobuf/com/pants/examples/imports',
-  ]
-)

pants.ini

@@ -3,6 +3,7 @@
 #   homedir: the current user's home directory
 #   buildroot: the root of this repo
 #   pants_bootstrapdir: the global pants scratch space primarily used for caches
+#   pants_configdir: the global pants storage for config files
 #   pants_supportdir: pants support files for this repo go here; for example: ivysettings.xml
 #   pants_distdir: user visible artifacts for this repo go here
 #   pants_workdir: the scratch space used to for live builds in this repo
@@ -180,11 +181,6 @@ repos: [
 indices: ["https://pypi.python.org/simple/"]
 
 
-[python-ipython]
-entry_point: IPython:start_ipython
-requirements: ["ipython==1.0.0"]
-
-
 [backends]
 packages: [
     "internal_backend.optional",
@@ -195,7 +191,7 @@ packages: [
   ]
 
 
-[android-keystore-location]
+[sign]
 # Default to debug keystore installed with SDK.
 # You can change this to point to a config.ini holding the definition of your keys.
-keystore_config_location: %(pants_bootstrapdir)s/android/keystore/default_config.ini
+keystore_config_location: %(pants_configdir)s/android/keystore/default_config.ini

src/python/pants/backend/android/keystore/keystore_resolver.py

@@ -24,19 +24,18 @@ class KeystoreResolver(object):
   class Error(Exception):
     """Indicates an invalid android distribution."""
 
-  _CONFIG_SECTION = 'android-keystore-location'
-
   @classmethod
   def resolve(cls, config_file):
     """Parse a keystore config file and return a list of Keystore objects."""
 
+    config_file = os.path.expanduser(config_file)
     config = Config.create_parser()
     try:
       with open(config_file, 'rb') as keystore_config:
         config.readfp(keystore_config)
-    except IOError:
-      raise KeystoreResolver.Error('The \'--{0}\' option must point at a valid .ini file holding '
-                                   'keystore definitions.'.format(cls._CONFIG_SECTION))
+    except IOError as e:
+      raise KeystoreResolver.Error('Problem parsing config at {}: {}'.format(config_file, e))
+
     parser = SingleFileConfig(config_file, config)
     key_names = config.sections()
     keys = {}
@@ -83,7 +82,7 @@ def __init__(self,
 
     self.keystore_name = keystore_name
     # The os call is robust against None b/c it was validated in KeyResolver with get_required().
-    self.keystore_location = os.path.expandvars(keystore_location)
+    self.keystore_location = os.path.expanduser(keystore_location)
     self.keystore_alias = keystore_alias
     self.keystore_password = keystore_password
     self.key_password = key_password
@@ -97,8 +96,8 @@ def build_type(self):
     if self._type is None:
       keystore_type = self._build_type.lower()
       if keystore_type not in ('release', 'debug'):
-        raise ValueError(self, 'The build_type must be one of (debug, release) '
-                               'instead of: {0}.'.format(self._build_type))
+        raise ValueError('The build_type of Android keystores must be one of (debug, release) '
+                         'instead of: {0}.'.format(self._build_type))
       else:
         self._type = keystore_type
     return self._type

src/python/pants/backend/android/tasks/BUILD

@@ -85,7 +85,7 @@ python_library(
     'src/python/pants/backend/android:keystore_resolver',
     'src/python/pants/backend/android/targets:android',
     'src/python/pants/backend/core/tasks:common',
-    'src/python/pants/base:config',
+    'src/python/pants/base:build_environment',
     'src/python/pants/base:exceptions',
     'src/python/pants/base:workunit',
     'src/python/pants/java:distribution',

src/python/pants/backend/android/tasks/dx_compile.py

@@ -22,7 +22,6 @@ class DxCompile(AndroidTask, NailgunTask):
 
   # name of output file. "Output name must end with one of: .dex .jar .zip .apk or be a directory."
   DEX_NAME = 'classes.dex'
-  _CONFIG_SECTION = 'dx-tool'
 
   @staticmethod
   def is_dextarget(target):
@@ -54,10 +53,6 @@ def __init__(self, *args, **kwargs):
 
     self.setup_artifact_cache()
 
-  @property
-  def config_section(self):
-    return self._CONFIG_SECTION
-
   def _render_args(self, outdir, classes):
     dex_file = os.path.join(outdir, self.DEX_NAME)
     args = []

src/python/pants/backend/android/tasks/sign_apk.py

@@ -13,7 +13,7 @@
 from pants.backend.android.keystore.keystore_resolver import KeystoreResolver
 from pants.backend.android.targets.android_binary import AndroidBinary
 from pants.backend.core.tasks.task import Task
-from pants.base.config import Config
+from pants.base.build_environment import get_pants_configdir
 from pants.base.exceptions import TaskError
 from pants.base.workunit import WorkUnit
 from pants.java.distribution.distribution import Distribution
@@ -27,8 +27,6 @@ class SignApkTask(Task):
   """Sign Android packages with keystores using the jarsigner tool."""
 
   _DEFAULT_KEYSTORE_CONFIG = 'android/keystore/default_config.ini'
-  _CONFIG_SECTION = 'android-keystore-location'
-  _CONFIG_OPTION = 'keystore_config_location'
 
   @classmethod
   def register_options(cls, register):
@@ -49,29 +47,49 @@ def is_signtarget(cls, target):
   def product_types(cls):
     return ['release_apk']
 
+  @classmethod
+  def setup_default_config(cls, path):
+    """Create the default keystore config file for Android targets.
+
+    :param string path: Full path for the created default config file.
+    """
+    # TODO(mateor): Hook into pants global setup instead of relying on building an Android target.
+    try:
+      AndroidConfigUtil.setup_keystore_config(path)
+    except AndroidConfigUtil.AndroidConfigError as e:
+      raise TaskError('Failed to setup default keystore config: {0}'.format(e))
+
+  @classmethod
+  def signed_package_name(cls, target, build_type):
+    """Get package name with 'build_type', a string KeyResolver mandates is in (debug, release)."""
+    return '{0}.{1}.signed.apk'.format(target.app_name, build_type)
+
   def __init__(self, *args, **kwargs):
     super(SignApkTask, self).__init__(*args, **kwargs)
     self._config_file = self.get_options().keystore_config_location
-    self._dist = None
     self._distdir = self.get_options().pants_distdir
+    self._configdir = get_pants_configdir()
+    self._dist = None
 
   @property
   def config_file(self):
-    """Path of .ini file containing definitions for backend.android.keystore_resolver.Keystore."""
-    if self._config_file in (None, ""):
-      try:
-        self._config_file = self.context.config.get_required(self._CONFIG_SECTION,
-                                                             self._CONFIG_OPTION )
-      except Config.ConfigError:
-       raise TaskError('The "[{0}]: {1}" option must declare the location of an .ini file '
-                             'holding keystore definitions.'.format(self._CONFIG_SECTION,
-                                                                    self._CONFIG_OPTION))
-    return self._config_file
+    if not self._config_file:
+      raise TaskError('The "[sign]: keystore_config_location" option must declare the location '
+                      'of an .ini file holding keystore definitions.')
+    return os.path.expanduser(self._config_file)
+
+  @property
+  def default_config_location(self):
+    """Return the path where pants creates the default keystore config file.
+
+    This location will hold the well-known definition of the debug keystore installed with the SDK.
+    """
+    return os.path.join(self._configdir, self._DEFAULT_KEYSTORE_CONFIG)
 
   @property
   def distribution(self):
     if self._dist is None:
-      # Currently no Java 8 for Android. I considered max=1.7.0_50. See comment in render_args().
+      # Currently no Java 8 for Android. I considered max=1.7.0_50. See comment in _render_args().
       self._dist = Distribution.cached(minimum_version='1.6.0_00',
                                        maximum_version='1.7.0_99',
                                        jdk=True)
@@ -98,23 +116,19 @@ def _render_args(self, target, key, unsigned_apk, outdir):
     args.extend(['-keystore', key.keystore_location])
     args.extend(['-storepass', key.keystore_password])
     args.extend(['-keypass', key.key_password])
-    args.extend(['-signedjar', os.path.join(outdir, self.package_name(target, key.build_type))])
+    args.extend(['-signedjar',
+                 os.path.join(outdir, self.signed_package_name(target, key.build_type))])
     args.append(unsigned_apk)
     args.append(key.keystore_alias)
     logger.debug('Executing: {0}'.format(' '.join(args)))
     return args
 
   def execute(self):
-    targets = self.context.targets(self.is_signtarget)
-    # Check for Android keystore config file (where the default keystore definition is kept).
-    config_file = os.path.join(self.context.config.getdefault('pants_bootstrapdir'),
-                               self._DEFAULT_KEYSTORE_CONFIG)
-    if not os.path.isfile(config_file):
-      try:
-        AndroidConfigUtil.setup_keystore_config(config_file)
-      except AndroidConfigUtil.AndroidConfigError as e:
-        raise TaskError('Failed to setup keystore config: {0}'.format(e))
+    # One time setup of the default keystore config file.
+    if not os.path.isfile(self.default_config_location):
+      self.setup_default_config(self.default_config_location)
 
+    targets = self.context.targets(self.is_signtarget)
     with self.invalidated(targets) as invalidation_check:
       invalid_targets = []
       for vt in invalidation_check.invalid_vts:
@@ -133,8 +147,8 @@ def get_products_path(target):
         packages = list(get_products_path(target))
         for unsigned_apk in packages:
           keystores = KeystoreResolver.resolve(self.config_file)
-          for key in keystores:
 
+          for key in keystores:
             outdir = self.sign_apk_out(target, keystores[key].build_type)
             safe_mkdir(outdir)
             args = self._render_args(target, keystores[key], unsigned_apk, outdir)
@@ -148,16 +162,11 @@ def get_products_path(target):
 
     for target in targets:
       release_path = self.sign_apk_out(target, 'release')
-      release_apk = self.package_name(target, 'release')
+      release_apk = self.signed_package_name(target, 'release')
 
       if os.path.isfile(os.path.join(release_path, release_apk)):
         self.context.products.get('release_apk').add(target, release_path).append(release_apk)
 
-
-  def package_name(self, target, build_type):
-    """Get package name with 'build_type', a string KeyResolver mandates is in (debug, release)."""
-    return '{0}.{1}.signed.apk'.format(target.app_name, build_type)
-
   def sign_apk_out(self, target, build_type):
     """Compute the outdir for a target."""
     if build_type == 'release':

src/python/pants/backend/codegen/tasks/protobuf_gen.py

@@ -168,6 +168,13 @@ def _proto_path_imports(self, proto_targets):
   def genlang(self, lang, targets):
     sources_by_base = self._calculate_sources(targets)
     sources = OrderedSet(itertools.chain.from_iterable(sources_by_base.values()))
+
+    # TODO(Eric Ayers) Push this check up to a superclass so all of codegen can share it?
+    if not sources:
+      formatted_targets = "\n".join([t.address.spec for t in targets])
+      raise TaskError("Had {count} targets but no sources?\n targets={targets}"
+                            .format(count=len(targets), targets=formatted_targets))
+
     bases = OrderedSet(sources_by_base.keys())
     bases.update(self._proto_path_imports(targets))
     check_duplicate_conflicting_protos(sources_by_base, sources, self.context.log)

src/python/pants/backend/core/tasks/reporting_server.py

@@ -15,7 +15,6 @@
 from pants import binary_util
 from pants.backend.core.tasks.task import QuietTaskMixin, Task
 from pants.base.build_environment import get_buildroot
-from pants.base.run_info import RunInfo
 from pants.reporting.reporting_server import ReportingServer, ReportingServerManager
 
 
@@ -36,6 +35,10 @@ def register_options(cls, register):
                   'your source code is exposed to all allowed clients!')
     register('--open', action='store_true', default=False,
              help='Attempt to open the server web ui in a browser.')
+    register('--template-dir', advanced=True,
+             help='Use templates from this dir instead of the defaults.')
+    register('--assets-dir', advanced=True,
+             help='Use assets from this dir instead of the defaults.')
 
   def execute(self):
     DONE = '__done_reporting'
@@ -66,11 +69,13 @@ def done_reporting():
         # but is allowed to block indefinitely on the server loop.
         if not os.fork():
           # Child process.
-          info_dir = RunInfo.dir(self.context.config)
+          # The server finds run-specific info dirs by looking at the subdirectories of info_dir,
+          # which is conveniently and obviously the parent dir of the current run's info dir.
+          info_dir = os.path.dirname(self.context.run_tracker.run_info_dir)
           # If these are specified explicitly in the config, use those. Otherwise
           # they will be None, and we'll use the ones baked into this package.
-          template_dir = self.context.config.get('reporting', 'reports_template_dir')
-          assets_dir = self.context.config.get('reporting', 'reports_assets_dir')
+          template_dir = self.get_options().template_dir
+          assets_dir = self.get_options().assets_dir
           settings = ReportingServer.Settings(info_dir=info_dir, template_dir=template_dir,
                                               assets_dir=assets_dir, root=get_buildroot(),
                                               allowed_clients=self.get_options().allowed_clients)

src/python/pants/backend/core/tasks/task.py

@@ -140,21 +140,23 @@ def __init__(self, *args, **kwargs):
     """
     self.context = context
     self._workdir = workdir
+    # TODO: It would be nice to use self.get_options().cache_key_gen_version here, because then
+    # we could have a separate value for each scope if we really wanted to. However we can't
+    # access per-task options in Task.__init__ because GroupTask.__init__ calls it with the
+    # group task's scope, which isn't currently in the known scopes we generate options for.
     self._cache_key_generator = CacheKeyGenerator(
-        context.config.getdefault('cache_key_gen_version', default='200'))
+      self.context.options.for_global_scope().cache_key_gen_version)
     self._read_artifact_cache_spec = None
     self._write_artifact_cache_spec = None
     self._artifact_cache = None
     self._artifact_cache_setup_lock = threading.Lock()
 
     self._cache_key_errors = set()
 
-    default_invalidator_root = os.path.join(
-      self.context.options.for_global_scope().pants_workdir, 'build_invalidator')
-    suffix_type = self.__class__.__name__
     self._build_invalidator_dir = os.path.join(
-        context.config.get('tasks', 'build_invalidator', default=default_invalidator_root),
-        suffix_type)
+      self.context.options.for_global_scope().pants_workdir,
+      'build_invalidator',
+      self.__class__.__name__)
 
   def get_options(self):
     """Returns the option values for this task's scope."""
@@ -198,7 +200,7 @@ def _setup_artifact_cache_from_specs(self, read_spec, write_spec):
   def _create_artifact_cache(self, spec, action):
     if len(spec) > 0:
       pants_workdir = self.context.options.for_global_scope().pants_workdir
-      compression = self.context.config.getint('cache', 'compression', default=5)
+      compression = self.get_options().cache_compression
       my_name = self.__class__.__name__
       return create_artifact_cache(
         log=self.context.log,

src/python/pants/backend/core/wrapped_globs.py

@@ -74,7 +74,7 @@ class Globs(FilesetRelPathWrapper):
 class RGlobs(FilesetRelPathWrapper):
   """Recursive ``globs``, returns Fileset matching files in this directory and its descendents.
 
-  E.g., ``bundle().add(rglobs('config/*')),`` to bundle up all files in
+  E.g., ``bundle(fileset=rglobs('config/*')),`` to bundle up all files in
   the config, config/foo, config/foo/bar directories.
 
   :param exclude: a list of {,r,z}globs objects, strings, or lists of

src/python/pants/backend/jvm/targets/BUILD

@@ -28,6 +28,7 @@ python_library(
     '3rdparty/python/twitter/commons:twitter.common.collections',
     '3rdparty/python/twitter/commons:twitter.common.dirutil',
     'src/python/pants/backend/core/targets:common',
+    'src/python/pants/base:address_lookup_error',
     'src/python/pants/base:build_environment',
     'src/python/pants/base:build_manual',
     'src/python/pants/base:deprecated',