Upgrade Pants (#60)

* Update to latest virtualenv for installing Python helpers

* Update to Pants 1.15.0

This pulls in a few updated code quality checks that requires a few
adjustments of our code base.

* Run Travis tests on Xenial, in line with our Vagrant environment

.travis.yml

@@ -1,10 +1,7 @@
-language: python
-python:
-- '2.7'
-
-sudo: required
-before_install:
-- jdk_switcher use oraclejdk8
+dist: xenial
+language: java
+jdk:
+- 'openjdk8'
 
 script:
 - TERM=dumb ./build-support/jenkins/build.sh

build-support/thrift/thriftw

@@ -103,8 +103,7 @@ if [[ -z "${thrift}" ]]; then
   # Without reverse engineering pants custom os-specific path names,
   # find a suitable thrift compiler
   readonly pants_cache_dir="$(get_pants_option GLOBAL pants_bootstrapdir)"
-  readonly thrift_bin_dir="$(get_pants_option thrift-binary supportdir)"
-  for candidate in "${pants_cache_dir}/${thrift_bin_dir}"/*/*/"${expected_version}"/thrift; do
+  for candidate in "${pants_cache_dir}"/bin/thrift/*/*/"${expected_version}"/thrift; do
     if compatible_thrift "${candidate}"; then
       thrift="${candidate}"
       break

pants

@@ -32,7 +32,7 @@ PYTHON=${PYTHON:-$(which python2.7)}
 PANTS_HOME="${PANTS_HOME:-${HOME}/.cache/pants/setup}"
 PANTS_BOOTSTRAP="${PANTS_HOME}/bootstrap-$(uname -s)-$(uname -m)"
 
-VENV_VERSION=16.0.0
+VENV_VERSION=16.6.1
 
 VENV_PACKAGE=virtualenv-${VENV_VERSION}
 VENV_TARBALL=${VENV_PACKAGE}.tar.gz

pants.ini

@@ -12,7 +12,7 @@
 # limitations under the License.
 
 [GLOBAL]
-pants_version: 1.6.0
+pants_version: 1.15.0
 
 plugins: [
     'pantsbuild.pants.contrib.python.checks==%(pants_version)s',
@@ -59,7 +59,7 @@ skip: True
 skip: True
 
 
-[pycheck-pep8]
+[pycheck-pycodestyle]
 # Code reference is here: http://pep8.readthedocs.org/en/latest/intro.html#error-codes
 ignore: [
     # Aurora custom ignores:
@@ -69,8 +69,10 @@ ignore: [
     'E126',  # continuation line over-indented for hanging indent
     'E129',  # visually indented line with same indent as next logical line
     'E131',  # continuation line unaligned for hanging indent
+    'E306',  # blank line before a nested definition
     'E731',  # do not assign a lambda expression, use a def
     'W503',  # line break before binary operator
+    'W504',  # line break after binary operator
 
     # These are a subset of the standard ignores pre-packaged for pycheck-pep8/pep8, but we need to
     # repeat here since we add our own above:

src/main/python/apache/aurora/admin/aurora_admin.py

@@ -27,6 +27,7 @@
 def main():
   app.help()
 
+
 try:
   from apache.aurora.kerberos.auth_module import KerberosAuthModule
   register_auth_module(KerberosAuthModule())

src/main/python/apache/aurora/client/base.py

@@ -175,6 +175,7 @@ def get_update_page(api, jobkey, update_id):
   return synthesize_url(api.scheduler_proxy.scheduler_client().url, jobkey.role,
                         jobkey.env, jobkey.name, update_id)
 
+
 AURORA_V2_USER_AGENT_NAME = 'Aurora V2'
 AURORA_ADMIN_USER_AGENT_NAME = 'Aurora Admin'
 

src/main/python/apache/aurora/client/cli/client.py

@@ -147,5 +147,6 @@ def proxy_main():
     sys.argv.append('-h')
   sys.exit(client.execute(sys.argv[1:]))
 
+
 if __name__ == '__main__':
   proxy_main()

src/main/python/apache/aurora/client/cli/jobs.py

@@ -83,6 +83,7 @@ def arg_type_jobkey(key):
     parts.append('*')
   return PartialJobKey(*parts)
 
+
 WILDCARD_JOBKEY_OPTION = CommandOption("jobspec", type=arg_type_jobkey,
         metavar="cluster[/role[/env[/name]]]",
         help="A jobkey, optionally containing wildcards")

src/main/python/apache/aurora/client/config.py

@@ -28,7 +28,6 @@
 from apache.aurora.client.base import die
 from apache.aurora.config import AuroraConfig
 
-
 ANNOUNCE_WARNING = """
 Announcer specified primary port as '%(primary_port)s' but no processes have bound that port.
 If you would like to utilize this port, you should listen on {{thermos.ports[%(primary_port)s]}}

src/main/python/apache/aurora/config/resource.py

@@ -12,9 +12,8 @@
 # limitations under the License.
 #
 from collections import namedtuple
-from numbers import Number
-
 from enum import Enum, unique
+from numbers import Number
 
 from gen.apache.aurora.api.ttypes import Resource
 

src/main/python/apache/aurora/tools/java/organize_imports.py

@@ -44,7 +44,7 @@ def index_by_group(import_statements):
 
 
 IMPORT_CLASS_RE = re.compile(
-    'import(?: static)? (?P<outer>[^A-Z]*[A-Z]\w+)(?:\.(?P<inners>[\w][^;]*))?')
+    r'import(?: static)? (?P<outer>[^A-Z]*[A-Z]\w+)(?:\.(?P<inners>[\w][^;]*))?')
 
 
 def get_all_group_lines(import_groups):
@@ -80,6 +80,7 @@ def comparator(x, y):
     all_lines += get_group_lines(group)
   return all_lines
 
+
 BEFORE_IMPORTS = 'before_imports'
 IMPORTS = 'imports'
 STATIC_IMPORTS = 'static_imports'

src/main/python/apache/thermos/common/path.py

@@ -38,7 +38,7 @@ class TaskPath(object):
       path_glob = pathspec.given(task_id = "*").getpath(task_type)
       matching_paths = glob.glob(path_glob)
 
-      path_re = pathspec.given(task_id = "(\S+)").getpath(task_type)
+      path_re = pathspec.given(task_id = "(\\S+)").getpath(task_type)
       path_re = re.compile(path_re)
 
       ids = []

src/main/python/apache/thermos/monitoring/detector.py

@@ -100,8 +100,8 @@ def __get_task_ids_patterns(self, state):
     ).getpath('task_path')
     path_regex = self._pathspec.given(
         root=re.escape(self._root_dir),
-        task_id="(\S+)",
-        state='(\S+)'
+        task_id=r'(\S+)',
+        state=r'(\S+)'
     ).getpath('task_path')
     return path_glob, re.compile(path_regex)
 
@@ -128,8 +128,8 @@ def __get_process_runs_patterns(self, task_id, log_dir):
         root=re.escape(self._root_dir),
         task_id=re.escape(task_id),
         log_dir=log_dir,
-        process='(\S+)',
-        run='(\d+)'
+        process=r'(\S+)',
+        run=r'(\d+)'
     ).getpath('process_logdir')
     return path_glob, re.compile(path_regex)
 
@@ -167,7 +167,7 @@ def __get_process_checkpoints_patterns(self, task_id):
     path_regex = self._pathspec.given(
         root=re.escape(self._root_dir),
         task_id=re.escape(task_id),
-        process='(\S+)',
+        process=r'(\S+)',
     ).getpath('process_checkpoint')
     return path_glob, re.compile(path_regex)
 

src/test/python/apache/aurora/client/test_binding_helper.py

@@ -20,7 +20,6 @@
 from apache.aurora.client.binding_helper import BindingHelper, CachingBindingHelper
 from apache.aurora.config import AuroraConfig
 
-
 GENERIC_CONFIG = """
 main_job = Job(
   name = 'hello_world',

src/test/python/apache/aurora/common/test_clusters.py

@@ -21,7 +21,6 @@
 from apache.aurora.common.cluster import Cluster
 from apache.aurora.common.clusters import Clusters
 
-
 CLUSTERS = '''[
   {
     "name": "cluster1",

src/test/python/apache/aurora/executor/common/test_health_checker.py

@@ -45,7 +45,6 @@
 
 from gen.apache.aurora.api.ttypes import AssignedTask, ExecutorConfig, JobKey, TaskConfig
 
-
 FAKE_MESOS_CONTAINERIZER_BINARY = '''#!/bin/sh
 if [[ $# == 1 ]]; then
   echo "command_info" >&2

src/test/python/apache/aurora/executor/test_thermos_task_runner.py

@@ -316,7 +316,7 @@ def status(self):
       assert signaler.mock_calls == [
         call('/quitquitquit', use_post_method=True),
         call('/abortabortabort', use_post_method=True)]
-      assert task_runner.kill_called() == True
+      assert task_runner.kill_called() is True
 
   def test_thermos_normal_exit_status(self):
     with self.exit_with_status(0, TaskState.SUCCESS) as task_runner:

src/test/python/apache/thermos/cli/test_common.py

@@ -20,7 +20,6 @@
 
 from apache.thermos.cli.common import get_task_from_options
 
-
 TASK_CONFIG = """
 proc = Process(name = 'process', cmdline = 'echo hello world')
 task = Task(name = 'task', processes = [proc])