diff --git a/src/python/pants/java/distribution/distribution.py b/src/python/pants/java/distribution/distribution.py index 7b113e06d4c..42b2b6c515e 100644 --- a/src/python/pants/java/distribution/distribution.py +++ b/src/python/pants/java/distribution/distribution.py @@ -14,6 +14,7 @@ from collections import namedtuple from contextlib import contextmanager +from future.utils import PY3 from six import string_types from pants.base.revision import Revision @@ -182,7 +183,7 @@ def binary(self, name): If this distribution has no valid command of the given name raises Distribution.Error. If this distribution is a JDK checks both `bin` and `jre/bin` for the binary. """ - if not isinstance(name, string_types): + if not isinstance(name, str): raise ValueError('name must be a binary name, given {} of type {}'.format(name, type(name))) self.validate() return self._validated_executable(name) @@ -234,17 +235,17 @@ def _get_version(self, java): def _get_system_properties(self, java): if not self._system_properties: with temporary_dir() as classpath: - with open(os.path.join(classpath, 'SystemProperties.class'), 'w+') as fp: + with open(os.path.join(classpath, 'SystemProperties.class'), 'w+b') as fp: fp.write(pkgutil.get_data(__name__, 'SystemProperties.class')) cmd = [java, '-cp', classpath, 'SystemProperties'] process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.returncode != 0: raise self.Error('Failed to determine java system properties for {} with {} - exit code' - ' {}: {}'.format(java, ' '.join(cmd), process.returncode, stderr)) + ' {}: {}'.format(java, ' '.join(cmd), process.returncode, stderr.decode('utf-8'))) props = {} - for line in stdout.split(os.linesep): + for line in stdout.decode('utf-8').split(os.linesep): key, _, val = line.partition('=') props[key] = val self._system_properties = props @@ -369,7 +370,8 @@ def jvm_locations(self): if os.path.exists(self._osx_java_home_exe): try: plist = subprocess.check_output([self._osx_java_home_exe, '--failfast', '--xml']) - for distribution in plistlib.readPlistFromString(plist): + plist_results = plistlib.loads(plist) if PY3 else plistlib.readPlistFromString(plist) + for distribution in plist_results: home = distribution['JVMHomePath'] yield self.Location.from_home(home) except subprocess.CalledProcessError: diff --git a/src/python/pants/java/jar/manifest.py b/src/python/pants/java/jar/manifest.py index 1f3769c23d0..8d65fc74b7f 100644 --- a/src/python/pants/java/jar/manifest.py +++ b/src/python/pants/java/jar/manifest.py @@ -6,8 +6,7 @@ from builtins import object from contextlib import closing - -from six import StringIO +from io import BytesIO class Manifest(object): @@ -20,7 +19,7 @@ class Manifest(object): @staticmethod def _wrap(text): text = text.encode('ascii') - with closing(StringIO(text)) as fp: + with closing(BytesIO(text)) as fp: yield fp.read(70) while True: chunk = fp.read(69) @@ -42,12 +41,11 @@ def addentry(self, header, value): if len(header) > 68: raise ValueError('Header name must be 68 characters or less, given {}'.format(header)) if self._contents: - self._contents += '\n' - self._contents += '\n'.join(self._wrap('{header}: {value}'.format(header=header, value=value))) + self._contents += b'\n' + self._contents += b'\n'.join(self._wrap('{header}: {value}'.format(header=header, value=value))) def contents(self): - padded = self._contents + '\n' - return padded.encode('ascii') + return self._contents + b'\n' def is_empty(self): if self._contents.strip(): diff --git a/src/python/pants/java/nailgun_client.py b/src/python/pants/java/nailgun_client.py index a3d0eafb2de..3586eb1468d 100644 --- a/src/python/pants/java/nailgun_client.py +++ b/src/python/pants/java/nailgun_client.py @@ -201,7 +201,8 @@ def execute(self, main_class, cwd=None, *args, **environment): :param dict environment: an env mapping made available to native nails via the nail context :returns: the exit code of the main_class. """ - environment = dict(self.ENV_DEFAULTS.items() + environment.items()) + environment = dict(**environment) + environment.update(self.ENV_DEFAULTS) cwd = cwd or self._workdir # N.B. This can throw NailgunConnectionError (catchable via NailgunError). diff --git a/src/python/pants/java/nailgun_executor.py b/src/python/pants/java/nailgun_executor.py index 8472759aada..d9cb057dac5 100644 --- a/src/python/pants/java/nailgun_executor.py +++ b/src/python/pants/java/nailgun_executor.py @@ -76,7 +76,7 @@ class NailgunExecutor(Executor, FingerprintedProcessManager): _NAILGUN_SPAWN_LOCK = threading.Lock() _SELECT_WAIT = 1 - _PROCESS_NAME = b'java' + _PROCESS_NAME = 'java' def __init__(self, identity, workdir, nailgun_classpath, distribution, connect_timeout=10, connect_attempts=5, metadata_base_dir=None): @@ -103,10 +103,10 @@ def __str__(self): def _create_owner_arg(self, workdir): # Currently the owner is identified via the full path to the workdir. - return '='.join((self._PANTS_OWNER_ARG_PREFIX, workdir)) + return b'='.join((self._PANTS_OWNER_ARG_PREFIX, workdir)) def _create_fingerprint_arg(self, fingerprint): - return '='.join((self.FINGERPRINT_CMD_KEY, fingerprint)) + return b'='.join((self.FINGERPRINT_CMD_KEY, fingerprint)) @staticmethod def _fingerprint(jvm_options, classpath, java_version): @@ -119,9 +119,9 @@ def _fingerprint(jvm_options, classpath, java_version): """ digest = hashlib.sha1() # TODO(John Sirois): hash classpath contents? - [digest.update(item) for item in (''.join(sorted(jvm_options)), - ''.join(sorted(classpath)), - repr(java_version))] + [digest.update(item) for item in (b''.join(sorted(jvm_options)), + b''.join(sorted(classpath)), + repr(java_version).encode('utf-8'))] return digest.hexdigest() def _runner(self, classpath, main, jvm_options, args, cwd=None): @@ -226,8 +226,8 @@ def ensure_connectable(self, nailgun): def _spawn_nailgun_server(self, fingerprint, jvm_options, classpath, stdout, stderr, stdin): """Synchronously spawn a new nailgun server.""" # Truncate the nailguns stdout & stderr. - safe_file_dump(self._ng_stdout, '') - safe_file_dump(self._ng_stderr, '') + safe_file_dump(self._ng_stdout, b'') + safe_file_dump(self._ng_stderr, b'') jvm_options = jvm_options + [self._PANTS_NG_BUILDROOT_ARG, self._create_owner_arg(self._workdir), diff --git a/src/python/pants/java/nailgun_protocol.py b/src/python/pants/java/nailgun_protocol.py index 3ad96e32a12..1770809af87 100644 --- a/src/python/pants/java/nailgun_protocol.py +++ b/src/python/pants/java/nailgun_protocol.py @@ -8,9 +8,6 @@ import struct from builtins import bytes, object, str, zip -import six -from future.utils import binary_type - STDIO_DESCRIPTORS = (0, 1, 2) @@ -147,9 +144,9 @@ def write_chunk(cls, sock, chunk_type, payload=b''): @classmethod def construct_chunk(cls, chunk_type, payload, encoding='utf-8'): """Construct and return a single chunk.""" - if isinstance(payload, six.text_type): + if isinstance(payload, str): payload = payload.encode(encoding) - elif not isinstance(payload, six.binary_type): + elif not isinstance(payload, bytes): raise TypeError('cannot encode type: {}'.format(type(payload))) header = struct.pack(cls.HEADER_FMT, len(payload), chunk_type) @@ -228,7 +225,7 @@ def send_stderr(cls, sock, payload): cls.write_chunk(sock, ChunkType.STDERR, payload) @classmethod - def send_exit(cls, sock, payload=''): + def send_exit(cls, sock, payload=b''): """Send the Exit chunk over the specified socket.""" cls.write_chunk(sock, ChunkType.EXIT, payload) @@ -249,9 +246,9 @@ def isatty_to_env(cls, stdin, stdout, stderr): def gen_env_vars(): for fd_id, fd in zip(STDIO_DESCRIPTORS, (stdin, stdout, stderr)): is_atty = fd.isatty() - yield (cls.TTY_ENV_TMPL.format(fd_id), binary_type(str(int(is_atty)))) + yield (cls.TTY_ENV_TMPL.format(fd_id), str(int(is_atty)).encode('utf-8')) if is_atty: - yield (cls.TTY_PATH_ENV.format(fd_id), os.ttyname(fd.fileno()) or '') + yield (cls.TTY_PATH_ENV.format(fd_id), os.ttyname(fd.fileno()) or b'') return dict(gen_env_vars()) @classmethod diff --git a/src/python/pants/option/BUILD b/src/python/pants/option/BUILD index 2b43d343d70..5c2296c49e9 100644 --- a/src/python/pants/option/BUILD +++ b/src/python/pants/option/BUILD @@ -3,6 +3,7 @@ python_library( dependencies=[ + '3rdparty/python:configparser', '3rdparty/python:future', '3rdparty/python:ansicolors', '3rdparty/python:setuptools', diff --git a/src/python/pants/option/config.py b/src/python/pants/option/config.py index 0d8dba02fd8..1b4e147dad7 100644 --- a/src/python/pants/option/config.py +++ b/src/python/pants/option/config.py @@ -4,6 +4,7 @@ from __future__ import absolute_import, division, print_function, unicode_literals +import configparser import getpass import io import itertools @@ -11,7 +12,6 @@ from contextlib import contextmanager import six -from six.moves import configparser from twitter.common.collections import OrderedSet from pants.base.build_environment import get_buildroot, get_pants_cachedir, get_pants_configdir @@ -85,7 +85,7 @@ def _meta_load(cls, open_ctx, config_items, seed_values=None): for config_item in config_items: parser = cls._create_parser(seed_values) with open_ctx(config_item) as ini: - parser.readfp(ini) + parser.read_file(ini) config_path = config_item.path if hasattr(config_item, 'path') else config_item single_file_configs.append(_SingleFileConfig(config_path, parser)) @@ -120,7 +120,7 @@ def update_dir_from_seed_values(key, default): update_dir_from_seed_values('pants_supportdir', 'build-support') update_dir_from_seed_values('pants_distdir', 'dist') - return configparser.SafeConfigParser(all_seed_values) + return configparser.ConfigParser(all_seed_values) def get(self, section, option, type_=six.string_types, default=None): """Retrieves option from the specified section (or 'DEFAULT') and attempts to parse it as type. diff --git a/src/python/pants/pantsd/pants_daemon.py b/src/python/pants/pantsd/pants_daemon.py index 3fb99abed20..ccccbd7883d 100644 --- a/src/python/pants/pantsd/pants_daemon.py +++ b/src/python/pants/pantsd/pants_daemon.py @@ -326,9 +326,9 @@ def _run_services(self, services): self._logger.info('starting service {}'.format(service)) try: service_thread.start() - except (RuntimeError, service.ServiceError): + except (RuntimeError, FSEventService.ServiceError): self.shutdown(service_thread_map) - raise self.StartupFailure('service {} failed to start, shutting down!'.format(service)) + raise PantsDaemon.StartupFailure('service {} failed to start, shutting down!'.format(service)) # Once all services are started, write our pid. self.write_pid() @@ -339,7 +339,7 @@ def _run_services(self, services): for service, service_thread in service_thread_map.items(): if not service_thread.is_alive(): self.shutdown(service_thread_map) - raise self.RuntimeFailure('service failure for {}, shutting down!'.format(service)) + raise PantsDaemon.RuntimeFailure('service failure for {}, shutting down!'.format(service)) else: # Avoid excessive CPU utilization. service_thread.join(self.JOIN_TIMEOUT_SECONDS) diff --git a/src/python/pants/pantsd/process_manager.py b/src/python/pants/pantsd/process_manager.py index dccb9217397..f1d86bdc57e 100644 --- a/src/python/pants/pantsd/process_manager.py +++ b/src/python/pants/pantsd/process_manager.py @@ -176,7 +176,8 @@ def read_metadata_by_name(self, name, metadata_key, caster=None): """ file_path = self._metadata_file_path(name, metadata_key) try: - return self._maybe_cast(read_file(file_path).strip(), caster) + metadata = read_file(file_path, binary_mode=False).strip() + return self._maybe_cast(metadata, caster) except (IOError, OSError): return None @@ -189,7 +190,7 @@ def write_metadata_by_name(self, name, metadata_key, metadata_value): """ self._maybe_init_metadata_dir_by_name(name) file_path = self._metadata_file_path(name, metadata_key) - safe_file_dump(file_path, metadata_value) + safe_file_dump(file_path, metadata_value, binary_mode=False) def await_metadata_by_name(self, name, metadata_key, timeout, caster=None): """Block up to a timeout for process metadata to arrive on disk. @@ -215,7 +216,7 @@ def purge_metadata_by_name(self, name): try: rm_rf(meta_dir) except OSError as e: - raise self.MetadataError('failed to purge metadata directory {}: {!r}'.format(meta_dir, e)) + raise ProcessMetadataManager.MetadataError('failed to purge metadata directory {}: {!r}'.format(meta_dir, e)) class ProcessManager(ProcessMetadataManager): @@ -319,7 +320,7 @@ def get_subprocess_output(cls, command, ignore_stderr=True, **kwargs): kwargs.setdefault('stderr', subprocess.STDOUT) try: - return subprocess.check_output(command, **kwargs) + return subprocess.check_output(command, **kwargs).decode('utf-8').strip() except (OSError, subprocess.CalledProcessError) as e: subprocess_output = getattr(e, 'output', '').strip() raise cls.ExecutionError(str(e), subprocess_output) @@ -400,7 +401,7 @@ def purge_metadata(self, force=False): :raises: `ProcessManager.MetadataError` when OSError is encountered on metadata dir removal. """ if not force and self.is_alive(): - raise self.MetadataError('cannot purge metadata for a running process!') + raise ProcessMetadataManager.MetadataError('cannot purge metadata for a running process!') super(ProcessManager, self).purge_metadata_by_name(self._name) @@ -434,7 +435,7 @@ def terminate(self, signal_chain=KILL_CHAIN, kill_wait=KILL_WAIT_SEC, purge=True pass if alive: - raise self.NonResponsiveProcess('failed to kill pid {pid} with signals {chain}' + raise ProcessManager.NonResponsiveProcess('failed to kill pid {pid} with signals {chain}' .format(pid=self.pid, chain=signal_chain)) if purge: diff --git a/src/python/pants/pantsd/service/fs_event_service.py b/src/python/pants/pantsd/service/fs_event_service.py index 11e0e391c47..b79bd9dd1b9 100644 --- a/src/python/pants/pantsd/service/fs_event_service.py +++ b/src/python/pants/pantsd/service/fs_event_service.py @@ -123,7 +123,7 @@ def run(self): """Main service entrypoint. Called via Thread.start() via PantsDaemon.run().""" if not (self._watchman and self._watchman.is_alive()): - raise self.ServiceError('watchman is not running, bailing!') + raise PantsService.ServiceError('watchman is not running, bailing!') # Enable watchman for the build root. self._watchman.watch_project(self._build_root) diff --git a/src/python/pants/pantsd/watchman.py b/src/python/pants/pantsd/watchman.py index 58f7e976e2d..114184c4a5c 100644 --- a/src/python/pants/pantsd/watchman.py +++ b/src/python/pants/pantsd/watchman.py @@ -76,7 +76,7 @@ def _is_valid_executable(self, binary_path): def _normalize_watchman_path(self, watchman_path): if not self._is_valid_executable(watchman_path): - raise self.ExecutionError('invalid watchman binary at {}!'.format(watchman_path)) + raise ProcessManager.ExecutionError('invalid watchman binary at {}!'.format(watchman_path)) return os.path.abspath(watchman_path) def _maybe_init_metadata(self): @@ -100,11 +100,11 @@ def _parse_pid_from_output(self, output): except ValueError: # JSON parse failure. self._logger.critical('invalid output from watchman!\n{output!s}'.format(output=output)) - raise self.InvalidCommandOutput(output) + raise ProcessManager.InvalidCommandOutput(output) except KeyError: # Key access error on 'pid' - bad output from watchman. self._logger.critical('no pid from watchman!') - raise self.InvalidCommandOutput(output) + raise ProcessManager.InvalidCommandOutput(output) def launch(self): """Launch and synchronously write metadata. @@ -134,7 +134,7 @@ def launch(self): # to give the server-side socket setup a few chances to quiesce before potentially orphaning it. get_output = functools.partial(self.get_subprocess_output, cmd) - output = retry_on_exception(get_output, 3, (self.ExecutionError,), lambda n: n * .5) + output = retry_on_exception(get_output, 3, (ProcessManager.ExecutionError,), lambda n: n * .5) # Parse the watchman PID from the cli output. pid = self._parse_pid_from_output(output) diff --git a/src/python/pants/pantsd/watchman_launcher.py b/src/python/pants/pantsd/watchman_launcher.py index 28129acf9a3..9095b665621 100644 --- a/src/python/pants/pantsd/watchman_launcher.py +++ b/src/python/pants/pantsd/watchman_launcher.py @@ -90,7 +90,7 @@ def maybe_launch(self): self._logger.debug('launching watchman') try: self.watchman.launch() - except (self.watchman.ExecutionError, self.watchman.InvalidCommandOutput) as e: + except (Watchman.ExecutionError, Watchman.InvalidCommandOutput) as e: self._logger.fatal('failed to launch watchman: {!r})'.format(e)) raise diff --git a/tests/python/pants_test/binaries/BUILD b/tests/python/pants_test/binaries/BUILD index 2cb46e0e525..70e62a35183 100644 --- a/tests/python/pants_test/binaries/BUILD +++ b/tests/python/pants_test/binaries/BUILD @@ -11,5 +11,6 @@ python_tests( 'src/python/pants/util:contextutil', 'src/python/pants/util:dirutil', 'tests/python/pants_test:test_base', + 'tests/python/pants_test:base_test', ] ) diff --git a/tests/python/pants_test/binaries/test_binary_util.py b/tests/python/pants_test/binaries/test_binary_util.py index 42cae52f6f0..1325ee672c5 100644 --- a/tests/python/pants_test/binaries/test_binary_util.py +++ b/tests/python/pants_test/binaries/test_binary_util.py @@ -9,6 +9,7 @@ from builtins import object, str import mock +from future.utils import PY2, PY3 from pants.binaries.binary_util import (BinaryRequest, BinaryToolFetcher, BinaryToolUrlGenerator, BinaryUtil) @@ -170,25 +171,25 @@ def test_support_url_fallback(self): ('bin/ivy', '4.3.7', 'ivy'), ('bin/bash', '4.4.3', 'bash'))} fetcher = self.MapFetcher({ - fake_url(binaries, bases[0], 'protoc'): 'SEEN PROTOC', - fake_url(binaries, bases[0], 'ivy'): 'SEEN IVY', - fake_url(binaries, bases[1], 'bash'): 'SEEN BASH', - fake_url(binaries, bases[1], 'protoc'): 'UNSEEN PROTOC 1', - fake_url(binaries, bases[2], 'protoc'): 'UNSEEN PROTOC 2', - fake_url(binaries, bases[2], 'ivy'): 'UNSEEN IVY 2', + fake_url(binaries, bases[0], 'protoc'): b'SEEN PROTOC', + fake_url(binaries, bases[0], 'ivy'): b'SEEN IVY', + fake_url(binaries, bases[1], 'bash'): b'SEEN BASH', + fake_url(binaries, bases[1], 'protoc'): b'UNSEEN PROTOC 1', + fake_url(binaries, bases[2], 'protoc'): b'UNSEEN PROTOC 2', + fake_url(binaries, bases[2], 'ivy'): b'UNSEEN IVY 2', }) binary_util = self._gen_binary_util( baseurls=bases, fetcher=fetcher) - unseen = [item for item in fetcher.values() if item.startswith('SEEN ')] + unseen = [item for item in fetcher.values() if item.startswith(b'SEEN ')] for supportdir, version, name in binaries.values(): binary_path_abs = binary_util.select_binary( supportdir=supportdir, version=version, name=name) - expected_content = 'SEEN {}'.format(name.upper()) + expected_content = 'SEEN {}'.format(name.upper()).encode('utf-8') self.assertEqual(expected_content, self._read_file(binary_path_abs)) unseen.remove(expected_content) self.assertEqual(0, len(unseen)) # Make sure we've seen all the SEENs. @@ -227,11 +228,14 @@ def uname_func(): the_raised_exception_message = str(cm.exception) self.assertIn(BinaryUtil.MissingMachineInfo.__name__, the_raised_exception_message) - expected_msg = ( + expected_msg_prefix = ( "Error resolving binary request BinaryRequest(supportdir=supportdir, version=version, " "name=name, platform_dependent=True, external_url_generator=None, archiver=None): " "Pants could not resolve binaries for the current host: platform 'vms' was not recognized. " - "Recognized platforms are: [u'darwin', u'linux'].") + "Recognized platforms are: ") + expected_msg = expected_msg_prefix + ("dict_keys(['linux', 'darwin'])." + if PY3 else + "[u'darwin', u'linux'].") self.assertIn(expected_msg, the_raised_exception_message) def test_select_binary_base_path_missing_version(self): @@ -249,7 +253,10 @@ def uname_func(): "Error resolving binary request BinaryRequest(supportdir=mysupportdir, version=myversion, " "name=myname, platform_dependent=True, external_url_generator=None, archiver=None): Pants could not " "resolve binaries for the current host. Update --binaries-path-by-id to find binaries for " - "the current host platform (u\'darwin\', u\'999\').\\n--binaries-path-by-id was:") + "the current host platform ({unicode_literal}\'darwin\', {unicode_literal}\'999\').\\n--binaries-path-by-id was:" + ).format(unicode_literal='u' if PY2 else '') + # expected_msg_postfix = ".\\n--binaries-path-by-id was:" + # expected_msg = (expected_msg_prefix + () + expected_msg_postfix) self.assertIn(expected_msg, the_raised_exception_message) def test_select_script_missing_version(self): @@ -268,7 +275,9 @@ def uname_func(): # platform_dependent=False when doing select_script() "name=myname, platform_dependent=False, external_url_generator=None, archiver=None): Pants " "could not resolve binaries for the current host. Update --binaries-path-by-id to find " - "binaries for the current host platform (u\'darwin\', u\'999\').\\n--binaries-path-by-id was:") + "binaries for the current host platform ({unicode_literal}\'darwin\', {unicode_literal}\'999\')." + "\\n--binaries-path-by-id was:" + ).format(unicode_literal='u' if PY2 else '') self.assertIn(expected_msg, the_raised_exception_message) def test_select_binary_base_path_override(self): diff --git a/tests/python/pants_test/build_graph/test_build_file_parser.py b/tests/python/pants_test/build_graph/test_build_file_parser.py index 7b0dcd224dc..f43b6933e07 100644 --- a/tests/python/pants_test/build_graph/test_build_file_parser.py +++ b/tests/python/pants_test/build_graph/test_build_file_parser.py @@ -9,6 +9,8 @@ from collections import namedtuple from textwrap import dedent +from future.utils import PY3 + from pants.base.build_file import BuildFile from pants.base.file_system_project_tree import FileSystemProjectTree from pants.build_graph.address import BuildFileAddress @@ -70,8 +72,10 @@ def test_name_injection(self): def test_addressable_exceptions(self): self.add_to_build_file('b/BUILD', 'java_library(name="foo", "bad_arg")') build_file_b = self.create_buildfile('b/BUILD') - self.assert_parser_error(build_file_b, - 'non-keyword arg after keyword arg') + expected_msg = ('positional argument follows keyword argument' + if PY3 else + 'non-keyword arg after keyword arg') + self.assert_parser_error(build_file_b, expected_msg) self.add_to_build_file('d/BUILD', dedent( """ @@ -95,6 +99,7 @@ def test_noop_parse(self): def test_invalid_unicode_in_build_file(self): """Demonstrate that unicode characters causing parse errors raise real parse errors.""" + # TODO(python3port): remove ensure_binary once safe_open() uses backport. This test fails on Py3. self.add_to_build_file('BUILD', ensure_binary(dedent( """ jvm_binary(name = ‘hello’, # Parse error due to smart quotes (non ascii characters) @@ -104,10 +109,11 @@ def test_invalid_unicode_in_build_file(self): """ ))) build_file = self.create_buildfile('BUILD') - self.assert_parser_error(build_file, 'invalid syntax') + self.assert_parser_error(build_file, 'invalid character' if PY3 else 'invalid syntax') def test_unicode_string_in_build_file(self): """Demonstrates that a string containing unicode should work in a BUILD file.""" + # TODO(python3port): remove ensure_binary once safe_open() uses backport. This test fails on Py3. self.add_to_build_file('BUILD', ensure_binary(dedent( """ java_library( @@ -319,7 +325,7 @@ def test_context_aware_object_factories(self): make_lib("com.foo.test", "does_not_exists", "1.0") path_util("baz") """) - self.create_file('3rdparty/BUILD', contents) + self.create_file('3rdparty/BUILD', contents, mode='w') build_file = BuildFile(FileSystemProjectTree(self.build_root), '3rdparty/BUILD') address_map = self.build_file_parser.parse_build_file(build_file) diff --git a/tests/python/pants_test/java/distribution/test_distribution.py b/tests/python/pants_test/java/distribution/test_distribution.py index ea1f7b20652..17505040f5a 100644 --- a/tests/python/pants_test/java/distribution/test_distribution.py +++ b/tests/python/pants_test/java/distribution/test_distribution.py @@ -303,7 +303,7 @@ class DistributionOSXLocationTest(unittest.TestCase): def java_home_exe(self): with distribution(executables=EXE('bin/java', version='1')) as jdk1_home: with distribution(executables=EXE('bin/java', version='2')) as jdk2_home: - with temporary_file() as osx_java_home_exe: + with temporary_file(binary_mode=False) as osx_java_home_exe: osx_java_home_exe.write(textwrap.dedent(""" #!/bin/sh echo ' @@ -378,7 +378,7 @@ def exe_path(name): stdout, _ = process.communicate() if process.returncode != 0: return None - path = stdout.strip() + path = stdout.decode('utf-8').strip() return path if os.path.exists(path) and os.access(path, os.X_OK) else None diff --git a/tests/python/pants_test/java/jar/test_manifest.py b/tests/python/pants_test/java/jar/test_manifest.py index c155193dc31..c77e6f5bb86 100644 --- a/tests/python/pants_test/java/jar/test_manifest.py +++ b/tests/python/pants_test/java/jar/test_manifest.py @@ -21,7 +21,7 @@ def test_addentry(self): manifest = Manifest() manifest.addentry('Header', 'value') self.assertEqual( - 'Header: value\n', manifest.contents()) + b'Header: value\n', manifest.contents()) def test_too_long_entry(self): manifest = Manifest() diff --git a/tests/python/pants_test/java/test_executor.py b/tests/python/pants_test/java/test_executor.py index 145cf44b072..1a97500fe29 100644 --- a/tests/python/pants_test/java/test_executor.py +++ b/tests/python/pants_test/java/test_executor.py @@ -42,7 +42,7 @@ def do_test_jre_env_var(self, env_var, env_value, scrubbed=True): stderr=subprocess.PIPE) _, stderr = process.communicate() self.assertEqual(0, process.returncode) - self.assertEqual('' if scrubbed else env_value, stderr.strip()) + self.assertEqual('' if scrubbed else env_value, stderr.decode('utf-8').strip()) def test_not_scrubbed(self): self.do_test_jre_env_var('FRED', 'frog', scrubbed=False) diff --git a/tests/python/pants_test/java/test_nailgun_client.py b/tests/python/pants_test/java/test_nailgun_client.py index 9351f3a140d..3256edd4a93 100644 --- a/tests/python/pants_test/java/test_nailgun_client.py +++ b/tests/python/pants_test/java/test_nailgun_client.py @@ -35,10 +35,10 @@ def flush(self): class TestNailgunClientSession(unittest.TestCase): BAD_CHUNK_TYPE = b';' - TEST_PAYLOAD = 't e s t' + TEST_PAYLOAD = b't e s t' TEST_WORKING_DIR = '/test_working_dir' TEST_MAIN_CLASS = 'test_main_class' - TEST_ARGUMENTS = ['t', 'e', 's', 't'] + TEST_ARGUMENTS = [b't', b'e', b's', b't'] TEST_ENVIRON = dict(TEST_ENV_VAR='xyz') def setUp(self): diff --git a/tests/python/pants_test/java/test_nailgun_executor.py b/tests/python/pants_test/java/test_nailgun_executor.py index 964a79da09e..6ae6252f8ff 100644 --- a/tests/python/pants_test/java/test_nailgun_executor.py +++ b/tests/python/pants_test/java/test_nailgun_executor.py @@ -35,7 +35,7 @@ def test_is_alive_override(self): name='java', pid=3, status=psutil.STATUS_IDLE, - cmdline=['java', '-arg', NailgunExecutor._PANTS_NG_BUILDROOT_ARG] + cmdline=[b'java', b'-arg', NailgunExecutor._PANTS_NG_BUILDROOT_ARG] ) self.assertTrue(self.executor.is_alive()) mock_as_process.assert_called_with(self.executor) @@ -46,7 +46,7 @@ def test_is_alive_override_not_my_process(self): name='java', pid=3, status=psutil.STATUS_IDLE, - cmdline=['java', '-arg', '-arg2'] + cmdline=[b'java', b'-arg', b'-arg2'] ) self.assertFalse(self.executor.is_alive()) mock_as_process.assert_called_with(self.executor) diff --git a/tests/python/pants_test/java/test_nailgun_protocol.py b/tests/python/pants_test/java/test_nailgun_protocol.py index c648971b8d1..0ca5a21c334 100644 --- a/tests/python/pants_test/java/test_nailgun_protocol.py +++ b/tests/python/pants_test/java/test_nailgun_protocol.py @@ -68,10 +68,10 @@ def test_send_and_parse_request_bad_chunktype(self): NailgunProtocol.parse_request(self.server_sock) def test_read_until(self): - recv_chunks = ['1', '234', '56', '789', '0'] + recv_chunks = [b'1', b'234', b'56', b'789', b'0'] mock_socket = mock.Mock() mock_socket.recv.side_effect = recv_chunks - self.assertEqual(NailgunProtocol._read_until(mock_socket, 10), '1234567890') + self.assertEqual(NailgunProtocol._read_until(mock_socket, 10), b'1234567890') self.assertEqual(mock_socket.recv.call_count, len(recv_chunks)) def test_read_until_truncated_recv(self): diff --git a/tests/python/pants_test/java/test_util.py b/tests/python/pants_test/java/test_util.py index cd114d6b8b3..ab93d061d87 100644 --- a/tests/python/pants_test/java/test_util.py +++ b/tests/python/pants_test/java/test_util.py @@ -126,8 +126,9 @@ def test_safe_classpath(self): self.assertEqual(synthetic_jar_dir, os.path.dirname(safe_jar)) with open_zip(safe_jar) as synthetic_jar: - self.assertListEqual([Manifest.PATH], synthetic_jar.namelist()) + self.assertEqual([Manifest.PATH], synthetic_jar.namelist()) # manifest should contain the relative path of both jar and resource directory - self.assertEqual('{}: ../{}/{} ../{}/{}/\n'.format(Manifest.CLASS_PATH, LIB_DIR, JAR_FILE, - LIB_DIR, RESOURCES), - synthetic_jar.read(Manifest.PATH).replace('\n ', '')) + expected = ('{}: ../{}/{} ../{}/{}/\n' + .format(Manifest.CLASS_PATH, LIB_DIR, JAR_FILE, LIB_DIR, RESOURCES) + .encode('utf-8')) + self.assertEqual(expected, synthetic_jar.read(Manifest.PATH).replace(b'\n ', b'')) diff --git a/tests/python/pants_test/option/test_config.py b/tests/python/pants_test/option/test_config.py index a32b02db612..62a33bf4492 100644 --- a/tests/python/pants_test/option/test_config.py +++ b/tests/python/pants_test/option/test_config.py @@ -53,11 +53,11 @@ def setUp(self): """ ) - with temporary_file() as ini1: + with temporary_file(binary_mode=False) as ini1: ini1.write(self.ini1_content) ini1.close() - with temporary_file() as ini2: + with temporary_file(binary_mode=False) as ini2: ini2.write(self.ini2_content) ini2.close() self.config = Config.load(config_paths=[ini1.name, ini2.name]) diff --git a/tests/python/pants_test/option/test_custom_types.py b/tests/python/pants_test/option/test_custom_types.py index 4feda86a694..708a8d4834a 100644 --- a/tests/python/pants_test/option/test_custom_types.py +++ b/tests/python/pants_test/option/test_custom_types.py @@ -9,7 +9,6 @@ from pants.option.custom_types import ListValueComponent, UnsetBool, dict_option, list_option from pants.option.errors import ParseError -from pants.util.strutil import ensure_binary class CustomTypesTest(unittest.TestCase): @@ -116,12 +115,12 @@ def test_unicode_comments(self): """ self._do_test( ['Hi there!', 'This is an element in a list of strings.'], - ensure_binary(dedent(u""" + dedent(""" [ 'Hi there!', # This is a comment with ‘sneaky‘ unicode characters. 'This is an element in a list of strings.', # This is a comment with an obvious unicode character ☺. ] - """).strip()), + """).strip(), ) diff --git a/tests/python/pants_test/option/test_options.py b/tests/python/pants_test/option/test_options.py index 7bf78c11219..eff6d163031 100644 --- a/tests/python/pants_test/option/test_options.py +++ b/tests/python/pants_test/option/test_options.py @@ -7,7 +7,6 @@ import io import os import shlex -import tempfile import unittest import warnings from builtins import str @@ -686,7 +685,7 @@ def test_designdoc_example(self): self.assertEqual(4, options.for_scope('compile.java').c) def test_file_spec_args(self): - with tempfile.NamedTemporaryFile() as tmp: + with temporary_file(binary_mode=False) as tmp: tmp.write(dedent( """ foo @@ -814,7 +813,7 @@ def warnings_catcher(self): def assertWarning(self, w, option_string): self.assertEqual(1, len(w)) self.assertTrue(issubclass(w[-1].category, DeprecationWarning)) - warning_message = str(w[-1].message) + warning_message = str(w[-1].message.args) self.assertIn("will be removed in version", warning_message) self.assertIn(option_string, warning_message) @@ -1116,7 +1115,7 @@ def test_fingerprintable_inverted(self): def assert_fromfile(self, parse_func, expected_append=None, append_contents=None): def _do_assert_fromfile(dest, expected, contents): - with temporary_file() as fp: + with temporary_file(binary_mode=False) as fp: fp.write(contents) fp.close() options = parse_func(dest, fp.name) @@ -1292,7 +1291,7 @@ class DummyOptionable2(Optionable): # Check that we got a warning, but not for the inherited option. self.assertEqual(1, len(w)) self.assertTrue(isinstance(w[0].message, DeprecationWarning)) - self.assertNotIn('inherited', w[0].message) + self.assertNotIn('inherited', w[0].message.args) # Check values. # Deprecated scope takes precedence at equal rank. @@ -1307,7 +1306,7 @@ class DummyOptionable2(Optionable): # Check that we got a warning. self.assertEqual(1, len(w)) self.assertTrue(isinstance(w[0].message, DeprecationWarning)) - self.assertNotIn('inherited', w[0].message) + self.assertNotIn('inherited', w[0].message.args) # Check values. self.assertEqual('uu', vals2.qux) diff --git a/tests/python/pants_test/option/test_options_bootstrapper.py b/tests/python/pants_test/option/test_options_bootstrapper.py index 7de30758a97..926658caa80 100644 --- a/tests/python/pants_test/option/test_options_bootstrapper.py +++ b/tests/python/pants_test/option/test_options_bootstrapper.py @@ -29,7 +29,7 @@ def _config_path(self, path): return ["--pants-config-files=['{}']".format(path)] def _test_bootstrap_options(self, config, env, args, **expected_entries): - with temporary_file() as fp: + with temporary_file(binary_mode=False) as fp: fp.write('[DEFAULT]\n') if config: for k, v in config.items(): @@ -105,7 +105,7 @@ def test_bootstrap_bool_option_values(self): def test_create_bootstrapped_options(self): # Check that we can set a bootstrap option from a cmd-line flag and have that interpolate # correctly into regular config. - with temporary_file() as fp: + with temporary_file(binary_mode=False) as fp: fp.write(dedent(""" [foo] bar: %(pants_workdir)s/baz @@ -136,7 +136,7 @@ def test_create_bootstrapped_options(self): def do_test_create_bootstrapped_multiple_config(self, create_options_bootstrapper): # check with multiple config files, the latest values always get taken # in this case worker_count will be overwritten, while fruit stays the same - with temporary_file() as fp: + with temporary_file(binary_mode=False) as fp: fp.write(dedent(""" [compile.apt] worker_count: 1 @@ -163,7 +163,7 @@ def do_test_create_bootstrapped_multiple_config(self, create_options_bootstrappe self.assertEqual('1', opts_single_config.for_scope('compile.apt').worker_count) self.assertEqual('red', opts_single_config.for_scope('fruit').apple) - with temporary_file() as fp2: + with temporary_file(binary_mode=False) as fp2: fp2.write(dedent(""" [compile.apt] worker_count: 2 @@ -280,9 +280,9 @@ def test_setting_pants_config_in_config(self): config1 = os.path.join(tmpdir, 'config1') config2 = os.path.join(tmpdir, 'config2') with open(config1, 'w') as out1: - out1.write(b"[DEFAULT]\npants_config_files: ['{}']\nlogdir: logdir1\n".format(config2)) + out1.write("[DEFAULT]\npants_config_files: ['{}']\nlogdir: logdir1\n".format(config2)) with open(config2, 'w') as out2: - out2.write(b'[DEFAULT]\nlogdir: logdir2\n') + out2.write('[DEFAULT]\nlogdir: logdir2\n') ob = OptionsBootstrapper(env={}, args=["--pants-config-files=['{}']".format(config1)]) logdir = ob.get_bootstrap_options().for_global_scope().logdir diff --git a/tests/python/pants_test/pantsd/service/test_fs_event_service.py b/tests/python/pants_test/pantsd/service/test_fs_event_service.py index e51f9472b32..c9adcd66ec7 100644 --- a/tests/python/pants_test/pantsd/service/test_fs_event_service.py +++ b/tests/python/pants_test/pantsd/service/test_fs_event_service.py @@ -68,7 +68,7 @@ def mocked_run(self, asserts=True): def test_run_raise_on_failure_isalive(self): self.mock_watchman.is_alive.return_value = False - with self.mocked_run(False), self.assertRaises(self.service.ServiceError): + with self.mocked_run(False), self.assertRaises(FSEventService.ServiceError): self.service.run() def test_run(self): diff --git a/tests/python/pants_test/pantsd/test_process_manager.py b/tests/python/pants_test/pantsd/test_process_manager.py index cf4a6f66eab..5bd3825d5cc 100644 --- a/tests/python/pants_test/pantsd/test_process_manager.py +++ b/tests/python/pants_test/pantsd/test_process_manager.py @@ -140,7 +140,7 @@ def test_readwrite_metadata_by_name(self): ) def test_deadline_until(self): - with self.assertRaises(self.pmm.Timeout): + with self.assertRaises(ProcessMetadataManager.Timeout): with self.captured_logging(logging.INFO) as captured: self.pmm._deadline_until(lambda: False, 'the impossible', timeout=.5, info_interval=.1) self.assertTrue(4 <= len(captured.infos()) <= 6, @@ -149,12 +149,12 @@ def test_deadline_until(self): def test_wait_for_file(self): with temporary_dir() as td: test_filename = os.path.join(td, 'test.out') - safe_file_dump(test_filename, 'test') + safe_file_dump(test_filename, 'test', binary_mode=False) self.pmm._wait_for_file(test_filename, timeout=.1) def test_wait_for_file_timeout(self): with temporary_dir() as td: - with self.assertRaises(self.pmm.Timeout): + with self.assertRaises(ProcessMetadataManager.Timeout): self.pmm._wait_for_file(os.path.join(td, 'non_existent_file'), timeout=.1) def test_await_metadata_by_name(self): @@ -175,7 +175,7 @@ def test_purge_metadata(self): def test_purge_metadata_error(self): with mock.patch('pants.pantsd.process_manager.rm_rf') as mock_rm: mock_rm.side_effect = OSError(errno.EACCES, os.strerror(errno.EACCES)) - with self.assertRaises(ProcessManager.MetadataError): + with self.assertRaises(ProcessMetadataManager.MetadataError): self.pmm.purge_metadata_by_name(self.NAME) self.assertGreater(mock_rm.call_count, 0) @@ -223,11 +223,11 @@ def test_get_subprocess_output_interleaved(self): self.assertEqual(self.pm.get_subprocess_output(cmd, stderr=subprocess.STDOUT), '939393') def test_get_subprocess_output_oserror_exception(self): - with self.assertRaises(self.pm.ExecutionError): + with self.assertRaises(ProcessManager.ExecutionError): self.pm.get_subprocess_output(['i_do_not_exist']) def test_get_subprocess_output_failure_exception(self): - with self.assertRaises(self.pm.ExecutionError): + with self.assertRaises(ProcessManager.ExecutionError): self.pm.get_subprocess_output(['false']) def test_await_pid(self): @@ -315,7 +315,7 @@ def extra_check(process): def test_purge_metadata_aborts(self): with mock.patch.object(ProcessManager, 'is_alive', return_value=True): - with self.assertRaises(self.pm.MetadataError): + with self.assertRaises(ProcessManager.MetadataError): self.pm.purge_metadata() def test_purge_metadata_alive_but_forced(self): @@ -369,7 +369,7 @@ def test_terminate_already_dead(self): def test_terminate_no_kill(self): with self.setup_terminate() as (mock_kill, mock_alive, mock_purge): mock_alive.return_value = True - with self.assertRaises(self.pm.NonResponsiveProcess): + with self.assertRaises(ProcessManager.NonResponsiveProcess): self.pm.terminate(kill_wait=.1, purge=True) self.assertEqual(mock_kill.call_count, len(ProcessManager.KILL_CHAIN)) self.assertEqual(mock_purge.call_count, 0) diff --git a/tests/python/pants_test/pantsd/test_watchman.py b/tests/python/pants_test/pantsd/test_watchman.py index 06e9a0c9f9a..9aa66a37f17 100644 --- a/tests/python/pants_test/pantsd/test_watchman.py +++ b/tests/python/pants_test/pantsd/test_watchman.py @@ -90,12 +90,12 @@ def test_parse_pid_from_output(self): def test_parse_pid_from_output_bad_output(self): output = '{bad JSON.,/#!' - with self.assertRaises(self.watchman.InvalidCommandOutput): + with self.assertRaises(Watchman.InvalidCommandOutput): self.watchman._parse_pid_from_output(output) def test_parse_pid_from_output_no_pid(self): output = json.dumps(dict(nothing=True)) - with self.assertRaises(self.watchman.InvalidCommandOutput): + with self.assertRaises(Watchman.InvalidCommandOutput): self.watchman._parse_pid_from_output(output) def test_launch(self):