Fix Python 3 binary vs unicode integration test issues (#6724)

src/python/pants/releases/reversion.py

@@ -57,7 +57,8 @@ def fingerprint_file(workspace, filename):
   """
   content = read_file(os.path.join(workspace, filename))
   fingerprint = hashlib.sha256(content)
-  return 'sha256={}'.format(base64.b64encode(fingerprint.digest())), str(len(content))
+  b64_encoded = base64.b64encode(fingerprint.digest())
+  return 'sha256={}'.format(b64_encoded.decode('utf-8')), str(len(content))
 
 
 def rewrite_record_file(workspace, src_record_file, mutated_file_tuples):
@@ -77,7 +78,8 @@ def rewrite_record_file(workspace, src_record_file, mutated_file_tuples):
     raise Exception('Malformed whl or bad globs: `{}` was not rewritten.'.format(src_record_file))
 
   output_records = []
-  for line in read_file(os.path.join(workspace, dst_record_file)).splitlines():
+  file_name = os.path.join(workspace, dst_record_file)
+  for line in read_file(file_name, binary_mode=False).splitlines():
     filename, fingerprint_str, size_str = line.rsplit(',', 3)
     if filename in mutated_files:
       fingerprint_str, size_str = fingerprint_file(workspace, filename)
@@ -86,7 +88,7 @@ def rewrite_record_file(workspace, src_record_file, mutated_file_tuples):
       output_line = line
     output_records.append(output_line)
 
-  safe_file_dump(os.path.join(workspace, dst_record_file), '\r\n'.join(output_records) + '\r\n', binary_mode=False)
+  safe_file_dump(file_name, '\r\n'.join(output_records) + '\r\n', binary_mode=False)
 
 
 # The wheel METADATA file will contain a line like: `Version: 1.11.0.dev3+7951ec01`.

tests/python/pants_test/backend/jvm/tasks/jvm_compile/java/test_zinc_compile_integration.py

@@ -412,7 +412,7 @@ def test_hermetic_binary_with_capturing_off(self):
       },
     }
     with self.temporary_workdir() as workdir:
-      with self.temporary_file_content("readme.txt", "yo"):
+      with self.temporary_file_content("readme.txt", b"yo"):
         pants_run = self.run_pants_with_workdir(
           [
             'run',
@@ -437,7 +437,7 @@ def test_hermetic_binary_with_3rdparty_dependencies_ivy(self):
     }
 
     with self.temporary_workdir() as workdir:
-      with self.temporary_file_content("readme.txt", "yo"):
+      with self.temporary_file_content("readme.txt", b"yo"):
         pants_run = self.run_pants_with_workdir(
           [
             'run',
@@ -466,7 +466,7 @@ def test_hermetic_binary_with_3rdparty_dependencies_coursier(self):
     }
 
     with self.temporary_workdir() as workdir:
-      with self.temporary_file_content("readme.txt", "yo"):
+      with self.temporary_file_content("readme.txt", b"yo"):
         pants_run = self.run_pants_with_workdir(
           [
             'run',

tests/python/pants_test/backend/jvm/tasks/test_jar_publish_integration.py

@@ -248,8 +248,8 @@ def publish(resource_content):
               self.assertEqual(resource_content, jar_entry.read())
 
       # Publish the same target twice with different resource content.
-      publish('one')
-      publish('two')
+      publish(b'one')
+      publish(b'two')
 
   def publish_test(self, target, artifacts, pushdb_files, extra_options=None, extra_config=None,
                    extra_env=None, success_expected=True, assert_publish_config_contents=False):

tests/python/pants_test/backend/python/tasks/test_ctypes_integration.py

@@ -80,7 +80,7 @@ def test_ctypes_binary(self):
 
       # Execute the binary and ensure its output is correct.
       binary_run_output = invoke_pex_for_output(pex)
-      self.assertEqual('x=3, f(x)=17\n', binary_run_output)
+      self.assertEqual(b'x=3, f(x)=17\n', binary_run_output)
 
   def test_ctypes_native_language_interop(self):
     # TODO: consider making this mock_buildroot/run_pants_with_workdir into a

tests/python/pants_test/base/test_exception_sink_integration.py

@@ -60,9 +60,9 @@ def test_logs_unhandled_exception(self):
 """)
       pid_specific_log_file, shared_log_file = self._get_log_file_paths(tmpdir, pants_run)
       self._assert_unhandled_exception_log_matches(
-        pants_run.pid, read_file(pid_specific_log_file))
+        pants_run.pid, read_file(pid_specific_log_file, binary_mode=False))
       self._assert_unhandled_exception_log_matches(
-        pants_run.pid, read_file(shared_log_file))
+        pants_run.pid, read_file(shared_log_file, binary_mode=False))
 
   def _assert_graceful_signal_log_matches(self, pid, signum, contents):
     self.assertRegexpMatches(contents, """\
@@ -110,23 +110,23 @@ def test_dumps_logs_on_terminate(self):
       # Check that the logs show a graceful exit by SIGTERM.
       pid_specific_log_file, shared_log_file = self._get_log_file_paths(workdir, waiter_run)
       self._assert_graceful_signal_log_matches(
-        waiter_run.pid, signal.SIGTERM, read_file(pid_specific_log_file))
+        waiter_run.pid, signal.SIGTERM, read_file(pid_specific_log_file, binary_mode=False))
       self._assert_graceful_signal_log_matches(
-          waiter_run.pid, signal.SIGTERM, read_file(shared_log_file))
+          waiter_run.pid, signal.SIGTERM, read_file(shared_log_file, binary_mode=False))
 
   def test_dumps_traceback_on_sigabrt(self):
     # SIGABRT sends a traceback to the log file for the current process thanks to
     # faulthandler.enable().
     with self._send_signal_to_waiter_handle(signal.SIGABRT) as (workdir, waiter_run):
       # Check that the logs show an abort signal and the beginning of a traceback.
       pid_specific_log_file, shared_log_file = self._get_log_file_paths(workdir, waiter_run)
-      self.assertRegexpMatches(read_file(pid_specific_log_file), """\
+      self.assertRegexpMatches(read_file(pid_specific_log_file, binary_mode=False), """\
 Fatal Python error: Aborted
 
 Thread [^\n]+ \\(most recent call first\\):
 """)
       # faulthandler.enable() only allows use of a single logging file at once for fatal tracebacks.
-      self.assertEqual('', read_file(shared_log_file))
+      self.assertEqual('', read_file(shared_log_file, binary_mode=False))
 
   def test_prints_traceback_on_sigusr2(self):
     with self._make_waiter_handle() as (workdir, waiter_handle):
@@ -198,4 +198,4 @@ def test_reset_interactive_output_stream(self):
       # The Exiter prints the final error message to whatever the interactive output stream is set
       # to, so when it's redirected it won't be in stderr.
       self.assertNotIn('erroneous!', redirected_pants_run.stderr_data)
-      self.assertIn('erroneous!', read_file(some_file))
+      self.assertIn('erroneous!', read_file(some_file, binary_mode=False))

tests/python/pants_test/engine/legacy/test_console_rule_integration.py

@@ -78,7 +78,7 @@ def test_v2_list_loop(self):
       rel_tmpdir = fast_relpath(tmpdir, get_buildroot())
 
       def dump(content):
-        safe_file_dump(os.path.join(tmpdir, 'BUILD'), content)
+        safe_file_dump(os.path.join(tmpdir, 'BUILD'), content, mode="w")
 
       # Dump an initial target before starting the loop.
       dump('target(name="one")')