Skip to content

Commit

Permalink
Fix encoding of workunits under pantsd (#6505)
Browse files Browse the repository at this point in the history
### Problem

On python2 under `pantsd`, workunits containing unicode are currently not encoded to bytes because the synthetic stdout that is created in that environment does not have "std" in its name. 

### Solution

Ensure that all output files in use with `PlainTextReporter` accept writes of raw bytes, and then universally encode non-`binary_type` values. Mark an integration test `@ensure_daemon` to cover this.
  • Loading branch information
Stu Hood committed Sep 17, 2018
1 parent 606fc83 commit 5a9b6b9
Show file tree
Hide file tree
Showing 3 changed files with 13 additions and 15 deletions.
Expand Up @@ -6,17 +6,20 @@

from builtins import range

from pants_test.pants_run_integration_test import PantsRunIntegrationTest
from pants_test.pants_run_integration_test import PantsRunIntegrationTest, ensure_daemon


class NodeTestIntegrationTest(PantsRunIntegrationTest):

@ensure_daemon
def test_test_simple(self):
command = ['test',
'contrib/node/examples/src/node/server-project:unit']
pants_run = self.run_pants(command=command)

self.assert_success(pants_run)
# NB: This test is marked `@ensure_daemon` to provide coverage for unicode-emitting workunits.
self.assertIn('✓', pants_run.stdout_data)

def test_test_target_with_non_default_script_name(self):
command = ['test',
Expand Down
17 changes: 6 additions & 11 deletions src/python/pants/reporting/plaintext_reporter.py
Expand Up @@ -9,13 +9,13 @@

import six
from colors import cyan, green, red, yellow
from future.utils import PY2

from pants.base.workunit import WorkUnit, WorkUnitLabel
from pants.reporting.plaintext_reporter_base import PlainTextReporterBase
from pants.reporting.report import Report
from pants.reporting.reporter import Reporter, ReporterDestination
from pants.util.memo import memoized_method
from pants.util.strutil import ensure_binary


class ToolOutputFormat(object):
Expand Down Expand Up @@ -94,6 +94,10 @@ class PlainTextReporter(PlainTextReporterBase):

def __init__(self, run_tracker, settings):
super(PlainTextReporter, self).__init__(run_tracker, settings)

# We eagerly validate that our output accepts raw bytes.
settings.outfile.write(b'')

for key, value in settings.label_format.items():
if key not in WorkUnitLabel.keys():
self.emit('*** Got invalid key {} for --reporting-console-label-format. Expected one of {}\n'
Expand Down Expand Up @@ -183,16 +187,7 @@ def handle_output(self, workunit, label, s):
self.flush()

def emit(self, s, dest=ReporterDestination.OUT):
# In Py2, sys.stdout tries to coerce into ASCII, and will fail in coercing Unicode. So,
# we encode prematurely to handle unicode.
# In Py3, sys.stdout takes unicode, so will work normally.
#
# `self.settings.outfile` can also be `io.StringIO` instead of an std stream, in which case it only
# accepts unicode, so `s` does not need to be modified.
# TODO(python3port): Figure out if there's a better way to do this, like opening `sys.stderr` in different mode.
# Part of https://github.com/pantsbuild/pants/issues/6071.
if PY2 and 'std' in str(self.settings.outfile):
s = s.encode('utf-8')
s = ensure_binary(s)
if dest == ReporterDestination.OUT:
self.settings.outfile.write(s)
elif dest == ReporterDestination.ERR:
Expand Down
6 changes: 3 additions & 3 deletions src/python/pants/reporting/reporting.py
Expand Up @@ -7,7 +7,7 @@
import os
import sys
from builtins import open
from io import StringIO
from io import BytesIO

from pants.base.workunit import WorkUnitLabel
from pants.reporting.html_reporter import HtmlReporter
Expand Down Expand Up @@ -62,8 +62,8 @@ def initialize(self, run_tracker, start_time=None):

# Capture initial console reporting into a buffer. We'll do something with it once
# we know what the cmd-line flag settings are.
outfile = StringIO()
errfile = StringIO()
outfile = BytesIO()
errfile = BytesIO()
capturing_reporter_settings = PlainTextReporter.Settings(
outfile=outfile, errfile=errfile, log_level=Report.INFO,
color=False, indent=True, timing=False,
Expand Down

0 comments on commit 5a9b6b9

Please sign in to comment.