Fix logs trying to output unicode (#902)

* updated print statements to python3 format

* made json.loads encode in utf-8

* made precommit add print_function

* Revert "made json.loads encode in utf-8"

This reverts commit de7a78f.

* import print_function everywhere

* updated pre-commit hooks

* added paasta_print

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 -   repo: https://github.com/pre-commit/pre-commit-hooks.git
-    sha: 18d7035de5388cc7775be57f529c154bf541aab9
+    sha: 7539d8bd1a00a3c1bfd34cdb606d3a6372e83469
     hooks:
     -   id: trailing-whitespace
         language_version: python2.7
@@ -31,6 +31,11 @@
     hooks:
     -   id: reorder-python-imports
         language_version: python2.7
+        args:
+        - --add-import
+        - from __future__ import unicode_literals
+        - --add-import
+        - from __future__ import absolute_import
 -   repo: local
     hooks:
     -   id: patch-enforce-autospec
@@ -41,6 +46,6 @@
         files: ^tests/.*\.py$
     -   id: no-catchall-except
         name: Prevent catchall except
-        entry: 'except:$'
+        entry: except:$
         language: pcre
-        files: '\.py$'
+        files: \.py$

docs/source/conf.py

@@ -10,9 +10,13 @@
 #
 # All configuration values have a default; values that are commented out
 # serve to show the default.
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
 import os
 import sys
 
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.

general_itests/environment.py

@@ -11,6 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
 import shutil
 
 from behave_pytest.hook import install_pytest_asserts

general_itests/steps/deployments_json_steps.py

@@ -11,6 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
 import contextlib
 import os
 import tempfile
@@ -34,23 +37,24 @@
 from paasta_tools.utils import format_timestamp
 from paasta_tools.utils import get_paasta_tag_from_deploy_group
 from paasta_tools.utils import load_deployments_json
+from paasta_tools.utils import paasta_print
 
 
-@given(u'a test git repo is setup with commits')
+@given('a test git repo is setup with commits')
 def step_impl_given(context):
     context.test_git_repo_dir = tempfile.mkdtemp('paasta_tools_deployments_json_itest')
     context.test_git_repo = Repo.init(context.test_git_repo_dir)
-    print 'Temp repo in %s' % context.test_git_repo_dir
+    paasta_print('Temp repo in %s' % context.test_git_repo_dir)
 
-    blob = Blob.from_string("My file content\n")
+    blob = Blob.from_string(b"My file content\n")
     tree = Tree()
-    tree.add("spam", 0100644, blob.id)
+    tree.add(b"spam", 0100644, blob.id)
 
     commit = Commit()
-    commit.author = commit.committer = "itest author"
+    commit.author = commit.committer = b"itest author"
     commit.commit_time = commit.author_time = int(time())
     commit.commit_timezone = commit.author_timezone = parse_timezone('-0200')[0]
-    commit.message = "Initial commit"
+    commit.message = b"Initial commit"
     commit.tree = tree.id
 
     object_store = context.test_git_repo.object_store
@@ -62,7 +66,7 @@ def step_impl_given(context):
     context.expected_commit = commit.id
 
 
-@when(u'paasta mark-for-deployments is run against the repo')
+@when('paasta mark-for-deployments is run against the repo')
 def step_paasta_mark_for_deployments_when(context):
     fake_args = mock.MagicMock(
         deploy_group='test_cluster.test_instance',
@@ -87,7 +91,7 @@ def step_paasta_mark_for_deployments_when(context):
             pass
 
 
-@when(u'paasta stop is run against the repo')
+@when('paasta stop is run against the repo')
 def step_paasta_stop_when(context):
     fake_args = mock.MagicMock(
         clusters='test_cluster',
@@ -111,7 +115,7 @@ def step_paasta_stop_when(context):
             pass
 
 
-@when(u'we generate deployments.json for that service')
+@when('we generate deployments.json for that service')
 def step_impl_when(context):
     context.deployments_file = os.path.join('fake_soa_configs', 'fake_deployments_json_service', 'deployments.json')
     try:
@@ -135,7 +139,7 @@ def step_impl_when(context):
         generate_deployments_for_service.main()
 
 
-@then(u'that deployments.json can be read back correctly')
+@then('that deployments.json can be read back correctly')
 def step_impl_then(context):
     deployments = load_deployments_json('fake_deployments_json_service', soa_dir='fake_soa_configs')
     expected_deployments = {
@@ -153,15 +157,15 @@ def step_impl_then(context):
     assert expected_deployments == deployments, "actual: %s\nexpected:%s" % (deployments, expected_deployments)
 
 
-@then(u'that deployments.json has a desired_state of "{expected_state}"')
+@then('that deployments.json has a desired_state of "{expected_state}"')
 def step_impl_then_desired_state(context, expected_state):
     deployments = load_deployments_json('fake_deployments_json_service', soa_dir='fake_soa_configs')
     latest = sorted(deployments.iteritems(), key=lambda(key, value): value['force_bounce'], reverse=True)[0][1]
     desired_state = latest['desired_state']
     assert desired_state == expected_state, "actual: %s\nexpected: %s" % (desired_state, expected_state)
 
 
-@then(u'the repository should be correctly tagged')
+@then('the repository should be correctly tagged')
 def step_impl_then_correctly_tagged(context):
     with contextlib.nested(
         mock.patch('paasta_tools.utils.format_timestamp', autosepc=True,

general_itests/steps/fsm_steps.py

@@ -11,6 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
 import contextlib
 import os
 import shutil
@@ -26,7 +29,7 @@
 from paasta_tools.utils import SystemPaastaConfig
 
 
-@given(u'a fake yelpsoa-config-root with an existing service')
+@given('a fake yelpsoa-config-root with an existing service')
 def step_impl_given(context):
     # Cleaned up in after_scenario()
     context.tmpdir = tempfile.mkdtemp('paasta_tools_fsm_itest')
@@ -49,7 +52,7 @@ def _load_yelpsoa_configs(context, service):
     context.my_config = all_services[service]
 
 
-@when(u'we fsm a new service')
+@when('we fsm a new service')
 def step_impl_when_fsm_auto(context):
     service = "my-cool-service"
 
@@ -70,7 +73,7 @@ def step_impl_when_fsm_auto(context):
     _load_yelpsoa_configs(context, service)
 
 
-@then(u'the new yelpsoa-configs directory has a valid smartstack proxy_port')
+@then('the new yelpsoa-configs directory has a valid smartstack proxy_port')
 def step_impl_then_proxy_port(context):
     port = context.my_config['smartstack']['main']['proxy_port']
     assert port >= 20000

general_itests/steps/http_drain_method_steps.py

@@ -11,6 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
 import BaseHTTPServer
 import threading
 
@@ -20,6 +23,7 @@
 from behave import when
 
 from paasta_tools import drain_lib
+from paasta_tools.utils import paasta_print
 
 
 @given('a fake HTTP server')
@@ -91,9 +95,9 @@ class FakeHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
     status_code = 200
 
     def do_GET(self):
-        print "Got GET for %s" % self.path
+        paasta_print("Got GET for %s" % self.path)
         try:
             FakeHTTPServer.paths.append(self.path)
             self.send_response(self.status_code)
         except Exception as e:
-            print e
+            paasta_print(e)

general_itests/steps/local_run_steps.py

@@ -11,6 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
 import os
 
 from behave import given
@@ -19,16 +22,17 @@
 from path import Path
 
 from paasta_tools.utils import _run
+from paasta_tools.utils import paasta_print
 
 
-@given(u'a simple service to test')
+@given('a simple service to test')
 def given_simple_service(context):
     context.fake_service_name = "fake_simple_service"
     assert os.path.isfile(os.path.join(context.fake_service_name, "Dockerfile"))
     assert os.path.isfile(os.path.join(context.fake_service_name, "Makefile"))
 
 
-@when(u'we run paasta local-run on a Marathon service in non-interactive mode '
+@when('we run paasta local-run on a Marathon service in non-interactive mode '
       'with environment variable "{var}" set to "{val}"')
 def non_interactive_local_run(context, var, val):
     with Path("fake_simple_service"):
@@ -48,19 +52,19 @@ def non_interactive_local_run(context, var, val):
         context.local_run_return_code, context.local_run_output = _run(command=localrun_cmd, timeout=60)
 
 
-@then(u'we should see the expected return code')
+@then('we should see the expected return code')
 def see_expected_return_code(context):
-    print context.local_run_output
-    print context.local_run_return_code
+    paasta_print(context.local_run_output)
+    paasta_print(context.local_run_return_code)
     assert context.local_run_return_code == 42
 
 
-@then(u'we should see the environment variable "{var}" with the value "{val}" in the ouput')
+@then('we should see the environment variable "{var}" with the value "{val}" in the ouput')
 def env_var_in_output(context, var, val):
     assert "%s=%s" % (var, val) in context.local_run_output
 
 
-@when(u'we run paasta local-run in non-interactive mode on a chronos job')
+@when('we run paasta local-run in non-interactive mode on a chronos job')
 def local_run_on_chronos_job(context):
     with Path("fake_simple_service"):
         # The local-run invocation here is designed to run and return a sentinel
@@ -79,7 +83,7 @@ def local_run_on_chronos_job(context):
         context.local_run_return_code, context.local_run_output = _run(command=local_run_cmd, timeout=60)
 
 
-@when(u'we run paasta local-run on an interactive job')
+@when('we run paasta local-run on an interactive job')
 def local_run_on_adhoc_job(context):
     with Path("fake_simple_service"):
         local_run_cmd = ("paasta local-run "
@@ -91,7 +95,7 @@ def local_run_on_adhoc_job(context):
         context.local_run_return_code, context.local_run_output = _run(command=local_run_cmd, timeout=60)
 
 
-@when(u'we run paasta local-run in non-interactive mode on a chronos job with cmd set to \'echo hello && sleep 5\'')
+@when('we run paasta local-run in non-interactive mode on a chronos job with cmd set to \'echo hello && sleep 5\'')
 def local_run_on_chronos_job_with_cmd(context):
     with Path("fake_simple_service"):
         local_run_cmd = ("paasta local-run "

general_itests/steps/paasta_execute_docker_command.py

@@ -11,23 +11,27 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
 from behave import given
 from behave import then
 from behave import when
 from docker.errors import APIError
 
 from paasta_tools.utils import _run
 from paasta_tools.utils import get_docker_client
+from paasta_tools.utils import paasta_print
 
 
-@given(u'Docker is available')
+@given('Docker is available')
 def docker_is_available(context):
     docker_client = get_docker_client()
     assert docker_client.ping()
     context.docker_client = docker_client
 
 
-@given(u'a running docker container with task id {task_id} and image {image_name}')
+@given('a running docker container with task id {task_id} and image {image_name}')
 def create_docker_container(context, task_id, image_name):
     container_name = 'paasta-itest-execute-in-containers'
     try:
@@ -45,21 +49,21 @@ def create_docker_container(context, task_id, image_name):
     context.running_container_id = container.get('Id')
 
 
-@when(u'we paasta_execute_docker_command a command with exit code {code} in container with task id {task_id}')
+@when('we paasta_execute_docker_command a command with exit code {code} in container with task id {task_id}')
 def run_command_in_container(context, code, task_id):
     cmd = '../paasta_tools/paasta_execute_docker_command.py -i %s -c "exit %s"' % (task_id, code)
-    print 'Running cmd %s' % cmd
+    paasta_print('Running cmd %s' % cmd)
     exit_code, output = _run(cmd)
-    print 'Got exitcode %s with output:\n%s' % (exit_code, output)
+    paasta_print('Got exitcode %s with output:\n%s' % (exit_code, output))
     context.return_code = exit_code
 
 
-@then(u'the exit code is {code}')
+@then('the exit code is {code}')
 def paasta_execute_docker_command_result(context, code):
     assert int(code) == int(context.return_code)
 
 
-@then(u'the docker container has at most {num} exec instances')
+@then('the docker container has at most {num} exec instances')
 def check_container_exec_instances(context, num):
     """Modern docker versions remove ExecIDs after they finished, but older
     docker versions leave ExecIDs behind. This test is for assering that
@@ -69,5 +73,5 @@ def check_container_exec_instances(context, num):
         execs = []
     else:
         execs = container_info['ExecIDs']
-    print 'Container info:\n%s' % container_info
+    paasta_print('Container info:\n%s' % container_info)
     assert len(execs) <= int(num)

general_itests/steps/run_steps.py

@@ -11,6 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
 import signal
 
 from behave import then
@@ -19,12 +22,12 @@
 from paasta_tools.utils import _run
 
 
-@when(u'we run a trivial command with timeout {timeout} seconds')
+@when('we run a trivial command with timeout {timeout} seconds')
 def run_command(context, timeout):
     fake_cmd = 'sleep 1'
     context.rc, context.output = _run(fake_cmd, timeout=float(timeout))
 
 
-@then(u'the command is killed with signal {killsignal}')
+@then('the command is killed with signal {killsignal}')
 def check_exit_code(context, killsignal):
     assert context.rc == -1 * getattr(signal, killsignal)