From 5e365a502f696b76dac637df5f55d1c01cac3a07 Mon Sep 17 00:00:00 2001 From: Simon Jess Date: Mon, 18 Nov 2019 09:06:04 +0100 Subject: [PATCH] subprocess.Popen -> cmk.utils.cmk_subprocess.Popen (2) Change-Id: I5db33b02d12759142c052d88e130a5f1cc7beec0 --- cmk/gui/multitar.py | 32 +++++++++++++-------- cmk/gui/plugins/wato/omd_configuration.py | 34 +++++++++++++---------- cmk/gui/watolib/snapshots.py | 23 +++++++++------ cmk/utils/crash_reporting.py | 16 ++++++----- cmk_base/console.py | 14 +++++++++- cmk_base/nagios_utils.py | 13 ++++++--- doc/treasures/multisite_to_mrpe | 26 +++++++++++------ 7 files changed, 104 insertions(+), 54 deletions(-) diff --git a/cmk/gui/multitar.py b/cmk/gui/multitar.py index 4d256cd16aeed..f30d19386a8dd 100644 --- a/cmk/gui/multitar.py +++ b/cmk/gui/multitar.py @@ -35,12 +35,12 @@ import cStringIO import glob import fnmatch -import subprocess import traceback import itertools import multiprocessing import cmk.utils.paths +import cmk.utils.cmk_subprocess as subprocess from cmk.gui.log import logger from cmk.gui.i18n import _ @@ -222,12 +222,14 @@ def _execute_bash_commands(self, commands, debug=False): if debug: self._logger.debug(" ".join(command)) try: - p = subprocess.Popen(command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=False, - close_fds=True) + p = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, + encoding="utf-8", + ) stdout, stderr = p.communicate() if p.returncode != 0: raise MKGeneralException(_("Activate changes error. Unable to prepare site snapshots. Failed command: %r; StdOut: %r; StdErr: %s") %\ @@ -480,9 +482,12 @@ def check_exists_or_writable(path_tokens): # Older versions of python tarfile handle empty subtar archives :( # This won't work: subtar = tarfile.open("%s/%s" % (restore_dir, tar_member.name)) - p = subprocess.Popen(["tar", "tzf", "%s/%s" % (restore_dir, tar_member.name)], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + p = subprocess.Popen( + ["tar", "tzf", "%s/%s" % (restore_dir, tar_member.name)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8", + ) stdout, stderr = p.communicate() if stderr: errors.append(_("Contains corrupt file %s") % tar_member.name) @@ -535,7 +540,12 @@ def extract_domain(domain, tar_member): tar.extract(tar_member, restore_dir) command = ["tar", "xzf", "%s/%s" % (restore_dir, tar_member.name), "-C", target_dir] - p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8", + ) _stdout, stderr = p.communicate() exit_code = p.wait() if exit_code: diff --git a/cmk/gui/plugins/wato/omd_configuration.py b/cmk/gui/plugins/wato/omd_configuration.py index 493312d873864..89776b5943871 100644 --- a/cmk/gui/plugins/wato/omd_configuration.py +++ b/cmk/gui/plugins/wato/omd_configuration.py @@ -26,12 +26,12 @@ import os import glob -import subprocess import traceback from pathlib2 import Path import cmk.utils.paths import cmk.utils.store as store +import cmk.utils.cmk_subprocess as subprocess from cmk.gui.log import logger from cmk.gui.i18n import _ @@ -422,14 +422,16 @@ def activate(self): try: self._write_config_file() - p = subprocess.Popen(["omd", "reload", "apache"], - shell=False, - stdin=open(os.devnull), - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - close_fds=True) + p = subprocess.Popen( + ["omd", "reload", "apache"], + stdin=open(os.devnull), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + close_fds=True, + encoding="utf-8", + ) - stdout = p.communicate()[0] + stdout, _stderr = p.communicate() if p.returncode != 0: raise Exception(stdout) @@ -539,14 +541,16 @@ def activate(self): try: self._write_config_file() - p = subprocess.Popen(["omd", "restart", "rrdcached"], - shell=False, - stdin=open(os.devnull), - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - close_fds=True) + p = subprocess.Popen( + ["omd", "restart", "rrdcached"], + stdin=open(os.devnull), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + close_fds=True, + encoding="utf-8", + ) - stdout = p.communicate()[0] + stdout, _stderr = p.communicate() if p.returncode != 0: raise Exception(stdout) diff --git a/cmk/gui/watolib/snapshots.py b/cmk/gui/watolib/snapshots.py index e2982ca73ee04..eaf8e914fda0f 100644 --- a/cmk/gui/watolib/snapshots.py +++ b/cmk/gui/watolib/snapshots.py @@ -29,13 +29,13 @@ import shutil import traceback import tarfile -import subprocess import cStringIO from hashlib import sha256 from typing import Any, Dict # pylint: disable=unused-import import cmk.utils import cmk.utils.store as store +import cmk.utils.cmk_subprocess as subprocess import cmk.gui.config as config import cmk.gui.multitar as multitar @@ -119,12 +119,15 @@ def get_basic_tarinfo(name): "tar", "czf", path_subtar, "--ignore-failed-read", "--force-local", "-C", prefix ] + paths - proc = subprocess.Popen(command, - stdin=None, - close_fds=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=prefix) + proc = subprocess.Popen( + command, + stdin=None, + close_fds=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=prefix, + encoding="utf-8", + ) _stdout, stderr = proc.communicate() exit_code = proc.wait() # Allow exit codes 0 and 1 (files changed during backup) @@ -139,7 +142,11 @@ def get_basic_tarinfo(name): # Append tar.gz subtar to snapshot command = ["tar", "--append", "--file=" + filename_work, filename_subtar] - proc = subprocess.Popen(command, cwd=work_dir, close_fds=True) + proc = subprocess.Popen( + command, + cwd=work_dir, + close_fds=True, + ) proc.communicate() exit_code = proc.wait() diff --git a/cmk/utils/crash_reporting.py b/cmk/utils/crash_reporting.py index 3d630be3c91f9..fb5485fb13453 100644 --- a/cmk/utils/crash_reporting.py +++ b/cmk/utils/crash_reporting.py @@ -37,7 +37,6 @@ import sys import time import traceback -import subprocess import json import uuid import urllib @@ -55,6 +54,7 @@ import cmk.utils.paths import cmk.utils.store import cmk.utils.plugin_registry +import cmk.utils.cmk_subprocess as subprocess @contextlib.contextmanager @@ -301,12 +301,14 @@ def _get_os_info(): def _current_monitoring_core(): # type: () -> Text try: - p = subprocess.Popen(["omd", "config", "show", "CORE"], - close_fds=True, - shell=False, - stdin=open(os.devnull), - stdout=subprocess.PIPE, - stderr=open(os.devnull, "w")) + p = subprocess.Popen( + ["omd", "config", "show", "CORE"], + close_fds=True, + stdin=open(os.devnull), + stdout=subprocess.PIPE, + stderr=open(os.devnull, "w"), + encoding="utf-8", + ) return p.communicate()[0] except OSError as e: # Allow running unit tests on systems without omd installed (e.g. on travis) diff --git a/cmk_base/console.py b/cmk_base/console.py index 437c6ad0962bc..eb9ad533d44d6 100644 --- a/cmk_base/console.py +++ b/cmk_base/console.py @@ -28,9 +28,15 @@ import logging import sys +from typing import AnyStr, Text, IO # pylint: disable=unused-import +import six from cmk.utils.log import VERBOSE import cmk.utils.tty as tty +from cmk.utils.encoding import ( + ensure_bytestr, + ensure_unicode, +) # NOTE: This is a hack! We abuse the global logger just to pass around the # verbosity setting. @@ -44,13 +50,19 @@ # would rather use "def output(text, *args, stream=sys.stdout)", but this is not possible # with python 2.7 def output(text, *args, **kwargs): + # type: (AnyStr, *AnyStr, **IO[Text]) -> None if args: text = text % args + if six.PY3: + ensured_text = ensure_unicode(text) # type: unicode + else: + ensured_text = ensure_bytestr(text) # type: bytes + stream = kwargs.get("stream", sys.stdout) try: - stream.write(text) + stream.write(ensured_text) stream.flush() except Exception: # TODO: Way to generic! diff --git a/cmk_base/nagios_utils.py b/cmk_base/nagios_utils.py index 9ee3a738adc92..1cc384b362e95 100644 --- a/cmk_base/nagios_utils.py +++ b/cmk_base/nagios_utils.py @@ -24,10 +24,9 @@ # to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, # Boston, MA 02110-1301 USA. -import subprocess - import cmk.utils.paths import cmk.utils.tty as tty +import cmk.utils.cmk_subprocess as subprocess import cmk_base.console @@ -37,8 +36,14 @@ def do_check_nagiosconfig(): cmk_base.console.verbose("Running '%s'\n" % subprocess.list2cmdline(command)) cmk_base.console.output("Validating Nagios configuration...") - p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) - (stdout, stderr) = p.communicate() + p = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, + encoding="utf-8", + ) + stdout, stderr = p.communicate() exit_status = p.returncode if not exit_status: cmk_base.console.output(tty.ok + "\n") diff --git a/doc/treasures/multisite_to_mrpe b/doc/treasures/multisite_to_mrpe index 5776f01fdd609..f3128d95f9f84 100755 --- a/doc/treasures/multisite_to_mrpe +++ b/doc/treasures/multisite_to_mrpe @@ -33,7 +33,8 @@ # from __future__ import print_function -import os, sys, getopt, time, urllib, subprocess +import os, sys, getopt, time, urllib +import cmk.utils.cmk_subprocess as subprocess omd_site = os.getenv("OMD_SITE") omd_root = os.getenv("OMD_ROOT") @@ -45,7 +46,7 @@ def convert_from_multisite(response): def bail_out(reason): - sys.stderr.write(reason + "\n") + sys.stderr.write((reason + "\n").encode("utf-8")) sys.exit(1) @@ -173,8 +174,13 @@ try: arguments.append(url) verbose("Calling: " + " ".join(arguments)) - process = subprocess.Popen(arguments, executable='curl', stdout=subprocess.PIPE) - answer, stderrdata = process.communicate() + process = subprocess.Popen( + arguments, + executable='curl', + stdout=subprocess.PIPE, + encoding="utf-8", + ) + answer, _stderr = process.communicate() result = process.wait() print(result) if answer.startswith("ERROR:"): @@ -195,7 +201,11 @@ sys.stdout.write('<<>>\n') for service in services: state = {"OK": 0, "WARN": 1, "CRIT": 2, "UNKNOWN": 3}.get(service['service_state']) if state is not None: # skip pending services - sys.stdout.write( - "(%s) %s %d %s|%s\n" % - (service["svc_check_command"].replace(" ", "_"), service["service_description"].replace( - " ", "_"), state, service["svc_plugin_output"], service["svc_perf_data"])) + msg = "(%s) %s %d %s|%s\n" % ( + service["svc_check_command"].replace(" ", "_"), + service["service_description"].replace(" ", "_"), + state, + service["svc_plugin_output"], + service["svc_perf_data"], + ) + sys.stdout.write(msg.encode("utf-8"))