#91 added sending logs in failure alerts + reimplemented process outp…

…ut sending to reactive streams

conf/logging.json

@@ -9,39 +9,24 @@
   "handlers": {
     "console": {
       "class": "logging.StreamHandler",
-      "level": "INFO",
+      "level": "DEBUG",
       "formatter": "simple",
       "stream": "ext://sys.stdout"
     },
     "file": {
       "class": "logging.FileHandler",
-      "level": "DEBUG",
+      "level": "INFO",
       "formatter": "simple",
       "filename": "logs/server.log"
     }
   },
   "loggers": {
-    "scriptServer": {
-      "level": "DEBUG",
-      "handlers": [
-        "console",
-        "file"
-      ],
-      "propagate": false
-    },
-    "execution": {
-      "level": "DEBUG",
-      "handlers": [
-        "console",
-        "file"
-      ],
-      "propagate": false
-    }
   },
   "root": {
     "level": "DEBUG",
     "handlers": [
-      "console"
+      "console",
+      "file"
     ]
   }
 }

launcher.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
-import os, sys
+import os
+import sys
 
 sys.path.append("src")
 os.chdir(os.path.dirname(os.path.realpath(__file__)))

src/alerts/destination_base.py

@@ -3,5 +3,5 @@
 
 class Destination(metaclass=abc.ABCMeta):
     @abc.abstractmethod
-    def send(self, title, body):
+    def send(self, title, body, logs=None):
         pass

src/alerts/destination_email.py

@@ -1,4 +1,8 @@
 import smtplib
+from email.mime.application import MIMEApplication
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.utils import formatdate
 
 from alerts import destination_base
 from model import model_helper
@@ -45,7 +49,7 @@ def __init__(self, params_dict):
 
         if self.auth_enabled is None:
             self.auth_enabled = self.password or self.login
-        elif (self.auth_enabled == True) or (self.auth_enabled.lower() == 'true'):
+        elif (self.auth_enabled is True) or (self.auth_enabled.lower() == 'true'):
             self.auth_enabled = True
         else:
             self.auth_enabled = False
@@ -63,8 +67,14 @@ def read_password(params_dict):
 
         return password
 
-    def send(self, title, body):
-        message = 'Subject: {}\n\n{}'.format(title, body)
+    def send(self, title, body, logs=None):
+        message = MIMEMultipart()
+        message['From'] = self.from_address
+        message['To'] = ','.join(self.to_addresses)
+        message['Date'] = formatdate(localtime=True)
+        message['Subject'] = title
+
+        message.attach(MIMEText(body))
 
         server = smtplib.SMTP(self.server)
         server.ehlo()
@@ -75,7 +85,13 @@ def send(self, title, body):
         if self.auth_enabled:
             server.login(self.login, self.password)
 
-        server.sendmail(self.from_address, self.to_addresses, message)
+        if logs:
+            logs_filename = 'log.txt'
+            part = MIMEApplication(logs, Name=logs_filename)
+            part['Content-Disposition'] = 'attachment; filename="%s"' % logs_filename
+            message.attach(part)
+
+        server.sendmail(self.from_address, self.to_addresses, message.as_string())
         server.quit()
 
     def __str__(self, *args, **kwargs):

src/alerts/destination_http.py

@@ -1,4 +1,4 @@
-import urllib.request
+import requests
 
 import alerts.destination_base as destination_base
 
@@ -15,11 +15,15 @@ def __init__(self, params_dict):
         if not self.url.strip().lower().startswith('http'):
             self.url = 'http://' + self.url.strip()
 
-    def send(self, title, body):
+    def send(self, title, body, logs=None):
         message = title + '\n' + body
-        urlopen = urllib.request.urlopen(self.url, data=message.encode('utf-8'))
-        urlopen.read()
-        urlopen.close()
+        data = {'message': message.encode('utf-8')}
+
+        files = {}
+        if logs:
+            files['log'] = ('log.txt', logs)
+
+        requests.post(self.url, data=data, files=files)
 
     def __str__(self, *args, **kwargs):
         return 'Web-hook at ' + self.url

src/auth/auth_ldap.py

@@ -10,6 +10,7 @@
     "user name is mandatory in simple bind",
     "password is mandatory in simple bind"]
 
+LOGGER = logging.getLogger('script_server.LdapAuthorizer')
 
 class LdapAuthorizer(auth_base.Authorizer):
     url = None
@@ -29,8 +30,6 @@ def __init__(self, params_dict):
             self.version = 3
 
     def authenticate(self, username, password):
-        logger = logging.getLogger("authorization")
-
         if self.username_template:
             user = self.username_template.substitute(username=username)
         else:
@@ -58,7 +57,7 @@ def authenticate(self, username, password):
             error = str(e)
 
             if error not in KNOWN_REJECTIONS:
-                logger.exception("Error occurred while ldap authentication")
+                LOGGER.exception("Error occurred while ldap authentication")
 
         if error in KNOWN_REJECTIONS:
             raise auth_base.AuthRejectedError("Invalid credentials")

src/execution/executor.py

@@ -0,0 +1,173 @@
+import logging
+import re
+import sys
+
+from execution import process_popen, process_base
+from model import model_helper
+from utils import file_utils, process_utils, os_utils
+
+TIME_BUFFER_MS = 100
+
+LOGGER = logging.getLogger('script_server.ScriptExecutor')
+
+mock_process = False
+
+
+class ScriptExecutor:
+    def __init__(self, config, parameter_values, audit_name):
+        self.config = config
+        self.parameter_values = parameter_values
+        self.audit_name = audit_name
+
+        self.working_directory = self.get_working_directory()
+        self.script_base_command = process_utils.split_command(
+            self.config.get_script_command(),
+            self.working_directory)
+        self.secure_replacements = self.__init_secure_replacements()
+
+        self.process_wrapper = None  # type: process_base.ProcessWrapper
+        self.output_stream = None
+        self.secure_output_stream = None
+
+    def get_working_directory(self):
+        working_directory = self.config.get_working_directory()
+        if working_directory is not None:
+            working_directory = file_utils.normalize_path(working_directory)
+        return working_directory
+
+    def start(self, process_constructor=None):
+        if self.process_wrapper is not None:
+            raise Exception('Executor already started')
+
+        script_args = build_command_args(self.parameter_values, self.config)
+        command = self.script_base_command + script_args
+
+        if process_constructor:
+            process_wrapper = process_constructor(command, self.working_directory)
+        else:
+            run_pty = self.config.is_requires_terminal()
+            if run_pty and not os_utils.is_pty_supported():
+                LOGGER.warning(
+                    "Requested PTY mode, but it's not supported for this OS (" + sys.platform + '). Falling back to POpen')
+                run_pty = False
+
+            if run_pty:
+                from execution import process_pty
+                process_wrapper = process_pty.PtyProcessWrapper(command, self.working_directory)
+            else:
+                process_wrapper = process_popen.POpenProcessWrapper(command, self.working_directory)
+
+        process_wrapper.start()
+
+        self.process_wrapper = process_wrapper
+
+        self.output_stream = process_wrapper.output_stream \
+            .time_buffered(TIME_BUFFER_MS, _concat_output)
+
+        if self.secure_replacements:
+            self.secure_output_stream = self.output_stream \
+                .map(self.__replace_secure_variables)
+        else:
+            self.secure_output_stream = self.output_stream
+
+        return process_wrapper.get_process_id()
+
+    def __init_secure_replacements(self):
+        word_replacements = {}
+        for parameter in self.config.parameters:
+            if not parameter.secure:
+                continue
+
+            value = self.parameter_values.get(parameter.name)
+            if (value is None) or (value == ''):
+                continue
+
+            value_string = str(value)
+            if not value_string.strip():
+                continue
+
+            value_pattern = '\\b' + re.escape(value_string) + '\\b'
+            word_replacements[value_pattern] = model_helper.SECURE_MASK
+
+        return word_replacements
+
+    def __replace_secure_variables(self, output):
+        result = output
+
+        replacements = self.secure_replacements
+
+        if replacements:
+            for word, replacement in replacements.items():
+                result = re.sub(word, replacement, result)
+
+        return result
+
+    def get_secure_command(self):
+        audit_script_args = build_command_args(
+            self.parameter_values,
+            self.config,
+            model_helper.value_to_str)
+
+        command = self.script_base_command + audit_script_args
+        return ' '.join(command)
+
+    def get_secure_output_stream(self):
+        return self.secure_output_stream
+
+    def get_unsecure_output_stream(self):
+        return self.output_stream
+
+    def get_return_code(self):
+        return self.process_wrapper.get_return_code()
+
+    def add_finish_listener(self, listener):
+        self.process_wrapper.add_finish_listener(listener)
+
+    def write_to_input(self, text):
+        if self.process_wrapper.is_finished():
+            LOGGER.warning('process already finished, ignoring input')
+            return
+
+        self.process_wrapper.write_to_input(text)
+
+    def kill(self):
+        if not self.process_wrapper.is_finished():
+            self.process_wrapper.kill()
+
+    def stop(self):
+        if not self.process_wrapper.is_finished():
+            self.process_wrapper.stop()
+
+
+def build_command_args(param_values, config, stringify=lambda value, param: value):
+    result = []
+
+    for parameter in config.get_parameters():
+        name = parameter.get_name()
+
+        if parameter.is_constant():
+            param_values[parameter.name] = model_helper.get_default(parameter)
+
+        if name in param_values:
+            value = param_values[name]
+
+            if parameter.is_no_value():
+                # do not replace == True, since REST service can start accepting boolean as string
+                if (value is True) or (value == 'true'):
+                    result.append(parameter.get_param())
+            else:
+                if value:
+                    if parameter.get_param():
+                        result.append(parameter.get_param())
+
+                    value_string = stringify(value, parameter)
+                    result.append(value_string)
+
+    return result
+
+
+def _concat_output(output_chunks):
+    if not output_chunks:
+        return output_chunks
+
+    return [''.join(output_chunks)]