setup/core/web - setup bug-fixes

code/core/config/db/link.py

@@ -2,7 +2,7 @@
 # gets connection settings passed from GaDataDb instance
 
 from core.utils.process import subprocess
-from core.utils.debug import log
+from core.utils.debug import log, censor
 from core.config import shared as config
 
 import mysql.connector
@@ -138,7 +138,7 @@ def _error(self, msg: str):
         except (UnboundLocalError, AttributeError):
             pass
 
-        raise ConnectionError(msg)
+        raise ConnectionError(censor(msg))
 
     @lru_cache(maxsize=16)
     def _read_cache(self, query: str):
@@ -153,23 +153,28 @@ def _read_cache(self, query: str):
     @staticmethod
     def _unix_sock():
         try:
-            sock = None
+            sock = config.AGENT.sql_socket
 
-            with open(config.MARIADB_CONFIG_FILE, 'r') as _:
-                for line in _.readlines():
-                    if line.find('socket') != -1:
-                        sock = line.split('=')[1].strip()
-                        break
+            if os_path.exists(sock):
+                return sock
 
-                if sock is None:
-                    sock = config.MARIADB_SOCKET_DEFAULT
-
-            if os_path.exists(sock) is False:
-                if subprocess(command=f"systemctl status {config.MARIADB_SVC} | grep 'Active:'").find('Active: inactive') != -1:
-                    if subprocess(command=f'systemctl start {config.MARIADB_SVC}').find('Not able to start') != -1:
+            else:
+                if subprocess(command=f"systemctl status {config.AGENT.sql_service} | grep 'Active:'").find('Active: inactive') != -1:
+                    if subprocess(command=f'systemctl start {config.AGENT.sql_service}').find('Not able to start') != -1:
                         return False
 
                     time_sleep(3)
+                    if os_path.exists(sock):
+                        return sock
+
+                    else:
+                        sock = False
+
+                        with open(config.AGENT.sql_config, 'r') as _:
+                            for line in _.readlines():
+                                if line.find('socket') != -1:
+                                    sock = line.split('=')[1].strip()
+                                    break
 
             return sock
 

code/core/config/object/base.py

@@ -3,6 +3,8 @@
 
 from core.config.object.helper import SETTING_DICT_EXCEPTION, SETTING_DICT_ERROR, set_attribute
 
+from core.utils.debug import censor
+
 
 class GaBase(object):
     reserved = ['name', 'description', 'state', 'object_id']
@@ -51,7 +53,7 @@ def __init__(self, parent_instance, setting_dict: dict, **kwargs):
         try:
             self.device_enabled = setting_dict['enabled']  # not dynamically set because of device-only attribute
         except SETTING_DICT_EXCEPTION as error_msg:
-            raise SETTING_DICT_EXCEPTION(SETTING_DICT_ERROR % error_msg)
+            raise SETTING_DICT_EXCEPTION(censor(SETTING_DICT_ERROR % error_msg))
 
     @property
     def enabled(self):

code/core/config/object/core/system.py

@@ -8,7 +8,7 @@
 class GaAgent(GaBase):
     setting_list = [
         'sql_server', 'sql_port', 'sql_user', 'sql_secret', 'sql_database', 'log_level', 'debug', 'device_fail_count', 'device_fail_sleep', 'device_log',
-        'version', 'version_detail', 'path_root', 'path_home', 'path_log', 'svc_interval_status', 'svc_interval_reload', 'subprocess_timeout'
+        'path_root', 'path_home', 'path_log', 'svc_interval_status', 'svc_interval_reload', 'subprocess_timeout', 'version', 'version_detail',
     ]
 
     def __init__(self, setting_dict: dict, **kwargs):

code/core/config/object/data/db.py

@@ -5,7 +5,7 @@
 from core.config.db.link import Go as Link
 from core.config.db.check import Go as Check
 from core.config import shared as config
-from core.utils.debug import log
+from core.utils.debug import log, censor
 
 
 class GaDataDb:
@@ -50,4 +50,4 @@ def put(self, command: [str, list]) -> bool:
     @staticmethod
     def _error(msg):
         log(f"Received error \"{msg}\"")
-        raise ConnectionError(msg)
+        raise ConnectionError(censor(msg))

code/core/config/object/helper.py

@@ -1,5 +1,7 @@
 # helper functions for object initialization
 
+from core.utils.debug import censor
+
 SETTING_DICT_ERROR = "The required setting %s for instance \"%s\" (id \"%s\") of the object \"%s\" was not defined " \
                      "in its settings: \"%s\""
 SETTING_DICT_EXCEPTION = KeyError
@@ -60,8 +62,7 @@ def set_attribute(setting_dict: dict, setting_list: list, instance, obj) -> None
                 setattr(instance, key, setting_dict[key])
 
     except SETTING_DICT_EXCEPTION as error_msg:
-        raise SETTING_DICT_EXCEPTION(SETTING_DICT_ERROR % (error_msg, instance.name, instance.object_id, obj,
-                                                           setting_dict))
+        raise SETTING_DICT_EXCEPTION(censor(SETTING_DICT_ERROR % (error_msg, instance.name, instance.object_id, obj, setting_dict)))
 
 
 def overwrite_inherited_attribute(child_setting_dict: dict, setting_list: list, child_instance, obj) -> None:
@@ -83,16 +84,14 @@ def overwrite_inherited_attribute(child_setting_dict: dict, setting_list: list,
                     elif getattr(child_instance, key) == child_setting_dict[key]:
                         pass
                     else:
-                        raise AttributeError(f"Unable to set attribute since it already exists: current value \"{getattr(child_instance, key)}\", "
-                                             f"new value \"{child_setting_dict[key]}\"")
+                        raise AttributeError(censor(f"Unable to set attribute since it already exists: current value \"{getattr(child_instance, key)}\", new value \"{child_setting_dict[key]}\""))
                 except NameError:  # if it already exists as property and should be overwritten
                     setattr(child_instance, key, child_setting_dict[key])
             else:
                 set_property(obj=obj, key=key)
 
     except SETTING_DICT_EXCEPTION as error_msg:
-        raise SETTING_DICT_EXCEPTION(SETTING_DICT_ERROR % (error_msg, child_instance.name, child_instance.object_id,
-                                                           obj, child_setting_dict))
+        raise SETTING_DICT_EXCEPTION(censor(SETTING_DICT_ERROR % (error_msg, child_instance.name, child_instance.object_id, obj, child_setting_dict)))
 
 
 def set_inherited_attribute(child_setting_dict: dict, setting_list: list, child_instance, obj) -> None:
@@ -115,14 +114,12 @@ def set_inherited_attribute(child_setting_dict: dict, setting_list: list, child_
                 elif getattr(child_instance, key) == child_setting_dict[key]:
                     pass
                 else:
-                    raise AttributeError(f"Unable to set attribute since it already exists: current value \"{getattr(child_instance, key)}\", "
-                                         f"new value \"{child_setting_dict[key]}\"")
+                    raise AttributeError(censor(f"Unable to set attribute since it already exists: current value \"{getattr(child_instance, key)}\", new value \"{child_setting_dict[key]}\""))
             else:
                 raise AttributeError("Unable to set attribute since it doesn't exist on neither child nor parent!")
 
     except SETTING_DICT_EXCEPTION as error_msg:
-        raise SETTING_DICT_EXCEPTION(SETTING_DICT_ERROR % (error_msg, child_instance.name, child_instance.object_id,
-                                                           obj, child_setting_dict))
+        raise SETTING_DICT_EXCEPTION(censor(SETTING_DICT_ERROR % (error_msg, child_instance.name, child_instance.object_id, obj, child_setting_dict)))
 
 
 def set_parent_attribute(child_instance, setting_list: list, obj) -> None:
@@ -139,5 +136,4 @@ def set_parent_attribute(child_instance, setting_list: list, obj) -> None:
             set_property(obj=obj, key=key)
 
     except SETTING_DICT_EXCEPTION as error_msg:
-        raise SETTING_DICT_EXCEPTION(SETTING_DICT_ERROR % (error_msg, child_instance.name, child_instance.object_id,
-                                                           obj, setting_list))
+        raise SETTING_DICT_EXCEPTION(censor(SETTING_DICT_ERROR % (error_msg, child_instance.name, child_instance.object_id, obj, setting_list)))

code/core/config/shared.py

@@ -37,11 +37,6 @@ def init():
 THREAD_JOIN_TIMEOUT = 5
 THREAD_DEFAULT_SLEEP_TIME = 600
 
-# db settings
-MARIADB_CONFIG_FILE = '/etc/mysql/mariadb.conf.d/50-server.cnf'
-MARIADB_SOCKET_DEFAULT = '/run/mysqld/mysqld.sock'
-MARIADB_SVC = 'mariadb.service'
-
 # log settings
 LOG_MAX_TRACEBACK_LENGTH = 5000
 LOG_TIMESTAMP_FORMAT = '%Y-%m-%d %H:%M:%S:%f'

code/core/device/output/condition/link.py

@@ -186,7 +186,7 @@ def _get_single_link_member_list(self, group: GaConditionGroup) -> list:
         device_log(f"Condition \"{group.name}\" has the following single link members \"{slm_list}\"", add=self.name, level=8)
 
         if len(slm_list) == 0:
-            raise ValueError("It looks like you have a configuration error.")
+            raise ValueError('It looks like you have a configuration error.')
 
         return slm_list
 

code/core/factory/forge/system/main.py

@@ -1,6 +1,6 @@
 from core.factory import config
 from core.config.object.core.system import GaServer, GaAgent
-from core.utils.debug import log
+from core.utils.debug import log, censor
 
 
 class Go:
@@ -36,7 +36,7 @@ def get(self) -> dict:
 
             except (IndexError, KeyError) as error:
                 log(f"Factory wasn't able to pull {key}-data from database! Make sure the configuration exists!", level=1)
-                raise KeyError(error)
+                raise KeyError(censor(error))
 
         # output_dict.update({
         #     self.key_object_task: TaskFactory(

code/core/service/prestart.py

@@ -48,7 +48,6 @@ def __init__(self):
         signal.signal(signal.SIGINT, self._stop)
         signal.signal(signal.SIGUSR1, self._stop)
         self.debug = False
-        self.logger = None
         self.log_cache = []
 
         group = config.GA_GROUP
@@ -61,8 +60,6 @@ def start(self):
         if self._check_file_config():
             startup_shared_vars.init()
             if self._check_file_log():
-                from core.utils.debug import Log, FileAndSystemd
-                self.logger = FileAndSystemd(Log())
                 if self._check_networking():
                     if self._check_database():
                         if self._check_factory():
@@ -268,17 +265,18 @@ def _error(self, msg: str):
         raise SystemError(msg)
 
     def _log(self, output, level: int = 1):
-        if self.logger is not None:
+        try:
+            from core.utils.debug import fns_log, log
+
             if len(self.log_cache) > 0:
-                from core.utils.debug import log
-                for _log in self.log_cache:
-                    log(output=_log['output'], level=_log['level'])
+                for msg in self.log_cache:
+                    log(output=msg['output'], level=msg['level'])
 
                 self.log_cache = []
 
-            self.logger.write(output, level=level)
+            fns_log(output=output, level=level)
 
-        else:
+        except Exception:
             self.log_cache.append({'level': level, 'output': output})
 
             if level == 1:

code/core/utils/debug.py

@@ -19,12 +19,6 @@ def now(time_format: str):
 
 
 class Log:
-    try:
-        SECRET_DATA = [config.AGENT.sql_secret]
-
-    except AttributeError:
-        SECRET_DATA = []
-
     def __init__(self, typ: str = 'core', addition: str = None, src_file: str = None):
         self.name = src_file
         if src_file is None:
@@ -49,7 +43,7 @@ def write(self, output: str, level: int = 1, system: bool = False) -> bool:
         if config.AGENT.debug == 0 and (level > self.log_level or not self.status):
             return False
 
-        output = self._censor(str(output))
+        output = censor(str(output))
 
         if system:
             journal.send(output)
@@ -79,28 +73,6 @@ def _check(self) -> bool:
             print(f"LOG ERROR: Unable to access/modify log file '{self.log_file}'")
             return False
 
-    def _censor(self, output: str) -> str:
-        for setting in config.LOG_SECRET_SETTINGS:
-            if output.find(setting) != -1:
-                split_output = output.split(setting)
-                updated_list = [split_output[0]]
-
-                for data in split_output[1:]:
-                    try:
-                        updated_list.append(f"{setting}': \"{config.LOG_CENSOR_OUTPUT}\",{data.split(',', 1)[1]}")
-
-                    except IndexError:
-                        output = f"LOG ERROR: 'Output has sensitive data (\"{setting}\") in it that must be censored. " \
-                                 f"But we were not able to safely censor it. Output was completely replaced.'"
-
-                if output.find('LOG ERROR') == -1:
-                    output = ''.join(updated_list)
-
-        for data in self.SECRET_DATA:
-            output.replace(data, config.LOG_CENSOR_OUTPUT)
-
-        return output
-
     @staticmethod
     def _debugger(command):
         if config.AGENT.debug == 1:
@@ -159,3 +131,34 @@ def device_log(output: str, add: str, level: int = 1) -> bool:
 
     else:
         return Log(src_file=_src).write(output=output, level=level)
+
+
+def censor(output: str) -> str:
+    output = str(output)
+
+    try:
+        secrets = [config.AGENT.sql_secret]
+
+    except AttributeError:
+        secrets = []
+
+    for setting in config.LOG_SECRET_SETTINGS:
+        if output.find(setting) != -1:
+            split_output = output.split(setting)
+            updated_list = [split_output[0]]
+
+            for data in split_output[1:]:
+                try:
+                    updated_list.append(f"{setting}': \"{config.LOG_CENSOR_OUTPUT}\",{data.split(',', 1)[1]}")
+
+                except IndexError:
+                    output = f"LOG ERROR: 'Output has sensitive data (\"{setting}\") in it that must be censored. " \
+                             f"But we were not able to safely censor it. Output was completely replaced.'"
+
+            if output.find('LOG ERROR') == -1:
+                output = ''.join(updated_list)
+
+    for secret in secrets:
+        output.replace(secret, config.LOG_CENSOR_OUTPUT)
+
+    return output

code/web/base/base/settings.py

@@ -92,7 +92,7 @@
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.1/howto/static-files/
 STATIC_URL = '/static/'
-STATICFILES_DIRS = (os_path.join(BASE_DIR, 'static/'), )
+STATICFILES_DIRS = [os_path.join(BASE_DIR, 'static/')]
 LOGIN_REDIRECT_URL = '/'
 LOGOUT_REDIRECT_URL = '/'
 handler500 = 'ga.subviews.handlers.handler500'

code/web/base/ga/config/model.py

@@ -14,6 +14,9 @@
 SYS_DEFAULT_TZ = 'UTC'
 SYS_DEFAULT_FAIL_COUNT = 3
 SYS_DEFAULT_FAIL_SLEEP = 3600
+SYS_DEFAULT_SQL_SOCKET = '/run/mysqld/mysqld.sock'
+SYS_DEFAULT_SQL_SVC = 'mariadb.service'
+SYS_DEFAULT_SQL_CONFIG = '/etc/mysql/mariadb.conf.d/50-server.cnf'
 
 # dashboard parameters
 DB_MAX_ROWS = 100

code/web/base/ga/submodels/device.py

@@ -3,6 +3,8 @@
 from .condition_min import GroupConditionModel
 
 
+# todo: replace script-fields with "models.FilePathField(path=f"{get_agent()['path_root']}/device/connection")"
+
 # connection
 
 class ObjectConnectionModel(BaseDeviceObjectModel):

code/web/base/ga/submodels/system.py

@@ -22,7 +22,7 @@
 class SystemAgentModel(BaseModel):
     field_list = [
         'name', 'description', 'sql_server', 'sql_port', 'sql_user', 'sql_secret', 'sql_database', 'log_level', 'debug', 'device_fail_count', 'device_fail_sleep', 'device_log',
-        'svc_interval_status', 'svc_interval_reload', 'subprocess_timeout'
+        'svc_interval_status', 'svc_interval_reload', 'subprocess_timeout', 'sql_socket', 'sql_service', 'sql_config',
     ]
 
     version = models.FloatField()
@@ -37,6 +37,9 @@ class SystemAgentModel(BaseModel):
     sql_user = models.CharField(max_length=50, default=config.SYS_DEFAULT_SQL_USER)
     sql_secret = models.CharField(max_length=255, default='o1Qhr6zm1INEZcKjBIVB')
     sql_database = models.CharField(max_length=50, default=config.SYS_DEFAULT_SQL_DB)
+    sql_socket = models.CharField(max_length=255, default=config.SYS_DEFAULT_SQL_SOCKET)
+    sql_service = models.CharField(max_length=50, default=config.SYS_DEFAULT_SQL_SVC)
+    sql_config = models.CharField(max_length=255, default=config.SYS_DEFAULT_SQL_CONFIG)
 
     log_level = models.PositiveSmallIntegerField(default=2, choices=LOG_LEVEL_CHOICES)
     debug = models.BooleanField(choices=BOOLEAN_CHOICES, default=False)
@@ -53,7 +56,7 @@ class SystemAgentModel(BaseModel):
 class SystemServerModel(BaseModel):
     field_list = [
         'name', 'description', 'sql_server', 'sql_port', 'sql_user', 'sql_secret', 'sql_database', 'log_level', 'debug', 'security', 'timezone', 'web_cdn', 'web_warn', 'ga_cloud',
-        'ga_cloud_ddns',
+        'ga_cloud_ddns', 'sql_service',
     ]
 
     version = models.FloatField()
@@ -71,6 +74,7 @@ class SystemServerModel(BaseModel):
     sql_user = models.CharField(max_length=50, default=config.SYS_DEFAULT_SQL_USER)
     sql_secret = models.CharField(max_length=255, default='o1Qhr6zm1INEZcKjBIVB')
     sql_database = models.CharField(max_length=50, default=config.SYS_DEFAULT_SQL_DB)
+    sql_service = models.CharField(max_length=50, default=config.SYS_DEFAULT_SQL_SVC)
 
     log_level = models.PositiveSmallIntegerField(default=2, choices=LOG_LEVEL_CHOICES)
     debug = models.BooleanField(choices=BOOLEAN_CHOICES, default=False)

setup/roles/core/tasks/devices.yml

@@ -11,7 +11,6 @@
     virtualenv: "{{ ga_core_path_venv }}"
     virtualenv_python: "python{{ ga_python_version }}"
     virtualenv_command: '/usr/bin/virtualenv'
-    executable: pip3
     state: present
 
 - name: GA | Core | Copying device-code