Add basic logging functionality

Signed-off-by: Rafael te Boekhorst <boekhorst@b1-systems.de>
Signed-off-by: Tobias Wolf <wolf@b1-systems.de>
Co-authored-by: Tobias Wolf <wolf@b1-systems.de>

Makefile

@@ -29,6 +29,11 @@ setup-venv:
 	source ./.venv/bin/activate && \
 	pip install --ignore-installed -r requirements.txt
 
+.PHONY: run-precommit
+run-precommit: ## Run pre-commit to check if all files running through
+	pre-commit run --all-files
+
+
 .PHONY: update-requirements
 update-requirements: ## Update the requirements.txt with newer versions of pip packages
 	source ./.venv/bin/activate && \
@@ -50,3 +55,8 @@ run-local-rookify: ## Runs rookify in the local development environment (require
 	$(eval PYTHONPATH="${PYTHONPATH}:$(pwd)/src")
 	source ./.venv/bin/activate && \
 	cd src && python3 -m rookify
+
+download-ceph-folder-from-testbed:
+	ssh testbed-node-0 'sudo cp -r /etc/ceph ~/ceph_configs && sudo chown -R $$USER:$$USER ~/ceph_configs'
+	scp -r testbed-node-0:ceph_configs ./.ceph
+	ssh testbed-node-0 'rm -rf ~/ceph_configs'

requirements.txt

@@ -25,6 +25,7 @@ requests==2.31.0
 requests-oauthlib==1.3.1
 rsa==4.9
 six==1.16.0
+structlog==24.1.0
 urllib3==2.2.1
 websocket-client==1.7.0
 wrapt==1.16.0

src/config.example.yaml

@@ -1,8 +1,14 @@
 general:
   module_data_file: data.yaml
 
+logging:
+  level: INFO # level at which logging should start
+  format:
+    time: "%Y-%m-%d %H:%M.%S" # other example: "iso"
+    renderer: console # or: json
+
 ceph:
-  conf_file: ../.ceph/ceph.conf
+  config: ../.ceph/ceph.conf
   keyring: ../.ceph/ceph.client.admin.keyring
 
 ssh:

src/rookify/__main__.py

@@ -5,18 +5,31 @@
 
 from types import MappingProxyType
 from .yaml import load_yaml, save_yaml
+from rookify.logger import configure_logging, get_logger
 
 
 def main() -> None:
+    # Load configuration file
     try:
         config = load_yaml("config.yaml")
     except FileNotFoundError as err:
         raise SystemExit(f"Could not load config: {err}")
+
+    # Configure logging
+    try:
+        configure_logging(config["logging"])
+    except Exception as e:
+        raise SystemExit(f"Error configuring logging: {e}")
+
+    log = get_logger()
+    log.debug("Executing Rookify")
+
     preflight_modules, migration_modules = rookify.modules.load_modules(
         config["migration_modules"]
     )
 
     module_data = dict()
+
     try:
         module_data.update(load_yaml(config["general"]["module_data_file"]))
     except FileNotFoundError:
@@ -59,6 +72,8 @@ def main() -> None:
 
     save_yaml(config["general"]["module_data_file"], module_data)
 
+    log.info("Data was updated to module_data_file.")
+
 
 if __name__ == "__main__":
     main()

src/rookify/logger.py

@@ -0,0 +1,37 @@
+import structlog
+import logging
+
+from typing import Any, Dict
+
+
+def configure_logging(config: Dict[str, Any]) -> None:
+    LOG_LEVEL = getattr(logging, config["level"], logging.INFO)
+    LOG_TIME_FORMAT = config["format"]["time"]
+    LOG_RENDERER = config["format"]["renderer"]
+
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(LOG_LEVEL),
+        processors=[
+            structlog.processors.TimeStamper(fmt=LOG_TIME_FORMAT),
+            structlog.processors.add_log_level,
+        ],
+    )
+
+    if LOG_RENDERER == "console":
+        structlog.configure(
+            processors=[
+                *structlog.get_config()["processors"],
+                structlog.dev.ConsoleRenderer(),
+            ]
+        )
+    else:
+        structlog.configure(
+            processors=[
+                *structlog.get_config()["processors"],
+                structlog.processors.JSONRenderer(),
+            ]
+        )
+
+
+def get_logger() -> structlog.getLogger:
+    return structlog.get_logger()

src/rookify/modules/analyze_ceph/main.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 
 from ..module import ModuleHandler
-
 from typing import Any, Dict
 
 
@@ -20,10 +19,11 @@ def run(self) -> Any:
                     leaf[part] = self.ceph.mon_command(command)
                 leaf = leaf[part]
 
+        self.logger.info("Dictionary created")
         results["ssh"] = dict()
         results["ssh"]["osd"] = dict()
         for node, values in results["node"]["ls"]["osd"].items():
             devices = self.ssh.command(node, "find /dev/ceph-*/*").stdout.splitlines()
             results["ssh"]["osd"][node] = {"devices": devices}
-
+        self.logger.info("AnalyzeCephHandler ran successfully.")
         return results

src/rookify/modules/migrate_monitors/main.py

@@ -1,7 +1,10 @@
 # -*- coding: utf-8 -*-
 
 from ..module import ModuleHandler
+from typing import Dict, Any
 
 
 class MigrateMonitorsHandler(ModuleHandler):
-    pass
+    def run(self) -> Dict[str, Any]:
+        self.logger.info("MigrateMonitorsHandler ran successfully.")
+        return {}

src/rookify/modules/migrate_osds/main.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 
 from ..module import ModuleHandler
-
 from typing import Any, Dict
 
 
@@ -34,5 +33,5 @@ def run(self) -> Any:
                         osd["device"] = device
                         break
 
-        print(osd_config)
+        self.logger.info(osd_config)
         return {}

src/rookify/modules/module.py

@@ -8,6 +8,8 @@
 import kubernetes
 import fabric
 import jinja2
+import structlog
+from rookify.logger import get_logger
 from typing import Any, Dict, List, Optional
 
 
@@ -162,6 +164,9 @@ def __init__(self, config: Dict[str, Any], data: Dict[str, Any], module_path: st
         self.__ceph: Optional[ModuleHandler.__Ceph] = None
         self.__k8s: Optional[ModuleHandler.__K8s] = None
         self.__ssh: Optional[ModuleHandler.__SSH] = None
+        self.__logger = get_logger()
+
+        self.__logger.debug("Executing {0}", self.__class__.__name__)
 
     @abc.abstractmethod
     def preflight(self) -> None:
@@ -185,6 +190,10 @@ def ceph(self) -> __Ceph:
             self.__ceph = ModuleHandler.__Ceph(self._config["ceph"])
         return self.__ceph
 
+    @property
+    def logger(self) -> structlog.getLogger:
+        return self.__logger
+
     @property
     def k8s(self) -> __K8s:
         if self.__k8s is None: