Skip to content

Commit

Permalink
Devel (#25)
Browse files Browse the repository at this point in the history 
* Changed Pydantic for JSON config files.

* Fixed poetry entry-point.

It threw the error 'EntryPoint must be in 'name=module:attrs [extras]' format'. Fixed it changing the '-' in 'monitor-agent' for a '_'.

* Implemented receiving settings as a JSON ->FILE<- (not text).

Need to fix testing of data types when a JSON is received

* Reformatted code of 'settings' module

* Implemented alert manager.

Need to check if it's the best way to store the thresholds.

* Fix #24.

* SECURITY: Removed GET Settings.

* Remote Work

* Settings reformmated and FINALLY FIXED. Removed Thresholds POST

Thresholds will now be passed through the config file instead of saving it as a dictionary.

* Bump Dependency Versions
  • Loading branch information
@n0nuser
n0nuser committed Mar 26, 2022
1 parent 9613535 commit 084325b
Show file tree
Hide file tree
Showing 5 changed files with 189 additions and 220 deletions.
57 changes: 56 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,57 @@
# Monitor - Agent
Final Degree Project - Host to monitor

Final Degree Project - Host to monitor

## Uvicorn configuration

In `settings.json` you could pass this parameters to Uvicorn server:

```python
host: str
port: int
uds: str
fd: int
loop: str
http: str
ws: str
ws_max_size: int
ws_ping_interval: float
ws_ping_timeout: float
ws_per_message_deflate: bool
lifespan: str
debug: bool
reload: bool
reload_dirs: typing.List[str]
reload_includes: typing.List[str]
reload_excludes: typing.List[str]
reload_delay: float
workers: int
env_file: str
log_config: str
log_level: str
access_log: bool
proxy_headers: bool
server_header: bool
date_header: bool
forwarded_allow_ips: str
root_path: str
limit_concurrency: int
backlog: int
limit_max_requests: int
timeout_keep_alive: int
ssl_keyfile: str
ssl_certfile: str
ssl_keyfile_password: str
ssl_version: int
ssl_cert_reqs: int
ssl_ca_certs: str
ssl_ciphers: str
headers: typing.List[str]
use_colors: bool
app_dir: str
factory: bool
```

More information in their webpage - [Uvicorn Settings](https://www.uvicorn.org/settings/).

> :warning: If any parameters is incorrect the server will crash.
107 changes: 30 additions & 77 deletions monitor_agent/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,17 @@
import uvicorn
import logging
import requests
from urllib3.exceptions import MaxRetryError, NewConnectionError
from fastapi import FastAPI, UploadFile, File
from fastapi_utils.tasks import repeat_every
from monitor_agent.core.helper import save2log
from .settings import Settings
from .core.metricFunctions import send_metrics, send_metrics_adapter, static, dynamic
from .core.command import Command
from fastapi import FastAPI, UploadFile
from fastapi_utils.tasks import repeat_every
from .core.metricFunctions import send_metrics, send_metrics_adapter, static, dynamic

config_file = Settings()
try:
CONFIG = Settings()
except json.decoder.JSONDecodeError as msg:
print('Error in "settings.json".', msg, file=sys.stderr)
exit()

logger = logging.getLogger(__name__)
api = FastAPI()
Expand All @@ -25,10 +27,6 @@
"thresholds": "/thresholds",
}

thresholds_dict = {
"cpu_percent": 50,
"ram_percent": 30,
}

# GET
@api.get(endpoints["root"])
Expand All @@ -38,10 +36,10 @@ async def root():

@api.get(endpoints["thresholds"])
async def thresholds():
return {"thresholds": thresholds_dict}
return {"thresholds": CONFIG.thresholds.__dict__}


if config_file.metric_endpoint:
if CONFIG.metrics.get_endpoint:

@api.get(endpoints["metrics"])
async def metrics_endpoint():
Expand All @@ -52,56 +50,32 @@ async def metrics_endpoint():
# POST
@api.post(endpoints["command"])
async def command(command: str, timeout: int):
# if token:
# blablabla
return Command(command, timeout).__dict__


@api.post(endpoints["settings"])
async def mod_settings(settings: UploadFile):
# if token:
# blablabla
global CONFIG
data: str = settings.file.read().decode()
return config_file.write_settings(data)


@api.post(endpoints["thresholds"])
async def mod_settings(
cpu_percent: float = thresholds_dict["cpu_percent"],
ram_percent: float = thresholds_dict["ram_percent"],
):
# if token:
# blablabla
thresholds_dict["cpu_percent"] = cpu_percent
thresholds_dict["ram_percent"] = ram_percent
return {"thresholds": thresholds_dict}
msg = CONFIG.write_settings(data)
CONFIG = Settings()
return msg


@api.on_event("startup")
@repeat_every(seconds=config_file.post_interval, logger=logger, wait_first=True)
@repeat_every(seconds=CONFIG.metrics.post_interval, logger=logger, wait_first=True)
def periodic():
# https://github.com/tiangolo/fastapi/issues/520
# https://fastapi-utils.davidmontague.xyz/user-guide/repeated-tasks/#the-repeat_every-decorator
# Changed Timeloop for this
elapsed_time, data = send_metrics_adapter([static, dynamic])
try:
send_metrics(
url=config_file.post_metric_url,
elapsed_time=elapsed_time,
data=data,
file_enabled=config_file.metric_enable_file,
file_path=config_file.metric_file,
)
except (
ConnectionRefusedError,
requests.exceptions.ConnectionError,
NewConnectionError,
MaxRetryError,
) as msg:
save2log(
type="ERROR",
data=f'Could not connect to "{config_file.post_metric_url}" ({msg})',
)
send_metrics(
url=CONFIG.metrics.post_url,
elapsed_time=elapsed_time,
data=data,
file_enabled=CONFIG.metrics.enable_logfile,
file_path=CONFIG.metrics.log_filename,
)

alert = {}
if data["cpu_percent"] >= thresholds_dict["cpu_percent"]:
Expand All @@ -114,36 +88,15 @@ def periodic():
except KeyError as msg:
pass
if alert:
try:
r = requests.post(config_file.post_alert_url, json={"alert": alert})
except requests.exceptions.MissingSchema:
# If invalid URL is provided
save2log(
type="ERROR",
data=f'Invalid POST URL for Alerts ("{config_file.post_alert_url}")',
)
r = requests.post(CONFIG.alerts.url, json={"alert": alert})


def start():
"""Launched with `poetry run start` at root level"""
uvicorn.run(
"monitor_agent.main:api",
host=config_file.host,
port=config_file.port,
reload=config_file.reload,
workers=config_file.workers,
log_level=config_file.log_level,
interface="asgi3",
debug=config_file.debug,
backlog=config_file.backlog,
timeout_keep_alive=config_file.timeout_keep_alive,
limit_concurrency=config_file.limit_concurrency,
limit_max_requests=config_file.limit_max_requests,
ssl_keyfile=config_file.ssl_keyfile,
ssl_keyfile_password=config_file.ssl_keyfile_password,
ssl_certfile=config_file.ssl_certfile,
ssl_version=config_file.ssl_version,
ssl_cert_reqs=config_file.ssl_cert_reqs,
ssl_ca_certs=config_file.ssl_ca_certs,
ssl_ciphers=config_file.ssl_ciphers,
)
uviconfig = {"app": "monitor_agent.main:api", "interface": "asgi3"}
uviconfig.update(CONFIG.uvicorn.__dict__)
uviconfig.pop("__module__", None)
uviconfig.pop("__dict__", None)
uviconfig.pop("__weakref__", None)
uviconfig.pop("__doc__", None)
uvicorn.run(**uviconfig)
47 changes: 24 additions & 23 deletions monitor_agent/settings.json
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,26 @@
{
"backlog": 2048,
"debug": true,
"host": "0.0.0.0",
"limit_concurrency": null,
"limit_max_requests": null,
"log_level": "trace",
"metric_enable_file": false,
"metric_endpoint": true,
"metric_file": "metrics.json",
"port": 8080,
"post_alert_url": "",
"post_interval": 60,
"post_metric_url": "http://httpbin.org/post",
"reload": true,
"ssl_ca_certs": null,
"ssl_cert_reqs": null,
"ssl_certfile": null,
"ssl_ciphers": null,
"ssl_keyfile": null,
"ssl_keyfile_password": null,
"ssl_version": null,
"timeout_keep_alive": 5,
"workers": 4
"alerts": {
"url": "127.0.0.1:8000/alerts"
},
"metrics": {
"enable_logfile": false,
"get_endpoint": true,
"log_filename": "metrics.json",
"post_interval": 60,
"post_url": "http://httpbin.org/post"
},
"thresholds": {
"cpu_percent": 50,
"ram_percent": 30
},
"uvicorn": {
"backlog": 2048,
"debug": true,
"host": "0.0.0.0",
"log_level": "trace",
"port": 8000,
"reload": true,
"timeout_keep_alive": 5,
"workers": 4
}
}
87 changes: 24 additions & 63 deletions monitor_agent/settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,52 +4,26 @@
from monitor_agent.core.helper import save2log


main_parameters = [
"host",
"port",
"workers",
"reload",
"debug",
"log_level",
"backlog",
"timeout_keep_alive",
"metric_endpoint",
"post_metric_url",
"post_interval",
"metric_enable_file",
"metric_file",
"post_alert_url",
]

optional_parameters = [
"ssl_keyfile",
"ssl_keyfile_password",
"ssl_certfile",
"ssl_version",
"ssl_cert_reqs",
"ssl_ca_certs",
"ssl_ciphers",
"limit_concurrency",
"limit_max_requests",
]

config_parameters = main_parameters + optional_parameters
rel_path = "settings.json"
dir = os.path.dirname(__file__) # <-- absolute dir the script is in
abs_file_path = os.path.join(dir, rel_path)


class Settings:
def __init__(self):
for key, value in self._read_settings_file().items():
setattr(self, key, value)
self.as_dict: dict = self._read_settings_file()
obj = toObj(self.as_dict)
self.alerts = obj.alerts
self.metrics = obj.metrics
self.thresholds = obj.thresholds
self.uvicorn = obj.uvicorn

def _read_settings_file(self):
try:
f = open(abs_file_path, "r")
data = f.read()
data_dict = _validate_json(data)
data_str, data_dict = _format_json_file(data_dict, abs_file_path)
data_dict = json.loads(data)
data_str, data_dict = _write_file(data_dict, abs_file_path)
except (json.JSONDecodeError, ValueError, FileNotFoundError) as msg:
print(f"ERROR: Invalid JSON settings file - {msg}", file=sys.stderr)
save2log(type="ERROR", data=f"Invalid JSON file - {msg}")
Expand All @@ -59,8 +33,8 @@ def _read_settings_file(self):

def write_settings(self, data: str):
try:
data_dict = _validate_json(data)
data_str, data_dict = _format_json_file(data_dict, abs_file_path)
data_dict = json.loads(data)
data_str, data_dict = _write_file(data_dict, abs_file_path)
except (json.JSONDecodeError, ValueError) as msg:
return {"status": f"Error: {msg}", "data": data}

Expand All @@ -73,36 +47,23 @@ def write_settings(self, data: str):
######################


def dict_keys_iterator(dictionary: dict):
for key, value in dictionary.items():
if isinstance(value, dict):
dict_keys_iterator(value)
yield key
def toObj(item):
"""Jakub Dóka: https://stackoverflow.com/a/65969444"""
if isinstance(item, dict):
obj = type("__object", (object,), {})

for key, value in item.items():
setattr(obj, key, toObj(value))

def _validate_json(data: str):
data_dict = json.loads(data)
return obj
elif isinstance(item, list):
return map(toObj, item)
else:
return item

keys = [key for key in dict_keys_iterator(data_dict)]

diff_total = list(set(keys) - set(config_parameters))
if len(diff_total) > 0:
raise ValueError(f"Config file contains invalid parameters: {diff_total}")

diff_main = list(set(main_parameters) - set(keys))
if len(diff_main) > 0:
raise ValueError(f"Config file does not contain main parameters: {diff_main}")
return data_dict

# Need to validate type of data


def _format_json_file(data: dict, path: str):
data_dict = data.copy()
for key in optional_parameters:
if key not in data_dict.keys():
data_dict[key] = None
def _write_file(data: dict, path: str):
f = open(path, "w")
data_str: str = json.dumps(data_dict, indent=4, sort_keys=True)
data_str: str = json.dumps(data, indent=4, sort_keys=True)
f.write(data_str)
return data_str, data_dict
return data_str, data
Loading

0 comments on commit 084325b

Please sign in to comment.