From 13b2fea8f72fd47263d932aa75c75a5c0331fee2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Gonz=C3=A1lez=20Rubio?= Date: Thu, 24 Feb 2022 00:08:17 +0100 Subject: [PATCH 01/15] Changed Pydantic for JSON config files. --- .pre-commit-config.yaml | 16 --- monitor-agent/core/metricFunctions.py | 6 +- monitor-agent/main.py | 41 +++--- monitor-agent/settings.json | 31 +++++ monitor-agent/settings.py | 47 ++----- poetry.lock | 175 +------------------------- pyproject.toml | 3 - 7 files changed, 68 insertions(+), 251 deletions(-) delete mode 100644 .pre-commit-config.yaml create mode 100644 monitor-agent/settings.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 8541b7d..0000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 - hooks: - - id: check-yaml - - id: check-toml - - id: check-json - - id: end-of-file-fixer - - id: trailing-whitespace - - id: check-case-conflict - - id: check-docstring-first - -- repo: https://github.com/psf/black - rev: 22.1.0 - hooks: - - id: black diff --git a/monitor-agent/core/metricFunctions.py b/monitor-agent/core/metricFunctions.py index 177c68a..5158951 100644 --- a/monitor-agent/core/metricFunctions.py +++ b/monitor-agent/core/metricFunctions.py @@ -1,4 +1,5 @@ import time +import json import typing import requests from ..settings import settings @@ -45,5 +46,6 @@ def send_metrics(url: str, elapsed_time: dict, data: dict): r = requests.post(url, json=json_request) # DEBUG print(r.status_code) - # with open(settings.metrics_file, "w") as f: - # f.write(json.dumps(json_request, indent=4, sort_keys=True)) + if settings.metrics.enable_file: + with open(settings.metrics.file, "w") as f: + f.write(json.dumps(json_request, indent=4, sort_keys=True)) diff --git a/monitor-agent/main.py b/monitor-agent/main.py index 0fd2b1a..0349c09 100644 --- a/monitor-agent/main.py +++ b/monitor-agent/main.py @@ -28,7 +28,7 @@ async def command(command: str, timeout: int): return Command(command, timeout).__dict__ -if settings.metrics_endpoint: +if settings.metrics.endpoint: @api.get(endpoints["metrics_endpoint"]) async def metrics_endpoint(): @@ -37,36 +37,35 @@ async def metrics_endpoint(): @api.on_event("startup") -@repeat_every(seconds=settings.post_task_interval, logger=logger, wait_first=True) +@repeat_every(seconds=settings.metrics.post_interval, logger=logger, wait_first=True) def periodic(): # https://github.com/tiangolo/fastapi/issues/520 # https://fastapi-utils.davidmontague.xyz/user-guide/repeated-tasks/#the-repeat_every-decorator # Changed Timeloop for this elapsed_time, data = send_metrics_adapter([static, dynamic]) - send_metrics(url=settings.metrics_URL, elapsed_time=elapsed_time, data=data) + send_metrics(url=settings.metrics.url, elapsed_time=elapsed_time, data=data) def start(): """Launched with `poetry run start` at root level""" uvicorn.run( "monitor-agent.main:api", - host=settings.host, - port=settings.port, - reload=settings.reload, - workers=settings.workers, - log_level=settings.log_level, + host=settings.uvicorn.host, + port=settings.uvicorn.port, + reload=settings.uvicorn.reload, + workers=settings.uvicorn.workers, + log_level=settings.uvicorn.log_level, interface="asgi3", - # workers=settings.workers, - debug=settings.debug, - ssl_keyfile=settings.ssl_keyfile, - ssl_keyfile_password=settings.ssl_keyfile_password, - ssl_certfile=settings.ssl_certfile, - ssl_version=settings.ssl_version, - ssl_cert_reqs=settings.ssl_cert_reqs, - ssl_ca_certs=settings.ssl_ca_certs, - ssl_ciphers=settings.ssl_ciphers, - limit_concurrency=settings.limit_concurrency, - limit_max_requests=settings.limit_max_requests, - backlog=settings.backlog, - timeout_keep_alive=settings.timeout_keep_alive, + debug=settings.uvicorn.debug, + ssl_keyfile=settings.uvicorn.ssl_keyfile, + ssl_keyfile_password=settings.uvicorn.ssl_keyfile_password, + ssl_certfile=settings.uvicorn.ssl_certfile, + ssl_version=settings.uvicorn.ssl_version, + ssl_cert_reqs=settings.uvicorn.ssl_cert_reqs, + ssl_ca_certs=settings.uvicorn.ssl_ca_certs, + ssl_ciphers=settings.uvicorn.ssl_ciphers, + limit_concurrency=settings.uvicorn.limit_concurrency, + limit_max_requests=settings.uvicorn.limit_max_requests, + backlog=settings.uvicorn.backlog, + timeout_keep_alive=settings.uvicorn.timeout_keep_alive, ) diff --git a/monitor-agent/settings.json b/monitor-agent/settings.json new file mode 100644 index 0000000..41494d9 --- /dev/null +++ b/monitor-agent/settings.json @@ -0,0 +1,31 @@ +{ + "uvicorn": { + "host":"0.0.0.0", + "port":8000, + "workers":4, + "reload":true, + "debug":true, + "log_level":"trace", + "ssl_keyfile":null, + "ssl_keyfile_password":null, + "ssl_certfile":null, + "ssl_version":3, + "ssl_cert_reqs":null, + "ssl_ca_certs":null, + "ssl_ciphers":"TLSv1", + "limit_concurrency":null, + "limit_max_requests":null, + "backlog":2048, + "timeout_keep_alive":5 + }, + "metrics": { + "endpoint":true, + "url":"http://httpbin.org/post", + "post_interval":60, + "enable_file":false, + "metrics_file":"metrics.json" + }, + "alerts":{ + "url":"" + } +} diff --git a/monitor-agent/settings.py b/monitor-agent/settings.py index 1239bd9..bc1386f 100644 --- a/monitor-agent/settings.py +++ b/monitor-agent/settings.py @@ -1,35 +1,12 @@ -from pydantic import BaseSettings - -# https://www.uvicorn.org/settings/ -class Settings(BaseSettings): - host: str = "0.0.0.0" - port: int = 8000 - workers: int = 4 - - reload: bool = True - debug: bool = True - log_level: str = "trace" - - ssl_keyfile: str = None - ssl_keyfile_password: str = None - ssl_certfile: str = None - ssl_version: int = 3 - ssl_cert_reqs: int = None - ssl_ca_certs: str = None - ssl_ciphers: str = "TLSv1" - - limit_concurrency: int = None - limit_max_requests: int = None - backlog: int = 2048 - - timeout_keep_alive: int = 5 - - metrics_endpoint: bool = True - metrics_URL: str = "http://httpbin.org/post" - post_task_interval: int = 60 - # metrics_file: str = "metrics.json" - - alerts_URL: str = "" - - -settings = Settings() +import os +import json +from types import SimpleNamespace + +dir = os.path.dirname(__file__) # <-- absolute dir the script is in +rel_path = "settings.json" +abs_file_path = os.path.join(dir, rel_path) +f = open(abs_file_path, "r") +data = f.read() +# Parse JSON into an object with attributes corresponding to dict keys. +# https://stackoverflow.com/a/15882054 +settings = json.loads(data, object_hook=lambda d: SimpleNamespace(**d)) diff --git a/poetry.lock b/poetry.lock index 5b45684..08a9a48 100644 --- a/poetry.lock +++ b/poetry.lock @@ -60,14 +60,6 @@ category = "main" optional = false python-versions = "*" -[[package]] -name = "cfgv" -version = "3.3.1" -description = "Validate configuration and produce human readable error messages." -category = "dev" -optional = false -python-versions = ">=3.6.1" - [[package]] name = "charset-normalizer" version = "2.0.12" @@ -172,17 +164,6 @@ python-versions = ">=3.6" [package.dependencies] typing-extensions = {version = "*", markers = "python_version < \"3.8\""} -[[package]] -name = "identify" -version = "2.4.10" -description = "File identification library for Python" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -license = ["ukkonen"] - [[package]] name = "idna" version = "3.3" @@ -216,14 +197,6 @@ category = "dev" optional = false python-versions = "*" -[[package]] -name = "nodeenv" -version = "1.6.0" -description = "Node.js virtual environment builder" -category = "dev" -optional = false -python-versions = "*" - [[package]] name = "packaging" version = "21.3" @@ -262,35 +235,6 @@ importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] -[[package]] -name = "pre-commit" -version = "2.17.0" -description = "A framework for managing and maintaining multi-language pre-commit hooks." -category = "dev" -optional = false -python-versions = ">=3.6.1" - -[package.dependencies] -cfgv = ">=2.0.0" -identify = ">=1.0.0" -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} -nodeenv = ">=0.11.1" -pyyaml = ">=5.1" -toml = "*" -virtualenv = ">=20.0.8" - -[[package]] -name = "pre-commit-hooks" -version = "4.1.0" -description = "Some out-of-the-box hooks for pre-commit." -category = "dev" -optional = false -python-versions = ">=3.6.1" - -[package.dependencies] -"ruamel.yaml" = ">=0.15" -toml = "*" - [[package]] name = "psutil" version = "5.9.0" @@ -358,14 +302,6 @@ tomli = ">=1.0.0" [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] -[[package]] -name = "pyyaml" -version = "6.0" -description = "YAML parser and emitter for Python" -category = "dev" -optional = false -python-versions = ">=3.6" - [[package]] name = "requests" version = "2.27.1" @@ -384,29 +320,6 @@ urllib3 = ">=1.21.1,<1.27" socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] -[[package]] -name = "ruamel.yaml" -version = "0.17.21" -description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" -category = "dev" -optional = false -python-versions = ">=3" - -[package.dependencies] -"ruamel.yaml.clib" = {version = ">=0.2.6", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.11\""} - -[package.extras] -docs = ["ryd"] -jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] - -[[package]] -name = "ruamel.yaml.clib" -version = "0.2.6" -description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" -category = "dev" -optional = false -python-versions = ">=3.5" - [[package]] name = "six" version = "1.16.0" @@ -582,7 +495,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = ">=3.7,<4.0.0" -content-hash = "863b3f9b6b377b783c7474bdac3864ae17dde1f8e348f55348f57ead72a1f5f6" +content-hash = "a2c8e6f013762686c1ad828c7656801b6bb8c83fceba639398b65f24e7689353" [metadata.files] anyio = [ @@ -605,10 +518,6 @@ certifi = [ {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, ] -cfgv = [ - {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, - {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, -] charset-normalizer = [ {file = "charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"}, {file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"}, @@ -698,10 +607,6 @@ h11 = [ {file = "h11-0.13.0-py3-none-any.whl", hash = "sha256:8ddd78563b633ca55346c8cd41ec0af27d3c79931828beffb46ce70a379e7442"}, {file = "h11-0.13.0.tar.gz", hash = "sha256:70813c1135087a248a4d38cc0e1a0181ffab2188141a93eaf567940c3957ff06"}, ] -identify = [ - {file = "identify-2.4.10-py2.py3-none-any.whl", hash = "sha256:7d10baf6ba6f1912a0a49f4c1c2c49fa1718765c3a37d72d13b07779567c5b85"}, - {file = "identify-2.4.10.tar.gz", hash = "sha256:e12b2aea3cf108de73ae055c2260783bde6601de09718f6768cf8e9f6f6322a6"}, -] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, @@ -714,10 +619,6 @@ iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] -nodeenv = [ - {file = "nodeenv-1.6.0-py2.py3-none-any.whl", hash = "sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7"}, - {file = "nodeenv-1.6.0.tar.gz", hash = "sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b"}, -] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, @@ -730,14 +631,6 @@ pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -pre-commit = [ - {file = "pre_commit-2.17.0-py2.py3-none-any.whl", hash = "sha256:725fa7459782d7bec5ead072810e47351de01709be838c2ce1726b9591dad616"}, - {file = "pre_commit-2.17.0.tar.gz", hash = "sha256:c1a8040ff15ad3d648c70cc3e55b93e4d2d5b687320955505587fd79bbaed06a"}, -] -pre-commit-hooks = [ - {file = "pre_commit_hooks-4.1.0-py2.py3-none-any.whl", hash = "sha256:ba95316b79038e56ce998cdacb1ce922831ac0e41744c77bcc2b9677bf183206"}, - {file = "pre_commit_hooks-4.1.0.tar.gz", hash = "sha256:b6361865d1877c5da5ac3a944aab19ce6bd749a534d2ede28e683d07194a57e1"}, -] psutil = [ {file = "psutil-5.9.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:55ce319452e3d139e25d6c3f85a1acf12d1607ddedea5e35fb47a552c051161b"}, {file = "psutil-5.9.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:7336292a13a80eb93c21f36bde4328aa748a04b68c13d01dfddd67fc13fd0618"}, @@ -821,76 +714,10 @@ pytest = [ {file = "pytest-7.0.1-py3-none-any.whl", hash = "sha256:9ce3ff477af913ecf6321fe337b93a2c0dcf2a0a1439c43f5452112c1e4280db"}, {file = "pytest-7.0.1.tar.gz", hash = "sha256:e30905a0c131d3d94b89624a1cc5afec3e0ba2fbdb151867d8e0ebd49850f171"}, ] -pyyaml = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, -] requests = [ {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, ] -"ruamel.yaml" = [ - {file = "ruamel.yaml-0.17.21-py3-none-any.whl", hash = "sha256:742b35d3d665023981bd6d16b3d24248ce5df75fdb4e2924e93a05c1f8b61ca7"}, - {file = "ruamel.yaml-0.17.21.tar.gz", hash = "sha256:8b7ce697a2f212752a35c1ac414471dc16c424c9573be4926b56ff3f5d23b7af"}, -] -"ruamel.yaml.clib" = [ - {file = "ruamel.yaml.clib-0.2.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6e7be2c5bcb297f5b82fee9c665eb2eb7001d1050deaba8471842979293a80b0"}, - {file = "ruamel.yaml.clib-0.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:221eca6f35076c6ae472a531afa1c223b9c29377e62936f61bc8e6e8bdc5f9e7"}, - {file = "ruamel.yaml.clib-0.2.6-cp310-cp310-win32.whl", hash = "sha256:1070ba9dd7f9370d0513d649420c3b362ac2d687fe78c6e888f5b12bf8bc7bee"}, - {file = "ruamel.yaml.clib-0.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:77df077d32921ad46f34816a9a16e6356d8100374579bc35e15bab5d4e9377de"}, - {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:cfdb9389d888c5b74af297e51ce357b800dd844898af9d4a547ffc143fa56751"}, - {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:7b2927e92feb51d830f531de4ccb11b320255ee95e791022555971c466af4527"}, - {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-win32.whl", hash = "sha256:ada3f400d9923a190ea8b59c8f60680c4ef8a4b0dfae134d2f2ff68429adfab5"}, - {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-win_amd64.whl", hash = "sha256:de9c6b8a1ba52919ae919f3ae96abb72b994dd0350226e28f3686cb4f142165c"}, - {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d67f273097c368265a7b81e152e07fb90ed395df6e552b9fa858c6d2c9f42502"}, - {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:72a2b8b2ff0a627496aad76f37a652bcef400fd861721744201ef1b45199ab78"}, - {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-win32.whl", hash = "sha256:9efef4aab5353387b07f6b22ace0867032b900d8e91674b5d8ea9150db5cae94"}, - {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-win_amd64.whl", hash = "sha256:846fc8336443106fe23f9b6d6b8c14a53d38cef9a375149d61f99d78782ea468"}, - {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0847201b767447fc33b9c235780d3aa90357d20dd6108b92be544427bea197dd"}, - {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:78988ed190206672da0f5d50c61afef8f67daa718d614377dcd5e3ed85ab4a99"}, - {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-win32.whl", hash = "sha256:a49e0161897901d1ac9c4a79984b8410f450565bbad64dbfcbf76152743a0cdb"}, - {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-win_amd64.whl", hash = "sha256:bf75d28fa071645c529b5474a550a44686821decebdd00e21127ef1fd566eabe"}, - {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a32f8d81ea0c6173ab1b3da956869114cae53ba1e9f72374032e33ba3118c233"}, - {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7f7ecb53ae6848f959db6ae93bdff1740e651809780822270eab111500842a84"}, - {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-win32.whl", hash = "sha256:89221ec6d6026f8ae859c09b9718799fea22c0e8da8b766b0b2c9a9ba2db326b"}, - {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-win_amd64.whl", hash = "sha256:31ea73e564a7b5fbbe8188ab8b334393e06d997914a4e184975348f204790277"}, - {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dc6a613d6c74eef5a14a214d433d06291526145431c3b964f5e16529b1842bed"}, - {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:1866cf2c284a03b9524a5cc00daca56d80057c5ce3cdc86a52020f4c720856f0"}, - {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-win32.whl", hash = "sha256:3fb9575a5acd13031c57a62cc7823e5d2ff8bc3835ba4d94b921b4e6ee664104"}, - {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-win_amd64.whl", hash = "sha256:825d5fccef6da42f3c8eccd4281af399f21c02b32d98e113dbc631ea6a6ecbc7"}, - {file = "ruamel.yaml.clib-0.2.6.tar.gz", hash = "sha256:4ff604ce439abb20794f05613c374759ce10e3595d1867764dd1ae675b85acbd"}, -] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, diff --git a/pyproject.toml b/pyproject.toml index dc11559..8ce5df3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,13 +19,10 @@ psutil = "^5.9.0" fastapi = "^0.74.1" fastapi-utils = "^0.2.1" uvicorn = "^0.17.4" -pydantic = "^1.9.0" [tool.poetry.dev-dependencies] pytest = "^7.0.0" tox = "^3.24.5" -pre-commit-hooks = "^4.1.0" -pre-commit = "^2.17.0" [build-system] requires = ["poetry-core>=1.0.0"] From 1f12ed4450be68159da3003452589ec8c82426ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Gonz=C3=A1lez=20Rubio?= Date: Thu, 24 Feb 2022 14:49:06 +0100 Subject: [PATCH 02/15] Fixed poetry entry-point. It threw the error 'EntryPoint must be in 'name=module:attrs [extras]' format'. Fixed it changing the '-' in 'monitor-agent' for a '_'. --- monitor-agent/main.py | 71 ------------- monitor-agent/settings.json | 31 ------ monitor-agent/settings.py | 12 --- {monitor-agent => monitor_agent}/__init__.py | 0 .../core/__init__.py | 0 .../core/command.py | 0 .../core/metricFunctions.py | 9 +- .../core/models/__init__.py | 0 .../core/models/metricModel.py | 0 .../core/schemas/__init__.py | 0 monitor_agent/main.py | 100 ++++++++++++++++++ monitor_agent/settings.json | 25 +++++ monitor_agent/settings.py | 54 ++++++++++ .../tests/__init__.py | 0 poetry.lock | 24 ++++- pyproject.toml | 10 +- 16 files changed, 209 insertions(+), 127 deletions(-) delete mode 100644 monitor-agent/main.py delete mode 100644 monitor-agent/settings.json delete mode 100644 monitor-agent/settings.py rename {monitor-agent => monitor_agent}/__init__.py (100%) rename {monitor-agent => monitor_agent}/core/__init__.py (100%) rename {monitor-agent => monitor_agent}/core/command.py (100%) rename {monitor-agent => monitor_agent}/core/metricFunctions.py (88%) rename {monitor-agent => monitor_agent}/core/models/__init__.py (100%) rename {monitor-agent => monitor_agent}/core/models/metricModel.py (100%) rename {monitor-agent => monitor_agent}/core/schemas/__init__.py (100%) create mode 100644 monitor_agent/main.py create mode 100644 monitor_agent/settings.json create mode 100644 monitor_agent/settings.py rename {monitor-agent => monitor_agent}/tests/__init__.py (100%) diff --git a/monitor-agent/main.py b/monitor-agent/main.py deleted file mode 100644 index 0349c09..0000000 --- a/monitor-agent/main.py +++ /dev/null @@ -1,71 +0,0 @@ -import uvicorn -import logging -from fastapi import FastAPI -from fastapi_utils.tasks import repeat_every -from .settings import settings -from .core.metricFunctions import send_metrics, send_metrics_adapter, static, dynamic -from .core.command import Command - -logger = logging.getLogger(__name__) -api = FastAPI() - -endpoints = { - "root": "/", - "command": "/command", - "metrics_endpoint": "/metrics", -} - - -@api.get(endpoints["root"]) -async def root(): - return {"message": "Hello World"} - - -@api.post(endpoints["command"]) -async def command(command: str, timeout: int): - # if token: - # blablabla - return Command(command, timeout).__dict__ - - -if settings.metrics.endpoint: - - @api.get(endpoints["metrics_endpoint"]) - async def metrics_endpoint(): - elapsed_time, data = send_metrics_adapter([static, dynamic]) - return {"data": data, "elapsed_time": elapsed_time} - - -@api.on_event("startup") -@repeat_every(seconds=settings.metrics.post_interval, logger=logger, wait_first=True) -def periodic(): - # https://github.com/tiangolo/fastapi/issues/520 - # https://fastapi-utils.davidmontague.xyz/user-guide/repeated-tasks/#the-repeat_every-decorator - # Changed Timeloop for this - elapsed_time, data = send_metrics_adapter([static, dynamic]) - send_metrics(url=settings.metrics.url, elapsed_time=elapsed_time, data=data) - - -def start(): - """Launched with `poetry run start` at root level""" - uvicorn.run( - "monitor-agent.main:api", - host=settings.uvicorn.host, - port=settings.uvicorn.port, - reload=settings.uvicorn.reload, - workers=settings.uvicorn.workers, - log_level=settings.uvicorn.log_level, - interface="asgi3", - debug=settings.uvicorn.debug, - ssl_keyfile=settings.uvicorn.ssl_keyfile, - ssl_keyfile_password=settings.uvicorn.ssl_keyfile_password, - ssl_certfile=settings.uvicorn.ssl_certfile, - ssl_version=settings.uvicorn.ssl_version, - ssl_cert_reqs=settings.uvicorn.ssl_cert_reqs, - ssl_ca_certs=settings.uvicorn.ssl_ca_certs, - ssl_ciphers=settings.uvicorn.ssl_ciphers, - limit_concurrency=settings.uvicorn.limit_concurrency, - limit_max_requests=settings.uvicorn.limit_max_requests, - backlog=settings.uvicorn.backlog, - timeout_keep_alive=settings.uvicorn.timeout_keep_alive, - ) diff --git a/monitor-agent/settings.json b/monitor-agent/settings.json deleted file mode 100644 index 41494d9..0000000 --- a/monitor-agent/settings.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "uvicorn": { - "host":"0.0.0.0", - "port":8000, - "workers":4, - "reload":true, - "debug":true, - "log_level":"trace", - "ssl_keyfile":null, - "ssl_keyfile_password":null, - "ssl_certfile":null, - "ssl_version":3, - "ssl_cert_reqs":null, - "ssl_ca_certs":null, - "ssl_ciphers":"TLSv1", - "limit_concurrency":null, - "limit_max_requests":null, - "backlog":2048, - "timeout_keep_alive":5 - }, - "metrics": { - "endpoint":true, - "url":"http://httpbin.org/post", - "post_interval":60, - "enable_file":false, - "metrics_file":"metrics.json" - }, - "alerts":{ - "url":"" - } -} diff --git a/monitor-agent/settings.py b/monitor-agent/settings.py deleted file mode 100644 index bc1386f..0000000 --- a/monitor-agent/settings.py +++ /dev/null @@ -1,12 +0,0 @@ -import os -import json -from types import SimpleNamespace - -dir = os.path.dirname(__file__) # <-- absolute dir the script is in -rel_path = "settings.json" -abs_file_path = os.path.join(dir, rel_path) -f = open(abs_file_path, "r") -data = f.read() -# Parse JSON into an object with attributes corresponding to dict keys. -# https://stackoverflow.com/a/15882054 -settings = json.loads(data, object_hook=lambda d: SimpleNamespace(**d)) diff --git a/monitor-agent/__init__.py b/monitor_agent/__init__.py similarity index 100% rename from monitor-agent/__init__.py rename to monitor_agent/__init__.py diff --git a/monitor-agent/core/__init__.py b/monitor_agent/core/__init__.py similarity index 100% rename from monitor-agent/core/__init__.py rename to monitor_agent/core/__init__.py diff --git a/monitor-agent/core/command.py b/monitor_agent/core/command.py similarity index 100% rename from monitor-agent/core/command.py rename to monitor_agent/core/command.py diff --git a/monitor-agent/core/metricFunctions.py b/monitor_agent/core/metricFunctions.py similarity index 88% rename from monitor-agent/core/metricFunctions.py rename to monitor_agent/core/metricFunctions.py index 5158951..9f0b221 100644 --- a/monitor-agent/core/metricFunctions.py +++ b/monitor_agent/core/metricFunctions.py @@ -2,7 +2,6 @@ import json import typing import requests -from ..settings import settings from .models.metricModel import Status, MetricDynamic, MetricStatic # Typing Tuple is used to overcome Python ^3.6 until Python 3.10 problem @@ -40,12 +39,14 @@ def send_metrics_adapter(function_list: list) -> typing.Tuple[dict, dict]: return elapsed_time, data -def send_metrics(url: str, elapsed_time: dict, data: dict): +def send_metrics( + url: str, elapsed_time: dict, data: dict, file_enabled: bool, file_path: str +): status = Status(elapsed=elapsed_time).__dict__ json_request = {"data": data, "status": status} r = requests.post(url, json=json_request) # DEBUG print(r.status_code) - if settings.metrics.enable_file: - with open(settings.metrics.file, "w") as f: + if file_enabled: + with open(file_path, "w") as f: f.write(json.dumps(json_request, indent=4, sort_keys=True)) diff --git a/monitor-agent/core/models/__init__.py b/monitor_agent/core/models/__init__.py similarity index 100% rename from monitor-agent/core/models/__init__.py rename to monitor_agent/core/models/__init__.py diff --git a/monitor-agent/core/models/metricModel.py b/monitor_agent/core/models/metricModel.py similarity index 100% rename from monitor-agent/core/models/metricModel.py rename to monitor_agent/core/models/metricModel.py diff --git a/monitor-agent/core/schemas/__init__.py b/monitor_agent/core/schemas/__init__.py similarity index 100% rename from monitor-agent/core/schemas/__init__.py rename to monitor_agent/core/schemas/__init__.py diff --git a/monitor_agent/main.py b/monitor_agent/main.py new file mode 100644 index 0000000..c9fea71 --- /dev/null +++ b/monitor_agent/main.py @@ -0,0 +1,100 @@ +import uvicorn +import logging +from fastapi import FastAPI, UploadFile, File +from typing import List +from fastapi_utils.tasks import repeat_every +from .settings import Settings, SettingsPydantic +from .core.metricFunctions import send_metrics, send_metrics_adapter, static, dynamic +from .core.command import Command + +logger = logging.getLogger(__name__) +config_file = Settings() +api = FastAPI() + +endpoints = { + "root": "/", + "command": "/command", + "metrics_endpoint": "/metrics", + "settings": "/settings", +} + +# GET +@api.get(endpoints["root"]) +async def root(): + return {"message": "Hello World"} + + +@api.get(endpoints["settings"]) +async def mod_settings(): + # if token: + # blablabla + return config_file.read_settings() + + +if config_file.settings.metric_endpoint: + + @api.get(endpoints["metrics_endpoint"]) + async def metrics_endpoint(): + elapsed_time, data = send_metrics_adapter([static, dynamic]) + return {"data": data, "elapsed_time": elapsed_time} + + +# POST +@api.post(endpoints["command"]) +async def command(command: str, timeout: int): + # if token: + # blablabla + return Command(command, timeout).__dict__ + + +@api.post(endpoints["settings"]) +async def mod_settings(settings: List[UploadFile]): + # if token: + # blablabla + print(settings) + settings = await settings.json() + print(settings) + return config_file.write_settings(settings) + + +@api.on_event("startup") +@repeat_every( + seconds=config_file.settings.post_interval, logger=logger, wait_first=True +) +def periodic(): + # https://github.com/tiangolo/fastapi/issues/520 + # https://fastapi-utils.davidmontague.xyz/user-guide/repeated-tasks/#the-repeat_every-decorator + # Changed Timeloop for this + elapsed_time, data = send_metrics_adapter([static, dynamic]) + send_metrics( + url=config_file.settings.post_metric_url, + elapsed_time=elapsed_time, + data=data, + file_enabled=config_file.settings.metric_enable_file, + file_path=config_file.abs_file_path, + ) + + +def start(): + """Launched with `poetry run start` at root level""" + uvicorn.run( + "monitor_agent.main:api", + host=config_file.settings.host, + port=config_file.settings.port, + reload=config_file.settings.reload, + workers=config_file.settings.workers, + log_level=config_file.settings.log_level, + interface="asgi3", + debug=config_file.settings.debug, + ssl_keyfile=config_file.settings.ssl_keyfile, + ssl_keyfile_password=config_file.settings.ssl_keyfile_password, + ssl_certfile=config_file.settings.ssl_certfile, + ssl_version=config_file.settings.ssl_version, + ssl_cert_reqs=config_file.settings.ssl_cert_reqs, + ssl_ca_certs=config_file.settings.ssl_ca_certs, + ssl_ciphers=config_file.settings.ssl_ciphers, + limit_concurrency=config_file.settings.limit_concurrency, + limit_max_requests=config_file.settings.limit_max_requests, + backlog=config_file.settings.backlog, + timeout_keep_alive=config_file.settings.timeout_keep_alive, + ) diff --git a/monitor_agent/settings.json b/monitor_agent/settings.json new file mode 100644 index 0000000..557e139 --- /dev/null +++ b/monitor_agent/settings.json @@ -0,0 +1,25 @@ +{ + "host":"0.0.0.0", + "port":8000, + "workers":4, + "reload":true, + "debug":true, + "log_level":"trace", + "ssl_keyfile":null, + "ssl_keyfile_password":null, + "ssl_certfile":null, + "ssl_version":3, + "ssl_cert_reqs":null, + "ssl_ca_certs":null, + "ssl_ciphers":"TLSv1", + "limit_concurrency":null, + "limit_max_requests":null, + "backlog":2048, + "timeout_keep_alive":5, + "metric_endpoint":true, + "post_metric_url":"http://httpbin.org/post", + "post_interval":60, + "metric_enable_file":false, + "metrics_file":"metrics.json", + "alert_url":"" +} diff --git a/monitor_agent/settings.py b/monitor_agent/settings.py new file mode 100644 index 0000000..a1bd0c2 --- /dev/null +++ b/monitor_agent/settings.py @@ -0,0 +1,54 @@ +import os +import json +from types import SimpleNamespace +from pydantic import BaseSettings + +# https://www.uvicorn.org/settings/ +class SettingsPydantic(BaseSettings): + host: str + port: int + workers: int + reload: bool + debug: bool + log_level: str + ssl_keyfile: str | None = None + ssl_keyfile_password: str | None = None + ssl_certfile: str | None = None + ssl_version: int | None = None + ssl_cert_reqs: int | None = None + ssl_ca_certs: str | None = None + ssl_ciphers: str | None = None + limit_concurrency: int + limit_max_requests: int + backlog: int + timeout_keep_alive: int + metric_endpoint: bool + post_metric_url: str + post_interval: int + metric_enable_file: bool + metric_file: str + alerts_URL: str + + +class Settings: + def __init__(self, rel_path: str = "settings.json"): + dir = os.path.dirname(__file__) # <-- absolute dir the script is in + self.abs_file_path = os.path.join(dir, rel_path) + self.settings = self.read_settings() + + def read_settings(self): + f = open(self.abs_file_path, "r") + data = f.read() + # Parse JSON into an object with attributes corresponding to dict keys. + # https://stackoverflow.com/a/15882054 + return json.loads(data, object_hook=lambda d: SimpleNamespace(**d)) + + def write_settings(self, new_data): + try: + json.loads(new_data) + except json.JSONDecodeError as msg: + return {"status": msg.msg, "data": new_data} + f = open(self.abs_file_path, "w") + f.write(new_data) + self.settings = self.read_settings() + return {"status": "success", "data": new_data} diff --git a/monitor-agent/tests/__init__.py b/monitor_agent/tests/__init__.py similarity index 100% rename from monitor-agent/tests/__init__.py rename to monitor_agent/tests/__init__.py diff --git a/poetry.lock b/poetry.lock index 08a9a48..4a70dcc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -302,6 +302,17 @@ tomli = ">=1.0.0" [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +[[package]] +name = "python-multipart" +version = "0.0.5" +description = "A streaming multipart parser for Python" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = ">=1.4.0" + [[package]] name = "requests" version = "2.27.1" @@ -324,7 +335,7 @@ use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" @@ -463,7 +474,7 @@ standard = ["websockets (>=10.0)", "httptools (>=0.2.0,<0.4.0)", "watchgod (>=0. [[package]] name = "virtualenv" -version = "20.13.1" +version = "20.13.2" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -495,7 +506,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = ">=3.7,<4.0.0" -content-hash = "a2c8e6f013762686c1ad828c7656801b6bb8c83fceba639398b65f24e7689353" +content-hash = "44aee2afc7d30d00b6f66d462ae6f2f3793c76c8789b188753eee882c577355f" [metadata.files] anyio = [ @@ -714,6 +725,9 @@ pytest = [ {file = "pytest-7.0.1-py3-none-any.whl", hash = "sha256:9ce3ff477af913ecf6321fe337b93a2c0dcf2a0a1439c43f5452112c1e4280db"}, {file = "pytest-7.0.1.tar.gz", hash = "sha256:e30905a0c131d3d94b89624a1cc5afec3e0ba2fbdb151867d8e0ebd49850f171"}, ] +python-multipart = [ + {file = "python-multipart-0.0.5.tar.gz", hash = "sha256:f7bb5f611fc600d15fa47b3974c8aa16e93724513b49b5f95c81e6624c83fa43"}, +] requests = [ {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, @@ -793,8 +807,8 @@ uvicorn = [ {file = "uvicorn-0.17.5.tar.gz", hash = "sha256:c04a9c069111489c324f427501b3840d306c6b91a77b00affc136a840a3f45f1"}, ] virtualenv = [ - {file = "virtualenv-20.13.1-py2.py3-none-any.whl", hash = "sha256:45e1d053cad4cd453181ae877c4ffc053546ae99e7dd049b9ff1d9be7491abf7"}, - {file = "virtualenv-20.13.1.tar.gz", hash = "sha256:e0621bcbf4160e4e1030f05065c8834b4e93f4fcc223255db2a823440aca9c14"}, + {file = "virtualenv-20.13.2-py2.py3-none-any.whl", hash = "sha256:e7b34c9474e6476ee208c43a4d9ac1510b041c68347eabfe9a9ea0c86aa0a46b"}, + {file = "virtualenv-20.13.2.tar.gz", hash = "sha256:01f5f80744d24a3743ce61858123488e91cb2dd1d3bdf92adaf1bba39ffdedf0"}, ] zipp = [ {file = "zipp-3.7.0-py3-none-any.whl", hash = "sha256:b47250dd24f92b7dd6a0a8fc5244da14608f3ca90a5efcd37a3b1642fac9a375"}, diff --git a/pyproject.toml b/pyproject.toml index 8ce5df3..0d04135 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,16 +1,16 @@ [tool.poetry] -name = "monitor-agent" +name = "monitor_agent" version = "0.1.0" description = "Monitor agent that sends and receives metrics and info from the REST API server." authors = ["Pablo González Rubio "] license = "GNU General Public License v3.0" packages = [ - { include = "monitor-agent"}, - { include = "monitor-agent/**/*.py" }, + { include = "monitor_agent"}, + { include = "monitor_agent/**/*.py" }, ] [tool.poetry.scripts] -start = "monitor-agent.main:start" +start = "monitor_agent.main:start" [tool.poetry.dependencies] python = ">=3.7,<4.0.0" @@ -19,6 +19,8 @@ psutil = "^5.9.0" fastapi = "^0.74.1" fastapi-utils = "^0.2.1" uvicorn = "^0.17.4" +pydantic = "^1.9.0" +python-multipart = "^0.0.5" [tool.poetry.dev-dependencies] pytest = "^7.0.0" From c936940ff56aa5c2a20b08fadeaeea90bce1cb63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Gonz=C3=A1lez=20Rubio?= Date: Thu, 24 Feb 2022 23:45:23 +0100 Subject: [PATCH 03/15] Implemented receiving settings as a JSON ->FILE<- (not text). Need to fix testing of data types when a JSON is received --- monitor_agent/core/models/metricModel.py | 10 +- monitor_agent/main.py | 30 +++--- monitor_agent/settings.json | 48 ++++----- monitor_agent/settings.py | 132 ++++++++++++++++------- poetry.lock | 2 +- pyproject.toml | 1 - 6 files changed, 144 insertions(+), 79 deletions(-) diff --git a/monitor_agent/core/models/metricModel.py b/monitor_agent/core/models/metricModel.py index 39d9265..8b0e133 100644 --- a/monitor_agent/core/models/metricModel.py +++ b/monitor_agent/core/models/metricModel.py @@ -106,11 +106,15 @@ def _process(ram: int, pc_cpu_percent): "name": p.name(), "cpu_percent": cpu_percent, "ram_percent": ram_percent, - "username": p.username(), "ppid": p.ppid(), - # Requires elevated permissions - # "path": p.exe() } + try: + # Requires elevated permissions SOMETIMES + process[p.pid]["username"] = p.username() + # Requires elevated permissions + process[p.pid]["path"] = p.exe() + except (PermissionError, psutil.AccessDenied): + pass return process diff --git a/monitor_agent/main.py b/monitor_agent/main.py index c9fea71..57f60fd 100644 --- a/monitor_agent/main.py +++ b/monitor_agent/main.py @@ -1,14 +1,20 @@ +import sys +import json import uvicorn import logging from fastapi import FastAPI, UploadFile, File -from typing import List from fastapi_utils.tasks import repeat_every -from .settings import Settings, SettingsPydantic +from .settings import Settings from .core.metricFunctions import send_metrics, send_metrics_adapter, static, dynamic from .core.command import Command +try: + config_file = Settings() +except json.decoder.JSONDecodeError as msg: + print('Error in "settings.json".', msg, file=sys.stderr) + exit() + logger = logging.getLogger(__name__) -config_file = Settings() api = FastAPI() endpoints = { @@ -28,7 +34,7 @@ async def root(): async def mod_settings(): # if token: # blablabla - return config_file.read_settings() + return config_file.read_settings_file(config_file.abs_file_path) if config_file.settings.metric_endpoint: @@ -48,13 +54,11 @@ async def command(command: str, timeout: int): @api.post(endpoints["settings"]) -async def mod_settings(settings: List[UploadFile]): +async def mod_settings(settings: UploadFile): # if token: # blablabla - print(settings) - settings = await settings.json() - print(settings) - return config_file.write_settings(settings) + data: str = settings.file.read().decode() + return config_file.write_settings(data) @api.on_event("startup") @@ -86,6 +90,10 @@ def start(): log_level=config_file.settings.log_level, interface="asgi3", debug=config_file.settings.debug, + backlog=config_file.settings.backlog, + timeout_keep_alive=config_file.settings.timeout_keep_alive, + limit_concurrency=config_file.settings.limit_concurrency, + limit_max_requests=config_file.settings.limit_max_requests, ssl_keyfile=config_file.settings.ssl_keyfile, ssl_keyfile_password=config_file.settings.ssl_keyfile_password, ssl_certfile=config_file.settings.ssl_certfile, @@ -93,8 +101,4 @@ def start(): ssl_cert_reqs=config_file.settings.ssl_cert_reqs, ssl_ca_certs=config_file.settings.ssl_ca_certs, ssl_ciphers=config_file.settings.ssl_ciphers, - limit_concurrency=config_file.settings.limit_concurrency, - limit_max_requests=config_file.settings.limit_max_requests, - backlog=config_file.settings.backlog, - timeout_keep_alive=config_file.settings.timeout_keep_alive, ) diff --git a/monitor_agent/settings.json b/monitor_agent/settings.json index 557e139..3d73910 100644 --- a/monitor_agent/settings.json +++ b/monitor_agent/settings.json @@ -1,25 +1,25 @@ { - "host":"0.0.0.0", - "port":8000, - "workers":4, - "reload":true, - "debug":true, - "log_level":"trace", - "ssl_keyfile":null, - "ssl_keyfile_password":null, - "ssl_certfile":null, - "ssl_version":3, - "ssl_cert_reqs":null, - "ssl_ca_certs":null, - "ssl_ciphers":"TLSv1", - "limit_concurrency":null, - "limit_max_requests":null, - "backlog":2048, - "timeout_keep_alive":5, - "metric_endpoint":true, - "post_metric_url":"http://httpbin.org/post", - "post_interval":60, - "metric_enable_file":false, - "metrics_file":"metrics.json", - "alert_url":"" -} + "alert_url": "", + "backlog": 2048, + "debug": true, + "host": "0.0.0.0", + "limit_concurrency": null, + "limit_max_requests": null, + "log_level": "trace", + "metric_enable_file": false, + "metric_endpoint": true, + "metric_file": "metrics.json", + "port": 8000, + "post_interval": 60, + "post_metric_url": "http://httpbin.org/post", + "reload": true, + "ssl_ca_certs": null, + "ssl_cert_reqs": null, + "ssl_certfile": null, + "ssl_ciphers": "TLSv1", + "ssl_keyfile": null, + "ssl_keyfile_password": null, + "ssl_version": 3, + "timeout_keep_alive": 5, + "workers": 4 +} \ No newline at end of file diff --git a/monitor_agent/settings.py b/monitor_agent/settings.py index a1bd0c2..ca971bb 100644 --- a/monitor_agent/settings.py +++ b/monitor_agent/settings.py @@ -1,54 +1,112 @@ +from multiprocessing.sharedctypes import Value import os +import sys import json from types import SimpleNamespace -from pydantic import BaseSettings - -# https://www.uvicorn.org/settings/ -class SettingsPydantic(BaseSettings): - host: str - port: int - workers: int - reload: bool - debug: bool - log_level: str - ssl_keyfile: str | None = None - ssl_keyfile_password: str | None = None - ssl_certfile: str | None = None - ssl_version: int | None = None - ssl_cert_reqs: int | None = None - ssl_ca_certs: str | None = None - ssl_ciphers: str | None = None - limit_concurrency: int - limit_max_requests: int - backlog: int - timeout_keep_alive: int - metric_endpoint: bool - post_metric_url: str - post_interval: int - metric_enable_file: bool - metric_file: str - alerts_URL: str + +main_parameters = [ + "host", + "port", + "workers", + "reload", + "debug", + "log_level", + "backlog", + "timeout_keep_alive", + "metric_endpoint", + "post_metric_url", + "post_interval", + "metric_enable_file", + "metric_file", + "alert_url", +] + +optional_parameters = [ + "ssl_keyfile", + "ssl_keyfile_password", + "ssl_certfile", + "ssl_version", + "ssl_cert_reqs", + "ssl_ca_certs", + "ssl_ciphers", + "limit_concurrency", + "limit_max_requests", +] + +config_parameters = main_parameters + optional_parameters + + +def dict_keys_iterator(dictionary: dict): + for key, value in dictionary.items(): + if isinstance(value, dict): + dict_keys_iterator(value) + yield key class Settings: def __init__(self, rel_path: str = "settings.json"): dir = os.path.dirname(__file__) # <-- absolute dir the script is in self.abs_file_path = os.path.join(dir, rel_path) - self.settings = self.read_settings() + self.settings = self.read_settings_file(self.abs_file_path) + + def _validate_json(self, config_parameters: list, data: dict): + keys = [key for key in dict_keys_iterator(data)] + + diff_total = list(set(keys) - set(config_parameters)) + if len(diff_total) > 0: + raise ValueError(f"Config file contains invalid parameters: {diff_total}") + + diff_main = list(set(main_parameters) - set(keys)) + if len(diff_main) > 0: + raise ValueError( + f"Config file does not contain main parameters: {diff_main}" + ) + + # Need to validate type of data + + def _fill_json(self, optional_parameters: list, json_dict: dict): + fill_dict = json_dict.copy() + for key in optional_parameters: + if key not in fill_dict.keys(): + fill_dict[key] = None + return fill_dict - def read_settings(self): - f = open(self.abs_file_path, "r") + def read_settings_file(self, file_path: str): + try: + f = open(file_path, "r") + except FileNotFoundError as msg: + print(msg, file=sys.stderr) + exit() data = f.read() - # Parse JSON into an object with attributes corresponding to dict keys. + + try: + json_dict = json.loads(data) + self._validate_json(config_parameters, json_dict) + except (json.JSONDecodeError, ValueError) as msg: + print(msg, file=sys.stderr) + exit() + + json_dict = self._fill_json(optional_parameters, json_dict) + f = open(self.abs_file_path, "w") + f.write(json.dumps(json_dict, indent=4, sort_keys=True)) + + return self.read_settings(data) + + def read_settings(self, data: str): + """Parse JSON into an object with attributes corresponding to dict keys.""" # https://stackoverflow.com/a/15882054 return json.loads(data, object_hook=lambda d: SimpleNamespace(**d)) - def write_settings(self, new_data): + def write_settings(self, new_data: str): try: - json.loads(new_data) - except json.JSONDecodeError as msg: - return {"status": msg.msg, "data": new_data} + json_dict = json.loads(new_data) + self._validate_json(config_parameters, json_dict) + except (json.JSONDecodeError, ValueError) as msg: + return {"status": f"Error: {msg}", "data": new_data} + + json_dict = self._fill_json(optional_parameters, json_dict) f = open(self.abs_file_path, "w") f.write(new_data) - self.settings = self.read_settings() - return {"status": "success", "data": new_data} + + self.settings = self.read_settings(new_data) + return {"status": "success", "data": json_dict} diff --git a/poetry.lock b/poetry.lock index 4a70dcc..d3b2f32 100644 --- a/poetry.lock +++ b/poetry.lock @@ -506,7 +506,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = ">=3.7,<4.0.0" -content-hash = "44aee2afc7d30d00b6f66d462ae6f2f3793c76c8789b188753eee882c577355f" +content-hash = "3204773141e19c071ab6a4aab1971e304cc99eb0b9bdd05717f21c1e271412ff" [metadata.files] anyio = [ diff --git a/pyproject.toml b/pyproject.toml index 0d04135..b4a8284 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,6 @@ psutil = "^5.9.0" fastapi = "^0.74.1" fastapi-utils = "^0.2.1" uvicorn = "^0.17.4" -pydantic = "^1.9.0" python-multipart = "^0.0.5" [tool.poetry.dev-dependencies] From 77d48819485317e23de79d1a5d34e1889de3da51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Gonz=C3=A1lez=20Rubio?= Date: Fri, 25 Feb 2022 19:13:54 +0100 Subject: [PATCH 04/15] Reformatted code of 'settings' module --- monitor_agent/main.py | 44 +++++++------- monitor_agent/settings.json | 4 +- monitor_agent/settings.py | 118 +++++++++++++++++------------------- 3 files changed, 80 insertions(+), 86 deletions(-) diff --git a/monitor_agent/main.py b/monitor_agent/main.py index 57f60fd..9499878 100644 --- a/monitor_agent/main.py +++ b/monitor_agent/main.py @@ -37,7 +37,7 @@ async def mod_settings(): return config_file.read_settings_file(config_file.abs_file_path) -if config_file.settings.metric_endpoint: +if config_file.metric_endpoint: @api.get(endpoints["metrics_endpoint"]) async def metrics_endpoint(): @@ -62,19 +62,17 @@ async def mod_settings(settings: UploadFile): @api.on_event("startup") -@repeat_every( - seconds=config_file.settings.post_interval, logger=logger, wait_first=True -) +@repeat_every(seconds=config_file.post_interval, logger=logger, wait_first=True) def periodic(): # https://github.com/tiangolo/fastapi/issues/520 # https://fastapi-utils.davidmontague.xyz/user-guide/repeated-tasks/#the-repeat_every-decorator # Changed Timeloop for this elapsed_time, data = send_metrics_adapter([static, dynamic]) send_metrics( - url=config_file.settings.post_metric_url, + url=config_file.post_metric_url, elapsed_time=elapsed_time, data=data, - file_enabled=config_file.settings.metric_enable_file, + file_enabled=config_file.metric_enable_file, file_path=config_file.abs_file_path, ) @@ -83,22 +81,22 @@ def start(): """Launched with `poetry run start` at root level""" uvicorn.run( "monitor_agent.main:api", - host=config_file.settings.host, - port=config_file.settings.port, - reload=config_file.settings.reload, - workers=config_file.settings.workers, - log_level=config_file.settings.log_level, + host=config_file.host, + port=config_file.port, + reload=config_file.reload, + workers=config_file.workers, + log_level=config_file.log_level, interface="asgi3", - debug=config_file.settings.debug, - backlog=config_file.settings.backlog, - timeout_keep_alive=config_file.settings.timeout_keep_alive, - limit_concurrency=config_file.settings.limit_concurrency, - limit_max_requests=config_file.settings.limit_max_requests, - ssl_keyfile=config_file.settings.ssl_keyfile, - ssl_keyfile_password=config_file.settings.ssl_keyfile_password, - ssl_certfile=config_file.settings.ssl_certfile, - ssl_version=config_file.settings.ssl_version, - ssl_cert_reqs=config_file.settings.ssl_cert_reqs, - ssl_ca_certs=config_file.settings.ssl_ca_certs, - ssl_ciphers=config_file.settings.ssl_ciphers, + debug=config_file.debug, + backlog=config_file.backlog, + timeout_keep_alive=config_file.timeout_keep_alive, + limit_concurrency=config_file.limit_concurrency, + limit_max_requests=config_file.limit_max_requests, + ssl_keyfile=config_file.ssl_keyfile, + ssl_keyfile_password=config_file.ssl_keyfile_password, + ssl_certfile=config_file.ssl_certfile, + ssl_version=config_file.ssl_version, + ssl_cert_reqs=config_file.ssl_cert_reqs, + ssl_ca_certs=config_file.ssl_ca_certs, + ssl_ciphers=config_file.ssl_ciphers, ) diff --git a/monitor_agent/settings.json b/monitor_agent/settings.json index 3d73910..0f190fc 100644 --- a/monitor_agent/settings.json +++ b/monitor_agent/settings.json @@ -16,10 +16,10 @@ "ssl_ca_certs": null, "ssl_cert_reqs": null, "ssl_certfile": null, - "ssl_ciphers": "TLSv1", + "ssl_ciphers": null, "ssl_keyfile": null, "ssl_keyfile_password": null, - "ssl_version": 3, + "ssl_version": null, "timeout_keep_alive": 5, "workers": 4 } \ No newline at end of file diff --git a/monitor_agent/settings.py b/monitor_agent/settings.py index ca971bb..0019dcf 100644 --- a/monitor_agent/settings.py +++ b/monitor_agent/settings.py @@ -1,3 +1,4 @@ +from asyncore import read from multiprocessing.sharedctypes import Value import os import sys @@ -34,79 +35,74 @@ ] config_parameters = main_parameters + optional_parameters - - -def dict_keys_iterator(dictionary: dict): - for key, value in dictionary.items(): - if isinstance(value, dict): - dict_keys_iterator(value) - yield key +rel_path = "settings.json" +dir = os.path.dirname(__file__) # <-- absolute dir the script is in +abs_file_path = os.path.join(dir, rel_path) class Settings: - def __init__(self, rel_path: str = "settings.json"): - dir = os.path.dirname(__file__) # <-- absolute dir the script is in - self.abs_file_path = os.path.join(dir, rel_path) - self.settings = self.read_settings_file(self.abs_file_path) - - def _validate_json(self, config_parameters: list, data: dict): - keys = [key for key in dict_keys_iterator(data)] - - diff_total = list(set(keys) - set(config_parameters)) - if len(diff_total) > 0: - raise ValueError(f"Config file contains invalid parameters: {diff_total}") - - diff_main = list(set(main_parameters) - set(keys)) - if len(diff_main) > 0: - raise ValueError( - f"Config file does not contain main parameters: {diff_main}" - ) - - # Need to validate type of data - - def _fill_json(self, optional_parameters: list, json_dict: dict): - fill_dict = json_dict.copy() - for key in optional_parameters: - if key not in fill_dict.keys(): - fill_dict[key] = None - return fill_dict - - def read_settings_file(self, file_path: str): + def __init__(self): + for key, value in self._read_settings_file().items(): + setattr(self, key, value) + + def _read_settings_file(self): try: - f = open(file_path, "r") - except FileNotFoundError as msg: + f = open(abs_file_path, "r") + data = f.read() + data_dict = _validate_json(data) + data_str, data_dict = _format_json_file(data_dict, abs_file_path) + except (json.JSONDecodeError, ValueError, FileNotFoundError) as msg: print(msg, file=sys.stderr) exit() - data = f.read() + return data_dict + + def write_settings(self, data: str): try: - json_dict = json.loads(data) - self._validate_json(config_parameters, json_dict) + data_dict = _validate_json(data) + data_str, data_dict = _format_json_file(data_dict, abs_file_path) except (json.JSONDecodeError, ValueError) as msg: - print(msg, file=sys.stderr) - exit() + return {"status": f"Error: {msg}", "data": data} - json_dict = self._fill_json(optional_parameters, json_dict) - f = open(self.abs_file_path, "w") - f.write(json.dumps(json_dict, indent=4, sort_keys=True)) + self.settings = data_dict + return {"status": "success", "data": data_dict} - return self.read_settings(data) - def read_settings(self, data: str): - """Parse JSON into an object with attributes corresponding to dict keys.""" - # https://stackoverflow.com/a/15882054 - return json.loads(data, object_hook=lambda d: SimpleNamespace(**d)) +###################### +## Helper functions ## +###################### - def write_settings(self, new_data: str): - try: - json_dict = json.loads(new_data) - self._validate_json(config_parameters, json_dict) - except (json.JSONDecodeError, ValueError) as msg: - return {"status": f"Error: {msg}", "data": new_data} - json_dict = self._fill_json(optional_parameters, json_dict) - f = open(self.abs_file_path, "w") - f.write(new_data) +def dict_keys_iterator(dictionary: dict): + for key, value in dictionary.items(): + if isinstance(value, dict): + dict_keys_iterator(value) + yield key + + +def _validate_json(data: str): + data_dict = json.loads(data) + + keys = [key for key in dict_keys_iterator(data_dict)] + + diff_total = list(set(keys) - set(config_parameters)) + if len(diff_total) > 0: + raise ValueError(f"Config file contains invalid parameters: {diff_total}") + + diff_main = list(set(main_parameters) - set(keys)) + if len(diff_main) > 0: + raise ValueError(f"Config file does not contain main parameters: {diff_main}") + return data_dict + + # Need to validate type of data + - self.settings = self.read_settings(new_data) - return {"status": "success", "data": json_dict} +def _format_json_file(data: dict, path: str): + data_dict = data.copy() + for key in optional_parameters: + if key not in data_dict.keys(): + data_dict[key] = None + f = open(path, "w") + data_str: str = json.dumps(data_dict, indent=4, sort_keys=True) + f.write(data_str) + return data_str, data_dict From 5ea3e725e2c06e62b18a642677f62c84054f0fc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Gonz=C3=A1lez=20Rubio?= Date: Sat, 26 Feb 2022 15:13:42 +0100 Subject: [PATCH 05/15] Implemented alert manager. Need to check if it's the best way to store the thresholds. --- monitor_agent/main.py | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/monitor_agent/main.py b/monitor_agent/main.py index 9499878..bda9e78 100644 --- a/monitor_agent/main.py +++ b/monitor_agent/main.py @@ -2,6 +2,7 @@ import json import uvicorn import logging +import requests from fastapi import FastAPI, UploadFile, File from fastapi_utils.tasks import repeat_every from .settings import Settings @@ -20,14 +21,25 @@ endpoints = { "root": "/", "command": "/command", - "metrics_endpoint": "/metrics", + "metrics": "/metrics", "settings": "/settings", + "thresholds": "/thresholds", +} + +thresholds = { + "cpu_percent": 50, + "ram_percent": 30, } # GET @api.get(endpoints["root"]) async def root(): - return {"message": "Hello World"} + return {"endpoints": endpoints} + + +@api.get(endpoints["thresholds"]) +async def root(): + return {"thresholds": thresholds} @api.get(endpoints["settings"]) @@ -39,7 +51,7 @@ async def mod_settings(): if config_file.metric_endpoint: - @api.get(endpoints["metrics_endpoint"]) + @api.get(endpoints["metrics"]) async def metrics_endpoint(): elapsed_time, data = send_metrics_adapter([static, dynamic]) return {"data": data, "elapsed_time": elapsed_time} @@ -61,6 +73,18 @@ async def mod_settings(settings: UploadFile): return config_file.write_settings(data) +@api.post(endpoints["thresholds"]) +async def mod_settings( + cpu_percent: float = thresholds["cpu_percent"], + ram_percent: float = thresholds["ram_percent"], +): + # if token: + # blablabla + thresholds["cpu_percent"] = cpu_percent + thresholds["ram_percent"] = ram_percent + return {"thresholds": thresholds} + + @api.on_event("startup") @repeat_every(seconds=config_file.post_interval, logger=logger, wait_first=True) def periodic(): @@ -73,8 +97,17 @@ def periodic(): elapsed_time=elapsed_time, data=data, file_enabled=config_file.metric_enable_file, - file_path=config_file.abs_file_path, + file_path=config_file.metric_file, ) + alert = {} + if data["cpu_percent"] >= thresholds["cpu_percent"]: + alert["cpu_percent"] = data["cpu_percent"] + if data["ram"]["percent"] >= thresholds["ram_percent"]: + alert["ram_percent"] = data["ram"]["percent"] + if data["process"]: + alert["processes"] = data["process"] + if alert: + r = requests.post(config_file.alert_url, json={"alert": alert}) def start(): From 865cf0c3d243ecca6539c3d75c4b92e8000e1d16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Gonz=C3=A1lez=20Rubio?= Date: Tue, 22 Mar 2022 23:04:03 +0100 Subject: [PATCH 06/15] Fix #24. --- monitor_agent/main.py | 52 ++++++++++++++--------------- monitor_agent/settings.json | 52 ++++++++++++++++------------- monitor_agent/settings.py | 65 ++++--------------------------------- 3 files changed, 62 insertions(+), 107 deletions(-) diff --git a/monitor_agent/main.py b/monitor_agent/main.py index afb7ecb..a9c586e 100644 --- a/monitor_agent/main.py +++ b/monitor_agent/main.py @@ -10,7 +10,7 @@ from .core.command import Command try: - config_file = Settings() + config = Settings() except json.decoder.JSONDecodeError as msg: print('Error in "settings.json".', msg, file=sys.stderr) exit() @@ -46,10 +46,10 @@ async def thresholds(): async def mod_settings(): # if token: # blablabla - return config_file.read_settings_file(config_file.abs_file_path) + return config.read_settings_file(config.abs_file_path) -if config_file.metric_endpoint: +if config.metrics.get_endpoint: @api.get(endpoints["metrics"]) async def metrics_endpoint(): @@ -70,7 +70,7 @@ async def mod_settings(settings: UploadFile): # if token: # blablabla data: str = settings.file.read().decode() - return config_file.write_settings(data) + return config.write_settings(data) @api.post(endpoints["thresholds"]) @@ -86,18 +86,18 @@ async def mod_settings( @api.on_event("startup") -@repeat_every(seconds=config_file.post_interval, logger=logger, wait_first=True) +@repeat_every(seconds=config.metrics.post_interval, logger=logger, wait_first=True) def periodic(): # https://github.com/tiangolo/fastapi/issues/520 # https://fastapi-utils.davidmontague.xyz/user-guide/repeated-tasks/#the-repeat_every-decorator # Changed Timeloop for this elapsed_time, data = send_metrics_adapter([static, dynamic]) send_metrics( - url=config_file.post_metric_url, + url=config.metrics.post_url, elapsed_time=elapsed_time, data=data, - file_enabled=config_file.metric_enable_file, - file_path=config_file.metric_file, + file_enabled=config.metrics.enable_logfile, + file_path=config.metrics.log_filename, ) alert = {} if data["cpu_percent"] >= thresholds["cpu_percent"]: @@ -107,29 +107,29 @@ def periodic(): if data["process"]: alert["processes"] = data["process"] if alert: - r = requests.post(config_file.alert_url, json={"alert": alert}) + r = requests.post(config.alerts.url, json={"alert": alert}) def start(): """Launched with `poetry run start` at root level""" uvicorn.run( "monitor_agent.main:api", - host=config_file.host, - port=config_file.port, - reload=config_file.reload, - workers=config_file.workers, - log_level=config_file.log_level, + host=config.uvicorn.host, + port=config.uvicorn.port, + reload=config.uvicorn.reload, + workers=config.uvicorn.workers, + log_level=config.uvicorn.log_level, interface="asgi3", - debug=config_file.debug, - backlog=config_file.backlog, - timeout_keep_alive=config_file.timeout_keep_alive, - limit_concurrency=config_file.limit_concurrency, - limit_max_requests=config_file.limit_max_requests, - ssl_keyfile=config_file.ssl_keyfile, - ssl_keyfile_password=config_file.ssl_keyfile_password, - ssl_certfile=config_file.ssl_certfile, - ssl_version=config_file.ssl_version, - ssl_cert_reqs=config_file.ssl_cert_reqs, - ssl_ca_certs=config_file.ssl_ca_certs, - ssl_ciphers=config_file.ssl_ciphers, + debug=config.uvicorn.debug, + backlog=config.uvicorn.backlog, + timeout_keep_alive=config.uvicorn.timeout_keep_alive, + limit_concurrency=config.uvicorn.limit_concurrency, + limit_max_requests=config.uvicorn.limit_max_requests, + ssl_keyfile=config.uvicorn.ssl_keyfile, + ssl_keyfile_password=config.uvicorn.ssl_keyfile_password, + ssl_certfile=config.uvicorn.ssl_certfile, + ssl_version=config.uvicorn.ssl_version, + ssl_cert_reqs=config.uvicorn.ssl_cert_reqs, + ssl_ca_certs=config.uvicorn.ssl_ca_certs, + ssl_ciphers=config.uvicorn.ssl_ciphers, ) diff --git a/monitor_agent/settings.json b/monitor_agent/settings.json index 0f190fc..3a4df9f 100644 --- a/monitor_agent/settings.json +++ b/monitor_agent/settings.json @@ -1,25 +1,31 @@ { - "alert_url": "", - "backlog": 2048, - "debug": true, - "host": "0.0.0.0", - "limit_concurrency": null, - "limit_max_requests": null, - "log_level": "trace", - "metric_enable_file": false, - "metric_endpoint": true, - "metric_file": "metrics.json", - "port": 8000, - "post_interval": 60, - "post_metric_url": "http://httpbin.org/post", - "reload": true, - "ssl_ca_certs": null, - "ssl_cert_reqs": null, - "ssl_certfile": null, - "ssl_ciphers": null, - "ssl_keyfile": null, - "ssl_keyfile_password": null, - "ssl_version": null, - "timeout_keep_alive": 5, - "workers": 4 + "uvicorn": { + "host":"0.0.0.0", + "port":8000, + "workers":4, + "reload":true, + "debug":true, + "log_level":"trace", + "ssl_keyfile":null, + "ssl_keyfile_password":null, + "ssl_certfile":"None", + "ssl_version":3, + "ssl_cert_reqs":null, + "ssl_ca_certs":null, + "ssl_ciphers":"TLSv1", + "limit_concurrency":null, + "limit_max_requests":null, + "backlog":2048, + "timeout_keep_alive":5 + }, + "metrics": { + "get_endpoint":true, + "post_url":"http://httpbin.org/post", + "post_interval":60, + "enable_logfile":false, + "log_filename":"metrics.json" + }, + "alerts":{ + "url":"127.0.0.1:8000/alerts" + } } \ No newline at end of file diff --git a/monitor_agent/settings.py b/monitor_agent/settings.py index 0718a9a..200a047 100644 --- a/monitor_agent/settings.py +++ b/monitor_agent/settings.py @@ -2,36 +2,6 @@ import sys import json -main_parameters = [ - "host", - "port", - "workers", - "reload", - "debug", - "log_level", - "backlog", - "timeout_keep_alive", - "metric_endpoint", - "post_metric_url", - "post_interval", - "metric_enable_file", - "metric_file", - "alert_url", -] - -optional_parameters = [ - "ssl_keyfile", - "ssl_keyfile_password", - "ssl_certfile", - "ssl_version", - "ssl_cert_reqs", - "ssl_ca_certs", - "ssl_ciphers", - "limit_concurrency", - "limit_max_requests", -] - -config_parameters = main_parameters + optional_parameters rel_path = "settings.json" dir = os.path.dirname(__file__) # <-- absolute dir the script is in abs_file_path = os.path.join(dir, rel_path) @@ -46,8 +16,8 @@ def _read_settings_file(self): try: f = open(abs_file_path, "r") data = f.read() - data_dict = _validate_json(data) - data_str, data_dict = _format_json_file(data_dict, abs_file_path) + data_dict = json.loads(data) + data_str, data_dict = _write_file(data_dict, abs_file_path) except (json.JSONDecodeError, ValueError, FileNotFoundError) as msg: print(msg, file=sys.stderr) exit() @@ -56,8 +26,8 @@ def _read_settings_file(self): def write_settings(self, data: str): try: - data_dict = _validate_json(data) - data_str, data_dict = _format_json_file(data_dict, abs_file_path) + data_dict = json.loads(data) + data_str, data_dict = _write_file(data_dict, abs_file_path) except (json.JSONDecodeError, ValueError) as msg: return {"status": f"Error: {msg}", "data": data} @@ -77,29 +47,8 @@ def dict_keys_iterator(dictionary: dict): yield key -def _validate_json(data: str): - data_dict = json.loads(data) - - keys = [key for key in dict_keys_iterator(data_dict)] - - diff_total = list(set(keys) - set(config_parameters)) - if len(diff_total) > 0: - raise ValueError(f"Config file contains invalid parameters: {diff_total}") - - diff_main = list(set(main_parameters) - set(keys)) - if len(diff_main) > 0: - raise ValueError(f"Config file does not contain main parameters: {diff_main}") - return data_dict - - # Need to validate type of data - - -def _format_json_file(data: dict, path: str): - data_dict = data.copy() - for key in optional_parameters: - if key not in data_dict.keys(): - data_dict[key] = None +def _write_file(data: dict, path: str): f = open(path, "w") - data_str: str = json.dumps(data_dict, indent=4, sort_keys=True) + data_str: str = json.dumps(data, indent=4, sort_keys=True) f.write(data_str) - return data_str, data_dict + return data_str, data From c343967c296491dd5a9251d52d9d4d05f4663081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Gonz=C3=A1lez=20Rubio?= Date: Tue, 22 Mar 2022 23:11:59 +0100 Subject: [PATCH 07/15] SECURITY: Removed GET Settings. --- monitor_agent/main.py | 9 +--- poetry.lock | 111 +++++++++++++++++++++--------------------- 2 files changed, 56 insertions(+), 64 deletions(-) diff --git a/monitor_agent/main.py b/monitor_agent/main.py index a9c586e..4bd4a03 100644 --- a/monitor_agent/main.py +++ b/monitor_agent/main.py @@ -3,7 +3,7 @@ import uvicorn import logging import requests -from fastapi import FastAPI, UploadFile, File +from fastapi import FastAPI, UploadFile from fastapi_utils.tasks import repeat_every from .settings import Settings from .core.metricFunctions import send_metrics, send_metrics_adapter, static, dynamic @@ -42,13 +42,6 @@ async def thresholds(): return {"thresholds": thresholds} -@api.get(endpoints["settings"]) -async def mod_settings(): - # if token: - # blablabla - return config.read_settings_file(config.abs_file_path) - - if config.metrics.get_endpoint: @api.get(endpoints["metrics"]) diff --git a/poetry.lock b/poetry.lock index d3b2f32..d985077 100644 --- a/poetry.lock +++ b/poetry.lock @@ -174,7 +174,7 @@ python-versions = ">=3.5" [[package]] name = "importlib-metadata" -version = "4.11.1" +version = "4.11.3" description = "Read metadata from Python packages" category = "main" optional = false @@ -185,7 +185,7 @@ typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] perf = ["ipython"] testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] @@ -282,11 +282,11 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" -version = "7.0.1" +version = "7.1.1" description = "pytest: simple powerful testing with Python" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} @@ -349,7 +349,7 @@ python-versions = ">=3.5" [[package]] name = "sqlalchemy" -version = "1.4.31" +version = "1.4.32" description = "Database Abstraction Library" category = "main" optional = false @@ -444,20 +444,20 @@ python-versions = ">=3.6" [[package]] name = "urllib3" -version = "1.26.8" +version = "1.26.9" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" [package.extras] -brotli = ["brotlipy (>=0.6.0)"] +brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "uvicorn" -version = "0.17.5" +version = "0.17.6" description = "The lightning-fast ASGI server." category = "main" optional = false @@ -470,11 +470,11 @@ h11 = ">=0.8" typing-extensions = {version = "*", markers = "python_version < \"3.8\""} [package.extras] -standard = ["websockets (>=10.0)", "httptools (>=0.2.0,<0.4.0)", "watchgod (>=0.6)", "python-dotenv (>=0.13)", "PyYAML (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "colorama (>=0.4)"] +standard = ["websockets (>=10.0)", "httptools (>=0.4.0)", "watchgod (>=0.6)", "python-dotenv (>=0.13)", "PyYAML (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "colorama (>=0.4)"] [[package]] name = "virtualenv" -version = "20.13.2" +version = "20.13.4" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -623,8 +623,8 @@ idna = [ {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, ] importlib-metadata = [ - {file = "importlib_metadata-4.11.1-py3-none-any.whl", hash = "sha256:e0bc84ff355328a4adfc5240c4f211e0ab386f80aa640d1b11f0618a1d282094"}, - {file = "importlib_metadata-4.11.1.tar.gz", hash = "sha256:175f4ee440a0317f6e8d81b7f8d4869f93316170a65ad2b007d2929186c8052c"}, + {file = "importlib_metadata-4.11.3-py3-none-any.whl", hash = "sha256:1208431ca90a8cca1a6b8af391bb53c1a2db74e5d1cef6ddced95d4b2062edc6"}, + {file = "importlib_metadata-4.11.3.tar.gz", hash = "sha256:ea4c597ebf37142f827b8f39299579e31685c31d3a438b59f469406afd0f2539"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -722,8 +722,8 @@ pyparsing = [ {file = "pyparsing-3.0.7.tar.gz", hash = "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea"}, ] pytest = [ - {file = "pytest-7.0.1-py3-none-any.whl", hash = "sha256:9ce3ff477af913ecf6321fe337b93a2c0dcf2a0a1439c43f5452112c1e4280db"}, - {file = "pytest-7.0.1.tar.gz", hash = "sha256:e30905a0c131d3d94b89624a1cc5afec3e0ba2fbdb151867d8e0ebd49850f171"}, + {file = "pytest-7.1.1-py3-none-any.whl", hash = "sha256:92f723789a8fdd7180b6b06483874feca4c48a5c76968e03bb3e7f806a1869ea"}, + {file = "pytest-7.1.1.tar.gz", hash = "sha256:841132caef6b1ad17a9afde46dc4f6cfa59a05f9555aae5151f73bdf2820ca63"}, ] python-multipart = [ {file = "python-multipart-0.0.5.tar.gz", hash = "sha256:f7bb5f611fc600d15fa47b3974c8aa16e93724513b49b5f95c81e6624c83fa43"}, @@ -741,42 +741,41 @@ sniffio = [ {file = "sniffio-1.2.0.tar.gz", hash = "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de"}, ] sqlalchemy = [ - {file = "SQLAlchemy-1.4.31-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:c3abc34fed19fdeaead0ced8cf56dd121f08198008c033596aa6aae7cc58f59f"}, - {file = "SQLAlchemy-1.4.31-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:8d0949b11681380b4a50ac3cd075e4816afe9fa4a8c8ae006c1ca26f0fa40ad8"}, - {file = "SQLAlchemy-1.4.31-cp27-cp27m-win32.whl", hash = "sha256:f3b7ec97e68b68cb1f9ddb82eda17b418f19a034fa8380a0ac04e8fe01532875"}, - {file = "SQLAlchemy-1.4.31-cp27-cp27m-win_amd64.whl", hash = "sha256:81f2dd355b57770fdf292b54f3e0a9823ec27a543f947fa2eb4ec0df44f35f0d"}, - {file = "SQLAlchemy-1.4.31-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:4ad31cec8b49fd718470328ad9711f4dc703507d434fd45461096da0a7135ee0"}, - {file = "SQLAlchemy-1.4.31-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:05fa14f279d43df68964ad066f653193187909950aa0163320b728edfc400167"}, - {file = "SQLAlchemy-1.4.31-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dccff41478050e823271642837b904d5f9bda3f5cf7d371ce163f00a694118d6"}, - {file = "SQLAlchemy-1.4.31-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57205844f246bab9b666a32f59b046add8995c665d9ecb2b7b837b087df90639"}, - {file = "SQLAlchemy-1.4.31-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea8210090a816d48a4291a47462bac750e3bc5c2442e6d64f7b8137a7c3f9ac5"}, - {file = "SQLAlchemy-1.4.31-cp310-cp310-win32.whl", hash = "sha256:2e216c13ecc7fcdcbb86bb3225425b3ed338e43a8810c7089ddb472676124b9b"}, - {file = "SQLAlchemy-1.4.31-cp310-cp310-win_amd64.whl", hash = "sha256:e3a86b59b6227ef72ffc10d4b23f0fe994bef64d4667eab4fb8cd43de4223bec"}, - {file = "SQLAlchemy-1.4.31-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:2fd4d3ca64c41dae31228b80556ab55b6489275fb204827f6560b65f95692cf3"}, - {file = "SQLAlchemy-1.4.31-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f22c040d196f841168b1456e77c30a18a3dc16b336ddbc5a24ce01ab4e95ae0"}, - {file = "SQLAlchemy-1.4.31-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c0c7171aa5a57e522a04a31b84798b6c926234cb559c0939840c3235cf068813"}, - {file = "SQLAlchemy-1.4.31-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d046a9aeba9bc53e88a41e58beb72b6205abb9a20f6c136161adf9128e589db5"}, - {file = "SQLAlchemy-1.4.31-cp36-cp36m-win32.whl", hash = "sha256:d86132922531f0dc5a4f424c7580a472a924dd737602638e704841c9cb24aea2"}, - {file = "SQLAlchemy-1.4.31-cp36-cp36m-win_amd64.whl", hash = "sha256:ca68c52e3cae491ace2bf39b35fef4ce26c192fd70b4cd90f040d419f70893b5"}, - {file = "SQLAlchemy-1.4.31-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:cf2cd387409b12d0a8b801610d6336ee7d24043b6dd965950eaec09b73e7262f"}, - {file = "SQLAlchemy-1.4.31-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb4b15fb1f0aafa65cbdc62d3c2078bea1ceecbfccc9a1f23a2113c9ac1191fa"}, - {file = "SQLAlchemy-1.4.31-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c317ddd7c586af350a6aef22b891e84b16bff1a27886ed5b30f15c1ed59caeaa"}, - {file = "SQLAlchemy-1.4.31-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c7ed6c69debaf6198fadb1c16ae1253a29a7670bbf0646f92582eb465a0b999"}, - {file = "SQLAlchemy-1.4.31-cp37-cp37m-win32.whl", hash = "sha256:6a01ec49ca54ce03bc14e10de55dfc64187a2194b3b0e5ac0fdbe9b24767e79e"}, - {file = "SQLAlchemy-1.4.31-cp37-cp37m-win_amd64.whl", hash = "sha256:330eb45395874cc7787214fdd4489e2afb931bc49e0a7a8f9cd56d6e9c5b1639"}, - {file = "SQLAlchemy-1.4.31-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:5e9c7b3567edbc2183607f7d9f3e7e89355b8f8984eec4d2cd1e1513c8f7b43f"}, - {file = "SQLAlchemy-1.4.31-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de85c26a5a1c72e695ab0454e92f60213b4459b8d7c502e0be7a6369690eeb1a"}, - {file = "SQLAlchemy-1.4.31-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:975f5c0793892c634c4920057da0de3a48bbbbd0a5c86f5fcf2f2fedf41b76da"}, - {file = "SQLAlchemy-1.4.31-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5c20c8415173b119762b6110af64448adccd4d11f273fb9f718a9865b88a99c"}, - {file = "SQLAlchemy-1.4.31-cp38-cp38-win32.whl", hash = "sha256:b35dca159c1c9fa8a5f9005e42133eed82705bf8e243da371a5e5826440e65ca"}, - {file = "SQLAlchemy-1.4.31-cp38-cp38-win_amd64.whl", hash = "sha256:b7b20c88873675903d6438d8b33fba027997193e274b9367421e610d9da76c08"}, - {file = "SQLAlchemy-1.4.31-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:85e4c244e1de056d48dae466e9baf9437980c19fcde493e0db1a0a986e6d75b4"}, - {file = "SQLAlchemy-1.4.31-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e79e73d5ee24196d3057340e356e6254af4d10e1fc22d3207ea8342fc5ffb977"}, - {file = "SQLAlchemy-1.4.31-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:15a03261aa1e68f208e71ae3cd845b00063d242cbf8c87348a0c2c0fc6e1f2ac"}, - {file = "SQLAlchemy-1.4.31-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ddc5e5ccc0160e7ad190e5c61eb57560f38559e22586955f205e537cda26034"}, - {file = "SQLAlchemy-1.4.31-cp39-cp39-win32.whl", hash = "sha256:289465162b1fa1e7a982f8abe59d26a8331211cad4942e8031d2b7db1f75e649"}, - {file = "SQLAlchemy-1.4.31-cp39-cp39-win_amd64.whl", hash = "sha256:9e4fb2895b83993831ba2401b6404de953fdbfa9d7d4fa6a4756294a83bbc94f"}, - {file = "SQLAlchemy-1.4.31.tar.gz", hash = "sha256:582b59d1e5780a447aada22b461e50b404a9dc05768da1d87368ad8190468418"}, + {file = "SQLAlchemy-1.4.32-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:4b2bcab3a914715d332ca783e9bda13bc570d8b9ef087563210ba63082c18c16"}, + {file = "SQLAlchemy-1.4.32-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:159c2f69dd6efd28e894f261ffca1100690f28210f34cfcd70b895e0ea7a64f3"}, + {file = "SQLAlchemy-1.4.32-cp27-cp27m-win_amd64.whl", hash = "sha256:d7e483f4791fbda60e23926b098702340504f7684ce7e1fd2c1bf02029288423"}, + {file = "SQLAlchemy-1.4.32-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:4aa96e957141006181ca58e792e900ee511085b8dae06c2d08c00f108280fb8a"}, + {file = "SQLAlchemy-1.4.32-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:576684771456d02e24078047c2567025f2011977aa342063468577d94e194b00"}, + {file = "SQLAlchemy-1.4.32-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fff677fa4522dafb5a5e2c0cf909790d5d367326321aeabc0dffc9047cb235bd"}, + {file = "SQLAlchemy-1.4.32-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8679f9aba5ac22e7bce54ccd8a77641d3aea3e2d96e73e4356c887ebf8ff1082"}, + {file = "SQLAlchemy-1.4.32-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c7046f7aa2db445daccc8424f50b47a66c4039c9f058246b43796aa818f8b751"}, + {file = "SQLAlchemy-1.4.32-cp310-cp310-win32.whl", hash = "sha256:bedd89c34ab62565d44745212814e4b57ef1c24ad4af9b29c504ce40f0dc6558"}, + {file = "SQLAlchemy-1.4.32-cp310-cp310-win_amd64.whl", hash = "sha256:199dc6d0068753b6a8c0bd3aceb86a3e782df118260ebc1fa981ea31ee054674"}, + {file = "SQLAlchemy-1.4.32-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:8e1e5d96b744a4f91163290b01045430f3f32579e46d87282449e5b14d27d4ac"}, + {file = "SQLAlchemy-1.4.32-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edfcf93fd92e2f9eef640b3a7a40db20fe3c1d7c2c74faa41424c63dead61b76"}, + {file = "SQLAlchemy-1.4.32-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:04164e0063feb7aedd9d073db0fd496edb244be40d46ea1f0d8990815e4b8c34"}, + {file = "SQLAlchemy-1.4.32-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ba59761c19b800bc2e1c9324da04d35ef51e4ee9621ff37534bc2290d258f71"}, + {file = "SQLAlchemy-1.4.32-cp36-cp36m-win32.whl", hash = "sha256:708973b5d9e1e441188124aaf13c121e5b03b6054c2df59b32219175a25aa13e"}, + {file = "SQLAlchemy-1.4.32-cp36-cp36m-win_amd64.whl", hash = "sha256:316270e5867566376e69a0ac738b863d41396e2b63274616817e1d34156dff0e"}, + {file = "SQLAlchemy-1.4.32-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:9a0195af6b9050c9322a97cf07514f66fe511968e623ca87b2df5e3cf6349615"}, + {file = "SQLAlchemy-1.4.32-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7e4a3c0c3c596296b37f8427c467c8e4336dc8d50f8ed38042e8ba79507b2c9"}, + {file = "SQLAlchemy-1.4.32-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:bca714d831e5b8860c3ab134c93aec63d1a4f493bed20084f54e3ce9f0a3bf99"}, + {file = "SQLAlchemy-1.4.32-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9a680d9665f88346ed339888781f5236347933906c5a56348abb8261282ec48"}, + {file = "SQLAlchemy-1.4.32-cp37-cp37m-win32.whl", hash = "sha256:9cb5698c896fa72f88e7ef04ef62572faf56809093180771d9be8d9f2e264a13"}, + {file = "SQLAlchemy-1.4.32-cp37-cp37m-win_amd64.whl", hash = "sha256:8b9a395122770a6f08ebfd0321546d7379f43505882c7419d7886856a07caa13"}, + {file = "SQLAlchemy-1.4.32-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:3f88a4ee192142eeed3fe173f673ea6ab1f5a863810a9d85dbf6c67a9bd08f97"}, + {file = "SQLAlchemy-1.4.32-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd93162615870c976dba43963a24bb418b28448fef584f30755990c134a06a55"}, + {file = "SQLAlchemy-1.4.32-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5a2e73508f939175363d8a4be9dcdc84cf16a92578d7fa86e6e4ca0e6b3667b2"}, + {file = "SQLAlchemy-1.4.32-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfec934aac7f9fa95fc82147a4ba5db0a8bdc4ebf1e33b585ab8860beb10232f"}, + {file = "SQLAlchemy-1.4.32-cp38-cp38-win32.whl", hash = "sha256:bb42f9b259c33662c6a9b866012f6908a91731a419e69304e1261ba3ab87b8d1"}, + {file = "SQLAlchemy-1.4.32-cp38-cp38-win_amd64.whl", hash = "sha256:7ff72b3cc9242d1a1c9b84bd945907bf174d74fc2519efe6184d6390a8df478b"}, + {file = "SQLAlchemy-1.4.32-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:5dc9801ae9884e822ba942ca493642fb50f049c06b6dbe3178691fce48ceb089"}, + {file = "SQLAlchemy-1.4.32-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4607d2d16330757818c9d6fba322c2e80b4b112ff24295d1343a80b876eb0ed"}, + {file = "SQLAlchemy-1.4.32-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:20e9eba7fd86ef52e0df25bea83b8b518dfdf0bce09b336cfe51671f52aaaa3f"}, + {file = "SQLAlchemy-1.4.32-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:290cbdf19129ae520d4bdce392648c6fcdbee763bc8f750b53a5ab51880cb9c9"}, + {file = "SQLAlchemy-1.4.32-cp39-cp39-win32.whl", hash = "sha256:1bbac3e8293b34c4403d297e21e8f10d2a57756b75cff101dc62186adec725f5"}, + {file = "SQLAlchemy-1.4.32-cp39-cp39-win_amd64.whl", hash = "sha256:b3f1d9b3aa09ab9adc7f8c4b40fc3e081eb903054c9a6f9ae1633fe15ae503b4"}, + {file = "SQLAlchemy-1.4.32.tar.gz", hash = "sha256:6fdd2dc5931daab778c2b65b03df6ae68376e028a3098eb624d0909d999885bc"}, ] starlette = [ {file = "starlette-0.17.1-py3-none-any.whl", hash = "sha256:26a18cbda5e6b651c964c12c88b36d9898481cd428ed6e063f5f29c418f73050"}, @@ -799,16 +798,16 @@ typing-extensions = [ {file = "typing_extensions-4.1.1.tar.gz", hash = "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42"}, ] urllib3 = [ - {file = "urllib3-1.26.8-py2.py3-none-any.whl", hash = "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed"}, - {file = "urllib3-1.26.8.tar.gz", hash = "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"}, + {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"}, + {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"}, ] uvicorn = [ - {file = "uvicorn-0.17.5-py3-none-any.whl", hash = "sha256:8adddf629b79857b48b999ae1b14d6c92c95d4d7840bd86461f09bee75f1653e"}, - {file = "uvicorn-0.17.5.tar.gz", hash = "sha256:c04a9c069111489c324f427501b3840d306c6b91a77b00affc136a840a3f45f1"}, + {file = "uvicorn-0.17.6-py3-none-any.whl", hash = "sha256:19e2a0e96c9ac5581c01eb1a79a7d2f72bb479691acd2b8921fce48ed5b961a6"}, + {file = "uvicorn-0.17.6.tar.gz", hash = "sha256:5180f9d059611747d841a4a4c4ab675edf54c8489e97f96d0583ee90ac3bfc23"}, ] virtualenv = [ - {file = "virtualenv-20.13.2-py2.py3-none-any.whl", hash = "sha256:e7b34c9474e6476ee208c43a4d9ac1510b041c68347eabfe9a9ea0c86aa0a46b"}, - {file = "virtualenv-20.13.2.tar.gz", hash = "sha256:01f5f80744d24a3743ce61858123488e91cb2dd1d3bdf92adaf1bba39ffdedf0"}, + {file = "virtualenv-20.13.4-py2.py3-none-any.whl", hash = "sha256:c3e01300fb8495bc00ed70741f5271fc95fed067eb7106297be73d30879af60c"}, + {file = "virtualenv-20.13.4.tar.gz", hash = "sha256:ce8901d3bbf3b90393498187f2d56797a8a452fb2d0d7efc6fd837554d6f679c"}, ] zipp = [ {file = "zipp-3.7.0-py3-none-any.whl", hash = "sha256:b47250dd24f92b7dd6a0a8fc5244da14608f3ca90a5efcd37a3b1642fac9a375"}, From 002081b79ab40d1d514b347f1486c73ac8a2be98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Gonz=C3=A1lez=20Rubio?= Date: Thu, 24 Mar 2022 21:39:11 +0100 Subject: [PATCH 08/15] Remote Work --- monitor_agent/main.py | 20 +++----------- monitor_agent/settings.json | 54 ++++++++++++++++++++----------------- monitor_agent/settings.py | 27 +++++++++++++------ 3 files changed, 51 insertions(+), 50 deletions(-) diff --git a/monitor_agent/main.py b/monitor_agent/main.py index 4bd4a03..824620d 100644 --- a/monitor_agent/main.py +++ b/monitor_agent/main.py @@ -11,6 +11,7 @@ try: config = Settings() + print(dir(config)) except json.decoder.JSONDecodeError as msg: print('Error in "settings.json".', msg, file=sys.stderr) exit() @@ -26,11 +27,6 @@ "thresholds": "/thresholds", } -thresholds = { - "cpu_percent": 50, - "ram_percent": 30, -} - # GET @api.get(endpoints["root"]) async def root(): @@ -39,11 +35,10 @@ async def root(): @api.get(endpoints["thresholds"]) async def thresholds(): - return {"thresholds": thresholds} + return {"thresholds": config.thresholds.__dict__} if config.metrics.get_endpoint: - @api.get(endpoints["metrics"]) async def metrics_endpoint(): elapsed_time, data = send_metrics_adapter([static, dynamic]) @@ -53,26 +48,17 @@ async def metrics_endpoint(): # POST @api.post(endpoints["command"]) async def command(command: str, timeout: int): - # if token: - # blablabla return Command(command, timeout).__dict__ @api.post(endpoints["settings"]) async def mod_settings(settings: UploadFile): - # if token: - # blablabla data: str = settings.file.read().decode() return config.write_settings(data) @api.post(endpoints["thresholds"]) -async def mod_settings( - cpu_percent: float = thresholds["cpu_percent"], - ram_percent: float = thresholds["ram_percent"], -): - # if token: - # blablabla +async def mod_settings(cpu_percent: float, ram_percent: float): thresholds["cpu_percent"] = cpu_percent thresholds["ram_percent"] = ram_percent return {"thresholds": thresholds} diff --git a/monitor_agent/settings.json b/monitor_agent/settings.json index 3a4df9f..ade1c06 100644 --- a/monitor_agent/settings.json +++ b/monitor_agent/settings.json @@ -1,31 +1,35 @@ { - "uvicorn": { - "host":"0.0.0.0", - "port":8000, - "workers":4, - "reload":true, - "debug":true, - "log_level":"trace", - "ssl_keyfile":null, - "ssl_keyfile_password":null, - "ssl_certfile":"None", - "ssl_version":3, - "ssl_cert_reqs":null, - "ssl_ca_certs":null, - "ssl_ciphers":"TLSv1", - "limit_concurrency":null, - "limit_max_requests":null, - "backlog":2048, - "timeout_keep_alive":5 + "alerts": { + "url": "127.0.0.1:8000/alerts" }, "metrics": { - "get_endpoint":true, - "post_url":"http://httpbin.org/post", - "post_interval":60, - "enable_logfile":false, - "log_filename":"metrics.json" + "enable_logfile": false, + "get_endpoint": true, + "log_filename": "metrics.json", + "post_interval": 60, + "post_url": "http://httpbin.org/post" + }, + "thresholds": { + "cpu_percent": 50, + "ram_percent": 30 }, - "alerts":{ - "url":"127.0.0.1:8000/alerts" + "uvicorn": { + "backlog": 2048, + "debug": true, + "host": "0.0.0.0", + "limit_concurrency": null, + "limit_max_requests": null, + "log_level": "trace", + "port": 8000, + "reload": true, + "ssl_ca_certs": null, + "ssl_cert_reqs": null, + "ssl_certfile": "None", + "ssl_ciphers": "TLSv1", + "ssl_keyfile": null, + "ssl_keyfile_password": null, + "ssl_version": 3, + "timeout_keep_alive": 5, + "workers": 4 } } \ No newline at end of file diff --git a/monitor_agent/settings.py b/monitor_agent/settings.py index 200a047..d338585 100644 --- a/monitor_agent/settings.py +++ b/monitor_agent/settings.py @@ -6,11 +6,16 @@ dir = os.path.dirname(__file__) # <-- absolute dir the script is in abs_file_path = os.path.join(dir, rel_path) - class Settings: def __init__(self): - for key, value in self._read_settings_file().items(): - setattr(self, key, value) + self.as_dict : dict = self._read_settings_file() + # DARLE UNA VUELTA + # for key, value in toObj(self.as_dict): + # self.alerts = value if key == "alerts" else None + # self.metrics = value if key == "metrics" else None + # self.thresholds = value if key == "thresholds" else None + # self.uvicorn = value if key == "uvicorn" else None + def _read_settings_file(self): try: @@ -39,13 +44,19 @@ def write_settings(self, data: str): ## Helper functions ## ###################### +def toObj(item): + """Jakub Dóka: https://stackoverflow.com/a/65969444""" + if isinstance(item, dict): + obj = type('__object', (object,), {}) -def dict_keys_iterator(dictionary: dict): - for key, value in dictionary.items(): - if isinstance(value, dict): - dict_keys_iterator(value) - yield key + for key, value in item.items(): + setattr(obj, key, toObj(value)) + return obj + elif isinstance(item, list): + return map(toObj, item) + else: + return item def _write_file(data: dict, path: str): f = open(path, "w") From dbc7f23d310d3ffefa97c1215b3383dec3f1c13c Mon Sep 17 00:00:00 2001 From: n0nuser Date: Sat, 26 Mar 2022 12:45:11 +0100 Subject: [PATCH 09/15] Settings reformmated and FINALLY FIXED. Removed Thresholds POST Thresholds will now be passed through the config file instead of saving it as a dictionary. --- README.md | 57 ++++++++++++++++++++++++++++++++- monitor_agent/main.py | 64 ++++++++++++++----------------------- monitor_agent/settings.json | 9 ------ monitor_agent/settings.py | 19 +++++------ 4 files changed, 90 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 74c26ac..4cbbb4a 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,57 @@ # Monitor - Agent - Final Degree Project - Host to monitor + +Final Degree Project - Host to monitor + +## Uvicorn configuration + +In `settings.json` you could pass this parameters to Uvicorn server: + +```python +host: str +port: int +uds: str +fd: int +loop: str +http: str +ws: str +ws_max_size: int +ws_ping_interval: float +ws_ping_timeout: float +ws_per_message_deflate: bool +lifespan: str +debug: bool +reload: bool +reload_dirs: typing.List[str] +reload_includes: typing.List[str] +reload_excludes: typing.List[str] +reload_delay: float +workers: int +env_file: str +log_config: str +log_level: str +access_log: bool +proxy_headers: bool +server_header: bool +date_header: bool +forwarded_allow_ips: str +root_path: str +limit_concurrency: int +backlog: int +limit_max_requests: int +timeout_keep_alive: int +ssl_keyfile: str +ssl_certfile: str +ssl_keyfile_password: str +ssl_version: int +ssl_cert_reqs: int +ssl_ca_certs: str +ssl_ciphers: str +headers: typing.List[str] +use_colors: bool +app_dir: str +factory: bool +``` + +More information in their webpage - [Uvicorn Settings](https://www.uvicorn.org/settings/). + +> :warning: If any parameters is incorrect the server will crash. diff --git a/monitor_agent/main.py b/monitor_agent/main.py index 824620d..0fa9e44 100644 --- a/monitor_agent/main.py +++ b/monitor_agent/main.py @@ -1,17 +1,18 @@ import sys +import ssl import json import uvicorn import logging import requests +from .settings import Settings +from .core.command import Command from fastapi import FastAPI, UploadFile from fastapi_utils.tasks import repeat_every -from .settings import Settings +from typing import Awaitable, Callable, Dict, List, Optional, Tuple, Type, Union from .core.metricFunctions import send_metrics, send_metrics_adapter, static, dynamic -from .core.command import Command try: - config = Settings() - print(dir(config)) + CONFIG = Settings() except json.decoder.JSONDecodeError as msg: print('Error in "settings.json".', msg, file=sys.stderr) exit() @@ -35,10 +36,11 @@ async def root(): @api.get(endpoints["thresholds"]) async def thresholds(): - return {"thresholds": config.thresholds.__dict__} + return {"thresholds": CONFIG.thresholds.__dict__} -if config.metrics.get_endpoint: +if CONFIG.metrics.get_endpoint: + @api.get(endpoints["metrics"]) async def metrics_endpoint(): elapsed_time, data = send_metrics_adapter([static, dynamic]) @@ -53,30 +55,26 @@ async def command(command: str, timeout: int): @api.post(endpoints["settings"]) async def mod_settings(settings: UploadFile): + global CONFIG data: str = settings.file.read().decode() - return config.write_settings(data) - - -@api.post(endpoints["thresholds"]) -async def mod_settings(cpu_percent: float, ram_percent: float): - thresholds["cpu_percent"] = cpu_percent - thresholds["ram_percent"] = ram_percent - return {"thresholds": thresholds} + msg = CONFIG.write_settings(data) + CONFIG = Settings() + return msg @api.on_event("startup") -@repeat_every(seconds=config.metrics.post_interval, logger=logger, wait_first=True) +@repeat_every(seconds=CONFIG.metrics.post_interval, logger=logger, wait_first=True) def periodic(): # https://github.com/tiangolo/fastapi/issues/520 # https://fastapi-utils.davidmontague.xyz/user-guide/repeated-tasks/#the-repeat_every-decorator # Changed Timeloop for this elapsed_time, data = send_metrics_adapter([static, dynamic]) send_metrics( - url=config.metrics.post_url, + url=CONFIG.metrics.post_url, elapsed_time=elapsed_time, data=data, - file_enabled=config.metrics.enable_logfile, - file_path=config.metrics.log_filename, + file_enabled=CONFIG.metrics.enable_logfile, + file_path=CONFIG.metrics.log_filename, ) alert = {} if data["cpu_percent"] >= thresholds["cpu_percent"]: @@ -86,29 +84,15 @@ def periodic(): if data["process"]: alert["processes"] = data["process"] if alert: - r = requests.post(config.alerts.url, json={"alert": alert}) + r = requests.post(CONFIG.alerts.url, json={"alert": alert}) def start(): """Launched with `poetry run start` at root level""" - uvicorn.run( - "monitor_agent.main:api", - host=config.uvicorn.host, - port=config.uvicorn.port, - reload=config.uvicorn.reload, - workers=config.uvicorn.workers, - log_level=config.uvicorn.log_level, - interface="asgi3", - debug=config.uvicorn.debug, - backlog=config.uvicorn.backlog, - timeout_keep_alive=config.uvicorn.timeout_keep_alive, - limit_concurrency=config.uvicorn.limit_concurrency, - limit_max_requests=config.uvicorn.limit_max_requests, - ssl_keyfile=config.uvicorn.ssl_keyfile, - ssl_keyfile_password=config.uvicorn.ssl_keyfile_password, - ssl_certfile=config.uvicorn.ssl_certfile, - ssl_version=config.uvicorn.ssl_version, - ssl_cert_reqs=config.uvicorn.ssl_cert_reqs, - ssl_ca_certs=config.uvicorn.ssl_ca_certs, - ssl_ciphers=config.uvicorn.ssl_ciphers, - ) + uviconfig = {"app": "monitor_agent.main:api", "interface": "asgi3"} + uviconfig.update(CONFIG.uvicorn.__dict__) + uviconfig.pop("__module__", None) + uviconfig.pop("__dict__", None) + uviconfig.pop("__weakref__", None) + uviconfig.pop("__doc__", None) + uvicorn.run(**uviconfig) diff --git a/monitor_agent/settings.json b/monitor_agent/settings.json index ade1c06..8c6e5ac 100644 --- a/monitor_agent/settings.json +++ b/monitor_agent/settings.json @@ -17,18 +17,9 @@ "backlog": 2048, "debug": true, "host": "0.0.0.0", - "limit_concurrency": null, - "limit_max_requests": null, "log_level": "trace", "port": 8000, "reload": true, - "ssl_ca_certs": null, - "ssl_cert_reqs": null, - "ssl_certfile": "None", - "ssl_ciphers": "TLSv1", - "ssl_keyfile": null, - "ssl_keyfile_password": null, - "ssl_version": 3, "timeout_keep_alive": 5, "workers": 4 } diff --git a/monitor_agent/settings.py b/monitor_agent/settings.py index d338585..aec9cb1 100644 --- a/monitor_agent/settings.py +++ b/monitor_agent/settings.py @@ -6,16 +6,15 @@ dir = os.path.dirname(__file__) # <-- absolute dir the script is in abs_file_path = os.path.join(dir, rel_path) + class Settings: def __init__(self): - self.as_dict : dict = self._read_settings_file() - # DARLE UNA VUELTA - # for key, value in toObj(self.as_dict): - # self.alerts = value if key == "alerts" else None - # self.metrics = value if key == "metrics" else None - # self.thresholds = value if key == "thresholds" else None - # self.uvicorn = value if key == "uvicorn" else None - + self.as_dict: dict = self._read_settings_file() + obj = toObj(self.as_dict) + self.alerts = obj.alerts + self.metrics = obj.metrics + self.thresholds = obj.thresholds + self.uvicorn = obj.uvicorn def _read_settings_file(self): try: @@ -44,10 +43,11 @@ def write_settings(self, data: str): ## Helper functions ## ###################### + def toObj(item): """Jakub Dóka: https://stackoverflow.com/a/65969444""" if isinstance(item, dict): - obj = type('__object', (object,), {}) + obj = type("__object", (object,), {}) for key, value in item.items(): setattr(obj, key, toObj(value)) @@ -58,6 +58,7 @@ def toObj(item): else: return item + def _write_file(data: dict, path: str): f = open(path, "w") data_str: str = json.dumps(data, indent=4, sort_keys=True) From aa815e6f85417745f5acaab2dcbb6b80aa0c98a9 Mon Sep 17 00:00:00 2001 From: n0nuser Date: Sat, 26 Mar 2022 12:47:30 +0100 Subject: [PATCH 10/15] Bump Dependency Versions --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index d985077..5c42c28 100644 --- a/poetry.lock +++ b/poetry.lock @@ -474,7 +474,7 @@ standard = ["websockets (>=10.0)", "httptools (>=0.4.0)", "watchgod (>=0.6)", "p [[package]] name = "virtualenv" -version = "20.13.4" +version = "20.14.0" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -806,8 +806,8 @@ uvicorn = [ {file = "uvicorn-0.17.6.tar.gz", hash = "sha256:5180f9d059611747d841a4a4c4ab675edf54c8489e97f96d0583ee90ac3bfc23"}, ] virtualenv = [ - {file = "virtualenv-20.13.4-py2.py3-none-any.whl", hash = "sha256:c3e01300fb8495bc00ed70741f5271fc95fed067eb7106297be73d30879af60c"}, - {file = "virtualenv-20.13.4.tar.gz", hash = "sha256:ce8901d3bbf3b90393498187f2d56797a8a452fb2d0d7efc6fd837554d6f679c"}, + {file = "virtualenv-20.14.0-py2.py3-none-any.whl", hash = "sha256:1e8588f35e8b42c6ec6841a13c5e88239de1e6e4e4cedfd3916b306dc826ec66"}, + {file = "virtualenv-20.14.0.tar.gz", hash = "sha256:8e5b402037287126e81ccde9432b95a8be5b19d36584f64957060a3488c11ca8"}, ] zipp = [ {file = "zipp-3.7.0-py3-none-any.whl", hash = "sha256:b47250dd24f92b7dd6a0a8fc5244da14608f3ca90a5efcd37a3b1642fac9a375"}, From d451ddce39504bebbcfebbeafbd68896c5ea6f5a Mon Sep 17 00:00:00 2001 From: n0nuser Date: Sat, 26 Mar 2022 15:22:18 +0100 Subject: [PATCH 11/15] Improved logging Needs testing with Postman. --- monitor_agent/core/command.py | 2 - monitor_agent/core/helper.py | 46 ++++-- monitor_agent/core/metricFunctions.py | 12 +- .../core/{models/metricModel.py => models.py} | 8 +- monitor_agent/core/models/__init__.py | 0 monitor_agent/core/schemas/__init__.py | 0 monitor_agent/main.py | 23 +-- monitor_agent/monitor.log | 138 ++++++++++++++++++ monitor_agent/settings.json | 8 + monitor_agent/settings.py | 17 ++- monitor_agent/settings_localhost.json | 25 ---- 11 files changed, 213 insertions(+), 66 deletions(-) rename monitor_agent/core/{models/metricModel.py => models.py} (96%) delete mode 100644 monitor_agent/core/models/__init__.py delete mode 100644 monitor_agent/core/schemas/__init__.py create mode 100644 monitor_agent/monitor.log delete mode 100644 monitor_agent/settings_localhost.json diff --git a/monitor_agent/core/command.py b/monitor_agent/core/command.py index 59ee4dd..8c68830 100644 --- a/monitor_agent/core/command.py +++ b/monitor_agent/core/command.py @@ -80,7 +80,6 @@ def _executeCommand(command: str, timeout: int) -> typing.Tuple[str, str, int, f # https://stackoverflow.com/questions/64948722/error-unicodedecodeerror-utf-8-codec-cant-decode-byte-0xbe-in-position-2-in # shell=True MUST NOT BE USED: # https://stackoverflow.com/questions/48763362/python-subprocess-kill-with-timeout - # print("Windows CMD built-in.", command) process = subprocess.run( command, capture_output=True, @@ -92,7 +91,6 @@ def _executeCommand(command: str, timeout: int) -> typing.Tuple[str, str, int, f check=True, ) else: - # print("Linux or Windows command:", command) process = subprocess.run( command.split(), capture_output=True, diff --git a/monitor_agent/core/helper.py b/monitor_agent/core/helper.py index d9addbc..856827f 100644 --- a/monitor_agent/core/helper.py +++ b/monitor_agent/core/helper.py @@ -1,14 +1,40 @@ -import sys -from datetime import datetime +import logging -LOG_FILE = "monitor-agent.log" -MODE = "a+" +def getLogger(CONFIG): + log_level = "" + tranlation = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, + } + try: + if CONFIG.logging.level in tranlation.keys(): + log_level = tranlation[CONFIG.logging.level] + else: + logging.warning("Level not established in Settings.json") + log_level = logging.info + except AttributeError as e: + logging.warning( + "Level not established in Settings.json.\nDefault Info level will be used.", + exc_info=True, + ) + log_level = logging.info -def save2log(filename: str = LOG_FILE, mode: str = MODE, data="", type=""): + log_filename = "" try: - with open(filename, mode) as f: - timestamp = datetime.now().strftime("%d/%m/%Y:%H:%M:%S") - f.write(f"{type} - - [{timestamp}] : {data}\n") - except OSError as msg: - print(f"ERROR: Couldn't open file {filename} - {msg}", file=sys.stderr) + log_filename = CONFIG.logging.filename + except AttributeError as e: + logging.warning( + 'Log filename not established in Settings.json.\nDefault "monitor.log" file will be used.', + exc_info=True, + ) + log_filename = "monitor.log" + + return logging.basicConfig( + level=log_level, + filename=log_filename, + format="%(asctime)s - %(levelname)s - %(message)s", + ) diff --git a/monitor_agent/core/metricFunctions.py b/monitor_agent/core/metricFunctions.py index f129f66..1a51b97 100644 --- a/monitor_agent/core/metricFunctions.py +++ b/monitor_agent/core/metricFunctions.py @@ -2,12 +2,10 @@ import json import typing import requests +import logging -from monitor_agent.core.helper import save2log -from .models.metricModel import Status, MetricDynamic, MetricStatic - -# Typing Tuple is used to overcome Python ^3.6 until Python 3.10 problem -# with Tuples not being a standard type +from monitor_agent.main import LOGGER +from core.models import Status, MetricDynamic, MetricStatic def execution_time_decorator(function) -> typing.Tuple[float, dict]: @@ -36,7 +34,7 @@ def send_metrics_adapter(function_list: list) -> typing.Tuple[dict, dict]: elapsed_time.update(f_time) data.update(f_data) except TypeError as msg: - save2log(type="WARNING", data=f"TypeError: {msg}") + LOGGER.warning(f"TypeError: {msg}", exc_info=True) continue return elapsed_time, data @@ -48,7 +46,7 @@ def send_metrics( json_request = {"data": data, "status": status} r = requests.post(url, json=json_request) # DEBUG - print(r.status_code) + logging.debug(f"Status Code: {r.status_code}") if file_enabled: with open(file_path, "w") as f: f.write(json.dumps(json_request, indent=4, sort_keys=True)) diff --git a/monitor_agent/core/models/metricModel.py b/monitor_agent/core/models.py similarity index 96% rename from monitor_agent/core/models/metricModel.py rename to monitor_agent/core/models.py index d92f319..8bf7466 100644 --- a/monitor_agent/core/models/metricModel.py +++ b/monitor_agent/core/models.py @@ -2,9 +2,8 @@ import platform import time import datetime -import os -from monitor_agent.core.helper import save2log +from monitor_agent.main import LOGGER class Status: @@ -116,9 +115,8 @@ def _process(ram: int, pc_cpu_percent): # Requires elevated permissions process[p.pid]["path"] = p.exe() except (PermissionError, psutil.AccessDenied): - save2log( - type="WARNING", - data=f"Could not get Username or Path for process {p.name()}", + LOGGER.warning( + f"Could not get Username or Path for process {p.name()}" ) return process diff --git a/monitor_agent/core/models/__init__.py b/monitor_agent/core/models/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monitor_agent/core/schemas/__init__.py b/monitor_agent/core/schemas/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/monitor_agent/main.py b/monitor_agent/main.py index fc74fd4..eec410b 100644 --- a/monitor_agent/main.py +++ b/monitor_agent/main.py @@ -1,6 +1,5 @@ -import re -import sys import json +from monitor_agent.core.helper import getLogger import uvicorn import logging import requests @@ -8,15 +7,18 @@ from .core.command import Command from fastapi import FastAPI, UploadFile from fastapi_utils.tasks import repeat_every -from .core.metricFunctions import send_metrics, send_metrics_adapter, static, dynamic + try: CONFIG = Settings() except json.decoder.JSONDecodeError as msg: - print('Error in "settings.json".', msg, file=sys.stderr) + logging.critical(f'Error in "settings.json". {msg}') exit() -logger = logging.getLogger(__name__) +LOGGER = getLogger(CONFIG) + +from .core.metricFunctions import send_metrics, send_metrics_adapter, static, dynamic + api = FastAPI() endpoints = { @@ -63,7 +65,7 @@ async def mod_settings(settings: UploadFile): @api.on_event("startup") -@repeat_every(seconds=CONFIG.metrics.post_interval, logger=logger, wait_first=True) +@repeat_every(seconds=CONFIG.metrics.post_interval, logger=LOGGER, wait_first=True) def periodic(): # https://github.com/tiangolo/fastapi/issues/520 # https://fastapi-utils.davidmontague.xyz/user-guide/repeated-tasks/#the-repeat_every-decorator @@ -78,9 +80,9 @@ def periodic(): ) alert = {} - if data["cpu_percent"] >= thresholds_dict["cpu_percent"]: + if data["cpu_percent"] >= CONFIG.thresholds.cpu_percent: alert["cpu_percent"] = data["cpu_percent"] - if data["ram"]["percent"] >= thresholds_dict["ram_percent"]: + if data["ram"]["percent"] >= CONFIG.thresholds.ram_percent: alert["ram_percent"] = data["ram"]["percent"] try: if data["process"]: @@ -99,4 +101,7 @@ def start(): uviconfig.pop("__dict__", None) uviconfig.pop("__weakref__", None) uviconfig.pop("__doc__", None) - uvicorn.run(**uviconfig) + try: + uvicorn.run(**uviconfig) + except: + LOGGER.critical("Unable to run server.", exc_info=True) diff --git a/monitor_agent/monitor.log b/monitor_agent/monitor.log new file mode 100644 index 0000000..488747e --- /dev/null +++ b/monitor_agent/monitor.log @@ -0,0 +1,138 @@ +2022-03-26 15:10:14,033 - INFO - Will watch for changes in these directories: ['/home/pablo/devel/monitor_agent/monitor_agent'] +2022-03-26 15:10:14,033 - INFO - Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) +2022-03-26 15:10:14,033 - INFO - Started reloader process [35964] using statreload +2022-03-26 15:10:14,306 - INFO - Started server process [35969] +2022-03-26 15:10:14,306 - INFO - Waiting for application startup. +2022-03-26 15:10:14,306 - TRACE - ASGI [1] Started scope={'type': 'lifespan', 'asgi': {'version': '3.0', 'spec_version': '2.0'}} +2022-03-26 15:10:14,306 - TRACE - ASGI [1] Receive {'type': 'lifespan.startup'} +2022-03-26 15:10:14,306 - TRACE - ASGI [1] Send {'type': 'lifespan.startup.complete'} +2022-03-26 15:10:14,307 - INFO - Application startup complete. +2022-03-26 15:10:28,957 - WARNING - StatReload detected file change in 'core/helper.py'. Reloading... +2022-03-26 15:10:28,991 - INFO - Shutting down +2022-03-26 15:10:29,092 - INFO - Waiting for application shutdown. +2022-03-26 15:10:29,092 - TRACE - ASGI [1] Receive {'type': 'lifespan.shutdown'} +2022-03-26 15:10:29,093 - TRACE - ASGI [1] Send {'type': 'lifespan.shutdown.complete'} +2022-03-26 15:10:29,093 - TRACE - ASGI [1] Completed +2022-03-26 15:10:29,094 - INFO - Application shutdown complete. +2022-03-26 15:10:29,094 - INFO - Finished server process [35969] +2022-03-26 15:10:29,492 - INFO - Started server process [35993] +2022-03-26 15:10:29,493 - INFO - Waiting for application startup. +2022-03-26 15:10:29,493 - TRACE - ASGI [1] Started scope={'type': 'lifespan', 'asgi': {'version': '3.0', 'spec_version': '2.0'}} +2022-03-26 15:10:29,493 - TRACE - ASGI [1] Receive {'type': 'lifespan.startup'} +2022-03-26 15:10:29,493 - TRACE - ASGI [1] Send {'type': 'lifespan.startup.complete'} +2022-03-26 15:10:29,493 - INFO - Application startup complete. +2022-03-26 15:10:33,738 - WARNING - StatReload detected file change in 'core/helper.py'. Reloading... +2022-03-26 15:10:33,815 - INFO - Shutting down +2022-03-26 15:10:33,917 - INFO - Waiting for application shutdown. +2022-03-26 15:10:33,918 - TRACE - ASGI [1] Receive {'type': 'lifespan.shutdown'} +2022-03-26 15:10:33,918 - TRACE - ASGI [1] Send {'type': 'lifespan.shutdown.complete'} +2022-03-26 15:10:33,919 - TRACE - ASGI [1] Completed +2022-03-26 15:10:33,919 - INFO - Application shutdown complete. +2022-03-26 15:10:33,919 - INFO - Finished server process [35993] +2022-03-26 15:10:34,262 - INFO - Started server process [36005] +2022-03-26 15:10:34,262 - INFO - Waiting for application startup. +2022-03-26 15:10:34,262 - TRACE - ASGI [1] Started scope={'type': 'lifespan', 'asgi': {'version': '3.0', 'spec_version': '2.0'}} +2022-03-26 15:10:34,262 - TRACE - ASGI [1] Receive {'type': 'lifespan.startup'} +2022-03-26 15:10:34,263 - TRACE - ASGI [1] Send {'type': 'lifespan.startup.complete'} +2022-03-26 15:10:34,263 - INFO - Application startup complete. +2022-03-26 15:10:37,679 - INFO - Shutting down +2022-03-26 15:10:37,780 - INFO - Waiting for application shutdown. +2022-03-26 15:10:37,781 - TRACE - ASGI [1] Receive {'type': 'lifespan.shutdown'} +2022-03-26 15:10:37,781 - TRACE - ASGI [1] Send {'type': 'lifespan.shutdown.complete'} +2022-03-26 15:10:37,782 - TRACE - ASGI [1] Completed +2022-03-26 15:10:37,782 - INFO - Application shutdown complete. +2022-03-26 15:10:37,783 - INFO - Finished server process [36005] +2022-03-26 15:10:37,879 - INFO - Stopping reloader process [35964] +2022-03-26 15:10:40,672 - INFO - Will watch for changes in these directories: ['/home/pablo/devel/monitor_agent/monitor_agent'] +2022-03-26 15:10:40,673 - INFO - Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) +2022-03-26 15:10:40,673 - INFO - Started reloader process [36045] using statreload +2022-03-26 15:10:40,944 - INFO - Started server process [36050] +2022-03-26 15:10:40,944 - INFO - Waiting for application startup. +2022-03-26 15:10:40,945 - TRACE - ASGI [1] Started scope={'type': 'lifespan', 'asgi': {'version': '3.0', 'spec_version': '2.0'}} +2022-03-26 15:10:40,945 - TRACE - ASGI [1] Receive {'type': 'lifespan.startup'} +2022-03-26 15:10:40,945 - TRACE - ASGI [1] Send {'type': 'lifespan.startup.complete'} +2022-03-26 15:10:40,945 - INFO - Application startup complete. +2022-03-26 15:10:51,286 - WARNING - StatReload detected file change in 'main.py'. Reloading... +2022-03-26 15:10:51,299 - INFO - Shutting down +2022-03-26 15:10:51,400 - INFO - Waiting for application shutdown. +2022-03-26 15:10:51,401 - TRACE - ASGI [1] Receive {'type': 'lifespan.shutdown'} +2022-03-26 15:10:51,401 - TRACE - ASGI [1] Send {'type': 'lifespan.shutdown.complete'} +2022-03-26 15:10:51,402 - TRACE - ASGI [1] Completed +2022-03-26 15:10:51,402 - INFO - Application shutdown complete. +2022-03-26 15:10:51,402 - INFO - Finished server process [36050] +2022-03-26 15:10:51,798 - INFO - Started server process [36066] +2022-03-26 15:10:51,798 - INFO - Waiting for application startup. +2022-03-26 15:10:51,798 - TRACE - ASGI [1] Started scope={'type': 'lifespan', 'asgi': {'version': '3.0', 'spec_version': '2.0'}} +2022-03-26 15:10:51,799 - TRACE - ASGI [1] Receive {'type': 'lifespan.startup'} +2022-03-26 15:10:51,799 - TRACE - ASGI [1] Send {'type': 'lifespan.startup.complete'} +2022-03-26 15:10:51,799 - INFO - Application startup complete. +2022-03-26 15:11:46,805 - WARNING - StatReload detected file change in 'main.py'. Reloading... +2022-03-26 15:11:46,839 - INFO - Shutting down +2022-03-26 15:11:46,941 - INFO - Waiting for application shutdown. +2022-03-26 15:11:46,941 - TRACE - ASGI [1] Receive {'type': 'lifespan.shutdown'} +2022-03-26 15:11:46,942 - TRACE - ASGI [1] Send {'type': 'lifespan.shutdown.complete'} +2022-03-26 15:11:46,942 - TRACE - ASGI [1] Completed +2022-03-26 15:11:46,943 - INFO - Application shutdown complete. +2022-03-26 15:11:46,943 - INFO - Finished server process [36066] +2022-03-26 15:11:47,302 - INFO - Started server process [36085] +2022-03-26 15:11:47,302 - INFO - Waiting for application startup. +2022-03-26 15:11:47,302 - TRACE - ASGI [1] Started scope={'type': 'lifespan', 'asgi': {'version': '3.0', 'spec_version': '2.0'}} +2022-03-26 15:11:47,303 - TRACE - ASGI [1] Receive {'type': 'lifespan.startup'} +2022-03-26 15:11:47,303 - TRACE - ASGI [1] Send {'type': 'lifespan.startup.complete'} +2022-03-26 15:11:47,303 - INFO - Application startup complete. +2022-03-26 15:12:05,661 - INFO - Shutting down +2022-03-26 15:12:05,762 - INFO - Waiting for application shutdown. +2022-03-26 15:12:05,762 - TRACE - ASGI [1] Receive {'type': 'lifespan.shutdown'} +2022-03-26 15:12:05,762 - TRACE - ASGI [1] Send {'type': 'lifespan.shutdown.complete'} +2022-03-26 15:12:05,762 - TRACE - ASGI [1] Completed +2022-03-26 15:12:05,763 - INFO - Application shutdown complete. +2022-03-26 15:12:05,763 - INFO - Finished server process [36085] +2022-03-26 15:12:05,813 - INFO - Stopping reloader process [36045] +2022-03-26 15:12:38,060 - INFO - Will watch for changes in these directories: ['/home/pablo/devel/monitor_agent/monitor_agent'] +2022-03-26 15:12:38,061 - INFO - Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) +2022-03-26 15:12:38,061 - INFO - Started reloader process [36505] using statreload +2022-03-26 15:12:38,332 - INFO - Started server process [36510] +2022-03-26 15:12:38,333 - INFO - Waiting for application startup. +2022-03-26 15:12:38,333 - TRACE - ASGI [1] Started scope={'type': 'lifespan', 'asgi': {'version': '3.0', 'spec_version': '2.0'}} +2022-03-26 15:12:38,333 - TRACE - ASGI [1] Receive {'type': 'lifespan.startup'} +2022-03-26 15:12:38,333 - TRACE - ASGI [1] Send {'type': 'lifespan.startup.complete'} +2022-03-26 15:12:38,333 - INFO - Application startup complete. +2022-03-26 15:12:39,642 - INFO - Shutting down +2022-03-26 15:12:39,743 - INFO - Waiting for application shutdown. +2022-03-26 15:12:39,744 - TRACE - ASGI [1] Receive {'type': 'lifespan.shutdown'} +2022-03-26 15:12:39,744 - TRACE - ASGI [1] Send {'type': 'lifespan.shutdown.complete'} +2022-03-26 15:12:39,745 - TRACE - ASGI [1] Completed +2022-03-26 15:12:39,745 - INFO - Application shutdown complete. +2022-03-26 15:12:39,746 - INFO - Finished server process [36510] +2022-03-26 15:12:39,815 - INFO - Stopping reloader process [36505] +2022-03-26 15:12:44,043 - INFO - Will watch for changes in these directories: ['/home/pablo/devel/monitor_agent/monitor_agent'] +2022-03-26 15:12:44,044 - INFO - Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) +2022-03-26 15:12:44,044 - INFO - Started reloader process [36527] using statreload +2022-03-26 15:12:44,314 - INFO - Started server process [36532] +2022-03-26 15:12:44,315 - INFO - Waiting for application startup. +2022-03-26 15:12:44,315 - TRACE - ASGI [1] Started scope={'type': 'lifespan', 'asgi': {'version': '3.0', 'spec_version': '2.0'}} +2022-03-26 15:12:44,315 - TRACE - ASGI [1] Receive {'type': 'lifespan.startup'} +2022-03-26 15:12:44,315 - TRACE - ASGI [1] Send {'type': 'lifespan.startup.complete'} +2022-03-26 15:12:44,315 - INFO - Application startup complete. +2022-03-26 15:12:45,816 - WARNING - StatReload detected file change in 'main.py'. Reloading... +2022-03-26 15:12:45,824 - INFO - Shutting down +2022-03-26 15:12:45,925 - INFO - Waiting for application shutdown. +2022-03-26 15:12:45,925 - TRACE - ASGI [1] Receive {'type': 'lifespan.shutdown'} +2022-03-26 15:12:45,926 - TRACE - ASGI [1] Send {'type': 'lifespan.shutdown.complete'} +2022-03-26 15:12:45,926 - TRACE - ASGI [1] Completed +2022-03-26 15:12:45,926 - INFO - Application shutdown complete. +2022-03-26 15:12:45,927 - INFO - Finished server process [36532] +2022-03-26 15:12:46,320 - INFO - Started server process [36543] +2022-03-26 15:12:46,320 - INFO - Waiting for application startup. +2022-03-26 15:12:46,321 - TRACE - ASGI [1] Started scope={'type': 'lifespan', 'asgi': {'version': '3.0', 'spec_version': '2.0'}} +2022-03-26 15:12:46,321 - TRACE - ASGI [1] Receive {'type': 'lifespan.startup'} +2022-03-26 15:12:46,321 - TRACE - ASGI [1] Send {'type': 'lifespan.startup.complete'} +2022-03-26 15:12:46,321 - INFO - Application startup complete. +2022-03-26 15:12:47,830 - INFO - Shutting down +2022-03-26 15:12:47,931 - INFO - Waiting for application shutdown. +2022-03-26 15:12:47,932 - TRACE - ASGI [1] Receive {'type': 'lifespan.shutdown'} +2022-03-26 15:12:47,932 - TRACE - ASGI [1] Send {'type': 'lifespan.shutdown.complete'} +2022-03-26 15:12:47,933 - TRACE - ASGI [1] Completed +2022-03-26 15:12:47,933 - INFO - Application shutdown complete. +2022-03-26 15:12:47,933 - INFO - Finished server process [36543] +2022-03-26 15:12:47,982 - INFO - Stopping reloader process [36527] diff --git a/monitor_agent/settings.json b/monitor_agent/settings.json index 8c6e5ac..7cbb829 100644 --- a/monitor_agent/settings.json +++ b/monitor_agent/settings.json @@ -2,6 +2,14 @@ "alerts": { "url": "127.0.0.1:8000/alerts" }, + "auth": { + "agent_token": "", + "api_token": "" + }, + "logging": { + "filename": "monitor.log", + "level": "info" + }, "metrics": { "enable_logfile": false, "get_endpoint": true, diff --git a/monitor_agent/settings.py b/monitor_agent/settings.py index a598721..57eb402 100644 --- a/monitor_agent/settings.py +++ b/monitor_agent/settings.py @@ -1,7 +1,7 @@ import os import sys import json -from monitor_agent.core.helper import save2log +import logging rel_path = "settings.json" @@ -12,11 +12,13 @@ class Settings: def __init__(self): self.as_dict: dict = self._read_settings_file() - obj = toObj(self.as_dict) - self.alerts = obj.alerts - self.metrics = obj.metrics - self.thresholds = obj.thresholds - self.uvicorn = obj.uvicorn + _obj = toObj(self.as_dict) + self.alerts = _obj.alerts + self.auth = _obj.auth + self.logging = _obj.logging + self.metrics = _obj.metrics + self.thresholds = _obj.thresholds + self.uvicorn = _obj.uvicorn def _read_settings_file(self): try: @@ -25,8 +27,7 @@ def _read_settings_file(self): data_dict = json.loads(data) data_str, data_dict = _write_file(data_dict, abs_file_path) except (json.JSONDecodeError, ValueError, FileNotFoundError) as msg: - print(f"ERROR: Invalid JSON settings file - {msg}", file=sys.stderr) - save2log(type="ERROR", data=f"Invalid JSON file - {msg}") + logging.error(f"Invalid JSON settings file.", exc_info=True) exit() return data_dict diff --git a/monitor_agent/settings_localhost.json b/monitor_agent/settings_localhost.json deleted file mode 100644 index e57e224..0000000 --- a/monitor_agent/settings_localhost.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "backlog": 2048, - "debug": true, - "host": "0.0.0.0", - "limit_concurrency": null, - "limit_max_requests": null, - "log_level": "trace", - "metric_enable_file": false, - "metric_endpoint": true, - "metric_file": "metrics.json", - "port": 8080, - "post_alert_url": "", - "post_interval": 60, - "post_metric_url": "http://localhost/api/metrics", - "reload": true, - "ssl_ca_certs": null, - "ssl_cert_reqs": null, - "ssl_certfile": null, - "ssl_ciphers": null, - "ssl_keyfile": null, - "ssl_keyfile_password": null, - "ssl_version": null, - "timeout_keep_alive": 5, - "workers": 4 -} \ No newline at end of file From c88fd8f377fb69d05a57e62fb9329459913a2a9f Mon Sep 17 00:00:00 2001 From: n0nuser Date: Sat, 26 Mar 2022 18:23:43 +0100 Subject: [PATCH 12/15] Added Name to Auth in Settings. Fix #27 Logger saves the Uvicorn logs too --- .gitignore | 4 +- monitor_agent/monitor.log | 138 ------------------------------------ monitor_agent/settings.json | 5 +- 3 files changed, 5 insertions(+), 142 deletions(-) delete mode 100644 monitor_agent/monitor.log diff --git a/.gitignore b/.gitignore index a30f048..5b476a0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -__pycache__/ -*/__pycache__/ +__pycache__ *.pyc monitor-agent.log +monitor.log diff --git a/monitor_agent/monitor.log b/monitor_agent/monitor.log deleted file mode 100644 index 488747e..0000000 --- a/monitor_agent/monitor.log +++ /dev/null @@ -1,138 +0,0 @@ -2022-03-26 15:10:14,033 - INFO - Will watch for changes in these directories: ['/home/pablo/devel/monitor_agent/monitor_agent'] -2022-03-26 15:10:14,033 - INFO - Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) -2022-03-26 15:10:14,033 - INFO - Started reloader process [35964] using statreload -2022-03-26 15:10:14,306 - INFO - Started server process [35969] -2022-03-26 15:10:14,306 - INFO - Waiting for application startup. -2022-03-26 15:10:14,306 - TRACE - ASGI [1] Started scope={'type': 'lifespan', 'asgi': {'version': '3.0', 'spec_version': '2.0'}} -2022-03-26 15:10:14,306 - TRACE - ASGI [1] Receive {'type': 'lifespan.startup'} -2022-03-26 15:10:14,306 - TRACE - ASGI [1] Send {'type': 'lifespan.startup.complete'} -2022-03-26 15:10:14,307 - INFO - Application startup complete. -2022-03-26 15:10:28,957 - WARNING - StatReload detected file change in 'core/helper.py'. Reloading... -2022-03-26 15:10:28,991 - INFO - Shutting down -2022-03-26 15:10:29,092 - INFO - Waiting for application shutdown. -2022-03-26 15:10:29,092 - TRACE - ASGI [1] Receive {'type': 'lifespan.shutdown'} -2022-03-26 15:10:29,093 - TRACE - ASGI [1] Send {'type': 'lifespan.shutdown.complete'} -2022-03-26 15:10:29,093 - TRACE - ASGI [1] Completed -2022-03-26 15:10:29,094 - INFO - Application shutdown complete. -2022-03-26 15:10:29,094 - INFO - Finished server process [35969] -2022-03-26 15:10:29,492 - INFO - Started server process [35993] -2022-03-26 15:10:29,493 - INFO - Waiting for application startup. -2022-03-26 15:10:29,493 - TRACE - ASGI [1] Started scope={'type': 'lifespan', 'asgi': {'version': '3.0', 'spec_version': '2.0'}} -2022-03-26 15:10:29,493 - TRACE - ASGI [1] Receive {'type': 'lifespan.startup'} -2022-03-26 15:10:29,493 - TRACE - ASGI [1] Send {'type': 'lifespan.startup.complete'} -2022-03-26 15:10:29,493 - INFO - Application startup complete. -2022-03-26 15:10:33,738 - WARNING - StatReload detected file change in 'core/helper.py'. Reloading... -2022-03-26 15:10:33,815 - INFO - Shutting down -2022-03-26 15:10:33,917 - INFO - Waiting for application shutdown. -2022-03-26 15:10:33,918 - TRACE - ASGI [1] Receive {'type': 'lifespan.shutdown'} -2022-03-26 15:10:33,918 - TRACE - ASGI [1] Send {'type': 'lifespan.shutdown.complete'} -2022-03-26 15:10:33,919 - TRACE - ASGI [1] Completed -2022-03-26 15:10:33,919 - INFO - Application shutdown complete. -2022-03-26 15:10:33,919 - INFO - Finished server process [35993] -2022-03-26 15:10:34,262 - INFO - Started server process [36005] -2022-03-26 15:10:34,262 - INFO - Waiting for application startup. -2022-03-26 15:10:34,262 - TRACE - ASGI [1] Started scope={'type': 'lifespan', 'asgi': {'version': '3.0', 'spec_version': '2.0'}} -2022-03-26 15:10:34,262 - TRACE - ASGI [1] Receive {'type': 'lifespan.startup'} -2022-03-26 15:10:34,263 - TRACE - ASGI [1] Send {'type': 'lifespan.startup.complete'} -2022-03-26 15:10:34,263 - INFO - Application startup complete. -2022-03-26 15:10:37,679 - INFO - Shutting down -2022-03-26 15:10:37,780 - INFO - Waiting for application shutdown. -2022-03-26 15:10:37,781 - TRACE - ASGI [1] Receive {'type': 'lifespan.shutdown'} -2022-03-26 15:10:37,781 - TRACE - ASGI [1] Send {'type': 'lifespan.shutdown.complete'} -2022-03-26 15:10:37,782 - TRACE - ASGI [1] Completed -2022-03-26 15:10:37,782 - INFO - Application shutdown complete. -2022-03-26 15:10:37,783 - INFO - Finished server process [36005] -2022-03-26 15:10:37,879 - INFO - Stopping reloader process [35964] -2022-03-26 15:10:40,672 - INFO - Will watch for changes in these directories: ['/home/pablo/devel/monitor_agent/monitor_agent'] -2022-03-26 15:10:40,673 - INFO - Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) -2022-03-26 15:10:40,673 - INFO - Started reloader process [36045] using statreload -2022-03-26 15:10:40,944 - INFO - Started server process [36050] -2022-03-26 15:10:40,944 - INFO - Waiting for application startup. -2022-03-26 15:10:40,945 - TRACE - ASGI [1] Started scope={'type': 'lifespan', 'asgi': {'version': '3.0', 'spec_version': '2.0'}} -2022-03-26 15:10:40,945 - TRACE - ASGI [1] Receive {'type': 'lifespan.startup'} -2022-03-26 15:10:40,945 - TRACE - ASGI [1] Send {'type': 'lifespan.startup.complete'} -2022-03-26 15:10:40,945 - INFO - Application startup complete. -2022-03-26 15:10:51,286 - WARNING - StatReload detected file change in 'main.py'. Reloading... -2022-03-26 15:10:51,299 - INFO - Shutting down -2022-03-26 15:10:51,400 - INFO - Waiting for application shutdown. -2022-03-26 15:10:51,401 - TRACE - ASGI [1] Receive {'type': 'lifespan.shutdown'} -2022-03-26 15:10:51,401 - TRACE - ASGI [1] Send {'type': 'lifespan.shutdown.complete'} -2022-03-26 15:10:51,402 - TRACE - ASGI [1] Completed -2022-03-26 15:10:51,402 - INFO - Application shutdown complete. -2022-03-26 15:10:51,402 - INFO - Finished server process [36050] -2022-03-26 15:10:51,798 - INFO - Started server process [36066] -2022-03-26 15:10:51,798 - INFO - Waiting for application startup. -2022-03-26 15:10:51,798 - TRACE - ASGI [1] Started scope={'type': 'lifespan', 'asgi': {'version': '3.0', 'spec_version': '2.0'}} -2022-03-26 15:10:51,799 - TRACE - ASGI [1] Receive {'type': 'lifespan.startup'} -2022-03-26 15:10:51,799 - TRACE - ASGI [1] Send {'type': 'lifespan.startup.complete'} -2022-03-26 15:10:51,799 - INFO - Application startup complete. -2022-03-26 15:11:46,805 - WARNING - StatReload detected file change in 'main.py'. Reloading... -2022-03-26 15:11:46,839 - INFO - Shutting down -2022-03-26 15:11:46,941 - INFO - Waiting for application shutdown. -2022-03-26 15:11:46,941 - TRACE - ASGI [1] Receive {'type': 'lifespan.shutdown'} -2022-03-26 15:11:46,942 - TRACE - ASGI [1] Send {'type': 'lifespan.shutdown.complete'} -2022-03-26 15:11:46,942 - TRACE - ASGI [1] Completed -2022-03-26 15:11:46,943 - INFO - Application shutdown complete. -2022-03-26 15:11:46,943 - INFO - Finished server process [36066] -2022-03-26 15:11:47,302 - INFO - Started server process [36085] -2022-03-26 15:11:47,302 - INFO - Waiting for application startup. -2022-03-26 15:11:47,302 - TRACE - ASGI [1] Started scope={'type': 'lifespan', 'asgi': {'version': '3.0', 'spec_version': '2.0'}} -2022-03-26 15:11:47,303 - TRACE - ASGI [1] Receive {'type': 'lifespan.startup'} -2022-03-26 15:11:47,303 - TRACE - ASGI [1] Send {'type': 'lifespan.startup.complete'} -2022-03-26 15:11:47,303 - INFO - Application startup complete. -2022-03-26 15:12:05,661 - INFO - Shutting down -2022-03-26 15:12:05,762 - INFO - Waiting for application shutdown. -2022-03-26 15:12:05,762 - TRACE - ASGI [1] Receive {'type': 'lifespan.shutdown'} -2022-03-26 15:12:05,762 - TRACE - ASGI [1] Send {'type': 'lifespan.shutdown.complete'} -2022-03-26 15:12:05,762 - TRACE - ASGI [1] Completed -2022-03-26 15:12:05,763 - INFO - Application shutdown complete. -2022-03-26 15:12:05,763 - INFO - Finished server process [36085] -2022-03-26 15:12:05,813 - INFO - Stopping reloader process [36045] -2022-03-26 15:12:38,060 - INFO - Will watch for changes in these directories: ['/home/pablo/devel/monitor_agent/monitor_agent'] -2022-03-26 15:12:38,061 - INFO - Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) -2022-03-26 15:12:38,061 - INFO - Started reloader process [36505] using statreload -2022-03-26 15:12:38,332 - INFO - Started server process [36510] -2022-03-26 15:12:38,333 - INFO - Waiting for application startup. -2022-03-26 15:12:38,333 - TRACE - ASGI [1] Started scope={'type': 'lifespan', 'asgi': {'version': '3.0', 'spec_version': '2.0'}} -2022-03-26 15:12:38,333 - TRACE - ASGI [1] Receive {'type': 'lifespan.startup'} -2022-03-26 15:12:38,333 - TRACE - ASGI [1] Send {'type': 'lifespan.startup.complete'} -2022-03-26 15:12:38,333 - INFO - Application startup complete. -2022-03-26 15:12:39,642 - INFO - Shutting down -2022-03-26 15:12:39,743 - INFO - Waiting for application shutdown. -2022-03-26 15:12:39,744 - TRACE - ASGI [1] Receive {'type': 'lifespan.shutdown'} -2022-03-26 15:12:39,744 - TRACE - ASGI [1] Send {'type': 'lifespan.shutdown.complete'} -2022-03-26 15:12:39,745 - TRACE - ASGI [1] Completed -2022-03-26 15:12:39,745 - INFO - Application shutdown complete. -2022-03-26 15:12:39,746 - INFO - Finished server process [36510] -2022-03-26 15:12:39,815 - INFO - Stopping reloader process [36505] -2022-03-26 15:12:44,043 - INFO - Will watch for changes in these directories: ['/home/pablo/devel/monitor_agent/monitor_agent'] -2022-03-26 15:12:44,044 - INFO - Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) -2022-03-26 15:12:44,044 - INFO - Started reloader process [36527] using statreload -2022-03-26 15:12:44,314 - INFO - Started server process [36532] -2022-03-26 15:12:44,315 - INFO - Waiting for application startup. -2022-03-26 15:12:44,315 - TRACE - ASGI [1] Started scope={'type': 'lifespan', 'asgi': {'version': '3.0', 'spec_version': '2.0'}} -2022-03-26 15:12:44,315 - TRACE - ASGI [1] Receive {'type': 'lifespan.startup'} -2022-03-26 15:12:44,315 - TRACE - ASGI [1] Send {'type': 'lifespan.startup.complete'} -2022-03-26 15:12:44,315 - INFO - Application startup complete. -2022-03-26 15:12:45,816 - WARNING - StatReload detected file change in 'main.py'. Reloading... -2022-03-26 15:12:45,824 - INFO - Shutting down -2022-03-26 15:12:45,925 - INFO - Waiting for application shutdown. -2022-03-26 15:12:45,925 - TRACE - ASGI [1] Receive {'type': 'lifespan.shutdown'} -2022-03-26 15:12:45,926 - TRACE - ASGI [1] Send {'type': 'lifespan.shutdown.complete'} -2022-03-26 15:12:45,926 - TRACE - ASGI [1] Completed -2022-03-26 15:12:45,926 - INFO - Application shutdown complete. -2022-03-26 15:12:45,927 - INFO - Finished server process [36532] -2022-03-26 15:12:46,320 - INFO - Started server process [36543] -2022-03-26 15:12:46,320 - INFO - Waiting for application startup. -2022-03-26 15:12:46,321 - TRACE - ASGI [1] Started scope={'type': 'lifespan', 'asgi': {'version': '3.0', 'spec_version': '2.0'}} -2022-03-26 15:12:46,321 - TRACE - ASGI [1] Receive {'type': 'lifespan.startup'} -2022-03-26 15:12:46,321 - TRACE - ASGI [1] Send {'type': 'lifespan.startup.complete'} -2022-03-26 15:12:46,321 - INFO - Application startup complete. -2022-03-26 15:12:47,830 - INFO - Shutting down -2022-03-26 15:12:47,931 - INFO - Waiting for application shutdown. -2022-03-26 15:12:47,932 - TRACE - ASGI [1] Receive {'type': 'lifespan.shutdown'} -2022-03-26 15:12:47,932 - TRACE - ASGI [1] Send {'type': 'lifespan.shutdown.complete'} -2022-03-26 15:12:47,933 - TRACE - ASGI [1] Completed -2022-03-26 15:12:47,933 - INFO - Application shutdown complete. -2022-03-26 15:12:47,933 - INFO - Finished server process [36543] -2022-03-26 15:12:47,982 - INFO - Stopping reloader process [36527] diff --git a/monitor_agent/settings.json b/monitor_agent/settings.json index 7cbb829..7dfe43d 100644 --- a/monitor_agent/settings.json +++ b/monitor_agent/settings.json @@ -3,8 +3,9 @@ "url": "127.0.0.1:8000/alerts" }, "auth": { - "agent_token": "", - "api_token": "" + "id_token": "", + "api_token": "", + "name": "" }, "logging": { "filename": "monitor.log", From e44625125933b12fb123f24eae85048b09228564 Mon Sep 17 00:00:00 2001 From: n0nuser Date: Mon, 28 Mar 2022 01:06:55 +0200 Subject: [PATCH 13/15] Modified logging. Fixed Settings file. --- monitor_agent/core/helper.py | 10 +++++----- monitor_agent/core/metricFunctions.py | 21 ++++++++++++++------- monitor_agent/core/models.py | 4 ++-- monitor_agent/main.py | 19 ++++++++++++++----- monitor_agent/settings.json | 14 +++++++------- monitor_agent/settings.py | 1 - 6 files changed, 42 insertions(+), 27 deletions(-) diff --git a/monitor_agent/core/helper.py b/monitor_agent/core/helper.py index 856827f..b787bbe 100644 --- a/monitor_agent/core/helper.py +++ b/monitor_agent/core/helper.py @@ -1,7 +1,7 @@ import logging -def getLogger(CONFIG): +def getLogger(level: str, filename: str): log_level = "" tranlation = { "debug": logging.DEBUG, @@ -11,8 +11,8 @@ def getLogger(CONFIG): "critical": logging.CRITICAL, } try: - if CONFIG.logging.level in tranlation.keys(): - log_level = tranlation[CONFIG.logging.level] + if level in tranlation.keys(): + log_level = tranlation[level] else: logging.warning("Level not established in Settings.json") log_level = logging.info @@ -25,7 +25,7 @@ def getLogger(CONFIG): log_filename = "" try: - log_filename = CONFIG.logging.filename + log_filename = filename except AttributeError as e: logging.warning( 'Log filename not established in Settings.json.\nDefault "monitor.log" file will be used.', @@ -33,7 +33,7 @@ def getLogger(CONFIG): ) log_filename = "monitor.log" - return logging.basicConfig( + logging.basicConfig( level=log_level, filename=log_filename, format="%(asctime)s - %(levelname)s - %(message)s", diff --git a/monitor_agent/core/metricFunctions.py b/monitor_agent/core/metricFunctions.py index 1a51b97..a53d8b0 100644 --- a/monitor_agent/core/metricFunctions.py +++ b/monitor_agent/core/metricFunctions.py @@ -4,8 +4,7 @@ import requests import logging -from monitor_agent.main import LOGGER -from core.models import Status, MetricDynamic, MetricStatic +from monitor_agent.core.models import Status, MetricDynamic, MetricStatic def execution_time_decorator(function) -> typing.Tuple[float, dict]: @@ -34,18 +33,26 @@ def send_metrics_adapter(function_list: list) -> typing.Tuple[dict, dict]: elapsed_time.update(f_time) data.update(f_data) except TypeError as msg: - LOGGER.warning(f"TypeError: {msg}", exc_info=True) + logging.warning(f"TypeError: {msg}", exc_info=True) continue return elapsed_time, data def send_metrics( - url: str, elapsed_time: dict, data: dict, file_enabled: bool, file_path: str + url: str, elapsed_time: dict, data: dict, file_enabled: bool, file_path: str, auth: dict, port:int ): + auth.update({"port": port}) + auth.pop("__module__", None) + auth.pop("__dict__", None) + auth.pop("__weakref__", None) + auth.pop("__doc__", None) status = Status(elapsed=elapsed_time).__dict__ - json_request = {"data": data, "status": status} - r = requests.post(url, json=json_request) - # DEBUG + json_request = {"auth": auth, "data": data, "status": status} + try: + r = requests.post(url, json=json_request, headers={'Authorization': auth["api_token"]}) + except requests.exceptions.InvalidSchema as e: + logging.critical(f"Agent could not send metrics to server {url}", exc_info=True) + logging.debug(f"Response: {r.text}") logging.debug(f"Status Code: {r.status_code}") if file_enabled: with open(file_path, "w") as f: diff --git a/monitor_agent/core/models.py b/monitor_agent/core/models.py index 8bf7466..1a34858 100644 --- a/monitor_agent/core/models.py +++ b/monitor_agent/core/models.py @@ -3,7 +3,7 @@ import time import datetime -from monitor_agent.main import LOGGER +from monitor_agent.main import logging class Status: @@ -115,7 +115,7 @@ def _process(ram: int, pc_cpu_percent): # Requires elevated permissions process[p.pid]["path"] = p.exe() except (PermissionError, psutil.AccessDenied): - LOGGER.warning( + logging.warning( f"Could not get Username or Path for process {p.name()}" ) return process diff --git a/monitor_agent/main.py b/monitor_agent/main.py index eec410b..2af1844 100644 --- a/monitor_agent/main.py +++ b/monitor_agent/main.py @@ -1,5 +1,4 @@ import json -from monitor_agent.core.helper import getLogger import uvicorn import logging import requests @@ -7,6 +6,7 @@ from .core.command import Command from fastapi import FastAPI, UploadFile from fastapi_utils.tasks import repeat_every +from monitor_agent.core.helper import getLogger try: @@ -15,7 +15,7 @@ logging.critical(f'Error in "settings.json". {msg}') exit() -LOGGER = getLogger(CONFIG) +getLogger(CONFIG.logging.level, CONFIG.logging.filename) from .core.metricFunctions import send_metrics, send_metrics_adapter, static, dynamic @@ -63,21 +63,26 @@ async def mod_settings(settings: UploadFile): CONFIG = Settings() return msg +logging.debug(CONFIG.metrics.post_interval) @api.on_event("startup") -@repeat_every(seconds=CONFIG.metrics.post_interval, logger=LOGGER, wait_first=True) +@repeat_every(seconds=CONFIG.metrics.post_interval, logger=logging, wait_first=True) def periodic(): # https://github.com/tiangolo/fastapi/issues/520 # https://fastapi-utils.davidmontague.xyz/user-guide/repeated-tasks/#the-repeat_every-decorator # Changed Timeloop for this elapsed_time, data = send_metrics_adapter([static, dynamic]) + logging.debug("Sending metrics") send_metrics( url=CONFIG.metrics.post_url, elapsed_time=elapsed_time, data=data, file_enabled=CONFIG.metrics.enable_logfile, file_path=CONFIG.metrics.log_filename, + auth=dict(CONFIG.auth.__dict__), + port=CONFIG.uvicorn.port ) + logging.debug("Metrics sent!") alert = {} if data["cpu_percent"] >= CONFIG.thresholds.cpu_percent: @@ -90,7 +95,11 @@ def periodic(): except KeyError as msg: pass if alert: - r = requests.post(CONFIG.alerts.url, json={"alert": alert}) + try: + r = requests.post(CONFIG.alerts.url, json={"alert": alert}) + except requests.exceptions.InvalidSchema as e: + logging.error(f"Agent could not send an alert to {CONFIG.alerts.url}", exc_info=True) + def start(): @@ -104,4 +113,4 @@ def start(): try: uvicorn.run(**uviconfig) except: - LOGGER.critical("Unable to run server.", exc_info=True) + logging.critical("Unable to run server.", exc_info=True) diff --git a/monitor_agent/settings.json b/monitor_agent/settings.json index 7dfe43d..62f6144 100644 --- a/monitor_agent/settings.json +++ b/monitor_agent/settings.json @@ -1,22 +1,22 @@ { "alerts": { - "url": "127.0.0.1:8000/alerts" + "url": "http://127.0.0.1:8000/api/alerts/" }, "auth": { - "id_token": "", - "api_token": "", - "name": "" + "api_token": "7320730b62807c221d0058eff8d13fa26b714db6", + "id_token": "79cff2298c9a54d0ccef292fe8938e4055faa708", + "name": "NUC Server" }, "logging": { "filename": "monitor.log", - "level": "info" + "level": "debug" }, "metrics": { "enable_logfile": false, "get_endpoint": true, "log_filename": "metrics.json", "post_interval": 60, - "post_url": "http://httpbin.org/post" + "post_url": "http://127.0.0.1:8000/api/metrics/" }, "thresholds": { "cpu_percent": 50, @@ -27,7 +27,7 @@ "debug": true, "host": "0.0.0.0", "log_level": "trace", - "port": 8000, + "port": 8080, "reload": true, "timeout_keep_alive": 5, "workers": 4 diff --git a/monitor_agent/settings.py b/monitor_agent/settings.py index 57eb402..a7aaca0 100644 --- a/monitor_agent/settings.py +++ b/monitor_agent/settings.py @@ -1,5 +1,4 @@ import os -import sys import json import logging From ccde1684ae2309f3611ce3e100d98cd461dd0a06 Mon Sep 17 00:00:00 2001 From: n0nuser Date: Tue, 29 Mar 2022 11:55:11 +0200 Subject: [PATCH 14/15] Improved Settings.json. Sends metrics finally --- monitor_agent/core/metricFunctions.py | 57 +++++++++++++++++++++------ monitor_agent/main.py | 35 ++++++++-------- monitor_agent/settings.json | 13 +++--- monitor_agent/settings.py | 1 + 4 files changed, 72 insertions(+), 34 deletions(-) diff --git a/monitor_agent/core/metricFunctions.py b/monitor_agent/core/metricFunctions.py index a53d8b0..4fdfa1f 100644 --- a/monitor_agent/core/metricFunctions.py +++ b/monitor_agent/core/metricFunctions.py @@ -1,8 +1,10 @@ -import time import json +import time import typing import requests import logging +from requests.exceptions import ConnectionError, InvalidSchema +from urllib3.exceptions import MaxRetryError, NewConnectionError from monitor_agent.core.models import Status, MetricDynamic, MetricStatic @@ -39,21 +41,50 @@ def send_metrics_adapter(function_list: list) -> typing.Tuple[dict, dict]: def send_metrics( - url: str, elapsed_time: dict, data: dict, file_enabled: bool, file_path: str, auth: dict, port:int + elapsed_time: dict, + metrics: dict, + file_enabled: bool, + file_path: str, + metric_endpoint: str, + agent_endpoint: str, + user_token: str, + agent_token: str, + name: str, ): - auth.update({"port": port}) - auth.pop("__module__", None) - auth.pop("__dict__", None) - auth.pop("__weakref__", None) - auth.pop("__doc__", None) status = Status(elapsed=elapsed_time).__dict__ - json_request = {"auth": auth, "data": data, "status": status} + if not agent_endpoint.endswith("/"): + agent_endpoint = agent_endpoint + "/" + if not metric_endpoint.endswith("/"): + metric_endpoint = metric_endpoint + "/" + + agent_token = f"{agent_endpoint}{agent_token}/" + json_request = { + "agent": agent_token, + "name": name, + "metrics": metrics, + "status": status, + } + logging.debug(f"Agent token: {agent_token}") + logging.debug(f"Metric endpoint: {metric_endpoint}") + try: - r = requests.post(url, json=json_request, headers={'Authorization': auth["api_token"]}) - except requests.exceptions.InvalidSchema as e: - logging.critical(f"Agent could not send metrics to server {url}", exc_info=True) - logging.debug(f"Response: {r.text}") - logging.debug(f"Status Code: {r.status_code}") + r = requests.post( + metric_endpoint, + json=json_request, + headers={"Authorization": f"Token {user_token}"}, + ) + logging.debug(f"Response: {r.text}") + logging.debug(f"Status Code: {r.status_code}") + except ( + MaxRetryError, + NewConnectionError, + ConnectionRefusedError, + ConnectionError, + InvalidSchema, + ) as e: + logging.critical( + f"Agent could not send metrics to server {metric_endpoint}", exc_info=True + ) if file_enabled: with open(file_path, "w") as f: f.write(json.dumps(json_request, indent=4, sort_keys=True)) diff --git a/monitor_agent/main.py b/monitor_agent/main.py index 2af1844..d26503f 100644 --- a/monitor_agent/main.py +++ b/monitor_agent/main.py @@ -63,7 +63,9 @@ async def mod_settings(settings: UploadFile): CONFIG = Settings() return msg -logging.debug(CONFIG.metrics.post_interval) + +logging.debug(f"POST Interval: {CONFIG.metrics.post_interval}") + @api.on_event("startup") @repeat_every(seconds=CONFIG.metrics.post_interval, logger=logging, wait_first=True) @@ -71,35 +73,36 @@ def periodic(): # https://github.com/tiangolo/fastapi/issues/520 # https://fastapi-utils.davidmontague.xyz/user-guide/repeated-tasks/#the-repeat_every-decorator # Changed Timeloop for this - elapsed_time, data = send_metrics_adapter([static, dynamic]) - logging.debug("Sending metrics") + elapsed_time, metrics = send_metrics_adapter([static, dynamic]) send_metrics( - url=CONFIG.metrics.post_url, elapsed_time=elapsed_time, - data=data, + metrics=metrics, file_enabled=CONFIG.metrics.enable_logfile, file_path=CONFIG.metrics.log_filename, - auth=dict(CONFIG.auth.__dict__), - port=CONFIG.uvicorn.port + metric_endpoint=CONFIG.endpoints.metric_endpoint, + agent_endpoint=CONFIG.endpoints.agent_endpoint, + agent_token=CONFIG.auth.agent_token, + user_token=CONFIG.auth.user_token, + name=CONFIG.auth.name, ) - logging.debug("Metrics sent!") alert = {} - if data["cpu_percent"] >= CONFIG.thresholds.cpu_percent: - alert["cpu_percent"] = data["cpu_percent"] - if data["ram"]["percent"] >= CONFIG.thresholds.ram_percent: - alert["ram_percent"] = data["ram"]["percent"] + if metrics["cpu_percent"] >= CONFIG.thresholds.cpu_percent: + alert["cpu_percent"] = metrics["cpu_percent"] + if metrics["ram"]["percent"] >= CONFIG.thresholds.ram_percent: + alert["ram_percent"] = metrics["ram"]["percent"] try: - if data["process"]: - alert["processes"] = data["process"] + if metrics["process"]: + alert["processes"] = metrics["process"] except KeyError as msg: pass if alert: try: r = requests.post(CONFIG.alerts.url, json={"alert": alert}) except requests.exceptions.InvalidSchema as e: - logging.error(f"Agent could not send an alert to {CONFIG.alerts.url}", exc_info=True) - + logging.error( + f"Agent could not send an alert to {CONFIG.alerts.url}", exc_info=True + ) def start(): diff --git a/monitor_agent/settings.json b/monitor_agent/settings.json index 62f6144..a5b6a7f 100644 --- a/monitor_agent/settings.json +++ b/monitor_agent/settings.json @@ -3,9 +3,13 @@ "url": "http://127.0.0.1:8000/api/alerts/" }, "auth": { - "api_token": "7320730b62807c221d0058eff8d13fa26b714db6", - "id_token": "79cff2298c9a54d0ccef292fe8938e4055faa708", - "name": "NUC Server" + "agent_token": "31dc98714ebef0f1feadae06efaee156ab9210fa", + "name": "localhost", + "user_token": "690b8d290d08fa26eecf3d69b6d2b76fee28b430" + }, + "endpoints": { + "agent_endpoint": "http://localhost:8000/api/agents/", + "metric_endpoint": "http://127.0.0.1:8000/api/metrics/" }, "logging": { "filename": "monitor.log", @@ -15,8 +19,7 @@ "enable_logfile": false, "get_endpoint": true, "log_filename": "metrics.json", - "post_interval": 60, - "post_url": "http://127.0.0.1:8000/api/metrics/" + "post_interval": 10 }, "thresholds": { "cpu_percent": 50, diff --git a/monitor_agent/settings.py b/monitor_agent/settings.py index a7aaca0..4a9a51f 100644 --- a/monitor_agent/settings.py +++ b/monitor_agent/settings.py @@ -16,6 +16,7 @@ def __init__(self): self.auth = _obj.auth self.logging = _obj.logging self.metrics = _obj.metrics + self.endpoints = _obj.endpoints self.thresholds = _obj.thresholds self.uvicorn = _obj.uvicorn From eb556c7e5b566700b4e5880ef736d1d9bb4bd949 Mon Sep 17 00:00:00 2001 From: n0nuser Date: Thu, 7 Apr 2022 23:03:30 +0200 Subject: [PATCH 15/15] Remote Work --- monitor_agent/core/helper.py | 6 +++--- monitor_agent/settings.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/monitor_agent/core/helper.py b/monitor_agent/core/helper.py index b787bbe..d01a194 100644 --- a/monitor_agent/core/helper.py +++ b/monitor_agent/core/helper.py @@ -3,7 +3,7 @@ def getLogger(level: str, filename: str): log_level = "" - tranlation = { + translation = { "debug": logging.DEBUG, "info": logging.INFO, "warning": logging.WARNING, @@ -11,8 +11,8 @@ def getLogger(level: str, filename: str): "critical": logging.CRITICAL, } try: - if level in tranlation.keys(): - log_level = tranlation[level] + if level in translation.keys(): + log_level = translation[level] else: logging.warning("Level not established in Settings.json") log_level = logging.info diff --git a/monitor_agent/settings.json b/monitor_agent/settings.json index a5b6a7f..3489fb9 100644 --- a/monitor_agent/settings.json +++ b/monitor_agent/settings.json @@ -3,9 +3,9 @@ "url": "http://127.0.0.1:8000/api/alerts/" }, "auth": { - "agent_token": "31dc98714ebef0f1feadae06efaee156ab9210fa", + "agent_token": "1b08ef6ec3af737bcc443050a3556638251abfe6", "name": "localhost", - "user_token": "690b8d290d08fa26eecf3d69b6d2b76fee28b430" + "user_token": "609d79c47e1beb3ec0425ec2e3ad5bbea79994f4" }, "endpoints": { "agent_endpoint": "http://localhost:8000/api/agents/", @@ -19,7 +19,7 @@ "enable_logfile": false, "get_endpoint": true, "log_filename": "metrics.json", - "post_interval": 10 + "post_interval": 60 }, "thresholds": { "cpu_percent": 50,