diff --git a/packaging/siterm-site-fe/scripts/DBCleaner-service b/packaging/siterm-site-fe/scripts/DBCleaner-service new file mode 100644 index 00000000..bc397bc6 --- /dev/null +++ b/packaging/siterm-site-fe/scripts/DBCleaner-service @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +""" +This part of code is taken from: + https://web.archive.org/web/20160305151936/http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/ +Please respect developer (Sander Marechal) and always keep a reference to URL and also as kudos to him +Changes applied to this code: + Dedention (Justas Balcas 07/12/2017) + pylint fixes: with open, split imports, var names, old style class (Justas Balcas 07/12/2017) +""" +import sys +from SiteRMLibs.Daemonizer import Daemon, getParser, validateArgs +from SiteRMLibs.DBCleaner import DBCleaner as dbcleaner + +COMPONENT = 'DBCleaner' +DESCRIPTION = "DBCleaner responsible for cleaning database of old entries." +DAEMON = None + +class MyDaemon(Daemon): + """ My own Deamon override """ + + def getThreads(self): + """Multi threading. Allow multiple sites under single FE""" + for sitename in self.config.get('general', 'sites'): + if sitename not in self.runThreads: + thr = dbcleaner(self.config, sitename) + self.runThreads[sitename] = thr + self.runThreads[sitename].refreshthread() + + +if __name__ == "__main__": + parser = getParser(DESCRIPTION) + if len(sys.argv) == 1: + parser.print_help() + inargs = parser.parse_args(sys.argv[1:]) + validateArgs(inargs) + DAEMON = MyDaemon(COMPONENT, inargs) + DAEMON.command() diff --git a/packaging/siterm-site-fe/scripts/DBCleaner.py b/packaging/siterm-site-fe/scripts/DBCleaner.py deleted file mode 100755 index c90853d1..00000000 --- a/packaging/siterm-site-fe/scripts/DBCleaner.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 -"""Database cleaner""" -import time -import mariadb -from SiteRMLibs.MainUtilities import getVal -from SiteRMLibs.MainUtilities import getDBConn -from SiteRMLibs.MainUtilities import getUTCnow -from SiteRMLibs.GitConfig import getGitConfig - -class DBCleaner: - """DBCleaner class""" - def __init__(self): - self.config = getGitConfig() - self.sites = self.config.get('general', 'sites') - for site in self.sites: - self.dbI = {} - self.dbI[site] = getVal(getDBConn('List', self), **{'sitename': site}) - - def clean(self, dbtable, olderthan): - """Clean the database""" - for site in self.sites: - print(f"Cleaning {dbtable} for {site}") - try: - data = self.dbI[site].get(dbtable, limit=100, orderby=["insertdate", "ASC"]) - except mariadb.OperationalError: - data = self.dbI[site].get(dbtable, limit=100, orderby=["updatedate", "ASC"]) - for item in data: - if 'updatedate' in item: - if item['updatedate'] < int(getUTCnow() - olderthan): - print(f"Deleting {item['id']} from {dbtable}") - self.dbI[site].delete(dbtable, [["id", item["id"]]]) - elif 'insertdate' in item: - if item['insertdate'] < int(getUTCnow() - olderthan): - print(f"Deleting {item['id']} from {dbtable}") - self.dbI[site].delete(dbtable, [["id", item["id"]]]) - else: - print(f"Item {item['id']} from {dbtable} does not have timestamp. Ignoring") - - - def start(self): - """Start the cleaner""" - print('Starting cleaner') - for table in ['debugrequests', 'deltas', 'deltatimestates', 'hosts', 'models', 'servicestates']: - self.clean(table, 86400*7) - print('Cleaner finished') - -if __name__ == "__main__": - dbcleaner = DBCleaner() - while True: - try: - dbcleaner.start() - except Exception as err: - print(f"Error in DB Cleaner: {err}") - print('Sleeping for 24 hour before next cleaning cycle') - time.sleep(86400) diff --git a/src/python/SiteRMLibs/DBCleaner.py b/src/python/SiteRMLibs/DBCleaner.py new file mode 100644 index 00000000..c9399be9 --- /dev/null +++ b/src/python/SiteRMLibs/DBCleaner.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Database cleaner""" +import mariadb +from SiteRMLibs.MainUtilities import (getDBConn, getLoggingObject, getVal, getUTCnow) +from SiteRMLibs.GitConfig import getGitConfig + + +class DBCleaner: + """DBCleaner class""" + def __init__(self, config, sitename): + self.config = config + self.sitename = sitename + self.logger = getLoggingObject(config=self.config, service="DBCleaner") + self.dbI = getVal(getDBConn("DBWorker", self), **{"sitename": self.sitename}) + self.nextRun = int(getUTCnow() - 1) # Make sure runs first time + + def clean(self, dbtable, olderthan): + """Clean the database""" + print(f"Cleaning {dbtable} for {self.sitename}") + try: + data = self.dbI.get(dbtable, limit=100, orderby=["insertdate", "ASC"]) + except mariadb.OperationalError: + data = self.dbI.get(dbtable, limit=100, orderby=["updatedate", "ASC"]) + for item in data: + if 'updatedate' in item: + if item['updatedate'] < int(getUTCnow() - olderthan): + print(f"Deleting {item['id']} from {dbtable}") + self.dbI.delete(dbtable, [["id", item["id"]]]) + elif 'insertdate' in item: + if item['insertdate'] < int(getUTCnow() - olderthan): + print(f"Deleting {item['id']} from {dbtable}") + self.dbI.delete(dbtable, [["id", item["id"]]]) + else: + print(f"Item {item['id']} from {dbtable} does not have timestamp. Ignoring") + + def startwork(self): + """Start the cleaner""" + if self.nextRun > getUTCnow(): + return + self.nextRun = int(getUTCnow() + 7200) # Run every 2 hours + print('Starting cleaner') + for table in ['debugrequests', 'deltas', 'deltatimestates', 'hosts', 'models', 'servicestates', 'states', + 'hoststates', 'hoststateshistory', 'parsed', 'switch', 'snmpmon', 'serviceaction', 'activeDeltas']: + print(f"Cleaning {table}") + self.clean(table, 86400*7) + print('Cleaner finished') + + +if __name__ == "__main__": + logObj = getLoggingObject(logType='StreamLogger', service='DBCleaner') + gconfig = getGitConfig() + for siteName in gconfig.get("general", "sites"): + dbworker = DBCleaner(gconfig, siteName) + dbworker.startwork()