From 6ba394fa0910bfd814b054f9f9efe7d2acf1c509 Mon Sep 17 00:00:00 2001 From: Travis <35282992+blink-zero@users.noreply.github.com> Date: Sat, 22 Jun 2024 16:48:30 +1000 Subject: [PATCH] updated log rotation and clearing of older than 30 days logs --- run.py | 98 +++++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 73 insertions(+), 25 deletions(-) diff --git a/run.py b/run.py index e038814..faba9e5 100644 --- a/run.py +++ b/run.py @@ -3,7 +3,6 @@ import time import json import logging -from logging.handlers import TimedRotatingFileHandler import os from flask_migrate import Migrate from flask import render_template, session @@ -134,15 +133,15 @@ def inject_host_status(): # Set up logging log_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') -# Handler for app.log (standard logs) -file_handler = TimedRotatingFileHandler('logs/app.log', when='midnight', interval=1, backupCount=30) -file_handler.suffix = "%Y-%m-%d_%H-%M-%S" +# Separate log files for regular and JSON logs +log_file_path = 'logs/app.log' +json_log_file_path = 'logs/app_json.log' + +file_handler = logging.FileHandler(log_file_path) file_handler.setFormatter(log_formatter) file_handler.setLevel(logging.INFO) -# Handler for app_json.log (JSON logs) -json_file_handler = TimedRotatingFileHandler('logs/app_json.log', when='midnight', interval=1, backupCount=30) -json_file_handler.suffix = "%Y-%m-%d_%H-%M-%S" +json_file_handler = logging.FileHandler(json_log_file_path) json_file_handler.setFormatter(JSONFormatter()) json_file_handler.setLevel(logging.INFO) @@ -158,33 +157,82 @@ def inject_host_status(): json_logger.setLevel(logging.INFO) json_logger.addHandler(json_file_handler) -# Clean up old log files -log_directory = 'logs' -log_retention_days = 30 -log_file_pattern = re.compile(r'app\.log\.\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}') +# Flags to ensure only one instance of each thread is running +log_cleanup_thread_running = False +json_log_cleanup_thread_running = False -for log_file in os.listdir(log_directory): - file_path = os.path.join(log_directory, log_file) - if os.path.isfile(file_path) and log_file_pattern.match(log_file): - file_creation_time = os.path.getctime(file_path) - if (time.time() - file_creation_time) // (24 * 3600) >= log_retention_days: - os.remove(file_path) +# Function to clean up old log entries +def clean_old_log_entries(log_file, retention_days=30): + global log_cleanup_thread_running + log_cleanup_thread_running = True + while True: + now = datetime.now() + # Calculate the time until the next midnight + next_midnight = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0) + sleep_time = (next_midnight - now).total_seconds() + time.sleep(sleep_time) -# Function to wait until the next midnight and log a message -def log_at_midnight(): + cutoff_date = datetime.now() - timedelta(days=retention_days) + with open(log_file, 'r') as f: + lines = f.readlines() + + with open(log_file, 'w') as f: + for line in lines: + try: + log_entry = json.loads(line) + log_date = datetime.strptime(log_entry['timestamp'], '%Y-%m-%d %H:%M:%S') + if log_date > cutoff_date: + f.write(line) + except json.JSONDecodeError: + # Handle non-JSON formatted logs + match = re.match(r'^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})', line) + if match: + log_date = datetime.strptime(match.group(1), '%Y-%m-%d %H:%M:%S') + if log_date > cutoff_date: + f.write(line) + log_cleanup_thread_running = False + +# Function to clean up old JSON log entries +def clean_old_json_log_entries(log_file, retention_days=30): + global json_log_cleanup_thread_running + json_log_cleanup_thread_running = True while True: now = datetime.now() # Calculate the time until the next midnight next_midnight = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0) sleep_time = (next_midnight - now).total_seconds() time.sleep(sleep_time) - app.logger.info("Midnight log entry to trigger log rotation.") - json_logger.info("Midnight JSON log entry to trigger log rotation.") -# Start the midnight logging in a separate thread -midnight_thread = threading.Thread(target=log_at_midnight) -midnight_thread.daemon = True -midnight_thread.start() + cutoff_date = datetime.now() - timedelta(days=retention_days) + with open(log_file, 'r') as f: + lines = f.readlines() + + with open(log_file, 'w') as f: + for line in lines: + try: + log_entry = json.loads(line) + log_date = datetime.strptime(log_entry['timestamp'], '%Y-%m-%d %H:%M:%S') + if log_date > cutoff_date: + f.write(line) + except json.JSONDecodeError: + # Handle non-JSON formatted logs + match = re.match(r'^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})', line) + if match: + log_date = datetime.strptime(match.group(1), '%Y-%m-%d %H:%M:%S') + if log_date > cutoff_date: + f.write(line) + json_log_cleanup_thread_running = False + +# Start the log cleanup in a separate thread if not already running +if not log_cleanup_thread_running: + log_cleanup_thread = threading.Thread(target=clean_old_log_entries, args=(log_file_path,)) + log_cleanup_thread.daemon = True + log_cleanup_thread.start() + +if not json_log_cleanup_thread_running: + json_log_cleanup_thread = threading.Thread(target=clean_old_json_log_entries, args=(json_log_file_path,)) + json_log_cleanup_thread.daemon = True + json_log_cleanup_thread.start() if __name__ == "__main__": app.run(host='0.0.0.0')