sb_log_manager.py

"""Collects log files generated by other SB apps for import into Django

"""
import configparser
from datetime import datetime, timedelta
from django.core.wsgi import get_wsgi_application
from email.mime.text import MIMEText
from email.utils import formataddr
import json
import logging
import logging.config
import os
import pytz
import smtplib
import re

import sys
from unipath import Path, FILES

def determine_next_date(minute_code, hour_code, day_code, month_code, weekday_code):
    """Determines next update date based on the provided criteria"""
    def convert_month_code(month_code):
        """Converts provided month code to numerical list"""
        log.debug("Provided Month Code: {}".format(month_code))

        # Split comma separated codes into a list
        split_codes = month_code.split(",")

        # Convert any */# codes or ABC codes into specific months
        codes = set()

        alpha_months = {
            "JAN": 1,
            "FEB": 2,
            "MAR": 3,
            "APR": 4,
            "MAY": 5,
            "JUN": 6,
            "JUL": 7,
            "AUG": 8,
            "SEP": 9,
            "OCT": 10,
            "NOV": 11,
            "DEC": 12
        }

        for c in split_codes:
            code = c.upper().strip()

            # Run the regex matches
            re_num = patterns["num"].match(code)
            re_range = patterns["range"].match(code)
            re_mod = patterns["mod"].match(code)
            re_mod_range = patterns["mod_range"].match(code)
            re_alpha = patterns["alpha"].match(code)
            re_alpha_range = patterns["alpha_range"].match(code)
            
            if re_num:
                # Code is numeric and be directly added
                codes.add(int(code))
            elif re_range:
                # Code is range of numbers
                first_num = int(re_range.group(1))
                last_num = int(re_range.group(2)) + 1
                
                for i in range(first_num, last_num):
                    codes.add(i)
            elif re_mod:
                # Add all months that match the mod function
                i = 1
                interval = int(re_mod.group(1))

                while i <= 12:
                    codes.add(i)    

                    i = i + interval
            elif re_mod_range:
                # Add all months that match the mod in the specified range
                i = int(re_mod_range.group(1))
                last_num = int(re_mod_range.group(2)) + 1
                interval = int(re_mod_range.group(3))

                while i < last_num:
                    codes.add(i)

                    i = i + interval
            elif re_alpha:
                # Add the numerical value of the month
                codes.add(alpha_months[int(code)])
            elif re_alpha_range:
                # Add all the numerical values of the month range
                first_num = alpha_months[re_alpha_range.group(1)]
                last_num = alpha_months[re_alpha_range.group(2)] + 1

                for i in range(first_num, last_num):
                    codes.add(i)
            else:
                # No regex match found, add all months
                for i in range(1,13):
                    codes.add(i)
        
        log.debug("Formatted Month Codes: {}".format(codes))

        return list(codes)

    def convert_weekday_code(weekday_code):
        """Converts provided weekday code to numerical list"""
        log.debug("Provided Weekday Code: {}".format(weekday_code))

        split_codes = weekday_code.split(",")

        # Converts any alpha codes into numbers
        codes = set()

        alpha_weekdays = {
            "SUN": 0,
            "MON": 1,
            "TUE": 2,
            "WED": 3,
            "THU": 4,
            "FRI": 5,
            "SAT": 6
        }

        for c in split_codes:
            code = c.upper().strip()

            # Run the regex matches
            re_num = patterns["num"].match(code)
            re_range = patterns["range"].match(code)
            re_mod = patterns["mod"].match(code)
            re_mod_range = patterns["mod_range"].match(code)
            re_alpha = patterns["alpha"].match(code)
            re_alpha_range = patterns["alpha_range"].match(code)
            
            if re_num:
                # Code is numeric and can be directly added
                codes.add(int(code))
            elif re_range:
                # Code is range of numbers that can be added
                first_num = int(re_range.group(1))
                last_num = int(re_range.group(2)) + 1
                
                for i in range(first_num, last_num):
                    codes.add(i)
            elif re_mod:
                # Add all matching mod values
                i = 0
                interval = int(re_mod.group(1))

                while i < 7:
                    codes.add(i)

                    i = i + interval
            elif re_mod_range:
                # Add all matching mod values in specified range
                i = int(re_mod_range.group(1))
                last_num = int(re_mod_range.group(2)) + 1
                interval = int(re_mod_range.group(3))

                while i < last_num:
                    codes.add(i)

                    i = i + interval
            elif re_alpha:
                # Add the numerical value of the DOW
                codes.add(alpha_weekdays[code])
            elif re_alpha_range:
                # Add all numerical values of the DOW range
                first_num = alpha_months[re_alpha_range.group(1)]
                last_num = alpha_months[re_alpha_range.group(2)] + 1

                for i in range(first_num, last_num):
                    codes.add(i)
            else:
                # No regex match found, add all weekdays
                for i in range(0,7):
                    codes.add(i)
        
        log.debug("Formatted Weekday Codes: {}".format(codes))

        return list(codes)

    def convert_day_code(day_code):
        """Converts provided day code to numerical list"""
        log.debug("Provided Day Code: {}".format(day_code))

        split_codes = day_code.split(",")

        codes = set()

        for c in split_codes:
            code = c.strip()

            # Run the regex matches
            re_num = patterns["num"].match(code)
            re_range = patterns["range"].match(code)
            re_mod = patterns["mod"].match(code)
            re_mod_range = patterns["mod_range"].match(code)

            if re_num:
                # Code is numeric and can be directly added
                codes.add(int(code))
            elif re_range:
                # Code is range of numbers that can be added
                first_num = int(re_range.group(1))
                last_num = int(re_range.group(2)) + 1
                
                for i in range(first_num, last_num):
                    codes.add(i)
            elif re_mod:
                # Add all matching mod values
                i = 1
                interval = int(re_mod.group(1))

                while i < 32:
                    codes.add(i)

                    i = i + interval
            elif re_mod_range:
                # Add all matching mod values in specified range
                i = int(re_mod_range.group(1))
                last_num = int(re_mod_range.group(2)) + 1
                interval = int(re_mod_range.group(3))

                while i < last_num:
                    codes.add(i)

                    i = i + interval
            else:
                # No regex match found, add all days
                for i in range(1,32):
                    codes.add(i)

        log.debug("Formatted Day Codes: {}".format(codes))

        return list(codes)

    def convert_hour_code(hour_code):
        """Converts provide hour code to numerical list"""
        log.debug("Provided Hour Code: {}".format(hour_code))

        split_codes = hour_code.split(",")

        codes = set()

        for c in split_codes:
            code = c.strip()

            # Run the regex matches
            re_num = patterns["num"].match(code)
            re_range = patterns["range"].match(code)
            re_mod = patterns["mod"].match(code)
            re_mod_range = patterns["mod_range"].match(code)

            if re_num:
                # Code is numeric and can be directly added
                codes.add(int(code))
            elif re_range:
                # Code is range of numbers that can be added
                first_num = int(re_range.group(1))
                last_num = int(re_range.group(2)) + 1
                
                for i in range(first_num, last_num):
                    codes.add(i)
            elif re_mod:
                # Add all matching mod values
                i = 0
                interval = int(re_mod.group(1))

                while i < 24:
                    codes.add(i)

                    i = i + interval
            elif re_mod_range:
                # Add all matching mod values in specified range
                i = int(re_mod_range.group(1))
                last_num = int(re_mod_range.group(2)) + 1
                interval = int(re_mod_range.group(3))

                while i < last_num:
                    codes.add(i)

                    i = i + interval
            else:
                # No regex match found, add all hours
                for i in range(0,24):
                    codes.add(i)

        log.debug("Formatted Hour Codes: {}".format(codes))

        return list(codes)

    def convert_minute_code(minute_code):
        """Converts provided minute code to numerical list"""
        log.debug("Provided Minute Code: {}".format(minute_code))

        split_codes = minute_code.split(",")
        
        codes = set()

        for c in split_codes:
            code = c.strip()

            # Run the regex matches
            re_num = patterns["num"].match(code)
            re_range = patterns["range"].match(code)
            re_mod = patterns["mod"].match(code)
            re_mod_range = patterns["mod_range"].match(code)

            if re_num:
                # Code is numeric and can be directly added
                codes.add(int(code))
            elif re_range:
                # Code is range of numbers that can be added
                first_num = int(re_range.group(1))
                last_num = int(re_range.group(2)) + 1
                
                for i in range(first_num, last_num):
                    codes.add(i)
            elif re_mod:
                # Add all matching mod values
                i = 0
                interval = int(re_mod.group(1))

                while i < 60:
                    codes.add(i)

                    i = i + interval
            elif re_mod_range:
                # Add all matching mod values in specified range
                i = int(re_mod_range.group(1))
                last_num = int(re_mod_range.group(2)) + 1
                interval = int(re_mod_range.group(3))

                while i < last_num:
                    codes.add(i)

                    i = i + interval
            else:
                # No regex match found, add all minutes
                for i in range(0,60):
                    codes.add(i)

        log.debug("Formatted Minute Codes: {}".format(codes))

        return list(codes)
    
    # Compile all the required regex patterns
    patterns = {
        "num": re.compile(r"^\d+$"),
        "range": re.compile(r"^(\d+)\-(\d+)$"),
        "mod": re.compile(r"^\*\/(\d+)$"),
        "mod_range": re.compile(r"^(\d+)\-(\d+)\/(\d+)$"),
        "alpha": re.compile(r"^[A-Z]{3}$"),
        "alpha_range": re.compile(r"^([A-Z]{3})\-([A-Z]{3})$")
    }

    # Properly format all the codes
    month_codes = convert_month_code(month_code)
    weekday_codes = convert_weekday_code(weekday_code)
    day_codes = convert_day_code(day_code)
    hour_codes = convert_hour_code(hour_code)
    minute_codes = convert_minute_code(minute_code)
    
    utc = pytz.timezone("UTC")
    now = datetime.now(utc)

    # Reset the seconds of the time
    now = now.replace(second=0, microsecond=0)
    
    # Iterate through days until match is found (month, weekday, day)
    for d in range(1, 4020):
        test_day = now + timedelta(days=d)

        # Test for matching month
        if test_day.month in month_codes:
            month_match = True
        else:
            month_match = False

        # Test for matching weekday
        if test_day.weekday() in weekday_codes:
            weekday_match = True
        else:
            weekday_match = False

        # Test for matching day
        if test_day.day in day_codes:
            day_match = True
        else:
            day_match = False
        
        if month_match and weekday_match and day_match:
            # All criteria matched, end loop
            break

    # Record the day on which the loop ended
    next_day = d

    # If we need to advance day, start the current time from 00:00
    if next_day > 0:
        now = now.replace(hour=0, minute=0)

    # Iterate through 24 hours to find hour match
    for h in range(0,24):
        test_hour = (now + timedelta(hours=h)).hour
            
        if test_hour in hour_codes:
            # Match found, end loop
            break

    # Record the hour which ended the loop
    next_hour = h
    
    # Iterate through 60 minutes to find minute match
    for m in range(0,60):
        test_minute = (now + timedelta(minutes=m)).minute
            
        if test_minute in minute_codes:
            # Match found, end loop
            break

    # Record the minute which ended the loop
    next_minute = m

    # Add all the recorded values to determine when the next job should run
    next_datetime = now + timedelta(days=next_day, hours=next_hour, minutes=next_minute)

    log.debug("Current time = {}".format(now))
    log.debug("Next review due = {}".format(next_datetime))

    return next_datetime

def email_start_failure(app_name, config):
    log.debug("Notifying owner of app failing to start")

    from_name = config.get("email", "from_name")
    from_email = config.get("email", "from_email")
    from_address = formataddr((from_name, from_email))

    to_name = config.get("email", "to_name")
    to_email = config.get("email", "to_email")
    to_address = formataddr((to_name, to_email))
       
    subject = "{} Application Failed To Start".format(app_name)
        
    message = (
        "This email is to notify you that the {} application did not "
        "start on time. Please view the monitored logs for further details."
    ).format(app_name)

    content = MIMEText(message)
    content['From'] = from_address
    content['To'] = to_address
    content['Subject'] = subject

    # Attempt to send email
    try:
        if config.getboolean("email", "debug"):
            log.debug(content.as_string())
        else:
            server = smtplib.SMTP(config.get("email", "server"))

            server.ehlo()
            server.starttls()
            server.sendmail(from_address, to_address, content.as_string())
            server.quit()
    except Exception:
        log.error("Unable to send notification to user", exc_info=True)
        
def email_error(app_name, config):
    log.debug("Notifying owner of errors with monitored applications")
    
    from_name = config.get("email", "from_name")
    from_email = config.get("email", "from_email")
    from_address = formataddr((from_name, from_email))

    to_name = config.get("email", "to_name")
    to_email = config.get("email", "to_email")
    to_address = formataddr((to_name, to_email))
       
    subject = "Errors in {} Application".format(app_name)
        
    message = (
        "This email is to notify you that there we errors noted in the "
        "{} application. Please view the monitored logs for further details."
    ).format(app_name)

    content = MIMEText(message)
    content['From'] = from_address
    content['To'] = to_address
    content['Subject'] = subject

    # Attempt to send email
    try:
        if config.getboolean("email", "debug"):
            log.debug(content.as_string())
        else:
            server = smtplib.SMTP(config.get("email", "server"))

            server.ehlo()
            server.starttls()
            server.sendmail(from_address, to_address, content.as_string())
            server.quit()
    except Exception:
        log.error("Unable to send notification to user", exc_info=True)
        
# Setup the root path
root = Path(sys.argv[1])

# Setup connection to the config file
config = configparser.ConfigParser()
config.read(Path(root.parent, "config", "sb_log_manager.cfg"))

# Setup logging for this application
logging.config.fileConfig(
    Path(root.parent, "config", "sb_log_manager_logging.cfg")
)
log = logging.getLogger(__name__)

# Setup connection to the Django application
djangoApp = config.get("django", "location")

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "studybuffalo.settings")
sys.path.append(djangoApp)
application = get_wsgi_application()

from log_manager.models import AppData, LogEntry

# Retrieve a list of the applications to monitor
log.info("Retrieving application list with reviews due")
now = datetime.now(pytz.timezone("UTC"))
app_list = AppData.objects.filter(next_review__lte=now)

# Retrieve each applicable application log
for app in app_list:
    log.info("Retrieving log data for {}".format(app.name))

    # Set up the directory containing all the logs
    log_directory = Path(app.log_location)

    log.debug("Log Location = {}".format(log_directory))

    # Cycle through all retrieved files looking for files 
    # matching the file name regex
    log.debug("File name regex = {}".format(app.file_name))

    name_regex = r"{}".format(app.file_name)
    app_logs = []

    for file in log_directory.listdir(names_only=True):
        log.debug("File name in log directory = {}".format(file))

        if re.match(name_regex, file):
            log.debug("Regex matches file name")

            app_logs.append(log_directory.child(file))
    
    # Open the log(s) and form them into a single json file
    json_log = []

    # Cycle through all the log files and create a list of the entries
    for app_log in app_logs:
        log.debug("Opening log file: {}".format(app_log))

        try:
            with open(app_log, "r") as file:
                for line in file:
                    json_log.append(json.loads(line))
        except IOError:
            log.critical(
                "Unable to open log file (is file in use?)", 
                exc_info=True
            )
        except FileNotFoundError:
            log.critical(
                "Unable to find the log at the specified location", 
                exc_info=True
            )
        except Exception:
            log.critical(
                "Error processing the provided JSON log", 
                exc_info=True
            )

    # If there are no JSON logs and application start is monitored, 
    # notify user of possible failure for app to start
    if app.monitor_start and len(json_log) == 0:
        log.debug("Emailing user about failed application start")
        email_start_failure(app.name, config)

    # Retrieve the asc_time format and timezone for this app
    asc_time_format = app.asc_time_format

    if app.log_timezone:
        log_timezone = pytz.timezone(app.log_timezone)
    else:
        log_timezone = pytz.timezone("UTC")

    # Process the JSON log and import into the django database
    log.info("Uploading log entries")

    # Whether a log entry has a level of warning or higher
    warn_log_entry = False 

    # Placeholder declartion to keep it in the needed scope
    asc_time = log_timezone.localize(datetime.utcnow())

    for entry in json_log:
        # Extract all the json values
        asc_time = entry["asctime"] if "asctime" in entry else None
        created = entry["created"] if "created" in entry else None
        exc_info = entry["exc_info"] if "exc_info" in entry else None
        file_name = entry["filename"] if "filename" in entry else None
        func_name = entry["funcName"] if "funcName" in entry else None
        level_name = entry["levelname"] if "levelname" in entry else None
        level_no = entry["levelno"] if "levelno" in entry else None
        line_no = entry["lineno"] if "lineno" in entry else None
        message = entry["message"] if "message" in entry else None
        module = entry["module"] if "module" in entry else None
        msecs = entry["msecs"] if "msecs" in entry else None
        name = entry["name"] if "name" in entry else None
        path_name = entry["pathname"] if "pathname" in entry else None
        process = entry["process"] if "process" in entry else None
        process_name = entry["processName"] if "processName" in entry else None
        relative_created = entry["relativeCreated"] if "relativeCreated" in entry else None
        stack_info = entry["stack_info"] if "stack_info" in entry else None
        thread = entry["thread"] if "thread" in entry else None
        thread_name = entry["threadName"] if "threadName" in entry else None
        
        # Convert the asc_time into a proper timezone aware datetime
        if asc_time:
            asc_time = log_timezone.localize(
                datetime.strptime(asc_time, asc_time_format)
            )
        else:
            # Default to now if not specified
            asc_time = log_timezone.localize(datetime.utcnow())

        # Check for an error level of warning or higher
        if level_no >= 30:
            warn_log_entry = True

        # Create a new entry for the LogEntry module
        new_entry = LogEntry(
            app_name=app,
            asc_time=asc_time,
            created=created,
            exc_info=exc_info,
            file_name=file_name,
            func_name=func_name,
            level_name=level_name,
            level_no=level_no,
            line_no=line_no,
            message=message,
            module=module,
            msecs=msecs,
            name=name,
            path_name=path_name,
            process=process,
            process_name=process_name,
            relative_created=relative_created,
            stack_info=stack_info,
            thread=thread,
            thread_name=thread_name,
        )
        
        try:
            new_entry.save()
        except Exception:
            log.error(
                (
                    "Unable to save log entry for following values: "
                    "app_name = {} asc_time = {} created = {} exc_info = {} "
                    "file_name = {} func_name = {} level_name = {} level_no = {} "
                    "line_no = {} message = {} module = {} msecs = {} name = {} "
                    "path_name = {} process = {} process_name = {} "
                    "relative_created = {} stack_info = {} thread = {} "
                    "thread_name = {}"
                ).format(
                    app,asc_time,created,exc_info,file_name,func_name,level_name,
                    level_no,line_no,message,module,msecs,name,path_name,process,
                    process_name,relative_created,stack_info,thread,thread_name
                ),
                exc_info=True)

    # Notify user if an error is present
    if warn_log_entry:
        log.debug("Emailing user about detected error in logs")
        email_error(app.name, config)

    # Update the app last_reviewed_log date and next review date
    log.debug("Updating the review and next review dates")

    app.last_reviewed_log = asc_time
    app.next_review = determine_next_date(
        app.review_minute,
        app.review_hour,
        app.review_day,
        app.review_month,
        app.review_weekday
    )

    try:
        app.save()
    except Exception:
        log.error("Unable to update AppData object", exc_info=True)

    # Clear the log file contents now that they have been saved in the database
    for app_log in app_logs:
        log.debug("Clearing log file: {}".format(app_log))

        try:
            with open(app_log, "w") as file:
                file.truncate()
        except Exception:
            log.error("Unable to clear log file", exc_info=True)