Skip to content
Cannot retrieve contributors at this time
131 lines (110 sloc) 4.73 KB
# Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
# This source code is licensed under both the GPLv2 (found in the
# COPYING file in the root directory) and Apache 2.0 License
# (found in the LICENSE.Apache file in the root directory).
from abc import ABC, abstractmethod
from calendar import timegm
from enum import Enum
import glob
import re
import time
class DataSource(ABC):
class Type(Enum):
LOG = 1
def __init__(self, type):
self.type = type
def check_and_trigger_conditions(self, conditions):
class Log:
def is_new_log(log_line):
# The assumption is that a new log will start with a date printed in
# the below regex format.
date_regex = '\d{4}/\d{2}/\d{2}-\d{2}:\d{2}:\d{2}\.\d{6}'
return re.match(date_regex, log_line)
def __init__(self, log_line, column_families):
token_list = log_line.strip().split()
self.time = token_list[0]
self.context = token_list[1]
self.message = " ".join(token_list[2:])
self.column_family = None
# example log for 'default' column family:
# "2018/07/25-17:29:05.176080 7f969de68700 [db/]
# [default] [JOB 3] Compacting 24@0 + 16@1 files to L1, score 6.00\n"
for col_fam in column_families:
search_for_str = '\[' + col_fam + '\]'
if, self.message):
self.column_family = col_fam
if not self.column_family:
self.column_family = NO_COL_FAMILY
def get_human_readable_time(self):
# example from a log line: '2018/07/25-11:25:45.782710'
return self.time
def get_column_family(self):
return self.column_family
def get_context(self):
return self.context
def get_message(self):
return self.message
def append_message(self, remaining_log):
self.message = self.message + '\n' + remaining_log.strip()
def get_timestamp(self):
# example: '2018/07/25-11:25:45.782710' will be converted to the GMT
# Unix timestamp 1532517945 (note: this method assumes that self.time
# is in GMT)
hr_time = self.time + 'GMT'
timestamp = timegm(time.strptime(hr_time, "%Y/%m/%d-%H:%M:%S.%f%Z"))
return timestamp
def __repr__(self):
return (
'time: ' + self.time + '; context: ' + self.context +
'; col_fam: ' + self.column_family +
'; message: ' + self.message
class DatabaseLogs(DataSource):
def __init__(self, logs_path_prefix, column_families):
self.logs_path_prefix = logs_path_prefix
self.column_families = column_families
def trigger_conditions_for_log(self, conditions, log):
# For a LogCondition object, trigger is:
# Dict[column_family_name, List[Log]]. This explains why the condition
# was triggered and for which column families.
for cond in conditions:
if, log.get_message(), re.IGNORECASE):
trigger = cond.get_trigger()
if not trigger:
trigger = {}
if log.get_column_family() not in trigger:
trigger[log.get_column_family()] = []
def check_and_trigger_conditions(self, conditions):
for file_name in glob.glob(self.logs_path_prefix + '*'):
# TODO(poojam23): find a way to distinguish between log files
# - generated in the current experiment but are labeled 'old'
# because they LOGs exceeded the file size limit AND
# - generated in some previous experiment that are also labeled
# 'old' and were not deleted for some reason
if'old', file_name, re.IGNORECASE):
with open(file_name, 'r') as db_logs:
new_log = None
for line in db_logs:
if Log.is_new_log(line):
if new_log:
conditions, new_log
new_log = Log(line, self.column_families)
# To account for logs split into multiple lines
# Check for the last log in the file.
if new_log:
self.trigger_conditions_for_log(conditions, new_log)