Browse files

support for automatic format detection. this allows us to map between a

source format and destination layout (i.e for "plain logging" that still
supports filters)
  • Loading branch information...
1 parent 4057d4d commit 3cabe72f1fb0cdb5bdd8e88947c0c6a6e32085a4 @marshall committed Nov 21, 2012
Showing with 342 additions and 177 deletions.
  1. +37 −21 logcat-color
  2. +10 −2 logcatcolor/column.py
  3. +10 −9 logcatcolor/config.py
  4. +137 −0 logcatcolor/format.py
  5. +40 −107 logcatcolor/layout.py
  6. +22 −15 logcatcolor/profile.py
  7. +86 −23 logcatcolor/reader.py
View
58 logcat-color
@@ -24,8 +24,8 @@ from logcatcolor.reader import LogcatReader
class LogcatColor(object):
def __init__(self):
- self.width = self.get_term_width()
self.parse_args()
+ self.width = self.get_term_width()
self.config = LogcatColorConfig(self.options)
self.profile = None
@@ -37,19 +37,21 @@ class LogcatColor(object):
if not self.profile:
self.logcat_args.extend(self.args)
- self.layout = "brief"
+ self.format = None
+ if self.options.format:
+ self.format = self.options.format
+ elif self.profile and self.profile.format:
+ self.format = self.profile.format
+
+ self.layout = self.format
if self.options.plain:
self.layout = "raw"
- elif self.options.format:
- self.layout = self.options.format
- elif self.profile and self.profile.format:
- self.layout = self.profile.format
def get_term_width(self):
- stdout_fd = sys.stdout.fileno()
- if os.isatty(stdout_fd):
+ out_fd = self.output.fileno()
+ if os.isatty(out_fd):
# unpack the current terminal width / height
- data = fcntl.ioctl(stdout_fd, termios.TIOCGWINSZ, '1234')
+ data = fcntl.ioctl(out_fd, termios.TIOCGWINSZ, '1234')
height, width = struct.unpack('hh', data)
else:
# store a large width when the output of this script is being piped
@@ -70,6 +72,13 @@ class LogcatColor(object):
parser.add_option("--no-wrap", action="store_false", dest="wrap",
default=None, help="don't wrap console text into a column " +
"(makes for better copy/paste)")
+ parser.add_option("-i", "--input", metavar="FILE", dest="input",
+ default=None,
+ help="read input from FILE, instead of starting adb. this is " +
+ "equivalent to piping FILE to logcat-color. (default: start " +
+ "adb, and read from it's stdout)")
+ parser.add_option("-o", "--output", metavar="FILE", dest="output",
+ default=None, help="write output to FILE (default: stdout)")
# ADB options
parser.add_option("-d", "--device", action="store_const",
@@ -117,9 +126,19 @@ class LogcatColor(object):
self.options = options
self.args = args
+ if options.config and not os.path.isfile(options.config):
+ parser.error("Config file does not exist: %s" % options.config)
+
+ self.input = sys.stdin
+ if options.input:
+ self.input = open(options.input, "r")
+
+ self.output = sys.stdout
+ if options.output:
+ self.output = open(options.output, "w")
+
self.adb_device = options.adb_device
self.logcat_args = options.logcat_args or []
- self.logcat_format = options.format
if options.buffers:
for buf in options.buffers:
@@ -165,29 +184,26 @@ class LogcatColor(object):
def get_logcat_args(self):
logcat_args = self.logcat_args[:]
- if self.logcat_format:
- logcat_args.extend(["-v", self.logcat_format])
+ if self.format:
+ logcat_args.extend(["-v", self.format])
return logcat_args
def start(self):
if self.profile:
- buffers = self.profile.get_buffers()
+ buffers = self.profile.buffers
if buffers:
for b in buffers: self.logcat_args.extend(["-b", b])
- if self.profile.format and not self.logcat_format:
- self.logcat_format = self.profile.format
-
# if someone is piping, use stdin as input. if not, invoke adb logcat
- input = sys.stdin
- if os.isatty(sys.stdin.fileno()):
+ if self.input.isatty():
adb_command = self.get_adb_args()
adb_command.append("logcat")
adb_command.extend(self.get_logcat_args())
- input = Popen(adb_command, stdout=PIPE).stdout
+ self.input = Popen(adb_command, stdout=PIPE).stdout
- reader = LogcatReader(input, self.config, profile=self.profile,
- layout=self.layout, width=self.width)
+ reader = LogcatReader(self.input, self.config, profile=self.profile,
+ format=self.format, layout=self.layout, writer=self.output,
+ width=self.width)
try:
asyncore.loop()
except KeyboardInterrupt, e:
View
12 logcatcolor/column.py
@@ -1,3 +1,11 @@
+"""
+logcat-color
+
+Copyright 2012, Marshall Culpepper
+Licensed under the Apache License, Version 2.0
+
+Columns for displaying logcat log data
+"""
import colorama
from colorama import Fore, Back, Style
import StringIO
@@ -62,7 +70,7 @@ def __init__(self, layout):
tag_colors = None
if layout.profile:
- tag_colors = layout.profile.get_tag_colors()
+ tag_colors = layout.profile.tag_colors
self.tag_colors = tag_colors or {}
self.last_used = self.COLOR_MAP.values()[:]
@@ -117,7 +125,7 @@ class MessageColumn(Column):
def __init__(self, layout):
self.width = None
self.left = layout.total_column_width
- if layout.config.wrap and (not layout.profile or layout.profile.wrap):
+ if layout.config.get_wrap() and (not layout.profile or layout.profile.wrap):
self.width = layout.width - self.left
def format(self, message):
View
19 logcatcolor/config.py
@@ -15,20 +15,23 @@ def __init__(self, options):
self.options = options
self.path = options.config or self.get_default_config()
self.filters = {}
- self.wrap = True
- self.config = { "Profile": Profile }
+ self.config = {
+ "Profile": Profile,
+ "__file__": self.path
+ }
+
self.config.update(TagColumn.COLOR_MAP)
- if os.path.exists(self.path):
+ if os.path.exists(self.path) and os.path.isfile(self.path):
# config file is just a python script that globals are imported from
try:
execfile(self.path, self.config)
except:
self.report_config_error()
sys.exit(1)
- self.load_config()
+ self.post_load()
def report_config_error(self):
config_error = """
@@ -52,11 +55,9 @@ def get_default_config(self):
home_dir = os.environ[env_key]
return os.path.join(home_dir, ".logcat-color")
- def load_config(self):
- if self.options.wrap is None:
- self.wrap = True
- if "wrap" in self.config:
- self.wrap = self.config["wrap"]
+ def post_load(self):
+ if self.options.wrap is not None:
+ self.config["wrap"] = self.options.wrap
def get_default_layout(self):
return self.config.get("default_layout", self.DEFAULT_LAYOUT)
View
137 logcatcolor/format.py
@@ -0,0 +1,137 @@
+"""
+logcat-color
+
+Copyright 2012, Marshall Culpepper
+Licensed under the Apache License, Version 2.0
+
+Support for reading various logcat logging formats into an easier to consume
+data map.
+"""
+import re
+
+def format(cls):
+ Format.TYPES[cls.NAME] = cls
+ Format.REGEXES[cls.NAME] = re.compile(cls.PATTERN) if cls.PATTERN else None
+ return cls
+
+class Format(object):
+ TYPES = {}
+ REGEXES = {}
+ MARKER_REGEX = re.compile(r"^--------- beginning of")
+
+ def __init__(self):
+ self.data = {}
+ self.regex = self.REGEXES[self.NAME]
+
+ def match(self, line):
+ if not self.regex:
+ return True
+
+ self.data["line"] = line
+ match = self.regex.match(line)
+ if not match:
+ return False
+
+ for name, value in match.groupdict().iteritems():
+ self.data[name] = value.strip()
+ return True
+
+ def get(self, name):
+ return self.data.get(name)
+
+ def include(self, profile):
+ if profile and not profile.include(self.data):
+ return False
+ return True
+
+@format
+class BriefFormat(Format):
+ "I/Tag( PID): message"
+ NAME = "brief"
+ PRIORITY_PATTERN = r"(?P<priority>[A-Z])"
+ PRIORITY_TAG_PATTERN = PRIORITY_PATTERN + r"/" + r"(?P<tag>[^\(]*?)"
+
+ PID_PATTERN = r"(?P<pid>\d+)"
+ PID_PAREN_PATTERN = r"\(\s*" + PID_PATTERN + r"\)"
+ MESSAGE_PATTERN = r"(?P<message>.*?)"
+
+ BRIEF_PATTERN = PRIORITY_TAG_PATTERN + \
+ PID_PAREN_PATTERN + r": " + \
+ MESSAGE_PATTERN
+ PATTERN = r"^" + BRIEF_PATTERN + r"$"
+
+@format
+class ProcessFormat(Format):
+ "I( PID) message (Tag)"
+ NAME = "process"
+ PATTERN = r"^" + BriefFormat.PRIORITY_PATTERN + \
+ BriefFormat.PID_PAREN_PATTERN + r" " + \
+ BriefFormat.MESSAGE_PATTERN + r" " + \
+ r"\((?P<tag>.+)\)$"
+
+@format
+class TagFormat(Format):
+ "I/Tag : message"
+ NAME = "tag"
+ PATTERN = r"^" + BriefFormat.PRIORITY_TAG_PATTERN + r": " + \
+ BriefFormat.MESSAGE_PATTERN + r"$"
+
+@format
+class ThreadFormat(Format):
+ "I( PID:TID) message"
+ NAME = "thread"
+ TID_HEX_PATTERN = r"(?P<tid>0x[0-9a-f]+)"
+ PID_TID_HEX_PATTERN = BriefFormat.PID_PATTERN + r":" + TID_HEX_PATTERN
+
+ PATTERN = r"^" + BriefFormat.PRIORITY_PATTERN + \
+ r"\(\s*" + PID_TID_HEX_PATTERN + r"\) " + \
+ BriefFormat.MESSAGE_PATTERN + r"$"
+
+@format
+class TimeFormat(Format):
+ "MM-DD HH:MM:SS.mmm D/Tag( PID): message"
+ NAME = "time"
+ DATE_TIME_PATTERN = r"(?P<date>\d\d-\d\d)\s(?P<time>\d\d:\d\d:\d\d\.\d\d\d)"
+ PATTERN = r"^" + DATE_TIME_PATTERN + r" " + BriefFormat.BRIEF_PATTERN + r"$"
+
+@format
+class ThreadTimeFormat(Format):
+ "MM-DD HH:MM:SS.mmm PID TID I ONCRPC : rpc_handle_rpc_call: Find Status: 0 Xid: 7062"
+ NAME = "threadtime"
+ PATTERN = r"^" + TimeFormat.DATE_TIME_PATTERN + r"\s+" + \
+ BriefFormat.PID_PATTERN + r"\s+" + \
+ r"(?P<tid>\d+)\s+" + \
+ BriefFormat.PRIORITY_PATTERN + r"\s+" + \
+ r"(?P<tag>.*?)\s*: " + \
+ BriefFormat.MESSAGE_PATTERN + r"$"
+
+@format
+class LongFormat(Format):
+ "[ MM-DD HH:MM:SS.mmm PID:TID I/Tag ]\nmessage"
+ NAME = "long"
+ PATTERN = r"^\[ " + TimeFormat.DATE_TIME_PATTERN + r"\s+" + \
+ ThreadFormat.PID_TID_HEX_PATTERN + r"\s+" + \
+ BriefFormat.PRIORITY_TAG_PATTERN + r"\s+\]$"
+
+ def match(self, line):
+ if not Format.match(self, line):
+ self.data["message"] = line
+
+ return "message" in self.data and "tag" in self.data
+
+"""
+A helper to detect the log format from a list of lines
+"""
+def detect_format(lines):
+ if len(lines) == 0:
+ return None
+
+ for line in lines:
+ if Format.MARKER_REGEX.match(line):
+ continue
+
+ for name, regex in Format.REGEXES.iteritems():
+ if regex.match(line):
+ return name
+
+ return None
View
147 logcatcolor/layout.py
@@ -1,168 +1,101 @@
+"""
+logcat-color
+
+Copyright 2012, Marshall Culpepper
+Licensed under the Apache License, Version 2.0
+
+Layouts for mapping logcat log data into a colorful terminal interface
+"""
from colorama import Fore, Back, Style
from logcatcolor.column import *
+from logcatcolor.format import Format
import re
-import StringIO
+from cStringIO import StringIO
+
+def layout(cls):
+ Layout.TYPES[cls.NAME] = cls
+ return cls
class Layout(object):
- MARKER_REGEX = re.compile(r"^--------- beginning of")
- MARKER_FORMAT = Fore.WHITE + Back.BLACK + Style.DIM + "%s" + Style.RESET_ALL
- def __init__(self, config, profile, width):
+ TYPES = {}
+ MARKER_LAYOUT = Fore.WHITE + Back.BLACK + Style.DIM + "%s" + Style.RESET_ALL
+
+ def __init__(self, config=None, profile=None, width=2000):
self.columns = []
self.config = config
- self.data = {}
self.profile = profile
self.width = width
self.total_column_width = 0
if self.COLUMNS:
# first get the total column width, then construct each column
for ColumnType in self.COLUMNS:
- self.total_column_width += config.get_column_width(ColumnType)
+ if config:
+ self.total_column_width += config.get_column_width(ColumnType)
+ else:
+ self.total_column_width += ColumnType.DEFAULT_WIDTH
for ColumnType in self.COLUMNS:
column = ColumnType(self)
self.columns.append(column)
self.column_count = len(self.columns)
- self.regex = None
- if self.PATTERN:
- self.regex = re.compile(self.PATTERN)
-
- def match_marker(self, line):
- return self.MARKER_REGEX.match(line)
-
- def match_data(self, line):
- if not self.regex:
- return True
-
- match = self.regex.match(line)
- if not match:
- return False
-
- for name, value in match.groupdict().iteritems():
- self.data[name] = value.strip()
- return True
-
- def clear_data(self):
- self.data.clear()
-
- def include(self):
- if self.profile and not self.profile.include(self.data):
- return False
-
- return True
-
- def layout(self, line):
- if len(line) == 0:
- return None
-
- if self.match_marker(line):
- return self.layout_marker(line)
-
- if not self.match_data(line):
- return None
-
- if not self.include():
- self.clear_data()
- return None
-
- line = self.layout_columns(line)
- self.clear_data()
- return line
-
def layout_marker(self, line):
- return self.MARKER_FORMAT % line
+ return self.MARKER_LAYOUT % line
- def layout_columns(self, line):
- formatted = StringIO.StringIO()
+ def layout_data(self, data):
+ formatted = StringIO()
for index in range(0, self.column_count):
column = self.columns[index]
- data = self.data[column.NAME]
- formatted.write(column.format(data))
+ formatted.write(column.format(data[column.NAME]))
if index < self.column_count - 1:
formatted.write(" ")
return formatted.getvalue()
+@layout
class RawLayout(Layout):
NAME = "raw"
- PATTERN = None
COLUMNS = None
- def layout_columns(self, line):
- return line
-class BriefLayout(Layout):
- "I/Tag( PID): message"
- NAME = "brief"
- PRIORITY_PATTERN = r"(?P<priority>[A-Z])"
- PRIORITY_TAG_PATTERN = PRIORITY_PATTERN + r"/" + r"(?P<tag>.*?)"
+ def layout_marker(self, line):
+ return line
- PID_PATTERN = r"(?P<pid>\d+)"
- PID_PAREN_PATTERN = r"\(\s*" + PID_PATTERN + r"\)"
- MESSAGE_PATTERN = r"(?P<message>.*?)"
+ def layout_data(self, data):
+ return data["line"]
- BRIEF_PATTERN = PRIORITY_TAG_PATTERN + \
- PID_PAREN_PATTERN + r": " + \
- MESSAGE_PATTERN
- PATTERN = r"^" + BRIEF_PATTERN + r"$"
+@layout
+class BriefLayout(Layout):
+ NAME = "brief"
COLUMNS = (PIDColumn, TagColumn, PriorityColumn, MessageColumn)
+@layout
class ProcessLayout(Layout):
- "I( PID) message (Tag)"
NAME = "process"
- PATTERN = r"^" + BriefLayout.PRIORITY_PATTERN + \
- BriefLayout.PID_PAREN_PATTERN + r" " + \
- BriefLayout.MESSAGE_PATTERN + r" " + \
- r"\((?P<tag>.+)\)$"
COLUMNS = BriefLayout.COLUMNS
+@layout
class TagLayout(Layout):
- "I/Tag : message"
NAME = "tag"
- PATTERN = r"^" + BriefLayout.PRIORITY_TAG_PATTERN + r": " + \
- BriefLayout.MESSAGE_PATTERN + r"$"
COLUMNS = (TagColumn, PriorityColumn, MessageColumn)
+@layout
class ThreadLayout(Layout):
- "I( PID:TID) message"
NAME = "thread"
- TID_HEX_PATTERN = r"(?P<tid>0x[0-9a-f]+)"
- PID_TID_HEX_PATTERN = BriefLayout.PID_PATTERN + r":" + TID_HEX_PATTERN
-
- PATTERN = r"^" + BriefLayout.PRIORITY_PATTERN + \
- r"\(\s*" + PID_TID_HEX_PATTERN + r"\) " + \
- BriefLayout.MESSAGE_PATTERN + r"$"
COLUMNS = (PIDColumn, TIDColumn, PriorityColumn, MessageColumn)
+@layout
class TimeLayout(Layout):
- "MM-DD HH:MM:SS.mmm D/Tag( PID): message"
NAME = "time"
- DATE_TIME_PATTERN = r"(?P<date>\d\d-\d\d)\s(?P<time>\d\d:\d\d:\d\d\.\d\d\d)"
- PATTERN = r"^" + DATE_TIME_PATTERN + r" " + BriefLayout.BRIEF_PATTERN + r"$"
COLUMNS = (DateColumn, TimeColumn, ) + BriefLayout.COLUMNS
+@layout
class ThreadTimeLayout(Layout):
- "MM-DD HH:MM:SS.mmm PID TID I ONCRPC : rpc_handle_rpc_call: Find Status: 0 Xid: 7062"
NAME = "threadtime"
- PATTERN = r"^" + TimeLayout.DATE_TIME_PATTERN + r"\s+" + \
- BriefLayout.PID_PATTERN + r"\s+" + \
- r"(?P<tid>\d+)\s+" + \
- BriefLayout.PRIORITY_PATTERN + r"\s+" + \
- r"(?P<tag>.*?)\s*: " + \
- BriefLayout.MESSAGE_PATTERN + r"$"
COLUMNS = (DateColumn, TimeColumn, PIDColumn, TIDColumn, TagColumn, PriorityColumn, MessageColumn)
+@layout
class LongLayout(Layout):
- "[ MM-DD HH:MM:SS.mmm PID:TID I/Tag ]\nmessage"
NAME = "long"
- PATTERN = r"^\[ " + TimeLayout.DATE_TIME_PATTERN + r"\s+" + \
- ThreadLayout.PID_TID_HEX_PATTERN + r"\s+" + \
- BriefLayout.PRIORITY_TAG_PATTERN + r"\s+\]$"
COLUMNS = ThreadTimeLayout.COLUMNS
-
- def match_data(self, line):
- if not Layout.match_data(self, line):
- self.data["message"] = line
-
- return "message" in self.data and "tag" in self.data
View
37 logcatcolor/profile.py
@@ -1,5 +1,7 @@
import re
+RegexType = type(re.compile(""))
+
class Profile(object):
__profiles__ = {}
@@ -47,36 +49,41 @@ def init_filters(self, filters):
if not filters:
return
+ if not isinstance(filters, (list, tuple)):
+ filters = [filters]
+
for filter in filters:
- if isinstance(filter, str):
+ if isinstance(filter, (str, RegexType)):
self.filters.append(self.regex_filter(filter))
else:
self.filters.append(filter)
def regex_filter(self, regex):
- pattern = re.compile(regex)
+ pattern = regex
+ if not isinstance(regex, RegexType):
+ pattern = re.compile(regex)
+
def __filter(data):
if "message" not in data:
return True
- return re.search(pattern, data["message"])
+ return pattern.search(data["message"])
return __filter
def include(self, data):
- if self.tags and data["tag"] not in self.tags:
- return False
+ if not data:
+ raise Exception("data should not be None")
- if self.priorities and data["priority"] not in self.priorities:
+ if self.tags and data.get("tag") not in self.tags:
return False
- if self.filters:
- for filter in self.filters:
- if not filter(data):
- return False
+ if self.priorities and data.get("priority") not in self.priorities:
+ return False
- return True
+ if not self.filters:
+ return True
- def get_buffers(self):
- return self.buffers
+ for filter in self.filters:
+ if not filter(data):
+ return False
- def get_tag_colors(self):
- return self.tag_colors
+ return True
View
109 logcatcolor/reader.py
@@ -1,8 +1,18 @@
+"""
+logcat-color
+
+Copyright 2012, Marshall Culpepper
+Licensed under the Apache License, Version 2.0
+
+Logcat I/O stream readers and helpers
+"""
import asyncore
import asynchat
+from cStringIO import StringIO
import fcntl
import inspect
-import logcatcolor.layout
+from logcatcolor.format import Format, detect_format
+from logcatcolor.layout import Layout
import os
import sys
import traceback
@@ -14,7 +24,7 @@ class FileLineReader(asynchat.async_chat):
def __init__(self, fd):
asynchat.async_chat.__init__(self)
self.connected = True
- self.log_buffer = []
+ self.log_buffer = StringIO()
self.set_file(fd)
self.set_terminator(self.LINE_TERMINATOR)
@@ -35,40 +45,93 @@ def set_file(self, fd):
fcntl.fcntl(fd, fcntl.F_SETFL, flags)
def collect_incoming_data(self, data):
- self.log_buffer.append(data)
+ self.log_buffer.write(data)
def found_terminator(self):
- line = "".join(self.log_buffer)
+ line = self.log_buffer.getvalue()
try:
self.process_line(line)
except:
traceback.print_exc()
sys.exit(1)
- self.log_buffer = []
+ self.log_buffer = StringIO()
def process_line(self):
pass
class LogcatReader(FileLineReader):
- LAYOUTS = {}
- def __init__(self, file, config, profile=None, layout="brief", width=80):
+ DETECT_COUNT = 3
+
+ def __init__(self, file, config, profile=None, format=None, layout=None,
+ writer=None, width=80):
FileLineReader.__init__(self, file)
- LayoutType = self.LAYOUTS[layout]
- self.layout = LayoutType(config, profile, width)
+ self.detect_lines = []
+ self.config = config
+ self.profile = profile
+ self.width = width
+ self.writer = writer or sys.stdout
+
+ self.format = None
+ if format is not None:
+ FormatType = Format.TYPES[format]
+ self.format = FormatType()
+
+ self.layout = None
+ if layout is not None:
+ LayoutType = Layout.TYPES[layout]
+ self.layout = LayoutType(config, profile, width)
+
+ def __del__(self):
+ # Clear the "detect" lines if we weren't able to detect a format
+ if len(self.detect_lines) > 0 and not self.format:
+ self.format = BriefFormat()
+ if not self.layout:
+ self.layout = BriefLayout()
+
+ for line in self.detect_lines:
+ self.layout_line(line)
+
+ def detect_format(self, line):
+ if len(self.detect_lines) < self.DETECT_COUNT:
+ self.detect_lines.append(line)
+ return False
+
+ format_name = detect_format(self.detect_lines) or "brief"
+ self.format = Format.TYPES[format_name]()
+ if not self.layout:
+ self.layout = Layout.TYPES[format_name](self.config, self.profile,
+ self.width)
+
+ for line in self.detect_lines:
+ self.layout_line(line)
+
+ self.detect_lines = []
+ return True
def process_line(self, line):
- formatted = self.layout.layout(line.strip())
- if formatted:
- print formatted
-
-for member in inspect.getmembers(logcatcolor.layout):
- LayoutType = member[1]
- if LayoutType == logcatcolor.layout.Layout:
- continue
- if not inspect.isclass(LayoutType):
- continue
- if not logcatcolor.layout.Layout in LayoutType.__bases__:
- continue
-
- LogcatReader.LAYOUTS[LayoutType.NAME] = LayoutType
+ line = line.strip()
+ if not self.format:
+ if not self.detect_format(line):
+ return
+
+ self.layout_line(line)
+
+ def layout_line(self, line):
+ if Format.MARKER_REGEX.match(line):
+ result = self.layout.layout_marker(line)
+ if result:
+ self.writer.write(result)
+ return
+
+ try:
+ if not self.format.match(line) or not self.format.include(self.profile):
+ return
+
+ result = self.layout.layout_data(self.format.data)
+ if not result:
+ return
+
+ self.writer.write(result + "\n")
+ finally:
+ self.format.data.clear()

0 comments on commit 3cabe72

Please sign in to comment.