generalize and test replay, fix comments

.gitignore

@@ -0,0 +1,3 @@
+*.pyc
+*.pyo
+.idea

src/ooi/logging/replay.py

@@ -31,17 +31,14 @@
     to by the running application.
 """
 
-import graypy
 import os
 import sys
 import logging
 
-class GELFReplayer(object):
-    def __init__(self, server, port=12201, delete=True, level=logging.DEBUG):
-        self._sender = graypy.GELFHander(server, port)
-        self._sender.setLevel(level)
+class Replayer(object):
+    def __init__(self, handler, delete=True):
         self._delete_file = delete
-
+        self._sender = handler
     def relay(self, *filenames):
         """ process each of the filenames given by forwarding log record contents to the GELF server.
             handles files in reverse order so rotated logs can be specified by wildcard (file.log.*) 
@@ -50,8 +47,9 @@ def relay(self, *filenames):
         # function needed by the exec below
         handle_record_dict = self._handle
 
-        filenames.reverse() # work oldest to newest
-        for filename in filenames:
+        name_list = [ name for name in filenames ] # convert tuple to list
+        name_list.reverse() # work oldest to newest
+        for filename in name_list:
             with open(filename, 'r') as f:
                 contents = f.read()
             exec contents ## DANGER, DANGER!
@@ -64,10 +62,16 @@ def _handle(self, dict):
         self._sender.emit(record)
 
 if __name__ == '__main__':
+    """ when run as application, replay raw logs to a GELF server """
     if len(sys.argv)<3:
         print 'USAGE: python ' + __file__ + " server logfile..."
         exit(1)
     server = sys.argv[1]
     filenames = sys.argv[2:]
-    obj = GELFReplayer(server)
+
+    import graypy
+    handler = graypy.GELFHander(server, 12201)
+    handler.setLevel(logging.DEBUG)
+
+    obj = Replayer(handler)
     obj.relay(*filenames)

test/ooi/logging/logging.yml

@@ -1,12 +1,9 @@
-# Configuration file for setting log levels.
-# To define local overrides, add a file "logging.local.yml"
-
-# Required.  Do not touch or alter.
-version: 1
-
-# Definition of the output format for the handlers.
+###
+# logging configuration file for unit tests
+#
+# do not change without testing that it doesn't break any of the unit tests in this project!
 #
-# Users should generally not touch this section.
+version: 1
 formatters:
   default:
     format: '%(asctime)s %(levelname)-8s %(threadName)s %(name)-15s:%(lineno)d %(message)s'
@@ -32,6 +29,11 @@ handlers:
     formatter: raw
     level: DEBUG
     filename: /tmp/unittest-raw.log
+  replay:
+    class: logging.handlers.RotatingFileHandler
+    formatter: stack
+    level: DEBUG
+    filename: /tmp/unittest-replay.log
 
 loggers:
   stack:
@@ -43,3 +45,6 @@ loggers:
   raw:
     handlers: [raw]
     level: INFO
+  replay:
+    handlers: [replay]
+    level: DEBUG

test/ooi/logging/test_raw_logging.py

@@ -32,15 +32,15 @@ def read_file(self):
         exec contents
 
     def test_simple_message(self):
-        """ simple message is just one line """
+        """ make sure message makes file grow """
         self.assertEquals(0, os.path.getsize(LOGFILE))
         self.log.info("short message")
         self.assertTrue(os.path.getsize(LOGFILE)>0)
         self.read_file()
         self.assertEquals(1, self.count)
 
     def test_multiple_messages(self):
-        """ simple message is just one line """
+        """ multiple log statements can be read back as multiple dictionaries """
         self.assertEquals(0, os.path.getsize(LOGFILE))
         self.log.info("short message")
         self.log.info("another short message")
@@ -50,7 +50,7 @@ def test_multiple_messages(self):
         self.assertEquals(3, self.count)
 
     def test_multi_line_messages(self):
-        """ simple message is just one line """
+        """ newline won't break parsing back to dict """
         self.assertEquals(0, os.path.getsize(LOGFILE))
         self.log.info("short message")
         self.log.info("this message\nspans multiple\nlines of text")
@@ -59,7 +59,7 @@ def test_multi_line_messages(self):
         self.assertEquals(2, self.count)
 
     def test_messages_with_stack(self):
-        """ simple message is just one line """
+        """ record can contain stack trace """
         self.assertEquals(0, os.path.getsize(LOGFILE))
         try:
             awjrht()

test/ooi/logging/test_replay.py

@@ -0,0 +1,86 @@
+
+from ooi.logging import config
+import logging
+import unittest
+from unittest.case import TestCase
+import os.path
+import os
+import ooi.exception
+import ooi.logging.replay
+
+RAW_LOGFILE='/tmp/unittest-raw.log'
+REPLAY_LOGFILE='/tmp/unittest-replay.log'
+CONFIGFILE='logging.yml'
+
+class TestRawLogger(TestCase):
+    def setUp(self):
+        # clear file
+        try: os.remove(RAW_LOGFILE)
+        except: pass
+        try: os.remove(REPLAY_LOGFILE)
+        except: pass
+        # configure logging system
+        path = os.path.dirname(__file__) + '/' + CONFIGFILE
+        config.replace_configuration(path)
+        self.log = logging.getLogger('raw')
+
+        replay_log = logging.getLogger('replay')
+        handler = replay_log.handlers[0]
+        self.replay = ooi.logging.replay.Replayer(handler)
+
+    def tearDown(self):
+        try: os.remove(RAW_LOGFILE)
+        except: pass
+        try: os.remove(REPLAY_LOGFILE)
+        except: pass
+
+    def read_file(self):
+        with open(REPLAY_LOGFILE, 'r') as f:
+            contents = f.readlines()
+        self.count = len(contents)
+
+    def test_simple_message(self):
+        """ make sure message makes file grow """
+        self.assertEquals(0, os.path.getsize(RAW_LOGFILE))
+        self.log.info("short message")
+        self.replay.relay(RAW_LOGFILE)
+        self.assertTrue(os.path.getsize(REPLAY_LOGFILE)>0)
+        self.read_file()
+        self.assertEquals(1, self.count)
+
+    def test_multiple_messages(self):
+        """ multiple log statements can be read back as multiple dictionaries """
+        self.assertEquals(0, os.path.getsize(RAW_LOGFILE))
+        self.log.info("short message")
+        self.log.info("another short message")
+        self.log.info("a third short message")
+        self.replay.relay(RAW_LOGFILE)
+        self.assertTrue(os.path.getsize(REPLAY_LOGFILE)>0)
+        self.read_file()
+        self.assertEquals(3, self.count)
+
+    def test_multi_line_messages(self):
+        """ newline won't break parsing back to dict """
+        self.assertEquals(0, os.path.getsize(RAW_LOGFILE))
+        self.log.info("short message")
+        self.log.info("this message\nspans multiple\nlines of text")
+        self.replay.relay(RAW_LOGFILE)
+        self.assertTrue(os.path.getsize(REPLAY_LOGFILE)>0)
+        self.read_file()
+        self.assertTrue(self.count>2)
+
+    def test_messages_with_stack(self):
+        """ record can contain stack trace """
+        self.assertEquals(0, os.path.getsize(RAW_LOGFILE))
+        try:
+            awjrht()
+        except:
+            self.log.info("message and stack", exc_info=True)
+        self.replay.relay(RAW_LOGFILE)
+        self.assertTrue(os.path.getsize(REPLAY_LOGFILE)>0)
+        self.read_file()
+        self.assertTrue(self.count>1)
+
+
+if __name__ == '__main__':
+    unittest.main()

test/ooi/logging/test_stack_logging.py

@@ -35,6 +35,7 @@ def test_simple_message(self):
         self.assertEquals(1, len(self.get_lines()))
 
     def test_generic_exception(self):
+        """ stack trace should be compact form """
         try:
             splotgorpsh()
         except:
@@ -46,7 +47,7 @@ def test_generic_exception(self):
         self.assertTrue("splotgorpsh" in lines[2])
 
     def test_chained_exception(self):
-        thrown=0
+        """ chained stack traces should be compact, columns should be aligned """
         try:
             try:
                 splotgorpsh()