Add hooks around updating the offset file.

README.md

@@ -23,6 +23,9 @@ From the command line:
       -p, --paranoid        Update the offset file every time we read a line
                             (as opposed to only when we reach the end of the
                             file).
+      -n N, --every-n=N     Update the offset file every N'th time we read a
+                            line (as opposed to only when we reach the end of
+                            the file).
       --no-copytruncate     Don't support copytruncate-style log rotation.
                             Instead, if the log file shrinks, print a warning.
 

pygtail/core.py

@@ -27,11 +27,10 @@
 from os.path import exists, getsize
 import sys
 import glob
-import string
 import gzip
 from optparse import OptionParser
 
-__version__ = '0.5.3'
+__version__ = '0.5.4'
 
 
 PY3 = sys.version_info[0] == 3
@@ -55,16 +54,23 @@ class Pygtail(object):
     Keyword arguments:
     offset_file   File to which offset data is written (default: <logfile>.offset).
     paranoid      Update the offset file every time we read a line (as opposed to
-                  only when we reach the end of the file (default: False)
+                  only when we reach the end of the file (default: False))
+    every_n       Update the offset file every n'th line (as opposed to only when
+                  we reach the end of the file (default: 0))
+    on_update     Execute this function when offset data is written (default False)
     copytruncate  Support copytruncate-style log rotation (default: True)
     """
-    def __init__(self, filename, offset_file=None, paranoid=False, copytruncate=True):
+    def __init__(self, filename, offset_file=None, paranoid=False, copytruncate=True,
+                 every_n=0, on_update=False):
         self.filename = filename
         self.paranoid = paranoid
+        self.every_n = every_n
+        self.on_update = on_update
         self.copytruncate = copytruncate
         self._offset_file = offset_file or "%s.offset" % self.filename
         self._offset_file_inode = 0
         self._offset = 0
+        self._since_update = 0
         self._fh = None
         self._rotated_logfile = None
 
@@ -114,6 +120,8 @@ def next(self):
 
         if self.paranoid:
             self._update_offset_file()
+        elif self.every_n and self.every_n <= self._since_update:
+            self._update_offset_file()
 
         return line
 
@@ -171,11 +179,14 @@ def _update_offset_file(self):
         """
         Update the offset file with the current inode and offset.
         """
+        if self.on_update:
+            self.on_update()
         offset = self._filehandle().tell()
         inode = stat(self.filename).st_ino
         fh = open(self._offset_file, "w")
         fh.write("%s\n%s\n" % (inode, offset))
         fh.close()
+        self._since_update = 0
 
     def _determine_rotated_logfile(self):
         """
@@ -241,8 +252,10 @@ def _get_next_line(self):
         line = self._filehandle().readline()
         if not line:
             raise StopIteration
+        self._since_update += 1
         return line
 
+
 def main():
     # command-line parsing
     cmdline = OptionParser(usage="usage: %prog [options] logfile",
@@ -252,6 +265,9 @@ def main():
     cmdline.add_option("--paranoid", "-p", action="store_true",
         help="Update the offset file every time we read a line (as opposed to"
              " only when we reach the end of the file).")
+    cmdline.add_option("--every-n", "-n", action="store",
+        help="Update the offset file every n'th time we read a line (as opposed to"
+             " only when we reach the end of the file).")
     cmdline.add_option("--no-copytruncate", action="store_true",
         help="Don't support copytruncate-style log rotation. Instead, if the log file"
              " shrinks, print a warning.")
@@ -267,9 +283,12 @@ def main():
     if (len(args) != 1):
         cmdline.error("Please provide a logfile to read.")
 
+    if options.every_n:
+        options.every_n = int(options.every_n)
     pygtail = Pygtail(args[0],
                       offset_file=options.offset_file,
                       paranoid=options.paranoid,
+                      every_n=options.every_n
                       copytruncate=not options.no_copytruncate)
 
     for line in pygtail:

pygtail/test/test_pygtail.py

@@ -17,7 +17,6 @@
 class PygtailTest(unittest.TestCase):
     # TODO:
     # - test for non-default offset file
-    # - test for paranoid flag
     # - test for savelog and datext rotation schemes
 
     def setUp(self):
@@ -156,6 +155,53 @@ def test_offset_file(self):
         self.assertEqual(inode, log_inode)
         self.assertEqual(offset, 6)
 
+    def test_on_update_with_paranoid(self):
+        updates = [0]
+        def record_update():
+            updates[0] += 1
+        pygtail = Pygtail(self.logfile.name, paranoid=True,
+                          on_update=record_update)
+
+        self.assertEqual(updates[0], 0)
+        next(pygtail)
+        self.assertEqual(updates[0], 1)
+        next(pygtail)
+        self.assertEqual(updates[0], 2)
+        next(pygtail)
+        self.assertEqual(updates[0], 3)
+
+
+    def test_on_update_without_paranoid(self):
+        updates = [0]
+
+        def record_update():
+            updates[0] += 1
+
+        pygtail = Pygtail(self.logfile.name, on_update=record_update)
+
+        self.assertEqual(updates[0], 0)
+        for line in pygtail:
+            self.assertEqual(updates[0], 0)
+        self.assertEqual(updates[0], 1)
+
+    def test_every_n(self):
+        updates = [0]
+        # We save before returning the second line.
+        # We save at the end of the file with all 3 recorded.
+        expected = [1, 3]
+        previous_lines = 0
+
+        def record_update():
+            self.assertEqual(previous_lines, expected[updates[0]])
+            updates[0] += 1
+
+        pygtail = Pygtail(self.logfile.name, every_n=2, on_update=record_update)
+
+        self.assertEqual(updates[0], 0)
+        for line in pygtail:
+            previous_lines += 1
+
+
 def main():
     unittest.main(buffer=True)