From d4bbc1899d12faeb5c1154e2f0b717bd16ac1d81 Mon Sep 17 00:00:00 2001 From: Ben Tilly Date: Mon, 24 Aug 2015 20:47:23 +0000 Subject: [PATCH] Add hooks around updating the offset file. --- README.md | 3 +++ pygtail/core.py | 27 +++++++++++++++++--- pygtail/test/test_pygtail.py | 48 +++++++++++++++++++++++++++++++++++- 3 files changed, 73 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 310980a..1baf970 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,9 @@ From the command line: -p, --paranoid Update the offset file every time we read a line (as opposed to only when we reach the end of the file). + -n N, --every-n=N Update the offset file every N'th time we read a + line (as opposed to only when we reach the end of + the file). --no-copytruncate Don't support copytruncate-style log rotation. Instead, if the log file shrinks, print a warning. diff --git a/pygtail/core.py b/pygtail/core.py index 15eb905..a964176 100755 --- a/pygtail/core.py +++ b/pygtail/core.py @@ -27,11 +27,10 @@ from os.path import exists, getsize import sys import glob -import string import gzip from optparse import OptionParser -__version__ = '0.5.3' +__version__ = '0.5.4' PY3 = sys.version_info[0] == 3 @@ -55,16 +54,23 @@ class Pygtail(object): Keyword arguments: offset_file File to which offset data is written (default: .offset). paranoid Update the offset file every time we read a line (as opposed to - only when we reach the end of the file (default: False) + only when we reach the end of the file (default: False)) + every_n Update the offset file every n'th line (as opposed to only when + we reach the end of the file (default: 0)) + on_update Execute this function when offset data is written (default False) copytruncate Support copytruncate-style log rotation (default: True) """ - def __init__(self, filename, offset_file=None, paranoid=False, copytruncate=True): + def __init__(self, filename, offset_file=None, paranoid=False, copytruncate=True, + every_n=0, on_update=False): self.filename = filename self.paranoid = paranoid + self.every_n = every_n + self.on_update = on_update self.copytruncate = copytruncate self._offset_file = offset_file or "%s.offset" % self.filename self._offset_file_inode = 0 self._offset = 0 + self._since_update = 0 self._fh = None self._rotated_logfile = None @@ -114,6 +120,8 @@ def next(self): if self.paranoid: self._update_offset_file() + elif self.every_n and self.every_n <= self._since_update: + self._update_offset_file() return line @@ -171,11 +179,14 @@ def _update_offset_file(self): """ Update the offset file with the current inode and offset. """ + if self.on_update: + self.on_update() offset = self._filehandle().tell() inode = stat(self.filename).st_ino fh = open(self._offset_file, "w") fh.write("%s\n%s\n" % (inode, offset)) fh.close() + self._since_update = 0 def _determine_rotated_logfile(self): """ @@ -241,8 +252,10 @@ def _get_next_line(self): line = self._filehandle().readline() if not line: raise StopIteration + self._since_update += 1 return line + def main(): # command-line parsing cmdline = OptionParser(usage="usage: %prog [options] logfile", @@ -252,6 +265,9 @@ def main(): cmdline.add_option("--paranoid", "-p", action="store_true", help="Update the offset file every time we read a line (as opposed to" " only when we reach the end of the file).") + cmdline.add_option("--every-n", "-n", action="store", + help="Update the offset file every n'th time we read a line (as opposed to" + " only when we reach the end of the file).") cmdline.add_option("--no-copytruncate", action="store_true", help="Don't support copytruncate-style log rotation. Instead, if the log file" " shrinks, print a warning.") @@ -267,9 +283,12 @@ def main(): if (len(args) != 1): cmdline.error("Please provide a logfile to read.") + if options.every_n: + options.every_n = int(options.every_n) pygtail = Pygtail(args[0], offset_file=options.offset_file, paranoid=options.paranoid, + every_n=options.every_n copytruncate=not options.no_copytruncate) for line in pygtail: diff --git a/pygtail/test/test_pygtail.py b/pygtail/test/test_pygtail.py index fbd0e1d..0648e8b 100644 --- a/pygtail/test/test_pygtail.py +++ b/pygtail/test/test_pygtail.py @@ -17,7 +17,6 @@ class PygtailTest(unittest.TestCase): # TODO: # - test for non-default offset file - # - test for paranoid flag # - test for savelog and datext rotation schemes def setUp(self): @@ -156,6 +155,53 @@ def test_offset_file(self): self.assertEqual(inode, log_inode) self.assertEqual(offset, 6) + def test_on_update_with_paranoid(self): + updates = [0] + def record_update(): + updates[0] += 1 + pygtail = Pygtail(self.logfile.name, paranoid=True, + on_update=record_update) + + self.assertEqual(updates[0], 0) + next(pygtail) + self.assertEqual(updates[0], 1) + next(pygtail) + self.assertEqual(updates[0], 2) + next(pygtail) + self.assertEqual(updates[0], 3) + + + def test_on_update_without_paranoid(self): + updates = [0] + + def record_update(): + updates[0] += 1 + + pygtail = Pygtail(self.logfile.name, on_update=record_update) + + self.assertEqual(updates[0], 0) + for line in pygtail: + self.assertEqual(updates[0], 0) + self.assertEqual(updates[0], 1) + + def test_every_n(self): + updates = [0] + # We save before returning the second line. + # We save at the end of the file with all 3 recorded. + expected = [1, 3] + previous_lines = 0 + + def record_update(): + self.assertEqual(previous_lines, expected[updates[0]]) + updates[0] += 1 + + pygtail = Pygtail(self.logfile.name, every_n=2, on_update=record_update) + + self.assertEqual(updates[0], 0) + for line in pygtail: + previous_lines += 1 + + def main(): unittest.main(buffer=True)