Permalink
Browse files

Created project structure.

Added base parsers.
Added utils.
Added tests.
Created console interface to logs parsing.
  • Loading branch information...
1 parent 93838d4 commit 142a4b67c9a83a88f063edd9f1882e85e4008b98 @Lispython committed May 8, 2012
Showing with 784 additions and 0 deletions.
  1. +12 −0 AUTHORS
  2. 0 CHANGES.md
  3. +33 −0 LICENSE
  4. +10 −0 MANIFEST.in
  5. +23 −0 Makefile
  6. +73 −0 README.rst
  7. 0 __init__.py
  8. +22 −0 alstat/__init__.py
  9. +67 −0 alstat/console.py
  10. +15 −0 alstat/parsers/__init__.py
  11. +153 −0 alstat/parsers/base.py
  12. +17 −0 alstat/parsers/nginx.py
  13. +113 −0 alstat/utils.py
  14. +2 −0 setup.cfg
  15. +119 −0 setup.py
  16. +125 −0 tests/__init__.py
View
12 AUTHORS
@@ -0,0 +1,12 @@
+alstats is written and maintained by Alexandr Lispython and
+various contributors:
+
+Development Lead
+~~~~~~~~~~~~~~~~
+
+- Alex Lispython <alex@obout.ru>
+
+Patches and Suggestions
+~~~~~~~~~~~~~~~~~~~~~~~
+
+http://github.com/lispython/alstats/contributors
View
0 CHANGES.md
No changes.
View
33 LICENSE
@@ -0,0 +1,33 @@
+Copyright (c) 2011 by Alexandr Lispython (alex@obout.ru) and contributors.
+See AUTHORS for more details.
+
+Some rights reserved.
+
+Redistribution and use in source and binary forms of the software as well
+as documentation, with or without modification, are permitted provided
+that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+* The names of the contributors may not be used to endorse or
+ promote products derived from this software without specific
+ prior written permission.
+
+THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
+NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE AND DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
View
10 MANIFEST.in
@@ -0,0 +1,10 @@
+include *.txt *.md *.rst Makefile
+recursive-include docs *.txt *.py *.rst *.md
+recursive-include docs *
+recursive-exclude docs *.pyc
+recursive-exclude docs *.pyo
+recursive-exclude tests *.pyc
+recursive-exclude tests *.pyo
+recursive-exclude examples *.pyc
+recursive-exclude examples *.pyo
+
View
23 Makefile
@@ -0,0 +1,23 @@
+all: clean-pyc test
+
+test:
+ python setup.py nosetests --stop --tests tests.py
+
+
+coverage:
+ python setup.py nosetests --with-coverage --cover-package=alstat --cover-html --cover-html-dir=coverage_out coverage
+
+
+shell:
+ ../venv/bin/ipython
+
+audit:
+ python setup.py autdit
+
+release:
+ python setup.py sdist upload
+
+clean-pyc:
+ find . -name '*.pyc' -exec rm -f {} +
+ find . -name '*.pyo' -exec rm -f {} +
+ find . -name '*~' -exec rm -f {} +
View
73 README.rst
@@ -0,0 +1,73 @@
+Welcome to alstat's documentation!
+==================================================
+
+alstat is advances logs statistics.
+It's collection of utils to analyze logs.
+
+Features
+--------
+
+- Unpack gzipped logfiles
+- Fast
+
+
+Usage
+-----
+
+This commant print all lines from all log files in directory /var/log/nginx
+if format `http_method status http_referer`
+
+ alstat -d /var/log/nginx/ -p "*access*" -f "base" http_method status http_referer
+
+ GET 200 http://google.com
+ .... to many lines
+ GET 404 http://ya.ru/
+ PUT 200 http://yandex.com/
+
+
+You can view fields list that you can use to display:
+
+ alstat -d /var/log/nginx/ -p "*access*" -l
+
+ Alstat v0.0.1 start at Tue May 8 23:25:24 2012
+ You can use fieldnames: status, http_protocol, http_method, http_referer, remote_addr, url, time_local, http_user_agent, remote_user, size
+
+
+
+INSTALLATION
+------------
+
+To use alstat use pip or easy_install:
+
+`pip install alstat`
+
+or
+
+`easy_install alstat`
+
+
+TODO
+----
+- Add group by fields and count
+- Web interface with reports
+
+
+CONTRIBUTE
+----------
+
+Fork https://github.com/Lispython/alstat/ , create commit and pull request.
+
+THANKS
+------
+
+To David M. Beazley for `generators`_ examples.
+
+
+SEE ALSO
+--------
+
+- `python pypi`_.
+
+.. _`python pypi`: http://pypi.python.org
+
+.. _`generators`: http://www.dabeaz.com/generators/
View
0 __init__.py
No changes.
View
22 alstat/__init__.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+alstat
+~~~~~~
+
+Collection of utils to analyze log and build statistics reports
+
+:copyright: (c) 2012 by Alexandr Lispython (alex@obout.ru).
+:license: BSD, see LICENSE for more details.
+"""
+
+__all__ = ('get_version', )
+__author__ = "Alex Lispython (alex@obout.ru)"
+__license__ = "BSD, see LICENSE for more details"
+__version_info__ = (0, 0, 1)
+__version__ = ".".join(map(str, __version_info__))
+__maintainer__ = "Alexandr Lispython (alex@obout.ru)"
+
+
+def get_version():
+ return __version__
View
67 alstat/console.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+alstat.console
+~~~~~~~~~~~~~~
+
+Console interface for alstat
+
+:copyright: (c) 2012 by Alexandr Lispython (alex@obout.ru).
+:license: BSD, see LICENSE for more details.
+"""
+
+import time
+import logging
+import sys
+
+from alstat import get_version
+from alstat.parsers import NginxParser
+
+logger = logging.getLogger('alstat')
+
+
+def main():
+ t = time.time()
+
+ from optparse import OptionParser
+ usage = "%prog [options] field1, field2, field3 ... "
+ parser = OptionParser(usage)
+
+ parser.add_option("-d", "--logs-dir", dest="dirname",
+ help="Logs dir", metavar="DIR")
+ parser.add_option("-p", "--names-pattern", help="Log files name pattern",
+ action="store", type="string", dest="pattern", metavar="PATTERN", default="*")
+
+ parser.add_option("-f", "--log_format",
+ action="store", type="string", dest="log_format", metavar="LOG FORMAT",
+ help="Can be nginx|base", default="nginx")
+ parser.add_option("-l", action="store_true", dest="fields_list", help="Show list of available fields")
+ parser.add_option("-v", action="store_true", dest="verbose", help="Verbose mode")
+
+ (options, args) = parser.parse_args()
+
+ if options.verbose:
+ print("Alstat v%s start at %s" % (get_version(), time.ctime(t)))
+
+ if options.fields_list:
+ print("You can use fieldnames: " + ", ".join(NginxParser.PATTERNS_MAP.keys()))
+ sys.exit(1)
+
+ if not args:
+ parser.error("You need specify field name")
+
+ if options.log_format not in ('nginx', 'base'):
+ parser.error("Log format can be nginx | base")
+
+ if not options.dirname:
+ parser.error("You need to specify logs dir")
+
+ log_parser = NginxParser(options.dirname, options.pattern)
+ for x in log_parser.get_fields():
+ print(" ".join([v for k, v in x.iteritems() if k in args]))
+
+ if options.verbose:
+ print("Analyze completed %s sec at %s " % ((time.time()-t) * 10, time.ctime()))
+ sys.exit(1)
+
View
15 alstat/parsers/__init__.py
@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+alstat.parsers
+~~~~~~~~~~~~~~
+
+Collection of parsers to different logformats
+
+:copyright: (c) 2012 by Alexandr Lispython (alex@obout.ru).
+:license: BSD, see LICENSE for more details.
+"""
+
+from alstat.parsers.base import BaseParser
+from alstat.parsers.nginx import NginxParser
+
View
153 alstat/parsers/base.py
@@ -0,0 +1,153 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+loganalyzer.base
+~~~~~~~~~~~~~~~~~
+
+Base logs parser
+
+:copyright: (c) 2012 by Alexandr Lispython (alex@obout.ru).
+:license: BSD, see LICENSE for more details.
+"""
+import bz2
+import fnmatch
+import gzip
+import os
+import re
+
+
+class BaseParser(object):
+ """
+ Base log parser
+ """
+
+ LOG_FORMAT = r'^{remote_addr}.*'\
+ r'\[{time_local}\].*'\
+ r'"{http_method}\s'\
+ r'{url}\s'\
+ r'{http_protocol}"\s'\
+ r'{status}\s{size}\s'\
+ r'"{http_referer}"\s'\
+ r'"{http_user_agent}".*$'
+
+ PATTERNS_MAP = {
+ "url": r".*",
+ "http_protocol": r".*",
+ "http_method": r"POST|GET",
+ "remote_addr": r"(?:\d{1,3}\.){3}\d{1,3}",
+ "remote_user": r"\S+",
+ "time_local": r".+",
+ "size": r'[0-9]*',
+ "status": r'\d{3}',
+ "http_referer": r".*",
+ "http_user_agent": r".*",
+ }
+
+ def __init__(self, root_path, file_name_pattern):
+ self.root_path = root_path
+ self.file_name_pattern = file_name_pattern
+ self._compiled_log_format = None
+ self._compiled_log_format_re = None
+
+ @property
+ def compiled_log_format(self):
+ if self._compiled_log_format:
+ return self._compiled_log_format
+ else:
+ log_format = self.LOG_FORMAT.format(**dict([(k, r"(?P<{0}>{1})".format(k, v)) for k, v in self.PATTERNS_MAP.iteritems()]))
+ self._compiled_log_format = log_format
+ return self._compiled_log_format
+
+ def setup_log_format(self, log_format):
+ self.LOG_FORMAT = log_format
+
+ def open_file(self, filename):
+ """Analyze filename and open it if it's have gzip format
+ """
+ if filename.endswith(".gz"):
+ return gzip.open(filename)
+ elif filename.endswith(".bz2"):
+ return bz2.BZ2File(filename)
+ else:
+ return open(filename)
+
+ def get_files(self):
+ """Open all files in ``root_path`` matche by ``file_name_pattern``
+ and return generator
+ """
+ for path, dirlist, filelist in os.walk(self.root_path):
+ for name in fnmatch.filter(filelist, self.file_name_pattern):
+ yield self.open_file(os.path.join(path, name))
+
+ def get_lines(self):
+ """Read file lines in generator with files concatenation
+ """
+ for source in self.get_files():
+ for item in source:
+ yield item
+
+ def grep(self, pattern):
+ """
+ Find all lines matched by ``pattern``
+ """
+ compiled_pattern = re.compile(pattern)
+ for line in self.get_lines():
+ if compiled_pattern.search(line):
+ yield line
+
+ def apply_func(self, func):
+ """Read lines and apply ``func``
+ """
+ for line in self.get_lines():
+ yield func(line)
+
+ @property
+ def parse_re(self):
+ """Parse lines and return generator of dicts
+ """
+ if self._compiled_log_format_re:
+ return self._compiled_log_format_re
+
+ self._compiled_log_format_re = re.compile(self.compiled_log_format)
+
+ return self._compiled_log_format_re
+
+ def get_fields(self):
+ """Get generator of fields
+ """
+ for item in self.apply_func(self.parse_re.match):
+ if not item:
+ continue
+ yield item.groupdict()
+
+
+ def field_map(self, field_name, func):
+ """Apply func to ``name`` field
+
+ Arguments:
+ - `name` - field name
+ - `func` - function applyed to item field value
+ """
+ if field_name not in self.PATTERNS_MAP.keys():
+ raise RuntimeError("Not a valid field name")
+
+ for d in self.get_fields():
+ d[field_name] = func(d[field_name])
+ yield d
+
+ def map_fn(self, field_name, map_func):
+ if field_name not in self.PATTERNS_MAP.keys():
+ raise RuntimeError("Not a valid field name")
+
+ for item in self.get_fields():
+ yield map_func(item[field_name])
+
+ def reduce_fn(self, field_name, map_func, reduce_func):
+ if field_name not in self.PATTERNS_MAP.keys():
+ raise RuntimeError("Not a valid field name")
+
+ result = {}
+ for x in self.map_fn(field_name, map_func):
+ #result[field_name] = reduce_func(x[field_name])
+ yield reduce_func(x)
View
17 alstat/parsers/nginx.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+loganalyzer.nginx
+~~~~~~~~~~~~~~~~~
+
+Nginx logs parser
+
+:copyright: (c) 2012 by Alexandr Lispython (alex@obout.ru).
+:license: BSD, see LICENSE for more details.
+"""
+
+from alstat.parsers.base import BaseParser
+
+class NginxParser(BaseParser):
+ pass
View
113 alstat/utils.py
@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+loganalyzer.utils
+~~~~~~~~~~~~~~~~~
+
+Utils to analyze logfiles
+
+:copyright: (c) 2012 by David M. Beazley, Alexandr Lispython (alex@obout.ru).
+:license: BSD, see LICENSE for more details.
+"""
+
+import fnmatch
+import gzip, bz2
+import os
+import re
+
+
+def open_files(filenames):
+ """Open given filenames
+
+ Attributes:
+ - `filenames` - list of filenames with full path
+
+ """
+ for name in filenames:
+ if name.endswith(".gz"):
+ yield gzip.open(name)
+ elif name.endswith(".bz2"):
+ yield bz2.BZ2File(name)
+ else:
+ yield open(name)
+
+
+def find_files(file_pattern, top):
+ """Find logfiles by pattern
+ Attributes:
+ - `file_pattern` - filename pattern
+ - `top` - parent directory to search files
+ Usage:
+ files = find_files("access-log*", '/var/log/nginx/')
+ """
+ for path, dirlist, filelist in os.walk(top):
+ for name in fnmatch.filter(filelist, file_pattern):
+ yield os.path.join(path, name)
+
+
+def concatenator(sources):
+ """Concatenate multiple generators into a single sequence
+
+ Attributes:
+ - `sources` - list of iterable sources
+ """
+ for source in sources:
+ for item in source:
+ yield item
+
+
+def grep(pattern, lines):
+ """Grep a sequence of lines that match a re pattern
+
+ Attributes:
+ - `pattern` - pattern to match
+ - `lines` - iterable seq with lines
+
+ Usage:
+ matched_lines = grep(r'ply-.*\.gz', lines)
+
+
+ """
+ compiled_pattern = re.compile(pattern)
+ for line in lines:
+ if compiled_pattern.search(line):
+ yield line
+
+
+def parse_line(func, lines):
+ """Parse lines by given function
+
+ Attributes:
+ - `lines` - iterable seq with lines
+ - `func` - function to parse line
+ """
+ for line in lines:
+ yield func(line)
+
+
+def field_map(dictseq, name, func):
+ """Take a sequence of dictionaries and remap one of the fields
+
+ Attributes:
+ - `dictseq` - sequence of parsed lines
+ - `name` - field to apply ``func``
+ - `func` - map function
+ """
+ for d in dictseq:
+ d[name] = func(d[name])
+ yield d
+
+
+def build_log_re(patterns_dict, log_format):
+ """Build re by given patterns_dict with format
+
+ Attributes:
+ - `patterns_dict` - dict of names and re patterns
+ - `format` - string format to replace
+
+ """
+ for name, pattern in patterns_dict.iteritems():
+ log_format = log_format.replace(name, r"(?P<%s>%s)" % (name, pattern))
+
+ return re.compile(log_format)
View
2 setup.cfg
@@ -0,0 +1,2 @@
+[nosetests]
+verbosity=3
View
119 setup.py
@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+alstats
+~~~~~~~
+
+Collection of tools to analyze logs
+
+:copyright: (c) 2012 by Alexandr Lispython (alex@obout.ru).
+:license: BSD, see LICENSE for more details.
+"""
+
+
+import sys
+import os
+try:
+ import subprocess
+ has_subprocess = True
+except:
+ has_subprocess = False
+
+from setuptools import Command, setup
+
+__version__ = (0, 0, 1)
+
+
+try:
+ readme_content = open(os.path.join(os.path.abspath(
+ os.path.dirname(__file__)), "README.rst")).read()
+except Exception, e:
+ print(e)
+ readme_content = __doc__
+
+
+class DebugManage(Command):
+ description = "Interface to django manage command from setup.py"
+ user_options = []
+
+ def initialize_options(self):
+ all = None
+
+ def finalize_options(self):
+ pass
+
+ def run(self):
+ from django.core.management import execute_manager
+ from tests.manage import settings as manage_settings
+
+ execute_manager(manage_settings, argv=sys.argv)
+
+
+class run_audit(Command):
+ """Audits source code using PyFlakes for following issues:
+ - Names which are used but not defined or used before they are defined.
+ - Names which are redefined without having been used.
+ """
+ description = "Audit source code with PyFlakes"
+ user_options = []
+
+ def initialize_options(self):
+ all = None
+
+ def finalize_options(self):
+ pass
+
+ def run(self):
+ try:
+ import pyflakes.scripts.pyflakes as flakes
+ except ImportError:
+ print "Audit requires PyFlakes installed in your system."""
+ sys.exit(-1)
+
+ dirs = ['alstats']
+ # Add example directories
+ for dir in []:
+ dirs.append(os.path.join('examples', dir))
+ # TODO: Add test subdirectories
+ warns = 0
+ for dir in dirs:
+ for filename in os.listdir(dir):
+ if filename.endswith('.py') and filename != '__init__.py':
+ warns += flakes.checkPath(os.path.join(dir, filename))
+ if warns > 0:
+ print ("Audit finished with total %d warnings." % warns)
+ else:
+ print ("No problems found in sourcecode.")
+
+
+setup(
+ name="alstats",
+ version=".".join(map(str, __version__)),
+ description="Advanced tools to analyze logs",
+ long_description=readme_content,
+ author="Alex Lispython",
+ author_email="alex@obout.ru",
+ maintainer = "Alexandr Lispython",
+ maintainer_email = "alex@obout.ru",
+ url="https://github.com/lispython/alstat",
+ install_requires=[
+# 'tornadoweb',
+ ],
+ license="BSD",
+# test_suite="nose.collector",
+ platforms = ['Linux', 'Mac'],
+ classifiers=[
+ "Environment :: Web Environment",
+ "License :: OSI Approved :: BSD License",
+ "Programming Language :: Python",
+ "Operating System :: MacOS :: MacOS X",
+ "Operating System :: POSIX",
+ "Topic :: Internet",
+ "Topic :: Software Development :: Libraries"
+ ],
+ cmdclass={'audit': run_audit},
+ entry_points=dict(
+ console_scripts=['alstat=alstat.console:main']
+ ),
+ test_suite = 'tests.suite'
+ )
View
125 tests/__init__.py
@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+tests
+~~~~~
+
+alstat unittests module
+
+:copyright: (c) 2012 by Alexandr Lispython (alex@obout.ru).
+:license: BSD, see LICENSE for more details.
+"""
+
+import unittest
+import os
+import re
+from collections import defaultdict
+
+from alstat.utils import find_files, concatenator, open_files, parse_line
+from alstat.parsers import BaseParser, NginxParser
+
+
+PROJECT_ROOT = os.path.normpath(os.path.dirname(__file__))
+
+
+def rel(*parts):
+ return os.path.join(PROJECT_ROOT, *parts)
+
+
+class UtilsTestCase(unittest.TestCase):
+
+ def setUp(self):
+ self.log_dir = rel('logs')
+ self.nginx_default_log_format = r'remote_addr - remote_user [time_local] "request" status bytes_sent "http_referer" "http_user_agent" "gzip_ratio"'
+
+ def test_utils(self):
+ print("test utils")
+
+ def test_find_files(self):
+ self.assertEquals(len([x for x in find_files("access.nginx.*", self.log_dir)]), 1)
+
+ def test_parser(self):
+ files = open_files(find_files("access.nginx.*", self.log_dir))
+ lines = concatenator(files)
+
+ for x in parse_line(lambda x: x[:20], lines):
+ self.assertEquals(len(x), 20)
+
+ def test_base_parser(self):
+ base_parser = BaseParser(self.log_dir, "access.nginx.*")
+
+ self.assertEquals(base_parser.root_path, self.log_dir)
+ self.assertEquals(base_parser.file_name_pattern, "access.nginx.*")
+ self.assertEquals(len([x for x in base_parser.get_files()]), 1)
+
+ self.assertEquals(len([x for x in base_parser.get_lines()]), 13)
+ self.assertEquals(len([x for x in base_parser.grep(r'\[.*\]')]), 13)
+
+ for x in base_parser.apply_func(lambda x: x[:20]):
+ self.assertEquals(len(x), 20)
+
+ def test_nginx_parser(self):
+ nginx_parser = NginxParser(self.log_dir, "access.nginx.*")
+ control_re_pattern = r'^(?P<remote_addr>(?:\d{1,3}\.){3}\d{1,3}).*'\
+ r'\[(?P<time_local>.+)\].*'\
+ r'"(?P<http_method>POST|GET)\s'\
+ r'(?P<url>.*)\s'\
+ r'(?P<http_protocol>.*)"\s'\
+ r'(?P<status>\d{3})\s(?P<size>[0-9]*)\s'\
+ r'"(?P<http_referer>.*)"\s'\
+ r'"(?P<http_user_agent>.*)".*$'
+
+ self.assertEquals(nginx_parser.compiled_log_format, control_re_pattern)
+ for x in nginx_parser.apply_func(nginx_parser.parse_re.match):
+ self.assertEquals(len(x.groupdict()), 9)
+
+ for x in nginx_parser.get_fields():
+ self.assertEquals(len(x), 9)
+ ## with self.assertRaises(RuntimeError):
+ ## for x in nginx_parser.field_map('not_valid_field', str):
+ ## print x
+
+ log = nginx_parser.field_map('status', lambda x: "bbb")
+ for x in log:
+ self.assertEquals(x['status'], "bbb")
+
+ for x in nginx_parser.map_fn('status', lambda x: (x, 1)):
+ self.assertEquals(x[1], 1)
+
+ def sort_fn(i_list):
+ new = defaultdict(list)
+ map(lambda x: new[x[0]].append(x[1]), i_list)
+ return new.items()
+
+ for x in sort_fn(nginx_parser.map_fn('status', lambda x: (x, 1))):
+ pass
+
+ ## for x in nginx_parser.reduce_fn('status', lambda x: (x, 1), lambda x: [x[0], sum([x[1]])]):
+ ## print x
+
+ ## for x in nginx_parser.reduce_fn('status', lambda x: (x, 1), lambda i: [i[0], sum(i[1])]):
+ ## print(x)
+
+ def test_re(self):
+ log_line = '66.249.73.219 - - [07/May/2012:16:50:48 -0400] "GET /users/791/ HTTP/1.1" 200 3318 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"'
+ pattern = r'^(?P<remote_addr>(?:\d{1,3}\.){3}\d{1,3}).*'\
+ r'\[(?P<time_local>.+)\].*'\
+ r'"(?P<http_method>POST|GET)\s'\
+ r'(?P<url>.*)\s'\
+ r'(?P<http_protocol>.*)"\s'\
+ r'(?P<status>\d{3})\s(?P<size>[0-9]*)\s'\
+ r'"(?P<http_referer>.*)"\s'\
+ r'"(?P<http_user_agent>.*)".*$'
+ pattern_re = re.compile(pattern, re.U)
+ matched = pattern_re.match(log_line)
+ self.assertTrue(matched)
+
+
+def suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(UtilsTestCase))
+ return suite
+
+
+if __name__ == '__main__':
+ unittest.main(defaultT='suite')

0 comments on commit 142a4b6

Please sign in to comment.