convert log parser to python3

Author: usmannasir

CPScripts/access-logparser.py

@@ -0,0 +1,230 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# Originally based on code from: https://leancrew.com/all-this/2013/07/parsing-my-apache-logs/
+
+import os
+import re
+import sys
+from collections import Counter
+from datetime import datetime, date, timedelta
+
+
+# print('version is', sys.version)
+
+def detectcontrolpanel():
+    global controlpanel
+    try:
+        if os.path.isfile('/usr/local/cpanel/cpanel'):
+            controlpanel = 'cpanel'
+    except:
+        controlpanel = 'Control Panel not found'
+
+    try:
+        if os.path.isfile('/usr/bin/cyberpanel'):
+            controlpanel = 'cyberpanel'
+    except:
+        controlpanel = 'Control Panel not found'
+    return controlpanel
+
+
+def main():
+    script = sys.argv[0]
+    filename = sys.argv[2]
+    # filenametest = "/home/example.com.access_log"
+
+    # Define the day of interest in the Apache common log format.
+    try:
+        daysAgo = int(sys.argv[1])
+        # daysAgo = 2
+    except:
+        daysAgo = 1
+    theDay = date.today() - timedelta(daysAgo)
+    apacheDay = theDay.strftime('[%d/%b/%Y:')
+
+    # Regex for the Apache common log format.
+    parts = [  # host %h  			:ip/hostname of the client 	172.68.142.138
+        # indent %l (unused) 	:client identity via client's identd configuration 	-
+        # user %u 			:HTTP authenticated user ID 	-
+        # time %t 			:timestamp 	[09/Mar/2019:00:38:03 -0600]
+        # request "%r" 		:request method of request, resource requested, & protocol 	"POST /wp-login.php HTTP/1.1"
+        # status %>s 		:Apache status code 	404
+        # size %b (careful,can be'-'):size of request in bytes, excluding headers 	3767
+        # referrer "%{Referer}i"	:Referer 	"https://www.google.com/"
+        # user agent "%{User-agent}i":User-Agent 	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0"
+        r'(?P<host>\S+)',
+        r'\S+',
+        r'(?P<user>\S+)',
+        r'\[(?P<time>.+)\]',
+        r'"(?P<request>.*)"',
+        r'(?P<status>[0-9]+)',
+        r'(?P<size>\S+)',
+        r'"(?P<referrer>.*)"',
+        r'"(?P<agent>.*)"',
+    ]
+    pattern = re.compile(r'\s+'.join(parts) + r'\s*\Z')
+
+    # Regex for a feed request.
+    feed = re.compile(r'/all-this/(\d\d\d\d/\d\d/[^/]+/)?feed/(atom/)?')
+
+    # Regexes for internal and Google search referrers.
+
+    internal = re.compile(r'https?://(www\.)?example\.com.*')
+    google = re.compile(r'https?://(www\.)?google\..*')
+
+    # Regexes for Uptime Monitoring Robots
+    uptimeroboturl = re.compile(r'https?://(www\.)?uptimerobot\..*')
+    uptimerobot = re.compile(r'UptimeRobot')
+
+    # WordPress CMS Regex
+    wordpresslogin = re.compile(r'wp-login\.php.*')
+    wordpressadmin = re.compile(r'wp-admin')
+    wordpresscron = re.compile(r'wp-cron\.php.*')
+    wordpressxmlrpc = re.compile(r'xmlrpc\.php')
+    wordpressajax = re.compile(r'admin-ajax\.php')
+
+    # Change Apache log items into Python types.
+
+    def pythonized(d):
+        # Clean up the request.
+
+        d['request'] = d['request'].split()[1]
+
+        # Some dashes become None.
+
+        for k in ('user', 'referrer', 'agent'):
+            if d[k] == '-':
+                d[k] = None
+
+        # The size dash becomes 0.
+
+        if d['size'] == '-':
+            d['size'] = 0
+        else:
+            d['size'] = int(d['size'])
+
+        # Convert the timestamp into a datetime object. Accept the server's time zone.
+
+        (time, zone) = d['time'].split()
+        d['time'] = datetime.strptime(time, '%d/%b/%Y:%H:%M:%S')
+
+        return d
+
+    # Is this hit a page?
+
+    def ispage(hit):
+        # Failures and redirects.
+
+        hit['status'] = int(hit['status'])
+        if hit['status'] < 200 or hit['status'] >= 300:
+            return False
+
+        # Feed requests.
+
+        if feed.search(hit['request']):
+            return False
+
+        # Requests that aren't GET.
+
+        #    if (hit['request'])[0:3] != 'GET':
+        #        return False
+
+        # Images, sounds, etc.
+
+        if hit['request'].split()[1][-1] != '/':
+            return False
+
+        # Requests that aren't Head type. AKA uptime monitoring
+
+        if (hit['request'])[0:3] == 'HEAD':
+            return False
+
+        # Must be a page.
+
+        return True
+
+    # Is the referrer interesting? Internal and Google referrers are not.
+    def goodref(hit):
+        if hit['referrer']:
+            return not (google.search(hit['referrer'])
+                        or internal.search(hit['referrer']))
+        else:
+            return False
+
+    # Is the user agent interesting? An uptime monitoring robot is not.
+    def goodagent(hit):
+        if hit['agent']:
+            return not (uptimerobot.search(hit['agent'])
+                        or uptimeroboturl.search(hit['agent']))
+        else:
+            return False
+
+    # Is the request a Wordpress related login event?
+    def wordpressbrute(hit):
+        if hit['request']:
+            return (wordpresslogin.search(hit['request'])
+                    or wordpressadmin.search(hit['request']))
+        else:
+            return False
+
+    # Initialize.
+
+    pages = []
+
+    # Parse all the lines associated with the day of interest.
+
+    # Open file
+    log = open(filename)
+    for line in log:
+        if apacheDay in line:
+            m = pattern.match(line)
+            hit = m.groupdict()
+            if ispage(hit):
+                pages.append(pythonized(hit))
+        else:
+            continue
+    log.close()
+
+    # Show the top five pages and the total.
+
+    print ('Show top 10 pages %s' % theDay.strftime('%b %d, %Y'))
+    pageviews = Counter(x['request'] for x in pages if goodagent(x))
+    pagestop10 = pageviews.most_common(10)
+    for p in pagestop10:
+        print ('  %5d  %s' % p[::-1])
+    print ('  %5d  total' % len(pages))
+
+    # Show the top five referrers.
+
+    print ('''
+    Show top 10 referrers %s''' % theDay.strftime('%b %d, %Y'))
+    referrers = Counter(x['referrer'] for x in pages if goodref(x))
+    referrerstop10 = referrers.most_common(10)
+    for r in referrerstop10:
+        print ('  %5d  %s' % r[::-1])
+    print ('  %5d  total' % sum(referrers.values()))
+
+    # Show the top 10 IPs.
+    print ('''
+    Show Top 10 IPs %s''' % theDay.strftime('%b %d, %Y'))
+    iphits = Counter(x['host'] for x in pages if goodagent(x))
+    iptop10 = iphits.most_common(10)
+    for p in iptop10:
+        print ('  %5d  %s' % p[::-1])
+    print ('  %5d  total hits' % sum(iphits.values()))
+
+    # CMS Checks
+
+    # Wordpress Checks
+    # Wordpress Login Bruteforcing checks for wp-login.php
+    print ('''
+    Wordpress Bruteforce Logins for wp-login.php %s''' % theDay.strftime('%b %d, %Y'))
+    wordpressloginhits = Counter(x['request'] for x in pages if wordpressbrute(x))
+    # wordpresslogintop10 = wordpressloginhits.most_common(10)
+    # for p in wordpresslogintop10:
+    #    print '  %5d  %s' % p[::-1]
+    print ('  %5d  total' % sum(wordpressloginhits.values()))
+
+
+if __name__ == '__main__':
+    # detectcontrolpanel()
+    main()