Session evaluation (#41)

* add attack type to the session

* add status to the session

* add session analyzer - tool to evaluate session

dorks_manager.py

@@ -0,0 +1,6 @@
+import pickle
+
+
+class DorksManager:
+    with open('dorks.pickle', 'rb') as fh:
+        dorks = pickle.load(fh)

server.py

@@ -15,6 +15,7 @@
 from session_manager import SessionManager
 from xss_emulator import XssEmulator
 from lfi_emulator import LfiEmulator
+from dorks_manager import DorksManager
 
 
 class HttpRequestHandler(aiohttp.server.ServerHttpProtocol):
@@ -32,9 +33,6 @@ class HttpRequestHandler(aiohttp.server.ServerHttpProtocol):
 
     }
 
-    with open('dorks.pickle', 'rb') as fh:
-        dorks = pickle.load(fh)
-
     session_manager = SessionManager()
 
     def __init__(self, *args, **kwargs):
@@ -88,6 +86,7 @@ def handle_event(self, data):
                     lfi_result = self.lfi_emulator.handle(path)
                     detection['payload'] = lfi_result
 
+            session.set_attack_type(path, detection['name'])
             m = self._make_response(msg=dict(detection=detection))
             print(m)
 
@@ -100,7 +99,7 @@ def handle_request(self, message, payload):
         )
         if message.path == '/dorks':
             m = json.dumps(
-                dict(version=1, response=dict(dorks=random.sample(self.dorks, 50))),
+                dict(version=1, response=dict(dorks=random.sample(DorksManager.dorks, 50))),
                 sort_keys=True, indent=2
             ).encode('utf-8')
         elif message.path == '/event':
@@ -122,7 +121,6 @@ def handle_request(self, message, payload):
         lambda: HttpRequestHandler(debug=False, keep_alive=75),
         '0.0.0.0', int('8090'))
     srv = loop.run_until_complete(f)
-
     print('serving on', srv.sockets[0].getsockname())
     try:
         loop.run_forever()

session.py

@@ -13,18 +13,19 @@ def __init__(self, data):
             self.port = data['peer']['port']
             self.user_agent = data['headers']['user-agent']
             self.sensor = data['uuid']
-            self.paths = [{'path': data['path'], 'timestamp': time.time()}]
+            self.paths = [{'path': data['path'], 'timestamp': time.time(), 'response_status': data['status']}]
         except KeyError as e:
             raise
 
         self.uuid = uuid.uuid4()
+        self.start_timestamp = time.time()
         self.timestamp = time.time()
         self.count = 1
 
-    def update_session(self, path):
+    def update_session(self, data):
         self.timestamp = time.time()
         self.count += 1
-        self.paths.append({'path': path, 'timestamp': time.time()})
+        self.paths.append({'path': data['path'], 'timestamp': time.time(), 'response_status': data['status']})
 
     def is_expired(self):
         exp_time = self.timestamp + self.KEEP_ALIVE_TIME
@@ -36,11 +37,17 @@ def to_json(self):
                  user_agent=self.user_agent,
                  sensor=self.sensor,
                  uuid=self.uuid.hex,
-                 timestamp=self.timestamp,
+                 start_time=self.start_timestamp,
+                 end_time=self.timestamp,
                  count=self.count,
                  paths=self.paths
                  )
         return json.dumps(s)
 
+    def set_attack_type(self, path, attack_type):
+        for p in self.paths:
+            if p == path:
+                p.update({'attack_type': attack_type})
+
     def get_key(self):
         return self.uuid

session_analyzer.py

@@ -0,0 +1,107 @@
+import pickle
+
+import redis
+import json
+import asyncio
+import socket
+import operator
+from dorks_manager import DorksManager
+
+
+class SessionAnalyzer:
+    def __init__(self):
+        self.r = redis.StrictRedis(host='localhost', port=6379)
+
+    @asyncio.coroutine
+    def analyze(self):
+        session = None
+        session_key = yield
+        try:
+            session = self.r.get(session_key)
+            session = json.loads(session.decode('utf-8'))
+        except (redis.ConnectionError, TypeError) as e:
+            pass
+        stats = self.create_stats(session)
+
+    def create_stats(self, session):
+        sess_duration = session['end_time'] - session['start_time']
+        rps = sess_duration / session['count']
+        tbr, errors, hidden_links, attack_types = self.analyze_paths(session['paths'])
+
+        stats = dict(
+            uuid=session['uuid'],
+            peer_ip=session['peer']['ip'],
+            peer_port=session['peer']['port'],
+            user_agent=session['user_agent'],
+            sensor_uuid=session['sensor'],
+            start_time=session['start_time'],
+            end_time=session['end_time'],
+            requests_in_second=rps,
+            approx_time_between_requests=tbr,
+            accepted_paths=session['count'],
+            errors=errors,
+            hidden_links=hidden_links,
+            attack_types=attack_types,
+            paths=session['paths']
+        )
+
+        owner = self.choose_possible_owner(stats)
+        stats.update(owner)
+
+        return stats
+
+    def analyze_paths(self, paths):
+        tbr = []
+        attack_types = set()
+        current_path = paths[0]
+
+        for i, path in enumerate(paths, start=1):
+            tbr.append(path['timestamp'] - current_path['timestamp'])
+            current_path = path
+        tbr_average = sum(tbr) / float(len(tbr))
+
+        errors = 0
+        hidden_links = 0
+        for path in paths:
+            if path['response_status'] is not 200:
+                errors += 1
+            if path['path'] in DorksManager.dorks:
+                hidden_links += 1
+            if 'attack_type' in path:
+                attack_types.add(path['attack_type'])
+        return tbr_average, errors, hidden_links, attack_types
+
+    def choose_possible_owner(self, stats):
+        possible_owners = dict(
+            user=0,
+            tool=0,
+            crawler=0,
+            attacker=0
+        )
+        attacks = {'rfi', 'sqli', 'lfi', 'xss'}
+        bots_owner = ['Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
+                      'Googlebot/2.1 (+http://www.google.com/bot.html)'
+                      'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)'
+                      'Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)'
+                      'Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)']
+        if stats['user_agent'] in bots_owner:
+            hostname, aliaslist, ipaddrlist = socket.gethostbyaddr(stats['peer_ip'])
+            if 'search.msn.com' or 'googlebot.com' in hostname:
+                possible_owners['crawler'] += 1
+            else:
+                possible_owners['attacker'] += 1
+        if stats['requests_in_second'] >= 10:
+            possible_owners['tool'] += 1
+            possible_owners['crawler'] += 1
+        else:
+            possible_owners['user'] += 1
+            possible_owners['attacker'] += 1
+        if stats['hidden_links'] > 0:
+            possible_owners['crawler'] += 1
+            possible_owners['attacker'] += 1
+        if stats['attack_types'].intersection(attacks):
+            possible_owners['attacker'] += 1
+
+        maxval = max(possible_owners.items(), key=operator.itemgetter(1))[1]
+        owners = [k for k, v in possible_owners.items() if v == maxval]
+        return {'possible_owners': owners}

session_manager.py

@@ -2,12 +2,15 @@
 import redis
 
 from session import Session
+from session_analyzer import SessionAnalyzer
 
 
 class SessionManager:
     def __init__(self):
         self.sessions = []
         self.r = redis.StrictRedis(host='localhost', port=6379)
+        self.analyzer = SessionAnalyzer().analyze()
+        next(self.analyzer)
 
     @asyncio.coroutine
     def add_or_update_session(self, raw_data):
@@ -25,7 +28,7 @@ def add_or_update_session(self, raw_data):
             self.sessions.append(new_session)
             return new_session
         else:
-            session.update_session(valid_data['path'])
+            session.update_session(valid_data)
             return session
 
     def validate_data(self, data):
@@ -40,6 +43,8 @@ def validate_data(self, data):
             data['path'] = None
         if 'uuid' not in data:
             data['uuid'] = None
+        if 'status' not in data:
+            data['status'] = 200 if 'error' not in data else 500
         return data
 
     def get_session(self, data):
@@ -61,3 +66,5 @@ def delete_old_sessions(self):
                 self.r.set(sess.get_key(), sess.to_json())
             except redis.ConnectionError as e:
                 self.sessions.append(sess)
+            else:
+                self.analyzer.send(sess.get_key())

tests/test_server.py

@@ -15,7 +15,14 @@ def setUp(self):
         self.handler = self.MockedRequestHandler(debug=False, keep_alive=75)
         self.handler.writer = mock.Mock()
 
-        self.handler.session_manager.add_or_update_session = mock.Mock(return_value=(lambda: (yield None))())
+        @asyncio.coroutine
+        def foobar(data):
+            sess = mock.Mock()
+            sess.set_attack_type = mock.Mock()
+            return sess
+
+        self.handler.session_manager.add_or_update_session = foobar
+
         self.m = mock.Mock()
         self.m_eof = mock.Mock()
         self.m_eof.return_value = (lambda: (yield None))()

tests/test_session_analyzer.py

@@ -0,0 +1,34 @@
+import unittest
+import json
+import redis
+from unittest import mock
+from session_analyzer import SessionAnalyzer
+
+session = b'{"uuid": "c546114f97f548f982756495f963e280", "start_time": 1466091813.4780173, ' \
+          b'"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2767.4 Safari/537.36", ' \
+          b'"end_time": 1466091899.9854035, "sensor": "78e51180-bf0d-4757-8a04-f000e5efa179", "count": 24, ' \
+          b'"paths": [{"timestamp": 1466091813.4779778, "path": "/", "attack_type": "index", "response_status": 200}, ' \
+          b'{"timestamp": 1466091842.7088752, "path": "/fluent-python.html", "attack_type": "index", "response_status": 200}, ' \
+          b'{"timestamp": 1466091858.214475, "path": "/wow-movie.html?exec=/bin/bash", "attack_type": "index", "response_status": 200}, ' \
+          b'{"timestamp": 1466091871.9076045, "path": "/wow-movie.html?exec=/etc/passwd", "attack_type": "lfi", "response_status": 200}, ' \
+          b'{"timestamp": 1466091885.1003792, "path": "/wow-movie.html?exec=/bin/bash", "attack_type": "index", "response_status": 200}, ' \
+          b'{"timestamp": 1466091899.9854052, "path": "/wow-movie.html?exec=/../../../..///././././.../../../etc/passwd", "attack_type": "lfi", "response_status": 200}], ' \
+          b'"peer": {"port": 56970, "ip": "192.168.1.3"}}'
+
+
+class TestSessionAnalyzer(unittest.TestCase):
+    def setUp(self):
+        self.session = json.loads(session.decode('utf-8'))
+        self.handler = SessionAnalyzer()
+
+    def tests_load_session(self):
+        stats = None
+        analyzer = self.handler.analyze()
+        redis_mock = mock.Mock(return_value=session)
+        with mock.patch('redis.StrictRedis.get', redis_mock):
+            stats = yield from analyzer
+        self.assertIsNotNone(stats)
+
+    def test_create_stats(self):
+        stats = self.handler.create_stats(self.session)
+        self.assertEqual(stats['possible_owners'], ['attacker'])

tests/test_session_manager.py

@@ -1,14 +1,15 @@
 import unittest
 import session_manager
 import session
-import hashlib
 from unittest import mock
 
 
 class TestSessions(unittest.TestCase):
     def setUp(self):
         with mock.patch('redis.StrictRedis', mock.Mock(), create=True):
             self.handler = session_manager.SessionManager()
+            self.handler.analyzer = mock.Mock()
+            self.handler.analyzer.send = mock.Mock()
 
     def test_validate_missing_peer(self):
         data = {
@@ -25,7 +26,8 @@ def test_validate_missing_peer(self):
                 'user-agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'
             },
             'path': '/foo',
-            'uuid': None
+            'uuid': None,
+            'status': 200
         }
         data = self.handler.validate_data(data)
         self.assertDictEqual(data, assertion_data)
@@ -48,7 +50,8 @@ def test_validate_missing_user_agent(self):
             },
             'headers': {'user-agent': None},
             'path': '/foo',
-            'uuid': None
+            'uuid': None,
+            'status': 200
         }
         data = self.handler.validate_data(data)
         self.assertDictEqual(data, assertion_data)
@@ -69,7 +72,8 @@ def test_adding_new_session(self):
             },
             'headers': {'user-agent': None},
             'path': '/foo',
-            'uuid': None
+            'uuid': None,
+            'status': 200
         }
         assertion_session = session.Session(assertion_data)
         self.assertEquals(session, assertion_session)
@@ -82,7 +86,8 @@ def test_updating_session(self):
             },
             'headers': {'user-agent': None},
             'path': '/foo',
-            'uuid': None
+            'uuid': None,
+            'status': 200
         }
         sess = session.Session(data)
         self.handler.sessions.append(sess)
@@ -97,7 +102,8 @@ def test_deleting_sessions(self):
             },
             'headers': {'user-agent': None},
             'path': '/foo',
-            'uuid': None
+            'uuid': None,
+            'status': 200
         }
         sess = session.Session(data)
         sess.is_expired = mock.MagicMock(name='expired')
@@ -116,9 +122,9 @@ def test_get_key(self):
             },
             'headers': {'user-agent': None},
             'path': '/foo',
-            'uuid': None
+            'uuid': None,
+            'status': 200
         }
         sess = session.Session(data)
         key = sess.get_key()
         self.assertIsNotNone(key)
-