From b2c79ebb8468ff0efc178d773370131467df8582 Mon Sep 17 00:00:00 2001 From: afeena Date: Sun, 1 Mar 2020 19:03:01 +0100 Subject: [PATCH 01/14] store sessions in dict, run delete in a separate thread --- tanner/server.py | 6 ++- tanner/sessions/__init__.py | 0 tanner/{ => sessions}/session.py | 0 tanner/{ => sessions}/session_analyzer.py | 0 tanner/{ => sessions}/session_manager.py | 56 ++++++++++++----------- tanner/tests/test_base.py | 2 +- tanner/tests/test_session_analyzer.py | 2 +- tanner/tests/test_session_manager.py | 2 +- 8 files changed, 37 insertions(+), 31 deletions(-) create mode 100644 tanner/sessions/__init__.py rename tanner/{ => sessions}/session.py (100%) rename tanner/{ => sessions}/session_analyzer.py (100%) rename tanner/{ => sessions}/session_manager.py (70%) diff --git a/tanner/server.py b/tanner/server.py index d2d52310..cb5caeef 100644 --- a/tanner/server.py +++ b/tanner/server.py @@ -1,12 +1,14 @@ import asyncio import json import logging +import concurrent import uvloop import yarl from aiohttp import web -from tanner import dorks_manager, session_manager, redis_client +from tanner import dorks_manager, redis_client +from tanner.sessions import session_manager from tanner.config import TannerConfig from tanner.emulators import base from tanner.reporting.log_local import Reporting as local_report @@ -115,6 +117,8 @@ def create_app(self, loop): def start(self): loop = asyncio.get_event_loop() self.redis_client = loop.run_until_complete(redis_client.RedisClient.get_redis_client()) + with concurrent.futures.ThreadPoolExecutor() as pool: + await loop.run_in_executor(pool, self.session_manager.delete_old_sessions, self.redis_client) app = self.create_app(loop) host = TannerConfig.get('TANNER', 'host') port = TannerConfig.get('TANNER', 'port') diff --git a/tanner/sessions/__init__.py b/tanner/sessions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tanner/session.py b/tanner/sessions/session.py similarity index 100% rename from tanner/session.py rename to tanner/sessions/session.py diff --git a/tanner/session_analyzer.py b/tanner/sessions/session_analyzer.py similarity index 100% rename from tanner/session_analyzer.py rename to tanner/sessions/session_analyzer.py diff --git a/tanner/session_manager.py b/tanner/sessions/session_manager.py similarity index 70% rename from tanner/session_manager.py rename to tanner/sessions/session_manager.py index d3ea2f2e..f12797e1 100644 --- a/tanner/session_manager.py +++ b/tanner/sessions/session_manager.py @@ -1,35 +1,39 @@ import logging +import hashlib +import asyncio import aioredis -from tanner.session import Session -from tanner.session_analyzer import SessionAnalyzer +from tanner.sessions.session import Session +from tanner.sessions.session_analyzer import SessionAnalyzer class SessionManager: def __init__(self, loop=None): - self.sessions = [] + self.sessions = {} self.analyzer = SessionAnalyzer(loop=loop) self.logger = logging.getLogger(__name__) async def add_or_update_session(self, raw_data, redis_client): - # prepare the list of sessions - await self.delete_old_sessions(redis_client) + # handle raw data valid_data = self.validate_data(raw_data) # push snare uuid into redis. await redis_client.sadd('snare_ids', *[valid_data['uuid']]) - session = self.get_session(valid_data) - if session is None: + session_uuid = self.get_session_uuid(valid_data) + if session_uuid not in self.sessions: try: new_session = Session(valid_data) except KeyError as key_error: self.logger.exception('Error during session creation: %s', key_error) return - self.sessions.append(new_session) + self.sessions[session_uuid] = new_session return new_session - session.update_session(valid_data) - return session + else: + self.sessions[session_uuid].update_session(valid_data) + # prepare the list of sessions + await self.delete_old_sessions(redis_client) + return self.sessions[session_uuid] @staticmethod def validate_data(data): @@ -53,33 +57,31 @@ def validate_data(data): return data - def get_session(self, data): - session = None + def get_session_uuid(self, data): ip = data['peer']['ip'] user_agent = data['headers']['user-agent'] sess_uuid = data['cookies']['sess_uuid'] - for sess in self.sessions: - if sess.ip == ip and sess.user_agent == user_agent and sess_uuid == sess.get_uuid(): - session = sess - break - return session + + return hashlib.md5(ip+user_agent+sess_uuid).hexdigest() async def delete_old_sessions(self, redis_client): - for sess in self.sessions: - if not sess.is_expired(): - continue - is_deleted = await self.delete_session(sess, redis_client) - if is_deleted: - try: - self.sessions.remove(sess) - except ValueError: + while True: + for sess_uuid,session in self.sessions.items(): + if not session.is_expired(): continue + is_deleted = await self.delete_session(session, redis_client) + if is_deleted: + try: + del self.sessions[session] + except ValueError: + continue + await asyncio.sleep(5*60) async def delete_sessions_on_shutdown(self, redis_client): - for sess in self.sessions: + for _, sess in self.sessions.items(): is_deleted = await self.delete_session(sess, redis_client) if is_deleted: - self.sessions.remove(sess) + del self.sessions[sess] async def delete_session(self, sess, redis_client): await sess.remove_associated_db() diff --git a/tanner/tests/test_base.py b/tanner/tests/test_base.py index 92b6e4b1..dcdb0f9a 100644 --- a/tanner/tests/test_base.py +++ b/tanner/tests/test_base.py @@ -3,7 +3,7 @@ from unittest import mock from tanner.utils.asyncmock import AsyncMock -from tanner import session +from tanner.sessions import session from tanner.emulators import base from tanner import __version__ as tanner_version diff --git a/tanner/tests/test_session_analyzer.py b/tanner/tests/test_session_analyzer.py index f5396d64..3b5f1339 100644 --- a/tanner/tests/test_session_analyzer.py +++ b/tanner/tests/test_session_analyzer.py @@ -5,7 +5,7 @@ from unittest.mock import patch import geoip2 import aioredis -from tanner.session_analyzer import SessionAnalyzer +from tanner.sessions.session_analyzer import SessionAnalyzer session = b'{"sess_uuid": "c546114f97f548f982756495f963e280", "start_time": 1466091813.4780173, ' \ diff --git a/tanner/tests/test_session_manager.py b/tanner/tests/test_session_manager.py index 1925040c..105163a5 100644 --- a/tanner/tests/test_session_manager.py +++ b/tanner/tests/test_session_manager.py @@ -2,7 +2,7 @@ import unittest from unittest import mock -from tanner import session, session_manager +from tanner.sessions import session_manager, session class TestSessions(unittest.TestCase): From bb1899c4a634975dd27da26f53297681a0a5deed Mon Sep 17 00:00:00 2001 From: afeena Date: Sun, 1 Mar 2020 18:41:20 +0000 Subject: [PATCH 02/14] add app background task --- tanner/server.py | 13 +++++++++++-- tanner/sessions/session_manager.py | 11 +++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/tanner/server.py b/tanner/server.py index cb5caeef..abae6bef 100644 --- a/tanner/server.py +++ b/tanner/server.py @@ -114,12 +114,21 @@ def create_app(self, loop): self.setup_routes(app) return app + async def start_background_delete(self, app): + app['session_delete'] = asyncio.create_task(self.session_manager.delete_old_sessions(self.redis_client)) + + async def cleanup_background_tasks(self, app): + app['session_delete'].cancel() + await app['session_delete'] + def start(self): loop = asyncio.get_event_loop() self.redis_client = loop.run_until_complete(redis_client.RedisClient.get_redis_client()) - with concurrent.futures.ThreadPoolExecutor() as pool: - await loop.run_in_executor(pool, self.session_manager.delete_old_sessions, self.redis_client) + app = self.create_app(loop) + app.on_startup.append(self.start_background_delete) + app.on_cleanup.append(self.cleanup_background_tasks) + host = TannerConfig.get('TANNER', 'host') port = TannerConfig.get('TANNER', 'port') web.run_app(app, host=host, port=int(port)) diff --git a/tanner/sessions/session_manager.py b/tanner/sessions/session_manager.py index f12797e1..62d8d239 100644 --- a/tanner/sessions/session_manager.py +++ b/tanner/sessions/session_manager.py @@ -1,6 +1,7 @@ import logging import hashlib import asyncio +import time import aioredis @@ -62,10 +63,12 @@ def get_session_uuid(self, data): user_agent = data['headers']['user-agent'] sess_uuid = data['cookies']['sess_uuid'] - return hashlib.md5(ip+user_agent+sess_uuid).hexdigest() + return hashlib.md5((ip+user_agent+sess_uuid).encode()).hexdigest() async def delete_old_sessions(self, redis_client): + delete_timeout = 60*5 while True: + print("delete session") for sess_uuid,session in self.sessions.items(): if not session.is_expired(): continue @@ -75,13 +78,13 @@ async def delete_old_sessions(self, redis_client): del self.sessions[session] except ValueError: continue - await asyncio.sleep(5*60) + await asyncio.sleep(delete_timeout) async def delete_sessions_on_shutdown(self, redis_client): - for _, sess in self.sessions.items(): + for sess_uuid, sess in self.sessions.items(): is_deleted = await self.delete_session(sess, redis_client) if is_deleted: - del self.sessions[sess] + del self.sessions[sess_uuid] async def delete_session(self, sess, redis_client): await sess.remove_associated_db() From 481a96d477fddb274b0a8ab0ab4c9e3721e229da Mon Sep 17 00:00:00 2001 From: afeena Date: Sun, 1 Mar 2020 18:53:24 +0000 Subject: [PATCH 03/14] use sess_uuid as del key --- tanner/sessions/session_manager.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tanner/sessions/session_manager.py b/tanner/sessions/session_manager.py index 62d8d239..a54c2c46 100644 --- a/tanner/sessions/session_manager.py +++ b/tanner/sessions/session_manager.py @@ -33,7 +33,6 @@ async def add_or_update_session(self, raw_data, redis_client): else: self.sessions[session_uuid].update_session(valid_data) # prepare the list of sessions - await self.delete_old_sessions(redis_client) return self.sessions[session_uuid] @staticmethod @@ -60,22 +59,21 @@ def validate_data(data): def get_session_uuid(self, data): ip = data['peer']['ip'] - user_agent = data['headers']['user-agent'] - sess_uuid = data['cookies']['sess_uuid'] + user_agent = data['headers']['user-agent'] if data['headers']['user-agent'] is not None else "" + sess_uuid = data['cookies']['sess_uuid'] if data['cookies']['sess_uuid'] is not None else "" return hashlib.md5((ip+user_agent+sess_uuid).encode()).hexdigest() async def delete_old_sessions(self, redis_client): delete_timeout = 60*5 while True: - print("delete session") - for sess_uuid,session in self.sessions.items(): + for sess_uuid, session in self.sessions.items(): if not session.is_expired(): continue is_deleted = await self.delete_session(session, redis_client) if is_deleted: try: - del self.sessions[session] + del self.sessions[sess_uuid] except ValueError: continue await asyncio.sleep(delete_timeout) From 4c93a8da8a15d8e2abfecdc10b3ffffba59a7a19 Mon Sep 17 00:00:00 2001 From: afeena Date: Sun, 1 Mar 2020 19:56:04 +0100 Subject: [PATCH 04/14] delete timeout as a parameter --- tanner/sessions/session_manager.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tanner/sessions/session_manager.py b/tanner/sessions/session_manager.py index a54c2c46..6837c5b2 100644 --- a/tanner/sessions/session_manager.py +++ b/tanner/sessions/session_manager.py @@ -10,10 +10,11 @@ class SessionManager: - def __init__(self, loop=None): + def __init__(self, loop=None, delete_timeout=60*5): self.sessions = {} self.analyzer = SessionAnalyzer(loop=loop) self.logger = logging.getLogger(__name__) + self.delete_timeout = delete_timeout async def add_or_update_session(self, raw_data, redis_client): @@ -65,7 +66,7 @@ def get_session_uuid(self, data): return hashlib.md5((ip+user_agent+sess_uuid).encode()).hexdigest() async def delete_old_sessions(self, redis_client): - delete_timeout = 60*5 + while True: for sess_uuid, session in self.sessions.items(): if not session.is_expired(): @@ -76,7 +77,7 @@ async def delete_old_sessions(self, redis_client): del self.sessions[sess_uuid] except ValueError: continue - await asyncio.sleep(delete_timeout) + await asyncio.sleep(self.delete_timeout) async def delete_sessions_on_shutdown(self, redis_client): for sess_uuid, sess in self.sessions.items(): From 8245182ce91753d2190b25227c343c82b8d38896 Mon Sep 17 00:00:00 2001 From: afeena Date: Sun, 1 Mar 2020 19:56:48 +0100 Subject: [PATCH 05/14] formatting formatting (2) --- tanner/server.py | 4 ++-- tanner/sessions/session_manager.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tanner/server.py b/tanner/server.py index abae6bef..df4fb7ed 100644 --- a/tanner/server.py +++ b/tanner/server.py @@ -118,8 +118,8 @@ async def start_background_delete(self, app): app['session_delete'] = asyncio.create_task(self.session_manager.delete_old_sessions(self.redis_client)) async def cleanup_background_tasks(self, app): - app['session_delete'].cancel() - await app['session_delete'] + app['session_delete'].cancel() + await app['session_delete'] def start(self): loop = asyncio.get_event_loop() diff --git a/tanner/sessions/session_manager.py b/tanner/sessions/session_manager.py index 6837c5b2..d40e4f39 100644 --- a/tanner/sessions/session_manager.py +++ b/tanner/sessions/session_manager.py @@ -10,7 +10,7 @@ class SessionManager: - def __init__(self, loop=None, delete_timeout=60*5): + def __init__(self, loop=None, delete_timeout=60 * 5): self.sessions = {} self.analyzer = SessionAnalyzer(loop=loop) self.logger = logging.getLogger(__name__) @@ -60,10 +60,10 @@ def validate_data(data): def get_session_uuid(self, data): ip = data['peer']['ip'] - user_agent = data['headers']['user-agent'] if data['headers']['user-agent'] is not None else "" + user_agent = data['headers']['user-agent'] if data['headers']['user-agent'] is not None else "" sess_uuid = data['cookies']['sess_uuid'] if data['cookies']['sess_uuid'] is not None else "" - return hashlib.md5((ip+user_agent+sess_uuid).encode()).hexdigest() + return hashlib.md5((ip + user_agent + sess_uuid).encode()).hexdigest() async def delete_old_sessions(self, redis_client): From fc7637e63483f5180acba79c8033b1dd972b387a Mon Sep 17 00:00:00 2001 From: afeena Date: Sun, 1 Mar 2020 20:11:43 +0100 Subject: [PATCH 06/14] fix tests, move while True to other async func --- tanner/server.py | 9 +++++-- tanner/sessions/session_manager.py | 36 +++++++++++++--------------- tanner/tests/test_session_manager.py | 20 +++++++++++----- 3 files changed, 38 insertions(+), 27 deletions(-) diff --git a/tanner/server.py b/tanner/server.py index df4fb7ed..e64278e8 100644 --- a/tanner/server.py +++ b/tanner/server.py @@ -58,7 +58,7 @@ async def handle_event(self, request): self.logger.exception('error parsing request: %s', data) response_msg = self._make_response(msg=type(error).__name__) else: - session = await self.session_manager.add_or_update_session( + session, _ = await self.session_manager.add_or_update_session( data, self.redis_client ) self.logger.info('Requested path %s', path) @@ -102,6 +102,11 @@ async def on_shutdown(self, app): await self.session_manager.delete_sessions_on_shutdown(self.redis_client) self.redis_client.close() + async def delete_sessions(self): + while True: + await self.session_manager.delete_old_sessions(self.redis_client) + await asyncio.sleep(self.delete_timeout) + def setup_routes(self, app): app.router.add_route('*', '/', self.default_handler) app.router.add_post('/event', self.handle_event) @@ -115,7 +120,7 @@ def create_app(self, loop): return app async def start_background_delete(self, app): - app['session_delete'] = asyncio.create_task(self.session_manager.delete_old_sessions(self.redis_client)) + app['session_delete'] = asyncio.create_task(self.delete_sessions()) async def cleanup_background_tasks(self, app): app['session_delete'].cancel() diff --git a/tanner/sessions/session_manager.py b/tanner/sessions/session_manager.py index d40e4f39..4a17fd3f 100644 --- a/tanner/sessions/session_manager.py +++ b/tanner/sessions/session_manager.py @@ -22,19 +22,19 @@ async def add_or_update_session(self, raw_data, redis_client): valid_data = self.validate_data(raw_data) # push snare uuid into redis. await redis_client.sadd('snare_ids', *[valid_data['uuid']]) - session_uuid = self.get_session_uuid(valid_data) - if session_uuid not in self.sessions: + session_id = self.get_session_id(valid_data) + if session_id not in self.sessions: try: new_session = Session(valid_data) except KeyError as key_error: self.logger.exception('Error during session creation: %s', key_error) return - self.sessions[session_uuid] = new_session + self.sessions[session_id] = new_session return new_session else: - self.sessions[session_uuid].update_session(valid_data) + self.sessions[session_id].update_session(valid_data) # prepare the list of sessions - return self.sessions[session_uuid] + return self.sessions[session_id], session_id @staticmethod def validate_data(data): @@ -58,7 +58,7 @@ def validate_data(data): return data - def get_session_uuid(self, data): + def get_session_id(self, data): ip = data['peer']['ip'] user_agent = data['headers']['user-agent'] if data['headers']['user-agent'] is not None else "" sess_uuid = data['cookies']['sess_uuid'] if data['cookies']['sess_uuid'] is not None else "" @@ -66,24 +66,22 @@ def get_session_uuid(self, data): return hashlib.md5((ip + user_agent + sess_uuid).encode()).hexdigest() async def delete_old_sessions(self, redis_client): - - while True: - for sess_uuid, session in self.sessions.items(): - if not session.is_expired(): + for sess_id, session in self.sessions.items(): + if not session.is_expired(): + continue + is_deleted = await self.delete_session(session, redis_client) + if is_deleted: + try: + del self.sessions[sess_id] + except ValueError: continue - is_deleted = await self.delete_session(session, redis_client) - if is_deleted: - try: - del self.sessions[sess_uuid] - except ValueError: - continue - await asyncio.sleep(self.delete_timeout) + async def delete_sessions_on_shutdown(self, redis_client): - for sess_uuid, sess in self.sessions.items(): + for sess_id, sess in self.sessions.items(): is_deleted = await self.delete_session(sess, redis_client) if is_deleted: - del self.sessions[sess_uuid] + del self.sessions[sess_id] async def delete_session(self, sess, redis_client): await sess.remove_associated_db() diff --git a/tanner/tests/test_session_manager.py b/tanner/tests/test_session_manager.py index 105163a5..8423a680 100644 --- a/tanner/tests/test_session_manager.py +++ b/tanner/tests/test_session_manager.py @@ -1,6 +1,7 @@ import asyncio import unittest from unittest import mock +import hashlib from tanner.sessions import session_manager, session @@ -106,9 +107,9 @@ async def sess_sadd(key, value): redis_mock = mock.Mock() redis_mock.sadd = sess_sadd - sess = self.loop.run_until_complete(self.handler.add_or_update_session(data, redis_mock)) + sess, sess_id = self.loop.run_until_complete(self.handler.add_or_update_session(data, redis_mock)) - self.assertEquals([sess], self.handler.sessions) + self.assertDictEqual({sess_id : sess}, self.handler.sessions) def test_updating_session(self): async def sess_sadd(key, value): @@ -126,12 +127,19 @@ async def sess_sadd(key, value): 'cookies': {'sess_uuid': None} } sess = session.Session(data) + data['cookies']['sess_uuid'] = sess.get_uuid() + ip = data['peer']['ip'] + user_agent = data['headers']['user-agent'] if data['headers']['user-agent'] is not None else "" + sess_uuid = data['cookies']['sess_uuid'] if data['cookies']['sess_uuid'] is not None else "" + + sess_id = hashlib.md5((ip + user_agent + sess_uuid).encode()).hexdigest() + redis_mock = mock.Mock() redis_mock.sadd = sess_sadd - self.handler.sessions.append(sess) + self.handler.sessions[sess_id] = sess self.loop.run_until_complete(self.handler.add_or_update_session(data, redis_mock)) - self.assertEqual(self.handler.sessions[0].count, 2) + self.assertEqual(self.handler.sessions[sess_id].count, 2) def test_deleting_sessions(self): async def analyze(session_key, redis_client): @@ -155,11 +163,11 @@ async def sess_set(key, val): sess = session.Session(data) sess.is_expired = mock.MagicMock(name='expired') sess.is_expired.__bool__.reurned_value = True - self.handler.sessions.append(sess) + self.handler.sessions[sess] = sess redis_mock = mock.Mock() redis_mock.set = sess_set self.loop.run_until_complete(self.handler.delete_old_sessions(redis_mock)) - self.assertListEqual(self.handler.sessions, []) + self.assertDictEqual(self.handler.sessions, {}) def test_get_uuid(self): data = { From 76a2b0081418e3261c476804d2a91a9c5d699f0f Mon Sep 17 00:00:00 2001 From: afeena Date: Sun, 1 Mar 2020 20:18:49 +0100 Subject: [PATCH 07/14] use format instead of if --- tanner/sessions/session_manager.py | 8 +++++--- tanner/tests/test_session_manager.py | 9 +++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tanner/sessions/session_manager.py b/tanner/sessions/session_manager.py index 4a17fd3f..33cbb38a 100644 --- a/tanner/sessions/session_manager.py +++ b/tanner/sessions/session_manager.py @@ -60,10 +60,12 @@ def validate_data(data): def get_session_id(self, data): ip = data['peer']['ip'] - user_agent = data['headers']['user-agent'] if data['headers']['user-agent'] is not None else "" - sess_uuid = data['cookies']['sess_uuid'] if data['cookies']['sess_uuid'] is not None else "" + user_agent = data['headers']['user-agent'] + sess_uuid = data['cookies']['sess_uuid'] - return hashlib.md5((ip + user_agent + sess_uuid).encode()).hexdigest() + sess_id_string = "{ip}{user_agent}{sess_uuid}".format(ip=ip, user_agent=user_agent, sess_uuid=sess_uuid) + + return hashlib.md5(sess_id_string.encode()).hexdigest() async def delete_old_sessions(self, redis_client): for sess_id, session in self.sessions.items(): diff --git a/tanner/tests/test_session_manager.py b/tanner/tests/test_session_manager.py index 8423a680..e58ffd7c 100644 --- a/tanner/tests/test_session_manager.py +++ b/tanner/tests/test_session_manager.py @@ -128,12 +128,13 @@ async def sess_sadd(key, value): } sess = session.Session(data) - data['cookies']['sess_uuid'] = sess.get_uuid() ip = data['peer']['ip'] - user_agent = data['headers']['user-agent'] if data['headers']['user-agent'] is not None else "" - sess_uuid = data['cookies']['sess_uuid'] if data['cookies']['sess_uuid'] is not None else "" + user_agent = data['headers']['user-agent'] + sess_uuid = data['cookies']['sess_uuid'] - sess_id = hashlib.md5((ip + user_agent + sess_uuid).encode()).hexdigest() + sess_id_string = "{ip}{user_agent}{sess_uuid}".format(ip=ip, user_agent=user_agent, sess_uuid=sess_uuid) + + sess_id = hashlib.md5(sess_id_string.encode()).hexdigest() redis_mock = mock.Mock() redis_mock.sadd = sess_sadd From 4eaa43e2daeb74bb29facec808f87286b95ac6eb Mon Sep 17 00:00:00 2001 From: afeena Date: Sun, 1 Mar 2020 20:24:11 +0100 Subject: [PATCH 08/14] do not change dict size during iteration --- tanner/sessions/session_manager.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tanner/sessions/session_manager.py b/tanner/sessions/session_manager.py index 33cbb38a..0fc813bb 100644 --- a/tanner/sessions/session_manager.py +++ b/tanner/sessions/session_manager.py @@ -68,15 +68,19 @@ def get_session_id(self, data): return hashlib.md5(sess_id_string.encode()).hexdigest() async def delete_old_sessions(self, redis_client): + id_for_deletion = [] for sess_id, session in self.sessions.items(): if not session.is_expired(): continue is_deleted = await self.delete_session(session, redis_client) if is_deleted: - try: - del self.sessions[sess_id] - except ValueError: - continue + id_for_deletion.append(sess_id) + + for sess_id in id_for_deletion: + try: + del self.sessions[sess_id] + except ValueError: + continue async def delete_sessions_on_shutdown(self, redis_client): From f44232608847e31bd34c2adf9084448293ff5796 Mon Sep 17 00:00:00 2001 From: afeena Date: Sun, 1 Mar 2020 20:24:23 +0100 Subject: [PATCH 09/14] fix tests --- tanner/tests/test_base.py | 2 +- tanner/tests/test_server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tanner/tests/test_base.py b/tanner/tests/test_base.py index dcdb0f9a..8c1c424a 100644 --- a/tanner/tests/test_base.py +++ b/tanner/tests/test_base.py @@ -182,7 +182,7 @@ def test_set_injectable_page(self): paths = [{'path': '/python.html', 'timestamp': 1465851064.2740946}, {'path': '/python.php/?foo=bar', 'timestamp': 1465851065.2740946}, {'path': '/python.html/?foo=bar', 'timestamp': 1465851065.2740946}] - with mock.patch('tanner.session.Session') as mock_session: + with mock.patch('tanner.sessions.session.Session') as mock_session: mock_session.return_value.paths = paths sess = session.Session(None) injectable_page = self.handler.set_injectable_page(sess) diff --git a/tanner/tests/test_server.py b/tanner/tests/test_server.py index 87368bed..037a5da9 100644 --- a/tanner/tests/test_server.py +++ b/tanner/tests/test_server.py @@ -23,7 +23,7 @@ def setUp(self): with mock.patch('tanner.dorks_manager.DorksManager', mock.Mock()): with mock.patch('tanner.emulators.base.BaseHandler', mock.Mock(), create=True): - with mock.patch('tanner.session_manager.SessionManager', mock.Mock(), create=True): + with mock.patch('tanner.sessions.session_manager.SessionManager', mock.Mock(), create=True): self.serv = server.TannerServer() self.test_uuid = uuid.uuid4() From 4c1bfb746648cca173c2200a01ee4f4358dece65 Mon Sep 17 00:00:00 2001 From: afeena Date: Sun, 1 Mar 2020 20:32:37 +0100 Subject: [PATCH 10/14] return id for new session --- tanner/sessions/session_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tanner/sessions/session_manager.py b/tanner/sessions/session_manager.py index 0fc813bb..200be030 100644 --- a/tanner/sessions/session_manager.py +++ b/tanner/sessions/session_manager.py @@ -30,7 +30,7 @@ async def add_or_update_session(self, raw_data, redis_client): self.logger.exception('Error during session creation: %s', key_error) return self.sessions[session_id] = new_session - return new_session + return new_session, session_id else: self.sessions[session_id].update_session(valid_data) # prepare the list of sessions From d1fac3630b6bb98d9913393c1e68ff54540d7b38 Mon Sep 17 00:00:00 2001 From: afeena Date: Sun, 1 Mar 2020 20:39:02 +0100 Subject: [PATCH 11/14] update mock --- tanner/tests/test_server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tanner/tests/test_server.py b/tanner/tests/test_server.py index 037a5da9..ead982d2 100644 --- a/tanner/tests/test_server.py +++ b/tanner/tests/test_server.py @@ -1,5 +1,6 @@ import uuid from unittest import mock +import hashlib from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop @@ -31,9 +32,10 @@ def setUp(self): async def _add_or_update_mock(data, client): sess = mock.Mock() sess.set_attack_type = mock.Mock() + sess_id = hashlib.md5(b"foo") test_uuid = uuid sess.get_uuid = mock.Mock(return_value=str(self.test_uuid)) - return sess + return sess, sess_id async def _delete_sessions_mock(client): pass From 73f0c0900ba477f123f21c8d4c34339e13840f24 Mon Sep 17 00:00:00 2001 From: afeena Date: Sun, 1 Mar 2020 21:43:21 +0100 Subject: [PATCH 12/14] style split long lines get timeout from config style remove whitespaces --- tanner/config.py | 3 ++- tanner/server.py | 2 ++ tanner/sessions/session_manager.py | 4 +--- tanner/tests/test_session_manager.py | 10 ++++++---- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tanner/config.py b/tanner/config.py index a8b459f8..4f4c7cc5 100644 --- a/tanner/config.py +++ b/tanner/config.py @@ -33,7 +33,8 @@ 'LOCALLOG': {'enabled': False, 'PATH': '/tmp/tanner_report.json'}, 'CLEANLOG': {'enabled': False}, 'REMOTE_DOCKERFILE': {'GITHUB': "https://raw.githubusercontent.com/mushorg/tanner/master/docker/" - "tanner/template_injection/Dockerfile"} + "tanner/template_injection/Dockerfile"}, + 'SESSIONS': {"delete_timeout": 300} } diff --git a/tanner/server.py b/tanner/server.py index e64278e8..0c41275d 100644 --- a/tanner/server.py +++ b/tanner/server.py @@ -25,6 +25,8 @@ def __init__(self): db_name = TannerConfig.get('SQLI', 'db_name') self.session_manager = session_manager.SessionManager() + self.delete_timeout = TannerConfig.get('SESSIONS', 'delete_timeout') + self.dorks = dorks_manager.DorksManager() self.base_handler = base.BaseHandler(base_dir, db_name) self.logger = logging.getLogger(__name__) diff --git a/tanner/sessions/session_manager.py b/tanner/sessions/session_manager.py index 200be030..38a5296d 100644 --- a/tanner/sessions/session_manager.py +++ b/tanner/sessions/session_manager.py @@ -10,11 +10,10 @@ class SessionManager: - def __init__(self, loop=None, delete_timeout=60 * 5): + def __init__(self, loop=None): self.sessions = {} self.analyzer = SessionAnalyzer(loop=loop) self.logger = logging.getLogger(__name__) - self.delete_timeout = delete_timeout async def add_or_update_session(self, raw_data, redis_client): @@ -82,7 +81,6 @@ async def delete_old_sessions(self, redis_client): except ValueError: continue - async def delete_sessions_on_shutdown(self, redis_client): for sess_id, sess in self.sessions.items(): is_deleted = await self.delete_session(sess, redis_client) diff --git a/tanner/tests/test_session_manager.py b/tanner/tests/test_session_manager.py index e58ffd7c..91cd819b 100644 --- a/tanner/tests/test_session_manager.py +++ b/tanner/tests/test_session_manager.py @@ -18,7 +18,8 @@ def test_validate_missing_peer(self): data = { 'headers': { 'USER-AGENT': - 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36' + 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/41.0.2228.0 Safari/537.36' }, 'path': '/foo', 'uuid': None, @@ -29,7 +30,8 @@ def test_validate_missing_peer(self): 'peer': {'ip': None, 'port': None}, 'headers': { 'user-agent': - 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36' + 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/41.0.2228.0 Safari/537.36' }, 'path': '/foo', 'uuid': None, @@ -109,7 +111,7 @@ async def sess_sadd(key, value): redis_mock.sadd = sess_sadd sess, sess_id = self.loop.run_until_complete(self.handler.add_or_update_session(data, redis_mock)) - self.assertDictEqual({sess_id : sess}, self.handler.sessions) + self.assertDictEqual({sess_id: sess}, self.handler.sessions) def test_updating_session(self): async def sess_sadd(key, value): @@ -134,7 +136,7 @@ async def sess_sadd(key, value): sess_id_string = "{ip}{user_agent}{sess_uuid}".format(ip=ip, user_agent=user_agent, sess_uuid=sess_uuid) - sess_id = hashlib.md5(sess_id_string.encode()).hexdigest() + sess_id = hashlib.md5(sess_id_string.encode()).hexdigest() redis_mock = mock.Mock() redis_mock.sadd = sess_sadd From 198a03161b7f11467be95d403ffb8b2f51c55499 Mon Sep 17 00:00:00 2001 From: afeena Date: Mon, 2 Mar 2020 09:51:28 +0100 Subject: [PATCH 13/14] handle shutdown --- tanner/server.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tanner/server.py b/tanner/server.py index 0c41275d..6c31f740 100644 --- a/tanner/server.py +++ b/tanner/server.py @@ -103,11 +103,15 @@ async def handle_version(self, request): async def on_shutdown(self, app): await self.session_manager.delete_sessions_on_shutdown(self.redis_client) self.redis_client.close() + await self.redis_client.wait_closed() async def delete_sessions(self): - while True: - await self.session_manager.delete_old_sessions(self.redis_client) - await asyncio.sleep(self.delete_timeout) + try: + while True: + await self.session_manager.delete_old_sessions(self.redis_client) + await asyncio.sleep(self.delete_timeout) + except asyncio.CancelledError: + pass def setup_routes(self, app): app.router.add_route('*', '/', self.default_handler) From c437fcdae2ddac29ca598612e9fff7e5fde8a5e0 Mon Sep 17 00:00:00 2001 From: afeena Date: Mon, 2 Mar 2020 19:14:54 +0100 Subject: [PATCH 14/14] use AsyncMock for redis in test_server --- tanner/tests/test_server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tanner/tests/test_server.py b/tanner/tests/test_server.py index ead982d2..1de74d06 100644 --- a/tanner/tests/test_server.py +++ b/tanner/tests/test_server.py @@ -6,6 +6,7 @@ from tanner import server from tanner.config import TannerConfig +from tanner.utils.asyncmock import AsyncMock from tanner import __version__ as tanner_version @@ -50,8 +51,9 @@ async def choosed(client): dorks.choose_dorks = choosed dorks.extract_path = self._make_coroutine() - redis = mock.Mock() + redis = AsyncMock() redis.close = mock.Mock() + redis.wait_closed = AsyncMock() self.serv.dorks = dorks self.serv.redis_client = redis