-
Notifications
You must be signed in to change notification settings - Fork 277
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #204 from mokaddem/redis-feed-generator
Realtime feed generator
- Loading branch information
Showing
8 changed files
with
683 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,89 @@ | ||
import redis | ||
import json | ||
|
||
|
||
class MISPItemToRedis: | ||
"""This class provides a simple normalization to add MISP item to | ||
redis, so that they can easily be processed and added to MISP later on.""" | ||
SUFFIX_SIGH = '_sighting' | ||
SUFFIX_ATTR = '_attribute' | ||
SUFFIX_OBJ = '_object' | ||
SUFFIX_LIST = [SUFFIX_SIGH, SUFFIX_ATTR, SUFFIX_OBJ] | ||
|
||
def __init__(self, keyname, host='localhost', port=6379, db=0): | ||
self.host = host | ||
self.port = port | ||
self.db = db | ||
self.keyname = keyname | ||
self.serv = redis.StrictRedis(self.host, self.port, self.db) | ||
|
||
def push_json(self, jdata, keyname, action): | ||
all_action = [s.lstrip('_') for s in self.SUFFIX_LIST] | ||
if action not in all_action: | ||
raise('Error: Invalid action. (Allowed: {})'.format(all_action)) | ||
key = keyname + '_' + action | ||
self.serv.lpush(key, jdata) | ||
|
||
def push_attribute(self, type_value, value, category=None, to_ids=False, | ||
comment=None, distribution=None, proposal=False, **kwargs): | ||
to_push = {} | ||
to_push['type'] = type_value | ||
to_push['value'] = value | ||
if category is not None: | ||
to_push['category'] = category | ||
if to_ids is not None: | ||
to_push['to_ids'] = to_ids | ||
if comment is not None: | ||
to_push['comment'] = comment | ||
if distribution is not None: | ||
to_push['distribution'] = distribution | ||
if proposal is not None: | ||
to_push['proposal'] = proposal | ||
for k, v in kwargs.items(): | ||
to_push[k] = v | ||
key = self.keyname + self.SUFFIX_ATTR | ||
self.serv.lpush(key, json.dumps(to_push)) | ||
|
||
def push_attribute_obj(self, MISP_Attribute, keyname): | ||
key = keyname + self.SUFFIX_ATTR | ||
jdata = MISP_Attribute.to_json() | ||
self.serv.lpush(key, jdata) | ||
|
||
def push_object(self, dict_values): | ||
# check that 'name' field is present | ||
if 'name' not in dict_values: | ||
print("Error: JSON must contain the field 'name'") | ||
key = self.keyname + self.SUFFIX_OBJ | ||
self.serv.lpush(key, json.dumps(dict_values)) | ||
|
||
def push_object_obj(self, MISP_Object, keyname): | ||
key = keyname + self.SUFFIX_OBJ | ||
jdata = MISP_Object.to_json() | ||
self.serv.lpush(key, jdata) | ||
|
||
def push_sighting(self, value=None, uuid=None, id=None, source=None, | ||
type=0, timestamp=None, **kargs): | ||
to_push = {} | ||
if value is not None: | ||
to_push['value'] = value | ||
if uuid is not None: | ||
to_push['uuid'] = uuid | ||
if id is not None: | ||
to_push['id'] = id | ||
if source is not None: | ||
to_push['source'] = source | ||
if type is not None: | ||
to_push['type'] = type | ||
if timestamp is not None: | ||
to_push['timestamp'] = timestamp | ||
|
||
for k, v in kargs.items(): | ||
if v is not None: | ||
to_push[k] = v | ||
key = self.keyname + self.SUFFIX_SIGH | ||
self.serv.lpush(key, json.dumps(to_push)) | ||
|
||
def push_sighting_obj(self, MISP_Sighting, keyname): | ||
key = keyname + self.SUFFIX_SIGH | ||
jdata = MISP_Sighting.to_json() | ||
self.serv.lpush(key, jdata) |
32 changes: 32 additions & 0 deletions
32
examples/feed-generator-from-redis/ObjectConstructor/CowrieMISPObject.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
#!/usr/bin/env python3 | ||
|
||
import time | ||
|
||
from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator | ||
|
||
|
||
class CowrieMISPObject(AbstractMISPObjectGenerator): | ||
def __init__(self, dico_val, **kargs): | ||
self._dico_val = dico_val | ||
self.name = "cowrie" | ||
|
||
# Enforce attribute date with timestamp | ||
super(CowrieMISPObject, self).__init__('cowrie', | ||
default_attributes_parameters={'timestamp': int(time.time())}, | ||
**kargs) | ||
self.generate_attributes() | ||
|
||
def generate_attributes(self): | ||
skip_list = ['time', 'duration', 'isError', 'ttylog'] | ||
for object_relation, value in self._dico_val.items(): | ||
if object_relation in skip_list or 'log_' in object_relation: | ||
continue | ||
|
||
if object_relation == 'timestamp': | ||
# Date already in ISO format, removing trailing Z | ||
value = value.rstrip('Z') | ||
|
||
if isinstance(value, dict): | ||
self.add_attribute(object_relation, **value) | ||
else: | ||
self.add_attribute(object_relation, value=value) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,80 @@ | ||
# What | ||
|
||
- ``generator.py`` exposes a class allowing to generate a MISP feed in real time, where each items can be added on daily generated events. | ||
- ``fromredis.py`` uses ``generator.py`` to generate a MISP feed based on data stored in redis. | ||
- ``server.py`` is a simple script using *Flask_autoindex* to serve data to MISP. | ||
- ``MISPItemToRedis.py`` permits to push (in redis) items to be added in MISP by the ``fromredis.py`` script. | ||
|
||
|
||
# Installation | ||
|
||
```` | ||
# Feed generator | ||
git clone https://github.com/CIRCL/PyMISP | ||
cd examples/feed-generator-from-redis | ||
cp settings.default.py settings.py | ||
vi settings.py # adjust your settings | ||
python3 fromredis.py | ||
# Serving file to MISP | ||
bash install.sh | ||
. ./serv-env/bin/activate | ||
python3 server.py | ||
```` | ||
|
||
|
||
# Utilisation | ||
|
||
``` | ||
# Activate virtualenv | ||
. ./serv-env/bin/activate | ||
``` | ||
|
||
### Adding items to MISP | ||
|
||
``` | ||
# create helper object | ||
>>> helper = MISPItemToRedis("redis_list_keyname") | ||
# push an attribute to redis | ||
>>> helper.push_attribute("ip-src", "8.8.8.8", category="Network activity") | ||
# push an object to redis | ||
>>> helper.push_object({ "name": "cowrie", "session": "session_id", "username": "admin", "password": "admin", "protocol": "telnet" }) | ||
# push a sighting to redis | ||
>>> helper.push_sighting(uuid="5a9e9e26-fe40-4726-8563-5585950d210f") | ||
``` | ||
|
||
### Generate the feed | ||
|
||
``` | ||
# Create the FeedGenerator object using the configuration provided in the file settings.py | ||
# It will create daily event in which attributes and object will be added | ||
>>> generator = FeedGenerator() | ||
# Add an attribute to the daily event | ||
>>> attr_type = "ip-src" | ||
>>> attr_value = "8.8.8.8" | ||
>>> additional_data = {} | ||
>>> generator.add_attribute_to_event(attr_type, attr_value, **additional_data) | ||
# Add a cowrie object to the daily event | ||
>>> obj_name = "cowrie" | ||
>>> obj_data = { "session": "session_id", "username": "admin", "password": "admin", "protocol": "telnet" } | ||
>>> generator.add_object_to_event(obj_name, **obj_data) | ||
``` | ||
|
||
### Consume stored data in redis | ||
|
||
``` | ||
# Configuration provided in the file settings.py | ||
>>> python3 fromredis.py | ||
``` | ||
|
||
### Serve data to MISP | ||
|
||
``` | ||
>>> python3 server.py | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,131 @@ | ||
#!/usr/bin/env python | ||
# -*- coding: utf-8 -*- | ||
|
||
import sys | ||
import json | ||
import argparse | ||
import datetime | ||
import time | ||
import redis | ||
|
||
import settings | ||
|
||
from generator import FeedGenerator | ||
|
||
|
||
def beautyful_sleep(sleep, additional): | ||
length = 20 | ||
sleeptime = float(sleep) / float(length) | ||
for i in range(length): | ||
temp_string = '|'*i + ' '*(length-i-1) | ||
print('sleeping [{}]\t{}'.format(temp_string, additional), end='\r', sep='') | ||
sys.stdout.flush() | ||
time.sleep(sleeptime) | ||
|
||
|
||
class RedisToMISPFeed: | ||
SUFFIX_SIGH = '_sighting' | ||
SUFFIX_ATTR = '_attribute' | ||
SUFFIX_OBJ = '_object' | ||
SUFFIX_LIST = [SUFFIX_SIGH, SUFFIX_ATTR, SUFFIX_OBJ] | ||
|
||
def __init__(self): | ||
self.host = settings.host | ||
self.port = settings.port | ||
self.db = settings.db | ||
self.serv = redis.StrictRedis(self.host, self.port, self.db, decode_responses=True) | ||
|
||
self.generator = FeedGenerator() | ||
|
||
self.keynames = [] | ||
for k in settings.keyname_pop: | ||
for s in self.SUFFIX_LIST: | ||
self.keynames.append(k+s) | ||
|
||
self.keynameError = settings.keyname_error | ||
|
||
self.update_last_action("Init system") | ||
|
||
def consume(self): | ||
self.update_last_action("Started consuming redis") | ||
while True: | ||
for key in self.keynames: | ||
while True: | ||
data = self.pop(key) | ||
if data is None: | ||
break | ||
try: | ||
self.perform_action(key, data) | ||
except Exception as error: | ||
self.save_error_to_redis(error, data) | ||
|
||
beautyful_sleep(5, self.format_last_action()) | ||
|
||
def pop(self, key): | ||
popped = self.serv.rpop(key) | ||
if popped is None: | ||
return None | ||
try: | ||
popped = json.loads(popped) | ||
except ValueError as error: | ||
self.save_error_to_redis(error, popped) | ||
except ValueError as error: | ||
self.save_error_to_redis(error, popped) | ||
return popped | ||
|
||
def perform_action(self, key, data): | ||
# sighting | ||
if key.endswith(self.SUFFIX_SIGH): | ||
if self.generator.add_sighting_on_attribute(): | ||
self.update_last_action("Added sighting") | ||
else: | ||
self.update_last_action("Error while adding sighting") | ||
|
||
# attribute | ||
elif key.endswith(self.SUFFIX_ATTR): | ||
attr_type = data.pop('type') | ||
attr_value = data.pop('value') | ||
if self.generator.add_attribute_to_event(attr_type, attr_value, **data): | ||
self.update_last_action("Added attribute") | ||
else: | ||
self.update_last_action("Error while adding attribute") | ||
|
||
# object | ||
elif key.endswith(self.SUFFIX_OBJ): | ||
# create the MISP object | ||
obj_name = data.pop('name') | ||
if self.generator.add_object_to_event(obj_name, **data): | ||
self.update_last_action("Added object") | ||
else: | ||
self.update_last_action("Error while adding object") | ||
|
||
else: | ||
# Suffix not valid | ||
self.update_last_action("Redis key suffix not supported") | ||
|
||
# OTHERS | ||
def update_last_action(self, action): | ||
self.last_action = action | ||
self.last_action_time = datetime.datetime.now() | ||
|
||
def format_last_action(self): | ||
return "Last action: [{}] @ {}".format( | ||
self.last_action, | ||
self.last_action_time.isoformat().replace('T', ' '), | ||
) | ||
|
||
|
||
def save_error_to_redis(self, error, item): | ||
to_push = {'error': str(error), 'item': str(item)} | ||
print('Error:', str(error), '\nOn adding:', item) | ||
self.serv.lpush(self.keynameError, to_push) | ||
|
||
|
||
if __name__ == '__main__': | ||
parser = argparse.ArgumentParser(description="Pop item fom redis and add " | ||
+ "it to the MISP feed. By default, each action are pushed into a " | ||
+ "daily named event. Configuration taken from the file settings.py.") | ||
args = parser.parse_args() | ||
|
||
redisToMISP = RedisToMISPFeed() | ||
redisToMISP.consume() |
Oops, something went wrong.