diff --git a/examples/feed-generator-from-redis/MISPItemToRedis.py b/examples/feed-generator-from-redis/MISPItemToRedis.py new file mode 100644 index 000000000..7c038b4f6 --- /dev/null +++ b/examples/feed-generator-from-redis/MISPItemToRedis.py @@ -0,0 +1,89 @@ +import redis +import json + + +class MISPItemToRedis: + """This class provides a simple normalization to add MISP item to + redis, so that they can easily be processed and added to MISP later on.""" + SUFFIX_SIGH = '_sighting' + SUFFIX_ATTR = '_attribute' + SUFFIX_OBJ = '_object' + SUFFIX_LIST = [SUFFIX_SIGH, SUFFIX_ATTR, SUFFIX_OBJ] + + def __init__(self, keyname, host='localhost', port=6379, db=0): + self.host = host + self.port = port + self.db = db + self.keyname = keyname + self.serv = redis.StrictRedis(self.host, self.port, self.db) + + def push_json(self, jdata, keyname, action): + all_action = [s.lstrip('_') for s in self.SUFFIX_LIST] + if action not in all_action: + raise('Error: Invalid action. (Allowed: {})'.format(all_action)) + key = keyname + '_' + action + self.serv.lpush(key, jdata) + + def push_attribute(self, type_value, value, category=None, to_ids=False, + comment=None, distribution=None, proposal=False, **kwargs): + to_push = {} + to_push['type'] = type_value + to_push['value'] = value + if category is not None: + to_push['category'] = category + if to_ids is not None: + to_push['to_ids'] = to_ids + if comment is not None: + to_push['comment'] = comment + if distribution is not None: + to_push['distribution'] = distribution + if proposal is not None: + to_push['proposal'] = proposal + for k, v in kwargs.items(): + to_push[k] = v + key = self.keyname + self.SUFFIX_ATTR + self.serv.lpush(key, json.dumps(to_push)) + + def push_attribute_obj(self, MISP_Attribute, keyname): + key = keyname + self.SUFFIX_ATTR + jdata = MISP_Attribute.to_json() + self.serv.lpush(key, jdata) + + def push_object(self, dict_values): + # check that 'name' field is present + if 'name' not in dict_values: + print("Error: JSON must contain the field 'name'") + key = self.keyname + self.SUFFIX_OBJ + self.serv.lpush(key, json.dumps(dict_values)) + + def push_object_obj(self, MISP_Object, keyname): + key = keyname + self.SUFFIX_OBJ + jdata = MISP_Object.to_json() + self.serv.lpush(key, jdata) + + def push_sighting(self, value=None, uuid=None, id=None, source=None, + type=0, timestamp=None, **kargs): + to_push = {} + if value is not None: + to_push['value'] = value + if uuid is not None: + to_push['uuid'] = uuid + if id is not None: + to_push['id'] = id + if source is not None: + to_push['source'] = source + if type is not None: + to_push['type'] = type + if timestamp is not None: + to_push['timestamp'] = timestamp + + for k, v in kargs.items(): + if v is not None: + to_push[k] = v + key = self.keyname + self.SUFFIX_SIGH + self.serv.lpush(key, json.dumps(to_push)) + + def push_sighting_obj(self, MISP_Sighting, keyname): + key = keyname + self.SUFFIX_SIGH + jdata = MISP_Sighting.to_json() + self.serv.lpush(key, jdata) diff --git a/examples/feed-generator-from-redis/ObjectConstructor/CowrieMISPObject.py b/examples/feed-generator-from-redis/ObjectConstructor/CowrieMISPObject.py new file mode 100644 index 000000000..6c1a40b3f --- /dev/null +++ b/examples/feed-generator-from-redis/ObjectConstructor/CowrieMISPObject.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +import time + +from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator + + +class CowrieMISPObject(AbstractMISPObjectGenerator): + def __init__(self, dico_val, **kargs): + self._dico_val = dico_val + self.name = "cowrie" + + # Enforce attribute date with timestamp + super(CowrieMISPObject, self).__init__('cowrie', + default_attributes_parameters={'timestamp': int(time.time())}, + **kargs) + self.generate_attributes() + + def generate_attributes(self): + skip_list = ['time', 'duration', 'isError', 'ttylog'] + for object_relation, value in self._dico_val.items(): + if object_relation in skip_list or 'log_' in object_relation: + continue + + if object_relation == 'timestamp': + # Date already in ISO format, removing trailing Z + value = value.rstrip('Z') + + if isinstance(value, dict): + self.add_attribute(object_relation, **value) + else: + self.add_attribute(object_relation, value=value) diff --git a/examples/feed-generator-from-redis/README.md b/examples/feed-generator-from-redis/README.md new file mode 100644 index 000000000..4ccd91f45 --- /dev/null +++ b/examples/feed-generator-from-redis/README.md @@ -0,0 +1,80 @@ +# What + +- ``generator.py`` exposes a class allowing to generate a MISP feed in real time, where each items can be added on daily generated events. +- ``fromredis.py`` uses ``generator.py`` to generate a MISP feed based on data stored in redis. +- ``server.py`` is a simple script using *Flask_autoindex* to serve data to MISP. +- ``MISPItemToRedis.py`` permits to push (in redis) items to be added in MISP by the ``fromredis.py`` script. + + +# Installation + +```` +# Feed generator +git clone https://github.com/CIRCL/PyMISP +cd examples/feed-generator-from-redis +cp settings.default.py settings.py +vi settings.py # adjust your settings + +python3 fromredis.py + +# Serving file to MISP +bash install.sh +. ./serv-env/bin/activate +python3 server.py +```` + + +# Utilisation + +``` +# Activate virtualenv +. ./serv-env/bin/activate +``` + +### Adding items to MISP + +``` +# create helper object +>>> helper = MISPItemToRedis("redis_list_keyname") + +# push an attribute to redis +>>> helper.push_attribute("ip-src", "8.8.8.8", category="Network activity") + +# push an object to redis +>>> helper.push_object({ "name": "cowrie", "session": "session_id", "username": "admin", "password": "admin", "protocol": "telnet" }) + +# push a sighting to redis +>>> helper.push_sighting(uuid="5a9e9e26-fe40-4726-8563-5585950d210f") +``` + +### Generate the feed + +``` +# Create the FeedGenerator object using the configuration provided in the file settings.py +# It will create daily event in which attributes and object will be added +>>> generator = FeedGenerator() + +# Add an attribute to the daily event +>>> attr_type = "ip-src" +>>> attr_value = "8.8.8.8" +>>> additional_data = {} +>>> generator.add_attribute_to_event(attr_type, attr_value, **additional_data) + +# Add a cowrie object to the daily event +>>> obj_name = "cowrie" +>>> obj_data = { "session": "session_id", "username": "admin", "password": "admin", "protocol": "telnet" } +>>> generator.add_object_to_event(obj_name, **obj_data) +``` + +### Consume stored data in redis + +``` +# Configuration provided in the file settings.py +>>> python3 fromredis.py +``` + +### Serve data to MISP + +``` +>>> python3 server.py +``` diff --git a/examples/feed-generator-from-redis/fromredis.py b/examples/feed-generator-from-redis/fromredis.py new file mode 100755 index 000000000..e8d85841d --- /dev/null +++ b/examples/feed-generator-from-redis/fromredis.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import json +import argparse +import datetime +import time +import redis + +import settings + +from generator import FeedGenerator + + +def beautyful_sleep(sleep, additional): + length = 20 + sleeptime = float(sleep) / float(length) + for i in range(length): + temp_string = '|'*i + ' '*(length-i-1) + print('sleeping [{}]\t{}'.format(temp_string, additional), end='\r', sep='') + sys.stdout.flush() + time.sleep(sleeptime) + + +class RedisToMISPFeed: + SUFFIX_SIGH = '_sighting' + SUFFIX_ATTR = '_attribute' + SUFFIX_OBJ = '_object' + SUFFIX_LIST = [SUFFIX_SIGH, SUFFIX_ATTR, SUFFIX_OBJ] + + def __init__(self): + self.host = settings.host + self.port = settings.port + self.db = settings.db + self.serv = redis.StrictRedis(self.host, self.port, self.db, decode_responses=True) + + self.generator = FeedGenerator() + + self.keynames = [] + for k in settings.keyname_pop: + for s in self.SUFFIX_LIST: + self.keynames.append(k+s) + + self.keynameError = settings.keyname_error + + self.update_last_action("Init system") + + def consume(self): + self.update_last_action("Started consuming redis") + while True: + for key in self.keynames: + while True: + data = self.pop(key) + if data is None: + break + try: + self.perform_action(key, data) + except Exception as error: + self.save_error_to_redis(error, data) + + beautyful_sleep(5, self.format_last_action()) + + def pop(self, key): + popped = self.serv.rpop(key) + if popped is None: + return None + try: + popped = json.loads(popped) + except ValueError as error: + self.save_error_to_redis(error, popped) + except ValueError as error: + self.save_error_to_redis(error, popped) + return popped + + def perform_action(self, key, data): + # sighting + if key.endswith(self.SUFFIX_SIGH): + if self.generator.add_sighting_on_attribute(): + self.update_last_action("Added sighting") + else: + self.update_last_action("Error while adding sighting") + + # attribute + elif key.endswith(self.SUFFIX_ATTR): + attr_type = data.pop('type') + attr_value = data.pop('value') + if self.generator.add_attribute_to_event(attr_type, attr_value, **data): + self.update_last_action("Added attribute") + else: + self.update_last_action("Error while adding attribute") + + # object + elif key.endswith(self.SUFFIX_OBJ): + # create the MISP object + obj_name = data.pop('name') + if self.generator.add_object_to_event(obj_name, **data): + self.update_last_action("Added object") + else: + self.update_last_action("Error while adding object") + + else: + # Suffix not valid + self.update_last_action("Redis key suffix not supported") + + # OTHERS + def update_last_action(self, action): + self.last_action = action + self.last_action_time = datetime.datetime.now() + + def format_last_action(self): + return "Last action: [{}] @ {}".format( + self.last_action, + self.last_action_time.isoformat().replace('T', ' '), + ) + + + def save_error_to_redis(self, error, item): + to_push = {'error': str(error), 'item': str(item)} + print('Error:', str(error), '\nOn adding:', item) + self.serv.lpush(self.keynameError, to_push) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Pop item fom redis and add " + + "it to the MISP feed. By default, each action are pushed into a " + + "daily named event. Configuration taken from the file settings.py.") + args = parser.parse_args() + + redisToMISP = RedisToMISPFeed() + redisToMISP.consume() diff --git a/examples/feed-generator-from-redis/generator.py b/examples/feed-generator-from-redis/generator.py new file mode 100755 index 000000000..ed8874323 --- /dev/null +++ b/examples/feed-generator-from-redis/generator.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python + +import sys +import json +import os +import hashlib +import datetime +import time +import uuid + +from pymisp import MISPEvent + +import settings + + +def get_system_templates(): + """Fetch all MISP-Object template present on the local system. + + Returns: + dict: A dictionary listing all MISP-Object templates + + """ + misp_objects_path = os.path.join( + os.path.abspath(os.path.dirname(sys.modules['pymisp'].__file__)), + 'data', 'misp-objects', 'objects') + + templates = {} + for root, dirs, files in os.walk(misp_objects_path, topdown=False): + for def_file in files: + obj_name = root.split('/')[-1] + template_path = os.path.join(root, def_file) + with open(template_path, 'r') as f: + definition = json.load(f) + templates[obj_name] = definition + return templates + + +def gen_uuid(): + """Generate a random UUID and returns its string representation""" + return str(uuid.uuid4()) + + +class FeedGenerator: + """Helper object to create MISP feed. + + Configuration taken from the file settings.py""" + + def __init__(self): + """This object can be use to easily create a daily MISP-feed. + + It handles the event creation, manifest file and cache file + (hashes.csv). + + """ + self.sys_templates = get_system_templates() + self.constructor_dict = settings.constructor_dict + + self.flushing_interval = settings.flushing_interval + self.flushing_next = time.time() + self.flushing_interval + + self.manifest = {} + self.attributeHashes = [] + + self.daily_event_name = settings.daily_event_name + ' {}' + event_date_str, self.current_event_uuid, self.event_name = self.get_last_event_from_manifest() + temp = [int(x) for x in event_date_str.split('-')] + self.current_event_date = datetime.date(temp[0], temp[1], temp[2]) + self.current_event = self._get_event_from_id(self.current_event_uuid) + + def add_sighting_on_attribute(self, sight_type, attr_uuid, **data): + """Add a sighting on an attribute. + + Not supported for the moment.""" + self.update_daily_event_id() + self._after_addition() + return False + + def add_attribute_to_event(self, attr_type, attr_value, **attr_data): + """Add an attribute to the daily event""" + self.update_daily_event_id() + self.current_event.add_attribute(attr_type, attr_value, **attr_data) + self._add_hash(attr_type, attr_value) + self._after_addition() + return True + + def add_object_to_event(self, obj_name, **data): + """Add an object to the daily event""" + self.update_daily_event_id() + if obj_name not in self.sys_templates: + print('Unkown object template') + return False + + # Get MISP object constructor + obj_constr = self.constructor_dict.get(obj_name, None) + # Constructor not known, using the generic one + if obj_constr is None: + obj_constr = self.constructor_dict.get('generic') + misp_object = obj_constr(obj_name) + # Fill generic object + for k, v in data.items(): + # attribute is not in the object template definition + if k not in self.sys_templates[obj_name]['attributes']: + # add it with type text + misp_object.add_attribute(k, **{'value': v, 'type': 'text'}) + else: + misp_object.add_attribute(k, **{'value': v}) + + else: + misp_object = obj_constr(data) + + self.current_event.add_object(misp_object) + for attr_type, attr_value in data.items(): + self._add_hash(attr_type, attr_value) + + self._after_addition() + return True + + def _after_addition(self): + """Write event on disk""" + now = time.time() + if self.flushing_next <= now: + self.flush_event() + self.flushing_next = now + self.flushing_interval + + # Cache + def _add_hash(self, attr_type, attr_value): + if ('|' in attr_type or attr_type == 'malware-sample'): + split = attr_value.split('|') + self.attributeHashes.append([ + hashlib.md5(str(split[0]).encode("utf-8")).hexdigest(), + self.current_event_uuid + ]) + self.attributeHashes.append([ + hashlib.md5(str(split[1]).encode("utf-8")).hexdigest(), + self.current_event_uuid + ]) + else: + self.attributeHashes.append([ + hashlib.md5(str(attr_value).encode("utf-8")).hexdigest(), + self.current_event_uuid + ]) + + # Manifest + def _init_manifest(self): + # create an empty manifest + with open(os.path.join(settings.outputdir, 'manifest.json'), 'w'): + pass + # create new event and save manifest + self.create_daily_event() + + def flush_event(self, new_event=None): + print('Writting event on disk'+' '*50) + if new_event is not None: + event_uuid = new_event['uuid'] + event = new_event + else: + event_uuid = self.current_event_uuid + event = self.current_event + + eventFile = open(os.path.join(settings.outputdir, event_uuid+'.json'), 'w') + eventFile.write(event.to_json()) + eventFile.close() + + self.save_hashes() + + def save_manifest(self): + try: + manifestFile = open(os.path.join(settings.outputdir, 'manifest.json'), 'w') + manifestFile.write(json.dumps(self.manifest)) + manifestFile.close() + print('Manifest saved') + except Exception as e: + print(e) + sys.exit('Could not create the manifest file.') + + def save_hashes(self): + if len(self.attributeHashes) == 0: + return False + try: + hashFile = open(os.path.join(settings.outputdir, 'hashes.csv'), 'a') + for element in self.attributeHashes: + hashFile.write('{},{}\n'.format(element[0], element[1])) + hashFile.close() + self.attributeHashes = [] + print('Hash saved' + ' '*30) + except Exception as e: + print(e) + sys.exit('Could not create the quick hash lookup file.') + + def _addEventToManifest(self, event): + event_dict = event.to_dict()['Event'] + tags = [] + for eventTag in event_dict.get('EventTag', []): + tags.append({'name': eventTag['Tag']['name'], + 'colour': eventTag['Tag']['colour']}) + return { + 'Orgc': event_dict.get('Orgc', []), + 'Tag': tags, + 'info': event_dict['info'], + 'date': event_dict['date'], + 'analysis': event_dict['analysis'], + 'threat_level_id': event_dict['threat_level_id'], + 'timestamp': event_dict.get('timestamp', int(time.time())) + } + + def get_last_event_from_manifest(self): + """Retreive last event from the manifest. + + If the manifest doesn't exists or if it is empty, initialize it. + + """ + try: + manifest_path = os.path.join(settings.outputdir, 'manifest.json') + with open(manifest_path, 'r') as f: + man = json.load(f) + dated_events = [] + for event_uuid, event_json in man.items(): + # add events to manifest + self.manifest[event_uuid] = event_json + dated_events.append([ + event_json['date'], + event_uuid, + event_json['info'] + ]) + # Sort by date then by event name + dated_events.sort(key=lambda k: (k[0], k[2]), reverse=True) + return dated_events[0] + except FileNotFoundError as e: + print('Manifest not found, generating a fresh one') + self._init_manifest() + return self.get_last_event_from_manifest() + + # DAILY + def update_daily_event_id(self): + if self.current_event_date != datetime.date.today(): # create new event + # save current event on disk + self.flush_event() + self.current_event = self.create_daily_event() + self.current_event_date = datetime.date.today() + self.current_event_uuid = self.current_event.get('uuid') + self.event_name = self.current_event.info + + def _get_event_from_id(self, event_uuid): + with open(os.path.join(settings.outputdir, '%s.json' % event_uuid), 'r') as f: + event_dict = json.load(f)['Event'] + event = MISPEvent() + event.from_dict(**event_dict) + return event + + def create_daily_event(self): + new_uuid = gen_uuid() + today = str(datetime.date.today()) + event_dict = { + 'uuid': new_uuid, + 'id': len(self.manifest)+1, + 'Tag': settings.Tag, + 'info': self.daily_event_name.format(today), + 'analysis': settings.analysis, # [0-2] + 'threat_level_id': settings.threat_level_id, # [1-4] + 'published': settings.published, + 'date': today + } + event = MISPEvent() + event.from_dict(**event_dict) + + # reference org + org_dict = {} + org_dict['name'] = settings.org_name + org_dict['uui'] = settings.org_uuid + event['Orgc'] = org_dict + + # save event on disk + self.flush_event(new_event=event) + # add event to manifest + self.manifest[event['uuid']] = self._addEventToManifest(event) + self.save_manifest() + return event diff --git a/examples/feed-generator-from-redis/install.sh b/examples/feed-generator-from-redis/install.sh new file mode 100644 index 000000000..0e65dcb4f --- /dev/null +++ b/examples/feed-generator-from-redis/install.sh @@ -0,0 +1,4 @@ +#!/bin/bash +virtualenv -p python3 serv-env +. ./serv-env/bin/activate +pip3 install -U flask Flask-AutoIndex redis diff --git a/examples/feed-generator-from-redis/server.py b/examples/feed-generator-from-redis/server.py new file mode 100755 index 000000000..1dd68734a --- /dev/null +++ b/examples/feed-generator-from-redis/server.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 + +import os.path +from flask import Flask +from flask_autoindex import AutoIndex +from settings import outputdir + +app = Flask(__name__) +AutoIndex(app, browse_root=os.path.join(os.path.curdir, outputdir)) + +if __name__ == '__main__': + app.run(host='0.0.0.0') diff --git a/examples/feed-generator-from-redis/settings.default.py b/examples/feed-generator-from-redis/settings.default.py new file mode 100755 index 000000000..0f6457c35 --- /dev/null +++ b/examples/feed-generator-from-redis/settings.default.py @@ -0,0 +1,58 @@ +""" REDIS RELATED """ +# Your redis server +host='127.0.0.1' +port=6379 +db=0 +## The keynames to POP element from +#keyname_pop='misp_feed_generator_key' +keyname_pop=['cowrie'] + +# OTHERS +## How frequent the event should be written on disk +flushing_interval=5*60 +## The redis list keyname in which to put items that generated an error +keyname_error='feed-generation-error' + +""" FEED GENERATOR CONFIGURATION """ + +# The output dir for the feed. This will drop a lot of files, so make +# sure that you use a directory dedicated to the feed +outputdir = 'output' + +# Event meta data +## Required +### The organisation id that generated this feed +org_name='myOrg' +### Your organisation UUID +org_uuid='' +### The daily event name to be used in MISP. +### (e.g. honeypot_1, will produce each day an event of the form honeypot_1 dd-mm-yyyy) +daily_event_name='PyMISP default event name' + +## Optional +analysis=0 +threat_level_id=3 +published=False +Tag=[ + { + "colour": "#ffffff", + "name": "tlp:white" + }, + { + "colour": "#ff00ff", + "name": "my:custom:feed" + } +] + +# MISP Object constructor +from ObjectConstructor.CowrieMISPObject import CowrieMISPObject +from pymisp.tools import GenericObjectGenerator + +constructor_dict = { + 'cowrie': CowrieMISPObject, + 'generic': GenericObjectGenerator +} + +# Others +## Redis pooling time +sleep=60