Merge branch 'master' of github.com:MISP/PyMISP

CHANGELOG.txt

@@ -2,9 +2,80 @@ Changelog
 =========
 
 
+v2.4.119 (2019-12-02)
+---------------------
+
+Changes
+~~~~~~~
+- Bump version. [Raphaël Vinot]
+- Bump dependencies. [Raphaël Vinot]
+
+Fix
+~~~
+- Bump lief to 0.10.1. [Raphaël Vinot]
+- Update tests. [Raphaël Vinot]
+- Raise PyMISPError instead of Exception. [Raphaël Vinot]
+- Rename feed_meta_generator so it clearly fails with python<3.6.
+  [Raphaël Vinot]
+
+
+v2.4.117.3 (2019-11-25)
+-----------------------
+
+New
+~~~
+- Script to generate the metadata of a feed out of a directory. [Raphaël
+  Vinot]
+- Add to_feed export to MISPEvent. [Raphaël Vinot]
+- Validate object templates. [Raphaël Vinot]
+
+  fix https://github.com/MISP/misp-objects/issues/199
+- Test cases for restricted tags. [Raphaël Vinot]
+
+  Fix #483
+- Get Database Schema Diagnostic. [Raphaël Vinot]
+
+  Fix #492
+
+Changes
+~~~~~~~
+- Bump changelog. [Raphaël Vinot]
+- Bump version. [Raphaël Vinot]
+- Bump dependencies. [Raphaël Vinot]
+- Require stable version of lief again. [Raphaël Vinot]
+- Few more improvements on the feed export. [Raphaël Vinot]
+- Bump misp-objects. [Raphaël Vinot]
+- Make the feed generator more generic. [Raphaël Vinot]
+- Use New version of PyMISP in the feed generator. [Raphaël Vinot]
+- Bump misp-object. [Raphaël Vinot]
+- Allow to sort and indent the json output for objects. [Raphaël Vinot]
+- Bump objects. [Raphaël Vinot]
+- Bump dependencies. [Raphaël Vinot]
+- [test] feed test updated as botvrij is now TLS by default. [Alexandre
+  Dulaunoy]
+
+Fix
+~~~
+- Improve stability of feed output. [Raphaël Vinot]
+- Do not unitialize the uuid in MISPEvent. [Raphaël Vinot]
+- Bump url template version in test cases. [Raphaël Vinot]
+- Python 2.7 tests. [Raphaël Vinot]
+- Print the full json blob in debug mode. [Raphaël Vinot]
+
+  Related https://github.com/MISP/PyMISP/issues/462
+
+Other
+~~~~~
+- Cch: Bump misp-objects. [Raphaël Vinot]
+
+
 v2.4.117.2 (2019-10-30)
 -----------------------
 
+Changes
+~~~~~~~
+- Bump changelog. [Raphaël Vinot]
+
 Fix
 ~~~
 - Avoid exception on legacy MISP. [Raphaël Vinot]

examples/feed-generator/generate.py

@@ -4,151 +4,36 @@
 import sys
 import json
 import os
-import hashlib
-from pymisp import PyMISP
+from pymisp import ExpandedPyMISP
 from settings import url, key, ssl, outputdir, filters, valid_attribute_distribution_levels
 
-objectsFields = {
-    'Attribute': {
-        'uuid',
-        'value',
-        'category',
-        'type',
-        'comment',
-        'data',
-        'timestamp',
-        'to_ids',
-        'object_relation',
-        'disable_correlation'
-    },
-    'Event': {
-        'uuid',
-        'info',
-        'threat_level_id',
-        'analysis',
-        'timestamp',
-        'publish_timestamp',
-        'published',
-        'date',
-        'extends_uuid'
-    },
-    'Object': {
-        'name',
-        'meta-category',
-        'description',
-        'template_uuid',
-        'template_version',
-        'uuid',
-        'timestamp',
-        'distribution',
-        'sharing_group_id',
-        'comment'
-    },
-    'ObjectReference': {
-        'uuid',
-        'timestamp',
-        'relationship_type',
-        'comment',
-        'object_uuid',
-        'referenced_uuid'
-    },
-    'Orgc': {
-        'name',
-        'uuid'
-    },
-    'Tag': {
-        'name',
-        'colour',
-        'exportable'
-    }
-}
-
-objectsToSave = {
-    'Orgc': {},
-    'Tag': {},
-    'Attribute': {
-        'Tag': {}
-    },
-    'Object': {
-        'Attribute': {
-            'Tag': {}
-        },
-        'ObjectReference': {}
-    }
-}
-
 valid_attribute_distributions = []
 
-attributeHashes = []
-
 
 def init():
     # If we have an old settings.py file then this variable won't exist
     global valid_attribute_distributions
     try:
-        valid_attribute_distributions = valid_attribute_distribution_levels
+        valid_attribute_distributions = [int(v) for v in valid_attribute_distribution_levels]
     except Exception:
-        valid_attribute_distributions = ['0', '1', '2', '3', '4', '5']
-    return PyMISP(url, key, ssl)
-
-
-def recursiveExtract(container, containerType, leaf, eventUuid):
-    temp = {}
-    if containerType in ['Attribute', 'Object']:
-        if (__blockByDistribution(container)):
-            return False
-    for field in objectsFields[containerType]:
-        if field in container:
-            temp[field] = container[field]
-    if (containerType == 'Attribute'):
-        global attributeHashes
-        if ('|' in container['type'] or container['type'] == 'malware-sample'):
-            split = container['value'].split('|')
-            attributeHashes.append([hashlib.md5(split[0].encode("utf-8")).hexdigest(), eventUuid])
-            attributeHashes.append([hashlib.md5(split[1].encode("utf-8")).hexdigest(), eventUuid])
-        else:
-            attributeHashes.append([hashlib.md5(container['value'].encode("utf-8")).hexdigest(), eventUuid])
-    children = leaf.keys()
-    for childType in children:
-        childContainer = container.get(childType)
-        if (childContainer):
-            if (type(childContainer) is dict):
-                temp[childType] = recursiveExtract(childContainer, childType, leaf[childType], eventUuid)
-            else:
-                temp[childType] = []
-                for element in childContainer:
-                    processed = recursiveExtract(element, childType, leaf[childType], eventUuid)
-                    if (processed):
-                        temp[childType].append(processed)
-    return temp
-
+        valid_attribute_distributions = [0, 1, 2, 3, 4, 5]
+    return ExpandedPyMISP(url, key, ssl)
 
-def saveEvent(misp, uuid):
-    event = misp.get_event(uuid)
-    if not event.get('Event'):
-        print('Error while fetching event: {}'.format(event['message']))
-        sys.exit('Could not create file for event ' + uuid + '.')
-    event['Event'] = recursiveExtract(event['Event'], 'Event', objectsToSave, event['Event']['uuid'])
-    event = json.dumps(event)
-    eventFile = open(os.path.join(outputdir, uuid + '.json'), 'w')
-    eventFile.write(event)
-    eventFile.close()
 
-
-def __blockByDistribution(element):
-    if element['distribution'] not in valid_attribute_distributions:
-        return True
-    return False
+def saveEvent(event):
+    try:
+        with open(os.path.join(outputdir, f'{event["uuid"]}.json'), 'w') as f:
+            json.dump(event, f, indent=2)
+    except Exception as e:
+        print(e)
+        sys.exit('Could not create the event dump.')
 
 
-def saveHashes():
-    if not attributeHashes:
-        return False
+def saveHashes(hashes):
     try:
-        hashFile = open(os.path.join(outputdir, 'hashes.csv'), 'w')
-        for element in attributeHashes:
-            hashFile.write('{},{}\n'.format(element[0], element[1]))
-        hashFile.close()
+        with open(os.path.join(outputdir, 'hashes.csv'), 'w') as hashFile:
+            for element in hashes:
+                hashFile.write('{},{}\n'.format(element[0], element[1]))
     except Exception as e:
         print(e)
         sys.exit('Could not create the quick hash lookup file.')
@@ -164,41 +49,31 @@ def saveManifest(manifest):
         sys.exit('Could not create the manifest file.')
 
 
-def __addEventToManifest(event):
-    tags = []
-    for eventTag in event['EventTag']:
-        tags.append({'name': eventTag['Tag']['name'],
-                     'colour': eventTag['Tag']['colour']})
-    return {'Orgc': event['Orgc'],
-            'Tag': tags,
-            'info': event['info'],
-            'date': event['date'],
-            'analysis': event['analysis'],
-            'threat_level_id': event['threat_level_id'],
-            'timestamp': event['timestamp']
-            }
-
-
 if __name__ == '__main__':
     misp = init()
     try:
-        r = misp.get_index(filters)
-        events = r['response']
-        print(events[0])
+        events = misp.search(metadata=True, limit=200, **filters, pythonify=True)
     except Exception as e:
         print(e)
         sys.exit("Invalid response received from MISP.")
     if len(events) == 0:
         sys.exit("No events returned.")
     manifest = {}
+    hashes = []
     counter = 1
     total = len(events)
     for event in events:
-        saveEvent(misp, event['uuid'])
-        manifest[event['uuid']] = __addEventToManifest(event)
+        e = misp.get_event(event.uuid, pythonify=True)
+        e_feed = e.to_feed(valid_distributions=valid_attribute_distributions, with_meta=True)
+        if not e_feed:
+            print(f'Invalid distribution {e.distribution}, skipping')
+            continue
+        hashes += [[h, e.uuid] for h in e_feed.pop('_hashes')]
+        manifest.update(e_feed.pop('_manifest'))
+        saveEvent(e_feed)
         print("Event " + str(counter) + "/" + str(total) + " exported.")
         counter += 1
     saveManifest(manifest)
     print('Manifest saved.')
-    saveHashes()
+    saveHashes(hashes)
     print('Hashes saved. Feed creation completed.')

examples/generate_meta_feed.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from pymisp.tools import feed_meta_generator
+import argparse
+from pathlib import Path
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Build meta files for feed')
+    parser.add_argument("--feed", required=True, help="Path to directory containing the feed.")
+    args = parser.parse_args()
+
+    feed = Path(args.feed)
+
+    feed_meta_generator(feed)

pymisp/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '2.4.117.1'
+__version__ = '2.4.119'
 import logging
 import warnings
 import sys