Merge 7c42a5f into a6ecd52

Pipfile

@@ -10,6 +10,7 @@ codecov = "*"
 requests-mock = "*"
 pymisp = {editable = true,extras = ["fileobjects", "neo", "openioc", "virustotal", "pdfexport", "docs"],path = "."}
 docutils = "==0.15"
+memory-profiler = "*"
 
 [packages]
 pymisp = {editable = true,extras = ["fileobjects", "openioc", "virustotal", "pdfexport"],path = "."}

examples/generate_file_objects.py

@@ -5,7 +5,7 @@
 import json
 
 try:
-    from pymisp import MISPEncode, AbstractMISP
+    from pymisp import pymisp_json_default, AbstractMISP
     from pymisp.tools import make_binary_objects
 except ImportError:
     pass
@@ -51,7 +51,8 @@ def make_objects(path):
         to_return['objects'].append(fo)
         if fo.ObjectReference:
             to_return['references'] += fo.ObjectReference
-    return json.dumps(to_return, cls=MISPEncode)
+    return json.dumps(to_return, default=pymisp_json_default)
+
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Extract indicators out of binaries and returns MISP objects.')

pymisp/__init__.py

@@ -31,7 +31,7 @@ def warning_2020():
     warning_2020()
     from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse, PyMISPEmptyResponse  # noqa
     from .api import PyMISP  # noqa
-    from .abstract import AbstractMISP, MISPEncode, MISPTag, Distribution, ThreatLevel, Analysis  # noqa
+    from .abstract import AbstractMISP, MISPEncode, pymisp_json_default, MISPTag, Distribution, ThreatLevel, Analysis  # noqa
     from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting, MISPLog, MISPShadowAttribute, MISPWarninglist, MISPTaxonomy, MISPNoticelist, MISPObjectTemplate, MISPSharingGroup, MISPRole, MISPServer, MISPFeed, MISPEventDelegation # noqa
     from .tools import AbstractMISPObjectGenerator  # noqa
     from .tools import Neo4j  # noqa

pymisp/abstract.py

@@ -3,24 +3,40 @@
 
 import sys
 import datetime
-import json
+
+from deprecated import deprecated
 from json import JSONEncoder
+
+try:
+    from rapidjson import load
+    from rapidjson import loads
+    from rapidjson import dumps
+    import rapidjson
+    HAS_RAPIDJSON = True
+except ImportError:
+    from json import load
+    from json import loads
+    from json import dumps
+    import json
+    HAS_RAPIDJSON = False
+
 import logging
 from enum import Enum
 
 from .exceptions import PyMISPInvalidFormat
 
-# Try to import MutableMapping the python 3.3+ way
-try:
-    from collections.abc import MutableMapping
-except Exception:
-    pass
-
 
 logger = logging.getLogger('pymisp')
 
 if sys.version_info < (3, 0):
     from collections import MutableMapping
+    import os
+    import cachetools
+
+    resources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data')
+    misp_objects_path = os.path.join(resources_path, 'misp-objects', 'objects')
+    with open(os.path.join(resources_path, 'describeTypes.json'), 'r') as f:
+        describe_types = load(f)['result']
 
     # This is required because Python 2 is a pain.
     from datetime import tzinfo, timedelta
@@ -37,6 +53,53 @@ def tzname(self, dt):
         def dst(self, dt):
             return timedelta(0)
 
+    class MISPFileCache(object):
+        # cache up to 150 JSON structures in class attribute
+        __file_cache = cachetools.LFUCache(150)
+
+        @classmethod
+        def _load_json(cls, path):
+            # use root class attribute as global cache
+            file_cache = cls.__file_cache
+            # use modified time with path as cache key
+            mtime = os.path.getmtime(path)
+            if path in file_cache:
+                ctime, data = file_cache[path]
+                if ctime == mtime:
+                    return data
+            with open(path, 'rb') as f:
+                if OLD_PY3:
+                    data = loads(f.read().decode())
+                else:
+                    data = load(f)
+            file_cache[path] = (mtime, data)
+            return data
+
+else:
+    from collections.abc import MutableMapping
+    from functools import lru_cache
+    from pathlib import Path
+
+    resources_path = Path(__file__).parent / 'data'
+    misp_objects_path = resources_path / 'misp-objects' / 'objects'
+    with (resources_path / 'describeTypes.json').open('r') as f:
+        describe_types = load(f)['result']
+
+    class MISPFileCache(object):
+        # cache up to 150 JSON structures in class attribute
+
+        @staticmethod
+        @lru_cache(maxsize=150)
+        def _load_json(path):
+            with path.open('rb') as f:
+                data = load(f)
+            return data
+
+if (3, 0) <= sys.version_info < (3, 6):
+    OLD_PY3 = True
+else:
+    OLD_PY3 = False
+
 
 class Distribution(Enum):
     your_organisation_only = 0
@@ -68,8 +131,8 @@ def _int_to_str(d):
     return d
 
 
+@deprecated(reason=" Use method default=pymisp_json_default instead of cls=MISPEncode", version='2.4.117', action='default')
 class MISPEncode(JSONEncoder):
-
     def default(self, obj):
         if isinstance(obj, AbstractMISP):
             return obj.jsonable()
@@ -80,13 +143,37 @@ def default(self, obj):
         return JSONEncoder.default(self, obj)
 
 
-class AbstractMISP(MutableMapping):
+if HAS_RAPIDJSON:
+    def pymisp_json_default(obj):
+        if isinstance(obj, AbstractMISP):
+            return obj.jsonable()
+        elif isinstance(obj, (datetime.datetime, datetime.date)):
+            return obj.isoformat()
+        elif isinstance(obj, Enum):
+            return obj.value
+        return rapidjson.default(obj)
+else:
+    def pymisp_json_default(obj):
+        if isinstance(obj, AbstractMISP):
+            return obj.jsonable()
+        elif isinstance(obj, (datetime.datetime, datetime.date)):
+            return obj.isoformat()
+        elif isinstance(obj, Enum):
+            return obj.value
+        return json.default(obj)
+
+
+class AbstractMISP(MutableMapping, MISPFileCache):
+    __resources_path = resources_path
+    __misp_objects_path = misp_objects_path
+    __describe_types = describe_types
 
     def __init__(self, **kwargs):
         """Abstract class for all the MISP objects"""
         super(AbstractMISP, self).__init__()
         self.__edited = True  # As we create a new object, we assume it is edited
         self.__not_jsonable = []
+        self.__self_defined_describe_types = None
 
         if kwargs.get('force_timestamps') is not None:
             # Ignore the edited objects and keep the timestamps.
@@ -103,16 +190,28 @@ def __init__(self, **kwargs):
             setattr(AbstractMISP, 'tags', property(AbstractMISP.__get_tags, AbstractMISP.__set_tags))
 
     @property
-    def properties(self):
-        """All the class public properties that will be dumped in the dictionary, and the JSON export.
-        Note: all the properties starting with a `_` (private), or listed in __not_jsonable will be skipped.
-        """
-        to_return = []
-        for prop, value in vars(self).items():
-            if prop.startswith('_') or prop in self.__not_jsonable:
-                continue
-            to_return.append(prop)
-        return to_return
+    def describe_types(self):
+        if self.__self_defined_describe_types:
+            return self.__self_defined_describe_types
+        return self.__describe_types
+
+    @describe_types.setter
+    def describe_types(self, describe_types):
+        self.__self_defined_describe_types = describe_types
+
+    @property
+    def resources_path(self):
+        return self.__resources_path
+
+    @property
+    def misp_objects_path(self):
+        return self.__misp_objects_path
+
+    @misp_objects_path.setter
+    def misp_objects_path(self, misp_objects_path):
+        if sys.version_info >= (3, 0) and isinstance(misp_objects_path, str):
+            misp_objects_path = Path(misp_objects_path)
+        self.__misp_objects_path = misp_objects_path
 
     def from_dict(self, **kwargs):
         """Loading all the parameters as class properties, if they aren't `None`.
@@ -137,21 +236,21 @@ def set_not_jsonable(self, *args):
 
     def from_json(self, json_string):
         """Load a JSON string"""
-        self.from_dict(**json.loads(json_string))
+        self.from_dict(**loads(json_string))
 
     def to_dict(self):
-        """Dump the lass to a dictionary.
+        """Dump the class to a dictionary.
         This method automatically removes the timestamp recursively in every object
         that has been edited is order to let MISP update the event accordingly."""
+        is_edited = self.edited
         to_return = {}
-        for attribute in self.properties:
-            val = getattr(self, attribute, None)
+        for attribute, val in self.items():
             if val is None:
                 continue
             elif isinstance(val, list) and len(val) == 0:
                 continue
             if attribute == 'timestamp':
-                if not self.__force_timestamps and self.edited:
+                if not self.__force_timestamps and is_edited:
                     # In order to be accepted by MISP, the timestamp of an object
                     # needs to be either newer, or None.
                     # If the current object is marked as edited, the easiest is to
@@ -167,13 +266,15 @@ def jsonable(self):
         """This method is used by the JSON encoder"""
         return self.to_dict()
 
-    def to_json(self):
+    def to_json(self, sort_keys=False, indent=None):
         """Dump recursively any class of type MISPAbstract to a json string"""
-        return json.dumps(self, cls=MISPEncode, sort_keys=True, indent=2)
+        return dumps(self, default=pymisp_json_default, sort_keys=sort_keys, indent=indent)
 
     def __getitem__(self, key):
         try:
-            return getattr(self, key)
+            if key[0] != '_' and key not in self.__not_jsonable:
+                return self.__dict__[key]
+            raise KeyError
         except AttributeError:
             # Expected by pop and other dict-related methods
             raise KeyError
@@ -185,26 +286,25 @@ def __delitem__(self, key):
         delattr(self, key)
 
     def __iter__(self):
-        return iter(self.to_dict())
+        return iter({k: v for k, v in self.__dict__.items() if not (k[0] == '_' or k in self.__not_jsonable)})
 
     def __len__(self):
-        return len(self.to_dict())
+        return len([k for k in self.__dict__.keys() if not (k[0] == '_' or k in self.__not_jsonable)])
 
     @property
     def edited(self):
         """Recursively check if an object has been edited and update the flag accordingly
         to the parent objects"""
         if self.__edited:
             return self.__edited
-        for p in self.properties:
-            if self.__edited:
-                break
-            val = getattr(self, p)
+        for p, val in self.items():
             if isinstance(val, AbstractMISP) and val.edited:
                 self.__edited = True
+                break
             elif isinstance(val, list) and all(isinstance(a, AbstractMISP) for a in val):
                 if any(a.edited for a in val):
                     self.__edited = True
+                    break
         return self.__edited
 
     @edited.setter
@@ -216,7 +316,9 @@ def edited(self, val):
             raise Exception('edited can only be True or False')
 
     def __setattr__(self, name, value):
-        if name in self.properties:
+        if name[0] != '_' and not self.__edited and name in self.keys():
+            # The private members don't matter
+            # If we already have a key with that name, we're modifying it.
             self.__edited = True
         super(AbstractMISP, self).__setattr__(name, value)