From 1e591a827dfc02977df7546a51135de57d7e45ba Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 2 Nov 2017 21:29:25 -0400 Subject: [PATCH 01/11] ABC for DataSink, DataStore and DataSource. Fixes across the concrete objects --- stix2/sources/__init__.py | 139 ++++++++++++++++-------------------- stix2/sources/filesystem.py | 103 ++++++++++++++++++++++---- stix2/sources/memory.py | 131 ++++++++++++++++++++++++--------- stix2/sources/taxii.py | 101 ++++++++++++++++++++++---- 4 files changed, 336 insertions(+), 138 deletions(-) diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 1fe93911..231b7775 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -11,8 +11,11 @@ | """ +from abc import ABCMeta, abstractmethod import uuid +from six import with_metaclass + from stix2.utils import deduplicate @@ -20,95 +23,94 @@ def make_id(): return str(uuid.uuid4()) -class DataStore(object): +class DataStore(with_metaclass(ABCMeta)): """An implementer will create a concrete subclass from this class for the specific DataStore. Args: source (DataSource): An existing DataSource to use as this DataStore's DataSource component - sink (DataSink): An existing DataSink to use as this DataStore's DataSink component Attributes: id (str): A unique UUIDv4 to identify this DataStore. - source (DataSource): An object that implements DataSource class. - sink (DataSink): An object that implements DataSink class. """ def __init__(self, source=None, sink=None): + super(DataStore, self).__init__() self.id = make_id() self.source = source self.sink = sink - def get(self, stix_id, allow_custom=False): + @abstractmethod + def get(self, stix_id): # pragma: no cover """Retrieve the most recent version of a single STIX object by ID. Translate get() call to the appropriate DataSource call. Args: stix_id (str): the id of the STIX object to retrieve. - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: stix_obj: the single most recent version of the STIX object specified by the "id". """ - return self.source.get(stix_id, allow_custom=allow_custom) + return NotImplementedError() - def all_versions(self, stix_id, allow_custom=False): + @abstractmethod + def all_versions(self, stix_id): # pragma: no cover """Retrieve all versions of a single STIX object by ID. - Implement: Translate all_versions() call to the appropriate DataSource call + Implement: Define a function that performs any custom behavior before + calling the associated DataSource all_versions() method. Args: stix_id (str): the id of the STIX object to retrieve. - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: stix_objs (list): a list of STIX objects """ - return self.source.all_versions(stix_id, allow_custom=allow_custom) + return NotImplementedError() - def query(self, query=None, allow_custom=False): + @abstractmethod + def query(self, query=None): # pragma: no cover """Retrieve STIX objects matching a set of filters. Implement: Specific data source API calls, processing, functionality required for retrieving query from the data source. + Define custom behavior before calling the associated DataSource query() + Args: query (list): a list of filters (which collectively are the query) to conduct search on. - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: stix_objs (list): a list of STIX objects """ - return self.source.query(query=query) + return NotImplementedError() - def add(self, stix_objs, allow_custom=False): - """Store STIX objects. + @abstractmethod + def add(self, stix_objs): # pragma: no cover + """Method for storing STIX objects. - Translates add() to the appropriate DataSink call. + Define custom behavior before storing STIX objects using the associated + DataSink. Translates add() to the appropriate DataSink call. Args: stix_objs (list): a list of STIX objects - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. + """ - return self.sink.add(stix_objs, allow_custom=allow_custom) + return NotImplementedError() -class DataSink(object): +class DataSink(with_metaclass(ABCMeta)): """An implementer will create a concrete subclass from this class for the specific DataSink. @@ -117,10 +119,12 @@ class DataSink(object): """ def __init__(self): + super(DataSink, self).__init__() self.id = make_id() - def add(self, stix_objs, allow_custom=False): - """Store STIX objects. + @abstractmethod + def add(self, stix_objs): # pragma: no cover + """Method for storing STIX objects. Implement: Specific data sink API calls, processing, functionality required for adding data to the sink @@ -128,28 +132,27 @@ def add(self, stix_objs, allow_custom=False): Args: stix_objs (list): a list of STIX objects (where each object is a STIX object) - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. """ raise NotImplementedError() -class DataSource(object): +class DataSource(with_metaclass(ABCMeta)): """An implementer will create a concrete subclass from this class for the specific DataSource. Attributes: id (str): A unique UUIDv4 to identify this DataSource. - - _filters (set): A collection of filters attached to this DataSource. + filters (set): A collection of filters attached to this DataSource. """ def __init__(self): + super(DataSource, self).__init__() self.id = make_id() self.filters = set() - def get(self, stix_id, _composite_filters=None, allow_custom=False): + @abstractmethod + def get(self, stix_id): # pragma: no cover """ Implement: Specific data source API calls, processing, functionality required for retrieving data from the data source @@ -158,10 +161,6 @@ def get(self, stix_id, _composite_filters=None, allow_custom=False): stix_id (str): the id of the STIX 2.0 object to retrieve. Should return a single object, the most recent version of the object specified by the "id". - _composite_filters (set): set of filters passed from the parent - the CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: stix_obj: the STIX object @@ -169,10 +168,11 @@ def get(self, stix_id, _composite_filters=None, allow_custom=False): """ raise NotImplementedError() - def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): + @abstractmethod + def all_versions(self, stix_id): # pragma: no cover """ - Implement: Similar to get() except returns list of all object versions of - the specified "id". In addition, implement the specific data + Implement: Similar to get() except returns list of all object versions + of the specified "id". In addition, implement the specific data source API calls, processing, functionality required for retrieving data from the data source. @@ -180,10 +180,6 @@ def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): stix_id (str): The id of the STIX 2.0 object to retrieve. Should return a list of objects, all the versions of the object specified by the "id". - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: stix_objs (list): a list of STIX objects @@ -191,18 +187,15 @@ def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): """ raise NotImplementedError() - def query(self, query=None, _composite_filters=None, allow_custom=False): + @abstractmethod + def query(self, query=None): # pragma: no cover """ - Implement:Implement the specific data source API calls, processing, + Implement: The specific data source API calls, processing, functionality required for retrieving query from the data source Args: query (list): a list of filters (which collectively are the query) - to conduct search on - _composite_filters (set): a set of filters passed from the parent - CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. + to conduct search on. Returns: stix_objs (list): a list of STIX objects @@ -224,7 +217,7 @@ class CompositeDataSource(DataSource): Attributes: - data_sources (dict): A dictionary of DataSource objects; to be + data_sources (list): A dictionary of DataSource objects; to be controlled and used by the Data Source Controller object. """ @@ -237,7 +230,7 @@ def __init__(self): super(CompositeDataSource, self).__init__() self.data_sources = [] - def get(self, stix_id, _composite_filters=None, allow_custom=False): + def get(self, stix_id, _composite_filters=None): """Retrieve STIX object by STIX ID Federated retrieve method, iterates through all DataSources @@ -253,9 +246,7 @@ def get(self, stix_id, _composite_filters=None, allow_custom=False): stix_id (str): the id of the STIX object to retrieve. _composite_filters (list): a list of filters passed from a CompositeDataSource (i.e. if this CompositeDataSource is attached - to another parent CompositeDataSource), not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. + to another parent CompositeDataSource), not user supplied. Returns: stix_obj: the STIX object to be returned. @@ -273,7 +264,7 @@ def get(self, stix_id, _composite_filters=None, allow_custom=False): # for every configured Data Source, call its retrieve handler for ds in self.data_sources: - data = ds.get(stix_id=stix_id, _composite_filters=all_filters, allow_custom=allow_custom) + data = ds.get(stix_id=stix_id, _composite_filters=all_filters) if data: all_data.append(data) @@ -288,22 +279,20 @@ def get(self, stix_id, _composite_filters=None, allow_custom=False): return stix_obj - def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): - """Retrieve STIX objects by STIX ID + def all_versions(self, stix_id, _composite_filters=None): + """Retrieve all versions of a STIX object by STIX ID. - Federated all_versions retrieve method - iterates through all DataSources - defined in "data_sources" + Federated all_versions retrieve method - iterates through all + DataSources defined in "data_sources". A composite data source will pass its attached filters to - each configured data source, pushing filtering to them to handle + each configured data source, pushing filtering to them to handle. Args: - stix_id (str): id of the STIX objects to retrieve + stix_id (str): id of the STIX objects to retrieve. _composite_filters (list): a list of filters passed from a - CompositeDataSource (i.e. if this CompositeDataSource is attached - to a parent CompositeDataSource), not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. + CompositeDataSource (i.e. if this CompositeDataSource is + attached to a parent CompositeDataSource), not user supplied. Returns: all_data (list): list of STIX objects that have the specified id @@ -322,7 +311,7 @@ def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): # retrieve STIX objects from all configured data sources for ds in self.data_sources: - data = ds.all_versions(stix_id=stix_id, _composite_filters=all_filters, allow_custom=allow_custom) + data = ds.all_versions(stix_id=stix_id, _composite_filters=all_filters) all_data.extend(data) # remove exact duplicates (where duplicates are STIX 2.0 objects @@ -332,19 +321,17 @@ def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): return all_data - def query(self, query=None, _composite_filters=None, allow_custom=False): - """Retrieve STIX objects that match query + def query(self, query=None, _composite_filters=None): + """Retrieve STIX objects that match a query. Federate the query to all DataSources attached to the Composite Data Source. Args: - query (list): list of filters to search on + query (list): list of filters to search on. _composite_filters (list): a list of filters passed from a - CompositeDataSource (i.e. if this CompositeDataSource is attached - to a parent CompositeDataSource), not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. + CompositeDataSource (i.e. if this CompositeDataSource is + attached to a parent CompositeDataSource), not user supplied. Returns: all_data (list): list of STIX objects to be returned @@ -354,7 +341,7 @@ def query(self, query=None, _composite_filters=None, allow_custom=False): raise AttributeError('CompositeDataSource has no data sources') if not query: - # dont mess with the query (i.e. convert to a set, as thats done + # don't mess with the query (i.e. convert to a set, as that's done # within the specific DataSources that are called) query = [] @@ -369,7 +356,7 @@ def query(self, query=None, _composite_filters=None, allow_custom=False): # federate query to all attached data sources, # pass composite filters to id for ds in self.data_sources: - data = ds.query(query=query, _composite_filters=all_filters, allow_custom=allow_custom) + data = ds.query(query=query, _composite_filters=all_filters) all_data.extend(data) # remove exact duplicates (where duplicates are STIX 2.0 diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index eb83d8c8..4287cbf1 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -26,7 +26,7 @@ class FileSystemStore(DataStore): Default: False. Attributes: - source (FileSystemSource): FuleSystemSource + source (FileSystemSource): FileSystemSource sink (FileSystemSink): FileSystemSink """ @@ -35,6 +35,85 @@ def __init__(self, stix_dir, bundlify=False): self.source = FileSystemSource(stix_dir=stix_dir) self.sink = FileSystemSink(stix_dir=stix_dir, bundlify=bundlify) + def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None): + """Retrieve the most recent version of a single STIX object by ID. + + Translate get() call to the appropriate DataSource call. + + Args: + stix_id (str): the id of the STIX object to retrieve. + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. + + Returns: + stix_obj: the single most recent version of the STIX + object specified by the "id". + + """ + return self.source.get(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) + + def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None): + """Retrieve all versions of a single STIX object by ID. + + Implement: Translate all_versions() call to the appropriate DataSource + call. + + Args: + stix_id (str): the id of the STIX object to retrieve. + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. + + Returns: + stix_objs (list): a list of STIX objects + + """ + return self.source.all_versions(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) + + def query(self, query=None, allow_custom=False, version=None, _composite_filters=None): + """Retrieve STIX objects matching a set of filters. + + Implement: Specific data source API calls, processing, + functionality required for retrieving query from the data source. + + Args: + query (list): a list of filters (which collectively are the query) + to conduct search on. + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. + + Returns: + stix_objs (list): a list of STIX objects + + """ + return self.source.query(query=query, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) + + def add(self, stix_objs, allow_custom=False, version=None): + """Store STIX objects. + + Translates add() to the appropriate DataSink call. + + Args: + stix_objs (list): a list of STIX objects + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. + + """ + return self.sink.add(stix_objs, allow_custom=allow_custom, version=version) + class FileSystemSink(DataSink): """Interface for adding/pushing STIX objects to file directory of STIX @@ -99,11 +178,11 @@ def add(self, stix_data=None, allow_custom=False, version=None): self._check_path_and_write(stix_data) elif isinstance(stix_data, (str, dict)): - stix_data = parse(stix_data, allow_custom, version) + stix_data = parse(stix_data, allow_custom=allow_custom, version=version) if stix_data["type"] == "bundle": # extract STIX objects for stix_obj in stix_data.get("objects", []): - self.add(stix_obj) + self.add(stix_obj, allow_custom=allow_custom, version=version) else: # adding json-formatted STIX self._check_path_and_write(stix_data) @@ -111,12 +190,12 @@ def add(self, stix_data=None, allow_custom=False, version=None): elif isinstance(stix_data, Bundle): # recursively add individual STIX objects for stix_obj in stix_data.get("objects", []): - self.add(stix_obj) + self.add(stix_obj, allow_custom=allow_custom, version=version) elif isinstance(stix_data, list): # recursively add individual STIX objects for stix_obj in stix_data: - self.add(stix_obj) + self.add(stix_obj, allow_custom=allow_custom, version=version) else: raise TypeError("stix_data must be a STIX object (or list of), " @@ -146,7 +225,7 @@ def __init__(self, stix_dir): def stix_dir(self): return self._stix_dir - def get(self, stix_id, _composite_filters=None, allow_custom=False, version=None): + def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None): """Retrieve STIX object from file directory via STIX ID. Args: @@ -166,8 +245,7 @@ def get(self, stix_id, _composite_filters=None, allow_custom=False, version=None """ query = [Filter("id", "=", stix_id)] - all_data = self.query(query=query, _composite_filters=_composite_filters, - allow_custom=allow_custom, version=version) + all_data = self.query(query=query, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) if all_data: stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] @@ -176,7 +254,7 @@ def get(self, stix_id, _composite_filters=None, allow_custom=False, version=None return stix_obj - def all_versions(self, stix_id, _composite_filters=None, allow_custom=False, version=None): + def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None): """Retrieve STIX object from file directory via STIX ID, all versions. Note: Since FileSystem sources/sinks don't handle multiple versions @@ -197,10 +275,9 @@ def all_versions(self, stix_id, _composite_filters=None, allow_custom=False, ver a python STIX objects and then returned """ - return [self.get(stix_id=stix_id, _composite_filters=_composite_filters, - allow_custom=allow_custom, version=version)] + return [self.get(stix_id=stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters)] - def query(self, query=None, _composite_filters=None, allow_custom=False, version=None): + def query(self, query=None, allow_custom=False, version=None, _composite_filters=None): """Search and retrieve STIX objects based on the complete query. A "complete query" includes the filters from the query, the filters @@ -305,7 +382,7 @@ def query(self, query=None, _composite_filters=None, allow_custom=False, version all_data = deduplicate(all_data) # parse python STIX objects from the STIX object dicts - stix_objs = [parse(stix_obj_dict, allow_custom, version) for stix_obj_dict in all_data] + stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom, version=version) for stix_obj_dict in all_data] return stix_objs diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 2d1705dc..0179c451 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -24,7 +24,7 @@ from stix2.sources.filters import Filter, apply_common_filters -def _add(store, stix_data=None, allow_custom=False): +def _add(store, stix_data=None, allow_custom=False, version=None): """Add STIX objects to MemoryStore/Sink. Adds STIX objects to an in-memory dictionary for fast lookup. @@ -34,6 +34,8 @@ def _add(store, stix_data=None, allow_custom=False): stix_data (list OR dict OR STIX object): STIX objects to be added allow_custom (bool): whether to allow custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. """ if isinstance(stix_data, _STIXBase): @@ -44,25 +46,25 @@ def _add(store, stix_data=None, allow_custom=False): if stix_data["type"] == "bundle": # adding a json bundle - so just grab STIX objects for stix_obj in stix_data.get("objects", []): - _add(store, stix_obj, allow_custom=allow_custom) + _add(store, stix_obj, allow_custom=allow_custom, version=version) else: # adding a json STIX object store._data[stix_data["id"]] = stix_data elif isinstance(stix_data, str): # adding json encoded string of STIX content - stix_data = parse(stix_data, allow_custom=allow_custom) + stix_data = parse(stix_data, allow_custom=allow_custom, version=version) if stix_data["type"] == "bundle": # recurse on each STIX object in bundle for stix_obj in stix_data.get("objects", []): - _add(store, stix_obj, allow_custom=allow_custom) + _add(store, stix_obj, allow_custom=allow_custom, version=version) else: - _add(store, stix_data) + _add(store, stix_data, allow_custom=allow_custom, version=version) elif isinstance(stix_data, list): # STIX objects are in a list- recurse on each object for stix_obj in stix_data: - _add(store, stix_obj, allow_custom=allow_custom) + _add(store, stix_obj, allow_custom=allow_custom, version=version) else: raise TypeError("stix_data must be a STIX object (or list of), JSON formatted STIX (or list of), or a JSON formatted STIX bundle") @@ -81,6 +83,8 @@ class MemoryStore(DataStore): stix_data (list OR dict OR STIX object): STIX content to be added allow_custom (bool): whether to allow custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Attributes: _data (dict): the in-memory dict that holds STIX objects @@ -88,15 +92,15 @@ class MemoryStore(DataStore): sink (MemorySink): MemorySink """ - def __init__(self, stix_data=None, allow_custom=False): + def __init__(self, stix_data=None, allow_custom=False, version=None): super(MemoryStore, self).__init__() self._data = {} if stix_data: - _add(self, stix_data, allow_custom=allow_custom) + _add(self, stix_data, allow_custom=allow_custom, version=version) - self.source = MemorySource(stix_data=self._data, _store=True, allow_custom=allow_custom) - self.sink = MemorySink(stix_data=self._data, _store=True, allow_custom=allow_custom) + self.source = MemorySource(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True) + self.sink = MemorySink(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True) def save_to_file(self, file_path, allow_custom=False): """Write SITX objects from in-memory dictionary to JSON file, as a STIX @@ -110,7 +114,7 @@ def save_to_file(self, file_path, allow_custom=False): """ return self.sink.save_to_file(file_path=file_path, allow_custom=allow_custom) - def load_from_file(self, file_path, allow_custom=False): + def load_from_file(self, file_path, allow_custom=False, version=None): """Load STIX data from JSON file. File format is expected to be a single JSON @@ -120,9 +124,72 @@ def load_from_file(self, file_path, allow_custom=False): file_path (str): file path to load STIX data from allow_custom (bool): whether to allow custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. """ - return self.source.load_from_file(file_path=file_path, allow_custom=allow_custom) + return self.source.load_from_file(file_path=file_path, allow_custom=allow_custom, version=version) + + def get(self, stix_id, _composite_filters=None): + """Retrieve the most recent version of a single STIX object by ID. + + Translate get() call to the appropriate DataSource call. + + Args: + stix_id (str): the id of the STIX object to retrieve. + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied + + Returns: + stix_obj: the single most recent version of the STIX + object specified by the "id". + + """ + return self.source.get(stix_id, _composite_filters=_composite_filters) + + def all_versions(self, stix_id, _composite_filters=None): + """Retrieve all versions of a single STIX object by ID. + + Translate all_versions() call to the appropriate DataSource call. + + Args: + stix_id (str): the id of the STIX object to retrieve. + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied + + Returns: + stix_objs (list): a list of STIX objects + + """ + return self.source.all_versions(stix_id, _composite_filters=_composite_filters) + + def query(self, query=None, _composite_filters=None): + """Retrieve STIX objects matching a set of filters. + + Translates query() to appropriate DataStore call. + + Args: + query (list): a list of filters (which collectively are the query) + to conduct search on. + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied + + Returns: + stix_objs (list): a list of STIX objects + + """ + return self.source.query(query=query, _composite_filters=_composite_filters) + + def add(self, stix_objs, allow_custom=False, version=None): + """Store STIX objects. + + Translates add() to the appropriate DataSink call. + + Args: + stix_objs (list): a list of STIX objects + + """ + return self.sink.add(stix_objs, allow_custom=allow_custom, version=version) class MemorySink(DataSink): @@ -146,17 +213,17 @@ class MemorySink(DataSink): a MemorySource """ - def __init__(self, stix_data=None, _store=False, allow_custom=False): + def __init__(self, stix_data=None, allow_custom=False, version=None, _store=False): super(MemorySink, self).__init__() self._data = {} if _store: self._data = stix_data elif stix_data: - _add(self, stix_data, allow_custom=allow_custom) + _add(self, stix_data, allow_custom=allow_custom, version=version) - def add(self, stix_data, allow_custom=False): - _add(self, stix_data, allow_custom=allow_custom) + def add(self, stix_data, allow_custom=False, version=None): + _add(self, stix_data, allow_custom=allow_custom, version=version) add.__doc__ = _add.__doc__ def save_to_file(self, file_path, allow_custom=False): @@ -190,24 +257,22 @@ class MemorySource(DataSource): a MemorySink """ - def __init__(self, stix_data=None, _store=False, allow_custom=False): + def __init__(self, stix_data=None, allow_custom=False, version=None, _store=False): super(MemorySource, self).__init__() self._data = {} if _store: self._data = stix_data elif stix_data: - _add(self, stix_data, allow_custom=allow_custom) + _add(self, stix_data, allow_custom=allow_custom, version=version) - def get(self, stix_id, _composite_filters=None, allow_custom=False): + def get(self, stix_id, _composite_filters=None): """Retrieve STIX object from in-memory dict via STIX ID. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. - composite_filters (set): set of filters passed from the parent + _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: (dict OR STIX object): STIX object that has the supplied @@ -227,7 +292,7 @@ def get(self, stix_id, _composite_filters=None, allow_custom=False): # if there are filters from the composite level, process full query query = [Filter("id", "=", stix_id)] - all_data = self.query(query=query, _composite_filters=_composite_filters, allow_custom=allow_custom) + all_data = self.query(query=query, _composite_filters=_composite_filters) if all_data: # reduce to most recent version @@ -237,7 +302,7 @@ def get(self, stix_id, _composite_filters=None, allow_custom=False): else: return None - def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): + def all_versions(self, stix_id, _composite_filters=None): """Retrieve STIX objects from in-memory dict via STIX ID, all versions of it Note: Since Memory sources/sinks don't handle multiple versions of a @@ -245,10 +310,8 @@ def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): Args: stix_id (str): The STIX ID of the STIX 2 object to retrieve. - composite_filters (set): set of filters passed from the parent + _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: (list): list of STIX objects that has the supplied ID. As the @@ -257,9 +320,9 @@ def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): is returned in the same form as it as added """ - return [self.get(stix_id=stix_id, _composite_filters=_composite_filters, allow_custom=allow_custom)] + return [self.get(stix_id=stix_id, _composite_filters=_composite_filters)] - def query(self, query=None, _composite_filters=None, allow_custom=False): + def query(self, query=None, _composite_filters=None): """Search and retrieve STIX objects based on the complete query. A "complete query" includes the filters from the query, the filters @@ -268,10 +331,8 @@ def query(self, query=None, _composite_filters=None, allow_custom=False): Args: query (list): list of filters to search on - composite_filters (set): set of filters passed from the + _composite_filters (set): set of filters passed from the CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. Returns: (list): list of STIX objects that matches the supplied @@ -284,7 +345,7 @@ def query(self, query=None, _composite_filters=None, allow_custom=False): query = set() else: if not isinstance(query, list): - # make sure dont make set from a Filter object, + # make sure don't make set from a Filter object, # need to make a set from a list of Filter objects (even if just one Filter) query = [query] query = set(query) @@ -300,8 +361,8 @@ def query(self, query=None, _composite_filters=None, allow_custom=False): return all_data - def load_from_file(self, file_path, allow_custom=False): + def load_from_file(self, file_path, allow_custom=False, version=None): file_path = os.path.abspath(file_path) stix_data = json.load(open(file_path, "r")) - _add(self, stix_data, allow_custom=allow_custom) + _add(self, stix_data, allow_custom=allow_custom, version=version) load_from_file.__doc__ = MemoryStore.load_from_file.__doc__ diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 414e27fb..3fd5eff2 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -1,5 +1,5 @@ """ -Python STIX 2.x TaxiiCollectionStore +Python STIX 2.x TAXIICollectionStore """ from stix2.base import _STIXBase @@ -24,6 +24,71 @@ def __init__(self, collection): self.source = TAXIICollectionSource(collection) self.sink = TAXIICollectionSink(collection) + def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None): + """Retrieve the most recent version of a single STIX object by ID. + + Translate get() call to the appropriate DataSource call. + + Args: + stix_id (str): the id of the STIX object to retrieve. + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. + + Returns: + stix_obj: the single most recent version of the STIX + object specified by the "id". + + """ + return self.source.get(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) + + def all_versions(self, stix_id): + """Retrieve all versions of a single STIX object by ID. + + Translate all_versions() to the appropriate DataSource call. + + Args: + stix_id (str): the id of the STIX object to retrieve. + + Returns: + stix_objs (list): a list of STIX objects + + """ + return self.source.all_versions(stix_id) + + def query(self, query=None): + """Retrieve STIX objects matching a set of filters. + + Translate query() to the appropriate DataSource call. + + Args: + query (list): a list of filters (which collectively are the query) + to conduct search on. + + Returns: + stix_objs (list): a list of STIX objects + + """ + return self.source.query(query=query) + + def add(self, stix_objs, allow_custom=False, version=None): + """Store STIX objects. + + Translate add() to the appropriate DataSink call. + + Args: + stix_objs (list): a list of STIX objects + allow_custom (bool): whether to allow custom objects/properties or + not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. + + """ + return self.sink.add(stix_objs, allow_custom=allow_custom, version=version) + class TAXIICollectionSink(DataSink): """Provides an interface for pushing STIX objects to a local/remote @@ -37,7 +102,7 @@ def __init__(self, collection): super(TAXIICollectionSink, self).__init__() self.collection = collection - def add(self, stix_data, allow_custom=False): + def add(self, stix_data, allow_custom=False, version=None): """Add/push STIX content to TAXII Collection endpoint Args: @@ -46,6 +111,8 @@ def add(self, stix_data, allow_custom=False): json encoded string, or list of any of the following allow_custom (bool): whether to allow custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. """ if isinstance(stix_data, _STIXBase): @@ -62,11 +129,11 @@ def add(self, stix_data, allow_custom=False): elif isinstance(stix_data, list): # adding list of something - recurse on each for obj in stix_data: - self.add(obj, allow_custom=allow_custom) + self.add(obj, allow_custom=allow_custom, version=version) elif isinstance(stix_data, str): # adding json encoded string of STIX content - stix_data = parse(stix_data, allow_custom=allow_custom) + stix_data = parse(stix_data, allow_custom=allow_custom, version=version) if stix_data["type"] == "bundle": bundle = dict(stix_data) else: @@ -90,16 +157,18 @@ def __init__(self, collection): super(TAXIICollectionSource, self).__init__() self.collection = collection - def get(self, stix_id, _composite_filters=None, allow_custom=False): + def get(self, stix_id, _composite_filters=None, allow_custom=False, version=None): """Retrieve STIX object from local/remote STIX Collection endpoint. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. - composite_filters (set): set of filters passed from the parent + _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied allow_custom (bool): whether to retrieve custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: (STIX object): STIX object that has the supplied STIX ID. @@ -121,7 +190,7 @@ def get(self, stix_id, _composite_filters=None, allow_custom=False): stix_obj = list(apply_common_filters(stix_objs, query)) if len(stix_obj): - stix_obj = parse(stix_obj[0], allow_custom=allow_custom) + stix_obj = parse(stix_obj[0], allow_custom=allow_custom, version=version) if stix_obj.id != stix_id: # check - was added to handle erroneous TAXII servers stix_obj = None @@ -130,16 +199,18 @@ def get(self, stix_id, _composite_filters=None, allow_custom=False): return stix_obj - def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): + def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None): """Retrieve STIX object from local/remote TAXII Collection endpoint, all versions of it Args: stix_id (str): The STIX ID of the STIX objects to be retrieved. - composite_filters (set): set of filters passed from the parent + _composite_filters (set): set of filters passed from the parent CompositeDataSource, not user supplied allow_custom (bool): whether to retrieve custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: (see query() as all_versions() is just a wrapper) @@ -154,14 +225,14 @@ def all_versions(self, stix_id, _composite_filters=None, allow_custom=False): all_data = self.query(query=query, _composite_filters=_composite_filters, allow_custom=allow_custom) # parse STIX objects from TAXII returned json - all_data = [parse(stix_obj) for stix_obj in all_data] + all_data = [parse(stix_obj, allow_custom=allow_custom, version=version) for stix_obj in all_data] # check - was added to handle erroneous TAXII servers all_data_clean = [stix_obj for stix_obj in all_data if stix_obj.id == stix_id] return all_data_clean - def query(self, query=None, _composite_filters=None, allow_custom=False): + def query(self, query=None, allow_custom=False, version=None, _composite_filters=None): """Search and retreive STIX objects based on the complete query A "complete query" includes the filters from the query, the filters @@ -170,10 +241,12 @@ def query(self, query=None, _composite_filters=None, allow_custom=False): Args: query (list): list of filters to search on - composite_filters (set): set of filters passed from the + _composite_filters (set): set of filters passed from the CompositeDataSource, not user supplied allow_custom (bool): whether to retrieve custom objects/properties or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: (list): list of STIX objects that matches the supplied @@ -200,7 +273,7 @@ def query(self, query=None, _composite_filters=None, allow_custom=False): taxii_filters = self._parse_taxii_filters(query) # query TAXII collection - all_data = self.collection.get_objects(filters=taxii_filters, allow_custom=allow_custom)["objects"] + all_data = self.collection.get_objects(filters=taxii_filters)["objects"] # deduplicate data (before filtering as reduces wasted filtering) all_data = deduplicate(all_data) @@ -209,7 +282,7 @@ def query(self, query=None, _composite_filters=None, allow_custom=False): all_data = list(apply_common_filters(all_data, query)) # parse python STIX objects from the STIX object dicts - stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom) for stix_obj_dict in all_data] + stix_objs = [parse(stix_obj_dict, allow_custom=allow_custom, version=version) for stix_obj_dict in all_data] return stix_objs From cd6c7982afc5c430ca7302535904d26279b396b7 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 2 Nov 2017 21:31:56 -0400 Subject: [PATCH 02/11] Update tests related to Datastores --- stix2/test/test_data_sources.py | 58 ++++++++++++++------------------- stix2/test/test_filesystem.py | 6 ++-- stix2/test/test_memory.py | 6 ++-- 3 files changed, 31 insertions(+), 39 deletions(-) diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 3327ca96..d798a0c3 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -1,7 +1,7 @@ import pytest from taxii2client import Collection -from stix2 import Filter, MemorySource +from stix2 import Filter, MemorySink, MemorySource from stix2.sources import (CompositeDataSource, DataSink, DataSource, DataStore, make_id, taxii) from stix2.sources.filters import apply_common_filters @@ -20,11 +20,6 @@ def collection(): return Collection(COLLECTION_URL, MockTAXIIClient()) -@pytest.fixture -def ds(): - return DataSource() - - IND1 = { "created": "2017-01-27T13:49:53.935Z", "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", @@ -127,21 +122,17 @@ def ds(): def test_ds_abstract_class_smoke(): - ds1 = DataSource() - ds2 = DataSink() - ds3 = DataStore(source=ds1, sink=ds2) + with pytest.raises(TypeError): + DataStore() - with pytest.raises(NotImplementedError): - ds3.add(None) + with pytest.raises(TypeError): + DataStore.get() - with pytest.raises(NotImplementedError): - ds3.all_versions("malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111") + with pytest.raises(TypeError): + DataSource() - with pytest.raises(NotImplementedError): - ds3.get("malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111") - - with pytest.raises(NotImplementedError): - ds3.query([Filter("id", "=", "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111")]) + with pytest.raises(TypeError): + DataSink() def test_ds_taxii(collection): @@ -177,7 +168,8 @@ def test_parse_taxii_filters(): assert taxii_filters == expected_params -def test_add_get_remove_filter(ds): +def test_add_get_remove_filter(): + ds = taxii.TAXIICollectionSource(collection) # First 3 filters are valid, remaining properties are erroneous in some way valid_filters = [ @@ -226,7 +218,7 @@ def test_add_get_remove_filter(ds): ds.filters.update(valid_filters) -def test_apply_common_filters(ds): +def test_apply_common_filters(): stix_objs = [ { "created": "2017-01-27T13:49:53.997Z", @@ -374,35 +366,35 @@ def test_apply_common_filters(ds): assert len(resp) == 0 -def test_filters0(ds): +def test_filters0(): # "Return any object modified before 2017-01-28T13:49:53.935Z" resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<", "2017-01-28T13:49:53.935Z")])) assert resp[0]['id'] == STIX_OBJS2[1]['id'] assert len(resp) == 2 -def test_filters1(ds): +def test_filters1(): # "Return any object modified after 2017-01-28T13:49:53.935Z" resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", ">", "2017-01-28T13:49:53.935Z")])) assert resp[0]['id'] == STIX_OBJS2[0]['id'] assert len(resp) == 1 -def test_filters2(ds): +def test_filters2(): # "Return any object modified after or on 2017-01-28T13:49:53.935Z" resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", ">=", "2017-01-27T13:49:53.935Z")])) assert resp[0]['id'] == STIX_OBJS2[0]['id'] assert len(resp) == 3 -def test_filters3(ds): +def test_filters3(): # "Return any object modified before or on 2017-01-28T13:49:53.935Z" resp = list(apply_common_filters(STIX_OBJS2, [Filter("modified", "<=", "2017-01-27T13:49:53.935Z")])) assert resp[0]['id'] == STIX_OBJS2[1]['id'] assert len(resp) == 2 -def test_filters4(ds): +def test_filters4(): # Assert invalid Filter cannot be created with pytest.raises(ValueError) as excinfo: Filter("modified", "?", "2017-01-27T13:49:53.935Z") @@ -410,21 +402,21 @@ def test_filters4(ds): "for specified property: 'modified'") -def test_filters5(ds): +def test_filters5(): # "Return any object whose id is not indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f" resp = list(apply_common_filters(STIX_OBJS2, [Filter("id", "!=", "indicator--d81f86b8-975b-bc0b-775e-810c5ad45a4f")])) assert resp[0]['id'] == STIX_OBJS2[0]['id'] assert len(resp) == 1 -def test_filters6(ds): +def test_filters6(): # Test filtering on non-common property resp = list(apply_common_filters(STIX_OBJS2, [Filter("name", "=", "Malicious site hosting downloader")])) assert resp[0]['id'] == STIX_OBJS2[0]['id'] assert len(resp) == 3 -def test_filters7(ds): +def test_filters7(): # Test filtering on embedded property stix_objects = list(STIX_OBJS2) + [{ "type": "observed-data", @@ -463,7 +455,7 @@ def test_filters7(ds): assert len(resp) == 1 -def test_deduplicate(ds): +def test_deduplicate(): unique = deduplicate(STIX_OBJS1) # Only 3 objects are unique @@ -483,14 +475,14 @@ def test_deduplicate(ds): def test_add_remove_composite_datasource(): cds = CompositeDataSource() - ds1 = DataSource() - ds2 = DataSource() - ds3 = DataSink() + ds1 = MemorySource() + ds2 = MemorySource() + ds3 = MemorySink() with pytest.raises(TypeError) as excinfo: cds.add_data_sources([ds1, ds2, ds1, ds3]) assert str(excinfo.value) == ("DataSource (to be added) is not of type " - "stix2.DataSource. DataSource type is ''") + "stix2.DataSource. DataSource type is ''") cds.add_data_sources([ds1, ds2, ds1]) diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py index 7aaa3f59..16106152 100644 --- a/stix2/test/test_filesystem.py +++ b/stix2/test/test_filesystem.py @@ -340,7 +340,7 @@ def test_filesystem_object_with_custom_property(fs_store): fs_store.add(camp, True) - camp_r = fs_store.get(camp.id, True) + camp_r = fs_store.get(camp.id) assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -354,7 +354,7 @@ def test_filesystem_object_with_custom_property_in_bundle(fs_store): bundle = Bundle(camp, allow_custom=True) fs_store.add(bundle, True) - camp_r = fs_store.get(camp.id, True) + camp_r = fs_store.get(camp.id) assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -369,7 +369,7 @@ class NewObj(): newobj = NewObj(property1='something') fs_store.add(newobj, True) - newobj_r = fs_store.get(newobj.id, True) + newobj_r = fs_store.get(newobj.id) assert newobj_r.id == newobj.id assert newobj_r.property1 == 'something' diff --git a/stix2/test/test_memory.py b/stix2/test/test_memory.py index 0603bf78..fbdcaf70 100644 --- a/stix2/test/test_memory.py +++ b/stix2/test/test_memory.py @@ -235,7 +235,7 @@ def test_memory_store_object_with_custom_property(mem_store): mem_store.add(camp, True) - camp_r = mem_store.get(camp.id, True) + camp_r = mem_store.get(camp.id) assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -249,7 +249,7 @@ def test_memory_store_object_with_custom_property_in_bundle(mem_store): bundle = Bundle(camp, allow_custom=True) mem_store.add(bundle, True) - bundle_r = mem_store.get(bundle.id, True) + bundle_r = mem_store.get(bundle.id) camp_r = bundle_r['objects'][0] assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -265,6 +265,6 @@ class NewObj(): newobj = NewObj(property1='something') mem_store.add(newobj, True) - newobj_r = mem_store.get(newobj.id, True) + newobj_r = mem_store.get(newobj.id) assert newobj_r.id == newobj.id assert newobj_r.property1 == 'something' From 8c2af813b29b68b253075f4aec082379318732ad Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Thu, 2 Nov 2017 21:56:18 -0400 Subject: [PATCH 03/11] Define CustomProperty. Make sure to update _properties dict when allow_custom=True --- stix2/base.py | 16 ++++++++-------- stix2/properties.py | 12 ++++++++++++ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index b0cf6ff7..b26f044d 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -40,14 +40,7 @@ class _STIXBase(collections.Mapping): """Base class for STIX object types""" def object_properties(self): - props = set(self._properties.keys()) - custom_props = list(set(self._inner.keys()) - props) - custom_props.sort() - - all_properties = list(self._properties.keys()) - all_properties.extend(custom_props) # Any custom properties to the bottom - - return all_properties + return list(self._properties.keys()) def _check_property(self, prop_name, prop, kwargs): if prop_name not in kwargs: @@ -109,6 +102,13 @@ def __init__(self, allow_custom=False, **kwargs): extra_kwargs = list(set(kwargs) - set(cls._properties)) if extra_kwargs: raise ExtraPropertiesError(cls, extra_kwargs) + else: + from .properties import CustomProperty + + # The custom properties will get added to the bottom. + # Matched with a CustomProperty. + extra_kwargs = list(set(kwargs) - set(cls._properties)) + self._properties.update([(x, CustomProperty()) for x in extra_kwargs]) # Remove any keyword arguments whose value is None setting_kwargs = {} diff --git a/stix2/properties.py b/stix2/properties.py index ca7f04ce..5b480ac8 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -393,3 +393,15 @@ def clean(self, value): raise ValueError(str(errors[0])) return self.string_type(value) + + +class CustomProperty(Property): + """ + The custom property class can be used as a base to extend further + functionality of a custom property. + + Note: + This class is used internally to signal the use of any custom property + that is parsed by any object or `parse()` method and allow_custom=True. + """ + pass From d6c14139f3e94cce76e5abad0bad8244d00d418f Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 3 Nov 2017 08:02:32 -0400 Subject: [PATCH 04/11] Extend object serialization options for _STIXBase --- stix2/base.py | 42 +++++++++++++++++++++++++------- stix2/test/test_bundle.py | 51 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 82 insertions(+), 11 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index b26f044d..d7c75530 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -153,15 +153,7 @@ def __setattr__(self, name, value): super(_STIXBase, self).__setattr__(name, value) def __str__(self): - properties = self.object_properties() - - def sort_by(element): - return find_property_index(self, properties, element) - - # separators kwarg -> don't include spaces after commas. - return json.dumps(self, indent=4, cls=STIXJSONEncoder, - item_sort_key=sort_by, - separators=(",", ": ")) + return self.serialize(pretty=True) def __repr__(self): props = [(k, self[k]) for k in self.object_properties() if self.get(k)] @@ -185,6 +177,38 @@ def new_version(self, **kwargs): def revoke(self): return _revoke(self) + def serialize(self, pretty=False, **kwargs): + """ + Serialize a STIX object. + + Args: + pretty (bool): If True, output properties following the STIX specs + formatting. This includes indentation. Refer to notes for more + details. + **kwargs: The arguments for a json.dumps() call. + + Returns: + dict: The serialized JSON object. + + Note: + The argument ``pretty=True`` will output the STIX object following + spec order. Using this argument greatly impacts object serialization + performance. If your use case is centered across machine-to-machine + operation it is recommended to set ``pretty=False``. + + When ``pretty=True`` the following key-value pairs will be added or + overridden: indent=4, separators=(",", ": "), item_sort_key=sort_by. + """ + if pretty: + properties = self.object_properties() + + def sort_by(element): + return find_property_index(self, properties, element) + + kwargs.update({'indent': 4, 'separators': (",", ": "), 'item_sort_key': sort_by}) + + return json.dumps(self, cls=STIXJSONEncoder, **kwargs) + class _Observable(_STIXBase): diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index b1cffd01..262b0506 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -1,8 +1,9 @@ +import json + import pytest import stix2 - EXPECTED_BUNDLE = """{ "type": "bundle", "id": "bundle--00000000-0000-0000-0000-000000000004", @@ -41,6 +42,44 @@ ] }""" +EXPECTED_BUNDLE_DICT = { + "type": "bundle", + "id": "bundle--00000000-0000-0000-0000-000000000004", + "spec_version": "2.0", + "objects": [ + { + "type": "indicator", + "id": "indicator--00000000-0000-0000-0000-000000000001", + "created": "2017-01-01T12:34:56.000Z", + "modified": "2017-01-01T12:34:56.000Z", + "pattern": "[file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e']", + "valid_from": "2017-01-01T12:34:56Z", + "labels": [ + "malicious-activity" + ] + }, + { + "type": "malware", + "id": "malware--00000000-0000-0000-0000-000000000002", + "created": "2017-01-01T12:34:56.000Z", + "modified": "2017-01-01T12:34:56.000Z", + "name": "Cryptolocker", + "labels": [ + "ransomware" + ] + }, + { + "type": "relationship", + "id": "relationship--00000000-0000-0000-0000-000000000003", + "created": "2017-01-01T12:34:56.000Z", + "modified": "2017-01-01T12:34:56.000Z", + "relationship_type": "indicates", + "source_ref": "indicator--01234567-89ab-cdef-0123-456789abcdef", + "target_ref": "malware--fedcba98-7654-3210-fedc-ba9876543210" + } + ] +} + def test_empty_bundle(): bundle = stix2.Bundle() @@ -82,10 +121,18 @@ def test_bundle_with_wrong_spec_version(): assert str(excinfo.value) == "Invalid value for Bundle 'spec_version': must equal '2.0'." -def test_create_bundle(indicator, malware, relationship): +def test_create_bundle1(indicator, malware, relationship): bundle = stix2.Bundle(objects=[indicator, malware, relationship]) assert str(bundle) == EXPECTED_BUNDLE + assert bundle.serialize(pretty=True) == EXPECTED_BUNDLE + + +def test_create_bundle2(indicator, malware, relationship): + bundle = stix2.Bundle(objects=[indicator, malware, relationship]) + + print(repr(bundle)) + assert json.loads(bundle.serialize()) == EXPECTED_BUNDLE_DICT def test_create_bundle_with_positional_args(indicator, malware, relationship): From d31b110330fd5b6fa5dc4ef89479975e71101817 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 3 Nov 2017 08:58:47 -0400 Subject: [PATCH 05/11] Update TAXII datastores --- stix2/sources/taxii.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 3fd5eff2..9242ed2f 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -45,21 +45,27 @@ def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None """ return self.source.get(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) - def all_versions(self, stix_id): + def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None): """Retrieve all versions of a single STIX object by ID. Translate all_versions() to the appropriate DataSource call. Args: stix_id (str): the id of the STIX object to retrieve. + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: stix_objs (list): a list of STIX objects """ - return self.source.all_versions(stix_id) + return self.source.all_versions(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) - def query(self, query=None): + def query(self, query=None, allow_custom=False, version=None, _composite_filters=None): """Retrieve STIX objects matching a set of filters. Translate query() to the appropriate DataSource call. @@ -67,12 +73,18 @@ def query(self, query=None): Args: query (list): a list of filters (which collectively are the query) to conduct search on. + _composite_filters (set): set of filters passed from the parent + CompositeDataSource, not user supplied + allow_custom (bool): whether to retrieve custom objects/properties + or not. Default: False. + version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If + None, use latest version. Returns: stix_objs (list): a list of STIX objects """ - return self.source.query(query=query) + return self.source.query(query=query, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) def add(self, stix_objs, allow_custom=False, version=None): """Store STIX objects. @@ -157,7 +169,7 @@ def __init__(self, collection): super(TAXIICollectionSource, self).__init__() self.collection = collection - def get(self, stix_id, _composite_filters=None, allow_custom=False, version=None): + def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None): """Retrieve STIX object from local/remote STIX Collection endpoint. @@ -222,7 +234,7 @@ def all_versions(self, stix_id, allow_custom=False, version=None, _composite_fil Filter("match[version]", "=", "all") ] - all_data = self.query(query=query, _composite_filters=_composite_filters, allow_custom=allow_custom) + all_data = self.query(query=query, allow_custom=allow_custom, _composite_filters=_composite_filters) # parse STIX objects from TAXII returned json all_data = [parse(stix_obj, allow_custom=allow_custom, version=version) for stix_obj in all_data] From 79475586d86ec6741d651db0dff6a556b992064e Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 3 Nov 2017 14:17:36 -0400 Subject: [PATCH 06/11] Revert object_properties() to #85 fix. Update tests accordingly --- stix2/base.py | 16 +++++++-------- stix2/test/test_data_sources.py | 35 +++++++++++++++++++++++++-------- stix2/test/test_filesystem.py | 10 +++++----- stix2/test/test_identity.py | 11 +++++++++++ 4 files changed, 51 insertions(+), 21 deletions(-) diff --git a/stix2/base.py b/stix2/base.py index d7c75530..76b07b8e 100644 --- a/stix2/base.py +++ b/stix2/base.py @@ -40,7 +40,14 @@ class _STIXBase(collections.Mapping): """Base class for STIX object types""" def object_properties(self): - return list(self._properties.keys()) + props = set(self._properties.keys()) + custom_props = list(set(self._inner.keys()) - props) + custom_props.sort() + + all_properties = list(self._properties.keys()) + all_properties.extend(custom_props) # Any custom properties to the bottom + + return all_properties def _check_property(self, prop_name, prop, kwargs): if prop_name not in kwargs: @@ -102,13 +109,6 @@ def __init__(self, allow_custom=False, **kwargs): extra_kwargs = list(set(kwargs) - set(cls._properties)) if extra_kwargs: raise ExtraPropertiesError(cls, extra_kwargs) - else: - from .properties import CustomProperty - - # The custom properties will get added to the bottom. - # Matched with a CustomProperty. - extra_kwargs = list(set(kwargs) - set(cls._properties)) - self._properties.update([(x, CustomProperty()) for x in extra_kwargs]) # Remove any keyword arguments whose value is None setting_kwargs = {} diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index d798a0c3..514ba409 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -498,29 +498,48 @@ def test_composite_datasource_operations(): objects=STIX_OBJS1, spec_version="2.0", type="bundle") - cds = CompositeDataSource() - ds1 = MemorySource(stix_data=BUNDLE1) - ds2 = MemorySource(stix_data=STIX_OBJS2) + cds1 = CompositeDataSource() + ds1_1 = MemorySource(stix_data=BUNDLE1) + ds1_2 = MemorySource(stix_data=STIX_OBJS2) + + cds2 = CompositeDataSource() + ds2_1 = MemorySource(stix_data=BUNDLE1) + ds2_2 = MemorySource(stix_data=STIX_OBJS2) - cds.add_data_sources([ds1, ds2]) + cds1.add_data_sources([ds1_1, ds1_2]) + cds2.add_data_sources([ds2_1, ds2_2]) - indicators = cds.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + indicators = cds1.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") # In STIX_OBJS2 changed the 'modified' property to a later time... assert len(indicators) == 2 - indicator = cds.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + cds1.add_data_sources([cds2]) + + indicator = cds1.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" assert indicator["modified"] == "2017-01-31T13:49:53.935Z" assert indicator["type"] == "indicator" - query = [ + query1 = [ Filter("type", "=", "indicator") ] - results = cds.query(query) + query2 = [ + Filter("valid_from", "=", "2017-01-27T13:49:53.935382Z") + ] + + cds1.filters.update(query2) + + results = cds1.query(query1) # STIX_OBJS2 has indicator with later time, one with different id, one with # original time in STIX_OBJS1 assert len(results) == 3 + + indicator = cds1.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + + assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" + assert indicator["modified"] == "2017-01-31T13:49:53.935Z" + assert indicator["type"] == "indicator" diff --git a/stix2/test/test_filesystem.py b/stix2/test/test_filesystem.py index 16106152..85f69668 100644 --- a/stix2/test/test_filesystem.py +++ b/stix2/test/test_filesystem.py @@ -340,7 +340,7 @@ def test_filesystem_object_with_custom_property(fs_store): fs_store.add(camp, True) - camp_r = fs_store.get(camp.id) + camp_r = fs_store.get(camp.id, allow_custom=True) assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -352,9 +352,9 @@ def test_filesystem_object_with_custom_property_in_bundle(fs_store): allow_custom=True) bundle = Bundle(camp, allow_custom=True) - fs_store.add(bundle, True) + fs_store.add(bundle, allow_custom=True) - camp_r = fs_store.get(camp.id) + camp_r = fs_store.get(camp.id, allow_custom=True) assert camp_r.id == camp.id assert camp_r.x_empire == camp.x_empire @@ -367,9 +367,9 @@ class NewObj(): pass newobj = NewObj(property1='something') - fs_store.add(newobj, True) + fs_store.add(newobj, allow_custom=True) - newobj_r = fs_store.get(newobj.id) + newobj_r = fs_store.get(newobj.id, allow_custom=True) assert newobj_r.id == newobj.id assert newobj_r.property1 == 'something' diff --git a/stix2/test/test_identity.py b/stix2/test/test_identity.py index a9415fe0..8e3dd429 100644 --- a/stix2/test/test_identity.py +++ b/stix2/test/test_identity.py @@ -62,4 +62,15 @@ def test_parse_no_type(): "identity_class": "individual" }""") + +def test_identity_with_custom(): + identity = stix2.Identity( + name="John Smith", + identity_class="individual", + custom_properties={'x_foo': 'bar'} + ) + + assert identity.x_foo == "bar" + assert "x_foo" in identity.object_properties() + # TODO: Add other examples From 6f1ab52aa5a465837151f9a3487bd3c1402f07dc Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 3 Nov 2017 14:22:15 -0400 Subject: [PATCH 07/11] Update Jupyter Notebook documentation --- docs/guide/custom.ipynb | 111 ++++++------------------------------ docs/guide/ts_support.ipynb | 15 +++-- 2 files changed, 30 insertions(+), 96 deletions(-) diff --git a/docs/guide/custom.ipynb b/docs/guide/custom.ipynb index 2254fa8c..48b9ffe0 100644 --- a/docs/guide/custom.ipynb +++ b/docs/guide/custom.ipynb @@ -99,101 +99,28 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, "metadata": {}, "outputs": [ { - "data": { - "text/html": [ - "
{\n",
-       "    "x_foo": "bar",\n",
-       "    "type": "identity",\n",
-       "    "id": "identity--8d7f0697-e589-4e3b-aa57-cae798d2d138",\n",
-       "    "created": "2017-09-26T21:02:19.465Z",\n",
-       "    "modified": "2017-09-26T21:02:19.465Z",\n",
-       "    "name": "John Smith",\n",
-       "    "identity_class": "individual"\n",
-       "}\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"type\": \"identity\",\n", + " \"id\": \"identity--10761df2-93f6-4eb4-9d02-4fccfe5dc91d\",\n", + " \"created\": \"2017-11-03T18:20:48.145Z\",\n", + " \"modified\": \"2017-11-03T18:20:48.145Z\",\n", + " \"name\": \"John Smith\",\n", + " \"identity_class\": \"individual\",\n", + " \"x_foo\": \"bar\"\n", + "}\n" + ] } ], "source": [ + "from stix2 import Identity\n", + "\n", "identity = Identity(name=\"John Smith\",\n", " identity_class=\"individual\",\n", " custom_properties={\n", @@ -923,14 +850,14 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" + "pygments_lexer": "ipython3", + "version": "3.5.2" } }, "nbformat": 4, diff --git a/docs/guide/ts_support.ipynb b/docs/guide/ts_support.ipynb index f98d7b52..0c035484 100644 --- a/docs/guide/ts_support.ipynb +++ b/docs/guide/ts_support.ipynb @@ -114,7 +114,6 @@ "import stix2\n", "\n", "stix2.v20.Indicator()\n", - "\n", "stix2.v21.Indicator()" ] }, @@ -169,9 +168,17 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n \"type\": \"indicator\",\n \"id\": \"indicator--dbcbd659-c927-4f9a-994f-0a2632274394\",\n \"created\": \"2017-09-26T23:33:39.829Z\",\n \"modified\": \"2017-09-26T23:33:39.829Z\",\n \"name\": \"File hash for malware variant\",\n \"pattern\": \"[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']\",\n \"valid_from\": \"2017-09-26T23:33:39.829952Z\",\n \"labels\": [\n \"malicious-activity\"\n ]\n}\n" + ] + } + ], "source": [ "from stix2 import parse\n", "\n", From 0d5859b9062adf62f19c56f40aa958c89fcbeeb6 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Fri, 3 Nov 2017 15:13:00 -0400 Subject: [PATCH 08/11] Test other CompositeDataSource operations --- stix2/test/test_data_sources.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 514ba409..0bbc858a 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -543,3 +543,13 @@ def test_composite_datasource_operations(): assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" assert indicator["modified"] == "2017-01-31T13:49:53.935Z" assert indicator["type"] == "indicator" + + # There is only one indicator with different ID. Since we use the same data + # when deduplicated, only two indicators (one with different modified). + results = cds1.all_versions("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") + assert len(results) == 2 + + # Since we have filters already associated with our CompositeSource providing + # nothing returns the same as cds1.query(query1) (the associated query is query2) + results = cds1.query([]) + assert len(results) == 3 From 19818c85736b01ce0c19df6a793f1f7f50e49f94 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Mon, 6 Nov 2017 14:21:29 -0500 Subject: [PATCH 09/11] Remove stray print() --- stix2/test/test_bundle.py | 1 - 1 file changed, 1 deletion(-) diff --git a/stix2/test/test_bundle.py b/stix2/test/test_bundle.py index 262b0506..8b14172c 100644 --- a/stix2/test/test_bundle.py +++ b/stix2/test/test_bundle.py @@ -131,7 +131,6 @@ def test_create_bundle1(indicator, malware, relationship): def test_create_bundle2(indicator, malware, relationship): bundle = stix2.Bundle(objects=[indicator, malware, relationship]) - print(repr(bundle)) assert json.loads(bundle.serialize()) == EXPECTED_BUNDLE_DICT From da66f101473cf20f5b56957ca18d783d2c5c0d28 Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 8 Nov 2017 13:53:21 -0500 Subject: [PATCH 10/11] Make DataStore a regular class, remove unwanted overrides, update tests. Remove CustomProperty since it is no longer needed --- stix2/properties.py | 12 ----- stix2/sources/__init__.py | 46 +++++++----------- stix2/sources/filesystem.py | 86 ++------------------------------- stix2/sources/memory.py | 68 ++------------------------ stix2/sources/taxii.py | 84 ++------------------------------ stix2/test/test_data_sources.py | 10 +--- 6 files changed, 31 insertions(+), 275 deletions(-) diff --git a/stix2/properties.py b/stix2/properties.py index 5b480ac8..ca7f04ce 100644 --- a/stix2/properties.py +++ b/stix2/properties.py @@ -393,15 +393,3 @@ def clean(self, value): raise ValueError(str(errors[0])) return self.string_type(value) - - -class CustomProperty(Property): - """ - The custom property class can be used as a base to extend further - functionality of a custom property. - - Note: - This class is used internally to signal the use of any custom property - that is parsed by any object or `parse()` method and allow_custom=True. - """ - pass diff --git a/stix2/sources/__init__.py b/stix2/sources/__init__.py index 231b7775..b3e8a298 100644 --- a/stix2/sources/__init__.py +++ b/stix2/sources/__init__.py @@ -23,9 +23,9 @@ def make_id(): return str(uuid.uuid4()) -class DataStore(with_metaclass(ABCMeta)): - """An implementer will create a concrete subclass from - this class for the specific DataStore. +class DataStore(object): + """An implementer can subclass to create custom behavior from + this class for the specific DataStores. Args: source (DataSource): An existing DataSource to use @@ -45,8 +45,7 @@ def __init__(self, source=None, sink=None): self.source = source self.sink = sink - @abstractmethod - def get(self, stix_id): # pragma: no cover + def get(self, *args, **kwargs): """Retrieve the most recent version of a single STIX object by ID. Translate get() call to the appropriate DataSource call. @@ -59,14 +58,12 @@ def get(self, stix_id): # pragma: no cover object specified by the "id". """ - return NotImplementedError() + return self.source.get(*args, **kwargs) - @abstractmethod - def all_versions(self, stix_id): # pragma: no cover + def all_versions(self, *args, **kwargs): """Retrieve all versions of a single STIX object by ID. - Implement: Define a function that performs any custom behavior before - calling the associated DataSource all_versions() method. + Translate all_versions() call to the appropriate DataSource call. Args: stix_id (str): the id of the STIX object to retrieve. @@ -75,16 +72,12 @@ def all_versions(self, stix_id): # pragma: no cover stix_objs (list): a list of STIX objects """ - return NotImplementedError() + return self.source.all_versions(*args, **kwargs) - @abstractmethod - def query(self, query=None): # pragma: no cover + def query(self, *args, **kwargs): """Retrieve STIX objects matching a set of filters. - Implement: Specific data source API calls, processing, - functionality required for retrieving query from the data source. - - Define custom behavior before calling the associated DataSource query() + Translate query() call to the appropriate DataSource call. Args: query (list): a list of filters (which collectively are the query) @@ -94,10 +87,9 @@ def query(self, query=None): # pragma: no cover stix_objs (list): a list of STIX objects """ - return NotImplementedError() + return self.source.query(*args, **kwargs) - @abstractmethod - def add(self, stix_objs): # pragma: no cover + def add(self, *args, **kwargs): """Method for storing STIX objects. Define custom behavior before storing STIX objects using the associated @@ -107,7 +99,7 @@ def add(self, stix_objs): # pragma: no cover stix_objs (list): a list of STIX objects """ - return NotImplementedError() + return self.sink.add(*args, **kwargs) class DataSink(with_metaclass(ABCMeta)): @@ -123,7 +115,7 @@ def __init__(self): self.id = make_id() @abstractmethod - def add(self, stix_objs): # pragma: no cover + def add(self, stix_objs): """Method for storing STIX objects. Implement: Specific data sink API calls, processing, @@ -134,7 +126,6 @@ def add(self, stix_objs): # pragma: no cover STIX object) """ - raise NotImplementedError() class DataSource(with_metaclass(ABCMeta)): @@ -152,7 +143,7 @@ def __init__(self): self.filters = set() @abstractmethod - def get(self, stix_id): # pragma: no cover + def get(self, stix_id): """ Implement: Specific data source API calls, processing, functionality required for retrieving data from the data source @@ -166,10 +157,9 @@ def get(self, stix_id): # pragma: no cover stix_obj: the STIX object """ - raise NotImplementedError() @abstractmethod - def all_versions(self, stix_id): # pragma: no cover + def all_versions(self, stix_id): """ Implement: Similar to get() except returns list of all object versions of the specified "id". In addition, implement the specific data @@ -185,10 +175,9 @@ def all_versions(self, stix_id): # pragma: no cover stix_objs (list): a list of STIX objects """ - raise NotImplementedError() @abstractmethod - def query(self, query=None): # pragma: no cover + def query(self, query=None): """ Implement: The specific data source API calls, processing, functionality required for retrieving query from the data source @@ -201,7 +190,6 @@ def query(self, query=None): # pragma: no cover stix_objs (list): a list of STIX objects """ - raise NotImplementedError() class CompositeDataSource(DataSource): diff --git a/stix2/sources/filesystem.py b/stix2/sources/filesystem.py index 4287cbf1..e92c525b 100644 --- a/stix2/sources/filesystem.py +++ b/stix2/sources/filesystem.py @@ -31,88 +31,10 @@ class FileSystemStore(DataStore): """ def __init__(self, stix_dir, bundlify=False): - super(FileSystemStore, self).__init__() - self.source = FileSystemSource(stix_dir=stix_dir) - self.sink = FileSystemSink(stix_dir=stix_dir, bundlify=bundlify) - - def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None): - """Retrieve the most recent version of a single STIX object by ID. - - Translate get() call to the appropriate DataSource call. - - Args: - stix_id (str): the id of the STIX object to retrieve. - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. - version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If - None, use latest version. - - Returns: - stix_obj: the single most recent version of the STIX - object specified by the "id". - - """ - return self.source.get(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) - - def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None): - """Retrieve all versions of a single STIX object by ID. - - Implement: Translate all_versions() call to the appropriate DataSource - call. - - Args: - stix_id (str): the id of the STIX object to retrieve. - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. - version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If - None, use latest version. - - Returns: - stix_objs (list): a list of STIX objects - - """ - return self.source.all_versions(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) - - def query(self, query=None, allow_custom=False, version=None, _composite_filters=None): - """Retrieve STIX objects matching a set of filters. - - Implement: Specific data source API calls, processing, - functionality required for retrieving query from the data source. - - Args: - query (list): a list of filters (which collectively are the query) - to conduct search on. - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. - version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If - None, use latest version. - - Returns: - stix_objs (list): a list of STIX objects - - """ - return self.source.query(query=query, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) - - def add(self, stix_objs, allow_custom=False, version=None): - """Store STIX objects. - - Translates add() to the appropriate DataSink call. - - Args: - stix_objs (list): a list of STIX objects - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. - version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If - None, use latest version. - - """ - return self.sink.add(stix_objs, allow_custom=allow_custom, version=version) + super(FileSystemStore, self).__init__( + source=FileSystemSource(stix_dir=stix_dir), + sink=FileSystemSink(stix_dir=stix_dir, bundlify=bundlify) + ) class FileSystemSink(DataSink): diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index 5b44edba..a069747f 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -93,14 +93,15 @@ class MemoryStore(DataStore): """ def __init__(self, stix_data=None, allow_custom=False, version=None): - super(MemoryStore, self).__init__() self._data = {} if stix_data: _add(self, stix_data, allow_custom=allow_custom, version=version) - self.source = MemorySource(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True) - self.sink = MemorySink(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True) + super(MemoryStore, self).__init__( + source=MemorySource(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True), + sink=MemorySink(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True) + ) def save_to_file(self, file_path, allow_custom=False): """Write SITX objects from in-memory dictionary to JSON file, as a STIX @@ -130,67 +131,6 @@ def load_from_file(self, file_path, allow_custom=False, version=None): """ return self.source.load_from_file(file_path=file_path, allow_custom=allow_custom, version=version) - def get(self, stix_id, _composite_filters=None): - """Retrieve the most recent version of a single STIX object by ID. - - Translate get() call to the appropriate DataSource call. - - Args: - stix_id (str): the id of the STIX object to retrieve. - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - - Returns: - stix_obj: the single most recent version of the STIX - object specified by the "id". - - """ - return self.source.get(stix_id, _composite_filters=_composite_filters) - - def all_versions(self, stix_id, _composite_filters=None): - """Retrieve all versions of a single STIX object by ID. - - Translate all_versions() call to the appropriate DataSource call. - - Args: - stix_id (str): the id of the STIX object to retrieve. - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - - Returns: - stix_objs (list): a list of STIX objects - - """ - return self.source.all_versions(stix_id, _composite_filters=_composite_filters) - - def query(self, query=None, _composite_filters=None): - """Retrieve STIX objects matching a set of filters. - - Translates query() to appropriate DataStore call. - - Args: - query (list): a list of filters (which collectively are the query) - to conduct search on. - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - - Returns: - stix_objs (list): a list of STIX objects - - """ - return self.source.query(query=query, _composite_filters=_composite_filters) - - def add(self, stix_objs, allow_custom=False, version=None): - """Store STIX objects. - - Translates add() to the appropriate DataSink call. - - Args: - stix_objs (list): a list of STIX objects - - """ - return self.sink.add(stix_objs, allow_custom=allow_custom, version=version) - class MemorySink(DataSink): """Interface for adding/pushing STIX objects to an in-memory dictionary. diff --git a/stix2/sources/taxii.py b/stix2/sources/taxii.py index 9242ed2f..8eb50695 100644 --- a/stix2/sources/taxii.py +++ b/stix2/sources/taxii.py @@ -20,86 +20,10 @@ class TAXIICollectionStore(DataStore): collection (taxii2.Collection): TAXII Collection instance """ def __init__(self, collection): - super(TAXIICollectionStore, self).__init__() - self.source = TAXIICollectionSource(collection) - self.sink = TAXIICollectionSink(collection) - - def get(self, stix_id, allow_custom=False, version=None, _composite_filters=None): - """Retrieve the most recent version of a single STIX object by ID. - - Translate get() call to the appropriate DataSource call. - - Args: - stix_id (str): the id of the STIX object to retrieve. - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. - version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If - None, use latest version. - - Returns: - stix_obj: the single most recent version of the STIX - object specified by the "id". - - """ - return self.source.get(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) - - def all_versions(self, stix_id, allow_custom=False, version=None, _composite_filters=None): - """Retrieve all versions of a single STIX object by ID. - - Translate all_versions() to the appropriate DataSource call. - - Args: - stix_id (str): the id of the STIX object to retrieve. - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. - version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If - None, use latest version. - - Returns: - stix_objs (list): a list of STIX objects - - """ - return self.source.all_versions(stix_id, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) - - def query(self, query=None, allow_custom=False, version=None, _composite_filters=None): - """Retrieve STIX objects matching a set of filters. - - Translate query() to the appropriate DataSource call. - - Args: - query (list): a list of filters (which collectively are the query) - to conduct search on. - _composite_filters (set): set of filters passed from the parent - CompositeDataSource, not user supplied - allow_custom (bool): whether to retrieve custom objects/properties - or not. Default: False. - version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If - None, use latest version. - - Returns: - stix_objs (list): a list of STIX objects - - """ - return self.source.query(query=query, allow_custom=allow_custom, version=version, _composite_filters=_composite_filters) - - def add(self, stix_objs, allow_custom=False, version=None): - """Store STIX objects. - - Translate add() to the appropriate DataSink call. - - Args: - stix_objs (list): a list of STIX objects - allow_custom (bool): whether to allow custom objects/properties or - not. Default: False. - version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If - None, use latest version. - - """ - return self.sink.add(stix_objs, allow_custom=allow_custom, version=version) + super(TAXIICollectionStore, self).__init__( + source=TAXIICollectionSource(collection), + sink=TAXIICollectionSink(collection) + ) class TAXIICollectionSink(DataSink): diff --git a/stix2/test/test_data_sources.py b/stix2/test/test_data_sources.py index 0bbc858a..ef0cf26e 100644 --- a/stix2/test/test_data_sources.py +++ b/stix2/test/test_data_sources.py @@ -2,8 +2,8 @@ from taxii2client import Collection from stix2 import Filter, MemorySink, MemorySource -from stix2.sources import (CompositeDataSource, DataSink, DataSource, - DataStore, make_id, taxii) +from stix2.sources import (CompositeDataSource, DataSink, DataSource, make_id, + taxii) from stix2.sources.filters import apply_common_filters from stix2.utils import deduplicate @@ -122,12 +122,6 @@ def collection(): def test_ds_abstract_class_smoke(): - with pytest.raises(TypeError): - DataStore() - - with pytest.raises(TypeError): - DataStore.get() - with pytest.raises(TypeError): DataSource() From 258ea4b82caf9728da1d79e62c82bc68a3fccb4c Mon Sep 17 00:00:00 2001 From: Emmanuelle Vargas-Gonzalez Date: Wed, 8 Nov 2017 14:01:56 -0500 Subject: [PATCH 11/11] Some cleanup in MemoryStore --- stix2/sources/memory.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stix2/sources/memory.py b/stix2/sources/memory.py index a069747f..308d0d05 100644 --- a/stix2/sources/memory.py +++ b/stix2/sources/memory.py @@ -103,7 +103,7 @@ def __init__(self, stix_data=None, allow_custom=False, version=None): sink=MemorySink(stix_data=self._data, allow_custom=allow_custom, version=version, _store=True) ) - def save_to_file(self, file_path, allow_custom=False): + def save_to_file(self, *args, **kwargs): """Write SITX objects from in-memory dictionary to JSON file, as a STIX Bundle. @@ -113,9 +113,9 @@ def save_to_file(self, file_path, allow_custom=False): not. Default: False. """ - return self.sink.save_to_file(file_path=file_path, allow_custom=allow_custom) + return self.sink.save_to_file(*args, **kwargs) - def load_from_file(self, file_path, allow_custom=False, version=None): + def load_from_file(self, *args, **kwargs): """Load STIX data from JSON file. File format is expected to be a single JSON @@ -129,7 +129,7 @@ def load_from_file(self, file_path, allow_custom=False, version=None): None, use latest version. """ - return self.source.load_from_file(file_path=file_path, allow_custom=allow_custom, version=version) + return self.source.load_from_file(*args, **kwargs) class MemorySink(DataSink):