diff --git a/docs/api/utils/nsparser.rst b/docs/api/utils/nsparser.rst index a39c808f..766f7ab1 100644 --- a/docs/api/utils/nsparser.rst +++ b/docs/api/utils/nsparser.rst @@ -3,22 +3,15 @@ .. module:: stix.utils.nsparser -Classes -------- +Functions +--------- -.. autoclass:: NamespaceParser - :show-inheritance: - :members: +.. autofunction:: get_namespaces +.. autofunction:: get_namespace_schemalocation_dict +.. autofunction:: get_xmlns_str +.. autofunction:: get_schemaloc_str Constants --------- -.. autodata:: XML_NAMESPACES - -.. autodata:: STIX_NS_TO_SCHEMALOCATION - -.. autodata:: EXT_NS_TO_SCHEMALOCATION - .. autodata:: DEFAULT_STIX_NS_TO_PREFIX - -.. autodata:: DEFAULT_EXT_TO_PREFIX diff --git a/stix/base.py b/stix/base.py index f9ef7ebc..9d5bc98a 100644 --- a/stix/base.py +++ b/stix/base.py @@ -1,17 +1,17 @@ # Copyright (c) 2015, The MITRE Corporation. All rights reserved. # See LICENSE.txt for complete terms. -# builtin -import json import collections import itertools +import json import StringIO +from mixbox import entities from mixbox import idgen from mixbox.binding_utils import save_encoding from mixbox.cache import Cached +from mixbox.datautils import is_sequence -# internal from . import utils @@ -19,7 +19,7 @@ def _override(*args, **kwargs): raise NotImplementedError() -class Entity(object): +class Entity(entities.Entity): """Base class for all classes in the STIX API.""" _namespace = None _XSI_TYPE = None @@ -145,7 +145,6 @@ def to_xml(self, include_namespaces=True, include_schemalocs=False, """ from .utils import nsparser - parser = nsparser.NamespaceParser() if auto_namespace: ns_info = nsparser.NamespaceInfo() @@ -176,11 +175,11 @@ def to_xml(self, include_namespaces=True, include_schemalocs=False, namespace_def = "" if include_namespaces: - xmlns = parser.get_xmlns_str(ns_info.finalized_namespaces) + xmlns = nsparser.get_xmlns_str(ns_info.finalized_namespaces) namespace_def += ("\n\t" + xmlns) if include_schemalocs and include_namespaces: - schemaloc = parser.get_schemaloc_str(ns_info.finalized_schemalocs) + schemaloc = nsparser.get_schemaloc_str(ns_info.finalized_schemalocs) namespace_def += ("\n\t" + schemaloc) if not pretty: @@ -269,75 +268,15 @@ def find(self, id_): return entity -class EntityList(collections.MutableSequence, Entity): +# TODO: For now, this subclasses mixbox.entities.EntityList and stix.Entity. +# Ultimately we want to remove the latter. +class EntityList(entities.EntityList, Entity): _binding_class = _override _binding_var = None _contained_type = _override _inner_name = None _dict_as_list = False - def __init__(self, *args): - super(EntityList, self).__init__() - self._inner = [] - - if not any(args): - return - - for arg in args: - if utils.is_sequence(arg): - self.extend(arg) - else: - self.append(arg) - - def __nonzero__(self): - return bool(self._inner) - - def __getitem__(self, key): - return self._inner.__getitem__(key) - - def __setitem__(self, key, value): - if not self._is_valid(value): - value = self._fix_value(value) - self._inner.__setitem__(key, value) - - def __delitem__(self, key): - self._inner.__delitem__(key) - - def __len__(self): - return len(self._inner) - - def insert(self, idx, value): - if not value: - return - if not self._is_valid(value): - value = self._fix_value(value) - self._inner.insert(idx, value) - - def _is_valid(self, value): - """Check if this is a valid object to add to the list.""" - # Subclasses can override this function, but if it becomes common, it's - # probably better to use self._contained_type.istypeof(value) - return isinstance(value, self._contained_type) - - def _fix_value(self, value): - """Attempt to coerce value into the correct type. - - Subclasses can override this function. - """ - try: - new_value = self._contained_type(value) - except: - error = "Can't put '{0}' ({1}) into a {2}. Expected a {3} object." - error = error.format( - value, # Input value - type(value), # Type of input value - type(self), # Type of collection - self._contained_type # Expected type of input value - ) - raise ValueError(error) - - return new_value - # The next four functions can be overridden, but otherwise define the # default behavior for EntityList subclasses which define the following # class-level members: @@ -357,9 +296,6 @@ def to_obj(self, return_obj=None, ns_info=None): return return_obj - def to_list(self): - return [h.to_dict() for h in self] - def to_dict(self): if self._dict_as_list: return self.to_list() @@ -392,7 +328,7 @@ def from_obj(cls, obj, return_obj=None, contained_type=None, @classmethod def from_list(cls, list_repr, return_obj=None, contained_type=None): - if not utils.is_sequence(list_repr): + if not is_sequence(list_repr): return None if return_obj is None: @@ -494,7 +430,7 @@ def from_obj(cls, obj_list, contained_type=None): if not contained_type: contained_type = cls._contained_type - if not utils.is_sequence(obj_list): + if not is_sequence(obj_list): obj_list = [obj_list] items = (contained_type.from_obj(x) for x in obj_list) @@ -506,7 +442,7 @@ def from_list(cls, list_repr, contained_type=None): if not list_repr: return None - if isinstance(list_repr, dict) or not utils.is_sequence(list_repr): + if isinstance(list_repr, dict) or not is_sequence(list_repr): list_repr = [list_repr] if not contained_type: @@ -539,7 +475,7 @@ def _initialize_inner(self, *args): return for arg in args: - if utils.is_sequence(arg): + if is_sequence(arg): self.extend(arg) else: self.append(arg) diff --git a/stix/common/information_source.py b/stix/common/information_source.py index 44d9dfef..41acd009 100644 --- a/stix/common/information_source.py +++ b/stix/common/information_source.py @@ -3,15 +3,13 @@ from __future__ import absolute_import -# external +from mixbox.datautils import is_sequence +# import cybox.common -# internal import stix -import stix.utils as utils import stix.bindings.stix_common as stix_common_binding -# relative from . import vocabs, VocabString from .identity import Identity from .structured_text import StructuredTextList @@ -52,7 +50,7 @@ def references(self, value): if not value: return - elif utils.is_sequence(value): + elif is_sequence(value): for v in value: self.add_reference(v) else: diff --git a/stix/common/structured_text.py b/stix/common/structured_text.py index c1257751..e1f738ec 100644 --- a/stix/common/structured_text.py +++ b/stix/common/structured_text.py @@ -1,12 +1,14 @@ # Copyright (c) 2015, The MITRE Corporation. All rights reserved. # See LICENSE.txt for complete terms. -import itertools -import contextlib import collections +import contextlib +import itertools + +from mixbox.datautils import is_sequence +from mixbox.exceptions import ignored import stix -import stix.utils as utils import stix.bindings.stix_common as stix_common_binding #: Default ordinality value for StructuredText. @@ -199,7 +201,7 @@ def _initialize_inner(self, *args): return for arg in args: - if utils.is_sequence(arg): + if is_sequence(arg): self.update(arg) else: self.add(arg) @@ -323,7 +325,7 @@ def add(self, value): value.ordinality = self.next_ordinality # Remove the existing item if there is one. - with utils.ignored(KeyError): + with ignored(KeyError): del self[value.ordinality] self._inner.append(value) diff --git a/stix/core/ttps.py b/stix/core/ttps.py index fbbf74a4..7630c2b1 100644 --- a/stix/core/ttps.py +++ b/stix/core/ttps.py @@ -1,8 +1,9 @@ # Copyright (c) 2015, The MITRE Corporation. All rights reserved. # See LICENSE.txt for complete terms. +from mixbox.datautils import is_sequence + import stix -import stix.utils as utils from stix.ttp import TTP from stix.common.kill_chains import KillChains from stix.bindings import stix_core as core_binding @@ -10,6 +11,7 @@ # deprecation warnings from stix.utils.deprecated import idref_deprecated + class TTPs(stix.EntityList): _binding = core_binding _binding_class = _binding.TTPsType @@ -33,7 +35,7 @@ def ttps(self): def ttps(self, value): self._inner = [] - if utils.is_sequence(value): + if is_sequence(value): self.extend(value) else: self.append(value) diff --git a/stix/exploit_target/vulnerability.py b/stix/exploit_target/vulnerability.py index 89c0563b..9b962174 100644 --- a/stix/exploit_target/vulnerability.py +++ b/stix/exploit_target/vulnerability.py @@ -1,6 +1,8 @@ # Copyright (c) 2015, The MITRE Corporation. All rights reserved. # See LICENSE.txt for complete terms. +from mixbox.datautils import is_sequence + import stix import stix.utils as utils import stix.bindings.exploit_target as exploit_target_binding @@ -198,7 +200,7 @@ def references(self, value): if not value: return - elif utils.is_sequence(value): + elif is_sequence(value): self._references.extend(x for x in value if x) else: self._references.append(value) diff --git a/stix/extensions/identity/ciq_identity_3_0.py b/stix/extensions/identity/ciq_identity_3_0.py index e20e6185..60bd7dc0 100644 --- a/stix/extensions/identity/ciq_identity_3_0.py +++ b/stix/extensions/identity/ciq_identity_3_0.py @@ -3,8 +3,9 @@ import lxml.etree as et +from mixbox.datautils import is_sequence + import stix -import stix.utils as utils import stix.common as common import stix.bindings.extensions.identity.ciq_identity_3_0 as ciq_identity_binding @@ -162,7 +163,7 @@ def addresses(self, value): self._addresses = [] if not value: return - elif utils.is_sequence(value): + elif is_sequence(value): for v in value: self.add_address(v) else: @@ -185,7 +186,7 @@ def languages(self, value): self._languages = [] if not value: return - elif utils.is_sequence(value): + elif is_sequence(value): for v in value: self.add_language(v) else: @@ -221,7 +222,7 @@ def electronic_address_identifiers(self, value): self._electronic_address_identifiers = [] if not value: return - elif utils.is_sequence(value): + elif is_sequence(value): for v in value: self.add_electronic_address_identifier(v) else: @@ -244,7 +245,7 @@ def free_text_lines(self, value): self._free_text_lines = [] if not value: return - elif utils.is_sequence(value): + elif is_sequence(value): for v in value: self.add_free_text_line(v) else: @@ -267,7 +268,7 @@ def contact_numbers(self, value): self._contact_numbers = [] if not value: return - elif utils.is_sequence(value): + elif is_sequence(value): for v in value: self.add_contact_number(v) else: @@ -290,7 +291,7 @@ def nationalities(self, value): self._nationalities = [] if not value: return - elif utils.is_sequence(value): + elif is_sequence(value): for v in value: self.add_nationality(v) else: @@ -544,7 +545,7 @@ def name_elements(self, value): self._name_elements = [] if not value: return - elif utils.is_sequence(value): + elif is_sequence(value): for v in value: self.add_name_element(v) else: @@ -616,7 +617,7 @@ def name_elements(self, value): self._name_elements = [] if not value: return - elif utils.is_sequence(value): + elif is_sequence(value): for v in value: self.add_name_element(v) else: @@ -732,7 +733,7 @@ def address_lines(self, value): if value is None or len(value) == 0: return - elif utils.is_sequence(value): + elif is_sequence(value): self._address_lines = value else: self._address_lines.append(value) @@ -1077,7 +1078,7 @@ def name_elements(self, value): self._name_elements = [] if not value: return - elif utils.is_sequence(value): + elif is_sequence(value): for v in value: self.add_organisation_name_element(v) else: @@ -1100,7 +1101,7 @@ def subdivision_names(self, value): self._subdivision_names = [] if not value: return - elif utils.is_sequence(value): + elif is_sequence(value): for v in value: self.add_subdivision_name(v) else: @@ -1679,7 +1680,7 @@ def contact_number_elements(self, value): self._contact_number_elements = [] if not value: return - elif utils.is_sequence(value): + elif is_sequence(value): for v in value: self.add_contact_number_element(v) else: diff --git a/stix/indicator/indicator.py b/stix/indicator/indicator.py index c9c0dd42..99c264a3 100644 --- a/stix/indicator/indicator.py +++ b/stix/indicator/indicator.py @@ -1,7 +1,8 @@ # Copyright (c) 2015, The MITRE Corporation. All rights reserved. # See LICENSE.txt for complete terms. -# external +from mixbox.datautils import is_sequence + from cybox.core import Observable, ObservableComposition from cybox.common import Time @@ -340,7 +341,7 @@ def alternative_id(self, value): self._alternative_id = [] if not value: return - elif utils.is_sequence(value): + elif is_sequence(value): self._alternative_id.extend(x for x in value if x) else: self._alternative_id.append(value) diff --git a/stix/test/utils/utils_test.py b/stix/test/utils/utils_test.py deleted file mode 100644 index a42e21c1..00000000 --- a/stix/test/utils/utils_test.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) 2015, The MITRE Corporation. All rights reserved. -# See LICENSE.txt for complete terms. - -# stdlib -import unittest - -# internal -from stix import utils - - -class UtilsTests(unittest.TestCase): - def test_is_sequence(self): - self.assertTrue(utils.is_sequence([1,2,3])) - self.assertTrue(utils.is_sequence((1,2,3))) - self.assertTrue(utils.is_sequence(set([1,2,3]))) - self.assertTrue(utils.is_sequence({1:1})) - - # Make sure that strings are not sequences. - self.assertEqual(False, utils.is_sequence("abc")) diff --git a/stix/utils/__init__.py b/stix/utils/__init__.py index 869c030b..742739d9 100644 --- a/stix/utils/__init__.py +++ b/stix/utils/__init__.py @@ -1,38 +1,28 @@ # Copyright (c) 2015, The MITRE Corporation. All rights reserved. # See LICENSE.txt for complete terms. -import contextlib import functools import keyword import lxml.etree +from mixbox import datautils from mixbox.entities import Entity, EntityList import mixbox.xml +from cybox.common import ObjectProperties +from cybox.core import Object, Observable + import stix # relative from . import dates - CDATA_START = "" CONFLICTING_NAMES = keyword.kwlist + ['id', 'type', 'range'] -@contextlib.contextmanager -def ignored(*exceptions): - """Allows you to ignore exceptions cleanly using context managers. This - exists in Python 3. - - """ - try: - yield - except exceptions: - pass - - def raise_warnings(func): """Function decorator that causes all Python warnings to be raised as exceptions in the wrapped function. @@ -131,24 +121,20 @@ def cdata(text): def is_stix(entity): - """Returns true if `entity` is an instance of :class:`.Entity`.""" + """Returns true if `entity` is an instance of :class:`stix.Entity`.""" return isinstance(entity, stix.Entity) def is_cybox(entity): - """Returns true if `entity` is an instance of - :class:`mixbox.entities.Entity`. + """Returns true if `entity` is a CybOX Object, Observable, or + ObjectProperties subclass. """ - # TODO: once all entities subclass from mixbox.entities.Entity, how do we - # do this? - return isinstance(entity, Entity) + return isinstance(entity, (Object, Observable, ObjectProperties)) def is_entity(entity): - """Returns true if `entity` is an instance of :class:`.Entity` or - :class:`mixbox.Entity`. - """ - return isinstance(entity, (Entity, stix.Entity)) + """Returns true if `entity` is :class:`mixbox.entities.Entity`.""" + return isinstance(entity, Entity) def is_entitylist(entity): @@ -216,20 +202,12 @@ def key_name(name): return name -def is_sequence(item): - """Returns ``True`` if `value` is a sequence type (e.g., ``list``, or - ``tuple``). String types will return ``False``. - - """ - return hasattr(item, "__iter__") - - def check_version(expected, found): """Raises ValueError if `found` is not equal to or found within `expected`. """ - if is_sequence(expected): + if datautils.is_sequence(expected): is_good = found in expected else: is_good = (found == expected) @@ -319,7 +297,7 @@ def dict_iter(items): d[key] = dates.serialize_date(field) elif mixbox.xml.is_element(field) or mixbox.xml.is_etree(field): d[key] = lxml.etree.tostring(field) - elif is_sequence(field): + elif datautils.is_sequence(field): d[key] = dict_iter(field) else: d[key] = field @@ -376,7 +354,7 @@ def remove_entries(d, keys): # Namespace flattening -from .nsparser import * # noqa from .dates import * # noqa +from .nsparser import * # noqa from .parser import * # noqa from .walk import * # noqa diff --git a/stix/utils/deprecated.py b/stix/utils/deprecated.py index 7d56fb96..f5d9fc0a 100644 --- a/stix/utils/deprecated.py +++ b/stix/utils/deprecated.py @@ -3,7 +3,7 @@ import warnings -from . import is_sequence +from mixbox.datautils import is_sequence def idref_deprecated(entity): diff --git a/stix/utils/nsparser.py b/stix/utils/nsparser.py index 2804482f..810b76fc 100644 --- a/stix/utils/nsparser.py +++ b/stix/utils/nsparser.py @@ -7,17 +7,18 @@ from mixbox import idgen from mixbox.entities import Entity +from mixbox.exceptions import ignored +from mixbox.namespaces import Namespace, NamespaceSet, register_namespace +from mixbox.namespaces import (get_full_ns_map, get_full_prefix_map, + get_full_schemaloc_map) -import cybox -import cybox.core -import cybox.common -from cybox.utils.nsparser import CYBOX_NAMESPACES +import cybox.utils.nsparser -# internal -import stix +try: + import maec.utils.nsparser +except ImportError: + pass -# relative -from . import ignored from .walk import iterwalk @@ -72,12 +73,11 @@ def update(self, ns_info): def _parse_collected_classes(self): collected = self._collected_classes - entity_klasses = (stix.Entity, Entity) - # Generator which yields all stix.Entity and mixbox.Entity subclasses + # Generator which yields all Entity subclasses # that were collected. entity_subclasses = ( - klass for klass in collected if issubclass(klass, entity_klasses) + klass for klass in collected if issubclass(klass, Entity) ) # Local function for adding namespaces that have no defined prefix @@ -101,7 +101,7 @@ def _parse_collected_classes(self): self._collected_namespaces[alias] = ns continue - # Many stix/cybox entity classes have an _XSI_TYPE attribute that + # Many Entity classes have an _XSI_TYPE attribute that # contains a `prefix:namespace` formatted QNAME for the # associated xsi:type. xsi_type = getattr(klass, "_XSI_TYPE", None) @@ -284,7 +284,7 @@ def _finalize_schemalocs(self, schemaloc_dict=None): schemaloc_dict[ns] = DEFAULT_STIX_SCHEMALOCATIONS[ns] elif ns in schemaloc_dict: continue - elif (ns == id_ns) or (ns in XML_NAMESPACES): + elif (ns == id_ns) or (ns in NO_SCHEMALOC_NEEDED): continue else: error = "Unable to map namespace '{0}' to schemaLocation" @@ -337,208 +337,120 @@ def collect(self, entity): self._input_schemalocs.update(entity.__input_schemalocations__) -class NamespaceParser(object): - def __init__(self): - pass - - def get_namespaces(self, entity, ns_dict=None): - ns_info = NamespaceInfo() - - for node in iterwalk(entity): - ns_info.collect(node) - - ns_info.finalize(ns_dict=ns_dict) - return ns_info.finalized_namespaces - - def get_namespace_schemalocation_dict(self, entity, ns_dict=None, schemaloc_dict=None): - ns_info = NamespaceInfo() - - for node in iterwalk(entity): - ns_info.collect(node) - - ns_info.finalize(ns_dict=ns_dict, schemaloc_dict=schemaloc_dict) - return ns_info.finalized_schemalocs - - def get_xmlns_str(self, ns_dict): - pairs = sorted(ns_dict.iteritems()) - return "\n\t".join( - 'xmlns:%s="%s"' % (alias, ns) for alias, ns in pairs - ) +def get_namespaces(entity, ns_dict=None): + ns_info = NamespaceInfo() - def get_schemaloc_str(self, schemaloc_dict): - if not schemaloc_dict: - return "" + for node in iterwalk(entity): + ns_info.collect(node) - schemaloc_str_start = 'xsi:schemaLocation="\n\t' - schemaloc_str_end = '"' + ns_info.finalize(ns_dict=ns_dict) + return ns_info.finalized_namespaces - pairs = sorted(schemaloc_dict.iteritems()) - schemaloc_str_content = "\n\t".join( - "%s %s" % (ns, loc) for ns, loc in pairs - ) - return schemaloc_str_start + schemaloc_str_content + schemaloc_str_end +def get_namespace_schemalocation_dict(entity, ns_dict=None, schemaloc_dict=None): + ns_info = NamespaceInfo() - def get_namespace_def_str(self, namespaces, schemaloc_dict): - if not any((namespaces, schemaloc_dict)): - return "" + for node in iterwalk(entity): + ns_info.collect(node) - parts = ( - self.get_xmlns_str(namespaces), - self.get_schemaloc_str(schemaloc_dict) - ) + ns_info.finalize(ns_dict=ns_dict, schemaloc_dict=schemaloc_dict) + return ns_info.finalized_schemalocs - return "\n\t".join(parts) - - -#: Schema locations for standard XML namespaces -XML_NAMESPACES = { - 'http://www.w3.org/2001/XMLSchema-instance': 'xsi', - 'http://www.w3.org/2001/XMLSchema': 'xs', - 'http://www.w3.org/1999/xlink': 'xlink', - 'http://www.w3.org/2000/09/xmldsig#': 'ds' -} - -#: Schema locations for namespaces defined by the STIX language -STIX_NS_TO_SCHEMALOCATION = { - 'http://data-marking.mitre.org/Marking-1': 'http://stix.mitre.org/XMLSchema/data_marking/1.2/data_marking.xsd', - 'http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1': 'http://stix.mitre.org/XMLSchema/extensions/marking/simple/1.2/simple_marking.xsd', - 'http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1': 'http://stix.mitre.org/XMLSchema/extensions/marking/tlp/1.2/tlp_marking.xsd', - 'http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1': 'http://stix.mitre.org/XMLSchema/extensions/marking/terms_of_use/1.1/terms_of_use_marking.xsd', - 'http://stix.mitre.org/Campaign-1': 'http://stix.mitre.org/XMLSchema/campaign/1.2/campaign.xsd', - 'http://stix.mitre.org/CourseOfAction-1': 'http://stix.mitre.org/XMLSchema/course_of_action/1.2/course_of_action.xsd', - 'http://stix.mitre.org/ExploitTarget-1': 'http://stix.mitre.org/XMLSchema/exploit_target/1.2/exploit_target.xsd', - 'http://stix.mitre.org/Incident-1': 'http://stix.mitre.org/XMLSchema/incident/1.2/incident.xsd', - 'http://stix.mitre.org/Indicator-2': 'http://stix.mitre.org/XMLSchema/indicator/2.2/indicator.xsd', - 'http://stix.mitre.org/Report-1': 'http://stix.mitre.org/XMLSchema/report/1.0/report.xsd', - 'http://stix.mitre.org/TTP-1': 'http://stix.mitre.org/XMLSchema/ttp/1.2/ttp.xsd', - 'http://stix.mitre.org/ThreatActor-1': 'http://stix.mitre.org/XMLSchema/threat_actor/1.2/threat_actor.xsd', - 'http://stix.mitre.org/common-1': 'http://stix.mitre.org/XMLSchema/common/1.2/stix_common.xsd', - 'http://stix.mitre.org/default_vocabularies-1': 'http://stix.mitre.org/XMLSchema/default_vocabularies/1.2.0/stix_default_vocabularies.xsd', - 'http://stix.mitre.org/extensions/AP#CAPEC2.7-1': 'http://stix.mitre.org/XMLSchema/extensions/attack_pattern/capec_2.7/1.1/capec_2.7_attack_pattern.xsd', - 'http://stix.mitre.org/extensions/Address#CIQAddress3.0-1': 'http://stix.mitre.org/XMLSchema/extensions/address/ciq_3.0/1.2/ciq_3.0_address.xsd', - 'http://stix.mitre.org/extensions/Identity#CIQIdentity3.0-1': 'http://stix.mitre.org/XMLSchema/extensions/identity/ciq_3.0/1.2/ciq_3.0_identity.xsd', - 'http://stix.mitre.org/extensions/Malware#MAEC4.1-1': 'http://stix.mitre.org/XMLSchema/extensions/malware/maec_4.1/1.1/maec_4.1_malware.xsd', - 'http://stix.mitre.org/extensions/StructuredCOA#Generic-1': 'http://stix.mitre.org/XMLSchema/extensions/structured_coa/generic/1.2/generic_structured_coa.xsd', - 'http://stix.mitre.org/extensions/TestMechanism#Generic-1': 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/generic/1.2/generic_test_mechanism.xsd', - 'http://stix.mitre.org/extensions/TestMechanism#OVAL5.10-1': 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/oval_5.10/1.2/oval_5.10_test_mechanism.xsd', - 'http://stix.mitre.org/extensions/TestMechanism#OpenIOC2010-1': 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/open_ioc_2010/1.2/open_ioc_2010_test_mechanism.xsd', - 'http://stix.mitre.org/extensions/TestMechanism#Snort-1': 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/snort/1.2/snort_test_mechanism.xsd', - 'http://stix.mitre.org/extensions/TestMechanism#YARA-1': 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/yara/1.2/yara_test_mechanism.xsd', - 'http://stix.mitre.org/extensions/Vulnerability#CVRF-1': 'http://stix.mitre.org/XMLSchema/extensions/vulnerability/cvrf_1.1/1.2/cvrf_1.1_vulnerability.xsd', - 'http://stix.mitre.org/stix-1': 'http://stix.mitre.org/XMLSchema/core/1.2/stix_core.xsd' -} - -#: Schema locations for namespaces defined by the CybOX language -CYBOX_NS_TO_SCHEMALOCATION = dict( - (x.name, x.schema_location) for x in CYBOX_NAMESPACES if x.schema_location -) - -#: Schema locations for namespaces not defined by STIX, but hosted on the STIX website -EXT_NS_TO_SCHEMALOCATION = { - 'urn:oasis:names:tc:ciq:xal:3': 'http://stix.mitre.org/XMLSchema/external/oasis_ciq_3.0/xAL.xsd', - 'urn:oasis:names:tc:ciq:xpil:3': 'http://stix.mitre.org/XMLSchema/external/oasis_ciq_3.0/xPIL.xsd', - 'urn:oasis:names:tc:ciq:xnl:3': 'http://stix.mitre.org/XMLSchema/external/oasis_ciq_3.0/xNL.xsd' -} -#: Default namespace->alias mappings. These can be overriden by user-provided dictionaries on export. -DEFAULT_STIX_NS_TO_PREFIX = { - 'http://cybox.mitre.org/common-2': 'cyboxCommon', - 'http://cybox.mitre.org/cybox-2': 'cybox', - 'http://data-marking.mitre.org/Marking-1': 'marking', - 'http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1': 'simpleMarking', - 'http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1': 'tlpMarking', - 'http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1': 'TOUMarking', - 'http://stix.mitre.org/Campaign-1': 'campaign', - 'http://stix.mitre.org/CourseOfAction-1': 'coa', - 'http://stix.mitre.org/ExploitTarget-1': 'et', - 'http://stix.mitre.org/Incident-1': 'incident', - 'http://stix.mitre.org/Indicator-2': 'indicator', - 'http://stix.mitre.org/TTP-1': 'ttp', - 'http://stix.mitre.org/ThreatActor-1': 'ta', - 'http://stix.mitre.org/Report-1': 'report', - 'http://stix.mitre.org/stix-1': 'stix', - 'http://stix.mitre.org/common-1': 'stixCommon', - 'http://stix.mitre.org/default_vocabularies-1': 'stixVocabs', - 'http://stix.mitre.org/extensions/AP#CAPEC2.7-1': 'stix-capec', - 'http://stix.mitre.org/extensions/Address#CIQAddress3.0-1': 'stix-ciqaddress', - 'http://stix.mitre.org/extensions/Identity#CIQIdentity3.0-1': 'ciqIdentity', - 'http://stix.mitre.org/extensions/Malware#MAEC4.1-1': 'stix-maec', - 'http://stix.mitre.org/extensions/StructuredCOA#Generic-1': 'genericStructuredCOA', - 'http://stix.mitre.org/extensions/TestMechanism#Generic-1': 'genericTM', - 'http://stix.mitre.org/extensions/TestMechanism#OVAL5.10-1': 'stix-oval', - 'http://stix.mitre.org/extensions/TestMechanism#OpenIOC2010-1': 'stix-openioc', - 'http://stix.mitre.org/extensions/TestMechanism#Snort-1': 'snortTM', - 'http://stix.mitre.org/extensions/TestMechanism#YARA-1': 'yaraTM', - 'http://stix.mitre.org/extensions/Vulnerability#CVRF-1': 'stix-cvrf' -} - -#: Mapping of extension namespaces to their (typical) prefixes. -DEFAULT_EXT_TO_PREFIX = { - 'http://capec.mitre.org/capec-2': 'capec', - 'http://maec.mitre.org/XMLSchema/maec-package-2': 'maecPackage', - 'http://oval.mitre.org/XMLSchema/oval-definitions-5': 'oval-def', - 'http://oval.mitre.org/XMLSchema/oval-variables-5': 'oval-var', - 'http://schemas.mandiant.com/2010/ioc': 'ioc', - 'http://schemas.mandiant.com/2010/ioc/TR/': 'ioc-tr', - 'http://www.icasi.org/CVRF/schema/cvrf/1.1': 'cvrf', - 'urn:oasis:names:tc:ciq:xal:3': 'xal', - 'urn:oasis:names:tc:ciq:xpil:3': 'xpil', - 'urn:oasis:names:tc:ciq:xnl:3': 'xnl' -} - -#: Mapping of CybOX namespaces to default aliases -DEFAULT_CYBOX_NAMESPACES = dict( - (x.name, x.prefix) for x in CYBOX_NAMESPACES -) - - -#: Mapping of all STIX/STIX Extension/CybOX/XML namespaces -DEFAULT_STIX_NAMESPACES = dict( - itertools.chain( - DEFAULT_CYBOX_NAMESPACES.iteritems(), - XML_NAMESPACES.iteritems(), - DEFAULT_STIX_NS_TO_PREFIX.iteritems(), - DEFAULT_EXT_TO_PREFIX.iteritems() +def get_xmlns_str(ns_dict): + pairs = sorted(ns_dict.iteritems()) + return "\n\t".join( + 'xmlns:%s="%s"' % (alias, ns) for alias, ns in pairs ) -) - -#: Prefix-to-namespace mapping of the `DEFAULT_STIX_NAMESPACES` mapping -DEFAULT_STIX_PREFIX_TO_NAMESPACE = dict( - (alias, ns) for ns, alias in DEFAULT_STIX_NAMESPACES.iteritems() -) - -#: Tuple of all keys found in `DEFAULT_STIX_NAMESPACES` mapping. -DEFAULT_STIX_NAMESPACES_TUPLE = tuple(DEFAULT_STIX_NAMESPACES.keys()) - -#: Mapping of STIX/CybOX/STIX Extension namespaces to canonical schema locations -DEFAULT_STIX_SCHEMALOCATIONS = dict( - itertools.chain( - STIX_NS_TO_SCHEMALOCATION.iteritems(), - EXT_NS_TO_SCHEMALOCATION.iteritems(), - CYBOX_NS_TO_SCHEMALOCATION.iteritems(), - ) -) -# python-maec support code -with ignored(ImportError): - from maec.utils.nsparser import MAEC_NAMESPACES - ns_to_prefix = dict( - (x.name, x.prefix) for x in MAEC_NAMESPACES - ) +def get_schemaloc_str(schemaloc_dict): + if not schemaloc_dict: + return "" - del ns_to_prefix['http://maec.mitre.org/default_vocabularies-1'] + schemaloc_str_start = 'xsi:schemaLocation="\n\t' + schemaloc_str_end = '"' - prefix_to_ns = dict( - (prefix, ns) for (ns, prefix) in ns_to_prefix.iteritems() + pairs = sorted(schemaloc_dict.iteritems()) + schemaloc_str_content = "\n\t".join( + "%s %s" % (ns, loc) for ns, loc in pairs ) - ns_to_schemalocation = dict( - (x.name, x.schema_location) for x in MAEC_NAMESPACES if x.schema_location - ) + return schemaloc_str_start + schemaloc_str_content + schemaloc_str_end + + +# These namespaces don't need to have a schemalocation included during export. +NO_SCHEMALOC_NEEDED = [ + 'http://www.w3.org/2001/XMLSchema-instance', + 'http://www.w3.org/2001/XMLSchema', + 'http://www.w3.org/1999/xlink', + 'http://www.w3.org/2000/09/xmldsig#', +] + +#: Data Marking Namespaces +NS_MARKING = Namespace('http://data-marking.mitre.org/Marking-1', 'marking', 'http://stix.mitre.org/XMLSchema/data_marking/1.2/data_marking.xsd') +NS_MARKING_SIMPLE = Namespace('http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1', 'simpleMarking', 'http://stix.mitre.org/XMLSchema/extensions/marking/simple/1.2/simple_marking.xsd') +NS_MARKING_TLP = Namespace('http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1', 'tlpMarking', 'http://stix.mitre.org/XMLSchema/extensions/marking/tlp/1.2/tlp_marking.xsd') +NS_MARKING_TERMSOFUSE = Namespace('http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1', 'TOUMarking', 'http://stix.mitre.org/XMLSchema/extensions/marking/terms_of_use/1.1/terms_of_use_marking.xsd') + +#: STIX Namespaces +NS_STIX_CORE = Namespace('http://stix.mitre.org/stix-1', 'stix', 'http://stix.mitre.org/XMLSchema/core/1.2/stix_core.xsd') +NS_STIX_COMMON = Namespace('http://stix.mitre.org/common-1', 'stixCommon', 'http://stix.mitre.org/XMLSchema/common/1.2/stix_common.xsd') +NS_STIX_VOCABS = Namespace('http://stix.mitre.org/default_vocabularies-1', 'stixVocabs', 'http://stix.mitre.org/XMLSchema/default_vocabularies/1.2.0/stix_default_vocabularies.xsd') +NS_STIX_CAMPAIGN = Namespace('http://stix.mitre.org/Campaign-1', 'campaign', 'http://stix.mitre.org/XMLSchema/campaign/1.2/campaign.xsd') +NS_STIX_COURSE_OF_ACTION = Namespace('http://stix.mitre.org/CourseOfAction-1', 'coa', 'http://stix.mitre.org/XMLSchema/course_of_action/1.2/course_of_action.xsd') +NS_STIX_EXPLOIT_TARGET = Namespace('http://stix.mitre.org/ExploitTarget-1', 'et', 'http://stix.mitre.org/XMLSchema/exploit_target/1.2/exploit_target.xsd') +NS_STIX_INCIDENT = Namespace('http://stix.mitre.org/Incident-1', 'incident', 'http://stix.mitre.org/XMLSchema/incident/1.2/incident.xsd') +NS_STIX_INDICATOR = Namespace('http://stix.mitre.org/Indicator-2', 'indicator', 'http://stix.mitre.org/XMLSchema/indicator/2.2/indicator.xsd') +NS_STIX_REPORT = Namespace('http://stix.mitre.org/Report-1', 'report', 'http://stix.mitre.org/XMLSchema/report/1.0/report.xsd') +NS_STIX_TTP = Namespace('http://stix.mitre.org/TTP-1', 'ttp', 'http://stix.mitre.org/XMLSchema/ttp/1.2/ttp.xsd') +NS_STIX_THREAT_ACTOR = Namespace('http://stix.mitre.org/ThreatActor-1', 'ta', 'http://stix.mitre.org/XMLSchema/threat_actor/1.2/threat_actor.xsd') +NS_STIX_EXT_AP_CAPEC = Namespace('http://stix.mitre.org/extensions/AP#CAPEC2.7-1', 'stix-capec', 'http://stix.mitre.org/XMLSchema/extensions/attack_pattern/capec_2.7/1.1/capec_2.7_attack_pattern.xsd') +NS_STIX_EXT_ADDRESS_CIQ= Namespace('http://stix.mitre.org/extensions/Address#CIQAddress3.0-1', 'stix-ciqaddress', 'http://stix.mitre.org/XMLSchema/extensions/address/ciq_3.0/1.2/ciq_3.0_address.xsd') +NS_STIX_EXT_IDENTITY_CIQ= Namespace('http://stix.mitre.org/extensions/Identity#CIQIdentity3.0-1', 'ciqIdentity', 'http://stix.mitre.org/XMLSchema/extensions/identity/ciq_3.0/1.2/ciq_3.0_identity.xsd') +NS_STIX_EXT_MALWARE_MAEC = Namespace('http://stix.mitre.org/extensions/Malware#MAEC4.1-1', 'stix-maec', 'http://stix.mitre.org/XMLSchema/extensions/malware/maec_4.1/1.1/maec_4.1_malware.xsd') +NS_STIX_EXT_COA_GENERIC = Namespace('http://stix.mitre.org/extensions/StructuredCOA#Generic-1', 'genericStructuredCOA', 'http://stix.mitre.org/XMLSchema/extensions/structured_coa/generic/1.2/generic_structured_coa.xsd') +NS_STIX_EXT_TEST_GENERIC = Namespace('http://stix.mitre.org/extensions/TestMechanism#Generic-1', 'genericTM', 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/generic/1.2/generic_test_mechanism.xsd') +NS_STIX_EXT_TEST_OVAL = Namespace('http://stix.mitre.org/extensions/TestMechanism#OVAL5.10-1', 'stix-oval', 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/oval_5.10/1.2/oval_5.10_test_mechanism.xsd') +NS_STIX_EXT_TEST_OPENIOC = Namespace('http://stix.mitre.org/extensions/TestMechanism#OpenIOC2010-1', 'stix-openioc', 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/open_ioc_2010/1.2/open_ioc_2010_test_mechanism.xsd') +NS_STIX_EXT_TEST_SNORT = Namespace('http://stix.mitre.org/extensions/TestMechanism#Snort-1', 'snortTM', 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/snort/1.2/snort_test_mechanism.xsd') +NS_STIX_EXT_TEST_YARA = Namespace('http://stix.mitre.org/extensions/TestMechanism#YARA-1', 'yaraTM', 'http://stix.mitre.org/XMLSchema/extensions/test_mechanism/yara/1.2/yara_test_mechanism.xsd') +NS_STIX_EXT_VULN_CVRF = Namespace('http://stix.mitre.org/extensions/Vulnerability#CVRF-1', 'stix-cvrf', 'http://stix.mitre.org/XMLSchema/extensions/vulnerability/cvrf_1.1/1.2/cvrf_1.1_vulnerability.xsd') + +#: Namespaces used by external schemas that are incorporated in STIX. +NS_CAPEC = Namespace('http://capec.mitre.org/capec-2', 'capec', '') +NS_MAEC_PACKAGE = Namespace('http://maec.mitre.org/XMLSchema/maec-package-2', 'maecPackage', '') +NS_OVAL_DEF = Namespace('http://oval.mitre.org/XMLSchema/oval-definitions-5', 'oval-def', '') +NS_OVAL_VAR = Namespace('http://oval.mitre.org/XMLSchema/oval-variables-5', 'oval-var', '') +NS_OPENIOC = Namespace('http://schemas.mandiant.com/2010/ioc', 'ioc', '') +NS_OPENIOC_TR = Namespace('http://schemas.mandiant.com/2010/ioc/TR/', 'ioc-tr', '') +NS_CVRF = Namespace('http://www.icasi.org/CVRF/schema/cvrf/1.1', 'cvrf', '') +# NOTE: These three schemas are hosted on the STIX site despite them being +# defined outside of STIX +NS_OASIS_CIQ_XAL = Namespace('urn:oasis:names:tc:ciq:xal:3', 'xal', 'http://stix.mitre.org/XMLSchema/external/oasis_ciq_3.0/xAL.xsd') +NS_OASIS_CIQ_XPIL = Namespace('urn:oasis:names:tc:ciq:xpil:3', 'xpil', 'http://stix.mitre.org/XMLSchema/external/oasis_ciq_3.0/xPIL.xsd') +NS_OASIS_CIQ_XNL = Namespace('urn:oasis:names:tc:ciq:xnl:3', 'xnl', 'http://stix.mitre.org/XMLSchema/external/oasis_ciq_3.0/xNL.xsd') + + +STIX_NAMESPACES = NamespaceSet() +EXTENSION_NAMESPACES = NamespaceSet() + +# Magic to automatically register all Namespaces defined in this module. +for k, v in dict(globals()).items(): + if k.startswith('NS_'): + register_namespace(v) + if k.startswith('NS_MARKING_') or k.startswith('NS_STIX_'): + STIX_NAMESPACES.add(v) + else: + EXTENSION_NAMESPACES.add(v) - DEFAULT_STIX_NAMESPACES.update(ns_to_prefix) - DEFAULT_STIX_PREFIX_TO_NAMESPACE.update(prefix_to_ns) - DEFAULT_STIX_NAMESPACES_TUPLE = tuple(DEFAULT_STIX_NAMESPACES.iterkeys()) - DEFAULT_STIX_SCHEMALOCATIONS.update(ns_to_schemalocation) + +#: Default namespace->alias mappings. These can be overriden by user-provided dictionaries on export. +DEFAULT_STIX_NS_TO_PREFIX = dict((x.name, x.prefix) for x in STIX_NAMESPACES) +# NOTE: CybOX Core and Common are no longer in this mapping. Is this OK? +# 'http://cybox.mitre.org/common-2': 'cyboxCommon', +# 'http://cybox.mitre.org/cybox-2': 'cybox', + +DEFAULT_STIX_NAMESPACES = get_full_ns_map() +DEFAULT_STIX_PREFIX_TO_NAMESPACE = get_full_prefix_map() +DEFAULT_STIX_SCHEMALOCATIONS = get_full_schemaloc_map() diff --git a/stix/utils/walk.py b/stix/utils/walk.py index fe1261ea..29a8de13 100644 --- a/stix/utils/walk.py +++ b/stix/utils/walk.py @@ -4,11 +4,13 @@ # stdlib import itertools +from mixbox.datautils import is_sequence + # external from cybox.common import ObjectProperties # internal -from . import is_entity, is_entitylist, attr_name, is_sequence +from . import is_entity, is_entitylist, attr_name def _is_skippable(owner, varname, varobj):