Fix Python 3 support

.travis.yml

@@ -1,11 +1,17 @@
 language: python
+python:
+  - 3.5
+
 env:
   - TOXENV=py26
   - TOXENV=py27
+  - TOXENV=py33
+  - TOXENV=py34
+  - TOXENV=py35
   - TOXENV=lxml23
 
 install:
-  - pip install tox
+  - pip install -U tox
 
 script:
   - tox

stix/base.py

@@ -5,16 +5,16 @@
 import json
 import collections
 import itertools
-import StringIO
+from sys import version_info
 
 from mixbox import idgen
 from mixbox.binding_utils import save_encoding
 from mixbox.cache import Cached
+from mixbox.vendor.six import StringIO, iteritems, itervalues, text_type, binary_type
 
 # internal
 from . import utils
 
-
 def _override(*args, **kwargs):
     raise NotImplementedError()
 
@@ -51,7 +51,7 @@ def _set_var(self, klass, try_cast=True, arg=None, **kwargs):
                 and the field value is the value.
 
         """
-        name, item = kwargs.iteritems().next()
+        name, item = next(iteritems(kwargs))
         attr = utils.private_name(name)  # 'title' => '_title'
 
         if item is None:
@@ -86,7 +86,7 @@ def _set_vocab(self, klass=None, **kwargs):
         from stix.common import VocabString
 
         klass = klass or VocabString
-        item  = kwargs.itervalues().next()
+        item  = next(itervalues(kwargs))
 
         if isinstance(item, VocabString):
             self._set_var(VocabString, **kwargs)
@@ -114,8 +114,8 @@ def to_xml(self, include_namespaces=True, include_schemalocs=False,
         """Serializes a :class:`Entity` instance to an XML string.
 
         The default character encoding is ``utf-8`` and can be set via the
-        `encoding` parameter. If `encoding` is ``None``, a unicode string
-        is returned.
+        `encoding` parameter. If `encoding` is ``None``, a string (unicode in
+        Python 2, str in Python 3) is returned.
 
         Args:
             auto_namespace: Automatically discover and export XML namespaces
@@ -136,7 +136,8 @@ def to_xml(self, include_namespaces=True, include_schemalocs=False,
                 only be included if `auto_namespace` is ``False``.
             pretty: Pretty-print the XML.
             encoding: The output character encoding. Default is ``utf-8``. If
-                `encoding` is set to ``None``, a unicode string is returned.
+                `encoding` is set to ``None``, a string (unicode in Python 2,
+                str in Python 3) is returned.
 
         Returns:
             An XML string for this
@@ -169,8 +170,8 @@ def to_xml(self, include_namespaces=True, include_schemalocs=False,
             ns_info.finalized_schemalocs = schemaloc_dict or {}
             obj_ns_dict = dict(
                 itertools.chain(
-                    ns_dict.iteritems(),
-                    nsparser.DEFAULT_STIX_NAMESPACES.iteritems()
+                    iteritems(ns_dict),
+                    iteritems(nsparser.DEFAULT_STIX_NAMESPACES)
                 )
             )
 
@@ -187,7 +188,7 @@ def to_xml(self, include_namespaces=True, include_schemalocs=False,
             namespace_def = namespace_def.replace('\n\t', ' ')
 
         with save_encoding(encoding):
-            sio = StringIO.StringIO()
+            sio = StringIO()
             obj.export(
                 sio.write,                    # output buffer
                 0,                            # output level
@@ -197,7 +198,7 @@ def to_xml(self, include_namespaces=True, include_schemalocs=False,
             )
 
         # Ensure that the StringIO buffer is unicode
-        s = unicode(sio.getvalue())
+        s = text_type(sio.getvalue())
 
         if encoding:
             return s.encode(encoding)

stix/bindings/extensions/address/ciq_address_3_0.py

@@ -88,7 +88,7 @@ def exportChildren(self, lwrite, level, nsmap, namespace_=XML_NS, name_='CIQAddr
             eol_ = ''
         if self.Location is not None:
             showIndent(lwrite, level, pretty_print)
-            lwrite(etree_.tostring(self.Location, pretty_print=pretty_print))
+            lwrite(etree_.tostring(self.Location, pretty_print=pretty_print).decode())
             #self.Location.export(lwrite, level, nsmap, namespace_, name_='Location', pretty_print=pretty_print)
     def build(self, node):
         already_processed = set()

stix/bindings/extensions/attack_pattern/capec_2_7.py

@@ -87,7 +87,7 @@ def exportChildren(self, lwrite, level, nsmap, namespace_=XML_NS, name_='CAPEC2.
             eol_ = ''
         if self.CAPEC is not None:
             showIndent(lwrite, level, pretty_print)
-            lwrite(etree_.tostring(self.CAPEC, pretty_print=pretty_print))
+            lwrite(etree_.tostring(self.CAPEC, pretty_print=pretty_print).decode())
             #self.CAPEC.export(lwrite, level, nsmap, namespace_, name_='CAPEC', pretty_print=pretty_print)
     def build(self, node):
         already_processed = set()

stix/bindings/extensions/identity/ciq_identity_3_0.py

@@ -99,7 +99,7 @@ def exportChildren(self, lwrite, level, nsmap, namespace_=XML_NS, name_='CIQIden
             eol_ = ''
         if self.Specification is not None:
             showIndent(lwrite, level, pretty_print)
-            lwrite(etree_.tostring(self.Specification, pretty_print=pretty_print))
+            lwrite(etree_.tostring(self.Specification, pretty_print=pretty_print).decode())
             #self.Specification.export(lwrite, level, nsmap, namespace_, name_='Specification', pretty_print=pretty_print)
         for Role_ in self.Role:
             showIndent(lwrite, level, pretty_print)

stix/bindings/extensions/malware/maec_4_1.py

@@ -92,7 +92,7 @@ def exportChildren(self, lwrite, level, nsmap, namespace_=XML_NS, name_='MAEC4.1
                 self.MAEC.export(lwrite, level, namespace_='stix-maec:', name_='MAEC', pretty_print=pretty_print)
             else:
                 showIndent(lwrite, level, pretty_print)
-                lwrite(etree_.tostring(self.MAEC, pretty_print=pretty_print))
+                lwrite(etree_.tostring(self.MAEC, pretty_print=pretty_print).decode())
     def build(self, node):
         already_processed = set()
         self.buildAttributes(node, node.attrib, already_processed)

stix/bindings/extensions/test_mechanism/open_ioc_2010.py

@@ -87,7 +87,7 @@ def exportChildren(self, lwrite, level, nsmap, namespace_=XML_NS, name_='OpenIOC
             eol_ = ''
         if self.ioc is not None:
             showIndent(lwrite, level, pretty_print)
-            lwrite(etree_.tostring(self.ioc, pretty_print=pretty_print))
+            lwrite(etree_.tostring(self.ioc, pretty_print=pretty_print).decode())
             #self.ioc.export(lwrite, level, nsmap, namespace_, name_='ioc', pretty_print=pretty_print)
     def build(self, node):
         already_processed = set()

stix/bindings/extensions/test_mechanism/oval_5_10.py

@@ -91,11 +91,11 @@ def exportChildren(self, lwrite, level, nsmap, namespace_=XML_NS, name_='OVAL5.1
             eol_ = ''
         if self.oval_definitions is not None:
             showIndent(lwrite, level, pretty_print)
-            lwrite(etree_.tostring(self.oval_definitions, pretty_print=pretty_print))
+            lwrite(etree_.tostring(self.oval_definitions, pretty_print=pretty_print).decode())
             #self.oval_definitions.export(lwrite, level, nsmap, namespace_, name_='oval_definitions', pretty_print=pretty_print)
         if self.oval_variables is not None:
             showIndent(lwrite, level, pretty_print)
-            lwrite(etree_.tostring(self.oval_variables, pretty_print=pretty_print))
+            lwrite(etree_.tostring(self.oval_variables, pretty_print=pretty_print).decode())
             #self.oval_variables.export(lwrite, level, nsmap, namespace_, name_='oval_variables', pretty_print=pretty_print)
     def build(self, node):
         already_processed = set()

stix/bindings/extensions/vulnerability/cvrf_1_1.py

@@ -88,7 +88,7 @@ def exportChildren(self, lwrite, level, nsmap, namespace_=XML_NS, name_='CVRF1.1
             eol_ = ''
         if self.cvrfdoc is not None:
             showIndent(lwrite, level, pretty_print)
-            lwrite(etree_.tostring(self.cvrfdoc, pretty_print=pretty_print))
+            lwrite(etree_.tostring(self.cvrfdoc, pretty_print=pretty_print).decode())
             #self.cvrfdoc.export(lwrite, level, nsmap, namespace_, name_='cvrfdoc', pretty_print=pretty_print)
     def build(self, node):
         already_processed = set()

stix/bindings/incident.py

@@ -2764,8 +2764,8 @@ def parseEtree(inFileName):
     return rootObj, rootElement
 
 def parseString(inString):
-    from mixbox.vendor.six import StringIO
-    doc = parsexml_(StringIO(inString))
+    from mixbox.vendor.six import BytesIO
+    doc = parsexml_(BytesIO(inString.encode('utf-8')))
     rootNode = doc.getroot()
     rootTag, rootClass = get_root_tag(rootNode)
     if rootClass is None:

stix/common/__init__.py

@@ -2,6 +2,7 @@
 # See LICENSE.txt for complete terms.
 
 from __future__ import absolute_import
+from sys import version_info
 
 from .structured_text import StructuredText, StructuredTextList  # noqa
 from .vocabs import VocabString   # noqa
@@ -128,7 +129,10 @@ def to_dict(self):
         return d
 
     def __str__(self):
-        return self.__unicode__().encode("utf-8")
+        if version_info < (3,):
+            return self.__unicode__().encode("utf-8")
+        else:
+            return self.__unicode__()
 
     def __unicode__(self):
         return text_type(self.value)

stix/common/structured_text.py

@@ -4,10 +4,12 @@
 import itertools
 import contextlib
 import collections
+from sys import version_info
 
 import stix
 import stix.utils as utils
 import stix.bindings.stix_common as stix_common_binding
+from mixbox.vendor.six import text_type
 
 #: Default ordinality value for StructuredText.
 DEFAULT_ORDINALITY = 1
@@ -150,13 +152,16 @@ def __str__(self):
         """Returns a UTF-8 encoded string representation of the ``value``.
 
         """
-        return self.__unicode__().encode("utf-8")
+        if version_info < (3,):
+            return self.__unicode__().encode("utf-8")
+        else:
+            return self.__unicode__()
 
     def __unicode__(self):
         """Returns a ``unicode`` string representation of the ``value``.
 
         """
-        return unicode(self.value)
+        return text_type(self.value)
 
 
 @contextlib.contextmanager

stix/extensions/malware/maec_4_1_malware.py

@@ -7,7 +7,7 @@
 # external
 from lxml import etree
 import mixbox.xml
-from mixbox.vendor.six import StringIO, iteritems
+from mixbox.vendor.six import BytesIO, iteritems
 
 # internal
 import stix
@@ -206,7 +206,7 @@ def from_dict(cls, d, return_obj=None):
                 return_obj.maec = cls._maec_from_dict(maec)
             else:
                 parser = mixbox.xml.get_xml_parser()
-                return_obj.maec = etree.parse(StringIO(maec), parser=parser)
+                return_obj.maec = etree.parse(BytesIO(maec), parser=parser)
 
         return return_obj
 

stix/extensions/test_mechanism/open_ioc_2010_test_mechanism.py

@@ -4,7 +4,7 @@
 # external
 from lxml import etree
 import mixbox.xml
-from mixbox.vendor.six import StringIO, iteritems
+from mixbox.vendor.six import BytesIO, iteritems
 
 # internal
 import stix
@@ -98,7 +98,7 @@ def from_dict(cls, d, return_obj=None):
         super(OpenIOCTestMechanism, cls).from_dict(d, return_obj)
         if 'ioc' in d:
             parser = mixbox.xml.get_xml_parser()
-            return_obj.ioc = etree.parse(StringIO(d['ioc']), parser=parser)
+            return_obj.ioc = etree.parse(BytesIO(d['ioc']), parser=parser)
 
         return return_obj
 

stix/test/core/stix_package_test.py

@@ -2,9 +2,10 @@
 # See LICENSE.txt for complete terms.
 
 import copy
-import StringIO
 import unittest
 
+from mixbox.vendor.six import BytesIO
+
 from stix.test import EntityTestCase, assert_warnings
 from stix.test import report_test
 from stix.test.common import kill_chains_test, related_test
@@ -126,7 +127,7 @@ def test_deepcopy(self):
 
         """
         package = core.STIXPackage.from_xml(
-            StringIO.StringIO(
+            BytesIO(
                 core.STIXPackage().to_xml()
             )
         )

stix/test/encoding_test.py

@@ -5,7 +5,7 @@
 """Tests for various encoding issues throughout the library"""
 
 import unittest
-from mixbox.vendor.six import StringIO, text_type
+from mixbox.vendor.six import BytesIO, text_type
 from mixbox import binding_utils
 
 from stix.core import STIXHeader, STIXPackage
@@ -147,15 +147,15 @@ def test_to_xml_no_encoding(self):
     @silence_warnings
     def test_from_xml_utf16_encoded(self):
         utf16_xml = XML.encode('utf-16')
-        sio = StringIO(utf16_xml)
+        sio = BytesIO(utf16_xml)
         sp = STIXPackage.from_xml(sio, encoding='utf-16')
         header = sp.stix_header
         self.assertEqual(header.title, UNICODE_STR)
 
     @silence_warnings
     def test_from_xml_default_encoded(self):
         utf8_xml = XML.encode('utf-8')
-        sio = StringIO(utf8_xml)
+        sio = BytesIO(utf8_xml)
         sp = STIXPackage.from_xml(sio)
         header = sp.stix_header
         self.assertEqual(header.title, UNICODE_STR)
@@ -171,7 +171,7 @@ def test_utf16_roundtrip(self):
         xml16 = sp.to_xml(encoding='utf-16')
 
         # deserialize as utf-16
-        sp2 = STIXPackage.from_xml(StringIO(xml16), encoding='utf-16')
+        sp2 = STIXPackage.from_xml(BytesIO(xml16), encoding='utf-16')
         sh2 = sp2.stix_header
 
         # check that the titles align

stix/test/extensions/malware/maec_4_1_malware_test.py

@@ -1,5 +1,5 @@
 import unittest
-from mixbox.vendor.six import StringIO
+from mixbox.vendor.six import StringIO, BytesIO
 
 from lxml import etree
 import mixbox.xml
@@ -73,7 +73,7 @@ class PythonMAECEtreeTests(unittest.TestCase):
     def _test_xml(self, obj):
         xml = obj.to_xml()
         parser = mixbox.xml.get_xml_parser()
-        tree = etree.parse(StringIO(xml), parser=parser)
+        tree = etree.parse(BytesIO(xml), parser=parser)
         root = tree.getroot()
 
         xpath = "//cyboxCommon:Type"

stix/test/extensions/test_mechanisms/openioc_test.py

@@ -7,7 +7,7 @@
 
 from mixbox import idgen
 from mixbox.namespaces import Namespace
-from mixbox.vendor.six import StringIO
+from mixbox.vendor.six import StringIO, BytesIO
 import mixbox.xml
 
 from stix.test import EntityTestCase
@@ -41,12 +41,11 @@ class OpenIOCTestMechanismTests(EntityTestCase, unittest.TestCase):
 class OpenIOCEtreeTests(unittest.TestCase):
     DESCRIPTION = "Finds Zeus variants, twexts, sdra64, ntos"
     XML = (
-        """
+        r"""
         <stix-openioc:ioc
             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
             xmlns:xsd="http://www.w3.org/2001/XMLSchema"
             xmlns:mandiant-openioc="http://schemas.mandiant.com/2010/ioc"
-            xmlns="http://schemas.mandiant.com/2010/ioc"
             xmlns:stix-openioc="http://stix.mitre.org/extensions/TestMechanism#OpenIOC2010-1"
             id="mandiant:6d2a1b03-b216-4cd8-9a9e-8827af6ebf93" last-modified="2011-10-28T19:28:20">
             <short_description>Zeus</short_description>
@@ -120,8 +119,8 @@ class OpenIOCEtreeTests(unittest.TestCase):
     )
 
     def setUp(self):
-        ioc_ns = Namespace("http://schemas.mandiant.com/2010/ioc",
-                           "mandiant-openioc", '')
+        ioc_ns = Namespace("http://stix.mitre.org/extensions/TestMechanism#OpenIOC2010-1",
+                           "stix-openioc", '')
         idgen.set_id_namespace(ioc_ns)
 
     def tearDown(self):
@@ -130,11 +129,11 @@ def tearDown(self):
     def _test_xml(self, obj):
         xml = obj.to_xml()
         parser = mixbox.xml.get_xml_parser()
-        tree = lxml.etree.parse(StringIO(xml), parser=parser)
+        tree = lxml.etree.parse(BytesIO(xml), parser=parser)
         root = tree.getroot()
 
-        xpath = "//openioc:description"
-        nodes = root.xpath(xpath, namespaces={'openioc': 'http://schemas.mandiant.com/2010/ioc'})
+        xpath = "//stix-openioc:ioc//description"
+        nodes = root.xpath(xpath, namespaces={'stix-openioc': 'http://stix.mitre.org/extensions/TestMechanism#OpenIOC2010-1'})
 
         self.assertTrue(nodes is not None)
         self.assertEqual(len(nodes), 1)