diff --git a/README.rst b/README.rst index 3ea6b4b4..7c3eb1c9 100644 --- a/README.rst +++ b/README.rst @@ -5,13 +5,17 @@ .. image:: https://coveralls.io/repos/github/G-Node/python-odml/badge.svg?branch=master :target: https://coveralls.io/github/G-Node/python-odml?branch=master -odML libraries and editor -========================= +odML (Open metaData Markup Language) core library +================================================= -The Python-odML library (including the odML-Editor) is available on -`GitHub `_. If you are not familiar with -the version control system **git**, but still want to use it, have a look at -the documentation available on the `git-scm website `_. +The open metadata Markup Language is a file based format (XML, JSON, YAML) for storing +metadata in an organised human- and machine-readable way. odML is an initiative to define +and establish an open, flexible, and easy-to-use format to transport metadata. + +The Python-odML library can be easily installed via :code:`pip`. The source code is freely +available on `GitHub `_. If you are not familiar +with the version control system **git**, but still want to use it, have a look at the +documentation available on the `git-scm website `_. Dependencies ------------ @@ -85,6 +89,5 @@ Bugs & Questions Should you find a behaviour that is likely a bug, please file a bug report at `the github bug tracker `_. -If you have questions regarding the use of the library or the editor, feel free to -join the `#gnode `_ IRC channel -on freenode. +If you have questions regarding the use of the library, feel free to join the +`#gnode `_ IRC channel on freenode. diff --git a/odml/base.py b/odml/base.py index 39408805..abe7b5aa 100644 --- a/odml/base.py +++ b/odml/base.py @@ -2,11 +2,11 @@ """ Collects common base functionality """ - +import collections import posixpath from . import terminology -from .tools.doc_inherit import inherit_docstring, allow_inherit_docstring +from .tools.doc_inherit import allow_inherit_docstring class _baseobj(object): @@ -169,8 +169,8 @@ def append(self, *obj_tuple): @allow_inherit_docstring class sectionable(baseobject): def __init__(self): - from odml.section import Section - self._sections = SmartList(Section) + from odml.section import BaseSection + self._sections = SmartList(BaseSection) self._repository = None @property @@ -178,11 +178,11 @@ def document(self): """ Returns the parent-most node (if its a document instance) or None """ + from odml.doc import BaseDocument p = self while p.parent: p = p.parent - import odml.doc as doc - if isinstance(p, doc.Document): + if isinstance(p, BaseDocument): return p @property @@ -192,29 +192,57 @@ def sections(self): def insert(self, position, section): """ - Adds the section to the section-list and makes this document the - section’s parent. + Insert a Section at the child-list position. A ValueError will be raised, + if a Section with the same name already exists in the child-list. - Currently just appends the section and does not insert at the - specified *position* + :param position: index at which the object should be inserted. + :param section: odML Section object. """ - self._sections.append(section) - section._parent = self + from odml.section import BaseSection + if isinstance(section, BaseSection): + if section.name in self._sections: + raise ValueError("Section with name '%s' already exists." % section.name) - def append(self, *vsection_tuple): + self._sections.insert(position, section) + section._parent = self + else: + raise ValueError("Can only insert objects of type Section.") + + def append(self, section): """ - Adds the section to the section-list and makes this document the - section’s parent. + Method appends a single Section to the section child-lists of the current Object. + + :param section: odML Section object. """ from odml.section import BaseSection - from odml.doc import BaseDocument - for vsection in vsection_tuple: - if (not isinstance(vsection, BaseSection)) & \ - isinstance(self, BaseDocument): - raise KeyError("Object " + str(vsection) + - " is not a Section.") - self._sections.append(vsection) - vsection._parent = self + if isinstance(section, BaseSection): + self._sections.append(section) + section._parent = self + elif isinstance(section, collections.Iterable) and not isinstance(section, str): + raise ValueError("Use extend to add a list of Sections.") + else: + raise ValueError("Can only append objects of type Section.") + + def extend(self, sec_list): + """ + Method adds Sections to the section child-list of the current object. + + :param sec_list: Iterable containing odML Section entries. + """ + from odml.section import BaseSection + if not isinstance(sec_list, collections.Iterable): + raise TypeError("'%s' object is not iterable" % type(sec_list).__name__) + + # Make sure only Sections with unique names will be added. + for sec in sec_list: + if not isinstance(sec, BaseSection): + raise ValueError("Can only extend objects of type Section.") + + elif isinstance(sec, BaseSection) and sec.name in self._sections: + raise KeyError("Section with name '%s' already exists." % sec.name) + + for sec in sec_list: + self.append(sec) def remove(self, section): """ Removes the specified child-section """ @@ -254,7 +282,7 @@ def itersections(self, recursive=True, yield_self=False, if self == self.document and ((max_depth is None) or (max_depth > 0)): for sec in self.sections: stack.append((sec, 1)) # (
, ) - elif not self == self.document: + elif self != self.document: stack.append((self, 0)) # (
, ) while len(stack) > 0: @@ -319,12 +347,11 @@ def contains(self, obj): if obj.name == i.name and obj.type == i.type: return i - # FIXME type arguments renamed to dtype? - def _matches(self, obj, key=None, type=None, include_subtype=False): + def _matches(self, obj, key=None, otype=None, include_subtype=False): """ Find out * if the *key* matches obj.name (if key is not None) - * or if *type* matches obj.type (if type is not None) + * or if *otype* matches obj.type (if type is not None) * if type does not match exactly, test for subtype. (e.g.stimulus/white_noise) comparisons are case-insensitive, however both key and type @@ -333,18 +360,16 @@ def _matches(self, obj, key=None, type=None, include_subtype=False): name_match = (key is None or ( key is not None and hasattr(obj, "name") and obj.name == key)) - exact_type_match = (type is None or (type is not None and - hasattr(obj, "type") and - obj.type.lower() == type)) + exact_type_match = (otype is None or (otype is not None and + hasattr(obj, "type") and + obj.type.lower() == otype)) if not include_subtype: return name_match and exact_type_match - subtype_match = type is None or (type is not None and - hasattr(obj, "type") and - type in obj.type - .lower().split('/')[:-1]) - # TODO : Break the above line more elegantly + subtype_match = (otype is None or + (otype is not None and hasattr(obj, "type") and + otype in obj.type.lower().split('/')[:-1])) return name_match and (exact_type_match or subtype_match) @@ -539,10 +564,10 @@ def clone(self, children=True): Clone this object recursively allowing to copy it independently to another document """ - from odml.section import Section + from odml.section import BaseSection obj = super(sectionable, self).clone(children) obj._parent = None - obj._sections = SmartList(Section) + obj._sections = SmartList(BaseSection) if children: for s in self._sections: obj.append(s.clone()) diff --git a/odml/doc.py b/odml/doc.py index 280c37a2..02c3f8a3 100644 --- a/odml/doc.py +++ b/odml/doc.py @@ -15,7 +15,7 @@ class Document(base._baseobj): @allow_inherit_docstring class BaseDocument(base.sectionable, Document): """ - A represenation of an odML document in memory. + A representation of an odML document in memory. Its odml attributes are: *author*, *date*, *version* and *repository*. A Document behaves very much like a section, except that it cannot hold properties. @@ -69,6 +69,8 @@ def author(self): @author.setter def author(self, new_value): + if new_value == "": + new_value = None self._author = new_value @property @@ -81,6 +83,8 @@ def version(self): @version.setter def version(self, new_value): + if new_value == "": + new_value = None self._version = new_value @property @@ -88,11 +92,15 @@ def date(self): """ The date the document was created. """ - return dtypes.set(self._date, "date") + return self._date @date.setter def date(self, new_value): - self._date = dtypes.get(new_value, "date") + if not new_value: + new_value = None + else: + new_value = dtypes.date_set(new_value) + self._date = new_value @property def parent(self): @@ -125,4 +133,4 @@ def get_terminology_equivalent(self): if self.repository is None: return None term = terminology.load(self.repository) - return term \ No newline at end of file + return term diff --git a/odml/property.py b/odml/property.py index a23340bd..a8ec7c56 100644 --- a/odml/property.py +++ b/odml/property.py @@ -61,6 +61,8 @@ def __init__(self, name, value=None, parent=None, unit=None, except ValueError as e: print(e) self._id = str(uuid.uuid4()) + + self._parent = None self._name = name self._value_origin = value_origin self._unit = unit @@ -79,7 +81,6 @@ def __init__(self, name, value=None, parent=None, unit=None, self._value = [] self.value = value - self._parent = None self.parent = parent @property diff --git a/odml/section.py b/odml/section.py index 4c32a32a..8a9e9edc 100644 --- a/odml/section.py +++ b/odml/section.py @@ -46,6 +46,7 @@ def __init__(self, name, type=None, parent=None, print(e) self._id = str(uuid.uuid4()) + self._parent = None self._name = name self._definition = definition self._reference = reference @@ -55,7 +56,6 @@ def __init__(self, name, type=None, parent=None, # this may fire a change event, so have the section setup then self.type = type - self._parent = None self.parent = parent def __repr__(self): diff --git a/test/test_doc.py b/test/test_doc.py index b619d4dd..d5983170 100644 --- a/test/test_doc.py +++ b/test/test_doc.py @@ -1,12 +1,38 @@ +import datetime +import os import unittest +try: + from urllib.request import pathname2url +except ImportError: + from urllib import pathname2url -from odml import Document +from odml import Document, Section, Property +from odml.doc import BaseDocument +from odml.dtypes import FORMAT_DATE class TestSection(unittest.TestCase): def setUp(self): pass + def test_simple_attributes(self): + author = "HPL" + version = "4.8.15" + doc = Document(author=author, version=version) + + self.assertEqual(doc.author, author) + self.assertEqual(doc.version, version) + + doc.author = "" + doc.version = "" + self.assertIsNone(doc.author) + self.assertIsNone(doc.version) + + doc.author = author + doc.version = version + self.assertEqual(doc.author, author) + self.assertEqual(doc.version, version) + def test_id(self): doc = Document() self.assertIsNotNone(doc.id) @@ -38,3 +64,135 @@ def test_new_id(self): # Test invalid custom id exception. with self.assertRaises(ValueError): doc.new_id("crash and burn") + + def test_date(self): + datestring = "2000-01-02" + doc = Document(date=datestring) + + self.assertIsInstance(doc.date, datetime.date) + self.assertEqual(doc.date, + datetime.datetime.strptime(datestring, FORMAT_DATE).date()) + + doc.date = None + self.assertIsNone(doc.date) + + doc.date = datestring + self.assertIsInstance(doc.date, datetime.date) + self.assertEqual(doc.date, + datetime.datetime.strptime(datestring, FORMAT_DATE).date()) + + doc.date = [] + self.assertIsNone(doc.date) + doc.date = {} + self.assertIsNone(doc.date) + doc.date = () + self.assertIsNone(doc.date) + doc.date = "" + self.assertIsNone(doc.date) + + with self.assertRaises(ValueError): + doc.date = "some format" + + def test_get_terminology_equivalent(self): + dir_path = os.path.dirname(os.path.realpath(__file__)) + repo_file = os.path.join(dir_path, "resources", + "local_repository_file_v1.1.xml") + local_url = "file://%s" % pathname2url(repo_file) + + doc = Document(repository=local_url) + + teq = doc.get_terminology_equivalent() + self.assertIsInstance(teq, BaseDocument) + self.assertEqual(len(teq), 1) + self.assertEqual(teq.sections[0].name, "Repository test") + + doc.repository = None + self.assertIsNone(doc.get_terminology_equivalent()) + + def test_append(self): + doc = Document() + self.assertListEqual(doc.sections, []) + + # Test append Section + sec = Section(name="sec_one") + doc.append(sec) + self.assertEqual(len(doc.sections), 1) + self.assertEqual(sec.parent, doc) + + # Test fail on Section list or tuple append + with self.assertRaises(ValueError): + doc.append([Section(name="sec_two"), Section(name="sec_three")]) + with self.assertRaises(ValueError): + doc.append((Section(name="sec_two"), Section(name="sec_three"))) + self.assertEqual(len(doc.sections), 1) + + # Test fail on unsupported value + with self.assertRaises(ValueError): + doc.append(Document()) + with self.assertRaises(ValueError): + doc.append("Section") + with self.assertRaises(ValueError): + doc.append(Property(name="prop")) + + # Test fail on same name entities + with self.assertRaises(KeyError): + doc.append(Section(name="sec_one")) + self.assertEqual(len(doc.sections), 1) + + def test_extend(self): + doc = Document() + self.assertListEqual(doc.sections, []) + + # Test extend with Section list + doc.extend([Section(name="sec_one"), Section(name="sec_two")]) + self.assertEqual(len(doc), 2) + self.assertEqual(len(doc.sections), 2) + self.assertEqual(doc.sections[0].name, "sec_one") + + # Test fail on non iterable + with self.assertRaises(TypeError): + doc.extend(1) + self.assertEqual(len(doc.sections), 2) + + # Test fail on non Section entry + with self.assertRaises(ValueError): + doc.extend([Document()]) + with self.assertRaises(ValueError): + doc.extend([Property(name="prop")]) + with self.assertRaises(ValueError): + doc.extend([5]) + self.assertEqual(len(doc.sections), 2) + + # Test fail on same name entities + with self.assertRaises(KeyError): + doc.extend([Section(name="sec_three"), Section(name="sec_one")]) + self.assertEqual(len(doc.sections), 2) + + def test_insert(self): + doc = Document() + + sec_one = Section(name="sec_one", parent=doc) + sec_two = Section(name="sec_two", parent=doc) + subsec = Section(name="sec_three") + + self.assertNotEqual(doc.sections[1].name, subsec.name) + doc.insert(1, subsec) + self.assertEqual(len(doc.sections), 3) + self.assertEqual(doc.sections[1].name, subsec.name) + self.assertEqual(doc.sections[0].name, sec_one.name) + self.assertEqual(doc.sections[2].name, sec_two.name) + self.assertEqual(subsec.parent, doc) + + # Test invalid object + with self.assertRaises(ValueError): + doc.insert(1, Document()) + with self.assertRaises(ValueError): + doc.insert(1, Property(name="prop_one")) + with self.assertRaises(ValueError): + doc.insert(1, "some info") + self.assertEqual(len(doc), 3) + + # Test same name entries + with self.assertRaises(ValueError): + doc.insert(0, subsec) + self.assertEqual(len(doc), 3) diff --git a/test/test_doc_integration.py b/test/test_doc_integration.py new file mode 100644 index 00000000..aed6dbd2 --- /dev/null +++ b/test/test_doc_integration.py @@ -0,0 +1,166 @@ +""" +This file tests proper creation, saving and loading +of odML Documents with all supported odML parsers. +""" + +import os +import shutil +import tempfile +import unittest + +import odml + + +class TestDocumentIntegration(unittest.TestCase): + + def setUp(self): + # Set up test environment + self.tmp_dir = tempfile.mkdtemp(suffix=".odml") + + self.json_file = os.path.join(self.tmp_dir, "test.json") + self.xml_file = os.path.join(self.tmp_dir, "test.xml") + self.yaml_file = os.path.join(self.tmp_dir, "test.yaml") + + # Set up odML document stub + doc = odml.Document() + self.doc = doc + + def tearDown(self): + if os.path.exists(self.tmp_dir): + shutil.rmtree(self.tmp_dir) + + def save_load(self): + """ + Helper method to save and load the current state of the document + with all supported parsers. + :return: jdoc ... document loaded from JSON file + xdoc ... document loaded from XML file + ydoc ... document loaded from YAML file + """ + odml.save(self.doc, self.json_file, "JSON") + jdoc = odml.load(self.json_file, "JSON") + + odml.save(self.doc, self.xml_file) + xdoc = odml.load(self.xml_file) + + odml.save(self.doc, self.yaml_file, "YAML") + ydoc = odml.load(self.yaml_file, "YAML") + + return jdoc, xdoc, ydoc + + def test_id(self): + """ + This test checks the correct writing and loading of + autogenerated and assigned document id. + """ + # Test correct save and load of generated id. + jdoc, xdoc, ydoc = self.save_load() + + self.assertEqual(jdoc.id, self.doc.id) + self.assertEqual(xdoc.id, self.doc.id) + self.assertEqual(ydoc.id, self.doc.id) + + # Test correct save and load of assigned id. + assigned_id = "79b613eb-a256-46bf-84f6-207df465b8f7" + self.doc = odml.Document(id=assigned_id) + jdoc, xdoc, ydoc = self.save_load() + + self.assertEqual(jdoc.id, assigned_id) + self.assertEqual(xdoc.id, assigned_id) + self.assertEqual(ydoc.id, assigned_id) + + def test_simple_attributes(self): + """ + This test checks correct writing and loading of 'simple' + Document format attributes. + """ + author = "HPL" + version = "ver64" + date = "1890-08-20" + repository = "invalid" + + self.doc = odml.Document(author, date, version, repository) + jdoc, xdoc, ydoc = self.save_load() + + # Test correct JSON save and load. + self.assertEqual(jdoc.author, author) + self.assertEqual(jdoc.version, version) + self.assertEqual(str(jdoc.date), date) + self.assertEqual(jdoc.repository, repository) + + # Test correct XML save and load. + self.assertEqual(xdoc.author, author) + self.assertEqual(xdoc.version, version) + self.assertEqual(str(xdoc.date), date) + self.assertEqual(xdoc.repository, repository) + + # Test correct YAML save and load. + self.assertEqual(ydoc.author, author) + self.assertEqual(ydoc.version, version) + self.assertEqual(str(ydoc.date), date) + self.assertEqual(ydoc.repository, repository) + + def test_children(self): + """ + This test checks the correct saving and loading of Section children of a Document. + """ + # Lvl 1 child Sections + sec_lvl_11 = odml.Section(name="sec_11", parent=self.doc) + _ = odml.Section(name="sec_12", parent=self.doc) + + # Lvl 2 child Sections + sec_lvl_21 = odml.Section(name="sec_21", parent=sec_lvl_11) + _ = odml.Section(name="sec_22", parent=sec_lvl_11) + _ = odml.Section(name="sec_23", parent=sec_lvl_11) + + # Lvl 2 child Properties + _ = odml.Property(name="prop_21", parent=sec_lvl_11) + _ = odml.Property(name="prop_22", parent=sec_lvl_11) + _ = odml.Property(name="prop_23", parent=sec_lvl_11) + + # Lvl 3 child Sections + _ = odml.Section(name="sec_31", parent=sec_lvl_21) + _ = odml.Section(name="sec_32", parent=sec_lvl_21) + _ = odml.Section(name="sec_33", parent=sec_lvl_21) + _ = odml.Section(name="sec_34", parent=sec_lvl_21) + + # Lvl 3 child Properties + _ = odml.Property(name="prop_31", parent=sec_lvl_21) + _ = odml.Property(name="prop_32", parent=sec_lvl_21) + _ = odml.Property(name="prop_33", parent=sec_lvl_21) + _ = odml.Property(name="prop_34", parent=sec_lvl_21) + + jdoc, xdoc, ydoc = self.save_load() + + # Test correct JSON save and load. + self.assertEqual(len(jdoc.sections), 2) + + jsec_lvl_1 = jdoc[sec_lvl_11.name] + self.assertEqual(len(jsec_lvl_1.sections), 3) + self.assertEqual(len(jsec_lvl_1.properties), 3) + + jsec_lvl_2 = jsec_lvl_1[sec_lvl_21.name] + self.assertEqual(len(jsec_lvl_2.sections), 4) + self.assertEqual(len(jsec_lvl_2.properties), 4) + + # Test correct XML save and load. + self.assertEqual(len(xdoc.sections), 2) + + xsec_lvl_1 = xdoc[sec_lvl_11.name] + self.assertEqual(len(xsec_lvl_1.sections), 3) + self.assertEqual(len(xsec_lvl_1.properties), 3) + + xsec_lvl_2 = xsec_lvl_1[sec_lvl_21.name] + self.assertEqual(len(xsec_lvl_2.sections), 4) + self.assertEqual(len(xsec_lvl_2.properties), 4) + + # Test correct YAML save and load. + self.assertEqual(len(ydoc.sections), 2) + + ysec_lvl_1 = ydoc[sec_lvl_11.name] + self.assertEqual(len(ysec_lvl_1.sections), 3) + self.assertEqual(len(ysec_lvl_1.properties), 3) + + ysec_lvl_2 = ysec_lvl_1[sec_lvl_21.name] + self.assertEqual(len(ysec_lvl_2.sections), 4) + self.assertEqual(len(ysec_lvl_2.properties), 4) diff --git a/test/test_mdoc.py b/test/test_mdoc.py deleted file mode 100644 index 76a2fb6c..00000000 --- a/test/test_mdoc.py +++ /dev/null @@ -1,39 +0,0 @@ -import odml -import unittest -import datetime as dt - - -class TestMultiAppendDoc(unittest.TestCase): - - def test_mdoc(self): - jordan = odml.Document( - author="Michael Jordan", - date=dt.date(1991, 9, 1), - version=0.01 - ) - - mjordan = odml.Document( - author="Michael Jordan", - date=dt.date(1991, 9, 1), - version=0.01 - ) - - section_bulls = odml.Section( - name="Chicago_Bulls", - definition="NBA team based in Chicago, IL", - type="team" - ) - - section_nc = odml.Section( - name="North_Caroline", - definition="NCAA team based in Wilmington, NC", - type="team" - ) - jordan.append(section_bulls) - jordan.append(section_nc) - mjordan.append( - section_bulls, - section_nc, - ) - self.assertTrue(jordan.sections[0].name == mjordan.sections[0].name) - self.assertTrue(jordan.sections[1].name == mjordan.sections[1].name) diff --git a/test/test_mdoc_property_add.py b/test/test_mdoc_property_add.py deleted file mode 100644 index 891e1f5a..00000000 --- a/test/test_mdoc_property_add.py +++ /dev/null @@ -1,21 +0,0 @@ -import odml -import unittest -import datetime as dt - - -class TestAppendProperty(unittest.TestCase): - - def test_mdoc_property_add(self): - jordan = odml.Document( - author="Michael Jordan", - date=dt.date(1991, 9, 1), - version=0.01 - ) - - prop1 = odml.Property( - name="Coach", - value="Phil Jackson", - definition="Run the team." - ) - - self.assertRaises(KeyError, lambda: jordan.append(prop1))