Merge pull request #48 from sissaschool/develop

New major release v3.0.0

.github/workflows/test-elementpath.yml

@@ -44,13 +44,8 @@ jobs:
       - name: Lint with mypy if Python version != 3.7
         if: ${{ matrix.python-version != '3.7' }}
         run: |
-          pip install mypy==0.950
+          pip install mypy==0.961 lxml-stubs
           mypy --show-error-codes --strict elementpath
-      - name: Lint with mypy if Python version == 3.7
-        if: ${{ matrix.python-version == '3.7' }}
-        run: |
-          pip install mypy==0.950
-          mypy --show-error-codes --no-warn-redundant-casts --no-warn-unused-ignores --no-warn-return-any --strict elementpath
       - name: Test with unittest
         run: |
           pip install lxml xmlschema>=1.9.0

.gitignore

@@ -17,3 +17,4 @@ dist/
 build/
 development/
 out/
+profiling/out/

CHANGELOG.rst

@@ -2,6 +2,12 @@
 CHANGELOG
 *********
 
+`v3.0.0`_ (2022-07-16)
+======================
+* Transition to full XPath node implementation (more memory usage but
+  better control and overall faster)
+* Add etree.py module with a safe XML parser (ported from xmlschema)
+
 `v2.5.3`_ (2022-05-30)
 ======================
 * Fix unary path step operator (issue #46)
@@ -367,3 +373,4 @@ CHANGELOG
 .. _v2.5.1: https://github.com/sissaschool/elementpath/compare/v2.5.0...v2.5.1
 .. _v2.5.2: https://github.com/sissaschool/elementpath/compare/v2.5.1...v2.5.2
 .. _v2.5.3: https://github.com/sissaschool/elementpath/compare/v2.5.2...v2.5.3
+.. _v3.0.0: https://github.com/sissaschool/elementpath/compare/v2.5.3...v3.0.0

doc/conf.py

@@ -29,10 +29,9 @@
 author = 'Davide Brunato'
 
 # The short X.Y version
-version = '2.5'
+version = '3.0'
 # The full version, including alpha/beta/rc tags
-release = '2.5.3'
-
+release = '3.0.0'
 
 # -- General configuration ---------------------------------------------------
 

doc/xpath_api.rst

@@ -27,7 +27,6 @@ XPath parsers
 
     .. autoattribute:: DEFAULT_NAMESPACES
     .. autoattribute:: version
-    .. autoattribute:: default_namespace
 
     Helper methods for defining token classes:
 
@@ -104,22 +103,45 @@ implementing concrete interfaces to other types of XML Schema processors.
     .. automethod:: is_instance
     .. automethod:: cast_as
     .. automethod:: iter_atomic_types
-    .. automethod:: get_primitive_type
 
 
 XPath nodes
 ===========
 
 XPath nodes are processed using a set of classes derived from
-:class:`elementpath.XPathNode`:
+:class:`elementpath.XPathNode`. This class hierarchy is as simple
+as possible, with a focus on speed a low memory consumption.
 
 .. autoclass:: elementpath.XPathNode
 
+The seven XPath node types:
+
 .. autoclass:: elementpath.AttributeNode
-.. autoclass:: elementpath.TextNode
-.. autoclass:: elementpath.TypedAttribute
-.. autoclass:: elementpath.TypedElement
 .. autoclass:: elementpath.NamespaceNode
+.. autoclass:: elementpath.TextNode
+.. autoclass:: elementpath.CommentNode
+.. autoclass:: elementpath.ProcessingInstructionNode
+.. autoclass:: elementpath.ElementNode
+.. autoclass:: elementpath.DocumentNode
+
+There are also other two specialized versions of ElementNode
+usable on specific cases:
+
+.. autoclass:: elementpath.LazyElementNode
+.. autoclass:: elementpath.SchemaElementNode
+
+
+Node tree builders
+==================
+
+Node trees are automatically created during the initialization of an
+:class:`elementpath.XPathContext`. But if you need to process the same XML data
+more times there is an helper API for creating document or element based node trees:
+
+.. autofunction:: elementpath.get_node_tree
+.. autofunction:: elementpath.build_node_tree
+.. autofunction:: elementpath.build_lxml_node_tree
+.. autofunction:: elementpath.build_schema_node_tree
 
 
 XPath regular expressions
@@ -134,14 +156,15 @@ Exception classes
 .. autoexception:: elementpath.ElementPathError
 .. autoexception:: elementpath.MissingContextError
 .. autoexception:: elementpath.RegexError
+.. autoexception:: elementpath.ElementPathLocaleError
 
 There are also other exceptions, multiple derived from the base exception
 :class:`elementpath.ElementPathError` and Python built-in exceptions:
 
 .. autoexception:: elementpath.ElementPathKeyError
-.. autoexception:: elementpath.ElementPathLocaleError
 .. autoexception:: elementpath.ElementPathNameError
 .. autoexception:: elementpath.ElementPathOverflowError
+.. autoexception:: elementpath.ElementPathRuntimeError
 .. autoexception:: elementpath.ElementPathSyntaxError
 .. autoexception:: elementpath.ElementPathTypeError
 .. autoexception:: elementpath.ElementPathValueError

elementpath/__init__.py

@@ -7,36 +7,47 @@
 #
 # @author Davide Brunato <brunato@sissa.it>
 #
-__version__ = '2.5.3'
+__version__ = '3.0.0'
 __author__ = "Davide Brunato"
 __contact__ = "brunato@sissa.it"
 __copyright__ = "Copyright 2018-2022, SISSA"
 __license__ = "MIT"
 __status__ = "Production/Stable"
 
+# Imports here are considered as stable API, other internal calls may change.
 
-from .exceptions import ElementPathError, MissingContextError, \
-    ElementPathSyntaxError, ElementPathNameError, ElementPathKeyError, \
-    ElementPathTypeError, ElementPathLocaleError, ElementPathValueError, \
-    ElementPathOverflowError, ElementPathZeroDivisionError
+from . import datatypes  # XSD datatypes
+from . import etree      # Safe parser and helper functions for ElementTree
+from . import protocols  # Protocols for type annotations
+
+from .exceptions import ElementPathError, MissingContextError, ElementPathKeyError, \
+    ElementPathZeroDivisionError, ElementPathNameError, ElementPathOverflowError, \
+    ElementPathRuntimeError, ElementPathSyntaxError, ElementPathTypeError, \
+    ElementPathValueError, ElementPathLocaleError
 
-from . import datatypes
 from .xpath_context import XPathContext, XPathSchemaContext
-from .xpath_nodes import XPathNode, AttributeNode, TextNode, \
-    NamespaceNode, TypedElement, TypedAttribute
+from .xpath_nodes import XPathNode, DocumentNode, ElementNode, AttributeNode, \
+    NamespaceNode,  CommentNode, ProcessingInstructionNode, TextNode, \
+    LazyElementNode, SchemaElementNode
+from .tree_builders import get_node_tree, build_node_tree, build_lxml_node_tree, \
+    build_schema_node_tree
 from .xpath_token import XPathToken, XPathFunction
 from .xpath1 import XPath1Parser
 from .xpath2 import XPath2Parser
 from .xpath_selectors import select, iter_select, Selector
 from .schema_proxy import AbstractSchemaProxy
 from .regex import RegexError, translate_pattern
 
+TypedElement = ElementNode  # for backward compatibility with xmlschema<=1.10.0
 
-__all__ = ['ElementPathError', 'MissingContextError', 'ElementPathSyntaxError',
-           'ElementPathKeyError', 'ElementPathLocaleError', 'ElementPathNameError',
-           'ElementPathOverflowError', 'ElementPathValueError', 'ElementPathTypeError',
-           'ElementPathZeroDivisionError', 'datatypes', 'XPathContext', 'XPathSchemaContext',
-           'XPathNode', 'AttributeNode', 'TextNode', 'NamespaceNode', 'TypedAttribute',
-           'TypedElement', 'XPathToken', 'XPathFunction', 'XPath1Parser', 'XPath2Parser',
-           'select', 'iter_select', 'Selector', 'AbstractSchemaProxy',
-           'RegexError', 'translate_pattern']
+__all__ = ['datatypes', 'protocols', 'etree', 'ElementPathError', 'MissingContextError',
+           'ElementPathKeyError', 'ElementPathZeroDivisionError', 'ElementPathNameError',
+           'ElementPathOverflowError', 'ElementPathRuntimeError', 'ElementPathSyntaxError',
+           'ElementPathTypeError', 'ElementPathValueError', 'ElementPathLocaleError',
+           'XPathContext', 'XPathSchemaContext', 'XPathNode', 'DocumentNode',
+           'ElementNode', 'AttributeNode', 'NamespaceNode', 'CommentNode',
+           'ProcessingInstructionNode', 'TextNode', 'LazyElementNode',
+           'SchemaElementNode', 'TypedElement', 'get_node_tree', 'build_node_tree',
+           'build_lxml_node_tree', 'build_schema_node_tree', 'XPathToken',
+           'XPathFunction', 'XPath1Parser', 'XPath2Parser', 'select', 'iter_select',
+           'Selector', 'AbstractSchemaProxy', 'RegexError', 'translate_pattern']

elementpath/datatypes/__init__.py

@@ -13,9 +13,11 @@
 exceptions in order to be reusable in other packages.
 """
 from decimal import Decimal
-from typing import Dict, Union
+from typing import Dict, Optional, Union
 
 from ..helpers import QNAME_PATTERN  # For backward compatibility
+from ..namespaces import XSD_NAMESPACE
+from ..protocols import XsdTypeProtocol
 
 from .atomic_types import xsd10_atomic_types, xsd11_atomic_types, \
     AtomicTypeMeta, AnyAtomicType
@@ -71,58 +73,73 @@
                         UntypedAtomic, DatetimeValueType]
 
 
-ATOMIC_VALUES: Dict[str, AtomicValueType] = {
-    'untypedAtomic': UntypedAtomic('1'),
-    'anyType': UntypedAtomic('1'),
-    'anySimpleType': UntypedAtomic('1'),
-    'anyAtomicType': UntypedAtomic('1'),
-    'boolean': True,
-    'decimal': Decimal('1.0'),
-    'double': 1.0,
-    'float': Float10(1.0),
-    'string': '  alpha\t',
-    'date': Date.fromstring('2000-01-01'),
-    'dateTime': DateTime.fromstring('2000-01-01T12:00:00'),
-    'gDay': GregorianDay.fromstring('---31'),
-    'gMonth': GregorianMonth.fromstring('--12'),
-    'gMonthDay': GregorianMonthDay.fromstring('--12-01'),
-    'gYear': GregorianYear.fromstring('1999'),
-    'gYearMonth': GregorianYearMonth.fromstring('1999-09'),
-    'time': Time.fromstring('09:26:54'),
-    'duration': Duration.fromstring('P1MT1S'),
-    'dayTimeDuration': DayTimeDuration.fromstring('P1DT1S'),
-    'yearMonthDuration': YearMonthDuration.fromstring('P1Y1M'),
-    'QName': QName("http://www.w3.org/2001/XMLSchema", 'xs:element'),
-    'anyURI': AnyURI('https://example.com'),
-    'normalizedString': NormalizedString(' alpha  '),
-    'token': XsdToken('a token'),
-    'language': Language('en-US'),
-    'Name': Name('_a.name::'),
-    'NCName': NCName('nc-name'),
-    'ID': Id('id1'),
-    'IDREF': Idref('id_ref1'),
-    'ENTITY': Entity('entity1'),
-    'NMTOKEN': NMToken('a_token'),
-    'base64Binary': Base64Binary(b'YWxwaGE='),
-    'hexBinary': HexBinary(b'31'),
-    'dateTimeStamp': DateTimeStamp.fromstring('2000-01-01T12:00:00+01:00'),
-    'integer': Integer(1),
-    'long': Long(1),
-    'int': Int(1),
-    'short': Short(1),
-    'byte': Byte(1),
-    'positiveInteger': PositiveInteger(1),
-    'negativeInteger': NegativeInteger(-1),
-    'nonPositiveInteger': NonPositiveInteger(0),
-    'nonNegativeInteger': NonNegativeInteger(0),
-    'unsignedLong': UnsignedLong(1),
-    'unsignedInt': UnsignedInt(1),
-    'unsignedShort': UnsignedShort(1),
-    'unsignedByte': UnsignedByte(1),
+ATOMIC_VALUES: Dict[Optional[str], AtomicValueType] = {
+    f'{{{XSD_NAMESPACE}}}untypedAtomic': UntypedAtomic('1'),
+    f'{{{XSD_NAMESPACE}}}anyType': UntypedAtomic('1'),
+    f'{{{XSD_NAMESPACE}}}anySimpleType': UntypedAtomic('1'),
+    f'{{{XSD_NAMESPACE}}}anyAtomicType': UntypedAtomic('1'),
+    f'{{{XSD_NAMESPACE}}}boolean': True,
+    f'{{{XSD_NAMESPACE}}}decimal': Decimal('1.0'),
+    f'{{{XSD_NAMESPACE}}}double': 1.0,
+    f'{{{XSD_NAMESPACE}}}float': Float10(1.0),
+    f'{{{XSD_NAMESPACE}}}string': '  alpha\t',
+    f'{{{XSD_NAMESPACE}}}date': Date.fromstring('2000-01-01'),
+    f'{{{XSD_NAMESPACE}}}dateTime': DateTime.fromstring('2000-01-01T12:00:00'),
+    f'{{{XSD_NAMESPACE}}}gDay': GregorianDay.fromstring('---31'),
+    f'{{{XSD_NAMESPACE}}}gMonth': GregorianMonth.fromstring('--12'),
+    f'{{{XSD_NAMESPACE}}}gMonthDay': GregorianMonthDay.fromstring('--12-01'),
+    f'{{{XSD_NAMESPACE}}}gYear': GregorianYear.fromstring('1999'),
+    f'{{{XSD_NAMESPACE}}}gYearMonth': GregorianYearMonth.fromstring('1999-09'),
+    f'{{{XSD_NAMESPACE}}}time': Time.fromstring('09:26:54'),
+    f'{{{XSD_NAMESPACE}}}duration': Duration.fromstring('P1MT1S'),
+    f'{{{XSD_NAMESPACE}}}dayTimeDuration': DayTimeDuration.fromstring('P1DT1S'),
+    f'{{{XSD_NAMESPACE}}}yearMonthDuration': YearMonthDuration.fromstring('P1Y1M'),
+    f'{{{XSD_NAMESPACE}}}QName': QName("http://www.w3.org/2001/XMLSchema", 'xs:element'),
+    f'{{{XSD_NAMESPACE}}}anyURI': AnyURI('https://example.com'),
+    f'{{{XSD_NAMESPACE}}}normalizedString': NormalizedString(' alpha  '),
+    f'{{{XSD_NAMESPACE}}}token': XsdToken('a token'),
+    f'{{{XSD_NAMESPACE}}}language': Language('en-US'),
+    f'{{{XSD_NAMESPACE}}}Name': Name('_a.name::'),
+    f'{{{XSD_NAMESPACE}}}NCName': NCName('nc-name'),
+    f'{{{XSD_NAMESPACE}}}ID': Id('id1'),
+    f'{{{XSD_NAMESPACE}}}IDREF': Idref('id_ref1'),
+    f'{{{XSD_NAMESPACE}}}ENTITY': Entity('entity1'),
+    f'{{{XSD_NAMESPACE}}}NMTOKEN': NMToken('a_token'),
+    f'{{{XSD_NAMESPACE}}}base64Binary': Base64Binary(b'YWxwaGE='),
+    f'{{{XSD_NAMESPACE}}}hexBinary': HexBinary(b'31'),
+    f'{{{XSD_NAMESPACE}}}dateTimeStamp': DateTimeStamp.fromstring('2000-01-01T12:00:00+01:00'),
+    f'{{{XSD_NAMESPACE}}}integer': Integer(1),
+    f'{{{XSD_NAMESPACE}}}long': Long(1),
+    f'{{{XSD_NAMESPACE}}}int': Int(1),
+    f'{{{XSD_NAMESPACE}}}short': Short(1),
+    f'{{{XSD_NAMESPACE}}}byte': Byte(1),
+    f'{{{XSD_NAMESPACE}}}positiveInteger': PositiveInteger(1),
+    f'{{{XSD_NAMESPACE}}}negativeInteger': NegativeInteger(-1),
+    f'{{{XSD_NAMESPACE}}}nonPositiveInteger': NonPositiveInteger(0),
+    f'{{{XSD_NAMESPACE}}}nonNegativeInteger': NonNegativeInteger(0),
+    f'{{{XSD_NAMESPACE}}}unsignedLong': UnsignedLong(1),
+    f'{{{XSD_NAMESPACE}}}unsignedInt': UnsignedInt(1),
+    f'{{{XSD_NAMESPACE}}}unsignedShort': UnsignedShort(1),
+    f'{{{XSD_NAMESPACE}}}unsignedByte': UnsignedByte(1),
 }
 
-__all__ = ['xsd10_atomic_types', 'xsd11_atomic_types', 'AtomicTypeMeta', 'AnyAtomicType',
-           'ATOMIC_VALUES', 'XSD_BUILTIN_TYPES', 'NumericProxy', 'ArithmeticProxy',
+
+def get_atomic_value(xsd_type: Optional[XsdTypeProtocol]) -> AtomicValueType:
+    """Gets an atomic value for an XSD type instance. Used for schema contexts."""
+    if xsd_type is None:
+        return UntypedAtomic('1')
+
+    try:
+        return ATOMIC_VALUES[xsd_type.name]
+    except KeyError:
+        try:
+            return ATOMIC_VALUES[xsd_type.root_type.name]
+        except KeyError:
+            return UntypedAtomic('1')
+
+
+__all__ = ['xsd10_atomic_types', 'xsd11_atomic_types', 'get_atomic_value', 'AtomicTypeMeta',
+           'AnyAtomicType', 'XSD_BUILTIN_TYPES', 'NumericProxy', 'ArithmeticProxy',
            'QNAME_PATTERN', 'AbstractDateTime', 'DateTime10', 'DateTime', 'DateTimeStamp',
            'Date10', 'Date', 'Time', 'GregorianDay', 'GregorianMonth', 'GregorianMonthDay',
            'GregorianYear10', 'GregorianYear', 'GregorianYearMonth10', 'GregorianYearMonth',

elementpath/datatypes/qname.py

@@ -11,6 +11,7 @@
 
 from ..helpers import QNAME_PATTERN
 from .atomic_types import AtomicTypeMeta
+from .untyped import UntypedAtomic
 
 
 class AbstractQName(metaclass=AtomicTypeMeta):
@@ -74,6 +75,8 @@ def __hash__(self) -> int:
     def __eq__(self, other: object) -> bool:
         if isinstance(other, AbstractQName):
             return self.uri == other.uri and self.local_name == other.local_name
+        elif isinstance(other, (str, UntypedAtomic)):
+            return other == self.qname
         raise TypeError("cannot compare {!r} to {!r}".format(type(self), type(other)))