FeaturedBytesIO added, Struct etc updated

construct/core.py

@@ -123,6 +123,26 @@ def _tell_stream(stream):
         raise StreamError("stream.tell() failed")
 
 
+def _size_stream(stream):
+    fallback = stream.tell()
+    end = stream.seek(0, 2)
+    stream.seek(fallback)
+    return end
+
+
+def _iseof_stream(stream):
+    fallback = stream.tell()
+    data = stream.read(1)
+    stream.seek(fallback)
+    return not data
+
+
+def FeaturedBytesIO(stream):
+    stream.size = lambda: _size_stream(stream)
+    stream.eof = lambda: _iseof_stream(stream)
+    return stream
+
+
 class CodeGen:
     def __init__(self):
         self.blocks = []
@@ -2057,8 +2077,8 @@ def __getattr__(self, name):
         raise AttributeError
 
     def _parse(self, stream, context, path):
-        obj = Container()
-        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons)
+        obj = Container(_stream = FeaturedBytesIO(stream))
+        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
         for sc in self.subcons:
             try:
                 subobj = sc._parsereport(stream, context, path)
@@ -2072,7 +2092,7 @@ def _parse(self, stream, context, path):
     def _build(self, obj, stream, context, path):
         if obj is None:
             obj = Container()
-        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons)
+        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
         context.update(obj)
         for sc in self.subcons:
             try:
@@ -2193,7 +2213,7 @@ def __getattr__(self, name):
 
     def _parse(self, stream, context, path):
         obj = ListContainer()
-        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons)
+        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
         for i,sc in enumerate(self.subcons):
             try:
                 subobj = sc._parsereport(stream, context, path)
@@ -2207,7 +2227,7 @@ def _parse(self, stream, context, path):
     def _build(self, obj, stream, context, path):
         if obj is None:
             obj = ListContainer([None for sc in self.subcons])
-        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons)
+        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
         retlist = ListContainer()
         for i,(sc,subobj) in enumerate(zip(self.subcons, obj)):
             try:
@@ -2989,7 +3009,7 @@ def __getattr__(self, name):
         raise AttributeError
 
     def _parse(self, stream, context, path):
-        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons)
+        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
         parsebuildfrom = evaluate(self.parsebuildfrom, context)
         for i,sc in enumerate(self.subcons):
             parseret = sc._parsereport(stream, context, path)
@@ -3000,7 +3020,7 @@ def _parse(self, stream, context, path):
         return finalret
 
     def _build(self, obj, stream, context, path):
-        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons)
+        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
         parsebuildfrom = evaluate(self.parsebuildfrom, context)
         context[parsebuildfrom] = obj
         for i,sc in enumerate(self.subcons):
@@ -3140,6 +3160,7 @@ def __init__(self, tuplename, tuplefields, subcon):
 
     def _decode(self, obj, context, path):
         if isinstance(self.subcon, Struct):
+            del obj["_stream"]
             return self.factory(**obj)
         if isinstance(self.subcon, (Sequence,Array,GreedyRange)):
             return self.factory(*obj)
@@ -3434,7 +3455,7 @@ def __getattr__(self, name):
 
     def _parse(self, stream, context, path):
         obj = Container()
-        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons)
+        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
         fallback = _tell_stream(stream)
         forwards = {}
         for i,sc in enumerate(self.subcons):
@@ -3454,7 +3475,7 @@ def _parse(self, stream, context, path):
         return obj
 
     def _build(self, obj, stream, context, path):
-        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons)
+        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
         context.update(obj)
         for sc in self.subcons:
             if sc.flagbuildnone:
@@ -5308,7 +5329,7 @@ def __getattr__(self, name):
         raise AttributeError
 
     def _parse(self, stream, context, path):
-        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons)
+        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
         offset = _tell_stream(stream)
         offsets = {0: offset}
         values = {}
@@ -5329,7 +5350,7 @@ def _build(self, obj, stream, context, path):
         # exact copy from Struct class
         if obj is None:
             obj = Container()
-        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons)
+        context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
         context.update(obj)
         for sc in self.subcons:
             try:

construct/lib/containers.py

@@ -205,14 +205,16 @@ def __eq__(self, other):
             return True
         if not isinstance(other, dict):
             return False
-        if len(self) != len(other):
-            return False
         def isequal(v1, v2):
             if v1.__class__.__name__ == "ndarray" or v2.__class__.__name__ == "ndarray":
                 import numpy
                 return numpy.array_equal(v1, v2)
             return v1 == v2
         for k,v in self.items():
+            if isinstance(k, unicodestringtype) and k.startswith(u"_"):
+                continue
+            if isinstance(k, bytestringtype) and k.startswith(b"_"):
+                continue
             if k not in other or not isequal(v, other[k]):
                 return False
         return True

docs/compilation.rst

@@ -28,7 +28,7 @@ Sizeof is applied during compilation (not during parsing and building)
 
 Exceptions do not include `path` information
 
-Struct Sequence FocusedSeq Union LazyStruct do not support `_parsing _building _subcons` context entries
+Struct Sequence FocusedSeq Union LazyStruct do not support `_parsing _building _subcons _stream` context entries
 
 Parsed hooks are not supported, ignored
 

docs/transition29.rst

@@ -43,9 +43,9 @@ Mapping replaced SymmetricMapping, and dropped `default` parameter (`see API pag
 
 Struct Sequence FocusedSeq Union LazyStruct have new embedding semantics (`see tutorial page <https://construct.readthedocs.io/en/latest/meta.html#nesting-and-embedding>`_)
 
-Struct Sequence FocusedSeq Union LazyStruct are exposing subcons, as attributes and in the context (`see tutorial page <https://construct.readthedocs.io/en/latest/meta.html#refering-to-inlined-constructs>`_)
+Struct Sequence FocusedSeq Union LazyStruct are exposing subcons, as attributes and in _subcons context entry (`see tutorial page <https://construct.readthedocs.io/en/latest/meta.html#refering-to-inlined-constructs>`_)
 
-Struct Sequence FocusedSeq Union LazyStruct are exposing _parsing _building boolean entries in the context (`see tutorial page <https://construct.readthedocs.io/en/latest/misc.html#ifthenelse>`_)
+Struct Sequence FocusedSeq Union LazyStruct are exposing _parsing _building boolean entries and _stream entry in the context (`see tutorial page <https://construct.readthedocs.io/en/latest/misc.html#ifthenelse>`_)
 
 EmbeddedBitStruct removed, instead use BitStruct with Bytewise-wrapped fields (`see tutorial page <https://construct.readthedocs.io/en/latest/bitwise.html#fields-that-work-with-bytes>`_)
 

tests/lib/test_containers_dict.py

@@ -249,8 +249,7 @@ def test_falseflags():
     setGlobalPrintFalseFlags()
 
 def test_privateentries():
-    d = Struct("_private" / Byte)
-    c = d.parse(b"\x01")
+    c = Container(_private = 1)
 
     setGlobalPrintPrivateEntries(True)
     assert str(c) == "Container: \n    _private = 1"

tests/test_core.py

@@ -1807,6 +1807,32 @@ def test_isparsingbuilding():
     )
     d.sizeof()
 
+def test_struct_stream():
+    d = Struct(
+        'fixed' / FixedSized(10, Struct(
+            'data' / GreedyBytes,
+            # check a substream
+            Check(lambda this: this._stream.size() == 10),
+            Check(lambda this: this._stream.eof()),
+            # checks parent original stream
+            Check(lambda this: this._._stream.size() == 20),
+            Check(lambda this: not this._._stream.eof()),
+        )),
+        # checks mid-parsing
+        Check(lambda this: this._stream.tell() == 10),
+        Check(lambda this: this._stream.size() == 20),
+        Check(lambda this: not this._stream.eof()),
+        'rest' / GreedyBytes,
+        # checks after parsed to EOF
+        Check(lambda this: this._stream.tell() == 20),
+        Check(lambda this: this._stream.size() == 20),
+        Check(lambda this: this._stream.eof()),
+        Check(lambda this: this._stream.seek(0,2) == 20),
+        # checks nested struct stream
+        Check(lambda this: this.fixed._stream.size() == 10),
+    )
+    d.parse(bytes(20))
+
 def test_parsedhook_repeatersdiscard():
     outputs = []
     def printobj(obj, ctx):