Skip to content

Commit

Permalink
Struct etc updated, _root _topmost context entries
Browse files Browse the repository at this point in the history
  • Loading branch information
@arekbulski
arekbulski committed Mar 28, 2018
1 parent 20015e9 commit 9105331
Show file tree
Hide file tree
Showing 5 changed files with 116 additions and 38 deletions.
75 changes: 39 additions & 36 deletions construct/core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,7 @@ def parse_stream(self, stream, **contextkw):
context = Container(**contextkw)
context._parsing = True
context._building = False
context._root = context
try:
return self._parsereport(stream, context, "(parsing)")
except CancelParsing:
Expand Down Expand Up @@ -361,6 +362,7 @@ def build_stream(self, obj, stream, **contextkw):
context = Container(**contextkw)
context._parsing = False
context._building = True
context._root = context
self._build(obj, stream, context, "(building)")

def build_file(self, obj, filename, **contextkw):
Expand Down Expand Up @@ -393,6 +395,7 @@ def sizeof(self, **contextkw):
context = Container(**contextkw)
context._parsing = False
context._building = False
context._root = context
return self._sizeof(context, "(sizeof)")

def _sizeof(self, context, path):
Expand Down Expand Up @@ -443,8 +446,6 @@ def reuse(obj, func):
""")
code.append("""
def parseall(io, this):
this['_parsing'] = True
this['_building'] = False
return %s
compiled = Compiled(None, None, parseall)
""" % (self._compileparse(code),))
Expand Down Expand Up @@ -2078,7 +2079,8 @@ def __getattr__(self, name):

def _parse(self, stream, context, path):
obj = Container(_stream = FeaturedBytesIO(stream))
context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context = Container(_ = context, _root = context._root, _topmost = None, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context._topmost = context._.get("_topmost", context)
for sc in self.subcons:
try:
subobj = sc._parsereport(stream, context, path)
Expand All @@ -2092,7 +2094,8 @@ def _parse(self, stream, context, path):
def _build(self, obj, stream, context, path):
if obj is None:
obj = Container()
context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context = Container(_ = context, _root = context._root, _topmost = None, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context._topmost = context._.get("_topmost", context)
context.update(obj)
for sc in self.subcons:
try:
Expand All @@ -2112,10 +2115,8 @@ def _build(self, obj, stream, context, path):
return context

def _sizeof(self, context, path):
context = Container(_ = context)
context._parsing = False
context._building = False
context._subcons = self._subcons
context = Container(_ = context, _root = context._root, _topmost = None, _parsing = context._parsing, _building = context._building, _subcons = self._subcons)
context._topmost = context._.get("_topmost", context)
try:
return sum(sc._sizeof(context, path) for sc in self.subcons)
except (KeyError, AttributeError):
Expand All @@ -2126,9 +2127,8 @@ def _emitparse(self, code):
block = """
def %s(io, this):
result = Container()
this = Container(_ = this)
this['_parsing'] = True
this['_building'] = False
this = Container(_ = this, _root = this['_root'], _topmost = None, _parsing = True, _building = False)
this['_topmost'] = this['_'].get('_topmost', this)
try:
""" % (fname, )
for sc in self.subcons:
Expand Down Expand Up @@ -2213,7 +2213,8 @@ def __getattr__(self, name):

def _parse(self, stream, context, path):
obj = ListContainer()
context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context = Container(_ = context, _root = context._root, _topmost = None, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context._topmost = context._.get("_topmost", context)
for i,sc in enumerate(self.subcons):
try:
subobj = sc._parsereport(stream, context, path)
Expand All @@ -2227,7 +2228,8 @@ def _parse(self, stream, context, path):
def _build(self, obj, stream, context, path):
if obj is None:
obj = ListContainer([None for sc in self.subcons])
context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context = Container(_ = context, _root = context._root, _topmost = None, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context._topmost = context._.get("_topmost", context)
retlist = ListContainer()
for i,(sc,subobj) in enumerate(zip(self.subcons, obj)):
try:
Expand All @@ -2243,10 +2245,8 @@ def _build(self, obj, stream, context, path):
return retlist

def _sizeof(self, context, path):
context = Container(_ = context)
context._parsing = False
context._building = False
context._subcons = self._subcons
context = Container(_ = context, _root = context._root, _topmost = None, _parsing = context._parsing, _building = context._building, _subcons = self._subcons)
context._topmost = context._.get("_topmost", context)
try:
return sum(sc._sizeof(context, path) for sc in self.subcons)
except (KeyError, AttributeError):
Expand All @@ -2257,7 +2257,8 @@ def _emitparse(self, code):
block = """
def %s(io, this):
result = ListContainer()
this = Container(_ = this)
this = Container(_ = this, _root = this['_root'], _topmost = None, _parsing = True, _building = False)
this['_topmost'] = this['_'].get('_topmost', this)
try:
""" % (fname,)
for sc in self.subcons:
Expand Down Expand Up @@ -3009,7 +3010,8 @@ def __getattr__(self, name):
raise AttributeError

def _parse(self, stream, context, path):
context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context = Container(_ = context, _root = context._root, _topmost = None, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context._topmost = context._.get("_topmost", context)
parsebuildfrom = evaluate(self.parsebuildfrom, context)
for i,sc in enumerate(self.subcons):
parseret = sc._parsereport(stream, context, path)
Expand All @@ -3020,7 +3022,8 @@ def _parse(self, stream, context, path):
return finalret

def _build(self, obj, stream, context, path):
context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context = Container(_ = context, _root = context._root, _topmost = None, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context._topmost = context._.get("_topmost", context)
parsebuildfrom = evaluate(self.parsebuildfrom, context)
context[parsebuildfrom] = obj
for i,sc in enumerate(self.subcons):
Expand All @@ -3032,10 +3035,8 @@ def _build(self, obj, stream, context, path):
return finalret

def _sizeof(self, context, path):
context = Container(_ = context)
context._parsing = False
context._building = False
context._subcons = self._subcons
context = Container(_ = context, _root = context._root, _topmost = None, _parsing = context._parsing, _building = context._building, _subcons = self._subcons)
context._topmost = context._.get("_topmost", context)
try:
return sum(sc._sizeof(context, path) for sc in self.subcons)
except (KeyError, AttributeError):
Expand All @@ -3046,7 +3047,8 @@ def _emitparse(self, code):
block = """
def %s(io, this):
result = []
this = Container(_ = this)
this = Container(_ = this, _root = this['_root'], _topmost = None, _parsing = True, _building = False)
this['_topmost'] = this['_'].get('_topmost', this)
""" % (fname, )
for sc in self.subcons:
block += """
Expand Down Expand Up @@ -3455,7 +3457,8 @@ def __getattr__(self, name):

def _parse(self, stream, context, path):
obj = Container()
context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context = Container(_ = context, _root = context._root, _topmost = None, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context._topmost = context._.get("_topmost", context)
fallback = _tell_stream(stream)
forwards = {}
for i,sc in enumerate(self.subcons):
Expand All @@ -3475,7 +3478,8 @@ def _parse(self, stream, context, path):
return obj

def _build(self, obj, stream, context, path):
context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context = Container(_ = context, _root = context._root, _topmost = None, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context._topmost = context._.get("_topmost", context)
context.update(obj)
for sc in self.subcons:
if sc.flagbuildnone:
Expand All @@ -3496,8 +3500,6 @@ def _build(self, obj, stream, context, path):
raise UnionError("cannot build, none of subcons were found in the dictionary %r" % (obj, ))

def _sizeof(self, context, path):
context._parsing = False
context._building = False
raise SizeofError("Union builds depending on actual object dict, size is unknown")

def _emitparse(self, code):
Expand All @@ -3506,7 +3508,8 @@ def _emitparse(self, code):
fname = "parse_union_%s" % code.allocateId()
block = """
def %s(io, this):
this = Container(_ = this)
this = Container(_ = this, _root = this['_root'], _topmost = None, _parsing = True, _building = False)
this['_topmost'] = this['_'].get('_topmost', this)
fallback = io.tell()
""" % (fname, )
if isinstance(self.parsefrom, type(None)):
Expand Down Expand Up @@ -5329,7 +5332,8 @@ def __getattr__(self, name):
raise AttributeError

def _parse(self, stream, context, path):
context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context = Container(_ = context, _root = context._root, _topmost = None, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context._topmost = context._.get("_topmost", context)
offset = _tell_stream(stream)
offsets = {0: offset}
values = {}
Expand All @@ -5350,7 +5354,8 @@ def _build(self, obj, stream, context, path):
# exact copy from Struct class
if obj is None:
obj = Container()
context = Container(_ = context, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context = Container(_ = context, _root = context._root, _topmost = None, _parsing = context._parsing, _building = context._building, _subcons = self._subcons, _stream = FeaturedBytesIO(stream))
context._topmost = context._.get("_topmost", context)
context.update(obj)
for sc in self.subcons:
try:
Expand All @@ -5371,10 +5376,8 @@ def _build(self, obj, stream, context, path):

def _sizeof(self, context, path):
# exact copy from Struct class
context = Container(_ = context)
context._parsing = False
context._building = False
context._subcons = self._subcons
context = Container(_ = context, _root = context._root, _topmost = None, _parsing = context._parsing, _building = context._building, _subcons = self._subcons)
context._topmost = context._.get("_topmost", context)
try:
return sum(sc._sizeof(context, path) for sc in self.subcons)
except (KeyError, AttributeError):
Expand Down
60 changes: 60 additions & 0 deletions docs/basics.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,66 @@ Container(data=b'1234')
Embedded structs should not be named, see :class:`~construct.core.Embedded` .


Hidden context entries
----------------------

There are few additional, hidden entries in the context. They are mostly used internally so they are not very well documented.

::

>>> d = Struct(
... 'x' / Computed(1),
... 'inner' / Struct(
... 'inner2' / Struct(
... 'x' / Computed(this._topmost.x),
... 'z' / Computed(this._root.z),
... 'zz' / Computed(this._topmost._.z),
... ),
... ),
... Probe(),
... )
>>> d.parse(b'', z=2)
--------------------------------------------------
Probe, path is (parsing), into is None
Container:
_ = Container:
z = 2
_parsing = True
_building = False
_root = <recursion detected>
_root = Container:
z = 2
_parsing = True
_building = False
_root = <recursion detected>
_topmost = <recursion detected>
_parsing = True
_building = False
_subcons = Container:
x = <Renamed x +nonbuild <Computed +nonbuild>>
inner = <Renamed inner +nonbuild <Struct +nonbuild>>
_stream = <_io.BytesIO object at 0x7f7e35a5fdb0>
x = 1
inner = Container:
_stream = <_io.BytesIO object at 0x7f7e35a5fdb0>
inner2 = Container:
_stream = <_io.BytesIO object at 0x7f7e35a5fdb0>
x = 1
z = 2
zz = 2
--------------------------------------------------

Explanation as follows:

* `_` means up-level in the context stack, every Struct does context nesting
* `_root` is the level on which externally provided values reside, those passed as parse() keyword arguments
* `_topmost` is the outer-most Struct, this entry might not exist if you do not use Structs
* `_parsing _building` are boolean values that are set by `parse build sizeof` public API methods
* `_subcons` is a list of Construct instances, this Struct members
* `_stream` is a BytesIO instance that has additional `size eof` methods as well as standard `seek tell` methods
* (parsed members are also added under matching names)


Sequences
=========

Expand Down
2 changes: 1 addition & 1 deletion docs/compilation.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ Sizeof is applied during compilation (not during parsing and building)

Exceptions do not include `path` information

Struct Sequence FocusedSeq Union LazyStruct do not support `_parsing _building _subcons _stream` context entries
Struct Sequence FocusedSeq Union LazyStruct do not support `_subcons _stream` context entries

Parsed hooks are not supported, ignored

Expand Down
4 changes: 3 additions & 1 deletion docs/transition29.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ Struct Sequence FocusedSeq Union LazyStruct have new embedding semantics (`see t

Struct Sequence FocusedSeq Union LazyStruct are exposing subcons, as attributes and in _subcons context entry (`see tutorial page <https://construct.readthedocs.io/en/latest/meta.html#refering-to-inlined-constructs>`_)

Struct Sequence FocusedSeq Union LazyStruct are exposing _parsing _building boolean entries and _stream entry in the context (`see tutorial page <https://construct.readthedocs.io/en/latest/misc.html#ifthenelse>`_)
Struct Sequence FocusedSeq Union LazyStruct are exposing _ _root _topmost _parsing _building _subcons _stream entries in the context (`see tutorial page <https://construct.readthedocs.io/en/latest/misc.html#ifthenelse>`_)

EmbeddedBitStruct removed, instead use BitStruct with Bytewise-wrapped fields (`see tutorial page <https://construct.readthedocs.io/en/latest/bitwise.html#fields-that-work-with-bytes>`_)

Expand Down Expand Up @@ -107,6 +107,8 @@ Support classes

Container updated, uses `globalPrintFullStrings globalPrintFalseFlags globalPrintPrivateEntries`

Container updated, equality does not check hidden keys like _private or keys order

FlagsContainer removed

RestreamedBytesIO supports reading till EOF, enabling GreedyBytes GreedyString inside Bitwise Bytewise
Expand Down
13 changes: 13 additions & 0 deletions tests/test_core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1833,6 +1833,19 @@ def test_struct_stream():
)
d.parse(bytes(20))

def test_struct_root_topmost():
d = Struct(
'x' / Computed(1),
'inner' / Struct(
'inner2' / Struct(
'x' / Computed(this._topmost.x),
'z' / Computed(this._root.z),
'zz' / Computed(this._topmost._.z),
),
),
)
assert d.parse(b"", z=2) == Container(x=1, inner=Container(inner2=Container(x=1,z=2,zz=2)))

def test_parsedhook_repeatersdiscard():
outputs = []
def printobj(obj, ctx):
Expand Down

0 comments on commit 9105331

Please sign in to comment.