From 94d4d9f713b1ced3a7858eed5adc1e8b647621cc Mon Sep 17 00:00:00 2001 From: HansBug Date: Tue, 28 Dec 2021 12:28:26 +0800 Subject: [PATCH 01/26] dev(hansbug): add delayed proxy into treevalue base --- test/tree/common/test_delay.py | 68 +++++++++++++++++++++++++++++++ treevalue/tree/common/__init__.py | 1 + treevalue/tree/common/delay.pxd | 25 ++++++++++++ treevalue/tree/common/delay.pyx | 64 +++++++++++++++++++++++++++++ treevalue/tree/common/storage.pxd | 2 +- treevalue/tree/common/storage.pyx | 25 ++++++++++-- 6 files changed, 180 insertions(+), 5 deletions(-) create mode 100644 test/tree/common/test_delay.py create mode 100644 treevalue/tree/common/delay.pxd create mode 100644 treevalue/tree/common/delay.pyx diff --git a/test/tree/common/test_delay.py b/test/tree/common/test_delay.py new file mode 100644 index 0000000000..5a151df9f1 --- /dev/null +++ b/test/tree/common/test_delay.py @@ -0,0 +1,68 @@ +import pytest + +from treevalue.tree.common import DelayedProxy, delayed_partial + + +@pytest.mark.unittest +class TestTreeDelay: + def test_delayed_partial_simple(self): + cnt = 0 + + def f(): + nonlocal cnt + cnt += 1 + return 1 + + pv = delayed_partial(f) + assert cnt == 0 + assert isinstance(pv, DelayedProxy) + + assert pv.value() == 1 + assert cnt == 1 + + assert pv.value() == 1 + assert cnt == 1 + + def test_delayed_partial_func(self): + cnt = 0 + + def f(x, y): + nonlocal cnt + cnt += 1 + return x + y * 2 + 1 + + pv = delayed_partial(f, 2, y=3) + assert cnt == 0 + assert isinstance(pv, DelayedProxy) + + assert pv.value() == 9 + assert cnt == 1 + + assert pv.value() == 9 + assert cnt == 1 + + def test_delayed_partial_complex(self): + cnt1, cnt2 = 0, 0 + + def f1(): + nonlocal cnt1 + cnt1 += 1 + return 1 + + def f2(x, y): + nonlocal cnt2 + cnt2 += 1 + return (x + 1) ** 2 + y + 2 + + pv = delayed_partial(f2, delayed_partial(f1), delayed_partial(f1)) + assert cnt1 == 0 + assert cnt2 == 0 + assert isinstance(pv, DelayedProxy) + + assert pv.value() == 7 + assert cnt1 == 2 + assert cnt2 == 1 + + assert pv.value() == 7 + assert cnt1 == 2 + assert cnt2 == 1 diff --git a/treevalue/tree/common/__init__.py b/treevalue/tree/common/__init__.py index 0a74c28add..77f1121aa2 100644 --- a/treevalue/tree/common/__init__.py +++ b/treevalue/tree/common/__init__.py @@ -1,2 +1,3 @@ from .base import raw, unraw, RawWrapper +from .delay import DelayedProxy, delayed_partial, unwrap_proxy from .storage import TreeStorage, create_storage diff --git a/treevalue/tree/common/delay.pxd b/treevalue/tree/common/delay.pxd new file mode 100644 index 0000000000..2ac7cd81be --- /dev/null +++ b/treevalue/tree/common/delay.pxd @@ -0,0 +1,25 @@ +# distutils:language=c++ +# cython:language_level=3 + +from libcpp cimport bool + +cdef class DelayedProxy: + cpdef object value(self) + +cdef class DelayedValueProxy(DelayedProxy): + cdef readonly object func + cdef readonly bool calculated + cdef object val + + cpdef object value(self) + +cdef class DelayedFuncProxy(DelayedProxy): + cdef readonly object func + cdef readonly tuple args + cdef readonly dict kwargs + cdef readonly bool calculated + cdef object val + + cpdef object value(self) + +cpdef object unwrap_proxy(object proxy) diff --git a/treevalue/tree/common/delay.pyx b/treevalue/tree/common/delay.pyx new file mode 100644 index 0000000000..ac4f53beb9 --- /dev/null +++ b/treevalue/tree/common/delay.pyx @@ -0,0 +1,64 @@ +# distutils:language=c++ +# cython:language_level=3 + +import cython + +cdef class DelayedProxy: + cpdef object value(self): + raise NotImplementedError # pragma: no cover + +cdef class DelayedValueProxy(DelayedProxy): + def __cinit__(self, object func): + self.func = func + self.calculated = False + self.val = None + + cpdef object value(self): + cdef object f + if not self.calculated: + f = unwrap_proxy(self.func) + self.val = f() + self.calculated = True + + return self.val + +cdef class DelayedFuncProxy(DelayedProxy): + def __cinit__(self, object func, tuple args, dict kwargs): + self.func = func + self.args = args + self.kwargs = kwargs + self.calculated = False + self.val = None + + cpdef object value(self): + cdef list pas + cdef dict pks + cdef str key + cdef object item + cdef object f + if not self.calculated: + f = unwrap_proxy(self.func) + pas = [] + pks = {} + for item in self.args: + pas.append(unwrap_proxy(item)) + for key, item in self.kwargs.items(): + pks[key] = unwrap_proxy(item) + + self.val = f(*pas, **pks) + self.calculated = True + + return self.val + +def delayed_partial(func, *args, **kwargs): + if args or kwargs: + return DelayedFuncProxy(func, args, kwargs) + else: + return DelayedValueProxy(func) + +@cython.binding(True) +cpdef inline object unwrap_proxy(object proxy): + if isinstance(proxy, DelayedProxy): + return unwrap_proxy(proxy.value()) + else: + return proxy diff --git a/treevalue/tree/common/storage.pxd b/treevalue/tree/common/storage.pxd index e9a097440b..50153aaa4b 100644 --- a/treevalue/tree/common/storage.pxd +++ b/treevalue/tree/common/storage.pxd @@ -4,7 +4,7 @@ ctypedef unsigned char boolean ctypedef unsigned int uint -cdef void _key_validate(const char*key) except * +cdef void _key_validate(const char *key) except * cdef class TreeStorage: cdef readonly dict map diff --git a/treevalue/tree/common/storage.pyx b/treevalue/tree/common/storage.pyx index 8c87df520c..f598943831 100644 --- a/treevalue/tree/common/storage.pyx +++ b/treevalue/tree/common/storage.pyx @@ -6,11 +6,12 @@ from copy import deepcopy from libc.string cimport strlen from .base cimport raw, unraw +from .delay cimport unwrap_proxy, DelayedProxy cdef inline object _keep_object(object obj): return obj -cdef inline void _key_validate(const char*key) except *: +cdef inline void _key_validate(const char *key) except *: cdef int n = strlen(key) if n < 1: raise KeyError(f'Key {repr(key)} is too short, minimum length is 1 but {n} found.') @@ -35,8 +36,15 @@ cdef class TreeStorage: self.map[key] = value cpdef public object get(self, str key): + cdef object v, newv try: - return self.map[key] + v = self.map[key] + newv = unwrap_proxy(v) + if newv is not v: + self.map[key] = v + return newv + else: + return v except KeyError: raise KeyError(f"Key {repr(key)} not found in this tree.") @@ -70,11 +78,16 @@ cdef class TreeStorage: cpdef public dict jsondumpx(self, copy_func, object need_raw): cdef dict result = {} cdef str k - cdef object v, obj + cdef object v, obj, newv for k, v in self.map.items(): if isinstance(v, TreeStorage): result[k] = v.jsondumpx(copy_func, need_raw) else: + newv = unwrap_proxy(v) + if newv is not v: + v = newv + self.map[k] = v + obj = copy_func(v) if need_raw: obj = raw(obj) @@ -106,7 +119,7 @@ cdef class TreeStorage: cdef set keys = set(self.map.keys()) | set(detached.keys()) cdef str k - cdef object + cdef object v, nv cdef TreeStorage newv for k in keys: if k in detached: @@ -119,6 +132,10 @@ cdef class TreeStorage: newv.copy_from(v) self.map[k] = newv else: + nv = unwrap_proxy(v) + if nv is not v: + v = nv + detached[k] = v self.map[k] = copy_func(v) else: del self.map[k] From 00896958048ba2f753266486f13455fa965dc529 Mon Sep 17 00:00:00 2001 From: HansBug Date: Tue, 28 Dec 2021 14:35:54 +0800 Subject: [PATCH 02/26] dev(hansbug): add all delay support in storage --- test/tree/common/test_storage.py | 116 +++++++++++++++++++++++++++++- treevalue/tree/common/base.pyx | 4 +- treevalue/tree/common/delay.pyx | 11 +-- treevalue/tree/common/storage.pyx | 67 +++++++++++------ 4 files changed, 170 insertions(+), 28 deletions(-) diff --git a/test/tree/common/test_storage.py b/test/tree/common/test_storage.py index a55ea5d80e..1b538348e5 100644 --- a/test/tree/common/test_storage.py +++ b/test/tree/common/test_storage.py @@ -2,7 +2,7 @@ import pytest -from treevalue.tree.common import create_storage, raw, TreeStorage +from treevalue.tree.common import create_storage, raw, TreeStorage, delayed_partial # noinspection PyArgumentList,DuplicatedCode @@ -23,6 +23,52 @@ def test_get(self): with pytest.raises(KeyError): _ = t.get('fff') + cnt1, cnt2, cnt3 = 0, 0, 0 + + def f1(): + nonlocal cnt1 + cnt1 += 1 + return 2 + + def f2(x, y): + nonlocal cnt2 + cnt2 += 1 + return {'x': x, 'y': y} + + def f3(x, y): + nonlocal cnt3 + cnt3 += 1 + return create_storage({'x': x, 'y': raw(y)}) + + t2 = create_storage({ + 'a': 1, + 'b': delayed_partial(f1), + 'c': delayed_partial(f2, delayed_partial(f1), 3), + 'd': delayed_partial(f3, 3, delayed_partial(f2, 3, 4)) + }) + + assert t2.get('a') == 1 + + assert cnt1 == 0 + assert t2.get('b') == 2 + assert cnt1 == 1 + assert t2.get('b') == 2 + assert cnt1 == 1 + + assert (cnt1, cnt2) == (1, 0) + assert t2.get('c') == {'x': 2, 'y': 3} + assert (cnt1, cnt2) == (2, 1) + assert t2.get('c') == {'x': 2, 'y': 3} + assert (cnt1, cnt2) == (2, 1) + + assert (cnt1, cnt2, cnt3) == (2, 1, 0) + assert t2.get('d').get('x') == 3 + assert t2.get('d').get('y') == {'x': 3, 'y': 4} + assert (cnt1, cnt2, cnt3) == (2, 2, 1) + assert t2.get('d').get('x') == 3 + assert t2.get('d').get('y') == {'x': 3, 'y': 4} + assert (cnt1, cnt2, cnt3) == (2, 2, 1) + def test_get_or_default(self): t = create_storage({'a': 1, 'b': 2, 'c': raw({'x': 3, 'y': 4}), 'd': {'x': 3, 'y': 4}}) assert t.get_or_default('a', 233) == 1 @@ -122,6 +168,19 @@ def test_dump(self): assert _dumped['d']['x'] == 3 assert _dumped['d']['y'] == 4 + t2 = create_storage({ + 'a': 1, + 'b': delayed_partial(lambda x: x + 1, 1), + 'c': delayed_partial(lambda: h1), + 'd': delayed_partial(lambda: create_storage(h2)), + }) + _dumped = t2.dump() + assert _dumped['a'] == 1 + assert _dumped['b'] == 2 + assert _dumped['c'].value() is h1 + assert _dumped['d']['x'] == 3 + assert _dumped['d']['y'] == 4 + def test_deepdump(self): h1 = {'x': 3, 'y': 4} h2 = {'x': 3, 'y': 4} @@ -232,6 +291,19 @@ def test_copy_from(self): assert t1.get('f').get('y') == 4 assert t1.get('f') is not t.get('f') + t2 = create_storage({ + 'a': delayed_partial(lambda: 11), + 'b': delayed_partial(lambda: 22), + 'c': delayed_partial(lambda: {'x': 3, 'y': 5}), + 'd': delayed_partial(lambda: create_storage({'x': 3, 'y': 7})), + }) + t1.copy_from(t2) + assert t1.get('a') == 11 + assert t1.get('b') == 22 + assert t1.get('c') == {'x': 3, 'y': 5} + assert t1.get('d').get('x') == 3 + assert t1.get('d').get('y') == 7 + def test_deepcopy_from(self): h1 = {'x': 3, 'y': 4} h2 = {'x': 3, 'y': 4} @@ -272,6 +344,21 @@ def test_eq(self): assert t == t assert t == t1 assert t != t2 + assert t != None + + t3 = create_storage({ + 'a': delayed_partial(lambda: 11), + 'b': delayed_partial(lambda: 22), + 'c': delayed_partial(lambda: {'x': 3, 'y': 5}), + 'd': delayed_partial(lambda: create_storage({'x': 3, 'y': 7})), + }) + t4 = create_storage({ + 'a': delayed_partial(lambda: t3.get('a')), + 'b': delayed_partial(lambda: t3.get('b')), + 'c': delayed_partial(lambda: t3.get('c')), + 'd': delayed_partial(lambda: t3.get('d')), + }) + assert t3 == t4 def test_keys(self): h1 = {'x': 3, 'y': 4} @@ -286,10 +373,20 @@ def test_values(self): t = create_storage({'a': 1, 'b': 2, 'd': h1}) assert set(t.get('d').values()) == {3, 4} - assert len(t.values()) == 3 + assert len(list(t.values())) == 3 assert 1 in t.values() assert 2 in t.values() + t1 = create_storage({ + 'a': delayed_partial(lambda: t.get('a')), + 'b': delayed_partial(lambda: t.get('b')), + 'd': delayed_partial(lambda: t.get('d')), + }) + assert set(t1.get('d').values()) == {3, 4} + assert len(list(t1.values())) == 3 + assert 1 in t1.values() + assert 2 in t1.values() + def test_items(self): h1 = {'x': 3, 'y': 4} t = create_storage({'a': 1, 'b': 2, 'd': raw(h1)}) @@ -303,3 +400,18 @@ def test_items(self): assert v == h1 else: pytest.fail('Should not reach here.') + + t1 = create_storage({ + 'a': delayed_partial(lambda: t.get('a')), + 'b': delayed_partial(lambda: t.get('b')), + 'd': delayed_partial(lambda: t.get('d')), + }) + for k, v in t1.items(): + if k == 'a': + assert v == 1 + elif k == 'b': + assert v == 2 + elif k == 'd': + assert v == h1 + else: + pytest.fail('Should not reach here.') diff --git a/treevalue/tree/common/base.pyx b/treevalue/tree/common/base.pyx index 12f1b6c584..e25b38612c 100644 --- a/treevalue/tree/common/base.pyx +++ b/treevalue/tree/common/base.pyx @@ -39,7 +39,7 @@ cdef class RawWrapper: self.val = state @cython.binding(True) -cpdef public object raw(object obj): +cpdef inline object raw(object obj): """ Overview: Try wrap the given ``obj`` to raw wrapper. @@ -57,7 +57,7 @@ cpdef public object raw(object obj): return obj @cython.binding(True) -cpdef public object unraw(object wrapped): +cpdef inline object unraw(object wrapped): """ Overview: Try unwrap the given ``wrapped`` to original object. diff --git a/treevalue/tree/common/delay.pyx b/treevalue/tree/common/delay.pyx index ac4f53beb9..7b8f7b6f82 100644 --- a/treevalue/tree/common/delay.pyx +++ b/treevalue/tree/common/delay.pyx @@ -3,6 +3,8 @@ import cython +from .base cimport unraw + cdef class DelayedProxy: cpdef object value(self): raise NotImplementedError # pragma: no cover @@ -58,7 +60,8 @@ def delayed_partial(func, *args, **kwargs): @cython.binding(True) cpdef inline object unwrap_proxy(object proxy): - if isinstance(proxy, DelayedProxy): - return unwrap_proxy(proxy.value()) - else: - return proxy + cdef object p = proxy + while isinstance(p, DelayedProxy): + p = p.value() + + return unraw(p) diff --git a/treevalue/tree/common/storage.pyx b/treevalue/tree/common/storage.pyx index f598943831..1e19359e6e 100644 --- a/treevalue/tree/common/storage.pyx +++ b/treevalue/tree/common/storage.pyx @@ -6,7 +6,7 @@ from copy import deepcopy from libc.string cimport strlen from .base cimport raw, unraw -from .delay cimport unwrap_proxy, DelayedProxy +from .delay cimport unwrap_proxy cdef inline object _keep_object(object obj): return obj @@ -36,13 +36,13 @@ cdef class TreeStorage: self.map[key] = value cpdef public object get(self, str key): - cdef object v, newv + cdef object v, nv try: v = self.map[key] - newv = unwrap_proxy(v) - if newv is not v: - self.map[key] = v - return newv + nv = unwrap_proxy(v) + if nv is not v: + self.map[key] = nv + return nv else: return v except KeyError: @@ -78,16 +78,16 @@ cdef class TreeStorage: cpdef public dict jsondumpx(self, copy_func, object need_raw): cdef dict result = {} cdef str k - cdef object v, obj, newv + cdef object v, obj, nv for k, v in self.map.items(): + nv = unwrap_proxy(v) + if nv is not v: + v = nv + self.map[k] = v + if isinstance(v, TreeStorage): result[k] = v.jsondumpx(copy_func, need_raw) else: - newv = unwrap_proxy(v) - if newv is not v: - v = newv - self.map[k] = v - obj = copy_func(v) if need_raw: obj = raw(obj) @@ -124,6 +124,11 @@ cdef class TreeStorage: for k in keys: if k in detached: v = detached[k] + nv = unwrap_proxy(v) + if nv is not v: + v = nv + detached[k] = v + if isinstance(v, TreeStorage): if k in self.map and isinstance(self.map[k], TreeStorage): self.map[k].copy_from(v) @@ -132,10 +137,6 @@ cdef class TreeStorage: newv.copy_from(v) self.map[k] = newv else: - nv = unwrap_proxy(v) - if nv is not v: - v = nv - detached[k] = v self.map[k] = copy_func(v) else: del self.map[k] @@ -162,12 +163,22 @@ cdef class TreeStorage: cdef list other_keys = sorted(other.detach().keys()) cdef str key - cdef object self_v - cdef object other_v + cdef object self_v, self_nv + cdef object other_v, other_nv if self_keys == other_keys: for key in self_keys: self_v = self.map[key] + self_nv = unwrap_proxy(self_v) + if self_nv is not self_v: + self_v = self_nv + self.map[key] = self_v + other_v = other_map[key] + other_nv = unwrap_proxy(other_v) + if other_nv is not other_v: + other_v = other_nv + other_map[key] = other_v + if self_v != other_v: return False return True @@ -187,10 +198,26 @@ cdef class TreeStorage: return self.map.keys() def values(self): - return self.map.values() + cdef str k + cdef object v, nv + for k, v in self.map.items(): + nv = unwrap_proxy(v) + if nv is not v: + v = nv + self.map[k] = v + + yield v def items(self): - return self.map.items() + cdef str k + cdef object v, nv + for k, v in self.map.items(): + nv = unwrap_proxy(v) + if nv is not v: + v = nv + self.map[k] = v + + yield k, v cpdef object create_storage(dict value): cdef dict _map = {} From 683b07eb314ef6bbab62d6b9ba93586269906a6d Mon Sep 17 00:00:00 2001 From: HansBug Date: Tue, 28 Dec 2021 15:19:42 +0800 Subject: [PATCH 03/26] dev(hansbug): update get_or_default && add test for __hash__ of TreeStorage --- test/tree/common/test_storage.py | 40 ++++++++++++++++++++++++++++++- treevalue/tree/common/storage.pyx | 12 ++++++++-- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/test/tree/common/test_storage.py b/test/tree/common/test_storage.py index 1b538348e5..00aca123e2 100644 --- a/test/tree/common/test_storage.py +++ b/test/tree/common/test_storage.py @@ -5,7 +5,7 @@ from treevalue.tree.common import create_storage, raw, TreeStorage, delayed_partial -# noinspection PyArgumentList,DuplicatedCode +# noinspection PyArgumentList,DuplicatedCode,PyTypeChecker @pytest.mark.unittest class TestTreeStorage: def test_init(self): @@ -80,6 +80,23 @@ def test_get_or_default(self): assert t.get_or_default('fff', 233) == 233 + t1 = create_storage({ + 'a': delayed_partial(lambda: t.get('a')), + 'b': delayed_partial(lambda: t.get('b')), + 'c': delayed_partial(lambda: t.get('c')), + 'd': delayed_partial(lambda: t.get('d')), + }) + assert t1.get_or_default('a', 233) == 1 + assert t1.get_or_default('b', 233) == 2 + assert t1.get_or_default('c', 233) == {'x': 3, 'y': 4} + assert isinstance(t1.get_or_default('d', 233), TreeStorage) + assert t1.get_or_default('d', 233).get_or_default('x', 233) == 3 + assert t1.get_or_default('d', 233).get_or_default('y', 233) == 4 + + assert t1.get_or_default('fff', 233) == 233 + assert t1.get_or_default('fff', delayed_partial(lambda: 2345)) == 2345 + assert not t1.contains('fff') + def test_set(self): t = create_storage({}) t.set('a', 1) @@ -415,3 +432,24 @@ def test_items(self): assert v == h1 else: pytest.fail('Should not reach here.') + + def test_hash(self): + h = {} + + h1 = {'x': 3, 'y': 4} + t = create_storage({'a': 1, 'b': 2, 'd': h1}) + t1 = create_storage({ + 'a': delayed_partial(lambda: t.get('a')), + 'b': delayed_partial(lambda: t.get('b')), + 'd': delayed_partial(lambda: t.get('d')), + }) + t2 = create_storage({ + 'a': delayed_partial(lambda: t.get('a')), + 'b': delayed_partial(lambda: 3), + 'd': delayed_partial(lambda: t.get('d')), + }) + + h[t] = 1 + assert t1 in h + assert h[t1] == 1 + assert t2 not in h diff --git a/treevalue/tree/common/storage.pyx b/treevalue/tree/common/storage.pyx index 1e19359e6e..b942b1c9bd 100644 --- a/treevalue/tree/common/storage.pyx +++ b/treevalue/tree/common/storage.pyx @@ -49,7 +49,15 @@ cdef class TreeStorage: raise KeyError(f"Key {repr(key)} not found in this tree.") cpdef public object get_or_default(self, str key, object default): - return self.map.get(key, default) + cdef object v, nv + v = self.map.get(key, default) + nv = unwrap_proxy(v) + if nv is not v: + v = nv + if key in self.map: + self.map[key] = v + + return v cpdef public void del_(self, str key) except *: try: @@ -189,7 +197,7 @@ cdef class TreeStorage: cdef str k cdef object v cdef list _items = [] - for k, v in sorted(self.map.items(), key=lambda x: x[0]): + for k, v in sorted(self.items(), key=lambda x: x[0]): _items.append((k, v)) return hash(tuple(_items)) From 787b27730b7650d784e9c0690290920482f1596a Mon Sep 17 00:00:00 2001 From: HansBug Date: Tue, 28 Dec 2021 17:22:20 +0800 Subject: [PATCH 04/26] dev(hansbug): fix bug of deepcopyx && add lazy delay when no-deep copy --- test/tree/common/test_storage.py | 15 ++++++++- treevalue/tree/common/storage.pxd | 8 +++-- treevalue/tree/common/storage.pyx | 51 +++++++++++++++++-------------- treevalue/tree/tree/service.pyx | 14 +++++++-- 4 files changed, 58 insertions(+), 30 deletions(-) diff --git a/test/tree/common/test_storage.py b/test/tree/common/test_storage.py index 00aca123e2..9b895a4034 100644 --- a/test/tree/common/test_storage.py +++ b/test/tree/common/test_storage.py @@ -253,7 +253,7 @@ def test_deepcopyx(self): h2 = {'x': 3, 'y': 4} t = create_storage({'a': 1, 'b': 2, 'c': raw(h1), 'd': h2}) - t1 = t.deepcopyx(lambda x: -x if isinstance(x, int) else {'holy': 'shit'}) + t1 = t.deepcopyx(lambda x: -x if isinstance(x, int) else {'holy': 'shit'}, False) assert t1.get('a') == -1 assert t1.get('b') == -2 assert t1.get('c') == {'holy': 'shit'} @@ -342,6 +342,19 @@ def test_deepcopy_from(self): assert t1.get('f').get('y') == 4 assert t1.get('f') is not t.get('f') + t2 = create_storage({ + 'a': delayed_partial(lambda: 11), + 'b': delayed_partial(lambda: 22), + 'c': delayed_partial(lambda: {'x': 3, 'y': 5}), + 'd': delayed_partial(lambda: create_storage({'x': 3, 'y': 7})), + }) + t1.deepcopy_from(t2) + assert t1.get('a') == 11 + assert t1.get('b') == 22 + assert t1.get('c') == {'x': 3, 'y': 5} + assert t1.get('d').get('x') == 3 + assert t1.get('d').get('y') == 7 + def test_repr(self): h1 = {'x': 3, 'y': 4} h2 = {'x': 3, 'y': 4} diff --git a/treevalue/tree/common/storage.pxd b/treevalue/tree/common/storage.pxd index 50153aaa4b..986cb527be 100644 --- a/treevalue/tree/common/storage.pxd +++ b/treevalue/tree/common/storage.pxd @@ -1,6 +1,8 @@ # distutils:language=c++ # cython:language_level=3 +from libcpp cimport bool + ctypedef unsigned char boolean ctypedef unsigned int uint @@ -19,13 +21,13 @@ cdef class TreeStorage: cpdef public dict dump(self) cpdef public dict deepdump(self) cpdef public dict deepdumpx(self, copy_func) - cpdef public dict jsondumpx(self, copy_func, object need_raw) + cpdef public dict jsondumpx(self, copy_func, bool need_raw, bool allow_delayed) cpdef public TreeStorage copy(self) cpdef public TreeStorage deepcopy(self) - cpdef public TreeStorage deepcopyx(self, copy_func) + cpdef public TreeStorage deepcopyx(self, copy_func, bool allow_delayed) cpdef public dict detach(self) cpdef public void copy_from(self, TreeStorage ts) cpdef public void deepcopy_from(self, TreeStorage ts) - cpdef public void deepcopyx_from(self, TreeStorage ts, copy_func) + cpdef public void deepcopyx_from(self, TreeStorage ts, copy_func, bool allow_delayed) cpdef public object create_storage(dict value) diff --git a/treevalue/tree/common/storage.pyx b/treevalue/tree/common/storage.pyx index b942b1c9bd..421e791fea 100644 --- a/treevalue/tree/common/storage.pyx +++ b/treevalue/tree/common/storage.pyx @@ -4,6 +4,7 @@ from copy import deepcopy from libc.string cimport strlen +from libcpp cimport bool from .base cimport raw, unraw from .delay cimport unwrap_proxy @@ -81,22 +82,23 @@ cdef class TreeStorage: return self.deepdumpx(deepcopy) cpdef public dict deepdumpx(self, copy_func): - return self.jsondumpx(copy_func, True) + return self.jsondumpx(copy_func, True, False) - cpdef public dict jsondumpx(self, copy_func, object need_raw): + cpdef public dict jsondumpx(self, copy_func, bool need_raw, bool allow_delayed): cdef dict result = {} cdef str k cdef object v, obj, nv for k, v in self.map.items(): - nv = unwrap_proxy(v) - if nv is not v: - v = nv - self.map[k] = v + if not allow_delayed: + nv = unwrap_proxy(v) + if nv is not v: + v = nv + self.map[k] = v if isinstance(v, TreeStorage): - result[k] = v.jsondumpx(copy_func, need_raw) + result[k] = v.jsondumpx(copy_func, need_raw, allow_delayed) else: - obj = copy_func(v) + obj = copy_func(v) if not allow_delayed else v if need_raw: obj = raw(obj) result[k] = obj @@ -104,25 +106,24 @@ cdef class TreeStorage: return result cpdef public TreeStorage copy(self): - return self.deepcopyx(_keep_object) + return self.deepcopyx(_keep_object, True) cpdef public TreeStorage deepcopy(self): - return self.deepcopyx(deepcopy) + return self.deepcopyx(deepcopy, False) - cpdef public TreeStorage deepcopyx(self, copy_func): - cdef type cls = type(self) - return create_storage(self.deepdumpx(copy_func)) + cpdef public TreeStorage deepcopyx(self, copy_func, bool allow_delayed): + return create_storage(self.jsondumpx(copy_func, True, allow_delayed)) cpdef public dict detach(self): return self.map cpdef public void copy_from(self, TreeStorage ts): - self.deepcopyx_from(ts, _keep_object) + self.deepcopyx_from(ts, _keep_object, True) cpdef public void deepcopy_from(self, TreeStorage ts): - self.deepcopyx_from(ts, deepcopy) + self.deepcopyx_from(ts, deepcopy, False) - cpdef public void deepcopyx_from(self, TreeStorage ts, copy_func): + cpdef public void deepcopyx_from(self, TreeStorage ts, copy_func, bool allow_delayed): cdef dict detached = ts.detach() cdef set keys = set(self.map.keys()) | set(detached.keys()) @@ -132,20 +133,24 @@ cdef class TreeStorage: for k in keys: if k in detached: v = detached[k] - nv = unwrap_proxy(v) - if nv is not v: - v = nv - detached[k] = v + if not allow_delayed: + nv = unwrap_proxy(v) + if nv is not v: + v = nv + detached[k] = v if isinstance(v, TreeStorage): if k in self.map and isinstance(self.map[k], TreeStorage): - self.map[k].copy_from(v) + self.map[k].deepcopyx_from(v, copy_func, allow_delayed) else: newv = TreeStorage({}) - newv.copy_from(v) + newv.deepcopyx_from(v, copy_func, allow_delayed) self.map[k] = newv else: - self.map[k] = copy_func(v) + if not allow_delayed: + self.map[k] = copy_func(v) + else: + self.map[k] = v else: del self.map[k] diff --git a/treevalue/tree/tree/service.pyx b/treevalue/tree/tree/service.pyx index bd68f4af86..6d21c9436a 100644 --- a/treevalue/tree/tree/service.pyx +++ b/treevalue/tree/tree/service.pyx @@ -29,7 +29,7 @@ cpdef object jsonify(TreeValue val): Example: >>> jsonify(TreeValue({'a': 1, 'b': 2, 'x': {'c': 3, 'd': 4}})) # {'a': 1, 'b': 2, 'x': {'c': 3, 'd': 4}} """ - return val._detach().jsondumpx(_keep_object, False) + return val._detach().jsondumpx(_keep_object, False, False) @cython.binding(True) cpdef TreeValue clone(TreeValue t, object copy_value=None): @@ -50,10 +50,18 @@ cpdef TreeValue clone(TreeValue t, object copy_value=None): >>> t = TreeValue({'a': 1, 'b': 2, 'x': {'c': 3, 'd': 4}}) >>> clone(t.x) # TreeValue({'c': 3, 'd': 4}) """ + cdef bool need_copy if not callable(copy_value): - copy_value = copy.deepcopy if copy_value else _keep_object + if copy_value: + need_copy = True + copy_value = copy.deepcopy + else: + need_copy = False + copy_value = _keep_object + else: + need_copy = True - return type(t)(t._detach().deepcopyx(copy_value)) + return type(t)(t._detach().deepcopyx(copy_value, not need_copy)) @cython.binding(True) cpdef TreeValue typetrans(TreeValue t, object return_type): From da028d58b88fc6a793e4d799dc8c13036a7c2f71 Mon Sep 17 00:00:00 2001 From: HansBug Date: Tue, 28 Dec 2021 22:53:27 +0800 Subject: [PATCH 05/26] dev(hansbug): add detached delay proxy --- test/tree/tree/test_tree.py | 16 ++++++++++++- treevalue/tree/common/__init__.py | 2 +- treevalue/tree/common/delay.pxd | 4 +++- treevalue/tree/common/delay.pyx | 33 +++++++++++++++---------- treevalue/tree/common/storage.pyx | 18 +++++++------- treevalue/tree/tree/__init__.py | 2 +- treevalue/tree/tree/tree.pxd | 11 +++++++++ treevalue/tree/tree/tree.pyx | 40 ++++++++++++++++++++++++++++++- 8 files changed, 99 insertions(+), 27 deletions(-) diff --git a/test/tree/tree/test_tree.py b/test/tree/tree/test_tree.py index 6f27c6d169..9bd03b9d33 100644 --- a/test/tree/tree/test_tree.py +++ b/test/tree/tree/test_tree.py @@ -3,7 +3,7 @@ import pytest -from treevalue import raw, TreeValue +from treevalue import raw, TreeValue, delayed class _Container: @@ -121,6 +121,20 @@ def test_tree_value_repr(self): assert "c --> )" in repr(tv2) + tv3 = TreeValue({ + 'a': delayed(lambda: tv1.a), + 'b': delayed(lambda: tv1.b), + 'c': delayed(lambda: tv1.c), + }) + + assert re.match(r"", repr(tv3)) + assert re.match(r"", repr(tv3.c)) + assert "a --> 1" in str(tv3) + assert "b --> 2" in str(tv3) + assert "x --> 2" in str(tv3) + assert "y --> 3" in str(tv3) + assert "c --> ' if isinstance(v, TreeStorage): @@ -404,3 +410,35 @@ cdef object _build_tree(TreeStorage st, object type_, str prefix, dict id_pool, self_repr = _prefix_fix(self_repr, prefix) return self_repr, children + +cdef class DetachedDelayedProxy(DelayedProxy): + def __init__(self, DelayedProxy proxy): + self.proxy = proxy + self.calculated = False + self.val = None + + cpdef object value(self): + if not self.calculated: + self.val = undelay(self.proxy, False) + self.calculated = True + + return self.val + + cpdef object fvalue(self): + cdef object v = self.value() + if isinstance(v, TreeValue): + v = v._detach() + return v + +@cython.binding(True) +def delayed(func, *args, **kwargs): + r""" + Overview: + Use delayed function in treevalue. + + Arguments: + - func: Delayed function. + - args: Positional arguments. + - kwargs: Key-word arguments. + """ + return DetachedDelayedProxy(_c_delayed_partial(func, args, kwargs)) From 82b32076401149ca252505af8fe45ee7795f729b Mon Sep 17 00:00:00 2001 From: HansBug Date: Tue, 28 Dec 2021 23:01:26 +0800 Subject: [PATCH 06/26] dev(hansbug): add delay support in flatten.pyx --- treevalue/tree/tree/flatten.pyx | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/treevalue/tree/tree/flatten.pyx b/treevalue/tree/tree/flatten.pyx index 0db0fc71e1..aa38e822f1 100644 --- a/treevalue/tree/tree/flatten.pyx +++ b/treevalue/tree/tree/flatten.pyx @@ -6,6 +6,7 @@ import cython from .tree cimport TreeValue +from ..common.delay cimport undelay from ..common.storage cimport TreeStorage cdef void _c_flatten(TreeStorage st, tuple path, list res) except *: @@ -13,8 +14,13 @@ cdef void _c_flatten(TreeStorage st, tuple path, list res) except *: cdef tuple curpath cdef str k - cdef object v + cdef object v, nv for k, v in data.items(): + nv = undelay(v) + if nv is not v: + v = nv + data[k] = v + curpath = path + (k,) if isinstance(v, TreeStorage): _c_flatten(v, curpath, res) @@ -47,8 +53,13 @@ cdef void _c_flatten_values(TreeStorage st, list res) except *: cdef dict data = st.detach() cdef str k - cdef object v + cdef object v, nv for k, v in data.items(): + nv = undelay(v) + if nv is not v: + v = nv + data[k] = v + if isinstance(v, TreeStorage): _c_flatten_values(v, res) else: @@ -75,8 +86,13 @@ cdef void _c_flatten_keys(TreeStorage st, tuple path, list res) except *: cdef tuple curpath cdef str k - cdef object v + cdef object v, nv for k, v in data.items(): + nv = undelay(v) + if nv is not v: + v = nv + data[k] = v + curpath = path + (k,) if isinstance(v, TreeStorage): _c_flatten_keys(v, curpath, res) From 6e13476458a2ef197790e0b38ebcb48018033a4a Mon Sep 17 00:00:00 2001 From: HansBug Date: Tue, 28 Dec 2021 23:10:47 +0800 Subject: [PATCH 07/26] dev(hansbug): test delayed flatten --- test/tree/tree/test_flatten.py | 46 +++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/test/tree/tree/test_flatten.py b/test/tree/tree/test_flatten.py index ac47d9000f..564f221872 100644 --- a/test/tree/tree/test_flatten.py +++ b/test/tree/tree/test_flatten.py @@ -1,6 +1,6 @@ import pytest -from treevalue.tree import TreeValue, raw, flatten, unflatten, flatten_values, flatten_keys +from treevalue.tree import TreeValue, raw, flatten, unflatten, flatten_values, flatten_keys, delayed class MyTreeValue(TreeValue): @@ -22,15 +22,37 @@ def test_flatten(self): (('d', 'y'), 4) ] + t1 = TreeValue({ + 'a': delayed(lambda: t.a), + 'b': delayed(lambda: t.b), + 'c': delayed(lambda: t.c), + 'd': delayed(lambda: t.d), + }) + flatted = sorted(flatten(t1)) + assert flatted == [ + (('a',), 1), + (('b',), 2), + (('c',), {'x': 3, 'y': 4}), + (('d', 'x'), 3), + (('d', 'y'), 4) + ] + def test_flatten_values(self): t = TreeValue({'a': 1, 'b': 5, 'c': {'x': 3, 'y': 4}, 'd': {'x': 3, 'y': 4}}) - flatted_values = sorted(flatten_values(t)) assert flatted_values == [1, 3, 3, 4, 4, 5] + t1 = TreeValue({ + 'a': delayed(lambda: t.a), + 'b': delayed(lambda: t.b), + 'c': delayed(lambda: t.c), + 'd': delayed(lambda: t.d), + }) + flatted_values = sorted(flatten_values(t1)) + assert flatted_values == [1, 3, 3, 4, 4, 5] + def test_flatten_keys(self): t = TreeValue({'a': 1, 'd': {'x': 3, 'y': 4}, 'e': raw({'x': 3, 'y': 4}), 'b': 5, 'c': {'x': 3, 'y': 4}}) - flatted_keys = sorted(flatten_keys(t)) assert flatted_keys == [ ('a',), @@ -42,6 +64,24 @@ def test_flatten_keys(self): ('e',), ] + t1 = TreeValue({ + 'a': delayed(lambda: t.a), + 'b': delayed(lambda: t.b), + 'c': delayed(lambda: t.c), + 'd': delayed(lambda: t.d), + 'e': delayed(lambda: t.e), + }) + flatted_keys = sorted(flatten_keys(t1)) + assert flatted_keys == [ + ('a',), + ('b',), + ('c', 'x',), + ('c', 'y',), + ('d', 'x',), + ('d', 'y',), + ('e',), + ] + def test_unflatten(self): flatted = [ (('a',), 1), From 9e515b948a180f692a1c72537ca81ccc2cf8d106 Mon Sep 17 00:00:00 2001 From: HansBug Date: Tue, 28 Dec 2021 23:19:35 +0800 Subject: [PATCH 08/26] dev(hansbug): add delay support for service part --- test/tree/tree/test_service.py | 34 ++++++++++++++++++++++++++++++++- treevalue/tree/tree/service.pyx | 8 +++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/test/tree/tree/test_service.py b/test/tree/tree/test_service.py index 2745918085..f3c3509ad9 100644 --- a/test/tree/tree/test_service.py +++ b/test/tree/tree/test_service.py @@ -1,6 +1,6 @@ import pytest -from treevalue.tree import jsonify, TreeValue, clone, typetrans, raw, walk +from treevalue.tree import jsonify, TreeValue, clone, typetrans, raw, walk, delayed # noinspection DuplicatedCode @@ -19,6 +19,18 @@ def test_jsonify(self): } assert tv2.c == {'x': 2, 'y': 3} + tv3 = TreeValue({ + 'a': delayed(lambda: tv1.a), + 'b': delayed(lambda: tv1.b), + 'c1': delayed(lambda: tv1.c), + 'c2': delayed(lambda: tv2.c), + }) + assert jsonify(tv3) == { + 'a': 1, 'b': 2, 'c1': {'x': 2, 'y': 3}, 'c2': {'x': 2, 'y': 3} + } + assert tv3.c1 == TreeValue({'x': 2, 'y': 3}) + assert tv3.c2 == {'x': 2, 'y': 3} + def test_clone(self): tv1 = TreeValue({'a': 1, 'b': 2, 'c': {'x': 2, 'y': 3}}) tv2 = clone(tv1) @@ -63,6 +75,14 @@ def test_clone(self): assert tv5.x.c is not tv3.x.c assert tv5.x.d is not tv3.x.d + tv6 = TreeValue({ + 'a': delayed(lambda: tv3.a), + 'b': delayed(lambda: tv3.b), + 'x': delayed(lambda: tv3.x), + }) + tv7 = clone(tv6, lambda x: x) + assert tv7 == tv3 + def test_typetrans(self): class MyTreeValue(TreeValue): pass @@ -99,3 +119,15 @@ class MyTreeValue(TreeValue): ('c', 'x',): 2, ('c', 'y',): 3, } + + tv2 = MyTreeValue({ + 'a': delayed(lambda: tv1.a), + 'b': delayed(lambda: tv1.b), + 'c': delayed(lambda: tv1.c), + }) + assert dict(walk(tv2)) == { + ('a',): 1, + ('b',): 2, + ('c', 'x',): 2, + ('c', 'y',): 3, + } diff --git a/treevalue/tree/tree/service.pyx b/treevalue/tree/tree/service.pyx index 6d21c9436a..ed373e02f4 100644 --- a/treevalue/tree/tree/service.pyx +++ b/treevalue/tree/tree/service.pyx @@ -9,6 +9,7 @@ import cython from libcpp cimport bool from .tree cimport TreeValue +from ..common.delay cimport undelay from ..common.storage cimport TreeStorage cdef object _keep_object(object obj): @@ -95,9 +96,14 @@ def _p_walk(TreeStorage tree, object type_, tuple path, bool include_nodes): cdef dict data = tree.detach() cdef str k - cdef object v + cdef object v, nv cdef tuple curpath for k, v in data.items(): + nv = undelay(v) + if nv is not v: + v = nv + data[k] = v + curpath = path + (k,) if isinstance(v, TreeStorage): yield from _p_walk(v, type_, curpath, include_nodes) From cd9eab68e515ba626bb99d365e5a21572d252273 Mon Sep 17 00:00:00 2001 From: HansBug Date: Tue, 28 Dec 2021 23:40:59 +0800 Subject: [PATCH 09/26] dev(hansbug): add delay support for function treelize --- test/tree/func/test_func.py | 41 ++++++++++++++++++++++++++++++++++- treevalue/tree/func/cfunc.pyx | 23 +++++++++++++++----- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/test/tree/func/test_func.py b/test/tree/func/test_func.py index 124b2342ad..6cd86b500b 100644 --- a/test/tree/func/test_func.py +++ b/test/tree/func/test_func.py @@ -3,7 +3,7 @@ import pytest -from treevalue.tree import func_treelize, TreeValue, method_treelize, classmethod_treelize +from treevalue.tree import func_treelize, TreeValue, method_treelize, classmethod_treelize, delayed # noinspection DuplicatedCode @@ -257,3 +257,42 @@ def append(arr: list, *args): 'f': [4], }, }) + + def test_delay_support(self): + @func_treelize(return_type=TreeValue) + def f(x, y, z): + return x + y * 2 + z * 3 + + t1 = TreeValue({ + 'a': 1, + 'b': delayed(lambda x: x ** 2, 3), + 'c': {'x': 2, 'y': delayed(lambda: 4)}, + }) + t2 = TreeValue({ + 'a': delayed(lambda x: x + 1, t1.a), + 'b': delayed(lambda: t1.c.y), + 'c': delayed(lambda: 5), + }) + t3 = delayed(lambda: 6) + + assert f(t1, t2, t3) == TreeValue({ + 'a': 23, 'b': 35, + 'c': {'x': 30, 'y': 32}, + }) + + t1 = TreeValue({ + 'a': 1, + 'b': delayed(lambda x: x ** 2, 3), + 'c': {'x': 2, 'y': delayed(lambda: 4)}, + }) + t2 = TreeValue({ + 'a': delayed(lambda x: x + 1, t1.a), + 'b': delayed(lambda: t1.c.y), + 'c': delayed(lambda: 5), + }) + t3 = delayed(lambda: 6) + + assert f(x=t1, y=t2, z=t3) == TreeValue({ + 'a': 23, 'b': 35, + 'c': {'x': 30, 'y': 32}, + }) diff --git a/treevalue/tree/func/cfunc.pyx b/treevalue/tree/func/cfunc.pyx index 3c6f0a36a3..38e8f985d6 100644 --- a/treevalue/tree/func/cfunc.pyx +++ b/treevalue/tree/func/cfunc.pyx @@ -8,6 +8,7 @@ from hbutils.design import SingletonMark from libcpp cimport bool from .modes cimport _e_tree_mode, _c_keyset, _c_load_mode, _c_check +from ..common.delay cimport undelay from ..common.storage cimport TreeStorage from ..tree.structural cimport _c_subside, _c_rise from ..tree.tree cimport TreeValue @@ -21,7 +22,7 @@ cdef object _c_func_treelize_run(object func, list args, dict kwargs, cdef bool has_tree = False cdef str k - cdef object v + cdef object v, nv for v in args: if isinstance(v, TreeStorage): ck_args.append((v.detach(), True)) @@ -66,7 +67,13 @@ cdef object _c_func_treelize_run(object func, list args, dict kwargs, for i, (av, at) in enumerate(ck_args): if at: try: - _l_args.append(av[k]) + v = av[k] + nv = undelay(v) + if nv is not v: + v = nv + av[k] = v + + _l_args.append(v) except KeyError: if allow_missing: _l_args.append(_VALUE_IS_MISSING) @@ -76,7 +83,7 @@ cdef object _c_func_treelize_run(object func, list args, dict kwargs, )) else: if inherit: - _l_args.append(av) + _l_args.append(undelay(av)) else: raise TypeError("Inherit is off, tree value expected but {type} found in args {index}.".format( type=repr(type(av).__name__), index=repr(i), @@ -86,7 +93,13 @@ cdef object _c_func_treelize_run(object func, list args, dict kwargs, for ak, av, at in ck_kwargs: if at: try: - _d_kwargs[ak] = av[k] + v = av[k] + nv = undelay(v) + if nv is not v: + v = nv + av[k] = v + + _d_kwargs[ak] = v except KeyError: if allow_missing: _d_kwargs[ak] = _VALUE_IS_MISSING @@ -96,7 +109,7 @@ cdef object _c_func_treelize_run(object func, list args, dict kwargs, )) else: if inherit: - _d_kwargs[ak] = av + _d_kwargs[ak] = undelay(av) else: raise TypeError("Inherit is off, tree value expected but {type} found in args {index}.".format( type=repr(type(av).__name__), index=repr(ak), From f39907577451afe5cd7581b9538e3d4ad1417c15 Mon Sep 17 00:00:00 2001 From: HansBug Date: Wed, 29 Dec 2021 00:03:21 +0800 Subject: [PATCH 10/26] dev(hansbug): add delayed support in functional part --- test/tree/tree/test_functional.py | 19 ++++++++++++++++- treevalue/tree/tree/functional.pyx | 33 ++++++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/test/tree/tree/test_functional.py b/test/tree/tree/test_functional.py index 8fcd9ccbcb..7b5ff49ee8 100644 --- a/test/tree/tree/test_functional.py +++ b/test/tree/tree/test_functional.py @@ -3,7 +3,7 @@ import pytest -from treevalue.tree import TreeValue, mapping, raw, mask, filter_, reduce_ +from treevalue.tree import TreeValue, mapping, raw, mask, filter_, reduce_, delayed # noinspection DuplicatedCode @@ -30,6 +30,11 @@ def test_mapping(self): }) assert tv6 == TreeValue({'a': 1.0, 'b': 2.0, 'c': {'x': 2.0, 'y': 3.0}}) + tv8 = TreeValue({'v': delayed(lambda: tv1)}) + assert mapping(tv8, lambda x: x + 2) == TreeValue({'v': { + 'a': 3, 'b': 4, 'c': {'x': 4, 'y': 5} + }}) + def test_mask(self): class MyTreeValue(TreeValue): pass @@ -46,6 +51,10 @@ class MyTreeValue(TreeValue): with pytest.raises(TypeError): assert mask(t2, m2) + t1 = TreeValue({'v': delayed(lambda: t)}) + m11 = TreeValue({'v': delayed(lambda: m1)}) + assert mask(t1, m11) == TreeValue({'v': {'a': 1}}) + def test_filter(self): class MyTreeValue(TreeValue): pass @@ -56,6 +65,9 @@ class MyTreeValue(TreeValue): assert filter_(t, lambda x: x < 3, remove_empty=False) == MyTreeValue({'a': 1, 'b': 2, 'x': {}}) assert filter_(t, lambda x: x % 2 == 1) == MyTreeValue({'a': 1, 'x': {'c': 3}}) + t2 = TreeValue({'v': delayed(lambda: t)}) + assert filter_(t2, lambda x: x < 3) == TreeValue({'v': {'a': 1, 'b': 2}}) + def test_reduce(self): class MyTreeValue(TreeValue): pass @@ -74,3 +86,8 @@ class MyTreeValue(TreeValue): assert reduce_(t2, lambda **kwargs: TreeValue( {k + k: (v ** 2 if not isinstance(v, TreeValue) else v) for k, v in kwargs.items()})) == MyTreeValue( {'aa': 1, 'bb': 4, 'xx': {'cc': 9, 'dd': 16}}) + + t1 = MyTreeValue({'a': 1, 'b': 2, 'x': {'c': 3, 'd': 4}}) + t3 = TreeValue({'v': delayed(lambda: t1), 'v2': delayed(lambda: t1)}) + assert reduce_(t3, lambda **kwargs: sum(kwargs.values())) == 20 + assert reduce_(t3, lambda **kwargs: reduce(__mul__, list(kwargs.values()))) == 576 diff --git a/treevalue/tree/tree/functional.pyx b/treevalue/tree/tree/functional.pyx index 80a7c83c4a..03c651bdf3 100644 --- a/treevalue/tree/tree/functional.pyx +++ b/treevalue/tree/tree/functional.pyx @@ -7,6 +7,7 @@ import cython from libcpp cimport bool from .tree cimport TreeValue +from ..common.delay cimport undelay from ..common.storage cimport TreeStorage cdef class _ValuePathFuncWrapper: @@ -32,9 +33,14 @@ cdef TreeStorage _c_mapping(TreeStorage st, object func, tuple path): cdef dict _d_res = {} cdef str k - cdef object v + cdef object v, nv cdef tuple curpath for k, v in _d_st.items(): + nv = undelay(v) + if nv is not v: + v = nv + _d_st[k] = v + curpath = path + (k,) if isinstance(v, TreeStorage): _d_res[k] = _c_mapping(v, func, curpath) @@ -81,10 +87,15 @@ cdef TreeStorage _c_filter_(TreeStorage st, object func, tuple path, bool remove cdef dict _d_res = {} cdef str k - cdef object v + cdef object v, nv cdef tuple curpath cdef TreeStorage curst for k, v in _d_st.items(): + nv = undelay(v) + if nv is not v: + v = nv + _d_st[k] = v + curpath = path + (k,) if isinstance(v, TreeStorage): curst = _c_filter_(v, func, curpath, remove_empty) @@ -138,15 +149,24 @@ cdef object _c_mask(TreeStorage st, object sm, tuple path, bool remove_empty): cdef dict _d_res = {} cdef str k - cdef object v, mv + cdef object v, mv, nv, nmv cdef tuple curpath cdef object curres for k, v in _d_st.items(): + nv = undelay(v) + if nv is not v: + v = nv + _d_st[k] = v + curpath = path + (k,) if _b_tree_mask: mv = _d_sm[k] else: mv = sm + nmv = undelay(mv) + if nmv is not mv: + mv = nmv + _d_sm[k] = mv if isinstance(v, TreeStorage): curres = _c_mask(v, mv, curpath, remove_empty) @@ -188,10 +208,15 @@ cdef object _c_reduce(TreeStorage st, object func, tuple path, object return_typ cdef dict _d_kwargs = {} cdef str k - cdef object v + cdef object v, nv cdef tuple curpath cdef object curst for k, v in _d_st.items(): + nv = undelay(v) + if nv is not v: + v = nv + _d_st[k] = v + curpath = path + (k,) if isinstance(v, TreeStorage): curst = _c_reduce(v, func, curpath, return_type) From 5490a788907ab96cbb5cd8147543cdb2fe3ba51d Mon Sep 17 00:00:00 2001 From: HansBug Date: Wed, 29 Dec 2021 00:13:06 +0800 Subject: [PATCH 11/26] dev(hansbug): add delayed support for structural part --- test/tree/tree/test_structural.py | 25 ++++++++++++++++++++++++- treevalue/tree/tree/structural.pyx | 8 +++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/test/tree/tree/test_structural.py b/test/tree/tree/test_structural.py index d41b973831..8e2ffdc213 100644 --- a/test/tree/tree/test_structural.py +++ b/test/tree/tree/test_structural.py @@ -1,6 +1,6 @@ import pytest -from treevalue.tree import TreeValue, mapping, union, raw, subside, rise +from treevalue.tree import TreeValue, mapping, union, raw, subside, rise, delayed # noinspection DuplicatedCode @@ -20,6 +20,11 @@ class MyTreeValue(TreeValue): assert union(1, 2) == (1, 2) assert union(1, 2, return_type=TreeValue) == (1, 2) + tp = MyTreeValue({'v': delayed(lambda: t)}) + tp1 = TreeValue({'v': delayed(lambda: t1)}) + assert union(tp, tp1) == MyTreeValue({'v': {'a': (1, 1), 'b': (2, 2), 'x': {'c': (3, 3), 'd': (4, 4)}}}) + assert union(tp1, tp) == TreeValue({'v': {'a': (1, 1), 'b': (2, 2), 'x': {'c': (3, 3), 'd': (4, 4)}}}) + def test_subside(self): assert subside({'a': (1, 2), 'b': [3, 4]}) == {'a': (1, 2), 'b': [3, 4]} assert subside({'a': (1, 2), 'b': [3, 4]}, return_type=TreeValue) == {'a': (1, 2), 'b': [3, 4]} @@ -203,3 +208,21 @@ class MyTreeValue(TreeValue): rise(t5, template=[object, object, object, object, object, ...]) assert rise(1) == 1 + + t1 = TreeValue({'x': raw({'a': [1, 2], 'b': [2, 3]}), 'y': raw({'a': [5, 6, 7], 'b': [7, 8]})}) + assert rise(t1) == { + 'a': TreeValue({'x': [1, 2], 'y': [5, 6, 7]}), + 'b': [ + TreeValue({'x': 2, 'y': 7}), + TreeValue({'x': 3, 'y': 8}), + ] + } + + t10 = MyTreeValue({'v': delayed(lambda: t1)}) + assert rise(t10) == { + 'a': MyTreeValue({'v': {'x': [1, 2], 'y': [5, 6, 7]}}), + 'b': [ + MyTreeValue({'v': {'x': 2, 'y': 7}}), + MyTreeValue({'v': {'x': 3, 'y': 8}}), + ] + } diff --git a/treevalue/tree/tree/structural.pyx b/treevalue/tree/tree/structural.pyx index fc94c97b3c..318d31361e 100644 --- a/treevalue/tree/tree/structural.pyx +++ b/treevalue/tree/tree/structural.pyx @@ -9,6 +9,7 @@ import cython from libcpp cimport bool from .tree cimport TreeValue +from ..common.delay cimport undelay from ..common.storage cimport TreeStorage from ..func.cfunc cimport _c_func_treelize_run from ..func.modes cimport _c_load_mode @@ -200,7 +201,7 @@ cdef object _c_rise_tree_builder(tuple p, object it): cdef tuple _c_rise_tree_process(object t): cdef str k - cdef object v + cdef object v, nv cdef list _l_items, _l_values cdef object _i_item, _i_value cdef dict detached @@ -209,6 +210,11 @@ cdef tuple _c_rise_tree_process(object t): _l_items = [] _l_values = [] for k, v in detached.items(): + nv = undelay(v) + if nv is not v: + v = nv + detached[k] = v + _i_item, _i_value = _c_rise_tree_process(v) _l_items.append((k, _i_item)) _l_values.append(_i_value) From 3b089d282b0a1ffb4dcd25c106da8d8466a84af9 Mon Sep 17 00:00:00 2001 From: HansBug Date: Wed, 29 Dec 2021 10:15:36 +0800 Subject: [PATCH 12/26] dev(hansbug): add object to _c_common_value --- treevalue/tree/func/cfunc.pxd | 2 +- treevalue/tree/func/cfunc.pyx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/treevalue/tree/func/cfunc.pxd b/treevalue/tree/func/cfunc.pxd index d7024aeae0..7f799fb572 100644 --- a/treevalue/tree/func/cfunc.pxd +++ b/treevalue/tree/func/cfunc.pxd @@ -10,6 +10,6 @@ cdef object _c_func_treelize_run(object func, list args, dict kwargs, cpdef object _d_func_treelize(object func, object mode, object return_type, bool inherit, object missing, object subside, object rise) -cdef _c_common_value(object item) +cdef object _c_common_value(object item) cpdef object func_treelize(object mode= *, object return_type= *, bool inherit= *, object missing= *, object subside= *, object rise= *) diff --git a/treevalue/tree/func/cfunc.pyx b/treevalue/tree/func/cfunc.pyx index 38e8f985d6..f68d2af87a 100644 --- a/treevalue/tree/func/cfunc.pyx +++ b/treevalue/tree/func/cfunc.pyx @@ -159,7 +159,7 @@ def _w_func_treelize_run(*args, object __w_func, _e_tree_mode __w_mode, object _ MISSING_NOT_ALLOW = SingletonMark("missing_not_allow") -cdef _c_common_value(object item): +cdef object _c_common_value(object item): return item # build-time function From c102d03530d9e133e8dece207d63c28b92c70129 Mon Sep 17 00:00:00 2001 From: HansBug Date: Wed, 29 Dec 2021 12:37:57 +0800 Subject: [PATCH 13/26] dev(hansbug): add missing option in subside --- treevalue/tree/func/cfunc.pxd | 1 + treevalue/tree/func/cfunc.pyx | 23 +++++++++++++++-------- treevalue/tree/tree/structural.pxd | 4 ++-- treevalue/tree/tree/structural.pyx | 24 ++++++++++++++++-------- 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/treevalue/tree/func/cfunc.pxd b/treevalue/tree/func/cfunc.pxd index 7f799fb572..1dd90f44de 100644 --- a/treevalue/tree/func/cfunc.pxd +++ b/treevalue/tree/func/cfunc.pxd @@ -11,5 +11,6 @@ cdef object _c_func_treelize_run(object func, list args, dict kwargs, cpdef object _d_func_treelize(object func, object mode, object return_type, bool inherit, object missing, object subside, object rise) cdef object _c_common_value(object item) +cdef tuple _c_missing_process(object missing) cpdef object func_treelize(object mode= *, object return_type= *, bool inherit= *, object missing= *, object subside= *, object rise= *) diff --git a/treevalue/tree/func/cfunc.pyx b/treevalue/tree/func/cfunc.pyx index f68d2af87a..846a794801 100644 --- a/treevalue/tree/func/cfunc.pyx +++ b/treevalue/tree/func/cfunc.pyx @@ -14,6 +14,7 @@ from ..tree.structural cimport _c_subside, _c_rise from ..tree.tree cimport TreeValue _VALUE_IS_MISSING = SingletonMark('value_is_missing') +MISSING_NOT_ALLOW = SingletonMark("missing_not_allow") cdef object _c_func_treelize_run(object func, list args, dict kwargs, _e_tree_mode mode, bool inherit, bool allow_missing, object missing_func): @@ -120,8 +121,9 @@ cdef object _c_func_treelize_run(object func, list args, dict kwargs, return TreeStorage(_d_res) -def _w_subside_func(object value, bool dict_=True, bool list_=True, bool tuple_=True, bool inherit=True): - return _c_subside(value, dict_, list_, tuple_, inherit)[0] +def _w_subside_func(object value, bool dict_=True, bool list_=True, bool tuple_=True, bool inherit=True, + object missing=MISSING_NOT_ALLOW): + return _c_subside(value, dict_, list_, tuple_, inherit, missing)[0] def _w_rise_func(object tree, bool dict_=True, bool list_=True, bool tuple_=True, object template=None): return _c_rise(tree, dict_, list_, tuple_, template) @@ -157,15 +159,10 @@ def _w_func_treelize_run(*args, object __w_func, _e_tree_mode __w_mode, object _ else: return None -MISSING_NOT_ALLOW = SingletonMark("missing_not_allow") - cdef object _c_common_value(object item): return item -# build-time function -cpdef object _d_func_treelize(object func, object mode, object return_type, bool inherit, object missing, - object subside, object rise): - cdef _e_tree_mode _v_mode = _c_load_mode(mode) +cdef inline tuple _c_missing_process(object missing): cdef bool allow_missing cdef object missing_func if missing is MISSING_NOT_ALLOW: @@ -175,6 +172,16 @@ cpdef object _d_func_treelize(object func, object mode, object return_type, bool allow_missing = True missing_func = missing if callable(missing) else partial(_c_common_value, missing) + return allow_missing, missing_func + +# build-time function +cpdef object _d_func_treelize(object func, object mode, object return_type, bool inherit, object missing, + object subside, object rise): + cdef _e_tree_mode _v_mode = _c_load_mode(mode) + cdef bool allow_missing + cdef object missing_func + allow_missing, missing_func = _c_missing_process(missing) + cdef object _v_subside, _v_rise if subside is not None and not isinstance(subside, dict): _v_subside = {} if subside else None diff --git a/treevalue/tree/tree/structural.pxd b/treevalue/tree/tree/structural.pxd index 50ce97745e..bab5262519 100644 --- a/treevalue/tree/tree/structural.pxd +++ b/treevalue/tree/tree/structural.pxd @@ -11,10 +11,10 @@ cdef class _SubsideCall: cdef object _c_subside_process(tuple value, object it) cdef tuple _c_subside_build(object value, bool dict_, bool list_, bool tuple_) cdef void _c_subside_missing() -cdef object _c_subside(object value, bool dict_, bool list_, bool tuple_, bool inherit) +cdef object _c_subside(object value, bool dict_, bool list_, bool tuple_, bool inherit, object missing) cdef object _c_subside_keep_type(object t) cpdef object subside(object value, bool dict_= *, bool list_= *, bool tuple_= *, - object return_type= *, bool inherit= *) + object return_type= *, bool inherit= *, object missing= *) cdef object _c_rise_tree_builder(tuple p, object it) cdef tuple _c_rise_tree_process(object t) diff --git a/treevalue/tree/tree/structural.pyx b/treevalue/tree/tree/structural.pyx index 318d31361e..de25cbcee9 100644 --- a/treevalue/tree/tree/structural.pyx +++ b/treevalue/tree/tree/structural.pyx @@ -6,14 +6,17 @@ from itertools import chain import cython +from hbutils.design import SingletonMark from libcpp cimport bool from .tree cimport TreeValue from ..common.delay cimport undelay from ..common.storage cimport TreeStorage -from ..func.cfunc cimport _c_func_treelize_run +from ..func.cfunc cimport _c_func_treelize_run, _c_missing_process from ..func.modes cimport _c_load_mode +MISSING_NOT_ALLOW = SingletonMark("missing_not_allow") + cdef object _c_subside_process(tuple value, object it): cdef type type_ cdef list items @@ -82,20 +85,25 @@ STRICT = _c_load_mode('STRICT') cdef inline void _c_subside_missing(): pass -cdef object _c_subside(object value, bool dict_, bool list_, bool tuple_, bool inherit): +cdef object _c_subside(object value, bool dict_, bool list_, bool tuple_, bool inherit, + object missing): cdef object builder, _i_args, _i_types builder, _i_args, _i_types = _c_subside_build(value, dict_, list_, tuple_) cdef list args = list(_i_args) + cdef bool allow_missing + cdef object missing_func + allow_missing, missing_func = _c_missing_process(missing) + return _c_func_treelize_run(_SubsideCall(builder), args, {}, - STRICT, inherit, False, _c_subside_missing), _i_types + STRICT, inherit, allow_missing, missing_func), _i_types cdef inline object _c_subside_keep_type(object t): return t @cython.binding(True) cpdef object subside(object value, bool dict_=True, bool list_=True, bool tuple_=True, - object return_type=None, bool inherit=True): + object return_type=None, bool inherit=True, object missing=MISSING_NOT_ALLOW): """ Overview: Drift down the structures (list, tuple, dict) down to the tree's value. @@ -133,7 +141,7 @@ cpdef object subside(object value, bool dict_=True, bool list_=True, bool tuple_ >>> #}), all structures above the tree values are subsided to the bottom of the tree. """ cdef object result, _i_types - result, _i_types = _c_subside(value, dict_, list_, tuple_, inherit) + result, _i_types = _c_subside(value, dict_, list_, tuple_, inherit, missing) cdef object type_ cdef set types @@ -151,7 +159,7 @@ cpdef object subside(object value, bool dict_=True, bool list_=True, bool tuple_ return type_(result) @cython.binding(True) -def union(*trees, object return_type=None, bool inherit=True): +def union(*trees, object return_type=None, bool inherit=True, object missing=MISSING_NOT_ALLOW): """ Overview: Union tree values together. @@ -170,7 +178,7 @@ def union(*trees, object return_type=None, bool inherit=True): >>> union(t, tx) # TreeValue({'a': (1, True), 'b': (2, False), 'x': {'c': (3, True), 'd': (4, False)}}) """ cdef object result, _i_types - result, _i_types = _c_subside(tuple(trees), True, True, True, inherit) + result, _i_types = _c_subside(tuple(trees), True, True, True, inherit, missing) cdef object type_ cdef list types @@ -268,7 +276,7 @@ cdef tuple _c_rise_struct_process(list objs, object template): _l_obj_0 = len(objs[0]) if _l_obj_0 < _l_temp - 2: raise ValueError(f"At least {repr(_l_temp - 2)} value expected due to template " - f"{repr(template)}, but length is {repr(_l_obj_0)}.") + f"{repr(template)}, but length is {repr(_l_obj_0)}.") _a_template = type(template)(chain(template[:-2], (template[-2],) * (_l_obj_0 - _l_temp + 2))) else: From bb3fb6ad6282ca78e0c1b15dbd807f24d2d7105b Mon Sep 17 00:00:00 2001 From: HansBug Date: Wed, 29 Dec 2021 12:55:19 +0800 Subject: [PATCH 14/26] dev(hansbug): add mode, missing function to subside and union --- test/tree/tree/test_structural.py | 14 ++++++++++++++ treevalue/tree/func/cfunc.pyx | 4 ++-- treevalue/tree/tree/structural.pxd | 5 +++-- treevalue/tree/tree/structural.pyx | 22 ++++++++++++++-------- 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/test/tree/tree/test_structural.py b/test/tree/tree/test_structural.py index 8e2ffdc213..4196c14903 100644 --- a/test/tree/tree/test_structural.py +++ b/test/tree/tree/test_structural.py @@ -16,6 +16,8 @@ class MyTreeValue(TreeValue): t1 = MyTreeValue({'a': 1, 'b': 2, 'x': {'c': 3, 'd': 4}}) assert union(t, t1) == TreeValue({'a': (1, 1), 'b': (2, 2), 'x': {'c': (3, 3), 'd': (4, 4)}}) + assert union(t, t1, return_type=MyTreeValue) == MyTreeValue( + {'a': (1, 1), 'b': (2, 2), 'x': {'c': (3, 3), 'd': (4, 4)}}) assert union(t1, t) == MyTreeValue({'a': (1, 1), 'b': (2, 2), 'x': {'c': (3, 3), 'd': (4, 4)}}) assert union(1, 2) == (1, 2) assert union(1, 2, return_type=TreeValue) == (1, 2) @@ -25,6 +27,18 @@ class MyTreeValue(TreeValue): assert union(tp, tp1) == MyTreeValue({'v': {'a': (1, 1), 'b': (2, 2), 'x': {'c': (3, 3), 'd': (4, 4)}}}) assert union(tp1, tp) == TreeValue({'v': {'a': (1, 1), 'b': (2, 2), 'x': {'c': (3, 3), 'd': (4, 4)}}}) + t = MyTreeValue({'a': 1, 'b': 2, 'x': {'c': 3}}) + t1 = TreeValue({ + 'a': delayed(lambda: t.x.c), + 'x': { + 'c': delayed(lambda: t.a), + 'd': delayed(lambda: t.b), + } + }) + assert union(t, t1, mode='outer', missing=None) == MyTreeValue({ + 'a': (1, 3), 'b': (2, None), 'x': {'c': (3, 1), 'd': (None, 2)}, + }) + def test_subside(self): assert subside({'a': (1, 2), 'b': [3, 4]}) == {'a': (1, 2), 'b': [3, 4]} assert subside({'a': (1, 2), 'b': [3, 4]}, return_type=TreeValue) == {'a': (1, 2), 'b': [3, 4]} diff --git a/treevalue/tree/func/cfunc.pyx b/treevalue/tree/func/cfunc.pyx index 846a794801..d24f2ec888 100644 --- a/treevalue/tree/func/cfunc.pyx +++ b/treevalue/tree/func/cfunc.pyx @@ -122,8 +122,8 @@ cdef object _c_func_treelize_run(object func, list args, dict kwargs, return TreeStorage(_d_res) def _w_subside_func(object value, bool dict_=True, bool list_=True, bool tuple_=True, bool inherit=True, - object missing=MISSING_NOT_ALLOW): - return _c_subside(value, dict_, list_, tuple_, inherit, missing)[0] + object mode='strict', object missing=MISSING_NOT_ALLOW): + return _c_subside(value, dict_, list_, tuple_, inherit, mode, missing)[0] def _w_rise_func(object tree, bool dict_=True, bool list_=True, bool tuple_=True, object template=None): return _c_rise(tree, dict_, list_, tuple_, template) diff --git a/treevalue/tree/tree/structural.pxd b/treevalue/tree/tree/structural.pxd index bab5262519..a84eab61ed 100644 --- a/treevalue/tree/tree/structural.pxd +++ b/treevalue/tree/tree/structural.pxd @@ -11,10 +11,11 @@ cdef class _SubsideCall: cdef object _c_subside_process(tuple value, object it) cdef tuple _c_subside_build(object value, bool dict_, bool list_, bool tuple_) cdef void _c_subside_missing() -cdef object _c_subside(object value, bool dict_, bool list_, bool tuple_, bool inherit, object missing) +cdef object _c_subside(object value, bool dict_, bool list_, bool tuple_, bool inherit, + object mode, object missing) cdef object _c_subside_keep_type(object t) cpdef object subside(object value, bool dict_= *, bool list_= *, bool tuple_= *, - object return_type= *, bool inherit= *, object missing= *) + object return_type= *, bool inherit= *, object mode= *, object missing= *) cdef object _c_rise_tree_builder(tuple p, object it) cdef tuple _c_rise_tree_process(object t) diff --git a/treevalue/tree/tree/structural.pyx b/treevalue/tree/tree/structural.pyx index de25cbcee9..1a2db91fa1 100644 --- a/treevalue/tree/tree/structural.pyx +++ b/treevalue/tree/tree/structural.pyx @@ -80,13 +80,11 @@ cdef tuple _c_subside_build(object value, bool dict_, bool list_, bool tuple_): else: return (object, None), (value,), () -STRICT = _c_load_mode('STRICT') - cdef inline void _c_subside_missing(): pass cdef object _c_subside(object value, bool dict_, bool list_, bool tuple_, bool inherit, - object missing): + object mode, object missing): cdef object builder, _i_args, _i_types builder, _i_args, _i_types = _c_subside_build(value, dict_, list_, tuple_) cdef list args = list(_i_args) @@ -96,14 +94,15 @@ cdef object _c_subside(object value, bool dict_, bool list_, bool tuple_, bool i allow_missing, missing_func = _c_missing_process(missing) return _c_func_treelize_run(_SubsideCall(builder), args, {}, - STRICT, inherit, allow_missing, missing_func), _i_types + _c_load_mode(mode), inherit, allow_missing, missing_func), _i_types cdef inline object _c_subside_keep_type(object t): return t @cython.binding(True) cpdef object subside(object value, bool dict_=True, bool list_=True, bool tuple_=True, - object return_type=None, bool inherit=True, object missing=MISSING_NOT_ALLOW): + object return_type=None, bool inherit=True, + object mode='strict', object missing=MISSING_NOT_ALLOW): """ Overview: Drift down the structures (list, tuple, dict) down to the tree's value. @@ -117,6 +116,9 @@ cpdef object subside(object value, bool dict_=True, bool list_=True, bool tuple_ will be auto detected when there is exactly one tree value type in this original value, \ otherwise the default will be `TreeValue`. - inherit (:obj:`bool`): Allow inherit in wrapped function, default is `True`. + - mode (:obj:`str`): Mode of the wrapping, default is `strict`. + - missing (:obj:`Union[Any, Callable]`): Missing value or lambda generator of when missing, \ + default is `MISSING_NOT_ALLOW`, which means raise `KeyError` when missing detected. Returns: - return (:obj:`_TreeValue`): Subsided tree value. @@ -141,7 +143,7 @@ cpdef object subside(object value, bool dict_=True, bool list_=True, bool tuple_ >>> #}), all structures above the tree values are subsided to the bottom of the tree. """ cdef object result, _i_types - result, _i_types = _c_subside(value, dict_, list_, tuple_, inherit, missing) + result, _i_types = _c_subside(value, dict_, list_, tuple_, inherit, mode, missing) cdef object type_ cdef set types @@ -159,7 +161,8 @@ cpdef object subside(object value, bool dict_=True, bool list_=True, bool tuple_ return type_(result) @cython.binding(True) -def union(*trees, object return_type=None, bool inherit=True, object missing=MISSING_NOT_ALLOW): +def union(*trees, object return_type=None, bool inherit=True, + object mode='strict', object missing=MISSING_NOT_ALLOW): """ Overview: Union tree values together. @@ -168,6 +171,9 @@ def union(*trees, object return_type=None, bool inherit=True, object missing=MIS - trees (:obj:`_TreeValue`): Tree value objects. - return_type (:obj:`Optional[Type[_ClassType]]`): Return type of the wrapped function, default is `TreeValue`. - inherit (:obj:`bool`): Allow inherit in wrapped function, default is `True`. + - mode (:obj:`str`): Mode of the wrapping, default is `strict`. + - missing (:obj:`Union[Any, Callable]`): Missing value or lambda generator of when missing, \ + default is `MISSING_NOT_ALLOW`, which means raise `KeyError` when missing detected. Returns: - result (:obj:`TreeValue`): Unionised tree value. @@ -178,7 +184,7 @@ def union(*trees, object return_type=None, bool inherit=True, object missing=MIS >>> union(t, tx) # TreeValue({'a': (1, True), 'b': (2, False), 'x': {'c': (3, True), 'd': (4, False)}}) """ cdef object result, _i_types - result, _i_types = _c_subside(tuple(trees), True, True, True, inherit, missing) + result, _i_types = _c_subside(tuple(trees), True, True, True, inherit, mode, missing) cdef object type_ cdef list types From b6ad686ab001889b867bcb28de29c300062e8739 Mon Sep 17 00:00:00 2001 From: HansBug Date: Wed, 29 Dec 2021 13:01:19 +0800 Subject: [PATCH 15/26] dev(hansbug): add doc for delayed --- treevalue/tree/tree/tree.pyx | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/treevalue/tree/tree/tree.pyx b/treevalue/tree/tree/tree.pyx index 2e0244173a..fc032e9c44 100644 --- a/treevalue/tree/tree/tree.pyx +++ b/treevalue/tree/tree/tree.pyx @@ -435,10 +435,42 @@ def delayed(func, *args, **kwargs): r""" Overview: Use delayed function in treevalue. + The given ``func`` will not be called until its value is accessed, and \ + it will be only called once, after that the delayed node will be replaced by the actual value. Arguments: - func: Delayed function. - args: Positional arguments. - kwargs: Key-word arguments. + + Examples:: + >>> from treevalue import TreeValue, delayed + >>> + >>> def f(x): + >>> print('f is called, x is', x) + >>> return x ** x + >>> + >>> t = TreeValue({'a': delayed(f, 2), 'x': delayed(f, 3)}) + >>> t.a + f is called, x is 2 + 4 + >>> t.x + f is called, x is 3 + 27 + >>> t.a + 4 + >>> t.x + 27 + >>> t = TreeValue({'a': delayed(f, 2), 'x': delayed(f, 3)}) + >>> print(t) + f is called, x is 2 + f is called, x is 3 + + ├── a --> 4 + └── x --> 27 + >>> print(t) + + ├── a --> 4 + └── x --> 27 """ return DetachedDelayedProxy(_c_delayed_partial(func, args, kwargs)) From 007df0be72c1c9b8757aad2a779da8be4cdbafa2 Mon Sep 17 00:00:00 2001 From: HansBug Date: Wed, 29 Dec 2021 13:23:57 +0800 Subject: [PATCH 16/26] dev(hansbug): complete doc for delayed part --- docs/source/api_doc/tree/common.rst | 43 ++++++++++++++++++ docs/source/api_doc/tree/tree.rst | 8 ++++ requirements-doc.txt | 2 +- treevalue/tree/common/__init__.py | 2 +- treevalue/tree/common/delay.pyx | 68 +++++++++++++++++++++++++++++ 5 files changed, 121 insertions(+), 2 deletions(-) diff --git a/docs/source/api_doc/tree/common.rst b/docs/source/api_doc/tree/common.rst index 6df0dbe4aa..5eeacc107e 100644 --- a/docs/source/api_doc/tree/common.rst +++ b/docs/source/api_doc/tree/common.rst @@ -36,3 +36,46 @@ RawWrapper .. autoclass:: RawWrapper :members: __init__, value + +.. _apidoc_tree_common_delayed_partial: + +delayed_partial +---------------------- + +.. autofunction:: delayed_partial + + +.. _apidoc_tree_common_undelay: + +undelay +--------------- + +.. autofunction:: undelay + + +.. _apidoc_tree_common_delayedproxy: + +DelayedProxy +------------------- + +.. autoclass:: DelayedProxy + :members: value, fvalue + + +.. _apidoc_tree_common_delayedvalueproxy: + +DelayedValueProxy +------------------- + +.. autoclass:: DelayedValueProxy + :members: __cinit__, value, fvalue + + +.. _apidoc_tree_common_delayedfuncproxy: + +DelayedFuncProxy +------------------- + +.. autoclass:: DelayedFuncProxy + :members: __cinit__, value, fvalue + diff --git a/docs/source/api_doc/tree/tree.rst b/docs/source/api_doc/tree/tree.rst index c009c0879f..b1f92bdec6 100644 --- a/docs/source/api_doc/tree/tree.rst +++ b/docs/source/api_doc/tree/tree.rst @@ -12,6 +12,14 @@ TreeValue :members: __init__, __getattribute__, __setattr__, __delattr__, __contains__, __repr__, __iter__, __hash__, __eq__, _attr_extern, __len__, __bool__, __str__, __getstate__, __setstate__, get +.. _apidoc_tree_tree_delayed: + +delayed +--------------- + +.. autofunction:: delayed + + .. _apidoc_tree_tree_jsonify: jsonify diff --git a/requirements-doc.txt b/requirements-doc.txt index 04c64a0122..6ab91ae465 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -2,7 +2,7 @@ sphinx~=3.2.0 sphinx_rtd_theme~=0.4.3 enum_tools sphinx-toolbox -plantumlcli>=0.0.2 +plantumlcli>=0.0.4 packaging sphinx-multiversion~=0.2.4 where~=1.0.2 diff --git a/treevalue/tree/common/__init__.py b/treevalue/tree/common/__init__.py index 2d23f6dd03..482621824b 100644 --- a/treevalue/tree/common/__init__.py +++ b/treevalue/tree/common/__init__.py @@ -1,3 +1,3 @@ from .base import raw, unraw, RawWrapper -from .delay import DelayedProxy, delayed_partial, undelay +from .delay import DelayedProxy, delayed_partial, undelay, DelayedValueProxy, DelayedFuncProxy from .storage import TreeStorage, create_storage diff --git a/treevalue/tree/common/delay.pyx b/treevalue/tree/common/delay.pyx index 8f1a7a2bb1..e6d45c42fa 100644 --- a/treevalue/tree/common/delay.pyx +++ b/treevalue/tree/common/delay.pyx @@ -5,14 +5,47 @@ import cython from libcpp cimport bool cdef class DelayedProxy: + """ + Overview: + Base class of all the delayed proxy class. + """ cpdef object value(self): + r""" + Overview: + Get value of the delayed proxy. + Should make sure the result is cached. + Can be accessed in :func:`treevalue.tree.common.undelay` when ``is_final`` is ``False``. + + Returns: + - value (:obj:`object`): Calculation result. + """ raise NotImplementedError # pragma: no cover cpdef object fvalue(self): + r""" + Overview: + Get value of the delayed proxy. + Can be accessed in :func:`treevalue.tree.common.undelay` when ``is_final`` is ``True``. + + Returns: + - value (:obj:`object`): Calculation result. + """ return self.value() cdef class DelayedValueProxy(DelayedProxy): + """ + Overview: + Simple function delayed proxy. + """ def __cinit__(self, object func): + """ + Overview: + Constructor of class :class:`treevalue.tree.common.DelayedValueProxy`. + + Arguments: + - func (:obj:`object`): Function to be called, which can be called without arguments. \ + Delayed proxy is supported. + """ self.func = func self.calculated = False self.val = None @@ -27,7 +60,21 @@ cdef class DelayedValueProxy(DelayedProxy): return self.val cdef class DelayedFuncProxy(DelayedProxy): + """ + Overview: + Simple function delayed proxy. + """ def __cinit__(self, object func, tuple args, dict kwargs): + """ + Overview: + Constructor of class :class:`treevalue.tree.common.DelayedFuncProxy`. + + Arguments: + - func (:obj:`object`): Function to be called, which can be called with given arguments. \ + Delayed proxy is supported. + - args (:obj:`tuple`): Positional arguments to be used, delayed proxy is supported. + - kwargs (:obj:`dict`): Key-word arguments to be used, delayed proxy is supported. + """ self.func = func self.args = args self.kwargs = kwargs @@ -60,11 +107,32 @@ cdef inline DelayedProxy _c_delayed_partial(func, args, kwargs): else: return DelayedValueProxy(func) +@cython.binding(True) def delayed_partial(func, *args, **kwargs): + """ + Overview: + Build a delayed partial object. + Similar to :func:`functools.partial`. + + Returns: + - delayed: Delayed object. + """ return _c_delayed_partial(func, args, kwargs) @cython.binding(True) cpdef inline object undelay(object p, bool is_final=True): + r""" + Overview: + Get the value of a given object, it can be a delayed proxy, a simple object or \ + a nested delayed proxy. + + Arguments: + - p (:obj:`object`): Given object to be undelay. + - is_final (:obj:`bool`): Is final value getting or not, default is ``True``. + + Returns: + - value (:obj:`object): Actual value of the given ``p``. + """ if isinstance(p, DelayedProxy): if is_final: return p.fvalue() From 38d94a89c18a797dbb4b79930e886aa6721c2a43 Mon Sep 17 00:00:00 2001 From: HansBug Date: Wed, 29 Dec 2021 16:43:51 +0800 Subject: [PATCH 17/26] dev(hansbug): add delayed mode to mapping --- test/tree/tree/test_functional.py | 48 ++++++++++++++++++++++++++++++ treevalue/tree/tree/functional.pxd | 4 +-- treevalue/tree/tree/functional.pyx | 34 +++++++++++++++------ 3 files changed, 75 insertions(+), 11 deletions(-) diff --git a/test/tree/tree/test_functional.py b/test/tree/tree/test_functional.py index 7b5ff49ee8..4c5c845906 100644 --- a/test/tree/tree/test_functional.py +++ b/test/tree/tree/test_functional.py @@ -35,6 +35,54 @@ def test_mapping(self): 'a': 3, 'b': 4, 'c': {'x': 4, 'y': 5} }}) + def test_mapping_delayed(self): + tv1 = TreeValue({'a': 1, 'b': 2, 'c': {'x': 2, 'y': 3}}) + tv8 = TreeValue({'v': delayed(lambda: tv1)}) + assert mapping(tv8, lambda x: x + 2, delayed=True) == TreeValue({'v': { + 'a': 3, 'b': 4, 'c': {'x': 4, 'y': 5} + }}) + + cnt_f, cnt_v = 0, 0 + + def f(x): + nonlocal cnt_f + cnt_f += 1 + return TreeValue({ + 'a': x * 2, 'b': x + 1, 'c': x ** 2, + }) + + def v(): + nonlocal cnt_v + cnt_v += 1 + return 3 + + t = TreeValue({ + 'a': 1, 'b': delayed(f, 1), + 'x': {'c': delayed(v), 'd': 4, }, + 'y': delayed(f, 3), + }) + t1 = mapping(t, lambda x: (x + 3) ** 2, delayed=True) + assert cnt_v == 0 + assert cnt_f == 0 + + assert t1 == TreeValue({ + 'a': 16, + 'b': {'a': 25, 'b': 25, 'c': 16, }, + 'x': {'c': 36, 'd': 49, }, + 'y': {'a': 81, 'b': 49, 'c': 144, }, + }) + assert cnt_v == 1 + assert cnt_f == 2 + + assert t == TreeValue({ + 'a': 1, + 'b': {'a': 2, 'b': 2, 'c': 1, }, + 'x': {'c': 3, 'd': 4, }, + 'y': {'a': 6, 'b': 4, 'c': 9, }, + }) + assert cnt_v == 1 + assert cnt_f == 2 + def test_mask(self): class MyTreeValue(TreeValue): pass diff --git a/treevalue/tree/tree/functional.pxd b/treevalue/tree/tree/functional.pxd index 44a28b6a1f..c84209796c 100644 --- a/treevalue/tree/tree/functional.pxd +++ b/treevalue/tree/tree/functional.pxd @@ -12,8 +12,8 @@ cdef class _ValuePathFuncWrapper: cdef readonly object func cdef int index -cdef TreeStorage _c_mapping(TreeStorage st, object func, tuple path) -cpdef TreeValue mapping(TreeValue tree, object func) +cdef TreeStorage _c_mapping(TreeStorage st, object func, tuple path, bool delayed) +cpdef TreeValue mapping(TreeValue tree, object func, bool delayed= *) cdef TreeStorage _c_filter_(TreeStorage st, object func, tuple path, bool remove_empty) cpdef TreeValue filter_(TreeValue tree, object func, bool remove_empty= *) cdef object _c_mask(TreeStorage st, object sm, tuple path, bool remove_empty) diff --git a/treevalue/tree/tree/functional.pyx b/treevalue/tree/tree/functional.pyx index 03c651bdf3..c6c05d4132 100644 --- a/treevalue/tree/tree/functional.pyx +++ b/treevalue/tree/tree/functional.pyx @@ -7,7 +7,9 @@ import cython from libcpp cimport bool from .tree cimport TreeValue +from .tree import delayed as _func_delayed from ..common.delay cimport undelay +from ..common.delay import delayed_partial from ..common.storage cimport TreeStorage cdef class _ValuePathFuncWrapper: @@ -28,7 +30,17 @@ cdef class _ValuePathFuncWrapper: except TypeError: self.index -= 1 -cdef TreeStorage _c_mapping(TreeStorage st, object func, tuple path): +def _p_delayed_mapping(object so, object func, tuple path, bool delayed): + cdef object nso = undelay(so) + if isinstance(nso, TreeValue): + nso = nso._detach() + + if isinstance(nso, TreeStorage): + return _c_mapping(nso, func, path, delayed) + else: + return func(nso, path) + +cdef TreeStorage _c_mapping(TreeStorage st, object func, tuple path, bool delayed): cdef dict _d_st = st.detach() cdef dict _d_res = {} @@ -36,21 +48,25 @@ cdef TreeStorage _c_mapping(TreeStorage st, object func, tuple path): cdef object v, nv cdef tuple curpath for k, v in _d_st.items(): - nv = undelay(v) - if nv is not v: - v = nv - _d_st[k] = v + if not delayed: + nv = undelay(v) + if nv is not v: + v = nv + _d_st[k] = v curpath = path + (k,) if isinstance(v, TreeStorage): - _d_res[k] = _c_mapping(v, func, curpath) + _d_res[k] = _c_mapping(v, func, curpath, delayed) else: - _d_res[k] = func(v, curpath) + if delayed: + _d_res[k] = delayed_partial(_p_delayed_mapping, v, func, curpath, delayed) + else: + _d_res[k] = func(v, curpath) return TreeStorage(_d_res) @cython.binding(True) -cpdef TreeValue mapping(TreeValue tree, object func): +cpdef TreeValue mapping(TreeValue tree, object func, bool delayed=False): """ Overview: Do mapping on every value in this tree. @@ -80,7 +96,7 @@ cpdef TreeValue mapping(TreeValue tree, object func): >>> mapping(t, lambda: 1) # TreeValue({'a': 1, 'b': 1, 'x': {'c': 1, 'd': 1}}) >>> mapping(t, lambda x, p: p) # TreeValue({'a': ('a',), 'b': ('b',), 'x': {'c': ('x', 'c'), 'd': ('x', 'd')}}) """ - return type(tree)(_c_mapping(tree._detach(), _ValuePathFuncWrapper(func), ())) + return type(tree)(_c_mapping(tree._detach(), _ValuePathFuncWrapper(func), (), delayed)) cdef TreeStorage _c_filter_(TreeStorage st, object func, tuple path, bool remove_empty): cdef dict _d_st = st.detach() From 5844679f2c56067332f6ea4641634e3c54236f42 Mon Sep 17 00:00:00 2001 From: HansBug Date: Wed, 29 Dec 2021 16:47:05 +0800 Subject: [PATCH 18/26] dev(hansbug): _c_delayed_mapping use cdef instead of def --- treevalue/tree/tree/functional.pxd | 1 + treevalue/tree/tree/functional.pyx | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/treevalue/tree/tree/functional.pxd b/treevalue/tree/tree/functional.pxd index c84209796c..d9096f7e63 100644 --- a/treevalue/tree/tree/functional.pxd +++ b/treevalue/tree/tree/functional.pxd @@ -12,6 +12,7 @@ cdef class _ValuePathFuncWrapper: cdef readonly object func cdef int index +cdef object _c_delayed_mapping(object so, object func, tuple path, bool delayed) cdef TreeStorage _c_mapping(TreeStorage st, object func, tuple path, bool delayed) cpdef TreeValue mapping(TreeValue tree, object func, bool delayed= *) cdef TreeStorage _c_filter_(TreeStorage st, object func, tuple path, bool remove_empty) diff --git a/treevalue/tree/tree/functional.pyx b/treevalue/tree/tree/functional.pyx index c6c05d4132..4ef0dd5e8d 100644 --- a/treevalue/tree/tree/functional.pyx +++ b/treevalue/tree/tree/functional.pyx @@ -30,7 +30,7 @@ cdef class _ValuePathFuncWrapper: except TypeError: self.index -= 1 -def _p_delayed_mapping(object so, object func, tuple path, bool delayed): +cdef object _c_delayed_mapping(object so, object func, tuple path, bool delayed): cdef object nso = undelay(so) if isinstance(nso, TreeValue): nso = nso._detach() @@ -59,7 +59,7 @@ cdef TreeStorage _c_mapping(TreeStorage st, object func, tuple path, bool delaye _d_res[k] = _c_mapping(v, func, curpath, delayed) else: if delayed: - _d_res[k] = delayed_partial(_p_delayed_mapping, v, func, curpath, delayed) + _d_res[k] = delayed_partial(_c_delayed_mapping, v, func, curpath, delayed) else: _d_res[k] = func(v, curpath) From a95cdaa7c50f675b89cb3ee985d5f3f568012bda Mon Sep 17 00:00:00 2001 From: HansBug Date: Wed, 29 Dec 2021 19:09:09 +0800 Subject: [PATCH 19/26] dev(hansbug): add delayed support to map method --- test/tree/general/base.py | 34 +++++++++++++++++++++++++++++-- treevalue/tree/general/general.py | 5 +++-- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/test/tree/general/base.py b/test/tree/general/base.py index 75ce513726..b68712696f 100644 --- a/test/tree/general/base.py +++ b/test/tree/general/base.py @@ -5,7 +5,7 @@ import numpy as np import pytest -from treevalue.tree import func_treelize, TreeValue, raw, mapping +from treevalue.tree import func_treelize, TreeValue, raw, mapping, delayed def get_tree_test(tree_value_clazz: Type[TreeValue]): @@ -356,8 +356,38 @@ def test_call(self): assert t2.add(t1) == tree_value_clazz({'a': 2, 'b': 4, 'x': {'c': 6, 'd': 8}}) def test_map(self): + cnt = 0 + + def f(x): + nonlocal cnt + cnt += 1 + return x + 2 + t1 = tree_value_clazz({'a': 1, 'b': 2, 'x': {'c': 3, 'd': 4}}) - assert t1.map(lambda x: x + 2) == tree_value_clazz({'a': 3, 'b': 4, 'x': {'c': 5, 'd': 6}}) + assert cnt == 0 + t2 = t1.map(f) + assert cnt == 4 + assert t2 == tree_value_clazz({'a': 3, 'b': 4, 'x': {'c': 5, 'd': 6}}) + + cnt = 0 + t3 = tree_value_clazz({ + 'a': delayed(lambda: t1.a), + 'b': delayed(lambda: t1.b), + 'x': delayed(lambda: t1.x), + }) + assert cnt == 0 + + t4 = t3.map(f, delayed=True) + assert cnt == 0 + + assert t4.a == 3 + assert cnt == 1 + + assert t4 == tree_value_clazz({'a': 3, 'b': 4, 'x': {'c': 5, 'd': 6}}) + assert cnt == 4 + + assert t4.a == 3 + assert cnt == 4 def test_type(self): t1 = tree_value_clazz({'a': 1, 'b': 2, 'x': {'c': 3, 'd': 4}}) diff --git a/treevalue/tree/general/general.py b/treevalue/tree/general/general.py index c7c4b72908..9015f509ab 100644 --- a/treevalue/tree/general/general.py +++ b/treevalue/tree/general/general.py @@ -147,13 +147,14 @@ def type(self, clazz: Type[_TreeValue]) -> _TreeValue: return typetrans(self, clazz) @_decorate_method - def map(self, mapper): + def map(self, mapper, delayed=False): """ Overview: Do mapping on every value in this tree. Arguments: - func (:obj:): Function for mapping + - delayed (:obj:`bool`): Enable delayed mode for this mapping. Returns: - tree (:obj:`_TreeValue`): Mapped tree value object. @@ -164,7 +165,7 @@ def map(self, mapper): >>> t.map(lambda: 1) # FastTreeValue({'a': 1, 'b': 1, 'x': {'c': 1, 'd': 1}}) >>> t.map(lambda x, p: p) # FastTreeValue({'a': ('a',), 'b': ('b',), 'x': {'c': ('x', 'c'), 'd': ('x', 'd')}}) """ - return mapping(self, mapper) + return mapping(self, mapper, delayed) @_decorate_method def mask(self, mask_: TreeValue, remove_empty: bool = True): From 7f63f2de24663d18a8a5727cac0f97a5c32b92a4 Mon Sep 17 00:00:00 2001 From: HansBug Date: Thu, 30 Dec 2021 20:06:08 +0800 Subject: [PATCH 20/26] dev(hansbug): remove useless cimport line --- treevalue/tree/tree/functional.pyx | 1 - 1 file changed, 1 deletion(-) diff --git a/treevalue/tree/tree/functional.pyx b/treevalue/tree/tree/functional.pyx index 65e5def1b8..66b9987298 100644 --- a/treevalue/tree/tree/functional.pyx +++ b/treevalue/tree/tree/functional.pyx @@ -8,7 +8,6 @@ from functools import partial from libcpp cimport bool from .tree cimport TreeValue -from .tree import delayed as _func_delayed from ..common.delay cimport undelay from ..common.delay import delayed_partial from ..common.storage cimport TreeStorage From 3a738c40aa6c1db48d1682ece1c7f166406f3cb4 Mon Sep 17 00:00:00 2001 From: HansBug Date: Sat, 1 Jan 2022 10:22:04 +0800 Subject: [PATCH 21/26] dev(hansbug): add base delayed option (not implemented) --- treevalue/tree/func/cfunc.pxd | 8 +++++--- treevalue/tree/func/cfunc.pyx | 18 ++++++++++-------- treevalue/tree/func/func.py | 10 +++++----- treevalue/tree/tree/structural.pyx | 2 +- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/treevalue/tree/func/cfunc.pxd b/treevalue/tree/func/cfunc.pxd index 1dd90f44de..5f529bb19f 100644 --- a/treevalue/tree/func/cfunc.pxd +++ b/treevalue/tree/func/cfunc.pxd @@ -6,11 +6,13 @@ from libcpp cimport bool from .modes cimport _e_tree_mode cdef object _c_func_treelize_run(object func, list args, dict kwargs, - _e_tree_mode mode, bool inherit, bool allow_missing, object missing_func) + _e_tree_mode mode, bool inherit, + bool allow_missing, object missing_func, + bool delayed) cpdef object _d_func_treelize(object func, object mode, object return_type, bool inherit, object missing, - object subside, object rise) + bool delayed, object subside, object rise) cdef object _c_common_value(object item) cdef tuple _c_missing_process(object missing) cpdef object func_treelize(object mode= *, object return_type= *, bool inherit= *, object missing= *, - object subside= *, object rise= *) + bool delayed= *, object subside= *, object rise= *) diff --git a/treevalue/tree/func/cfunc.pyx b/treevalue/tree/func/cfunc.pyx index d24f2ec888..6a1ca35c96 100644 --- a/treevalue/tree/func/cfunc.pyx +++ b/treevalue/tree/func/cfunc.pyx @@ -17,7 +17,9 @@ _VALUE_IS_MISSING = SingletonMark('value_is_missing') MISSING_NOT_ALLOW = SingletonMark("missing_not_allow") cdef object _c_func_treelize_run(object func, list args, dict kwargs, - _e_tree_mode mode, bool inherit, bool allow_missing, object missing_func): + _e_tree_mode mode, bool inherit, + bool allow_missing, object missing_func, + bool delayed): cdef list ck_args = [] cdef list ck_kwargs = [] cdef bool has_tree = False @@ -117,7 +119,7 @@ cdef object _c_func_treelize_run(object func, list args, dict kwargs, )) _d_res[k] = _c_func_treelize_run(func, _l_args, _d_kwargs, - mode, inherit, allow_missing, missing_func) + mode, inherit, allow_missing, missing_func, delayed) return TreeStorage(_d_res) @@ -131,7 +133,7 @@ def _w_rise_func(object tree, bool dict_=True, bool list_=True, bool tuple_=True # runtime function def _w_func_treelize_run(*args, object __w_func, _e_tree_mode __w_mode, object __w_return_type, bool __w_inherit, bool __w_allow_missing, object __w_missing_func, - object __w_subside, object __w_rise, **kwargs): + bool __w_delayed, object __w_subside, object __w_rise, **kwargs): cdef list _a_args = [(item._detach() if isinstance(item, TreeValue) else item) for item in args] cdef dict _a_kwargs = {k: (v._detach() if isinstance(v, TreeValue) else v) for k, v in kwargs.items()} @@ -140,7 +142,7 @@ def _w_func_treelize_run(*args, object __w_func, _e_tree_mode __w_mode, object _ _a_kwargs = {key: _w_subside_func(value, **__w_subside) for key, value in _a_kwargs.items()} cdef object _st_res = _c_func_treelize_run(__w_func, _a_args, _a_kwargs, __w_mode, - __w_inherit, __w_allow_missing, __w_missing_func) + __w_inherit, __w_allow_missing, __w_missing_func, __w_delayed) cdef object _o_res if __w_return_type is not None: @@ -176,7 +178,7 @@ cdef inline tuple _c_missing_process(object missing): # build-time function cpdef object _d_func_treelize(object func, object mode, object return_type, bool inherit, object missing, - object subside, object rise): + bool delayed, object subside, object rise): cdef _e_tree_mode _v_mode = _c_load_mode(mode) cdef bool allow_missing cdef object missing_func @@ -195,12 +197,12 @@ cpdef object _d_func_treelize(object func, object mode, object return_type, bool _c_check(_v_mode, return_type, inherit, allow_missing, missing_func) return partial(_w_func_treelize_run, __w_func=func, __w_mode=_v_mode, __w_return_type=return_type, __w_inherit=inherit, __w_allow_missing=allow_missing, __w_missing_func=missing_func, - __w_subside=_v_subside, __w_rise=_v_rise) + __w_delayed=delayed, __w_subside=_v_subside, __w_rise=_v_rise) @cython.binding(True) cpdef object func_treelize(object mode='strict', object return_type=TreeValue, bool inherit=True, object missing=MISSING_NOT_ALLOW, - object subside=None, object rise=None): + bool delayed=False, object subside=None, object rise=None): """ Overview: Wrap a common function to tree-supported function. @@ -233,4 +235,4 @@ cpdef object func_treelize(object mode='strict', object return_type=TreeValue, >>> ssum(t1, t2) # TreeValue({'a': 12, 'b': 24, 'x': {'c': 36, 'd': 9}}) """ return partial(_d_func_treelize, mode=mode, return_type=return_type, - inherit=inherit, missing=missing, subside=subside, rise=rise) + inherit=inherit, missing=missing, delayed=delayed, subside=subside, rise=rise) diff --git a/treevalue/tree/func/func.py b/treevalue/tree/func/func.py index fb8ac0caa5..3c44ef3794 100644 --- a/treevalue/tree/func/func.py +++ b/treevalue/tree/func/func.py @@ -12,7 +12,7 @@ def func_treelize(mode: str = 'strict', return_type: Optional[Type[TreeClassType_]] = TreeValue, - inherit: bool = True, missing: Union[Any, Callable] = MISSING_NOT_ALLOW, + inherit: bool = True, missing: Union[Any, Callable] = MISSING_NOT_ALLOW, delayed: bool = False, subside: Union[Mapping, bool, None] = None, rise: Union[Mapping, bool, None] = None): """ Overview: @@ -21,7 +21,7 @@ def func_treelize(mode: str = 'strict', return_type: Optional[Type[TreeClassType Arguments: - mode (:obj:`str`): Mode of the wrapping, default is `strict`. - return_type (:obj:`Optional[Type[TreeClassType_]]`): Return type of the wrapped function, default is `TreeValue`. - - inherit (:obj:`bool`): Allow inherit in wrapped function, default is `True`. + - inherit (:obj:`bool`): Allow inheriting in wrapped function, default is `True`. - missing (:obj:`Union[Any, Callable]`): Missing value or lambda generator of when missing, \ default is `MISSING_NOT_ALLOW`, which means raise `KeyError` when missing detected. - subside (:obj:`Union[Mapping, bool, None]`): Subside enabled to function's arguments or not, \ @@ -47,7 +47,7 @@ def func_treelize(mode: str = 'strict', return_type: Optional[Type[TreeClassType """ def _decorator(func): - _treelized = _c_func_treelize(mode, return_type, inherit, missing, subside, rise)(func) + _treelized = _c_func_treelize(mode, return_type, inherit, missing, delayed, subside, rise)(func) @wraps(func) def _new_func(*args, **kwargs): @@ -82,7 +82,7 @@ def method_treelize(mode: str = 'strict', return_type: Optional[Type[TreeClassTy - mode (:obj:`str`): Mode of the wrapping, default is `strict`. - return_type (:obj:`Optional[Type[TreeClassType_]]`): Return type of the wrapped function, \ default is `AUTO_DETECT_RETURN_VALUE`, which means automatically use the decorated method's class. - - inherit (:obj:`bool`): Allow inherit in wrapped function, default is `True`. + - inherit (:obj:`bool`): Allow inheriting in wrapped function, default is `True`. - missing (:obj:`Union[Any, Callable]`): Missing value or lambda generator of when missing, \ default is `MISSING_NOT_ALLOW`, which means raise `KeyError` when missing detected. - subside (:obj:`Union[Mapping, bool, None]`): Subside enabled to function's arguments or not, \ @@ -154,7 +154,7 @@ def classmethod_treelize(mode: str = 'strict', return_type: Optional[Type[TreeCl - mode (:obj:`str`): Mode of the wrapping, default is `strict`. - return_type (:obj:`Optional[Type[TreeClassType_]]`): Return type of the wrapped function, \ default is `AUTO_DETECT_RETURN_VALUE`, which means automatically use the decorated method's class. - - inherit (:obj:`bool`): Allow inherit in wrapped function, default is `True`. + - inherit (:obj:`bool`): Allow inheriting in wrapped function, default is `True`. - missing (:obj:`Union[Any, Callable]`): Missing value or lambda generator of when missing, \ default is `MISSING_NOT_ALLOW`, which means raise `KeyError` when missing detected. - subside (:obj:`Union[Mapping, bool, None]`): Subside enabled to function's arguments or not, \ diff --git a/treevalue/tree/tree/structural.pyx b/treevalue/tree/tree/structural.pyx index 1a2db91fa1..3d67cf72d7 100644 --- a/treevalue/tree/tree/structural.pyx +++ b/treevalue/tree/tree/structural.pyx @@ -94,7 +94,7 @@ cdef object _c_subside(object value, bool dict_, bool list_, bool tuple_, bool i allow_missing, missing_func = _c_missing_process(missing) return _c_func_treelize_run(_SubsideCall(builder), args, {}, - _c_load_mode(mode), inherit, allow_missing, missing_func), _i_types + _c_load_mode(mode), inherit, allow_missing, missing_func, False), _i_types cdef inline object _c_subside_keep_type(object t): return t From 33ac4c6e03b544cee9d0a66b5071da4d5bc330d4 Mon Sep 17 00:00:00 2001 From: HansBug Date: Sat, 1 Jan 2022 11:11:11 +0800 Subject: [PATCH 22/26] dev(hansbug): complete delayed mode for func_treelize --- test/tree/func/test_func.py | 105 ++++++++++++++++++++++++++++++ test/tree/func/test_outer.py | 43 ++++++++++++ treevalue/tree/func/cfunc.pxd | 8 +-- treevalue/tree/func/cfunc.pyx | 86 ++++++++++++++++++------ treevalue/tree/func/func.py | 2 + treevalue/tree/general/general.py | 9 +-- 6 files changed, 225 insertions(+), 28 deletions(-) diff --git a/test/tree/func/test_func.py b/test/tree/func/test_func.py index 6cd86b500b..1a56fe13d4 100644 --- a/test/tree/func/test_func.py +++ b/test/tree/func/test_func.py @@ -296,3 +296,108 @@ def f(x, y, z): 'a': 23, 'b': 35, 'c': {'x': 30, 'y': 32}, }) + + def test_delayed_treelize(self): + t1 = TreeValue({ + 'a': 1, 'b': 2, 'x': {'c': 3, 'd': 4}, + }) + t2 = TreeValue({ + 'a': 11, 'b': 23, 'x': {'c': 35, 'd': 47}, + }) + + cnt_1 = 0 + + @func_treelize(delayed=True) + def total(a, b): + nonlocal cnt_1 + cnt_1 += 1 + return a + b + + # positional + t3 = total(t1, t2) + assert cnt_1 == 0 + + assert t3.a == 12 + assert cnt_1 == 1 + assert t3.x == TreeValue({'c': 38, 'd': 51}) + assert cnt_1 == 3 + assert t3 == TreeValue({ + 'a': 12, 'b': 25, 'x': {'c': 38, 'd': 51} + }) + assert cnt_1 == 4 + + # keyword + cnt_1 = 0 + t3 = total(a=t1, b=t2) + assert cnt_1 == 0 + + assert t3.a == 12 + assert cnt_1 == 1 + assert t3.x == TreeValue({'c': 38, 'd': 51}) + assert cnt_1 == 3 + assert t3 == TreeValue({ + 'a': 12, 'b': 25, 'x': {'c': 38, 'd': 51} + }) + assert cnt_1 == 4 + + # positional, with constant + cnt_1 = 0 + t3 = total(1, t2) + assert cnt_1 == 0 + + assert t3.a == 12 + assert cnt_1 == 1 + assert t3.x == TreeValue({'c': 36, 'd': 48}) + assert cnt_1 == 3 + assert t3 == TreeValue({ + 'a': 12, 'b': 24, 'x': {'c': 36, 'd': 48} + }) + assert cnt_1 == 4 + + # keyword, with constant + cnt_1 = 0 + t3 = total(b=1, a=t2) + assert cnt_1 == 0 + + assert t3.a == 12 + assert cnt_1 == 1 + assert t3.x == TreeValue({'c': 36, 'd': 48}) + assert cnt_1 == 3 + assert t3 == TreeValue({ + 'a': 12, 'b': 24, 'x': {'c': 36, 'd': 48} + }) + assert cnt_1 == 4 + + # positional, with delay + cnt_1 = 0 + t4 = TreeValue({'v': delayed(lambda: t1)}) + t5 = TreeValue({'v': delayed(lambda: t2)}) + + t6 = total(t4, t5) + assert cnt_1 == 0 + + assert t6.v.a == 12 + assert cnt_1 == 1 + assert t6.v.x == TreeValue({'c': 38, 'd': 51}) + assert cnt_1 == 3 + assert t6 == TreeValue({ + 'v': {'a': 12, 'b': 25, 'x': {'c': 38, 'd': 51}}, + }) + assert cnt_1 == 4 + + # keyword, with delay + cnt_1 = 0 + t4 = TreeValue({'v': delayed(lambda: t1)}) + t5 = TreeValue({'v': delayed(lambda: t2)}) + + t6 = total(a=t4, b=t5) + assert cnt_1 == 0 + + assert t6.v.a == 12 + assert cnt_1 == 1 + assert t6.v.x == TreeValue({'c': 38, 'd': 51}) + assert cnt_1 == 3 + assert t6 == TreeValue({ + 'v': {'a': 12, 'b': 25, 'x': {'c': 38, 'd': 51}}, + }) + assert cnt_1 == 4 diff --git a/test/tree/func/test_outer.py b/test/tree/func/test_outer.py index 145b31f8f4..622b29c34a 100644 --- a/test/tree/func/test_outer.py +++ b/test/tree/func/test_outer.py @@ -38,3 +38,46 @@ def ssum(*args): with pytest.raises(KeyError): _ = ssum(t1, t3) + + def test_delayed_treelize(self): + t1 = TreeValue({ + 'a': 1, 'x': {'c': 3, 'd': 4}, + }) + t2 = TreeValue({ + 'a': 11, 'b': 23, 'x': {'c': 35, }, + }) + + cnt_1 = 0 + + @func_treelize(delayed=True, mode='outer', missing=0) + def total(a, b): + nonlocal cnt_1 + cnt_1 += 1 + return a + b + + # positional + t3 = total(t1, t2) + assert cnt_1 == 0 + + assert t3.a == 12 + assert cnt_1 == 1 + assert t3.x == TreeValue({'c': 38, 'd': 4}) + assert cnt_1 == 3 + assert t3 == TreeValue({ + 'a': 12, 'b': 23, 'x': {'c': 38, 'd': 4} + }) + assert cnt_1 == 4 + + # keyword + cnt_1 = 0 + t3 = total(a=t1, b=t2) + assert cnt_1 == 0 + + assert t3.a == 12 + assert cnt_1 == 1 + assert t3.x == TreeValue({'c': 38, 'd': 4}) + assert cnt_1 == 3 + assert t3 == TreeValue({ + 'a': 12, 'b': 23, 'x': {'c': 38, 'd': 4} + }) + assert cnt_1 == 4 diff --git a/treevalue/tree/func/cfunc.pxd b/treevalue/tree/func/cfunc.pxd index 5f529bb19f..05aec98403 100644 --- a/treevalue/tree/func/cfunc.pxd +++ b/treevalue/tree/func/cfunc.pxd @@ -5,10 +5,10 @@ from libcpp cimport bool from .modes cimport _e_tree_mode -cdef object _c_func_treelize_run(object func, list args, dict kwargs, - _e_tree_mode mode, bool inherit, - bool allow_missing, object missing_func, - bool delayed) +cdef object _c_wrap_func_treelize_run(object func, list args, dict kwargs, _e_tree_mode mode, bool inherit, + bool allow_missing, object missing_func, bool delayed) +cdef object _c_func_treelize_run(object func, list args, dict kwargs, _e_tree_mode mode, bool inherit, + bool allow_missing, object missing_func, bool delayed) cpdef object _d_func_treelize(object func, object mode, object return_type, bool inherit, object missing, bool delayed, object subside, object rise) diff --git a/treevalue/tree/func/cfunc.pyx b/treevalue/tree/func/cfunc.pyx index 6a1ca35c96..c59cf7ad92 100644 --- a/treevalue/tree/func/cfunc.pyx +++ b/treevalue/tree/func/cfunc.pyx @@ -8,6 +8,7 @@ from hbutils.design import SingletonMark from libcpp cimport bool from .modes cimport _e_tree_mode, _c_keyset, _c_load_mode, _c_check +from ..common.delay import delayed_partial from ..common.delay cimport undelay from ..common.storage cimport TreeStorage from ..tree.structural cimport _c_subside, _c_rise @@ -16,10 +17,31 @@ from ..tree.tree cimport TreeValue _VALUE_IS_MISSING = SingletonMark('value_is_missing') MISSING_NOT_ALLOW = SingletonMark("missing_not_allow") -cdef object _c_func_treelize_run(object func, list args, dict kwargs, - _e_tree_mode mode, bool inherit, - bool allow_missing, object missing_func, - bool delayed): +cdef inline object _c_wrap_func_treelize_run(object func, list args, dict kwargs, _e_tree_mode mode, bool inherit, + bool allow_missing, object missing_func, bool delayed): + cdef list _l_args = [] + cdef dict _d_kwargs = {} + cdef str k, ak + cdef object av, v, nv + for av, k, v in args: + nv = undelay(v) + if nv is not v and k is not None: + av[k] = nv + + _l_args.append(nv) + + for ak, (av, k, v) in kwargs.items(): + nv = undelay(v) + if nv is not v and k is not None: + av[k] = nv + + _d_kwargs[ak] = nv + + return _c_func_treelize_run(func, _l_args, _d_kwargs, + mode, inherit, allow_missing, missing_func, delayed) + +cdef object _c_func_treelize_run(object func, list args, dict kwargs, _e_tree_mode mode, bool inherit, + bool allow_missing, object missing_func, bool delayed): cdef list ck_args = [] cdef list ck_kwargs = [] cdef bool has_tree = False @@ -71,22 +93,31 @@ cdef object _c_func_treelize_run(object func, list args, dict kwargs, if at: try: v = av[k] - nv = undelay(v) - if nv is not v: - v = nv - av[k] = v + if delayed: + _l_args.append((av, k, v)) + else: + nv = undelay(v) + if nv is not v: + v = nv + av[k] = v - _l_args.append(v) + _l_args.append(v) except KeyError: if allow_missing: - _l_args.append(_VALUE_IS_MISSING) + if delayed: + _l_args.append((None, None, _VALUE_IS_MISSING)) + else: + _l_args.append(_VALUE_IS_MISSING) else: raise KeyError("Missing is off, key {key} not found in {item}.".format( key=repr(k), item=repr(av), )) else: if inherit: - _l_args.append(undelay(av)) + if delayed: + _l_args.append((None, None, av)) + else: + _l_args.append(undelay(av)) else: raise TypeError("Inherit is off, tree value expected but {type} found in args {index}.".format( type=repr(type(av).__name__), index=repr(i), @@ -97,29 +128,42 @@ cdef object _c_func_treelize_run(object func, list args, dict kwargs, if at: try: v = av[k] - nv = undelay(v) - if nv is not v: - v = nv - av[k] = v + if delayed: + _d_kwargs[ak] = (av, k, v) + else: + nv = undelay(v) + if nv is not v: + v = nv + av[k] = v - _d_kwargs[ak] = v + _d_kwargs[ak] = v except KeyError: if allow_missing: - _d_kwargs[ak] = _VALUE_IS_MISSING + if delayed: + _d_kwargs[ak] = (None, None, _VALUE_IS_MISSING) + else: + _d_kwargs[ak] = _VALUE_IS_MISSING else: raise KeyError("Missing is off, key {key} not found in {item}.".format( key=repr(k), item=repr(av), )) else: if inherit: - _d_kwargs[ak] = undelay(av) + if delayed: + _d_kwargs[ak] = (None, None, av) + else: + _d_kwargs[ak] = undelay(av) else: raise TypeError("Inherit is off, tree value expected but {type} found in args {index}.".format( type=repr(type(av).__name__), index=repr(ak), )) - _d_res[k] = _c_func_treelize_run(func, _l_args, _d_kwargs, - mode, inherit, allow_missing, missing_func, delayed) + if delayed: + _d_res[k] = delayed_partial(_c_wrap_func_treelize_run, func, _l_args, _d_kwargs, + mode, inherit, allow_missing, missing_func, delayed) + else: + _d_res[k] = _c_func_treelize_run(func, _l_args, _d_kwargs, + mode, inherit, allow_missing, missing_func, delayed) return TreeStorage(_d_res) @@ -213,6 +257,8 @@ cpdef object func_treelize(object mode='strict', object return_type=TreeValue, - inherit (:obj:`bool`): Allow inherit in wrapped function, default is `True`. - missing (:obj:`Union[Any, Callable]`): Missing value or lambda generator of when missing, \ default is `MISSING_NOT_ALLOW`, which means raise `KeyError` when missing detected. + - delayed (:obj:`bool`): Enable delayed mode or not, the calculation will be delayed when enabled, \ + default is ``False``, which means to all the calculation at once. - subside (:obj:`Union[Mapping, bool, None]`): Subside enabled to function's arguments or not, \ and subside configuration, default is `None` which means do not use subside. \ When subside is `True`, it will use all the default arguments in `subside` function. diff --git a/treevalue/tree/func/func.py b/treevalue/tree/func/func.py index 3c44ef3794..1a9b200a30 100644 --- a/treevalue/tree/func/func.py +++ b/treevalue/tree/func/func.py @@ -24,6 +24,8 @@ def func_treelize(mode: str = 'strict', return_type: Optional[Type[TreeClassType - inherit (:obj:`bool`): Allow inheriting in wrapped function, default is `True`. - missing (:obj:`Union[Any, Callable]`): Missing value or lambda generator of when missing, \ default is `MISSING_NOT_ALLOW`, which means raise `KeyError` when missing detected. + - delayed (:obj:`bool`): Enable delayed mode or not, the calculation will be delayed when enabled, \ + default is ``False``, which means to all the calculation at once. - subside (:obj:`Union[Mapping, bool, None]`): Subside enabled to function's arguments or not, \ and subside configuration, default is `None` which means do not use subside. \ When subside is `True`, it will use all the default arguments in `subside` function. diff --git a/treevalue/tree/general/general.py b/treevalue/tree/general/general.py index 9015f509ab..9612bece58 100644 --- a/treevalue/tree/general/general.py +++ b/treevalue/tree/general/general.py @@ -309,9 +309,8 @@ def graph(self, root: Optional[str] = None, title: Optional[str] = None, @classmethod @_decorate_method def func(cls, mode: str = 'strict', inherit: bool = True, - missing: Union[Any, Callable] = MISSING_NOT_ALLOW, - subside: Union[Mapping, bool, None] = None, - rise: Union[Mapping, bool, None] = None): + missing: Union[Any, Callable] = MISSING_NOT_ALLOW, delayed: bool = False, + subside: Union[Mapping, bool, None] = None, rise: Union[Mapping, bool, None] = None): """ Overview: Wrap a common function to tree-supported function based on this type. @@ -321,6 +320,8 @@ def func(cls, mode: str = 'strict', inherit: bool = True, - inherit (:obj:`bool`): Allow inherit in wrapped function, default is `True`. - missing (:obj:`Union[Any, Callable]`): Missing value or lambda generator of when missing, \ default is `MISSING_NOT_ALLOW`, which means raise `KeyError` when missing detected. + - delayed (:obj:`bool`): Enable delayed mode or not, the calculation will be delayed when enabled, \ + default is ``False``, which means to all the calculation at once. - subside (:obj:`Union[Mapping, bool, None]`): Subside enabled to function's arguments or not, \ and subside configuration, default is `None` which means do not use subside. \ When subside is `True`, it will use all the default arguments in `subside` function. @@ -344,7 +345,7 @@ def func(cls, mode: str = 'strict', inherit: bool = True, >>> ssum(1, 2) # 3 >>> ssum(t1, t2) # FastTreeValue({'a': 12, 'b': 24, 'x': {'c': 36, 'd': 9}}) """ - return func_treelize(mode, cls, inherit, missing, subside, rise) + return func_treelize(mode, cls, inherit, missing, delayed, subside, rise) @classmethod @_decorate_method From 401b495f597ed5a553234f82d3330aa668cc1dfa Mon Sep 17 00:00:00 2001 From: HansBug Date: Sat, 1 Jan 2022 11:23:56 +0800 Subject: [PATCH 23/26] dev(hansbug): add delayed for subside and union --- treevalue/tree/func/cfunc.pyx | 10 ++++++---- treevalue/tree/tree/structural.pxd | 4 ++-- treevalue/tree/tree/structural.pyx | 18 +++++++++++------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/treevalue/tree/func/cfunc.pyx b/treevalue/tree/func/cfunc.pyx index c59cf7ad92..47cc69a148 100644 --- a/treevalue/tree/func/cfunc.pyx +++ b/treevalue/tree/func/cfunc.pyx @@ -168,8 +168,8 @@ cdef object _c_func_treelize_run(object func, list args, dict kwargs, _e_tree_mo return TreeStorage(_d_res) def _w_subside_func(object value, bool dict_=True, bool list_=True, bool tuple_=True, bool inherit=True, - object mode='strict', object missing=MISSING_NOT_ALLOW): - return _c_subside(value, dict_, list_, tuple_, inherit, mode, missing)[0] + object mode='strict', object missing=MISSING_NOT_ALLOW, bool delayed=False): + return _c_subside(value, dict_, list_, tuple_, inherit, mode, missing, delayed)[0] def _w_rise_func(object tree, bool dict_=True, bool list_=True, bool tuple_=True, object template=None): return _c_rise(tree, dict_, list_, tuple_, template) @@ -181,9 +181,11 @@ def _w_func_treelize_run(*args, object __w_func, _e_tree_mode __w_mode, object _ cdef list _a_args = [(item._detach() if isinstance(item, TreeValue) else item) for item in args] cdef dict _a_kwargs = {k: (v._detach() if isinstance(v, TreeValue) else v) for k, v in kwargs.items()} + cdef dict _w_subside_cfg if __w_subside is not None: - _a_args = [_w_subside_func(item, **__w_subside) for item in _a_args] - _a_kwargs = {key: _w_subside_func(value, **__w_subside) for key, value in _a_kwargs.items()} + _w_subside_cfg = {'delayed': __w_delayed, **__w_subside} + _a_args = [_w_subside_func(item, **_w_subside_cfg) for item in _a_args] + _a_kwargs = {key: _w_subside_func(value, **_w_subside_cfg) for key, value in _a_kwargs.items()} cdef object _st_res = _c_func_treelize_run(__w_func, _a_args, _a_kwargs, __w_mode, __w_inherit, __w_allow_missing, __w_missing_func, __w_delayed) diff --git a/treevalue/tree/tree/structural.pxd b/treevalue/tree/tree/structural.pxd index a84eab61ed..3fa1204e0a 100644 --- a/treevalue/tree/tree/structural.pxd +++ b/treevalue/tree/tree/structural.pxd @@ -12,10 +12,10 @@ cdef object _c_subside_process(tuple value, object it) cdef tuple _c_subside_build(object value, bool dict_, bool list_, bool tuple_) cdef void _c_subside_missing() cdef object _c_subside(object value, bool dict_, bool list_, bool tuple_, bool inherit, - object mode, object missing) + object mode, object missing, bool delayed) cdef object _c_subside_keep_type(object t) cpdef object subside(object value, bool dict_= *, bool list_= *, bool tuple_= *, - object return_type= *, bool inherit= *, object mode= *, object missing= *) + object return_type= *, bool inherit= *, object mode= *, object missing= *, bool delayed= *) cdef object _c_rise_tree_builder(tuple p, object it) cdef tuple _c_rise_tree_process(object t) diff --git a/treevalue/tree/tree/structural.pyx b/treevalue/tree/tree/structural.pyx index 3d67cf72d7..9d30b6397f 100644 --- a/treevalue/tree/tree/structural.pyx +++ b/treevalue/tree/tree/structural.pyx @@ -84,7 +84,7 @@ cdef inline void _c_subside_missing(): pass cdef object _c_subside(object value, bool dict_, bool list_, bool tuple_, bool inherit, - object mode, object missing): + object mode, object missing, bool delayed): cdef object builder, _i_args, _i_types builder, _i_args, _i_types = _c_subside_build(value, dict_, list_, tuple_) cdef list args = list(_i_args) @@ -94,7 +94,7 @@ cdef object _c_subside(object value, bool dict_, bool list_, bool tuple_, bool i allow_missing, missing_func = _c_missing_process(missing) return _c_func_treelize_run(_SubsideCall(builder), args, {}, - _c_load_mode(mode), inherit, allow_missing, missing_func, False), _i_types + _c_load_mode(mode), inherit, allow_missing, missing_func, delayed), _i_types cdef inline object _c_subside_keep_type(object t): return t @@ -102,7 +102,7 @@ cdef inline object _c_subside_keep_type(object t): @cython.binding(True) cpdef object subside(object value, bool dict_=True, bool list_=True, bool tuple_=True, object return_type=None, bool inherit=True, - object mode='strict', object missing=MISSING_NOT_ALLOW): + object mode='strict', object missing=MISSING_NOT_ALLOW, bool delayed=False): """ Overview: Drift down the structures (list, tuple, dict) down to the tree's value. @@ -119,6 +119,8 @@ cpdef object subside(object value, bool dict_=True, bool list_=True, bool tuple_ - mode (:obj:`str`): Mode of the wrapping, default is `strict`. - missing (:obj:`Union[Any, Callable]`): Missing value or lambda generator of when missing, \ default is `MISSING_NOT_ALLOW`, which means raise `KeyError` when missing detected. + - delayed (:obj:`bool`): Enable delayed mode or not, the calculation will be delayed when enabled, \ + default is ``False``, which means to all the calculation at once. Returns: - return (:obj:`_TreeValue`): Subsided tree value. @@ -143,7 +145,7 @@ cpdef object subside(object value, bool dict_=True, bool list_=True, bool tuple_ >>> #}), all structures above the tree values are subsided to the bottom of the tree. """ cdef object result, _i_types - result, _i_types = _c_subside(value, dict_, list_, tuple_, inherit, mode, missing) + result, _i_types = _c_subside(value, dict_, list_, tuple_, inherit, mode, missing, delayed) cdef object type_ cdef set types @@ -162,7 +164,7 @@ cpdef object subside(object value, bool dict_=True, bool list_=True, bool tuple_ @cython.binding(True) def union(*trees, object return_type=None, bool inherit=True, - object mode='strict', object missing=MISSING_NOT_ALLOW): + object mode='strict', object missing=MISSING_NOT_ALLOW, bool delayed=False): """ Overview: Union tree values together. @@ -170,10 +172,12 @@ def union(*trees, object return_type=None, bool inherit=True, Arguments: - trees (:obj:`_TreeValue`): Tree value objects. - return_type (:obj:`Optional[Type[_ClassType]]`): Return type of the wrapped function, default is `TreeValue`. - - inherit (:obj:`bool`): Allow inherit in wrapped function, default is `True`. + - inherit (:obj:`bool`): Allow inheriting in wrapped function, default is `True`. - mode (:obj:`str`): Mode of the wrapping, default is `strict`. - missing (:obj:`Union[Any, Callable]`): Missing value or lambda generator of when missing, \ default is `MISSING_NOT_ALLOW`, which means raise `KeyError` when missing detected. + - delayed (:obj:`bool`): Enable delayed mode or not, the calculation will be delayed when enabled, \ + default is ``False``, which means to all the calculation at once. Returns: - result (:obj:`TreeValue`): Unionised tree value. @@ -184,7 +188,7 @@ def union(*trees, object return_type=None, bool inherit=True, >>> union(t, tx) # TreeValue({'a': (1, True), 'b': (2, False), 'x': {'c': (3, True), 'd': (4, False)}}) """ cdef object result, _i_types - result, _i_types = _c_subside(tuple(trees), True, True, True, inherit, mode, missing) + result, _i_types = _c_subside(tuple(trees), True, True, True, inherit, mode, missing, delayed) cdef object type_ cdef list types From 1213bcd7ee059ef3352c52b4cee954735e4cb21f Mon Sep 17 00:00:00 2001 From: HansBug Date: Sat, 1 Jan 2022 12:24:06 +0800 Subject: [PATCH 24/26] dev(hansbug): add delay test for FastTreeValue.func --- test/tree/general/base.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/tree/general/base.py b/test/tree/general/base.py index b68712696f..64c9eeea8f 100644 --- a/test/tree/general/base.py +++ b/test/tree/general/base.py @@ -602,4 +602,23 @@ def ssum(x, y): assert ssum(t1, t2) == tree_value_clazz({'a': 12, 'b': 22, 'x': {'c': 36, 'd': 52}}) + cnt_1 = 0 + + @tree_value_clazz.func(delayed=True) + def ssumx(x, y): + nonlocal cnt_1 + cnt_1 += 1 + return x + y + + cnt_1 = 0 + t3 = ssumx(t1, t2) + assert cnt_1 == 0 + + assert t3.a == 12 + assert cnt_1 == 1 + assert t3.x == tree_value_clazz({'c': 36, 'd': 52}) + assert cnt_1 == 3 + assert t3 == tree_value_clazz({'a': 12, 'b': 22, 'x': {'c': 36, 'd': 52}}) + assert cnt_1 == 4 + return _TestClass From 40911839ec349f2d716d9b36f4e4c8f55e858d95 Mon Sep 17 00:00:00 2001 From: HansBug Date: Sat, 1 Jan 2022 12:27:52 +0800 Subject: [PATCH 25/26] test(hansbug): add delayed test for union and subside --- test/tree/tree/test_structural.py | 32 +++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/tree/tree/test_structural.py b/test/tree/tree/test_structural.py index 4196c14903..b9ffdcb700 100644 --- a/test/tree/tree/test_structural.py +++ b/test/tree/tree/test_structural.py @@ -39,6 +39,14 @@ class MyTreeValue(TreeValue): 'a': (1, 3), 'b': (2, None), 'x': {'c': (3, 1), 'd': (None, 2)}, }) + def test_union_delayed(self): + class MyTreeValue(TreeValue): + pass + + t = TreeValue({'a': 1, 'b': 2, 'x': {'c': 3, 'd': 4}}) + t1 = MyTreeValue({'a': 11, 'b': 22, 'x': {'c': 33, 'd': 44}}) + assert union(t, t1, delayed=True) == TreeValue({'a': (1, 11), 'b': (2, 22), 'x': {'c': (3, 33), 'd': (4, 44)}}) + def test_subside(self): assert subside({'a': (1, 2), 'b': [3, 4]}) == {'a': (1, 2), 'b': [3, 4]} assert subside({'a': (1, 2), 'b': [3, 4]}, return_type=TreeValue) == {'a': (1, 2), 'b': [3, 4]} @@ -107,6 +115,30 @@ class MyTreeValue(TreeValue): assert subside({'a': 1, 'b': 2, 'x': {'c': 3, 'd': 4}, 'e': [3, 4, 5]}) == \ {'a': 1, 'b': 2, 'x': {'c': 3, 'd': 4}, 'e': [3, 4, 5]} + def test_subside_delayed(self): + class MyTreeValue(TreeValue): + pass + + original2 = { + 'a': TreeValue({'a': 1, 'b': 2}), + 'x': { + 'c': MyTreeValue({'a': 3, 'b': 4}), + 'd': [ + MyTreeValue({'a': 5, 'b': 6}), + MyTreeValue({'a': 7, 'b': 8}), + ] + }, + 'k': '233' + } + assert subside(original2, delayed=True) == TreeValue({ + 'a': raw({'a': 1, 'k': '233', 'x': {'c': 3, 'd': [5, 7]}}), + 'b': raw({'a': 2, 'k': '233', 'x': {'c': 4, 'd': [6, 8]}}), + }) + assert subside(original2, return_type=MyTreeValue, delayed=True) == MyTreeValue({ + 'a': raw({'a': 1, 'k': '233', 'x': {'c': 3, 'd': [5, 7]}}), + 'b': raw({'a': 2, 'k': '233', 'x': {'c': 4, 'd': [6, 8]}}), + }) + def test_rise(self): t1 = TreeValue({'x': raw({'a': [1, 2], 'b': [2, 3]}), 'y': raw({'a': [5, 6, 7], 'b': [7, 8]})}) assert rise(t1) == { From 17c2ac4f5a50e5e2163a45485892bf3aa42467e4 Mon Sep 17 00:00:00 2001 From: HansBug Date: Sun, 2 Jan 2022 10:45:53 +0800 Subject: [PATCH 26/26] dev(hansbug): use _c_undelay_data when checkout datas --- treevalue/tree/common/storage.pxd | 3 ++ treevalue/tree/common/storage.pyx | 65 ++++++++++++------------------ treevalue/tree/func/cfunc.pyx | 28 ++++--------- treevalue/tree/tree/flatten.pyx | 21 ++-------- treevalue/tree/tree/functional.pyx | 32 ++++----------- treevalue/tree/tree/service.pyx | 9 +---- treevalue/tree/tree/structural.pyx | 9 +---- treevalue/tree/tree/tree.pyx | 8 +--- 8 files changed, 53 insertions(+), 122 deletions(-) diff --git a/treevalue/tree/common/storage.pxd b/treevalue/tree/common/storage.pxd index 986cb527be..4713bc650e 100644 --- a/treevalue/tree/common/storage.pxd +++ b/treevalue/tree/common/storage.pxd @@ -31,3 +31,6 @@ cdef class TreeStorage: cpdef public void deepcopyx_from(self, TreeStorage ts, copy_func, bool allow_delayed) cpdef public object create_storage(dict value) +cdef object _c_undelay_data(dict data, object k, object v) +cdef object _c_undelay_not_none_data(dict data, object k, object v) +cdef object _c_undelay_check_data(dict data, object k, object v) diff --git a/treevalue/tree/common/storage.pyx b/treevalue/tree/common/storage.pyx index 6f28c7d49a..ffd698db72 100644 --- a/treevalue/tree/common/storage.pyx +++ b/treevalue/tree/common/storage.pyx @@ -40,25 +40,14 @@ cdef class TreeStorage: cdef object v, nv try: v = self.map[key] - nv = undelay(v) - if nv is not v: - self.map[key] = nv - return nv - else: - return v + return _c_undelay_data(self.map, key, v) except KeyError: raise KeyError(f"Key {repr(key)} not found in this tree.") cpdef public object get_or_default(self, str key, object default): cdef object v, nv v = self.map.get(key, default) - nv = undelay(v) - if nv is not v: - v = nv - if key in self.map: - self.map[key] = v - - return v + return _c_undelay_check_data(self.map, key, v) cpdef public void del_(self, str key) except *: try: @@ -90,10 +79,7 @@ cdef class TreeStorage: cdef object v, obj, nv for k, v in self.map.items(): if not allow_delayed: - nv = undelay(v) - if nv is not v: - v = nv - self.map[k] = v + v = _c_undelay_data(self.map, k, v) if isinstance(v, TreeStorage): result[k] = v.jsondumpx(copy_func, need_raw, allow_delayed) @@ -134,10 +120,7 @@ cdef class TreeStorage: if k in detached: v = detached[k] if not allow_delayed: - nv = undelay(v) - if nv is not v: - v = nv - detached[k] = v + v = _c_undelay_data(detached, k, v) if isinstance(v, TreeStorage): if k in self.map and isinstance(self.map[k], TreeStorage): @@ -181,16 +164,10 @@ cdef class TreeStorage: if self_keys == other_keys: for key in self_keys: self_v = self.map[key] - self_nv = undelay(self_v) - if self_nv is not self_v: - self_v = self_nv - self.map[key] = self_v + self_v = _c_undelay_data(self.map, key, self_v) other_v = other_map[key] - other_nv = undelay(other_v) - if other_nv is not other_v: - other_v = other_nv - other_map[key] = other_v + other_v = _c_undelay_data(other_map, key, other_v) if self_v != other_v: return False @@ -214,21 +191,13 @@ cdef class TreeStorage: cdef str k cdef object v, nv for k, v in self.map.items(): - nv = undelay(v) - if nv is not v: - v = nv - self.map[k] = v - - yield v + yield _c_undelay_data(self.map, k, v) def items(self): cdef str k cdef object v, nv for k, v in self.map.items(): - nv = undelay(v) - if nv is not v: - v = nv - self.map[k] = v + v = _c_undelay_data(self.map, k, v) yield k, v @@ -244,3 +213,21 @@ cpdef object create_storage(dict value): _map[k] = unraw(v) return TreeStorage(_map) + +cdef inline object _c_undelay_data(dict data, object k, object v): + cdef object nv = undelay(v) + if nv is not v: + data[k] = nv + return nv + +cdef inline object _c_undelay_not_none_data(dict data, object k, object v): + cdef object nv = undelay(v) + if nv is not v and k is not None: + data[k] = nv + return nv + +cdef inline object _c_undelay_check_data(dict data, object k, object v): + cdef object nv = undelay(v) + if nv is not v and k in data: + data[k] = nv + return nv \ No newline at end of file diff --git a/treevalue/tree/func/cfunc.pyx b/treevalue/tree/func/cfunc.pyx index 47cc69a148..535fc77b70 100644 --- a/treevalue/tree/func/cfunc.pyx +++ b/treevalue/tree/func/cfunc.pyx @@ -10,13 +10,15 @@ from libcpp cimport bool from .modes cimport _e_tree_mode, _c_keyset, _c_load_mode, _c_check from ..common.delay import delayed_partial from ..common.delay cimport undelay -from ..common.storage cimport TreeStorage +from ..common.storage cimport TreeStorage, _c_undelay_not_none_data, _c_undelay_data from ..tree.structural cimport _c_subside, _c_rise from ..tree.tree cimport TreeValue _VALUE_IS_MISSING = SingletonMark('value_is_missing') MISSING_NOT_ALLOW = SingletonMark("missing_not_allow") + + cdef inline object _c_wrap_func_treelize_run(object func, list args, dict kwargs, _e_tree_mode mode, bool inherit, bool allow_missing, object missing_func, bool delayed): cdef list _l_args = [] @@ -24,18 +26,10 @@ cdef inline object _c_wrap_func_treelize_run(object func, list args, dict kwargs cdef str k, ak cdef object av, v, nv for av, k, v in args: - nv = undelay(v) - if nv is not v and k is not None: - av[k] = nv - - _l_args.append(nv) + _l_args.append(_c_undelay_not_none_data(av, k, v)) for ak, (av, k, v) in kwargs.items(): - nv = undelay(v) - if nv is not v and k is not None: - av[k] = nv - - _d_kwargs[ak] = nv + _d_kwargs[ak] = _c_undelay_not_none_data(av, k, v) return _c_func_treelize_run(func, _l_args, _d_kwargs, mode, inherit, allow_missing, missing_func, delayed) @@ -96,11 +90,7 @@ cdef object _c_func_treelize_run(object func, list args, dict kwargs, _e_tree_mo if delayed: _l_args.append((av, k, v)) else: - nv = undelay(v) - if nv is not v: - v = nv - av[k] = v - + v = _c_undelay_data(av, k, v) _l_args.append(v) except KeyError: if allow_missing: @@ -131,11 +121,7 @@ cdef object _c_func_treelize_run(object func, list args, dict kwargs, _e_tree_mo if delayed: _d_kwargs[ak] = (av, k, v) else: - nv = undelay(v) - if nv is not v: - v = nv - av[k] = v - + v = _c_undelay_data(av, k, v) _d_kwargs[ak] = v except KeyError: if allow_missing: diff --git a/treevalue/tree/tree/flatten.pyx b/treevalue/tree/tree/flatten.pyx index aa38e822f1..54449a8c2e 100644 --- a/treevalue/tree/tree/flatten.pyx +++ b/treevalue/tree/tree/flatten.pyx @@ -6,8 +6,7 @@ import cython from .tree cimport TreeValue -from ..common.delay cimport undelay -from ..common.storage cimport TreeStorage +from ..common.storage cimport TreeStorage, _c_undelay_data cdef void _c_flatten(TreeStorage st, tuple path, list res) except *: cdef dict data = st.detach() @@ -16,11 +15,7 @@ cdef void _c_flatten(TreeStorage st, tuple path, list res) except *: cdef str k cdef object v, nv for k, v in data.items(): - nv = undelay(v) - if nv is not v: - v = nv - data[k] = v - + v = _c_undelay_data(data, k, v) curpath = path + (k,) if isinstance(v, TreeStorage): _c_flatten(v, curpath, res) @@ -55,11 +50,7 @@ cdef void _c_flatten_values(TreeStorage st, list res) except *: cdef str k cdef object v, nv for k, v in data.items(): - nv = undelay(v) - if nv is not v: - v = nv - data[k] = v - + v = _c_undelay_data(data, k, v) if isinstance(v, TreeStorage): _c_flatten_values(v, res) else: @@ -88,11 +79,7 @@ cdef void _c_flatten_keys(TreeStorage st, tuple path, list res) except *: cdef str k cdef object v, nv for k, v in data.items(): - nv = undelay(v) - if nv is not v: - v = nv - data[k] = v - + v = _c_undelay_data(data, k, v) curpath = path + (k,) if isinstance(v, TreeStorage): _c_flatten_keys(v, curpath, res) diff --git a/treevalue/tree/tree/functional.pyx b/treevalue/tree/tree/functional.pyx index 66b9987298..b94b7300b9 100644 --- a/treevalue/tree/tree/functional.pyx +++ b/treevalue/tree/tree/functional.pyx @@ -10,7 +10,7 @@ from libcpp cimport bool from .tree cimport TreeValue from ..common.delay cimport undelay from ..common.delay import delayed_partial -from ..common.storage cimport TreeStorage +from ..common.storage cimport TreeStorage, _c_undelay_data cdef inline object _c_no_arg(object func, object v, object p): return func() @@ -54,10 +54,7 @@ cdef TreeStorage _c_mapping(TreeStorage st, object func, tuple path, bool delaye cdef tuple curpath for k, v in _d_st.items(): if not delayed: - nv = undelay(v) - if nv is not v: - v = nv - _d_st[k] = v + v = _c_undelay_data(_d_st, k, v) curpath = path + (k,) if isinstance(v, TreeStorage): @@ -112,11 +109,7 @@ cdef TreeStorage _c_filter_(TreeStorage st, object func, tuple path, bool remove cdef tuple curpath cdef TreeStorage curst for k, v in _d_st.items(): - nv = undelay(v) - if nv is not v: - v = nv - _d_st[k] = v - + v = _c_undelay_data(_d_st, k, v) curpath = path + (k,) if isinstance(v, TreeStorage): curst = _c_filter_(v, func, curpath, remove_empty) @@ -170,24 +163,17 @@ cdef object _c_mask(TreeStorage st, object sm, tuple path, bool remove_empty): cdef dict _d_res = {} cdef str k - cdef object v, mv, nv, nmv + cdef object v, mv cdef tuple curpath cdef object curres for k, v in _d_st.items(): - nv = undelay(v) - if nv is not v: - v = nv - _d_st[k] = v - + v = _c_undelay_data(_d_st, k, v) curpath = path + (k,) if _b_tree_mask: mv = _d_sm[k] + mv = _c_undelay_data(_d_sm, k, mv) else: mv = sm - nmv = undelay(mv) - if nmv is not mv: - mv = nmv - _d_sm[k] = mv if isinstance(v, TreeStorage): curres = _c_mask(v, mv, curpath, remove_empty) @@ -233,11 +219,7 @@ cdef object _c_reduce(TreeStorage st, object func, tuple path, object return_typ cdef tuple curpath cdef object curst for k, v in _d_st.items(): - nv = undelay(v) - if nv is not v: - v = nv - _d_st[k] = v - + v = _c_undelay_data(_d_st, k, v) curpath = path + (k,) if isinstance(v, TreeStorage): curst = _c_reduce(v, func, curpath, return_type) diff --git a/treevalue/tree/tree/service.pyx b/treevalue/tree/tree/service.pyx index ed373e02f4..466e277607 100644 --- a/treevalue/tree/tree/service.pyx +++ b/treevalue/tree/tree/service.pyx @@ -9,8 +9,7 @@ import cython from libcpp cimport bool from .tree cimport TreeValue -from ..common.delay cimport undelay -from ..common.storage cimport TreeStorage +from ..common.storage cimport TreeStorage, _c_undelay_data cdef object _keep_object(object obj): return obj @@ -99,11 +98,7 @@ def _p_walk(TreeStorage tree, object type_, tuple path, bool include_nodes): cdef object v, nv cdef tuple curpath for k, v in data.items(): - nv = undelay(v) - if nv is not v: - v = nv - data[k] = v - + v = _c_undelay_data(data, k, v) curpath = path + (k,) if isinstance(v, TreeStorage): yield from _p_walk(v, type_, curpath, include_nodes) diff --git a/treevalue/tree/tree/structural.pyx b/treevalue/tree/tree/structural.pyx index 9d30b6397f..1c5aa049a4 100644 --- a/treevalue/tree/tree/structural.pyx +++ b/treevalue/tree/tree/structural.pyx @@ -10,8 +10,7 @@ from hbutils.design import SingletonMark from libcpp cimport bool from .tree cimport TreeValue -from ..common.delay cimport undelay -from ..common.storage cimport TreeStorage +from ..common.storage cimport TreeStorage, _c_undelay_data from ..func.cfunc cimport _c_func_treelize_run, _c_missing_process from ..func.modes cimport _c_load_mode @@ -228,11 +227,7 @@ cdef tuple _c_rise_tree_process(object t): _l_items = [] _l_values = [] for k, v in detached.items(): - nv = undelay(v) - if nv is not v: - v = nv - detached[k] = v - + v = _c_undelay_data(detached, k, v) _i_item, _i_value = _c_rise_tree_process(v) _l_items.append((k, _i_item)) _l_values.append(_i_value) diff --git a/treevalue/tree/tree/tree.pyx b/treevalue/tree/tree/tree.pyx index fc032e9c44..ced2beb4fa 100644 --- a/treevalue/tree/tree/tree.pyx +++ b/treevalue/tree/tree/tree.pyx @@ -8,7 +8,7 @@ import cython from hbutils.design import SingletonMark from ..common.delay cimport undelay, _c_delayed_partial, DelayedProxy -from ..common.storage cimport TreeStorage, create_storage +from ..common.storage cimport TreeStorage, create_storage, _c_undelay_data from ...utils import format_tree _GET_NO_DEFAULT = SingletonMark('get_no_default') @@ -396,11 +396,7 @@ cdef object _build_tree(TreeStorage st, object type_, str prefix, dict id_pool, id_pool[nid] = path data = st.detach() for k, v in sorted(data.items()): - nv = undelay(v) - if nv is not v: - v = nv - data[k] = v - + v = _c_undelay_data(data, k, v) curpath = path + (k,) _t_prefix = f'{k} --> ' if isinstance(v, TreeStorage):