From 3f77286964f3f610e64173285ee9322d01d32e29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Thu, 2 Sep 2021 12:58:11 +0200 Subject: [PATCH] Adds pseudo recursive getstate --- _unittests/ut_pycode/test_extunittest.py | 3 +- _unittests/ut_pycode/test_profiling.py | 15 ++- _unittests/ut_pycode/test_unittest_assert.py | 97 +++++++++++++++++++ .../jenkinshelper/yaml_helper.py | 2 +- src/pyquickhelper/loghelper/process_helper.py | 4 +- src/pyquickhelper/pycode/_pylint_common.py | 7 +- src/pyquickhelper/pycode/unittest_assert.py | 86 ++++++++++++++++ src/pyquickhelper/pycode/unittest_cst.py | 4 +- 8 files changed, 205 insertions(+), 13 deletions(-) create mode 100644 _unittests/ut_pycode/test_unittest_assert.py create mode 100644 src/pyquickhelper/pycode/unittest_assert.py diff --git a/_unittests/ut_pycode/test_extunittest.py b/_unittests/ut_pycode/test_extunittest.py index 4a564c26c..3c5f3cf14 100644 --- a/_unittests/ut_pycode/test_extunittest.py +++ b/_unittests/ut_pycode/test_extunittest.py @@ -228,7 +228,8 @@ def simple(): rootrem = os.path.normpath(os.path.abspath( os.path.join(os.path.dirname(rootfile), '..'))) - ps, res = self.profile(simple, rootrem=rootrem) # pylint: disable=W0632 + ps, res = self.profile( + simple, rootrem=rootrem) # pylint: disable=W0632 res = res.replace('\\', '/') self.assertIn('pyquickhelper/pandashelper/tblformat.py', res) self.assertNotEmpty(ps) diff --git a/_unittests/ut_pycode/test_profiling.py b/_unittests/ut_pycode/test_profiling.py index e16e7aa7b..24908fbc9 100644 --- a/_unittests/ut_pycode/test_profiling.py +++ b/_unittests/ut_pycode/test_profiling.py @@ -46,7 +46,8 @@ def simple2(): rootrem = os.path.normpath(os.path.abspath( os.path.join(os.path.dirname(rootfile), '..'))) - ps, df = profile(simple, rootrem=rootrem, as_df=True) # pylint: disable=W0632 + ps, df = profile(simple, rootrem=rootrem, + as_df=True) # pylint: disable=W0632 self.assertIsInstance(df, pandas.DataFrame) self.assertEqual(df.loc[0, 'namefct'].split('-')[-1], 'simple2') self.assertNotEmpty(ps) @@ -59,18 +60,22 @@ def simple(): df2rst(df) return df2rst(df) - ps, res = profile(simple, pyinst_format='text') # pylint: disable=W0632 + ps, res = profile( + simple, pyinst_format='text') # pylint: disable=W0632 self.assertIn('.py', res) self.assertNotEmpty(ps) - ps, res = profile(simple, pyinst_format='textu') # pylint: disable=W0632 + ps, res = profile( + simple, pyinst_format='textu') # pylint: disable=W0632 self.assertIn('Recorded', res) self.assertNotEmpty(ps) - ps, res = profile(simple, pyinst_format='html') # pylint: disable=W0632 + ps, res = profile( + simple, pyinst_format='html') # pylint: disable=W0632 self.assertIn("", res) self.assertNotEmpty(ps) self.assertRaise(lambda: profile( simple, pyinst_format='htmlgg'), ValueError) - ps, res = profile(simple, pyinst_format='json') # pylint: disable=W0632 + ps, res = profile( + simple, pyinst_format='json') # pylint: disable=W0632 self.assertIn('"start_time"', res) self.assertNotEmpty(ps) diff --git a/_unittests/ut_pycode/test_unittest_assert.py b/_unittests/ut_pycode/test_unittest_assert.py new file mode 100644 index 000000000..ba950a3de --- /dev/null +++ b/_unittests/ut_pycode/test_unittest_assert.py @@ -0,0 +1,97 @@ +""" +@brief test tree node (time=5s) +""" +import unittest +from pyquickhelper.pycode import ExtTestCase +from pyquickhelper.pycode.unittest_assert import getstate + + +class AAAA: + + def __init__(self): + self.g = ['h'] + + +class AAAA2: + + def __init__(self): + self.g = ['H'] + + +class AAA: + + def __init__(self): + self.glist = [AAAA()] + self.gtuple = (AAAA2(), ) + self.gdict = {0: AAAA()} + + +class BBB: + + def __init__(self): + pass + + def __getstate__(self): + raise NotImplementedError() + + +class TestUnitTestAssert(ExtTestCase): + + def test_getstate(self): + d = dict(a=1, b=2) + r = getstate(d) + self.assertEqual(d, r) + + d = ['R', 1] + r = getstate(d) + self.assertEqual(d, r) + + d = ('R', 1) + r = getstate(d) + self.assertEqual(d, r) + + d = {'R', 1} + r = getstate(d) + self.assertEqual(d, r) + + d = None + r = getstate(d) + self.assertEqual(d, r) + + def test_getstate_recursion(self): + d = dict(a=1, b=2) + d['d'] = d + r = getstate(d) + del r['d'] + del d['d'] + self.assertEqual(d, r) + + def test_getstate2(self): + d = dict(a=1, b=2, c=[5]) + r = getstate(d) + self.assertEqual(d, r) + + d = ['R', 1, (6,)] + r = getstate(d) + self.assertEqual(d, r) + + d = ('R', 1, {1: 2}) + r = getstate(d) + self.assertEqual(d, r) + + def test_getstate_exc(self): + d = [BBB()] + self.assertRaise(lambda: getstate(d), NotImplementedError) + + def test_getstate_obj(self): + d = [AAA()] + r = getstate(d) + expected = [{'glist': [{'g': ['h']}], + 'gtuple': ({'g': ['H']},), + 'gdict': {0: {'g': ['h']}}}] + self.assertEqual(expected, r) + + +if __name__ == "__main__": + TestUnitTestAssert().test_getstate_obj() + unittest.main() diff --git a/src/pyquickhelper/jenkinshelper/yaml_helper.py b/src/pyquickhelper/jenkinshelper/yaml_helper.py index db449d6d6..5cf0efe0a 100644 --- a/src/pyquickhelper/jenkinshelper/yaml_helper.py +++ b/src/pyquickhelper/jenkinshelper/yaml_helper.py @@ -507,7 +507,7 @@ def add_path_win(rows, interpreter, platform, root_project): nbrem = v.split("-")[-1] try: nbrem = int(nbrem) - except ValueError as e: # pragma: no cover + except ValueError: # pragma: no cover raise ValueError( "Unable to interpret '{0}'".format(v)) else: diff --git a/src/pyquickhelper/loghelper/process_helper.py b/src/pyquickhelper/loghelper/process_helper.py index c0dfa0789..f766d6e0b 100644 --- a/src/pyquickhelper/loghelper/process_helper.py +++ b/src/pyquickhelper/loghelper/process_helper.py @@ -39,12 +39,12 @@ def on_terminate(proc): if alive: # send SIGKILL for p in alive: - fLOG("process {} survived SIGTERM; trying SIGKILL" % p) + fLOG("process {} survived SIGTERM; trying SIGKILL".format(p)) p.kill() _, alive = psutil.wait_procs( alive, timeout=timeout, callback=on_terminate) if alive: # give up for p in alive: - fLOG("process {} survived SIGKILL; giving up" % p) + fLOG("process {} survived SIGKILL; giving up".format(p)) return killed diff --git a/src/pyquickhelper/pycode/_pylint_common.py b/src/pyquickhelper/pycode/_pylint_common.py index d9a8cfa2b..cb36e9bba 100644 --- a/src/pyquickhelper/pycode/_pylint_common.py +++ b/src/pyquickhelper/pycode/_pylint_common.py @@ -28,7 +28,9 @@ def _private_test_style_src(fLOG, run_lint, verbose=False, pattern=".*[.]py$"): 'R0911', 'R0916', 'C0200', 'W0223', 'W0122', 'E1003', 'R0205', 'E0001', 'W0143', 'W0107', 'C0415', 'W1202', - 'W0707', 'R1725', 'R1732'), + 'W0707', 'R1725', 'R1732', 'W1514', + 'C0123', 'C0208', 'W1514', 'R1735', + 'R1734'), skip=["windows_scripts.py", "Redefining built-in 'format'", "bokeh_plot.py", @@ -82,7 +84,8 @@ def _private_test_style_test(fLOG, run_lint, verbose=False, pattern=".*[.]py$"): 'W0201', 'E0702', 'W1503', 'C0102', 'W0223', 'W0611', 'R1705', 'W0631', 'W0102', 'R0205', 'W0107', 'C0415', 'W1202', 'W0707', 'R1725', - 'R1732', 'W1514', 'C0208'), + 'R1732', 'W1514', 'C0208', 'R1735', 'W0632', + 'R1734'), skip=["skip_' imported but unused", "skip__' imported but unused", "skip___' imported but unused", diff --git a/src/pyquickhelper/pycode/unittest_assert.py b/src/pyquickhelper/pycode/unittest_assert.py new file mode 100644 index 000000000..7f0eaa1e8 --- /dev/null +++ b/src/pyquickhelper/pycode/unittest_assert.py @@ -0,0 +1,86 @@ +""" +@file +@brief Helpers for unit tests. +""" +import numpy +import pandas + + +def getstate(obj, recursive=True, type_stop=None, type_stack=None, done=None): + """ + Returns the state of an objects. It cannot be used + to restore the object. + + :param obj: object + :param recursive: unfold every object + :param type_stop: stop recursion when type is in this list + :param type_stack: list of types if an exception is raised + :param done: already processed objects (avoids infinite recursion) + :return: state (always as dictionay) + """ + if done is not None and id(obj) in done: + return id(obj) + if obj is None: + return None + if type_stop is None: + type_stop = ( + int, float, str, bytes, + numpy.ndarray, pandas.DataFrame) + if numpy.isscalar(obj): + return obj + if isinstance(obj, type_stop): + return obj + + if type_stack is None: + type_stack = [type(obj)] + else: + type_stack = type_stack.copy() + type_stack.append(type(obj)) + + if done is None: + done = set() + done.add(id(obj)) + + if type(obj) == list: + state = [getstate(o, recursive=recursive, type_stop=type_stop, + type_stack=type_stack, done=done) + for o in obj] + return state + if type(obj) == set: + state = set(getstate(o, recursive=recursive, type_stop=type_stop, + type_stack=type_stack, done=done) + for o in obj) + return state + if type(obj) == tuple: + state = tuple(getstate(o, recursive=recursive, type_stop=type_stop, + type_stack=type_stack, done=done) + for o in obj) + return state + if type(obj) == dict: + state = {a: getstate(o, recursive=recursive, type_stop=type_stop, + type_stack=type_stack, done=done) + for a, o in obj.items()} + return state + + try: + state = obj.__getstate__() + except AttributeError as e: + try: + state = obj.__dict__.copy() + except AttributeError as e: + raise NotImplementedError( + "Unable to retrieve state of object %r, type_stack=%r." + "" % (type(obj), ", ".join(map(str, type_stack)))) from e + except Exception as e: + raise NotImplementedError( + "Unable to retrieve state of object %r, type_stack=%r." + "" % (type(obj), ", ".join(map(str, type_stack)))) from e + except Exception as e: + raise NotImplementedError( + "Unable to retrieve state of object %r, type_stack=%r." + "" % (type(obj), ", ".join(map(str, type_stack)))) from e + + if not recursive: + return state + return getstate(state, recursive=True, type_stop=type_stop, + type_stack=type_stack, done=done) diff --git a/src/pyquickhelper/pycode/unittest_cst.py b/src/pyquickhelper/pycode/unittest_cst.py index 639a2983f..97d3f808f 100644 --- a/src/pyquickhelper/pycode/unittest_cst.py +++ b/src/pyquickhelper/pycode/unittest_cst.py @@ -10,7 +10,7 @@ def compress_cst(data, length=70, as_text=False): """ - Transform a huge constant into a sequence of compressed binary strings. + Transforms a huge constant into a sequence of compressed binary strings. :param data: data :param length: line length @@ -45,7 +45,7 @@ def compress_cst(data, length=70, as_text=False): def decompress_cst(data): """ - Transform a huge constant produced by function @see fn compress_cst + Transforms a huge constant produced by function @see fn compress_cst into the original value. :param data: data