Add function profile2df

_unittests/ut_pycode/test_profiling.py

@@ -6,11 +6,10 @@
 import unittest
 import warnings
 import pandas
-
 from pyquickhelper.pycode import ExtTestCase
 from pyquickhelper.pandashelper import df2rst
 from pyquickhelper import __file__ as rootfile
-from pyquickhelper.pycode.profiling import profile
+from pyquickhelper.pycode.profiling import profile, profile2df
 
 
 class TestProfiling(ExtTestCase):
@@ -51,6 +50,11 @@ def simple2():
         self.assertIsInstance(df, pandas.DataFrame)
         self.assertEqual(df.loc[0, 'namefct'].split('-')[-1], 'simple2')
         self.assertNotEmpty(ps)
+        df = profile2df(ps, False)
+        self.assertIsInstance(df, list)
+        self.assertIsInstance(df[0], dict)
+        df = profile2df(ps, True)
+        self.assertIsInstance(df, pandas.DataFrame)
 
     def test_profile_pyinst(self):
         def simple():

src/pyquickhelper/jenkinshelper/yaml_helper.py

@@ -460,7 +460,8 @@ def add_path_win(rows, interpreter, platform, root_project):
                         "export PATH={0}:$PATH".format(venv_interpreter))
                 pat = '"{0}" -m virtualenv {1}'
                 if isinstance(value, dict):
-                    system_site_packages = value.get('system_site_packages', True)
+                    system_site_packages = value.get(
+                        'system_site_packages', True)
                 else:
                     system_site_packages = True
                 if system_site_packages:

src/pyquickhelper/pycode/profiling.py

@@ -46,24 +46,57 @@ def add_rows(rows, d):
     return rows
 
 
+def profile2df(ps, as_df=True):
+    """
+    Converts profiling statistics into a Dataframe.
+
+    :param ps: an instance of `pstats
+        <https://docs.python.org/3/library/profile.html#pstats.Stats>`_
+    :param as_df: returns the results as a dataframe (True)
+        or a list of dictionaries (False)
+    :return: a DataFrame
+
+    ::
+
+        import pstats
+        from pyquickhelper.pycode.profiling import profile2df
+
+        ps = pstats.Stats('c:/temp/bench_ortmodule_nn_gpu6')
+        print(ps.strip_dirs().sort_stats(SortKey.TIME).print_stats())
+        df = profile2df(pd)
+        print(df)
+    """
+    rows = _process_pstats(ps, lambda x: x)
+    if not as_df:
+        return rows
+
+    import pandas
+    df = pandas.DataFrame(rows)
+    df = df[['fct', 'file', 'ncalls1', 'ncalls2', 'tin', 'cum_tin',
+             'tall', 'cum_tall']]
+    df = df.groupby(['fct', 'file'], as_index=False).sum().sort_values(
+        'cum_tall', ascending=False).reset_index(drop=True)
+    return df.copy()
+
+
 def profile(fct, sort='cumulative', rootrem=None, as_df=False,
             pyinst_format=None, return_results=False, **kwargs):
     """
     Profiles the execution of a function.
 
-    @param      fct             function to profile
-    @param      sort            see `sort_stats <https://docs.python.org/3/library/
-                                profile.html#pstats.Stats.sort_stats>`_
-    @param      rootrem         root to remove in filenames
-    @param      as_df           return the results as a dataframe and not text
-    @param      pyinst_format   format for :epkg:`pyinstrument`, if not empty,
-                                the function uses this module or raises an exception if not
-                                installed, the options are *text*, *textu* (text with colors),
-                                *json*, *html*
-    @param      return_results  if True, return results as well
-                                (in the first position)
-    @param      kwargs          additional parameters used to create the profiler
-    @return                     raw results, statistics text dump (or dataframe is *as_df* is True)
+    :param fct: function to profile
+    :param sort: see `sort_stats <https://docs.python.org/3/library/
+        profile.html#pstats.Stats.sort_stats>`_
+    :param rootrem: root to remove in filenames
+    :param as_df: return the results as a dataframe and not text
+    :param pyinst_format: format for :epkg:`pyinstrument`, if not empty,
+        the function uses this module or raises an exception if not
+        installed, the options are *text*, *textu* (text with colors),
+        *json*, *html*
+    :param return_results: if True, return results as well
+        (in the first position)
+    :param kwargs: additional parameters used to create the profiler
+    :return: raw results, statistics text dump (or dataframe is *as_df* is True)
 
     .. plot::
 

src/pyquickhelper/sphinxext/sphinx_md_builder.py

@@ -979,7 +979,7 @@ def depart_todo_node(self, node):
     def unknown_visit(self, node):
         logger = logging.getLogger("MdBuilder")
         logger.warning("[md] unknown visit node: '{0}' - '{1}'".format(
-                node.__class__.__name__, node))
+            node.__class__.__name__, node))
 
 
 class MdBuilder(Builder):

src/pyquickhelper/sphinxext/sphinx_rst_builder.py

@@ -1099,7 +1099,7 @@ def unknown_visit(self, node):
             return
         logger = logging.getLogger("RstBuilder")
         logger.warning("[rst] unknown visit node: '{0}' - '{1}'".format(
-                node.__class__.__name__, node))
+            node.__class__.__name__, node))
 
     def unknown_departure(self, node):
         classname = node.__class__.__name__
@@ -1112,7 +1112,7 @@ def unknown_departure(self, node):
             return
         logger = logging.getLogger("RstBuilder")
         logger.warning("[rst] unknown depart node: '{0}' - '{1}'".format(
-                node.__class__.__name__, node))
+            node.__class__.__name__, node))
 
 
 class _BodyPlaceholder: