ENH: More comprehensive error statements;

examples/example_01_first_steps.py

@@ -23,7 +23,6 @@ def multiply(traj):
 filename = os.path.join('hdf5','example_01.hdf5')
 env = Environment(trajectory='Multiplication',
                   filename=filename,
-                  overwrite_file=True,
                   file_title='Example_01_First_Steps',
                   comment='The first example!',
                   large_overview_tables=True,  # To see a nice overview of all

pypet/backends/mongodb.py

@@ -904,7 +904,7 @@ def _srvc_flush_tree_db(self):
             try:
                 self._retry_write()
             except:
-                self._logger.error('Bulk write error with bul `%s`' % str(self._bulk))
+                self._logger.error('Bulk write error with bulk `%s`' % str(self._bulk))
                 raise
             self._bulk = []
 

pypet/environment.py

@@ -1899,7 +1899,8 @@ def _finish_sumatra(self):
 
         finish_time = self._start_timestamp - self._finish_timestamp
         self._sumatra_record.duration = finish_time
-        self._sumatra_record.output_data = self._sumatra_record.datastore.find_new_data(self._sumatra_record.timestamp)
+        self._sumatra_record.output_data = self._sumatra_record.datastore.find_new_data(
+                                                                self._sumatra_record.timestamp)
         self._loaded_sumatatra_project.add_record(self._sumatra_record)
         self._loaded_sumatatra_project.save()
         sumatra_label = self._sumatra_record.label

pypet/naturalnaming.py

@@ -2442,21 +2442,45 @@ class NNGroupNode(NNTreeNode, KnowsTrajectory):
 
     """
 
-    __slots__ = ('_children', '_links', '_groups', '_leaves',
+    __slots__ = ('_children_', '_links_', '_groups_', '_leaves_',
                  '_nn_interface', '_kids')
 
     def __init__(self, full_name='', trajectory=None, comment=''):
         super(NNGroupNode, self).__init__(full_name, comment=comment, is_leaf=False)
-        self._children = {}
-        self._links = {}
-        self._groups = {}
-        self._leaves = {}
+        self._children_ = None
+        self._links_ = None
+        self._groups_ = None
+        self._leaves_ = None
         self._kids = None
         if trajectory is not None:
             self._nn_interface = trajectory._nn_interface
         else:
             self._nn_interface = None
 
+    @property
+    def _children(self):
+        if self._children_ is None:
+            self._children_ = {}
+        return self._children_
+
+    @property
+    def _links(self):
+        if self._links_ is None:
+            self._links_ = {}
+        return self._links_
+
+    @property
+    def _groups(self):
+        if self._groups_ is None:
+            self._groups_ = {}
+        return self._groups_
+
+    @property
+    def _leaves(self):
+        if self._leaves_ is None:
+            self._leaves_ = {}
+        return self._leaves_
+
     @property
     def kids(self):
         """Alternative naming, you can use `node.kids.name` instead of `node.name`

pypet/storageservice.py

@@ -2471,7 +2471,11 @@ def _trj_store_trajectory(self, traj, only_init=False, store_data=pypetconstants
         # We do not want to mess up the stored trajectory but raise an Error
         if not traj._stored and self._trajectory_group is not None:
             raise RuntimeError('You want to store a completely new trajectory with name'
-                               ' `%s` but this trajectory is already found in file `%s`' %
+                               ' `%s` but this trajectory is already found in file `%s`.'
+                               'Did you try to accidentally overwrite existing data? If '
+                               'you DO want to override existing data, use `overwrite_file=True`.'
+                               'Note that this deletes the whole HDF5 file not just the particular '
+                               'trajectroy therein! ' %
                                (traj.v_name, self._filename))
 
         # Extract HDF5 settings from the trajectory

pypet/tests/profiling/profiling_loading_many_nodes.py

@@ -0,0 +1,118 @@
+__author__ = 'Robert Meyer'
+
+import logging
+import os
+
+import numpy as np
+import scipy.sparse as spsp
+from pycallgraph import PyCallGraph, Config, GlobbingFilter
+from pycallgraph.output import GraphvizOutput
+from pycallgraph.color import Color
+
+
+class CustomOutput(GraphvizOutput):
+    def node_color(self, node):
+        value = float(node.time.fraction)
+        return Color.hsv(value / 2 + .5, value, 0.9)
+
+    def edge_color(self, edge):
+        value = float(edge.time.fraction)
+        return Color.hsv(value / 2 + .5, value, 0.7)
+
+
+from pypet import Environment, Parameter, load_trajectory, cartesian_product
+
+from pypet.tests.testutils.ioutils import make_temp_dir
+from pypet.tests.testutils.data import create_param_dict, add_params, simple_calculations
+
+filename = None
+
+
+def explore(traj):
+    explored ={'Normal.trial': range(300),
+        'Numpy.double': [np.array([1.0,2.0,3.0,4.0]), np.array([-1.0,3.0,5.0,7.0])],
+        'csr_mat' :[spsp.csr_matrix((2222,22)), spsp.csr_matrix((2222,22))]}
+
+    explored['csr_mat'][0][1,2]=44.0
+    explored['csr_mat'][1][2,2]=33
+
+    traj.f_explore(cartesian_product(explored))
+
+
+def test_run():
+
+    global filename
+
+
+    np.random.seed()
+    trajname = 'profiling_many_nodes'
+    filename = make_temp_dir(os.path.join('hdf5', 'test%s.hdf5' % trajname))
+
+    env = Environment(trajectory=trajname, filename=filename,
+                      file_title=trajname,
+                      log_stdout=False,
+                      results_per_run=5,
+                      derived_parameters_per_run=5,
+                      multiproc=True,
+                      ncores=2,
+                      wrap_mode='LOCK',
+                      use_pool=False,
+                      overwrite_file=True)
+
+    traj = env.v_trajectory
+
+    traj.v_standard_parameter=Parameter
+
+    ## Create some parameters
+    param_dict={}
+    create_param_dict(param_dict)
+    ### Add some parameter:
+    add_params(traj,param_dict)
+
+    #remember the trajectory and the environment
+    traj = traj
+    env = env
+
+    traj.f_add_parameter('TEST', 'test_run')
+    ###Explore
+    explore(traj)
+
+    ### Make a test run
+    simple_arg = -13
+    simple_kwarg= 13.0
+    env.f_run(simple_calculations,simple_arg,simple_kwarg=simple_kwarg)
+
+    size=os.path.getsize(filename)
+    size_in_mb = size/1000000.
+    print('Size is %sMB' % str(size_in_mb))
+
+
+def test_load():
+    newtraj = load_trajectory(index=-1, filename=filename, load_data=1)
+
+
+if __name__ == '__main__':
+    if not os.path.isdir('./tmp'):
+        os.mkdir('tmp')
+    graphviz = CustomOutput()
+    graphviz.output_file = './tmp/run_profile_traj_slots.png'
+    service_filter = GlobbingFilter(include=['*storageservice.*', '*ptcompat.*',
+                                              '*naturalnaming.*', '*parameter.*',
+                                              '*trajectory.*'])
+    #service_filter = GlobbingFilter(include=['*naturalnaming.*', '*trajectory.*'])
+
+    config = Config(groups=True, verbose=True)
+    config.trace_filter = service_filter
+
+    print('RUN PROFILE')
+    #with PyCallGraph(config=config, output=graphviz):
+    test_run()
+    print('DONE RUN PROFILE')
+
+    graphviz = CustomOutput()
+    graphviz.output_file = './tmp/load_mode_1_profile_many_nodes.png'
+
+    print('LOAD PROFILE')
+    with PyCallGraph(config=config, output=graphviz):
+        test_load()
+    print('DONE LOAD PROFILE')

pypet/utils/helpful_functions.py

@@ -194,7 +194,10 @@ def __call__(self, index, total, percentage_step=5, logger='print', log_level=lo
                 statement = fmt_string % statement
             if logger == 'print':
                 if reprint and not ending:
-                    print(statement, end='\r')
+                    if compat.python_major == 3:
+                        print(statement, end='\r', flush=True)
+                    else:
+                        print(statement, end='\r')
                 else:
                     print(statement)
             elif logger is not None: