Refactored storage service, the environment now takes more arguments

CHANGES.txt

@@ -5,6 +5,10 @@ pypet 0.1b5
 *   BUG FIX: When parameters are emptied, the default value is set to None (and no longer kept)
 
 *   Now items are only saved once, if the node already exist on disk, storage is refused
+    (Previously new data was added if it was not within the leaf before, but this can lead to
+    strange inconsistencies).
+
+
 
 pypet 0.1b.4
 

doc/source/cookbook/environment.rst

@@ -38,10 +38,20 @@ You start your simulations by creating an environment object:
                  add_time=True,
                  comment='',
                  dynamically_imported_classes=None,
+                 fast_access=True,
+                 shortcuts=True,
+                 backwards_search=True,
+                 iter_recursive=True,
+                 store_before_runs=True,
                  log_folder=None,
-                 log_level=logging.DEBUG,
+                 log_level=logging.INFO,
+                 log_stdout=True,
                  multiproc=False,
                  ncores=1,
+                 use_pool=False,
+                 cpu_cap=1.0,
+                 memory_cap=1.0,
+                 swap_cap=1.0,
                  wrap_mode=pypetconstants.WRAP_MODE_LOCK,
                  continuable=1,
                  use_hdf5=True,
@@ -51,18 +61,21 @@ You start your simulations by creating an environment object:
                  complib='zlib',
                  shuffle=True,
                  fletcher32=False,
-                 pandas_format='table',
-                 pandas_append=True,
+                 pandas_format='fixed',
+                 pandas_append=False,
                  purge_duplicate_comments=True,
-                 summary_tables=True,
+                 summary_tables = True,
                  small_overview_tables=True,
-                 large_overview_tables=True,
+                 large_overview_tables=False,
                  results_per_run=0,
                  derived_parameters_per_run=0,
                  git_repository = None,
-                 git_message=''):
+                 git_message='',
+                 do_single_runs=True,
+                 lazy_debug=False)::
 
-You can pass the following arguments:
+You can pass the following arguments. Note usually you only have to change very few of these
+because most of the time the default settings are sufficient.
 
 * `trajectory`
 
@@ -82,6 +95,8 @@ You can pass the following arguments:
 
 * `dynamically_imported_classes`
 
+    Only considered if a new trajectory is created.
+
     The argument `dynamically_imported_classes` is important
     if you have written your own *parameter* or *result* classes, you can pass these either
     as class variables `MyCustomParameterClass` or as strings leading to the classes in your package:
@@ -94,6 +109,39 @@ You can pass the following arguments:
     two classes named `'MyCustomParameterClass'` in two different python modules!
     The identification of the class is based only on its name and not its path in your packages.
 
+* `fast_access`
+
+    Only considered if a new trajectory is created.
+    If *fast access* should be enabled for the new trajectory.
+    Will simply call `traj.v_fast_access = fast_access` on the new trajectory.
+    Can be changed later on during runtime.
+
+* `shortcuts`
+
+    Only considered if a new trajectory is created.
+    If *shortcuts* should be enabled for the new trajectory.
+    Will simply call `traj.v_shortcuts = shortcuts` on the new trajectory.
+    Can be changed later on during runtime.
+
+* `backwards_search`
+
+    Only considered if a new trajectory is created.
+    If *backwards search* should be enabled for the new trajectory.
+    Will simply call `traj.v_backwards_search = backwards_search` on the new trajectory.
+    Can be changed later on during runtime.
+
+* `iter_recursive`
+
+    Only considered if a new trajectory is created.
+    If recursive iterations should be enabled for the new trajectory.
+    Will simply call `traj.v_iter_recursive = iter_recursive` on the new trajectory.
+    Can be changed later on during runtime.
+
+* `store_before_runs`
+
+    I the whole trajectory should be stored before the runs are started. Otherwise
+    the storage will be only initialised.
+
 * `log_folder`
 
     The `log_folder` specifies where all log files will be stored.

pypet/environment.py

@@ -198,6 +198,7 @@ class Environment(object):
 
     :param dynamically_imported_classes:
 
+          Only considered if a new trajectory is created.
           If you've written custom parameters or results that need to be loaded
           dynamically during runtime, the module containing the class
           needs to be specified here as a list of classes or strings
@@ -211,6 +212,39 @@ class Environment(object):
           the list brackets:
           `dynamically_imported_classes = 'pypet.parameter.PickleParameter'`
 
+    :param fast_access:
+
+        Only considered if a new trajectory is created.
+        If *fast access* should be enabled for the new trajectory.
+        Will simply call `traj.v_fast_access = fast_access` on the new trajectory.
+        Can be changed later on during runtime.
+
+    :param shortcuts:
+
+        Only considered if a new trajectory is created.
+        If *shortcuts* should be enabled for the new trajectory.
+        Will simply call `traj.v_shortcuts = shortcuts` on the new trajectory.
+        Can be changed later on during runtime.
+
+    :param backwards_search:
+
+        Only considered if a new trajectory is created.
+        If *backwards search* should be enabled for the new trajectory.
+        Will simply call `traj.v_backwards_search = backwards_search` on the new trajectory.
+        Can be changed later on during runtime.
+
+    :param iter_recursive:
+
+        Only considered if a new trajectory is created.
+        If recursive iterations should be enabled for the new trajectory.
+        Will simply call `traj.v_iter_recursive = iter_recursive` on the new trajectory.
+        Can be changed later on during runtime.
+
+    :param store_before_runs:
+
+        I the whole trajectory should be stored before the runs are started. Otherwise
+        the storage will be only initialised.
+
     :param log_folder:
 
         Path to a folder where all log files will be stored. If none is specified the default
@@ -557,6 +591,11 @@ def __init__(self, trajectory='trajectory',
                  add_time=True,
                  comment='',
                  dynamically_imported_classes=None,
+                 fast_access=True,
+                 shortcuts=True,
+                 backwards_search=True,
+                 iter_recursive=True,
+                 store_before_runs=True,
                  log_folder=None,
                  log_level=logging.INFO,
                  log_stdout=True,
@@ -599,7 +638,7 @@ def __init__(self, trajectory='trajectory',
         self._git_message=git_message
 
         # Check if a novel trajectory needs to be created.
-        if isinstance(trajectory,str):
+        if isinstance(trajectory, basestring):
             # Create a new trajectory
             self._traj = Trajectory(trajectory,
                                     add_time=add_time,
@@ -609,6 +648,11 @@ def __init__(self, trajectory='trajectory',
             self._timestamp = self.v_trajectory.v_timestamp # Timestamp of creation
             self._time = self.v_trajectory.v_time # Formatted timestamp
 
+            self._traj.v_fast_access = fast_access
+            self._traj.v_backwards_search = backwards_search
+            self._traj.v_iter_recursive = iter_recursive
+            self._traj.v_shortcuts = shortcuts
+
         else:
 
             self._traj = trajectory
@@ -722,13 +766,15 @@ def __init__(self, trajectory='trajectory',
                                   str(self._hexsha))
 
         self._do_single_runs = do_single_runs
+        self._store_before_runs = store_before_runs
 
         if self._do_single_runs:
             config_name='environment.%s.multiproc' % self.v_name
             self._traj.f_add_config(config_name, multiproc,
                                     comment= 'Whether or not to use multiprocessing. If yes'
                                              ' than everything must be pickable.')
 
+
             if self._traj.f_get('config.environment.%s.multiproc' % self.v_name).f_get():
                 config_name='environment.%s.use_pool' % self.v_name
                 self._traj.f_add_config(config_name, use_pool,
@@ -1119,15 +1165,23 @@ def f_run(self, runfunc, *args,**kwargs):
 
         # Add the amount to be run to the trajectory
         config_name='environment.%s.normal_runs' % self.v_name
-        if not config_name in self._traj:
+        if not self._traj.f_contains('config.'+config_name, shortcuts=False):
             self._traj.f_add_config(config_name, count,
                                     comment ='Added if trajectory was explored normally and not '
                                              'continued.')
 
+        config_name='evironment.%s.store_before_runs' % self.v_name
+        if not self._traj.f_contains('config.'+config_name, shortcuts=False):
+            self._traj.f_add_config(config_name, self._store_before_runs,
+                    comment='If the trajectory was automatically be saved before the '
+                            'single runs start, otherwise the store is only initialised. '
+                            'Only added if runs are started, not continued.')
+
         # Make some preparations (locking of parameters etc) and store the trajectory
         self._logger.info('I am preparing the Trajectory for the experiment and store it.')
         self._traj._prepare_experiment()
-        self._traj.f_store()
+
+        self._traj.f_store(only_init= (not self._store_before_runs))
         self._logger.info('Trajectory successfully stored.')
 
         # Make the trajectory continuable in case the user wants that

pypet/storageservice.py

@@ -626,6 +626,7 @@ def load(self,msg,stuff_to_load,*args,**kwargs):
         :raises: NoSuchServiceError if message or data is not understood
 
         """
+        opened = True
         try:
 
             self._srvc_extract_file_information(kwargs)
@@ -651,7 +652,7 @@ def load(self,msg,stuff_to_load,*args,**kwargs):
 
             self._srvc_closing_routine(opened)
         except:
-            self._srvc_closing_routine(True)
+            self._srvc_closing_routine(opened)
             self._logger.error('Failed loading  `%s`' % str(stuff_to_load))
             raise
 
@@ -878,6 +879,7 @@ def store(self,msg,stuff_to_store,*args,**kwargs):
         :raises: NoSuchServiceError if message or data is not understood
 
         """
+        opened = True
         try:
 
             self._srvc_extract_file_information(kwargs)
@@ -927,7 +929,7 @@ def store(self,msg,stuff_to_store,*args,**kwargs):
             self._srvc_closing_routine(opened)
 
         except:
-            self._srvc_closing_routine(True)
+            self._srvc_closing_routine(opened)
             self._logger.error('Failed storing `%s`' % str(stuff_to_store))
             raise
 
@@ -1523,6 +1525,9 @@ def _trj_prepare_merge(self, traj, changed_parameters):
 
         """
 
+        if not traj._stored:
+            traj.f_store()
+
         # Update meta information
         infotable = getattr(self._overview_group,'info')
         insert_dict = self._all_extract_insert_dict(traj,infotable.colnames)
@@ -1534,9 +1539,13 @@ def _trj_prepare_merge(self, traj, changed_parameters):
         for param_name in changed_parameters:
             param = traj.f_get(param_name)
             # First we delete the parameter
-            self.store(pypetconstants.DELETE, param)
+            try:
+                self.store(pypetconstants.DELETE, param)
+            except pt.NoSuchNodeError:
+                pass # We can end up here if the parameter was never stored
+
             # And then we add it again
-            self.store(pypetconstants.LEAF,param)
+            self.store(pypetconstants.LEAF, param)
 
         # Increase the run table by the number of new runs
         run_table = getattr(self._overview_group,'runs')
@@ -4016,6 +4025,15 @@ def _all_delete_parameter_or_result_or_group(self, instance,
 
         if delete_only is None:
 
+            try:
+                the_node = self._hdf5file.get_node(where=where, name=node_name)
+            except AttributeError:
+                the_node = self._hdf5file.getNode(where=where, name=node_name)
+
+            if not instance.v_is_leaf:
+                if len(the_node._v_groups) != 0:
+                    raise TypeError('You cannot remove a group that is not empty!')
+
             if instance.v_is_leaf:
                 # If we delete a leaf we need to take care about overview tables
                 base_group = split_name[0]
@@ -4039,15 +4057,6 @@ def _all_delete_parameter_or_result_or_group(self, instance,
 
                 self._prm_meta_remove_summary(instance)
 
-            try:
-                the_node = self._hdf5file.get_node(where=where, name=node_name)
-            except AttributeError:
-                the_node = self._hdf5file.getNode(where=where, name=node_name)
-
-            if not instance.v_is_leaf:
-                if len(the_node._v_groups) != 0:
-                    raise TypeError('You cannot remove a group that is not empty!')
-
             the_node._f_remove(recursive=True)
 
             if remove_empty_groups:

pypet/trajectory.py

@@ -608,7 +608,7 @@ def f_get_results(self, fast_access=False, copy=True):
         """
         return self._return_item_dictionary(self._results, fast_access, copy)
 
-    def f_store(self, *args, **kwargs):
+    def f_store(self):
         """Stores the single run to disk."""
         self._storage_service.store(pypetconstants.SINGLE_RUN, self,
                                     trajectory_name=self.v_trajectory_name)
@@ -2169,8 +2169,13 @@ def _check_if_both_have_same_parameters(self, other_trajectory,
         # If not ignored, add also the trajectory derived parameters to check for merging
         if not ignore_trajectory_derived_parameters and 'derived_parameters' in self:
             my_traj_dpars = self._get_traj_dpars_or_results(self, 'derived_parameters')
+            if self._stored:
+                self.f_load_items(my_traj_dpars.itervalues(), only_empties=True)
             allmyparams.update(my_traj_dpars)
             other_traj_dpars = self._get_traj_dpars_or_results(other_trajectory, 'derived_parameters')
+            if other_trajectory._stored:
+                other_trajectory.f_load_items(other_traj_dpars.values(),
+                                          only_empties=True)
             allotherparams.update(other_traj_dpars)