refactor the super deep for loop, and rename get_img_filenames to mor…

…e generic get_base_filenames

mdig/mdig/actions.py

@@ -523,9 +523,8 @@ def do_me(self,mdig_model):
                                 (i.get_index(), r.r_index))
                         maps = r.get_saved_maps(ls)
                         stats = g.get_univariate_stats(maps)
-                        fn = os.path.split(r.get_img_filenames(ls=ls, extension=False,
-                                gif=True))[:-5]
-                        fn = os.path.join(fn[0],self.options.analysis_filename_base + fn[1])
+                        fn = os.path.split(r.get_base_filenames(ls, single_file=True))
+                        fn = os.path.join(fn[0], self.options.analysis_filename_base + fn[1])
                         self.write_stats_to_file(stats,fn)
             else:
                 # just run on last map
@@ -887,10 +886,9 @@ def do_rep(self,i,r):
         output_images = self.options.output_gif or self.options.output_image 
         output_maps = self.options.output_map_pack
         if output_maps:
-            rep_filenames = r.get_img_filenames(ls, extension=False,
-                dir=self.options.outdir) 
+            rep_filenames = r.get_base_filenames(ls, output_dir=self.options.outdir) 
         elif output_images:
-            rep_filenames = r.get_img_filenames(ls, dir=self.options.outdir) 
+            rep_filenames = r.get_base_filenames(ls, extension='.png', output_dir=self.options.outdir) 
         for t in times:
             m = saved_maps[t]
             if output_images:
@@ -900,9 +898,11 @@ def do_rep(self,i,r):
                 map_list.append(self.export_map(m, rep_filenames[t]))
                 self.update_listeners_map_pack(None, r, ls, t)
         if self.options.output_gif:
-            self.create_gif(map_list, r.get_img_filenames(ls, gif=True, dir=self.options.outdir))
+            self.create_gif(map_list, r.get_base_filenames(ls, extension='_anim.gif',
+                single_file=True, output_dir=self.options.outdir))
         elif output_maps:
-            zip_fn = r.get_img_filenames(ls, extension=False, gif=True, dir=self.options.outdir)[:-5]
+            zip_fn = r.get_base_filenames(ls, extension='.zip', single_file=True,
+                    output_dir=self.options.outdir)
             self.zip_maps(map_list, zip_fn)
         return map_list
 

mdig/mdig/replicate.py

@@ -200,22 +200,22 @@ def get_map_name_base(self):
         fn = i.get_map_name_base()
         fn += "_rep_" + str(i.replicates.index(self))
         return fn
-
-    def get_img_filenames(self, ls="all", extension=True, gif=False, dir=None):
-        """ Get a dict of time:image_filename pairs for outputting maps to.
 
-        If gif is true, then returns a single string.
-        Warning: doesn't check that ls is an actual lifestage.
+    def get_base_filenames(self, ls="all", extension='', single_file=False, output_dir=None):
+        """ Get a dict of time:base_filename pairs for outputting things to.
+
+        If single_file is true, then returns a single string.
+        Warning: This doesn't check that ls is an actual lifestage.
         """
-        if dir is None:
+        if output_dir is None:
             output_dir = os.path.join(self.instance.experiment.base_dir, "output")
         else:
-            output_dir = os.path.normpath(dir)
+            output_dir = os.path.normpath(output_dir)
         fn = outputformats.create_filename(self)
-        if gif:
-            result = os.path.join(output_dir, fn + "_ls_" + ls + "_anim")
+        if single_file:
+            result = os.path.join(output_dir, fn + "_ls_" + ls)
             if extension:
-                result += '.gif'
+                result += extension
         else: 
             result = {}
             env = self.get_saved_maps(ls)
@@ -227,7 +227,7 @@ def get_img_filenames(self, ls="all", extension=True, gif=False, dir=None):
             for t in times:
                 result[t] = os.path.join(output_dir, fn + "_ls_" + ls + "_" + str(t))
                 if extension:
-                    result[t] += '.png'
+                    result[t] += extension
         return result
 
     def set_seed(self,s):
@@ -418,27 +418,50 @@ def run(self, remove_null=False):
 
     def save_metrics(self):
         """
-        Looks like:
-        {'all': {'events': {},
-         'treatments': {0: {'AREA_EVALUATED': {'1988-1': '7',
-                                               '1989-1': '11',
-                                               '1990-1': '13',
-                                               '1991-1': '11',
-                                               '1992-1': '13'}}}}}
+        The metrics objects looks like:
+
+        {
+         'all': {
+         'events': {},
+         'treatments': {
+            0: {'AREA_EVALUATED': {'1988-1': '7',
+                                   '1989-1': '11',
+                                   '1990-1': '13',
+                                   '1991-1': '11',
+                                   '1992-1': '13'}}}}}
+
+        We implement this by flattening the nested dictionary to lists of keys
+        leading to leaf dictionaries (those with the time series). This flattening
+        is preformed by _flatten_and_map. We then run _save_data on each of these
+        leaves.
         """
-        for ls, t_and_e in self.metrics.iteritems():
-            for event_type, metrics in t_and_e.iteritems():
-                for event_idx, metric in metrics.iteritems():
-                    for metric_name, time_series in metric.iteritems():
-                        base_name = self.get_img_filenames(ls, extension=False, gif=True)
-                        # TODO: add event's command name
-                        # TODO: grep for get_img_filename and make it more generic
-                        base_name += '%s_%d_%s.dat' % (event_type, event_idx, metric_name)
-                        f = open(base_name, 'w')
-                        f.write('time, interval, value\n')
-                        for t, val in sorted(time_series.iteritems(), key=itemgetter(0)):
-                            f.write('%s, %s, %s\n' % (str(t[0]), str(t[1]), val))
-                        f.close()
+        def _flatten_and_map(the_dict, function, arg_acc=None, depth=4):
+            if arg_acc is None:
+                arg_acc = []
+            if depth == 0:
+                arg_acc.append(the_dict)
+                function(*arg_acc)
+                return
+            for key, val in the_dict.iteritems():
+                arg_acc.append(key)
+                _flatten_and_map(val, function, arg_acc, depth=depth-1)
+
+        def _save_data(ls, event_type, event, metric_name, time_series):
+            base_name = self.get_base_filenames(ls, single_file=True)
+            event_idx, event_cmd = event
+            cmd = event_cmd.replace('.', '_')
+            base_name += '_%s_%d_%s_%s.dat' % (
+                    event_type, event_idx, cmd, metric_name)
+            f = open(base_name, 'w')
+            f.write('time, interval, value\n')
+            for t, val in sorted(time_series.iteritems(), key=itemgetter(0)):
+                if int(val) > 25000000:
+                    print 'metric is very large = ', val
+                    import pdb; pdb.set_trace()
+                f.write('%s, %s, %s\n' % (str(t[0]), str(t[1]), val))
+            f.close()
+
+        _flatten_and_map(self.metrics, _save_data)
 
     def add_event_metrics(self, ls, event, metrics, interval, treatment=None):
         if not metrics:
@@ -451,11 +474,14 @@ def add_event_metrics(self, ls, event, metrics, interval, treatment=None):
         self.metrics[ls.name].setdefault('events', dict())
 
         if treatment:
-            self.metrics[ls.name]['treatments'].setdefault(treatment.index, dict())
-            store_in = self.metrics[ls.name]['treatments'][treatment.index]
+            event_cmd = treatment.get_event().get_command()
+            ev_key = (treatment.index, event_cmd)
+            self.metrics[ls.name]['treatments'].setdefault(ev_key, dict())
+            store_in = self.metrics[ls.name]['treatments'][ev_key]
         else:
-            self.metrics[ls.name]['events'].setdefault(ls.events.index(event), dict())
-            store_in = self.metrics[ls.name]['events'][ls.events.index(event)]
+            ev_key = (ls.events.index(event), event.get_command())
+            self.metrics[ls.name]['events'].setdefault(ev_key, dict())
+            store_in = self.metrics[ls.name]['events'][ev_key]
 
         for metric, val in metrics.iteritems():
             store_in.setdefault(metric, dict())

mdig/mdig/tests/test_actions.py

@@ -87,7 +87,7 @@ def test_do_instance_images(self,m_rm,m_g):
         # create mock replicate
         #instances[0].replicates = [Mock()]
         #instances[0].replicates[0].get_saved_maps.return_value = {'1':'xx','2':'yy'}
-        #instances[0].replicates[0].get_img_filenames.return_value = {'1':'xx','2':'yy'}
+        #instances[0].replicates[0].get_base_filenames.return_value = {'1':'xx','2':'yy'}
         # ea.do_instance_images(instances[0])
 	m.remove_log_handler()
 

mdig/mdig/tests/test_replicate.py

@@ -102,14 +102,14 @@ def test_map_name_base(self):
         name = i.replicates[0].get_map_name_base()
         self.assertEqual('variables_complete_region_a_i1_rep_0', name)
 
-    def test_get_img_filenames(self):
+    def test_get_base_filenames(self):
         i = self.m_variables_complete.get_instances()[0]
         # return a dictionary of time:names
-        img_fns = i.replicates[0].get_img_filenames()
+        img_fns = i.replicates[0].get_base_filenames()
         self.assertEqual(len(img_fns),6)
         # return a single filename
-        img_fn = i.replicates[0].get_img_filenames(gif = True)
-        self.assertTrue('variables_complete_region_a_i0_rep_0_ls_all_anim.gif' in img_fn)
+        img_fn = i.replicates[0].get_base_filenames(single_file=True)
+        self.assertTrue('variables_complete_region_a_i0_rep_0_ls_all' in img_fn)
 
     def test_get_initial_maps(self):
         i = self.m_variables_complete.get_instances()[0]

mdig/mdig/webui.py

@@ -434,13 +434,12 @@ def show_replicate(model, instance, replicate):
         abort(404, "No such replicate, or replicate doesn't exist yet")
     rep = instance.replicates[rep_num]
     error = None
-    m_name = dm.get_name()
 
     # Scan output dir to see if gifs have been generated for this instance
     gifs_present = []
     for ls_id in instance.experiment.get_lifestage_ids():
         # if there is an envelope generated then collect it
-        fn = rep.get_img_filenames(ls=ls_id, gif=True)
+        fn = rep.get_base_filenames(ls=ls_id, single_file=True, extension='_anim.gif')
         if os.path.isfile(fn):
             # get creation time
             mtime = datetime.datetime.fromtimestamp(os.stat(fn).st_mtime)
@@ -451,8 +450,7 @@ def show_replicate(model, instance, replicate):
     # Scan output dir to see if map_packs have been generated for this instance
     map_packs_present = []
     for ls_id in instance.experiment.get_lifestage_ids():
-        fn = rep.get_img_filenames(
-            ls=ls_id, extension=False, gif=True)[:-5] + '.zip'
+        fn = rep.get_base_filenames(ls=ls_id, extension='.zip', single_file=True)
         if os.path.isfile(fn):
             # get creation time
             mtime = datetime.datetime.fromtimestamp(os.stat(fn).st_mtime)
@@ -654,7 +652,7 @@ def replicate_spread_gif(model, instance, replicate, ls_id):
         abort(404, "No such replicate, or replicate doesn't exist yet")
     r = instance.replicates[replicate]
     # do per lifestage!
-    fn = r.get_img_filenames(ls=ls_id, gif=True)
+    fn = r.get_base_filenames(ls=ls_id, extension='_anim.gif', single_file=True)
     root_dir = os.path.dirname(fn)
     send_file(os.path.basename(fn), root=root_dir)
 
@@ -676,8 +674,7 @@ def create_replicate_map_pack(model, instance, replicate, ls_id):
     submit_replicate_job(dm, idx, replicate, ls_id, action)
     purge_oldest_map_packs()
     r = instance.replicates[replicate]
-    fn = r.get_img_filenames(ls=ls_id, extension=False, gif=True)
-    fn += '.zip'
+    fn = r.get_base_filenames(ls=ls_id, extension='_anim.zip', single_file=True)
     add_to_map_pack_lfu(fn, nodate=True)
     redirect('/models/%s/instances/%d/replicates/%d' %
             (model.get_name(), idx, replicate))
@@ -696,8 +693,7 @@ def replicate_map_pack(model, instance, replicate, ls_id):
         abort(404, "No such replicate, or replicate doesn't exist yet")
     r = instance.replicates[replicate]
     # do per lifestage!
-    fn = r.get_img_filenames(ls=ls_id, extension=False, gif=True)
-    fn += '.zip'
+    fn = r.get_base_filenames(ls=ls_id, extension='_anim.zip', single_file=True)
     if os.path.isfile(fn):
         add_to_map_pack_lfu(fn)
         root_dir = os.path.dirname(fn)