Add update functions to jobs and esd

Add functions to process the data,
to find and store the coincidences

django_publicdb/histograms/esd.py

@@ -109,6 +109,24 @@ def _create_progressbar_from_iterable(self, iterable, length=None):
         return lambda x: x
 
 
+def process_coincidences_and_store_in_esd(summary):
+    """Determine coincidences for events from Event Summary Data
+
+    Events from all non-test stations in the ESD are processed
+    for coincidences, the results of which are stored in the
+    coincidences group.
+
+    :param summary: summary of data source (station and date)
+    :type summary: histograms.models.Summary instance
+
+    """
+    date = summary.date
+
+    filepath = get_esd_data_path(date)
+    with tables.openFile(filepath, 'r') as data:
+        pass
+
+
 def process_events_and_store_temporary_esd(summary):
     """Process events from datastore and save temporary Event Summary Data
 

django_publicdb/histograms/jobs.py

@@ -162,6 +162,7 @@ def update_all_histograms():
 
 def perform_update_tasks():
     update_esd()
+    update_coincidences()
     update_histograms()
 
 
@@ -182,8 +183,26 @@ def update_esd():
     worker_pool.join()
 
 
+def update_coincidences():
+    worker_pool = multiprocessing.Pool()
+    summaries = NetworkSummary.objects.filter(needs_update=True).reverse()
+    results = worker_pool.imap_unordered(
+        process_and_store_coincidences_for_network_summary, summaries)
+
+    for summary in results:
+        summary.needs_update = False
+        summary.save()
+
+    worker_pool.close()
+    worker_pool.join()
+
+
 def process_and_store_temporary_esd_for_summary(summary):
     django.db.close_connection()
+        for tmpfile_path, node_path in tmp_locations:
+            esd.copy_temporary_esd_node_to_esd(summary, tmpfile_path,
+                                               node_path)
+
     tmp_locations = []
     if summary.needs_update_events:
         tmp_locations.append(process_events_and_store_esd(summary))
@@ -192,6 +211,13 @@ def process_and_store_temporary_esd_for_summary(summary):
     return summary, tmp_locations
 
 
+def process_and_store_coincidences_for_network_summary(summary):
+    django.db.close_connection()
+    if summary.needs_update_coincidences:
+        process_coincidences_and_store_esd(summary)
+    return summary
+
+
 def update_histograms():
     """Update all histograms"""
 
@@ -315,6 +341,14 @@ def update_pulseintegral_histogram(summary):
     save_histograms(summary, 'pulseintegral', bins, histograms)
 
 
+def process_coincidences_and_store_esd(summary):
+    logger.debug("Processing coincidences and storing ESD for %s", summary)
+    t0 = time.time()
+    esd.process_coincidences_and_store_in_esd(summary)
+    t1 = time.time()
+    logger.debug("Processing took %.1f s.", t1 - t0)
+
+
 def process_events_and_store_esd(summary):
     logger.debug("Processing events and storing ESD for %s", summary)
     t0 = time.time()