Improve publicdb.download_data

Add support for new tables.
Filter each table with timestamp column by timestamp.
Append multiple rows at once instead of one at a time.
Open temporary source data in read mode.

sapphire/publicdb.py

@@ -11,11 +11,13 @@
 import datetime
 import tables
 import os
-import calendar
 import re
 
 import logging
 
+from .transformations.clock import datetime_to_gps
+
+
 logger = logging.getLogger('hisparc.publicdb')
 
 # PUBLICDB_XMLRPC_URL = 'http://localhost:8000/raw_data/rpc'
@@ -80,11 +82,8 @@ def _store_data(dst_file, dst_group, src_filename, t0, t1):
     This function takes a file containing downloaded data and copies it to
     the destination file, based on start and end timestamps.
 
-    This can be further optimized at the expense of spaghetti code.
-
     """
-    # Open in rw mode, need to update blob idxs, if necessary
-    with tables.open_file(src_filename, 'a') as src_file:
+    with tables.open_file(src_filename, 'r') as src_file:
         src_group = src_file.list_nodes('/')[0]
         dst_group = _get_or_create_group(dst_file, dst_group)
 
@@ -100,36 +99,32 @@ def _store_data(dst_file, dst_group, src_filename, t0, t1):
                 for row in node:
                     dst_node.append(row)
 
-            elif node.name == 'events':
-                if not t1:
-                    cond = 'timestamp >= %d' % \
-                        calendar.timegm(t0.utctimetuple())
-                else:
-                    cond = '(%d <= timestamp) & (timestamp <= %d)' % \
-                        (calendar.timegm(t0.utctimetuple()),
-                         calendar.timegm(t1.utctimetuple()))
-
-                if len_blobs:
-                    for row in node.read_where(cond):
-                        row['traces'] += len_blobs
-                        dst_node.append([tuple(row)])
+            elif node.name in ['events', 'errors', 'config', 'comparator',
+                               'singles', 'satellites', 'weather',
+                               'weather_error', 'weather_config']:
+                if t1 is None:
+                    cond = 'timestamp >= %d' % datetime_to_gps(t0)
                 else:
-                    for row in node.read_where(cond):
-                        dst_node.append([tuple(row)])
+                    cond = ('(%d <= timestamp) & (timestamp <= %d)' %
+                            (datetime_to_gps(t0), datetime_to_gps(t1)))
 
-            elif node.name == 'errors' and len_blobs:
-                for row in node:
-                    row['messages'] += len_blobs
-                    dst_node.append([row[:]])
+                rows = node.read_where(cond)
 
-            elif node.name == 'config' and len_blobs:
-                for row in node:
-                    row['mas_version'] += len_blobs
-                    row['slv_version'] += len_blobs
-                    row['password'] += len_blobs
-                    row['buffer'] += len_blobs
-                    dst_node.append([row[:]])
+                if len_blobs:
+                    if node.name == 'events':
+                        rows['traces'] += len_blobs
+                    elif node.name in ['errors', 'weather_error']:
+                        rows['messages'] += len_blobs
+                    elif node.name == 'config':
+                        rows['mas_version'] += len_blobs
+                        rows['slv_version'] += len_blobs
+                        rows['password'] += len_blobs
+                        rows['buffer'] += len_blobs
+                    elif node.name == 'weather_config':
+                        rows['help_url'] += len_blobs
+                        rows['database_name'] += len_blobs
 
+                dst_node.append(rows)
             else:
                 rows = node.read()
                 dst_node.append(rows)

sapphire/tests/validate_results.py

@@ -13,8 +13,8 @@ def validate_results(test, expected_path, actual_path):
     :param actual_path: path to the output from the test.
 
     """
-    with tables.open_file(expected_path) as expected_file, \
-            tables.open_file(actual_path) as actual_file:
+    with tables.open_file(expected_path, 'r') as expected_file, \
+            tables.open_file(actual_path, 'r') as actual_file:
         for expected_node in expected_file.walk_nodes('/', 'Leaf'):
             try:
                 actual_node = actual_file.get_node(expected_node._v_pathname)
@@ -44,8 +44,8 @@ def validate_results_node(test, expected_path, actual_path, expected_node,
     :param actual_node: path to the output node from the test.
 
     """
-    with tables.open_file(expected_path) as expected_file, \
-            tables.open_file(actual_path) as actual_file:
+    with tables.open_file(expected_path, 'r') as expected_file, \
+            tables.open_file(actual_path, 'r') as actual_file:
         expected = expected_file.get_node(expected_node)
         try:
             actual = actual_file.get_node(actual_node)