improve checks for int_time column

tango-controls · Apr 25, 2019 · 84e11f4 · 84e11f4
1 parent 25cee7b
commit 84e11f4
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 12 deletions.
diff --git a/PyTangoArchiving/dbs.py b/PyTangoArchiving/dbs.py
@@ -321,8 +321,11 @@ def mysqldump_by_date(schema, user, passwd, folder, start, stop,
         filename = ('%s/%s-%s-%s-%s.dmp' 
             % (folder,schema,t,start.split()[0],stop.split()[0]))
         cols = db.getTableCols(t)
-        col = [c for c in ('time','data_time') if c in cols] 
-        if col:
+        col = [c for c in ('int_time','time','data_time') if c in cols] 
+        if col and col[0] == 'int_time':
+            where = " %s >= %s and %s < %s " % (
+                col[0],fn.str2time(start),col[0],fn.str2time(stop))
+        elif col:
             where = " %s >= '%s' and %s < '%s' " % (col[0],start,col[0],stop)
         else:
             where = ""

diff --git a/PyTangoArchiving/hdbpp/create_int_column.py b/PyTangoArchiving/hdbpp/create_int_column.py
@@ -4,24 +4,35 @@
 assert sys.argv[1:], 'Start date required (e.g. 2017-08-01)'
 
 tables = {
+### BUT, NOT ALL TABLES ARE IN THIS LIST!
+# I'm partitioning only the big ones, and ignoring the others
+# boolean, encoded, enum, long64 uchar ulong64, ulong, ushort
+# b, e, n, l64, ul6, ul, us, uc
 
     'att_array_devdouble_ro':'adr',
+    'att_array_devfloat_ro':'afr',
     'att_array_devlong_ro':'alr',
+    'att_array_devlong_rw':'alw',    
     'att_array_devshort_ro':'ahr',
+    'att_array_devboolean_ro':'abr',    
+    'att_array_devboolean_rw':'abw',        
     'att_array_devstring_ro':'asr',
     'att_array_devstate_ro':'atr',
 
     'att_scalar_devdouble_ro':'sdr',
     'att_scalar_devdouble_rw':'sdw',
-
     'att_scalar_devfloat_ro':'sfr',
     'att_scalar_devlong_ro':'slr',
     'att_scalar_devlong_rw':'slw',
+    'att_scalar_devshort_ro':'shr',
+    'att_scalar_devshort_rw':'shw',    
+    'att_scalar_devboolean_ro':'sbr',
+    'att_scalar_devboolean_rw':'sbw',
+
     'att_scalar_devstate_ro':'str',
     'att_scalar_devstring_ro':'ssr',
-    'att_scalar_devshort_ro':'shr',
-    'att_scalar_devshort_rw':'shw',
-
+    'att_scalar_devushort_ro':'sur',
+    'att_scalar_devuchar_ro':'scr',
     }
 
 start_date = sys.argv[1] # '2017-08-01'
@@ -43,15 +54,15 @@ def inc_months(date,count):
 newc = ("alter table %s add column int_time INT "
     "generated always as (TO_SECONDS(data_time)-62167222800) PERSISTENT;")
 
-newi = ("create index i%s on %s(att_conf_id, int_time);")
-newi += ("\ndrop index att_conf_id_data_time on %s;")
+newi = ("drop index att_conf_id_data_time on %s;")
+newi += ("\ncreate index i%s on %s(att_conf_id, int_time);")
 head = "ALTER TABLE %s PARTITION BY RANGE(int_time) ("
 line = "PARTITION %s%s VALUES LESS THAN (TO_SECONDS('%s')-62167222800)"
 lines = []
 
 for t,p in tables.items():
     lines.append(newc%t)
-    lines.append(newi%(p,t,t))
+    lines.append(newi%(t,p,t))
     lines.append(head%t)
     for i in range(0,npartitions):
         date = inc_months(start_date,i)

diff --git a/PyTangoArchiving/hdbpp/query.py b/PyTangoArchiving/hdbpp/query.py
@@ -47,7 +47,7 @@ def get_last_attribute_values(self,table,n=1,
             start = fn.str2time(
                 fn.time2str().split()[0].rsplit('-',1)[0]+'-01')
         vals = self.get_attribute_values(table, N=n, human=True, desc=True,
-                                            start_date=start, stop_date=epoch)
+                        start_date=start, stop_date=epoch)
         if len(vals):
             return vals[0] if abs(n)==1 else vals
         else: 
@@ -92,8 +92,9 @@ def get_attribute_values(self,table,start_date=None,stop_date=None,
         start_date and stop_date must be in a format valid for SQL
         """
         t0 = time.time()
-        self.debug('HDBpp.get_attribute_values(%s,%s,%s,%s,decimate=%s,%s)'
-              %(table,start_date,stop_date,N,decimate,kwargs))
+        self.debug('HDBpp.get_attribute_values(%s,%s,%s,%s,decimate=%s,'
+                   'int_time=%s,%s)'
+              %(table,start_date,stop_date,N,decimate,int_time,kwargs))
         if fn.isSequence(table):
             aid,tid,table = table
         else:
@@ -123,6 +124,7 @@ def get_attribute_values(self,table,start_date=None,stop_date=None,
         interval = 'where att_conf_id = %s'%aid if aid is not None \
                                                 else 'where att_conf_id >= 0 '
 
+        #self.info('%s : %s' % (table, self.getTableCols(table)))
         int_time = int_time and 'int_time' in self.getTableCols(table)
         if int_time:
             self.info('Using int_time indexing for %s' % table)
@@ -159,9 +161,13 @@ def str2mysqlsecs(date):
                 d = int(decimate) or 1
             else:
                 d = int((stop_time-start_time)/10800) or 1
+            def next_power_of_2(x):  
+                return 1 if x == 0 else 2**int(x - 1).bit_length()
+            d = next_power_of_2(d/2)
             # decimation on server side
             query += ' group by FLOOR(%s/%d)' % (
                 'int_time' if int_time else 'UNIX_TIMESTAMP(data_time)',d)
+
         query += ' order by %s' % ('int_time' if int_time else 'data_time')
 
         if N == 1: