Skip to content
Permalink
Browse files

add arguments to create_partitions, apply alias to all schemas

  • Loading branch information...
sergirubio committed Oct 3, 2019
1 parent 6ed30e7 commit a7d2099ec5887a3ef3b098adf12b799d93c39b6c
@@ -39,7 +39,16 @@
"""

RELEASE = (6,4,0)
try:
import os
f = open(os.path.dirname(os.path.abspath(__file__))+'/VERSION')
v = f.read().strip().split('.')
f.close()
RELEASE = tuple(map(int,v))
except:
import traceback
traceback.print_exc()
print('Unable to read PyTangoArchiving.VERSION')

ARCHIVING_TYPES = ['hdb','tdb','snap']
ARCHIVING_CLASSES = ['HdbArchiver','TdbArchiver','SnapArchiver',
@@ -270,6 +270,11 @@ def clean_attribute_modes(self,date):
return False
return True

def clean_old_tables(self, tables, date):
"""
Cleanup all unactive modes from DB if stop_date is older than date
"""

def get_table_updates(self,name=''):
if name and not str(name).startswith('att_'):
n = self.get_table_name(name if isinstance(name,int) else self.get_attribute_ID(name))
@@ -82,7 +82,7 @@ def __init__(self,db_name='',host='',user='',
else:
sch = Schemas.getSchema(db_name)
if sch:
print('HDBpp(): Loading from Schemas')
#print('HDBpp(): Loading from Schemas')
db_name = sch.get('dbname',sch.get('db_name'))
host = host or sch.get('host')
user = user or sch.get('user')
@@ -9,30 +9,31 @@
# boolean, encoded, enum, long64 uchar ulong64, ulong, ushort
# b, e, n, l64, ul6, ul, us, uc

#'att_array_devdouble_ro':'adr',
#'att_array_devfloat_ro':'afr',
#'att_array_devlong_ro':'alr',
#'att_array_devlong_rw':'alw',
#'att_array_devshort_ro':'ahr',
#'att_array_devboolean_ro':'abr',
#'att_array_devboolean_rw':'abw',
#'att_array_devstring_ro':'asr',
#'att_array_devstate_ro':'atr',
'att_array_devdouble_ro':'adr',
'att_array_devfloat_ro':'afr',
'att_array_devlong_ro':'alr',
'att_array_devlong_rw':'alw',
'att_array_devshort_ro':'ahr',
'att_array_devboolean_ro':'abr',
'att_array_devboolean_rw':'abw',
'att_array_devstring_ro':'asr',
'att_array_devstate_ro':'atr',

#'att_scalar_devdouble_ro':'sdr',
#'att_scalar_devdouble_rw':'sdw',
#'att_scalar_devfloat_ro':'sfr',
#'att_scalar_devlong_ro':'slr',
#'att_scalar_devlong_rw':'slw',
#'att_scalar_devshort_ro':'shr',
#'att_scalar_devshort_rw':'shw',
#'att_scalar_devboolean_ro':'sbr',
#'att_scalar_devboolean_rw':'sbw',
'att_scalar_devdouble_ro':'sdr',
'att_scalar_devdouble_rw':'sdw',

'att_scalar_devfloat_ro':'sfr',
'att_scalar_devlong_ro':'slr',
'att_scalar_devlong_rw':'slw',
'att_scalar_devshort_ro':'shr',
'att_scalar_devshort_rw':'shw',
'att_scalar_devboolean_ro':'sbr',
'att_scalar_devboolean_rw':'sbw',

#'att_scalar_devstate_ro':'str',
#'att_scalar_devstring_ro':'ssr',
#'att_scalar_devushort_ro':'sur',
#'att_scalar_devuchar_ro':'scr',
'att_scalar_devstate_ro':'str',
'att_scalar_devstring_ro':'ssr',
'att_scalar_devushort_ro':'sur',
'att_scalar_devuchar_ro':'scr',

'att_array_devfloat_rw':'afw',
'att_scalar_devstring_ro':'ssw',
@@ -1,34 +1,40 @@
import sys
import fandango as fd
import PyTangoArchiving

assert sys.argv[1:], 'Start date required (e.g. 2017-08-01)'

tables = {
from argparse import ArgumentParser
parser = ArgumentParser()
parser.add_argument('--tables', metavar='N', type=str, nargs='+',
help='optional, tables to partition')
parser.add_argument('--start', type=str,
help='MANDATORY, first partition to create')
parser.add_argument('--api', type=str, help='if provided, it will generate '
'partitions from the existing point')
parser.add_argument('--nparts', type=int, help='partitions to add at the end,'
' if not set, nothing is done except if --add_last 1')
parser.add_argument('--int_time', type=bool,
help='partition using INT_TIME column')
parser.add_argument('--add_last', type=bool,
help='add a prefix_last partition at the end')

#'att_array_devdouble_ro':'adr',
#'att_array_devlong_ro':'alr',
#'att_array_devshort_ro':'ahr',
#'att_array_devstring_ro':'asr',
#'att_array_devstate_ro':'atr',
args = parser.parse_args()
if not args.start and not args.add_last:
raise Exception('--start or --add_last argument required (e.g. 2017-08-01)')

#'att_scalar_devdouble_ro':'sdr',
#'att_scalar_devdouble_rw':'sdw',

#'att_scalar_devfloat_ro':'sfr',
#'att_scalar_devlong_ro':'slr',
#'att_scalar_devlong_rw':'slw',
#'att_scalar_devstate_ro':'str',
#'att_scalar_devstring_ro':'ssr',
#'att_scalar_devshort_ro':'shr',
#'att_scalar_devshort_rw':'shw',

'att_array_devfloat_rw':'afw',
'att_scalar_devstring_rw':'ssw',

}

start_date = sys.argv[1] # '2017-08-01'
npartitions = 30
if args.api:
api = PyTangoArchiving.api(args.api)
else:
api = None

tables = PyTangoArchiving.hdbpp.query.partition_prefixes
### BUT, NOT ALL TABLES ARE IN THIS LIST!
# I'm partitioning only the big ones, and ignoring the others
# boolean, encoded, enum, long64 uchar ulong64, ulong, ushort
# b, e, n, l64, ul6, ul, us, uc

start_date = args.start or fd.time2str().split()[0]
npartitions = args.nparts or 0
counter = 0

def inc_months(date,count):
@@ -42,20 +48,56 @@ def inc_months(date,count):
y += int(m/12)-1
m = 12
return '%04d-%02d-%02d'%(y,m,d)

intcol = 'int_time'

if args.int_time:
newc = ("alter table %s add column int_time INT "
"generated always as (TO_SECONDS(data_time)-62167222800) PERSISTENT;")

newi = ("drop index att_conf_id_data_time on %s;")
newi += ("\ncreate index i%s on %s(att_conf_id, int_time);")
head = "ALTER TABLE %s "
comm = "PARTITION BY RANGE(int_time) ("
line = "PARTITION %s%s VALUES LESS THAN (TO_SECONDS('%s')-62167222800)"
else:
head = "ALTER TABLE %s "
comm = "PARTITION BY RANGE(TO_DAYS(data_time)) ("
line = "PARTITION %s%s VALUES LESS THAN (TO_DAYS('%s'))"

head = "ALTER TABLE %s PARTITION BY RANGE(TO_DAYS(data_time)) ("
line = "PARTITION %s%s VALUES LESS THAN (TO_DAYS('%s'))"
lines = []

for t,p in tables.items():
tlist = [t for t in tables if not args.tables or t in args.tables]

for t in tlist:
p = tables[t]
if args.int_time and (not api or not intcol in api.getTableCols(t)):
lines.append(newc%t)
lines.append(newi%(t,p,t))
eparts = [] if not api else api.getTablePartitions(t)
lines.append(head%t)
if not any(eparts):
lines.append(comm)
elif p+'_last' in eparts:
lines.append('REORGANIZE PARTITION %s INTO (' % (p+'_last'))
else:
lines.append('ADD PARTITION (')

for i in range(0,npartitions):
date = inc_months(start_date,i)
end = inc_months(date,1)
pp = p+date.replace('-','')
l = line%(p,date.replace('-',''),end)
if i<(npartitions-1): l+=','
lines.append(l)
if i<(npartitions-1):
l+=','
if pp not in eparts:
lines.append(l)

if args.add_last and p+'_last' not in eparts or 'REORGANIZE' in str(lines):
if not lines[-1][-1] in ('(',','):
lines[-1] += ','
lines.append('PARTITION %s_last VALUES LESS THAN (MAXVALUE)'%p)

lines.append(');\n\n')

print('\n'.join(lines))

0 comments on commit a7d2099

Please sign in to comment.
You can’t perform that action at this time.