fixes #15, improves debug handling and adds schema module

lodstorage/schema.py

@@ -0,0 +1,119 @@
+'''
+Created on 2021-01-26
+
+@author: wf
+'''
+from collections import Counter
+
+class SchemaManager(object,):
+    '''
+    a manager for schemas
+    '''
+    def __init__(self,schemaDefs=None,baseUrl:str=None):
+        '''
+        constructor
+            Args:
+                schemaDefs(dict): a dictionary of schema names
+                baseUrl(str): the base url to use for links
+        '''
+        self.baseUrl=baseUrl
+        self.schemasByName={}
+        if schemaDefs is None:
+            schemaDefs={}
+        for key,name in schemaDefs.items():
+            self.schemasByName[key]=Schema(key,name) 
+        pass
+
+class Schema(object):
+    '''
+    a relational Schema
+    '''
+
+    def __init__(self,name:str,title:str):
+        '''
+        Constructor
+        
+        Args:
+            name(str): the name of the schema
+            title(str): the title of the schema
+        '''
+        self.name=name
+        self.title=title
+        self.propsByName={}
+
+    @staticmethod    
+    def generalizeColumn(tableList,colName:str):
+        ''' 
+        remove the column with the given name from all tables in the tablelist and
+        return it
+        
+        Args:
+            tableList(list): a list of Tables
+            colName(string): the name of the column to generalize
+        
+        Returns:
+            string: the column having been generalized and removed
+        '''
+        gCol=None
+        for table in tableList:
+            for col in table['columns']: 
+                if col['name']==colName:
+                    gCol=col.copy()
+                    # no linking yet @FIXME - will need this later
+                    if 'link' in gCol:
+                        gCol.pop('link')
+                    # is generalization protected for this column?
+                    if not 'special' in col or not col['special']:
+                        table['columns'].remove(col)
+        return gCol                
+
+    @staticmethod    
+    def getGeneral(tableList,name:str,debug:bool=False):
+        '''
+        derive a general table from the given table list
+        Args:
+            tableList(list): a list of tables
+            name(str): name of the general table
+            debug(bool): True if column names should be shown
+            
+        Returns:
+            at table dict for the generalized table
+        '''
+        general={'name':name,'columns':[]}
+        colCount=Counter()
+        for table in tableList:
+            for col in table['columns']:
+                columnId="%s.%s" % (col['name'],col['type'])
+                if debug:
+                    print (columnId) 
+                colCount[columnId]+=1
+        for columnId,count in colCount.items():
+            if count==len(tableList): 
+                colName=columnId.split('.')[0]
+                generalCol=Schema.generalizeColumn(tableList, colName)
+                general['columns'].append(generalCol)    
+        return general
+
+    @staticmethod
+    def getGeneralViewDDL(tableList,name:str,debug=False)->str:
+        ''' 
+        get the DDL statement to create a general view
+        
+        Args:
+            tableList: the list of tables
+            name(str): the name of the view
+            debug(bool): True if debug should be set
+        '''
+        general=Schema.getGeneral(tableList, name, debug)
+        cols=""
+        delim=""
+        for col in general['columns']:
+            cols+="%s%s" % (delim,col['name'])
+            delim=","
+        ddl="CREATE VIEW %s AS \n" % name
+        delim=""
+        for table in tableList:
+            ddl+="%s  SELECT %s FROM %s" % (delim,cols,table['name'])
+            delim="\nUNION\n"
+        return ddl
+

lodstorage/sql.py

@@ -227,14 +227,20 @@ def getTableList(self):
             columns=self.query(columnQuery)
             table['columns']=columns
         return tableList
+
+    def restoreProgress(self,status,remaining,total):
+        self.progress("Restore",status,remaining,total)
+
+    def backupProgress(self,status,remaining,total):
+        self.progress("Backup",status,remaining,total)
 
-    def progress(self,status, remaining, total):
+    def progress(self,action,status, remaining, total):
         '''
         show progress
         '''
-        print('Backup %s at %5.0f%%' % ("... " if status==0 else "done",(total-remaining)/total*100)) 
+        print('%s %s at %5.0f%%' % (action,"... " if status==0 else "done",(total-remaining)/total*100)) 
 
-    def backup(self,backupDB,action="Backup",profile=False,showProgress=200,doClose=True):
+    def backup(self,backupDB,action="Backup",profile=False,showProgress:int=200,doClose=True):
         '''
         create backup of this SQLDB to the given backup db
         
@@ -251,7 +257,10 @@ def backup(self,backupDB,action="Backup",profile=False,showProgress=200,doClose=
         startTime=time.time()
         bck=sqlite3.connect(backupDB)
         if showProgress>0:
-            progress=self.progress
+            if action=="Restore":
+                progress=self.restoreProgress
+            else:
+                progress=self.backupProgress
         else:
             progress=None
         with bck:

lodstorage/uml.py

@@ -3,7 +3,8 @@
 
 @author: wf
 '''
-from collections import Counter
+from lodstorage.schema import Schema
+
 class UML(object):
     '''
     UML diagrams via plantuml
@@ -99,55 +100,7 @@ def __init__(self, debug=False):
         '''
         self.debug=debug
 
-    def generalizeColumn(self,tableList,colName):
-        ''' 
-        remove the column with the given name from all tables in the tablelist and
-        return it
-        
-        Args:
-            tableList(list): a list of Tables
-            colName(string): the name of the column to generalize
-        
-        Returns:
-            string: the column having been generalized and removed
-        '''
-        gCol=None
-        for table in tableList:
-            for col in table['columns']: 
-                if col['name']==colName:
-                    gCol=col.copy()
-                    # no linking yet @FIXME - will need this later
-                    if 'link' in gCol:
-                        gCol.pop('link')
-                    # is generalization protected for this column?
-                    if not 'special' in col or not col['special']:
-                        table['columns'].remove(col)
-        return gCol                
-
-    def getGeneral(self,tableList,name):
-        '''
-        derive a general table from the given table list
-        Args:
-            tableList(list): a list of tables
-            name(string): name of the general table
-            
-        Returns:
-            at table dict for the generalized table
-        '''
-        general={'name':name,'columns':[]}
-        colCount=Counter()
-        for table in tableList:
-            for col in table['columns']:
-                columnId="%s.%s" % (col['name'],col['type'])
-                if self.debug:
-                    print (columnId) 
-                colCount[columnId]+=1
-        for columnId,count in colCount.items():
-            if count==len(tableList): 
-                colName=columnId.split('.')[0]
-                generalCol=self.generalizeColumn(tableList, colName)
-                general['columns'].append(generalCol)    
-        return general
+
 
     def tableListToPlantUml(self,tableList,title=None,packageName=None,generalizeTo=None,withSkin=True):
         '''
@@ -172,7 +125,7 @@ def tableListToPlantUml(self,tableList,title=None,packageName=None,generalizeTo=
             uml+="package %s {\n" % packageName
             indent="  "
         if generalizeTo is not None:
-            generalTable=self.getGeneral(tableList,generalizeTo)
+            generalTable=Schema.getGeneral(tableList,generalizeTo)
             for table in tableList:
                 inherit+="%s%s <|-- %s\n" % (indent,generalizeTo,table['name'])
             tableList.insert(0,generalTable)

tests/testJson.py

@@ -15,8 +15,8 @@ class TestJsonAble(unittest.TestCase):
     '''
 
     def setUp(self):
-        self.profile=True
-        self.debug=True
+        self.profile=False
+        self.debug=False
         pass
 
     def tearDown(self):
@@ -54,11 +54,14 @@ def testSingleQuoteToDoubleQuoteStackoverflow(self):
         singleQuotedExamples=[
             '''{'cities': [{'name': "Upper Hell's Gate"}, {'name': "N'zeto"}]''']
         for example in singleQuotedExamples:
-            print (example)
+            if self.debug:
+                print (example)
             for useRegex in [False,True]:
                 doubleQuoted=JSONAble.singleQuoteToDoubleQuote(example,useRegex=useRegex)
-                print(doubleQuoted)
-            print
+                if self.debug:
+                    print(doubleQuoted)
+            if self.debug:
+                print
 
     def dumpListOfDicts(self,listOfDicts,limit):
         if self.debug:
@@ -123,13 +126,15 @@ def testRoyals(self):
         royals1=Royals(load=True)
         self.assertEqual(4,len(royals1.royals))
         json=royals1.toJSON()
-        print(json)
+        if self.debug:
+            print(json)
         types=Types.forTable(royals1, "royals",debug=True)
         royals2=Royals()
         royals2.fromJson(json,types=types)
         self.assertEqual(4,len(royals2.royals))
-        print(royals1.royals)
-        print(royals2.royals)
+        if self.debug:
+            print(royals1.royals)
+            print(royals2.royals)
         self.assertEqual(royals1.royals,royals2.royals)
 
     def testTypes(self):
@@ -139,7 +144,8 @@ def testTypes(self):
         royals1=Royals(load=True)
         types1=Types.forTable(royals1, "royals")
         json=types1.toJSON()
-        print(json)
+        if self.debug:
+            print(json)
         types2=Types("Royals")
         types2.fromJson(json)
         self.assertEqual(types1.typeMap,types2.typeMap)