add alter() & preview_alter()

datajoint/table.py

@@ -3,16 +3,18 @@
 import inspect
 import platform
 import numpy as np
+import pyparsing as pp
+import re
 import pandas
 import pymysql
 import logging
 import warnings
 from pymysql import OperationalError, InternalError, IntegrityError
 from .settings import config
-from .declare import declare
+from .declare import declare, is_foreign_key, attribute_parser, compile_foreign_key
 from .expression import QueryExpression
 from .blob import pack
-from .utils import user_choice
+from .utils import user_choice, to_camel_case
 from .heading import Heading
 from .errors import server_error_codes, DataJointError, DuplicateError
 from .version import __version__ as version
@@ -72,7 +74,190 @@ def declare(self, context=None):
                 raise
         else:
             self._log('Declared ' + self.full_table_name)
-
+
+    def make_alter(self,new_definition=None):
+        """
+        Returns changes required to go from current defition to new defition.
+        Does not execute any changes. Use alter() method to finalize any changes.
+        Only supports table comment, and SECONDARY attribute changes(ADD,DROP,CHANGE).
+        Reordering currently not supported.
+        Use # {<old_name>} <comment> to rename given attribute.(insert values between '< >') 
+
+        :param new_definition: string, same format used to create the table.
+        :return: alter sql. 
+        """
+
+        alter_sql = '' #alter sql
+
+        #change in table comment
+        new_definition = re.split(r'\s*\n\s*', new_definition.strip())
+        new_table_comment = new_definition.pop(0)[1:].strip() if new_definition[0].startswith('#') else ''
+
+        if (self.heading.table_info['comment'] != new_table_comment):    
+            alter_sql += ('COMMENT = "{new_table_comment}", '.format(new_table_comment = new_table_comment))
+
+        old_attributes = self.heading.as_sql.replace('`','').split(',') #used only for reordering
+        new_attributes = [] #non foreign, non index dependent attributes
+        in_key = True
+
+        #check for unsupported modifications
+        old_unsupported = []
+        for line in re.split(r'\s*\n\s*', self.definition.strip()):
+            if line.startswith('---') or line.startswith('___'):
+                in_key = False 
+            if is_foreign_key(line) or re.match(r'^(unique\s+)?index[^:]*$', line, re.I) or (in_key and not line.startswith('#')):
+                line = line.replace(' ','')
+                old_unsupported.append(line.replace('"','\''))
+        in_key = True
+        new_unsupported = []
+        for line in new_definition:
+            if line.startswith('---') or line.startswith('___'):
+                in_key = False 
+            if is_foreign_key(line) or re.match(r'^(unique\s+)?index[^:]*$', line, re.I) or (in_key and not line.startswith('#')):
+                line = line.replace(' ','')
+                new_unsupported.append(line.replace('"','\''))
+        if new_unsupported != old_unsupported:
+            raise DataJointError('Unsupported changes(PK,FK,Index) detected.')
+
+        after = self.heading.primary_key[-1] if self.heading.primary_key else 'FIRST'
+        in_key = True
+        for line in new_definition:
+            if line.startswith('#'):
+                continue
+            elif line.startswith('---') or line.startswith('___'):
+                in_key = False
+            elif is_foreign_key(line):
+                atts = []
+                not_needed = []
+                compile_foreign_key(line,self.connection.schemas[self.database].context,atts,not_needed,not_needed,not_needed,not_needed)
+                for att in atts[::-1]:
+                    if att not in new_attributes:
+                        new_attributes.append({'old_name':att,'name':att})
+                        after = atts[-1] if atts else after
+                        continue
+            elif re.match(r'^(unique\s+)?index[^:]*$', line, re.I): # index
+                continue
+            else:
+                if not in_key:
+                    # change in secondary attributes
+                    try:
+                        attr = attribute_parser.parseString(line+'#', parseAll=True)
+                    except pp.ParseException as err:
+                        raise DataJointError('Declaration error in position {pos} in line:\n  {line}\n{msg}'.format(
+                            line=err.args[0], pos=err.args[1], msg=err.args[2]))
+
+                    #external not supported
+                    if attr['type'].startswith('external'):
+                        continue
+
+                    #old_name and name are the same if no rename
+                    attr['old_name'] = attr['name']
+                    attr['comment'] = attr['comment'].rstrip('#')
+
+                    #extract rename, and new comment if necessary. brackets after any alphanumerical are ignored.
+                    rename = re.match(r'\s*{\s*(?P<name>[a-z][a-z0-9_]*)\s*}\s*(?P<comment>.*)', attr['comment'])
+                    if rename:
+                        attr['old_name'] = rename.group('name')
+                        attr['comment'] = rename.group('comment')
+
+                    attr['comment'] = attr['comment'].replace('"', '\\"')   # escape double quotes in comment
+
+                    #Extract changes in default, and nullable
+                    if 'default' not in attr:
+                        attr['default'] = ''
+                    attr = {k: v.strip() for k, v in attr.items()}
+                    attr['nullable'] = attr['default'].lower() == 'null'
+                    literals = ['CURRENT_TIMESTAMP'] 
+                    if attr['nullable']:
+                        if in_key:
+                            raise DataJointError('Primary key attributes cannot be nullable in line %s' % line)
+                        attr['default'] = 'NULL'
+                    else:
+                        if attr['default']:
+                            quote = attr['default'].upper() not in literals and attr['default'][0] in '"\''
+                            attr['default'] = ('"'+attr['default'][1:len(attr['default'])-1]+'"' if quote else attr['default'])
+                        else:
+                            attr['default'] = None
+
+                    after = new_attributes[-1]['name'] if new_attributes else after
+                    new_attributes.append(attr)
+
+                    #add attribute
+                    if attr['name'] not in self.heading.attributes and not rename:
+                        column_definition = ('{type}{null}{default}{comment}'.format(
+                                            type=attr['type'],
+                                            null=' NOT NULL' if not attr['nullable'] else '',
+                                            default=' DEFAULT {default}'.format(default=attr['default']) if attr['default'] else '',
+                                            comment=' COMMENT "{comment}"'.format(comment=attr['comment']) if attr['comment'] else ''))
+                        alter_sql += ('ADD COLUMN {name} {col_def} {aftercol}, '.format(
+                                        name=attr['name'], 
+                                        col_def=column_definition,
+                                        aftercol='AFTER {after}'.format(after=after)))
+                        old_attributes[old_attributes.index(after):old_attributes.index(after)] = [attr['name']]
+                        continue
+
+                    #change attribute
+                    if (rename
+                        or after != old_attributes[max(0, old_attributes.index(attr['old_name']) - 1)]
+                        or any(getattr(self.heading.attributes[attr['old_name']],attr_def) != attr[attr_def] 
+                            for attr_def in ('type','nullable','default','comment'))):
+                        #both enums?
+                        if (re.match('\s*enum',self.heading.attributes[attr['old_name']].type)
+                            and re.match('\s*enum',attr['type'])):
+                            #outer ' -> " (per enum) 
+                            old_enum = re.sub(r'(?<!\w)[\']|[\'](?!\w)','"',self.heading.attributes[attr['old_name']].type)
+                            attr['type'] = re.sub(r'(?<!\w)[\']|[\'](?!\w)','"',attr['type'])
+                            #no difference
+                            if set(re.findall(r'(?<!\w)"(.+?)"(?!\w)',old_enum)) == set(re.findall(r'(?<!\w)"(.+?)"(?!\w)',attr['type'])):
+                                continue
+                        column_definition = ('{type}{null}{default}{comment}'.format(
+                                            type=attr['type'],
+                                            null=' NOT NULL' if not attr['nullable'] else '',
+                                            default=' DEFAULT {default}'.format(default=attr['default']) if attr['default'] else '',
+                                            comment=' COMMENT "{comment}"'.format(comment=attr['comment']) if attr['comment'] else ''))
+                        alter_sql += ('CHANGE COLUMN {old_name} {name} {column_definition} {aftercol}, '.format(
+                                        old_name=attr['old_name'],
+                                        name=attr['name'],
+                                        column_definition=column_definition,
+                                        aftercol='AFTER {after}'.format(after=after)))
+                        old_attributes.pop(old_attributes.index(attr['old_name']))
+                        old_attributes[old_attributes.index(after)+1:old_attributes.index(after)+1] = [attr['name']]
+
+
+        #Drop attribute
+        for old_attribute in self.heading.dependent_attributes:
+            if (all(old_attribute != new_attribute['old_name'] for new_attribute in new_attributes) 
+                and not self.heading.attributes[old_attribute].type.startswith('external') 
+                and not any(old_attribute in tup for tup in self.heading.indexes.keys())):
+                alter_sql += ('DROP COLUMN {old_name}, '.format(old_name=old_attribute))
+
+        if alter_sql:
+            alter_sql = 'ALTER TABLE %s %s;' % (self.full_table_name, alter_sql[:-2])
+
+        return alter_sql or None
+
+    def alter(self, new_definition = None, alter_statement = None):
+        """
+        Execute an ALTER TABLE statement.
+        :param new_definition: new definition.
+        :param alter_statement: optional alter statements.
+        if provided, alter_statement overrides new_definition.
+        """
+
+        if new_definition is None and alter_statement is None:
+            raise DataJointError("No alter specification provided.")
+
+        if self.connection.in_transaction:
+            raise DataJointError("Table definition cannot be altered during a transaction.")
+
+        if alter_statement is None:
+            alter_statement = self.make_alter(new_definition)
+
+        if alter_statement:        
+            self.connection.query(alter_statement)
+            self.heading.init_from_database(self.connection, self.database, self.table_name)
+            self.heading.init_from_database(self.connection, self.database, self.table_name)
+
     @property
     def from_clause(self):
         """

datajoint/user_tables.py

@@ -14,7 +14,7 @@
 supported_class_attrs = {
     'key_source', 'describe', 'heading', 'populate', 'progress', 'primary_key', 'proj', 'aggr',
     'fetch', 'fetch1','head', 'tail',
-    'insert', 'insert1', 'drop', 'drop_quick', 'delete', 'delete_quick'}
+    'insert', 'insert1', 'drop', 'drop_quick', 'delete', 'delete_quick', 'make_alter', 'alter'}
 
 
 class OrderedClass(type):

tests/test_relation.py

@@ -2,7 +2,7 @@
 import re
 import pandas
 import numpy as np
-from nose.tools import assert_equal, assert_not_equal, assert_true, assert_list_equal, raises
+from nose.tools import assert_equal, assert_not_equal, assert_true, assert_false, assert_list_equal, raises
 from pymysql import InternalError
 import datajoint as dj
 from datajoint import utils, DataJointError
@@ -242,3 +242,58 @@ def test_table_size(self):
 
     def test_repr_html(self):
         assert_true(self.ephys._repr_html_().strip().startswith("<style"))
+    @raises(DataJointError)
+    def test_alter_unsupported(self):
+        """alter unsupported(primary, foreign key, index) test on dj.lookup with data"""
+
+        definition = """  
+        # Basic information about animal subjects used in experiments
+        # removed 'unique index (real_id, species)' from secondary
+        subject_id   :int  #  {op} no change. unique subject id
+        new_pri: int # shouldn't exist
+        ---
+        -> dep: voila   # shouldn't matter
+        real_id            :varchar(40)  # real-world name. Omit if the same as subject_id
+        species = "mouse"  :enum('mouse', 'monkey', 'human')
+        date_of_birth      :date
+        subject_notes      :varchar(4000)
+        """ 
+        self.subject.make_alter(definition)
+
+    def test_alter_supported(self):
+        """alter supported test on dj.lookup with data"""
+        #asser (TABLE COMMENT,CHANGE, CHANGE(rename), ADD, DROP) in that order
+        new_definition = """
+        subject_id   :int  #  unique subject id
+        ---
+        real_id            :varchar(40)  # real-world name. Omit if the same as subject_id
+        species = 'monkey' :enum('mouse', 'monkey', 'human', 'fish')
+        dob                :date #{date_of_birth}
+        salary             :int # minimum wage for monkeys
+        unique index (real_id, species)
+        """
+        old_definition = """
+        # Basic information about animal subjects used in experiments
+        subject_id   :int  #  unique subject id
+        ---
+        real_id            :varchar(40)  # real-world name. Omit if the same as subject_id
+        species = "mouse"  :enum("mouse", "monkey", "human")
+        date_of_birth      :date #{dob}
+        subject_notes      :varchar(4000)
+        unique index (real_id, species)
+        """
+        self.subject.alter(new_definition)
+        assert_false(self.subject.heading.table_info['comment'])
+        assert_true('fish' in self.subject.heading.attributes['species'].type
+                and 'monkey' == self.subject.heading.attributes['species'].default.strip('"'))
+        assert_true('date_of_birth' not in self.subject.heading.attributes
+                    and 'dob' in self.subject.heading.attributes)
+        assert_true('salary' in self.subject.heading.attributes
+                    and self.subject.heading.attributes['salary'].type.strip('"') == 'int'
+                    and not self.subject.heading.attributes['salary'].nullable
+                    and not self.subject.heading.attributes['salary'].default)
+        assert_true('subject_notes' not in self.subject.heading.attributes)
+
+        #revert
+        self.subject.alter(old_definition)
+        self.subject.insert(self.subject.contents,replace=True)