Erd

Makefile

@@ -2,7 +2,7 @@ all:
 	@echo 'MakeFile for DataJoint packaging                                              '
 	@echo '                                                                              '
 	@echo 'make sdist                              Creates source distribution           '
-	@echo 'make wheel                              Creates Wheel dstribution             '
+	@echo 'make wheel                              Creates Wheel distribution            '
 	@echo 'make pypi                               Package and upload to PyPI            '
 	@echo 'make pypitest                           Package and upload to PyPI test server'
 	@echo 'make purge                              Remove all build related directories  '

datajoint/__init__.py

@@ -15,7 +15,7 @@
 __version__ = "0.2"
 __all__ = ['__author__', '__version__',
            'config', 'conn', 'kill',
-           'Connection', 'Heading', 'Relation', 'FreeRelation', 'Not', 'schema',
+           'Connection', 'Heading', 'BaseRelation', 'FreeRelation', 'Not', 'schema',
            'Manual', 'Lookup', 'Imported', 'Computed', 'Part']
 
 
@@ -52,9 +52,9 @@ class DataJointError(Exception):
 
 # ------------- flatten import hierarchy -------------------------
 from .connection import conn, Connection
-from .relation import Relation
+from .base_relation import BaseRelation
 from .user_relations import Manual, Lookup, Imported, Computed, Part
 from .relational_operand import Not
 from .heading import Heading
-from .schema import schema
+from .schema import Schema as schema
 from .kill import kill

datajoint/autopopulate.py

@@ -5,7 +5,7 @@
 import random
 from .relational_operand import RelationalOperand
 from . import DataJointError
-from .relation import FreeRelation
+from .base_relation import FreeRelation
 
 # noinspection PyExceptionInherit,PyCallingNonCallable
 
@@ -60,20 +60,21 @@ def populate(self, restriction=None, suppress_errors=False,
 
         :param restriction: restriction on rel.populated_from - target
         :param suppress_errors: suppresses error if true
-        :param reserve_jobs: currently not implemented
-        :param batch: batch size of a single job
+        :param reserve_jobs: if true, reserves job to populate in asynchronous fashion
         :param order: "original"|"reverse"|"random"  - the order of execution
         """
-        if not isinstance(self.populated_from, RelationalOperand):
-            raise DataJointError('Invalid populated_from value')
-
         if self.connection.in_transaction:
             raise DataJointError('Populate cannot be called during a transaction.')
 
         valid_order = ['original', 'reverse', 'random']
         if order not in valid_order:
             raise DataJointError('The order argument must be one of %s' % str(valid_order))
 
+        self.connection.dependencies.load()
+
+        if not isinstance(self.populated_from, RelationalOperand):
+            raise DataJointError('Invalid populated_from value')
+
         error_list = [] if suppress_errors else None
 
         jobs = self.connection.jobs[self.target.database]
@@ -112,6 +113,7 @@ def populate(self, restriction=None, suppress_errors=False,
                             jobs.complete(table_name, key)
         return error_list
 
+
     def progress(self, restriction=None, display=True):
         """
         report progress of populating this table

datajoint/relation.py → datajoint/base_relation.py

@@ -16,15 +16,16 @@
 logger = logging.getLogger(__name__)
 
 
-class Relation(RelationalOperand, metaclass=abc.ABCMeta):
+class BaseRelation(RelationalOperand, metaclass=abc.ABCMeta):
     """
-    Relation is an abstract class that represents a base relation, i.e. a table in the database.
+    BaseRelation is an abstract class that represents a base relation, i.e. a table in the database.
     To make it a concrete class, override the abstract properties specifying the connection,
     table name, database, context, and definition.
     A Relation implements insert and delete methods in addition to inherited relational operators.
     """
     _heading = None
     _context = None
+    database = None
 
     # ---------- abstract properties ------------ #
     @property
@@ -58,8 +59,10 @@ def heading(self):
         """
         if self._heading is None:
             self._heading = Heading()  # instance-level heading
-        if not self._heading:
+        if not self._heading:   # heading is not initialized
             self.declare()
+            self._heading.init_from_database(self.connection, self.database, self.table_name)
+
         return self._heading
 
     def declare(self):
@@ -69,9 +72,6 @@ def declare(self):
         if not self.is_declared:
             self.connection.query(
                 declare(self.full_table_name, self.definition, self._context))
-        if self.is_declared:
-            self.connection.erm.load_dependencies(self.full_table_name)
-            self._heading.init_from_database(self.connection, self.database, self.table_name)
 
     @property
     def from_clause(self):
@@ -87,42 +87,54 @@ def select_fields(self):
         """
         return '*'
 
-    def erd(self, *args, **kwargs):
+    def erd(self, *args, fill=True, mode='updown', **kwargs):
         """
+        :param mode: diffent methods of creating a graph pertaining to this relation.
+        Currently options includes the following:
+        * 'updown': Contains this relation and all other nodes that can be reached within specific
+        number of ups and downs in the graph. ups(=2) and downs(=2) are optional keyword arguments
+        * 'ancestors': Returs
         :return: the entity relationship diagram object of this relation
         """
         erd = self.connection.erd()
-        nodes = erd.up_down_neighbors(self.full_table_name)
-        return erd.restrict_by_tables(nodes)
+        if mode == 'updown':
+            nodes = erd.up_down_neighbors(self.full_table_name, *args, **kwargs)
+        elif mode == 'ancestors':
+            nodes = erd.ancestors(self.full_table_name, *args, **kwargs)
+        elif mode == 'descendants':
+            nodes = erd.descendants(self.full_table_name, *args, **kwargs)
+        else:
+            raise DataJointError('Unsupported erd mode', 'Mode "%s" is currently not supported' % mode)
+        return erd.restrict_by_tables(nodes, fill=fill)
 
     # ------------- dependencies ---------- #
     @property
     def parents(self):
         """
         :return: the parent relation of this relation
         """
-        return self.connection.erm.parents[self.full_table_name]
+        return self.connection.dependencies.parents[self.full_table_name]
 
     @property
     def children(self):
         """
         :return: the child relations of this relation
         """
-        return self.connection.erm.children[self.full_table_name]
+        return self.connection.dependencies.children[self.full_table_name]
 
     @property
     def references(self):
         """
         :return: list of tables that this tables refers to
         """
-        return self.connection.erm.references[self.full_table_name]
+        return self.connection.dependencies.references[self.full_table_name]
 
     @property
     def referenced(self):
         """
         :return: list of tables for which this table is referenced by
         """
-        return self.connection.erm.referenced[self.full_table_name]
+        return self.connection.dependencies.referenced[self.full_table_name]
 
     @property
     def descendants(self):
@@ -134,10 +146,13 @@ def descendants(self):
         :return: list of descendants
         """
         relations = (FreeRelation(self.connection, table)
-                     for table in self.connection.erm.get_descendants(self.full_table_name))
+                     for table in self.connection.dependencies.get_descendants(self.full_table_name))
         return [relation for relation in relations if relation.is_declared]
 
     def _repr_helper(self):
+        """
+        :return: String representation of this object
+        """
         return "%s.%s()" % (self.__module__, self.__class__.__name__)
 
     # --------- SQL functionality --------- #
@@ -162,7 +177,7 @@ def insert(self, rows, **kwargs):
         """
         Insert a collection of rows. Additional keyword arguments are passed to insert1.
 
-        :param iter: Must be an iterator that generates a sequence of valid arguments for insert.
+        :param rows: An iterable where an element is a valid arguments for insert1.
         """
         for row in rows:
             self.insert1(row, **kwargs)
@@ -172,9 +187,9 @@ def insert1(self, tup, replace=False, ignore_errors=False, skip_duplicates=False
         Insert one data record or one Mapping (like a dict).
 
         :param tup: Data record, a Mapping (like a dict), or a list or tuple with ordered values.
-        :param replace=False: Replaces data tuple if True.
+        :param replace=False: If True, replaces the matching data tuple in the table if it exists.
         :param ignore_errors=False: If True, ignore errors: e.g. constraint violations.
-        :param skip_dublicates=False: If True, ignore duplicate inserts.
+        :param skip_dublicates=False: If True, silently skip duplicate inserts.
 
         Example::
 
@@ -185,14 +200,18 @@ def insert1(self, tup, replace=False, ignore_errors=False, skip_duplicates=False
         heading = self.heading
 
         def check_fields(fields):
+            """
+            Validates that all items in `fields` are valid attributes in the heading
+            """
             for field in fields:
                 if field not in heading:
                     raise KeyError(u'{0:s} is not in the attribute list'.format(field))
 
         def make_attribute(name, value):
             """
-            For a given attribute, return its value or value placeholder as a string to be included
-            in the query and the value, if any to be submitted for processing by mysql API.
+            For a given attribute `name` with `value, return its processed value or value placeholder
+            as a string to be included in the query and the value, if any to be submitted for
+            processing by mysql API.
             """
             if heading[name].is_blob:
                 value = pack(value)
@@ -249,15 +268,18 @@ def make_attribute(name, value):
 
     def delete_quick(self):
         """
-        Deletes the table without cascading and without user prompt.
+        Deletes the table without cascading and without user prompt. If this table has any dependent
+        table(s), this will fail.
         """
+        #TODO: give a better exception message
         self.connection.query('DELETE FROM ' + self.from_clause + self.where_clause)
 
     def delete(self):
         """
         Deletes the contents of the table and its dependent tables, recursively.
         User is prompted for confirmation if config['safemode'] is set to True.
         """
+        self.connection.dependencies.load()
 
         # construct a list (OrderedDict) of relations to delete
         relations = OrderedDict((r.full_table_name, r) for r in self.descendants)
@@ -281,7 +303,7 @@ def delete(self):
         for name, r in relations.items():
             if restrictions[name]:  # do not restrict by an empty list
                 r.restrict([r.project() if isinstance(r, RelationalOperand) else r
-                            for r in restrictions[name]])  # project 
+                            for r in restrictions[name]])  # project
 
         # execute
         do_delete = False  # indicate if there is anything to delete
@@ -308,19 +330,21 @@ def delete(self):
     def drop_quick(self):
         """
         Drops the table associated with this relation without cascading and without user prompt.
+        If the table has any dependent table(s), this call will fail with an error.
         """
+        #TODO: give a better exception message
         if self.is_declared:
             self.connection.query('DROP TABLE %s' % self.full_table_name)
-            self.connection.erm.clear_dependencies(self.full_table_name)
-            if self._heading:
-                self._heading.reset()
             logger.info("Dropped table %s" % self.full_table_name)
+        else:
+            logger.info("Nothing to drop: table %s is not declared" % self.full_table_name)
 
     def drop(self):
         """
         Drop the table and all tables that reference it, recursively.
         User is prompted for confirmation if config['safemode'] is set to True.
         """
+        self.connection.dependencies.load()
         do_drop = True
         relations = self.descendants
         if config['safemode']:
@@ -351,9 +375,10 @@ def _prepare(self):
         pass
 
 
-class FreeRelation(Relation):
+class FreeRelation(BaseRelation):
     """
-    A base relation without a dedicated class.  The table name is explicitly set.
+    A base relation without a dedicated class. Each instance is associated with a table
+    specified by full_table_name.
     """
     def __init__(self, connection, full_table_name, definition=None, context=None):
         self.database, self._table_name = (s.strip('`') for s in full_table_name.split('.'))

datajoint/connection.py

@@ -8,7 +8,8 @@
 import logging
 from . import config
 from . import DataJointError
-from .erd import ERM
+from datajoint.erd import ERD
+from .dependencies import Dependencies
 from .jobs import JobManager
 
 logger = logging.getLogger(__name__)
@@ -67,7 +68,8 @@ def __init__(self, host, user, passwd, init_fun=None):
         self._conn.autocommit(True)
         self._in_transaction = False
         self.jobs = JobManager(self)
-        self.erm = ERM(self)
+        self.schemas = dict()
+        self.dependencies = Dependencies(self)
 
     def __del__(self):
         logger.info('Disconnecting {user}@{host}:{port}'.format(**self.conn_info))
@@ -81,8 +83,8 @@ def __repr__(self):
         return "DataJoint connection ({connected}) {user}@{host}:{port}".format(
             connected=connected, **self.conn_info)
 
-    def erd(self, *args, **kwargs):
-        return self.erm.copy_graph(*args, **kwargs)
+    def register(self, schema):
+        self.schemas[schema.database] = schema
 
     @property
     def is_connected(self):
@@ -91,6 +93,10 @@ def is_connected(self):
         """
         return self._conn.ping()
 
+    def erd(self):
+        self.dependencies.load()
+        return ERD.create_from_dependencies(self.dependencies)
+
     def query(self, query, args=(), as_dict=False):
         """
         Execute the specified query and return the tuple generator (cursor).

datajoint/declare.py

@@ -38,8 +38,21 @@ def declare(full_table_name, definition, context):
         elif line.startswith('---') or line.startswith('___'):
             in_key = False  # start parsing dependent attributes
         elif line.startswith('->'):
-            # foreign key
-            ref = eval(line[2:], context)()  # TODO: surround this with try...except... to give a better error message
+            # foreign
+            # TODO: clean up import order
+            from .base_relation import BaseRelation
+            # TODO: break this step into finer steps, checking the type of reference before calling it
+            try:
+                ref = eval(line[2:], context)()
+            except NameError:
+                raise DataJointError('Foreign key reference %s could not be resolved' % line[2:])
+            except TypeError:
+                raise DataJointError('Foreign key reference %s could not be instantiated.'
+                                     'Make sure %s is a valid BaseRelation subclass' % line[2:])
+            # TODO: consider the case where line[2:] is a function that returns an instance of BaseRelation
+            if not isinstance(ref, BaseRelation):
+                    raise DataJointError('Foreign key reference %s must be a subclass of BaseRelation' % line[2:])
+
             foreign_key_sql.append(
                 'FOREIGN KEY ({primary_key})'
                 ' REFERENCES {ref} ({primary_key})'