Redesigned Dependencies, completed ERD, implemented relation U.

.travis.yml

@@ -1,14 +1,15 @@
 sudo: required
 language: python
 env:
-  - DJ_TEST_HOST="127.0.0.1" DJ_TEST_USER="root" DJ_TEST_PASSWORD="" DJ_HOST="127.0.0.1" DJ_USER="root" DJ_PASSWORD=""
+  - DJ_TEST_HOST="127.0.0.1" DJ_TEST_USER="datajoint" DJ_TEST_PASSWORD="datajoint" DJ_HOST="127.0.0.1" DJ_USER="datajoint" DJ_PASSWORD="datajoint"
 python:
   - "3.4"
   - "3.5"
 services: mysql
 before_install:
   - sudo apt-get -qq update
   - sudo apt-get install -y libblas-dev liblapack-dev libatlas-dev gfortran
+  - mysql -e "create user 'datajoint'@'%' identified by 'datajoint'; GRANT ALL PRIVILEGES ON \`djtest\_%\`.* TO 'datajoint'@'%';" -uroot
 install:
   - travis_wait 30 pip install -r requirements.txt
   - pip install nose nose-cov python-coveralls

README.md

@@ -15,6 +15,7 @@ DataJoint for Python is a high-level programming interface for relational databa
 
 DataJoint was initially developed in 2009 by Dimitri Yatsenko in Andreas Tolias' Lab for the distributed processing and management of large volumes of data streaming from regular experiments. Starting in 2011, DataJoint has been available as an open-source project adopted by other labs and improved through contributions from several developers.
 
+
 ## Quick start guide
 The current pip version is ancient. We will update it as soon as datajoint release 1.0 is out. 
 ~~To install datajoint using `pip` just run:~~
@@ -27,3 +28,26 @@ pip install datajoint
 
 ~~However, please be aware that DataJoint for Python is still undergoing major changes, and thus what's available on PyPI via `pip` is in **pre-release state**!~~
 
+## Tutorial 
+1. [Setup](tutorial-notebooks/Primer00.ipynb)
+1. [Connect](tutorial-notebooks/Primer01.ipynb)
+1. [Create a schema, define a table](tutorial-notebooks/Primer02.ipynb)
+1. [Dependencies](tutorial-notebooks/Primer03.ipynb)
+1. [Schemas as Python modules](tutorial-notebooks/Primer04.ipynb)
+1. [Lookup tables](tutorial-notebooks/Primer05.ipynb)
+1. [Queries 1: restrictions and joins](tutorial-notebooks/Primer06.ipynb)
+1. [Dependencies 2: non-primary](tutorial-notebooks/Primer07.ipynb) 
+1. [Queries 2: projections](tutorial-notebooks/Primer08.ipynb)
+1. [Dependencies 3: aliased foreign keys](tutorial-notebooks/Primer09.ipynb)
+1. [Computations](tutorial-notebooks/Primer10.ipynb)
+1. [Parameterized Computations](tutorial-notebooks/Primer11.ipynb)
+1. [Master-part relationships](tutorial-notebooks/Primer12.ipynb)
+1. [Understanding transactions](tutorial-notebooks/Primer13.ipynb)
+1. [Job management for distributed computation](tutorial-notebooks/Primer14.ipynb)
+1. [Projection and aggregation](tutorial-notebooks/Primer15.ipynb)
+1. [Relation U](tutorial-notebooks/Primer16.ipynb)
+1. [Dependencies 4: mapped dependencies](tutorial-notebooks/Primer17.ipynb)
+1. [Representing graphs](tutorial-notebooks/Primer18.ipynb)
+1. [Customizing computations](tutorial-notebooks/Primer19.ipynb)
+1. [BOSS interface](tutorial-notebooks/Primer20.ipynb)
+1. [Web interfaces](tutorial-notebooks/Primer21.ipynb)

datajoint/__init__.py

@@ -6,18 +6,25 @@
 
 DataJoint is free software under the LGPL License. In addition, we request
 that any use of DataJoint leading to a publication be acknowledged in the publication.
+
+Please cite:
+    http://biorxiv.org/content/early/2015/11/14/031658
+    http://dx.doi.org/10.1101/031658
 """
 
 import logging
 import os
 
 __author__ = "Dimitri Yatsenko, Edgar Walker, and Fabian Sinz at Baylor College of Medicine"
-__version__ = "0.2"
+__version__ = "0.2.1"
+__date__ = "June 1, 2016"
 __all__ = ['__author__', '__version__',
            'config', 'conn', 'kill',
            'Connection', 'Heading', 'BaseRelation', 'FreeRelation', 'Not', 'schema',
            'Manual', 'Lookup', 'Imported', 'Computed', 'Part',
-           'AndList', 'OrList']
+           'AndList', 'OrList', 'ERD', 'U']
+
+print('DataJoint', __version__, '('+__date__+')')
 
 
 class key:
@@ -65,12 +72,12 @@ class DataJointError(Exception):
 
 logger.setLevel(log_levels[config['loglevel']])
 
-
 # ------------- flatten import hierarchy -------------------------
 from .connection import conn, Connection
 from .base_relation import BaseRelation
 from .user_relations import Manual, Lookup, Imported, Computed, Part
-from .relational_operand import Not, AndList, OrList
+from .relational_operand import Not, AndList, OrList, U
 from .heading import Heading
 from .schema import Schema as schema
 from .kill import kill
+from .erd import ERD

datajoint/autopopulate.py

@@ -3,6 +3,7 @@
 import logging
 import datetime
 import random
+from pymysql import OperationalError
 from .relational_operand import RelationalOperand, AndList
 from . import DataJointError
 from .base_relation import FreeRelation
@@ -12,41 +13,40 @@
 logger = logging.getLogger(__name__)
 
 
-class AutoPopulate(metaclass=abc.ABCMeta):
+class AutoPopulate:
     """
     AutoPopulate is a mixin class that adds the method populate() to a Relation class.
     Auto-populated relations must inherit from both Relation and AutoPopulate,
-    must define the property populated_from, and must define the callback method _make_tuples.
+    must define the property `key_source`, and must define the callback method _make_tuples.
     """
     _jobs = None
     _populated_from = None
 
     @property
-    def populated_from(self):
+    def key_source(self):
         """
         :return: the relation whose primary key values are passed, sequentially, to the
                 `_make_tuples` method when populate() is called.The default value is the
                 join of the parent relations. Users may override to change the granularity
                 or the scope of populate() calls.
         """
         if self._populated_from is None:
-            self.connection.dependencies.load()
-            parents = [FreeRelation(self.target.connection, rel) for rel in self.target.parents]
+            self.connection.dependencies.load(self.full_table_name)
+            parents = self.target.parents(primary=True)
             if not parents:
                 raise DataJointError('A relation must have parent relations to be able to be populated')
-            ret = parents.pop(0)
+            self._populated_from = FreeRelation(self.connection, parents.pop(0)).proj()
             while parents:
-                ret *= parents.pop(0)
-            self._populated_from = ret
+                self._populated_from *= FreeRelation(self.connection, parents.pop(0)).proj()
         return self._populated_from
 
-    @abc.abstractmethod
     def _make_tuples(self, key):
         """
         Derived classes must implement method _make_tuples that fetches data from tables that are
         above them in the dependency hierarchy, restricting by the given key, computes dependent
         attributes, and inserts the new tuples into self.
         """
+        raise NotImplementedError('Subclasses of AutoPopulate must implement the method "_make_tuples"')
 
     @property
     def target(self):
@@ -58,10 +58,10 @@ def target(self):
 
     def populate(self, *restrictions, suppress_errors=False, reserve_jobs=False, order="original"):
         """
-        rel.populate() calls rel._make_tuples(key) for every primary key in self.populated_from
+        rel.populate() calls rel._make_tuples(key) for every primary key in self.key_source
         for which there is not already a tuple in rel.
 
-        :param restrictions: a list of restrictions each restrict (rel.populated_from - target.proj())
+        :param restrictions: a list of restrictions each restrict (rel.key_source - target.proj())
         :param suppress_errors: suppresses error if true
         :param reserve_jobs: if true, reserves job to populate in asynchronous fashion
         :param order: "original"|"reverse"|"random"  - the order of execution
@@ -73,10 +73,10 @@ def populate(self, *restrictions, suppress_errors=False, reserve_jobs=False, ord
         if order not in valid_order:
             raise DataJointError('The order argument must be one of %s' % str(valid_order))
 
-        todo = self.populated_from
+        todo = self.key_source
         if not isinstance(todo, RelationalOperand):
-            raise DataJointError('Invalid populated_from value')
-        todo.restrict(AndList(restrictions))
+            raise DataJointError('Invalid key_source value')
+        todo = todo & AndList(restrictions)
 
         error_list = [] if suppress_errors else None
 
@@ -104,7 +104,11 @@ def populate(self, *restrictions, suppress_errors=False, reserve_jobs=False, ord
                     try:
                         self._make_tuples(dict(key))
                     except Exception as error:
-                        self.connection.cancel_transaction()
+                        try:
+                            self.connection.cancel_transaction()
+                        except OperationalError:
+                            pass
+
                         if reserve_jobs:
                             jobs.error(self.target.table_name, key, error_message=str(error))
                         if not suppress_errors:
@@ -118,14 +122,14 @@ def populate(self, *restrictions, suppress_errors=False, reserve_jobs=False, ord
                             jobs.complete(self.target.table_name, key)
         return error_list
 
-    def progress(self, restriction=None, display=True):
+    def progress(self, *restrictions, display=True):
         """
         report progress of populating this table
         :return: remaining, total -- tuples to be populated
         """
-        todo = self.populated_from & restriction
+        todo = self.key_source & AndList(restrictions)
         total = len(todo)
-        remaining = len(todo - self.target.project())
+        remaining = len(todo - self.target.proj())
         if display:
             print('%-20s' % self.__class__.__name__, flush=True, end=': ')
             print('Completed %d of %d (%2.1f%%)   %s' %