From 1557a98be3d71cb2b79d0228dd3c128229d431a1 Mon Sep 17 00:00:00 2001
From: Dimitri Yatsenko <dimitri.yatsenko@gmail.com>
Date: Tue, 25 Aug 2015 14:03:16 -0500
Subject: [PATCH 01/12] minor cleanup

---
 datajoint/__init__.py       | 12 ++++++------
 datajoint/schema.py         | 12 ++++++------
 datajoint/user_relations.py | 13 +++++++------
 3 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/datajoint/__init__.py b/datajoint/__init__.py
index aea688382..0163fc343 100644
--- a/datajoint/__init__.py
+++ b/datajoint/__init__.py
@@ -16,17 +16,17 @@
 __all__ = ['__author__', '__version__',
            'config',
            'Connection', 'Heading', 'Relation', 'FreeRelation', 'Not',
-           'Relation', 'schema',
-           'Manual', 'Lookup', 'Imported', 'Computed', 'Part',
+           'schema',
+           'Relation', 'Manual', 'Lookup', 'Imported', 'Computed', 'Part',
            'conn', 'kill']
 
 
-# define an object that identifies the primary key in RelationalOperand.__getitem__
-class PrimaryKey:
+class key:
+    """
+    object that allows requesting the primary key in Fetch.__getitem__
+    """
     pass
 
-key = PrimaryKey
-
 
 class DataJointError(Exception):
     """
diff --git a/datajoint/schema.py b/datajoint/schema.py
index bdf985c61..8e0727a37 100644
--- a/datajoint/schema.py
+++ b/datajoint/schema.py
@@ -45,15 +45,15 @@ def __call__(self, cls):
         :param cls: class to be decorated
         """
 
-        def process_relation_class(class_object, context):
+        def process_relation_class(relation_class, context):
             """
             assign schema properties to the relation class and declare the table
             """
-            class_object.database = self.database
-            class_object._connection = self.connection
-            class_object._heading = Heading()
-            class_object._context = context
-            instance = class_object()
+            relation_class.database = self.database
+            relation_class._connection = self.connection
+            relation_class._heading = Heading()
+            relation_class._context = context
+            instance = relation_class()
             instance.heading  # trigger table declaration
             instance._prepare()
 
diff --git a/datajoint/user_relations.py b/datajoint/user_relations.py
index 2e140f62f..b8adb0f44 100644
--- a/datajoint/user_relations.py
+++ b/datajoint/user_relations.py
@@ -2,13 +2,14 @@
 Hosts the table tiers, user relations should be derived from.
 """
 
-from datajoint.relation import Relation
+import abc
+from .relation import Relation
 from .autopopulate import AutoPopulate
 from .utils import from_camel_case
 from . import DataJointError
 
 
-class Part(Relation):
+class Part(Relation, metaclass=abc.ABCMeta):
 
     @property
     def master(self):
@@ -22,7 +23,7 @@ def table_name(self):
         return self.master().table_name + '__' + from_camel_case(self.__class__.__name__)
 
 
-class Manual(Relation):
+class Manual(Relation, metaclass=abc.ABCMeta):
     """
     Inherit from this class if the table's values are entered manually.
     """
@@ -35,7 +36,7 @@ def table_name(self):
         return from_camel_case(self.__class__.__name__)
 
 
-class Lookup(Relation):
+class Lookup(Relation, metaclass=abc.ABCMeta):
     """
     Inherit from this class if the table's values are for lookup. This is
     currently equivalent to defining the table as Manual and serves semantic
@@ -57,7 +58,7 @@ def _prepare(self):
             self.insert(self.contents, ignore_errors=True)
 
 
-class Imported(Relation, AutoPopulate):
+class Imported(Relation, AutoPopulate, metaclass=abc.ABCMeta):
     """
     Inherit from this class if the table's values are imported from external data sources.
     The inherited class must at least provide the function `_make_tuples`.
@@ -71,7 +72,7 @@ def table_name(self):
         return "_" + from_camel_case(self.__class__.__name__)
 
 
-class Computed(Relation, AutoPopulate):
+class Computed(Relation, AutoPopulate, metaclass=abc.ABCMeta):
     """
     Inherit from this class if the table's values are computed from other relations in the schema.
     The inherited class must at least provide the function `_make_tuples`.

From fc4c0391795f7a81ca249896858798cc3bcd22e3 Mon Sep 17 00:00:00 2001
From: Dimitri Yatsenko <dimitri.yatsenko@gmail.com>
Date: Tue, 25 Aug 2015 14:06:08 -0500
Subject: [PATCH 02/12] minor cleanup

---
 datajoint/__init__.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/datajoint/__init__.py b/datajoint/__init__.py
index 0163fc343..47ce89b30 100644
--- a/datajoint/__init__.py
+++ b/datajoint/__init__.py
@@ -14,11 +14,9 @@
 __author__ = "Dimitri Yatsenko, Edgar Walker, and Fabian Sinz at Baylor College of Medicine"
 __version__ = "0.2"
 __all__ = ['__author__', '__version__',
-           'config',
-           'Connection', 'Heading', 'Relation', 'FreeRelation', 'Not',
-           'schema',
-           'Relation', 'Manual', 'Lookup', 'Imported', 'Computed', 'Part',
-           'conn', 'kill']
+           'config', 'conn', 'kill',
+           'Connection', 'Heading', 'Relation', 'FreeRelation', 'Not', 'schema',
+           'Manual', 'Lookup', 'Imported', 'Computed', 'Part']
 
 
 class key:

From 3f5fdee79d594c56b094e0bf6328f117ce5e672d Mon Sep 17 00:00:00 2001
From: Dimitri Yatsenko <dimitri.yatsenko@gmail.com>
Date: Tue, 25 Aug 2015 15:08:24 -0500
Subject: [PATCH 03/12] Now Relation._prepare() is called after all the Part
 relations are declared.

---
 datajoint/relation.py | 41 ++++++++++++++++++++++++-----------------
 datajoint/schema.py   | 12 +++++++++---
 2 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/datajoint/relation.py b/datajoint/relation.py
index c93826c3a..2b2133cf4 100644
--- a/datajoint/relation.py
+++ b/datajoint/relation.py
@@ -56,14 +56,20 @@ def heading(self):
         if self._heading is None:
             self._heading = Heading()  # instance-level heading
         if not self._heading:
-            if not self.is_declared:
-                self.connection.query(
-                    declare(self.full_table_name, self.definition, self._context))
-            if self.is_declared:
-                self.connection.erm.load_dependencies(self.full_table_name)
-                self._heading.init_from_database(self.connection, self.database, self.table_name)
+            self.declare()
         return self._heading
 
+    def declare(self):
+        """
+        load the table heading. If the table is not declared, use self.definition to declare
+        """
+        if not self.is_declared:
+            self.connection.query(
+                declare(self.full_table_name, self.definition, self._context))
+        if self.is_declared:
+            self.connection.erm.load_dependencies(self.full_table_name)
+            self._heading.init_from_database(self.connection, self.database, self.table_name)
+
     @property
     def from_clause(self):
         """
@@ -115,7 +121,6 @@ def descendants(self):
                      for table in self.connection.erm.get_descendants(self.full_table_name))
         return [relation for relation in relations if relation.is_declared]
 
-
     def _repr_helper(self):
         return "%s.%s()" % (self.__module__, self.__class__.__name__)
 
@@ -133,7 +138,7 @@ def full_table_name(self):
 
     def insert(self, rows, **kwargs):
         """
-        Inserts a collection of tuples. Additional keyword arguments are passed to insert1.
+        Insert a collection of tuples. Additional keyword arguments are passed to insert1.
 
         :param iter: Must be an iterator that generates a sequence of valid arguments for insert.
         """
@@ -154,19 +159,19 @@ def insert1(self, tup, replace=False, ignore_errors=False):
         heading = self.heading
 
         if isinstance(tup, np.void):    # np.array insert
-            for fieldname in tup.dtype.fields:
-                if fieldname not in heading:
-                    raise KeyError(u'{0:s} is not in the attribute list'.format(fieldname))
+            for field in tup.dtype.fields:
+                if field not in heading:
+                    raise KeyError(u'{0:s} is not in the attribute list'.format(field))
             value_list = ','.join([repr(tup[name]) if not heading[name].is_blob else '%s'
                                    for name in heading if name in tup.dtype.fields])
             args = tuple(pack(tup[name]) for name in heading
                          if name in tup.dtype.fields and heading[name].is_blob)
             attribute_list = '`' + '`,`'.join(q for q in heading if q in tup.dtype.fields) + '`'
 
-        elif isinstance(tup, Mapping):   #  dict-based insert
-            for fieldname in tup.keys():
-                if fieldname not in heading:
-                    raise KeyError(u'{0:s} is not in the attribute list'.format(fieldname))
+        elif isinstance(tup, Mapping):  # dict-based insert
+            for field in tup.keys():
+                if field not in heading:
+                    raise KeyError(u'{0:s} is not in the attribute list'.format(field))
             value_list = ','.join(repr(tup[name]) if not heading[name].is_blob else '%s'
                                   for name in heading if name in tup)
             args = tuple(pack(tup[name]) for name in heading
@@ -175,9 +180,11 @@ def insert1(self, tup, replace=False, ignore_errors=False):
 
         else:    # positional insert
             try:
-                if len(tup) != len(self.heading):
+                if len(tup) != len(heading):
                     raise DataJointError(
-                        'Tuple size does not match the number of relation attributes')
+                        'Incorrect number of attributes: '
+                        '{given} given; {expected} expected'.format(
+                            given=len(tup), expected=len(heading)))
             except TypeError:
                 raise DataJointError('Datatype %s cannot be inserted' % type(tup))
             else:
diff --git a/datajoint/schema.py b/datajoint/schema.py
index 8e0727a37..2d82aa25c 100644
--- a/datajoint/schema.py
+++ b/datajoint/schema.py
@@ -53,9 +53,7 @@ def process_relation_class(relation_class, context):
             relation_class._connection = self.connection
             relation_class._heading = Heading()
             relation_class._context = context
-            instance = relation_class()
-            instance.heading  # trigger table declaration
-            instance._prepare()
+            relation_class().declare()
 
         if issubclass(cls, Part):
             raise DataJointError('The schema decorator should not apply to part relations')
@@ -63,6 +61,7 @@ def process_relation_class(relation_class, context):
         process_relation_class(cls, context=self.context)
 
         #  Process subordinate relations
+        parts = list()
         for name in (name for name in dir(cls) if not name.startswith('_')):
             part = getattr(cls, name)
             try:
@@ -71,10 +70,17 @@ def process_relation_class(relation_class, context):
                 pass
             else:
                 if is_sub:
+                    parts.append(part)
                     part._master = cls
                     process_relation_class(part, context=dict(self.context, **{cls.__name__: cls}))
                 elif issubclass(part, Relation):
                     raise DataJointError('Part relations must subclass from datajoint.Part')
+
+        # invoke _prepare()
+        cls()._prepare()
+        for part in parts:
+            part()._prepare()
+
         return cls
 
     @property

From 4c046dee85b40c40f8fcfce1fa2c1f71b2a37f9e Mon Sep 17 00:00:00 2001
From: Dimitri Yatsenko <dimitri.yatsenko@gmail.com>
Date: Thu, 27 Aug 2015 10:54:53 -0500
Subject: [PATCH 04/12] added the ability to control the order of _make_tuples
 calls from AutoPopulate.populate

---
 datajoint/autopopulate.py | 21 ++++++++++++++++++---
 datajoint/kill.py         |  2 +-
 datajoint/relation.py     | 16 +++++++++++-----
 datajoint/schema.py       |  2 +-
 4 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/datajoint/autopopulate.py b/datajoint/autopopulate.py
index 7d2b43f95..4ac656cc3 100644
--- a/datajoint/autopopulate.py
+++ b/datajoint/autopopulate.py
@@ -2,6 +2,7 @@
 import abc
 import logging
 import datetime
+import random
 from .relational_operand import RelationalOperand
 from . import DataJointError
 from .relation import FreeRelation
@@ -52,7 +53,8 @@ def target(self):
         """
         return self
 
-    def populate(self, restriction=None, suppress_errors=False, reserve_jobs=False):
+    def populate(self, restriction=None, suppress_errors=False,
+                 reserve_jobs=False, order="original"):
         """
         rel.populate() calls rel._make_tuples(key) for every primary key in self.populated_from
         for which there is not already a tuple in rel.
@@ -61,18 +63,31 @@ def populate(self, restriction=None, suppress_errors=False, reserve_jobs=False):
         :param suppress_errors: suppresses error if true
         :param reserve_jobs: currently not implemented
         :param batch: batch size of a single job
+        :param order: "original"|"reverse"|"random"  - the order of execution
         """
-        error_list = [] if suppress_errors else None
         if not isinstance(self.populated_from, RelationalOperand):
             raise DataJointError('Invalid populated_from value')
 
         if self.connection.in_transaction:
             raise DataJointError('Populate cannot be called during a transaction.')
 
+        valid_order = ['original', 'reverse', 'random']
+        if order not in valid_order:
+            raise DataJointError('The order argument must be one of %s' % str(valid_order))
+
+        error_list = [] if suppress_errors else None
+
         jobs = self.connection.jobs[self.target.database]
         table_name = self.target.table_name
         unpopulated = (self.populated_from & restriction) - self.target.project()
-        for key in unpopulated.fetch.keys():
+        keys = unpopulated.fetch.keys()
+        if order == "reverse":
+            keys = list(keys).reverse()
+        elif order == "random":
+            keys = list(keys)
+            random.shuffle(keys)
+
+        for key in keys:
             if not reserve_jobs or jobs.reserve(table_name, key):
                 self.connection.start_transaction()
                 if key in self.target:  # already populated
diff --git a/datajoint/kill.py b/datajoint/kill.py
index 82c9ac38d..b4c0c526c 100644
--- a/datajoint/kill.py
+++ b/datajoint/kill.py
@@ -32,7 +32,7 @@ def kill(restriction=None, connection=None):
             except TypeError as err:
                 print(process)
 
-        response = input('process to kill or "q" to quit)')
+        response = input('process to kill or "q" to quit > ')
         if response == 'q':
             break
         if response:
diff --git a/datajoint/relation.py b/datajoint/relation.py
index 2b2133cf4..80eb570f7 100644
--- a/datajoint/relation.py
+++ b/datajoint/relation.py
@@ -229,7 +229,8 @@ def delete(self):
                 relations[dep] &= r.project() if name in restrict_by_me else r.restrictions
 
         do_delete = False  # indicate if there is anything to delete
-        print('The contents of the following tables are about to be deleted:')
+        if config['safemode']:
+            print('The contents of the following tables are about to be deleted:')
         for relation in relations.values():
             count = len(relation)
             if count:
@@ -238,10 +239,15 @@ def delete(self):
                     print(relation.full_table_name, '(%d tuples)' % count)
             else:
                 relations.pop(relation.full_table_name)
-        if do_delete and (not config['safemode'] or user_choice("Proceed?", default='no') == 'yes'):
-            with self.connection.transaction:
-                for r in reversed(list(relations.values())):
-                    r.delete_quick()
+        if not do_delete:
+            if config['safemode']:
+                print('Nothing to delete')
+        else:
+            if not config['safemode'] or user_choice("Proceed?", default='no') == 'yes':
+                with self.connection.transaction:
+                    for r in reversed(list(relations.values())):
+                        r.delete_quick()
+                print('Done')
 
     def drop_quick(self):
         """
diff --git a/datajoint/schema.py b/datajoint/schema.py
index 2d82aa25c..a96f1678e 100644
--- a/datajoint/schema.py
+++ b/datajoint/schema.py
@@ -76,7 +76,7 @@ def process_relation_class(relation_class, context):
                 elif issubclass(part, Relation):
                     raise DataJointError('Part relations must subclass from datajoint.Part')
 
-        # invoke _prepare()
+        # invoke Relation._prepare() on class and its part relations.
         cls()._prepare()
         for part in parts:
             part()._prepare()

From c8e0474253cb509db77febc0c7af129ba42e7fb4 Mon Sep 17 00:00:00 2001
From: Dimitri Yatsenko <dimitri@dimitris-mbp-2.ad.bcm.edu>
Date: Thu, 27 Aug 2015 12:58:16 -0500
Subject: [PATCH 05/12] minor

---
 datajoint/relation.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/datajoint/relation.py b/datajoint/relation.py
index 097e6c15d..5442e4d44 100644
--- a/datajoint/relation.py
+++ b/datajoint/relation.py
@@ -274,8 +274,7 @@ def size_on_disk(self):
         """
         ret = self.connection.query(
             'SHOW TABLE STATUS FROM `{database}` WHERE NAME="{table}"'.format(
-                database=self.database, table=self.table_name), as_dict=True
-        ).fetchone()
+                database=self.database, table=self.table_name), as_dict=True).fetchone()
         return ret['Data_length'] + ret['Index_length']
 
     # --------- functionality used by the decorator ---------

From ef7296645de8f7d8c8b1b818ba211e21842e9830 Mon Sep 17 00:00:00 2001
From: Fabian Sinz <sinz@bcm.edu>
Date: Fri, 28 Aug 2015 18:17:21 -0500
Subject: [PATCH 06/12] bugfix

---
 datajoint/relation.py  | 6 +++---
 tests/schema.py        | 9 ++++++++-
 tests/test_relation.py | 8 ++++++++
 3 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/datajoint/relation.py b/datajoint/relation.py
index 097e6c15d..bc01932b4 100644
--- a/datajoint/relation.py
+++ b/datajoint/relation.py
@@ -178,10 +178,9 @@ def insert1(self, tup, replace=False, ignore_errors=False, skip_duplicates=False
             except TypeError:
                 raise DataJointError('Datatype %s cannot be inserted' % type(tup))
             else:
-                pairs = zip(heading, tup)
-                values = ['%s' if heading[name].is_blob else value for name, value in pairs]
+                values = ['%s' if heading[name].is_blob else value for name, value in zip(heading, tup)]
                 attributes = heading.names
-                args = tuple(pack(value) for name, value in pairs if heading[name].is_blob)
+                args = tuple(pack(value) for name, value in zip(heading, tup) if heading[name].is_blob)
 
         value_list = ','.join(map(lambda elem: repr(elem) if elem != '%s' else elem , values))
         attribute_list = '`' + '`,`'.join(attributes) + '`'
@@ -198,6 +197,7 @@ def insert1(self, tup, replace=False, ignore_errors=False, skip_duplicates=False
             logger.info(sql)
             self.connection.query(sql, args=args)
 
+
     def delete_quick(self):
         """
         delete without cascading and without user prompt
diff --git a/tests/schema.py b/tests/schema.py
index 564b0fcd4..0eb6bfb63 100644
--- a/tests/schema.py
+++ b/tests/schema.py
@@ -160,4 +160,11 @@ def _make_tuples(self, key):
                      channel=channel,
                      voltage=np.float32(np.random.randn(number_samples))))
 
-
+@schema
+class Image(dj.Manual):
+    definition = """
+    # table for testing blob inserts
+    id           : int # image identifier
+    ---
+    img             : longblob # image
+    """
\ No newline at end of file
diff --git a/tests/test_relation.py b/tests/test_relation.py
index decac5e3d..9a619936c 100644
--- a/tests/test_relation.py
+++ b/tests/test_relation.py
@@ -20,6 +20,7 @@ def __init__(self):
         self.trial = schema.Trial()
         self.ephys = schema.Ephys()
         self.channel = schema.Ephys.Channel()
+        self.img = schema.Image()
 
     def test_contents(self):
         """
@@ -68,3 +69,10 @@ def test_not_skip_duplicate(self):
             (1, 'Peter', 'mouse', '2015-01-01', '')],
             dtype=self.subject.heading.as_dtype)
         self.subject.insert(tmp, skip_duplicates=False)
+
+
+    def test_blob_insert(self):
+        X = np.random.randn(20,10)
+        self.img.insert1((1,X))
+        Y = self.img.fetch()[0]['img']
+        assert_true(np.all(X == Y), 'Inserted and retrieved image are not identical')
\ No newline at end of file

From eb2fed3fe722662b01e37f4eeebe6f1ff315c2c5 Mon Sep 17 00:00:00 2001
From: Fabian Sinz <fabee@epagoge.de>
Date: Fri, 28 Aug 2015 23:06:55 -0500
Subject: [PATCH 07/12] test_relation 100%

---
 datajoint/relation.py  |  1 -
 tests/__init__.py      |  1 -
 tests/schema.py        | 24 +++++++++++++++++++++++-
 tests/test_relation.py | 23 ++++++++++++++++++-----
 4 files changed, 41 insertions(+), 8 deletions(-)

diff --git a/datajoint/relation.py b/datajoint/relation.py
index 1ee7dc725..08ec6b133 100644
--- a/datajoint/relation.py
+++ b/datajoint/relation.py
@@ -271,7 +271,6 @@ def drop(self):
         do_drop = True
         relations = self.descendants
         if config['safemode']:
-            print('The following tables are about to be dropped:')
             for relation in relations:
                 print(relation.full_table_name, '(%d tuples)' % len(relation))
             do_drop = user_choice("Proceed?", default='no') == 'yes'
diff --git a/tests/__init__.py b/tests/__init__.py
index 2bf44d4cf..149175c82 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -24,7 +24,6 @@
 # Prefix for all databases used during testing
 PREFIX = environ.get('DJ_TEST_DB_PREFIX', 'djtest')
 
-
 def setup_package():
     """
     Package-level unit test setup
diff --git a/tests/schema.py b/tests/schema.py
index 0eb6bfb63..f5f680338 100644
--- a/tests/schema.py
+++ b/tests/schema.py
@@ -167,4 +167,26 @@ class Image(dj.Manual):
     id           : int # image identifier
     ---
     img             : longblob # image
-    """
\ No newline at end of file
+    """
+
+
+@schema
+class UberTrash(dj.Manual):
+    definition = """
+    id : int
+    ---
+    """
+
+
+@schema
+class UnterTrash(dj.Manual):
+    definition = """
+    -> UberTrash
+    my_id   : int
+    ---
+    """
+
+    def _prepare(self):
+        UberTrash().insert1((1,), skip_duplicates=True)
+        self.insert1((1, 1), skip_duplicates=True)
+        self.insert1((1, 2), skip_duplicates=True)
diff --git a/tests/test_relation.py b/tests/test_relation.py
index 9a619936c..c7c357e23 100644
--- a/tests/test_relation.py
+++ b/tests/test_relation.py
@@ -5,7 +5,10 @@
     assert_tuple_equal, assert_dict_equal, raises
 
 from . import schema
-from pymysql import IntegrityError
+from pymysql import IntegrityError, ProgrammingError
+import datajoint as dj
+from datajoint import utils
+from unittest.mock import patch
 
 
 class TestRelation:
@@ -21,6 +24,7 @@ def __init__(self):
         self.ephys = schema.Ephys()
         self.channel = schema.Ephys.Channel()
         self.img = schema.Image()
+        self.trash = schema.UberTrash()
 
     def test_contents(self):
         """
@@ -70,9 +74,18 @@ def test_not_skip_duplicate(self):
             dtype=self.subject.heading.as_dtype)
         self.subject.insert(tmp, skip_duplicates=False)
 
-
     def test_blob_insert(self):
-        X = np.random.randn(20,10)
-        self.img.insert1((1,X))
+        """Tests inserting and retrieving blobs."""
+        X = np.random.randn(20, 10)
+        self.img.insert1((1, X))
         Y = self.img.fetch()[0]['img']
-        assert_true(np.all(X == Y), 'Inserted and retrieved image are not identical')
\ No newline at end of file
+        assert_true(np.all(X == Y), 'Inserted and retrieved image are not identical')
+
+    @raises(ProgrammingError)
+    def test_drop(self):
+        """Tests dropping tables"""
+        dj.config['safemode'] = True
+        with patch.object(utils, "input", create=True, return_value='yes'):
+            self.trash.drop()
+        dj.config['safemode'] = False
+        self.trash.fetch()

From 0a4555213ec5f1d78a6c17dcf868b9cb624ea4b5 Mon Sep 17 00:00:00 2001
From: Fabian Sinz <fabee@epagoge.de>
Date: Sat, 29 Aug 2015 16:03:59 -0500
Subject: [PATCH 08/12] new tests for fetch

---
 tests/test_fetch.py | 46 +++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 44 insertions(+), 2 deletions(-)

diff --git a/tests/test_fetch.py b/tests/test_fetch.py
index 4accfc679..0f010b993 100644
--- a/tests/test_fetch.py
+++ b/tests/test_fetch.py
@@ -1,9 +1,9 @@
 from operator import itemgetter, attrgetter
 import itertools
-from nose.tools import assert_true
+from nose.tools import assert_true, raises
 from numpy.testing import assert_array_equal, assert_equal
 import numpy as np
-
+import warnings
 from . import schema
 import datajoint as dj
 
@@ -135,3 +135,45 @@ def test_repr(self):
         # 3 lines are used for headers (2) and summary statement (1)
         assert_true(n - 3 <= limit)
 
+    @raises(dj.DataJointError)
+    def test_prepare_attributes(self):
+        """Test preparing attributes for getitem"""
+        self.lang.fetch[None]
+
+    def test_asdict(self):
+        """Test returns as dictionaries"""
+        d = self.lang.fetch.as_dict()
+        for dd in d:
+            assert_true(isinstance(dd, dict))
+
+    def test_asdict_with_call(self):
+        """Test returns as dictionaries with call."""
+        d = self.lang.fetch.as_dict()()
+        for dd in d:
+            assert_true(isinstance(dd, dict))
+
+    def test_offset(self):
+        """Tests offset"""
+        cur = self.lang.fetch.limit(4).offset(1)(order_by=['language', 'name DESC'])
+        langs = self.lang.contents
+        langs.sort(key=itemgetter(0), reverse=True)
+        langs.sort(key=itemgetter(1), reverse=False)
+        assert_equal(len(cur), 4, 'Length is not correct')
+        for c, l in list(zip(cur, langs[1:]))[:4]:
+            assert_true(np.all([cc == ll for cc, ll in zip(c, l)]), 'Sorting order is different')
+
+
+    def test_limit_warning(self):
+        """Tests whether warning is raised if offset is used without limit."""
+        with warnings.catch_warnings(record=True) as w:
+            self.lang.fetch.offset(1)()
+            assert_true(len(w) > 0, "Warning war not raised")
+
+    def test_len(self):
+        """Tests __len__"""
+        assert_true(len(self.lang.fetch) == len(self.lang),'__len__ is not behaving properly')
+
+    @raises(dj.DataJointError)
+    def test_fetch1(self):
+        """Tests whether fetch1 raises error"""
+        self.lang.fetch1()
\ No newline at end of file

From 11fa2dda39952a8cb040f18e30500ae2e93bb840 Mon Sep 17 00:00:00 2001
From: Fabian Sinz <fabee@epagoge.de>
Date: Sat, 29 Aug 2015 16:19:55 -0500
Subject: [PATCH 09/12] utils documented

---
 datajoint/utils.py   | 22 +++++++++++++++-------
 doc/source/index.rst |  2 ++
 doc/source/utils.rst |  5 +++++
 3 files changed, 22 insertions(+), 7 deletions(-)
 create mode 100644 doc/source/utils.rst

diff --git a/datajoint/utils.py b/datajoint/utils.py
index 1d5a8e5a6..dcf0cb5a7 100644
--- a/datajoint/utils.py
+++ b/datajoint/utils.py
@@ -6,7 +6,8 @@
 def user_choice(prompt, choices=("yes", "no"), default=None):
     """
     Prompts the user for confirmation.  The default value, if any, is capitalized.
-    :parsam prompt: Information to display to the user.
+
+    :param prompt: Information to display to the user.
     :param choices: an iterable of possible choices.
     :param default: default choice
     :return: the user's choice
@@ -36,11 +37,15 @@ def group_by(rel, *attributes, sortby=None):
 
 def to_camel_case(s):
     """
-    Convert names with under score (_) separation
-    into camel case names.
+    Convert names with under score (_) separation into camel case names.
+
+    :param s: string in under_score notation
+    :returns: string in CamelCase notation
+
     Example:
-    >>> to_camel_case("table_name")
-        "TableName"
+
+    >>> to_camel_case("table_name") # yields "TableName"
+
     """
 
     def to_upper(match):
@@ -53,9 +58,12 @@ def from_camel_case(s):
     """
     Convert names in camel case into underscore (_) separated names
 
+    :param s: string in CamelCase notation
+    :returns: string in under_score notation
+
     Example:
-    >>> from_camel_case("TableName")
-        "table_name"
+
+    >>> from_camel_case("TableName") # yields "table_name"
     """
 
     def convert(match):
diff --git a/doc/source/index.rst b/doc/source/index.rst
index d70f80705..5f6c389b3 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -13,9 +13,11 @@ API:
 
    tiers.rst
    relation.rst
+   relational_operand.rst
    connection.rst
    blob.rst
    declare.rst
+   utils.rst
 
 Indices and tables
 ==================
diff --git a/doc/source/utils.rst b/doc/source/utils.rst
new file mode 100644
index 000000000..fee986ed6
--- /dev/null
+++ b/doc/source/utils.rst
@@ -0,0 +1,5 @@
+Utils
+=====
+
+.. automodule:: datajoint.utils
+    :members:

From 9357bd8d13703fe336baf3d24673f8caa45a330d Mon Sep 17 00:00:00 2001
From: Fabian Sinz <fabee@epagoge.de>
Date: Sat, 29 Aug 2015 19:13:21 -0500
Subject: [PATCH 10/12] fetch documented

---
 datajoint/fetch.py   | 83 +++++++++++++++++++++++++++++++++++++++-----
 doc/source/fetch.rst |  5 +++
 doc/source/index.rst |  1 +
 tests/test_fetch.py  |  2 +-
 4 files changed, 81 insertions(+), 10 deletions(-)
 create mode 100644 doc/source/fetch.rst

diff --git a/datajoint/fetch.py b/datajoint/fetch.py
index 5d7a079b5..955116892 100644
--- a/datajoint/fetch.py
+++ b/datajoint/fetch.py
@@ -9,6 +9,15 @@
 
 
 def prepare_attributes(relation, item):
+    """
+    Used by fetch.__getitem__ to deal with slices
+
+    :param relation: the relation that created the fetch object
+    :param item: the item passed to __getitem__. Can be a string, a tuple, a list, or a slice.
+
+    :return: a tuple of items to fetch, a list of the corresponding attributes
+    :raise DataJointError: if item does not match one of the datatypes above
+    """
     if isinstance(item, str) or item is PRIMARY_KEY:
         item = (item,)
     elif isinstance(item, int):
@@ -27,7 +36,7 @@ def prepare_attributes(relation, item):
 
 def copy_first(f):
     """
-    decorates methods that return an altered copy of self
+    Decorates methods that return an altered copy of self
     """
     @wraps(f)
     def ret(*args, **kwargs):
@@ -39,6 +48,11 @@ def ret(*args, **kwargs):
 
 
 class Fetch:
+    """
+    A fetch object that handles retrieving elements from the database table.
+
+    :param relation: relation the fetch object retrieves data from.
+    """
 
     def __init__(self, relation):
         if isinstance(relation, Fetch):  # copy constructor
@@ -52,22 +66,57 @@ def __init__(self, relation):
 
     @copy_first
     def order_by(self, *args):
+        """
+        Changes the state of the fetch object to order the results by a particular attribute.
+        The commands are handed down to mysql.
+
+        :param args: the attributes to sort by. If DESC is passed after the name, then the order is descending.
+        :return: a copy of the fetch object
+
+        Example:
+
+        >>> my_relation.fetch.order_by('language', 'name DESC')
+
+        """
         if len(args) > 0:
             self.behavior['order_by'] = args
         return self
 
+    @property
     @copy_first
     def as_dict(self):
+        """
+        Changes the state of the fetch object to return dictionaries.
+
+        :return: a copy of the fetch object
+
+        Example:
+
+        >>> my_relation.fetch.as_dict()
+
+        """
         self.behavior['as_dict'] = True
         return self
 
     @copy_first
     def limit(self, limit):
+        """
+        Limits the number of items fetched.
+
+        :param limit: limit on the number of items
+        :return: a copy of the fetch object
+        """
         self.behavior['limit'] = limit
         return self
 
     @copy_first
     def offset(self, offset):
+        """
+        Offsets the number of itms fetched. Needs to be applied with limit.
+
+        :param offset: offset
+        :return: a copy of the fetch object
+        """
         if self.behavior['limit'] is None:
             warnings.warn('You should supply a limit together with an offset,')
         self.behavior['offset'] = offset
@@ -75,6 +124,12 @@ def offset(self, offset):
 
     @copy_first
     def set_behavior(self, **kwargs):
+        """
+        Sets the behavior like offset, limit, or order_by via keywords arguments.
+
+        :param kwargs:  keyword arguments
+        :return: a copy of the fetch object
+        """
         self.behavior.update(kwargs)
         return self
 
@@ -146,10 +201,11 @@ def __getitem__(self, item):
         :return: tuple with an entry for each element of item
 
         Examples:
-        a, b = relation['a', 'b']
-        a, b, key = relation['a', 'b', datajoint.key]
-        results = relation['a':'z']    # return attributes a-z as a tuple
-        results = relation[:-1]   # return all but the last attribute
+
+        >>> a, b = relation['a', 'b']
+        >>> a, b, key = relation['a', 'b', datajoint.key]
+        >>> results = relation['a':'z']    # return attributes a-z as a tuple
+        >>> results = relation[:-1]   # return all but the last attribute
         """
         single_output = isinstance(item, str) or item is PRIMARY_KEY or isinstance(item, int)
         item, attributes = prepare_attributes(self._relation, item)
@@ -185,6 +241,11 @@ def __len__(self):
 
 
 class Fetch1:
+    """
+    Fetch object for fetching exactly one row.
+
+    :param relation: relation the fetch object fetches data from
+    """
 
     def __init__(self, relation):
         self._relation = relation
@@ -192,6 +253,7 @@ def __init__(self, relation):
     def __call__(self):
         """
         This version of fetch is called when self is expected to contain exactly one tuple.
+
         :return: the one tuple in the relation in the form of a dict
         """
         heading = self._relation.heading
@@ -208,13 +270,16 @@ def __getitem__(self, item):
         """
         Fetch attributes as separate outputs.
         datajoint.key is a special value that requests the entire primary key
+
         :return: tuple with an entry for each element of item
 
         Examples:
-        a, b = relation['a', 'b']
-        a, b, key = relation['a', 'b', datajoint.key]
-        results = relation['a':'z']    # return attributes a-z as a tuple
-        results = relation[:-1]   # return all but the last attribute
+
+        >>> a, b = relation['a', 'b']
+        >>> a, b, key = relation['a', 'b', datajoint.key]
+        >>> results = relation['a':'z']    # return attributes a-z as a tuple
+        >>> results = relation[:-1]   # return all but the last attribute
+
         """
         single_output = isinstance(item, str) or item is PRIMARY_KEY or isinstance(item, int)
         item, attributes = prepare_attributes(self._relation, item)
diff --git a/doc/source/fetch.rst b/doc/source/fetch.rst
new file mode 100644
index 000000000..b9d6b3cdf
--- /dev/null
+++ b/doc/source/fetch.rst
@@ -0,0 +1,5 @@
+Fetch objects
+=============
+
+.. automodule:: datajoint.fetch
+    :members:
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 5f6c389b3..de5adae87 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -18,6 +18,7 @@ API:
    blob.rst
    declare.rst
    utils.rst
+   fetch.rst
 
 Indices and tables
 ==================
diff --git a/tests/test_fetch.py b/tests/test_fetch.py
index 0f010b993..744eec369 100644
--- a/tests/test_fetch.py
+++ b/tests/test_fetch.py
@@ -148,7 +148,7 @@ def test_asdict(self):
 
     def test_asdict_with_call(self):
         """Test returns as dictionaries with call."""
-        d = self.lang.fetch.as_dict()()
+        d = self.lang.fetch.as_dict()
         for dd in d:
             assert_true(isinstance(dd, dict))
 

From e9e9f88b72e78bae1cefc4e63b9d65bac75aa5ab Mon Sep 17 00:00:00 2001
From: Fabian Sinz <fabee@epagoge.de>
Date: Sat, 29 Aug 2015 19:46:41 -0500
Subject: [PATCH 11/12] Fabian documented and tested his share

---
 datajoint/relation.py       | 62 +++++++++++++++++++++++++++++--------
 datajoint/settings.py       |  3 ++
 datajoint/user_relations.py |  6 ++++
 doc/source/index.rst        |  1 +
 doc/source/settings.rst     |  5 +++
 5 files changed, 64 insertions(+), 13 deletions(-)
 create mode 100644 doc/source/settings.rst

diff --git a/datajoint/relation.py b/datajoint/relation.py
index 08ec6b133..1129de7f0 100644
--- a/datajoint/relation.py
+++ b/datajoint/relation.py
@@ -44,14 +44,17 @@ def definition(self):
     # -------------- required by RelationalOperand ----------------- #
     @property
     def connection(self):
+        """
+        :return: the connection object of the relation
+        """
         return self._connection
 
     @property
     def heading(self):
         """
-        Get the table heading.
-        If the table is not declared, attempts to declare it and return heading.
-        :return:
+        Returns the table heading. If the table is not declared, attempts to declare it and return heading.
+
+        :return: table heading
         """
         if self._heading is None:
             self._heading = Heading()  # instance-level heading
@@ -61,7 +64,7 @@ def heading(self):
 
     def declare(self):
         """
-        load the table heading. If the table is not declared, use self.definition to declare
+        Loads the table heading. If the table is not declared, use self.definition to declare
         """
         if not self.is_declared:
             self.connection.query(
@@ -79,9 +82,15 @@ def from_clause(self):
 
     @property
     def select_fields(self):
+        """
+        :return: the selected attributes from the SQL SELECT statement.
+        """
         return '*'
 
     def erd(self, *args, **kwargs):
+        """
+        :return: the entity relationship diagram object of this relation
+        """
         erd = self.connection.erd()
         nodes = erd.up_down_neighbors(self.full_table_name)
         return erd.restrict_by_tables(nodes)
@@ -89,10 +98,16 @@ def erd(self, *args, **kwargs):
     # ------------- dependencies ---------- #
     @property
     def parents(self):
+        """
+        :return: the parent relation of this relation
+        """
         return self.connection.erm.parents[self.full_table_name]
 
     @property
     def children(self):
+        """
+        :return: the child relations of this relation
+        """
         return self.connection.erm.children[self.full_table_name]
 
     @property
@@ -112,10 +127,11 @@ def referenced(self):
     @property
     def descendants(self):
         """
-        :return: list of relation objects for all children and references, recursively,
-        in order of dependence.
-        Does not include self.
+        Returns a list of relation objects for all children and references, recursively,
+        in order of dependence. The returned values do not include self.
         This is helpful for cascading delete or drop operations.
+
+        :return: list of descendants
         """
         relations = (FreeRelation(self.connection, table)
                      for table in self.connection.erm.get_descendants(self.full_table_name))
@@ -127,6 +143,9 @@ def _repr_helper(self):
     # --------- SQL functionality --------- #
     @property
     def is_declared(self):
+        """
+        :return: True is the table is declared
+        """
         cur = self.connection.query(
             'SHOW TABLES in `{database}`LIKE "{table_name}"'.format(
                 database=self.database, table_name=self.table_name))
@@ -134,11 +153,14 @@ def is_declared(self):
 
     @property
     def full_table_name(self):
+        """
+        :return: full table name in the database
+        """
         return r"`{0:s}`.`{1:s}`".format(self.database, self.table_name)
 
     def insert(self, rows, **kwargs):
         """
-        Insert a collection of tuples. Additional keyword arguments are passed to insert1.
+        Insert a collection of rows. Additional keyword arguments are passed to insert1.
 
         :param iter: Must be an iterator that generates a sequence of valid arguments for insert.
         """
@@ -155,7 +177,9 @@ def insert1(self, tup, replace=False, ignore_errors=False, skip_duplicates=False
         :param skip_dublicates=False: If True, ignore duplicate inserts.
 
         Example::
-            relation.insert1(dict(subject_id=7, species="mouse", date_of_birth="2014-09-01"))
+
+        >>> relation.insert1(dict(subject_id=7, species="mouse", date_of_birth="2014-09-01"))
+
         """
         heading = self.heading
 
@@ -207,14 +231,15 @@ def insert1(self, tup, replace=False, ignore_errors=False, skip_duplicates=False
 
     def delete_quick(self):
         """
-        delete without cascading and without user prompt
+        Deletes the table without cascading and without user prompt.
         """
         self.connection.query('DELETE FROM ' + self.from_clause + self.where_clause)
 
     def delete(self):
         """
-        Delete the contents of the table and its dependent tables, recursively.
-        User is prompted for confirmation if config['safemode']
+        Deletes the contents of the table and its dependent tables, recursively.
+        User is prompted for confirmation if config['safemode'] is set to True.
+
         """
         relations = self.descendants
         restrict_by_me = set()
@@ -266,7 +291,7 @@ def drop_quick(self):
     def drop(self):
         """
         Drop the table and all tables that reference it, recursively.
-        User is prompted for confirmation if config['safemode']
+        User is prompted for confirmation if config['safemode'] is set to True.
         """
         do_drop = True
         relations = self.descendants
@@ -313,12 +338,23 @@ def __repr__(self):
 
     @property
     def definition(self):
+        """
+        Definition of the table.
+
+        :return: the definition
+        """
         return self._definition
 
     @property
     def connection(self):
+        """
+        :return: the connection object of the relation.
+        """
         return self._connection
 
     @property
     def table_name(self):
+        """
+        :return: the table name in the database
+        """
         return self._table_name
diff --git a/datajoint/settings.py b/datajoint/settings.py
index 7dac8186a..c3d022b17 100644
--- a/datajoint/settings.py
+++ b/datajoint/settings.py
@@ -54,6 +54,9 @@
 
 
 class Borg:
+    """
+    Class to implement an de facto singleton.
+    """
     _shared_state = {}
 
     def __init__(self):
diff --git a/datajoint/user_relations.py b/datajoint/user_relations.py
index 0d9a46544..8ffcd2e2f 100644
--- a/datajoint/user_relations.py
+++ b/datajoint/user_relations.py
@@ -10,6 +10,12 @@
 
 
 class Part(Relation, metaclass=abc.ABCMeta):
+    """
+    Inherit from this class if the table's values are details of an entry in another relation
+    and if this table is populated by this relation. For example, the entries inheriting from
+    dj.Part could be single entries of a matrix, while the parent table refers to the entire matrix.
+    Part relations are implemented as classes inside classes.
+    """
 
     @property
     def master(self):
diff --git a/doc/source/index.rst b/doc/source/index.rst
index de5adae87..bdc002df7 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -19,6 +19,7 @@ API:
    declare.rst
    utils.rst
    fetch.rst
+   settings.rst
 
 Indices and tables
 ==================
diff --git a/doc/source/settings.rst b/doc/source/settings.rst
new file mode 100644
index 000000000..a1be30413
--- /dev/null
+++ b/doc/source/settings.rst
@@ -0,0 +1,5 @@
+Config
+======
+
+.. automodule:: datajoint.settings
+    :members:

From 3d38a9c31d01ec1b0be30bdc12089af9736b886b Mon Sep 17 00:00:00 2001
From: Fabian Sinz <fabee@epagoge.de>
Date: Sat, 29 Aug 2015 20:03:09 -0500
Subject: [PATCH 12/12] ok, a few fetch tests were missing

---
 tests/test_fetch.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/test_fetch.py b/tests/test_fetch.py
index 744eec369..bac9eda77 100644
--- a/tests/test_fetch.py
+++ b/tests/test_fetch.py
@@ -11,6 +11,7 @@
 class TestFetch:
     def __init__(self):
         self.subject = schema.Subject()
+
         self.lang = schema.Language()
 
     def test_getitem(self):
@@ -35,6 +36,13 @@ def test_getitem(self):
         for column, field in zip(self.subject.fetch['subject_id'::2], [e[0] for e in tmp.dtype.descr][::2]):
             np.testing.assert_array_equal(sorted(tmp[field]), sorted(column), 'slice : does not work correctly')
 
+    def test_getitem_for_fetch1(self):
+        """Testing Fetch1.__getitem__"""
+        assert_true( (self.subject & "subject_id=10").fetch1['subject_id'] == 10)
+        assert_true( (self.subject & "subject_id=10").fetch1['subject_id','species'] == (10, 'monkey'))
+        assert_true( (self.subject & "subject_id=10").fetch1['subject_id':'species'] == (10, 'Curious George'))
+
+
     def test_order_by(self):
         """Tests order_by sorting order"""
         langs = schema.Language.contents