From d019c7603d57dbfbf05c4ff8c3d92d6b0ae02e4f Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <egar.walker@gmail.com>
Date: Thu, 2 Jul 2015 16:52:06 -0500
Subject: [PATCH 01/12] Add PyPI support along with a Makefile for automation

---
 .gitignore                  |  7 +++-
 Makefile                    | 28 +++++++++++++++
 Vagrantfile                 | 68 +++++++++++++++++++++++++++++++++++++
 datajoint/connection.py     | 17 +++++-----
 datajoint/declare.py        |  8 ++---
 datajoint/erd.py            |  6 ++--
 datajoint/heading.py        |  7 ++--
 datajoint/user_relations.py |  3 +-
 datajoint/utils.py          |  2 +-
 db_setup.sql                |  2 ++
 provision.sh                | 33 ++++++++++++++++++
 setup.py                    | 30 ++++++++++++----
 12 files changed, 182 insertions(+), 29 deletions(-)
 create mode 100644 Makefile
 create mode 100644 Vagrantfile
 create mode 100644 db_setup.sql
 create mode 100644 provision.sh

diff --git a/.gitignore b/.gitignore
index ecd81049f..9292f22d7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,5 +6,10 @@
 .DS_Store
 __*__
 .idea
-datajoint.egg-info/
 *.pyc
+.python-version
+*.egg-info/
+dist/
+build/
+MANIFEST
+.vagrant/
diff --git a/Makefile b/Makefile
new file mode 100644
index 000000000..77a432a28
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,28 @@
+all:
+	@echo 'MakeFile for DataJoint packaging                                              '
+	@echo '                                                                              '
+	@echo 'make sdist                              Creates source distribution           '
+	@echo 'make wheel                              Creates Wheel dstribution             '
+	@echo 'make pypi                               Package and upload to PyPI            '
+	@echo 'make pypitest                           Package and upload to PyPI test server'
+	@echo 'make purge                              Remove all build related directories  '
+	
+
+sdist:
+	python setup.py sdist >/dev/null 2>&1
+
+wheel:
+	python setup.py bdist_wheel >/dev/null 2>&1
+
+pypi:purge sdist wheel
+	twine upload dist/*
+	
+pypitest: purge sdist wheel
+	twine upload -r pypitest dist/*
+
+purge:
+	rm -rf dist && rm -rf build && rm -rf datajoint.egg-info
+
+
+
+
diff --git a/Vagrantfile b/Vagrantfile
new file mode 100644
index 000000000..8742d3eb3
--- /dev/null
+++ b/Vagrantfile
@@ -0,0 +1,68 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# All Vagrant configuration is done below. The "2" in Vagrant.configure
+# configures the configuration version (we support older styles for
+# backwards compatibility). Please don't change it unless you know what
+# you're doing.
+Vagrant.configure(2) do |config|
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://atlas.hashicorp.com/search.
+  config.vm.box = "ubuntu/trusty64"
+
+  # Disable automatic box update checking. If you disable this, then
+  # boxes will only be checked for updates when the user runs
+  # `vagrant box outdated`. This is not recommended.
+  # config.vm.box_check_update = false
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine. In the example below,
+  # accessing "localhost:8080" will access port 80 on the guest machine.
+  # config.vm.network "forwarded_port", guest: 80, host: 8080
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  config.vm.network "private_network", ip: "192.168.33.10"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network "public_network"
+
+  # Share an additional folder to the guest VM. The first argument is
+  # the path on the host to the actual folder. The second argument is
+  # the path on the guest to mount the folder. And the optional third
+  # argument is a set of non-required options.
+  # config.vm.synced_folder "../data", "/vagrant_data"
+
+  # Provider-specific configuration so you can fine-tune various
+  # backing providers for Vagrant. These expose provider-specific options.
+  # Example for VirtualBox:
+  #
+  # config.vm.provider "virtualbox" do |vb|
+  #   # Display the VirtualBox GUI when booting the machine
+  #   vb.gui = true
+  #
+  #   # Customize the amount of memory on the VM:
+  #   vb.memory = "1024"
+  # end
+  #
+  # View the documentation for the provider you are using for more
+  # information on available options.
+
+  # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies
+  # such as FTP and Heroku are also available. See the documentation at
+  # https://docs.vagrantup.com/v2/push/atlas.html for more information.
+  # config.push.define "atlas" do |push|
+  #   push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME"
+  # end
+
+  # Enable provisioning with a shell script. Additional provisioners such as
+  # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
+  # documentation for more information about their specific syntax and use.
+  config.vm.provision "shell", path: "provision.sh" 
+end
diff --git a/datajoint/connection.py b/datajoint/connection.py
index 42d70e21c..1218a60e2 100644
--- a/datajoint/connection.py
+++ b/datajoint/connection.py
@@ -62,7 +62,7 @@ def __init__(self, host, user, passwd, init_fun=None):
         self.conn_info = dict(host=host, port=port, user=user, passwd=passwd)
         self._conn = pymysql.connect(init_command=init_fun, **self.conn_info)
         if self.is_connected:
-            logger.info("Connected " + user + '@' + host + ':' + str(port))
+            logger.info("Connected {user}@{host}:{port}".format(**self.conn_info))
         else:
             raise DataJointError('Connection failed.')
         self._conn.autocommit(True)
@@ -76,6 +76,11 @@ def __del__(self):
     def __eq__(self, other):
         return self.conn_info == other.conn_info
 
+    def __repr__(self):
+        connected = "connected" if self.is_connected else "disconnected"
+        return "DataJoint connection ({connected}) {user}@{host}:{port}".format(
+            connected=connected, **self.conn_info)
+
     @property
     def is_connected(self):
         """
@@ -83,15 +88,9 @@ def is_connected(self):
         """
         return self._conn.ping()
 
-    def __repr__(self):
-        connected = "connected" if self.is_connected else "disconnected"
-        return "DataJoint connection ({connected}) {user}@{host}:{port}".format(
-            connected=connected, **self.conn_info)
-
-
     def query(self, query, args=(), as_dict=False):
         """
-        Execute the specified query and return the tuple generator.
+        Execute the specified query and return the tuple generator (cursor).
 
         :param query: mysql query
         :param args: additional arguments for the pymysql.cursor
@@ -168,4 +167,4 @@ def transaction(self):
             self.cancel_transaction()
             raise
         else:
-            self.commit_transaction()
\ No newline at end of file
+            self.commit_transaction()
diff --git a/datajoint/declare.py b/datajoint/declare.py
index eceb70b48..660081883 100644
--- a/datajoint/declare.py
+++ b/datajoint/declare.py
@@ -12,8 +12,7 @@
 logger = logging.getLogger(__name__)
 
 
-
-def declare(full_table_name,  definition, context):
+def declare(full_table_name, definition, context):
     """
     Parse declaration and create new SQL table accordingly.
 
@@ -24,6 +23,7 @@ def declare(full_table_name,  definition, context):
     # split definition into lines
     definition = re.split(r'\s*\n\s*', definition.strip())
 
+    # check for optional table comment
     table_comment = definition.pop(0)[1:].strip() if definition[0].startswith('#') else ''
 
     in_key = True  # parse primary keys
@@ -40,7 +40,7 @@ def declare(full_table_name,  definition, context):
             in_key = False  # start parsing dependent attributes
         elif line.startswith('->'):
             # foreign key
-            ref = eval(line[2:], context)()
+            ref = eval(line[2:], context)() # TODO: surround this with try...except... to give a better error message
             foreign_key_sql.append(
                 'FOREIGN KEY ({primary_key})'
                 ' REFERENCES {ref} ({primary_key})'
@@ -52,7 +52,7 @@ def declare(full_table_name,  definition, context):
                     primary_key.append(name)
                 if name not in attributes:
                     attributes.append(name)
-                    attribute_sql.append(ref.heading[name].sql())
+                    attribute_sql.append(ref.heading[name].sql)
         elif re.match(r'^(unique\s+)?index[^:]*$', line, re.I):   # index
             index_sql.append(line)  # the SQL syntax is identical to DataJoint's
         else:
diff --git a/datajoint/erd.py b/datajoint/erd.py
index a90f4b017..53b3a2f74 100644
--- a/datajoint/erd.py
+++ b/datajoint/erd.py
@@ -1,17 +1,17 @@
 import logging
-import pyparsing as pp
 import re
+from collections import defaultdict
+
+import pyparsing as pp
 import networkx as nx
 from networkx import DiGraph
 from networkx import pygraphviz_layout
 import numpy as np
 import matplotlib.pyplot as plt
 from matplotlib import transforms
-from collections import defaultdict
 
 from . import DataJointError
 
-
 logger = logging.getLogger(__name__)
 
 
diff --git a/datajoint/heading.py b/datajoint/heading.py
index 85d54ee3e..47c2bd534 100644
--- a/datajoint/heading.py
+++ b/datajoint/heading.py
@@ -83,6 +83,9 @@ def non_blobs(self):
     def computed(self):
         return [k for k, v in self.attributes.items() if v.computation]
 
+    def __bool__(self):
+        return self.attributes is not None
+
     def __getitem__(self, name):
         """shortcut to the attribute"""
         return self.attributes[name]
@@ -104,7 +107,7 @@ def as_dtype(self):
         """
         return np.dtype(dict(
             names=self.names,
-            formats=[v.dtype for k, v in self.attributes.items()]))
+            formats=[v.dtype for v in self.attributes.values()]))
 
     @property
     def as_sql(self):
@@ -262,4 +265,4 @@ def resolve(self):
         """
         Remove attribute computations after they have been resolved in a subquery
         """
-        return Heading([dict(v._asdict(), computation=None) for v in self.attributes.values()])
\ No newline at end of file
+        return Heading([dict(v._asdict(), computation=None) for v in self.attributes.values()])
diff --git a/datajoint/user_relations.py b/datajoint/user_relations.py
index 26a21b0db..c93827376 100644
--- a/datajoint/user_relations.py
+++ b/datajoint/user_relations.py
@@ -6,7 +6,7 @@
 import abc
 from datajoint.relation import Relation
 from .autopopulate import AutoPopulate
-from . import DataJointError
+from datajoint.utils import from_camel_case
 
 
 class Manual(Relation):
@@ -98,7 +98,6 @@ def _make_tuples(self, key):
         raise NotImplementedError('Subtables should not be populated directly.')
 
 
-# ---------------- utilities --------------------
 def from_camel_case(s):
     """
     Convert names in camel case into underscore (_) separated names
diff --git a/datajoint/utils.py b/datajoint/utils.py
index a142e2707..5dcb5da7a 100644
--- a/datajoint/utils.py
+++ b/datajoint/utils.py
@@ -28,4 +28,4 @@ def group_by(rel, *attributes, sortby=None):
         if len(nk) == 1:
             yield nk[0], rel & restr
         else:
-            yield nk, rel & restr
\ No newline at end of file
+            yield nk, rel & restr
diff --git a/db_setup.sql b/db_setup.sql
new file mode 100644
index 000000000..a8e7d3674
--- /dev/null
+++ b/db_setup.sql
@@ -0,0 +1,2 @@
+create user 'datajoint'@'localhost' identified by 'datajoint';
+grant all on `djtest\_%`.* to 'datajoint'@'localhost';
diff --git a/provision.sh b/provision.sh
new file mode 100644
index 000000000..6d9a070e6
--- /dev/null
+++ b/provision.sh
@@ -0,0 +1,33 @@
+BASE='/home/vagrant'
+PYENV_PATH="$BASE/.pyenv"
+PYTHON_VER="3.4.3"
+PROJECT="/vagrant"
+
+echo "Setting up Python environment"
+apt-get update >/dev/null 2>&1
+echo "Installing essential packages"
+apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
+  libreadline-dev libsqlite3-dev wget curl llvm libpng12-dev libfreetype6-dev \
+  pkg-config git vim >/dev/null 2>&1
+
+echo "Installing Python 3"
+apt-get install -y python3 python3-pip
+
+pip3 install --upgrade pip
+
+
+cd "$PROJECT"
+
+if [ -f "requirements.txt" ]; then
+  echo "Installing Python packages"
+  pip3 install -r requirements.txt
+fi
+
+echo "Setting up database connections"
+
+sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password password root'
+sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password root'
+sudo apt-get install -y mysql-server 2> /dev/null
+sudo apt-get install -y mysql-client 2> /dev/null
+
+mysql -uroot -proot < db_setup.sql
diff --git a/setup.py b/setup.py
index 5c56b483c..7e1d5d07e 100644
--- a/setup.py
+++ b/setup.py
@@ -1,15 +1,31 @@
 #!/usr/bin/env python
+from setuptools import setup, find_packages
+from os import path
+
+here = path.abspath(path.dirname(__file__))
+
+#with open(path.join(here, 'VERSION')) as version_file:
+#   version = version_file.read().strip()
+long_description="An object-relational mapping and relational algebra to facilitate data definition and data manipulation in MySQL databases."
 
-from distutils.core import setup
 
 setup(
     name='datajoint',
-    version='0.1',
-    author='Dimitri Yatsenko',  # todo: change that once deon
+    version='0.1.0.dev5',
+    description="An ORM with closed relational algebra",
+    long_description=long_description,
+    author='Dimitri Yatsenko',
     author_email='Dimitri.Yatsenko@gmail.com',
-    description='An object-relational mapping and relational algebra to facilitate data definition and data manipulation in MySQL databases.',
-    url='https://github.com/datajoint/datajoint-python',
-    packages=['datajoint'],
-    requires=['numpy', 'pymysql', 'networkx', 'matplotlib', 'sphinx_rtd_theme', 'mock', 'json'],
     license = "MIT",
+    url='https://github.com/datajoint/datajoint-python',
+    keywords='database organization',
+    packages=find_packages(exclude=['contrib', 'docs', 'tests*']),
+    install_requires=['numpy', 'pymysql', 'pyparsing', 'networkx', 'matplotlib', 'sphinx_rtd_theme', 'mock'],
+    classifiers=[
+        'Development Status :: 4 - Beta',
+        'Intended Audience :: Science/Research',
+        'Programming Language :: Python :: 3 :: Only',
+        'License :: OSI Approved :: MIT License',
+        'Topic :: Database :: Front-Ends',
+    ],
 )

From 92e63fe594db7cc70645ccfecebe29c6a6f060d1 Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Thu, 16 Jul 2015 01:09:23 -0500
Subject: [PATCH 02/12] Move setup files to directory misc and rename ERD to
 ERM

---
 .gitignore                        |  2 +-
 Vagrantfile                       |  2 +-
 datajoint/__init__.py             |  5 ++--
 datajoint/connection.py           |  7 +++--
 datajoint/declare.py              |  1 -
 datajoint/erd.py                  | 48 ++++++++++++++-----------------
 datajoint/heading.py              |  8 +++---
 datajoint/relation.py             | 19 ++++++------
 datajoint/relational_operand.py   |  3 +-
 datajoint/schema.py               |  3 +-
 datajoint/user_relations.py       | 20 +------------
 datajoint/utils.py                | 40 +++++++++++++++++++++++++-
 db_setup.sql => misc/db_setup.sql |  0
 provision.sh => misc/provision.sh |  6 ++--
 tests/test_settings.py            |  1 +
 tests/test_utils.py               |  3 +-
 16 files changed, 94 insertions(+), 74 deletions(-)
 rename db_setup.sql => misc/db_setup.sql (100%)
 rename provision.sh => misc/provision.sh (85%)

diff --git a/.gitignore b/.gitignore
index 9292f22d7..45250c9f6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,7 +5,7 @@
 */*.pyc
 .DS_Store
 __*__
-.idea
+.idea/
 *.pyc
 .python-version
 *.egg-info/
diff --git a/Vagrantfile b/Vagrantfile
index 8742d3eb3..130a8b0c9 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -64,5 +64,5 @@ Vagrant.configure(2) do |config|
   # Enable provisioning with a shell script. Additional provisioners such as
   # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
   # documentation for more information about their specific syntax and use.
-  config.vm.provision "shell", path: "provision.sh" 
+  config.vm.provision "shell", path: "misc/provision.sh" 
 end
diff --git a/datajoint/__init__.py b/datajoint/__init__.py
index 584995a6d..2709b823f 100644
--- a/datajoint/__init__.py
+++ b/datajoint/__init__.py
@@ -18,8 +18,7 @@
            'Connection', 'Heading', 'Relation', 'FreeRelation', 'Not',
            'Relation', 'schema',
            'Manual', 'Lookup', 'Imported', 'Computed',
-           'conn', 'DataJointError']
-
+           'conn']
 
 class DataJointError(Exception):
     """
@@ -27,7 +26,6 @@ class DataJointError(Exception):
     """
     pass
 
-
 # ----------- loads local configuration from file ----------------
 from .settings import Config, CONFIGVAR, LOCALCONFIG, logger, log_levels
 config = Config()
@@ -53,3 +51,4 @@ class DataJointError(Exception):
 from .relational_operand import Not
 from .heading import Heading
 from .schema import schema
+
diff --git a/datajoint/connection.py b/datajoint/connection.py
index 1218a60e2..365193496 100644
--- a/datajoint/connection.py
+++ b/datajoint/connection.py
@@ -7,8 +7,9 @@
 import pymysql
 import logging
 from collections import defaultdict
-from . import DataJointError, config
-from .erd import ERD
+from . import config
+from . import DataJointError
+from .erd import ERM
 from .jobs import JobManager
 
 logger = logging.getLogger(__name__)
@@ -53,7 +54,7 @@ class Connection:
     """
 
     def __init__(self, host, user, passwd, init_fun=None):
-        self.erd = ERD()
+        self.erm = ERM(self)
         if ':' in host:
             host, port = host.split(':')
             port = int(port)
diff --git a/datajoint/declare.py b/datajoint/declare.py
index 660081883..1e210aaa8 100644
--- a/datajoint/declare.py
+++ b/datajoint/declare.py
@@ -8,7 +8,6 @@
 
 from . import DataJointError
 
-
 logger = logging.getLogger(__name__)
 
 
diff --git a/datajoint/erd.py b/datajoint/erd.py
index 53b3a2f74..4286013ca 100644
--- a/datajoint/erd.py
+++ b/datajoint/erd.py
@@ -1,28 +1,38 @@
+from matplotlib import transforms
+
+import numpy as np
+
 import logging
 import re
 from collections import defaultdict
-
 import pyparsing as pp
 import networkx as nx
 from networkx import DiGraph
 from networkx import pygraphviz_layout
-import numpy as np
 import matplotlib.pyplot as plt
-from matplotlib import transforms
-
 from . import DataJointError
+from utils import to_camel_case
 
 logger = logging.getLogger(__name__)
 
 
-class ERD:
-    _checked_dependencies = set()
-    _parents = dict()
-    _referenced = dict()
-    _children = defaultdict(list)
-    _references = defaultdict(list)
+class ERM:
+    """
+    Entity Relation Map
+
+    Represents known relation between tables
+    """
+    #_checked_dependencies = set()
+
+    def __init__(self, conn):
+        self._conn = conn
+        self._parents = dict()
+        self._referenced = dict()
+        self._children = defaultdict(list)
+        self._references = defaultdict(list)
 
-    def load_dependencies(self, connection, full_table_name):
+
+    def load_dependencies(self, full_table_name):
         # check if already loaded.  Use clear_dependencies before reloading
         if full_table_name in self._parents:
             return
@@ -30,7 +40,7 @@ def load_dependencies(self, connection, full_table_name):
         self._referenced[full_table_name] = list()
 
         # fetch the CREATE TABLE statement
-        cur = connection.query('SHOW CREATE TABLE %s' % full_table_name)
+        cur = self._conn.query('SHOW CREATE TABLE %s' % full_table_name)
         create_statement = cur.fetchone()
         if not create_statement:
             raise DataJointError('Could not load the definition table %s' % full_table_name)
@@ -127,20 +137,6 @@ def recurse(full_table_name, level):
         return sorted(ret.keys(), key=ret.__getitem__)
 
 
-def to_camel_case(s):
-    """
-    Convert names with under score (_) separation
-    into camel case names.
-    Example:
-    >>>to_camel_case("table_name")
-        "TableName"
-    """
-    def to_upper(match):
-        return match.group(0)[-1].upper()
-    return re.sub('(^|[_\W])+[a-zA-Z]', to_upper, s)
-
-
-
 class RelGraph(DiGraph):
     """
     A directed graph representing relations between tables within and across
diff --git a/datajoint/heading.py b/datajoint/heading.py
index 47c2bd534..ee673af96 100644
--- a/datajoint/heading.py
+++ b/datajoint/heading.py
@@ -1,8 +1,7 @@
-
-import re
-from collections import OrderedDict, namedtuple
 import numpy as np
-from datajoint import DataJointError
+from . import DataJointError
+from collections import namedtuple, OrderedDict
+import re
 
 
 class Attribute(namedtuple('Attribute',
@@ -15,6 +14,7 @@ def _asdict(self):
         """
         return OrderedDict((name, self[i]) for i, name in enumerate(self._fields))
 
+    @property
     def sql(self):
         """
         Convert attribute tuple into its SQL CREATE TABLE clause.
diff --git a/datajoint/relation.py b/datajoint/relation.py
index 140b2858d..7368d610d 100644
--- a/datajoint/relation.py
+++ b/datajoint/relation.py
@@ -3,7 +3,8 @@
 import logging
 import abc
 
-from . import DataJointError, config
+from . import config
+from . import DataJointError
 from .declare import declare
 from .relational_operand import RelationalOperand
 from .blob import pack
@@ -48,7 +49,7 @@ def connection(self):
     @property
     def heading(self):
         """
-        Get the table headng.
+        Get the table heading.
         If the table is not declared, attempts to declare it and return heading.
         :return:
         """
@@ -59,7 +60,7 @@ def heading(self):
                 self.connection.query(
                     declare(self.full_table_name, self.definition, self._context))
             if self.is_declared:
-                self.connection.erd.load_dependencies(self.connection, self.full_table_name)
+                self.connection.erm.load_dependencies(self.full_table_name)
                 self._heading.init_from_database(self.connection, self.database, self.table_name)
         return self._heading
 
@@ -73,19 +74,19 @@ def from_clause(self):
     # ------------- dependencies ---------- #
     @property
     def parents(self):
-        return self.connection.erd.parents[self.full_table_name]
+        return self.connection.erm.parents[self.full_table_name]
 
     @property
     def children(self):
-        return self.connection.erd.children[self.full_table_name]
+        return self.connection.erm.children[self.full_table_name]
 
     @property
     def references(self):
-        return self.connection.erd.references[self.full_table_name]
+        return self.connection.erm.references[self.full_table_name]
 
     @property
     def referenced(self):
-        return self.connection.erd.referenced[self.full_table_name]
+        return self.connection.erm.referenced[self.full_table_name]
 
     @property
     def descendants(self):
@@ -95,7 +96,7 @@ def descendants(self):
         This is helpful for cascading delete or drop operations.
         """
         relations = (FreeRelation(self.connection, table)
-                     for table in self.connection.erd.get_descendants(self.full_table_name))
+                     for table in self.connection.erm.get_descendants(self.full_table_name))
         return [relation for relation in relations if relation.is_declared]
 
     # --------- SQL functionality --------- #
@@ -209,7 +210,7 @@ def drop_quick(self):
         """
         if self.is_declared:
             self.connection.query('DROP TABLE %s' % self.full_table_name)
-            self.connection.erd.clear_dependencies(self.full_table_name)
+            self.connection.erm.clear_dependencies(self.full_table_name)
             if self._heading:
                 self._heading.reset()
             logger.info("Dropped table %s" % self.full_table_name)
diff --git a/datajoint/relational_operand.py b/datajoint/relational_operand.py
index a8f633c28..1f4fa3d21 100644
--- a/datajoint/relational_operand.py
+++ b/datajoint/relational_operand.py
@@ -7,7 +7,8 @@
 import re
 from collections import OrderedDict
 from copy import copy
-from datajoint import DataJointError, config
+from . import config
+from . import DataJointError
 import logging
 
 from .blob import unpack
diff --git a/datajoint/schema.py b/datajoint/schema.py
index f1c82b351..f95143a91 100644
--- a/datajoint/schema.py
+++ b/datajoint/schema.py
@@ -1,7 +1,8 @@
 import pymysql
 import logging
 
-from . import DataJointError, conn
+from . import conn
+from . import DataJointError
 from .heading import Heading
 
 logger = logging.getLogger(__name__)
diff --git a/datajoint/user_relations.py b/datajoint/user_relations.py
index c93827376..34d425045 100644
--- a/datajoint/user_relations.py
+++ b/datajoint/user_relations.py
@@ -2,11 +2,9 @@
 Hosts the table tiers, user relations should be derived from.
 """
 
-import re
-import abc
 from datajoint.relation import Relation
 from .autopopulate import AutoPopulate
-from datajoint.utils import from_camel_case
+from utils import from_camel_case
 
 
 class Manual(Relation):
@@ -98,19 +96,3 @@ def _make_tuples(self, key):
         raise NotImplementedError('Subtables should not be populated directly.')
 
 
-def from_camel_case(s):
-    """
-    Convert names in camel case into underscore (_) separated names
-
-    Example:
-    >>>from_camel_case("TableName")
-        "table_name"
-    """
-
-    def convert(match):
-        return ('_' if match.groups()[0] else '') + match.group(0).lower()
-
-    if not re.match(r'[A-Z][a-zA-Z0-9]*', s):
-        raise DataJointError(
-            'ClassName must be alphanumeric in CamelCase, begin with a capital letter')
-    return re.sub(r'(\B[A-Z])|(\b[A-Z])', convert, s)
diff --git a/datajoint/utils.py b/datajoint/utils.py
index 5dcb5da7a..7ef0cab24 100644
--- a/datajoint/utils.py
+++ b/datajoint/utils.py
@@ -1,10 +1,13 @@
 import numpy as np
+import re
+from datajoint import DataJointError
+
 
 
 def user_choice(prompt, choices=("yes", "no"), default=None):
     """
     Prompts the user for confirmation.  The default value, if any, is capitalized.
-    :param prompt: Information to display to the user.
+    :parsam prompt: Information to display to the user.
     :param choices: an iterable of possible choices.
     :param default: default choice
     :return: the user's choice
@@ -29,3 +32,38 @@ def group_by(rel, *attributes, sortby=None):
             yield nk[0], rel & restr
         else:
             yield nk, rel & restr
+
+
+def to_camel_case(s):
+    """
+    Convert names with under score (_) separation
+    into camel case names.
+    Example:
+    >>>to_camel_case("table_name")
+        "TableName"
+    """
+    def to_upper(match):
+        return match.group(0)[-1].upper()
+    return re.sub('(^|[_\W])+[a-zA-Z]', to_upper, s)
+
+
+def from_camel_case(s):
+    """
+    Convert names in camel case into underscore (_) separated names
+
+    Example:
+    >>>from_camel_case("TableName")
+        "table_name"
+    """
+
+    def convert(match):
+        return ('_' if match.groups()[0] else '') + match.group(0).lower()
+
+    if not re.match(r'[A-Z][a-zA-Z0-9]*', s):
+        raise DataJointError(
+            'ClassName must be alphanumeric in CamelCase, begin with a capital letter')
+    return re.sub(r'(\B[A-Z])|(\b[A-Z])', convert, s)
+
+
+
+
diff --git a/db_setup.sql b/misc/db_setup.sql
similarity index 100%
rename from db_setup.sql
rename to misc/db_setup.sql
diff --git a/provision.sh b/misc/provision.sh
similarity index 85%
rename from provision.sh
rename to misc/provision.sh
index 6d9a070e6..f57f28e60 100644
--- a/provision.sh
+++ b/misc/provision.sh
@@ -18,9 +18,9 @@ pip3 install --upgrade pip
 
 cd "$PROJECT"
 
-if [ -f "requirements.txt" ]; then
+if [ -f "$PROJECT/misc/requirements.txt" ]; then
   echo "Installing Python packages"
-  pip3 install -r requirements.txt
+  pip3 install -r "$PROJECT/misc/requirements.txt"
 fi
 
 echo "Setting up database connections"
@@ -30,4 +30,4 @@ sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password_again p
 sudo apt-get install -y mysql-server 2> /dev/null
 sudo apt-get install -y mysql-client 2> /dev/null
 
-mysql -uroot -proot < db_setup.sql
+mysql -uroot -proot < "$PROJECT/misc/db_setup.sql"
diff --git a/tests/test_settings.py b/tests/test_settings.py
index 2e9f18328..a9720a075 100644
--- a/tests/test_settings.py
+++ b/tests/test_settings.py
@@ -3,6 +3,7 @@
 import random
 import string
 from datajoint import settings
+import utils
 
 __author__ = 'Fabian Sinz'
 
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 2973c8d81..827478103 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -2,8 +2,9 @@
 Collection of test cases to test core module.
 """
 from nose.tools import assert_true, assert_raises, assert_equal
-from datajoint.user_relations import from_camel_case
 from datajoint import DataJointError
+from utils import from_camel_case
+
 
 def setup():
     pass

From e161aa3727ebae9f9561000d0281e3b6c065d80b Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Thu, 16 Jul 2015 01:15:02 -0500
Subject: [PATCH 03/12] Update provision file and add development specific
 package requirements

---
 misc/dev-requirements.txt |  5 +++++
 misc/provision.sh         | 11 +++++++++--
 2 files changed, 14 insertions(+), 2 deletions(-)
 create mode 100644 misc/dev-requirements.txt

diff --git a/misc/dev-requirements.txt b/misc/dev-requirements.txt
new file mode 100644
index 000000000..9a1af7f16
--- /dev/null
+++ b/misc/dev-requirements.txt
@@ -0,0 +1,5 @@
+ipython
+jinja
+tornado
+jsonschema
+pyzmq
diff --git a/misc/provision.sh b/misc/provision.sh
index f57f28e60..91d4d7fb0 100644
--- a/misc/provision.sh
+++ b/misc/provision.sh
@@ -18,9 +18,16 @@ pip3 install --upgrade pip
 
 cd "$PROJECT"
 
-if [ -f "$PROJECT/misc/requirements.txt" ]; then
+# Install minimal requirement for running the package
+if [ -f "$PROJECT/requirements.txt" ]; then
   echo "Installing Python packages"
-  pip3 install -r "$PROJECT/misc/requirements.txt"
+  pip3 install -r "$PROJECT/requirements.txt"
+fi
+
+# Install additional development requirements
+if [ -f "$PROJECT/misc/dev-requirements.txt" ]; then
+  echo "Installing Python packages"
+  pip3 install -r "$PROJECT/misc/dev-requirements.txt"
 fi
 
 echo "Setting up database connections"

From 77a21d7ea230418b1c9db1723fbaa5b2ed6ffb07 Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Thu, 16 Jul 2015 01:27:35 -0500
Subject: [PATCH 04/12] Fix absolute import error for utils and add test case
 for to_camel_case

---
 datajoint/erd.py            | 2 +-
 datajoint/user_relations.py | 2 +-
 tests/test_utils.py         | 9 ++++++++-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/datajoint/erd.py b/datajoint/erd.py
index 4286013ca..e23643188 100644
--- a/datajoint/erd.py
+++ b/datajoint/erd.py
@@ -11,7 +11,7 @@
 from networkx import pygraphviz_layout
 import matplotlib.pyplot as plt
 from . import DataJointError
-from utils import to_camel_case
+from .utils import to_camel_case
 
 logger = logging.getLogger(__name__)
 
diff --git a/datajoint/user_relations.py b/datajoint/user_relations.py
index 34d425045..f3fb91928 100644
--- a/datajoint/user_relations.py
+++ b/datajoint/user_relations.py
@@ -4,7 +4,7 @@
 
 from datajoint.relation import Relation
 from .autopopulate import AutoPopulate
-from utils import from_camel_case
+from .utils import from_camel_case
 
 
 class Manual(Relation):
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 827478103..f8615d71d 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -3,7 +3,7 @@
 """
 from nose.tools import assert_true, assert_raises, assert_equal
 from datajoint import DataJointError
-from utils import from_camel_case
+from utils import from_camel_case, to_camel_case
 
 
 def setup():
@@ -24,3 +24,10 @@ def test_from_camel_case():
         from_camel_case('hello world')
     with assert_raises(DataJointError):
         from_camel_case('#baisc_names')
+
+
+def test_to_camel_case():
+    assert_equal(to_camel_case('all_groups'), 'AllGroups')
+    assert_equal(to_camel_case('hello'), 'Hello')
+    assert_equal(to_camel_case('this_is_a_sample_case'), 'ThisIsASampleCase')
+    assert_equal(to_camel_case('This_is_Mixed'), 'ThisIsMixed')

From 8ae5686a6dbe04e801d3a67a1f149065c864f9ba Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Thu, 16 Jul 2015 01:35:21 -0500
Subject: [PATCH 05/12] Fix remaining relative imports to util

---
 tests/test_settings.py | 1 -
 tests/test_utils.py    | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_settings.py b/tests/test_settings.py
index a9720a075..2e9f18328 100644
--- a/tests/test_settings.py
+++ b/tests/test_settings.py
@@ -3,7 +3,6 @@
 import random
 import string
 from datajoint import settings
-import utils
 
 __author__ = 'Fabian Sinz'
 
diff --git a/tests/test_utils.py b/tests/test_utils.py
index f8615d71d..fe1694173 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -3,7 +3,7 @@
 """
 from nose.tools import assert_true, assert_raises, assert_equal
 from datajoint import DataJointError
-from utils import from_camel_case, to_camel_case
+from datajoint.utils import from_camel_case, to_camel_case
 
 
 def setup():

From c0f114282bb566f47c62c4778775513909e1892f Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Thu, 16 Jul 2015 01:44:37 -0500
Subject: [PATCH 06/12] Update travis config to use new framework

---
 .travis.yml | 33 ++++++++++-----------------------
 1 file changed, 10 insertions(+), 23 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 0a1deb781..c68e34443 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,31 +1,18 @@
 language: python
+env:
+  - DJ_TEST_HOST="127.0.0.1"
+  - DJ_TEST_USER="root"
+  - DJ_TEST_PASSWORD=""
+  - DJ_HOST="127.0.0.1"
+  - DJ_USER="root"
+  - DJ_PASSWORD=""
 python: 3.4
 services: mysql
-env:
-  - DJ_TEST_HOST="127.0.0.1" DJ_TEST_USER="root" DJ_TEST_PASSWORD="" DJ_HOST="127.0.0.1" DJ_USER="root" DJ_PASSWORD=""
-before_install:
-  - sudo apt-get install -qq libatlas-dev libatlas-base-dev liblapack-dev gfortran
-  - sudo apt-get update
-  # You may want to periodically update this, although the conda update
-  # conda line below will keep everything up-to-date.  We do this
-  # conditionally because it saves us some downloading if the version is
-  # the same.
-  - wget http://repo.continuum.io/miniconda/Miniconda3-3.4.2-Linux-x86_64.sh -O miniconda.sh;
-  - bash miniconda.sh -b -p $HOME/miniconda
-  - export PATH="$HOME/miniconda/bin:$PATH"
-  - hash -r
-  - conda update -q --yes conda
-  - conda config --set always_yes yes --set changeps1 no
-  # Useful for debugging any issues with conda
-  - conda info -a
 install:
-  # Replace dep1 dep2 ... with your dependencies
-  - conda create -n test-environment python=$TRAVIS_PYTHON_VERSION numpy scipy setuptools pip
-  - source activate test-environment
-  - pip install nose nose-cov python-coveralls pymysql networkx matplotlib
-  - conda info -a
+  - pip install -r requirements.txt
+  - pip install nose nose-cv python-coveralls
 # command to run tests
 script:
   - nosetests -vv --with-coverage --cover-package=datajoint
 after_success:
-  - coveralls
+  - coveralls
\ No newline at end of file

From 0a84cd5a24661869091feec9ffb942428148f4d9 Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Thu, 16 Jul 2015 01:51:31 -0500
Subject: [PATCH 07/12] Fix env config for travis

---
 .travis.yml | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index c68e34443..607c45d05 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,11 +1,6 @@
 language: python
 env:
-  - DJ_TEST_HOST="127.0.0.1"
-  - DJ_TEST_USER="root"
-  - DJ_TEST_PASSWORD=""
-  - DJ_HOST="127.0.0.1"
-  - DJ_USER="root"
-  - DJ_PASSWORD=""
+  - DJ_TEST_HOST="127.0.0.1" DJ_TEST_USER="root" DJ_TEST_PASSWORD="" DJ_HOST="127.0.0.1" DJ_USER="root" DJ_PASSWORD=""
 python: 3.4
 services: mysql
 install:
@@ -15,4 +10,4 @@ install:
 script:
   - nosetests -vv --with-coverage --cover-package=datajoint
 after_success:
-  - coveralls
\ No newline at end of file
+  - coveralls

From 6cb67af10e2f93f682d8b4e3df824c74080da249 Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Thu, 16 Jul 2015 01:56:58 -0500
Subject: [PATCH 08/12] Fix package name in travis config

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 607c45d05..01cbecd88 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,7 +5,7 @@ python: 3.4
 services: mysql
 install:
   - pip install -r requirements.txt
-  - pip install nose nose-cv python-coveralls
+  - pip install nose nose-cov python-coveralls
 # command to run tests
 script:
   - nosetests -vv --with-coverage --cover-package=datajoint

From ac527de38c119ca0c49d220bfe2e3db65dd978e4 Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <eywalker@users.noreply.github.com>
Date: Thu, 16 Jul 2015 02:05:49 -0500
Subject: [PATCH 09/12] Update README.md

---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 7ed1f5f71..1f02b19b5 100644
--- a/README.md
+++ b/README.md
@@ -5,7 +5,8 @@ DataJoint for Python is a high-level programming interface for relational databa
 DataJoint was initially developed in 2009 by Dimitri Yatsenko in Andreas Tolias' Lab for the distributed processing and management of large volumes of data streaming from regular experiments. Starting in 2011, DataJoint has been available as an open-source project adopted by other labs and improved through contributions from several developers.
 
 To install datajoint using pip just run:
-```pip install git+https://github.com/datajoint/datajoint-python```
+```pip install datajoint```
+
+However, please be aware that DataJoint for Python is still under active development, and thus what's available on PyPI via `pip` is in **pre-release state**!
 
 
-[![Join the chat at https://gitter.im/datajoint/datajoint-python](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/datajoint/datajoint-python?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
\ No newline at end of file

From 9052dd146b004163306402af54183e6bfaabb7c2 Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <eywalker@users.noreply.github.com>
Date: Thu, 16 Jul 2015 02:14:17 -0500
Subject: [PATCH 10/12] Update README.md

Include badges and update installation instruction.
---
 README.md | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 1f02b19b5..aeebb225d 100644
--- a/README.md
+++ b/README.md
@@ -1,12 +1,22 @@
+[![Build Status](https://travis-ci.org/eywalker/datajoint-python.svg?branch=master)](https://travis-ci.org/eywalker/datajoint-python)
+[![Coverage Status](https://coveralls.io/repos/datajoint/datajoint-python/badge.svg?branch=master&service=github)](https://coveralls.io/github/datajoint/datajoint-python?branch=master)
+[![Join the chat at https://gitter.im/datajoint/datajoint-python](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/datajoint/datajoint-python?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+
+# Welcome to DataJoint for Python!
 The Python version of DataJoint is undergoing major revamping to match the features and capabilities of its more mature MATLAB counterpart. We expect to complete the revamp within a few weeks: August -- September, 2015.
 
 DataJoint for Python is a high-level programming interface for relational databases designed to support data processing chains in science labs. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, and querying data.
 
 DataJoint was initially developed in 2009 by Dimitri Yatsenko in Andreas Tolias' Lab for the distributed processing and management of large volumes of data streaming from regular experiments. Starting in 2011, DataJoint has been available as an open-source project adopted by other labs and improved through contributions from several developers.
 
-To install datajoint using pip just run:
-```pip install datajoint```
+## Quick start guide
+To install datajoint using `pip` just run:
+
+```
+pip install datajoint
+```
 
-However, please be aware that DataJoint for Python is still under active development, and thus what's available on PyPI via `pip` is in **pre-release state**!
+in your favorite terminal app.
 
+However, please be aware that DataJoint for Python is still undergoing major changes, and thus what's available on PyPI via `pip` is in **pre-release state**!
 

From 6dc27bf4b33019325ba17007c97115c917ad09f0 Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Thu, 16 Jul 2015 02:44:38 -0500
Subject: [PATCH 11/12] Test on 3.2, 3.3, and 3.4

---
 .travis.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 01cbecd88..d8f325830 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,7 +1,10 @@
 language: python
 env:
   - DJ_TEST_HOST="127.0.0.1" DJ_TEST_USER="root" DJ_TEST_PASSWORD="" DJ_HOST="127.0.0.1" DJ_USER="root" DJ_PASSWORD=""
-python: 3.4
+python:
+  - "3.2"
+  - "3.3"
+  - "3.4"
 services: mysql
 install:
   - pip install -r requirements.txt

From 1e489886da7e0a77dd8b797539ca4aafcf2eae88 Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Thu, 16 Jul 2015 02:49:39 -0500
Subject: [PATCH 12/12] Test only on 3.4

---
 .travis.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index d8f325830..20c2f651b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,8 +2,6 @@ language: python
 env:
   - DJ_TEST_HOST="127.0.0.1" DJ_TEST_USER="root" DJ_TEST_PASSWORD="" DJ_HOST="127.0.0.1" DJ_USER="root" DJ_PASSWORD=""
 python:
-  - "3.2"
-  - "3.3"
   - "3.4"
 services: mysql
 install: