Add PyPI and Vagrant support and use new Travis CI framework

.gitignore

@@ -5,6 +5,11 @@
 */*.pyc
 .DS_Store
 __*__
-.idea
-datajoint.egg-info/
+.idea/
 *.pyc
+.python-version
+*.egg-info/
+dist/
+build/
+MANIFEST
+.vagrant/

.travis.yml

@@ -1,29 +1,12 @@
 language: python
-python: 3.4
-services: mysql
 env:
   - DJ_TEST_HOST="127.0.0.1" DJ_TEST_USER="root" DJ_TEST_PASSWORD="" DJ_HOST="127.0.0.1" DJ_USER="root" DJ_PASSWORD=""
-before_install:
-  - sudo apt-get install -qq libatlas-dev libatlas-base-dev liblapack-dev gfortran
-  - sudo apt-get update
-  # You may want to periodically update this, although the conda update
-  # conda line below will keep everything up-to-date.  We do this
-  # conditionally because it saves us some downloading if the version is
-  # the same.
-  - wget http://repo.continuum.io/miniconda/Miniconda3-3.4.2-Linux-x86_64.sh -O miniconda.sh;
-  - bash miniconda.sh -b -p $HOME/miniconda
-  - export PATH="$HOME/miniconda/bin:$PATH"
-  - hash -r
-  - conda update -q --yes conda
-  - conda config --set always_yes yes --set changeps1 no
-  # Useful for debugging any issues with conda
-  - conda info -a
+python:
+  - "3.4"
+services: mysql
 install:
-  # Replace dep1 dep2 ... with your dependencies
-  - conda create -n test-environment python=$TRAVIS_PYTHON_VERSION numpy scipy setuptools pip
-  - source activate test-environment
-  - pip install nose nose-cov python-coveralls pymysql networkx matplotlib
-  - conda info -a
+  - pip install -r requirements.txt
+  - pip install nose nose-cov python-coveralls
 # command to run tests
 script:
   - nosetests -vv --with-coverage --cover-package=datajoint

Makefile

@@ -0,0 +1,28 @@
+all:
+	@echo 'MakeFile for DataJoint packaging                                              '
+	@echo '                                                                              '
+	@echo 'make sdist                              Creates source distribution           '
+	@echo 'make wheel                              Creates Wheel dstribution             '
+	@echo 'make pypi                               Package and upload to PyPI            '
+	@echo 'make pypitest                           Package and upload to PyPI test server'
+	@echo 'make purge                              Remove all build related directories  '
+
+
+sdist:
+	python setup.py sdist >/dev/null 2>&1
+
+wheel:
+	python setup.py bdist_wheel >/dev/null 2>&1
+
+pypi:purge sdist wheel
+	twine upload dist/*
+
+pypitest: purge sdist wheel
+	twine upload -r pypitest dist/*
+
+purge:
+	rm -rf dist && rm -rf build && rm -rf datajoint.egg-info
+
+
+
+

README.md

@@ -1,11 +1,22 @@
+[![Build Status](https://travis-ci.org/eywalker/datajoint-python.svg?branch=master)](https://travis-ci.org/eywalker/datajoint-python)
+[![Coverage Status](https://coveralls.io/repos/datajoint/datajoint-python/badge.svg?branch=master&service=github)](https://coveralls.io/github/datajoint/datajoint-python?branch=master)
+[![Join the chat at https://gitter.im/datajoint/datajoint-python](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/datajoint/datajoint-python?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+
+# Welcome to DataJoint for Python!
 The Python version of DataJoint is undergoing major revamping to match the features and capabilities of its more mature MATLAB counterpart. We expect to complete the revamp within a few weeks: August -- September, 2015.
 
 DataJoint for Python is a high-level programming interface for relational databases designed to support data processing chains in science labs. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, and querying data.
 
 DataJoint was initially developed in 2009 by Dimitri Yatsenko in Andreas Tolias' Lab for the distributed processing and management of large volumes of data streaming from regular experiments. Starting in 2011, DataJoint has been available as an open-source project adopted by other labs and improved through contributions from several developers.
 
-To install datajoint using pip just run:
-```pip install git+https://github.com/datajoint/datajoint-python```
+## Quick start guide
+To install datajoint using `pip` just run:
+
+```
+pip install datajoint
+```
+
+in your favorite terminal app.
 
+However, please be aware that DataJoint for Python is still undergoing major changes, and thus what's available on PyPI via `pip` is in **pre-release state**!
 
-[![Join the chat at https://gitter.im/datajoint/datajoint-python](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/datajoint/datajoint-python?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)

Vagrantfile

@@ -0,0 +1,68 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# All Vagrant configuration is done below. The "2" in Vagrant.configure
+# configures the configuration version (we support older styles for
+# backwards compatibility). Please don't change it unless you know what
+# you're doing.
+Vagrant.configure(2) do |config|
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://atlas.hashicorp.com/search.
+  config.vm.box = "ubuntu/trusty64"
+
+  # Disable automatic box update checking. If you disable this, then
+  # boxes will only be checked for updates when the user runs
+  # `vagrant box outdated`. This is not recommended.
+  # config.vm.box_check_update = false
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine. In the example below,
+  # accessing "localhost:8080" will access port 80 on the guest machine.
+  # config.vm.network "forwarded_port", guest: 80, host: 8080
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  config.vm.network "private_network", ip: "192.168.33.10"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network "public_network"
+
+  # Share an additional folder to the guest VM. The first argument is
+  # the path on the host to the actual folder. The second argument is
+  # the path on the guest to mount the folder. And the optional third
+  # argument is a set of non-required options.
+  # config.vm.synced_folder "../data", "/vagrant_data"
+
+  # Provider-specific configuration so you can fine-tune various
+  # backing providers for Vagrant. These expose provider-specific options.
+  # Example for VirtualBox:
+  #
+  # config.vm.provider "virtualbox" do |vb|
+  #   # Display the VirtualBox GUI when booting the machine
+  #   vb.gui = true
+  #
+  #   # Customize the amount of memory on the VM:
+  #   vb.memory = "1024"
+  # end
+  #
+  # View the documentation for the provider you are using for more
+  # information on available options.
+
+  # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies
+  # such as FTP and Heroku are also available. See the documentation at
+  # https://docs.vagrantup.com/v2/push/atlas.html for more information.
+  # config.push.define "atlas" do |push|
+  #   push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME"
+  # end
+
+  # Enable provisioning with a shell script. Additional provisioners such as
+  # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
+  # documentation for more information about their specific syntax and use.
+  config.vm.provision "shell", path: "misc/provision.sh" 
+end

datajoint/__init__.py

@@ -18,16 +18,14 @@
            'Connection', 'Heading', 'Relation', 'FreeRelation', 'Not',
            'Relation', 'schema',
            'Manual', 'Lookup', 'Imported', 'Computed',
-           'conn', 'DataJointError']
-
+           'conn']
 
 class DataJointError(Exception):
     """
     Base class for errors specific to DataJoint internal operation.
     """
     pass
 
-
 # ----------- loads local configuration from file ----------------
 from .settings import Config, CONFIGVAR, LOCALCONFIG, logger, log_levels
 config = Config()
@@ -53,3 +51,4 @@ class DataJointError(Exception):
 from .relational_operand import Not
 from .heading import Heading
 from .schema import schema
+

datajoint/connection.py

@@ -7,8 +7,9 @@
 import pymysql
 import logging
 from collections import defaultdict
-from . import DataJointError, config
-from .erd import ERD
+from . import config
+from . import DataJointError
+from .erd import ERM
 from .jobs import JobManager
 
 logger = logging.getLogger(__name__)
@@ -53,7 +54,7 @@ class Connection:
     """
 
     def __init__(self, host, user, passwd, init_fun=None):
-        self.erd = ERD()
+        self.erm = ERM(self)
         if ':' in host:
             host, port = host.split(':')
             port = int(port)
@@ -62,7 +63,7 @@ def __init__(self, host, user, passwd, init_fun=None):
         self.conn_info = dict(host=host, port=port, user=user, passwd=passwd)
         self._conn = pymysql.connect(init_command=init_fun, **self.conn_info)
         if self.is_connected:
-            logger.info("Connected " + user + '@' + host + ':' + str(port))
+            logger.info("Connected {user}@{host}:{port}".format(**self.conn_info))
         else:
             raise DataJointError('Connection failed.')
         self._conn.autocommit(True)
@@ -76,22 +77,21 @@ def __del__(self):
     def __eq__(self, other):
         return self.conn_info == other.conn_info
 
+    def __repr__(self):
+        connected = "connected" if self.is_connected else "disconnected"
+        return "DataJoint connection ({connected}) {user}@{host}:{port}".format(
+            connected=connected, **self.conn_info)
+
     @property
     def is_connected(self):
         """
         Returns true if the object is connected to the database server.
         """
         return self._conn.ping()
 
-    def __repr__(self):
-        connected = "connected" if self.is_connected else "disconnected"
-        return "DataJoint connection ({connected}) {user}@{host}:{port}".format(
-            connected=connected, **self.conn_info)
-
-
     def query(self, query, args=(), as_dict=False):
         """
-        Execute the specified query and return the tuple generator.
+        Execute the specified query and return the tuple generator (cursor).
 
         :param query: mysql query
         :param args: additional arguments for the pymysql.cursor
@@ -168,4 +168,4 @@ def transaction(self):
             self.cancel_transaction()
             raise
         else:
-            self.commit_transaction()
+            self.commit_transaction()

datajoint/declare.py

@@ -8,12 +8,10 @@
 
 from . import DataJointError
 
-
 logger = logging.getLogger(__name__)
 
 
-
-def declare(full_table_name,  definition, context):
+def declare(full_table_name, definition, context):
     """
     Parse declaration and create new SQL table accordingly.
 
@@ -24,6 +22,7 @@ def declare(full_table_name,  definition, context):
     # split definition into lines
     definition = re.split(r'\s*\n\s*', definition.strip())
 
+    # check for optional table comment
     table_comment = definition.pop(0)[1:].strip() if definition[0].startswith('#') else ''
 
     in_key = True  # parse primary keys
@@ -40,7 +39,7 @@ def declare(full_table_name,  definition, context):
             in_key = False  # start parsing dependent attributes
         elif line.startswith('->'):
             # foreign key
-            ref = eval(line[2:], context)()
+            ref = eval(line[2:], context)() # TODO: surround this with try...except... to give a better error message
             foreign_key_sql.append(
                 'FOREIGN KEY ({primary_key})'
                 ' REFERENCES {ref} ({primary_key})'
@@ -52,7 +51,7 @@ def declare(full_table_name,  definition, context):
                     primary_key.append(name)
                 if name not in attributes:
                     attributes.append(name)
-                    attribute_sql.append(ref.heading[name].sql())
+                    attribute_sql.append(ref.heading[name].sql)
         elif re.match(r'^(unique\s+)?index[^:]*$', line, re.I):   # index
             index_sql.append(line)  # the SQL syntax is identical to DataJoint's
         else:

datajoint/erd.py

@@ -1,36 +1,46 @@
+from matplotlib import transforms
+
+import numpy as np
+
 import logging
-import pyparsing as pp
 import re
+from collections import defaultdict
+import pyparsing as pp
 import networkx as nx
 from networkx import DiGraph
 from networkx import pygraphviz_layout
-import numpy as np
 import matplotlib.pyplot as plt
-from matplotlib import transforms
-from collections import defaultdict
-
 from . import DataJointError
-
+from .utils import to_camel_case
 
 logger = logging.getLogger(__name__)
 
 
-class ERD:
-    _checked_dependencies = set()
-    _parents = dict()
-    _referenced = dict()
-    _children = defaultdict(list)
-    _references = defaultdict(list)
+class ERM:
+    """
+    Entity Relation Map
+
+    Represents known relation between tables
+    """
+    #_checked_dependencies = set()
+
+    def __init__(self, conn):
+        self._conn = conn
+        self._parents = dict()
+        self._referenced = dict()
+        self._children = defaultdict(list)
+        self._references = defaultdict(list)
+
 
-    def load_dependencies(self, connection, full_table_name):
+    def load_dependencies(self, full_table_name):
         # check if already loaded.  Use clear_dependencies before reloading
         if full_table_name in self._parents:
             return
         self._parents[full_table_name] = list()
         self._referenced[full_table_name] = list()
 
         # fetch the CREATE TABLE statement
-        cur = connection.query('SHOW CREATE TABLE %s' % full_table_name)
+        cur = self._conn.query('SHOW CREATE TABLE %s' % full_table_name)
         create_statement = cur.fetchone()
         if not create_statement:
             raise DataJointError('Could not load the definition table %s' % full_table_name)
@@ -127,20 +137,6 @@ def recurse(full_table_name, level):
         return sorted(ret.keys(), key=ret.__getitem__)
 
 
-def to_camel_case(s):
-    """
-    Convert names with under score (_) separation
-    into camel case names.
-    Example:
-    >>>to_camel_case("table_name")
-        "TableName"
-    """
-    def to_upper(match):
-        return match.group(0)[-1].upper()
-    return re.sub('(^|[_\W])+[a-zA-Z]', to_upper, s)
-
-
-
 class RelGraph(DiGraph):
     """
     A directed graph representing relations between tables within and across