Update database interface to sqlalchemy

Also adds ORM models, updates tests and uses testing.postgresql
to test against a live database.

app/dbinterface.py

@@ -1,34 +1,19 @@
-import datetime
-import json
+import functools
 import os
 import random
 
-import dataset
-import pylibmc
-import tornado.options
-
+import sqlalchemy as sa
 from hashids import Hashids
 from twiggy import log
-log = log.name(__name__)
 
-tornado.options.define('db_file',
-                       default='/var/ipborgdb/ipborg.db',
-                       type=str)
-tornado.options.define('secret_salt', type=str)
-tornado.options.define('public_salt', type=str)
-
-JSONIZE_KEYS = {'python_version', 'code_cells', 'grid_data'}
-PUBLIC_TABLE = 'public_grids'
-SECRET_TABLE = 'secret_grids'
-HASH_MIN_LENGTH = 6
+from . import models
 
+log = log.name(__name__)
 
-def get_memcached():
-    host = os.environ.get('MC_PORT', '127.0.0.1').replace('tcp://', '')
-    log.fields(mc_host=host).debug('connecting to memcached')
-    return pylibmc.Client([host], binary=True)
+HASH_MIN_LENGTH = 6
 
 
+@functools.lru_cache(maxsize=2)
 def get_hashids(secret):
     """
     Return the appropriate Hashids instance depending whether it's
@@ -43,11 +28,14 @@ def get_hashids(secret):
     hashids : hashids.Hashids instance
 
     """
-    opts = tornado.options.options
-    salt = opts.secret_salt if secret else opts.public_salt
+    salt = (
+        os.environ['HASHIDS_SECRET_SALT']
+        if secret
+        else os.environ['HASHIDS_PUBLIC_SALT'])
     return Hashids(salt=salt, min_length=HASH_MIN_LENGTH)
 
 
+@functools.lru_cache()
 def encode_grid_id(grid_id, secret):
     """
     Turn an integer grid ID into a hash ID to be used in a URL.
@@ -66,6 +54,7 @@ def encode_grid_id(grid_id, secret):
     return hashids.encrypt(grid_id)
 
 
+@functools.lru_cache()
 def decode_hash_id(hash_id, secret):
     """
     Turn a hash ID from a URL into an integer grid ID for database lookup.
@@ -87,111 +76,40 @@ def decode_hash_id(hash_id, secret):
         return dec[0]
 
 
-def get_db():
-    """
-    Get a pointer to the databse.
-
-    Returns
-    -------
-    db : dataset.Database
-
-    """
-    return dataset.connect('sqlite:///' + tornado.options.options.db_file)
-
-
-def get_table(secret):
-    """
-    Return the appropriate dataset.Table instance.
-
-    Parameters
-    ----------
-    secret : bool
-        Whether it's a secret grid.
-
-    Returns
-    -------
-    table : dataset.Table
-
-    """
-    db = get_db()
-    return db[PUBLIC_TABLE] if not secret else db[SECRET_TABLE]
-
-
-def sqlize_grid_spec(grid_spec):
-    """
-    Not all of grid_spec's fields match sqlite types,
-    this will convert them to JSON strings.
-
-    Parameters
-    ----------
-    grid_spec : dict
-
-    Returns
-    -------
-    grid_entry : dict
-
-    """
-    grid_entry = grid_spec.copy()
-
-    for k in JSONIZE_KEYS:
-        grid_entry[k] = json.dumps(grid_entry[k])
-
-    return grid_entry
-
-
-def desqlize_grid_entry(grid_entry):
-    """
-    Not all of grid_spec's fields match sqlite types,
-    so some of them are stored as JSON strings.
-    This will convert them from the strings back to the useful types.
-
-    Parameters
-    ----------
-    grid_entry : dict
-
-    Returns
-    -------
-    grid_spec : dict
-
-    """
-    grid_spec = grid_entry.copy()
-
-    for k in JSONIZE_KEYS:
-        grid_spec[k] = json.loads(grid_spec[k])
-
-    return grid_spec
-
-
-def store_grid_entry(grid_spec):
+def store_grid_entry(session, grid_spec):
     """
     Add a grid spec to the database and return the grid's unique ID.
 
     Parameters
     ----------
+    session : sqlalchemy.orm.session.Session
     grid_spec : dict
 
     Returns
     -------
     hash_id : str
 
     """
-    grid_entry = sqlize_grid_spec(grid_spec)
-
-    llog = log.fields(secret=grid_entry['secret'])
+    llog = log.fields(secret=grid_spec['secret'])
     llog.debug('storing grid')
-    table = get_table(grid_entry['secret'])
-    grid_id = table.insert(grid_entry)
-    llog.fields(grid_id=grid_id).debug('grid stored')
 
-    return encode_grid_id(grid_id, grid_entry['secret'])
+    table = models.SecretGrid if grid_spec['secret'] else models.PublicGrid
+    new_grid = table(**grid_spec)
+    session.add(new_grid)
+    session.flush()
+
+    llog.fields(grid_id=new_grid.id).debug('grid stored')
+
+    return encode_grid_id(new_grid.id, grid_spec['secret'])
 
 
-def get_grid_entry(hash_id, secret=False):
+def get_grid_entry(session, hash_id, secret=False):
     """
     Get a specific grid entry.
 
     Parameters
     ----------
+    session : sqlalchemy.orm.session.Session
     hash_id : str
     secret : bool, optional
         Whether this is a secret grid.
@@ -209,43 +127,29 @@ def get_grid_entry(hash_id, secret=False):
         llog.debug('cannot decrypt hash')
         return
 
-    llog.debug('looking for grid')
-
-    mc = get_memcached()
-    mc_key = str((grid_id, secret))
-    if mc_key in mc:
-        llog.debug('pulling grid from memcached')
-        return mc[mc_key]
-
     llog.debug('pulling grid from database')
-    table = get_table(secret)
-    grid_spec = table.find_one(id=grid_id)
-
-    if grid_spec:
-        llog.debug('grid found')
-        grid_spec = desqlize_grid_entry(grid_spec)
-        mc[mc_key] = grid_spec
-
-    else:
-        llog.debug('grid not found')
-        return
+    table = models.SecretGrid if secret else models.PublicGrid
+    grid_spec = session.query(table).filter(table.id == grid_id).one_or_none()
 
     return grid_spec
 
 
-def get_random_hash_id():
+def get_random_hash_id(session):
     """
     Get a random, non-secret grid id.
 
+    Parameters
+    ----------
+    session : sqlalchemy.orm.session.Session
+
     Returns
     -------
     hash_id : str
 
     """
-    query = ('SELECT id '
-             'FROM {} '
-             'ORDER BY RANDOM() '
-             'LIMIT 1').format(PUBLIC_TABLE)
-    db = get_db()
-    cursor = db.query(query)
-    return encode_grid_id(tuple(cursor)[0]['id'], secret=False)
+    # this is okay right now because I know every ID up to the max is present
+    # in the database, but if I ever do cleanup and delete rows I'll need
+    # to switch to another strategy here.
+    max_id = session.query(sa.func.max(models.PublicGrid.id)).scalar()
+    random_grid_id = random.randint(1, max_id)
+    return encode_grid_id(random_grid_id, secret=False)

app/models.py

@@ -0,0 +1,37 @@
+"""Definition of SQL tables used to store ipythonblocks grid data"""
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql as pg
+from sqlalchemy.ext.declarative import declarative_base
+
+Base = declarative_base()
+
+
+class CommonColumnsMixin:
+    """Columns common to both public and secret grids"""
+    __table_args__ = {'schema': 'public'}  # "public" is postgres' default schema
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    ipb_version = sa.Column(sa.Text, nullable=False)
+    python_version = sa.Column(pg.JSONB, nullable=False)
+    grid_data = sa.Column(pg.JSONB, nullable=False)
+    code_cells = sa.Column(pg.JSONB)
+    ipb_class = sa.Column(sa.Text, nullable=False)
+    created_at = sa.Column(
+        sa.DateTime(timezone=True), nullable=False,
+        server_default=sa.text('NOW()'))
+
+
+class PublicGrid(CommonColumnsMixin, Base):
+    """Table to hold public grids (discoverable via "random")"""
+    __tablename__ = 'public_grids'
+
+    # no-op column, but put it here anyway
+    secret = sa.Column(sa.Boolean, nullable=False, default=False)
+
+
+class SecretGrid(CommonColumnsMixin, Base):
+    """Table to hold secret grids"""
+    __tablename__ = 'secret_grids'
+
+    # no-op column, but put it here anyway
+    secret = sa.Column(sa.Boolean, nullable=False, default=True)