Permalink
Browse files

Update database interface to sqlalchemy

Also adds ORM models, updates tests and uses testing.postgresql
to test against a live database.
  • Loading branch information...
jiffyclub committed Jul 8, 2017
1 parent 57d8226 commit d7412b4c3ee7e81a0a20662356f2a48ad0c7fea3
Showing with 126 additions and 195 deletions.
  1. +38 −134 app/dbinterface.py
  2. +37 −0 app/models.py
  3. +51 −61 app/test/test_dbinterface.py
View
@@ -1,34 +1,19 @@
import datetime
import json
import functools
import os
import random
import dataset
import pylibmc
import tornado.options
import sqlalchemy as sa
from hashids import Hashids
from twiggy import log
log = log.name(__name__)
tornado.options.define('db_file',
default='/var/ipborgdb/ipborg.db',
type=str)
tornado.options.define('secret_salt', type=str)
tornado.options.define('public_salt', type=str)
JSONIZE_KEYS = {'python_version', 'code_cells', 'grid_data'}
PUBLIC_TABLE = 'public_grids'
SECRET_TABLE = 'secret_grids'
HASH_MIN_LENGTH = 6
from . import models
log = log.name(__name__)
def get_memcached():
host = os.environ.get('MC_PORT', '127.0.0.1').replace('tcp://', '')
log.fields(mc_host=host).debug('connecting to memcached')
return pylibmc.Client([host], binary=True)
HASH_MIN_LENGTH = 6
@functools.lru_cache(maxsize=2)
def get_hashids(secret):
"""
Return the appropriate Hashids instance depending whether it's
@@ -43,11 +28,14 @@ def get_hashids(secret):
hashids : hashids.Hashids instance
"""
opts = tornado.options.options
salt = opts.secret_salt if secret else opts.public_salt
salt = (
os.environ['HASHIDS_SECRET_SALT']
if secret
else os.environ['HASHIDS_PUBLIC_SALT'])
return Hashids(salt=salt, min_length=HASH_MIN_LENGTH)
@functools.lru_cache()
def encode_grid_id(grid_id, secret):
"""
Turn an integer grid ID into a hash ID to be used in a URL.
@@ -66,6 +54,7 @@ def encode_grid_id(grid_id, secret):
return hashids.encrypt(grid_id)
@functools.lru_cache()
def decode_hash_id(hash_id, secret):
"""
Turn a hash ID from a URL into an integer grid ID for database lookup.
@@ -87,111 +76,40 @@ def decode_hash_id(hash_id, secret):
return dec[0]
def get_db():
"""
Get a pointer to the databse.
Returns
-------
db : dataset.Database
"""
return dataset.connect('sqlite:///' + tornado.options.options.db_file)
def get_table(secret):
"""
Return the appropriate dataset.Table instance.
Parameters
----------
secret : bool
Whether it's a secret grid.
Returns
-------
table : dataset.Table
"""
db = get_db()
return db[PUBLIC_TABLE] if not secret else db[SECRET_TABLE]
def sqlize_grid_spec(grid_spec):
"""
Not all of grid_spec's fields match sqlite types,
this will convert them to JSON strings.
Parameters
----------
grid_spec : dict
Returns
-------
grid_entry : dict
"""
grid_entry = grid_spec.copy()
for k in JSONIZE_KEYS:
grid_entry[k] = json.dumps(grid_entry[k])
return grid_entry
def desqlize_grid_entry(grid_entry):
"""
Not all of grid_spec's fields match sqlite types,
so some of them are stored as JSON strings.
This will convert them from the strings back to the useful types.
Parameters
----------
grid_entry : dict
Returns
-------
grid_spec : dict
"""
grid_spec = grid_entry.copy()
for k in JSONIZE_KEYS:
grid_spec[k] = json.loads(grid_spec[k])
return grid_spec
def store_grid_entry(grid_spec):
def store_grid_entry(session, grid_spec):
"""
Add a grid spec to the database and return the grid's unique ID.
Parameters
----------
session : sqlalchemy.orm.session.Session
grid_spec : dict
Returns
-------
hash_id : str
"""
grid_entry = sqlize_grid_spec(grid_spec)
llog = log.fields(secret=grid_entry['secret'])
llog = log.fields(secret=grid_spec['secret'])
llog.debug('storing grid')
table = get_table(grid_entry['secret'])
grid_id = table.insert(grid_entry)
llog.fields(grid_id=grid_id).debug('grid stored')
return encode_grid_id(grid_id, grid_entry['secret'])
table = models.SecretGrid if grid_spec['secret'] else models.PublicGrid
new_grid = table(**grid_spec)
session.add(new_grid)
session.flush()
llog.fields(grid_id=new_grid.id).debug('grid stored')
return encode_grid_id(new_grid.id, grid_spec['secret'])
def get_grid_entry(hash_id, secret=False):
def get_grid_entry(session, hash_id, secret=False):
"""
Get a specific grid entry.
Parameters
----------
session : sqlalchemy.orm.session.Session
hash_id : str
secret : bool, optional
Whether this is a secret grid.
@@ -209,43 +127,29 @@ def get_grid_entry(hash_id, secret=False):
llog.debug('cannot decrypt hash')
return
llog.debug('looking for grid')
mc = get_memcached()
mc_key = str((grid_id, secret))
if mc_key in mc:
llog.debug('pulling grid from memcached')
return mc[mc_key]
llog.debug('pulling grid from database')
table = get_table(secret)
grid_spec = table.find_one(id=grid_id)
if grid_spec:
llog.debug('grid found')
grid_spec = desqlize_grid_entry(grid_spec)
mc[mc_key] = grid_spec
else:
llog.debug('grid not found')
return
table = models.SecretGrid if secret else models.PublicGrid
grid_spec = session.query(table).filter(table.id == grid_id).one_or_none()
return grid_spec
def get_random_hash_id():
def get_random_hash_id(session):
"""
Get a random, non-secret grid id.
Parameters
----------
session : sqlalchemy.orm.session.Session
Returns
-------
hash_id : str
"""
query = ('SELECT id '
'FROM {} '
'ORDER BY RANDOM() '
'LIMIT 1').format(PUBLIC_TABLE)
db = get_db()
cursor = db.query(query)
return encode_grid_id(tuple(cursor)[0]['id'], secret=False)
# this is okay right now because I know every ID up to the max is present
# in the database, but if I ever do cleanup and delete rows I'll need
# to switch to another strategy here.
max_id = session.query(sa.func.max(models.PublicGrid.id)).scalar()
random_grid_id = random.randint(1, max_id)
return encode_grid_id(random_grid_id, secret=False)
View
@@ -0,0 +1,37 @@
"""Definition of SQL tables used to store ipythonblocks grid data"""
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql as pg
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class CommonColumnsMixin:
"""Columns common to both public and secret grids"""
__table_args__ = {'schema': 'public'} # "public" is postgres' default schema
id = sa.Column(sa.Integer, primary_key=True)
ipb_version = sa.Column(sa.Text, nullable=False)
python_version = sa.Column(pg.JSONB, nullable=False)
grid_data = sa.Column(pg.JSONB, nullable=False)
code_cells = sa.Column(pg.JSONB)
ipb_class = sa.Column(sa.Text, nullable=False)
created_at = sa.Column(
sa.DateTime(timezone=True), nullable=False,
server_default=sa.text('NOW()'))
class PublicGrid(CommonColumnsMixin, Base):
"""Table to hold public grids (discoverable via "random")"""
__tablename__ = 'public_grids'
# no-op column, but put it here anyway
secret = sa.Column(sa.Boolean, nullable=False, default=False)
class SecretGrid(CommonColumnsMixin, Base):
"""Table to hold secret grids"""
__tablename__ = 'secret_grids'
# no-op column, but put it here anyway
secret = sa.Column(sa.Boolean, nullable=False, default=True)
Oops, something went wrong.

0 comments on commit d7412b4

Please sign in to comment.