Use SQLAlchemy idiomatically and implement Response using specification.

Use SQLAlchemy connection pool using MySQL config file.
Use SQLAlchemy text function for SQL.
Move engine to Flask app's config.
Depend on webservcommon.
Depend on anaconda instead of flask
Fix some formatting.

bin/metaServer.py

@@ -31,12 +31,33 @@
 from flask import Flask, request
 import json
 import logging as log
+import os
 import sys
 
+import ConfigParser
+import sqlalchemy
+from sqlalchemy.engine.url import URL
+
 from lsst.metaserv import metaREST_v0
 
 app = Flask(__name__)
 
+def initEngine():
+    config = ConfigParser.ConfigParser()
+    defaults_file = os.path.expanduser("~/.lsst/dbAuth-dbServ.txt")
+    config.readfp(open(defaults_file))
+    db_config = dict(config.items("mysql"))
+    # Translate user name
+    db_config["username"] = db_config["user"]
+    del db_config["user"]
+    # SQLAlchemy part
+    url = URL("mysql",**db_config)
+    return sqlalchemy.create_engine(url)
+
+engine = initEngine()
+
+app.config["default_engine"] = engine
+
 @app.route('/')
 def getRoot():
     fmt = request.accept_mimetypes.best_match(['application/json', 'text/html'])

python/lsst/metaserv/metaREST_v0.py

@@ -24,134 +24,119 @@
 supported formats: json and html.
 
 @author  Jacek Becla, SLAC
-
-# todos:
-#  * known issue: it blocks commands such as "drop database", because
-#    it keeps connection option. Close connection per request?
-#  * migrate to db, and use execCommands etc from there.
-#  * generate proper html header
+@author Brian Van Klaveren, SLAC
 """
 
-from flask import Blueprint, request
+from flask import Blueprint, request, current_app, make_response
+from lsst.webservcommon import renderJsonResponse
+
+from httplib import OK, NOT_FOUND, INTERNAL_SERVER_ERROR
 import json
 import logging as log
+import re
+from sqlalchemy import text
+from sqlalchemy.exc import SQLAlchemyError
 
-from lsst.db.dbPool import DbPool
-
-metaREST = Blueprint('metaREST', __name__, template_folder='metaserv')
+SAFE_NAME_REGEX = r'[a-zA-Z0-9_]+$'
+SAFE_SCHEMA_PATTERN = re.compile(SAFE_NAME_REGEX)
+SAFE_TABLE_PATTERN = re.compile(SAFE_NAME_REGEX)
 
-# Connect to the metaserv database. Note that the metaserv typically runs for
-# a long time, and the connection can timeout if there long period of inactivity.
-# Use the DbPool, which will keep the connection alive.
-dbPool = DbPool()
-dbPool.addConn("c1", read_default_file="~/.lsst/dbAuth-metaServ.txt")
+metaREST = Blueprint('metaREST', __name__, template_folder="templates")
 
 
-def runDbQuery1(query, optParams=None, notFoundMsg='Not found'):
-    '''Runs query that returns one row. It can raise DbException or mysql
-    exception.'''
-    cursor = dbPool.getConn("c1").getCursor()
-    log.debug("Executing '%s', optParams: %s.", query, optParams)
-    cursor.execute(query, optParams)
-    row = cursor.fetchone()
-    log.debug("Got: %s", row)
-    fmt = request.accept_mimetypes.best_match(['application/json', 'text/html'])
-    if not row:
-        retStr = notFoundMsg
-    else:
-        retStr = ''
-        for x in range(0, len(row)):
-            if fmt == "text/html":
-                retStr += "%s: %s<br />" % (cursor.description[x][0], row[x])
-            else: # default format is application/json
-                retStr += "%s:%s " % (cursor.description[x][0], row[x])
-        if fmt == "application/json":
-            retStr = json.dumps(retStr)
-    return retStr
-
-def runDbQueryM(query, optParams=None, notFoundMsg='Not found'):
-    '''Runs query that returns many rows. It can raise DbException or mysql
-    exception.'''
-    rows = dbPool.getConn("c1").execCommandN(query, optParams)
-    fmt = request.accept_mimetypes.best_match(['application/json', 'text/html'])
-    if len(rows) == 0:
-        retStr = notFoundMsg
-    else:
-        if fmt == 'text/html':
-            retStr = "<br />".join(str(r[0]) for r in rows)
-        else: # default format is application/json
-            ret = " ".join(str(r[0]) for r in rows)
-            retStr = json.dumps(ret)
-    return retStr
-
 @metaREST.route('/', methods=['GET'])
 def getRoot():
     fmt = request.accept_mimetypes.best_match(['application/json', 'text/html'])
     if fmt == 'text/html':
-        return ("LSST Metadata Service v0 here. I currently support: "
-                "<a href='db'>/db</a> and <a href='image'>/image</a>.")
-    return "LSST Metadata Service v0 here. I currently support: /db and /image."
+        return "LSST Metadata Service v0. See: <a href='db'>/db</a> and <a href='image'>/image</a>."
+    return "LSST Metadata Service v0. See: /db and /image."
+
 
 @metaREST.route('/db', methods=['GET'])
 def getDb():
     '''Lists types of databases (that have at least one database).'''
-    # todo: this will currently fail if Repo table does not exist
-    return runDbQueryM(
-        "SELECT DISTINCT lsstLevel FROM Repo WHERE repoType = 'db'",
-        None,
-        "No types with at least one database found.")
+    query = "SELECT DISTINCT lsstLevel FROM Repo WHERE repoType = 'db'"
+    return _resultsOf(text(query), scalar=True)
+
 
 @metaREST.route('/db/<string:lsstLevel>', methods=['GET'])
 def getDbPerType(lsstLevel):
     '''Lists databases for a given type.'''
-    # todo: this will currently fail if Repo table does not exist
-    return runDbQueryM(
-        "SELECT dbName FROM Repo JOIN DbMeta on (repoId=dbMetaId) "
-        "WHERE lsstLevel = %s",
-        (lsstLevel,),
-        "No database found.")
+    query = "SELECT dbName FROM Repo JOIN DbMeta on (repoId=dbMetaId) WHERE lsstLevel = :lsstLevel"
+    return _resultsOf(text(query), paramMap={"lsstLevel": lsstLevel})
+
 
 @metaREST.route('/db/<string:lsstLevel>/<string:dbName>', methods=['GET'])
 def getDbPerTypeDbName(lsstLevel, dbName):
     '''Retrieves information about one database.'''
     # We don't use lsstLevel here because db names are unique across all types.
-    return runDbQuery1(
-        "SELECT Repo.*, DbMeta.* "
-        "FROM Repo JOIN DbMeta on (repoId=dbMetaId) "
-        "WHERE dbName = %s",
-        (dbName,),
-        "Database '%s' not found." % dbName)
+    query = "SELECT Repo.*, DbMeta.* " \
+            "FROM Repo JOIN DbMeta on (repoId=dbMetaId) WHERE dbName = :dbName"
+    return _resultsOf(text(query), paramMap={"dbName": dbName}, scalar=True)
+
 
 @metaREST.route('/db/<string:lsstLevel>/<string:dbName>/tables', methods=['GET'])
 def getDbPerTypeDbNameTables(lsstLevel, dbName):
     '''Lists table names in a given database.'''
-    return runDbQueryM(
-        "SELECT table_name FROM information_schema.tables "
-        "WHERE table_schema= %s ",
-        (dbName,),
-        "No tables found in database '%s'." % dbName)
+    query = "SELECT table_name FROM information_schema.tables WHERE table_schema=:dbName"
+    return _resultsOf(text(query), paramMap={"dbName": dbName})
+
 
 @metaREST.route('/db/<string:lsstLevel>/<string:dbName>/tables/'
                 '<string:tableName>', methods=['GET'])
 def getDbPerTypeDbNameTablesTableName(lsstLevel, dbName, tableName):
     '''Retrieves information about a table from a given database.'''
-    return runDbQuery1(
-        "SELECT DDT_Table.* FROM DDT_Table JOIN DbMeta USING (dbMetaId) "
-        "WHERE dbName=%s AND tableName=%s",
-        (dbName, tableName),
-        "Table '%s.%s'not found." % (dbName, tableName))
+    query = "SELECT DDT_Table.* FROM DDT_Table " \
+            "JOIN DbMeta USING (dbMetaId) " \
+            "WHERE dbName=:dbName AND tableName=:tableName"
+    return _resultsOf(text(query), paramMap={"dbName": dbName, "tableName": tableName}, scalar=True)
+
 
 @metaREST.route('/db/<string:lsstLevel>/<string:dbName>/' +
                 'tables/<string:tableName>/schema', methods=['GET'])
 def getDbPerTypeDbNameTablesTableNameSchema(lsstLevel, dbName, tableName):
     '''Retrieves schema for a given table.'''
-    return runDbQuery1(
-        "SHOW CREATE TABLE %s.%s" % (dbName, tableName),
-        None,
-        "Table '%s.%s'not found." % (dbName, tableName))
+    # Scalar
+    if SAFE_SCHEMA_PATTERN.match(dbName) and SAFE_TABLE_PATTERN.match(tableName):
+        query = "SHOW CREATE TABLE %s.%s" % (dbName, tableName)
+        return _resultsOf(query, scalar=True)
+    return _response(_error("ValueError", "Database name or Table name is not safe"), 400)
+
 
 @metaREST.route('/image', methods=['GET'])
 def getImage():
-    return ("meta/.../image not implemented. I am supposed to print list of "
-            "supported image types here, something like: raw, template, coadd, "
-            "jpeg, calexp, ... etc")
+    return "meta/.../image not implemented. I am supposed to print list of " \
+           "supported image types here, something like: raw, template, coadd, " \
+           "jpeg, calexp, ... etc"
+
+
+_error = lambda exception, message: {"exception": exception, "message": message}
+_vector = lambda results: {"results": results}
+_scalar = lambda result: {"result": result}
+
+
+def _resultsOf(query, paramMap=None, scalar=False):
+    status_code = OK
+    paramMap = paramMap or {}
+    try:
+        engine = current_app.config["default_engine"]
+        if scalar:
+            result = list(engine.execute(query, **paramMap).first())
+            response = _scalar(result)
+        else:
+            results = [list(result) for result in engine.execute(query, **paramMap)]
+            response = _vector(results)
+    except SQLAlchemyError as e:
+        log.debug("Encountered an error processing request: '%s'" % e.message)
+        status_code = INTERNAL_SERVER_ERROR
+        response = _error(type(e).__name__, e.message)
+    return _response(response, status_code)
+
+
+def _response(response, status_code):
+    fmt = request.accept_mimetypes.best_match(['application/json', 'text/html'])
+    if fmt == 'text/html':
+        response = renderJsonResponse(response=response, status_code=status_code)
+    else:
+        response = json.dumps(response)
+    return make_response(response, status_code)

ups/metaserv.table

@@ -1,5 +1,6 @@
 setupRequired(db)
-setupRequired(flask)
+setupRequired(webservcommon)
+setupRequired(anaconda)
 setupRequired(mysqlpython)
 setupRequired(sconsUtils)