Initial TAP-like implementation.

README.txt

@@ -10,13 +10,16 @@ sudo aptitude install python-flask
   python bin/dbServer.py
 
   # and run some queries:
-  curl 'http://localhost:5000/db'
-  curl 'http://localhost:5000/db/v0'
-  curl 'http://localhost:5000/db/v0/query'
-  curl 'http://localhost:5000/db/v0/query?sql=SHOW+DATABASES+LIKE+"%Stripe%"'
-  curl 'http://localhost:5000/db/v0/query?sql=SHOW+TABLES+IN+DC_W13_Stripe82'
-  curl 'http://localhost:5000/db/v0/query?sql=DESCRIBE+DC_W13_Stripe82.DeepForcedSource'
-  curl 'http://localhost:5000/db/v0/query?sql=DESCRIBE+DC_W13_Stripe82.Science_Ccd_Exposure'
-  curl 'http://localhost:5000/db/v0/query?sql=SELECT+deepForcedSourceId,scienceCcdExposureId+FROM+DC_W13_Stripe82.DeepForcedSource+LIMIT+10'
-  curl 'http://localhost:5000/db/v0/query?sql=SELECT+ra,decl,filterName+FROM+DC_W13_Stripe82.Science_Ccd_Exposure+WHERE+scienceCcdExposureId=125230127'
- curl 'http://localhost:5000/image/v0/raw/cutout?ra=7.90481567257&dec=-0.299951669961&filter=r&width=30.0&height=45.0'
+  curl 'http://localhost:5000/'
+  curl 'http://localhost:5000/tap'
+  curl 'http://localhost:5000/tap/sync'
+  curl -d 'query=SHOW+DATABASES+LIKE+"%Stripe%"' http://localhost:5000/tap/sync
+  curl -d 'query=SHOW+TABLES+IN+DC_W13_Stripe82' http://localhost:5000/tap/sync
+  curl -d 'query=DESCRIBE+DC_W13_Stripe82.DeepForcedSource' http://localhost:5000/tap/sync
+  curl -d 'query=DESCRIBE+DC_W13_Stripe82.Science_Ccd_Exposure' http://localhost:5000/tap/sync
+  curl -d 'query=SELECT+deepForcedSourceId,scienceCcdExposureId+FROM+DC_W13_Stripe82.DeepForcedSource+LIMIT+10' http://localhost:5000/tap/sync
+  curl -d 'query=SELECT+ra,decl,filterName+FROM+DC_W13_Stripe82.Science_Ccd_Exposure+WHERE+scienceCcdExposureId=125230127' http://localhost:5000/tap/sync
+
+  You can also use alternative Content types by adding the following flags to curl:
+  -H "Accept: text/html"
+  -H "Accept: application/x-votable+xml"

bin/dbServer.py

@@ -31,49 +31,29 @@
 from flask import Flask, request
 import json
 import logging as log
-import os
 import sys
-
-import ConfigParser
-import sqlalchemy
-from sqlalchemy.engine.url import URL
-
 from lsst.dax.dbserv import dbREST_v0
+from lsst.db.engineFactory import getEngineFromFile
 
 app = Flask(__name__)
 
-def initEngine():
-    config = ConfigParser.ConfigParser()
-    defaults_file = os.path.expanduser("~/.lsst/dbAuth-dbServ.txt")
-    config.readfp(open(defaults_file))
-    db_config = dict(config.items("mysql"))
-    # Translate user name
-    db_config["username"] = db_config["user"]
-    del db_config["user"]
-    # SQLAlchemy part
-    url = URL("mysql",**db_config)
-    return sqlalchemy.create_engine(url)
-
-engine = initEngine()
-
+# Configure Engine
+defaults_file = "~/.lsst/dbAuth-dbServ.ini"
+engine = getEngineFromFile(defaults_file)
 app.config["default_engine"] = engine
 
-@app.route('/')
-def getRoot():
-    return '''Test server for testing db. Try adding /db to URI.
-'''
 
-@app.route('/db')
-def getDb():
-    '''Lists supported versions for /db.'''
+@app.route('/')
+def application_root():
+    """In standalone mode, this handles requests above the tap service"""
     fmt = request.accept_mimetypes.best_match(['application/json', 'text/html'])
-    s = '''v0
-'''
+    s = "tap"
     if fmt == "text/html":
         return s
     return json.dumps(s)
 
-app.register_blueprint(dbREST_v0.dbREST, url_prefix='/db/v0')
+
+app.register_blueprint(dbREST_v0.dbREST, url_prefix='/tap')
 
 if __name__ == '__main__':
     log.basicConfig(

python/lsst/dax/dbserv/compat/fields.py

@@ -0,0 +1,88 @@
+# LSST Data Management System
+# Copyright 2016 AURA/LSST.
+#
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+
+"""
+This module is intended to help us introspect DBAPI cursor
+metadata and rows to determine field types for a given result set.
+"""
+
+from base64 import b64encode
+
+import MySQLdb
+from MySQLdb.constants.FLAG import BINARY
+
+
+class MySQLFieldHelper:
+    def __init__(self, description, flags, value):
+        """
+        Helper class to define a column, get it's type for conversion, and convert types if needed.
+        This class works on a best-effort basis. It's not guaranteed to be 100% correct.
+        @param flags: Flags from MySQLdb.constants.FLAGS
+        @param value: An example value type to help with inferring how to convert
+        """
+        self.datatype = None
+        self.xtype = None
+        self.converter = None
+        self.name = description[0]
+
+        type_code = description[1]
+        scale = description[5]
+
+        if type_code in MySQLdb.NUMBER:
+            # Use python types first, fallback on float otherwise (e.g. NoneType)
+            if isinstance(value, int):
+                self.datatype = "int"
+            else:
+                # If there's a scale, use double, otherwise assume long
+                self.datatype = "double" if scale else "long"
+
+        # Check datetime and date
+        if type_code in MySQLdb.DATETIME:
+            self.datatype = "text"
+            self.xtype = "timestamp"
+            self.converter = lambda x: x.isoformat()
+        if type_code in MySQLdb.DATE:
+            self.datatype = "text"
+            self.xtype = "date"
+            self.converter = lambda x: x.isoformat()
+
+        # Check if this is binary data and potentially unsafe for JSON
+        if not self.datatype and flags & BINARY and type_code not in MySQLdb.TIME:
+            # This needs to be checked BEFORE the next type check
+            self.datatype = "binary"
+            self.converter = b64encode
+        elif isinstance(value, str):
+            self.datatype = "text"
+
+        if not self.datatype:
+            # Just return a string and make sure to convert it to string if we don't know about
+            # this type. This may include datetime.time
+            self.datatype = "text"
+            self.converter = str
+
+    def check_value(self, value):
+        """
+        Check the value returned from the DBAPI.
+        @param value:
+        @return: The value itself, or a stringified version if it needs to be stringified.
+        """
+        if self.converter and value:
+            return self.converter(value)
+        return value

python/lsst/dax/dbserv/dbREST_v0.py

@@ -21,49 +21,49 @@
 # see <http://www.lsstcorp.org/LegalNotices/>.
 
 """
-This module implements the RESTful interface for Databaser Service.
-Corresponding URI: /db. Default output format is json. Currently
-supported formats: json and html.
+This module implements the TAP and TAP-like protocols for access
+to a database.
+
+Supported formats: json and html.
 
 @author  Jacek Becla, SLAC
 
-# todos:
-#  * migrate to db, and use execCommands etc from there.
-#  * generate proper html header
 """
 
-from base64 import b64encode
-from datetime import date, datetime, time, timedelta
-from flask import Blueprint, request, current_app, make_response
-from httplib import OK, INTERNAL_SERVER_ERROR
 import json
 import logging as log
-from lsst.dax.webservcommon import renderJsonResponse
-import MySQLdb
-from MySQLdb.constants.FLAG import BINARY
+from httplib import OK, INTERNAL_SERVER_ERROR
+
+from flask import Blueprint, request, current_app, make_response, render_template
 from sqlalchemy import text
 from sqlalchemy.exc import SQLAlchemyError
 
-dbREST = Blueprint('dbREST', __name__, template_folder='dax_dbserv')
+from lsst.dax.dbserv.compat.fields import MySQLFieldHelper
+from lsst.dax.webservcommon import render_response
+
+dbREST = Blueprint('dbREST', __name__, template_folder='templates')
 
 
 @dbREST.route('/', methods=['GET'])
-def getRoot():
+def root():
     fmt = request.accept_mimetypes.best_match(['application/json', 'text/html'])
     if fmt == 'text/html':
-        return "LSST Database Service v0 here. I currently support: " \
-               "<a href='query'>/query</a>."
-    return "LSST Database Service v0 here. I currently support: /query."
-
-_error = lambda exception, message: {"exception": exception, "message": message}
-_vector = lambda results, metadata: {"results": results, "metadata": metadata}
-
-@dbREST.route('/query', methods=['GET'])
-def getQuery():
-    '''If sql is not passed, it lists quries running for a given user.
-       If sql is passed, it runs a given query.'''
-    if 'sql' in request.args:
-        sql = request.args.get('sql').encode('utf8')
+        return "LSST TAP Service v0 here. I currently support: " \
+               "<a href='query'>/sync</a>."
+    return "LSST Database Service v0 here. I currently support: /sync."
+
+
+@dbREST.route('/sync', methods=['POST'])
+def sync_query():
+    """
+    If sql is not passed, it lists queries running for a given user.
+    If sql is passed, it runs a given query.
+    :return: A proper response object
+    """
+
+    query = request.args.get("query", request.form.get("query", None))
+    if query:
+        sql = query.encode('utf8')
         log.debug(sql)
         try:
             engine = current_app.config["default_engine"]
@@ -76,14 +76,20 @@ def getQuery():
                 # If this is the first time, build column definitions (use raw values to help)
                 if not helpers:
                     for desc, flags, val in zip(curs.description, curs.description_flags, result):
-                        helpers.append(ColumnHelper(desc, flags, val))
+                        helpers.append(MySQLFieldHelper(desc, flags, val))
 
                 # Not streaming...
-                results.append([helper.checkValue(val) for helper, val in zip(helpers, result)])
+                results.append([helper.check_value(val) for helper, val in zip(helpers, result)])
 
             status_code = OK
-            metadata = {"columnDefs": [{"name": cd.name, "type": cd.type} for cd in helpers]}
-            response = _vector(results, metadata)
+            elements = []
+            for helper in helpers:
+                field = dict(name=helper.name, datatype=helper.datatype)
+                if helper.xtype:
+                    field["xtype"] = helper.xtype
+                elements.append(field)
+
+            response = _result(dict(metadata=dict(elements=elements), data=results))
         except SQLAlchemyError as e:
             log.debug("Encountered an error processing request: '%s'" % e.message)
             response = _error(type(e).__name__, e.message)
@@ -93,67 +99,26 @@ def getQuery():
         return "Listing queries is not implemented."
 
 
+def _error(exception, message):
+    return dict(error=exception, message=message)
+
+
+def _result(table):
+    return dict(result=dict(table=table))
+
+
+votable_mappings = {
+    "text": "unicodeChar",
+    "binary": "unsignedByte"
+}
+
+
 def _response(response, status_code):
-    fmt = request.accept_mimetypes.best_match(['application/json', 'text/html'])
+    fmt = request.accept_mimetypes.best_match(['application/json', 'text/html', 'application/x-votable+xml'])
     if fmt == 'text/html':
-        response = renderJsonResponse(response=response, status_code=status_code)
+        response = render_response(response=response, status_code=status_code)
+    elif fmt == 'application/x-votable+xml':
+        response = render_template('votable.xml.j2', result=response["result"], mappings=votable_mappings)
     else:
         response = json.dumps(response)
     return make_response(response, status_code)
-
-
-class ColumnHelper:
-    def __init__(self, description, flags, value):
-        """
-        Helper class to define a column, get it's type for conversion, and convert types if needed.
-        This class works on a best-effort basis. It's not guaranteed to be 100% correct.
-        @param name: Column name
-        @param flags: Flags from MySQLdb.constants.FLAGS
-        @param value: An example value type to help with inferring how to convert
-        """
-        self.type = None
-        self.converter = None
-        self.name = description[0]
-
-        type_code = description[1]
-        scale = description[5]
-
-        if type_code in MySQLdb.NUMBER:
-            # Use python types first, fallback on float otherwise (e.g. NoneType)
-            if isinstance(value, int):
-                self.type = "int"
-            else:
-                # If there's a scale, use float, otherwise assume long
-                self.type = "float" if scale else "long"
-
-        # Check datetime and date
-        if type_code in MySQLdb.DATETIME:
-            self.type = "timestamp"
-            self.converter = lambda x: x.isoformat()
-        if type_code in MySQLdb.DATE:
-            self.type = "date"
-            self.converter = lambda x: x.isoformat()
-
-        # Check if this is binary data and potentially unsafe for JSON
-        if not self.type and flags & BINARY and type_code not in MySQLdb.TIME:
-            # This needs to be checked BEFORE the next type check
-            self.type = "binary"
-            self.converter = b64encode
-        elif isinstance(value, str):
-            self.type = "string"
-
-        if not self.type:
-            # Just return a string and make sure to convert it to string if we don't know about
-            # this type. This may include datetime.time
-            self.type = "string"
-            self.converter = str
-
-    def checkValue(self, value):
-        """
-        Check the value returned from the DBAPI.
-        @param value:
-        @return: The value itself, or a stringified version if it needs to be stringified.
-        """
-        if self.converter and value:
-            return self.converter(value)
-        return value

python/lsst/dax/dbserv/templates/votable.xml.j2

@@ -0,0 +1,32 @@
+{%- macro xml_attribute(meta, name) -%}
+{%- set value = meta.get(name) -%}{%- if value %} {{ name }}="{{ value }}"{%- endif -%}
+{%- endmacro -%}
+
+<?xml version="1.0"?>
+<VOTABLE version="1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://www.ivoa.net/xml/VOTable/v1.3">
+  <RESOURCE>
+      {%- set table = result["table"] %}
+      <TABLE>
+          {% for element in table["metadata"]["elements"] -%}
+          {%- set is_array = element["datatype"] in mappings -%}
+          {%- set xtype = element.get("xtype") -%}
+          {%- set datatype = mappings.get(element["datatype"], element["datatype"]) -%}
+          <FIELD name="{{ element["name"] }}" datatype="{{ datatype }}"
+                  {%- if is_array %} arraysize="*"{%- endif -%}
+                  {%- if xtype %} xtype="{{ xtype }}"{%- endif -%}/>
+          {%- endfor %}
+          <DATA>
+              <TABLEDATA>
+              {%- for row in table["data"] %}
+                  <TR>{%- for column in row -%}
+                      {%- set field = table["metadata"]["elements"][loop.index0] -%}
+                      {%- set b64 = field["datatype"] == "binary" -%}
+                      <TD {%- if b64 %} encoding="base64"{%- endif -%}>{{ column }}</TD>{%- endfor -%}
+                  </TR>
+              {%- endfor -%}
+              </TABLEDATA>
+          </DATA>
+      </TABLE>
+  </RESOURCE>
+</VOTABLE>