[api][m]: introduce new web server (in api.py) using cyclone.

* implements some approximation of what I think the scraperwiki api is (but probably not that close)
* write out own basic Client in test_api.py so no longer use the sw client stuff (really not sure how it works)
* README.rst: significant additions including plans for API going forward (more RESTful)
* webstore/datalib.py: minor tweaks to generalize writeback methods so not tied to sw setup (so can work with new frontend api server)

.hgignore

@@ -0,0 +1,2 @@
+syntax: glob
+*.egg-info/

README.rst

@@ -1,9 +1,88 @@
 webstore is a web-api enabled datastore backed onto sqlite or mongodb.
 
+Requirements
+============
+
+* Cyclone and twisted
+
+Run the web server::
+
+  python webstore/api.py
+
+Run tests (start server first!)::
+
+  nosetests test/test_api.py
+
+API
+===
+
+Current:
+
+/?owner=...&database=...&data={jsondict}
+
+Options:
+    /?data={json-data}
+    /jsonrpc
+
+Proposed
+--------
+
+Read
+~~~~
+
+GET: /{owner}/{db-name}/?sql=...
+GET: /{owner}/{db-name}/?table=...&attr=value&attr=value&limit=...
+
+Returns: 
+
+{
+    u'keys': [u'id', u'name'],
+    u'data': [
+        [1, u'jones'],
+        [u'aaa', u'jones']
+        ]
+}
+
+Write
+~~~~~
+
+POST to:
+
+/{owner/{database}/{table}
+
+Payload is json data structured as follows:
+
+{
+    unique_keys: [list of key attributes]
+    data: {dict of values}
+}
+
+
+Authentication and Authorization
+--------------------------------
+
+Authentication: use basic auth header.
+
+
+Authorization:
+
+    * Default: all read, owner can write
+    * Restricted: owner can read and write, everyone can do nothing
+
+Possible future: config file can specify a python method (TODO: method
+signature)
+
+
+Integration with Other Systems
+==============================
+
+Delegate authenatication to user database in some other system.
+
+
 Plan
 ====
 
-* Import existing uml/dataproxy stuff as per Francis' info
-* Get some tests (use existing scraperwiki frontend code)
-* Replace webstore/dataproxy.py with something simpler (probably cyclone based).
+* DONE. Import existing uml/dataproxy stuff as per Francis' info
+* DONE. Get some tests (use existing scraperwiki frontend code)
+* DONE. Replace webstore/dataproxy.py with something simpler (probably cyclone based).
 

setup.py

@@ -0,0 +1,27 @@
+from setuptools import setup, find_packages
+
+setup(
+    name = 'webstore',
+    version = '0.1',
+    packages = find_packages(),
+    install_requires = [
+        ],
+    # metadata for upload to PyPI
+    author = 'Open Knowledge Foundation',
+    author_email = 'info@okfn.org',
+    description = '',
+    license = 'MIT',
+    url = '',
+    download_url = '',
+    classifiers = [
+        'Development Status :: 3 - Alpha',
+        'Environment :: Console',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: MIT License',
+        'Operating System :: OS Independent',
+        'Programming Language :: Python',
+        'Topic :: Software Development :: Libraries :: Python Modules'
+    ],
+)
+
+

test/test_api.py

@@ -1,14 +1,82 @@
-import webstore.client
+# import webstore.client
 import json
+import urllib
+
+class Client(object):
+    def __init__(self, host, port, owner, database):
+        self.host = host
+        self.port = port
+        self.owner = owner
+        self.database = database
+
+    def save(self, table, keys, value):
+        data = {
+            'maincommand': 'save_sqlite',
+            'unique_keys': keys,
+            'data': value,
+            'swdatatblname': table
+            }
+        data = json.dumps(data)
+        url = 'http://%s:%s/?owner=%s&db=%s&data=%s' % (self.host, self.port, self.owner,
+                self.database, data)
+        fo = urllib.urlopen(url)
+        response = fo.read()
+        return json.loads(response)
+
+    def execute(self, query):
+        data = {
+            'maincommand': 'sqliteexecute',
+            'sqlquery': query,
+            'data': {},
+            'attachlist': []
+            }
+        data = json.dumps(data)
+        url = 'http://%s:%s/?owner=%s&db=%s&data=%s' % (self.host, self.port, self.owner,
+                self.database, data)
+        fo = urllib.urlopen(url)
+        response = fo.read()
+        return json.loads(response)
+
 
 def test_it_now():
     host = '127.0.0.1'
-    port = '9034'
-    webstore.client.create(host, port)
-    webstore.client.save(['id'], {'id': 1, 'name': 'jones'})
-    webstore.client.save(['id'], {'id': 'aaa', 'name': 'jones'})
-    print webstore.client.show_tables()
-    out = webstore.client.execute('select * from swdata')
-    assert len(out.keys()) == 2, data
-    assert out['keys'] == [u'id', u'name']
+    port = '8888'
+    client = Client(host, port, 'test', 'test')
+    table = 'defaulttbl'
+    client.save(table, ['id'], {'id': 1, 'name': 'jones'})
+    client.save(table, ['id'], {'id': 'aaa', 'name': 'jones'})
+    # print client.show_tables()
+    out = client.execute('select * from %s' % table)
+    assert len(out.keys()) == 2, out
+    print out
+    assert out['keys'] == [u'id', u'name'], out
+
+
+from webstore.datalib import SQLiteDatabase
+def test_db():
+    output = []
+    def echo(out):
+        print out
+        output.append(out)
+    resourcedir = '/tmp'
+    db = SQLiteDatabase(echo, resourcedir, 'webstoretest', 'xxxx', '1')
+    data = {
+            'maincommand': 'save_sqlite',
+            'unique_keys': ['id'],
+            'data': {'id': 'aaa'},
+            'swdatatblname': 'test'
+        }
+    out = db.process(data)
+    assert out['nrecords'] == 1
+    request = {
+            'maincommand': 'sqliteexecute',
+            'sqlquery': 'select * from test',
+            'data': {},
+            'attachlist': [],
+            'streamchunking': False
+        }
+    out = db.process(request)
+    print out
+    print output
+    assert out['keys'] == ['id']
 

webstore/api.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+import webstore
+import sys
+import cyclone.web
+import cyclone.escape
+from twisted.python import log
+from twisted.internet import reactor
+import webstore.datalib
+
+class IndexHandler(cyclone.web.RequestHandler):
+    def get(self):
+        doc = '''Read from a datastore.
+
+        :param owner: the datastore owner
+        :param database: the database name or id
+        :param table: the database table
+        '''
+        owner = self.request.arguments.get('owner', [])
+        if not owner:
+            self.write('<pre>' + doc + '</pre>')
+        else:
+            self._handle_request()
+
+    def _handle_request(self):
+        owner = self.request.arguments.get('owner', [])
+        owner = owner[0]
+        owner = self.request.arguments.get('owner', [])[0]
+        db = self.request.arguments.get('db', [])[0]
+        dataauth = 'anyoldthing'
+        runID = '1'
+        sqlite = webstore.datalib.SQLiteDatabase(self.write, '/tmp', db, dataauth, runID)
+        data = self.request.arguments.get('data', [])[0]
+        data = cyclone.escape.json_decode(data)
+        resp = sqlite.process(data)
+        self.write(cyclone.escape.json_encode(resp))
+
+
+class Application(cyclone.web.Application):
+    def __init__(self):
+        handlers = [
+            (r'/', IndexHandler),
+        ]
+
+        settings = {
+            'static_path': './static',
+        }
+
+        cyclone.web.Application.__init__(self, handlers, **settings)
+
+if __name__ == '__main__':
+    log.startLogging(sys.stdout)
+    reactor.listenTCP(8888, Application())
+    reactor.run()
+

webstore/datalib.py

@@ -46,8 +46,11 @@ def process(self):
 
 class SQLiteDatabase(Database):
 
-    def __init__(self, ldataproxy, resourcedir, short_name, dataauth, runID):
-        self.dataproxy = ldataproxy  # this is just to give access to self.dataproxy.connection.send()
+    def __init__(self, client_write, resourcedir, short_name, dataauth, runID):
+        # TODO: remove at some point
+        # old scraperwiki setup
+        # self.dataproxy = ldataproxy.connection.sendall
+        self.client_write = client_write
         self.m_resourcedir = resourcedir
         self.short_name = short_name
         self.dataauth = dataauth
@@ -62,6 +65,13 @@ def __init__(self, ldataproxy, resourcedir, short_name, dataauth, runID):
 
         self.logger = logging.getLogger('dataproxy')
 
+    def _write(self, data):
+        '''Write data back to client system'''
+        # TODO: delete at some point
+        # old scraperwiki
+        # self.dataproxy.connection.sendall(json.dumps(arg)+'\n')
+        self.client_write(data)
+
     def process(self, request):
         if type(request) != dict:
             res = {"error":'request must be dict', "content":str(request)}
@@ -220,7 +230,7 @@ def sqliteexecute(self, sqlquery, data, attachlist, streamchunking):
                     break
                 arg["moredata"] = True
                 self.logger.debug("midchunk %s %d" % (self.short_name, len(data)))
-                self.dataproxy.connection.sendall(json.dumps(arg)+'\n')
+                self._write(json.dumps(arg)+'\n')
             return arg