Browse files

Move to the new URIManager 'middleware' - existing webtests now work.…

… meanwhile httplib2 itself is somewhat less than stateless and a cause of interesting bugs - always del Http() instances after use unless you really intended otherwise
  • Loading branch information...
1 parent ccbd48f commit cb89d76a256da67eb823f4d3bbb597d2c4215703 @harryf committed Jun 13, 2008
Showing with 252 additions and 197 deletions.
  1. +2 −1 README
  2. +0 −18 dammit/http.py
  3. +75 −16 dammit/request.py
  4. +7 −2 dammit/uri.py
  5. +78 −39 dammit/webtests.py
  6. +90 −121 urldammit.py
View
3 README
@@ -62,5 +62,6 @@ now knows the page has been permanently moved to
http://example.com/articles/8, so you serve a redirect in the
response to the browser.
-API
+Installing
+Requires web.py and simplejson. Tested on python 2.5
View
18 dammit/http.py
@@ -1,18 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-statusmap = {
- 200: '200 OK',
- 201: '201 Created',
- 202: '202 Accepted',
- 204: '204 No Content',
- 300: '300 Multiple Choices',
- 301: '301 Moved Permanently',
- 302: '302 Found',
- 303: '303 See Other',
- 304: '304 Not Modified',
- 305: '305 Use Proxy',
- 306: '305 Switch Proxy',
- 307: '307 Temporary Redirect',
- 400: '400 Bad Request',
- 404: '404 Not Found',
- }
View
91 dammit/request.py
@@ -18,38 +18,97 @@ def is_pair(i):
return True
def unpack_tags(s):
+ """
+ >>> unpack_tags(None)
+ >>> unpack_tags('')
+ >>> unpack_tags('foobar')
+ >>> ["x", "1"] == unpack_tags('["x", 1]')
+ True
+ >>> ["x", "1"] == unpack_tags('["x", 1, {"y": 2}]')
+ True
+ """
+ if not s or s == '':
+ return None
try:
us = simplejson.loads(s)
except ValueError:
- logging.debug("couldnt unpack json: '%s'" % s)
- return []
+ return None
if not type(us) == list:
- logging.debug("not a list: '%s'" % str(us))
- return []
+ return None
- return [i for i in us if is_scalar(i)]
+ return [str(i) for i in us if is_scalar(i)]
def unpack_pairs(s):
+ """
+ >>> unpack_pairs(None)
+ >>> unpack_pairs('')
+ >>> unpack_pairs('foobar')
+ >>> {"y":"2"} == unpack_pairs('{"y": 2}')
+ True
+ >>> {"y":"2"} == unpack_pairs('{"y": 2, "x":[1,2]}')
+ True
+ """
+ if not s or s == '':
+ return None
+
try:
us = simplejson.loads(s)
except ValueError:
- logging.debug("couldnt unpack json: '%s'" % s)
- return {}
+ return None
if not type(us) == dict:
- logging.debug("not a dict: '%s'" % str(us))
- return {}
+ return None
out = {}
for k, v in us.items():
- if not is_scalar(k):
- next
- if not is_scalar(v):
- next
- out[k] = v
+ if not is_scalar(k) or not is_scalar(v):
+ continue
+ out[str(k)] = str(v)
return out
-def pack_response(d):
- return simplejson.dumps(d)
+def pack_response(u):
+ """
+ Construct an response as JSON
+ """
+ keys = ('uri','status','created','updated',
+ 'location','tags','pairs')
+
+ d = {}
+ for key in keys:
+ try:
+ d[key] = str(getattr(u, key))
+ except:
+ pass
+
+ try:
+ return simplejson.dumps(d)
+ except:
+ logging.error("Can't dump '%s'" % d)
+ return ''
+
+def _test():
+ import doctest
+ doctest.testmod()
+
+statusmap = {
+ 200: '200 OK',
+ 201: '201 Created',
+ 202: '202 Accepted',
+ 204: '204 No Content',
+ 300: '300 Multiple Choices',
+ 301: '301 Moved Permanently',
+ 302: '302 Found',
+ 303: '303 See Other',
+ 304: '304 Not Modified',
+ 305: '305 Use Proxy',
+ 306: '305 Switch Proxy',
+ 307: '307 Temporary Redirect',
+ 400: '400 Bad Request',
+ 404: '404 Not Found',
+ }
+
+
+if __name__ == '__main__':
+ _test()
View
9 dammit/uri.py
@@ -396,6 +396,9 @@ def fset(self, code):
self._status = code
+class URIError(Exception):
+ pass
+
class URIManager(object):
"""
Layer between the HTTP frontend and the backend
@@ -433,7 +436,8 @@ def register(self, uri, status = 200, **kwargs):
>>> um = URIManager(db)
>>> print um.register('http://local.ch/test1.html', \
status = 404)
- None
+ Traceback (most recent call last):
+ URIError: Cannot store 'http://local.ch/test1.html' with status '404': no status 200 record found
>>> u = um.register('http://local.ch/test1.html', \
tags = ['a','b'])
>>> u.uri == "http://local.ch/test1.html"
@@ -496,7 +500,8 @@ def register(self, uri, status = 200, **kwargs):
if 200 <= status < 300:
u = URI()
else:
- return None
+ msg = "Cannot store '%s' with status '%s': no status 200 record found" % ( uri, status )
+ raise URIError(msg)
def apply_attribute(uri, key, value):
updated = False
View
117 dammit/webtests.py
@@ -1,20 +1,38 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-import urllib, httplib2, unittest, sys
+import urllib, httplib2, unittest, sys, datetime
+
+REPORT = False
+
+def Report(test):
+ def runtest(self):
+ if REPORT: print "[%s] testing %s" % ( datetime.datetime.now(), test.__name__ )
+ test(self)
+ if REPORT: print "[%s] testing %s done" % ( datetime.datetime.now(), test.__name__ )
+ return runtest
+
class WebTests(unittest.TestCase):
def setUp(self):
+ self.http = None
self._init_http()
self.url = 'http://localhost:8080'
- self.headers = {'Content-type': 'application/x-www-form-urlencoded'}
+ self.headers = {
+ 'Content-type': 'application/x-www-form-urlencoded',
+ 'Connection': 'Close',
+ }
self.body = {
'uri': 'http://foobar.com/setup.html',
'status': '200',
}
+ def tearDown(self):
+ if self.http: del self.http
+
def _init_http(self):
- self.http = httplib2.Http()
+ if self.http: del self.http
+ self.http = httplib2.Http(timeout = 10)
def _post(self):
return self.http.request(
@@ -23,12 +41,13 @@ def _post(self):
body=urllib.urlencode(self.body)
)
+ @Report
def testHome(self):
response, content = self.http.request(self.url, 'GET')
self.assert_( response['status'] == '200' )
self.assert_("where's my url dammit?" in content)
-
+ @Report
def test404(self):
self.body['uri'] = 'http://foobar.com/%s.html'\
% sys._getframe().f_code.co_name
@@ -37,7 +56,7 @@ def test404(self):
)
self.assert_(response['status'] == '404')
-
+ @Report
def test303(self):
# test the direct response to a POST
# without following the redirect
@@ -49,6 +68,7 @@ def test303(self):
self.assert_( self.body['uri'] in content )
+ @Report
def testPOST(self):
# TODO test updated time
self.body['uri'] = 'http://foobar.com/%s.html'\
@@ -57,41 +77,14 @@ def testPOST(self):
self.assert_(response['status'] == '200')
self.assert_( self.body['uri'] in content )
- def testTags(self):
- # TODO test updated time
- self.body['uri'] = 'http://foobar.com/%s.html'\
- % sys._getframe().f_code.co_name
- self.body['tags'] = '["foo","bar"]'
- response, content = self._post()
- self.assert_( response['status'] == '200' )
- self.assert_( self.body['uri'] in content )
- self.assert_( '["foo", "bar"]' in content )
-
- def testTagChange(self):
- # TODO test updated time!
- self.body['uri'] = 'http://foobar.com/%s.html'\
- % sys._getframe().f_code.co_name
-
- self.body['tags'] = '["foo","bar"]'
- response, content = self._post()
- self.assert_( response['status'] == '200' )
- self.assert_( self.body['uri'] in content )
- self.assert_( '["foo", "bar"]' in content )
-
- self._init_http()
- self.body['tags'] = '["abc","xyz"]'
- response, content = self._post()
- self.assert_( response['status'] == '200' )
- self.assert_( self.body['uri'] in content )
- self.assert_( '["abc", "xyz"]' in content )
-
+ @Report
def testDELETE(self):
self.body['uri'] = 'http://foobar.com/%s.html'\
% sys._getframe().f_code.co_name
response, content = self._post()
self.assert_( response['status'] == '200' )
self.assert_( self.body['uri'] in content )
-
+
uri = response['content-location']
self._init_http()
@@ -107,27 +100,29 @@ def testDELETE(self):
)
self.assert_( response['status'] == '404' )
+ @Report
def testDeleteViaPost(self):
# with empty value in status field, record is deleted
self.body['uri'] = 'http://foobar.com/%s.html'\
- % sys._getframe().f_code.co_name
+ % sys._getframe().f_code.co_name
response, content = self._post()
self.assert_( response['status'] == '200' )
self.assert_( self.body['uri'] in content )
-
+
uri = response['content-location']
self._init_http()
- self.body['status'] = ''
+ self.body['delete'] = 'true'
response, content = self._post()
self.assert_( response['status'] == '204' )
-
+
self._init_http()
response, content = self.http.request(
uri, 'GET'
)
self.assert_( response['status'] == '404' )
+ @Report
def testHEAD(self):
self.body['uri'] = 'http://foobar.com/%s.html'\
% sys._getframe().f_code.co_name
@@ -142,13 +137,57 @@ def testHEAD(self):
self.assert_( response['status'] == '200' )
self.assert_( content.strip() == '' )
+ @Report
def testBadrequest(self):
self.body['uri'] = 'http://foobar.com/%s.html'\
% sys._getframe().f_code.co_name
self.body['status'] = 'abc'
response, content = self._post()
self.assert_( response['status'] == '400' )
-
+
+ @Report
+ def testTags(self):
+ # TODO test updated time
+ self.body['uri'] = 'http://foobar.com/%s.html'\
+ % sys._getframe().f_code.co_name
+ self.body['tags'] = '["foo","bar"]'
+ response, content = self._post()
+ self.assert_( response['status'] == '200' )
+ self.assert_( self.body['uri'] in content )
+ self.assert_( "['foo', 'bar']" in content )
+
+ @Report
+ def testPairs(self):
+ self._init_http()
+ # TODO test updated time
+ self.body['uri'] = 'http://foobar.com/%s.html'\
+ % sys._getframe().f_code.co_name
+ self.body['pairs'] = '{"x":1, "y" : 2}'
+ response, content = self._post()
+ self.assert_( response['status'] == '200' )
+ self.assert_( self.body['uri'] in content )
+ self.assert_( "'y': '2'" in content )
+ self.assert_( "'x': '1'" in content )
+
+ @Report
+ def testTagChange(self):
+ # TODO test updated time!
+ self.body['uri'] = 'http://foobar.com/%s.html'\
+ % sys._getframe().f_code.co_name
+
+ self.body['tags'] = '["foo","bar"]'
+ response, content = self._post()
+ self.assert_( response['status'] == '200' )
+ self.assert_( self.body['uri'] in content )
+ self.assert_( "['foo', 'bar']" in content )
+
+ self._init_http()
+ self.body['tags'] = '["abc","xyz"]'
+ response, content = self._post()
+ self.assert_( response['status'] == '200' )
+ self.assert_( self.body['uri'] in content )
+ self.assert_( "['abc', 'xyz']" in content )
+
if __name__ == '__main__':
unittest.main()
View
211 urldammit.py
@@ -1,15 +1,12 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-import web
-import urllib2
-import logging
-import re
-from dammit.couchutils import uri_to_id, put, delete
-from dammit.resource import *
+import urllib2, logging, re
+import web # web.py
from dammit.lrucache import LRUCache
-from couchdb import Database
-from dammit.http import statusmap
-from dammit.request import unpack_tags, unpack_pairs, pack_response
+from dammit.request import *
+from dammit.uri import *
+from dammit.db.mock import MockDB
+
import view
from view import render
import config
@@ -22,7 +19,8 @@
'/([0-9a-f]{40})', 'urldammit',
)
-db = Database(config.DBURL)
+db = MockDB()
+manager = URIManager(db)
# cache URIs we know about
known = LRUCache(config.KNOWN_CACHE_SIZE)
@@ -33,26 +31,30 @@
class urldammit:
- def HEAD(self, uri):
- r = self._locate(uri)
- if not r: return
- if self._redirect(r): return
- self._ok(r)
+ def HEAD(self, id):
+ u = self._locate(id)
+ if not u: return
+ if self._redirect(u): return
+ self._ok(u)
def GET(self, uri = None):
if not uri:
print "where's my url dammit?"
return
- r = self._locate(uri)
+ u = self._locate(uri)
+
+ if not u:
+ web.notfound()
+ return
- if not r: return
- if self._redirect(r): return
+ if self._redirect(u):
+ return
- self._ok(r)
- self._render(r)
+ self._ok(u)
+ self._render(u)
- validstatus = re.compile("^[1-5][0-9]{2}$")
+ validstatus = re.compile("^200|301|404$")
def PUT(self, uri):
pass
@@ -61,75 +63,69 @@ def POST(self):
"""
Update couch with status of uri
"""
+ def required(input, key):
+ val = None
+ try:
+ val = getattr(input, key)
+ except AttributeError:
+ web.ctx.status = statusmap[406]
+ print "%s parameter required" % key
+ return val
+
i = web.input()
- uri = self._reqd(i, 'uri')
+ uri = required(i, 'uri')
if uri is None: return
- status = self._reqd(i, 'status')
+ # allow an explicit delete using a delete
+ # parameter (i.e. allow delete via HTML form)
+ try:
+ delete = getattr(i, 'delete')
+ if delete == 'true':
+ self.DELETE(URI.hash(uri))
+ return
+ except:
+ pass
- # status not supplied? acts as a DELETE
- # (i.e. allow delete via HTML form)
- if status is None or status == "":
- logging.debug("proxy delete via post")
- self.DELETE(uri)
- return
+ # if it's not a delete, we require a status
+ status = required(i, 'status')
+ if status is None: return
if not self.validstatus.match(status):
self._badrequest("Bad value for status: '%s'" % status)
return
- try:
- logging.debug('tags: %s', i.tags)
- except:
- logging.debug('tags not found')
-
- tags = getattr(i, 'tags', [])
- if tags:
- tags = unpack_tags(tags)
-
- logging.debug('tags unpacked: %s', tags)
+ tags = unpack_tags(getattr(i, 'tags', []))
+ pairs = unpack_pairs(getattr(i, 'pairs', []))
+ location = getattr(i, 'location', [])
try:
- logging.debug('pairs: %s', i.tags)
- except:
- logging.debug('pairs not found')
-
- pairs = getattr(i, 'pairs', [])
- logging.debug(pairs)
- if pairs:
- pairs = unpack_pairs(pairs)
-
- logging.debug('pairs unpacked: %s', pairs)
-
- r = register_uri(db, uri = uri, status = int(status),\
- tags = tags, pairs = pairs )
+ u = manager.register(
+ uri,
+ int(status),
+ tags = tags,
+ pairs = pairs,
+ location = location
+ )
- id = uri_to_id(i.uri)
- if r:
- known[id] = r
+ known[u.id] = u
+ if u.id in unknown: del unknown[u.id]
+
web.http.seeother(
- "%s/%s" % ( web.ctx.home, id)
+ "%s/%s" % ( web.ctx.home, u.id)
)
- self._render(r)
+ self._render(u)
+
+ except URIError, e:
+ self._badrequest(e.message)
- def DELETE(self, uri):
- try:
- u = URI.load(db, uri_to_id(uri))
- if not u:
- logging.warn("DELETE called for id %s - not found" % uri)
- db.resource.delete(db, uri, u.rev)
- except Exception, e:
- """
- couchdb issue... (doesn't actually delete)
- 'Document rev/etag must be specified to delete'
- """
- logging.error(e)
-
+ def DELETE(self, id):
+ manager.delete(id)
+ if id in known: del known[id]
web.ctx.status = statusmap[204]
- def _locate(self, uri):
+ def _locate(self, id):
"""
See what we know about this uri...
uri is in fact a SHA-1 hash of the uri
@@ -139,73 +135,46 @@ def ithas(key, cache):
return cache[key]
return None
- u = ithas(uri, unknown)
+ u = ithas(id, unknown)
if u:
web.notfound()
return None
- r = ithas(uri, known)
- if not r:
- r = URI.load(db, uri)
+ u = ithas(id, known)
+ if not u:
+ u = manager.load(id)
- if not r:
- unknown[uri] = True
+ if not u:
+ unknown[id] = True
web.notfound()
return None
- known[uri] = r
+ known[id] = u
- return r
+ return u
- def _ok(self, r):
+ def _ok(self, u):
web.ctx.status = statusmap[200]
- web.http.lastmodified(r.updated)
+ web.http.lastmodified(u.updated)
- def _redirect(self, r):
+ def _redirect(self, u):
"""
- couch reports this url is now elsewhere
+ db reports this url is now elsewhere
return a redirect response
"""
- if 300 <= r.status < 400:
- if r.status in status:
- web.ctx.status = statusmap[r.status]
- web.header(
- 'Location',
- "%s/%s" % ( web.ctx.home, uri_to_id(r.location) )
- )
- return True
+ if u.status == 301:
+ web.ctx.status = statusmap[u.status]
+ web.header(
+ 'Location',
+ "%s/%s" % ( web.ctx.home, URI.hash(u.location) )
+ )
+ return True
+
return False
- def _reqd(self, input, key):
- val = None
- try:
- val = getattr(input,key)
- except AttributeError:
- web.ctx.status = statusmap[406]
- print "%s parameter required" % input
- return val
-
- def _render(self, r):
- if not r:
- return
-
- response = {
- 'uri': r.uri,
- 'status': r.status,
- 'created': str(r.created),
- 'updated': str(r.updated),
- 'tags': r.tags,
- 'pairs': r.pairs.list,
- }
-
- response['id'] = uri_to_id(r.uri)
-
- if r.location:
- response['location'] = r.location
- else:
- response['location'] = r.uri
-
- print pack_response(response)
+ def _render(self, u):
+ if not u: return
+ print pack_response(u)
def _badrequest(self, msg):
web.ctx.status = statusmap[400]

0 comments on commit cb89d76

Please sign in to comment.