Skip to content
This repository
Browse code

Move to the new URIManager 'middleware' - existing webtests now work.…

… meanwhile httplib2 itself is somewhat less than stateless and a cause of interesting bugs - always del Http() instances after use unless you really intended otherwise
  • Loading branch information...
commit cb89d76a256da67eb823f4d3bbb597d2c4215703 1 parent ccbd48f
Harry Fuecks authored
3  README
@@ -62,5 +62,6 @@ now knows the page has been permanently moved to
62 62 http://example.com/articles/8, so you serve a redirect in the
63 63 response to the browser.
64 64
65   -API
  65 +Installing
66 66
  67 +Requires web.py and simplejson. Tested on python 2.5
18 dammit/http.py
... ... @@ -1,18 +0,0 @@
1   -#!/usr/bin/env python
2   -# -*- coding: utf-8 -*-
3   -statusmap = {
4   - 200: '200 OK',
5   - 201: '201 Created',
6   - 202: '202 Accepted',
7   - 204: '204 No Content',
8   - 300: '300 Multiple Choices',
9   - 301: '301 Moved Permanently',
10   - 302: '302 Found',
11   - 303: '303 See Other',
12   - 304: '304 Not Modified',
13   - 305: '305 Use Proxy',
14   - 306: '305 Switch Proxy',
15   - 307: '307 Temporary Redirect',
16   - 400: '400 Bad Request',
17   - 404: '404 Not Found',
18   - }
91 dammit/request.py
@@ -18,38 +18,97 @@ def is_pair(i):
18 18 return True
19 19
20 20 def unpack_tags(s):
  21 + """
  22 + >>> unpack_tags(None)
  23 + >>> unpack_tags('')
  24 + >>> unpack_tags('foobar')
  25 + >>> ["x", "1"] == unpack_tags('["x", 1]')
  26 + True
  27 + >>> ["x", "1"] == unpack_tags('["x", 1, {"y": 2}]')
  28 + True
  29 + """
  30 + if not s or s == '':
  31 + return None
21 32 try:
22 33 us = simplejson.loads(s)
23 34 except ValueError:
24   - logging.debug("couldnt unpack json: '%s'" % s)
25   - return []
  35 + return None
26 36
27 37 if not type(us) == list:
28   - logging.debug("not a list: '%s'" % str(us))
29   - return []
  38 + return None
30 39
31   - return [i for i in us if is_scalar(i)]
  40 + return [str(i) for i in us if is_scalar(i)]
32 41
33 42 def unpack_pairs(s):
  43 + """
  44 + >>> unpack_pairs(None)
  45 + >>> unpack_pairs('')
  46 + >>> unpack_pairs('foobar')
  47 + >>> {"y":"2"} == unpack_pairs('{"y": 2}')
  48 + True
  49 + >>> {"y":"2"} == unpack_pairs('{"y": 2, "x":[1,2]}')
  50 + True
  51 + """
  52 + if not s or s == '':
  53 + return None
  54 +
34 55 try:
35 56 us = simplejson.loads(s)
36 57 except ValueError:
37   - logging.debug("couldnt unpack json: '%s'" % s)
38   - return {}
  58 + return None
39 59
40 60 if not type(us) == dict:
41   - logging.debug("not a dict: '%s'" % str(us))
42   - return {}
  61 + return None
43 62
44 63 out = {}
45 64 for k, v in us.items():
46   - if not is_scalar(k):
47   - next
48   - if not is_scalar(v):
49   - next
50   - out[k] = v
  65 + if not is_scalar(k) or not is_scalar(v):
  66 + continue
  67 + out[str(k)] = str(v)
51 68
52 69 return out
53 70
54   -def pack_response(d):
55   - return simplejson.dumps(d)
  71 +def pack_response(u):
  72 + """
  73 + Construct an response as JSON
  74 + """
  75 + keys = ('uri','status','created','updated',
  76 + 'location','tags','pairs')
  77 +
  78 + d = {}
  79 + for key in keys:
  80 + try:
  81 + d[key] = str(getattr(u, key))
  82 + except:
  83 + pass
  84 +
  85 + try:
  86 + return simplejson.dumps(d)
  87 + except:
  88 + logging.error("Can't dump '%s'" % d)
  89 + return ''
  90 +
  91 +def _test():
  92 + import doctest
  93 + doctest.testmod()
  94 +
  95 +statusmap = {
  96 + 200: '200 OK',
  97 + 201: '201 Created',
  98 + 202: '202 Accepted',
  99 + 204: '204 No Content',
  100 + 300: '300 Multiple Choices',
  101 + 301: '301 Moved Permanently',
  102 + 302: '302 Found',
  103 + 303: '303 See Other',
  104 + 304: '304 Not Modified',
  105 + 305: '305 Use Proxy',
  106 + 306: '305 Switch Proxy',
  107 + 307: '307 Temporary Redirect',
  108 + 400: '400 Bad Request',
  109 + 404: '404 Not Found',
  110 + }
  111 +
  112 +
  113 +if __name__ == '__main__':
  114 + _test()
9 dammit/uri.py
@@ -396,6 +396,9 @@ def fset(self, code):
396 396
397 397 self._status = code
398 398
  399 +class URIError(Exception):
  400 + pass
  401 +
399 402 class URIManager(object):
400 403 """
401 404 Layer between the HTTP frontend and the backend
@@ -433,7 +436,8 @@ def register(self, uri, status = 200, **kwargs):
433 436 >>> um = URIManager(db)
434 437 >>> print um.register('http://local.ch/test1.html', \
435 438 status = 404)
436   - None
  439 + Traceback (most recent call last):
  440 + URIError: Cannot store 'http://local.ch/test1.html' with status '404': no status 200 record found
437 441 >>> u = um.register('http://local.ch/test1.html', \
438 442 tags = ['a','b'])
439 443 >>> u.uri == "http://local.ch/test1.html"
@@ -496,7 +500,8 @@ def register(self, uri, status = 200, **kwargs):
496 500 if 200 <= status < 300:
497 501 u = URI()
498 502 else:
499   - return None
  503 + msg = "Cannot store '%s' with status '%s': no status 200 record found" % ( uri, status )
  504 + raise URIError(msg)
500 505
501 506 def apply_attribute(uri, key, value):
502 507 updated = False
117 dammit/webtests.py
... ... @@ -1,20 +1,38 @@
1 1 #!/usr/bin/env python
2 2 # -*- coding: utf-8 -*-
3   -import urllib, httplib2, unittest, sys
  3 +import urllib, httplib2, unittest, sys, datetime
  4 +
  5 +REPORT = False
  6 +
  7 +def Report(test):
  8 + def runtest(self):
  9 + if REPORT: print "[%s] testing %s" % ( datetime.datetime.now(), test.__name__ )
  10 + test(self)
  11 + if REPORT: print "[%s] testing %s done" % ( datetime.datetime.now(), test.__name__ )
  12 + return runtest
  13 +
4 14
5 15 class WebTests(unittest.TestCase):
6 16
7 17 def setUp(self):
  18 + self.http = None
8 19 self._init_http()
9 20 self.url = 'http://localhost:8080'
10   - self.headers = {'Content-type': 'application/x-www-form-urlencoded'}
  21 + self.headers = {
  22 + 'Content-type': 'application/x-www-form-urlencoded',
  23 + 'Connection': 'Close',
  24 + }
11 25 self.body = {
12 26 'uri': 'http://foobar.com/setup.html',
13 27 'status': '200',
14 28 }
15 29
  30 + def tearDown(self):
  31 + if self.http: del self.http
  32 +
16 33 def _init_http(self):
17   - self.http = httplib2.Http()
  34 + if self.http: del self.http
  35 + self.http = httplib2.Http(timeout = 10)
18 36
19 37 def _post(self):
20 38 return self.http.request(
@@ -23,12 +41,13 @@ def _post(self):
23 41 body=urllib.urlencode(self.body)
24 42 )
25 43
  44 + @Report
26 45 def testHome(self):
27 46 response, content = self.http.request(self.url, 'GET')
28 47 self.assert_( response['status'] == '200' )
29 48 self.assert_("where's my url dammit?" in content)
30 49
31   -
  50 + @Report
32 51 def test404(self):
33 52 self.body['uri'] = 'http://foobar.com/%s.html'\
34 53 % sys._getframe().f_code.co_name
@@ -37,7 +56,7 @@ def test404(self):
37 56 )
38 57 self.assert_(response['status'] == '404')
39 58
40   -
  59 + @Report
41 60 def test303(self):
42 61 # test the direct response to a POST
43 62 # without following the redirect
@@ -49,6 +68,7 @@ def test303(self):
49 68 self.assert_( self.body['uri'] in content )
50 69
51 70
  71 + @Report
52 72 def testPOST(self):
53 73 # TODO test updated time
54 74 self.body['uri'] = 'http://foobar.com/%s.html'\
@@ -57,41 +77,14 @@ def testPOST(self):
57 77 self.assert_(response['status'] == '200')
58 78 self.assert_( self.body['uri'] in content )
59 79
60   - def testTags(self):
61   - # TODO test updated time
62   - self.body['uri'] = 'http://foobar.com/%s.html'\
63   - % sys._getframe().f_code.co_name
64   - self.body['tags'] = '["foo","bar"]'
65   - response, content = self._post()
66   - self.assert_( response['status'] == '200' )
67   - self.assert_( self.body['uri'] in content )
68   - self.assert_( '["foo", "bar"]' in content )
69   -
70   - def testTagChange(self):
71   - # TODO test updated time!
72   - self.body['uri'] = 'http://foobar.com/%s.html'\
73   - % sys._getframe().f_code.co_name
74   -
75   - self.body['tags'] = '["foo","bar"]'
76   - response, content = self._post()
77   - self.assert_( response['status'] == '200' )
78   - self.assert_( self.body['uri'] in content )
79   - self.assert_( '["foo", "bar"]' in content )
80   -
81   - self._init_http()
82   - self.body['tags'] = '["abc","xyz"]'
83   - response, content = self._post()
84   - self.assert_( response['status'] == '200' )
85   - self.assert_( self.body['uri'] in content )
86   - self.assert_( '["abc", "xyz"]' in content )
87   -
  80 + @Report
88 81 def testDELETE(self):
89 82 self.body['uri'] = 'http://foobar.com/%s.html'\
90 83 % sys._getframe().f_code.co_name
91 84 response, content = self._post()
92 85 self.assert_( response['status'] == '200' )
93 86 self.assert_( self.body['uri'] in content )
94   -
  87 +
95 88 uri = response['content-location']
96 89
97 90 self._init_http()
@@ -107,27 +100,29 @@ def testDELETE(self):
107 100 )
108 101 self.assert_( response['status'] == '404' )
109 102
  103 + @Report
110 104 def testDeleteViaPost(self):
111 105 # with empty value in status field, record is deleted
112 106 self.body['uri'] = 'http://foobar.com/%s.html'\
113   - % sys._getframe().f_code.co_name
  107 + % sys._getframe().f_code.co_name
114 108 response, content = self._post()
115 109 self.assert_( response['status'] == '200' )
116 110 self.assert_( self.body['uri'] in content )
117   -
  111 +
118 112 uri = response['content-location']
119 113
120 114 self._init_http()
121   - self.body['status'] = ''
  115 + self.body['delete'] = 'true'
122 116 response, content = self._post()
123 117 self.assert_( response['status'] == '204' )
124   -
  118 +
125 119 self._init_http()
126 120 response, content = self.http.request(
127 121 uri, 'GET'
128 122 )
129 123 self.assert_( response['status'] == '404' )
130 124
  125 + @Report
131 126 def testHEAD(self):
132 127 self.body['uri'] = 'http://foobar.com/%s.html'\
133 128 % sys._getframe().f_code.co_name
@@ -142,13 +137,57 @@ def testHEAD(self):
142 137 self.assert_( response['status'] == '200' )
143 138 self.assert_( content.strip() == '' )
144 139
  140 + @Report
145 141 def testBadrequest(self):
146 142 self.body['uri'] = 'http://foobar.com/%s.html'\
147 143 % sys._getframe().f_code.co_name
148 144 self.body['status'] = 'abc'
149 145 response, content = self._post()
150 146 self.assert_( response['status'] == '400' )
151   -
  147 +
  148 + @Report
  149 + def testTags(self):
  150 + # TODO test updated time
  151 + self.body['uri'] = 'http://foobar.com/%s.html'\
  152 + % sys._getframe().f_code.co_name
  153 + self.body['tags'] = '["foo","bar"]'
  154 + response, content = self._post()
  155 + self.assert_( response['status'] == '200' )
  156 + self.assert_( self.body['uri'] in content )
  157 + self.assert_( "['foo', 'bar']" in content )
  158 +
  159 + @Report
  160 + def testPairs(self):
  161 + self._init_http()
  162 + # TODO test updated time
  163 + self.body['uri'] = 'http://foobar.com/%s.html'\
  164 + % sys._getframe().f_code.co_name
  165 + self.body['pairs'] = '{"x":1, "y" : 2}'
  166 + response, content = self._post()
  167 + self.assert_( response['status'] == '200' )
  168 + self.assert_( self.body['uri'] in content )
  169 + self.assert_( "'y': '2'" in content )
  170 + self.assert_( "'x': '1'" in content )
  171 +
  172 + @Report
  173 + def testTagChange(self):
  174 + # TODO test updated time!
  175 + self.body['uri'] = 'http://foobar.com/%s.html'\
  176 + % sys._getframe().f_code.co_name
  177 +
  178 + self.body['tags'] = '["foo","bar"]'
  179 + response, content = self._post()
  180 + self.assert_( response['status'] == '200' )
  181 + self.assert_( self.body['uri'] in content )
  182 + self.assert_( "['foo', 'bar']" in content )
  183 +
  184 + self._init_http()
  185 + self.body['tags'] = '["abc","xyz"]'
  186 + response, content = self._post()
  187 + self.assert_( response['status'] == '200' )
  188 + self.assert_( self.body['uri'] in content )
  189 + self.assert_( "['abc', 'xyz']" in content )
  190 +
152 191
153 192 if __name__ == '__main__':
154 193 unittest.main()
211 urldammit.py
... ... @@ -1,15 +1,12 @@
1 1 #!/usr/bin/env python
2 2 # -*- coding: utf-8 -*-
3   -import web
4   -import urllib2
5   -import logging
6   -import re
7   -from dammit.couchutils import uri_to_id, put, delete
8   -from dammit.resource import *
  3 +import urllib2, logging, re
  4 +import web # web.py
9 5 from dammit.lrucache import LRUCache
10   -from couchdb import Database
11   -from dammit.http import statusmap
12   -from dammit.request import unpack_tags, unpack_pairs, pack_response
  6 +from dammit.request import *
  7 +from dammit.uri import *
  8 +from dammit.db.mock import MockDB
  9 +
13 10 import view
14 11 from view import render
15 12 import config
@@ -22,7 +19,8 @@
22 19 '/([0-9a-f]{40})', 'urldammit',
23 20 )
24 21
25   -db = Database(config.DBURL)
  22 +db = MockDB()
  23 +manager = URIManager(db)
26 24
27 25 # cache URIs we know about
28 26 known = LRUCache(config.KNOWN_CACHE_SIZE)
@@ -33,26 +31,30 @@
33 31
34 32 class urldammit:
35 33
36   - def HEAD(self, uri):
37   - r = self._locate(uri)
38   - if not r: return
39   - if self._redirect(r): return
40   - self._ok(r)
  34 + def HEAD(self, id):
  35 + u = self._locate(id)
  36 + if not u: return
  37 + if self._redirect(u): return
  38 + self._ok(u)
41 39
42 40 def GET(self, uri = None):
43 41 if not uri:
44 42 print "where's my url dammit?"
45 43 return
46 44
47   - r = self._locate(uri)
  45 + u = self._locate(uri)
  46 +
  47 + if not u:
  48 + web.notfound()
  49 + return
48 50
49   - if not r: return
50   - if self._redirect(r): return
  51 + if self._redirect(u):
  52 + return
51 53
52   - self._ok(r)
53   - self._render(r)
  54 + self._ok(u)
  55 + self._render(u)
54 56
55   - validstatus = re.compile("^[1-5][0-9]{2}$")
  57 + validstatus = re.compile("^200|301|404$")
56 58
57 59 def PUT(self, uri):
58 60 pass
@@ -61,75 +63,69 @@ def POST(self):
61 63 """
62 64 Update couch with status of uri
63 65 """
  66 + def required(input, key):
  67 + val = None
  68 + try:
  69 + val = getattr(input, key)
  70 + except AttributeError:
  71 + web.ctx.status = statusmap[406]
  72 + print "%s parameter required" % key
  73 + return val
  74 +
64 75 i = web.input()
65 76
66   - uri = self._reqd(i, 'uri')
  77 + uri = required(i, 'uri')
67 78 if uri is None: return
68 79
69   - status = self._reqd(i, 'status')
  80 + # allow an explicit delete using a delete
  81 + # parameter (i.e. allow delete via HTML form)
  82 + try:
  83 + delete = getattr(i, 'delete')
  84 + if delete == 'true':
  85 + self.DELETE(URI.hash(uri))
  86 + return
  87 + except:
  88 + pass
70 89
71   - # status not supplied? acts as a DELETE
72   - # (i.e. allow delete via HTML form)
73   - if status is None or status == "":
74   - logging.debug("proxy delete via post")
75   - self.DELETE(uri)
76   - return
  90 + # if it's not a delete, we require a status
  91 + status = required(i, 'status')
  92 + if status is None: return
77 93
78 94 if not self.validstatus.match(status):
79 95 self._badrequest("Bad value for status: '%s'" % status)
80 96 return
81 97
82   - try:
83   - logging.debug('tags: %s', i.tags)
84   - except:
85   - logging.debug('tags not found')
86   -
87   - tags = getattr(i, 'tags', [])
88   - if tags:
89   - tags = unpack_tags(tags)
90   -
91   - logging.debug('tags unpacked: %s', tags)
  98 + tags = unpack_tags(getattr(i, 'tags', []))
  99 + pairs = unpack_pairs(getattr(i, 'pairs', []))
  100 + location = getattr(i, 'location', [])
92 101
93 102 try:
94   - logging.debug('pairs: %s', i.tags)
95   - except:
96   - logging.debug('pairs not found')
97   -
98   - pairs = getattr(i, 'pairs', [])
99   - logging.debug(pairs)
100   - if pairs:
101   - pairs = unpack_pairs(pairs)
102   -
103   - logging.debug('pairs unpacked: %s', pairs)
104   -
105   - r = register_uri(db, uri = uri, status = int(status),\
106   - tags = tags, pairs = pairs )
  103 + u = manager.register(
  104 + uri,
  105 + int(status),
  106 + tags = tags,
  107 + pairs = pairs,
  108 + location = location
  109 + )
107 110
108   - id = uri_to_id(i.uri)
109   - if r:
110   - known[id] = r
  111 + known[u.id] = u
  112 + if u.id in unknown: del unknown[u.id]
  113 +
111 114 web.http.seeother(
112   - "%s/%s" % ( web.ctx.home, id)
  115 + "%s/%s" % ( web.ctx.home, u.id)
113 116 )
114   - self._render(r)
  117 + self._render(u)
  118 +
  119 + except URIError, e:
  120 + self._badrequest(e.message)
115 121
116 122
117   - def DELETE(self, uri):
118   - try:
119   - u = URI.load(db, uri_to_id(uri))
120   - if not u:
121   - logging.warn("DELETE called for id %s - not found" % uri)
122   - db.resource.delete(db, uri, u.rev)
123   - except Exception, e:
124   - """
125   - couchdb issue... (doesn't actually delete)
126   - 'Document rev/etag must be specified to delete'
127   - """
128   - logging.error(e)
129   -
  123 + def DELETE(self, id):
  124 + manager.delete(id)
  125 + if id in known: del known[id]
130 126 web.ctx.status = statusmap[204]
131 127
132   - def _locate(self, uri):
  128 + def _locate(self, id):
133 129 """
134 130 See what we know about this uri...
135 131 uri is in fact a SHA-1 hash of the uri
@@ -139,73 +135,46 @@ def ithas(key, cache):
139 135 return cache[key]
140 136 return None
141 137
142   - u = ithas(uri, unknown)
  138 + u = ithas(id, unknown)
143 139 if u:
144 140 web.notfound()
145 141 return None
146 142
147   - r = ithas(uri, known)
148   - if not r:
149   - r = URI.load(db, uri)
  143 + u = ithas(id, known)
  144 + if not u:
  145 + u = manager.load(id)
150 146
151   - if not r:
152   - unknown[uri] = True
  147 + if not u:
  148 + unknown[id] = True
153 149 web.notfound()
154 150 return None
155 151
156   - known[uri] = r
  152 + known[id] = u
157 153
158   - return r
  154 + return u
159 155
160   - def _ok(self, r):
  156 + def _ok(self, u):
161 157 web.ctx.status = statusmap[200]
162   - web.http.lastmodified(r.updated)
  158 + web.http.lastmodified(u.updated)
163 159
164   - def _redirect(self, r):
  160 + def _redirect(self, u):
165 161 """
166   - couch reports this url is now elsewhere
  162 + db reports this url is now elsewhere
167 163 return a redirect response
168 164 """
169   - if 300 <= r.status < 400:
170   - if r.status in status:
171   - web.ctx.status = statusmap[r.status]
172   - web.header(
173   - 'Location',
174   - "%s/%s" % ( web.ctx.home, uri_to_id(r.location) )
175   - )
176   - return True
  165 + if u.status == 301:
  166 + web.ctx.status = statusmap[u.status]
  167 + web.header(
  168 + 'Location',
  169 + "%s/%s" % ( web.ctx.home, URI.hash(u.location) )
  170 + )
  171 + return True
  172 +
177 173 return False
178 174
179   - def _reqd(self, input, key):
180   - val = None
181   - try:
182   - val = getattr(input,key)
183   - except AttributeError:
184   - web.ctx.status = statusmap[406]
185   - print "%s parameter required" % input
186   - return val
187   -
188   - def _render(self, r):
189   - if not r:
190   - return
191   -
192   - response = {
193   - 'uri': r.uri,
194   - 'status': r.status,
195   - 'created': str(r.created),
196   - 'updated': str(r.updated),
197   - 'tags': r.tags,
198   - 'pairs': r.pairs.list,
199   - }
200   -
201   - response['id'] = uri_to_id(r.uri)
202   -
203   - if r.location:
204   - response['location'] = r.location
205   - else:
206   - response['location'] = r.uri
207   -
208   - print pack_response(response)
  175 + def _render(self, u):
  176 + if not u: return
  177 + print pack_response(u)
209 178
210 179 def _badrequest(self, msg):
211 180 web.ctx.status = statusmap[400]

0 comments on commit cb89d76

Please sign in to comment.
Something went wrong with that request. Please try again.