make cookies module all-bytes internally

* morsel name, keys and value are bytes-only
* encode cookie keys on set
* fix serialization on py2 (was returning unicode)
* change unquoting to b->b dict

tests/test_cookies.py

@@ -2,7 +2,6 @@
 from datetime import timedelta
 from webob import cookies
 from nose.tools import eq_
-from webob.compat import bytes_
 
 def test_cookie_empty():
     c = cookies.Cookie() # empty cookie
@@ -12,21 +11,22 @@ def test_cookie_one_value():
     c = cookies.Cookie('dismiss-top=6')
     vals = list(c.values())
     eq_(len(vals), 1)
-    eq_(vals[0].name, 'dismiss-top')
-    eq_(vals[0].value, '6')
+    eq_(vals[0].name, b'dismiss-top')
+    eq_(vals[0].value, b'6')
 
 def test_cookie_one_value_with_trailing_semi():
     c = cookies.Cookie('dismiss-top=6;')
     vals = list(c.values())
     eq_(len(vals), 1)
-    eq_(vals[0].name, 'dismiss-top')
-    eq_(vals[0].value, '6')
+    eq_(vals[0].name, b'dismiss-top')
+    eq_(vals[0].value, b'6')
     c = cookies.Cookie('dismiss-top=6;')
 
 def test_cookie_complex():
     c = cookies.Cookie('dismiss-top=6; CP=null*, '\
                        'PHPSESSID=0a539d42abc001cdc762809248d4beed, a="42,"')
-    c_dict = dict((k,v.value) for k,v in c.items())
+    d = lambda v: v.decode('ascii')
+    c_dict = dict((d(k),d(v.value)) for k,v in c.items())
     eq_(c_dict, {'a': '42,',
         'CP': 'null*',
         'PHPSESSID': '0a539d42abc001cdc762809248d4beed',
@@ -44,25 +44,25 @@ def test_cookie_load_multiple():
     c = cookies.Cookie('a=1; Secure=true')
     vals = list(c.values())
     eq_(len(vals), 1)
-    eq_(c['a']['secure'], 'true')
+    eq_(c[b'a'][b'secure'], b'true')
 
 def test_cookie_secure():
     c = cookies.Cookie()
-    c['foo'] = 'bar'
-    c['foo'].secure = True
+    c['foo'] = b'bar'
+    c[b'foo'].secure = True
     eq_(c.serialize(), 'foo=bar; secure')
 
 def test_cookie_httponly():
     c = cookies.Cookie()
-    c['foo'] = 'bar'
-    c['foo'].httponly = True
+    c['foo'] = b'bar'
+    c[b'foo'].httponly = True
     eq_(c.serialize(), 'foo=bar; HttpOnly')
 
 def test_cookie_reserved_keys():
     c = cookies.Cookie('dismiss-top=6; CP=null*; $version=42; a=42')
     assert '$version' not in c
     c = cookies.Cookie('$reserved=42; a=$42')
-    eq_(list(c.keys()), ['a'])
+    eq_(list(c.keys()), [b'a'])
 
 def test_serialize_cookie_date():
     """
@@ -73,35 +73,35 @@ def test_serialize_cookie_date():
               should continue the rest of the process
     """
     eq_(cookies.serialize_cookie_date('Tue, 04-Jan-2011 13:43:50 GMT'),
-        'Tue, 04-Jan-2011 13:43:50 GMT')
+        b'Tue, 04-Jan-2011 13:43:50 GMT')
     eq_(cookies.serialize_cookie_date(None), None)
     cdate_delta = cookies.serialize_cookie_date(timedelta(seconds=10))
     cdate_int = cookies.serialize_cookie_date(10)
     eq_(cdate_delta, cdate_int)
 
 def test_ch_unquote():
-    eq_(cookies._unquote('"hello world'), '"hello world')
-    eq_(cookies._unquote('hello world'), 'hello world')
-    eq_(cookies._unquote('"hello world"'), 'hello world')
+    eq_(cookies._unquote(b'"hello world'), b'"hello world')
+    eq_(cookies._unquote(b'hello world'), b'hello world')
+    eq_(cookies._unquote(b'"hello world"'), b'hello world')
     eq_(cookies._quote(b'hello world'), b'"hello world"')
     # quotation mark is escaped w/ backslash
-    eq_(cookies._unquote(r'"\""'), '"')
-    eq_(cookies._quote(b'"'), bytes_(r'"\""'))
+    eq_(cookies._unquote(b'"\\""'), b'"')
+    eq_(cookies._quote(b'"'), b'"\\""')
     # misc byte escaped as octal
-    eq_(cookies._unquote(r'"\377"'), '\xff')
-    eq_(cookies._quote(b'\xff'), bytes_(r'"\377"'))
+    eq_(cookies._unquote(b'"\\377"'), b'\xff')
+    eq_(cookies._quote(b'\xff'), b'"\\377"')
     # combination
-    eq_(cookies._unquote(r'"a\"\377"'), 'a"\xff')
-    eq_(cookies._quote(b'a"\xff'), bytes_(r'"a\"\377"'))
+    eq_(cookies._unquote(b'"a\\"\\377"'), b'a"\xff')
+    eq_(cookies._quote(b'a"\xff'), b'"a\\"\\377"')
 
 def test_cookie_invalid_name():
     c = cookies.Cookie()
     c['La Pe\xc3\xb1a'] = '1'
     eq_(len(c), 0)
 
 def test_morsel_serialize_with_expires():
-    morsel = cookies.Morsel('bleh', 'blah')
-    morsel.expires = 'Tue, 04-Jan-2011 13:43:50 GMT'
+    morsel = cookies.Morsel(b'bleh', b'blah')
+    morsel.expires = b'Tue, 04-Jan-2011 13:43:50 GMT'
     result = morsel.serialize()
     eq_(result, 'bleh=blah; expires=Tue, 04-Jan-2011 13:43:50 GMT')
 
@@ -123,21 +123,17 @@ def test_serialize_max_age_str():
 
 def test_escape_comma():
     c = cookies.Cookie()
-    c['x'] = '";,"'
+    c['x'] = b'";,"'
     eq_(c.serialize(True), r'x="\"\073\054\""')
 
 def test_parse_qmark_in_val():
     v = r'x="\"\073\054\""; expires=Sun, 12-Jun-2011 23:16:01 GMT'
     c = cookies.Cookie(v)
-    eq_(c['x'].value, r'";,"')
-
-def test_parse_expires_no_quoting():
-    v = r'x="\"\073\054\""; expires=Sun, 12-Jun-2011 23:16:01 GMT'
-    c = cookies.Cookie(v)
-    eq_(c['x'].expires, 'Sun, 12-Jun-2011 23:16:01 GMT')
+    eq_(c[b'x'].value, b'";,"')
+    eq_(c[b'x'].expires, b'Sun, 12-Jun-2011 23:16:01 GMT')
 
 def test_morsel_repr():
-    v = cookies.Morsel('a', 'b')
+    v = cookies.Morsel(b'a', b'b')
     result = repr(v)
     eq_(result, "<Morsel: a='b'>")
 

webob/cookies.py

@@ -9,9 +9,11 @@
 import time
 
 from webob.compat import (
+    PY3,
     binary_type,
+    text_type,
     bytes_,
-    PY3,
+    native_,
     text_,
     )
 
@@ -23,12 +25,14 @@ def __init__(self, input=None):
             self.load(input)
 
     def load(self, data):
+        if PY3:
+            data = data.encode('latin-1')
         ckey = None
         for key, val in _rx_cookie.findall(data):
             if key.lower() in _c_keys:
                 if ckey:
                     self[ckey][key] = _unquote(val)
-            elif key[0] == '$':
+            elif key[0] == _b_dollar_sign:
                 # RFC2109: NAMEs that begin with $ are reserved for other uses
                 # and must not be used by applications.
                 continue
@@ -37,6 +41,8 @@ def load(self, data):
                 ckey = key
 
     def __setitem__(self, key, val):
+        if not isinstance(key, binary_type):
+            key = key.encode('ascii', 'replace')
         if _valid_cookie_name(key):
             dict.__setitem__(self, key, Morsel(key, val))
 
@@ -69,71 +75,77 @@ def serialize_max_age(v):
 def serialize_cookie_date(v):
     if v is None:
         return None
-    elif isinstance(v, str):
+    elif isinstance(v, bytes):
         return v
+    elif isinstance(v, text_type):
+        return v.encode('ascii')
     elif isinstance(v, int):
         v = timedelta(seconds=v)
     if isinstance(v, timedelta):
         v = datetime.utcnow() + v
     if isinstance(v, (datetime, date)):
         v = v.timetuple()
     r = time.strftime('%%s, %d-%%s-%Y %H:%M:%S GMT', v)
-    return r % (weekdays[v[6]], months[v[1]])
+    return bytes_(r % (weekdays[v[6]], months[v[1]]), 'ascii')
 
 class Morsel(dict):
     __slots__ = ('name', 'value')
     def __init__(self, name, value):
-        assert name.lower() not in _c_keys
         assert _valid_cookie_name(name)
+        assert name.lower() not in _c_keys
+        assert isinstance(value, bytes)
         self.name = name
         self.value = value
         self.update(dict.fromkeys(_c_keys, None))
 
-    path = cookie_property('path')
-    domain = cookie_property('domain')
-    comment = cookie_property('comment')
-    expires = cookie_property('expires', serialize_cookie_date)
-    max_age = cookie_property('max-age', serialize_max_age)
-    httponly = cookie_property('httponly', bool)
-    secure = cookie_property('secure', bool)
+    path = cookie_property(b'path')
+    domain = cookie_property(b'domain')
+    comment = cookie_property(b'comment')
+    expires = cookie_property(b'expires', serialize_cookie_date)
+    max_age = cookie_property(b'max-age', serialize_max_age)
+    httponly = cookie_property(b'httponly', bool)
+    secure = cookie_property(b'secure', bool)
 
     def __setitem__(self, k, v):
-        k = k.lower()
+        k = bytes_(k.lower(), 'ascii')
         if k in _c_keys:
             dict.__setitem__(self, k, v)
 
     def serialize(self, full=True):
         result = []
         add = result.append
-        add(bytes_(self.name) + b'=' + _quote(bytes_(self.value, 'utf-8')))
+        add(self.name + b'=' + _quote(self.value))
         if full:
             for k in _c_valkeys:
                 v = self[k]
                 if v:
-                    add(bytes_(_c_renames[k]) + b'='+_quote(bytes_(v, 'utf-8')))
-            expires = self['expires']
+                    add(_c_renames[k] + b'='+_quote(bytes_(v, 'utf-8')))
+            expires = self[b'expires']
             if expires:
-                add(b'expires=' + bytes_(expires))
+                add(b'expires=' + expires)
             if self.secure:
                 add(b'secure')
             if self.httponly:
                 add(b'HttpOnly')
-        return text_(b'; '.join(result), 'ascii')
+        return native_(b'; '.join(result), 'ascii')
 
     __str__ = serialize
 
     def __repr__(self):
-        return '<%s: %s=%r>' % (self.__class__.__name__, self.name, self.value)
+        return '<%s: %s=%r>' % (self.__class__.__name__,
+            native_(self.name),
+            native_(self.value)
+        )
 
 _c_renames = {
-    "path" : "Path",
-    "comment" : "Comment",
-    "domain" : "Domain",
-    "max-age" : "Max-Age",
+    b"path" : b"Path",
+    b"comment" : b"Comment",
+    b"domain" : b"Domain",
+    b"max-age" : b"Max-Age",
 }
 _c_valkeys = sorted(_c_renames)
 _c_keys = set(_c_renames)
-_c_keys.update(['expires', 'secure', 'httponly'])
+_c_keys.update([b'expires', b'secure', b'httponly'])
 
 
 
@@ -151,23 +163,29 @@ def __repr__(self):
 _rx_cookie_str_val = r"(%s|%s|%s*)" % (_re_quoted, _re_expires_val,
                                        _re_legal_char)
 _rx_cookie_str = _rx_cookie_str_key + _rx_cookie_str_equal + _rx_cookie_str_val
-_rx_cookie = re.compile(_rx_cookie_str)
+_rx_cookie = re.compile(bytes_(_rx_cookie_str, 'ascii'))
 
-_rx_unquote = re.compile(r'\\([0-3][0-7][0-7]|.)')
+_rx_unquote = re.compile(bytes_(r'\\([0-3][0-7][0-7]|.)', 'ascii'))
+
+_bchr = (lambda i: bytes([i])) if PY3 else chr
+_ch_unquote_map = dict((bytes_('%03o' % i), _bchr(i))
+    for i in range(256)
+)
+_ch_unquote_map.update((v, v) for v in list(_ch_unquote_map.values()))
+
+_b_dollar_sign = 36 if PY3 else '$'
+_b_quote_mark = 34 if PY3 else '"'
 
 def _unquote(v):
-    if v and v[0] == v[-1] == '"':
+    assert isinstance(v, bytes)
+    if v and v[0] == v[-1] == _b_quote_mark:
         v = v[1:-1]
         def _ch_unquote(m):
-            v = m.group(1)
-            if v.isdigit():
-                return chr(int(v, 8))
-            return v
+            return _ch_unquote_map[m.group(1)]
         v = _rx_unquote.sub(_ch_unquote, v)
     return v
 
 
-
 #
 # serializing
 #
@@ -204,8 +222,6 @@ def _quote(v):
     return v
 
 def _valid_cookie_name(key):
-    if not isinstance(key, binary_type):
-        key = key.encode('ascii', 'replace')
-    return not _needs_quoting(key)
+    return isinstance(key, bytes) and not _needs_quoting(key)
 
 

webob/request.py

@@ -671,7 +671,7 @@ def cookies(self):
         if source:
             cookies = Cookie(source)
             for name in cookies:
-                vars[name] = cookies[name].value
+                vars[name.decode('ascii')] = cookies[name].value.decode('utf-8')
         env['webob._parsed_cookies'] = (vars, source)
         return vars
 

webob/response.py

@@ -51,7 +51,7 @@
     serialize_etag_response,
     serialize_int,
     )
-    
+
 from webob.headers import ResponseHeaders
 from webob.request import BaseRequest
 from webob.util import status_reasons
@@ -667,7 +667,10 @@ def set_cookie(self, key, value='', max_age=None,
             expires = datetime.utcnow() + max_age
         elif max_age is None and expires is not None:
             max_age = expires - datetime.utcnow()
-
+        if isinstance(value, text_type):
+            value = value.encode('utf8')
+        if isinstance(key, text_type):
+            key = key.encode('utf8')
         m = Morsel(key, value)
         m.path = path
         m.domain = domain
@@ -699,6 +702,8 @@ def unset_cookie(self, key, strict=True):
         cookies = Cookie()
         for header in existing:
             cookies.load(header)
+        if isinstance(key, text_type):
+            key = key.encode('utf8')
         if key in cookies:
             del cookies[key]
             del self.headers['Set-Cookie']