Permalink
Browse files

local WSGI Request and Response classes

This change replaces WebOb with a mostly compatible local library,
swift.common.swob.  Subtle changes to WebOb's API over the years have been a
huge headache.  Swift doesn't even run on the current version.

There are a few incompatibilities to simplify the implementation/interface:
 * It only implements the header properties we use.  More can be easily added.
 * Casts header values to str on assignment.
 * Response classes ("HTTPNotFound") are no longer subclasses, but partials
   on Response, so things like isinstance no longer work on them.
 * Unlike newer webob versions, will never return unicode objects.

Change-Id: I76617a0903ee2286b25a821b3c935c86ff95233f
  • Loading branch information...
1 parent f0bd91d commit 5e3e9a882de8b51b8e3b27628ba39f0dabfc78df @redbo redbo committed Sep 4, 2012
Showing with 1,448 additions and 225 deletions.
  1. +2 −2 doc/source/debian_package_guide.rst
  2. +8 −8 doc/source/development_auth.rst
  3. +2 −2 doc/source/development_saio.rst
  4. +0 −1 doc/source/getting_started.rst
  5. +5 −6 swift/account/server.py
  6. +1 −1 swift/common/constraints.py
  7. +5 −5 swift/common/db_replicator.py
  8. +0 −34 swift/common/http.py
  9. +1 −2 swift/common/internal_client.py
  10. +1 −2 swift/common/middleware/catch_errors.py
  11. +1 −2 swift/common/middleware/cname_lookup.py
  12. +1 −2 swift/common/middleware/domain_remap.py
  13. +1 −1 swift/common/middleware/healthcheck.py
  14. +5 −6 swift/common/middleware/keystoneauth.py
  15. +3 −2 swift/common/middleware/name_check.py
  16. +1 −2 swift/common/middleware/proxy_logging.py
  17. +2 −2 swift/common/middleware/ratelimit.py
  18. +1 −1 swift/common/middleware/recon.py
  19. +1 −2 swift/common/middleware/staticweb.py
  20. +7 −7 swift/common/middleware/tempauth.py
  21. +840 −0 swift/common/swob.py
  22. +24 −1 swift/common/utils.py
  23. +3 −3 swift/common/wsgi.py
  24. +5 −6 swift/container/server.py
  25. +6 −7 swift/obj/server.py
  26. +1 −3 swift/proxy/controllers/account.py
  27. +7 −8 swift/proxy/controllers/base.py
  28. +2 −2 swift/proxy/controllers/container.py
  29. +9 −9 swift/proxy/controllers/obj.py
  30. +7 −6 swift/proxy/server.py
  31. +1 −5 test/functional/tests.py
  32. +10 −10 test/unit/account/test_server.py
  33. +1 −2 test/unit/common/middleware/test_cname_lookup.py
  34. +1 −2 test/unit/common/middleware/test_domain_remap.py
  35. +1 −2 test/unit/common/middleware/test_except.py
  36. +1 −2 test/unit/common/middleware/test_formpost.py
  37. +1 −2 test/unit/common/middleware/test_healthcheck.py
  38. +13 −12 test/unit/common/middleware/test_keystoneauth.py
  39. +2 −2 test/unit/common/middleware/test_memcache.py
  40. +2 −1 test/unit/common/middleware/test_name_check.py
  41. +1 −2 test/unit/common/middleware/test_proxy_logging.py
  42. +1 −1 test/unit/common/middleware/test_ratelimit.py
  43. +3 −2 test/unit/common/middleware/test_recon.py
  44. +1 −2 test/unit/common/middleware/test_staticweb.py
  45. +1 −2 test/unit/common/middleware/test_tempauth.py
  46. +1 −2 test/unit/common/middleware/test_tempurl.py
  47. +30 −30 test/unit/common/test_constraints.py
  48. +398 −0 test/unit/common/test_swob.py
  49. +1 −1 test/unit/common/test_wsgi.py
  50. +1 −1 test/unit/container/test_server.py
  51. +1 −1 test/unit/obj/test_internal_client.py
  52. +21 −3 test/unit/obj/test_server.py
  53. +3 −2 test/unit/proxy/test_server.py
  54. +0 −1 tools/pip-requires
@@ -58,7 +58,7 @@ Instructions for Building Debian Packages for Swift
apt-get install python-software-properties
add-apt-repository ppa:swift-core/release
apt-get update
- apt-get install curl gcc bzr python-configobj python-coverage python-dev python-nose python-setuptools python-simplejson python-xattr python-webob python-eventlet python-greenlet debhelper python-sphinx python-all python-openssl python-pastedeploy python-netifaces bzr-builddeb
+ apt-get install curl gcc bzr python-configobj python-coverage python-dev python-nose python-setuptools python-simplejson python-xattr python-eventlet python-greenlet debhelper python-sphinx python-all python-openssl python-pastedeploy python-netifaces bzr-builddeb
* As you
@@ -105,7 +105,7 @@ Instructions for Deploying Debian Packages for Swift
#. Install dependencies::
- apt-get install rsync python-openssl python-setuptools python-webob
+ apt-get install rsync python-openssl python-setuptools
python-simplejson python-xattr python-greenlet python-eventlet
python-netifaces
@@ -63,15 +63,15 @@ Example Authentication with TempAuth:
Authorization is performed through callbacks by the Swift Proxy server to the
WSGI environment's swift.authorize value, if one is set. The swift.authorize
-value should simply be a function that takes a webob.Request as an argument and
+value should simply be a function that takes a Request as an argument and
returns None if access is granted or returns a callable(environ,
start_response) if access is denied. This callable is a standard WSGI callable.
Generally, you should return 403 Forbidden for requests by an authenticated
user and 401 Unauthorized for an unauthenticated request. For example, here's
an authorize function that only allows GETs (in this case you'd probably return
405 Method Not Allowed, but ignore that for the moment).::
- from webob.exc import HTTPForbidden, HTTPUnauthorized
+ from swift.common.swob import HTTPForbidden, HTTPUnauthorized
def authorize(req):
@@ -87,7 +87,7 @@ middleware as authentication and authorization are often paired together. But,
you could create separate authorization middleware that simply sets the
callback before passing on the request. To continue our example above::
- from webob.exc import HTTPForbidden, HTTPUnauthorized
+ from swift.common.swob import HTTPForbidden, HTTPUnauthorized
class Authorization(object):
@@ -127,7 +127,7 @@ then swift.authorize will be called once more. These are called delay_denial
requests and currently include container read requests and object read and
write requests. For these requests, the read or write access control string
(X-Container-Read and X-Container-Write) will be fetched and set as the 'acl'
-attribute in the webob.Request passed to swift.authorize.
+attribute in the Request passed to swift.authorize.
The delay_denial procedures allow skipping possibly expensive access control
string retrievals for requests that can be approved without that information,
@@ -138,7 +138,7 @@ control string set to same value as the authenticated user string. Note that
you probably wouldn't do this exactly as the access control string represents a
list rather than a single user, but it'll suffice for this example::
- from webob.exc import HTTPForbidden, HTTPUnauthorized
+ from swift.common.swob import HTTPForbidden, HTTPUnauthorized
class Authorization(object):
@@ -185,7 +185,7 @@ Let's continue our example to use parse_acl and referrer_allowed. Now we'll
only allow GETs after a referrer check and any requests after a group check::
from swift.common.middleware.acl import parse_acl, referrer_allowed
- from webob.exc import HTTPForbidden, HTTPUnauthorized
+ from swift.common.swob import HTTPForbidden, HTTPUnauthorized
class Authorization(object):
@@ -235,7 +235,7 @@ standard Swift format. Let's improve our example by making use of that::
from swift.common.middleware.acl import \
clean_acl, parse_acl, referrer_allowed
- from webob.exc import HTTPForbidden, HTTPUnauthorized
+ from swift.common.swob import HTTPForbidden, HTTPUnauthorized
class Authorization(object):
@@ -293,7 +293,7 @@ folks a start on their own code if they want to use repoze.what::
from swift.common.bufferedhttp import http_connect_raw as http_connect
from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed
from swift.common.utils import cache_from_env, split_path
- from webob.exc import HTTPForbidden, HTTPUnauthorized
+ from swift.common.swob import HTTPForbidden, HTTPUnauthorized
class DevAuthorization(object):
@@ -30,8 +30,8 @@ Installing dependencies and the core code
#. `apt-get update`
#. `apt-get install curl gcc git-core memcached python-configobj
python-coverage python-dev python-nose python-setuptools python-simplejson
- python-xattr sqlite3 xfsprogs python-webob python-eventlet
- python-greenlet python-pastedeploy python-netifaces python-pip`
+ python-xattr sqlite3 xfsprogs python-eventlet python-greenlet
+ python-pastedeploy python-netifaces python-pip`
#. `pip install mock`
#. Install anything else you want, like screen, ssh, vim, etc.
@@ -15,7 +15,6 @@ most Linux platforms with the following software:
And the following python libraries:
* Eventlet 0.9.8
-* WebOb 0.9.8
* Setuptools
* Simplejson
* Xattr
@@ -22,11 +22,6 @@
from xml.sax import saxutils
from eventlet import Timeout
-from webob import Request, Response
-from webob.exc import HTTPAccepted, HTTPBadRequest, \
- HTTPCreated, HTTPForbidden, HTTPInternalServerError, \
- HTTPMethodNotAllowed, HTTPNoContent, HTTPNotFound, \
- HTTPPreconditionFailed, HTTPConflict
import swift.common.db
from swift.common.db import AccountBroker
@@ -36,7 +31,11 @@
from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \
check_mount, check_float, check_utf8, FORMAT2CONTENT_TYPE
from swift.common.db_replicator import ReplicatorRpc
-from swift.common.http import HTTPInsufficientStorage
+from swift.common.swob import HTTPAccepted, HTTPBadRequest, \
+ HTTPCreated, HTTPForbidden, HTTPInternalServerError, \
+ HTTPMethodNotAllowed, HTTPNoContent, HTTPNotFound, \
+ HTTPPreconditionFailed, HTTPConflict, Request, Response, \
+ HTTPInsufficientStorage
DATADIR = 'accounts'
@@ -17,7 +17,7 @@
from ConfigParser import ConfigParser, NoSectionError, NoOptionError, \
RawConfigParser
-from webob.exc import HTTPBadRequest, HTTPLengthRequired, \
+from swift.common.swob import HTTPBadRequest, HTTPLengthRequired, \
HTTPRequestEntityTooLarge
constraints_conf = ConfigParser()
@@ -26,18 +26,18 @@
from eventlet import GreenPool, sleep, Timeout
from eventlet.green import subprocess
import simplejson
-from webob import Response
-from webob.exc import HTTPNotFound, HTTPNoContent, HTTPAccepted, \
- HTTPInsufficientStorage, HTTPBadRequest
import swift.common.db
from swift.common.utils import get_logger, whataremyips, storage_directory, \
renamer, mkdirs, lock_parent_directory, TRUE_VALUES, unlink_older_than, \
dump_recon_cache, rsync_ip
from swift.common import ring
+from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE
from swift.common.bufferedhttp import BufferedHTTPConnection
from swift.common.exceptions import DriveNotMounted, ConnectionTimeout
from swift.common.daemon import Daemon
+from swift.common.swob import Response, HTTPNotFound, HTTPNoContent, \
+ HTTPAccepted, HTTPInsufficientStorage, HTTPBadRequest
DEBUG_TIMINGS_THRESHOLD = 10
@@ -324,11 +324,11 @@ def _repl_to_node(self, node, broker, partition, info):
info['delete_timestamp'], info['metadata'])
if not response:
return False
- elif response.status == HTTPNotFound.code: # completely missing, rsync
+ elif response.status == HTTP_NOT_FOUND: # completely missing, rsync
self.stats['rsync'] += 1
self.logger.increment('rsyncs')
return self._rsync_db(broker, node, http, info['id'])
- elif response.status == HTTPInsufficientStorage.code:
+ elif response.status == HTTP_INSUFFICIENT_STORAGE:
raise DriveNotMounted()
elif response.status >= 200 and response.status < 300:
rinfo = simplejson.loads(response.data)
View
@@ -13,40 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from webob.exc import HTTPClientError,\
- HTTPInsufficientStorage as BaseHTTPInsufficientStorage
-
-
-class HTTPClientDisconnect(HTTPClientError):
- """
- subclass of :class:`~HTTPClientError`
-
- This code is introduced to log the case when the connection is closed by
- client while HTTP server is processing its request
-
- code: 499, title: Client Disconnect
- """
- code = 499
- title = 'Client Disconnect'
- explanation = (
- 'This code is introduced to log the case when the connection '
- 'is closed by client while HTTP server is processing its request')
-
-
-class HTTPInsufficientStorage(BaseHTTPInsufficientStorage):
- """
- subclass of :class:`~HTTPInsufficientStorage`
-
- The server is unable to store the representation needed to
- complete the request.
-
- code: 507, title: Insufficient Storage
- """
- def __init__(self, drive=None, *args, **kwargs):
- if drive:
- self.explanation = ('%s is not mounted' % drive)
- super(HTTPInsufficientStorage, self).__init__(*args, **kwargs)
-
def is_informational(status):
"""
@@ -19,12 +19,11 @@
import struct
from sys import exc_info
from urllib import quote
-from webob import Request
import zlib
from zlib import compressobj
-
from swift.common.http import HTTP_NOT_FOUND
+from swift.common.swob import Request
class UnexpectedResponse(Exception):
@@ -14,10 +14,9 @@
# limitations under the License.
from eventlet import Timeout
-from webob import Request
-from webob.exc import HTTPServerError
import uuid
+from swift.common.swob import Request, HTTPServerError
from swift.common.utils import get_logger
@@ -27,8 +27,6 @@
rewritten and the request is passed further down the WSGI chain.
"""
-from webob import Request
-from webob.exc import HTTPBadRequest
try:
import dns.resolver
from dns.exception import DNSException
@@ -39,6 +37,7 @@
else: # executed if the try block finishes with no errors
MODULE_DEPENDENCY_MET = True
+from swift.common.swob import Request, HTTPBadRequest
from swift.common.utils import cache_from_env, get_logger
@@ -49,8 +49,7 @@
sync destinations.
"""
-from webob import Request
-from webob.exc import HTTPBadRequest
+from swift.common.swob import Request, HTTPBadRequest
class DomainRemapMiddleware(object):
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from webob import Request, Response
+from swift.common.swob import Request, Response
class HealthCheckMiddleware(object):
@@ -14,10 +14,9 @@
# License for the specific language governing permissions and limitations
# under the License.
-import webob
-
from swift.common import utils as swift_utils
from swift.common.middleware import acl as swift_acl
+from swift.common.swob import HTTPNotFound, HTTPForbidden, HTTPUnauthorized
class KeystoneAuth(object):
@@ -153,7 +152,7 @@ def authorize(self, req):
part = swift_utils.split_path(req.path, 1, 4, True)
version, account, container, obj = part
except ValueError:
- return webob.exc.HTTPNotFound(request=req)
+ return HTTPNotFound(request=req)
user_roles = env_identity.get('roles', [])
@@ -226,7 +225,7 @@ def authorize_anonymous(self, req):
part = swift_utils.split_path(req.path, 1, 4, True)
version, account, container, obj = part
except ValueError:
- return webob.exc.HTTPNotFound(request=req)
+ return HTTPNotFound(request=req)
is_authoritative_authz = (account and
account.startswith(self.reseller_prefix))
@@ -274,9 +273,9 @@ def denied_response(self, req):
depending on whether the REMOTE_USER is set or not.
"""
if req.remote_user:
- return webob.exc.HTTPForbidden(request=req)
+ return HTTPForbidden(request=req)
else:
- return webob.exc.HTTPUnauthorized(request=req)
+ return HTTPUnauthorized(request=req)
def filter_factory(global_conf, **local_conf):
@@ -38,10 +38,11 @@
import re
from swift.common.utils import get_logger
-from webob import Request
-from webob.exc import HTTPBadRequest
from urllib2 import unquote
+from swift.common.swob import Request, HTTPBadRequest
+
+
FORBIDDEN_CHARS = "\'\"`<>"
MAX_LENGTH = 255
FORBIDDEN_REGEXP = "/\./|/\.\./|/\.$|/\.\.$"
@@ -40,8 +40,7 @@
import time
from urllib import quote, unquote
-from webob import Request
-
+from swift.common.swob import Request
from swift.common.utils import (get_logger, get_remote_client,
get_valid_utf8_str, TRUE_VALUES)
@@ -13,11 +13,11 @@
# limitations under the License.
import time
import eventlet
-from webob import Request, Response
from swift.common.utils import split_path, cache_from_env, get_logger
from swift.proxy.controllers.base import get_container_memcache_key
from swift.common.memcached import MemcacheConnectionError
+from swift.common.swob import Request, Response
class MaxSleepTimeHitError(Exception):
@@ -205,7 +205,7 @@ def handle_ratelimit(self, req, account_name, container_name, obj_name):
def __call__(self, env, start_response):
"""
WSGI entry point.
- Wraps env in webob.Request object and passes it down.
+ Wraps env in swob.Request object and passes it down.
:param env: WSGI environment dictionary
:param start_response: WSGI callable
@@ -16,7 +16,7 @@
import errno
import os
-from webob import Request, Response
+from swift.common.swob import Request, Response
from swift.common.utils import split_path, get_logger, TRUE_VALUES
from swift.common.constraints import check_mount
from resource import getpagesize
@@ -118,14 +118,13 @@
import time
from urllib import unquote, quote as urllib_quote
-from webob import Response
-from webob.exc import HTTPMovedPermanently, HTTPNotFound
from swift.common.utils import cache_from_env, get_logger, human_readable, \
split_path, TRUE_VALUES
from swift.common.wsgi import make_pre_authed_env, make_pre_authed_request, \
WSGIContext
from swift.common.http import is_success, is_redirection, HTTP_NOT_FOUND
+from swift.common.swob import Response, HTTPMovedPermanently, HTTPNotFound
def quote(value, safe='/'):
Oops, something went wrong. Retry.

0 comments on commit 5e3e9a8

Please sign in to comment.