Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Account and container info fixes and improvement.

- Fixes bug 1119282.
- Allow middleware accessing metadata of an account without having to
  store it separately in a new memcache namespace.
- Add tests for get_container_info that was previously missed.
- Add get_account_info method based on get_container_info, a function
  for other middleware to query accounts.
- Rename container_info['count'] as container_info['object_count'].

Change-Id: I43787916c7a812cb08d278edf45370521f12c912
  • Loading branch information...
commit b62299376a5a6ead5d547e6c00a74faf011442e5 1 parent e88ff34
@chmouel chmouel authored
View
1  .mailmap
@@ -17,6 +17,7 @@ Anne Gentle <anne@openstack.org> annegentle
Fujita Tomonori <fujita.tomonori@lab.ntt.co.jp>
Greg Lange <greglange@gmail.com> <glange@rackspace.com>
Greg Lange <greglange@gmail.com> <greglange+launchpad@gmail.com>
+Chmouel Boudjnah <chmouel@chmouel.com> <chmouel@enovance.com>
Gaurav B. Gangalwar <gaurav@gluster.com> gaurav@gluster.com <>
Joe Arnold <joe@swiftstack.com> <joe@cloudscaling.com>
Kapil Thangavelu <kapil.foss@gmail.com> kapil.foss@gmail.com <>
View
5 swift/common/middleware/container_quotas.py
@@ -42,7 +42,6 @@
+---------------------------------------------+-------------------------------+
"""
-from swift.common.utils import split_path
from swift.common.http import is_success
from swift.proxy.controllers.base import get_container_info
from swift.common.swob import Response, HTTPBadRequest, wsgify
@@ -93,9 +92,9 @@ def __call__(self, req):
if int(container_info['meta']['quota-bytes']) < new_size:
return self.bad_response(req, container_info)
if 'quota-count' in container_info.get('meta', {}) and \
- 'count' in container_info and \
+ 'object_count' in container_info and \
container_info['meta']['quota-count'].isdigit():
- new_count = int(container_info['count']) + 1
+ new_count = int(container_info['object_count']) + 1
if int(container_info['meta']['quota-count']) < new_count:
return self.bad_response(req, container_info)
View
71 swift/proxy/controllers/base.py
@@ -97,17 +97,33 @@ def get_container_memcache_key(account, container):
return 'container/%s/%s' % (account, container)
+def headers_to_account_info(headers, status_int=HTTP_OK):
+ """
+ Construct a cacheable dict of account info based on response headers.
+ """
+ headers = dict((k.lower(), v) for k, v in dict(headers).iteritems())
+ return {
+ 'status': status_int,
+ 'container_count': headers.get('x-account-container-count'),
+ 'total_object_count': headers.get('x-account-object-count'),
+ 'bytes': headers.get('x-account-bytes-used'),
+ 'meta': dict((key[15:], value)
+ for key, value in headers.iteritems()
+ if key.startswith('x-account-meta-'))
+ }
+
+
def headers_to_container_info(headers, status_int=HTTP_OK):
"""
Construct a cacheable dict of container info based on response headers.
"""
- headers = dict(headers)
+ headers = dict((k.lower(), v) for k, v in dict(headers).iteritems())
return {
'status': status_int,
'read_acl': headers.get('x-container-read'),
'write_acl': headers.get('x-container-write'),
'sync_key': headers.get('x-container-sync-key'),
- 'count': headers.get('x-container-object-count'),
+ 'object_count': headers.get('x-container-object-count'),
'bytes': headers.get('x-container-bytes-used'),
'versions': headers.get('x-versions-location'),
'cors': {
@@ -120,9 +136,9 @@ def headers_to_container_info(headers, status_int=HTTP_OK):
'max_age': headers.get(
'x-container-meta-access-control-max-age')
},
- 'meta': dict((key.lower()[17:], value)
+ 'meta': dict((key[17:], value)
for key, value in headers.iteritems()
- if key.lower().startswith('x-container-meta-'))
+ if key.startswith('x-container-meta-'))
}
@@ -198,8 +214,8 @@ def get_container_info(env, app, swift_source=None):
cache = cache_from_env(env)
if not cache:
return None
- (version, account, container, obj) = \
- split_path(env['PATH_INFO'], 2, 4, True)
+ (version, account, container, _) = \
+ split_path(env['PATH_INFO'], 3, 4, True)
cache_key = get_container_memcache_key(account, container)
# Use a unique environment cache key per container. If you copy this env
# to make a new request, it won't accidentally reuse the old container info
@@ -217,6 +233,33 @@ def get_container_info(env, app, swift_source=None):
return env[env_key]
+def get_account_info(env, app, swift_source=None):
+ """
+ Get the info structure for an account, based on env and app.
+ This is useful to middlewares.
+ """
+ cache = cache_from_env(env)
+ if not cache:
+ return None
+ (version, account, container, _) = \
+ split_path(env['PATH_INFO'], 2, 4, True)
+ cache_key = get_account_memcache_key(account)
+ # Use a unique environment cache key per account. If you copy this env
+ # to make a new request, it won't accidentally reuse the old account info
+ env_key = 'swift.%s' % cache_key
+ if env_key not in env:
+ account_info = cache.get(cache_key)
+ if not account_info:
+ resp = make_pre_authed_request(
+ env, 'HEAD', '/%s/%s' % (version, account),
+ swift_source=swift_source,
+ ).get_response(app)
+ account_info = headers_to_account_info(
+ resp.headers, resp.status_int)
+ env[env_key] = account_info
+ return env[env_key]
+
+
class Controller(object):
"""Base WSGI controller class for the proxy"""
server_type = 'Base'
@@ -326,6 +369,11 @@ def account_info(self, account, autocreate=False):
or (None, None, None) if it does not exist
"""
partition, nodes = self.app.account_ring.get_nodes(account)
+ account_info = {'status': 0,
+ 'container_count': 0,
+ 'total_object_count': None,
+ 'bytes': None,
+ 'meta': {}}
# 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses
if self.app.memcache:
cache_key = get_account_memcache_key(account)
@@ -341,7 +389,6 @@ def account_info(self, account, autocreate=False):
elif result_code == HTTP_NOT_FOUND and not autocreate:
return None, None, None
result_code = 0
- container_count = 0
attempts_left = len(nodes)
path = '/%s' % account
headers = {'x-trans-id': self.trans_id, 'Connection': 'close'}
@@ -362,8 +409,8 @@ def account_info(self, account, autocreate=False):
resp.read()
if is_success(resp.status):
result_code = HTTP_OK
- container_count = int(
- resp.getheader('x-account-container-count') or 0)
+ account_info.update(
+ headers_to_account_info(resp.getheaders()))
break
elif resp.status == HTTP_NOT_FOUND:
if result_code == 0:
@@ -398,12 +445,12 @@ def account_info(self, account, autocreate=False):
cache_timeout = self.app.recheck_account_existence
else:
cache_timeout = self.app.recheck_account_existence * 0.1
+ account_info.update(status=result_code)
self.app.memcache.set(cache_key,
- {'status': result_code,
- 'container_count': container_count},
+ account_info,
time=cache_timeout)
if result_code == HTTP_OK:
- return partition, nodes, container_count
+ return partition, nodes, account_info['container_count']
return None, None, None
def container_info(self, account, container, account_autocreate=False):
View
6 test/unit/common/middleware/test_quotas.py
@@ -87,7 +87,7 @@ def test_not_exceed_bytes_quota(self):
def test_exceed_counts_quota(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
- cache = FakeCache({'count': 1, 'meta': {'quota-count': '1'}})
+ cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
'CONTENT_LENGTH': '100'})
@@ -96,7 +96,7 @@ def test_exceed_counts_quota(self):
def test_not_exceed_counts_quota(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
- cache = FakeCache({'count': 1, 'meta': {'quota-count': '2'}})
+ cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
'CONTENT_LENGTH': '100'})
@@ -152,7 +152,7 @@ def test_missing_container(self):
def test_auth_fail(self):
app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {})
- cache = FakeCache({'count': 1, 'meta': {'quota-count': '1'},
+ cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'},
'write_acl': None})
req = Request.blank('/v1/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache,
View
121 test/unit/proxy/controllers/test_base.py
@@ -14,10 +14,101 @@
# limitations under the License.
import unittest
-from swift.proxy.controllers.base import headers_to_container_info
+
+import swift.proxy.controllers.base
+from swift.proxy.controllers.base import headers_to_container_info, \
+ headers_to_account_info, get_container_info, get_container_memcache_key, \
+ get_account_info, get_account_memcache_key
+from swift.common.swob import Request
+from swift.common.utils import split_path
+
+
+class FakeResponse(object):
+ def __init__(self, headers):
+ self.headers = headers
+ self.status_int = 201
+
+
+class FakeRequest(object):
+ def __init__(self, env, method, path, swift_source=None):
+ (version, account,
+ container, obj) = split_path(env['PATH_INFO'], 2, 4, True)
+ stype = container and 'container' or 'account'
+ self.headers = {'x-%s-object-count' % (stype): 1000,
+ 'x-%s-bytes-used' % (stype): 6666}
+
+ def get_response(self, app):
+ return FakeResponse(self.headers)
+
+
+class FakeCache(object):
+ def __init__(self, val):
+ self.val = val
+
+ def get(self, *args):
+ return self.val
class TestFuncs(unittest.TestCase):
+ def test_get_container_info_no_cache(self):
+ swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
+ req = Request.blank("/v1/AUTH_account/cont",
+ environ={'swift.cache': FakeCache({})})
+ resp = get_container_info(req.environ, 'xxx')
+ self.assertEquals(resp['bytes'], 6666)
+ self.assertEquals(resp['object_count'], 1000)
+
+ def test_get_container_info_cache(self):
+ swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
+ cached = {'status': 404,
+ 'bytes': 3333,
+ 'object_count': 10}
+ req = Request.blank("/v1/account/cont",
+ environ={'swift.cache': FakeCache(cached)})
+ resp = get_container_info(req.environ, 'xxx')
+ self.assertEquals(resp['bytes'], 3333)
+ self.assertEquals(resp['object_count'], 10)
+ self.assertEquals(resp['status'], 404)
+
+ def test_get_container_info_env(self):
+ cache_key = get_container_memcache_key("account", "cont")
+ env_key = 'swift.%s' % cache_key
+ req = Request.blank("/v1/account/cont",
+ environ={ env_key: {'bytes': 3867},
+ 'swift.cache': FakeCache({})})
+ resp = get_container_info(req.environ, 'xxx')
+ self.assertEquals(resp['bytes'], 3867)
+
+ def test_get_account_info_no_cache(self):
+ swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
+ req = Request.blank("/v1/AUTH_account",
+ environ={'swift.cache': FakeCache({})})
+ resp = get_account_info(req.environ, 'xxx')
+ print resp
+ self.assertEquals(resp['bytes'], 6666)
+ self.assertEquals(resp['total_object_count'], 1000)
+
+ def test_get_account_info_cache(self):
+ swift.proxy.controllers.base.make_pre_authed_request = FakeRequest
+ cached = {'status': 404,
+ 'bytes': 3333,
+ 'total_object_count': 10}
+ req = Request.blank("/v1/account/cont",
+ environ={'swift.cache': FakeCache(cached)})
+ resp = get_account_info(req.environ, 'xxx')
+ self.assertEquals(resp['bytes'], 3333)
+ self.assertEquals(resp['total_object_count'], 10)
+ self.assertEquals(resp['status'], 404)
+
+ def test_get_account_info_env(self):
+ cache_key = get_account_memcache_key("account")
+ env_key = 'swift.%s' % cache_key
+ req = Request.blank("/v1/account",
+ environ={ env_key: {'bytes': 3867},
+ 'swift.cache': FakeCache({})})
+ resp = get_account_info(req.environ, 'xxx')
+ self.assertEquals(resp['bytes'], 3867)
+
def test_headers_to_container_info_missing(self):
resp = headers_to_container_info({}, 404)
self.assertEquals(resp['status'], 404)
@@ -48,3 +139,31 @@ def test_headers_to_container_info_values(self):
self.assertEquals(
resp,
headers_to_container_info(headers.items(), 200))
+
+ def test_headers_to_account_info_missing(self):
+ resp = headers_to_account_info({}, 404)
+ self.assertEquals(resp['status'], 404)
+ self.assertEquals(resp['bytes'], None)
+ self.assertEquals(resp['container_count'], None)
+
+ def test_headers_to_account_info_meta(self):
+ headers = {'X-Account-Meta-Whatevs': 14,
+ 'x-account-meta-somethingelse': 0}
+ resp = headers_to_account_info(headers.items(), 200)
+ self.assertEquals(len(resp['meta']), 2)
+ self.assertEquals(resp['meta']['whatevs'], 14)
+ self.assertEquals(resp['meta']['somethingelse'], 0)
+
+ def test_headers_to_account_info_values(self):
+ headers = {
+ 'x-account-object-count': '10',
+ 'x-account-container-count': '20',
+ }
+ resp = headers_to_account_info(headers.items(), 200)
+ self.assertEquals(resp['total_object_count'], '10')
+ self.assertEquals(resp['container_count'], '20')
+
+ headers['x-unused-header'] = 'blahblahblah'
+ self.assertEquals(
+ resp,
+ headers_to_account_info(headers.items(), 200))
View
14 test/unit/proxy/test_server.py
@@ -466,7 +466,12 @@ def test_account_info_200(self):
self.assertEquals(count, 12345)
cache_key = get_account_memcache_key(self.account)
- self.assertEquals({'status': 200, 'container_count': 12345},
+ container_info = {'status': 200,
+ 'container_count': 12345,
+ 'total_object_count': None,
+ 'bytes': None,
+ 'meta': {}}
+ self.assertEquals(container_info,
self.memcache.get(cache_key))
set_http_connect()
@@ -485,7 +490,12 @@ def test_account_info_404(self):
self.assertEquals(count, None)
cache_key = get_account_memcache_key(self.account)
- self.assertEquals({'status': 404, 'container_count': 0},
+ container_info = {'status': 404,
+ 'container_count': 0,
+ 'total_object_count': None,
+ 'bytes': None,
+ 'meta': {}}
+ self.assertEquals(container_info,
self.memcache.get(cache_key))
set_http_connect()
Please sign in to comment.
Something went wrong with that request. Please try again.