diff --git a/swift/proxy/controllers/account.py b/swift/proxy/controllers/account.py index d3ddaa816f..e6e90d66ee 100644 --- a/swift/proxy/controllers/account.py +++ b/swift/proxy/controllers/account.py @@ -31,7 +31,7 @@ from swift.common.utils import public from swift.common.constraints import check_metadata, MAX_ACCOUNT_NAME_LENGTH from swift.common.http import HTTP_NOT_FOUND -from swift.proxy.controllers.base import Controller, get_account_memcache_key +from swift.proxy.controllers.base import Controller, clear_info_cache from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed @@ -84,9 +84,7 @@ def PUT(self, req): account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = self.generate_request_headers(req, transfer=True) - if self.app.memcache: - self.app.memcache.delete( - get_account_memcache_key(self.account_name)) + clear_info_cache(self.app, req.environ, self.account_name) resp = self.make_requests( req, self.app.account_ring, account_partition, 'PUT', req.path_info, [headers] * len(accounts)) @@ -106,14 +104,12 @@ def POST(self, req): account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = self.generate_request_headers(req, transfer=True) - if self.app.memcache: - self.app.memcache.delete( - get_account_memcache_key(self.account_name)) + clear_info_cache(self.app, req.environ, self.account_name) resp = self.make_requests( req, self.app.account_ring, account_partition, 'POST', req.path_info, [headers] * len(accounts)) if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate: - self.autocreate_account(self.account_name) + self.autocreate_account(req.environ, self.account_name) resp = self.make_requests( req, self.app.account_ring, account_partition, 'POST', req.path_info, [headers] * len(accounts)) @@ -134,9 +130,7 @@ def DELETE(self, req): account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = self.generate_request_headers(req) - if self.app.memcache: - self.app.memcache.delete( - get_account_memcache_key(self.account_name)) + clear_info_cache(self.app, req.environ, self.account_name) resp = self.make_requests( req, self.app.account_ring, account_partition, 'DELETE', req.path_info, [headers] * len(accounts)) diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index e0a86a8469..dfd51127f6 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -28,15 +28,15 @@ import time import functools import inspect +from urllib import quote from eventlet import spawn_n, GreenPile from eventlet.queue import Queue, Empty, Full from eventlet.timeout import Timeout -from swift.common.wsgi import make_pre_authed_request +from swift.common.wsgi import make_pre_authed_env from swift.common.utils import normalize_timestamp, config_true_value, \ - public, split_path, cache_from_env, list_from_csv, \ - GreenthreadSafeIterator + public, split_path, list_from_csv, GreenthreadSafeIterator from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ChunkReadTimeout, ConnectionTimeout from swift.common.http import is_informational, is_success, is_redirection, \ @@ -91,11 +91,15 @@ def wrapped(*a, **kw): def get_account_memcache_key(account): - return 'account/%s' % account + cache_key, env_key = _get_cache_key(account, None) + return cache_key def get_container_memcache_key(account, container): - return 'container/%s/%s' % (account, container) + if not container: + raise ValueError("container not provided") + cache_key, env_key = _get_cache_key(account, container) + return cache_key def headers_to_account_info(headers, status_int=HTTP_OK): @@ -105,6 +109,10 @@ def headers_to_account_info(headers, status_int=HTTP_OK): headers = dict((k.lower(), v) for k, v in dict(headers).iteritems()) return { 'status': status_int, + # 'container_count' anomaly: + # Previous code sometimes expects an int sometimes a string + # Current code aligns to str and None, yet translates to int in + # deprecated functions as needed 'container_count': headers.get('x-account-container-count'), 'total_object_count': headers.get('x-account-object-count'), 'bytes': headers.get('x-account-bytes-used'), @@ -162,7 +170,7 @@ def wrapped(*a, **kw): # Yes, this is a CORS request so test if the origin is allowed container_info = \ controller.container_info(controller.account_name, - controller.container_name) + controller.container_name, req) cors_info = container_info.get('cors', {}) # Call through to the decorated method @@ -207,28 +215,14 @@ def get_container_info(env, app, swift_source=None): Get the info structure for a container, based on env and app. This is useful to middlewares. Note: This call bypasses auth. Success does not imply that the - request has authorization to the container_info. + request has authorization to the account. """ - cache = cache_from_env(env) - if not cache: - return None (version, account, container, _) = \ split_path(env['PATH_INFO'], 3, 4, True) - cache_key = get_container_memcache_key(account, container) - # Use a unique environment cache key per container. If you copy this env - # to make a new request, it won't accidentally reuse the old container info - env_key = 'swift.%s' % cache_key - if env_key not in env: - container_info = cache.get(cache_key) - if not container_info: - resp = make_pre_authed_request( - env, 'HEAD', '/%s/%s/%s' % (version, account, container), - swift_source=swift_source, - ).get_response(app) - container_info = headers_to_container_info( - resp.headers, resp.status_int) - env[env_key] = container_info - return env[env_key] + info = get_info(app, env, account, container, ret_not_found=True) + if not info: + info = headers_to_container_info({}, 0) + return info def get_account_info(env, app, swift_source=None): @@ -236,28 +230,177 @@ def get_account_info(env, app, swift_source=None): Get the info structure for an account, based on env and app. This is useful to middlewares. Note: This call bypasses auth. Success does not imply that the - request has authorization to the account_info. + request has authorization to the container. """ - cache = cache_from_env(env) - if not cache: - return None (version, account, _junk, _junk) = \ split_path(env['PATH_INFO'], 2, 4, True) - cache_key = get_account_memcache_key(account) - # Use a unique environment cache key per account. If you copy this env - # to make a new request, it won't accidentally reuse the old account info + info = get_info(app, env, account, ret_not_found=True) + if not info: + info = headers_to_account_info({}, 0) + if info.get('container_count') is None: + info['container_count'] = 0 + else: + info['container_count'] = int(info['container_count']) + return info + + +def _get_cache_key(account, container): + """ + Get the keys for both memcache (cache_key) and env (env_key) + where info about accounts and containers is cached + :param account: The name of the account + :param container: The name of the container (or None if account) + :returns a tuple of (cache_key, env_key) + """ + + if container: + cache_key = 'container/%s/%s' % (account, container) + else: + cache_key = 'account/%s' % account + # Use a unique environment cache key per account and one container. + # This allows caching both account and container and ensures that when we + # copy this env to form a new request, it won't accidentally reuse the + # old container or account info env_key = 'swift.%s' % cache_key - if env_key not in env: - account_info = cache.get(cache_key) - if not account_info: - resp = make_pre_authed_request( - env, 'HEAD', '/%s/%s' % (version, account), - swift_source=swift_source, - ).get_response(app) - account_info = headers_to_account_info( - resp.headers, resp.status_int) - env[env_key] = account_info - return env[env_key] + return cache_key, env_key + + +def _set_info_cache(app, env, account, container, resp): + """ + Cache info in both memcache and env. + + Caching is used to avoid unnecessary calls to account & container servers. + This is a private function that is being called by GETorHEAD_base and + by clear_info_cache. + Any attempt to GET or HEAD from the container/account server should use + the GETorHEAD_base interface which would than set the cache. + + :param app: the application object + :param account: the unquoted account name + :param container: the unquoted containr name or None + :param resp: the response received or None if info cache should be cleared + """ + + if container: + cache_time = app.recheck_container_existence + else: + cache_time = app.recheck_account_existence + cache_key, env_key = _get_cache_key(account, container) + + if resp: + if resp.status_int == HTTP_NOT_FOUND: + cache_time *= 0.1 + elif not is_success(resp.status_int): + cache_time = None + else: + cache_time = None + + # Next actually set both memcache and the env chache + memcache = getattr(app, 'memcache', None) or env.get('swift.cache') + if not cache_time: + env.pop(env_key, None) + if memcache: + memcache.delete(cache_key) + return + + if container: + info = headers_to_container_info(resp.headers, resp.status_int) + else: + info = headers_to_account_info(resp.headers, resp.status_int) + if memcache: + memcache.set(cache_key, info, cache_time) + env[env_key] = info + + +def clear_info_cache(app, env, account, container=None): + """ + Clear the cached info in both memcache and env + + :param app: the application object + :param account: the account name + :param container: the containr name or None if setting info for containers + """ + _set_info_cache(app, env, account, container, None) + + +def _get_info_cache(app, env, account, container=None): + """ + Get the cached info from env or memcache (if used) in that order + Used for both account and container info + A private function used by get_info + + :param app: the application object + :param env: the environment used by the current request + :returns the cached info or None if not cached + """ + + cache_key, env_key = _get_cache_key(account, container) + if env_key in env: + return env[env_key] + memcache = getattr(app, 'memcache', None) or env.get('swift.cache') + if memcache: + info = memcache.get(cache_key) + if info: + env[env_key] = info + return info + return None + + +def _prepare_pre_auth_info_request(env, path): + """ + Prepares a pre authed request to obtain info using a HEAD. + + :param env: the environment used by the current request + :param path: The unquoted request path + :returns: the pre authed request + """ + # Set the env for the pre_authed call without a query string + newenv = make_pre_authed_env(env, 'HEAD', path, agent='Swift', + query_string='', swift_source='GET_INFO') + # Note that Request.blank expects quoted path + return Request.blank(quote(path), environ=newenv) + + +def get_info(app, env, account, container=None, ret_not_found=False): + """ + Get the info about accounts or containers + + Note: This call bypasses auth. Success does not imply that the + request has authorization to the info. + + :param app: the application object + :param env: the environment used by the current request + :param account: The unquoted name of the account + :param container: The unquoted name of the container (or None if account) + :returns: the cached info or None if cannot be retrieved + """ + info = _get_info_cache(app, env, account, container) + if info: + if ret_not_found or is_success(info['status']): + return info + return None + # Not in cached, let's try the account servers + path = '/v1/%s' % account + if container: + # Stop and check if we have an account? + if not get_info(app, env, account): + return None + path += '/' + container + + req = _prepare_pre_auth_info_request(env, path) + # Whenever we do a GET/HEAD, the GETorHEAD_base will set the info in + # the environment under environ[env_key] and in memcache. We will + # pick the one from environ[env_key] and use it to set the caller env + resp = req.get_response(app) + cache_key, env_key = _get_cache_key(account, container) + try: + info = resp.environ[env_key] + env[env_key] = info + if ret_not_found or is_success(info['status']): + return info + except (KeyError, AttributeError): + pass + return None class Controller(object): @@ -268,6 +411,11 @@ class Controller(object): pass_through_headers = [] def __init__(self, app): + """ + Creates a controller attached to an application instance + + :param app: the application instance + """ self.account_name = None self.app = app self.trans_id = '-' @@ -278,9 +426,21 @@ def __init__(self, app): self.allowed_methods.add(name) def _x_remove_headers(self): + """ + Returns a list of headers that must not be sent to the backend + + :returns: a list of header + """ return [] def transfer_headers(self, src_headers, dst_headers): + """ + Transfer legal headers from an original client request to dictionary + that will be used as headers by the backend request + + :param src_headers: A dictionary of the original client request headers + :param dst_headers: A dictionary of the backend request headers + """ st = self.server_type.lower() x_remove = 'x-remove-%s-meta-' % st @@ -297,6 +457,14 @@ def transfer_headers(self, src_headers, dst_headers): def generate_request_headers(self, orig_req=None, additional=None, transfer=False): + """ + Create a list of headers to be used in backend requets + + :param orig_req: the original request sent by the client to the proxy + :param additional: additional headers to send to the backend + :param transfer: If True, transfer headers from original client request + :returns: a dictionary of headers + """ # Use the additional headers first so they don't overwrite the headers # we require. headers = HeaderKeyDict(additional) if additional else HeaderKeyDict() @@ -385,101 +553,31 @@ def account_info(self, account, req=None): :param account: name of the account to get the info for :param req: caller's HTTP request context object (optional) - :param autocreate: whether or not to automatically create the given - account or not (optional, default: False) :returns: tuple of (account partition, account nodes, container_count) or (None, None, None) if it does not exist """ partition, nodes = self.app.account_ring.get_nodes(account) - account_info = {'status': 0, - 'container_count': 0, - 'total_object_count': None, - 'bytes': None, - 'meta': {}} - # 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses - if self.app.memcache: - cache_key = get_account_memcache_key(account) - cache_value = self.app.memcache.get(cache_key) - if not isinstance(cache_value, dict): - result_code = cache_value - container_count = 0 - else: - result_code = cache_value['status'] - try: - container_count = int(cache_value['container_count']) - except ValueError: - container_count = 0 - if result_code == HTTP_OK: - return partition, nodes, container_count - elif result_code == HTTP_NOT_FOUND: - return None, None, None - result_code = 0 - path = '/%s' % account - headers = self.generate_request_headers(req) - for node in self.iter_nodes(self.app.account_ring, partition): - try: - start_node_timing = time.time() - with ConnectionTimeout(self.app.conn_timeout): - conn = http_connect(node['ip'], node['port'], - node['device'], partition, 'HEAD', - path, headers) - self.app.set_node_timing(node, time.time() - start_node_timing) - with Timeout(self.app.node_timeout): - resp = conn.getresponse() - body = resp.read() - if is_success(resp.status): - result_code = HTTP_OK - account_info.update( - headers_to_account_info(resp.getheaders())) - break - elif resp.status == HTTP_NOT_FOUND: - if result_code == 0: - result_code = HTTP_NOT_FOUND - elif result_code != HTTP_NOT_FOUND: - result_code = -1 - elif resp.status == HTTP_INSUFFICIENT_STORAGE: - self.error_limit(node, _('ERROR Insufficient Storage')) - continue - else: - result_code = -1 - if is_server_error(resp.status): - self.error_occurred( - node, - _('ERROR %(status)d %(body)s From Account ' - 'Server') % - {'status': resp.status, 'body': body[:1024]}) - except (Exception, Timeout): - self.exception_occurred(node, _('Account'), - _('Trying to get account info for %s') - % path) - if self.app.memcache and result_code in (HTTP_OK, HTTP_NOT_FOUND): - if result_code == HTTP_OK: - cache_timeout = self.app.recheck_account_existence - else: - cache_timeout = self.app.recheck_account_existence * 0.1 - account_info.update(status=result_code) - self.app.memcache.set(cache_key, - account_info, - time=cache_timeout) - if result_code == HTTP_OK: - try: - container_count = int(account_info['container_count']) - except ValueError: - container_count = 0 - return partition, nodes, container_count - return None, None, None + if req: + env = getattr(req, 'environ', {}) + else: + env = {} + info = get_info(self.app, env, account) + if not info: + return None, None, None + if info.get('container_count') is None: + container_count = 0 + else: + container_count = int(info['container_count']) + return partition, nodes, container_count def container_info(self, account, container, req=None): """ Get container information and thusly verify container existence. - This will also make a call to account_info to verify that the - account exists. + This will also verify account existence. :param account: account name for the container :param container: container name to look up :param req: caller's HTTP request context object (optional) - :param account_autocreate: whether or not to automatically create the - given account or not (optional, default: False) :returns: dict containing at least container partition ('partition'), container nodes ('containers'), container read acl ('read_acl'), container write acl ('write_acl'), @@ -487,69 +585,19 @@ def container_info(self, account, container, req=None): Values are set to None if the container does not exist. """ part, nodes = self.app.container_ring.get_nodes(account, container) - path = '/%s/%s' % (account, container) - container_info = {'status': 0, 'read_acl': None, - 'write_acl': None, 'sync_key': None, - 'count': None, 'bytes': None, - 'versions': None, 'partition': None, - 'nodes': None} - if self.app.memcache: - cache_key = get_container_memcache_key(account, container) - cache_value = self.app.memcache.get(cache_key) - if isinstance(cache_value, dict): - if 'container_size' in cache_value: - cache_value['count'] = cache_value['container_size'] - if is_success(cache_value['status']): - container_info.update(cache_value) - container_info['partition'] = part - container_info['nodes'] = nodes - return container_info - if not self.account_info(account, req)[1]: - return container_info - headers = self.generate_request_headers(req) - for node in self.iter_nodes(self.app.container_ring, part): - try: - start_node_timing = time.time() - with ConnectionTimeout(self.app.conn_timeout): - conn = http_connect(node['ip'], node['port'], - node['device'], part, 'HEAD', - path, headers) - self.app.set_node_timing(node, time.time() - start_node_timing) - with Timeout(self.app.node_timeout): - resp = conn.getresponse() - body = resp.read() - if is_success(resp.status): - container_info.update( - headers_to_container_info(resp.getheaders())) - break - elif resp.status == HTTP_NOT_FOUND: - container_info['status'] = HTTP_NOT_FOUND - else: - container_info['status'] = -1 - if resp.status == HTTP_INSUFFICIENT_STORAGE: - self.error_limit(node, _('ERROR Insufficient Storage')) - elif is_server_error(resp.status): - self.error_occurred(node, _( - 'ERROR %(status)d %(body)s From Container ' - 'Server') % - {'status': resp.status, 'body': body[:1024]}) - except (Exception, Timeout): - self.exception_occurred( - node, _('Container'), - _('Trying to get container info for %s') % path) - if self.app.memcache: - if container_info['status'] == HTTP_OK: - self.app.memcache.set( - cache_key, container_info, - time=self.app.recheck_container_existence) - elif container_info['status'] == HTTP_NOT_FOUND: - self.app.memcache.set( - cache_key, container_info, - time=self.app.recheck_container_existence * 0.1) - if container_info['status'] == HTTP_OK: - container_info['partition'] = part - container_info['nodes'] = nodes - return container_info + if req: + env = getattr(req, 'environ', {}) + else: + env = {} + info = get_info(self.app, env, account, container) + if not info: + info = headers_to_container_info({}, 0) + info['partition'] = None + info['nodes'] = None + else: + info['partition'] = part + info['nodes'] = nodes + return info def iter_nodes(self, ring, partition): """ @@ -595,6 +643,23 @@ def iter_nodes(self, ring, partition): def _make_request(self, nodes, part, method, path, headers, query, logger_thread_locals): + """ + Sends an HTTP request to a single node and aggregates the result. + It attempts the primary node, then iterates over the handoff nodes + as needed. + + :param nodes: an iterator of the backend server and handoff servers + :param part: the partition number + :param method: the method to send to the backend + :param path: the path to send to the backend + :param headers: a list of dicts, where each dict represents one + backend request that should be made. + :param query: query string to send to the backend. + :param logger_thread_locals: The thread local values to be set on the + self.app.logger to retain transaction + logging information. + :returns: a swob.Response object + """ self.app.logger.thread_locals = logger_thread_locals for node in nodes: try: @@ -624,8 +689,14 @@ def make_requests(self, req, ring, part, method, path, headers, It attempts the primary nodes concurrently, then iterates over the handoff nodes as needed. + :param req: a request sent by the client + :param ring: the ring used for finding backend servers + :param part: the partition number + :param method: the method to send to the backend + :param path: the path to send to the backend :param headers: a list of dicts, where each dict represents one backend request that should be made. + :param query_string: optional query string to send to the backend :returns: a swob.Response object """ start_nodes = ring.get_part_nodes(part) @@ -675,12 +746,22 @@ def best_response(self, req, statuses, reasons, bodies, server_type, @public def GET(self, req): - """Handler for HTTP GET requests.""" + """ + Handler for HTTP GET requests. + + :param req: The client request + :returns: the response to the client + """ return self.GETorHEAD(req) @public def HEAD(self, req): - """Handler for HTTP HEAD requests.""" + """ + Handler for HTTP HEAD requests. + + :param req: The client request + :returns: the response to the client + """ return self.GETorHEAD(req) def _make_app_iter_reader(self, node, source, queue, logger_thread_locals): @@ -761,6 +842,11 @@ def _make_app_iter(self, node, source): raise def close_swift_conn(self, src): + """ + Force close the http connection to the backend. + + :param src: the response from the backend + """ try: src.swift_conn.close() except Exception: @@ -780,10 +866,19 @@ def is_good_source(self, src): """ Indicates whether or not the request made to the backend found what it was looking for. + + :param src: the response from the backend + :returns: True if found, False if not """ return is_success(src.status) or is_redirection(src.status) - def autocreate_account(self, account): + def autocreate_account(self, env, account): + """ + Autocreate an account + + :param env: the environment of the request leading to this autocreate + :param account: the unquoted account name + """ partition, nodes = self.app.account_ring.get_nodes(account) path = '/%s' % account headers = {'X-Timestamp': normalize_timestamp(time.time()), @@ -794,9 +889,7 @@ def autocreate_account(self, account): path, [headers] * len(nodes)) if is_success(resp.status_int): self.app.logger.info('autocreate account %r' % path) - if self.app.memcache: - self.app.memcache.delete( - get_account_memcache_key(account)) + clear_info_cache(self.app, env, account) else: self.app.logger.warning('Could not autocreate account %r' % path) @@ -862,6 +955,7 @@ def GETorHEAD_base(self, req, server_type, ring, partition, path): {'status': possible_source.status, 'body': bodies[-1][:1024], 'type': server_type}) + res = None if sources: sources.sort(key=lambda s: source_key(s[0])) source, node = sources.pop() @@ -884,9 +978,15 @@ def GETorHEAD_base(self, req, server_type, ring, partition, path): if source.getheader('Content-Type'): res.charset = None res.content_type = source.getheader('Content-Type') - return res - return self.best_response(req, statuses, reasons, bodies, - '%s %s' % (server_type, req.method)) + if not res: + res = self.best_response(req, statuses, reasons, bodies, + '%s %s' % (server_type, req.method)) + try: + (account, container) = split_path(req.path_info, 1, 2) + _set_info_cache(self.app, req.environ, account, container, res) + except ValueError: + pass + return res def is_origin_allowed(self, cors_info, origin): """ @@ -926,7 +1026,8 @@ def OPTIONS(self, req): # This is a CORS preflight request so check it's allowed try: container_info = \ - self.container_info(self.account_name, self.container_name) + self.container_info(self.account_name, + self.container_name, req) except AttributeError: # This should only happen for requests to the Account. A future # change could allow CORS requests to the Account level as well. diff --git a/swift/proxy/controllers/container.py b/swift/proxy/controllers/container.py index b62afc4570..1f04273968 100644 --- a/swift/proxy/controllers/container.py +++ b/swift/proxy/controllers/container.py @@ -30,7 +30,7 @@ from swift.common.constraints import check_metadata, MAX_CONTAINER_NAME_LENGTH from swift.common.http import HTTP_ACCEPTED from swift.proxy.controllers.base import Controller, delay_denial, \ - get_container_memcache_key, headers_to_container_info, cors_validation + cors_validation, clear_info_cache from swift.common.swob import HTTPBadRequest, HTTPForbidden, \ HTTPNotFound @@ -75,15 +75,6 @@ def GETorHEAD(self, req): self.account_name, self.container_name) resp = self.GETorHEAD_base( req, _('Container'), self.app.container_ring, part, req.path_info) - if self.app.memcache: - # set the memcache container size for ratelimiting - cache_key = get_container_memcache_key(self.account_name, - self.container_name) - self.app.memcache.set( - cache_key, - headers_to_container_info(resp.headers, resp.status_int), - time=self.app.recheck_container_existence) - if 'swift.authorize' in req.environ: req.acl = resp.headers.get('x-container-read') aresp = req.environ['swift.authorize'](req) @@ -124,11 +115,11 @@ def PUT(self, req): (len(self.container_name), MAX_CONTAINER_NAME_LENGTH) return resp account_partition, accounts, container_count = \ - self.account_info(self.account_name) + self.account_info(self.account_name, req) if not accounts and self.app.account_autocreate: - self.autocreate_account(self.account_name) + self.autocreate_account(req.environ, self.account_name) account_partition, accounts, container_count = \ - self.account_info(self.account_name) + self.account_info(self.account_name, req) if not accounts: return HTTPNotFound(request=req) if self.app.max_containers_per_account > 0 and \ @@ -142,10 +133,8 @@ def PUT(self, req): self.account_name, self.container_name) headers = self._backend_requests(req, len(containers), account_partition, accounts) - if self.app.memcache: - cache_key = get_container_memcache_key(self.account_name, - self.container_name) - self.app.memcache.delete(cache_key) + clear_info_cache(self.app, req.environ, + self.account_name, self.container_name) resp = self.make_requests( req, self.app.container_ring, container_partition, 'PUT', req.path_info, headers) @@ -160,15 +149,14 @@ def POST(self, req): if error_response: return error_response account_partition, accounts, container_count = \ - self.account_info(self.account_name) + self.account_info(self.account_name, req) if not accounts: return HTTPNotFound(request=req) container_partition, containers = self.app.container_ring.get_nodes( self.account_name, self.container_name) headers = self.generate_request_headers(req, transfer=True) - if self.app.memcache: - self.app.memcache.delete(get_container_memcache_key( - self.account_name, self.container_name)) + clear_info_cache(self.app, req.environ, + self.account_name, self.container_name) resp = self.make_requests( req, self.app.container_ring, container_partition, 'POST', req.path_info, [headers] * len(containers)) @@ -186,10 +174,8 @@ def DELETE(self, req): self.account_name, self.container_name) headers = self._backend_requests(req, len(containers), account_partition, accounts) - if self.app.memcache: - cache_key = get_container_memcache_key(self.account_name, - self.container_name) - self.app.memcache.delete(cache_key) + clear_info_cache(self.app, req.environ, + self.account_name, self.container_name) resp = self.make_requests( req, self.app.container_ring, container_partition, 'DELETE', req.path_info, headers) diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 3b1489d374..edfa682da1 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -568,7 +568,7 @@ def POST(self, req): if error_response: return error_response container_info = self.container_info( - self.account_name, self.container_name) + self.account_name, self.container_name, req) container_partition = container_info['partition'] containers = container_info['nodes'] req.acl = container_info['write_acl'] @@ -691,7 +691,7 @@ def _connect_put_node(self, nodes, part, path, headers, def PUT(self, req): """HTTP PUT request handler.""" container_info = self.container_info( - self.account_name, self.container_name) + self.account_name, self.container_name, req) container_partition = container_info['partition'] containers = container_info['nodes'] req.acl = container_info['write_acl'] diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 5a38d1c953..9dac5d35ed 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -17,6 +17,78 @@ import logging.handlers from httplib import HTTPException +class FakeRing(object): + + def __init__(self, replicas=3): + # 9 total nodes (6 more past the initial 3) is the cap, no matter if + # this is set higher, or R^2 for R replicas + self.replicas = replicas + self.max_more_nodes = 0 + self.devs = {} + + def set_replicas(self, replicas): + self.replicas = replicas + self.devs = {} + + @property + def replica_count(self): + return self.replicas + + def get_part(self, account, container=None, obj=None): + return 1 + + def get_nodes(self, account, container=None, obj=None): + devs = [] + for x in xrange(self.replicas): + devs.append(self.devs.get(x)) + if devs[x] is None: + self.devs[x] = devs[x] = \ + {'ip': '10.0.0.%s' % x, + 'port': 1000 + x, + 'device': 'sd' + (chr(ord('a') + x)), + 'id': x} + return 1, devs + + def get_part_nodes(self, part): + return self.get_nodes('blah')[1] + + def get_more_nodes(self, nodes): + # replicas^2 is the true cap + for x in xrange(self.replicas, min(self.replicas + self.max_more_nodes, + self.replicas * self.replicas)): + yield {'ip': '10.0.0.%s' % x, 'port': 1000 + x, 'device': 'sda'} + + +class FakeMemcache(object): + + def __init__(self): + self.store = {} + + def get(self, key): + return self.store.get(key) + + def keys(self): + return self.store.keys() + + def set(self, key, value, time=0): + self.store[key] = value + return True + + def incr(self, key, time=0): + self.store[key] = self.store.setdefault(key, 0) + 1 + return self.store[key] + + @contextmanager + def soft_lock(self, key, timeout=0, retries=5): + yield True + + def delete(self, key): + try: + del self.store[key] + except Exception: + pass + return True + def readuntil2crlfs(fd): rv = '' @@ -133,6 +205,7 @@ def stub_fn(self, *args, **kwargs): def exception(self, *args, **kwargs): self.log_dict['exception'].append((args, kwargs, str(exc_info()[1]))) + print 'FakeLogger Exception: %s' % self.log_dict # mock out the StatsD logging methods: increment = _store_in('increment') diff --git a/test/unit/common/middleware/test_account_quotas.py b/test/unit/common/middleware/test_account_quotas.py index 752bcc3ebe..0112a0fc2e 100644 --- a/test/unit/common/middleware/test_account_quotas.py +++ b/test/unit/common/middleware/test_account_quotas.py @@ -17,6 +17,9 @@ from swift.common.middleware import account_quotas +from swift.proxy.controllers.base import _get_cache_key, \ + headers_to_account_info + class FakeCache(object): def __init__(self, val): @@ -43,6 +46,9 @@ def __init__(self, headers=[]): self.headers = headers def __call__(self, env, start_response): + # Cache the account_info (same as a real application) + cache_key, env_key = _get_cache_key('a', None) + env[env_key] = headers_to_account_info(self.headers, 200) start_response('200 OK', self.headers) return [] diff --git a/test/unit/proxy/controllers/test_account.py b/test/unit/proxy/controllers/test_account.py new file mode 100644 index 0000000000..4cc840efea --- /dev/null +++ b/test/unit/proxy/controllers/test_account.py @@ -0,0 +1,47 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import unittest +import swift.proxy.controllers.base + +from contextlib import contextmanager +from swift.common.swob import Request +from swift.proxy import server as proxy_server +from swift.proxy.controllers.base import headers_to_account_info +from test.unit import fake_http_connect, FakeRing, FakeMemcache + + +class TestAccountController(unittest.TestCase): + def setUp(self): + self.app = proxy_server.Application(None, FakeMemcache(), + account_ring=FakeRing(), + container_ring=FakeRing(), + object_ring=FakeRing) + + def test_account_info_in_response_env(self): + controller = proxy_server.AccountController(self.app, 'AUTH_bob') + with mock.patch('swift.proxy.controllers.base.http_connect', + fake_http_connect(200, 200, body='')): + req = Request.blank('/AUTH_bob', {'PATH_INFO': '/AUTH_bob'}) + resp = controller.HEAD(req) + self.assertEqual(2, resp.status_int // 100) + self.assertTrue('swift.account/AUTH_bob' in resp.environ) + self.assertEqual(headers_to_account_info(resp.headers), + resp.environ['swift.account/AUTH_bob']) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/proxy/controllers/test_base.py b/test/unit/proxy/controllers/test_base.py index 1ed742372a..fc7ba7e103 100644 --- a/test/unit/proxy/controllers/test_base.py +++ b/test/unit/proxy/controllers/test_base.py @@ -14,31 +14,45 @@ # limitations under the License. import unittest - -import swift.proxy.controllers.base +from mock import patch from swift.proxy.controllers.base import headers_to_container_info, \ headers_to_account_info, get_container_info, get_container_memcache_key, \ - get_account_info, get_account_memcache_key + get_account_info, get_account_memcache_key, _get_cache_key, get_info, \ + Controller from swift.common.swob import Request from swift.common.utils import split_path +import swift.proxy.controllers.base +from test.unit import FakeLogger, fake_http_connect, FakeRing, FakeMemcache +from swift.proxy import server as proxy_server +FakeResponse_status_int = 201 class FakeResponse(object): - def __init__(self, headers): + def __init__(self, headers, env, account, container): self.headers = headers - self.status_int = 201 + self.status_int = FakeResponse_status_int + self.environ = env + cache_key, env_key = _get_cache_key(account, container) + if container: + info = headers_to_container_info(headers, FakeResponse_status_int) + else: + info = headers_to_account_info(headers, FakeResponse_status_int) + env[env_key] = info class FakeRequest(object): - def __init__(self, env, method, path, swift_source=None): - (version, account, - container, obj) = split_path(env['PATH_INFO'], 2, 4, True) + def __init__(self, env, path): + self.environ = env + (version, account, container, obj) = split_path(path, 2, 4, True) + self.account = account + self.container = container stype = container and 'container' or 'account' self.headers = {'x-%s-object-count' % (stype): 1000, 'x-%s-bytes-used' % (stype): 6666} def get_response(self, app): - return FakeResponse(self.headers) + return FakeResponse(self.headers, self.environ, self.account, + self.container) class FakeCache(object): @@ -50,22 +64,150 @@ def get(self, *args): class TestFuncs(unittest.TestCase): + def setUp(self): + self.app = proxy_server.Application(None, FakeMemcache(), + account_ring=FakeRing(), + container_ring=FakeRing(), + object_ring=FakeRing) + + def test_GETorHEAD_base(self): + base = Controller(self.app) + req = Request.blank('/a/c') + with patch('swift.proxy.controllers.base.' + 'http_connect', fake_http_connect(200)): + resp = base.GETorHEAD_base(req, 'container', FakeRing(), 'part', + '/a/c') + self.assertTrue('swift.container/a/c' in resp.environ) + self.assertEqual(resp.environ['swift.container/a/c']['status'], 200) + + req = Request.blank('/a') + with patch('swift.proxy.controllers.base.' + 'http_connect', fake_http_connect(200)): + resp = base.GETorHEAD_base(req, 'account', FakeRing(), 'part', + '/a') + self.assertTrue('swift.account/a' in resp.environ) + self.assertEqual(resp.environ['swift.account/a']['status'], 200) + + def test_get_info(self): + global FakeResponse_status_int + # Do a non cached call to account + env = {} + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + info_a = get_info(None, env, 'a') + # Check that you got proper info + self.assertEquals(info_a['status'], 201) + self.assertEquals(info_a['bytes'], 6666) + self.assertEquals(info_a['total_object_count'], 1000) + # Make sure the env cache is set + self.assertEquals(env, {'swift.account/a': info_a}) + + # Do an env cached call to account + info_a = get_info(None, env, 'a') + # Check that you got proper info + self.assertEquals(info_a['status'], 201) + self.assertEquals(info_a['bytes'], 6666) + self.assertEquals(info_a['total_object_count'], 1000) + # Make sure the env cache is set + self.assertEquals(env, {'swift.account/a': info_a}) + + # This time do env cached call to account and non cached to container + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + info_c = get_info(None, env, 'a', 'c') + # Check that you got proper info + self.assertEquals(info_a['status'], 201) + self.assertEquals(info_c['bytes'], 6666) + self.assertEquals(info_c['object_count'], 1000) + # Make sure the env cache is set + self.assertEquals(env['swift.account/a'], info_a) + self.assertEquals(env['swift.container/a/c'], info_c) + + # This time do a non cached call to account than non cached to container + env = {} # abandon previous call to env + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + info_c = get_info(None, env, 'a', 'c') + # Check that you got proper info + self.assertEquals(info_a['status'], 201) + self.assertEquals(info_c['bytes'], 6666) + self.assertEquals(info_c['object_count'], 1000) + # Make sure the env cache is set + self.assertEquals(env['swift.account/a'], info_a) + self.assertEquals(env['swift.container/a/c'], info_c) + + # This time do an env cached call to container while account is not cached + del(env['swift.account/a']) + info_c = get_info(None, env, 'a', 'c') + # Check that you got proper info + self.assertEquals(info_a['status'], 201) + self.assertEquals(info_c['bytes'], 6666) + self.assertEquals(info_c['object_count'], 1000) + # Make sure the env cache is set and account still not cached + self.assertEquals(env, {'swift.container/a/c': info_c}) + + # Do a non cached call to account not found with ret_not_found + env = {} + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + try: + FakeResponse_status_int = 404 + info_a = get_info(None, env, 'a', ret_not_found=True) + finally: + FakeResponse_status_int = 201 + # Check that you got proper info + self.assertEquals(info_a['status'], 404) + self.assertEquals(info_a['bytes'], 6666) + self.assertEquals(info_a['total_object_count'], 1000) + # Make sure the env cache is set + self.assertEquals(env, {'swift.account/a': info_a}) + + # Do a cached call to account not found with ret_not_found + info_a = get_info(None, env, 'a', ret_not_found=True) + # Check that you got proper info + self.assertEquals(info_a['status'], 404) + self.assertEquals(info_a['bytes'], 6666) + self.assertEquals(info_a['total_object_count'], 1000) + # Make sure the env cache is set + self.assertEquals(env, {'swift.account/a': info_a}) + + # Do a non cached call to account not found without ret_not_found + env = {} + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + try: + FakeResponse_status_int = 404 + info_a = get_info(None, env, 'a') + finally: + FakeResponse_status_int = 201 + # Check that you got proper info + self.assertEquals(info_a, None) + self.assertEquals(env['swift.account/a']['status'], 404) + + # Do a cached call to account not found without ret_not_found + info_a = get_info(None, env, 'a') + # Check that you got proper info + self.assertEquals(info_a, None) + self.assertEquals(env['swift.account/a']['status'], 404) + def test_get_container_info_no_cache(self): - swift.proxy.controllers.base.make_pre_authed_request = FakeRequest req = Request.blank("/v1/AUTH_account/cont", environ={'swift.cache': FakeCache({})}) - resp = get_container_info(req.environ, 'xxx') + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + resp = get_container_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 6666) self.assertEquals(resp['object_count'], 1000) def test_get_container_info_cache(self): - swift.proxy.controllers.base.make_pre_authed_request = FakeRequest cached = {'status': 404, 'bytes': 3333, 'object_count': 10} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cached)}) - resp = get_container_info(req.environ, 'xxx') + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + resp = get_container_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 3333) self.assertEquals(resp['object_count'], 10) self.assertEquals(resp['status'], 404) @@ -80,26 +222,45 @@ def test_get_container_info_env(self): self.assertEquals(resp['bytes'], 3867) def test_get_account_info_no_cache(self): - swift.proxy.controllers.base.make_pre_authed_request = FakeRequest req = Request.blank("/v1/AUTH_account", environ={'swift.cache': FakeCache({})}) - resp = get_account_info(req.environ, 'xxx') - print resp + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + resp = get_account_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 6666) self.assertEquals(resp['total_object_count'], 1000) def test_get_account_info_cache(self): - swift.proxy.controllers.base.make_pre_authed_request = FakeRequest + # The original test that we prefer to preserve cached = {'status': 404, 'bytes': 3333, 'total_object_count': 10} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cached)}) - resp = get_account_info(req.environ, 'xxx') + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + resp = get_account_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 3333) self.assertEquals(resp['total_object_count'], 10) self.assertEquals(resp['status'], 404) + # Here is a more realistic test + cached = {'status': 404, + 'bytes': '3333', + 'container_count': '234', + 'total_object_count': '10', + 'meta': {}} + req = Request.blank("/v1/account/cont", + environ={'swift.cache': FakeCache(cached)}) + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + resp = get_account_info(req.environ, 'xxx') + self.assertEquals(resp['status'], 404) + self.assertEquals(resp['bytes'], '3333') + self.assertEquals(resp['container_count'], 234) + self.assertEquals(resp['meta'], {}) + self.assertEquals(resp['total_object_count'], '10') + def test_get_account_info_env(self): cache_key = get_account_memcache_key("account") env_key = 'swift.%s' % cache_key diff --git a/test/unit/proxy/controllers/test_container.py b/test/unit/proxy/controllers/test_container.py new file mode 100644 index 0000000000..55fc6e391d --- /dev/null +++ b/test/unit/proxy/controllers/test_container.py @@ -0,0 +1,47 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import unittest +import swift.proxy.controllers.base + +from contextlib import contextmanager +from swift.common.swob import Request +from swift.proxy import server as proxy_server +from swift.proxy.controllers.base import headers_to_container_info +from test.unit import fake_http_connect, FakeRing, FakeMemcache + + +class TestContainerController(unittest.TestCase): + def setUp(self): + self.app = proxy_server.Application(None, FakeMemcache(), + account_ring=FakeRing(), + container_ring=FakeRing(), + object_ring=FakeRing) + + def test_container_info_in_response_env(self): + controller = proxy_server.ContainerController(self.app, 'a', 'c') + with mock.patch('swift.proxy.controllers.base.http_connect', + fake_http_connect(200, 200, body='')): + req = Request.blank('/a/c', {'PATH_INFO': '/a/c'}) + resp = controller.HEAD(req) + self.assertEqual(2, resp.status_int // 100) + self.assertTrue("swift.container/a/c" in resp.environ) + self.assertEqual(headers_to_container_info(resp.headers), + resp.environ['swift.container/a/c']) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index af1b50745a..cd9d59f826 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -34,7 +34,8 @@ from eventlet import sleep, spawn, wsgi, listen import simplejson -from test.unit import connect_tcp, readuntil2crlfs, FakeLogger, fake_http_connect +from test.unit import connect_tcp, readuntil2crlfs, FakeLogger, \ + fake_http_connect, FakeRing, FakeMemcache from swift.proxy import server as proxy_server from swift.account import server as account_server from swift.container import server as container_server @@ -195,79 +196,6 @@ def sortHeaderNames(headerNames): return ', '.join(headers) -class FakeRing(object): - - def __init__(self, replicas=3): - # 9 total nodes (6 more past the initial 3) is the cap, no matter if - # this is set higher, or R^2 for R replicas - self.replicas = replicas - self.max_more_nodes = 0 - self.devs = {} - - def set_replicas(self, replicas): - self.replicas = replicas - self.devs = {} - - @property - def replica_count(self): - return self.replicas - - def get_part(self, account, container=None, obj=None): - return 1 - - def get_nodes(self, account, container=None, obj=None): - devs = [] - for x in xrange(self.replicas): - devs.append(self.devs.get(x)) - if devs[x] is None: - self.devs[x] = devs[x] = \ - {'ip': '10.0.0.%s' % x, - 'port': 1000 + x, - 'device': 'sd' + (chr(ord('a') + x)), - 'id': x} - return 1, devs - - def get_part_nodes(self, part): - return self.get_nodes('blah')[1] - - def get_more_nodes(self, nodes): - # replicas^2 is the true cap - for x in xrange(self.replicas, min(self.replicas + self.max_more_nodes, - self.replicas * self.replicas)): - yield {'ip': '10.0.0.%s' % x, 'port': 1000 + x, 'device': 'sda'} - - -class FakeMemcache(object): - - def __init__(self): - self.store = {} - - def get(self, key): - return self.store.get(key) - - def keys(self): - return self.store.keys() - - def set(self, key, value, time=0): - self.store[key] = value - return True - - def incr(self, key, time=0): - self.store[key] = self.store.setdefault(key, 0) + 1 - return self.store[key] - - @contextmanager - def soft_lock(self, key, timeout=0, retries=5): - yield True - - def delete(self, key): - try: - del self.store[key] - except Exception: - pass - return True - - class FakeMemcacheReturnsNone(FakeMemcache): def get(self, key): @@ -392,9 +320,11 @@ def test_account_info_200(self): self.check_account_info_return(partition, nodes) self.assertEquals(count, 12345) + # Test the internal representation in memcache + # 'container_count' changed from int to str cache_key = get_account_memcache_key(self.account) container_info = {'status': 200, - 'container_count': 12345, + 'container_count': '12345', 'total_object_count': None, 'bytes': None, 'meta': {}} @@ -416,9 +346,11 @@ def test_account_info_404(self): self.check_account_info_return(partition, nodes, True) self.assertEquals(count, None) + # Test the internal representation in memcache + # 'container_count' changed from 0 to None cache_key = get_account_memcache_key(self.account) container_info = {'status': 404, - 'container_count': 0, + 'container_count': None, # internally keep None 'total_object_count': None, 'bytes': None, 'meta': {}} @@ -442,8 +374,9 @@ def test(*status_list): self.assertEquals(count, None) with save_globals(): - test(503, 404, 404) - test(404, 404, 503) + # We cache if we have two 404 responses - fail if only one + test(503, 503, 404) + test(504, 404, 503) test(404, 507, 503) test(503, 503, 503) @@ -481,14 +414,12 @@ def account_info(self, account, request, autocreate=False): # tests if 200 is cached and used def test_container_info_200(self): - def account_info(self, account, request, autocreate=False): - return True, True, 0 with save_globals(): headers = {'x-container-read': self.read_acl, 'x-container-write': self.write_acl} - swift.proxy.controllers.Controller.account_info = account_info - set_http_connect(200, headers=headers) + set_http_connect(200, # account_info is found + 200, headers=headers) # container_info is found ret = self.controller.container_info( self.account, self.container, self.request) self.check_container_info_return(ret) @@ -506,12 +437,28 @@ def account_info(self, account, request, autocreate=False): # tests if 404 is cached and used def test_container_info_404(self): - def account_info(self, account, request, autocreate=False): + def account_info(self, account, request): return True, True, 0 with save_globals(): - swift.proxy.controllers.Controller.account_info = account_info - set_http_connect(404, 404, 404) + set_http_connect(503, 204, # account_info found + 504, 404, 404) # container_info 'NotFound' + ret = self.controller.container_info( + self.account, self.container, self.request) + self.check_container_info_return(ret, True) + + cache_key = get_container_memcache_key(self.account, + self.container) + cache_value = self.memcache.get(cache_key) + self.assertTrue(isinstance(cache_value, dict)) + self.assertEquals(404, cache_value.get('status')) + + set_http_connect() + ret = self.controller.container_info( + self.account, self.container, self.request) + self.check_container_info_return(ret, True) + + set_http_connect(503, 404, 404)# account_info 'NotFound' ret = self.controller.container_info( self.account, self.container, self.request) self.check_container_info_return(ret, True) @@ -537,8 +484,9 @@ def test(*status_list): self.check_container_info_return(ret, True) with save_globals(): - test(503, 404, 404) - test(404, 404, 503) + # We cache if we have two 404 responses - fail if only one + test(503, 503, 404) + test(504, 404, 503) test(404, 507, 503) test(503, 503, 503) @@ -2328,36 +2276,50 @@ def test_acc_or_con_missing_returns_404(self): set_http_connect(404, 404, 404) # acct acct acct + # make sure to use a fresh request without cached env + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 404) set_http_connect(503, 404, 404) # acct acct acct + # make sure to use a fresh request without cached env + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 404) set_http_connect(503, 503, 404) # acct acct acct + # make sure to use a fresh request without cached env + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 404) set_http_connect(503, 503, 503) # acct acct acct + # make sure to use a fresh request without cached env + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 404) set_http_connect(200, 200, 204, 204, 204) # acct cont obj obj obj + # make sure to use a fresh request without cached env + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 204) set_http_connect(200, 404, 404, 404) # acct cont cont cont + # make sure to use a fresh request without cached env + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 404) set_http_connect(200, 503, 503, 503) # acct cont cont cont + # make sure to use a fresh request without cached env + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 404) @@ -2367,6 +2329,8 @@ def test_acc_or_con_missing_returns_404(self): set_http_connect(200) # acct [isn't actually called since everything # is error limited] + # make sure to use a fresh request without cached env + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 404) @@ -2378,6 +2342,8 @@ def test_acc_or_con_missing_returns_404(self): set_http_connect(200, 200) # acct cont [isn't actually called since # everything is error limited] + # make sure to use a fresh request without cached env + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 404) @@ -4633,10 +4599,10 @@ def assert_status_map(self, method, statuses, expected, def test_HEAD_GET(self): with save_globals(): - controller = proxy_server.ContainerController(self.app, 'account', - 'container') + controller = proxy_server.ContainerController(self.app, 'a', 'c') - def test_status_map(statuses, expected, **kwargs): + def test_status_map(statuses, expected, + c_expected=None, a_expected=None, **kwargs): set_http_connect(*statuses, **kwargs) self.app.memcache.store = {} req = Request.blank('/a/c', {}) @@ -4647,6 +4613,18 @@ def test_status_map(statuses, expected, **kwargs): if expected < 400: self.assert_('x-works' in res.headers) self.assertEquals(res.headers['x-works'], 'yes') + if c_expected: + self.assertTrue('swift.container/a/c' in res.environ) + self.assertEquals(res.environ['swift.container/a/c']['status'], + c_expected) + else: + self.assertTrue('swift.container/a/c' not in res.environ) + if a_expected: + self.assertTrue('swift.account/a' in res.environ) + self.assertEquals(res.environ['swift.account/a']['status'], + a_expected) + else: + self.assertTrue('swift.account/a' not in res.environ) set_http_connect(*statuses, **kwargs) self.app.memcache.store = {} @@ -4658,17 +4636,37 @@ def test_status_map(statuses, expected, **kwargs): if expected < 400: self.assert_('x-works' in res.headers) self.assertEquals(res.headers['x-works'], 'yes') - - test_status_map((200, 200, 404, 404), 200) - test_status_map((200, 200, 500, 404), 200) - test_status_map((200, 304, 500, 404), 304) - test_status_map((200, 404, 404, 404), 404) - test_status_map((200, 404, 404, 500), 404) - test_status_map((200, 500, 500, 500), 503) + if c_expected: + self.assertTrue('swift.container/a/c' in res.environ) + self.assertEquals(res.environ['swift.container/a/c']['status'], + c_expected) + else: + self.assertTrue('swift.container/a/c' not in res.environ) + if a_expected: + self.assertTrue('swift.account/a' in res.environ) + self.assertEquals(res.environ['swift.account/a']['status'], + a_expected) + else: + self.assertTrue('swift.account/a' not in res.environ) + # In all the following tests cache 200 for account + # return and ache vary for container + # return 200 and cache 200 for and container + test_status_map((200, 200, 404, 404), 200, 200, 200) + test_status_map((200, 200, 500, 404), 200, 200, 200) + # return 304 dont cache container + test_status_map((200, 304, 500, 404), 304, None, 200) + # return 404 and cache 404 for container + test_status_map((200, 404, 404, 404), 404, 404, 200) + test_status_map((200, 404, 404, 500), 404, 404, 200) + # return 503, dont cache container + test_status_map((200, 500, 500, 500), 503, None, 200) self.assertFalse(self.app.account_autocreate) - test_status_map((404, 404, 404), 404) - self.app.account_autocreate = True - test_status_map((404, 404, 404), 404) + + # In all the following tests cache 404 for account + # return 404 (as account is not found) and dont cache container + test_status_map((404, 404, 404), 404, None, 404) + self.app.account_autocreate = True # This should make no difference + test_status_map((404, 404, 404), 404, None, 404) def test_PUT(self): with save_globals(): @@ -4821,14 +4819,20 @@ def test_acc_missing_returns_404(self): self.assertEquals(resp.status_int, 200) set_http_connect(404, 404, 404, 200, 200, 200) + # Make sure it is a blank request wthout env caching + req = Request.blank('/a/c', environ={'REQUEST_METHOD': meth}) resp = getattr(controller, meth)(req) self.assertEquals(resp.status_int, 404) set_http_connect(503, 404, 404) + # Make sure it is a blank request wthout env caching + req = Request.blank('/a/c', environ={'REQUEST_METHOD': meth}) resp = getattr(controller, meth)(req) self.assertEquals(resp.status_int, 404) set_http_connect(503, 404, raise_exc=True) + # Make sure it is a blank request wthout env caching + req = Request.blank('/a/c', environ={'REQUEST_METHOD': meth}) resp = getattr(controller, meth)(req) self.assertEquals(resp.status_int, 404) @@ -4836,6 +4840,8 @@ def test_acc_missing_returns_404(self): dev['errors'] = self.app.error_suppression_limit + 1 dev['last_error'] = time.time() set_http_connect(200, 200, 200, 200, 200, 200) + # Make sure it is a blank request wthout env caching + req = Request.blank('/a/c', environ={'REQUEST_METHOD': meth}) resp = getattr(controller, meth)(req) self.assertEquals(resp.status_int, 404) @@ -5132,6 +5138,7 @@ def test_GET_no_content(self): req = Request.blank('/a/c') self.app.update_request(req) res = controller.GET(req) + self.assertEquals(res.environ['swift.container/a/c']['status'], 204) self.assertEquals(res.content_length, 0) self.assertTrue('transfer-encoding' not in res.headers) @@ -5149,6 +5156,7 @@ def authorize(req): req.environ['swift.authorize'] = authorize self.app.update_request(req) res = controller.GET(req) + self.assertEquals(res.environ['swift.container/a/c']['status'], 201) self.assert_(called[0]) def test_HEAD_calls_authorize(self): @@ -5444,18 +5452,24 @@ def setUp(self): container_ring=FakeRing(), object_ring=FakeRing) - def assert_status_map(self, method, statuses, expected): + def assert_status_map(self, method, statuses, expected, env_expected=None): with save_globals(): set_http_connect(*statuses) req = Request.blank('/a', {}) self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) + if env_expected: + self.assertEquals(res.environ['swift.account/a']['status'], + env_expected) set_http_connect(*statuses) req = Request.blank('/a/', {}) self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) + if env_expected: + self.assertEquals(res.environ['swift.account/a']['status'], + env_expected) def test_OPTIONS(self): with save_globals(): @@ -5501,23 +5515,23 @@ def test_GET(self): with save_globals(): controller = proxy_server.AccountController(self.app, 'account') # GET returns after the first successful call to an Account Server - self.assert_status_map(controller.GET, (200,), 200) - self.assert_status_map(controller.GET, (503, 200), 200) - self.assert_status_map(controller.GET, (503, 503, 200), 200) - self.assert_status_map(controller.GET, (204,), 204) - self.assert_status_map(controller.GET, (503, 204), 204) - self.assert_status_map(controller.GET, (503, 503, 204), 204) - self.assert_status_map(controller.GET, (404, 200), 200) - self.assert_status_map(controller.GET, (404, 404, 200), 200) - self.assert_status_map(controller.GET, (404, 503, 204), 204) + self.assert_status_map(controller.GET, (200,), 200, 200) + self.assert_status_map(controller.GET, (503, 200), 200, 200) + self.assert_status_map(controller.GET, (503, 503, 200), 200, 200) + self.assert_status_map(controller.GET, (204,), 204, 204) + self.assert_status_map(controller.GET, (503, 204), 204, 204) + self.assert_status_map(controller.GET, (503, 503, 204), 204, 204) + self.assert_status_map(controller.GET, (404, 200), 200, 200) + self.assert_status_map(controller.GET, (404, 404, 200), 200, 200) + self.assert_status_map(controller.GET, (404, 503, 204), 204, 204) # If Account servers fail, if autocreate = False, return majority # response - self.assert_status_map(controller.GET, (404, 404, 404), 404) - self.assert_status_map(controller.GET, (404, 404, 503), 404) + self.assert_status_map(controller.GET, (404, 404, 404), 404, 404) + self.assert_status_map(controller.GET, (404, 404, 503), 404, 404) self.assert_status_map(controller.GET, (404, 503, 503), 503) self.app.memcache = FakeMemcacheReturnsNone() - self.assert_status_map(controller.GET, (404, 404, 404), 404) + self.assert_status_map(controller.GET, (404, 404, 404), 404, 404) def test_GET_autocreate(self): @@ -5548,19 +5562,19 @@ def test_HEAD(self): # Same behaviour as GET with save_globals(): controller = proxy_server.AccountController(self.app, 'account') - self.assert_status_map(controller.HEAD, (200,), 200) - self.assert_status_map(controller.HEAD, (503, 200), 200) - self.assert_status_map(controller.HEAD, (503, 503, 200), 200) - self.assert_status_map(controller.HEAD, (204,), 204) - self.assert_status_map(controller.HEAD, (503, 204), 204) - self.assert_status_map(controller.HEAD, (204, 503, 503), 204) - self.assert_status_map(controller.HEAD, (204,), 204) - self.assert_status_map(controller.HEAD, (404, 404, 404), 404) - self.assert_status_map(controller.HEAD, (404, 404, 200), 200) - self.assert_status_map(controller.HEAD, (404, 200), 200) - self.assert_status_map(controller.HEAD, (404, 404, 503), 404) + self.assert_status_map(controller.HEAD, (200,), 200, 200) + self.assert_status_map(controller.HEAD, (503, 200), 200, 200) + self.assert_status_map(controller.HEAD, (503, 503, 200), 200, 200) + self.assert_status_map(controller.HEAD, (204,), 204, 204) + self.assert_status_map(controller.HEAD, (503, 204), 204, 204) + self.assert_status_map(controller.HEAD, (204, 503, 503), 204, 204) + self.assert_status_map(controller.HEAD, (204,), 204, 204) + self.assert_status_map(controller.HEAD, (404, 404, 404), 404, 404) + self.assert_status_map(controller.HEAD, (404, 404, 200), 200, 200) + self.assert_status_map(controller.HEAD, (404, 200), 200, 200) + self.assert_status_map(controller.HEAD, (404, 404, 503), 404, 404) self.assert_status_map(controller.HEAD, (404, 503, 503), 503) - self.assert_status_map(controller.HEAD, (404, 503, 204), 204) + self.assert_status_map(controller.HEAD, (404, 503, 204), 204, 204) def test_HEAD_autocreate(self): # Same behaviour as GET