From 93ab856f6640a2322846a6861a4dbe1085294667 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Wed, 4 Sep 2013 22:25:28 -0400 Subject: [PATCH 01/35] Add support for POST commit coverage runs Modeled on how keystone does it. Change-Id: Idad4f854f1bfb915a48ff69988553810a76accc0 Signed-off-by: Peter Portante --- tox.ini | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tox.ini b/tox.ini index 3dfdf78940..6c7e723e88 100644 --- a/tox.ini +++ b/tox.ini @@ -16,6 +16,13 @@ deps = -r{toxinidir}/test-requirements.txt commands = nosetests test/unit {posargs} +[testenv:cover] +setenv = VIRTUAL_ENV={envdir} + NOSE_WITH_COVERAGE=1 + NOSE_COVER_BRANCHES=1 + NOSE_COVER_HTML=1 + NOSE_COVER_HTML_DIR={toxinidir}/cover + [tox:jenkins] downloadcache = ~/cache/pip From 30b1590c3cbf066eee7d2411379654ceccd0a723 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Tue, 3 Sep 2013 23:08:14 -0400 Subject: [PATCH 02/35] Assume ETag is always in the metadata Currently the GET and HEAD calls always assume that the ETag is present in the on-disk metadata, as they index the metadata dictionary directly for the value to fill in the proper response header (a KeyError would be thrown if missing and turn into a 503). The close code that handles quarantining checked to see if an ETag is present in the metadata before making the comparison. However, a close operation would never even be attempted if an ETag was not present, since the response headers are filled in before the object is read. Change-Id: I5032251414eceb38079d235504cc9e589dea5f3e Signed-off-by: Peter Portante --- swift/obj/diskfile.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py index 80d806f265..94966884af 100644 --- a/swift/obj/diskfile.py +++ b/swift/obj/diskfile.py @@ -586,8 +586,7 @@ def _handle_close_quarantine(self): return if self.iter_etag and self.started_at_0 and self.read_to_eof and \ - 'ETag' in self._metadata and \ - self.iter_etag.hexdigest() != self._metadata.get('ETag'): + self.iter_etag.hexdigest() != self._metadata['ETag']: self.quarantine() def close(self, verify_file=True): From fffc95c3ccb01333becc86e0cb4c67cf5edf9725 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Tue, 16 Jul 2013 16:39:23 +0200 Subject: [PATCH 03/35] Handle X-Copy-From header in account_quota mw Content length of the copied object is checked before allowing the copy request according to the account quota set by Reseller. Fixes: bug #1200271 Change-Id: Ie4700f23466dd149ea5a497e6c72438cf52940fd --- swift/common/middleware/account_quotas.py | 19 ++- swift/proxy/controllers/base.py | 112 ++++++++++++++++++ .../common/middleware/test_account_quotas.py | 74 ++++++++++-- test/unit/proxy/controllers/test_base.py | 96 +++++++++++++-- 4 files changed, 278 insertions(+), 23 deletions(-) diff --git a/swift/common/middleware/account_quotas.py b/swift/common/middleware/account_quotas.py index 06b0d9aa6f..01a8336a0d 100644 --- a/swift/common/middleware/account_quotas.py +++ b/swift/common/middleware/account_quotas.py @@ -48,8 +48,7 @@ from swift.common.swob import HTTPForbidden, HTTPRequestEntityTooLarge, \ HTTPBadRequest, wsgify - -from swift.proxy.controllers.base import get_account_info +from swift.proxy.controllers.base import get_account_info, get_object_info class AccountQuotaMiddleware(object): @@ -68,7 +67,7 @@ def __call__(self, request): return self.app try: - request.split_path(2, 4, rest_with_last=True) + ver, acc, cont, obj = request.split_path(2, 4, rest_with_last=True) except ValueError: return self.app @@ -86,10 +85,22 @@ def __call__(self, request): if new_quota is not None: return HTTPForbidden() + copy_from = request.headers.get('X-Copy-From') + content_length = (request.content_length or 0) + + if obj and copy_from: + path = '/' + ver + '/' + acc + '/' + copy_from.lstrip('/') + object_info = get_object_info(request.environ, self.app, path) + if not object_info or not object_info['length']: + content_length = 0 + else: + content_length = int(object_info['length']) + account_info = get_account_info(request.environ, self.app) if not account_info or not account_info['bytes']: return self.app - new_size = int(account_info['bytes']) + (request.content_length or 0) + + new_size = int(account_info['bytes']) + content_length quota = int(account_info['meta'].get('quota-bytes', -1)) if 0 <= quota < new_size: diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 1e0b7146da..045bdfb5c9 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -152,6 +152,22 @@ def headers_to_container_info(headers, status_int=HTTP_OK): } +def headers_to_object_info(headers, status_int=HTTP_OK): + """ + Construct a cacheable dict of object info based on response headers. + """ + headers = dict((k.lower(), v) for k, v in dict(headers).iteritems()) + info = {'status': status_int, + 'length': headers.get('content-length'), + 'type': headers.get('content-type'), + 'etag': headers.get('etag'), + 'meta': dict((key[14:], value) + for key, value in headers.iteritems() + if key.startswith('x-object-meta-')) + } + return info + + def cors_validation(func): """ Decorator to check if the request is a CORS request and if so, if it's @@ -213,6 +229,21 @@ def wrapped(*a, **kw): return wrapped +def get_object_info(env, app, path=None, swift_source=None): + """ + Get the info structure for an object, based on env and app. + This is useful to middlewares. + Note: This call bypasses auth. Success does not imply that the + request has authorization to the object. + """ + (version, account, container, obj) = \ + split_path(path or env['PATH_INFO'], 4, 4, True) + info = _get_object_info(app, env, account, container, obj) + if not info: + info = headers_to_object_info({}, 0) + return info + + def get_container_info(env, app, swift_source=None): """ Get the info structure for a container, based on env and app. @@ -268,6 +299,19 @@ def _get_cache_key(account, container): return cache_key, env_key +def get_object_env_key(account, container, obj): + """ + Get the keys for env (env_key) where info about object is cached + :param account: The name of the account + :param container: The name of the container + :param obj: The name of the object + :returns a string env_key + """ + env_key = 'swift.object/%s/%s/%s' % (account, + container, obj) + return env_key + + def _set_info_cache(app, env, account, container, resp): """ Cache info in both memcache and env. @@ -315,6 +359,34 @@ def _set_info_cache(app, env, account, container, resp): env[env_key] = info +def _set_object_info_cache(app, env, account, container, obj, resp): + """ + Cache object info env. Do not cache object informations in + memcache. This is an intentional omission as it would lead + to cache pressure. This is a per-request cache. + + Caching is used to avoid unnecessary calls to object servers. + This is a private function that is being called by GETorHEAD_base. + Any attempt to GET or HEAD from the object server should use + the GETorHEAD_base interface which would then set the cache. + + :param app: the application object + :param account: the unquoted account name + :param container: the unquoted container name or None + :param object: the unquoted object name or None + :param resp: the response received or None if info cache should be cleared + """ + + env_key = get_object_env_key(account, container, obj) + + if not resp: + env.pop(env_key, None) + return + + info = headers_to_object_info(resp.headers, resp.status_int) + env[env_key] = info + + def clear_info_cache(app, env, account, container=None): """ Clear the cached info in both memcache and env @@ -406,6 +478,40 @@ def get_info(app, env, account, container=None, ret_not_found=False): return None +def _get_object_info(app, env, account, container, obj): + """ + Get the info about object + + Note: This call bypasses auth. Success does not imply that the + request has authorization to the info. + + :param app: the application object + :param env: the environment used by the current request + :param account: The unquoted name of the account + :param container: The unquoted name of the container + :param obj: The unquoted name of the object + :returns: the cached info or None if cannot be retrieved + """ + env_key = get_object_env_key(account, container, obj) + info = env.get(env_key) + if info: + return info + # Not in cached, let's try the object servers + path = '/v1/%s/%s/%s' % (account, container, obj) + req = _prepare_pre_auth_info_request(env, path) + # Whenever we do a GET/HEAD, the GETorHEAD_base will set the info in + # the environment under environ[env_key]. We will + # pick the one from environ[env_key] and use it to set the caller env + resp = req.get_response(app) + try: + info = resp.environ[env_key] + env[env_key] = info + return info + except (KeyError, AttributeError): + pass + return None + + class Controller(object): """Base WSGI controller class for the proxy""" server_type = 'Base' @@ -1017,6 +1123,12 @@ def GETorHEAD_base(self, req, server_type, ring, partition, path): _set_info_cache(self.app, req.environ, account, container, res) except ValueError: pass + try: + (account, container, obj) = split_path(req.path_info, 3, 3, True) + _set_object_info_cache(self.app, req.environ, account, + container, obj, res) + except ValueError: + pass return res def is_origin_allowed(self, cors_info, origin): diff --git a/test/unit/common/middleware/test_account_quotas.py b/test/unit/common/middleware/test_account_quotas.py index 30f5a70d04..137309df9f 100644 --- a/test/unit/common/middleware/test_account_quotas.py +++ b/test/unit/common/middleware/test_account_quotas.py @@ -18,7 +18,8 @@ from swift.common.middleware import account_quotas from swift.proxy.controllers.base import _get_cache_key, \ - headers_to_account_info + headers_to_account_info, get_object_env_key, \ + headers_to_object_info class FakeCache(object): @@ -46,17 +47,22 @@ def __init__(self, headers=[]): self.headers = headers def __call__(self, env, start_response): - # Cache the account_info (same as a real application) - cache_key, env_key = _get_cache_key('a', None) - env[env_key] = headers_to_account_info(self.headers, 200) - start_response('200 OK', self.headers) + if env['REQUEST_METHOD'] == "HEAD" and \ + env['PATH_INFO'] == '/v1/a/c2/o2': + env_key = get_object_env_key('a', 'c2', 'o2') + env[env_key] = headers_to_object_info(self.headers, 200) + start_response('200 OK', self.headers) + elif env['REQUEST_METHOD'] == "HEAD" and \ + env['PATH_INFO'] == '/v1/a/c2/o3': + start_response('404 Not Found', []) + else: + # Cache the account_info (same as a real application) + cache_key, env_key = _get_cache_key('a', None) + env[env_key] = headers_to_account_info(self.headers, 200) + start_response('200 OK', self.headers) return [] -def start_response(*args): - pass - - class TestAccountQuota(unittest.TestCase): def test_unauthorized(self): @@ -91,6 +97,43 @@ def test_exceed_bytes_quota(self): res = req.get_response(app) self.assertEquals(res.status_int, 413) + def test_exceed_bytes_quota_copy_from(self): + headers = [('x-account-bytes-used', '500'), + ('x-account-meta-quota-bytes', '1000'), + ('content-length', '1000')] + app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) + cache = FakeCache(None) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o2'}) + res = req.get_response(app) + self.assertEquals(res.status_int, 413) + + def test_not_exceed_bytes_quota_copy_from(self): + headers = [('x-account-bytes-used', '0'), + ('x-account-meta-quota-bytes', '1000')] + app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) + cache = FakeCache(None) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o2'}) + res = req.get_response(app) + self.assertEquals(res.status_int, 200) + + def test_quota_copy_from_no_src(self): + headers = [('x-account-bytes-used', '0'), + ('x-account-meta-quota-bytes', '1000')] + app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) + cache = FakeCache(None) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache}, + headers={'x-copy-from': '/c2/o3'}) + res = req.get_response(app) + self.assertEquals(res.status_int, 200) + def test_exceed_bytes_quota_reseller(self): headers = [('x-account-bytes-used', '1000'), ('x-account-meta-quota-bytes', '0')] @@ -103,6 +146,19 @@ def test_exceed_bytes_quota_reseller(self): res = req.get_response(app) self.assertEquals(res.status_int, 200) + def test_exceed_bytes_quota_reseller_copy_from(self): + headers = [('x-account-bytes-used', '1000'), + ('x-account-meta-quota-bytes', '0')] + app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) + cache = FakeCache(None) + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT', + 'swift.cache': cache, + 'reseller_request': True}, + headers={'x-copy-from': 'c2/o2'}) + res = req.get_response(app) + self.assertEquals(res.status_int, 200) + def test_bad_application_quota(self): headers = [] app = account_quotas.AccountQuotaMiddleware(FakeBadApp(headers)) diff --git a/test/unit/proxy/controllers/test_base.py b/test/unit/proxy/controllers/test_base.py index fde8f19532..9af1a9f1d1 100644 --- a/test/unit/proxy/controllers/test_base.py +++ b/test/unit/proxy/controllers/test_base.py @@ -16,9 +16,9 @@ import unittest from mock import patch from swift.proxy.controllers.base import headers_to_container_info, \ - headers_to_account_info, get_container_info, get_container_memcache_key, \ - get_account_info, get_account_memcache_key, _get_cache_key, get_info, \ - Controller + headers_to_account_info, headers_to_object_info, get_container_info, \ + get_container_memcache_key, get_account_info, get_account_memcache_key, \ + get_object_env_key, _get_cache_key, get_info, get_object_info, Controller from swift.common.swob import Request from swift.common.utils import split_path from test.unit import fake_http_connect, FakeRing, FakeMemcache @@ -29,12 +29,18 @@ class FakeResponse(object): - def __init__(self, headers, env, account, container): + def __init__(self, headers, env, account, container, obj): self.headers = headers self.status_int = FakeResponse_status_int self.environ = env - cache_key, env_key = _get_cache_key(account, container) - if container: + if obj: + env_key = get_object_env_key(account, container, obj) + else: + cache_key, env_key = _get_cache_key(account, container) + + if account and container and obj: + info = headers_to_object_info(headers, FakeResponse_status_int) + elif account and container: info = headers_to_container_info(headers, FakeResponse_status_int) else: info = headers_to_account_info(headers, FakeResponse_status_int) @@ -47,13 +53,18 @@ def __init__(self, env, path): (version, account, container, obj) = split_path(path, 2, 4, True) self.account = account self.container = container - stype = container and 'container' or 'account' - self.headers = {'x-%s-object-count' % (stype): 1000, - 'x-%s-bytes-used' % (stype): 6666} + self.obj = obj + if obj: + self.headers = {'content-length': 5555, + 'content-type': 'text/plain'} + else: + stype = container and 'container' or 'account' + self.headers = {'x-%s-object-count' % (stype): 1000, + 'x-%s-bytes-used' % (stype): 6666} def get_response(self, app): return FakeResponse(self.headers, self.environ, self.account, - self.container) + self.container, self.obj) class FakeCache(object): @@ -73,6 +84,21 @@ def setUp(self): def test_GETorHEAD_base(self): base = Controller(self.app) + req = Request.blank('/a/c/o/with/slashes') + with patch('swift.proxy.controllers.base.' + 'http_connect', fake_http_connect(200)): + resp = base.GETorHEAD_base(req, 'object', FakeRing(), 'part', + '/a/c/o/with/slashes') + self.assertTrue('swift.object/a/c/o/with/slashes' in resp.environ) + self.assertEqual( + resp.environ['swift.object/a/c/o/with/slashes']['status'], 200) + req = Request.blank('/a/c/o') + with patch('swift.proxy.controllers.base.' + 'http_connect', fake_http_connect(200)): + resp = base.GETorHEAD_base(req, 'object', FakeRing(), 'part', + '/a/c/o') + self.assertTrue('swift.object/a/c/o' in resp.environ) + self.assertEqual(resp.environ['swift.object/a/c/o']['status'], 200) req = Request.blank('/a/c') with patch('swift.proxy.controllers.base.' 'http_connect', fake_http_connect(200)): @@ -273,6 +299,28 @@ def test_get_account_info_env(self): resp = get_account_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 3867) + def test_get_object_info_env(self): + cached = {'status': 200, + 'length': 3333, + 'type': 'application/json', + 'meta': {}} + env_key = get_object_env_key("account", "cont", "obj") + req = Request.blank("/v1/account/cont/obj", + environ={env_key: cached, + 'swift.cache': FakeCache({})}) + resp = get_object_info(req.environ, 'xxx') + self.assertEquals(resp['length'], 3333) + self.assertEquals(resp['type'], 'application/json') + + def test_get_object_info_no_env(self): + req = Request.blank("/v1/account/cont/obj", + environ={'swift.cache': FakeCache({})}) + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + resp = get_object_info(req.environ, 'xxx') + self.assertEquals(resp['length'], 5555) + self.assertEquals(resp['type'], 'text/plain') + def test_headers_to_container_info_missing(self): resp = headers_to_container_info({}, 404) self.assertEquals(resp['status'], 404) @@ -331,3 +379,31 @@ def test_headers_to_account_info_values(self): self.assertEquals( resp, headers_to_account_info(headers.items(), 200)) + + def test_headers_to_object_info_missing(self): + resp = headers_to_object_info({}, 404) + self.assertEquals(resp['status'], 404) + self.assertEquals(resp['length'], None) + self.assertEquals(resp['etag'], None) + + def test_headers_to_object_info_meta(self): + headers = {'X-Object-Meta-Whatevs': 14, + 'x-object-meta-somethingelse': 0} + resp = headers_to_object_info(headers.items(), 200) + self.assertEquals(len(resp['meta']), 2) + self.assertEquals(resp['meta']['whatevs'], 14) + self.assertEquals(resp['meta']['somethingelse'], 0) + + def test_headers_to_object_info_values(self): + headers = { + 'content-length': '1024', + 'content-type': 'application/json', + } + resp = headers_to_object_info(headers.items(), 200) + self.assertEquals(resp['length'], '1024') + self.assertEquals(resp['type'], 'application/json') + + headers['x-unused-header'] = 'blahblahblah' + self.assertEquals( + resp, + headers_to_object_info(headers.items(), 200)) From d4b024ad7d64a854072517eef47b059c93bdfdd3 Mon Sep 17 00:00:00 2001 From: Pete Zaitcev Date: Tue, 10 Sep 2013 13:30:28 -0600 Subject: [PATCH 04/35] Split backends off swift/common/db.py The main purpose of this patch is to lay the groundwork for allowing the container and account servers to optionally use pluggable backend implementations. The backend.py files will eventually be the module where the backend APIs are defined via docstrings of this reference implementation. The swift/common/db.py module will remain an internal module used by the reference implementation. We have a raft of changes to docstrings staged for later, but this patch takes care to relocate ContainerBroker and AccountBroker into their new home intact. Change-Id: Ibab5c7605860ab768c8aa5a3161a705705689b04 --- doc/source/backends.rst | 16 + doc/source/index.rst | 1 + swift/account/auditor.py | 2 +- swift/account/backend.py | 416 +++++++ swift/account/reaper.py | 4 +- swift/account/replicator.py | 5 +- swift/account/server.py | 7 +- swift/common/db.py | 853 -------------- swift/container/auditor.py | 2 +- swift/container/backend.py | 496 ++++++++ swift/container/replicator.py | 5 +- swift/container/server.py | 9 +- swift/container/sync.py | 2 +- swift/container/updater.py | 2 +- test/unit/account/test_backend.py | 540 +++++++++ test/unit/common/test_db.py | 1704 +-------------------------- test/unit/container/test_backend.py | 1205 +++++++++++++++++++ test/unit/container/test_updater.py | 2 +- 18 files changed, 2699 insertions(+), 2572 deletions(-) create mode 100644 doc/source/backends.rst create mode 100644 swift/account/backend.py create mode 100644 swift/container/backend.py create mode 100644 test/unit/account/test_backend.py create mode 100644 test/unit/container/test_backend.py diff --git a/doc/source/backends.rst b/doc/source/backends.rst new file mode 100644 index 0000000000..4f37ffca1c --- /dev/null +++ b/doc/source/backends.rst @@ -0,0 +1,16 @@ +====================================== +Pluggable Back-ends: API Documentation +====================================== + +.. automodule:: swift.account.backend + :private-members: + :members: + :undoc-members: + +.. automodule:: swift.container.backend + :private-members: + :members: + :undoc-members: + +.. automodule:: swift.obj.diskfile + :members: diff --git a/doc/source/index.rst b/doc/source/index.rst index 1f55071e8b..9223e9a706 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -66,6 +66,7 @@ Developer Documentation development_guidelines development_saio development_auth + backends Administrator Documentation =========================== diff --git a/swift/account/auditor.py b/swift/account/auditor.py index 1259b75f23..5f81490d94 100644 --- a/swift/account/auditor.py +++ b/swift/account/auditor.py @@ -20,7 +20,7 @@ import swift.common.db from swift.account import server as account_server -from swift.common.db import AccountBroker +from swift.account.backend import AccountBroker from swift.common.utils import get_logger, audit_location_generator, \ config_true_value, dump_recon_cache, ratelimit_sleep from swift.common.daemon import Daemon diff --git a/swift/account/backend.py b/swift/account/backend.py new file mode 100644 index 0000000000..866d69d269 --- /dev/null +++ b/swift/account/backend.py @@ -0,0 +1,416 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Pluggable Back-end for Account Server +""" + +from __future__ import with_statement +import os +from uuid import uuid4 +import time +import cPickle as pickle +import errno + +import sqlite3 + +from swift.common.utils import normalize_timestamp, lock_parent_directory +from swift.common.db import DatabaseBroker, DatabaseConnectionError, \ + PENDING_CAP, PICKLE_PROTOCOL, utf8encode + + +class AccountBroker(DatabaseBroker): + """Encapsulates working with a account database.""" + db_type = 'account' + db_contains_type = 'container' + db_reclaim_timestamp = 'delete_timestamp' + + def _initialize(self, conn, put_timestamp): + """ + Create a brand new database (tables, indices, triggers, etc.) + + :param conn: DB connection object + :param put_timestamp: put timestamp + """ + if not self.account: + raise ValueError( + 'Attempting to create a new database with no account set') + self.create_container_table(conn) + self.create_account_stat_table(conn, put_timestamp) + + def create_container_table(self, conn): + """ + Create container table which is specific to the account DB. + + :param conn: DB connection object + """ + conn.executescript(""" + CREATE TABLE container ( + ROWID INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT, + put_timestamp TEXT, + delete_timestamp TEXT, + object_count INTEGER, + bytes_used INTEGER, + deleted INTEGER DEFAULT 0 + ); + + CREATE INDEX ix_container_deleted_name ON + container (deleted, name); + + CREATE TRIGGER container_insert AFTER INSERT ON container + BEGIN + UPDATE account_stat + SET container_count = container_count + (1 - new.deleted), + object_count = object_count + new.object_count, + bytes_used = bytes_used + new.bytes_used, + hash = chexor(hash, new.name, + new.put_timestamp || '-' || + new.delete_timestamp || '-' || + new.object_count || '-' || new.bytes_used); + END; + + CREATE TRIGGER container_update BEFORE UPDATE ON container + BEGIN + SELECT RAISE(FAIL, 'UPDATE not allowed; DELETE and INSERT'); + END; + + + CREATE TRIGGER container_delete AFTER DELETE ON container + BEGIN + UPDATE account_stat + SET container_count = container_count - (1 - old.deleted), + object_count = object_count - old.object_count, + bytes_used = bytes_used - old.bytes_used, + hash = chexor(hash, old.name, + old.put_timestamp || '-' || + old.delete_timestamp || '-' || + old.object_count || '-' || old.bytes_used); + END; + """) + + def create_account_stat_table(self, conn, put_timestamp): + """ + Create account_stat table which is specific to the account DB. + + :param conn: DB connection object + :param put_timestamp: put timestamp + """ + conn.executescript(""" + CREATE TABLE account_stat ( + account TEXT, + created_at TEXT, + put_timestamp TEXT DEFAULT '0', + delete_timestamp TEXT DEFAULT '0', + container_count INTEGER, + object_count INTEGER DEFAULT 0, + bytes_used INTEGER DEFAULT 0, + hash TEXT default '00000000000000000000000000000000', + id TEXT, + status TEXT DEFAULT '', + status_changed_at TEXT DEFAULT '0', + metadata TEXT DEFAULT '' + ); + + INSERT INTO account_stat (container_count) VALUES (0); + """) + + conn.execute(''' + UPDATE account_stat SET account = ?, created_at = ?, id = ?, + put_timestamp = ? + ''', (self.account, normalize_timestamp(time.time()), str(uuid4()), + put_timestamp)) + + def get_db_version(self, conn): + if self._db_version == -1: + self._db_version = 0 + for row in conn.execute(''' + SELECT name FROM sqlite_master + WHERE name = 'ix_container_deleted_name' '''): + self._db_version = 1 + return self._db_version + + def _delete_db(self, conn, timestamp, force=False): + """ + Mark the DB as deleted. + + :param conn: DB connection object + :param timestamp: timestamp to mark as deleted + """ + conn.execute(""" + UPDATE account_stat + SET delete_timestamp = ?, + status = 'DELETED', + status_changed_at = ? + WHERE delete_timestamp < ? """, (timestamp, timestamp, timestamp)) + + def _commit_puts_load(self, item_list, entry): + (name, put_timestamp, delete_timestamp, + object_count, bytes_used, deleted) = \ + pickle.loads(entry.decode('base64')) + item_list.append( + {'name': name, + 'put_timestamp': put_timestamp, + 'delete_timestamp': delete_timestamp, + 'object_count': object_count, + 'bytes_used': bytes_used, + 'deleted': deleted}) + + def empty(self): + """ + Check if the account DB is empty. + + :returns: True if the database has no active containers. + """ + self._commit_puts_stale_ok() + with self.get() as conn: + row = conn.execute( + 'SELECT container_count from account_stat').fetchone() + return (row[0] == 0) + + def put_container(self, name, put_timestamp, delete_timestamp, + object_count, bytes_used): + """ + Create a container with the given attributes. + + :param name: name of the container to create + :param put_timestamp: put_timestamp of the container to create + :param delete_timestamp: delete_timestamp of the container to create + :param object_count: number of objects in the container + :param bytes_used: number of bytes used by the container + """ + if delete_timestamp > put_timestamp and \ + object_count in (None, '', 0, '0'): + deleted = 1 + else: + deleted = 0 + record = {'name': name, 'put_timestamp': put_timestamp, + 'delete_timestamp': delete_timestamp, + 'object_count': object_count, + 'bytes_used': bytes_used, + 'deleted': deleted} + if self.db_file == ':memory:': + self.merge_items([record]) + return + if not os.path.exists(self.db_file): + raise DatabaseConnectionError(self.db_file, "DB doesn't exist") + pending_size = 0 + try: + pending_size = os.path.getsize(self.pending_file) + except OSError as err: + if err.errno != errno.ENOENT: + raise + if pending_size > PENDING_CAP: + self._commit_puts([record]) + else: + with lock_parent_directory(self.pending_file, + self.pending_timeout): + with open(self.pending_file, 'a+b') as fp: + # Colons aren't used in base64 encoding; so they are our + # delimiter + fp.write(':') + fp.write(pickle.dumps( + (name, put_timestamp, delete_timestamp, object_count, + bytes_used, deleted), + protocol=PICKLE_PROTOCOL).encode('base64')) + fp.flush() + + def is_deleted(self): + """ + Check if the account DB is considered to be deleted. + + :returns: True if the account DB is considered to be deleted, False + otherwise + """ + if self.db_file != ':memory:' and not os.path.exists(self.db_file): + return True + self._commit_puts_stale_ok() + with self.get() as conn: + row = conn.execute(''' + SELECT put_timestamp, delete_timestamp, container_count, status + FROM account_stat''').fetchone() + return row['status'] == 'DELETED' or ( + row['container_count'] in (None, '', 0, '0') and + row['delete_timestamp'] > row['put_timestamp']) + + def is_status_deleted(self): + """Only returns true if the status field is set to DELETED.""" + with self.get() as conn: + row = conn.execute(''' + SELECT status + FROM account_stat''').fetchone() + return (row['status'] == "DELETED") + + def get_info(self): + """ + Get global data for the account. + + :returns: dict with keys: account, created_at, put_timestamp, + delete_timestamp, container_count, object_count, + bytes_used, hash, id + """ + self._commit_puts_stale_ok() + with self.get() as conn: + return dict(conn.execute(''' + SELECT account, created_at, put_timestamp, delete_timestamp, + container_count, object_count, bytes_used, hash, id + FROM account_stat + ''').fetchone()) + + def list_containers_iter(self, limit, marker, end_marker, prefix, + delimiter): + """ + Get a list of containers sorted by name starting at marker onward, up + to limit entries. Entries will begin with the prefix and will not have + the delimiter after the prefix. + + :param limit: maximum number of entries to get + :param marker: marker query + :param end_marker: end marker query + :param prefix: prefix query + :param delimiter: delimiter for query + + :returns: list of tuples of (name, object_count, bytes_used, 0) + """ + (marker, end_marker, prefix, delimiter) = utf8encode( + marker, end_marker, prefix, delimiter) + self._commit_puts_stale_ok() + if delimiter and not prefix: + prefix = '' + orig_marker = marker + with self.get() as conn: + results = [] + while len(results) < limit: + query = """ + SELECT name, object_count, bytes_used, 0 + FROM container + WHERE deleted = 0 AND """ + query_args = [] + if end_marker: + query += ' name < ? AND' + query_args.append(end_marker) + if marker and marker >= prefix: + query += ' name > ? AND' + query_args.append(marker) + elif prefix: + query += ' name >= ? AND' + query_args.append(prefix) + if self.get_db_version(conn) < 1: + query += ' +deleted = 0' + else: + query += ' deleted = 0' + query += ' ORDER BY name LIMIT ?' + query_args.append(limit - len(results)) + curs = conn.execute(query, query_args) + curs.row_factory = None + + if prefix is None: + # A delimiter without a specified prefix is ignored + return [r for r in curs] + if not delimiter: + if not prefix: + # It is possible to have a delimiter but no prefix + # specified. As above, the prefix will be set to the + # empty string, so avoid performing the extra work to + # check against an empty prefix. + return [r for r in curs] + else: + return [r for r in curs if r[0].startswith(prefix)] + + # We have a delimiter and a prefix (possibly empty string) to + # handle + rowcount = 0 + for row in curs: + rowcount += 1 + marker = name = row[0] + if len(results) >= limit or not name.startswith(prefix): + curs.close() + return results + end = name.find(delimiter, len(prefix)) + if end > 0: + marker = name[:end] + chr(ord(delimiter) + 1) + dir_name = name[:end + 1] + if dir_name != orig_marker: + results.append([dir_name, 0, 0, 1]) + curs.close() + break + results.append(row) + if not rowcount: + break + return results + + def merge_items(self, item_list, source=None): + """ + Merge items into the container table. + + :param item_list: list of dictionaries of {'name', 'put_timestamp', + 'delete_timestamp', 'object_count', 'bytes_used', + 'deleted'} + :param source: if defined, update incoming_sync with the source + """ + with self.get() as conn: + max_rowid = -1 + for rec in item_list: + record = [rec['name'], rec['put_timestamp'], + rec['delete_timestamp'], rec['object_count'], + rec['bytes_used'], rec['deleted']] + query = ''' + SELECT name, put_timestamp, delete_timestamp, + object_count, bytes_used, deleted + FROM container WHERE name = ? + ''' + if self.get_db_version(conn) >= 1: + query += ' AND deleted IN (0, 1)' + curs = conn.execute(query, (rec['name'],)) + curs.row_factory = None + row = curs.fetchone() + if row: + row = list(row) + for i in xrange(5): + if record[i] is None and row[i] is not None: + record[i] = row[i] + if row[1] > record[1]: # Keep newest put_timestamp + record[1] = row[1] + if row[2] > record[2]: # Keep newest delete_timestamp + record[2] = row[2] + # If deleted, mark as such + if record[2] > record[1] and \ + record[3] in (None, '', 0, '0'): + record[5] = 1 + else: + record[5] = 0 + conn.execute(''' + DELETE FROM container WHERE name = ? AND + deleted IN (0, 1) + ''', (record[0],)) + conn.execute(''' + INSERT INTO container (name, put_timestamp, + delete_timestamp, object_count, bytes_used, + deleted) + VALUES (?, ?, ?, ?, ?, ?) + ''', record) + if source: + max_rowid = max(max_rowid, rec['ROWID']) + if source: + try: + conn.execute(''' + INSERT INTO incoming_sync (sync_point, remote_id) + VALUES (?, ?) + ''', (max_rowid, source)) + except sqlite3.IntegrityError: + conn.execute(''' + UPDATE incoming_sync SET sync_point=max(?, sync_point) + WHERE remote_id=? + ''', (max_rowid, source)) + conn.commit() diff --git a/swift/account/reaper.py b/swift/account/reaper.py index 8df2758b7b..90265b9ea5 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -24,7 +24,7 @@ import swift.common.db from swift.account.server import DATADIR -from swift.common.db import AccountBroker +from swift.account.backend import AccountBroker from swift.common.direct_client import ClientException, \ direct_delete_container, direct_delete_object, direct_get_container from swift.common.ring import Ring @@ -206,7 +206,7 @@ def reap_account(self, broker, partition, nodes): .. seealso:: - :class:`swift.common.db.AccountBroker` for the broker class. + :class:`swift.account.backend.AccountBroker` for the broker class. .. seealso:: diff --git a/swift/account/replicator.py b/swift/account/replicator.py index c7f93d9b90..a4ee4373b4 100644 --- a/swift/account/replicator.py +++ b/swift/account/replicator.py @@ -14,11 +14,12 @@ # limitations under the License. from swift.account import server as account_server -from swift.common import db, db_replicator +from swift.account.backend import AccountBroker +from swift.common import db_replicator class AccountReplicator(db_replicator.Replicator): server_type = 'account' - brokerclass = db.AccountBroker + brokerclass = AccountBroker datadir = account_server.DATADIR default_port = 6002 diff --git a/swift/account/server.py b/swift/account/server.py index dc4154cb39..99bfc58cd6 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -23,8 +23,9 @@ from eventlet import Timeout import swift.common.db +from swift.account.backend import AccountBroker from swift.account.utils import account_listing_response -from swift.common.db import AccountBroker, DatabaseConnectionError +from swift.common.db import DatabaseConnectionError, DatabaseAlreadyExists from swift.common.request_helpers import get_param, get_listing_content_type, \ split_and_validate_path from swift.common.utils import get_logger, hash_path, public, \ @@ -119,7 +120,7 @@ def PUT(self, req): try: broker.initialize(normalize_timestamp( req.headers.get('x-timestamp') or time.time())) - except swift.common.db.DatabaseAlreadyExists: + except DatabaseAlreadyExists: pass if req.headers.get('x-account-override-deleted', 'no').lower() != \ 'yes' and broker.is_deleted(): @@ -140,7 +141,7 @@ def PUT(self, req): try: broker.initialize(timestamp) created = True - except swift.common.db.DatabaseAlreadyExists: + except DatabaseAlreadyExists: pass elif broker.is_status_deleted(): return self._deleted_response(broker, req, HTTPForbidden, diff --git a/swift/common/db.py b/swift/common/db.py index 0027ca820c..f48c68ab13 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -23,7 +23,6 @@ from uuid import uuid4 import sys import time -import cPickle as pickle import errno from swift import gettext_ as _ from tempfile import mkstemp @@ -731,855 +730,3 @@ def update_put_timestamp(self, timestamp): ' WHERE put_timestamp < ?' % self.db_type, (timestamp, timestamp)) conn.commit() - - -class ContainerBroker(DatabaseBroker): - """Encapsulates working with a container database.""" - db_type = 'container' - db_contains_type = 'object' - db_reclaim_timestamp = 'created_at' - - def _initialize(self, conn, put_timestamp): - """Creates a brand new database (tables, indices, triggers, etc.)""" - if not self.account: - raise ValueError( - 'Attempting to create a new database with no account set') - if not self.container: - raise ValueError( - 'Attempting to create a new database with no container set') - self.create_object_table(conn) - self.create_container_stat_table(conn, put_timestamp) - - def create_object_table(self, conn): - """ - Create the object table which is specifc to the container DB. - - :param conn: DB connection object - """ - conn.executescript(""" - CREATE TABLE object ( - ROWID INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT, - created_at TEXT, - size INTEGER, - content_type TEXT, - etag TEXT, - deleted INTEGER DEFAULT 0 - ); - - CREATE INDEX ix_object_deleted_name ON object (deleted, name); - - CREATE TRIGGER object_insert AFTER INSERT ON object - BEGIN - UPDATE container_stat - SET object_count = object_count + (1 - new.deleted), - bytes_used = bytes_used + new.size, - hash = chexor(hash, new.name, new.created_at); - END; - - CREATE TRIGGER object_update BEFORE UPDATE ON object - BEGIN - SELECT RAISE(FAIL, 'UPDATE not allowed; DELETE and INSERT'); - END; - - CREATE TRIGGER object_delete AFTER DELETE ON object - BEGIN - UPDATE container_stat - SET object_count = object_count - (1 - old.deleted), - bytes_used = bytes_used - old.size, - hash = chexor(hash, old.name, old.created_at); - END; - """) - - def create_container_stat_table(self, conn, put_timestamp=None): - """ - Create the container_stat table which is specific to the container DB. - - :param conn: DB connection object - :param put_timestamp: put timestamp - """ - if put_timestamp is None: - put_timestamp = normalize_timestamp(0) - conn.executescript(""" - CREATE TABLE container_stat ( - account TEXT, - container TEXT, - created_at TEXT, - put_timestamp TEXT DEFAULT '0', - delete_timestamp TEXT DEFAULT '0', - object_count INTEGER, - bytes_used INTEGER, - reported_put_timestamp TEXT DEFAULT '0', - reported_delete_timestamp TEXT DEFAULT '0', - reported_object_count INTEGER DEFAULT 0, - reported_bytes_used INTEGER DEFAULT 0, - hash TEXT default '00000000000000000000000000000000', - id TEXT, - status TEXT DEFAULT '', - status_changed_at TEXT DEFAULT '0', - metadata TEXT DEFAULT '', - x_container_sync_point1 INTEGER DEFAULT -1, - x_container_sync_point2 INTEGER DEFAULT -1 - ); - - INSERT INTO container_stat (object_count, bytes_used) - VALUES (0, 0); - """) - conn.execute(''' - UPDATE container_stat - SET account = ?, container = ?, created_at = ?, id = ?, - put_timestamp = ? - ''', (self.account, self.container, normalize_timestamp(time.time()), - str(uuid4()), put_timestamp)) - - def get_db_version(self, conn): - if self._db_version == -1: - self._db_version = 0 - for row in conn.execute(''' - SELECT name FROM sqlite_master - WHERE name = 'ix_object_deleted_name' '''): - self._db_version = 1 - return self._db_version - - def _newid(self, conn): - conn.execute(''' - UPDATE container_stat - SET reported_put_timestamp = 0, reported_delete_timestamp = 0, - reported_object_count = 0, reported_bytes_used = 0''') - - def _delete_db(self, conn, timestamp): - """ - Mark the DB as deleted - - :param conn: DB connection object - :param timestamp: timestamp to mark as deleted - """ - conn.execute(""" - UPDATE container_stat - SET delete_timestamp = ?, - status = 'DELETED', - status_changed_at = ? - WHERE delete_timestamp < ? """, (timestamp, timestamp, timestamp)) - - def _commit_puts_load(self, item_list, entry): - (name, timestamp, size, content_type, etag, deleted) = \ - pickle.loads(entry.decode('base64')) - item_list.append({'name': name, - 'created_at': timestamp, - 'size': size, - 'content_type': content_type, - 'etag': etag, - 'deleted': deleted}) - - def empty(self): - """ - Check if the DB is empty. - - :returns: True if the database has no active objects, False otherwise - """ - self._commit_puts_stale_ok() - with self.get() as conn: - row = conn.execute( - 'SELECT object_count from container_stat').fetchone() - return (row[0] == 0) - - def delete_object(self, name, timestamp): - """ - Mark an object deleted. - - :param name: object name to be deleted - :param timestamp: timestamp when the object was marked as deleted - """ - self.put_object(name, timestamp, 0, 'application/deleted', 'noetag', 1) - - def put_object(self, name, timestamp, size, content_type, etag, deleted=0): - """ - Creates an object in the DB with its metadata. - - :param name: object name to be created - :param timestamp: timestamp of when the object was created - :param size: object size - :param content_type: object content-type - :param etag: object etag - :param deleted: if True, marks the object as deleted and sets the - deteleted_at timestamp to timestamp - """ - record = {'name': name, 'created_at': timestamp, 'size': size, - 'content_type': content_type, 'etag': etag, - 'deleted': deleted} - if self.db_file == ':memory:': - self.merge_items([record]) - return - if not os.path.exists(self.db_file): - raise DatabaseConnectionError(self.db_file, "DB doesn't exist") - pending_size = 0 - try: - pending_size = os.path.getsize(self.pending_file) - except OSError as err: - if err.errno != errno.ENOENT: - raise - if pending_size > PENDING_CAP: - self._commit_puts([record]) - else: - with lock_parent_directory(self.pending_file, - self.pending_timeout): - with open(self.pending_file, 'a+b') as fp: - # Colons aren't used in base64 encoding; so they are our - # delimiter - fp.write(':') - fp.write(pickle.dumps( - (name, timestamp, size, content_type, etag, deleted), - protocol=PICKLE_PROTOCOL).encode('base64')) - fp.flush() - - def is_deleted(self, timestamp=None): - """ - Check if the DB is considered to be deleted. - - :returns: True if the DB is considered to be deleted, False otherwise - """ - if self.db_file != ':memory:' and not os.path.exists(self.db_file): - return True - self._commit_puts_stale_ok() - with self.get() as conn: - row = conn.execute(''' - SELECT put_timestamp, delete_timestamp, object_count - FROM container_stat''').fetchone() - # leave this db as a tombstone for a consistency window - if timestamp and row['delete_timestamp'] > timestamp: - return False - # The container is considered deleted if the delete_timestamp - # value is greater than the put_timestamp, and there are no - # objects in the container. - return (row['object_count'] in (None, '', 0, '0')) and \ - (float(row['delete_timestamp']) > float(row['put_timestamp'])) - - def get_info(self): - """ - Get global data for the container. - - :returns: dict with keys: account, container, created_at, - put_timestamp, delete_timestamp, object_count, bytes_used, - reported_put_timestamp, reported_delete_timestamp, - reported_object_count, reported_bytes_used, hash, id, - x_container_sync_point1, and x_container_sync_point2. - """ - self._commit_puts_stale_ok() - with self.get() as conn: - data = None - trailing = 'x_container_sync_point1, x_container_sync_point2' - while not data: - try: - data = conn.execute(''' - SELECT account, container, created_at, put_timestamp, - delete_timestamp, object_count, bytes_used, - reported_put_timestamp, reported_delete_timestamp, - reported_object_count, reported_bytes_used, hash, - id, %s - FROM container_stat - ''' % (trailing,)).fetchone() - except sqlite3.OperationalError as err: - if 'no such column: x_container_sync_point' in str(err): - trailing = '-1 AS x_container_sync_point1, ' \ - '-1 AS x_container_sync_point2' - else: - raise - data = dict(data) - return data - - def set_x_container_sync_points(self, sync_point1, sync_point2): - with self.get() as conn: - orig_isolation_level = conn.isolation_level - try: - # We turn off auto-transactions to ensure the alter table - # commands are part of the transaction. - conn.isolation_level = None - conn.execute('BEGIN') - try: - self._set_x_container_sync_points(conn, sync_point1, - sync_point2) - except sqlite3.OperationalError as err: - if 'no such column: x_container_sync_point' not in \ - str(err): - raise - conn.execute(''' - ALTER TABLE container_stat - ADD COLUMN x_container_sync_point1 INTEGER DEFAULT -1 - ''') - conn.execute(''' - ALTER TABLE container_stat - ADD COLUMN x_container_sync_point2 INTEGER DEFAULT -1 - ''') - self._set_x_container_sync_points(conn, sync_point1, - sync_point2) - conn.execute('COMMIT') - finally: - conn.isolation_level = orig_isolation_level - - def _set_x_container_sync_points(self, conn, sync_point1, sync_point2): - if sync_point1 is not None and sync_point2 is not None: - conn.execute(''' - UPDATE container_stat - SET x_container_sync_point1 = ?, - x_container_sync_point2 = ? - ''', (sync_point1, sync_point2)) - elif sync_point1 is not None: - conn.execute(''' - UPDATE container_stat - SET x_container_sync_point1 = ? - ''', (sync_point1,)) - elif sync_point2 is not None: - conn.execute(''' - UPDATE container_stat - SET x_container_sync_point2 = ? - ''', (sync_point2,)) - - def reported(self, put_timestamp, delete_timestamp, object_count, - bytes_used): - """ - Update reported stats. - - :param put_timestamp: put_timestamp to update - :param delete_timestamp: delete_timestamp to update - :param object_count: object_count to update - :param bytes_used: bytes_used to update - """ - with self.get() as conn: - conn.execute(''' - UPDATE container_stat - SET reported_put_timestamp = ?, reported_delete_timestamp = ?, - reported_object_count = ?, reported_bytes_used = ? - ''', (put_timestamp, delete_timestamp, object_count, bytes_used)) - conn.commit() - - def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, - path=None): - """ - Get a list of objects sorted by name starting at marker onward, up - to limit entries. Entries will begin with the prefix and will not - have the delimiter after the prefix. - - :param limit: maximum number of entries to get - :param marker: marker query - :param end_marker: end marker query - :param prefix: prefix query - :param delimiter: delimiter for query - :param path: if defined, will set the prefix and delimter based on - the path - - :returns: list of tuples of (name, created_at, size, content_type, - etag) - """ - delim_force_gte = False - (marker, end_marker, prefix, delimiter, path) = utf8encode( - marker, end_marker, prefix, delimiter, path) - self._commit_puts_stale_ok() - if path is not None: - prefix = path - if path: - prefix = path = path.rstrip('/') + '/' - delimiter = '/' - elif delimiter and not prefix: - prefix = '' - orig_marker = marker - with self.get() as conn: - results = [] - while len(results) < limit: - query = '''SELECT name, created_at, size, content_type, etag - FROM object WHERE''' - query_args = [] - if end_marker: - query += ' name < ? AND' - query_args.append(end_marker) - if delim_force_gte: - query += ' name >= ? AND' - query_args.append(marker) - # Always set back to False - delim_force_gte = False - elif marker and marker >= prefix: - query += ' name > ? AND' - query_args.append(marker) - elif prefix: - query += ' name >= ? AND' - query_args.append(prefix) - if self.get_db_version(conn) < 1: - query += ' +deleted = 0' - else: - query += ' deleted = 0' - query += ' ORDER BY name LIMIT ?' - query_args.append(limit - len(results)) - curs = conn.execute(query, query_args) - curs.row_factory = None - - if prefix is None: - # A delimiter without a specified prefix is ignored - return [r for r in curs] - if not delimiter: - if not prefix: - # It is possible to have a delimiter but no prefix - # specified. As above, the prefix will be set to the - # empty string, so avoid performing the extra work to - # check against an empty prefix. - return [r for r in curs] - else: - return [r for r in curs if r[0].startswith(prefix)] - - # We have a delimiter and a prefix (possibly empty string) to - # handle - rowcount = 0 - for row in curs: - rowcount += 1 - marker = name = row[0] - if len(results) >= limit or not name.startswith(prefix): - curs.close() - return results - end = name.find(delimiter, len(prefix)) - if path is not None: - if name == path: - continue - if end >= 0 and len(name) > end + len(delimiter): - marker = name[:end] + chr(ord(delimiter) + 1) - curs.close() - break - elif end > 0: - marker = name[:end] + chr(ord(delimiter) + 1) - # we want result to be inclusinve of delim+1 - delim_force_gte = True - dir_name = name[:end + 1] - if dir_name != orig_marker: - results.append([dir_name, '0', 0, None, '']) - curs.close() - break - results.append(row) - if not rowcount: - break - return results - - def merge_items(self, item_list, source=None): - """ - Merge items into the object table. - - :param item_list: list of dictionaries of {'name', 'created_at', - 'size', 'content_type', 'etag', 'deleted'} - :param source: if defined, update incoming_sync with the source - """ - with self.get() as conn: - max_rowid = -1 - for rec in item_list: - query = ''' - DELETE FROM object - WHERE name = ? AND (created_at < ?) - ''' - if self.get_db_version(conn) >= 1: - query += ' AND deleted IN (0, 1)' - conn.execute(query, (rec['name'], rec['created_at'])) - query = 'SELECT 1 FROM object WHERE name = ?' - if self.get_db_version(conn) >= 1: - query += ' AND deleted IN (0, 1)' - if not conn.execute(query, (rec['name'],)).fetchall(): - conn.execute(''' - INSERT INTO object (name, created_at, size, - content_type, etag, deleted) - VALUES (?, ?, ?, ?, ?, ?) - ''', ([rec['name'], rec['created_at'], rec['size'], - rec['content_type'], rec['etag'], rec['deleted']])) - if source: - max_rowid = max(max_rowid, rec['ROWID']) - if source: - try: - conn.execute(''' - INSERT INTO incoming_sync (sync_point, remote_id) - VALUES (?, ?) - ''', (max_rowid, source)) - except sqlite3.IntegrityError: - conn.execute(''' - UPDATE incoming_sync SET sync_point=max(?, sync_point) - WHERE remote_id=? - ''', (max_rowid, source)) - conn.commit() - - -class AccountBroker(DatabaseBroker): - """Encapsulates working with a account database.""" - db_type = 'account' - db_contains_type = 'container' - db_reclaim_timestamp = 'delete_timestamp' - - def _initialize(self, conn, put_timestamp): - """ - Create a brand new database (tables, indices, triggers, etc.) - - :param conn: DB connection object - :param put_timestamp: put timestamp - """ - if not self.account: - raise ValueError( - 'Attempting to create a new database with no account set') - self.create_container_table(conn) - self.create_account_stat_table(conn, put_timestamp) - - def create_container_table(self, conn): - """ - Create container table which is specific to the account DB. - - :param conn: DB connection object - """ - conn.executescript(""" - CREATE TABLE container ( - ROWID INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT, - put_timestamp TEXT, - delete_timestamp TEXT, - object_count INTEGER, - bytes_used INTEGER, - deleted INTEGER DEFAULT 0 - ); - - CREATE INDEX ix_container_deleted_name ON - container (deleted, name); - - CREATE TRIGGER container_insert AFTER INSERT ON container - BEGIN - UPDATE account_stat - SET container_count = container_count + (1 - new.deleted), - object_count = object_count + new.object_count, - bytes_used = bytes_used + new.bytes_used, - hash = chexor(hash, new.name, - new.put_timestamp || '-' || - new.delete_timestamp || '-' || - new.object_count || '-' || new.bytes_used); - END; - - CREATE TRIGGER container_update BEFORE UPDATE ON container - BEGIN - SELECT RAISE(FAIL, 'UPDATE not allowed; DELETE and INSERT'); - END; - - - CREATE TRIGGER container_delete AFTER DELETE ON container - BEGIN - UPDATE account_stat - SET container_count = container_count - (1 - old.deleted), - object_count = object_count - old.object_count, - bytes_used = bytes_used - old.bytes_used, - hash = chexor(hash, old.name, - old.put_timestamp || '-' || - old.delete_timestamp || '-' || - old.object_count || '-' || old.bytes_used); - END; - """) - - def create_account_stat_table(self, conn, put_timestamp): - """ - Create account_stat table which is specific to the account DB. - - :param conn: DB connection object - :param put_timestamp: put timestamp - """ - conn.executescript(""" - CREATE TABLE account_stat ( - account TEXT, - created_at TEXT, - put_timestamp TEXT DEFAULT '0', - delete_timestamp TEXT DEFAULT '0', - container_count INTEGER, - object_count INTEGER DEFAULT 0, - bytes_used INTEGER DEFAULT 0, - hash TEXT default '00000000000000000000000000000000', - id TEXT, - status TEXT DEFAULT '', - status_changed_at TEXT DEFAULT '0', - metadata TEXT DEFAULT '' - ); - - INSERT INTO account_stat (container_count) VALUES (0); - """) - - conn.execute(''' - UPDATE account_stat SET account = ?, created_at = ?, id = ?, - put_timestamp = ? - ''', (self.account, normalize_timestamp(time.time()), str(uuid4()), - put_timestamp)) - - def get_db_version(self, conn): - if self._db_version == -1: - self._db_version = 0 - for row in conn.execute(''' - SELECT name FROM sqlite_master - WHERE name = 'ix_container_deleted_name' '''): - self._db_version = 1 - return self._db_version - - def _delete_db(self, conn, timestamp, force=False): - """ - Mark the DB as deleted. - - :param conn: DB connection object - :param timestamp: timestamp to mark as deleted - """ - conn.execute(""" - UPDATE account_stat - SET delete_timestamp = ?, - status = 'DELETED', - status_changed_at = ? - WHERE delete_timestamp < ? """, (timestamp, timestamp, timestamp)) - - def _commit_puts_load(self, item_list, entry): - (name, put_timestamp, delete_timestamp, - object_count, bytes_used, deleted) = \ - pickle.loads(entry.decode('base64')) - item_list.append( - {'name': name, - 'put_timestamp': put_timestamp, - 'delete_timestamp': delete_timestamp, - 'object_count': object_count, - 'bytes_used': bytes_used, - 'deleted': deleted}) - - def empty(self): - """ - Check if the account DB is empty. - - :returns: True if the database has no active containers. - """ - self._commit_puts_stale_ok() - with self.get() as conn: - row = conn.execute( - 'SELECT container_count from account_stat').fetchone() - return (row[0] == 0) - - def put_container(self, name, put_timestamp, delete_timestamp, - object_count, bytes_used): - """ - Create a container with the given attributes. - - :param name: name of the container to create - :param put_timestamp: put_timestamp of the container to create - :param delete_timestamp: delete_timestamp of the container to create - :param object_count: number of objects in the container - :param bytes_used: number of bytes used by the container - """ - if delete_timestamp > put_timestamp and \ - object_count in (None, '', 0, '0'): - deleted = 1 - else: - deleted = 0 - record = {'name': name, 'put_timestamp': put_timestamp, - 'delete_timestamp': delete_timestamp, - 'object_count': object_count, - 'bytes_used': bytes_used, - 'deleted': deleted} - if self.db_file == ':memory:': - self.merge_items([record]) - return - if not os.path.exists(self.db_file): - raise DatabaseConnectionError(self.db_file, "DB doesn't exist") - pending_size = 0 - try: - pending_size = os.path.getsize(self.pending_file) - except OSError as err: - if err.errno != errno.ENOENT: - raise - if pending_size > PENDING_CAP: - self._commit_puts([record]) - else: - with lock_parent_directory(self.pending_file, - self.pending_timeout): - with open(self.pending_file, 'a+b') as fp: - # Colons aren't used in base64 encoding; so they are our - # delimiter - fp.write(':') - fp.write(pickle.dumps( - (name, put_timestamp, delete_timestamp, object_count, - bytes_used, deleted), - protocol=PICKLE_PROTOCOL).encode('base64')) - fp.flush() - - def is_deleted(self): - """ - Check if the account DB is considered to be deleted. - - :returns: True if the account DB is considered to be deleted, False - otherwise - """ - if self.db_file != ':memory:' and not os.path.exists(self.db_file): - return True - self._commit_puts_stale_ok() - with self.get() as conn: - row = conn.execute(''' - SELECT put_timestamp, delete_timestamp, container_count, status - FROM account_stat''').fetchone() - return row['status'] == 'DELETED' or ( - row['container_count'] in (None, '', 0, '0') and - row['delete_timestamp'] > row['put_timestamp']) - - def is_status_deleted(self): - """Only returns true if the status field is set to DELETED.""" - with self.get() as conn: - row = conn.execute(''' - SELECT status - FROM account_stat''').fetchone() - return (row['status'] == "DELETED") - - def get_info(self): - """ - Get global data for the account. - - :returns: dict with keys: account, created_at, put_timestamp, - delete_timestamp, container_count, object_count, - bytes_used, hash, id - """ - self._commit_puts_stale_ok() - with self.get() as conn: - return dict(conn.execute(''' - SELECT account, created_at, put_timestamp, delete_timestamp, - container_count, object_count, bytes_used, hash, id - FROM account_stat - ''').fetchone()) - - def list_containers_iter(self, limit, marker, end_marker, prefix, - delimiter): - """ - Get a list of containers sorted by name starting at marker onward, up - to limit entries. Entries will begin with the prefix and will not have - the delimiter after the prefix. - - :param limit: maximum number of entries to get - :param marker: marker query - :param end_marker: end marker query - :param prefix: prefix query - :param delimiter: delimiter for query - - :returns: list of tuples of (name, object_count, bytes_used, 0) - """ - (marker, end_marker, prefix, delimiter) = utf8encode( - marker, end_marker, prefix, delimiter) - self._commit_puts_stale_ok() - if delimiter and not prefix: - prefix = '' - orig_marker = marker - with self.get() as conn: - results = [] - while len(results) < limit: - query = """ - SELECT name, object_count, bytes_used, 0 - FROM container - WHERE deleted = 0 AND """ - query_args = [] - if end_marker: - query += ' name < ? AND' - query_args.append(end_marker) - if marker and marker >= prefix: - query += ' name > ? AND' - query_args.append(marker) - elif prefix: - query += ' name >= ? AND' - query_args.append(prefix) - if self.get_db_version(conn) < 1: - query += ' +deleted = 0' - else: - query += ' deleted = 0' - query += ' ORDER BY name LIMIT ?' - query_args.append(limit - len(results)) - curs = conn.execute(query, query_args) - curs.row_factory = None - - if prefix is None: - # A delimiter without a specified prefix is ignored - return [r for r in curs] - if not delimiter: - if not prefix: - # It is possible to have a delimiter but no prefix - # specified. As above, the prefix will be set to the - # empty string, so avoid performing the extra work to - # check against an empty prefix. - return [r for r in curs] - else: - return [r for r in curs if r[0].startswith(prefix)] - - # We have a delimiter and a prefix (possibly empty string) to - # handle - rowcount = 0 - for row in curs: - rowcount += 1 - marker = name = row[0] - if len(results) >= limit or not name.startswith(prefix): - curs.close() - return results - end = name.find(delimiter, len(prefix)) - if end > 0: - marker = name[:end] + chr(ord(delimiter) + 1) - dir_name = name[:end + 1] - if dir_name != orig_marker: - results.append([dir_name, 0, 0, 1]) - curs.close() - break - results.append(row) - if not rowcount: - break - return results - - def merge_items(self, item_list, source=None): - """ - Merge items into the container table. - - :param item_list: list of dictionaries of {'name', 'put_timestamp', - 'delete_timestamp', 'object_count', 'bytes_used', - 'deleted'} - :param source: if defined, update incoming_sync with the source - """ - with self.get() as conn: - max_rowid = -1 - for rec in item_list: - record = [rec['name'], rec['put_timestamp'], - rec['delete_timestamp'], rec['object_count'], - rec['bytes_used'], rec['deleted']] - query = ''' - SELECT name, put_timestamp, delete_timestamp, - object_count, bytes_used, deleted - FROM container WHERE name = ? - ''' - if self.get_db_version(conn) >= 1: - query += ' AND deleted IN (0, 1)' - curs = conn.execute(query, (rec['name'],)) - curs.row_factory = None - row = curs.fetchone() - if row: - row = list(row) - for i in xrange(5): - if record[i] is None and row[i] is not None: - record[i] = row[i] - if row[1] > record[1]: # Keep newest put_timestamp - record[1] = row[1] - if row[2] > record[2]: # Keep newest delete_timestamp - record[2] = row[2] - # If deleted, mark as such - if record[2] > record[1] and \ - record[3] in (None, '', 0, '0'): - record[5] = 1 - else: - record[5] = 0 - conn.execute(''' - DELETE FROM container WHERE name = ? AND - deleted IN (0, 1) - ''', (record[0],)) - conn.execute(''' - INSERT INTO container (name, put_timestamp, - delete_timestamp, object_count, bytes_used, - deleted) - VALUES (?, ?, ?, ?, ?, ?) - ''', record) - if source: - max_rowid = max(max_rowid, rec['ROWID']) - if source: - try: - conn.execute(''' - INSERT INTO incoming_sync (sync_point, remote_id) - VALUES (?, ?) - ''', (max_rowid, source)) - except sqlite3.IntegrityError: - conn.execute(''' - UPDATE incoming_sync SET sync_point=max(?, sync_point) - WHERE remote_id=? - ''', (max_rowid, source)) - conn.commit() diff --git a/swift/container/auditor.py b/swift/container/auditor.py index 6da9f602b6..df2266c076 100644 --- a/swift/container/auditor.py +++ b/swift/container/auditor.py @@ -22,7 +22,7 @@ import swift.common.db from swift.container import server as container_server -from swift.common.db import ContainerBroker +from swift.container.backend import ContainerBroker from swift.common.utils import get_logger, audit_location_generator, \ config_true_value, dump_recon_cache, ratelimit_sleep from swift.common.daemon import Daemon diff --git a/swift/container/backend.py b/swift/container/backend.py new file mode 100644 index 0000000000..f16b3acc8e --- /dev/null +++ b/swift/container/backend.py @@ -0,0 +1,496 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Pluggable Back-ends for Container Server +""" + +from __future__ import with_statement +import os +from uuid import uuid4 +import time +import cPickle as pickle +import errno + +import sqlite3 + +from swift.common.utils import normalize_timestamp, lock_parent_directory +from swift.common.db import DatabaseBroker, DatabaseConnectionError, \ + PENDING_CAP, PICKLE_PROTOCOL, utf8encode + + +class ContainerBroker(DatabaseBroker): + """Encapsulates working with a container database.""" + db_type = 'container' + db_contains_type = 'object' + db_reclaim_timestamp = 'created_at' + + def _initialize(self, conn, put_timestamp): + """Creates a brand new database (tables, indices, triggers, etc.)""" + if not self.account: + raise ValueError( + 'Attempting to create a new database with no account set') + if not self.container: + raise ValueError( + 'Attempting to create a new database with no container set') + self.create_object_table(conn) + self.create_container_stat_table(conn, put_timestamp) + + def create_object_table(self, conn): + """ + Create the object table which is specifc to the container DB. + + :param conn: DB connection object + """ + conn.executescript(""" + CREATE TABLE object ( + ROWID INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT, + created_at TEXT, + size INTEGER, + content_type TEXT, + etag TEXT, + deleted INTEGER DEFAULT 0 + ); + + CREATE INDEX ix_object_deleted_name ON object (deleted, name); + + CREATE TRIGGER object_insert AFTER INSERT ON object + BEGIN + UPDATE container_stat + SET object_count = object_count + (1 - new.deleted), + bytes_used = bytes_used + new.size, + hash = chexor(hash, new.name, new.created_at); + END; + + CREATE TRIGGER object_update BEFORE UPDATE ON object + BEGIN + SELECT RAISE(FAIL, 'UPDATE not allowed; DELETE and INSERT'); + END; + + CREATE TRIGGER object_delete AFTER DELETE ON object + BEGIN + UPDATE container_stat + SET object_count = object_count - (1 - old.deleted), + bytes_used = bytes_used - old.size, + hash = chexor(hash, old.name, old.created_at); + END; + """) + + def create_container_stat_table(self, conn, put_timestamp=None): + """ + Create the container_stat table which is specific to the container DB. + + :param conn: DB connection object + :param put_timestamp: put timestamp + """ + if put_timestamp is None: + put_timestamp = normalize_timestamp(0) + conn.executescript(""" + CREATE TABLE container_stat ( + account TEXT, + container TEXT, + created_at TEXT, + put_timestamp TEXT DEFAULT '0', + delete_timestamp TEXT DEFAULT '0', + object_count INTEGER, + bytes_used INTEGER, + reported_put_timestamp TEXT DEFAULT '0', + reported_delete_timestamp TEXT DEFAULT '0', + reported_object_count INTEGER DEFAULT 0, + reported_bytes_used INTEGER DEFAULT 0, + hash TEXT default '00000000000000000000000000000000', + id TEXT, + status TEXT DEFAULT '', + status_changed_at TEXT DEFAULT '0', + metadata TEXT DEFAULT '', + x_container_sync_point1 INTEGER DEFAULT -1, + x_container_sync_point2 INTEGER DEFAULT -1 + ); + + INSERT INTO container_stat (object_count, bytes_used) + VALUES (0, 0); + """) + conn.execute(''' + UPDATE container_stat + SET account = ?, container = ?, created_at = ?, id = ?, + put_timestamp = ? + ''', (self.account, self.container, normalize_timestamp(time.time()), + str(uuid4()), put_timestamp)) + + def get_db_version(self, conn): + if self._db_version == -1: + self._db_version = 0 + for row in conn.execute(''' + SELECT name FROM sqlite_master + WHERE name = 'ix_object_deleted_name' '''): + self._db_version = 1 + return self._db_version + + def _newid(self, conn): + conn.execute(''' + UPDATE container_stat + SET reported_put_timestamp = 0, reported_delete_timestamp = 0, + reported_object_count = 0, reported_bytes_used = 0''') + + def _delete_db(self, conn, timestamp): + """ + Mark the DB as deleted + + :param conn: DB connection object + :param timestamp: timestamp to mark as deleted + """ + conn.execute(""" + UPDATE container_stat + SET delete_timestamp = ?, + status = 'DELETED', + status_changed_at = ? + WHERE delete_timestamp < ? """, (timestamp, timestamp, timestamp)) + + def _commit_puts_load(self, item_list, entry): + (name, timestamp, size, content_type, etag, deleted) = \ + pickle.loads(entry.decode('base64')) + item_list.append({'name': name, + 'created_at': timestamp, + 'size': size, + 'content_type': content_type, + 'etag': etag, + 'deleted': deleted}) + + def empty(self): + """ + Check if the DB is empty. + + :returns: True if the database has no active objects, False otherwise + """ + self._commit_puts_stale_ok() + with self.get() as conn: + row = conn.execute( + 'SELECT object_count from container_stat').fetchone() + return (row[0] == 0) + + def delete_object(self, name, timestamp): + """ + Mark an object deleted. + + :param name: object name to be deleted + :param timestamp: timestamp when the object was marked as deleted + """ + self.put_object(name, timestamp, 0, 'application/deleted', 'noetag', 1) + + def put_object(self, name, timestamp, size, content_type, etag, deleted=0): + """ + Creates an object in the DB with its metadata. + + :param name: object name to be created + :param timestamp: timestamp of when the object was created + :param size: object size + :param content_type: object content-type + :param etag: object etag + :param deleted: if True, marks the object as deleted and sets the + deteleted_at timestamp to timestamp + """ + record = {'name': name, 'created_at': timestamp, 'size': size, + 'content_type': content_type, 'etag': etag, + 'deleted': deleted} + if self.db_file == ':memory:': + self.merge_items([record]) + return + if not os.path.exists(self.db_file): + raise DatabaseConnectionError(self.db_file, "DB doesn't exist") + pending_size = 0 + try: + pending_size = os.path.getsize(self.pending_file) + except OSError as err: + if err.errno != errno.ENOENT: + raise + if pending_size > PENDING_CAP: + self._commit_puts([record]) + else: + with lock_parent_directory(self.pending_file, + self.pending_timeout): + with open(self.pending_file, 'a+b') as fp: + # Colons aren't used in base64 encoding; so they are our + # delimiter + fp.write(':') + fp.write(pickle.dumps( + (name, timestamp, size, content_type, etag, deleted), + protocol=PICKLE_PROTOCOL).encode('base64')) + fp.flush() + + def is_deleted(self, timestamp=None): + """ + Check if the DB is considered to be deleted. + + :returns: True if the DB is considered to be deleted, False otherwise + """ + if self.db_file != ':memory:' and not os.path.exists(self.db_file): + return True + self._commit_puts_stale_ok() + with self.get() as conn: + row = conn.execute(''' + SELECT put_timestamp, delete_timestamp, object_count + FROM container_stat''').fetchone() + # leave this db as a tombstone for a consistency window + if timestamp and row['delete_timestamp'] > timestamp: + return False + # The container is considered deleted if the delete_timestamp + # value is greater than the put_timestamp, and there are no + # objects in the container. + return (row['object_count'] in (None, '', 0, '0')) and \ + (float(row['delete_timestamp']) > float(row['put_timestamp'])) + + def get_info(self): + """ + Get global data for the container. + + :returns: dict with keys: account, container, created_at, + put_timestamp, delete_timestamp, object_count, bytes_used, + reported_put_timestamp, reported_delete_timestamp, + reported_object_count, reported_bytes_used, hash, id, + x_container_sync_point1, and x_container_sync_point2. + """ + self._commit_puts_stale_ok() + with self.get() as conn: + data = None + trailing = 'x_container_sync_point1, x_container_sync_point2' + while not data: + try: + data = conn.execute(''' + SELECT account, container, created_at, put_timestamp, + delete_timestamp, object_count, bytes_used, + reported_put_timestamp, reported_delete_timestamp, + reported_object_count, reported_bytes_used, hash, + id, %s + FROM container_stat + ''' % (trailing,)).fetchone() + except sqlite3.OperationalError as err: + if 'no such column: x_container_sync_point' in str(err): + trailing = '-1 AS x_container_sync_point1, ' \ + '-1 AS x_container_sync_point2' + else: + raise + data = dict(data) + return data + + def set_x_container_sync_points(self, sync_point1, sync_point2): + with self.get() as conn: + orig_isolation_level = conn.isolation_level + try: + # We turn off auto-transactions to ensure the alter table + # commands are part of the transaction. + conn.isolation_level = None + conn.execute('BEGIN') + try: + self._set_x_container_sync_points(conn, sync_point1, + sync_point2) + except sqlite3.OperationalError as err: + if 'no such column: x_container_sync_point' not in \ + str(err): + raise + conn.execute(''' + ALTER TABLE container_stat + ADD COLUMN x_container_sync_point1 INTEGER DEFAULT -1 + ''') + conn.execute(''' + ALTER TABLE container_stat + ADD COLUMN x_container_sync_point2 INTEGER DEFAULT -1 + ''') + self._set_x_container_sync_points(conn, sync_point1, + sync_point2) + conn.execute('COMMIT') + finally: + conn.isolation_level = orig_isolation_level + + def _set_x_container_sync_points(self, conn, sync_point1, sync_point2): + if sync_point1 is not None and sync_point2 is not None: + conn.execute(''' + UPDATE container_stat + SET x_container_sync_point1 = ?, + x_container_sync_point2 = ? + ''', (sync_point1, sync_point2)) + elif sync_point1 is not None: + conn.execute(''' + UPDATE container_stat + SET x_container_sync_point1 = ? + ''', (sync_point1,)) + elif sync_point2 is not None: + conn.execute(''' + UPDATE container_stat + SET x_container_sync_point2 = ? + ''', (sync_point2,)) + + def reported(self, put_timestamp, delete_timestamp, object_count, + bytes_used): + """ + Update reported stats. + + :param put_timestamp: put_timestamp to update + :param delete_timestamp: delete_timestamp to update + :param object_count: object_count to update + :param bytes_used: bytes_used to update + """ + with self.get() as conn: + conn.execute(''' + UPDATE container_stat + SET reported_put_timestamp = ?, reported_delete_timestamp = ?, + reported_object_count = ?, reported_bytes_used = ? + ''', (put_timestamp, delete_timestamp, object_count, bytes_used)) + conn.commit() + + def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, + path=None): + """ + Get a list of objects sorted by name starting at marker onward, up + to limit entries. Entries will begin with the prefix and will not + have the delimiter after the prefix. + + :param limit: maximum number of entries to get + :param marker: marker query + :param end_marker: end marker query + :param prefix: prefix query + :param delimiter: delimiter for query + :param path: if defined, will set the prefix and delimter based on + the path + + :returns: list of tuples of (name, created_at, size, content_type, + etag) + """ + delim_force_gte = False + (marker, end_marker, prefix, delimiter, path) = utf8encode( + marker, end_marker, prefix, delimiter, path) + self._commit_puts_stale_ok() + if path is not None: + prefix = path + if path: + prefix = path = path.rstrip('/') + '/' + delimiter = '/' + elif delimiter and not prefix: + prefix = '' + orig_marker = marker + with self.get() as conn: + results = [] + while len(results) < limit: + query = '''SELECT name, created_at, size, content_type, etag + FROM object WHERE''' + query_args = [] + if end_marker: + query += ' name < ? AND' + query_args.append(end_marker) + if delim_force_gte: + query += ' name >= ? AND' + query_args.append(marker) + # Always set back to False + delim_force_gte = False + elif marker and marker >= prefix: + query += ' name > ? AND' + query_args.append(marker) + elif prefix: + query += ' name >= ? AND' + query_args.append(prefix) + if self.get_db_version(conn) < 1: + query += ' +deleted = 0' + else: + query += ' deleted = 0' + query += ' ORDER BY name LIMIT ?' + query_args.append(limit - len(results)) + curs = conn.execute(query, query_args) + curs.row_factory = None + + if prefix is None: + # A delimiter without a specified prefix is ignored + return [r for r in curs] + if not delimiter: + if not prefix: + # It is possible to have a delimiter but no prefix + # specified. As above, the prefix will be set to the + # empty string, so avoid performing the extra work to + # check against an empty prefix. + return [r for r in curs] + else: + return [r for r in curs if r[0].startswith(prefix)] + + # We have a delimiter and a prefix (possibly empty string) to + # handle + rowcount = 0 + for row in curs: + rowcount += 1 + marker = name = row[0] + if len(results) >= limit or not name.startswith(prefix): + curs.close() + return results + end = name.find(delimiter, len(prefix)) + if path is not None: + if name == path: + continue + if end >= 0 and len(name) > end + len(delimiter): + marker = name[:end] + chr(ord(delimiter) + 1) + curs.close() + break + elif end > 0: + marker = name[:end] + chr(ord(delimiter) + 1) + # we want result to be inclusinve of delim+1 + delim_force_gte = True + dir_name = name[:end + 1] + if dir_name != orig_marker: + results.append([dir_name, '0', 0, None, '']) + curs.close() + break + results.append(row) + if not rowcount: + break + return results + + def merge_items(self, item_list, source=None): + """ + Merge items into the object table. + + :param item_list: list of dictionaries of {'name', 'created_at', + 'size', 'content_type', 'etag', 'deleted'} + :param source: if defined, update incoming_sync with the source + """ + with self.get() as conn: + max_rowid = -1 + for rec in item_list: + query = ''' + DELETE FROM object + WHERE name = ? AND (created_at < ?) + ''' + if self.get_db_version(conn) >= 1: + query += ' AND deleted IN (0, 1)' + conn.execute(query, (rec['name'], rec['created_at'])) + query = 'SELECT 1 FROM object WHERE name = ?' + if self.get_db_version(conn) >= 1: + query += ' AND deleted IN (0, 1)' + if not conn.execute(query, (rec['name'],)).fetchall(): + conn.execute(''' + INSERT INTO object (name, created_at, size, + content_type, etag, deleted) + VALUES (?, ?, ?, ?, ?, ?) + ''', ([rec['name'], rec['created_at'], rec['size'], + rec['content_type'], rec['etag'], rec['deleted']])) + if source: + max_rowid = max(max_rowid, rec['ROWID']) + if source: + try: + conn.execute(''' + INSERT INTO incoming_sync (sync_point, remote_id) + VALUES (?, ?) + ''', (max_rowid, source)) + except sqlite3.IntegrityError: + conn.execute(''' + UPDATE incoming_sync SET sync_point=max(?, sync_point) + WHERE remote_id=? + ''', (max_rowid, source)) + conn.commit() diff --git a/swift/container/replicator.py b/swift/container/replicator.py index 3d5aee9b73..77d0d77f7b 100644 --- a/swift/container/replicator.py +++ b/swift/container/replicator.py @@ -14,12 +14,13 @@ # limitations under the License. from swift.container import server as container_server -from swift.common import db, db_replicator +from swift.container.backend import ContainerBroker +from swift.common import db_replicator class ContainerReplicator(db_replicator.Replicator): server_type = 'container' - brokerclass = db.ContainerBroker + brokerclass = ContainerBroker datadir = container_server.DATADIR default_port = 6001 diff --git a/swift/container/server.py b/swift/container/server.py index 8c089fdf95..42aed48af1 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -25,7 +25,8 @@ from eventlet import Timeout import swift.common.db -from swift.common.db import ContainerBroker +from swift.container.backend import ContainerBroker +from swift.common.db import DatabaseAlreadyExists from swift.common.request_helpers import get_param, get_listing_content_type, \ split_and_validate_path from swift.common.utils import get_logger, hash_path, public, \ @@ -194,7 +195,7 @@ def DELETE(self, req): try: broker.initialize(normalize_timestamp( req.headers.get('x-timestamp') or time.time())) - except swift.common.db.DatabaseAlreadyExists: + except DatabaseAlreadyExists: pass if not os.path.exists(broker.db_file): return HTTPNotFound() @@ -241,7 +242,7 @@ def PUT(self, req): not os.path.exists(broker.db_file): try: broker.initialize(timestamp) - except swift.common.db.DatabaseAlreadyExists: + except DatabaseAlreadyExists: pass if not os.path.exists(broker.db_file): return HTTPNotFound() @@ -254,7 +255,7 @@ def PUT(self, req): try: broker.initialize(timestamp) created = True - except swift.common.db.DatabaseAlreadyExists: + except DatabaseAlreadyExists: pass else: created = broker.is_deleted() diff --git a/swift/container/sync.py b/swift/container/sync.py index 7125db3a3f..759248417b 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -24,9 +24,9 @@ from swift.container import server as container_server from swiftclient import ClientException, delete_object, put_object, \ quote +from swift.container.backend import ContainerBroker from swift.common.direct_client import direct_get_object from swift.common.ring import Ring -from swift.common.db import ContainerBroker from swift.common.utils import audit_location_generator, get_logger, \ hash_path, config_true_value, validate_sync_to, whataremyips, FileLikeIter from swift.common.daemon import Daemon diff --git a/swift/container/updater.py b/swift/container/updater.py index 552f5145d6..d6f0edbd6c 100644 --- a/swift/container/updater.py +++ b/swift/container/updater.py @@ -25,9 +25,9 @@ from eventlet import spawn, patcher, Timeout import swift.common.db +from swift.container.backend import ContainerBroker from swift.container.server import DATADIR from swift.common.bufferedhttp import http_connect -from swift.common.db import ContainerBroker from swift.common.exceptions import ConnectionTimeout from swift.common.ring import Ring from swift.common.utils import get_logger, config_true_value, dump_recon_cache diff --git a/test/unit/account/test_backend.py b/test/unit/account/test_backend.py new file mode 100644 index 0000000000..379598ba5f --- /dev/null +++ b/test/unit/account/test_backend.py @@ -0,0 +1,540 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Tests for swift.account.backend """ + +from __future__ import with_statement +import hashlib +import unittest +from time import sleep, time +from uuid import uuid4 + +from swift.account.backend import AccountBroker +from swift.common.utils import normalize_timestamp + + +class TestAccountBroker(unittest.TestCase): + """Tests for AccountBroker""" + + def test_creation(self): + # Test AccountBroker.__init__ + broker = AccountBroker(':memory:', account='a') + self.assertEqual(broker.db_file, ':memory:') + got_exc = False + try: + with broker.get() as conn: + pass + except Exception: + got_exc = True + self.assert_(got_exc) + broker.initialize(normalize_timestamp('1')) + with broker.get() as conn: + curs = conn.cursor() + curs.execute('SELECT 1') + self.assertEqual(curs.fetchall()[0][0], 1) + + def test_exception(self): + # Test AccountBroker throwing a conn away after exception + first_conn = None + broker = AccountBroker(':memory:', account='a') + broker.initialize(normalize_timestamp('1')) + with broker.get() as conn: + first_conn = conn + try: + with broker.get() as conn: + self.assertEquals(first_conn, conn) + raise Exception('OMG') + except Exception: + pass + self.assert_(broker.conn is None) + + def test_empty(self): + # Test AccountBroker.empty + broker = AccountBroker(':memory:', account='a') + broker.initialize(normalize_timestamp('1')) + self.assert_(broker.empty()) + broker.put_container('o', normalize_timestamp(time()), 0, 0, 0) + self.assert_(not broker.empty()) + sleep(.00001) + broker.put_container('o', 0, normalize_timestamp(time()), 0, 0) + self.assert_(broker.empty()) + + def test_reclaim(self): + broker = AccountBroker(':memory:', account='test_account') + broker.initialize(normalize_timestamp('1')) + broker.put_container('c', normalize_timestamp(time()), 0, 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 0").fetchone()[0], 1) + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 1").fetchone()[0], 0) + broker.reclaim(normalize_timestamp(time() - 999), time()) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 0").fetchone()[0], 1) + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 1").fetchone()[0], 0) + sleep(.00001) + broker.put_container('c', 0, normalize_timestamp(time()), 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 0").fetchone()[0], 0) + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 1").fetchone()[0], 1) + broker.reclaim(normalize_timestamp(time() - 999), time()) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 0").fetchone()[0], 0) + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 1").fetchone()[0], 1) + sleep(.00001) + broker.reclaim(normalize_timestamp(time()), time()) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 0").fetchone()[0], 0) + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 1").fetchone()[0], 0) + # Test reclaim after deletion. Create 3 test containers + broker.put_container('x', 0, 0, 0, 0) + broker.put_container('y', 0, 0, 0, 0) + broker.put_container('z', 0, 0, 0, 0) + broker.reclaim(normalize_timestamp(time()), time()) + # self.assertEquals(len(res), 2) + # self.assert_(isinstance(res, tuple)) + # containers, account_name = res + # self.assert_(containers is None) + # self.assert_(account_name is None) + # Now delete the account + broker.delete_db(normalize_timestamp(time())) + broker.reclaim(normalize_timestamp(time()), time()) + # self.assertEquals(len(res), 2) + # self.assert_(isinstance(res, tuple)) + # containers, account_name = res + # self.assertEquals(account_name, 'test_account') + # self.assertEquals(len(containers), 3) + # self.assert_('x' in containers) + # self.assert_('y' in containers) + # self.assert_('z' in containers) + # self.assert_('a' not in containers) + + def test_delete_container(self): + # Test AccountBroker.delete_container + broker = AccountBroker(':memory:', account='a') + broker.initialize(normalize_timestamp('1')) + broker.put_container('o', normalize_timestamp(time()), 0, 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 0").fetchone()[0], 1) + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 1").fetchone()[0], 0) + sleep(.00001) + broker.put_container('o', 0, normalize_timestamp(time()), 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 0").fetchone()[0], 0) + self.assertEquals(conn.execute( + "SELECT count(*) FROM container " + "WHERE deleted = 1").fetchone()[0], 1) + + def test_put_container(self): + # Test AccountBroker.put_container + broker = AccountBroker(':memory:', account='a') + broker.initialize(normalize_timestamp('1')) + + # Create initial container + timestamp = normalize_timestamp(time()) + broker.put_container('"{}"', timestamp, 0, 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM container").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT put_timestamp FROM container").fetchone()[0], + timestamp) + self.assertEquals(conn.execute( + "SELECT deleted FROM container").fetchone()[0], 0) + + # Reput same event + broker.put_container('"{}"', timestamp, 0, 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM container").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT put_timestamp FROM container").fetchone()[0], + timestamp) + self.assertEquals(conn.execute( + "SELECT deleted FROM container").fetchone()[0], 0) + + # Put new event + sleep(.00001) + timestamp = normalize_timestamp(time()) + broker.put_container('"{}"', timestamp, 0, 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM container").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT put_timestamp FROM container").fetchone()[0], + timestamp) + self.assertEquals(conn.execute( + "SELECT deleted FROM container").fetchone()[0], 0) + + # Put old event + otimestamp = normalize_timestamp(float(timestamp) - 1) + broker.put_container('"{}"', otimestamp, 0, 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM container").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT put_timestamp FROM container").fetchone()[0], + timestamp) + self.assertEquals(conn.execute( + "SELECT deleted FROM container").fetchone()[0], 0) + + # Put old delete event + dtimestamp = normalize_timestamp(float(timestamp) - 1) + broker.put_container('"{}"', 0, dtimestamp, 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM container").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT put_timestamp FROM container").fetchone()[0], + timestamp) + self.assertEquals(conn.execute( + "SELECT delete_timestamp FROM container").fetchone()[0], + dtimestamp) + self.assertEquals(conn.execute( + "SELECT deleted FROM container").fetchone()[0], 0) + + # Put new delete event + sleep(.00001) + timestamp = normalize_timestamp(time()) + broker.put_container('"{}"', 0, timestamp, 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM container").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT delete_timestamp FROM container").fetchone()[0], + timestamp) + self.assertEquals(conn.execute( + "SELECT deleted FROM container").fetchone()[0], 1) + + # Put new event + sleep(.00001) + timestamp = normalize_timestamp(time()) + broker.put_container('"{}"', timestamp, 0, 0, 0) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM container").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT put_timestamp FROM container").fetchone()[0], + timestamp) + self.assertEquals(conn.execute( + "SELECT deleted FROM container").fetchone()[0], 0) + + def test_get_info(self): + # Test AccountBroker.get_info + broker = AccountBroker(':memory:', account='test1') + broker.initialize(normalize_timestamp('1')) + + info = broker.get_info() + self.assertEquals(info['account'], 'test1') + self.assertEquals(info['hash'], '00000000000000000000000000000000') + + info = broker.get_info() + self.assertEquals(info['container_count'], 0) + + broker.put_container('c1', normalize_timestamp(time()), 0, 0, 0) + info = broker.get_info() + self.assertEquals(info['container_count'], 1) + + sleep(.00001) + broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0) + info = broker.get_info() + self.assertEquals(info['container_count'], 2) + + sleep(.00001) + broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0) + info = broker.get_info() + self.assertEquals(info['container_count'], 2) + + sleep(.00001) + broker.put_container('c1', 0, normalize_timestamp(time()), 0, 0) + info = broker.get_info() + self.assertEquals(info['container_count'], 1) + + sleep(.00001) + broker.put_container('c2', 0, normalize_timestamp(time()), 0, 0) + info = broker.get_info() + self.assertEquals(info['container_count'], 0) + + def test_list_containers_iter(self): + # Test AccountBroker.list_containers_iter + broker = AccountBroker(':memory:', account='a') + broker.initialize(normalize_timestamp('1')) + for cont1 in xrange(4): + for cont2 in xrange(125): + broker.put_container('%d-%04d' % (cont1, cont2), + normalize_timestamp(time()), 0, 0, 0) + for cont in xrange(125): + broker.put_container('2-0051-%04d' % cont, + normalize_timestamp(time()), 0, 0, 0) + + for cont in xrange(125): + broker.put_container('3-%04d-0049' % cont, + normalize_timestamp(time()), 0, 0, 0) + + listing = broker.list_containers_iter(100, '', None, None, '') + self.assertEquals(len(listing), 100) + self.assertEquals(listing[0][0], '0-0000') + self.assertEquals(listing[-1][0], '0-0099') + + listing = broker.list_containers_iter(100, '', '0-0050', None, '') + self.assertEquals(len(listing), 50) + self.assertEquals(listing[0][0], '0-0000') + self.assertEquals(listing[-1][0], '0-0049') + + listing = broker.list_containers_iter(100, '0-0099', None, None, '') + self.assertEquals(len(listing), 100) + self.assertEquals(listing[0][0], '0-0100') + self.assertEquals(listing[-1][0], '1-0074') + + listing = broker.list_containers_iter(55, '1-0074', None, None, '') + self.assertEquals(len(listing), 55) + self.assertEquals(listing[0][0], '1-0075') + self.assertEquals(listing[-1][0], '2-0004') + + listing = broker.list_containers_iter(10, '', None, '0-01', '') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0-0100') + self.assertEquals(listing[-1][0], '0-0109') + + listing = broker.list_containers_iter(10, '', None, '0-01', '-') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0-0100') + self.assertEquals(listing[-1][0], '0-0109') + + listing = broker.list_containers_iter(10, '', None, '0-', '-') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0-0000') + self.assertEquals(listing[-1][0], '0-0009') + + listing = broker.list_containers_iter(10, '', None, '', '-') + self.assertEquals(len(listing), 4) + self.assertEquals([row[0] for row in listing], + ['0-', '1-', '2-', '3-']) + + listing = broker.list_containers_iter(10, '2-', None, None, '-') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], ['3-']) + + listing = broker.list_containers_iter(10, '', None, '2', '-') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], ['2-']) + + listing = broker.list_containers_iter(10, '2-0050', None, '2-', '-') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '2-0051') + self.assertEquals(listing[1][0], '2-0051-') + self.assertEquals(listing[2][0], '2-0052') + self.assertEquals(listing[-1][0], '2-0059') + + listing = broker.list_containers_iter(10, '3-0045', None, '3-', '-') + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['3-0045-', '3-0046', '3-0046-', '3-0047', + '3-0047-', '3-0048', '3-0048-', '3-0049', + '3-0049-', '3-0050']) + + broker.put_container('3-0049-', normalize_timestamp(time()), 0, 0, 0) + listing = broker.list_containers_iter(10, '3-0048', None, None, None) + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', + '3-0050', '3-0050-0049', '3-0051', '3-0051-0049', + '3-0052', '3-0052-0049']) + + listing = broker.list_containers_iter(10, '3-0048', None, '3-', '-') + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['3-0048-', '3-0049', '3-0049-', '3-0050', + '3-0050-', '3-0051', '3-0051-', '3-0052', + '3-0052-', '3-0053']) + + listing = broker.list_containers_iter(10, None, None, '3-0049-', '-') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], + ['3-0049-', '3-0049-0049']) + + def test_double_check_trailing_delimiter(self): + # Test AccountBroker.list_containers_iter for an + # account that has an odd container with a trailing delimiter + broker = AccountBroker(':memory:', account='a') + broker.initialize(normalize_timestamp('1')) + broker.put_container('a', normalize_timestamp(time()), 0, 0, 0) + broker.put_container('a-', normalize_timestamp(time()), 0, 0, 0) + broker.put_container('a-a', normalize_timestamp(time()), 0, 0, 0) + broker.put_container('a-a-a', normalize_timestamp(time()), 0, 0, 0) + broker.put_container('a-a-b', normalize_timestamp(time()), 0, 0, 0) + broker.put_container('a-b', normalize_timestamp(time()), 0, 0, 0) + broker.put_container('b', normalize_timestamp(time()), 0, 0, 0) + broker.put_container('b-a', normalize_timestamp(time()), 0, 0, 0) + broker.put_container('b-b', normalize_timestamp(time()), 0, 0, 0) + broker.put_container('c', normalize_timestamp(time()), 0, 0, 0) + listing = broker.list_containers_iter(15, None, None, None, None) + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['a', 'a-', 'a-a', 'a-a-a', 'a-a-b', 'a-b', 'b', + 'b-a', 'b-b', 'c']) + listing = broker.list_containers_iter(15, None, None, '', '-') + self.assertEquals(len(listing), 5) + self.assertEquals([row[0] for row in listing], + ['a', 'a-', 'b', 'b-', 'c']) + listing = broker.list_containers_iter(15, None, None, 'a-', '-') + self.assertEquals(len(listing), 4) + self.assertEquals([row[0] for row in listing], + ['a-', 'a-a', 'a-a-', 'a-b']) + listing = broker.list_containers_iter(15, None, None, 'b-', '-') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], ['b-a', 'b-b']) + + def test_chexor(self): + broker = AccountBroker(':memory:', account='a') + broker.initialize(normalize_timestamp('1')) + broker.put_container('a', normalize_timestamp(1), + normalize_timestamp(0), 0, 0) + broker.put_container('b', normalize_timestamp(2), + normalize_timestamp(0), 0, 0) + hasha = hashlib.md5( + '%s-%s' % ('a', '0000000001.00000-0000000000.00000-0-0') + ).digest() + hashb = hashlib.md5( + '%s-%s' % ('b', '0000000002.00000-0000000000.00000-0-0') + ).digest() + hashc = \ + ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) + self.assertEquals(broker.get_info()['hash'], hashc) + broker.put_container('b', normalize_timestamp(3), + normalize_timestamp(0), 0, 0) + hashb = hashlib.md5( + '%s-%s' % ('b', '0000000003.00000-0000000000.00000-0-0') + ).digest() + hashc = \ + ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) + self.assertEquals(broker.get_info()['hash'], hashc) + + def test_merge_items(self): + broker1 = AccountBroker(':memory:', account='a') + broker1.initialize(normalize_timestamp('1')) + broker2 = AccountBroker(':memory:', account='a') + broker2.initialize(normalize_timestamp('1')) + broker1.put_container('a', normalize_timestamp(1), 0, 0, 0) + broker1.put_container('b', normalize_timestamp(2), 0, 0, 0) + id = broker1.get_info()['id'] + broker2.merge_items(broker1.get_items_since( + broker2.get_sync(id), 1000), id) + items = broker2.get_items_since(-1, 1000) + self.assertEquals(len(items), 2) + self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) + broker1.put_container('c', normalize_timestamp(3), 0, 0, 0) + broker2.merge_items(broker1.get_items_since( + broker2.get_sync(id), 1000), id) + items = broker2.get_items_since(-1, 1000) + self.assertEquals(len(items), 3) + self.assertEquals(['a', 'b', 'c'], + sorted([rec['name'] for rec in items])) + + +def premetadata_create_account_stat_table(self, conn, put_timestamp): + """ + Copied from AccountBroker before the metadata column was + added; used for testing with TestAccountBrokerBeforeMetadata. + + Create account_stat table which is specific to the account DB. + + :param conn: DB connection object + :param put_timestamp: put timestamp + """ + conn.executescript(''' + CREATE TABLE account_stat ( + account TEXT, + created_at TEXT, + put_timestamp TEXT DEFAULT '0', + delete_timestamp TEXT DEFAULT '0', + container_count INTEGER, + object_count INTEGER DEFAULT 0, + bytes_used INTEGER DEFAULT 0, + hash TEXT default '00000000000000000000000000000000', + id TEXT, + status TEXT DEFAULT '', + status_changed_at TEXT DEFAULT '0' + ); + + INSERT INTO account_stat (container_count) VALUES (0); + ''') + + conn.execute(''' + UPDATE account_stat SET account = ?, created_at = ?, id = ?, + put_timestamp = ? + ''', (self.account, normalize_timestamp(time()), str(uuid4()), + put_timestamp)) + + +class TestAccountBrokerBeforeMetadata(TestAccountBroker): + """ + Tests for AccountBroker against databases created before + the metadata column was added. + """ + + def setUp(self): + self._imported_create_account_stat_table = \ + AccountBroker.create_account_stat_table + AccountBroker.create_account_stat_table = \ + premetadata_create_account_stat_table + broker = AccountBroker(':memory:', account='a') + broker.initialize(normalize_timestamp('1')) + exc = None + with broker.get() as conn: + try: + conn.execute('SELECT metadata FROM account_stat') + except BaseException as err: + exc = err + self.assert_('no such column: metadata' in str(exc)) + + def tearDown(self): + AccountBroker.create_account_stat_table = \ + self._imported_create_account_stat_table + broker = AccountBroker(':memory:', account='a') + broker.initialize(normalize_timestamp('1')) + with broker.get() as conn: + conn.execute('SELECT metadata FROM account_stat') diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py index 58c7a6ecdf..5bd3fb9c1c 100644 --- a/test/unit/common/test_db.py +++ b/test/unit/common/test_db.py @@ -16,11 +16,9 @@ """Tests for swift.common.db""" from __future__ import with_statement -import hashlib import os import unittest from shutil import rmtree, copy -from time import sleep, time from uuid import uuid4 import simplejson @@ -28,8 +26,8 @@ from mock import patch import swift.common.db -from swift.common.db import AccountBroker, chexor, ContainerBroker, \ - DatabaseBroker, DatabaseConnectionError, dict_factory, get_db_connection +from swift.common.db import chexor, dict_factory, get_db_connection, \ + DatabaseBroker, DatabaseConnectionError, DatabaseAlreadyExists from swift.common.utils import normalize_timestamp from swift.common.exceptions import LockTimeout @@ -175,7 +173,7 @@ def my_ismount(*a, **kw): with patch('os.path.ismount', my_ismount): broker = DatabaseBroker(os.path.join(self.testdir, '1.db')) broker._initialize = stub - self.assertRaises(swift.common.db.DatabaseAlreadyExists, + self.assertRaises(DatabaseAlreadyExists, broker.initialize, normalize_timestamp('1')) def test_delete_db(self): @@ -635,1701 +633,5 @@ def reclaim(broker, timestamp): self.assert_('Second' not in broker.metadata) -class TestContainerBroker(unittest.TestCase): - """Tests for swift.common.db.ContainerBroker""" - - def test_creation(self): - # Test swift.common.db.ContainerBroker.__init__ - broker = ContainerBroker(':memory:', account='a', container='c') - self.assertEqual(broker.db_file, ':memory:') - broker.initialize(normalize_timestamp('1')) - with broker.get() as conn: - curs = conn.cursor() - curs.execute('SELECT 1') - self.assertEqual(curs.fetchall()[0][0], 1) - - def test_exception(self): - # Test swift.common.db.ContainerBroker throwing a conn away after - # unhandled exception - first_conn = None - broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1')) - with broker.get() as conn: - first_conn = conn - try: - with broker.get() as conn: - self.assertEquals(first_conn, conn) - raise Exception('OMG') - except Exception: - pass - self.assert_(broker.conn is None) - - def test_empty(self): - # Test swift.common.db.ContainerBroker.empty - broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1')) - self.assert_(broker.empty()) - broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', - 'd41d8cd98f00b204e9800998ecf8427e') - self.assert_(not broker.empty()) - sleep(.00001) - broker.delete_object('o', normalize_timestamp(time())) - self.assert_(broker.empty()) - - def test_reclaim(self): - broker = ContainerBroker(':memory:', account='test_account', - container='test_container') - broker.initialize(normalize_timestamp('1')) - broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', - 'd41d8cd98f00b204e9800998ecf8427e') - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT count(*) FROM object " - "WHERE deleted = 0").fetchone()[0], 1) - self.assertEquals(conn.execute( - "SELECT count(*) FROM object " - "WHERE deleted = 1").fetchone()[0], 0) - broker.reclaim(normalize_timestamp(time() - 999), time()) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT count(*) FROM object " - "WHERE deleted = 0").fetchone()[0], 1) - self.assertEquals(conn.execute( - "SELECT count(*) FROM object " - "WHERE deleted = 1").fetchone()[0], 0) - sleep(.00001) - broker.delete_object('o', normalize_timestamp(time())) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT count(*) FROM object " - "WHERE deleted = 0").fetchone()[0], 0) - self.assertEquals(conn.execute( - "SELECT count(*) FROM object " - "WHERE deleted = 1").fetchone()[0], 1) - broker.reclaim(normalize_timestamp(time() - 999), time()) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT count(*) FROM object " - "WHERE deleted = 0").fetchone()[0], 0) - self.assertEquals(conn.execute( - "SELECT count(*) FROM object " - "WHERE deleted = 1").fetchone()[0], 1) - sleep(.00001) - broker.reclaim(normalize_timestamp(time()), time()) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT count(*) FROM object " - "WHERE deleted = 0").fetchone()[0], 0) - self.assertEquals(conn.execute( - "SELECT count(*) FROM object " - "WHERE deleted = 1").fetchone()[0], 0) - # Test the return values of reclaim() - broker.put_object('w', normalize_timestamp(time()), 0, 'text/plain', - 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('x', normalize_timestamp(time()), 0, 'text/plain', - 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('y', normalize_timestamp(time()), 0, 'text/plain', - 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('z', normalize_timestamp(time()), 0, 'text/plain', - 'd41d8cd98f00b204e9800998ecf8427e') - # Test before deletion - broker.reclaim(normalize_timestamp(time()), time()) - broker.delete_db(normalize_timestamp(time())) - - def test_delete_object(self): - # Test swift.common.db.ContainerBroker.delete_object - broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1')) - broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', - 'd41d8cd98f00b204e9800998ecf8427e') - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT count(*) FROM object " - "WHERE deleted = 0").fetchone()[0], 1) - self.assertEquals(conn.execute( - "SELECT count(*) FROM object " - "WHERE deleted = 1").fetchone()[0], 0) - sleep(.00001) - broker.delete_object('o', normalize_timestamp(time())) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT count(*) FROM object " - "WHERE deleted = 0").fetchone()[0], 0) - self.assertEquals(conn.execute( - "SELECT count(*) FROM object " - "WHERE deleted = 1").fetchone()[0], 1) - - def test_put_object(self): - # Test swift.common.db.ContainerBroker.put_object - broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1')) - - # Create initial object - timestamp = normalize_timestamp(time()) - broker.put_object('"{}"', timestamp, 123, - 'application/x-test', - '5af83e3196bf99f440f31f2e1a6c9afe') - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT name FROM object").fetchone()[0], - '"{}"') - self.assertEquals(conn.execute( - "SELECT created_at FROM object").fetchone()[0], timestamp) - self.assertEquals(conn.execute( - "SELECT size FROM object").fetchone()[0], 123) - self.assertEquals(conn.execute( - "SELECT content_type FROM object").fetchone()[0], - 'application/x-test') - self.assertEquals(conn.execute( - "SELECT etag FROM object").fetchone()[0], - '5af83e3196bf99f440f31f2e1a6c9afe') - self.assertEquals(conn.execute( - "SELECT deleted FROM object").fetchone()[0], 0) - - # Reput same event - broker.put_object('"{}"', timestamp, 123, - 'application/x-test', - '5af83e3196bf99f440f31f2e1a6c9afe') - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT name FROM object").fetchone()[0], - '"{}"') - self.assertEquals(conn.execute( - "SELECT created_at FROM object").fetchone()[0], timestamp) - self.assertEquals(conn.execute( - "SELECT size FROM object").fetchone()[0], 123) - self.assertEquals(conn.execute( - "SELECT content_type FROM object").fetchone()[0], - 'application/x-test') - self.assertEquals(conn.execute( - "SELECT etag FROM object").fetchone()[0], - '5af83e3196bf99f440f31f2e1a6c9afe') - self.assertEquals(conn.execute( - "SELECT deleted FROM object").fetchone()[0], 0) - - # Put new event - sleep(.00001) - timestamp = normalize_timestamp(time()) - broker.put_object('"{}"', timestamp, 124, - 'application/x-test', - 'aa0749bacbc79ec65fe206943d8fe449') - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT name FROM object").fetchone()[0], - '"{}"') - self.assertEquals(conn.execute( - "SELECT created_at FROM object").fetchone()[0], timestamp) - self.assertEquals(conn.execute( - "SELECT size FROM object").fetchone()[0], 124) - self.assertEquals(conn.execute( - "SELECT content_type FROM object").fetchone()[0], - 'application/x-test') - self.assertEquals(conn.execute( - "SELECT etag FROM object").fetchone()[0], - 'aa0749bacbc79ec65fe206943d8fe449') - self.assertEquals(conn.execute( - "SELECT deleted FROM object").fetchone()[0], 0) - - # Put old event - otimestamp = normalize_timestamp(float(timestamp) - 1) - broker.put_object('"{}"', otimestamp, 124, - 'application/x-test', - 'aa0749bacbc79ec65fe206943d8fe449') - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT name FROM object").fetchone()[0], - '"{}"') - self.assertEquals(conn.execute( - "SELECT created_at FROM object").fetchone()[0], timestamp) - self.assertEquals(conn.execute( - "SELECT size FROM object").fetchone()[0], 124) - self.assertEquals(conn.execute( - "SELECT content_type FROM object").fetchone()[0], - 'application/x-test') - self.assertEquals(conn.execute( - "SELECT etag FROM object").fetchone()[0], - 'aa0749bacbc79ec65fe206943d8fe449') - self.assertEquals(conn.execute( - "SELECT deleted FROM object").fetchone()[0], 0) - - # Put old delete event - dtimestamp = normalize_timestamp(float(timestamp) - 1) - broker.put_object('"{}"', dtimestamp, 0, '', '', - deleted=1) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT name FROM object").fetchone()[0], - '"{}"') - self.assertEquals(conn.execute( - "SELECT created_at FROM object").fetchone()[0], timestamp) - self.assertEquals(conn.execute( - "SELECT size FROM object").fetchone()[0], 124) - self.assertEquals(conn.execute( - "SELECT content_type FROM object").fetchone()[0], - 'application/x-test') - self.assertEquals(conn.execute( - "SELECT etag FROM object").fetchone()[0], - 'aa0749bacbc79ec65fe206943d8fe449') - self.assertEquals(conn.execute( - "SELECT deleted FROM object").fetchone()[0], 0) - - # Put new delete event - sleep(.00001) - timestamp = normalize_timestamp(time()) - broker.put_object('"{}"', timestamp, 0, '', '', - deleted=1) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT name FROM object").fetchone()[0], - '"{}"') - self.assertEquals(conn.execute( - "SELECT created_at FROM object").fetchone()[0], timestamp) - self.assertEquals(conn.execute( - "SELECT deleted FROM object").fetchone()[0], 1) - - # Put new event - sleep(.00001) - timestamp = normalize_timestamp(time()) - broker.put_object('"{}"', timestamp, 123, - 'application/x-test', - '5af83e3196bf99f440f31f2e1a6c9afe') - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT name FROM object").fetchone()[0], - '"{}"') - self.assertEquals(conn.execute( - "SELECT created_at FROM object").fetchone()[0], timestamp) - self.assertEquals(conn.execute( - "SELECT size FROM object").fetchone()[0], 123) - self.assertEquals(conn.execute( - "SELECT content_type FROM object").fetchone()[0], - 'application/x-test') - self.assertEquals(conn.execute( - "SELECT etag FROM object").fetchone()[0], - '5af83e3196bf99f440f31f2e1a6c9afe') - self.assertEquals(conn.execute( - "SELECT deleted FROM object").fetchone()[0], 0) - - # We'll use this later - sleep(.0001) - in_between_timestamp = normalize_timestamp(time()) - - # New post event - sleep(.0001) - previous_timestamp = timestamp - timestamp = normalize_timestamp(time()) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT name FROM object").fetchone()[0], - '"{}"') - self.assertEquals(conn.execute( - "SELECT created_at FROM object").fetchone()[0], - previous_timestamp) - self.assertEquals(conn.execute( - "SELECT size FROM object").fetchone()[0], 123) - self.assertEquals(conn.execute( - "SELECT content_type FROM object").fetchone()[0], - 'application/x-test') - self.assertEquals(conn.execute( - "SELECT etag FROM object").fetchone()[0], - '5af83e3196bf99f440f31f2e1a6c9afe') - self.assertEquals(conn.execute( - "SELECT deleted FROM object").fetchone()[0], 0) - - # Put event from after last put but before last post - timestamp = in_between_timestamp - broker.put_object('"{}"', timestamp, 456, - 'application/x-test3', - '6af83e3196bf99f440f31f2e1a6c9afe') - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT name FROM object").fetchone()[0], - '"{}"') - self.assertEquals(conn.execute( - "SELECT created_at FROM object").fetchone()[0], timestamp) - self.assertEquals(conn.execute( - "SELECT size FROM object").fetchone()[0], 456) - self.assertEquals(conn.execute( - "SELECT content_type FROM object").fetchone()[0], - 'application/x-test3') - self.assertEquals(conn.execute( - "SELECT etag FROM object").fetchone()[0], - '6af83e3196bf99f440f31f2e1a6c9afe') - self.assertEquals(conn.execute( - "SELECT deleted FROM object").fetchone()[0], 0) - - def test_get_info(self): - # Test swift.common.db.ContainerBroker.get_info - broker = ContainerBroker(':memory:', account='test1', - container='test2') - broker.initialize(normalize_timestamp('1')) - - info = broker.get_info() - self.assertEquals(info['account'], 'test1') - self.assertEquals(info['container'], 'test2') - self.assertEquals(info['hash'], '00000000000000000000000000000000') - - info = broker.get_info() - self.assertEquals(info['object_count'], 0) - self.assertEquals(info['bytes_used'], 0) - - broker.put_object('o1', normalize_timestamp(time()), 123, 'text/plain', - '5af83e3196bf99f440f31f2e1a6c9afe') - info = broker.get_info() - self.assertEquals(info['object_count'], 1) - self.assertEquals(info['bytes_used'], 123) - - sleep(.00001) - broker.put_object('o2', normalize_timestamp(time()), 123, 'text/plain', - '5af83e3196bf99f440f31f2e1a6c9afe') - info = broker.get_info() - self.assertEquals(info['object_count'], 2) - self.assertEquals(info['bytes_used'], 246) - - sleep(.00001) - broker.put_object('o2', normalize_timestamp(time()), 1000, - 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') - info = broker.get_info() - self.assertEquals(info['object_count'], 2) - self.assertEquals(info['bytes_used'], 1123) - - sleep(.00001) - broker.delete_object('o1', normalize_timestamp(time())) - info = broker.get_info() - self.assertEquals(info['object_count'], 1) - self.assertEquals(info['bytes_used'], 1000) - - sleep(.00001) - broker.delete_object('o2', normalize_timestamp(time())) - info = broker.get_info() - self.assertEquals(info['object_count'], 0) - self.assertEquals(info['bytes_used'], 0) - - info = broker.get_info() - self.assertEquals(info['x_container_sync_point1'], -1) - self.assertEquals(info['x_container_sync_point2'], -1) - - def test_set_x_syncs(self): - broker = ContainerBroker(':memory:', account='test1', - container='test2') - broker.initialize(normalize_timestamp('1')) - - info = broker.get_info() - self.assertEquals(info['x_container_sync_point1'], -1) - self.assertEquals(info['x_container_sync_point2'], -1) - - broker.set_x_container_sync_points(1, 2) - info = broker.get_info() - self.assertEquals(info['x_container_sync_point1'], 1) - self.assertEquals(info['x_container_sync_point2'], 2) - - def test_get_report_info(self): - broker = ContainerBroker(':memory:', account='test1', - container='test2') - broker.initialize(normalize_timestamp('1')) - - info = broker.get_info() - self.assertEquals(info['account'], 'test1') - self.assertEquals(info['container'], 'test2') - self.assertEquals(info['object_count'], 0) - self.assertEquals(info['bytes_used'], 0) - self.assertEquals(info['reported_object_count'], 0) - self.assertEquals(info['reported_bytes_used'], 0) - - broker.put_object('o1', normalize_timestamp(time()), 123, 'text/plain', - '5af83e3196bf99f440f31f2e1a6c9afe') - info = broker.get_info() - self.assertEquals(info['object_count'], 1) - self.assertEquals(info['bytes_used'], 123) - self.assertEquals(info['reported_object_count'], 0) - self.assertEquals(info['reported_bytes_used'], 0) - - sleep(.00001) - broker.put_object('o2', normalize_timestamp(time()), 123, 'text/plain', - '5af83e3196bf99f440f31f2e1a6c9afe') - info = broker.get_info() - self.assertEquals(info['object_count'], 2) - self.assertEquals(info['bytes_used'], 246) - self.assertEquals(info['reported_object_count'], 0) - self.assertEquals(info['reported_bytes_used'], 0) - - sleep(.00001) - broker.put_object('o2', normalize_timestamp(time()), 1000, - 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') - info = broker.get_info() - self.assertEquals(info['object_count'], 2) - self.assertEquals(info['bytes_used'], 1123) - self.assertEquals(info['reported_object_count'], 0) - self.assertEquals(info['reported_bytes_used'], 0) - - put_timestamp = normalize_timestamp(time()) - sleep(.001) - delete_timestamp = normalize_timestamp(time()) - broker.reported(put_timestamp, delete_timestamp, 2, 1123) - info = broker.get_info() - self.assertEquals(info['object_count'], 2) - self.assertEquals(info['bytes_used'], 1123) - self.assertEquals(info['reported_put_timestamp'], put_timestamp) - self.assertEquals(info['reported_delete_timestamp'], delete_timestamp) - self.assertEquals(info['reported_object_count'], 2) - self.assertEquals(info['reported_bytes_used'], 1123) - - sleep(.00001) - broker.delete_object('o1', normalize_timestamp(time())) - info = broker.get_info() - self.assertEquals(info['object_count'], 1) - self.assertEquals(info['bytes_used'], 1000) - self.assertEquals(info['reported_object_count'], 2) - self.assertEquals(info['reported_bytes_used'], 1123) - - sleep(.00001) - broker.delete_object('o2', normalize_timestamp(time())) - info = broker.get_info() - self.assertEquals(info['object_count'], 0) - self.assertEquals(info['bytes_used'], 0) - self.assertEquals(info['reported_object_count'], 2) - self.assertEquals(info['reported_bytes_used'], 1123) - - def test_list_objects_iter(self): - # Test swift.common.db.ContainerBroker.list_objects_iter - broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1')) - for obj1 in xrange(4): - for obj2 in xrange(125): - broker.put_object('%d/%04d' % (obj1, obj2), - normalize_timestamp(time()), 0, 'text/plain', - 'd41d8cd98f00b204e9800998ecf8427e') - for obj in xrange(125): - broker.put_object('2/0051/%04d' % obj, - normalize_timestamp(time()), 0, 'text/plain', - 'd41d8cd98f00b204e9800998ecf8427e') - - for obj in xrange(125): - broker.put_object('3/%04d/0049' % obj, - normalize_timestamp(time()), 0, 'text/plain', - 'd41d8cd98f00b204e9800998ecf8427e') - - listing = broker.list_objects_iter(100, '', None, None, '') - self.assertEquals(len(listing), 100) - self.assertEquals(listing[0][0], '0/0000') - self.assertEquals(listing[-1][0], '0/0099') - - listing = broker.list_objects_iter(100, '', '0/0050', None, '') - self.assertEquals(len(listing), 50) - self.assertEquals(listing[0][0], '0/0000') - self.assertEquals(listing[-1][0], '0/0049') - - listing = broker.list_objects_iter(100, '0/0099', None, None, '') - self.assertEquals(len(listing), 100) - self.assertEquals(listing[0][0], '0/0100') - self.assertEquals(listing[-1][0], '1/0074') - - listing = broker.list_objects_iter(55, '1/0074', None, None, '') - self.assertEquals(len(listing), 55) - self.assertEquals(listing[0][0], '1/0075') - self.assertEquals(listing[-1][0], '2/0004') - - listing = broker.list_objects_iter(10, '', None, '0/01', '') - self.assertEquals(len(listing), 10) - self.assertEquals(listing[0][0], '0/0100') - self.assertEquals(listing[-1][0], '0/0109') - - listing = broker.list_objects_iter(10, '', None, '0/', '/') - self.assertEquals(len(listing), 10) - self.assertEquals(listing[0][0], '0/0000') - self.assertEquals(listing[-1][0], '0/0009') - - # Same as above, but using the path argument. - listing = broker.list_objects_iter(10, '', None, None, '', '0') - self.assertEquals(len(listing), 10) - self.assertEquals(listing[0][0], '0/0000') - self.assertEquals(listing[-1][0], '0/0009') - - listing = broker.list_objects_iter(10, '', None, '', '/') - self.assertEquals(len(listing), 4) - self.assertEquals([row[0] for row in listing], - ['0/', '1/', '2/', '3/']) - - listing = broker.list_objects_iter(10, '2', None, None, '/') - self.assertEquals(len(listing), 2) - self.assertEquals([row[0] for row in listing], ['2/', '3/']) - - listing = broker.list_objects_iter(10, '2/', None, None, '/') - self.assertEquals(len(listing), 1) - self.assertEquals([row[0] for row in listing], ['3/']) - - listing = broker.list_objects_iter(10, '2/0050', None, '2/', '/') - self.assertEquals(len(listing), 10) - self.assertEquals(listing[0][0], '2/0051') - self.assertEquals(listing[1][0], '2/0051/') - self.assertEquals(listing[2][0], '2/0052') - self.assertEquals(listing[-1][0], '2/0059') - - listing = broker.list_objects_iter(10, '3/0045', None, '3/', '/') - self.assertEquals(len(listing), 10) - self.assertEquals([row[0] for row in listing], - ['3/0045/', '3/0046', '3/0046/', '3/0047', - '3/0047/', '3/0048', '3/0048/', '3/0049', - '3/0049/', '3/0050']) - - broker.put_object('3/0049/', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - listing = broker.list_objects_iter(10, '3/0048', None, None, None) - self.assertEquals(len(listing), 10) - self.assertEquals( - [row[0] for row in listing], - ['3/0048/0049', '3/0049', '3/0049/', - '3/0049/0049', '3/0050', '3/0050/0049', '3/0051', '3/0051/0049', - '3/0052', '3/0052/0049']) - - listing = broker.list_objects_iter(10, '3/0048', None, '3/', '/') - self.assertEquals(len(listing), 10) - self.assertEquals( - [row[0] for row in listing], - ['3/0048/', '3/0049', '3/0049/', '3/0050', - '3/0050/', '3/0051', '3/0051/', '3/0052', '3/0052/', '3/0053']) - - listing = broker.list_objects_iter(10, None, None, '3/0049/', '/') - self.assertEquals(len(listing), 2) - self.assertEquals( - [row[0] for row in listing], - ['3/0049/', '3/0049/0049']) - - listing = broker.list_objects_iter(10, None, None, None, None, - '3/0049') - self.assertEquals(len(listing), 1) - self.assertEquals([row[0] for row in listing], ['3/0049/0049']) - - listing = broker.list_objects_iter(2, None, None, '3/', '/') - self.assertEquals(len(listing), 2) - self.assertEquals([row[0] for row in listing], ['3/0000', '3/0000/']) - - listing = broker.list_objects_iter(2, None, None, None, None, '3') - self.assertEquals(len(listing), 2) - self.assertEquals([row[0] for row in listing], ['3/0000', '3/0001']) - - def test_list_objects_iter_non_slash(self): - # Test swift.common.db.ContainerBroker.list_objects_iter using a - # delimiter that is not a slash - broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1')) - for obj1 in xrange(4): - for obj2 in xrange(125): - broker.put_object('%d:%04d' % (obj1, obj2), - normalize_timestamp(time()), 0, 'text/plain', - 'd41d8cd98f00b204e9800998ecf8427e') - for obj in xrange(125): - broker.put_object('2:0051:%04d' % obj, - normalize_timestamp(time()), 0, 'text/plain', - 'd41d8cd98f00b204e9800998ecf8427e') - - for obj in xrange(125): - broker.put_object('3:%04d:0049' % obj, - normalize_timestamp(time()), 0, 'text/plain', - 'd41d8cd98f00b204e9800998ecf8427e') - - listing = broker.list_objects_iter(100, '', None, None, '') - self.assertEquals(len(listing), 100) - self.assertEquals(listing[0][0], '0:0000') - self.assertEquals(listing[-1][0], '0:0099') - - listing = broker.list_objects_iter(100, '', '0:0050', None, '') - self.assertEquals(len(listing), 50) - self.assertEquals(listing[0][0], '0:0000') - self.assertEquals(listing[-1][0], '0:0049') - - listing = broker.list_objects_iter(100, '0:0099', None, None, '') - self.assertEquals(len(listing), 100) - self.assertEquals(listing[0][0], '0:0100') - self.assertEquals(listing[-1][0], '1:0074') - - listing = broker.list_objects_iter(55, '1:0074', None, None, '') - self.assertEquals(len(listing), 55) - self.assertEquals(listing[0][0], '1:0075') - self.assertEquals(listing[-1][0], '2:0004') - - listing = broker.list_objects_iter(10, '', None, '0:01', '') - self.assertEquals(len(listing), 10) - self.assertEquals(listing[0][0], '0:0100') - self.assertEquals(listing[-1][0], '0:0109') - - listing = broker.list_objects_iter(10, '', None, '0:', ':') - self.assertEquals(len(listing), 10) - self.assertEquals(listing[0][0], '0:0000') - self.assertEquals(listing[-1][0], '0:0009') - - # Same as above, but using the path argument, so nothing should be - # returned since path uses a '/' as a delimiter. - listing = broker.list_objects_iter(10, '', None, None, '', '0') - self.assertEquals(len(listing), 0) - - listing = broker.list_objects_iter(10, '', None, '', ':') - self.assertEquals(len(listing), 4) - self.assertEquals([row[0] for row in listing], - ['0:', '1:', '2:', '3:']) - - listing = broker.list_objects_iter(10, '2', None, None, ':') - self.assertEquals(len(listing), 2) - self.assertEquals([row[0] for row in listing], ['2:', '3:']) - - listing = broker.list_objects_iter(10, '2:', None, None, ':') - self.assertEquals(len(listing), 1) - self.assertEquals([row[0] for row in listing], ['3:']) - - listing = broker.list_objects_iter(10, '2:0050', None, '2:', ':') - self.assertEquals(len(listing), 10) - self.assertEquals(listing[0][0], '2:0051') - self.assertEquals(listing[1][0], '2:0051:') - self.assertEquals(listing[2][0], '2:0052') - self.assertEquals(listing[-1][0], '2:0059') - - listing = broker.list_objects_iter(10, '3:0045', None, '3:', ':') - self.assertEquals(len(listing), 10) - self.assertEquals([row[0] for row in listing], - ['3:0045:', '3:0046', '3:0046:', '3:0047', - '3:0047:', '3:0048', '3:0048:', '3:0049', - '3:0049:', '3:0050']) - - broker.put_object('3:0049:', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - listing = broker.list_objects_iter(10, '3:0048', None, None, None) - self.assertEquals(len(listing), 10) - self.assertEquals( - [row[0] for row in listing], - ['3:0048:0049', '3:0049', '3:0049:', - '3:0049:0049', '3:0050', '3:0050:0049', '3:0051', '3:0051:0049', - '3:0052', '3:0052:0049']) - - listing = broker.list_objects_iter(10, '3:0048', None, '3:', ':') - self.assertEquals(len(listing), 10) - self.assertEquals( - [row[0] for row in listing], - ['3:0048:', '3:0049', '3:0049:', '3:0050', - '3:0050:', '3:0051', '3:0051:', '3:0052', '3:0052:', '3:0053']) - - listing = broker.list_objects_iter(10, None, None, '3:0049:', ':') - self.assertEquals(len(listing), 2) - self.assertEquals( - [row[0] for row in listing], - ['3:0049:', '3:0049:0049']) - - # Same as above, but using the path argument, so nothing should be - # returned since path uses a '/' as a delimiter. - listing = broker.list_objects_iter(10, None, None, None, None, - '3:0049') - self.assertEquals(len(listing), 0) - - listing = broker.list_objects_iter(2, None, None, '3:', ':') - self.assertEquals(len(listing), 2) - self.assertEquals([row[0] for row in listing], ['3:0000', '3:0000:']) - - listing = broker.list_objects_iter(2, None, None, None, None, '3') - self.assertEquals(len(listing), 0) - - def test_list_objects_iter_prefix_delim(self): - # Test swift.common.db.ContainerBroker.list_objects_iter - broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1')) - - broker.put_object( - '/pets/dogs/1', normalize_timestamp(0), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object( - '/pets/dogs/2', normalize_timestamp(0), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object( - '/pets/fish/a', normalize_timestamp(0), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object( - '/pets/fish/b', normalize_timestamp(0), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object( - '/pets/fish_info.txt', normalize_timestamp(0), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object( - '/snakes', normalize_timestamp(0), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - - #def list_objects_iter(self, limit, marker, prefix, delimiter, - # path=None, format=None): - listing = broker.list_objects_iter(100, None, None, '/pets/f', '/') - self.assertEquals([row[0] for row in listing], - ['/pets/fish/', '/pets/fish_info.txt']) - listing = broker.list_objects_iter(100, None, None, '/pets/fish', '/') - self.assertEquals([row[0] for row in listing], - ['/pets/fish/', '/pets/fish_info.txt']) - listing = broker.list_objects_iter(100, None, None, '/pets/fish/', '/') - self.assertEquals([row[0] for row in listing], - ['/pets/fish/a', '/pets/fish/b']) - - def test_double_check_trailing_delimiter(self): - # Test swift.common.db.ContainerBroker.list_objects_iter for a - # container that has an odd file with a trailing delimiter - broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1')) - broker.put_object('a', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a/', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a/a', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a/a/a', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a/a/b', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a/b', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b/a', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b/b', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('c', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a/0', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0/', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('00', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0/0', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0/00', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0/1', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0/1/', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0/1/0', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('1', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('1/', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('1/0', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - listing = broker.list_objects_iter(25, None, None, None, None) - self.assertEquals(len(listing), 22) - self.assertEquals( - [row[0] for row in listing], - ['0', '0/', '0/0', '0/00', '0/1', '0/1/', '0/1/0', '00', '1', '1/', - '1/0', 'a', 'a/', 'a/0', 'a/a', 'a/a/a', 'a/a/b', 'a/b', 'b', - 'b/a', 'b/b', 'c']) - listing = broker.list_objects_iter(25, None, None, '', '/') - self.assertEquals(len(listing), 10) - self.assertEquals( - [row[0] for row in listing], - ['0', '0/', '00', '1', '1/', 'a', 'a/', 'b', 'b/', 'c']) - listing = broker.list_objects_iter(25, None, None, 'a/', '/') - self.assertEquals(len(listing), 5) - self.assertEquals( - [row[0] for row in listing], - ['a/', 'a/0', 'a/a', 'a/a/', 'a/b']) - listing = broker.list_objects_iter(25, None, None, '0/', '/') - self.assertEquals(len(listing), 5) - self.assertEquals( - [row[0] for row in listing], - ['0/', '0/0', '0/00', '0/1', '0/1/']) - listing = broker.list_objects_iter(25, None, None, '0/1/', '/') - self.assertEquals(len(listing), 2) - self.assertEquals( - [row[0] for row in listing], - ['0/1/', '0/1/0']) - listing = broker.list_objects_iter(25, None, None, 'b/', '/') - self.assertEquals(len(listing), 2) - self.assertEquals([row[0] for row in listing], ['b/a', 'b/b']) - - def test_double_check_trailing_delimiter_non_slash(self): - # Test swift.common.db.ContainerBroker.list_objects_iter for a - # container that has an odd file with a trailing delimiter - broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1')) - broker.put_object('a', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a:', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a:a', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a:a:a', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a:a:b', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a:b', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b:a', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b:b', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('c', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('a:0', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0:', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('00', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0:0', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0:00', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0:1', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0:1:', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('0:1:0', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('1', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('1:', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('1:0', normalize_timestamp(time()), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - listing = broker.list_objects_iter(25, None, None, None, None) - self.assertEquals(len(listing), 22) - self.assertEquals( - [row[0] for row in listing], - ['0', '00', '0:', '0:0', '0:00', '0:1', '0:1:', '0:1:0', '1', '1:', - '1:0', 'a', 'a:', 'a:0', 'a:a', 'a:a:a', 'a:a:b', 'a:b', 'b', - 'b:a', 'b:b', 'c']) - listing = broker.list_objects_iter(25, None, None, '', ':') - self.assertEquals(len(listing), 10) - self.assertEquals( - [row[0] for row in listing], - ['0', '00', '0:', '1', '1:', 'a', 'a:', 'b', 'b:', 'c']) - listing = broker.list_objects_iter(25, None, None, 'a:', ':') - self.assertEquals(len(listing), 5) - self.assertEquals( - [row[0] for row in listing], - ['a:', 'a:0', 'a:a', 'a:a:', 'a:b']) - listing = broker.list_objects_iter(25, None, None, '0:', ':') - self.assertEquals(len(listing), 5) - self.assertEquals( - [row[0] for row in listing], - ['0:', '0:0', '0:00', '0:1', '0:1:']) - listing = broker.list_objects_iter(25, None, None, '0:1:', ':') - self.assertEquals(len(listing), 2) - self.assertEquals( - [row[0] for row in listing], - ['0:1:', '0:1:0']) - listing = broker.list_objects_iter(25, None, None, 'b:', ':') - self.assertEquals(len(listing), 2) - self.assertEquals([row[0] for row in listing], ['b:a', 'b:b']) - - def test_chexor(self): - broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1')) - broker.put_object('a', normalize_timestamp(1), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker.put_object('b', normalize_timestamp(2), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - hasha = hashlib.md5('%s-%s' % ('a', '0000000001.00000')).digest() - hashb = hashlib.md5('%s-%s' % ('b', '0000000002.00000')).digest() - hashc = ''.join( - ('%2x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) - self.assertEquals(broker.get_info()['hash'], hashc) - broker.put_object('b', normalize_timestamp(3), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - hashb = hashlib.md5('%s-%s' % ('b', '0000000003.00000')).digest() - hashc = ''.join( - ('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) - self.assertEquals(broker.get_info()['hash'], hashc) - - def test_newid(self): - # test DatabaseBroker.newid - broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1')) - id = broker.get_info()['id'] - broker.newid('someid') - self.assertNotEquals(id, broker.get_info()['id']) - - def test_get_items_since(self): - # test DatabaseBroker.get_items_since - broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1')) - broker.put_object('a', normalize_timestamp(1), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - max_row = broker.get_replication_info()['max_row'] - broker.put_object('b', normalize_timestamp(2), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - items = broker.get_items_since(max_row, 1000) - self.assertEquals(len(items), 1) - self.assertEquals(items[0]['name'], 'b') - - def test_sync_merging(self): - # exercise the DatabaseBroker sync functions a bit - broker1 = ContainerBroker(':memory:', account='a', container='c') - broker1.initialize(normalize_timestamp('1')) - broker2 = ContainerBroker(':memory:', account='a', container='c') - broker2.initialize(normalize_timestamp('1')) - self.assertEquals(broker2.get_sync('12345'), -1) - broker1.merge_syncs([{'sync_point': 3, 'remote_id': '12345'}]) - broker2.merge_syncs(broker1.get_syncs()) - self.assertEquals(broker2.get_sync('12345'), 3) - - def test_merge_items(self): - broker1 = ContainerBroker(':memory:', account='a', container='c') - broker1.initialize(normalize_timestamp('1')) - broker2 = ContainerBroker(':memory:', account='a', container='c') - broker2.initialize(normalize_timestamp('1')) - broker1.put_object('a', normalize_timestamp(1), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker1.put_object('b', normalize_timestamp(2), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - id = broker1.get_info()['id'] - broker2.merge_items(broker1.get_items_since( - broker2.get_sync(id), 1000), id) - items = broker2.get_items_since(-1, 1000) - self.assertEquals(len(items), 2) - self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) - broker1.put_object('c', normalize_timestamp(3), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker2.merge_items(broker1.get_items_since( - broker2.get_sync(id), 1000), id) - items = broker2.get_items_since(-1, 1000) - self.assertEquals(len(items), 3) - self.assertEquals(['a', 'b', 'c'], - sorted([rec['name'] for rec in items])) - - def test_merge_items_overwrite(self): - # test DatabaseBroker.merge_items - broker1 = ContainerBroker(':memory:', account='a', container='c') - broker1.initialize(normalize_timestamp('1')) - id = broker1.get_info()['id'] - broker2 = ContainerBroker(':memory:', account='a', container='c') - broker2.initialize(normalize_timestamp('1')) - broker1.put_object('a', normalize_timestamp(2), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker1.put_object('b', normalize_timestamp(3), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker2.merge_items(broker1.get_items_since( - broker2.get_sync(id), 1000), id) - broker1.put_object('a', normalize_timestamp(4), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker2.merge_items(broker1.get_items_since( - broker2.get_sync(id), 1000), id) - items = broker2.get_items_since(-1, 1000) - self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) - for rec in items: - if rec['name'] == 'a': - self.assertEquals(rec['created_at'], normalize_timestamp(4)) - if rec['name'] == 'b': - self.assertEquals(rec['created_at'], normalize_timestamp(3)) - - def test_merge_items_post_overwrite_out_of_order(self): - # test DatabaseBroker.merge_items - broker1 = ContainerBroker(':memory:', account='a', container='c') - broker1.initialize(normalize_timestamp('1')) - id = broker1.get_info()['id'] - broker2 = ContainerBroker(':memory:', account='a', container='c') - broker2.initialize(normalize_timestamp('1')) - broker1.put_object('a', normalize_timestamp(2), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker1.put_object('b', normalize_timestamp(3), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker2.merge_items(broker1.get_items_since( - broker2.get_sync(id), 1000), id) - broker1.put_object('a', normalize_timestamp(4), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker2.merge_items(broker1.get_items_since( - broker2.get_sync(id), 1000), id) - items = broker2.get_items_since(-1, 1000) - self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) - for rec in items: - if rec['name'] == 'a': - self.assertEquals(rec['created_at'], normalize_timestamp(4)) - if rec['name'] == 'b': - self.assertEquals(rec['created_at'], normalize_timestamp(3)) - self.assertEquals(rec['content_type'], 'text/plain') - items = broker2.get_items_since(-1, 1000) - self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) - for rec in items: - if rec['name'] == 'a': - self.assertEquals(rec['created_at'], normalize_timestamp(4)) - if rec['name'] == 'b': - self.assertEquals(rec['created_at'], normalize_timestamp(3)) - broker1.put_object('b', normalize_timestamp(5), 0, - 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - broker2.merge_items(broker1.get_items_since( - broker2.get_sync(id), 1000), id) - items = broker2.get_items_since(-1, 1000) - self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) - for rec in items: - if rec['name'] == 'a': - self.assertEquals(rec['created_at'], normalize_timestamp(4)) - if rec['name'] == 'b': - self.assertEquals(rec['created_at'], normalize_timestamp(5)) - self.assertEquals(rec['content_type'], 'text/plain') - - -def premetadata_create_container_stat_table(self, conn, put_timestamp=None): - """ - Copied from swift.common.db.ContainerBroker before the metadata column was - added; used for testing with TestContainerBrokerBeforeMetadata. - - Create the container_stat table which is specifc to the container DB. - - :param conn: DB connection object - :param put_timestamp: put timestamp - """ - if put_timestamp is None: - put_timestamp = normalize_timestamp(0) - conn.executescript(''' - CREATE TABLE container_stat ( - account TEXT, - container TEXT, - created_at TEXT, - put_timestamp TEXT DEFAULT '0', - delete_timestamp TEXT DEFAULT '0', - object_count INTEGER, - bytes_used INTEGER, - reported_put_timestamp TEXT DEFAULT '0', - reported_delete_timestamp TEXT DEFAULT '0', - reported_object_count INTEGER DEFAULT 0, - reported_bytes_used INTEGER DEFAULT 0, - hash TEXT default '00000000000000000000000000000000', - id TEXT, - status TEXT DEFAULT '', - status_changed_at TEXT DEFAULT '0' - ); - - INSERT INTO container_stat (object_count, bytes_used) - VALUES (0, 0); - ''') - conn.execute(''' - UPDATE container_stat - SET account = ?, container = ?, created_at = ?, id = ?, - put_timestamp = ? - ''', (self.account, self.container, normalize_timestamp(time()), - str(uuid4()), put_timestamp)) - - -class TestContainerBrokerBeforeMetadata(TestContainerBroker): - """ - Tests for swift.common.db.ContainerBroker against databases created before - the metadata column was added. - """ - - def setUp(self): - self._imported_create_container_stat_table = \ - ContainerBroker.create_container_stat_table - ContainerBroker.create_container_stat_table = \ - premetadata_create_container_stat_table - broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1')) - exc = None - with broker.get() as conn: - try: - conn.execute('SELECT metadata FROM container_stat') - except BaseException as err: - exc = err - self.assert_('no such column: metadata' in str(exc)) - - def tearDown(self): - ContainerBroker.create_container_stat_table = \ - self._imported_create_container_stat_table - broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1')) - with broker.get() as conn: - conn.execute('SELECT metadata FROM container_stat') - - -def prexsync_create_container_stat_table(self, conn, put_timestamp=None): - """ - Copied from swift.common.db.ContainerBroker before the - x_container_sync_point[12] columns were added; used for testing with - TestContainerBrokerBeforeXSync. - - Create the container_stat table which is specifc to the container DB. - - :param conn: DB connection object - :param put_timestamp: put timestamp - """ - if put_timestamp is None: - put_timestamp = normalize_timestamp(0) - conn.executescript(""" - CREATE TABLE container_stat ( - account TEXT, - container TEXT, - created_at TEXT, - put_timestamp TEXT DEFAULT '0', - delete_timestamp TEXT DEFAULT '0', - object_count INTEGER, - bytes_used INTEGER, - reported_put_timestamp TEXT DEFAULT '0', - reported_delete_timestamp TEXT DEFAULT '0', - reported_object_count INTEGER DEFAULT 0, - reported_bytes_used INTEGER DEFAULT 0, - hash TEXT default '00000000000000000000000000000000', - id TEXT, - status TEXT DEFAULT '', - status_changed_at TEXT DEFAULT '0', - metadata TEXT DEFAULT '' - ); - - INSERT INTO container_stat (object_count, bytes_used) - VALUES (0, 0); - """) - conn.execute(''' - UPDATE container_stat - SET account = ?, container = ?, created_at = ?, id = ?, - put_timestamp = ? - ''', (self.account, self.container, normalize_timestamp(time()), - str(uuid4()), put_timestamp)) - - -class TestContainerBrokerBeforeXSync(TestContainerBroker): - """ - Tests for swift.common.db.ContainerBroker against databases created before - the x_container_sync_point[12] columns were added. - """ - - def setUp(self): - self._imported_create_container_stat_table = \ - ContainerBroker.create_container_stat_table - ContainerBroker.create_container_stat_table = \ - prexsync_create_container_stat_table - broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1')) - exc = None - with broker.get() as conn: - try: - conn.execute('''SELECT x_container_sync_point1 - FROM container_stat''') - except BaseException as err: - exc = err - self.assert_('no such column: x_container_sync_point1' in str(exc)) - - def tearDown(self): - ContainerBroker.create_container_stat_table = \ - self._imported_create_container_stat_table - broker = ContainerBroker(':memory:', account='a', container='c') - broker.initialize(normalize_timestamp('1')) - with broker.get() as conn: - conn.execute('SELECT x_container_sync_point1 FROM container_stat') - - -class TestAccountBroker(unittest.TestCase): - """Tests for swift.common.db.AccountBroker""" - - def test_creation(self): - # Test swift.common.db.AccountBroker.__init__ - broker = AccountBroker(':memory:', account='a') - self.assertEqual(broker.db_file, ':memory:') - got_exc = False - try: - with broker.get() as conn: - pass - except Exception: - got_exc = True - self.assert_(got_exc) - broker.initialize(normalize_timestamp('1')) - with broker.get() as conn: - curs = conn.cursor() - curs.execute('SELECT 1') - self.assertEqual(curs.fetchall()[0][0], 1) - - def test_exception(self): - # Test swift.common.db.AccountBroker throwing a conn away after - # exception - first_conn = None - broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) - with broker.get() as conn: - first_conn = conn - try: - with broker.get() as conn: - self.assertEquals(first_conn, conn) - raise Exception('OMG') - except Exception: - pass - self.assert_(broker.conn is None) - - def test_empty(self): - # Test swift.common.db.AccountBroker.empty - broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) - self.assert_(broker.empty()) - broker.put_container('o', normalize_timestamp(time()), 0, 0, 0) - self.assert_(not broker.empty()) - sleep(.00001) - broker.put_container('o', 0, normalize_timestamp(time()), 0, 0) - self.assert_(broker.empty()) - - def test_reclaim(self): - broker = AccountBroker(':memory:', account='test_account') - broker.initialize(normalize_timestamp('1')) - broker.put_container('c', normalize_timestamp(time()), 0, 0, 0) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT count(*) FROM container " - "WHERE deleted = 0").fetchone()[0], 1) - self.assertEquals(conn.execute( - "SELECT count(*) FROM container " - "WHERE deleted = 1").fetchone()[0], 0) - broker.reclaim(normalize_timestamp(time() - 999), time()) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT count(*) FROM container " - "WHERE deleted = 0").fetchone()[0], 1) - self.assertEquals(conn.execute( - "SELECT count(*) FROM container " - "WHERE deleted = 1").fetchone()[0], 0) - sleep(.00001) - broker.put_container('c', 0, normalize_timestamp(time()), 0, 0) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT count(*) FROM container " - "WHERE deleted = 0").fetchone()[0], 0) - self.assertEquals(conn.execute( - "SELECT count(*) FROM container " - "WHERE deleted = 1").fetchone()[0], 1) - broker.reclaim(normalize_timestamp(time() - 999), time()) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT count(*) FROM container " - "WHERE deleted = 0").fetchone()[0], 0) - self.assertEquals(conn.execute( - "SELECT count(*) FROM container " - "WHERE deleted = 1").fetchone()[0], 1) - sleep(.00001) - broker.reclaim(normalize_timestamp(time()), time()) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT count(*) FROM container " - "WHERE deleted = 0").fetchone()[0], 0) - self.assertEquals(conn.execute( - "SELECT count(*) FROM container " - "WHERE deleted = 1").fetchone()[0], 0) - # Test reclaim after deletion. Create 3 test containers - broker.put_container('x', 0, 0, 0, 0) - broker.put_container('y', 0, 0, 0, 0) - broker.put_container('z', 0, 0, 0, 0) - broker.reclaim(normalize_timestamp(time()), time()) - # self.assertEquals(len(res), 2) - # self.assert_(isinstance(res, tuple)) - # containers, account_name = res - # self.assert_(containers is None) - # self.assert_(account_name is None) - # Now delete the account - broker.delete_db(normalize_timestamp(time())) - broker.reclaim(normalize_timestamp(time()), time()) - # self.assertEquals(len(res), 2) - # self.assert_(isinstance(res, tuple)) - # containers, account_name = res - # self.assertEquals(account_name, 'test_account') - # self.assertEquals(len(containers), 3) - # self.assert_('x' in containers) - # self.assert_('y' in containers) - # self.assert_('z' in containers) - # self.assert_('a' not in containers) - - def test_delete_container(self): - # Test swift.common.db.AccountBroker.delete_container - broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) - broker.put_container('o', normalize_timestamp(time()), 0, 0, 0) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT count(*) FROM container " - "WHERE deleted = 0").fetchone()[0], 1) - self.assertEquals(conn.execute( - "SELECT count(*) FROM container " - "WHERE deleted = 1").fetchone()[0], 0) - sleep(.00001) - broker.put_container('o', 0, normalize_timestamp(time()), 0, 0) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT count(*) FROM container " - "WHERE deleted = 0").fetchone()[0], 0) - self.assertEquals(conn.execute( - "SELECT count(*) FROM container " - "WHERE deleted = 1").fetchone()[0], 1) - - def test_put_container(self): - # Test swift.common.db.AccountBroker.put_container - broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) - - # Create initial container - timestamp = normalize_timestamp(time()) - broker.put_container('"{}"', timestamp, 0, 0, 0) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT name FROM container").fetchone()[0], - '"{}"') - self.assertEquals(conn.execute( - "SELECT put_timestamp FROM container").fetchone()[0], - timestamp) - self.assertEquals(conn.execute( - "SELECT deleted FROM container").fetchone()[0], 0) - - # Reput same event - broker.put_container('"{}"', timestamp, 0, 0, 0) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT name FROM container").fetchone()[0], - '"{}"') - self.assertEquals(conn.execute( - "SELECT put_timestamp FROM container").fetchone()[0], - timestamp) - self.assertEquals(conn.execute( - "SELECT deleted FROM container").fetchone()[0], 0) - - # Put new event - sleep(.00001) - timestamp = normalize_timestamp(time()) - broker.put_container('"{}"', timestamp, 0, 0, 0) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT name FROM container").fetchone()[0], - '"{}"') - self.assertEquals(conn.execute( - "SELECT put_timestamp FROM container").fetchone()[0], - timestamp) - self.assertEquals(conn.execute( - "SELECT deleted FROM container").fetchone()[0], 0) - - # Put old event - otimestamp = normalize_timestamp(float(timestamp) - 1) - broker.put_container('"{}"', otimestamp, 0, 0, 0) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT name FROM container").fetchone()[0], - '"{}"') - self.assertEquals(conn.execute( - "SELECT put_timestamp FROM container").fetchone()[0], - timestamp) - self.assertEquals(conn.execute( - "SELECT deleted FROM container").fetchone()[0], 0) - - # Put old delete event - dtimestamp = normalize_timestamp(float(timestamp) - 1) - broker.put_container('"{}"', 0, dtimestamp, 0, 0) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT name FROM container").fetchone()[0], - '"{}"') - self.assertEquals(conn.execute( - "SELECT put_timestamp FROM container").fetchone()[0], - timestamp) - self.assertEquals(conn.execute( - "SELECT delete_timestamp FROM container").fetchone()[0], - dtimestamp) - self.assertEquals(conn.execute( - "SELECT deleted FROM container").fetchone()[0], 0) - - # Put new delete event - sleep(.00001) - timestamp = normalize_timestamp(time()) - broker.put_container('"{}"', 0, timestamp, 0, 0) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT name FROM container").fetchone()[0], - '"{}"') - self.assertEquals(conn.execute( - "SELECT delete_timestamp FROM container").fetchone()[0], - timestamp) - self.assertEquals(conn.execute( - "SELECT deleted FROM container").fetchone()[0], 1) - - # Put new event - sleep(.00001) - timestamp = normalize_timestamp(time()) - broker.put_container('"{}"', timestamp, 0, 0, 0) - with broker.get() as conn: - self.assertEquals(conn.execute( - "SELECT name FROM container").fetchone()[0], - '"{}"') - self.assertEquals(conn.execute( - "SELECT put_timestamp FROM container").fetchone()[0], - timestamp) - self.assertEquals(conn.execute( - "SELECT deleted FROM container").fetchone()[0], 0) - - def test_get_info(self): - # Test swift.common.db.AccountBroker.get_info - broker = AccountBroker(':memory:', account='test1') - broker.initialize(normalize_timestamp('1')) - - info = broker.get_info() - self.assertEquals(info['account'], 'test1') - self.assertEquals(info['hash'], '00000000000000000000000000000000') - - info = broker.get_info() - self.assertEquals(info['container_count'], 0) - - broker.put_container('c1', normalize_timestamp(time()), 0, 0, 0) - info = broker.get_info() - self.assertEquals(info['container_count'], 1) - - sleep(.00001) - broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0) - info = broker.get_info() - self.assertEquals(info['container_count'], 2) - - sleep(.00001) - broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0) - info = broker.get_info() - self.assertEquals(info['container_count'], 2) - - sleep(.00001) - broker.put_container('c1', 0, normalize_timestamp(time()), 0, 0) - info = broker.get_info() - self.assertEquals(info['container_count'], 1) - - sleep(.00001) - broker.put_container('c2', 0, normalize_timestamp(time()), 0, 0) - info = broker.get_info() - self.assertEquals(info['container_count'], 0) - - def test_list_containers_iter(self): - # Test swift.common.db.AccountBroker.list_containers_iter - broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) - for cont1 in xrange(4): - for cont2 in xrange(125): - broker.put_container('%d-%04d' % (cont1, cont2), - normalize_timestamp(time()), 0, 0, 0) - for cont in xrange(125): - broker.put_container('2-0051-%04d' % cont, - normalize_timestamp(time()), 0, 0, 0) - - for cont in xrange(125): - broker.put_container('3-%04d-0049' % cont, - normalize_timestamp(time()), 0, 0, 0) - - listing = broker.list_containers_iter(100, '', None, None, '') - self.assertEquals(len(listing), 100) - self.assertEquals(listing[0][0], '0-0000') - self.assertEquals(listing[-1][0], '0-0099') - - listing = broker.list_containers_iter(100, '', '0-0050', None, '') - self.assertEquals(len(listing), 50) - self.assertEquals(listing[0][0], '0-0000') - self.assertEquals(listing[-1][0], '0-0049') - - listing = broker.list_containers_iter(100, '0-0099', None, None, '') - self.assertEquals(len(listing), 100) - self.assertEquals(listing[0][0], '0-0100') - self.assertEquals(listing[-1][0], '1-0074') - - listing = broker.list_containers_iter(55, '1-0074', None, None, '') - self.assertEquals(len(listing), 55) - self.assertEquals(listing[0][0], '1-0075') - self.assertEquals(listing[-1][0], '2-0004') - - listing = broker.list_containers_iter(10, '', None, '0-01', '') - self.assertEquals(len(listing), 10) - self.assertEquals(listing[0][0], '0-0100') - self.assertEquals(listing[-1][0], '0-0109') - - listing = broker.list_containers_iter(10, '', None, '0-01', '-') - self.assertEquals(len(listing), 10) - self.assertEquals(listing[0][0], '0-0100') - self.assertEquals(listing[-1][0], '0-0109') - - listing = broker.list_containers_iter(10, '', None, '0-', '-') - self.assertEquals(len(listing), 10) - self.assertEquals(listing[0][0], '0-0000') - self.assertEquals(listing[-1][0], '0-0009') - - listing = broker.list_containers_iter(10, '', None, '', '-') - self.assertEquals(len(listing), 4) - self.assertEquals([row[0] for row in listing], - ['0-', '1-', '2-', '3-']) - - listing = broker.list_containers_iter(10, '2-', None, None, '-') - self.assertEquals(len(listing), 1) - self.assertEquals([row[0] for row in listing], ['3-']) - - listing = broker.list_containers_iter(10, '', None, '2', '-') - self.assertEquals(len(listing), 1) - self.assertEquals([row[0] for row in listing], ['2-']) - - listing = broker.list_containers_iter(10, '2-0050', None, '2-', '-') - self.assertEquals(len(listing), 10) - self.assertEquals(listing[0][0], '2-0051') - self.assertEquals(listing[1][0], '2-0051-') - self.assertEquals(listing[2][0], '2-0052') - self.assertEquals(listing[-1][0], '2-0059') - - listing = broker.list_containers_iter(10, '3-0045', None, '3-', '-') - self.assertEquals(len(listing), 10) - self.assertEquals([row[0] for row in listing], - ['3-0045-', '3-0046', '3-0046-', '3-0047', - '3-0047-', '3-0048', '3-0048-', '3-0049', - '3-0049-', '3-0050']) - - broker.put_container('3-0049-', normalize_timestamp(time()), 0, 0, 0) - listing = broker.list_containers_iter(10, '3-0048', None, None, None) - self.assertEquals(len(listing), 10) - self.assertEquals([row[0] for row in listing], - ['3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', - '3-0050', '3-0050-0049', '3-0051', '3-0051-0049', - '3-0052', '3-0052-0049']) - - listing = broker.list_containers_iter(10, '3-0048', None, '3-', '-') - self.assertEquals(len(listing), 10) - self.assertEquals([row[0] for row in listing], - ['3-0048-', '3-0049', '3-0049-', '3-0050', - '3-0050-', '3-0051', '3-0051-', '3-0052', - '3-0052-', '3-0053']) - - listing = broker.list_containers_iter(10, None, None, '3-0049-', '-') - self.assertEquals(len(listing), 2) - self.assertEquals([row[0] for row in listing], - ['3-0049-', '3-0049-0049']) - - def test_double_check_trailing_delimiter(self): - # Test swift.common.db.AccountBroker.list_containers_iter for an - # account that has an odd container with a trailing delimiter - broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) - broker.put_container('a', normalize_timestamp(time()), 0, 0, 0) - broker.put_container('a-', normalize_timestamp(time()), 0, 0, 0) - broker.put_container('a-a', normalize_timestamp(time()), 0, 0, 0) - broker.put_container('a-a-a', normalize_timestamp(time()), 0, 0, 0) - broker.put_container('a-a-b', normalize_timestamp(time()), 0, 0, 0) - broker.put_container('a-b', normalize_timestamp(time()), 0, 0, 0) - broker.put_container('b', normalize_timestamp(time()), 0, 0, 0) - broker.put_container('b-a', normalize_timestamp(time()), 0, 0, 0) - broker.put_container('b-b', normalize_timestamp(time()), 0, 0, 0) - broker.put_container('c', normalize_timestamp(time()), 0, 0, 0) - listing = broker.list_containers_iter(15, None, None, None, None) - self.assertEquals(len(listing), 10) - self.assertEquals([row[0] for row in listing], - ['a', 'a-', 'a-a', 'a-a-a', 'a-a-b', 'a-b', 'b', - 'b-a', 'b-b', 'c']) - listing = broker.list_containers_iter(15, None, None, '', '-') - self.assertEquals(len(listing), 5) - self.assertEquals([row[0] for row in listing], - ['a', 'a-', 'b', 'b-', 'c']) - listing = broker.list_containers_iter(15, None, None, 'a-', '-') - self.assertEquals(len(listing), 4) - self.assertEquals([row[0] for row in listing], - ['a-', 'a-a', 'a-a-', 'a-b']) - listing = broker.list_containers_iter(15, None, None, 'b-', '-') - self.assertEquals(len(listing), 2) - self.assertEquals([row[0] for row in listing], ['b-a', 'b-b']) - - def test_chexor(self): - broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) - broker.put_container('a', normalize_timestamp(1), - normalize_timestamp(0), 0, 0) - broker.put_container('b', normalize_timestamp(2), - normalize_timestamp(0), 0, 0) - hasha = hashlib.md5( - '%s-%s' % ('a', '0000000001.00000-0000000000.00000-0-0') - ).digest() - hashb = hashlib.md5( - '%s-%s' % ('b', '0000000002.00000-0000000000.00000-0-0') - ).digest() - hashc = \ - ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) - self.assertEquals(broker.get_info()['hash'], hashc) - broker.put_container('b', normalize_timestamp(3), - normalize_timestamp(0), 0, 0) - hashb = hashlib.md5( - '%s-%s' % ('b', '0000000003.00000-0000000000.00000-0-0') - ).digest() - hashc = \ - ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) - self.assertEquals(broker.get_info()['hash'], hashc) - - def test_merge_items(self): - broker1 = AccountBroker(':memory:', account='a') - broker1.initialize(normalize_timestamp('1')) - broker2 = AccountBroker(':memory:', account='a') - broker2.initialize(normalize_timestamp('1')) - broker1.put_container('a', normalize_timestamp(1), 0, 0, 0) - broker1.put_container('b', normalize_timestamp(2), 0, 0, 0) - id = broker1.get_info()['id'] - broker2.merge_items(broker1.get_items_since( - broker2.get_sync(id), 1000), id) - items = broker2.get_items_since(-1, 1000) - self.assertEquals(len(items), 2) - self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) - broker1.put_container('c', normalize_timestamp(3), 0, 0, 0) - broker2.merge_items(broker1.get_items_since( - broker2.get_sync(id), 1000), id) - items = broker2.get_items_since(-1, 1000) - self.assertEquals(len(items), 3) - self.assertEquals(['a', 'b', 'c'], - sorted([rec['name'] for rec in items])) - - -def premetadata_create_account_stat_table(self, conn, put_timestamp): - """ - Copied from swift.common.db.AccountBroker before the metadata column was - added; used for testing with TestAccountBrokerBeforeMetadata. - - Create account_stat table which is specific to the account DB. - - :param conn: DB connection object - :param put_timestamp: put timestamp - """ - conn.executescript(''' - CREATE TABLE account_stat ( - account TEXT, - created_at TEXT, - put_timestamp TEXT DEFAULT '0', - delete_timestamp TEXT DEFAULT '0', - container_count INTEGER, - object_count INTEGER DEFAULT 0, - bytes_used INTEGER DEFAULT 0, - hash TEXT default '00000000000000000000000000000000', - id TEXT, - status TEXT DEFAULT '', - status_changed_at TEXT DEFAULT '0' - ); - - INSERT INTO account_stat (container_count) VALUES (0); - ''') - - conn.execute(''' - UPDATE account_stat SET account = ?, created_at = ?, id = ?, - put_timestamp = ? - ''', (self.account, normalize_timestamp(time()), str(uuid4()), - put_timestamp)) - - -class TestAccountBrokerBeforeMetadata(TestAccountBroker): - """ - Tests for swift.common.db.AccountBroker against databases created before - the metadata column was added. - """ - - def setUp(self): - self._imported_create_account_stat_table = \ - AccountBroker.create_account_stat_table - AccountBroker.create_account_stat_table = \ - premetadata_create_account_stat_table - broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) - exc = None - with broker.get() as conn: - try: - conn.execute('SELECT metadata FROM account_stat') - except BaseException as err: - exc = err - self.assert_('no such column: metadata' in str(exc)) - - def tearDown(self): - AccountBroker.create_account_stat_table = \ - self._imported_create_account_stat_table - broker = AccountBroker(':memory:', account='a') - broker.initialize(normalize_timestamp('1')) - with broker.get() as conn: - conn.execute('SELECT metadata FROM account_stat') - - if __name__ == '__main__': unittest.main() diff --git a/test/unit/container/test_backend.py b/test/unit/container/test_backend.py new file mode 100644 index 0000000000..3bda3ccef3 --- /dev/null +++ b/test/unit/container/test_backend.py @@ -0,0 +1,1205 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Tests for swift.container.backend """ + +from __future__ import with_statement +import hashlib +import unittest +from time import sleep, time +from uuid import uuid4 + +from swift.container.backend import ContainerBroker +from swift.common.utils import normalize_timestamp + + +class TestContainerBroker(unittest.TestCase): + """Tests for ContainerBroker""" + + def test_creation(self): + # Test ContainerBroker.__init__ + broker = ContainerBroker(':memory:', account='a', container='c') + self.assertEqual(broker.db_file, ':memory:') + broker.initialize(normalize_timestamp('1')) + with broker.get() as conn: + curs = conn.cursor() + curs.execute('SELECT 1') + self.assertEqual(curs.fetchall()[0][0], 1) + + def test_exception(self): + # Test ContainerBroker throwing a conn away after + # unhandled exception + first_conn = None + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + with broker.get() as conn: + first_conn = conn + try: + with broker.get() as conn: + self.assertEquals(first_conn, conn) + raise Exception('OMG') + except Exception: + pass + self.assert_(broker.conn is None) + + def test_empty(self): + # Test ContainerBroker.empty + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + self.assert_(broker.empty()) + broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + self.assert_(not broker.empty()) + sleep(.00001) + broker.delete_object('o', normalize_timestamp(time())) + self.assert_(broker.empty()) + + def test_reclaim(self): + broker = ContainerBroker(':memory:', account='test_account', + container='test_container') + broker.initialize(normalize_timestamp('1')) + broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 0").fetchone()[0], 1) + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 1").fetchone()[0], 0) + broker.reclaim(normalize_timestamp(time() - 999), time()) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 0").fetchone()[0], 1) + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 1").fetchone()[0], 0) + sleep(.00001) + broker.delete_object('o', normalize_timestamp(time())) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 0").fetchone()[0], 0) + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 1").fetchone()[0], 1) + broker.reclaim(normalize_timestamp(time() - 999), time()) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 0").fetchone()[0], 0) + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 1").fetchone()[0], 1) + sleep(.00001) + broker.reclaim(normalize_timestamp(time()), time()) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 0").fetchone()[0], 0) + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 1").fetchone()[0], 0) + # Test the return values of reclaim() + broker.put_object('w', normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('x', normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('y', normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('z', normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + # Test before deletion + broker.reclaim(normalize_timestamp(time()), time()) + broker.delete_db(normalize_timestamp(time())) + + def test_delete_object(self): + # Test ContainerBroker.delete_object + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 0").fetchone()[0], 1) + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 1").fetchone()[0], 0) + sleep(.00001) + broker.delete_object('o', normalize_timestamp(time())) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 0").fetchone()[0], 0) + self.assertEquals(conn.execute( + "SELECT count(*) FROM object " + "WHERE deleted = 1").fetchone()[0], 1) + + def test_put_object(self): + # Test ContainerBroker.put_object + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + + # Create initial object + timestamp = normalize_timestamp(time()) + broker.put_object('"{}"', timestamp, 123, + 'application/x-test', + '5af83e3196bf99f440f31f2e1a6c9afe') + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM object").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT created_at FROM object").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT size FROM object").fetchone()[0], 123) + self.assertEquals(conn.execute( + "SELECT content_type FROM object").fetchone()[0], + 'application/x-test') + self.assertEquals(conn.execute( + "SELECT etag FROM object").fetchone()[0], + '5af83e3196bf99f440f31f2e1a6c9afe') + self.assertEquals(conn.execute( + "SELECT deleted FROM object").fetchone()[0], 0) + + # Reput same event + broker.put_object('"{}"', timestamp, 123, + 'application/x-test', + '5af83e3196bf99f440f31f2e1a6c9afe') + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM object").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT created_at FROM object").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT size FROM object").fetchone()[0], 123) + self.assertEquals(conn.execute( + "SELECT content_type FROM object").fetchone()[0], + 'application/x-test') + self.assertEquals(conn.execute( + "SELECT etag FROM object").fetchone()[0], + '5af83e3196bf99f440f31f2e1a6c9afe') + self.assertEquals(conn.execute( + "SELECT deleted FROM object").fetchone()[0], 0) + + # Put new event + sleep(.00001) + timestamp = normalize_timestamp(time()) + broker.put_object('"{}"', timestamp, 124, + 'application/x-test', + 'aa0749bacbc79ec65fe206943d8fe449') + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM object").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT created_at FROM object").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT size FROM object").fetchone()[0], 124) + self.assertEquals(conn.execute( + "SELECT content_type FROM object").fetchone()[0], + 'application/x-test') + self.assertEquals(conn.execute( + "SELECT etag FROM object").fetchone()[0], + 'aa0749bacbc79ec65fe206943d8fe449') + self.assertEquals(conn.execute( + "SELECT deleted FROM object").fetchone()[0], 0) + + # Put old event + otimestamp = normalize_timestamp(float(timestamp) - 1) + broker.put_object('"{}"', otimestamp, 124, + 'application/x-test', + 'aa0749bacbc79ec65fe206943d8fe449') + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM object").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT created_at FROM object").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT size FROM object").fetchone()[0], 124) + self.assertEquals(conn.execute( + "SELECT content_type FROM object").fetchone()[0], + 'application/x-test') + self.assertEquals(conn.execute( + "SELECT etag FROM object").fetchone()[0], + 'aa0749bacbc79ec65fe206943d8fe449') + self.assertEquals(conn.execute( + "SELECT deleted FROM object").fetchone()[0], 0) + + # Put old delete event + dtimestamp = normalize_timestamp(float(timestamp) - 1) + broker.put_object('"{}"', dtimestamp, 0, '', '', + deleted=1) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM object").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT created_at FROM object").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT size FROM object").fetchone()[0], 124) + self.assertEquals(conn.execute( + "SELECT content_type FROM object").fetchone()[0], + 'application/x-test') + self.assertEquals(conn.execute( + "SELECT etag FROM object").fetchone()[0], + 'aa0749bacbc79ec65fe206943d8fe449') + self.assertEquals(conn.execute( + "SELECT deleted FROM object").fetchone()[0], 0) + + # Put new delete event + sleep(.00001) + timestamp = normalize_timestamp(time()) + broker.put_object('"{}"', timestamp, 0, '', '', + deleted=1) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM object").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT created_at FROM object").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT deleted FROM object").fetchone()[0], 1) + + # Put new event + sleep(.00001) + timestamp = normalize_timestamp(time()) + broker.put_object('"{}"', timestamp, 123, + 'application/x-test', + '5af83e3196bf99f440f31f2e1a6c9afe') + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM object").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT created_at FROM object").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT size FROM object").fetchone()[0], 123) + self.assertEquals(conn.execute( + "SELECT content_type FROM object").fetchone()[0], + 'application/x-test') + self.assertEquals(conn.execute( + "SELECT etag FROM object").fetchone()[0], + '5af83e3196bf99f440f31f2e1a6c9afe') + self.assertEquals(conn.execute( + "SELECT deleted FROM object").fetchone()[0], 0) + + # We'll use this later + sleep(.0001) + in_between_timestamp = normalize_timestamp(time()) + + # New post event + sleep(.0001) + previous_timestamp = timestamp + timestamp = normalize_timestamp(time()) + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM object").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT created_at FROM object").fetchone()[0], + previous_timestamp) + self.assertEquals(conn.execute( + "SELECT size FROM object").fetchone()[0], 123) + self.assertEquals(conn.execute( + "SELECT content_type FROM object").fetchone()[0], + 'application/x-test') + self.assertEquals(conn.execute( + "SELECT etag FROM object").fetchone()[0], + '5af83e3196bf99f440f31f2e1a6c9afe') + self.assertEquals(conn.execute( + "SELECT deleted FROM object").fetchone()[0], 0) + + # Put event from after last put but before last post + timestamp = in_between_timestamp + broker.put_object('"{}"', timestamp, 456, + 'application/x-test3', + '6af83e3196bf99f440f31f2e1a6c9afe') + with broker.get() as conn: + self.assertEquals(conn.execute( + "SELECT name FROM object").fetchone()[0], + '"{}"') + self.assertEquals(conn.execute( + "SELECT created_at FROM object").fetchone()[0], timestamp) + self.assertEquals(conn.execute( + "SELECT size FROM object").fetchone()[0], 456) + self.assertEquals(conn.execute( + "SELECT content_type FROM object").fetchone()[0], + 'application/x-test3') + self.assertEquals(conn.execute( + "SELECT etag FROM object").fetchone()[0], + '6af83e3196bf99f440f31f2e1a6c9afe') + self.assertEquals(conn.execute( + "SELECT deleted FROM object").fetchone()[0], 0) + + def test_get_info(self): + # Test ContainerBroker.get_info + broker = ContainerBroker(':memory:', account='test1', + container='test2') + broker.initialize(normalize_timestamp('1')) + + info = broker.get_info() + self.assertEquals(info['account'], 'test1') + self.assertEquals(info['container'], 'test2') + self.assertEquals(info['hash'], '00000000000000000000000000000000') + + info = broker.get_info() + self.assertEquals(info['object_count'], 0) + self.assertEquals(info['bytes_used'], 0) + + broker.put_object('o1', normalize_timestamp(time()), 123, 'text/plain', + '5af83e3196bf99f440f31f2e1a6c9afe') + info = broker.get_info() + self.assertEquals(info['object_count'], 1) + self.assertEquals(info['bytes_used'], 123) + + sleep(.00001) + broker.put_object('o2', normalize_timestamp(time()), 123, 'text/plain', + '5af83e3196bf99f440f31f2e1a6c9afe') + info = broker.get_info() + self.assertEquals(info['object_count'], 2) + self.assertEquals(info['bytes_used'], 246) + + sleep(.00001) + broker.put_object('o2', normalize_timestamp(time()), 1000, + 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') + info = broker.get_info() + self.assertEquals(info['object_count'], 2) + self.assertEquals(info['bytes_used'], 1123) + + sleep(.00001) + broker.delete_object('o1', normalize_timestamp(time())) + info = broker.get_info() + self.assertEquals(info['object_count'], 1) + self.assertEquals(info['bytes_used'], 1000) + + sleep(.00001) + broker.delete_object('o2', normalize_timestamp(time())) + info = broker.get_info() + self.assertEquals(info['object_count'], 0) + self.assertEquals(info['bytes_used'], 0) + + info = broker.get_info() + self.assertEquals(info['x_container_sync_point1'], -1) + self.assertEquals(info['x_container_sync_point2'], -1) + + def test_set_x_syncs(self): + broker = ContainerBroker(':memory:', account='test1', + container='test2') + broker.initialize(normalize_timestamp('1')) + + info = broker.get_info() + self.assertEquals(info['x_container_sync_point1'], -1) + self.assertEquals(info['x_container_sync_point2'], -1) + + broker.set_x_container_sync_points(1, 2) + info = broker.get_info() + self.assertEquals(info['x_container_sync_point1'], 1) + self.assertEquals(info['x_container_sync_point2'], 2) + + def test_get_report_info(self): + broker = ContainerBroker(':memory:', account='test1', + container='test2') + broker.initialize(normalize_timestamp('1')) + + info = broker.get_info() + self.assertEquals(info['account'], 'test1') + self.assertEquals(info['container'], 'test2') + self.assertEquals(info['object_count'], 0) + self.assertEquals(info['bytes_used'], 0) + self.assertEquals(info['reported_object_count'], 0) + self.assertEquals(info['reported_bytes_used'], 0) + + broker.put_object('o1', normalize_timestamp(time()), 123, 'text/plain', + '5af83e3196bf99f440f31f2e1a6c9afe') + info = broker.get_info() + self.assertEquals(info['object_count'], 1) + self.assertEquals(info['bytes_used'], 123) + self.assertEquals(info['reported_object_count'], 0) + self.assertEquals(info['reported_bytes_used'], 0) + + sleep(.00001) + broker.put_object('o2', normalize_timestamp(time()), 123, 'text/plain', + '5af83e3196bf99f440f31f2e1a6c9afe') + info = broker.get_info() + self.assertEquals(info['object_count'], 2) + self.assertEquals(info['bytes_used'], 246) + self.assertEquals(info['reported_object_count'], 0) + self.assertEquals(info['reported_bytes_used'], 0) + + sleep(.00001) + broker.put_object('o2', normalize_timestamp(time()), 1000, + 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') + info = broker.get_info() + self.assertEquals(info['object_count'], 2) + self.assertEquals(info['bytes_used'], 1123) + self.assertEquals(info['reported_object_count'], 0) + self.assertEquals(info['reported_bytes_used'], 0) + + put_timestamp = normalize_timestamp(time()) + sleep(.001) + delete_timestamp = normalize_timestamp(time()) + broker.reported(put_timestamp, delete_timestamp, 2, 1123) + info = broker.get_info() + self.assertEquals(info['object_count'], 2) + self.assertEquals(info['bytes_used'], 1123) + self.assertEquals(info['reported_put_timestamp'], put_timestamp) + self.assertEquals(info['reported_delete_timestamp'], delete_timestamp) + self.assertEquals(info['reported_object_count'], 2) + self.assertEquals(info['reported_bytes_used'], 1123) + + sleep(.00001) + broker.delete_object('o1', normalize_timestamp(time())) + info = broker.get_info() + self.assertEquals(info['object_count'], 1) + self.assertEquals(info['bytes_used'], 1000) + self.assertEquals(info['reported_object_count'], 2) + self.assertEquals(info['reported_bytes_used'], 1123) + + sleep(.00001) + broker.delete_object('o2', normalize_timestamp(time())) + info = broker.get_info() + self.assertEquals(info['object_count'], 0) + self.assertEquals(info['bytes_used'], 0) + self.assertEquals(info['reported_object_count'], 2) + self.assertEquals(info['reported_bytes_used'], 1123) + + def test_list_objects_iter(self): + # Test ContainerBroker.list_objects_iter + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + for obj1 in xrange(4): + for obj2 in xrange(125): + broker.put_object('%d/%04d' % (obj1, obj2), + normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + for obj in xrange(125): + broker.put_object('2/0051/%04d' % obj, + normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + + for obj in xrange(125): + broker.put_object('3/%04d/0049' % obj, + normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + + listing = broker.list_objects_iter(100, '', None, None, '') + self.assertEquals(len(listing), 100) + self.assertEquals(listing[0][0], '0/0000') + self.assertEquals(listing[-1][0], '0/0099') + + listing = broker.list_objects_iter(100, '', '0/0050', None, '') + self.assertEquals(len(listing), 50) + self.assertEquals(listing[0][0], '0/0000') + self.assertEquals(listing[-1][0], '0/0049') + + listing = broker.list_objects_iter(100, '0/0099', None, None, '') + self.assertEquals(len(listing), 100) + self.assertEquals(listing[0][0], '0/0100') + self.assertEquals(listing[-1][0], '1/0074') + + listing = broker.list_objects_iter(55, '1/0074', None, None, '') + self.assertEquals(len(listing), 55) + self.assertEquals(listing[0][0], '1/0075') + self.assertEquals(listing[-1][0], '2/0004') + + listing = broker.list_objects_iter(10, '', None, '0/01', '') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0/0100') + self.assertEquals(listing[-1][0], '0/0109') + + listing = broker.list_objects_iter(10, '', None, '0/', '/') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0/0000') + self.assertEquals(listing[-1][0], '0/0009') + + # Same as above, but using the path argument. + listing = broker.list_objects_iter(10, '', None, None, '', '0') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0/0000') + self.assertEquals(listing[-1][0], '0/0009') + + listing = broker.list_objects_iter(10, '', None, '', '/') + self.assertEquals(len(listing), 4) + self.assertEquals([row[0] for row in listing], + ['0/', '1/', '2/', '3/']) + + listing = broker.list_objects_iter(10, '2', None, None, '/') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], ['2/', '3/']) + + listing = broker.list_objects_iter(10, '2/', None, None, '/') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], ['3/']) + + listing = broker.list_objects_iter(10, '2/0050', None, '2/', '/') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '2/0051') + self.assertEquals(listing[1][0], '2/0051/') + self.assertEquals(listing[2][0], '2/0052') + self.assertEquals(listing[-1][0], '2/0059') + + listing = broker.list_objects_iter(10, '3/0045', None, '3/', '/') + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['3/0045/', '3/0046', '3/0046/', '3/0047', + '3/0047/', '3/0048', '3/0048/', '3/0049', + '3/0049/', '3/0050']) + + broker.put_object('3/0049/', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + listing = broker.list_objects_iter(10, '3/0048', None, None, None) + self.assertEquals(len(listing), 10) + self.assertEquals( + [row[0] for row in listing], + ['3/0048/0049', '3/0049', '3/0049/', + '3/0049/0049', '3/0050', '3/0050/0049', '3/0051', '3/0051/0049', + '3/0052', '3/0052/0049']) + + listing = broker.list_objects_iter(10, '3/0048', None, '3/', '/') + self.assertEquals(len(listing), 10) + self.assertEquals( + [row[0] for row in listing], + ['3/0048/', '3/0049', '3/0049/', '3/0050', + '3/0050/', '3/0051', '3/0051/', '3/0052', '3/0052/', '3/0053']) + + listing = broker.list_objects_iter(10, None, None, '3/0049/', '/') + self.assertEquals(len(listing), 2) + self.assertEquals( + [row[0] for row in listing], + ['3/0049/', '3/0049/0049']) + + listing = broker.list_objects_iter(10, None, None, None, None, + '3/0049') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], ['3/0049/0049']) + + listing = broker.list_objects_iter(2, None, None, '3/', '/') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], ['3/0000', '3/0000/']) + + listing = broker.list_objects_iter(2, None, None, None, None, '3') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], ['3/0000', '3/0001']) + + def test_list_objects_iter_non_slash(self): + # Test ContainerBroker.list_objects_iter using a + # delimiter that is not a slash + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + for obj1 in xrange(4): + for obj2 in xrange(125): + broker.put_object('%d:%04d' % (obj1, obj2), + normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + for obj in xrange(125): + broker.put_object('2:0051:%04d' % obj, + normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + + for obj in xrange(125): + broker.put_object('3:%04d:0049' % obj, + normalize_timestamp(time()), 0, 'text/plain', + 'd41d8cd98f00b204e9800998ecf8427e') + + listing = broker.list_objects_iter(100, '', None, None, '') + self.assertEquals(len(listing), 100) + self.assertEquals(listing[0][0], '0:0000') + self.assertEquals(listing[-1][0], '0:0099') + + listing = broker.list_objects_iter(100, '', '0:0050', None, '') + self.assertEquals(len(listing), 50) + self.assertEquals(listing[0][0], '0:0000') + self.assertEquals(listing[-1][0], '0:0049') + + listing = broker.list_objects_iter(100, '0:0099', None, None, '') + self.assertEquals(len(listing), 100) + self.assertEquals(listing[0][0], '0:0100') + self.assertEquals(listing[-1][0], '1:0074') + + listing = broker.list_objects_iter(55, '1:0074', None, None, '') + self.assertEquals(len(listing), 55) + self.assertEquals(listing[0][0], '1:0075') + self.assertEquals(listing[-1][0], '2:0004') + + listing = broker.list_objects_iter(10, '', None, '0:01', '') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0:0100') + self.assertEquals(listing[-1][0], '0:0109') + + listing = broker.list_objects_iter(10, '', None, '0:', ':') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '0:0000') + self.assertEquals(listing[-1][0], '0:0009') + + # Same as above, but using the path argument, so nothing should be + # returned since path uses a '/' as a delimiter. + listing = broker.list_objects_iter(10, '', None, None, '', '0') + self.assertEquals(len(listing), 0) + + listing = broker.list_objects_iter(10, '', None, '', ':') + self.assertEquals(len(listing), 4) + self.assertEquals([row[0] for row in listing], + ['0:', '1:', '2:', '3:']) + + listing = broker.list_objects_iter(10, '2', None, None, ':') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], ['2:', '3:']) + + listing = broker.list_objects_iter(10, '2:', None, None, ':') + self.assertEquals(len(listing), 1) + self.assertEquals([row[0] for row in listing], ['3:']) + + listing = broker.list_objects_iter(10, '2:0050', None, '2:', ':') + self.assertEquals(len(listing), 10) + self.assertEquals(listing[0][0], '2:0051') + self.assertEquals(listing[1][0], '2:0051:') + self.assertEquals(listing[2][0], '2:0052') + self.assertEquals(listing[-1][0], '2:0059') + + listing = broker.list_objects_iter(10, '3:0045', None, '3:', ':') + self.assertEquals(len(listing), 10) + self.assertEquals([row[0] for row in listing], + ['3:0045:', '3:0046', '3:0046:', '3:0047', + '3:0047:', '3:0048', '3:0048:', '3:0049', + '3:0049:', '3:0050']) + + broker.put_object('3:0049:', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + listing = broker.list_objects_iter(10, '3:0048', None, None, None) + self.assertEquals(len(listing), 10) + self.assertEquals( + [row[0] for row in listing], + ['3:0048:0049', '3:0049', '3:0049:', + '3:0049:0049', '3:0050', '3:0050:0049', '3:0051', '3:0051:0049', + '3:0052', '3:0052:0049']) + + listing = broker.list_objects_iter(10, '3:0048', None, '3:', ':') + self.assertEquals(len(listing), 10) + self.assertEquals( + [row[0] for row in listing], + ['3:0048:', '3:0049', '3:0049:', '3:0050', + '3:0050:', '3:0051', '3:0051:', '3:0052', '3:0052:', '3:0053']) + + listing = broker.list_objects_iter(10, None, None, '3:0049:', ':') + self.assertEquals(len(listing), 2) + self.assertEquals( + [row[0] for row in listing], + ['3:0049:', '3:0049:0049']) + + # Same as above, but using the path argument, so nothing should be + # returned since path uses a '/' as a delimiter. + listing = broker.list_objects_iter(10, None, None, None, None, + '3:0049') + self.assertEquals(len(listing), 0) + + listing = broker.list_objects_iter(2, None, None, '3:', ':') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], ['3:0000', '3:0000:']) + + listing = broker.list_objects_iter(2, None, None, None, None, '3') + self.assertEquals(len(listing), 0) + + def test_list_objects_iter_prefix_delim(self): + # Test ContainerBroker.list_objects_iter + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + + broker.put_object( + '/pets/dogs/1', normalize_timestamp(0), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object( + '/pets/dogs/2', normalize_timestamp(0), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object( + '/pets/fish/a', normalize_timestamp(0), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object( + '/pets/fish/b', normalize_timestamp(0), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object( + '/pets/fish_info.txt', normalize_timestamp(0), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object( + '/snakes', normalize_timestamp(0), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + + #def list_objects_iter(self, limit, marker, prefix, delimiter, + # path=None, format=None): + listing = broker.list_objects_iter(100, None, None, '/pets/f', '/') + self.assertEquals([row[0] for row in listing], + ['/pets/fish/', '/pets/fish_info.txt']) + listing = broker.list_objects_iter(100, None, None, '/pets/fish', '/') + self.assertEquals([row[0] for row in listing], + ['/pets/fish/', '/pets/fish_info.txt']) + listing = broker.list_objects_iter(100, None, None, '/pets/fish/', '/') + self.assertEquals([row[0] for row in listing], + ['/pets/fish/a', '/pets/fish/b']) + + def test_double_check_trailing_delimiter(self): + # Test ContainerBroker.list_objects_iter for a + # container that has an odd file with a trailing delimiter + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + broker.put_object('a', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('a/', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('a/a', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('a/a/a', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('a/a/b', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('a/b', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('b', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('b/a', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('b/b', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('c', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('a/0', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('0', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('0/', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('00', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('0/0', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('0/00', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('0/1', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('0/1/', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('0/1/0', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('1', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('1/', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('1/0', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + listing = broker.list_objects_iter(25, None, None, None, None) + self.assertEquals(len(listing), 22) + self.assertEquals( + [row[0] for row in listing], + ['0', '0/', '0/0', '0/00', '0/1', '0/1/', '0/1/0', '00', '1', '1/', + '1/0', 'a', 'a/', 'a/0', 'a/a', 'a/a/a', 'a/a/b', 'a/b', 'b', + 'b/a', 'b/b', 'c']) + listing = broker.list_objects_iter(25, None, None, '', '/') + self.assertEquals(len(listing), 10) + self.assertEquals( + [row[0] for row in listing], + ['0', '0/', '00', '1', '1/', 'a', 'a/', 'b', 'b/', 'c']) + listing = broker.list_objects_iter(25, None, None, 'a/', '/') + self.assertEquals(len(listing), 5) + self.assertEquals( + [row[0] for row in listing], + ['a/', 'a/0', 'a/a', 'a/a/', 'a/b']) + listing = broker.list_objects_iter(25, None, None, '0/', '/') + self.assertEquals(len(listing), 5) + self.assertEquals( + [row[0] for row in listing], + ['0/', '0/0', '0/00', '0/1', '0/1/']) + listing = broker.list_objects_iter(25, None, None, '0/1/', '/') + self.assertEquals(len(listing), 2) + self.assertEquals( + [row[0] for row in listing], + ['0/1/', '0/1/0']) + listing = broker.list_objects_iter(25, None, None, 'b/', '/') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], ['b/a', 'b/b']) + + def test_double_check_trailing_delimiter_non_slash(self): + # Test ContainerBroker.list_objects_iter for a + # container that has an odd file with a trailing delimiter + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + broker.put_object('a', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('a:', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('a:a', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('a:a:a', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('a:a:b', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('a:b', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('b', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('b:a', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('b:b', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('c', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('a:0', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('0', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('0:', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('00', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('0:0', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('0:00', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('0:1', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('0:1:', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('0:1:0', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('1', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('1:', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('1:0', normalize_timestamp(time()), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + listing = broker.list_objects_iter(25, None, None, None, None) + self.assertEquals(len(listing), 22) + self.assertEquals( + [row[0] for row in listing], + ['0', '00', '0:', '0:0', '0:00', '0:1', '0:1:', '0:1:0', '1', '1:', + '1:0', 'a', 'a:', 'a:0', 'a:a', 'a:a:a', 'a:a:b', 'a:b', 'b', + 'b:a', 'b:b', 'c']) + listing = broker.list_objects_iter(25, None, None, '', ':') + self.assertEquals(len(listing), 10) + self.assertEquals( + [row[0] for row in listing], + ['0', '00', '0:', '1', '1:', 'a', 'a:', 'b', 'b:', 'c']) + listing = broker.list_objects_iter(25, None, None, 'a:', ':') + self.assertEquals(len(listing), 5) + self.assertEquals( + [row[0] for row in listing], + ['a:', 'a:0', 'a:a', 'a:a:', 'a:b']) + listing = broker.list_objects_iter(25, None, None, '0:', ':') + self.assertEquals(len(listing), 5) + self.assertEquals( + [row[0] for row in listing], + ['0:', '0:0', '0:00', '0:1', '0:1:']) + listing = broker.list_objects_iter(25, None, None, '0:1:', ':') + self.assertEquals(len(listing), 2) + self.assertEquals( + [row[0] for row in listing], + ['0:1:', '0:1:0']) + listing = broker.list_objects_iter(25, None, None, 'b:', ':') + self.assertEquals(len(listing), 2) + self.assertEquals([row[0] for row in listing], ['b:a', 'b:b']) + + def test_chexor(self): + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + broker.put_object('a', normalize_timestamp(1), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker.put_object('b', normalize_timestamp(2), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + hasha = hashlib.md5('%s-%s' % ('a', '0000000001.00000')).digest() + hashb = hashlib.md5('%s-%s' % ('b', '0000000002.00000')).digest() + hashc = ''.join( + ('%2x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) + self.assertEquals(broker.get_info()['hash'], hashc) + broker.put_object('b', normalize_timestamp(3), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + hashb = hashlib.md5('%s-%s' % ('b', '0000000003.00000')).digest() + hashc = ''.join( + ('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) + self.assertEquals(broker.get_info()['hash'], hashc) + + def test_newid(self): + # test DatabaseBroker.newid + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + id = broker.get_info()['id'] + broker.newid('someid') + self.assertNotEquals(id, broker.get_info()['id']) + + def test_get_items_since(self): + # test DatabaseBroker.get_items_since + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + broker.put_object('a', normalize_timestamp(1), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + max_row = broker.get_replication_info()['max_row'] + broker.put_object('b', normalize_timestamp(2), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + items = broker.get_items_since(max_row, 1000) + self.assertEquals(len(items), 1) + self.assertEquals(items[0]['name'], 'b') + + def test_sync_merging(self): + # exercise the DatabaseBroker sync functions a bit + broker1 = ContainerBroker(':memory:', account='a', container='c') + broker1.initialize(normalize_timestamp('1')) + broker2 = ContainerBroker(':memory:', account='a', container='c') + broker2.initialize(normalize_timestamp('1')) + self.assertEquals(broker2.get_sync('12345'), -1) + broker1.merge_syncs([{'sync_point': 3, 'remote_id': '12345'}]) + broker2.merge_syncs(broker1.get_syncs()) + self.assertEquals(broker2.get_sync('12345'), 3) + + def test_merge_items(self): + broker1 = ContainerBroker(':memory:', account='a', container='c') + broker1.initialize(normalize_timestamp('1')) + broker2 = ContainerBroker(':memory:', account='a', container='c') + broker2.initialize(normalize_timestamp('1')) + broker1.put_object('a', normalize_timestamp(1), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker1.put_object('b', normalize_timestamp(2), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + id = broker1.get_info()['id'] + broker2.merge_items(broker1.get_items_since( + broker2.get_sync(id), 1000), id) + items = broker2.get_items_since(-1, 1000) + self.assertEquals(len(items), 2) + self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) + broker1.put_object('c', normalize_timestamp(3), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker2.merge_items(broker1.get_items_since( + broker2.get_sync(id), 1000), id) + items = broker2.get_items_since(-1, 1000) + self.assertEquals(len(items), 3) + self.assertEquals(['a', 'b', 'c'], + sorted([rec['name'] for rec in items])) + + def test_merge_items_overwrite(self): + # test DatabaseBroker.merge_items + broker1 = ContainerBroker(':memory:', account='a', container='c') + broker1.initialize(normalize_timestamp('1')) + id = broker1.get_info()['id'] + broker2 = ContainerBroker(':memory:', account='a', container='c') + broker2.initialize(normalize_timestamp('1')) + broker1.put_object('a', normalize_timestamp(2), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker1.put_object('b', normalize_timestamp(3), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker2.merge_items(broker1.get_items_since( + broker2.get_sync(id), 1000), id) + broker1.put_object('a', normalize_timestamp(4), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker2.merge_items(broker1.get_items_since( + broker2.get_sync(id), 1000), id) + items = broker2.get_items_since(-1, 1000) + self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) + for rec in items: + if rec['name'] == 'a': + self.assertEquals(rec['created_at'], normalize_timestamp(4)) + if rec['name'] == 'b': + self.assertEquals(rec['created_at'], normalize_timestamp(3)) + + def test_merge_items_post_overwrite_out_of_order(self): + # test DatabaseBroker.merge_items + broker1 = ContainerBroker(':memory:', account='a', container='c') + broker1.initialize(normalize_timestamp('1')) + id = broker1.get_info()['id'] + broker2 = ContainerBroker(':memory:', account='a', container='c') + broker2.initialize(normalize_timestamp('1')) + broker1.put_object('a', normalize_timestamp(2), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker1.put_object('b', normalize_timestamp(3), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker2.merge_items(broker1.get_items_since( + broker2.get_sync(id), 1000), id) + broker1.put_object('a', normalize_timestamp(4), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker2.merge_items(broker1.get_items_since( + broker2.get_sync(id), 1000), id) + items = broker2.get_items_since(-1, 1000) + self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) + for rec in items: + if rec['name'] == 'a': + self.assertEquals(rec['created_at'], normalize_timestamp(4)) + if rec['name'] == 'b': + self.assertEquals(rec['created_at'], normalize_timestamp(3)) + self.assertEquals(rec['content_type'], 'text/plain') + items = broker2.get_items_since(-1, 1000) + self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) + for rec in items: + if rec['name'] == 'a': + self.assertEquals(rec['created_at'], normalize_timestamp(4)) + if rec['name'] == 'b': + self.assertEquals(rec['created_at'], normalize_timestamp(3)) + broker1.put_object('b', normalize_timestamp(5), 0, + 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') + broker2.merge_items(broker1.get_items_since( + broker2.get_sync(id), 1000), id) + items = broker2.get_items_since(-1, 1000) + self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) + for rec in items: + if rec['name'] == 'a': + self.assertEquals(rec['created_at'], normalize_timestamp(4)) + if rec['name'] == 'b': + self.assertEquals(rec['created_at'], normalize_timestamp(5)) + self.assertEquals(rec['content_type'], 'text/plain') + + +def premetadata_create_container_stat_table(self, conn, put_timestamp=None): + """ + Copied from ContainerBroker before the metadata column was + added; used for testing with TestContainerBrokerBeforeMetadata. + + Create the container_stat table which is specifc to the container DB. + + :param conn: DB connection object + :param put_timestamp: put timestamp + """ + if put_timestamp is None: + put_timestamp = normalize_timestamp(0) + conn.executescript(''' + CREATE TABLE container_stat ( + account TEXT, + container TEXT, + created_at TEXT, + put_timestamp TEXT DEFAULT '0', + delete_timestamp TEXT DEFAULT '0', + object_count INTEGER, + bytes_used INTEGER, + reported_put_timestamp TEXT DEFAULT '0', + reported_delete_timestamp TEXT DEFAULT '0', + reported_object_count INTEGER DEFAULT 0, + reported_bytes_used INTEGER DEFAULT 0, + hash TEXT default '00000000000000000000000000000000', + id TEXT, + status TEXT DEFAULT '', + status_changed_at TEXT DEFAULT '0' + ); + + INSERT INTO container_stat (object_count, bytes_used) + VALUES (0, 0); + ''') + conn.execute(''' + UPDATE container_stat + SET account = ?, container = ?, created_at = ?, id = ?, + put_timestamp = ? + ''', (self.account, self.container, normalize_timestamp(time()), + str(uuid4()), put_timestamp)) + + +class TestContainerBrokerBeforeMetadata(TestContainerBroker): + """ + Tests for ContainerBroker against databases created before + the metadata column was added. + """ + + def setUp(self): + self._imported_create_container_stat_table = \ + ContainerBroker.create_container_stat_table + ContainerBroker.create_container_stat_table = \ + premetadata_create_container_stat_table + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + exc = None + with broker.get() as conn: + try: + conn.execute('SELECT metadata FROM container_stat') + except BaseException as err: + exc = err + self.assert_('no such column: metadata' in str(exc)) + + def tearDown(self): + ContainerBroker.create_container_stat_table = \ + self._imported_create_container_stat_table + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + with broker.get() as conn: + conn.execute('SELECT metadata FROM container_stat') + + +def prexsync_create_container_stat_table(self, conn, put_timestamp=None): + """ + Copied from ContainerBroker before the + x_container_sync_point[12] columns were added; used for testing with + TestContainerBrokerBeforeXSync. + + Create the container_stat table which is specifc to the container DB. + + :param conn: DB connection object + :param put_timestamp: put timestamp + """ + if put_timestamp is None: + put_timestamp = normalize_timestamp(0) + conn.executescript(""" + CREATE TABLE container_stat ( + account TEXT, + container TEXT, + created_at TEXT, + put_timestamp TEXT DEFAULT '0', + delete_timestamp TEXT DEFAULT '0', + object_count INTEGER, + bytes_used INTEGER, + reported_put_timestamp TEXT DEFAULT '0', + reported_delete_timestamp TEXT DEFAULT '0', + reported_object_count INTEGER DEFAULT 0, + reported_bytes_used INTEGER DEFAULT 0, + hash TEXT default '00000000000000000000000000000000', + id TEXT, + status TEXT DEFAULT '', + status_changed_at TEXT DEFAULT '0', + metadata TEXT DEFAULT '' + ); + + INSERT INTO container_stat (object_count, bytes_used) + VALUES (0, 0); + """) + conn.execute(''' + UPDATE container_stat + SET account = ?, container = ?, created_at = ?, id = ?, + put_timestamp = ? + ''', (self.account, self.container, normalize_timestamp(time()), + str(uuid4()), put_timestamp)) + + +class TestContainerBrokerBeforeXSync(TestContainerBroker): + """ + Tests for ContainerBroker against databases created + before the x_container_sync_point[12] columns were added. + """ + + def setUp(self): + self._imported_create_container_stat_table = \ + ContainerBroker.create_container_stat_table + ContainerBroker.create_container_stat_table = \ + prexsync_create_container_stat_table + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + exc = None + with broker.get() as conn: + try: + conn.execute('''SELECT x_container_sync_point1 + FROM container_stat''') + except BaseException as err: + exc = err + self.assert_('no such column: x_container_sync_point1' in str(exc)) + + def tearDown(self): + ContainerBroker.create_container_stat_table = \ + self._imported_create_container_stat_table + broker = ContainerBroker(':memory:', account='a', container='c') + broker.initialize(normalize_timestamp('1')) + with broker.get() as conn: + conn.execute('SELECT x_container_sync_point1 FROM container_stat') diff --git a/test/unit/container/test_updater.py b/test/unit/container/test_updater.py index a7da07b757..6b6030bd86 100644 --- a/test/unit/container/test_updater.py +++ b/test/unit/container/test_updater.py @@ -26,7 +26,7 @@ from swift.common import utils from swift.container import updater as container_updater from swift.container import server as container_server -from swift.common.db import ContainerBroker +from swift.container.backend import ContainerBroker from swift.common.ring import RingData from swift.common.utils import normalize_timestamp From d51e87342331ded54de5424d888d9d143b132a7e Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Fri, 30 Aug 2013 18:08:24 -0700 Subject: [PATCH 05/35] Remove keep_data_fp argument from DiskFile constructor All access to the data_file fp for DiskFile is moved after the new "open" method. This prepares to move some additional smarts into DiskFile and reduce the surface area of the abstraction and the exposure of the underlying implementation in the object-server. Future work: * Consolidate put_metadata to DiskWriter * Add public "update_metdata" method to DiskFile * Create DiskReader class to gather all access of methods under "open" Change-Id: I4de2f265bf099a810c5f1c14b5278d89bd0b382d --- swift/common/exceptions.py | 4 ++ swift/obj/auditor.py | 4 +- swift/obj/diskfile.py | 68 +++++++++++++++------ swift/obj/server.py | 47 ++++++++------- test/unit/obj/test_diskfile.py | 106 ++++++++++++++++++++------------- test/unit/obj/test_server.py | 15 +++-- 6 files changed, 157 insertions(+), 87 deletions(-) diff --git a/swift/common/exceptions.py b/swift/common/exceptions.py index 2f37e7d938..2b9f8ea4b2 100644 --- a/swift/common/exceptions.py +++ b/swift/common/exceptions.py @@ -38,6 +38,10 @@ class DiskFileError(SwiftException): pass +class DiskFileNotOpenError(DiskFileError): + pass + + class DiskFileCollision(DiskFileError): pass diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py index 7468d80ee7..28f39f89da 100644 --- a/swift/obj/auditor.py +++ b/swift/obj/auditor.py @@ -166,8 +166,8 @@ def object_audit(self, path, device, partition): raise AuditException('Error when reading metadata: %s' % exc) _junk, account, container, obj = name.split('/', 3) df = diskfile.DiskFile(self.devices, device, partition, - account, container, obj, self.logger, - keep_data_fp=True) + account, container, obj, self.logger) + df.open() try: try: obj_size = df.get_data_file_size() diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py index 80d806f265..6dc9eb198d 100644 --- a/swift/obj/diskfile.py +++ b/swift/obj/diskfile.py @@ -38,7 +38,7 @@ fdatasync, drop_buffer_cache, ThreadPool, lock_path, write_pickle from swift.common.exceptions import DiskFileError, DiskFileNotExist, \ DiskFileCollision, DiskFileNoSpace, DiskFileDeviceUnavailable, \ - PathNotDir + PathNotDir, DiskFileNotOpenError from swift.common.swob import multi_range_iterator @@ -342,7 +342,6 @@ def put(self, metadata, extension='.data'): self.threadpool.force_run_in_thread( self._finalize_put, metadata, target_path) - self.disk_file.metadata = metadata class DiskFile(object): @@ -355,19 +354,17 @@ class DiskFile(object): :param account: account name for the object :param container: container name for the object :param obj: object name for the object - :param keep_data_fp: if True, don't close the fp, otherwise close it :param disk_chunk_size: size of chunks on file reads :param bytes_per_sync: number of bytes between fdatasync calls :param iter_hook: called when __iter__ returns a chunk :param threadpool: thread pool in which to do blocking operations - - :raises DiskFileCollision: on md5 collision """ def __init__(self, path, device, partition, account, container, obj, - logger, keep_data_fp=False, disk_chunk_size=65536, - bytes_per_sync=(512 * 1024 * 1024), iter_hook=None, - threadpool=None, obj_dir='objects', mount_check=False): + logger, disk_chunk_size=65536, + bytes_per_sync=(512 * 1024 * 1024), + iter_hook=None, threadpool=None, obj_dir='objects', + mount_check=False): if mount_check and not check_mount(path, device): raise DiskFileDeviceUnavailable() self.disk_chunk_size = disk_chunk_size @@ -380,27 +377,50 @@ def __init__(self, path, device, partition, account, container, obj, self.device_path = join(path, device) self.tmpdir = join(path, device, 'tmp') self.logger = logger - self._metadata = {} + self._metadata = None self.data_file = None + self._data_file_size = None self.fp = None self.iter_etag = None self.started_at_0 = False self.read_to_eof = False self.quarantined_dir = None - self.keep_cache = False self.suppress_file_closing = False + self._verify_close = False self.threadpool = threadpool or ThreadPool(nthreads=0) + # FIXME(clayg): this attribute is set after open and affects the + # behavior of the class (i.e. public interface) + self.keep_cache = False + + def open(self, verify_close=False): + """ + Open the file and read the metadata. + + This method must populate the _metadata attribute. + + :param verify_close: force implicit close to verify_file, no effect on + explicit close. + + :raises DiskFileCollision: on md5 collision + """ data_file, meta_file, ts_file = self._get_ondisk_file() if not data_file: if ts_file: self._construct_from_ts_file(ts_file) else: - fp = self._construct_from_data_file(data_file, meta_file) - if keep_data_fp: - self.fp = fp - else: - fp.close() + self.fp = self._construct_from_data_file(data_file, meta_file) + self._verify_close = verify_close + self._metadata = self._metadata or {} + return self + + def __enter__(self): + if self._metadata is None: + raise DiskFileNotOpenError() + return self + + def __exit__(self, t, v, tb): + self.close(verify_file=self._verify_close) def _get_ondisk_file(self): """ @@ -508,6 +528,8 @@ def _construct_from_data_file(self, data_file, meta_file): def __iter__(self): """Returns an iterator over the data file.""" + if self.fp is None: + raise DiskFileNotOpenError() try: dropped_cache = 0 read = 0 @@ -610,6 +632,9 @@ def close(self, verify_file=True): finally: self.fp.close() self.fp = None + self._metadata = None + self._data_file_size = None + self._verify_close = False def get_metadata(self): """ @@ -617,6 +642,8 @@ def get_metadata(self): :returns: object's metadata dictionary """ + if self._metadata is None: + raise DiskFileNotOpenError() return self._metadata def is_deleted(self): @@ -716,13 +743,20 @@ def get_data_file_size(self): :raises DiskFileError: on file size mismatch. :raises DiskFileNotExist: on file not existing (including deleted) """ + if self._data_file_size is None: + self._data_file_size = self._get_data_file_size() + return self._data_file_size + + def _get_data_file_size(self): + # ensure file is opened + metadata = self.get_metadata() try: file_size = 0 if self.data_file: file_size = self.threadpool.run_in_thread( getsize, self.data_file) - if 'Content-Length' in self._metadata: - metadata_size = int(self._metadata['Content-Length']) + if 'Content-Length' in metadata: + metadata_size = int(metadata['Content-Length']) if file_size != metadata_size: raise DiskFileError( 'Content-Length of %s does not match file size ' diff --git a/swift/obj/server.py b/swift/obj/server.py index d93384f0b7..3113628314 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -298,14 +298,15 @@ def POST(self, request): obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) - if disk_file.is_deleted() or disk_file.is_expired(): - return HTTPNotFound(request=request) - try: - disk_file.get_data_file_size() - except (DiskFileError, DiskFileNotExist): - disk_file.quarantine() - return HTTPNotFound(request=request) - orig_metadata = disk_file.get_metadata() + with disk_file.open(): + if disk_file.is_deleted() or disk_file.is_expired(): + return HTTPNotFound(request=request) + try: + disk_file.get_data_file_size() + except (DiskFileError, DiskFileNotExist): + disk_file.quarantine() + return HTTPNotFound(request=request) + orig_metadata = disk_file.get_metadata() orig_timestamp = orig_metadata.get('X-Timestamp', '0') if orig_timestamp >= request.headers['x-timestamp']: return HTTPConflict(request=request) @@ -355,7 +356,8 @@ def PUT(self, request): obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) - orig_metadata = disk_file.get_metadata() + with disk_file.open(): + orig_metadata = disk_file.get_metadata() old_delete_at = int(orig_metadata.get('X-Delete-At') or 0) orig_timestamp = orig_metadata.get('X-Timestamp') if orig_timestamp and orig_timestamp >= request.headers['x-timestamp']: @@ -432,9 +434,10 @@ def GET(self, request): split_and_validate_path(request, 5, 5, True) try: disk_file = self._diskfile(device, partition, account, container, - obj, keep_data_fp=True, iter_hook=sleep) + obj, iter_hook=sleep) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) + disk_file.open() if disk_file.is_deleted() or disk_file.is_expired(): if request.headers.get('if-match') == '*': return HTTPPreconditionFailed(request=request) @@ -510,15 +513,16 @@ def HEAD(self, request): obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) - if disk_file.is_deleted() or disk_file.is_expired(): - return HTTPNotFound(request=request) - try: - file_size = disk_file.get_data_file_size() - except (DiskFileError, DiskFileNotExist): - disk_file.quarantine() - return HTTPNotFound(request=request) + with disk_file.open(): + if disk_file.is_deleted() or disk_file.is_expired(): + return HTTPNotFound(request=request) + try: + file_size = disk_file.get_data_file_size() + except (DiskFileError, DiskFileNotExist): + disk_file.quarantine() + return HTTPNotFound(request=request) + metadata = disk_file.get_metadata() response = Response(request=request, conditional_response=True) - metadata = disk_file.get_metadata() response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): @@ -549,7 +553,10 @@ def DELETE(self, request): obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) - orig_metadata = disk_file.get_metadata() + with disk_file.open(): + orig_metadata = disk_file.get_metadata() + is_deleted = disk_file.is_deleted() + is_expired = disk_file.is_expired() if 'x-if-delete-at' in request.headers and \ int(request.headers['x-if-delete-at']) != \ int(orig_metadata.get('X-Delete-At') or 0): @@ -562,7 +569,7 @@ def DELETE(self, request): container, obj, request, device) orig_timestamp = orig_metadata.get('X-Timestamp', 0) req_timestamp = request.headers['X-Timestamp'] - if disk_file.is_deleted() or disk_file.is_expired(): + if is_deleted or is_expired: response_class = HTTPNotFound else: if orig_timestamp < req_timestamp: diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py index 176fe70805..7d0cbb58ea 100644 --- a/test/unit/obj/test_diskfile.py +++ b/test/unit/obj/test_diskfile.py @@ -377,14 +377,15 @@ def _create_ondisk_file(self, df, data, timestamp, ext='.data'): setxattr(f.fileno(), diskfile.METADATA_KEY, pickle.dumps(md, diskfile.PICKLE_PROTOCOL)) - def _create_test_file(self, data, keep_data_fp=True, timestamp=None): + def _create_test_file(self, data, timestamp=None): df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', FakeLogger()) if timestamp is None: timestamp = time() self._create_ondisk_file(df, data, timestamp) df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', - FakeLogger(), keep_data_fp=keep_data_fp) + FakeLogger()) + df.open() return df def test_get_metadata(self): @@ -397,33 +398,37 @@ def test_disk_file_default_disallowed_metadata(self): orig_metadata = {'X-Object-Meta-Key1': 'Value1', 'Content-Type': 'text/garbage'} df = self._get_disk_file(ts=41, extra_metadata=orig_metadata) - self.assertEquals('1024', df._metadata['Content-Length']) + with df.open(): + self.assertEquals('1024', df._metadata['Content-Length']) # write some new metadata (fast POST, don't send orig meta, ts 42) df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', FakeLogger()) df.put_metadata({'X-Timestamp': '42', 'X-Object-Meta-Key2': 'Value2'}) df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', FakeLogger()) - # non-fast-post updateable keys are preserved - self.assertEquals('text/garbage', df._metadata['Content-Type']) - # original fast-post updateable keys are removed - self.assert_('X-Object-Meta-Key1' not in df._metadata) - # new fast-post updateable keys are added - self.assertEquals('Value2', df._metadata['X-Object-Meta-Key2']) + with df.open(): + # non-fast-post updateable keys are preserved + self.assertEquals('text/garbage', df._metadata['Content-Type']) + # original fast-post updateable keys are removed + self.assert_('X-Object-Meta-Key1' not in df._metadata) + # new fast-post updateable keys are added + self.assertEquals('Value2', df._metadata['X-Object-Meta-Key2']) def test_disk_file_app_iter_corners(self): df = self._create_test_file('1234567890') self.assertEquals(''.join(df.app_iter_range(0, None)), '1234567890') df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', - FakeLogger(), keep_data_fp=True) - self.assertEqual(''.join(df.app_iter_range(5, None)), '67890') + FakeLogger()) + with df.open(): + self.assertEqual(''.join(df.app_iter_range(5, None)), '67890') def test_disk_file_app_iter_partial_closes(self): df = self._create_test_file('1234567890') - it = df.app_iter_range(0, 5) - self.assertEqual(''.join(it), '12345') - self.assertEqual(df.fp, None) + with df.open(): + it = df.app_iter_range(0, 5) + self.assertEqual(''.join(it), '12345') + self.assertEqual(df.fp, None) def test_disk_file_app_iter_ranges(self): df = self._create_test_file('012345678911234567892123456789') @@ -482,10 +487,11 @@ def test_disk_file_app_iter_ranges_empty(self): self.assertEqual(''.join(it), '') df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', - FakeLogger(), keep_data_fp=True) - it = df.app_iter_ranges(None, 'app/something', - '\r\n--someheader\r\n', 150) - self.assertEqual(''.join(it), '') + FakeLogger()) + with df.open(): + it = df.app_iter_ranges(None, 'app/something', + '\r\n--someheader\r\n', 150) + self.assertEqual(''.join(it), '') def test_disk_file_mkstemp_creates_dir(self): tmpdir = os.path.join(self.testdir, 'sda1', 'tmp') @@ -501,8 +507,9 @@ def hook(): hook_call_count[0] += 1 df = self._get_disk_file(fsize=65, csize=8, iter_hook=hook) - for _ in df: - pass + with df.open(): + for _ in df: + pass self.assertEquals(hook_call_count[0], 9) @@ -525,7 +532,8 @@ def test_quarantine_same_file(self): # have to remake the datadir and file self._create_ondisk_file(df, '', time()) # still empty df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', - FakeLogger(), keep_data_fp=True) + FakeLogger()) + df.open() double_uuid_path = df.quarantine() self.assert_(os.path.isdir(double_uuid_path)) self.assert_('-' in os.path.basename(double_uuid_path)) @@ -573,10 +581,10 @@ def _get_disk_file(self, invalid_type=None, obj_name='o', df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', obj_name, FakeLogger(), - keep_data_fp=True, disk_chunk_size=csize, + disk_chunk_size=csize, iter_hook=iter_hook, mount_check=mount_check) + df.open() if invalid_type == 'Zero-Byte': - os.remove(df.data_file) fp = open(df.data_file, 'w') fp.close() df.unit_test_len = fsize @@ -694,8 +702,9 @@ def err(): df = self._get_disk_file(fsize=1024 * 2) df._handle_close_quarantine = err - for chunk in df: - pass + with df.open(): + for chunk in df: + pass # close is called at the end of the iterator self.assertEquals(df.fp, None) self.assertEquals(len(df.logger.log_dict['error']), 1) @@ -726,10 +735,12 @@ def test_ondisk_search_loop_ts_meta_data(self): self._create_ondisk_file(df, 'A', ext='.data', timestamp=5) df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', FakeLogger()) - self.assertTrue('X-Timestamp' in df._metadata) - self.assertEquals(df._metadata['X-Timestamp'], normalize_timestamp(10)) - self.assertTrue('deleted' in df._metadata) - self.assertTrue(df._metadata['deleted']) + with df.open(): + self.assertTrue('X-Timestamp' in df._metadata) + self.assertEquals(df._metadata['X-Timestamp'], + normalize_timestamp(10)) + self.assertTrue('deleted' in df._metadata) + self.assertTrue(df._metadata['deleted']) def test_ondisk_search_loop_meta_ts_data(self): df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', @@ -742,9 +753,11 @@ def test_ondisk_search_loop_meta_ts_data(self): self._create_ondisk_file(df, 'A', ext='.data', timestamp=5) df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', FakeLogger()) - self.assertTrue('X-Timestamp' in df._metadata) - self.assertEquals(df._metadata['X-Timestamp'], normalize_timestamp(8)) - self.assertTrue('deleted' in df._metadata) + with df.open(): + self.assertTrue('X-Timestamp' in df._metadata) + self.assertEquals(df._metadata['X-Timestamp'], + normalize_timestamp(8)) + self.assertTrue('deleted' in df._metadata) def test_ondisk_search_loop_meta_data_ts(self): df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', @@ -757,9 +770,11 @@ def test_ondisk_search_loop_meta_data_ts(self): self._create_ondisk_file(df, '', ext='.ts', timestamp=5) df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', FakeLogger()) - self.assertTrue('X-Timestamp' in df._metadata) - self.assertEquals(df._metadata['X-Timestamp'], normalize_timestamp(10)) - self.assertTrue('deleted' not in df._metadata) + with df.open(): + self.assertTrue('X-Timestamp' in df._metadata) + self.assertEquals(df._metadata['X-Timestamp'], + normalize_timestamp(10)) + self.assertTrue('deleted' not in df._metadata) def test_ondisk_search_loop_data_meta_ts(self): df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', @@ -772,9 +787,11 @@ def test_ondisk_search_loop_data_meta_ts(self): self._create_ondisk_file(df, '', ext='.meta', timestamp=5) df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', FakeLogger()) - self.assertTrue('X-Timestamp' in df._metadata) - self.assertEquals(df._metadata['X-Timestamp'], normalize_timestamp(10)) - self.assertTrue('deleted' not in df._metadata) + with df.open(): + self.assertTrue('X-Timestamp' in df._metadata) + self.assertEquals(df._metadata['X-Timestamp'], + normalize_timestamp(10)) + self.assertTrue('deleted' not in df._metadata) def test_ondisk_search_loop_wayward_files_ignored(self): df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', @@ -788,9 +805,11 @@ def test_ondisk_search_loop_wayward_files_ignored(self): self._create_ondisk_file(df, '', ext='.meta', timestamp=5) df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', FakeLogger()) - self.assertTrue('X-Timestamp' in df._metadata) - self.assertEquals(df._metadata['X-Timestamp'], normalize_timestamp(10)) - self.assertTrue('deleted' not in df._metadata) + with df.open(): + self.assertTrue('X-Timestamp' in df._metadata) + self.assertEquals(df._metadata['X-Timestamp'], + normalize_timestamp(10)) + self.assertTrue('deleted' not in df._metadata) def test_ondisk_search_loop_listdir_error(self): df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', @@ -807,8 +826,9 @@ def mock_listdir_exp(*args, **kwargs): self._create_ondisk_file(df, '', ext='.ts', timestamp=7) self._create_ondisk_file(df, '', ext='.meta', timestamp=6) self._create_ondisk_file(df, '', ext='.meta', timestamp=5) - self.assertRaises(OSError, diskfile.DiskFile, self.testdir, 'sda1', - '0', 'a', 'c', 'o', FakeLogger()) + df = diskfile.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o', + FakeLogger()) + self.assertRaises(OSError, df.open) def test_exception_in_handle_close_quarantine(self): df = self._get_disk_file() diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index c5aed59a8b..17f82e8a19 100755 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -361,7 +361,8 @@ def test_POST_quarantine_zbyte(self): resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) objfile = diskfile.DiskFile(self.testdir, 'sda1', 'p', 'a', 'c', 'o', - FakeLogger(), keep_data_fp=True) + FakeLogger()) + objfile.open() file_name = os.path.basename(objfile.data_file) with open(objfile.data_file) as fp: @@ -718,7 +719,8 @@ def test_HEAD_quarantine_zbyte(self): resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) objfile = diskfile.DiskFile(self.testdir, 'sda1', 'p', 'a', 'c', 'o', - FakeLogger(), keep_data_fp=True) + FakeLogger()) + objfile.open() file_name = os.path.basename(objfile.data_file) with open(objfile.data_file) as fp: @@ -1016,7 +1018,8 @@ def test_GET_quarantine(self): resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) objfile = diskfile.DiskFile(self.testdir, 'sda1', 'p', 'a', 'c', 'o', - FakeLogger(), keep_data_fp=True) + FakeLogger()) + objfile.open() file_name = os.path.basename(objfile.data_file) etag = md5() etag.update('VERIF') @@ -1048,7 +1051,8 @@ def test_GET_quarantine_zbyte(self): resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) objfile = diskfile.DiskFile(self.testdir, 'sda1', 'p', 'a', 'c', 'o', - FakeLogger(), keep_data_fp=True) + FakeLogger()) + objfile.open() file_name = os.path.basename(objfile.data_file) with open(objfile.data_file) as fp: metadata = diskfile.read_metadata(fp) @@ -1076,7 +1080,8 @@ def test_GET_quarantine_range(self): resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) objfile = diskfile.DiskFile(self.testdir, 'sda1', 'p', 'a', 'c', 'o', - FakeLogger(), keep_data_fp=True) + FakeLogger()) + objfile.open() file_name = os.path.basename(objfile.data_file) etag = md5() etag.update('VERIF') From 84296aae924e21dd34e9b07505ba30b72aaf3422 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Wed, 21 Aug 2013 14:57:16 +0200 Subject: [PATCH 06/35] Improve unittest coverage of account reaper Add more unittests for account_reaper in order to improve code coverage. Fixes: bug #1210045 Change-Id: I73490f80ac70b814a0abc490dc932f2a86fca703 --- test/unit/account/test_reaper.py | 410 ++++++++++++++++++++++++++++++- 1 file changed, 407 insertions(+), 3 deletions(-) diff --git a/test/unit/account/test_reaper.py b/test/unit/account/test_reaper.py index 3861daf218..e0f438b80c 100644 --- a/test/unit/account/test_reaper.py +++ b/test/unit/account/test_reaper.py @@ -13,16 +13,53 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO(creiht): Tests - +import os +import time +import shutil +import tempfile import unittest +from logging import DEBUG +from mock import patch +from contextlib import nested + from swift.account import reaper +from swift.account.server import DATADIR from swift.common.utils import normalize_timestamp +from swift.common.direct_client import ClientException -class FakeBroker(object): +class FakeLogger(object): + def __init__(self, *args, **kwargs): + self.inc = {'return_codes.4': 0, + 'return_codes.2': 0, + 'objects_failures': 0, + 'objects_deleted': 0, + 'objects_remaining': 0, + 'objects_possibly_remaining': 0, + 'containers_failures': 0, + 'containers_deleted': 0, + 'containers_remaining': 0, + 'containers_possibly_remaining': 0} + self.exp = [] + + def info(self, msg, *args): + self.msg = msg + + def timing_since(*args, **kwargs): + pass + + def getEffectiveLevel(self): + return DEBUG + + def exception(self, *args): + self.exp.append(args) + + def increment(self, key): + self.inc[key] += 1 + +class FakeBroker(object): def __init__(self): self.info = {} @@ -30,8 +67,149 @@ def get_info(self): return self.info +class FakeAccountBroker(): + def __init__(self, containers): + self.containers = containers + + def get_info(self): + info = {'account': 'a', + 'delete_timestamp': time.time() - 10} + return info + + def list_containers_iter(self, *args): + for cont in self.containers: + yield cont, None, None, None + + def is_status_deleted(self): + return True + + def empty(self): + return False + + +class FakeRing(): + def __init__(self): + self.nodes = [{'id': '1', + 'ip': '10.10.10.1', + 'port': None, + 'device': None}, + {'id': '2', + 'ip': '10.10.10.1', + 'port': None, + 'device': None}, + {'id': '3', + 'ip': '10.10.10.1', + 'port': None, + 'device': None}, + ] + + def get_nodes(self, *args, **kwargs): + return ('partition', self.nodes) + + def get_part_nodes(self, *args, **kwargs): + return self.nodes + +acc_nodes = [{'device': 'sda1', + 'ip': '', + 'port': ''}, + {'device': 'sda1', + 'ip': '', + 'port': ''}, + {'device': 'sda1', + 'ip': '', + 'port': ''}] + +cont_nodes = [{'device': 'sda1', + 'ip': '', + 'port': ''}, + {'device': 'sda1', + 'ip': '', + 'port': ''}, + {'device': 'sda1', + 'ip': '', + 'port': ''}] + + class TestReaper(unittest.TestCase): + def setUp(self): + self.to_delete = [] + self.myexp = ClientException("", http_host=None, + http_port=None, + http_device=None, + http_status=404, + http_reason=None + ) + + def tearDown(self): + for todel in self.to_delete: + shutil.rmtree(todel) + + def fake_direct_delete_object(self, *args, **kwargs): + if self.amount_fail < self.max_fail: + self.amount_fail += 1 + raise self.myexp + + def fake_object_ring(self): + return FakeRing() + + def fake_direct_delete_container(self, *args, **kwargs): + if self.amount_delete_fail < self.max_delete_fail: + self.amount_delete_fail += 1 + raise self.myexp + + def fake_direct_get_container(self, *args, **kwargs): + if self.get_fail: + raise self.myexp + objects = [{'name': 'o1'}, + {'name': 'o2'}, + {'name': unicode('o3')}, + {'name': ''}] + return None, objects + + def fake_container_ring(self): + return FakeRing() + + def fake_reap_object(self, *args, **kwargs): + if self.reap_obj_fail: + raise Exception + + def prepare_data_dir(self, ts=False): + devices_path = tempfile.mkdtemp() + # will be deleted by teardown + self.to_delete.append(devices_path) + path = os.path.join(devices_path, 'sda1', DATADIR) + os.makedirs(path) + path = os.path.join(path, '100', + 'a86', 'a8c682d2472e1720f2d81ff8993aba6') + os.makedirs(path) + suffix = 'db' + if ts: + suffix = 'ts' + with open(os.path.join(path, 'a8c682203aba6.%s' % suffix), 'w') as fd: + fd.write('') + return devices_path + + def init_reaper(self, conf={}, myips=['10.10.10.1'], fakelogger=False): + r = reaper.AccountReaper(conf) + r.stats_return_codes = {} + r.stats_containers_deleted = 0 + r.stats_containers_remaining = 0 + r.stats_containers_possibly_remaining = 0 + r.stats_objects_deleted = 0 + r.stats_objects_remaining = 0 + r.stats_objects_possibly_remaining = 0 + r.myips = myips + if fakelogger: + r.logger = FakeLogger() + return r + + def fake_reap_account(self, *args, **kwargs): + self.called_amount += 1 + + def fake_account_ring(self): + return FakeRing() + def test_delay_reaping_conf_default(self): r = reaper.AccountReaper({}) self.assertEquals(r.delay_reaping, 0) @@ -81,6 +259,232 @@ def _time(): finally: reaper.time = time_orig + def test_reap_object(self): + r = self.init_reaper({}, fakelogger=True) + self.amount_fail = 0 + self.max_fail = 0 + with patch('swift.account.reaper.AccountReaper.get_object_ring', + self.fake_object_ring): + with patch('swift.account.reaper.direct_delete_object', + self.fake_direct_delete_object): + r.reap_object('a', 'c', 'partition', cont_nodes, 'o') + self.assertEqual(r.stats_objects_deleted, 3) + + def test_reap_object_fail(self): + r = self.init_reaper({}, fakelogger=True) + self.amount_fail = 0 + self.max_fail = 1 + ctx = [patch('swift.account.reaper.AccountReaper.get_object_ring', + self.fake_object_ring), + patch('swift.account.reaper.direct_delete_object', + self.fake_direct_delete_object)] + with nested(*ctx): + r.reap_object('a', 'c', 'partition', cont_nodes, 'o') + self.assertEqual(r.stats_objects_deleted, 1) + self.assertEqual(r.stats_objects_remaining, 1) + self.assertEqual(r.stats_objects_possibly_remaining, 1) + + def test_reap_container_get_object_fail(self): + r = self.init_reaper({}, fakelogger=True) + self.get_fail = True + self.reap_obj_fail = False + self.amount_delete_fail = 0 + self.max_delete_fail = 0 + ctx = [patch('swift.account.reaper.direct_get_container', + self.fake_direct_get_container), + patch('swift.account.reaper.direct_delete_container', + self.fake_direct_delete_container), + patch('swift.account.reaper.AccountReaper.get_container_ring', + self.fake_container_ring), + patch('swift.account.reaper.AccountReaper.reap_object', + self.fake_reap_object)] + with nested(*ctx): + r.reap_container('a', 'partition', acc_nodes, 'c') + self.assertEqual(r.logger.inc['return_codes.4'], 1) + self.assertEqual(r.stats_containers_deleted, 1) + + def test_reap_container_partial_fail(self): + r = self.init_reaper({}, fakelogger=True) + self.get_fail = False + self.reap_obj_fail = False + self.amount_delete_fail = 0 + self.max_delete_fail = 2 + ctx = [patch('swift.account.reaper.direct_get_container', + self.fake_direct_get_container), + patch('swift.account.reaper.direct_delete_container', + self.fake_direct_delete_container), + patch('swift.account.reaper.AccountReaper.get_container_ring', + self.fake_container_ring), + patch('swift.account.reaper.AccountReaper.reap_object', + self.fake_reap_object)] + with nested(*ctx): + r.reap_container('a', 'partition', acc_nodes, 'c') + self.assertEqual(r.logger.inc['return_codes.4'], 2) + self.assertEqual(r.stats_containers_possibly_remaining, 1) + + def test_reap_container_full_fail(self): + r = self.init_reaper({}, fakelogger=True) + self.get_fail = False + self.reap_obj_fail = False + self.amount_delete_fail = 0 + self.max_delete_fail = 3 + ctx = [patch('swift.account.reaper.direct_get_container', + self.fake_direct_get_container), + patch('swift.account.reaper.direct_delete_container', + self.fake_direct_delete_container), + patch('swift.account.reaper.AccountReaper.get_container_ring', + self.fake_container_ring), + patch('swift.account.reaper.AccountReaper.reap_object', + self.fake_reap_object)] + with nested(*ctx): + r.reap_container('a', 'partition', acc_nodes, 'c') + self.assertEqual(r.logger.inc['return_codes.4'], 3) + self.assertEqual(r.stats_containers_remaining, 1) + + def fake_reap_container(self, *args, **kwargs): + self.called_amount += 1 + self.r.stats_containers_deleted = 1 + self.r.stats_objects_deleted = 1 + self.r.stats_containers_remaining = 1 + self.r.stats_objects_remaining = 1 + self.r.stats_containers_possibly_remaining = 1 + self.r.stats_objects_possibly_remaining = 1 + + def test_reap_account(self): + containers = ('c1', 'c2', 'c3', '') + broker = FakeAccountBroker(containers) + self.called_amount = 0 + self.r = r = self.init_reaper({}, fakelogger=True) + r.start_time = time.time() + ctx = [patch('swift.account.reaper.AccountReaper.reap_container', + self.fake_reap_container), + patch('swift.account.reaper.AccountReaper.get_account_ring', + self.fake_account_ring)] + with nested(*ctx): + nodes = r.get_account_ring().get_part_nodes() + self.assertTrue(r.reap_account(broker, 'partition', nodes)) + self.assertEqual(self.called_amount, 4) + self.assertEqual(r.logger.msg.find('Completed pass'), 0) + self.assertTrue(r.logger.msg.find('1 containers deleted')) + self.assertTrue(r.logger.msg.find('1 objects deleted')) + self.assertTrue(r.logger.msg.find('1 containers remaining')) + self.assertTrue(r.logger.msg.find('1 objects remaining')) + self.assertTrue(r.logger.msg.find('1 containers possibly remaining')) + self.assertTrue(r.logger.msg.find('1 objects possibly remaining')) + + def test_reap_account_no_container(self): + broker = FakeAccountBroker(tuple()) + self.r = r = self.init_reaper({}, fakelogger=True) + self.called_amount = 0 + r.start_time = time.time() + ctx = [patch('swift.account.reaper.AccountReaper.reap_container', + self.fake_reap_container), + patch('swift.account.reaper.AccountReaper.get_account_ring', + self.fake_account_ring)] + with nested(*ctx): + nodes = r.get_account_ring().get_part_nodes() + self.assertTrue(r.reap_account(broker, 'partition', nodes)) + self.assertEqual(r.logger.msg.find('Completed pass'), 0) + self.assertEqual(self.called_amount, 0) + + def test_reap_device(self): + devices = self.prepare_data_dir() + self.called_amount = 0 + conf = {'devices': devices} + r = self.init_reaper(conf) + ctx = [patch('swift.account.reaper.AccountBroker', + FakeAccountBroker), + patch('swift.account.reaper.AccountReaper.get_account_ring', + self.fake_account_ring), + patch('swift.account.reaper.AccountReaper.reap_account', + self.fake_reap_account)] + with nested(*ctx): + r.reap_device('sda1') + self.assertEqual(self.called_amount, 1) + + def test_reap_device_with_ts(self): + devices = self.prepare_data_dir(ts=True) + self.called_amount = 0 + conf = {'devices': devices} + r = self.init_reaper(conf=conf) + ctx = [patch('swift.account.reaper.AccountBroker', + FakeAccountBroker), + patch('swift.account.reaper.AccountReaper.get_account_ring', + self.fake_account_ring), + patch('swift.account.reaper.AccountReaper.reap_account', + self.fake_reap_account)] + with nested(*ctx): + r.reap_device('sda1') + self.assertEqual(self.called_amount, 0) + + def test_reap_device_with_not_my_ip(self): + devices = self.prepare_data_dir() + self.called_amount = 0 + conf = {'devices': devices} + r = self.init_reaper(conf, myips=['10.10.1.2']) + ctx = [patch('swift.account.reaper.AccountBroker', + FakeAccountBroker), + patch('swift.account.reaper.AccountReaper.get_account_ring', + self.fake_account_ring), + patch('swift.account.reaper.AccountReaper.reap_account', + self.fake_reap_account)] + with nested(*ctx): + r.reap_device('sda1') + self.assertEqual(self.called_amount, 0) + + def test_run_once(self): + def prepare_data_dir(): + devices_path = tempfile.mkdtemp() + # will be deleted by teardown + self.to_delete.append(devices_path) + path = os.path.join(devices_path, 'sda1', DATADIR) + os.makedirs(path) + return devices_path + + def init_reaper(devices): + r = reaper.AccountReaper({'devices': devices}) + return r + + devices = prepare_data_dir() + r = init_reaper(devices) + + with patch('swift.account.reaper.os.path.ismount', lambda x: True): + with patch( + 'swift.account.reaper.AccountReaper.reap_device') as foo: + r.run_once() + self.assertEqual(foo.called, 1) + + with patch('swift.account.reaper.os.path.ismount', lambda x: False): + with patch( + 'swift.account.reaper.AccountReaper.reap_device') as foo: + r.run_once() + self.assertFalse(foo.called) + + def test_run_forever(self): + def fake_sleep(val): + self.val = val + + def fake_random(): + return 1 + + def fake_run_once(): + raise Exception('exit') + + def init_reaper(): + r = reaper.AccountReaper({'interval': 1}) + r.run_once = fake_run_once + return r + + r = init_reaper() + with patch('swift.account.reaper.sleep', fake_sleep): + with patch('swift.account.reaper.random.random', fake_random): + try: + r.run_forever() + except Exception, err: + pass + self.assertEqual(self.val, 1) + self.assertEqual(str(err), 'exit') + if __name__ == '__main__': unittest.main() From 83a6ec1683fa0824d82a889046132a7ac33463a8 Mon Sep 17 00:00:00 2001 From: Tobias Stevenson Date: Tue, 27 Aug 2013 16:03:58 -0500 Subject: [PATCH 07/35] Man page lintian errors and warnings Used groff to recreate the errors. I believe all the issues except `binary-without-manpage` are solved. Would like confirmation from someone using Lintian. Closes-Bug: #1210114 Change-Id: I533205c53efdb7cdf3645cc3e3dc487f9ee5640a --- doc/manpages/container-server.conf.5 | 10 ++++----- doc/manpages/object-server.conf.5 | 2 +- doc/manpages/proxy-server.conf.5 | 10 ++++----- doc/manpages/swift-orphans.1 | 18 ++++++++-------- doc/manpages/swift-recon.1 | 32 ++++++++++++++-------------- 5 files changed, 36 insertions(+), 36 deletions(-) diff --git a/doc/manpages/container-server.conf.5 b/doc/manpages/container-server.conf.5 index 1f9e808e7e..a6bb699758 100644 --- a/doc/manpages/container-server.conf.5 +++ b/doc/manpages/container-server.conf.5 @@ -119,11 +119,11 @@ This is normally \fBegg:swift#container\fR. Label used when logging. The default is container-server. .IP "\fBset log_facility\fR Syslog log facility. The default is LOG_LOCAL0. -.IP "\fB set log_level\fR +.IP "\fBset log_level\fR Logging level. The default is INFO. -.IP "\fB set log_requests\fR +.IP "\fBset log_requests\fR Enables request logging. The default is True. -.IP "\fB set log_address\fR +.IP "\fBset log_address\fR Logging address. The default is /dev/log. .IP \fBnode_timeout\fR Request timeout to external services. The default is 3 seconds. @@ -224,7 +224,7 @@ Number of reaper workers to spawn. The default is 4. Request timeout to external services. The default is 3 seconds. .IP \fBconn_timeout\fR Connection timeout to external services. The default is 0.5 seconds. -.IP \fBslowdown = 0.01\fR +.IP \fBslowdown\fR Slowdown will sleep that amount between containers. The default is 0.01 seconds. .IP \fBaccount_suppression_time\fR Seconds to suppress updating an account that has generated an error. The default is 60 seconds. @@ -286,4 +286,4 @@ and .SH "SEE ALSO" -.BR swift-container-server(1), +.BR swift-container-server(1) diff --git a/doc/manpages/object-server.conf.5 b/doc/manpages/object-server.conf.5 index 298a60e51b..b629416577 100644 --- a/doc/manpages/object-server.conf.5 +++ b/doc/manpages/object-server.conf.5 @@ -236,7 +236,7 @@ Number of reaper workers to spawn. The default is 1. Request timeout to external services. The default is 10 seconds. .IP \fBconn_timeout\fR Connection timeout to external services. The default is 0.5 seconds. -.IP \fBslowdown = 0.01\fR +.IP \fBslowdown\fR Slowdown will sleep that amount between objects. The default is 0.01 seconds. .RE .PD diff --git a/doc/manpages/proxy-server.conf.5 b/doc/manpages/proxy-server.conf.5 index 767fc182cc..305a6293a3 100644 --- a/doc/manpages/proxy-server.conf.5 +++ b/doc/manpages/proxy-server.conf.5 @@ -475,7 +475,7 @@ Default is localhost. Default is 8125. .IP \fBaccess_log_statsd_default_sample_rate\fR Default is 1. -.IP \fBaccess_log_statsd_metric_prefix = +.IP \fBaccess_log_statsd_metric_prefix\fR Default is "" (empty-string) .IP \fBaccess_log_headers\fR Default is False. @@ -499,13 +499,13 @@ that are acceptable within this section. .IP \fBuse\fR Entry point for paste.deploy for the proxy server. This is the reference to the installed python egg. This is normally \fBegg:swift#proxy\fR. -.IP \fBset log_name\fR +.IP "\fBset log_name\fR" Label used when logging. The default is proxy-server. -.IP \fBset log_facility\fR +.IP "\fBset log_facility\fR" Syslog log facility. The default is LOG_LOCAL0. -.IP \fB set log_level\fR +.IP "\fBset log_level\fR" Logging level. The default is INFO. -.IP \fB set log_address\fR +.IP "\fBset log_address\fR" Logging address. The default is /dev/log. .IP \fBlog_handoffs\fR Log when handoff locations are used. Default is True. diff --git a/doc/manpages/swift-orphans.1 b/doc/manpages/swift-orphans.1 index 3a6611e974..1ef3488f1d 100644 --- a/doc/manpages/swift-orphans.1 +++ b/doc/manpages/swift-orphans.1 @@ -14,31 +14,31 @@ .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. -.\" +.\" .TH swift-orphans 1 "3/15/2012" "Linux" "OpenStack Swift" -.SH NAME +.SH NAME .LP .B swift-orphans \- Openstack-swift orphans tool .SH SYNOPSIS .LP -.B swift-orphans +.B swift-orphans [-h|--help] [-a|--age] [-k|--kill] [-w|--wide] [-r|--run-dir] -.SH DESCRIPTION +.SH DESCRIPTION .PP Lists and optionally kills orphaned Swift processes. This is done by scanning -/var/run/swift or the directory specified to the -r switch for .pid files and +/var/run/swift or the directory specified to the \-r switch for .pid files and listing any processes that look like Swift processes but aren't associated with the pids in those .pid files. Any Swift processes running with the 'once' parameter are ignored, as those are usually for full-speed audit scans and such. -Example (sends SIGTERM to all orphaned Swift processes older than two hours): -swift-orphans -a 2 -k TERM +Example (sends SIGTERM to all orphaned Swift processes older than two hours): +swift-orphans \-a 2 \-k TERM The options are as follows: @@ -62,9 +62,9 @@ The options are as follows: .PD .RE - + .SH DOCUMENTATION .LP -More documentation about Openstack-Swift can be found at +More documentation about Openstack-Swift can be found at .BI http://swift.openstack.org/index.html diff --git a/doc/manpages/swift-recon.1 b/doc/manpages/swift-recon.1 index 5bfa661985..4311c99add 100644 --- a/doc/manpages/swift-recon.1 +++ b/doc/manpages/swift-recon.1 @@ -14,26 +14,26 @@ .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. -.\" +.\" .TH swift-recon 1 "8/26/2011" "Linux" "OpenStack Swift" -.SH NAME +.SH NAME .LP .B swift-recon \- Openstack-swift recon middleware cli tool .SH SYNOPSIS .LP -.B swift-recon +.B swift-recon \ [-v] [--suppress] [-a] [-r] [-u] [-d] [-l] [--md5] [--auditor] [--updater] [--expirer] [--sockstat] - -.SH DESCRIPTION + +.SH DESCRIPTION .PP The swift-recon cli tool can be used to retrieve various metrics and telemetry information about -a cluster that has been collected by the swift-recon middleware. +a cluster that has been collected by the swift-recon middleware. -In order to make use of the swift-recon middleware, update the object-server.conf file and -enable the recon middleware by adding a pipeline entry and setting its option(s). You can view +In order to make use of the swift-recon middleware, update the object-server.conf file and +enable the recon middleware by adding a pipeline entry and setting its option(s). You can view more information in the example section below. @@ -69,13 +69,13 @@ Get cluster quarantine stats .IP "\fB--md5\fR" Get md5sum of servers ring and compare to local cop .IP "\fB--all\fR" -Perform all checks. Equivalent to -arudlq --md5 +Perform all checks. Equivalent to \-arudlq \-\-md5 .IP "\fB-z ZONE, --zone=ZONE\fR" Only query servers in specified zone .IP "\fB--swiftdir=PATH\fR" Default = /etc/swift .PD -.RE +.RE @@ -84,16 +84,16 @@ Default = /etc/swift .PD 0 .RS 0 .IP "ubuntu:~$ swift-recon -q --zone 3" -.IP "===============================================================================" +.IP "=================================================================" .IP "[2011-10-18 19:36:00] Checking quarantine dirs on 1 hosts... " .IP "[Quarantined objects] low: 4, high: 4, avg: 4, total: 4 " .IP "[Quarantined accounts] low: 0, high: 0, avg: 0, total: 0 " .IP "[Quarantined containers] low: 0, high: 0, avg: 0, total: 0 " -.IP "===============================================================================" +.IP "=================================================================" .RE .RS 0 -Finally if you also wish to track asynchronous pending’s you will need to setup a +Finally if you also wish to track asynchronous pending’s you will need to setup a cronjob to run the swift-recon-cron script periodically: .IP "*/5 * * * * swift /usr/bin/swift-recon-cron /etc/swift/object-server.conf" @@ -104,9 +104,9 @@ cronjob to run the swift-recon-cron script periodically: .SH DOCUMENTATION .LP -More documentation about Openstack-Swift can be found at -.BI http://swift.openstack.org/index.html -Also more specific documentation about swift-recon can be found at +More documentation about Openstack-Swift can be found at +.BI http://swift.openstack.org/index.html +Also more specific documentation about swift-recon can be found at .BI http://swift.openstack.org/admin_guide.html#cluster-telemetry-and-monitoring From 2d0ceb1e50f08bc9059c4d8560a3827ed8259508 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Tue, 10 Sep 2013 15:56:09 -0700 Subject: [PATCH 08/35] Verbose functional test request failures. * test/__init__.py: Put safe_repr import/implementation here so that it is available to functional and unit tests. * test/functional/swift_test_client.py: When a request fails record why that request failed, how many requests failed, and what the request was when raising RequestError to aid in debugging. Makes use of safe_repr from test/__init__.py. * test/unit/common/test_constraints.py: Remove implementation of safe_repr and use the implementation in test/__init__.py. Change-Id: I6c957343fb4b8b95d3875fd5ca87b3cf28a5f47a --- test/__init__.py | 14 ++++++++++++++ test/functional/swift_test_client.py | 14 ++++++++++++-- test/unit/common/test_constraints.py | 16 +--------------- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/test/__init__.py b/test/__init__.py index 98d0b42ecd..72dff139bf 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -18,6 +18,20 @@ import sys import os +try: + from unittest.util import safe_repr +except ImportError: + # Probably py26 + _MAX_LENGTH = 80 + + def safe_repr(obj, short=False): + try: + result = repr(obj) + except Exception: + result = object.__repr__(obj) + if not short or len(result) < _MAX_LENGTH: + return result + return result[:_MAX_LENGTH] + ' [truncated]...' # make unittests pass on all locale import swift diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index f9e031b67b..061497a0b7 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -28,6 +28,8 @@ from xml.dom import minidom from swiftclient import get_auth +from test import safe_repr + class AuthenticationFailed(Exception): pass @@ -216,18 +218,22 @@ def try_request(): self.response = None try_count = 0 + fail_messages = [] while try_count < 5: try_count += 1 try: self.response = try_request() - except httplib.HTTPException: + except httplib.HTTPException as e: + fail_messages.append(safe_repr(e)) continue if self.response.status == 401: + fail_messages.append("Response 401") self.authenticate() continue elif self.response.status == 503: + fail_messages.append("Response 503") if try_count != 5: time.sleep(5) continue @@ -237,7 +243,11 @@ def try_request(): if self.response: return self.response.status - raise RequestError('Unable to complete http request') + request = "{method} {path} headers: {headers} data: {data}".format( + method=method, path=path, headers=headers, data=data) + raise RequestError('Unable to complete http request: %s. ' + 'Attempts: %s, Failures: %s' % + (request, len(fail_messages), fail_messages)) def put_start(self, path, hdrs={}, parms={}, cfg={}, chunked=False): self.http_connect() diff --git a/test/unit/common/test_constraints.py b/test/unit/common/test_constraints.py index 80f5ff7736..9d62b5968e 100644 --- a/test/unit/common/test_constraints.py +++ b/test/unit/common/test_constraints.py @@ -14,23 +14,9 @@ # limitations under the License. import unittest -try: - from unittest.util import safe_repr -except ImportError: - # Probably py26 - _MAX_LENGTH = 80 - - def safe_repr(obj, short=False): - try: - result = repr(obj) - except Exception: - result = object.__repr__(obj) - if not short or len(result) < _MAX_LENGTH: - return result - return result[:_MAX_LENGTH] + ' [truncated]...' - import mock +from test import safe_repr from test.unit import MockTrue from swift.common.swob import HTTPBadRequest, Request From d3bd30da4191e576f4e7b78498979ffc8892eeb5 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Tue, 6 Aug 2013 14:15:48 +0200 Subject: [PATCH 09/35] Fix sync test when localhost on port 80 is binded - When localhost:80 was binding the tests was trying to connect into it. - To test you can simply run sudo python -m SimpleHTTPServer 80 which should show : 1.0.0.127.in-addr.arpa - - [06/Aug/2013 14:10:42] code 501, message Unsupported method ('DELETE') 1.0.0.127.in-addr.arpa - - [06/Aug/2013 14:10:42] "DELETE /a/c/o HTTP/1.1" 501 - (the test was passing since 501 would raise ClientException). mock delete_object in the fourth test to fix that - Refactor the code to use mock.patch as well. Closes-Bug: 1208802 Change-Id: I5ddd4ac3a97879f51cf5883fcfc0fe0f0adaeff6 --- test/unit/container/test_sync.py | 148 ++++++++++++++++--------------- 1 file changed, 76 insertions(+), 72 deletions(-) diff --git a/test/unit/container/test_sync.py b/test/unit/container/test_sync.py index 163f4e8172..eaabdb327c 100644 --- a/test/unit/container/test_sync.py +++ b/test/unit/container/test_sync.py @@ -13,9 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re import unittest +from contextlib import nested + +import mock -import re from test.unit import FakeLogger from swift.container import sync from swift.common import utils @@ -407,26 +410,23 @@ def test_container_first_loop(self): cring = FakeRing() oring = FakeRing() cs = sync.ContainerSync({}, container_ring=cring, object_ring=oring) - orig_ContainerBroker = sync.ContainerBroker - orig_hash_path = sync.hash_path - orig_delete_object = sync.delete_object - try: - def fake_hash_path(account, container, obj, raw_digest=False): - # Ensures that no rows match for full syncing, ordinal is 0 and - # all hashes are 0 - return '\x00' * 16 - - sync.hash_path = fake_hash_path - fcb = FakeContainerBroker( - 'path', - info={'account': 'a', 'container': 'c', - 'x_container_sync_point1': 2, - 'x_container_sync_point2': -1}, - metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), - 'x-container-sync-key': ('key', 1)}, - items_since=[{'ROWID': 1, 'name': 'o'}]) - sync.ContainerBroker = lambda p: fcb + def fake_hash_path(account, container, obj, raw_digest=False): + # Ensures that no rows match for full syncing, ordinal is 0 and + # all hashes are 0 + return '\x00' * 16 + fcb = FakeContainerBroker( + 'path', + info={'account': 'a', 'container': 'c', + 'x_container_sync_point1': 2, + 'x_container_sync_point2': -1}, + metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), + 'x-container-sync-key': ('key', 1)}, + items_since=[{'ROWID': 1, 'name': 'o'}]) + with nested( + mock.patch('swift.container.sync.ContainerBroker', + lambda p: fcb), + mock.patch('swift.container.sync.hash_path', fake_hash_path)): cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match cs.allowed_sync_hosts = ['127.0.0.1'] @@ -437,21 +437,23 @@ def fake_hash_path(account, container, obj, raw_digest=False): self.assertEquals(fcb.sync_point1, None) self.assertEquals(fcb.sync_point2, -1) - def fake_hash_path(account, container, obj, raw_digest=False): - # Ensures that all rows match for full syncing, ordinal is 0 - # and all hashes are 1 - return '\x01' * 16 - - sync.hash_path = fake_hash_path - fcb = FakeContainerBroker( - 'path', - info={'account': 'a', 'container': 'c', - 'x_container_sync_point1': 1, - 'x_container_sync_point2': 1}, - metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), - 'x-container-sync-key': ('key', 1)}, - items_since=[{'ROWID': 1, 'name': 'o'}]) - sync.ContainerBroker = lambda p: fcb + def fake_hash_path(account, container, obj, raw_digest=False): + # Ensures that all rows match for full syncing, ordinal is 0 + # and all hashes are 1 + return '\x01' * 16 + fcb = FakeContainerBroker('path', info={'account': 'a', + 'container': 'c', + 'x_container_sync_point1': 1, + 'x_container_sync_point2': 1}, + metadata={'x-container-sync-to': + ('http://127.0.0.1/a/c', 1), + 'x-container-sync-key': + ('key', 1)}, + items_since=[{'ROWID': 1, 'name': 'o'}]) + with nested( + mock.patch('swift.container.sync.ContainerBroker', + lambda p: fcb), + mock.patch('swift.container.sync.hash_path', fake_hash_path)): cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match cs.allowed_sync_hosts = ['127.0.0.1'] @@ -462,15 +464,15 @@ def fake_hash_path(account, container, obj, raw_digest=False): self.assertEquals(fcb.sync_point1, -1) self.assertEquals(fcb.sync_point2, -1) - fcb = FakeContainerBroker( - 'path', - info={'account': 'a', 'container': 'c', - 'x_container_sync_point1': 2, - 'x_container_sync_point2': -1}, - metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), - 'x-container-sync-key': ('key', 1)}, - items_since=[{'ROWID': 1, 'name': 'o'}]) - sync.ContainerBroker = lambda p: fcb + fcb = FakeContainerBroker( + 'path', + info={'account': 'a', 'container': 'c', + 'x_container_sync_point1': 2, + 'x_container_sync_point2': -1}, + metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), + 'x-container-sync-key': ('key', 1)}, + items_since=[{'ROWID': 1, 'name': 'o'}]) + with mock.patch('swift.container.sync.ContainerBroker', lambda p: fcb): cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match cs.allowed_sync_hosts = ['127.0.0.1'] @@ -482,16 +484,22 @@ def fake_hash_path(account, container, obj, raw_digest=False): self.assertEquals(fcb.sync_point1, None) self.assertEquals(fcb.sync_point2, -1) - fcb = FakeContainerBroker( - 'path', - info={'account': 'a', 'container': 'c', - 'x_container_sync_point1': 2, - 'x_container_sync_point2': -1}, - metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), - 'x-container-sync-key': ('key', 1)}, - items_since=[{'ROWID': 1, 'name': 'o', 'created_at': '1.2', - 'deleted': True}]) - sync.ContainerBroker = lambda p: fcb + def fake_delete_object(*args, **kwargs): + raise ClientException + fcb = FakeContainerBroker( + 'path', + info={'account': 'a', 'container': 'c', + 'x_container_sync_point1': 2, + 'x_container_sync_point2': -1}, + metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), + 'x-container-sync-key': ('key', 1)}, + items_since=[{'ROWID': 1, 'name': 'o', 'created_at': '1.2', + 'deleted': True}]) + with nested( + mock.patch('swift.container.sync.ContainerBroker', + lambda p: fcb), + mock.patch('swift.container.sync.delete_object', + fake_delete_object)): cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match cs.allowed_sync_hosts = ['127.0.0.1'] @@ -502,20 +510,20 @@ def fake_hash_path(account, container, obj, raw_digest=False): self.assertEquals(fcb.sync_point1, None) self.assertEquals(fcb.sync_point2, -1) - def fake_delete_object(*args, **kwargs): - pass - - sync.delete_object = fake_delete_object - fcb = FakeContainerBroker( - 'path', - info={'account': 'a', 'container': 'c', - 'x_container_sync_point1': 2, - 'x_container_sync_point2': -1}, - metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), - 'x-container-sync-key': ('key', 1)}, - items_since=[{'ROWID': 1, 'name': 'o', 'created_at': '1.2', - 'deleted': True}]) - sync.ContainerBroker = lambda p: fcb + fcb = FakeContainerBroker( + 'path', + info={'account': 'a', 'container': 'c', + 'x_container_sync_point1': 2, + 'x_container_sync_point2': -1}, + metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), + 'x-container-sync-key': ('key', 1)}, + items_since=[{'ROWID': 1, 'name': 'o', 'created_at': '1.2', + 'deleted': True}]) + with nested( + mock.patch('swift.container.sync.ContainerBroker', + lambda p: fcb), + mock.patch('swift.container.sync.delete_object', + lambda *x, **y: None)): cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match cs.allowed_sync_hosts = ['127.0.0.1'] @@ -525,10 +533,6 @@ def fake_delete_object(*args, **kwargs): self.assertEquals(cs.container_skips, 0) self.assertEquals(fcb.sync_point1, None) self.assertEquals(fcb.sync_point2, 1) - finally: - sync.ContainerBroker = orig_ContainerBroker - sync.hash_path = orig_hash_path - sync.delete_object = orig_delete_object def test_container_second_loop(self): cring = FakeRing() From 2283c54d199be5d19a4ca5fce6d692b6df9619b6 Mon Sep 17 00:00:00 2001 From: Pete Zaitcev Date: Wed, 11 Sep 2013 19:46:34 -0600 Subject: [PATCH 10/35] Tinker with dockstrings in back-ends and related places Change-Id: If80509e2e19cec5d0b8c5cfccd15e6d893558de4 --- swift/account/backend.py | 6 ++++-- swift/common/db.py | 6 +++--- swift/container/backend.py | 11 ++++++++--- swift/obj/server.py | 1 + 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/swift/account/backend.py b/swift/account/backend.py index 866d69d269..a772ba2a5f 100644 --- a/swift/account/backend.py +++ b/swift/account/backend.py @@ -31,14 +31,14 @@ class AccountBroker(DatabaseBroker): - """Encapsulates working with a account database.""" + """Encapsulates working with an account database.""" db_type = 'account' db_contains_type = 'container' db_reclaim_timestamp = 'delete_timestamp' def _initialize(self, conn, put_timestamp): """ - Create a brand new database (tables, indices, triggers, etc.) + Create a brand new account database (tables, indices, triggers, etc.) :param conn: DB connection object :param put_timestamp: put timestamp @@ -103,6 +103,7 @@ def create_container_table(self, conn): def create_account_stat_table(self, conn, put_timestamp): """ Create account_stat table which is specific to the account DB. + Not a part of Pluggable Back-ends, internal to the baseline code. :param conn: DB connection object :param put_timestamp: put timestamp @@ -156,6 +157,7 @@ def _delete_db(self, conn, timestamp, force=False): WHERE delete_timestamp < ? """, (timestamp, timestamp, timestamp)) def _commit_puts_load(self, item_list, entry): + """See :func:`swift.common.db.DatabaseBroker._commit_puts_load`""" (name, put_timestamp, delete_timestamp, object_count, bytes_used, deleted) = \ pickle.loads(entry.decode('base64')) diff --git a/swift/common/db.py b/swift/common/db.py index f48c68ab13..cbe1c2ca35 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -536,7 +536,7 @@ def _commit_puts_load(self, item_list, entry): """ Unmarshall the :param:entry and append it to :param:item_list. This is implemented by a particular broker to be compatible - with its merge_items(). + with its :func:`merge_items`. """ raise NotImplementedError @@ -622,7 +622,7 @@ def update_metadata(self, metadata_updates): that key was set to that value. Key/values will only be overwritten if the timestamp is newer. To delete a key, set its value to ('', timestamp). These empty keys will eventually be removed by - :func:reclaim + :func:`reclaim` """ old_metadata = self.metadata if set(metadata_updates).issubset(set(old_metadata)): @@ -659,7 +659,7 @@ def reclaim(self, age_timestamp, sync_timestamp): from incoming_sync and outgoing_sync where the updated_at timestamp is < sync_timestamp. - In addition, this calls the DatabaseBroker's :func:_reclaim method. + In addition, this calls the DatabaseBroker's :func:`_reclaim` method. :param age_timestamp: max created_at timestamp of object rows to delete :param sync_timestamp: max update_at timestamp of sync rows to delete diff --git a/swift/container/backend.py b/swift/container/backend.py index f16b3acc8e..1723c9c93c 100644 --- a/swift/container/backend.py +++ b/swift/container/backend.py @@ -37,7 +37,9 @@ class ContainerBroker(DatabaseBroker): db_reclaim_timestamp = 'created_at' def _initialize(self, conn, put_timestamp): - """Creates a brand new database (tables, indices, triggers, etc.)""" + """ + Create a brand new container database (tables, indices, triggers, etc.) + """ if not self.account: raise ValueError( 'Attempting to create a new database with no account set') @@ -50,6 +52,7 @@ def _initialize(self, conn, put_timestamp): def create_object_table(self, conn): """ Create the object table which is specifc to the container DB. + Not a part of Pluggable Back-ends, internal to the baseline code. :param conn: DB connection object """ @@ -91,6 +94,7 @@ def create_object_table(self, conn): def create_container_stat_table(self, conn, put_timestamp=None): """ Create the container_stat table which is specific to the container DB. + Not a part of Pluggable Back-ends, internal to the baseline code. :param conn: DB connection object :param put_timestamp: put timestamp @@ -159,6 +163,7 @@ def _delete_db(self, conn, timestamp): WHERE delete_timestamp < ? """, (timestamp, timestamp, timestamp)) def _commit_puts_load(self, item_list, entry): + """See :func:`swift.common.db.DatabaseBroker._commit_puts_load`""" (name, timestamp, size, content_type, etag, deleted) = \ pickle.loads(entry.decode('base64')) item_list.append({'name': name, @@ -170,7 +175,7 @@ def _commit_puts_load(self, item_list, entry): def empty(self): """ - Check if the DB is empty. + Check if container DB is empty. :returns: True if the database has no active objects, False otherwise """ @@ -334,7 +339,7 @@ def _set_x_container_sync_points(self, conn, sync_point1, sync_point2): def reported(self, put_timestamp, delete_timestamp, object_count, bytes_used): """ - Update reported stats. + Update reported stats, available with container's `get_info`. :param put_timestamp: put_timestamp to update :param delete_timestamp: delete_timestamp to update diff --git a/swift/obj/server.py b/swift/obj/server.py index 3113628314..362f940197 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -217,6 +217,7 @@ def delete_at_update(self, op, delete_at, account, container, obj, Update the expiring objects container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') + :param delete_at: scheduled delete in UNIX seconds, int :param account: account name for the object :param container: container name for the object :param obj: object name From 64430da593b40186251b3de08b5ae18dcffb062b Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 11 Sep 2013 12:44:23 -0700 Subject: [PATCH 11/35] fix race in test_wait on busy server fix bug 1224208 Change-Id: I83c673a87c31214a7c54b6399ca53512885e6bc3 --- test/unit/common/test_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/common/test_manager.py b/test/unit/common/test_manager.py index 84716c8cc1..53679af21b 100644 --- a/test/unit/common/test_manager.py +++ b/test/unit/common/test_manager.py @@ -1055,7 +1055,7 @@ def sleep(self, *args, **kwargs): self.assertEquals(status, 1) self.assert_('failed' in pop_stream(f)) # test multiple procs - procs = [MockProcess() for i in range(3)] + procs = [MockProcess(delay=.5) for i in range(3)] for proc in procs: proc.start() server.procs = procs From 0afcc674cb698906f354d9e57f97378d37d9d0c0 Mon Sep 17 00:00:00 2001 From: David Goetz Date: Thu, 12 Sep 2013 08:07:59 -0700 Subject: [PATCH 12/35] remove useless if from slo Change-Id: Id232c6973dcb55c19233b6d517770e58c2737ba0 --- swift/common/middleware/slo.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py index 48760867b5..b6c3f93ff5 100644 --- a/swift/common/middleware/slo.py +++ b/swift/common/middleware/slo.py @@ -348,8 +348,7 @@ def get_segments_to_delete_iter(self, req): if len(sub_segments) > MAX_BUFFERED_SLO_SEGMENTS: raise HTTPBadRequest( 'Too many buffered slo segments to delete.') - if sub_segments: - seg_data = sub_segments.pop(0) + seg_data = sub_segments.pop(0) if seg_data.get('sub_slo'): new_env = req.environ.copy() new_env['REQUEST_METHOD'] = 'GET' From a44307940432af93d52dab0a2701c7d6d91beeed Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 11 Sep 2013 22:42:19 -0700 Subject: [PATCH 13/35] Ensure audit tests don't cause unwanted errors Because the audit code catches all Exceptions, bugs in the code, like misspelled names, bad logic, can result in errors which don't cause exceptions that kill the unit tests. Change-Id: Idb8e3b00a49863ab920161a46edb61a5c47fffe2 --- swift/obj/auditor.py | 18 ++++++++++++------ test/unit/obj/test_auditor.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py index 28f39f89da..78ab123474 100644 --- a/swift/obj/auditor.py +++ b/swift/obj/auditor.py @@ -78,7 +78,7 @@ def audit_all_objects(self, mode='once'): logger=self.logger) for path, device, partition in all_locs: loop_time = time.time() - self.object_audit(path, device, partition) + self.failsafe_object_audit(path, device, partition) self.logger.timing_since('timing', loop_time) self.files_running_time = ratelimit_sleep( self.files_running_time, self.max_files_per_second) @@ -151,6 +151,17 @@ def record_stats(self, obj_size): else: self.stats_buckets["OVER"] += 1 + def failsafe_object_audit(self, path, device, partition): + """ + Entrypoint to object_audit, with a failsafe generic exception handler. + """ + try: + self.object_audit(path, device, partition) + except (Exception, Timeout): + self.logger.increment('errors') + self.errors += 1 + self.logger.exception(_('ERROR Trying to audit %s'), path) + def object_audit(self, path, device, partition): """ Audits the given object path. @@ -204,11 +215,6 @@ def object_audit(self, path, device, partition): diskfile.quarantine_renamer( os.path.join(self.devices, device), path) return - except (Exception, Timeout): - self.logger.increment('errors') - self.errors += 1 - self.logger.exception(_('ERROR Trying to audit %s'), path) - return self.passes += 1 diff --git a/test/unit/obj/test_auditor.py b/test/unit/obj/test_auditor.py index 83b6e27f33..94a6c5b57b 100644 --- a/test/unit/obj/test_auditor.py +++ b/test/unit/obj/test_auditor.py @@ -141,6 +141,36 @@ def test_object_audit_no_meta(self): 'sda', '0') self.assertEquals(self.auditor.quarantines, pre_quarantines + 1) + def test_object_audit_will_not_swallow_errors_in_tests(self): + timestamp = str(normalize_timestamp(time.time())) + path = os.path.join(self.disk_file.datadir, timestamp + '.data') + mkdirs(self.disk_file.datadir) + with open(path, 'w') as f: + write_metadata(f, {'name': '/a/c/o'}) + self.auditor = auditor.AuditorWorker(self.conf, self.logger) + + def blowup(*args): + raise NameError('tpyo') + with mock.patch('swift.obj.diskfile.DiskFile', + blowup): + self.assertRaises(NameError, self.auditor.object_audit, + path, 'sda', '0') + + def test_failsafe_object_audit_will_swallow_errors_in_tests(self): + timestamp = str(normalize_timestamp(time.time())) + path = os.path.join(self.disk_file.datadir, timestamp + '.data') + mkdirs(self.disk_file.datadir) + with open(path, 'w') as f: + write_metadata(f, {'name': '/a/c/o'}) + self.auditor = auditor.AuditorWorker(self.conf, self.logger) + + def blowup(*args): + raise NameError('tpyo') + with mock.patch('swift.obj.diskfile.DiskFile', + blowup): + self.auditor.failsafe_object_audit(path, 'sda', '0') + self.assertEquals(self.auditor.errors, 1) + def test_generic_exception_handling(self): self.auditor = auditor.AuditorWorker(self.conf, self.logger) timestamp = str(normalize_timestamp(time.time())) From 0b949ebe90ebd4613105af0c0127949128c2ec3d Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Thu, 12 Sep 2013 16:00:49 -0400 Subject: [PATCH 14/35] Remove unused method iter_devices_partition Change-Id: I69a14bcb92490abd3ad1070799bf1580a2dcaa92 Signed-off-by: Peter Portante --- swift/common/utils.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/swift/common/utils.py b/swift/common/utils.py index bb6cbe8c42..08cf0b04ad 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -1295,32 +1295,6 @@ def compute_eta(start_time, current_value, final_value): return get_time_units(1.0 / completion * elapsed - elapsed) -def iter_devices_partitions(devices_dir, item_type): - """ - Iterate over partitions across all devices. - - :param devices_dir: Path to devices - :param item_type: One of 'accounts', 'containers', or 'objects' - :returns: Each iteration returns a tuple of (device, partition) - """ - devices = listdir(devices_dir) - shuffle(devices) - devices_partitions = [] - for device in devices: - partitions = listdir(os.path.join(devices_dir, device, item_type)) - shuffle(partitions) - devices_partitions.append((device, iter(partitions))) - yielded = True - while yielded: - yielded = False - for device, partitions in devices_partitions: - try: - yield device, partitions.next() - yielded = True - except StopIteration: - pass - - def unlink_older_than(path, mtime): """ Remove any file in a given path that that was last modified before mtime. From a30a7ced9cc7c6510bbca93529e62bcbc30bf7c0 Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Thu, 22 Aug 2013 19:23:29 +0000 Subject: [PATCH 15/35] Add handoffs_first and handoff_delete to obj-repl If handoffs_first is True, then the object replicator will give partitions that are not supposed to be on the node priority. If handoff_delete is set to a number (n), then it will delete a handoff partition if at least n replicas were successfully replicated Also fixed a couple of things in the object replicator unit tests and added some more DocImpact Change-Id: Icb9968953cf467be2a52046fb16f4b84eb5604e4 --- doc/source/deployment_guide.rst | 13 +++ swift/obj/replicator.py | 20 ++++- test/unit/obj/test_replicator.py | 136 ++++++++++++++++++++++++++++++- 3 files changed, 163 insertions(+), 6 deletions(-) diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index 8e2564e8a8..ed744f7f23 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -414,6 +414,19 @@ stats_interval 3600 Interval in seconds between logging replication statistics reclaim_age 604800 Time elapsed in seconds before an object can be reclaimed +handoffs_first false If set to True, partitions that are + not supposed to be on the node will be + replicated first. The default setting + should not be changed, except for + extreme situations. +handoff_delete auto By default handoff partitions will be + removed when it has successfully + replicated to all the cannonical nodes. + If set to an integer n, it will remove + the partition if it is successfully + replicated to n nodes. The default + setting should not be changed, except + for extremem situations. ================== ================= ======================================= [object-updater] diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index 61c3ddc74d..e7639d35e7 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -31,7 +31,7 @@ from swift.common.utils import whataremyips, unlink_older_than, \ compute_eta, get_logger, dump_recon_cache, \ rsync_ip, mkdirs, config_true_value, list_from_csv, get_hub, \ - tpool_reraise + tpool_reraise, config_auto_int_value from swift.common.bufferedhttp import http_connect from swift.common.daemon import Daemon from swift.common.http import HTTP_OK, HTTP_INSUFFICIENT_STORAGE @@ -83,6 +83,10 @@ def __init__(self, conf): 'user-agent': 'obj-replicator %s' % os.getpid()} self.rsync_error_log_line_length = \ int(conf.get('rsync_error_log_line_length', 0)) + self.handoffs_first = config_true_value(conf.get('handoffs_first', + False)) + self.handoff_delete = config_auto_int_value( + conf.get('handoff_delete', 'auto'), 0) def _rsync(self, args): """ @@ -212,8 +216,15 @@ def tpool_get_suffixes(path): '/' + '-'.join(suffixes), headers=self.headers) conn.getresponse().read() responses.append(success) - if not suffixes or (len(responses) == - len(job['nodes']) and all(responses)): + if self.handoff_delete: + # delete handoff if we have had handoff_delete successes + delete_handoff = len([resp for resp in responses if resp]) >= \ + self.handoff_delete + else: + # delete handoff if all syncs were successful + delete_handoff = len(responses) == len(job['nodes']) and \ + all(responses) + if not suffixes or delete_handoff: self.logger.info(_("Removing partition: %s"), job['path']) tpool.execute(shutil.rmtree, job['path'], ignore_errors=True) except (Exception, Timeout): @@ -412,6 +423,9 @@ def collect_jobs(self): except (ValueError, OSError): continue random.shuffle(jobs) + if self.handoffs_first: + # Move the handoff parts to the front of the list + jobs.sort(key=lambda job: not job['delete']) self.job_count = len(jobs) return jobs diff --git a/test/unit/obj/test_replicator.py b/test/unit/obj/test_replicator.py index 7d56c47dc7..c8805494f5 100644 --- a/test/unit/obj/test_replicator.py +++ b/test/unit/obj/test_replicator.py @@ -236,7 +236,7 @@ def test_collect_jobs(self): for job in jobs: jobs_by_part[job['partition']] = job self.assertEquals(len(jobs_to_delete), 1) - self.assertTrue('1', jobs_to_delete[0]['partition']) + self.assertEquals('1', jobs_to_delete[0]['partition']) self.assertEquals( [node['id'] for node in jobs_by_part['0']['nodes']], [1, 2]) self.assertEquals( @@ -251,6 +251,12 @@ def test_collect_jobs(self): self.assertEquals(jobs_by_part[part]['path'], os.path.join(self.objects, part)) + def test_collect_jobs_handoffs_first(self): + self.replicator.handoffs_first = True + jobs = self.replicator.collect_jobs() + self.assertTrue(jobs[0]['delete']) + self.assertEquals('1', jobs[0]['partition']) + def test_collect_jobs_removes_zbf(self): """ After running xfs_repair, a partition directory could become a @@ -292,13 +298,137 @@ def test_delete_partition(self): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): df = diskfile.DiskFile(self.devices, - 'sda', '0', 'a', 'c', 'o', FakeLogger()) + 'sda', '1', 'a', 'c', 'o', FakeLogger()) + mkdirs(df.datadir) + print df.datadir + f = open(os.path.join(df.datadir, + normalize_timestamp(time.time()) + '.data'), + 'wb') + f.write('1234567890') + f.close() + ohash = hash_path('a', 'c', 'o') + data_dir = ohash[-3:] + whole_path_from = os.path.join(self.objects, '1', data_dir) + part_path = os.path.join(self.objects, '1') + self.assertTrue(os.access(part_path, os.F_OK)) + nodes = [node for node in + self.ring.get_part_nodes(1) + if node['ip'] not in _ips()] + process_arg_checker = [] + for node in nodes: + rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], 1) + process_arg_checker.append( + (0, '', ['rsync', whole_path_from, rsync_mod])) + with _mock_process(process_arg_checker): + self.replicator.replicate() + self.assertFalse(os.access(part_path, os.F_OK)) + + def test_delete_partition_with_failures(self): + with mock.patch('swift.obj.replicator.http_connect', + mock_http_connect(200)): + df = diskfile.DiskFile(self.devices, + 'sda', '1', 'a', 'c', 'o', FakeLogger()) + mkdirs(df.datadir) + print df.datadir + f = open(os.path.join(df.datadir, + normalize_timestamp(time.time()) + '.data'), + 'wb') + f.write('1234567890') + f.close() + ohash = hash_path('a', 'c', 'o') + data_dir = ohash[-3:] + whole_path_from = os.path.join(self.objects, '1', data_dir) + part_path = os.path.join(self.objects, '1') + self.assertTrue(os.access(part_path, os.F_OK)) + nodes = [node for node in + self.ring.get_part_nodes(1) + if node['ip'] not in _ips()] + process_arg_checker = [] + for i, node in enumerate(nodes): + rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], 1) + if i == 0: + # force one of the rsync calls to fail + ret_code = 1 + else: + ret_code = 0 + process_arg_checker.append( + (ret_code, '', ['rsync', whole_path_from, rsync_mod])) + with _mock_process(process_arg_checker): + self.replicator.replicate() + # The path should still exist + self.assertTrue(os.access(part_path, os.F_OK)) + + def test_delete_partition_with_handoff_delete(self): + with mock.patch('swift.obj.replicator.http_connect', + mock_http_connect(200)): + self.replicator.handoff_delete = 2 + df = diskfile.DiskFile(self.devices, + 'sda', '1', 'a', 'c', 'o', FakeLogger()) mkdirs(df.datadir) + print df.datadir + f = open(os.path.join(df.datadir, + normalize_timestamp(time.time()) + '.data'), + 'wb') + f.write('1234567890') + f.close() + ohash = hash_path('a', 'c', 'o') + data_dir = ohash[-3:] + whole_path_from = os.path.join(self.objects, '1', data_dir) part_path = os.path.join(self.objects, '1') self.assertTrue(os.access(part_path, os.F_OK)) - self.replicator.replicate() + nodes = [node for node in + self.ring.get_part_nodes(1) + if node['ip'] not in _ips()] + process_arg_checker = [] + for i, node in enumerate(nodes): + rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], 1) + if i == 0: + # force one of the rsync calls to fail + ret_code = 1 + else: + ret_code = 0 + process_arg_checker.append( + (ret_code, '', ['rsync', whole_path_from, rsync_mod])) + with _mock_process(process_arg_checker): + self.replicator.replicate() self.assertFalse(os.access(part_path, os.F_OK)) + def test_delete_partition_with_handoff_delete_failures(self): + with mock.patch('swift.obj.replicator.http_connect', + mock_http_connect(200)): + self.replicator.handoff_delete = 2 + df = diskfile.DiskFile(self.devices, + 'sda', '1', 'a', 'c', 'o', FakeLogger()) + mkdirs(df.datadir) + print df.datadir + f = open(os.path.join(df.datadir, + normalize_timestamp(time.time()) + '.data'), + 'wb') + f.write('1234567890') + f.close() + ohash = hash_path('a', 'c', 'o') + data_dir = ohash[-3:] + whole_path_from = os.path.join(self.objects, '1', data_dir) + part_path = os.path.join(self.objects, '1') + self.assertTrue(os.access(part_path, os.F_OK)) + nodes = [node for node in + self.ring.get_part_nodes(1) + if node['ip'] not in _ips()] + process_arg_checker = [] + for i, node in enumerate(nodes): + rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], 1) + if i in (0, 1): + # force two of the rsync calls to fail + ret_code = 1 + else: + ret_code = 0 + process_arg_checker.append( + (ret_code, '', ['rsync', whole_path_from, rsync_mod])) + with _mock_process(process_arg_checker): + self.replicator.replicate() + # The file should still exist + self.assertTrue(os.access(part_path, os.F_OK)) + def test_delete_partition_override_params(self): df = diskfile.DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o', FakeLogger()) From 5b0788d37d61584ba4b8e9f5d729ff2c5474f66c Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Fri, 13 Sep 2013 09:50:14 -0700 Subject: [PATCH 16/35] Use an existing local var rather than doing alookup This should make ``Ring.get_more_nodes`` microscopically faster. Change-Id: Ibf0988fe0630ad94ac0c04040766d89ef86d1488 --- swift/common/ring/ring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/common/ring/ring.py b/swift/common/ring/ring.py index 440904f726..5f4dba83be 100644 --- a/swift/common/ring/ring.py +++ b/swift/common/ring/ring.py @@ -313,7 +313,7 @@ def get_more_nodes(self, part): dev_id = part2dev_id[handoff_part] dev = self._devs[dev_id] region = dev['region'] - zone = (dev['region'], dev['zone']) + zone = (region, dev['zone']) if dev_id not in used and region not in same_regions: yield dev used.add(dev_id) From 5026e4569115e7aef1d25ed5ab9adbd6c330f110 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Fri, 13 Sep 2013 10:36:54 -0700 Subject: [PATCH 17/35] Switched some relative imports to be absolute Implicit relative imports don't work on Python3, and are also prone to several different classes of errors. Change-Id: I7b62e9bfbe9c0b1fc9876413e3139fda019a4e57 --- swift/common/ring/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swift/common/ring/__init__.py b/swift/common/ring/__init__.py index 7027aab2db..687e7b466f 100644 --- a/swift/common/ring/__init__.py +++ b/swift/common/ring/__init__.py @@ -1,5 +1,5 @@ -from ring import RingData, Ring -from builder import RingBuilder +from swift.common.ring.ring import RingData, Ring +from swift.common.ring.builder import RingBuilder __all__ = [ 'RingData', From cfa4f9497d8903d340f05f8314f40c814e3dde1c Mon Sep 17 00:00:00 2001 From: Pete Zaitcev Date: Fri, 13 Sep 2013 13:55:10 -0600 Subject: [PATCH 18/35] Use a local variable auditor_worker This patch is only a cleanup, but our code makes me rage every time I read it. In short, we have a class with a variable self.auditor, and we assign ObjectAuditor to it at some ties, AuditorWorker at other times. So, whenever there's a mismerge or whatever, you cannot tell if self.auditor.broker_class makes sense or not. Since all cases of using self.auditor as AuditorWorker are purely local, just evict it to a local variable auditor_worker. Leave self.auditor to represent ObjectAuditor, because it's used as a class variable in places. Change-Id: I811b80ac6c69a4daacfed7a5918bc0b15761d72e --- test/unit/obj/test_auditor.py | 82 +++++++++++++++++------------------ 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/test/unit/obj/test_auditor.py b/test/unit/obj/test_auditor.py index 94a6c5b57b..bbd7070ba8 100644 --- a/test/unit/obj/test_auditor.py +++ b/test/unit/obj/test_auditor.py @@ -60,7 +60,7 @@ def tearDown(self): unit.xattr_data = {} def test_object_audit_extra_data(self): - self.auditor = auditor.AuditorWorker(self.conf, self.logger) + auditor_worker = auditor.AuditorWorker(self.conf, self.logger) data = '0' * 1024 etag = md5() with self.disk_file.create() as writer: @@ -74,21 +74,21 @@ def test_object_audit_extra_data(self): 'Content-Length': str(os.fstat(writer.fd).st_size), } writer.put(metadata) - pre_quarantines = self.auditor.quarantines + pre_quarantines = auditor_worker.quarantines - self.auditor.object_audit( + auditor_worker.object_audit( os.path.join(self.disk_file.datadir, timestamp + '.data'), 'sda', '0') - self.assertEquals(self.auditor.quarantines, pre_quarantines) + self.assertEquals(auditor_worker.quarantines, pre_quarantines) os.write(writer.fd, 'extra_data') - self.auditor.object_audit( + auditor_worker.object_audit( os.path.join(self.disk_file.datadir, timestamp + '.data'), 'sda', '0') - self.assertEquals(self.auditor.quarantines, pre_quarantines + 1) + self.assertEquals(auditor_worker.quarantines, pre_quarantines + 1) def test_object_audit_diff_data(self): - self.auditor = auditor.AuditorWorker(self.conf, self.logger) + auditor_worker = auditor.AuditorWorker(self.conf, self.logger) data = '0' * 1024 etag = md5() timestamp = str(normalize_timestamp(time.time())) @@ -102,16 +102,16 @@ def test_object_audit_diff_data(self): 'Content-Length': str(os.fstat(writer.fd).st_size), } writer.put(metadata) - pre_quarantines = self.auditor.quarantines + pre_quarantines = auditor_worker.quarantines # remake so it will have metadata self.disk_file = DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o', self.logger) - self.auditor.object_audit( + auditor_worker.object_audit( os.path.join(self.disk_file.datadir, timestamp + '.data'), 'sda', '0') - self.assertEquals(self.auditor.quarantines, pre_quarantines) + self.assertEquals(auditor_worker.quarantines, pre_quarantines) etag = md5() etag.update('1' + '0' * 1023) etag = etag.hexdigest() @@ -121,10 +121,10 @@ def test_object_audit_diff_data(self): writer.write(data) writer.put(metadata) - self.auditor.object_audit( + auditor_worker.object_audit( os.path.join(self.disk_file.datadir, timestamp + '.data'), 'sda', '0') - self.assertEquals(self.auditor.quarantines, pre_quarantines + 1) + self.assertEquals(auditor_worker.quarantines, pre_quarantines + 1) def test_object_audit_no_meta(self): timestamp = str(normalize_timestamp(time.time())) @@ -134,12 +134,12 @@ def test_object_audit_no_meta(self): fp.write('0' * 1024) fp.close() invalidate_hash(os.path.dirname(self.disk_file.datadir)) - self.auditor = auditor.AuditorWorker(self.conf, self.logger) - pre_quarantines = self.auditor.quarantines - self.auditor.object_audit( + auditor_worker = auditor.AuditorWorker(self.conf, self.logger) + pre_quarantines = auditor_worker.quarantines + auditor_worker.object_audit( os.path.join(self.disk_file.datadir, timestamp + '.data'), 'sda', '0') - self.assertEquals(self.auditor.quarantines, pre_quarantines + 1) + self.assertEquals(auditor_worker.quarantines, pre_quarantines + 1) def test_object_audit_will_not_swallow_errors_in_tests(self): timestamp = str(normalize_timestamp(time.time())) @@ -147,13 +147,13 @@ def test_object_audit_will_not_swallow_errors_in_tests(self): mkdirs(self.disk_file.datadir) with open(path, 'w') as f: write_metadata(f, {'name': '/a/c/o'}) - self.auditor = auditor.AuditorWorker(self.conf, self.logger) + auditor_worker = auditor.AuditorWorker(self.conf, self.logger) def blowup(*args): raise NameError('tpyo') with mock.patch('swift.obj.diskfile.DiskFile', blowup): - self.assertRaises(NameError, self.auditor.object_audit, + self.assertRaises(NameError, auditor_worker.object_audit, path, 'sda', '0') def test_failsafe_object_audit_will_swallow_errors_in_tests(self): @@ -162,19 +162,19 @@ def test_failsafe_object_audit_will_swallow_errors_in_tests(self): mkdirs(self.disk_file.datadir) with open(path, 'w') as f: write_metadata(f, {'name': '/a/c/o'}) - self.auditor = auditor.AuditorWorker(self.conf, self.logger) + auditor_worker = auditor.AuditorWorker(self.conf, self.logger) def blowup(*args): raise NameError('tpyo') with mock.patch('swift.obj.diskfile.DiskFile', blowup): - self.auditor.failsafe_object_audit(path, 'sda', '0') - self.assertEquals(self.auditor.errors, 1) + auditor_worker.failsafe_object_audit(path, 'sda', '0') + self.assertEquals(auditor_worker.errors, 1) def test_generic_exception_handling(self): - self.auditor = auditor.AuditorWorker(self.conf, self.logger) + auditor_worker = auditor.AuditorWorker(self.conf, self.logger) timestamp = str(normalize_timestamp(time.time())) - pre_errors = self.auditor.errors + pre_errors = auditor_worker.errors data = '0' * 1024 etag = md5() with self.disk_file.create() as writer: @@ -189,14 +189,14 @@ def test_generic_exception_handling(self): writer.put(metadata) with mock.patch('swift.obj.diskfile.DiskFile', lambda *_: 1 / 0): - self.auditor.audit_all_objects() - self.assertEquals(self.auditor.errors, pre_errors + 1) + auditor_worker.audit_all_objects() + self.assertEquals(auditor_worker.errors, pre_errors + 1) def test_object_run_once_pass(self): - self.auditor = auditor.AuditorWorker(self.conf, self.logger) - self.auditor.log_time = 0 + auditor_worker = auditor.AuditorWorker(self.conf, self.logger) + auditor_worker.log_time = 0 timestamp = str(normalize_timestamp(time.time())) - pre_quarantines = self.auditor.quarantines + pre_quarantines = auditor_worker.quarantines data = '0' * 1024 etag = md5() with self.disk_file.create() as writer: @@ -209,15 +209,15 @@ def test_object_run_once_pass(self): 'Content-Length': str(os.fstat(writer.fd).st_size), } writer.put(metadata) - self.auditor.audit_all_objects() - self.assertEquals(self.auditor.quarantines, pre_quarantines) - self.assertEquals(self.auditor.stats_buckets[1024], 1) - self.assertEquals(self.auditor.stats_buckets[10240], 0) + auditor_worker.audit_all_objects() + self.assertEquals(auditor_worker.quarantines, pre_quarantines) + self.assertEquals(auditor_worker.stats_buckets[1024], 1) + self.assertEquals(auditor_worker.stats_buckets[10240], 0) def test_object_run_once_no_sda(self): - self.auditor = auditor.AuditorWorker(self.conf, self.logger) + auditor_worker = auditor.AuditorWorker(self.conf, self.logger) timestamp = str(normalize_timestamp(time.time())) - pre_quarantines = self.auditor.quarantines + pre_quarantines = auditor_worker.quarantines data = '0' * 1024 etag = md5() with self.disk_file.create() as writer: @@ -231,13 +231,13 @@ def test_object_run_once_no_sda(self): } writer.put(metadata) os.write(writer.fd, 'extra_data') - self.auditor.audit_all_objects() - self.assertEquals(self.auditor.quarantines, pre_quarantines + 1) + auditor_worker.audit_all_objects() + self.assertEquals(auditor_worker.quarantines, pre_quarantines + 1) def test_object_run_once_multi_devices(self): - self.auditor = auditor.AuditorWorker(self.conf, self.logger) + auditor_worker = auditor.AuditorWorker(self.conf, self.logger) timestamp = str(normalize_timestamp(time.time())) - pre_quarantines = self.auditor.quarantines + pre_quarantines = auditor_worker.quarantines data = '0' * 10 etag = md5() with self.disk_file.create() as writer: @@ -250,7 +250,7 @@ def test_object_run_once_multi_devices(self): 'Content-Length': str(os.fstat(writer.fd).st_size), } writer.put(metadata) - self.auditor.audit_all_objects() + auditor_worker.audit_all_objects() self.disk_file = DiskFile(self.devices, 'sdb', '0', 'a', 'c', 'ob', self.logger) data = '1' * 10 @@ -266,8 +266,8 @@ def test_object_run_once_multi_devices(self): } writer.put(metadata) os.write(writer.fd, 'extra_data') - self.auditor.audit_all_objects() - self.assertEquals(self.auditor.quarantines, pre_quarantines + 1) + auditor_worker.audit_all_objects() + self.assertEquals(auditor_worker.quarantines, pre_quarantines + 1) def test_object_run_fast_track_non_zero(self): self.auditor = auditor.ObjectAuditor(self.conf) From 9cd7c6c15514794afba5ac019823578aa7fe92a9 Mon Sep 17 00:00:00 2001 From: Samuel Merritt Date: Fri, 13 Sep 2013 13:26:41 -0700 Subject: [PATCH 19/35] Optimizations to Ring.get_more_nodes() When we're looking for handoffs, we first try to span all the regions, then zones, then devices. However, the search loops would always go over a big subset of the ring, regardless of whether or not there was anything left to find. In particular, this would result in long runtimes for the first call to get_more_nodes() if you didn't have more regions than replicas, as the region-search loop would chew through a ton of devices and not find any (since there weren't any). Now we do a little arithmetic, think about the pigeonhole principle a little bit, and bail out when there's nothing left to find, rather than waiting until there's no space left to search. Similar changes were made for the different-zone and different-device search loops. On a 4800-device test ring (4 regions, 5 zones each, 20 node each, 12 drives each), the time to get all handoffs dropped to about 5% of its previous value (a 95% improvement). More usefully, on a 1200-device test ring (same as above but only 1 region), the time to get just the first 6 handoffs dropped to about 0.01% of its original runtime, or a 10,000x speedup. Fixes bug 1225018. Change-Id: I4c77094186f0032a3e19a099a1a0e71b2ba06719 --- swift/common/ring/ring.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/swift/common/ring/ring.py b/swift/common/ring/ring.py index 5f4dba83be..9d4b92a7df 100644 --- a/swift/common/ring/ring.py +++ b/swift/common/ring/ring.py @@ -162,6 +162,19 @@ def _reload(self, force=False): self._part_shift = ring_data._part_shift self._rebuild_tier_data() + # Do this now, when we know the data has changed, rather then + # doing it on every call to get_more_nodes(). + regions = set() + zones = set() + self._num_devs = 0 + for dev in self._devs: + if dev: + regions.add(dev['region']) + zones.add((dev['region'], dev['zone'])) + self._num_devs += 1 + self._num_regions = len(regions) + self._num_zones = len(zones) + def _rebuild_tier_data(self): self.tier2devs = defaultdict(list) for dev in self._devs: @@ -305,9 +318,14 @@ def get_more_nodes(self, part): inc = int(parts / 65536) or 1 # Multiple loops for execution speed; the checks and bookkeeping get # simpler as you go along + hit_all_regions = len(same_regions) == self._num_regions for handoff_part in chain(xrange(start, parts, inc), xrange(inc - ((parts - start) % inc), start, inc)): + if hit_all_regions: + # At this point, there are no regions left untouched, so we + # can stop looking. + break for part2dev_id in self._replica2part2dev_id: if handoff_part < len(part2dev_id): dev_id = part2dev_id[handoff_part] @@ -319,10 +337,18 @@ def get_more_nodes(self, part): used.add(dev_id) same_regions.add(region) same_zones.add(zone) + if len(same_regions) == self._num_regions: + hit_all_regions = True + break + hit_all_zones = len(same_zones) == self._num_zones for handoff_part in chain(xrange(start, parts, inc), xrange(inc - ((parts - start) % inc), start, inc)): + if hit_all_zones: + # Much like we stopped looking for fresh regions before, we + # can now stop looking for fresh zones; there are no more. + break for part2dev_id in self._replica2part2dev_id: if handoff_part < len(part2dev_id): dev_id = part2dev_id[handoff_part] @@ -332,13 +358,24 @@ def get_more_nodes(self, part): yield dev used.add(dev_id) same_zones.add(zone) + if len(same_zones) == self._num_zones: + hit_all_zones = True + break + hit_all_devs = len(used) == self._num_devs for handoff_part in chain(xrange(start, parts, inc), xrange(inc - ((parts - start) % inc), start, inc)): + if hit_all_devs: + # We've used every device we have, so let's stop looking for + # unused devices now. + break for part2dev_id in self._replica2part2dev_id: if handoff_part < len(part2dev_id): dev_id = part2dev_id[handoff_part] if dev_id not in used: yield self._devs[dev_id] used.add(dev_id) + if len(used) == self._num_devs: + hit_all_devs = True + break From a9c119d440c99440088012417f56b1fa9baab980 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Fri, 13 Sep 2013 15:38:01 -0700 Subject: [PATCH 20/35] Removed many unnecessary uses of dict.keys This has a few advantages: some of these locations will perform slightly faster, it's minutely less code, and dict.keys has a different behavior on Python 3. Also removes usage of dict.iterkeys, which is identical to just iterating over the dict, except it doesn't exist on Python 3 at all. Change-Id: Ia91e97232dc8d78cf63fa807de288fc25cf5425d --- swift/account/reaper.py | 2 +- swift/common/db.py | 4 ++-- swift/common/memcached.py | 2 +- swift/common/middleware/bulk.py | 4 ++-- swift/common/middleware/ratelimit.py | 2 +- swift/common/middleware/slo.py | 2 +- swift/common/ring/ring.py | 2 +- swift/common/swob.py | 2 +- swift/container/server.py | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/swift/account/reaper.py b/swift/account/reaper.py index 90265b9ea5..488f1c541e 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -267,7 +267,7 @@ def reap_account(self, broker, partition, nodes): self.stats_objects_possibly_remaining if self.stats_return_codes: log += _(', return codes: ') - for code in sorted(self.stats_return_codes.keys()): + for code in sorted(self.stats_return_codes): log += '%s %sxxs, ' % (self.stats_return_codes[code], code) log = log[:-2] log += _(', elapsed: %.02fs') % (time() - begin) diff --git a/swift/common/db.py b/swift/common/db.py index cbe1c2ca35..20fec973af 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -50,7 +50,7 @@ def utf8encode(*args): def utf8encodekeys(metadata): - uni_keys = [k for k in metadata.keys() if isinstance(k, unicode)] + uni_keys = [k for k in metadata if isinstance(k, unicode)] for k in uni_keys: sv = metadata[k] del metadata[k] @@ -274,7 +274,7 @@ def delete_db(self, timestamp): timestamp = normalize_timestamp(timestamp) # first, clear the metadata cleared_meta = {} - for k in self.metadata.iterkeys(): + for k in self.metadata: cleared_meta[k] = ('', timestamp) self.update_metadata(cleared_meta) # then mark the db as deleted diff --git a/swift/common/memcached.py b/swift/common/memcached.py index 9c5f96d487..678e66a668 100644 --- a/swift/common/memcached.py +++ b/swift/common/memcached.py @@ -106,7 +106,7 @@ def __init__(self, servers, connect_timeout=CONN_TIMEOUT, for i in xrange(NODE_WEIGHT): self._ring[md5hash('%s-%s' % (server, i))] = server self._tries = tries if tries <= len(servers) else len(servers) - self._sorted = sorted(self._ring.keys()) + self._sorted = sorted(self._ring) self._client_cache = dict(((server, []) for server in servers)) self._connect_timeout = connect_timeout self._io_timeout = io_timeout diff --git a/swift/common/middleware/bulk.py b/swift/common/middleware/bulk.py index 89971ed6b5..92abdbd556 100644 --- a/swift/common/middleware/bulk.py +++ b/swift/common/middleware/bulk.py @@ -56,7 +56,7 @@ def get_response_body(data_format, data_dict, error_list): return json.dumps(data_dict) if data_format and data_format.endswith('/xml'): output = '\n' - for key in sorted(data_dict.keys()): + for key in sorted(data_dict): xml_key = key.replace(' ', '_').lower() output += '<%s>%s\n' % (xml_key, data_dict[key], xml_key) output += '\n' @@ -69,7 +69,7 @@ def get_response_body(data_format, data_dict, error_list): return output output = '' - for key in sorted(data_dict.keys()): + for key in sorted(data_dict): output += '%s: %s\n' % (key, data_dict[key]) output += 'Errors:\n' output += '\n'.join( diff --git a/swift/common/middleware/ratelimit.py b/swift/common/middleware/ratelimit.py index 56efb411be..a139e8e2db 100644 --- a/swift/common/middleware/ratelimit.py +++ b/swift/common/middleware/ratelimit.py @@ -25,7 +25,7 @@ def interpret_conf_limits(conf, name_prefix): conf_limits = [] - for conf_key in conf.keys(): + for conf_key in conf: if conf_key.startswith(name_prefix): cont_size = int(conf_key[len(name_prefix):]) rate = float(conf[conf_key]) diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py index 48760867b5..186e28cc5c 100644 --- a/swift/common/middleware/slo.py +++ b/swift/common/middleware/slo.py @@ -164,7 +164,7 @@ def parse_input(raw_data): req_keys = set(['path', 'etag', 'size_bytes']) try: for seg_dict in parsed_data: - if (set(seg_dict.keys()) != req_keys or + if (set(seg_dict) != req_keys or '/' not in seg_dict['path'].lstrip('/')): raise HTTPBadRequest('Invalid SLO Manifest File') except (AttributeError, TypeError): diff --git a/swift/common/ring/ring.py b/swift/common/ring/ring.py index 5f4dba83be..9ed0b35bfd 100644 --- a/swift/common/ring/ring.py +++ b/swift/common/ring/ring.py @@ -171,7 +171,7 @@ def _rebuild_tier_data(self): self.tier2devs[tier].append(dev) tiers_by_length = defaultdict(list) - for tier in self.tier2devs.keys(): + for tier in self.tier2devs: tiers_by_length[len(tier)].append(tier) self.tiers_by_length = sorted(tiers_by_length.values(), key=lambda x: len(x[0])) diff --git a/swift/common/swob.py b/swift/common/swob.py index 7a396354f2..73efa60700 100644 --- a/swift/common/swob.py +++ b/swift/common/swob.py @@ -235,7 +235,7 @@ def __delitem__(self, key): def keys(self): keys = [key[5:].replace('_', '-').title() - for key in self.environ.iterkeys() if key.startswith('HTTP_')] + for key in self.environ if key.startswith('HTTP_')] if 'CONTENT_LENGTH' in self.environ: keys.append('Content-Length') if 'CONTENT_TYPE' in self.environ: diff --git a/swift/container/server.py b/swift/container/server.py index 42aed48af1..9013ce8c57 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -406,7 +406,7 @@ def GET(self, req): "last_modified"]: SubElement(obj_element, field).text = str( record.pop(field)).decode('utf-8') - for field in sorted(record.keys()): + for field in sorted(record): SubElement(obj_element, field).text = str( record[field]).decode('utf-8') ret.body = tostring(doc, encoding='UTF-8').replace( From 75086a13301f20a1f459cf2dbd684bc8a06726bb Mon Sep 17 00:00:00 2001 From: Pete Zaitcev Date: Sat, 14 Sep 2013 16:09:53 -0600 Subject: [PATCH 21/35] Supply correct arguments to __init__ of a base class Hopefuly this extra test case is not too inane and slows us down for nothing. It is verified to fail with the old code. Change-Id: I604eca09f7f9ae044a8ad75284cd82a37325f99c --- swift/common/internal_client.py | 2 +- test/unit/common/test_internal_client.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/swift/common/internal_client.py b/swift/common/internal_client.py index e56f391695..ff2bc67d51 100644 --- a/swift/common/internal_client.py +++ b/swift/common/internal_client.py @@ -36,7 +36,7 @@ class UnexpectedResponse(Exception): """ def __init__(self, message, resp): - super(UnexpectedResponse, self).__init__(self, message) + super(UnexpectedResponse, self).__init__(message) self.resp = resp diff --git a/test/unit/common/test_internal_client.py b/test/unit/common/test_internal_client.py index 4c021b8d67..f4d2504c8e 100644 --- a/test/unit/common/test_internal_client.py +++ b/test/unit/common/test_internal_client.py @@ -291,6 +291,12 @@ def fake_app(self, env, start_response): except Exception as err: pass self.assertEquals(200, err.resp.status_int) + try: + client.make_request('GET', '/', {}, (111,)) + except Exception as err: + self.assertTrue(str(err).startswith('Unexpected response')) + else: + self.fail("Expected the UnexpectedResponse") finally: internal_client.sleep = old_sleep From da78035043643d1d89efbab197803d84934e420a Mon Sep 17 00:00:00 2001 From: Jamie Lennox Date: Tue, 17 Sep 2013 11:46:04 +1000 Subject: [PATCH 22/35] Add a user variable to templates This is a common configuration option and allows devstack (and others) to configure a mod_wsgi user to run the daemon process. Change-Id: Idf134b3bc6b08e3c3a80dde8830d5a4f3da5a06c Fixes: bug 1226346 --- examples/apache2/account-server.template | 4 +++- examples/apache2/container-server.template | 4 +++- examples/apache2/object-server.template | 5 +++-- examples/apache2/proxy-server.template | 4 +++- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/examples/apache2/account-server.template b/examples/apache2/account-server.template index e39bf25b1b..d4da3d807d 100644 --- a/examples/apache2/account-server.template +++ b/examples/apache2/account-server.template @@ -2,17 +2,19 @@ # # Change %PORT% to the port that you wish to use on your system # Change %SERVICENAME% to the service name you are using +# Change %USER% to the system user that will run the daemon process # Change the debug level as you see fit # # For example: # Replace %PORT% by 6012 # Replace %SERVICENAME% by account-server-1 +# Replace %USER% with apache (or remove it for default) NameVirtualHost *:%PORT% Listen %PORT% - WSGIDaemonProcess %SERVICENAME% processes=5 threads=1 + WSGIDaemonProcess %SERVICENAME% processes=5 threads=1 user=%USER% WSGIProcessGroup %SERVICENAME% WSGIScriptAlias / /var/www/swift/%SERVICENAME%.wsgi WSGIApplicationGroup %{GLOBAL} diff --git a/examples/apache2/container-server.template b/examples/apache2/container-server.template index e4ff7ce2cc..82fa281d3e 100644 --- a/examples/apache2/container-server.template +++ b/examples/apache2/container-server.template @@ -2,17 +2,19 @@ # # Change %PORT% to the port that you wish to use on your system # Change %SERVICENAME% to the service name you are using +# Change %USER% to the system user that will run the daemon process # Change the debug level as you see fit # # For example: # Replace %PORT% by 6011 # Replace %SERVICENAME% by container-server-1 +# Replace %USER% with apache (or remove it for default) NameVirtualHost *:%PORT% Listen %PORT% - WSGIDaemonProcess %SERVICENAME% processes=5 threads=1 + WSGIDaemonProcess %SERVICENAME% processes=5 threads=1 user=%USER% WSGIProcessGroup %SERVICENAME% WSGIScriptAlias / /var/www/swift/%SERVICENAME%.wsgi WSGIApplicationGroup %{GLOBAL} diff --git a/examples/apache2/object-server.template b/examples/apache2/object-server.template index c8aaee1d52..a022389d4d 100644 --- a/examples/apache2/object-server.template +++ b/examples/apache2/object-server.template @@ -2,18 +2,19 @@ # # Change %PORT% to the port that you wish to use on your system # Change %SERVICENAME% to the service name you are using +# Change %USER% to the system user that will run the daemon process # Change the debug level as you see fit # # For example: # Replace %PORT% by 6010 # Replace %SERVICENAME% by object-server-1 - +# Replace %USER% with apache (or remove it for default) NameVirtualHost *:%PORT% Listen %PORT% - WSGIDaemonProcess %SERVICENAME% processes=5 threads=1 + WSGIDaemonProcess %SERVICENAME% processes=5 threads=1 user=%USER% WSGIProcessGroup %SERVICENAME% WSGIScriptAlias / /var/www/swift/%SERVICENAME%.wsgi WSGIApplicationGroup %{GLOBAL} diff --git a/examples/apache2/proxy-server.template b/examples/apache2/proxy-server.template index 4066ad60d2..959d76ef9c 100644 --- a/examples/apache2/proxy-server.template +++ b/examples/apache2/proxy-server.template @@ -2,11 +2,13 @@ # # Change %PORT% to the port that you wish to use on your system # Change %SERVICENAME% to the service name you are using +# Change %USER% to the system user that will run the daemon process # Change the debug level as you see fit # # For example: # Replace %PORT% by 8080 # Replace %SERVICENAME% by proxy-server +# Replace %USER% with apache (or remove it for default) NameVirtualHost *:%PORT% Listen %PORT% @@ -14,7 +16,7 @@ Listen %PORT% # The limit of an object size LimitRequestBody 5368709122 - WSGIDaemonProcess %SERVICENAME% processes=5 threads=1 + WSGIDaemonProcess %SERVICENAME% processes=5 threads=1 user=%USER% WSGIProcessGroup %SERVICENAME% WSGIScriptAlias / /var/www/swift/%SERVICENAME%.wsgi WSGIApplicationGroup %{GLOBAL} From 7760f41c3ce436cb23b4b8425db3749a3da33d32 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Thu, 12 Sep 2013 15:57:56 -0400 Subject: [PATCH 23/35] Refactor common/utils methods to common/ondisk Place all the methods related to on-disk layout and / or configuration into a new common module that can be shared by the various modules using the same on-disk layout. Change-Id: I27ffd4665d5115ffdde649c48a4d18e12017e6a9 Signed-off-by: Peter Portante --- doc/source/misc.rst | 9 + swift/account/auditor.py | 5 +- swift/account/backend.py | 3 +- swift/account/server.py | 5 +- swift/account/utils.py | 3 +- swift/common/daemon.py | 3 +- swift/common/db.py | 5 +- swift/common/db_replicator.py | 7 +- swift/common/direct_client.py | 3 +- swift/common/ondisk.py | 184 ++++++++++++++++++ swift/common/request_helpers.py | 3 +- swift/common/ring/ring.py | 5 +- swift/common/utils.py | 163 +--------------- swift/common/wsgi.py | 3 +- swift/container/auditor.py | 5 +- swift/container/backend.py | 3 +- swift/container/server.py | 5 +- swift/container/sync.py | 5 +- swift/obj/auditor.py | 5 +- swift/obj/diskfile.py | 7 +- swift/obj/server.py | 6 +- swift/proxy/controllers/base.py | 6 +- swift/proxy/controllers/obj.py | 6 +- test/unit/account/test_backend.py | 2 +- test/unit/account/test_reaper.py | 2 +- test/unit/account/test_server.py | 3 +- .../common/middleware/test_list_endpoints.py | 6 +- test/unit/common/ring/test_ring.py | 18 +- test/unit/common/test_daemon.py | 6 +- test/unit/common/test_db.py | 2 +- test/unit/common/test_db_replicator.py | 2 +- test/unit/common/test_ondisk.py | 139 +++++++++++++ test/unit/common/test_utils.py | 104 ---------- test/unit/container/test_backend.py | 2 +- test/unit/container/test_replicator.py | 2 +- test/unit/container/test_server.py | 3 +- test/unit/container/test_updater.py | 2 +- test/unit/obj/test_auditor.py | 3 +- test/unit/obj/test_diskfile.py | 9 +- test/unit/obj/test_replicator.py | 9 +- test/unit/obj/test_server.py | 23 +-- test/unit/obj/test_updater.py | 10 +- test/unit/proxy/test_server.py | 6 +- 43 files changed, 448 insertions(+), 354 deletions(-) create mode 100644 swift/common/ondisk.py create mode 100644 test/unit/common/test_ondisk.py diff --git a/doc/source/misc.rst b/doc/source/misc.rst index e42a91193d..e8921173f1 100644 --- a/doc/source/misc.rst +++ b/doc/source/misc.rst @@ -33,6 +33,15 @@ Utils :members: :show-inheritance: +.. _ondisk: + +OnDisk +====== + +.. automodule:: swift.common.ondisk + :members: + :show-inheritance: + .. _common_tempauth: TempAuth diff --git a/swift/account/auditor.py b/swift/account/auditor.py index 5f81490d94..a85839ad70 100644 --- a/swift/account/auditor.py +++ b/swift/account/auditor.py @@ -21,8 +21,9 @@ import swift.common.db from swift.account import server as account_server from swift.account.backend import AccountBroker -from swift.common.utils import get_logger, audit_location_generator, \ - config_true_value, dump_recon_cache, ratelimit_sleep +from swift.common.utils import get_logger, config_true_value, \ + dump_recon_cache, ratelimit_sleep +from swift.common.ondisk import audit_location_generator from swift.common.daemon import Daemon from eventlet import Timeout diff --git a/swift/account/backend.py b/swift/account/backend.py index a772ba2a5f..6a889fbb7f 100644 --- a/swift/account/backend.py +++ b/swift/account/backend.py @@ -25,7 +25,8 @@ import sqlite3 -from swift.common.utils import normalize_timestamp, lock_parent_directory +from swift.common.utils import lock_parent_directory +from swift.common.ondisk import normalize_timestamp from swift.common.db import DatabaseBroker, DatabaseConnectionError, \ PENDING_CAP, PICKLE_PROTOCOL, utf8encode diff --git a/swift/account/server.py b/swift/account/server.py index 99bfc58cd6..da27fe613f 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -28,9 +28,10 @@ from swift.common.db import DatabaseConnectionError, DatabaseAlreadyExists from swift.common.request_helpers import get_param, get_listing_content_type, \ split_and_validate_path -from swift.common.utils import get_logger, hash_path, public, \ - normalize_timestamp, storage_directory, config_true_value, \ +from swift.common.utils import get_logger, public, config_true_value, \ json, timing_stats, replication +from swift.common.ondisk import hash_path, normalize_timestamp, \ + storage_directory from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.db_replicator import ReplicatorRpc diff --git a/swift/account/utils.py b/swift/account/utils.py index 5565b46166..919c05c02d 100644 --- a/swift/account/utils.py +++ b/swift/account/utils.py @@ -17,7 +17,8 @@ from xml.sax import saxutils from swift.common.swob import HTTPOk, HTTPNoContent -from swift.common.utils import json, normalize_timestamp +from swift.common.utils import json +from swift.common.ondisk import normalize_timestamp class FakeAccountBroker(object): diff --git a/swift/common/daemon.py b/swift/common/daemon.py index 0b3effb542..25de076c66 100644 --- a/swift/common/daemon.py +++ b/swift/common/daemon.py @@ -22,6 +22,7 @@ import eventlet.debug from swift.common import utils +from swift.common import ondisk class Daemon(object): @@ -41,7 +42,7 @@ def run_forever(self, *args, **kwargs): def run(self, once=False, **kwargs): """Run the daemon""" - utils.validate_configuration() + ondisk.validate_configuration() utils.drop_privileges(self.conf.get('user', 'swift')) utils.capture_stdio(self.logger, **kwargs) diff --git a/swift/common/db.py b/swift/common/db.py index 20fec973af..0fd4900e01 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -30,8 +30,9 @@ from eventlet import sleep, Timeout import sqlite3 -from swift.common.utils import json, normalize_timestamp, renamer, \ - mkdirs, lock_parent_directory, fallocate +from swift.common.utils import json, renamer, mkdirs, lock_parent_directory, \ + fallocate +from swift.common.ondisk import normalize_timestamp from swift.common.exceptions import LockTimeout diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index 32fe485c8d..b0965296f2 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -30,9 +30,10 @@ import swift.common.db from swift.common.direct_client import quote -from swift.common.utils import get_logger, whataremyips, storage_directory, \ - renamer, mkdirs, lock_parent_directory, config_true_value, \ - unlink_older_than, dump_recon_cache, rsync_ip +from swift.common.utils import get_logger, whataremyips, renamer, mkdirs, \ + lock_parent_directory, config_true_value, unlink_older_than, \ + dump_recon_cache, rsync_ip +from swift.common.ondisk import storage_directory from swift.common import ring from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE from swift.common.bufferedhttp import BufferedHTTPConnection diff --git a/swift/common/direct_client.py b/swift/common/direct_client.py index 36e495ca79..ab283026d1 100644 --- a/swift/common/direct_client.py +++ b/swift/common/direct_client.py @@ -27,7 +27,8 @@ from swift.common.bufferedhttp import http_connect from swiftclient import ClientException, json_loads -from swift.common.utils import normalize_timestamp, FileLikeIter +from swift.common.utils import FileLikeIter +from swift.common.ondisk import normalize_timestamp from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \ is_success, is_server_error from swift.common.swob import HeaderKeyDict diff --git a/swift/common/ondisk.py b/swift/common/ondisk.py new file mode 100644 index 0000000000..8acad105c2 --- /dev/null +++ b/swift/common/ondisk.py @@ -0,0 +1,184 @@ +# Copyright (c) 2010-2013 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Methods & Attributes for shared 'on-disk' data layouts.""" + +import os +import sys +import errno + +from hashlib import md5 +from random import shuffle +from ConfigParser import ConfigParser, NoSectionError, NoOptionError + +from swift import gettext_ as _ +from swift.common.utils import listdir, quote + +# Used by hash_path to offer a bit more security when generating hashes for +# paths. It simply appends this value to all paths; guessing the hash a path +# will end up with would also require knowing this suffix. +_hash_conf = ConfigParser() +HASH_PATH_SUFFIX = '' +HASH_PATH_PREFIX = '' +if _hash_conf.read('/etc/swift/swift.conf'): + try: + HASH_PATH_SUFFIX = _hash_conf.get('swift-hash', + 'swift_hash_path_suffix') + except (NoSectionError, NoOptionError): + pass + try: + HASH_PATH_PREFIX = _hash_conf.get('swift-hash', + 'swift_hash_path_prefix') + except (NoSectionError, NoOptionError): + pass + + +def validate_configuration(): + if not HASH_PATH_SUFFIX and not HASH_PATH_PREFIX: + sys.exit("Error: [swift-hash]: both swift_hash_path_suffix " + "and swift_hash_path_prefix are missing " + "from /etc/swift/swift.conf") + + +def hash_path(account, container=None, object=None, raw_digest=False): + """ + Get the canonical hash for an account/container/object + + :param account: Account + :param container: Container + :param object: Object + :param raw_digest: If True, return the raw version rather than a hex digest + :returns: hash string + """ + if object and not container: + raise ValueError('container is required if object is provided') + paths = [account] + if container: + paths.append(container) + if object: + paths.append(object) + if raw_digest: + return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths) + + HASH_PATH_SUFFIX).digest() + else: + return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths) + + HASH_PATH_SUFFIX).hexdigest() + + +def normalize_timestamp(timestamp): + """ + Format a timestamp (string or numeric) into a standardized + xxxxxxxxxx.xxxxx (10.5) format. + + Note that timestamps using values greater than or equal to November 20th, + 2286 at 17:46 UTC will use 11 digits to represent the number of + seconds. + + :param timestamp: unix timestamp + :returns: normalized timestamp as a string + """ + return "%016.05f" % (float(timestamp)) + + +def validate_device_partition(device, partition): + """ + Validate that a device and a partition are valid and won't lead to + directory traversal when used. + + :param device: device to validate + :param partition: partition to validate + :raises: ValueError if given an invalid device or partition + """ + invalid_device = False + invalid_partition = False + if not device or '/' in device or device in ['.', '..']: + invalid_device = True + if not partition or '/' in partition or partition in ['.', '..']: + invalid_partition = True + + if invalid_device: + raise ValueError('Invalid device: %s' % quote(device or '')) + elif invalid_partition: + raise ValueError('Invalid partition: %s' % quote(partition or '')) + + +def storage_directory(datadir, partition, name_hash): + """ + Get the storage directory + + :param datadir: Base data directory + :param partition: Partition + :param name_hash: Account, container or object name hash + :returns: Storage directory + """ + return os.path.join(datadir, str(partition), name_hash[-3:], name_hash) + + +def audit_location_generator(devices, datadir, suffix='', + mount_check=True, logger=None): + ''' + Given a devices path and a data directory, yield (path, device, + partition) for all files in that directory + + :param devices: parent directory of the devices to be audited + :param datadir: a directory located under self.devices. This should be + one of the DATADIR constants defined in the account, + container, and object servers. + :param suffix: path name suffix required for all names returned + :param mount_check: Flag to check if a mount check should be performed + on devices + :param logger: a logger object + ''' + device_dir = listdir(devices) + # randomize devices in case of process restart before sweep completed + shuffle(device_dir) + for device in device_dir: + if mount_check and not \ + os.path.ismount(os.path.join(devices, device)): + if logger: + logger.debug( + _('Skipping %s as it is not mounted'), device) + continue + datadir_path = os.path.join(devices, device, datadir) + partitions = listdir(datadir_path) + for partition in partitions: + part_path = os.path.join(datadir_path, partition) + try: + suffixes = listdir(part_path) + except OSError as e: + if e.errno != errno.ENOTDIR: + raise + continue + for asuffix in suffixes: + suff_path = os.path.join(part_path, asuffix) + try: + hashes = listdir(suff_path) + except OSError as e: + if e.errno != errno.ENOTDIR: + raise + continue + for hsh in hashes: + hash_path = os.path.join(suff_path, hsh) + try: + files = sorted(listdir(hash_path), reverse=True) + except OSError as e: + if e.errno != errno.ENOTDIR: + raise + continue + for fname in files: + if suffix and not fname.endswith(suffix): + continue + path = os.path.join(hash_path, fname) + yield path, device, partition diff --git a/swift/common/request_helpers.py b/swift/common/request_helpers.py index b24c8b24c8..51c846d124 100644 --- a/swift/common/request_helpers.py +++ b/swift/common/request_helpers.py @@ -22,7 +22,8 @@ from swift.common.constraints import FORMAT2CONTENT_TYPE from swift.common.swob import HTTPBadRequest, HTTPNotAcceptable -from swift.common.utils import split_path, validate_device_partition +from swift.common.ondisk import validate_device_partition +from swift.common.utils import split_path from urllib import unquote diff --git a/swift/common/ring/ring.py b/swift/common/ring/ring.py index 9ed0b35bfd..22bdb4c442 100644 --- a/swift/common/ring/ring.py +++ b/swift/common/ring/ring.py @@ -25,7 +25,8 @@ from hashlib import md5 from itertools import chain -from swift.common.utils import hash_path, validate_configuration, json +from swift.common.utils import json +from swift.common.ondisk import hash_path, validate_configuration from swift.common.ring.utils import tiers_for_dev @@ -130,7 +131,7 @@ class Ring(object): """ def __init__(self, serialized_path, reload_time=15, ring_name=None): - # can't use the ring unless HASH_PATH_SUFFIX is set + # Can't use the ring unless the on-disk configuration is valid validate_configuration() if ring_name: self.serialized_path = os.path.join(serialized_path, diff --git a/swift/common/utils.py b/swift/common/utils.py index 08cf0b04ad..8173cbbdd2 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -26,14 +26,12 @@ import time import uuid import functools -from hashlib import md5 -from random import random, shuffle +from random import random from urllib import quote as _quote from contextlib import contextmanager, closing import ctypes import ctypes.util -from ConfigParser import ConfigParser, NoSectionError, NoOptionError, \ - RawConfigParser +from ConfigParser import ConfigParser, RawConfigParser from optparse import OptionParser from Queue import Queue, Empty from tempfile import mkstemp, NamedTemporaryFile @@ -80,24 +78,6 @@ # available being at or below this amount, in bytes. FALLOCATE_RESERVE = 0 -# Used by hash_path to offer a bit more security when generating hashes for -# paths. It simply appends this value to all paths; guessing the hash a path -# will end up with would also require knowing this suffix. -hash_conf = ConfigParser() -HASH_PATH_SUFFIX = '' -HASH_PATH_PREFIX = '' -if hash_conf.read('/etc/swift/swift.conf'): - try: - HASH_PATH_SUFFIX = hash_conf.get('swift-hash', - 'swift_hash_path_suffix') - except (NoSectionError, NoOptionError): - pass - try: - HASH_PATH_PREFIX = hash_conf.get('swift-hash', - 'swift_hash_path_prefix') - except (NoSectionError, NoOptionError): - pass - def backward(f, blocksize=4096): """ @@ -164,13 +144,6 @@ def noop_libc_function(*args): return 0 -def validate_configuration(): - if not HASH_PATH_SUFFIX and not HASH_PATH_PREFIX: - sys.exit("Error: [swift-hash]: both swift_hash_path_suffix " - "and swift_hash_path_prefix are missing " - "from /etc/swift/swift.conf") - - def load_libc_function(func_name, log_error=True): """ Attempt to find the function in libc, otherwise return a no-op func. @@ -422,21 +395,6 @@ def drop_buffer_cache(fd, offset, length): % (fd, offset, length, ret)) -def normalize_timestamp(timestamp): - """ - Format a timestamp (string or numeric) into a standardized - xxxxxxxxxx.xxxxx (10.5) format. - - Note that timestamps using values greater than or equal to November 20th, - 2286 at 17:46 UTC will use 11 digits to represent the number of - seconds. - - :param timestamp: unix timestamp - :returns: normalized timestamp as a string - """ - return "%016.05f" % (float(timestamp)) - - def mkdirs(path): """ Ensures the path is a directory or makes it if not. Errors if the path @@ -515,28 +473,6 @@ def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False): return segs -def validate_device_partition(device, partition): - """ - Validate that a device and a partition are valid and won't lead to - directory traversal when used. - - :param device: device to validate - :param partition: partition to validate - :raises: ValueError if given an invalid device or partition - """ - invalid_device = False - invalid_partition = False - if not device or '/' in device or device in ['.', '..']: - invalid_device = True - if not partition or '/' in partition or partition in ['.', '..']: - invalid_partition = True - - if invalid_device: - raise ValueError('Invalid device: %s' % quote(device or '')) - elif invalid_partition: - raise ValueError('Invalid partition: %s' % quote(partition or '')) - - class GreenthreadSafeIterator(object): """ Wrap an iterator to ensure that only one greenthread is inside its next() @@ -1140,43 +1076,6 @@ def whataremyips(): return addresses -def storage_directory(datadir, partition, name_hash): - """ - Get the storage directory - - :param datadir: Base data directory - :param partition: Partition - :param name_hash: Account, container or object name hash - :returns: Storage directory - """ - return os.path.join(datadir, str(partition), name_hash[-3:], name_hash) - - -def hash_path(account, container=None, object=None, raw_digest=False): - """ - Get the canonical hash for an account/container/object - - :param account: Account - :param container: Container - :param object: Object - :param raw_digest: If True, return the raw version rather than a hex digest - :returns: hash string - """ - if object and not container: - raise ValueError('container is required if object is provided') - paths = [account] - if container: - paths.append(container) - if object: - paths.append(object) - if raw_digest: - return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths) - + HASH_PATH_SUFFIX).digest() - else: - return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths) - + HASH_PATH_SUFFIX).hexdigest() - - @contextmanager def lock_path(directory, timeout=10): """ @@ -1491,64 +1390,6 @@ def remove_file(path): pass -def audit_location_generator(devices, datadir, suffix='', - mount_check=True, logger=None): - ''' - Given a devices path and a data directory, yield (path, device, - partition) for all files in that directory - - :param devices: parent directory of the devices to be audited - :param datadir: a directory located under self.devices. This should be - one of the DATADIR constants defined in the account, - container, and object servers. - :param suffix: path name suffix required for all names returned - :param mount_check: Flag to check if a mount check should be performed - on devices - :param logger: a logger object - ''' - device_dir = listdir(devices) - # randomize devices in case of process restart before sweep completed - shuffle(device_dir) - for device in device_dir: - if mount_check and not \ - os.path.ismount(os.path.join(devices, device)): - if logger: - logger.debug( - _('Skipping %s as it is not mounted'), device) - continue - datadir_path = os.path.join(devices, device, datadir) - partitions = listdir(datadir_path) - for partition in partitions: - part_path = os.path.join(datadir_path, partition) - try: - suffixes = listdir(part_path) - except OSError as e: - if e.errno != errno.ENOTDIR: - raise - continue - for asuffix in suffixes: - suff_path = os.path.join(part_path, asuffix) - try: - hashes = listdir(suff_path) - except OSError as e: - if e.errno != errno.ENOTDIR: - raise - continue - for hsh in hashes: - hash_path = os.path.join(suff_path, hsh) - try: - files = sorted(listdir(hash_path), reverse=True) - except OSError as e: - if e.errno != errno.ENOTDIR: - raise - continue - for fname in files: - if suffix and not fname.endswith(suffix): - continue - path = os.path.join(hash_path, fname) - yield path, device, partition - - def ratelimit_sleep(running_time, max_rate, incr_by=1, rate_buffer=5): ''' Will eventlet.sleep() for the appropriate time so that the max_rate diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index d8f7d3bcec..f8ff8c2e82 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -35,7 +35,8 @@ from swift.common.swob import Request from swift.common.utils import capture_stdio, disable_fallocate, \ drop_privileges, get_logger, NullLogger, config_true_value, \ - validate_configuration, get_hub, config_auto_int_value + get_hub, config_auto_int_value +from swift.common.ondisk import validate_configuration try: import multiprocessing diff --git a/swift/container/auditor.py b/swift/container/auditor.py index df2266c076..bfcefbc53a 100644 --- a/swift/container/auditor.py +++ b/swift/container/auditor.py @@ -23,8 +23,9 @@ import swift.common.db from swift.container import server as container_server from swift.container.backend import ContainerBroker -from swift.common.utils import get_logger, audit_location_generator, \ - config_true_value, dump_recon_cache, ratelimit_sleep +from swift.common.utils import get_logger, config_true_value, \ + dump_recon_cache, ratelimit_sleep +from swift.common.ondisk import audit_location_generator from swift.common.daemon import Daemon diff --git a/swift/container/backend.py b/swift/container/backend.py index 1723c9c93c..f81b946b89 100644 --- a/swift/container/backend.py +++ b/swift/container/backend.py @@ -25,7 +25,8 @@ import sqlite3 -from swift.common.utils import normalize_timestamp, lock_parent_directory +from swift.common.utils import lock_parent_directory +from swift.common.ondisk import normalize_timestamp from swift.common.db import DatabaseBroker, DatabaseConnectionError, \ PENDING_CAP, PICKLE_PROTOCOL, utf8encode diff --git a/swift/container/server.py b/swift/container/server.py index 9013ce8c57..2e08bba890 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -29,9 +29,10 @@ from swift.common.db import DatabaseAlreadyExists from swift.common.request_helpers import get_param, get_listing_content_type, \ split_and_validate_path -from swift.common.utils import get_logger, hash_path, public, \ - normalize_timestamp, storage_directory, validate_sync_to, \ +from swift.common.utils import get_logger, public, validate_sync_to, \ config_true_value, json, timing_stats, replication, parse_content_type +from swift.common.ondisk import hash_path, normalize_timestamp, \ + storage_directory from swift.common.constraints import CONTAINER_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.bufferedhttp import http_connect diff --git a/swift/container/sync.py b/swift/container/sync.py index 759248417b..c014914b59 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -27,8 +27,9 @@ from swift.container.backend import ContainerBroker from swift.common.direct_client import direct_get_object from swift.common.ring import Ring -from swift.common.utils import audit_location_generator, get_logger, \ - hash_path, config_true_value, validate_sync_to, whataremyips, FileLikeIter +from swift.common.utils import get_logger, config_true_value, \ + validate_sync_to, whataremyips, FileLikeIter +from swift.common.ondisk import audit_location_generator, hash_path from swift.common.daemon import Daemon from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py index 78ab123474..cd674b7e79 100644 --- a/swift/obj/auditor.py +++ b/swift/obj/auditor.py @@ -21,8 +21,9 @@ from swift.obj import diskfile from swift.obj import server as object_server -from swift.common.utils import get_logger, audit_location_generator, \ - ratelimit_sleep, config_true_value, dump_recon_cache, list_from_csv, json +from swift.common.utils import get_logger, ratelimit_sleep, \ + config_true_value, dump_recon_cache, list_from_csv, json +from swift.common.ondisk import audit_location_generator from swift.common.exceptions import AuditException, DiskFileError, \ DiskFileNotExist from swift.common.daemon import Daemon diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py index cec98c728a..b8326d3e45 100644 --- a/swift/obj/diskfile.py +++ b/swift/obj/diskfile.py @@ -33,9 +33,10 @@ from swift import gettext_ as _ from swift.common.constraints import check_mount -from swift.common.utils import mkdirs, normalize_timestamp, \ - storage_directory, hash_path, renamer, fallocate, fsync, \ - fdatasync, drop_buffer_cache, ThreadPool, lock_path, write_pickle +from swift.common.utils import mkdirs, renamer, fallocate, fsync, fdatasync, \ + drop_buffer_cache, ThreadPool, lock_path, write_pickle +from swift.common.ondisk import hash_path, normalize_timestamp, \ + storage_directory from swift.common.exceptions import DiskFileError, DiskFileNotExist, \ DiskFileCollision, DiskFileNoSpace, DiskFileDeviceUnavailable, \ PathNotDir, DiskFileNotOpenError diff --git a/swift/obj/server.py b/swift/obj/server.py index 362f940197..a89c8ff2b7 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -27,9 +27,9 @@ from eventlet import sleep, Timeout -from swift.common.utils import mkdirs, normalize_timestamp, public, \ - hash_path, get_logger, write_pickle, config_true_value, timing_stats, \ - ThreadPool, replication +from swift.common.utils import mkdirs, public, get_logger, write_pickle, \ + config_true_value, timing_stats, ThreadPool, replication +from swift.common.ondisk import normalize_timestamp, hash_path from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, check_mount, \ check_float, check_utf8 diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 1e0b7146da..f1c1640cd4 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -37,9 +37,9 @@ from eventlet.timeout import Timeout from swift.common.wsgi import make_pre_authed_env -from swift.common.utils import normalize_timestamp, config_true_value, \ - public, split_path, list_from_csv, GreenthreadSafeIterator, \ - quorum_size +from swift.common.utils import config_true_value, public, split_path, \ + list_from_csv, GreenthreadSafeIterator, quorum_size +from swift.common.ondisk import normalize_timestamp from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ChunkReadTimeout, ConnectionTimeout from swift.common.http import is_informational, is_success, is_redirection, \ diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 09e4016e46..ee6afdac1c 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -37,9 +37,9 @@ from eventlet.queue import Queue from eventlet.timeout import Timeout -from swift.common.utils import ContextPool, normalize_timestamp, \ - config_true_value, public, json, csv_append, GreenthreadSafeIterator, \ - quorum_size +from swift.common.utils import ContextPool, config_true_value, public, json, \ + csv_append, GreenthreadSafeIterator, quorum_size +from swift.common.ondisk import normalize_timestamp from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_metadata, check_object_creation, \ CONTAINER_LISTING_LIMIT, MAX_FILE_SIZE, MAX_BUFFERED_SLO_SEGMENTS diff --git a/test/unit/account/test_backend.py b/test/unit/account/test_backend.py index 379598ba5f..2f59f9a4a6 100644 --- a/test/unit/account/test_backend.py +++ b/test/unit/account/test_backend.py @@ -22,7 +22,7 @@ from uuid import uuid4 from swift.account.backend import AccountBroker -from swift.common.utils import normalize_timestamp +from swift.common.ondisk import normalize_timestamp class TestAccountBroker(unittest.TestCase): diff --git a/test/unit/account/test_reaper.py b/test/unit/account/test_reaper.py index e0f438b80c..81ea63b5e2 100644 --- a/test/unit/account/test_reaper.py +++ b/test/unit/account/test_reaper.py @@ -25,7 +25,7 @@ from swift.account import reaper from swift.account.server import DATADIR -from swift.common.utils import normalize_timestamp +from swift.common.ondisk import normalize_timestamp from swift.common.direct_client import ClientException diff --git a/test/unit/account/test_server.py b/test/unit/account/test_server.py index 8461071409..ba56b6d85b 100644 --- a/test/unit/account/test_server.py +++ b/test/unit/account/test_server.py @@ -25,7 +25,8 @@ from swift.common.swob import Request from swift.account.server import AccountController, ACCOUNT_LISTING_LIMIT -from swift.common.utils import normalize_timestamp, replication, public +from swift.common.utils import replication, public +from swift.common.ondisk import normalize_timestamp class TestAccountController(unittest.TestCase): diff --git a/test/unit/common/middleware/test_list_endpoints.py b/test/unit/common/middleware/test_list_endpoints.py index 6d9f6d3d44..8da02382ae 100644 --- a/test/unit/common/middleware/test_list_endpoints.py +++ b/test/unit/common/middleware/test_list_endpoints.py @@ -18,7 +18,7 @@ from shutil import rmtree import os -from swift.common import ring, utils +from swift.common import ring, ondisk from swift.common.utils import json from swift.common.swob import Request, Response from swift.common.middleware import list_endpoints @@ -35,8 +35,8 @@ def start_response(*args): class TestListEndpoints(unittest.TestCase): def setUp(self): - utils.HASH_PATH_SUFFIX = 'endcap' - utils.HASH_PATH_PREFIX = '' + ondisk.HASH_PATH_SUFFIX = 'endcap' + ondisk.HASH_PATH_PREFIX = '' self.testdir = os.path.join(os.path.dirname(__file__), 'ring') rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) diff --git a/test/unit/common/ring/test_ring.py b/test/unit/common/ring/test_ring.py index 3d5d8f2b1c..d47ee564f0 100644 --- a/test/unit/common/ring/test_ring.py +++ b/test/unit/common/ring/test_ring.py @@ -23,7 +23,7 @@ from shutil import rmtree from time import sleep, time -from swift.common import ring, utils +from swift.common import ring, ondisk class TestRingData(unittest.TestCase): @@ -99,8 +99,8 @@ def test_deterministic_serialization(self): class TestRing(unittest.TestCase): def setUp(self): - utils.HASH_PATH_SUFFIX = 'endcap' - utils.HASH_PATH_PREFIX = '' + ondisk.HASH_PATH_SUFFIX = 'endcap' + ondisk.HASH_PATH_PREFIX = '' self.testdir = os.path.join(os.path.dirname(__file__), 'ring') rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) @@ -146,15 +146,15 @@ def test_creation(self): self.assertEquals(self.ring.reload_time, self.intended_reload_time) self.assertEquals(self.ring.serialized_path, self.testgz) # test invalid endcap - _orig_hash_path_suffix = utils.HASH_PATH_SUFFIX - _orig_hash_path_prefix = utils.HASH_PATH_PREFIX + _orig_hash_path_suffix = ondisk.HASH_PATH_SUFFIX + _orig_hash_path_prefix = ondisk.HASH_PATH_PREFIX try: - utils.HASH_PATH_SUFFIX = '' - utils.HASH_PATH_PREFIX = '' + ondisk.HASH_PATH_SUFFIX = '' + ondisk.HASH_PATH_PREFIX = '' self.assertRaises(SystemExit, ring.Ring, self.testdir, 'whatever') finally: - utils.HASH_PATH_SUFFIX = _orig_hash_path_suffix - utils.HASH_PATH_PREFIX = _orig_hash_path_prefix + ondisk.HASH_PATH_SUFFIX = _orig_hash_path_suffix + ondisk.HASH_PATH_PREFIX = _orig_hash_path_prefix def test_has_changed(self): self.assertEquals(self.ring.has_changed(), False) diff --git a/test/unit/common/test_daemon.py b/test/unit/common/test_daemon.py index a0eb6caecb..246610bdf4 100644 --- a/test/unit/common/test_daemon.py +++ b/test/unit/common/test_daemon.py @@ -23,7 +23,7 @@ from test.unit import tmpfile from mock import patch -from swift.common import daemon, utils +from swift.common import daemon, utils, ondisk class MyDaemon(daemon.Daemon): @@ -63,8 +63,8 @@ def test_stubs(self): class TestRunDaemon(unittest.TestCase): def setUp(self): - utils.HASH_PATH_SUFFIX = 'endcap' - utils.HASH_PATH_PREFIX = 'startcap' + ondisk.HASH_PATH_SUFFIX = 'endcap' + ondisk.HASH_PATH_PREFIX = 'startcap' utils.drop_privileges = lambda *args: None utils.capture_stdio = lambda *args: None diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py index 5bd3fb9c1c..1ae67df3c1 100644 --- a/test/unit/common/test_db.py +++ b/test/unit/common/test_db.py @@ -28,7 +28,7 @@ import swift.common.db from swift.common.db import chexor, dict_factory, get_db_connection, \ DatabaseBroker, DatabaseConnectionError, DatabaseAlreadyExists -from swift.common.utils import normalize_timestamp +from swift.common.ondisk import normalize_timestamp from swift.common.exceptions import LockTimeout diff --git a/test/unit/common/test_db_replicator.py b/test/unit/common/test_db_replicator.py index 04173f5585..6913b691eb 100644 --- a/test/unit/common/test_db_replicator.py +++ b/test/unit/common/test_db_replicator.py @@ -26,7 +26,7 @@ import simplejson from swift.common import db_replicator -from swift.common.utils import normalize_timestamp +from swift.common.ondisk import normalize_timestamp from swift.container import server as container_server from swift.common.exceptions import DriveNotMounted diff --git a/test/unit/common/test_ondisk.py b/test/unit/common/test_ondisk.py new file mode 100644 index 0000000000..394d008ed0 --- /dev/null +++ b/test/unit/common/test_ondisk.py @@ -0,0 +1,139 @@ +# Copyright (c) 2010-2013 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for swift.common.ondisk""" + +from __future__ import with_statement +from test.unit import temptree + +import os + +import unittest + +from swift.common import ondisk + + +class TestOndisk(unittest.TestCase): + """Tests for swift.common.ondisk""" + + def setUp(self): + ondisk.HASH_PATH_SUFFIX = 'endcap' + ondisk.HASH_PATH_PREFIX = 'startcap' + + def test_normalize_timestamp(self): + # Test swift.common.ondisk.normalize_timestamp + self.assertEquals(ondisk.normalize_timestamp('1253327593.48174'), + "1253327593.48174") + self.assertEquals(ondisk.normalize_timestamp(1253327593.48174), + "1253327593.48174") + self.assertEquals(ondisk.normalize_timestamp('1253327593.48'), + "1253327593.48000") + self.assertEquals(ondisk.normalize_timestamp(1253327593.48), + "1253327593.48000") + self.assertEquals(ondisk.normalize_timestamp('253327593.48'), + "0253327593.48000") + self.assertEquals(ondisk.normalize_timestamp(253327593.48), + "0253327593.48000") + self.assertEquals(ondisk.normalize_timestamp('1253327593'), + "1253327593.00000") + self.assertEquals(ondisk.normalize_timestamp(1253327593), + "1253327593.00000") + self.assertRaises(ValueError, ondisk.normalize_timestamp, '') + self.assertRaises(ValueError, ondisk.normalize_timestamp, 'abc') + + def test_validate_device_partition(self): + # Test swift.common.ondisk.validate_device_partition + ondisk.validate_device_partition('foo', 'bar') + self.assertRaises(ValueError, + ondisk.validate_device_partition, '', '') + self.assertRaises(ValueError, + ondisk.validate_device_partition, '', 'foo') + self.assertRaises(ValueError, + ondisk.validate_device_partition, 'foo', '') + self.assertRaises(ValueError, + ondisk.validate_device_partition, 'foo/bar', 'foo') + self.assertRaises(ValueError, + ondisk.validate_device_partition, 'foo', 'foo/bar') + self.assertRaises(ValueError, + ondisk.validate_device_partition, '.', 'foo') + self.assertRaises(ValueError, + ondisk.validate_device_partition, '..', 'foo') + self.assertRaises(ValueError, + ondisk.validate_device_partition, 'foo', '.') + self.assertRaises(ValueError, + ondisk.validate_device_partition, 'foo', '..') + try: + ondisk.validate_device_partition('o\nn e', 'foo') + except ValueError as err: + self.assertEquals(str(err), 'Invalid device: o%0An%20e') + try: + ondisk.validate_device_partition('foo', 'o\nn e') + except ValueError as err: + self.assertEquals(str(err), 'Invalid partition: o%0An%20e') + + def test_storage_directory(self): + self.assertEquals(ondisk.storage_directory('objects', '1', 'ABCDEF'), + 'objects/1/DEF/ABCDEF') + + def test_hash_path(self): + _prefix = ondisk.HASH_PATH_PREFIX + ondisk.HASH_PATH_PREFIX = '' + # Yes, these tests are deliberately very fragile. We want to make sure + # that if someones changes the results hash_path produces, they know it + try: + self.assertEquals(ondisk.hash_path('a'), + '1c84525acb02107ea475dcd3d09c2c58') + self.assertEquals(ondisk.hash_path('a', 'c'), + '33379ecb053aa5c9e356c68997cbb59e') + self.assertEquals(ondisk.hash_path('a', 'c', 'o'), + '06fbf0b514e5199dfc4e00f42eb5ea83') + self.assertEquals( + ondisk.hash_path('a', 'c', 'o', raw_digest=False), + '06fbf0b514e5199dfc4e00f42eb5ea83') + self.assertEquals(ondisk.hash_path('a', 'c', 'o', raw_digest=True), + '\x06\xfb\xf0\xb5\x14\xe5\x19\x9d\xfcN' + '\x00\xf4.\xb5\xea\x83') + self.assertRaises(ValueError, ondisk.hash_path, 'a', object='o') + ondisk.HASH_PATH_PREFIX = 'abcdef' + self.assertEquals( + ondisk.hash_path('a', 'c', 'o', raw_digest=False), + '363f9b535bfb7d17a43a46a358afca0e') + finally: + ondisk.HASH_PATH_PREFIX = _prefix + + +class TestAuditLocationGenerator(unittest.TestCase): + def test_non_dir_contents(self): + with temptree([]) as tmpdir: + data = os.path.join(tmpdir, "drive", "data") + os.makedirs(data) + with open(os.path.join(data, "partition1"), "w"): + pass + partition = os.path.join(data, "partition2") + os.makedirs(partition) + with open(os.path.join(partition, "suffix1"), "w"): + pass + suffix = os.path.join(partition, "suffix2") + os.makedirs(suffix) + with open(os.path.join(suffix, "hash1"), "w"): + pass + locations = ondisk.audit_location_generator( + tmpdir, "data", mount_check=False + ) + self.assertEqual(list(locations), []) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index cf1d1a9155..c9408eab52 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -132,31 +132,6 @@ def reset_loggers(): class TestUtils(unittest.TestCase): """Tests for swift.common.utils """ - def setUp(self): - utils.HASH_PATH_SUFFIX = 'endcap' - utils.HASH_PATH_PREFIX = 'startcap' - - def test_normalize_timestamp(self): - # Test swift.common.utils.normalize_timestamp - self.assertEquals(utils.normalize_timestamp('1253327593.48174'), - "1253327593.48174") - self.assertEquals(utils.normalize_timestamp(1253327593.48174), - "1253327593.48174") - self.assertEquals(utils.normalize_timestamp('1253327593.48'), - "1253327593.48000") - self.assertEquals(utils.normalize_timestamp(1253327593.48), - "1253327593.48000") - self.assertEquals(utils.normalize_timestamp('253327593.48'), - "0253327593.48000") - self.assertEquals(utils.normalize_timestamp(253327593.48), - "0253327593.48000") - self.assertEquals(utils.normalize_timestamp('1253327593'), - "1253327593.00000") - self.assertEquals(utils.normalize_timestamp(1253327593), - "1253327593.00000") - self.assertRaises(ValueError, utils.normalize_timestamp, '') - self.assertRaises(ValueError, utils.normalize_timestamp, 'abc') - def test_backwards(self): # Test swift.common.utils.backward @@ -248,36 +223,6 @@ def test_split_path(self): except ValueError as err: self.assertEquals(str(err), 'Invalid path: o%0An%20e') - def test_validate_device_partition(self): - # Test swift.common.utils.validate_device_partition - utils.validate_device_partition('foo', 'bar') - self.assertRaises(ValueError, - utils.validate_device_partition, '', '') - self.assertRaises(ValueError, - utils.validate_device_partition, '', 'foo') - self.assertRaises(ValueError, - utils.validate_device_partition, 'foo', '') - self.assertRaises(ValueError, - utils.validate_device_partition, 'foo/bar', 'foo') - self.assertRaises(ValueError, - utils.validate_device_partition, 'foo', 'foo/bar') - self.assertRaises(ValueError, - utils.validate_device_partition, '.', 'foo') - self.assertRaises(ValueError, - utils.validate_device_partition, '..', 'foo') - self.assertRaises(ValueError, - utils.validate_device_partition, 'foo', '.') - self.assertRaises(ValueError, - utils.validate_device_partition, 'foo', '..') - try: - utils.validate_device_partition('o\nn e', 'foo') - except ValueError as err: - self.assertEquals(str(err), 'Invalid device: o%0An%20e') - try: - utils.validate_device_partition('foo', 'o\nn e') - except ValueError as err: - self.assertEquals(str(err), 'Invalid partition: o%0An%20e') - def test_NullLogger(self): # Test swift.common.utils.NullLogger sio = StringIO() @@ -637,10 +582,6 @@ def strip_value(sio): logger.logger.removeHandler(handler) reset_loggers() - def test_storage_directory(self): - self.assertEquals(utils.storage_directory('objects', '1', 'ABCDEF'), - 'objects/1/DEF/ABCDEF') - def test_whataremyips(self): myips = utils.whataremyips() self.assert_(len(myips) > 1) @@ -676,30 +617,6 @@ def my_ipv6_ifaddresses(interface): self.assertEquals(len(myips), 1) self.assertEquals(myips[0], test_ipv6_address) - def test_hash_path(self): - _prefix = utils.HASH_PATH_PREFIX - utils.HASH_PATH_PREFIX = '' - # Yes, these tests are deliberately very fragile. We want to make sure - # that if someones changes the results hash_path produces, they know it - try: - self.assertEquals(utils.hash_path('a'), - '1c84525acb02107ea475dcd3d09c2c58') - self.assertEquals(utils.hash_path('a', 'c'), - '33379ecb053aa5c9e356c68997cbb59e') - self.assertEquals(utils.hash_path('a', 'c', 'o'), - '06fbf0b514e5199dfc4e00f42eb5ea83') - self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=False), - '06fbf0b514e5199dfc4e00f42eb5ea83') - self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=True), - '\x06\xfb\xf0\xb5\x14\xe5\x19\x9d\xfcN' - '\x00\xf4.\xb5\xea\x83') - self.assertRaises(ValueError, utils.hash_path, 'a', object='o') - utils.HASH_PATH_PREFIX = 'abcdef' - self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=False), - '363f9b535bfb7d17a43a46a358afca0e') - finally: - utils.HASH_PATH_PREFIX = _prefix - def test_load_libc_function(self): self.assert_(callable( utils.load_libc_function('printf'))) @@ -2401,26 +2318,5 @@ def test_force_run_in_thread_without_threads(self): self.assertTrue(caught) -class TestAuditLocationGenerator(unittest.TestCase): - def test_non_dir_contents(self): - with temptree([]) as tmpdir: - data = os.path.join(tmpdir, "drive", "data") - os.makedirs(data) - with open(os.path.join(data, "partition1"), "w"): - pass - partition = os.path.join(data, "partition2") - os.makedirs(partition) - with open(os.path.join(partition, "suffix1"), "w"): - pass - suffix = os.path.join(partition, "suffix2") - os.makedirs(suffix) - with open(os.path.join(suffix, "hash1"), "w"): - pass - locations = utils.audit_location_generator( - tmpdir, "data", mount_check=False - ) - self.assertEqual(list(locations), []) - - if __name__ == '__main__': unittest.main() diff --git a/test/unit/container/test_backend.py b/test/unit/container/test_backend.py index 3bda3ccef3..ca7a841fa9 100644 --- a/test/unit/container/test_backend.py +++ b/test/unit/container/test_backend.py @@ -22,7 +22,7 @@ from uuid import uuid4 from swift.container.backend import ContainerBroker -from swift.common.utils import normalize_timestamp +from swift.common.ondisk import normalize_timestamp class TestContainerBroker(unittest.TestCase): diff --git a/test/unit/container/test_replicator.py b/test/unit/container/test_replicator.py index 9f1b0259a8..cb31a0e5f0 100644 --- a/test/unit/container/test_replicator.py +++ b/test/unit/container/test_replicator.py @@ -15,7 +15,7 @@ import unittest from swift.container import replicator -from swift.common.utils import normalize_timestamp +from swift.common.ondisk import normalize_timestamp class TestReplicator(unittest.TestCase): diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index dddff8f6f8..627d67cdac 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -29,7 +29,8 @@ from swift.common.swob import Request, HeaderKeyDict import swift.container from swift.container import server as container_server -from swift.common.utils import normalize_timestamp, mkdirs, public, replication +from swift.common.utils import mkdirs, public, replication +from swift.common.ondisk import normalize_timestamp from test.unit import fake_http_connect diff --git a/test/unit/container/test_updater.py b/test/unit/container/test_updater.py index 6b6030bd86..68bc9dd695 100644 --- a/test/unit/container/test_updater.py +++ b/test/unit/container/test_updater.py @@ -28,7 +28,7 @@ from swift.container import server as container_server from swift.container.backend import ContainerBroker from swift.common.ring import RingData -from swift.common.utils import normalize_timestamp +from swift.common.ondisk import normalize_timestamp class TestContainerUpdater(unittest.TestCase): diff --git a/test/unit/obj/test_auditor.py b/test/unit/obj/test_auditor.py index bbd7070ba8..deca216eae 100644 --- a/test/unit/obj/test_auditor.py +++ b/test/unit/obj/test_auditor.py @@ -25,7 +25,8 @@ from swift.obj import auditor from swift.obj.diskfile import DiskFile, write_metadata, invalidate_hash from swift.obj.server import DATADIR -from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \ +from swift.common.utils import mkdirs +from swift.common.ondisk import hash_path, normalize_timestamp, \ storage_directory diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py index 7d0cbb58ea..c5db32a668 100644 --- a/test/unit/obj/test_diskfile.py +++ b/test/unit/obj/test_diskfile.py @@ -36,8 +36,9 @@ from test.unit import FakeLogger, mock as unit_mock from test.unit import _setxattr as setxattr from swift.obj import diskfile -from swift.common import utils -from swift.common.utils import hash_path, mkdirs, normalize_timestamp +from swift.common import ondisk +from swift.common.utils import mkdirs +from swift.common.ondisk import hash_path, normalize_timestamp from swift.common import ring from swift.common.exceptions import DiskFileNotExist, DiskFileDeviceUnavailable @@ -72,8 +73,8 @@ def _create_test_ring(path): class TestDiskFileModuleMethods(unittest.TestCase): def setUp(self): - utils.HASH_PATH_SUFFIX = 'endcap' - utils.HASH_PATH_PREFIX = '' + ondisk.HASH_PATH_SUFFIX = 'endcap' + ondisk.HASH_PATH_PREFIX = '' # Setup a test ring (stolen from common/test_ring.py) self.testdir = tempfile.mkdtemp() self.devices = os.path.join(self.testdir, 'node') diff --git a/test/unit/obj/test_replicator.py b/test/unit/obj/test_replicator.py index c8805494f5..ac79f717cb 100644 --- a/test/unit/obj/test_replicator.py +++ b/test/unit/obj/test_replicator.py @@ -29,8 +29,9 @@ from eventlet import Timeout, tpool from test.unit import FakeLogger -from swift.common import utils -from swift.common.utils import hash_path, mkdirs, normalize_timestamp +from swift.common.utils import mkdirs +from swift.common import ondisk +from swift.common.ondisk import hash_path, normalize_timestamp from swift.common import ring from swift.obj import diskfile, replicator as object_replicator @@ -137,8 +138,8 @@ def _create_test_ring(path): class TestObjectReplicator(unittest.TestCase): def setUp(self): - utils.HASH_PATH_SUFFIX = 'endcap' - utils.HASH_PATH_PREFIX = '' + ondisk.HASH_PATH_SUFFIX = 'endcap' + ondisk.HASH_PATH_PREFIX = '' # Setup a test ring (stolen from common/test_ring.py) self.testdir = tempfile.mkdtemp() self.devices = os.path.join(self.testdir, 'node') diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 17f82e8a19..638ccb5793 100755 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -32,9 +32,10 @@ from test.unit import connect_tcp, readuntil2crlfs from swift.obj import server as object_server from swift.obj import diskfile -from swift.common import utils -from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \ - NullLogger, storage_directory, public, replication +from swift.common.utils import mkdirs, NullLogger, public, replication +from swift.common.ondisk import hash_path, normalize_timestamp, \ + storage_directory +from swift.common import ondisk from swift.common import constraints from eventlet import tpool from swift.common.swob import Request, HeaderKeyDict @@ -45,8 +46,8 @@ class TestObjectController(unittest.TestCase): def setUp(self): """Set up for testing swift.object.server.ObjectController""" - utils.HASH_PATH_SUFFIX = 'endcap' - utils.HASH_PATH_PREFIX = 'startcap' + ondisk.HASH_PATH_SUFFIX = 'endcap' + ondisk.HASH_PATH_PREFIX = 'startcap' self.testdir = \ os.path.join(mkdtemp(), 'tmp_test_object_server_ObjectController') mkdirs(os.path.join(self.testdir, 'sda1', 'tmp')) @@ -1904,8 +1905,8 @@ def read(self): 'x-trans-id': '-'})}) def test_async_update_saves_on_exception(self): - _prefix = utils.HASH_PATH_PREFIX - utils.HASH_PATH_PREFIX = '' + _prefix = ondisk.HASH_PATH_PREFIX + ondisk.HASH_PATH_PREFIX = '' def fake_http_connect(*args): raise Exception('test') @@ -1918,7 +1919,7 @@ def fake_http_connect(*args): {'x-timestamp': '1', 'x-out': 'set'}, 'sda1') finally: object_server.http_connect = orig_http_connect - utils.HASH_PATH_PREFIX = _prefix + ondisk.HASH_PATH_PREFIX = _prefix self.assertEquals( pickle.load(open(os.path.join( self.testdir, 'sda1', 'async_pending', 'a83', @@ -1928,8 +1929,8 @@ def fake_http_connect(*args): 'account': 'a', 'container': 'c', 'obj': 'o', 'op': 'PUT'}) def test_async_update_saves_on_non_2xx(self): - _prefix = utils.HASH_PATH_PREFIX - utils.HASH_PATH_PREFIX = '' + _prefix = ondisk.HASH_PATH_PREFIX + ondisk.HASH_PATH_PREFIX = '' def fake_http_connect(status): @@ -1963,7 +1964,7 @@ def read(self): 'op': 'PUT'}) finally: object_server.http_connect = orig_http_connect - utils.HASH_PATH_PREFIX = _prefix + ondisk.HASH_PATH_PREFIX = _prefix def test_async_update_does_not_save_on_2xx(self): diff --git a/test/unit/obj/test_updater.py b/test/unit/obj/test_updater.py index ebfac25bbb..97a07d059b 100644 --- a/test/unit/obj/test_updater.py +++ b/test/unit/obj/test_updater.py @@ -27,17 +27,17 @@ from swift.obj import updater as object_updater, server as object_server from swift.obj.server import ASYNCDIR from swift.common.ring import RingData -from swift.common import utils -from swift.common.utils import hash_path, normalize_timestamp, mkdirs, \ - write_pickle +from swift.common.utils import mkdirs, write_pickle +from swift.common import ondisk +from swift.common.ondisk import hash_path, normalize_timestamp from test.unit import FakeLogger class TestObjectUpdater(unittest.TestCase): def setUp(self): - utils.HASH_PATH_SUFFIX = 'endcap' - utils.HASH_PATH_PREFIX = '' + ondisk.HASH_PATH_SUFFIX = 'endcap' + ondisk.HASH_PATH_PREFIX = '' self.testdir = os.path.join(os.path.dirname(__file__), 'object_updater') rmtree(self.testdir, ignore_errors=1) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 75fd79b3f4..6a32dd30a1 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -47,7 +47,9 @@ MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \ MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH from swift.common import utils -from swift.common.utils import mkdirs, normalize_timestamp, NullLogger +from swift.common.utils import mkdirs, NullLogger +from swift.common import ondisk +from swift.common.ondisk import normalize_timestamp from swift.common.wsgi import monkey_patch_mimetools from swift.proxy.controllers.obj import SegmentedIterable from swift.proxy.controllers.base import get_container_memcache_key, \ @@ -73,7 +75,7 @@ def request_init(self, *args, **kwargs): def setup(): - utils.HASH_PATH_SUFFIX = 'endcap' + ondisk.HASH_PATH_SUFFIX = 'endcap' global _testdir, _test_servers, _test_sockets, \ _orig_container_listing_limit, _test_coros, _orig_SysLogHandler _orig_SysLogHandler = utils.SysLogHandler From a6443f7068cdeaeb680f92849241be042fdc561d Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Tue, 17 Sep 2013 12:24:29 +0530 Subject: [PATCH 24/35] Add HTTPSeeOther(303) and HTTPTemporaryRedirect(307) to swob RFC 1945 and RFC 2068 specify that the client is not allowed to change the method on the redirected request. However, most existing user agent implementations treat 302 as if it were a 303 response, performing a GET on the Location field-value regardless of the original request method. The status codes 303 and 307 have been added for servers that wish to make unambiguously clear which kind of reaction is expected of the client. HTTP/1.1 RFC for 302: http://tools.ietf.org/html/rfc2616#section-10.3.4 HTTP/1.1 RFC for 307: http://tools.ietf.org/html/rfc2616#section-10.3.8 Change-Id: I354e2f4f3e3eb6a1553b3d9c60b613d8f0c37531 Signed-off-by: Prashanth Pai --- swift/common/swob.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/swift/common/swob.py b/swift/common/swob.py index 73efa60700..a8114bb38d 100644 --- a/swift/common/swob.py +++ b/swift/common/swob.py @@ -60,7 +60,9 @@ 204: ('No Content', ''), 206: ('Partial Content', ''), 301: ('Moved Permanently', 'The resource has moved permanently.'), - 302: ('Found', ''), + 302: ('Found', 'The resource has moved temporarily.'), + 303: ('See Other', 'The response to the request can be found under a ' + 'different URI.'), 304: ('Not Modified', ''), 307: ('Temporary Redirect', 'The resource has moved temporarily.'), 400: ('Bad Request', 'The server could not comply with the request since ' @@ -1213,7 +1215,9 @@ def __getitem__(self, key): HTTPNoContent = status_map[204] HTTPMovedPermanently = status_map[301] HTTPFound = status_map[302] +HTTPSeeOther = status_map[303] HTTPNotModified = status_map[304] +HTTPTemporaryRedirect = status_map[307] HTTPBadRequest = status_map[400] HTTPUnauthorized = status_map[401] HTTPForbidden = status_map[403] From 58efcb3b3e70ba0502504124a1348ee14cc2687c Mon Sep 17 00:00:00 2001 From: gholt Date: Thu, 19 Sep 2013 17:14:35 +0000 Subject: [PATCH 25/35] Fix probe tests Change-Id: I03573bf24baf031b1874c3ff2e2d89d34473c266 --- test/probe/test_container_failures.py | 3 ++- test/probe/test_object_failures.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/probe/test_container_failures.py b/test/probe/test_container_failures.py index fe0feeed4d..7c2ba673ce 100755 --- a/test/probe/test_container_failures.py +++ b/test/probe/test_container_failures.py @@ -25,7 +25,8 @@ from swiftclient import client from swift.common import direct_client -from swift.common.utils import hash_path, readconf +from swift.common.ondisk import hash_path +from swift.common.utils import readconf from test.probe.common import get_to_final_state, kill_nonprimary_server, \ kill_server, kill_servers, reset_environment, start_server diff --git a/test/probe/test_object_failures.py b/test/probe/test_object_failures.py index e7f6bf5bf1..b46dd330bb 100755 --- a/test/probe/test_object_failures.py +++ b/test/probe/test_object_failures.py @@ -22,7 +22,8 @@ from swiftclient import client from swift.common import direct_client -from swift.common.utils import hash_path, readconf +from swift.common.ondisk import hash_path +from swift.common.utils import readconf from swift.obj.diskfile import write_metadata, read_metadata from test.probe.common import kill_servers, reset_environment From 285a3a88a1ede50014d4f4a124994a2a0e85705b Mon Sep 17 00:00:00 2001 From: Greg Lange Date: Tue, 17 Sep 2013 19:24:24 +0000 Subject: [PATCH 26/35] add seek() to CompressingFileReader Change-Id: I33d9119684497b53101db3bc380953e86bdf25f0 --- swift/common/internal_client.py | 20 +++++++++++++++--- test/unit/common/test_internal_client.py | 26 +++++++++++++++++++++++- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/swift/common/internal_client.py b/swift/common/internal_client.py index ff2bc67d51..53ca049d96 100644 --- a/swift/common/internal_client.py +++ b/swift/common/internal_client.py @@ -56,14 +56,23 @@ class CompressingFileReader(object): def __init__(self, file_obj, compresslevel=9, chunk_size=4096): self._f = file_obj + self.compresslevel = compresslevel + self.chunk_size = chunk_size + self.set_initial_state() + + def set_initial_state(self): + """ + Sets the object to the state needed for the first read. + """ + + self._f.seek(0) self._compressor = compressobj( - compresslevel, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, - 0) + self.compresslevel, zlib.DEFLATED, -zlib.MAX_WBITS, + zlib.DEF_MEM_LEVEL, 0) self.done = False self.first = True self.crc32 = 0 self.total_size = 0 - self.chunk_size = chunk_size def read(self, *a, **kw): """ @@ -105,6 +114,11 @@ def next(self): return chunk raise StopIteration + def seek(self, offset, whence=0): + if not (offset == 0 and whence == 0): + raise NotImplementedError('Seek implemented on offset 0 only') + self.set_initial_state() + class InternalClient(object): """ diff --git a/test/unit/common/test_internal_client.py b/test/unit/common/test_internal_client.py index f4d2504c8e..9569ad66de 100644 --- a/test/unit/common/test_internal_client.py +++ b/test/unit/common/test_internal_client.py @@ -141,7 +141,8 @@ def method(self, *args): def test_read(self): exp_data = 'abcdefghijklmnopqrstuvwxyz' - fobj = internal_client.CompressingFileReader(StringIO(exp_data)) + fobj = internal_client.CompressingFileReader( + StringIO(exp_data), chunk_size=5) data = '' d = zlib.decompressobj(16 + zlib.MAX_WBITS) @@ -150,6 +151,29 @@ def test_read(self): self.assertEquals(exp_data, data) + def test_seek(self): + exp_data = 'abcdefghijklmnopqrstuvwxyz' + fobj = internal_client.CompressingFileReader( + StringIO(exp_data), chunk_size=5) + + # read a couple of chunks only + for _ in range(2): + fobj.read() + + # read whole thing after seek and check data + fobj.seek(0) + data = '' + d = zlib.decompressobj(16 + zlib.MAX_WBITS) + for chunk in fobj.read(): + data += d.decompress(chunk) + self.assertEquals(exp_data, data) + + def test_seek_not_implemented_exception(self): + fobj = internal_client.CompressingFileReader( + StringIO(''), chunk_size=5) + self.assertRaises(NotImplementedError, fobj.seek, 10) + self.assertRaises(NotImplementedError, fobj.seek, 0, 10) + class TestInternalClient(unittest.TestCase): def test_init(self): From 4a5c2fa0c6865f8a6dde2cd08ceb08c31ae38fa9 Mon Sep 17 00:00:00 2001 From: gholt Date: Thu, 19 Sep 2013 21:05:46 +0000 Subject: [PATCH 27/35] Log x-copy-from when it could be useful Change-Id: Ia28a9b47213f848ab5ea59572e14ac710ed881e3 --- swift/proxy/controllers/obj.py | 3 +++ test/unit/proxy/controllers/test_obj.py | 33 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index ee6afdac1c..741cb9e6e0 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -928,6 +928,9 @@ def PUT(self, req): source_header = req.headers.get('X-Copy-From') source_resp = None if source_header: + if req.environ.get('swift.orig_req_method', req.method) != 'POST': + req.environ.setdefault('swift.log_info', []).append( + 'x-copy-from:%s' % source_header) source_header = unquote(source_header) acct = req.path_info.split('/', 2)[1] if isinstance(acct, unicode): diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index 707d952d39..a3f3539ff1 100755 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -17,6 +17,8 @@ import unittest from contextlib import contextmanager +import mock + import swift from swift.proxy import server as proxy_server from test.unit import FakeRing, FakeMemcache, fake_http_connect @@ -85,5 +87,36 @@ def test_connect_put_node_timeout(self): res = controller._connect_put_node(nodes, '', '', {}, ('', '')) self.assertTrue(res is None) + +class TestObjController(unittest.TestCase): + + def test_PUT_log_info(self): + # mock out enough to get to the area of the code we want to test + with mock.patch('swift.proxy.controllers.obj.check_object_creation', + mock.MagicMock(return_value=None)): + app = mock.MagicMock() + app.container_ring.get_nodes.return_value = (1, [2]) + app.object_ring.get_nodes.return_value = (1, [2]) + controller = proxy_server.ObjectController(app, 'a', 'c', 'o') + controller.container_info = mock.MagicMock(return_value={ + 'partition': 1, + 'nodes': [{}], + 'write_acl': None, + 'sync_key': None, + 'versions': None}) + # and now test that we add the header to log_info + req = swift.common.swob.Request.blank('/v1/a/c/o') + req.headers['x-copy-from'] = 'somewhere' + controller.PUT(req) + self.assertEquals( + req.environ.get('swift.log_info'), ['x-copy-from:somewhere']) + # and then check that we don't do that for originating POSTs + req = swift.common.swob.Request.blank('/v1/a/c/o') + req.method = 'POST' + req.headers['x-copy-from'] = 'elsewhere' + controller.PUT(req) + self.assertEquals(req.environ.get('swift.log_info'), None) + + if __name__ == '__main__': unittest.main() From 01f58d68262aab1ac63683c9fac91b4da764aa92 Mon Sep 17 00:00:00 2001 From: David Goetz Date: Tue, 10 Sep 2013 09:01:32 -0700 Subject: [PATCH 28/35] SLOs broken for range requests Change-Id: I21175a4be0cda9a8a98c425bff11c80895cd6d3e --- swift/common/utils.py | 18 +++ swift/container/server.py | 17 +-- swift/proxy/controllers/obj.py | 193 ++++++++++++++++++++------------- test/unit/common/test_utils.py | 20 ++++ test/unit/proxy/test_server.py | 119 ++++++++++++++++++-- 5 files changed, 270 insertions(+), 97 deletions(-) diff --git a/swift/common/utils.py b/swift/common/utils.py index 8173cbbdd2..94bac3b9f4 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -2076,6 +2076,24 @@ def parse_content_type(content_type): return content_type, parm_list +def override_bytes_from_content_type(listing_dict, logger=None): + """ + Takes a dict from a container listing and overrides the content_type, + bytes fields if swift_bytes is set. + """ + content_type, params = parse_content_type(listing_dict['content_type']) + for key, value in params: + if key == 'swift_bytes': + try: + listing_dict['bytes'] = int(value) + except ValueError: + if logger: + logger.exception("Invalid swift_bytes") + else: + content_type += ';%s=%s' % (key, value) + listing_dict['content_type'] = content_type + + def quote(value, safe='/'): """ Patched version of urllib.quote that encodes utf-8 strings before quoting diff --git a/swift/container/server.py b/swift/container/server.py index 2e08bba890..084b068943 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -30,7 +30,8 @@ from swift.common.request_helpers import get_param, get_listing_content_type, \ split_and_validate_path from swift.common.utils import get_logger, public, validate_sync_to, \ - config_true_value, json, timing_stats, replication, parse_content_type + config_true_value, json, timing_stats, replication, \ + override_bytes_from_content_type from swift.common.ondisk import hash_path, normalize_timestamp, \ storage_directory from swift.common.constraints import CONTAINER_LISTING_LIMIT, \ @@ -327,22 +328,14 @@ def update_data_record(self, record): (name, created, size, content_type, etag) = record if content_type is None: return {'subdir': name} - response = {'bytes': size, 'hash': etag, 'name': name} + response = {'bytes': size, 'hash': etag, 'name': name, + 'content_type': content_type} last_modified = datetime.utcfromtimestamp(float(created)).isoformat() # python isoformat() doesn't include msecs when zero if len(last_modified) < len("1970-01-01T00:00:00.000000"): last_modified += ".000000" response['last_modified'] = last_modified - content_type, params = parse_content_type(content_type) - for key, value in params: - if key == 'swift_bytes': - try: - response['bytes'] = int(value) - except ValueError: - self.logger.exception("Invalid swift_bytes") - else: - content_type += ';%s=%s' % (key, value) - response['content_type'] = content_type + override_bytes_from_content_type(response, logger=self.logger) return response @public diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index ee6afdac1c..6ac39b8ee6 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -32,17 +32,19 @@ from swift import gettext_ as _ from urllib import unquote, quote from hashlib import md5 +from sys import exc_info from eventlet import sleep, GreenPile from eventlet.queue import Queue from eventlet.timeout import Timeout from swift.common.utils import ContextPool, config_true_value, public, json, \ - csv_append, GreenthreadSafeIterator, quorum_size + csv_append, GreenthreadSafeIterator, quorum_size, split_path, \ + override_bytes_from_content_type, get_valid_utf8_str from swift.common.ondisk import normalize_timestamp from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_metadata, check_object_creation, \ - CONTAINER_LISTING_LIMIT, MAX_FILE_SIZE, MAX_BUFFERED_SLO_SEGMENTS + CONTAINER_LISTING_LIMIT, MAX_FILE_SIZE from swift.common.exceptions import ChunkReadTimeout, \ ChunkWriteTimeout, ConnectionTimeout, ListingIterNotFound, \ ListingIterNotAuthorized, ListingIterError, SegmentError @@ -55,34 +57,16 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \ HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \ HTTPServerError, HTTPServiceUnavailable, Request, Response, \ - HTTPClientDisconnect, HTTPNotImplemented + HTTPClientDisconnect, HTTPNotImplemented, HTTPException -class SegmentListing(object): - - def __init__(self, listing): - self.listing = iter(listing) - self._prepended_segments = [] - - def prepend_segments(self, new_segs): - """ - Will prepend given segments to listing when iterating. - :raises SegmentError: when # segments > MAX_BUFFERED_SLO_SEGMENTS - """ - new_segs.extend(self._prepended_segments) - if len(new_segs) > MAX_BUFFERED_SLO_SEGMENTS: - raise SegmentError('Too many unread slo segments in buffer') - self._prepended_segments = new_segs - - def listing_iter(self): - while True: - if self._prepended_segments: - seg_dict = self._prepended_segments.pop(0) - else: - seg_dict = self.listing.next() - if isinstance(seg_dict['name'], unicode): - seg_dict['name'] = seg_dict['name'].encode('utf-8') - yield seg_dict +def segment_listing_iter(listing): + listing = iter(listing) + while True: + seg_dict = listing.next() + if isinstance(seg_dict['name'], unicode): + seg_dict['name'] = seg_dict['name'].encode('utf-8') + yield seg_dict def copy_headers_into(from_r, to_r): @@ -134,8 +118,7 @@ def __init__(self, controller, container, listing, response=None, is_slo=False, max_lo_time=86400): self.controller = controller self.container = container - self.segment_listing = SegmentListing(listing) - self.listing = self.segment_listing.listing_iter() + self.listing = segment_listing_iter(listing) self.is_slo = is_slo self.max_lo_time = max_lo_time self.ratelimit_index = 0 @@ -178,7 +161,8 @@ def _load_next_segment(self): path = '/%s/%s/%s' % (self.controller.account_name, container, obj) req = Request.blank(path) if self.seek or (self.length and self.length > 0): - bytes_available = self.segment_dict['bytes'] - self.seek + bytes_available = \ + self.segment_dict['bytes'] - self.seek range_tail = '' if self.length: if bytes_available >= self.length: @@ -206,27 +190,16 @@ def _load_next_segment(self): 'Could not load object segment %(path)s:' ' %(status)s') % {'path': path, 'status': resp.status_int}) if self.is_slo: - if 'X-Static-Large-Object' in resp.headers: - # this segment is a nested slo object. read in the body - # and add its segments into this slo. - try: - sub_manifest = json.loads(resp.body) - self.segment_listing.prepend_segments(sub_manifest) - sub_etag = md5(''.join( - o['hash'] for o in sub_manifest)).hexdigest() - if sub_etag != self.segment_dict['hash']: - raise SegmentError(_( - 'Object segment does not match sub-slo: ' - '%(path)s etag: %(r_etag)s != %(s_etag)s.') % - {'path': path, 'r_etag': sub_etag, - 's_etag': self.segment_dict['hash']}) - return self._load_next_segment() - except ValueError: - raise SegmentError(_( - 'Sub SLO has invalid manifest: %s') % path) - - elif resp.etag != self.segment_dict['hash'] or \ - resp.content_length != self.segment_dict['bytes']: + if (resp.etag != self.segment_dict['hash'] or + (resp.content_length != self.segment_dict['bytes'] and + not req.range)): + # The content-length check is for security reasons. Seems + # possible that an attacker could upload a >1mb object and + # then replace it with a much smaller object with same + # etag. Then create a big nested SLO that calls that + # object many times which would hammer our obj servers. If + # this is a range request, don't check content-length + # because it won't match. raise SegmentError(_( 'Object segment no longer valid: ' '%(path)s etag: %(r_etag)s != %(s_etag)s or ' @@ -288,6 +261,9 @@ def __iter__(self): except StopIteration: raise except SegmentError: + # I have to save this error because yielding the ' ' below clears + # the exception from the current stack frame. + err = exc_info() if not self.have_yielded_data: # Normally, exceptions before any data has been yielded will # cause Eventlet to send a 5xx response. In this particular @@ -297,7 +273,7 @@ def __iter__(self): # Agreements and this SegmentError indicates the user has # created an invalid condition. yield ' ' - raise + raise err except (Exception, Timeout) as err: if not getattr(err, 'swift_logged', False): self.controller.app.logger.exception(_( @@ -376,6 +352,7 @@ def app_iter_range(self, start, stop): class ObjectController(Controller): """WSGI controller for object requests.""" server_type = 'Object' + max_slo_recusion_depth = 10 def __init__(self, app, account_name, container_name, object_name, **kwargs): @@ -383,6 +360,7 @@ def __init__(self, app, account_name, container_name, object_name, self.account_name = unquote(account_name) self.container_name = unquote(container_name) self.object_name = unquote(object_name) + self.slo_recursion_depth = 0 def _listing_iter(self, lcontainer, lprefix, env): for page in self._listing_pages_iter(lcontainer, lprefix, env): @@ -422,6 +400,67 @@ def _listing_pages_iter(self, lcontainer, lprefix, env): marker = sublisting[-1]['name'].encode('utf-8') yield sublisting + def _slo_listing_obj_iter(self, incoming_req, account, container, obj, + partition=None, initial_resp=None): + """ + The initial_resp indicated that this is a SLO manifest file. This will + create an iterable that will expand nested SLOs as it walks though the + listing. + :params incoming_req: The original GET request from client + :params initial_resp: the first resp from the above request + """ + + if initial_resp and initial_resp.status_int == HTTP_OK and \ + incoming_req.method == 'GET' and not incoming_req.range: + valid_resp = initial_resp + else: + new_req = incoming_req.copy_get() + new_req.method = 'GET' + new_req.range = None + if partition is None: + try: + partition = self.app.object_ring.get_part( + account, container, obj) + except ValueError: + raise HTTPException( + "Invalid path to whole SLO manifest: %s" % + new_req.path) + valid_resp = self.GETorHEAD_base( + new_req, _('Object'), self.app.object_ring, partition, + '/'.join(['', account, container, obj])) + + if 'swift.authorize' in incoming_req.environ: + incoming_req.acl = valid_resp.headers.get('x-container-read') + auth_resp = incoming_req.environ['swift.authorize'](incoming_req) + if auth_resp: + raise ListingIterNotAuthorized(auth_resp) + if valid_resp.status_int == HTTP_NOT_FOUND: + raise ListingIterNotFound() + elif not is_success(valid_resp.status_int): + raise ListingIterError() + try: + listing = json.loads(valid_resp.body) + except ValueError: + listing = [] + for seg_dict in listing: + if config_true_value(seg_dict.get('sub_slo')): + if incoming_req.method == 'HEAD': + override_bytes_from_content_type(seg_dict, + logger=self.app.logger) + yield seg_dict + continue + sub_path = get_valid_utf8_str(seg_dict['name']) + sub_cont, sub_obj = split_path(sub_path, 2, 2, True) + self.slo_recursion_depth += 1 + if self.slo_recursion_depth >= self.max_slo_recusion_depth: + raise ListingIterError("Max recursion depth exceeded") + for sub_seg_dict in self._slo_listing_obj_iter( + incoming_req, account, sub_cont, sub_obj): + yield sub_seg_dict + self.slo_recursion_depth -= 1 + else: + yield seg_dict + def _remaining_items(self, listing_iter): """ Returns an item-by-item iterator for a page-by-page iterator @@ -527,31 +566,29 @@ def GETorHEAD(self, req): req.params.get('multipart-manifest') != 'get' and \ self.app.allow_static_large_object: large_object = 'SLO' - listing_page1 = () - listing = [] lcontainer = None # container name is included in listing - if resp.status_int == HTTP_OK and \ - req.method == 'GET' and not req.range: - try: - listing = json.loads(resp.body) - except ValueError: - listing = [] - else: - # need to make a second request to get whole manifest - new_req = req.copy_get() - new_req.method = 'GET' - new_req.range = None - new_resp = self.GETorHEAD_base( - new_req, _('Object'), self.app.object_ring, partition, - req.path_info) - if new_resp.status_int // 100 == 2: - try: - listing = json.loads(new_resp.body) - except ValueError: - listing = [] - else: - return HTTPServiceUnavailable( - "Unable to load SLO manifest", request=req) + try: + seg_iter = iter(self._slo_listing_obj_iter( + req, self.account_name, self.container_name, + self.object_name, partition=partition, initial_resp=resp)) + listing_page1 = [] + for seg in seg_iter: + listing_page1.append(seg) + if len(listing_page1) >= CONTAINER_LISTING_LIMIT: + break + listing = itertools.chain(listing_page1, + self._remaining_items(seg_iter)) + except ListingIterNotFound: + return HTTPNotFound(request=req) + except ListingIterNotAuthorized, err: + return err.aresp + except ListingIterError: + return HTTPServerError(request=req) + except StopIteration: + listing_page1 = listing = () + except HTTPException: + return HTTPServiceUnavailable( + "Unable to load SLO manifest", request=req) if 'x-object-manifest' in resp.headers and \ req.params.get('multipart-manifest') != 'get': @@ -602,8 +639,8 @@ def head_response(environ, start_response): else: # For objects with a reasonable number of segments, we'll serve # them with a set content-length and computed etag. + listing = list(listing) if listing: - listing = list(listing) try: content_length = sum(o['bytes'] for o in listing) last_modified = \ diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index c9408eab52..64c6038757 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -51,6 +51,7 @@ ConnectionTimeout, LockTimeout) from swift.common import utils from swift.common.swob import Response +from test.unit import FakeLogger class MockOs(): @@ -1492,6 +1493,25 @@ def test_parse_content_type(self): utils.parse_content_type(r'text/plain; x="\""; a'), ('text/plain', [('x', r'"\""'), ('a', '')])) + def test_override_bytes_from_content_type(self): + listing_dict = { + 'bytes': 1234, 'hash': 'asdf', 'name': 'zxcv', + 'content_type': 'text/plain; hello="world"; swift_bytes=15'} + utils.override_bytes_from_content_type(listing_dict, + logger=FakeLogger()) + self.assertEquals(listing_dict['bytes'], 15) + self.assertEquals(listing_dict['content_type'], + 'text/plain;hello="world"') + + listing_dict = { + 'bytes': 1234, 'hash': 'asdf', 'name': 'zxcv', + 'content_type': 'text/plain; hello="world"; swift_bytes=hey'} + utils.override_bytes_from_content_type(listing_dict, + logger=FakeLogger()) + self.assertEquals(listing_dict['bytes'], 1234) + self.assertEquals(listing_dict['content_type'], + 'text/plain;hello="world"') + def test_quote(self): res = utils.quote('/v1/a/c3/subdirx/') assert res == '/v1/a/c3/subdirx/' diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 6a32dd30a1..37ff6b8b08 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -1046,7 +1046,7 @@ def test_GET_manifest_no_segments(self): response_bodies = ( '', # HEAD /a '', # HEAD /a/c - '', # GET manifest + simplejson.dumps([]), # GET manifest simplejson.dumps([])) # GET empty listing with save_globals(): @@ -1393,7 +1393,7 @@ def test_GET_nested_slo(self): {"hash": "8681fb3ada2715c8754706ee5f23d4f8", "last_modified": "2012-11-08T04:05:37.846710", "bytes": 4, - "name": "/d2/sub_manifest", + "name": u"/d2/sub_manifest \u2661", "sub_slo": True, "content_type": "application/octet-stream"}, {"hash": "419af6d362a14b7a789ba1c7e772bbae", "last_modified": "2012-11-08T04:05:37.866820", @@ -1416,8 +1416,8 @@ def test_GET_nested_slo(self): '', # HEAD /a '', # HEAD /a/c simplejson.dumps(listing), # GET manifest - 'Aa', # GET seg01 simplejson.dumps(sub_listing), # GET sub_manifest + 'Aa', # GET seg01 'Bb', # GET seg02 'Cc', # GET seg03 'Dd') # GET seg04 @@ -1439,12 +1439,12 @@ def capture_requested_paths(ipaddr, port, device, partition, 200, # HEAD /a 200, # HEAD /a/c 200, # GET listing1 - 200, # GET seg01 200, # GET sub listing1 + 200, # GET seg01 200, # GET seg02 200, # GET seg03 200, # GET seg04 - headers=[{}, {}, slob_headers, {}, slob_headers, {}, {}, {}], + headers=[{}, {}, slob_headers, slob_headers, {}, {}, {}, {}], body_iter=response_bodies, give_connect=capture_requested_paths) req = Request.blank('/a/c/manifest') @@ -1457,7 +1457,8 @@ def capture_requested_paths(ipaddr, port, device, partition, requested, [['HEAD', '/a', {}], ['HEAD', '/a/c', {}], - ['GET', '/a/c/manifest', {}]]) + ['GET', '/a/c/manifest', {}], + ['GET', '/a/d2/sub_manifest \xe2\x99\xa1', {}]]) # iterating over body will retrieve manifest and sub manifest's # objects self.assertEqual(resp.body, 'AaBbCcDd') @@ -1466,12 +1467,116 @@ def capture_requested_paths(ipaddr, port, device, partition, [['HEAD', '/a', {}], ['HEAD', '/a/c', {}], ['GET', '/a/c/manifest', {}], + ['GET', '/a/d2/sub_manifest \xe2\x99\xa1', {}], ['GET', '/a/d1/seg01', {}], - ['GET', '/a/d2/sub_manifest', {}], ['GET', '/a/d1/seg02', {}], ['GET', '/a/d2/seg03', {}], ['GET', '/a/d1/seg04', {}]]) + def test_GET_nested_manifest_slo_with_range(self): + """ + Original whole slo is Aa1234Bb where 1234 is a sub-manifests. I'm + pulling out 34Bb + """ + listing = [{"hash": "98568d540134639be4655198a36614a4", # Aa + "last_modified": "2012-11-08T04:05:37.866820", + "bytes": 2, + "name": "/d1/seg01", + "content_type": "application/octet-stream"}, + {"hash": "7b4b0ffa275d404bdc2fc6384916714f", # SubManifest1 + "last_modified": "2012-11-08T04:05:37.866820", + "bytes": 4, "sub_slo": True, + "name": "/d2/subManifest01", + "content_type": "application/octet-stream"}, + {"hash": "d526f1c8ef6c1e4e980e2b8471352d23", # Bb + "last_modified": "2012-11-08T04:05:37.866820", + "bytes": 2, + "name": "/d1/seg02", + "content_type": "application/octet-stream"}] + + sublisting = [{"hash": "c20ad4d76fe97759aa27a0c99bff6710", # 12 + "last_modified": "2012-11-08T04:05:37.866820", + "bytes": 2, + "name": "/d2/subSeg01", + "content_type": "application/octet-stream"}, + {"hash": "e369853df766fa44e1ed0ff613f563bd", # 34 + "last_modified": "2012-11-08T04:05:37.866820", + "bytes": 2, + "name": "/d2/subSeg02", + "content_type": "application/octet-stream"}] + + response_bodies = ( + '', # HEAD /a + '', # HEAD /a/c + simplejson.dumps(listing)[1:1], # GET incomplete manifest + simplejson.dumps(listing), # GET complete manifest + simplejson.dumps(sublisting), # GET complete submanifest + '34', # GET subseg02 + 'Bb') # GET seg02 + etag_iter = ['', '', '', '', '', + 'e369853df766fa44e1ed0ff613f563bd', # subSeg02 + 'd526f1c8ef6c1e4e980e2b8471352d23'] # seg02 + headers = [{}, {}, + {'X-Static-Large-Object': 'True', + 'content-type': 'text/html; swift_bytes=4'}, + {'X-Static-Large-Object': 'True', + 'content-type': 'text/html; swift_bytes=4'}, + {'X-Static-Large-Object': 'True', + 'content-type': 'text/html; swift_bytes=4'}, + {}, {}] + self.assertTrue(len(response_bodies) == len(etag_iter) == len(headers)) + with save_globals(): + controller = proxy_server.ObjectController( + self.app, 'a', 'c', 'manifest') + + requested = [] + + def capture_requested_paths(ipaddr, port, device, partition, + method, path, headers=None, + query_string=None): + qs_dict = dict(urlparse.parse_qsl(query_string or '')) + requested.append([method, path, qs_dict]) + + set_http_connect( + 200, # HEAD /a + 200, # HEAD /a/c + 206, # GET incomplete listing + 200, # GET complete listing + 200, # GET complete sublisting + 200, # GET subSeg02 + 200, # GET seg02 + headers=headers, + etags=etag_iter, + body_iter=response_bodies, + give_connect=capture_requested_paths) + + req = Request.blank('/a/c/manifest') + req.range = 'bytes=4-7' + resp = controller.GET(req) + got_called = [False, ] + + def fake_start_response(*args, **kwargs): + got_called[0] = True + self.assertTrue(args[0].startswith('206')) + + app_iter = resp(req.environ, fake_start_response) + resp_body = ''.join(app_iter) # read in entire resp + self.assertEqual(resp.status_int, 206) + self.assertEqual(resp_body, '34Bb') + self.assertTrue(got_called[0]) + self.assertEqual(resp.content_length, 4) + self.assertEqual(resp.content_type, 'text/html') + + self.assertEqual( + requested, + [['HEAD', '/a', {}], + ['HEAD', '/a/c', {}], + ['GET', '/a/c/manifest', {}], # for incomplete manifest + ['GET', '/a/c/manifest', {}], + ['GET', '/a/d2/subManifest01', {}], + ['GET', '/a/d2/subSeg02', {}], + ['GET', '/a/d1/seg02', {}]]) + def test_GET_bad_404_manifest_slo(self): listing = [{"hash": "98568d540134639be4655198a36614a4", "last_modified": "2012-11-08T04:05:37.866820", From 92ae497800d9e66795346019cf284026a751597e Mon Sep 17 00:00:00 2001 From: ZhiQiang Fan Date: Fri, 20 Sep 2013 23:34:06 +0800 Subject: [PATCH 29/35] Fix unsuitable assertTrue assertTrue accepts a parameter msg which will be printed when assertion fails, usually msg is a str. This patch fixes unsuitable usage of assertTrue which set msg to bool type True. Change-Id: I731f8ea553c935eba0e112ffded16f41a5ea86c0 Fixes-Bug: #1226374 --- test/unit/common/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index c9408eab52..06881a7e70 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -1617,14 +1617,14 @@ def test_close(self): iter_file = utils.FileLikeIter('abcdef') self.assertEquals(iter_file.next(), 'a') iter_file.close() - self.assertTrue(iter_file.closed, True) + self.assertTrue(iter_file.closed) self.assertRaises(ValueError, iter_file.next) self.assertRaises(ValueError, iter_file.read) self.assertRaises(ValueError, iter_file.readline) self.assertRaises(ValueError, iter_file.readlines) # Just make sure repeated close calls don't raise an Exception iter_file.close() - self.assertTrue(iter_file.closed, True) + self.assertTrue(iter_file.closed) class TestStatsdLogging(unittest.TestCase): From 3e6f9293b8882cecb151e87fe5bfbe24e605b847 Mon Sep 17 00:00:00 2001 From: "Brian D. Burns" Date: Thu, 1 Aug 2013 14:50:03 -0400 Subject: [PATCH 30/35] update SLO delete error handling * ensure all responses are 200 OK * report missing sub-SLO manifests or other error messages in bulk delete response Change-Id: Iaf88c94bc7114ff3c9751f9f31f8f748de911f8a --- swift/common/middleware/bulk.py | 30 ++++-- swift/common/middleware/slo.py | 117 ++++++++++++-------- test/unit/common/middleware/test_bulk.py | 54 +++++++++- test/unit/common/middleware/test_slo.py | 131 +++++++++++++++++++---- 4 files changed, 250 insertions(+), 82 deletions(-) diff --git a/swift/common/middleware/bulk.py b/swift/common/middleware/bulk.py index 92abdbd556..69086e6b3a 100644 --- a/swift/common/middleware/bulk.py +++ b/swift/common/middleware/bulk.py @@ -190,6 +190,8 @@ def __init__(self, app, conf): conf.get('max_containers_per_extraction', 10000)) self.max_failed_extractions = int( conf.get('max_failed_extractions', 1000)) + self.max_failed_deletes = int( + conf.get('max_failed_deletes', 1000)) self.max_deletes_per_request = int( conf.get('max_deletes_per_request', 10000)) self.yield_frequency = int(conf.get('yield_frequency', 60)) @@ -239,7 +241,8 @@ def get_objs_to_delete(self, req): while data_remaining: if '\n' in line: obj_to_delete, line = line.split('\n', 1) - objs_to_delete.append(unquote(obj_to_delete)) + objs_to_delete.append( + {'name': unquote(obj_to_delete)}) else: data = req.body_file.read(MAX_PATH_LENGTH) if data: @@ -247,7 +250,8 @@ def get_objs_to_delete(self, req): else: data_remaining = False if line.strip(): - objs_to_delete.append(unquote(line)) + objs_to_delete.append( + {'name': unquote(line)}) if len(objs_to_delete) > self.max_deletes_per_request: raise HTTPRequestEntityTooLarge( 'Maximum Bulk Deletes: %d per request' % @@ -304,13 +308,22 @@ def handle_delete_iter(self, req, objs_to_delete=None, separator = '\r\n\r\n' last_yield = time() yield ' ' - obj_to_delete = obj_to_delete.strip() - if not obj_to_delete: + obj_name = obj_to_delete['name'].strip() + if not obj_name: + continue + if len(failed_files) >= self.max_failed_deletes: + raise HTTPBadRequest('Max delete failures exceeded') + if obj_to_delete.get('error'): + if obj_to_delete['error']['code'] == HTTP_NOT_FOUND: + resp_dict['Number Not Found'] += 1 + else: + failed_files.append([quote(obj_name), + obj_to_delete['error']['message']]) continue delete_path = '/'.join(['', vrs, account, - obj_to_delete.lstrip('/')]) + obj_name.lstrip('/')]) if not check_utf8(delete_path): - failed_files.append([quote(obj_to_delete), + failed_files.append([quote(obj_name), HTTPPreconditionFailed().status]) continue new_env = req.environ.copy() @@ -327,13 +340,12 @@ def handle_delete_iter(self, req, objs_to_delete=None, elif resp.status_int == HTTP_NOT_FOUND: resp_dict['Number Not Found'] += 1 elif resp.status_int == HTTP_UNAUTHORIZED: - failed_files.append([quote(obj_to_delete), + failed_files.append([quote(obj_name), HTTPUnauthorized().status]) - raise HTTPUnauthorized(request=req) else: if resp.status_int // 100 == 5: failed_file_response_type = HTTPBadGateway - failed_files.append([quote(obj_to_delete), resp.status]) + failed_files.append([quote(obj_name), resp.status]) if failed_files: resp_dict['Response Status'] = \ diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py index 2be82896a6..df4f21d718 100644 --- a/swift/common/middleware/slo.py +++ b/swift/common/middleware/slo.py @@ -141,10 +141,11 @@ from hashlib import md5 from swift.common.swob import Request, HTTPBadRequest, HTTPServerError, \ HTTPMethodNotAllowed, HTTPRequestEntityTooLarge, HTTPLengthRequired, \ - HTTPOk, HTTPPreconditionFailed, HTTPException + HTTPOk, HTTPPreconditionFailed, HTTPException, HTTPNotFound, \ + HTTPUnauthorized from swift.common.utils import json, get_logger, config_true_value from swift.common.constraints import check_utf8, MAX_BUFFERED_SLO_SEGMENTS -from swift.common.http import HTTP_NOT_FOUND +from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED from swift.common.wsgi import WSGIContext from swift.common.middleware.bulk import get_response_body, \ ACCEPTABLE_FORMATS, Bulk @@ -216,8 +217,7 @@ def __init__(self, app, conf): 1024 * 1024 * 2)) self.min_segment_size = int(self.conf.get('min_segment_size', 1024 * 1024)) - self.bulk_deleter = Bulk( - app, {'max_deletes_per_request': self.max_manifest_segments}) + self.bulk_deleter = Bulk(app, {}) def handle_multipart_put(self, req, start_response): """ @@ -333,66 +333,91 @@ def get_segments_to_delete_iter(self, req): A generator function to be used to delete all the segments and sub-segments referenced in a manifest. - :raises HTTPBadRequest: on sub manifest not manifest anymore or - on too many buffered sub segments - :raises HTTPServerError: on unable to load manifest + :params req: a swob.Request with an SLO manifest in path + :raises HTTPPreconditionFailed: on invalid UTF8 in request path + :raises HTTPBadRequest: on too many buffered sub segments and + on invalid SLO manifest path """ + if not check_utf8(req.path_info): + raise HTTPPreconditionFailed( + request=req, body='Invalid UTF8 or contains NULL') try: vrs, account, container, obj = req.split_path(4, 4, True) except ValueError: - raise HTTPBadRequest('Not a SLO manifest') - sub_segments = [{ + raise HTTPBadRequest('Invalid SLO manifiest path') + + segments = [{ 'sub_slo': True, 'name': ('/%s/%s' % (container, obj)).decode('utf-8')}] - while sub_segments: - if len(sub_segments) > MAX_BUFFERED_SLO_SEGMENTS: + while segments: + if len(segments) > MAX_BUFFERED_SLO_SEGMENTS: raise HTTPBadRequest( 'Too many buffered slo segments to delete.') - seg_data = sub_segments.pop(0) + seg_data = segments.pop(0) if seg_data.get('sub_slo'): - new_env = req.environ.copy() - new_env['REQUEST_METHOD'] = 'GET' - del(new_env['wsgi.input']) - new_env['QUERY_STRING'] = 'multipart-manifest=get' - new_env['CONTENT_LENGTH'] = 0 - new_env['HTTP_USER_AGENT'] = \ - '%s MultipartDELETE' % new_env.get('HTTP_USER_AGENT') - new_env['swift.source'] = 'SLO' - new_env['PATH_INFO'] = ( - '/%s/%s/%s' % ( - vrs, account, - seg_data['name'].lstrip('/'))).encode('utf-8') - sub_resp = Request.blank('', new_env).get_response(self.app) - if sub_resp.is_success: - try: - # if its still a SLO, load its segments - if config_true_value( - sub_resp.headers.get('X-Static-Large-Object')): - sub_segments.extend(json.loads(sub_resp.body)) - except ValueError: - raise HTTPServerError('Unable to load SLO manifest') - # add sub-manifest back to be deleted after sub segments - # (even if obj is not a SLO) - seg_data['sub_slo'] = False - sub_segments.append(seg_data) - elif sub_resp.status_int != HTTP_NOT_FOUND: - # on deletes treat not found as success - raise HTTPServerError('Sub SLO unable to load.') + try: + segments.extend( + self.get_slo_segments(seg_data['name'], req)) + except HTTPException as err: + # allow bulk delete response to report errors + seg_data['error'] = {'code': err.status_int, + 'message': err.body} + + # add manifest back to be deleted after segments + seg_data['sub_slo'] = False + segments.append(seg_data) + else: + seg_data['name'] = seg_data['name'].encode('utf-8') + yield seg_data + + def get_slo_segments(self, obj_name, req): + """ + Performs a swob.Request and returns the SLO manifest's segments. + + :raises HTTPServerError: on unable to load obj_name or + on unable to load the SLO manifest data. + :raises HTTPBadRequest: on not an SLO manifest + :raises HTTPNotFound: on SLO manifest not found + :returns: SLO manifest's segments + """ + vrs, account, _junk = req.split_path(2, 3, True) + new_env = req.environ.copy() + new_env['REQUEST_METHOD'] = 'GET' + del(new_env['wsgi.input']) + new_env['QUERY_STRING'] = 'multipart-manifest=get' + new_env['CONTENT_LENGTH'] = 0 + new_env['HTTP_USER_AGENT'] = \ + '%s MultipartDELETE' % new_env.get('HTTP_USER_AGENT') + new_env['swift.source'] = 'SLO' + new_env['PATH_INFO'] = ( + '/%s/%s/%s' % ( + vrs, account, + obj_name.lstrip('/'))).encode('utf-8') + resp = Request.blank('', new_env).get_response(self.app) + + if resp.is_success: + if config_true_value(resp.headers.get('X-Static-Large-Object')): + try: + return json.loads(resp.body) + except ValueError: + raise HTTPServerError('Unable to load SLO manifest') else: - yield seg_data['name'].encode('utf-8') + raise HTTPBadRequest('Not an SLO manifest') + elif resp.status_int == HTTP_NOT_FOUND: + raise HTTPNotFound('SLO manifest not found') + elif resp.status_int == HTTP_UNAUTHORIZED: + raise HTTPUnauthorized('401 Unauthorized') + else: + raise HTTPServerError('Unable to load SLO manifest or segment.') def handle_multipart_delete(self, req): """ Will delete all the segments in the SLO manifest and then, if successful, will delete the manifest file. + :params req: a swob.Request with an obj in path - :raises HTTPServerError: on invalid manifest :returns: swob.Response whose app_iter set to Bulk.handle_delete_iter """ - if not check_utf8(req.path_info): - raise HTTPPreconditionFailed( - request=req, body='Invalid UTF8 or contains NULL') - resp = HTTPOk(request=req) out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS) if out_content_type: diff --git a/test/unit/common/middleware/test_bulk.py b/test/unit/common/middleware/test_bulk.py index 5f800e4c32..f5ae07a153 100644 --- a/test/unit/common/middleware/test_bulk.py +++ b/test/unit/common/middleware/test_bulk.py @@ -23,6 +23,7 @@ from mock import patch from swift.common.middleware import bulk from swift.common.swob import Request, Response, HTTPException +from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED from swift.common.utils import json @@ -35,6 +36,8 @@ def __init__(self): def __call__(self, env, start_response): self.calls += 1 if env['PATH_INFO'].startswith('/unauth/'): + if env['PATH_INFO'].endswith('/c/f_ok'): + return Response(status='204 No Content')(env, start_response) return Response(status=401)(env, start_response) if env['PATH_INFO'].startswith('/create_cont/'): if env['REQUEST_METHOD'] == 'HEAD': @@ -493,6 +496,29 @@ def handle_delete_and_iter(self, req, out_content_type='application/json'): req, out_content_type=out_content_type)) return resp_body + def test_bulk_delete_uses_predefined_object_errors(self): + req = Request.blank('/delete_works/AUTH_Acc') + objs_to_delete = [ + {'name': '/c/file_a'}, + {'name': '/c/file_b', 'error': {'code': HTTP_NOT_FOUND, + 'message': 'not found'}}, + {'name': '/c/file_c', 'error': {'code': HTTP_UNAUTHORIZED, + 'message': 'unauthorized'}}, + {'name': '/c/file_d'}] + resp_body = ''.join(self.bulk.handle_delete_iter( + req, objs_to_delete=objs_to_delete, + out_content_type='application/json')) + self.assertEquals( + self.app.delete_paths, ['/delete_works/AUTH_Acc/c/file_a', + '/delete_works/AUTH_Acc/c/file_d']) + self.assertEquals(self.app.calls, 2) + resp_data = json.loads(resp_body) + self.assertEquals(resp_data['Response Status'], '400 Bad Request') + self.assertEquals(resp_data['Number Deleted'], 2) + self.assertEquals(resp_data['Number Not Found'], 1) + self.assertEquals(resp_data['Errors'], + [['/c/file_c', 'unauthorized']]) + def test_bulk_delete_works(self): req = Request.blank('/delete_works/AUTH_Acc', body='/c/f\n/c/f404', headers={'Accept': 'application/json'}) @@ -538,13 +564,14 @@ def test_bulk_delete_get_objs(self): req.method = 'DELETE' with patch.object(self.bulk, 'max_deletes_per_request', 2): results = self.bulk.get_objs_to_delete(req) - self.assertEquals(results, ['1\r', '2\r']) + self.assertEquals(results, [{'name': '1\r'}, {'name': '2\r'}]) with patch.object(bulk, 'MAX_PATH_LENGTH', 2): results = [] req.environ['wsgi.input'] = StringIO('1\n2\n3') results = self.bulk.get_objs_to_delete(req) - self.assertEquals(results, ['1', '2', '3']) + self.assertEquals(results, + [{'name': '1'}, {'name': '2'}, {'name': '3'}]) with patch.object(self.bulk, 'max_deletes_per_request', 9): with patch.object(bulk, 'MAX_PATH_LENGTH', 1): @@ -611,14 +638,15 @@ def test_bulk_delete_no_files_in_body(self): self.assertTrue('400 Bad Request' in resp_body) def test_bulk_delete_unauth(self): - req = Request.blank('/unauth/AUTH_acc/', body='/c/f\n/c/f2\n', + req = Request.blank('/unauth/AUTH_acc/', body='/c/f\n/c/f_ok\n', headers={'Accept': 'application/json'}) req.method = 'DELETE' resp_body = self.handle_delete_and_iter(req) - self.assertEquals(self.app.calls, 1) + self.assertEquals(self.app.calls, 2) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Errors'], [['/c/f', '401 Unauthorized']]) - self.assertEquals(resp_data['Response Status'], '401 Unauthorized') + self.assertEquals(resp_data['Response Status'], '400 Bad Request') + self.assertEquals(resp_data['Number Deleted'], 1) def test_bulk_delete_500_resp(self): req = Request.blank('/broke/AUTH_acc/', body='/c/f\nc/f2\n', @@ -667,5 +695,21 @@ def test_bulk_delete_bad_file_over_twice_max_length(self): resp_body = self.handle_delete_and_iter(req) self.assertTrue('400 Bad Request' in resp_body) + def test_bulk_delete_max_failures(self): + req = Request.blank('/unauth/AUTH_Acc', body='/c/f1\n/c/f2\n/c/f3', + headers={'Accept': 'application/json'}) + req.method = 'DELETE' + with patch.object(self.bulk, 'max_failed_deletes', 2): + resp_body = self.handle_delete_and_iter(req) + self.assertEquals(self.app.calls, 2) + resp_data = json.loads(resp_body) + self.assertEquals(resp_data['Response Status'], '400 Bad Request') + self.assertEquals(resp_data['Response Body'], + 'Max delete failures exceeded') + self.assertEquals(resp_data['Errors'], + [['/c/f1', '401 Unauthorized'], + ['/c/f2', '401 Unauthorized']]) + + if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py index 731dcd6180..7586ff02e9 100644 --- a/test/unit/common/middleware/test_slo.py +++ b/test/unit/common/middleware/test_slo.py @@ -81,9 +81,18 @@ def __call__(self, env, start_response): return Response(status=200, body='lalala')(env, start_response) if env['PATH_INFO'].startswith('/test_delete_404/'): + good_data = json.dumps( + [{'name': '/c/a_1', 'hash': 'a', 'bytes': '1'}, + {'name': '/d/b_2', 'hash': 'b', 'bytes': '2'}]) self.req_method_paths.append((env['REQUEST_METHOD'], env['PATH_INFO'])) - return Response(status=404)(env, start_response) + if env['PATH_INFO'].endswith('/c/man_404'): + return Response(status=404)(env, start_response) + if env['PATH_INFO'].endswith('/c/a_1'): + return Response(status=404)(env, start_response) + return Response(status=200, + headers={'X-Static-Large-Object': 'True'}, + body=good_data)(env, start_response) if env['PATH_INFO'].startswith('/test_delete/'): good_data = json.dumps( @@ -115,6 +124,21 @@ def __call__(self, env, start_response): headers={'X-Static-Large-Object': 'True'}, body=good_data)(env, start_response) + if env['PATH_INFO'].startswith('/test_delete_nested_404/'): + good_data = json.dumps( + [{'name': '/a/a_1', 'hash': 'a', 'bytes': '1'}, + {'name': '/a/sub_nest', 'hash': 'a', 'bytes': '2', + 'sub_slo': True}, + {'name': '/d/d_3', 'hash': 'b', 'bytes': '3'}]) + self.req_method_paths.append((env['REQUEST_METHOD'], + env['PATH_INFO'])) + if 'sub_nest' in env['PATH_INFO']: + return Response(status=404)(env, start_response) + else: + return Response(status=200, + headers={'X-Static-Large-Object': 'True'}, + body=good_data)(env, start_response) + if env['PATH_INFO'].startswith('/test_delete_bad_json/'): self.req_method_paths.append((env['REQUEST_METHOD'], env['PATH_INFO'])) @@ -127,13 +151,13 @@ def __call__(self, env, start_response): env['PATH_INFO'])) return Response(status=200, body='')(env, start_response) - if env['PATH_INFO'].startswith('/test_delete_bad/'): + if env['PATH_INFO'].startswith('/test_delete_401/'): good_data = json.dumps( [{'name': '/c/a_1', 'hash': 'a', 'bytes': '1'}, {'name': '/d/b_2', 'hash': 'b', 'bytes': '2'}]) self.req_method_paths.append((env['REQUEST_METHOD'], env['PATH_INFO'])) - if env['PATH_INFO'].endswith('/c/a_1'): + if env['PATH_INFO'].endswith('/d/b_2'): return Response(status=401)(env, start_response) return Response(status=200, headers={'X-Static-Large-Object': 'True'}, @@ -368,13 +392,38 @@ def test_handle_multipart_delete_man(self): def test_handle_multipart_delete_whole_404(self): req = Request.blank( - '/test_delete_404/A/c/man?multipart-manifest=delete', - environ={'REQUEST_METHOD': 'DELETE'}) + '/test_delete_404/A/c/man_404?multipart-manifest=delete', + environ={'REQUEST_METHOD': 'DELETE', + 'HTTP_ACCEPT': 'application/json'}) app_iter = self.slo(req.environ, fake_start_response) - list(app_iter) # iterate through whole response + app_iter = list(app_iter) # iterate through whole response + resp_data = json.loads(app_iter[0]) self.assertEquals(self.app.calls, 1) self.assertEquals(self.app.req_method_paths, - [('GET', '/test_delete_404/A/c/man')]) + [('GET', '/test_delete_404/A/c/man_404')]) + self.assertEquals(resp_data['Response Status'], '200 OK') + self.assertEquals(resp_data['Response Body'], '') + self.assertEquals(resp_data['Number Deleted'], 0) + self.assertEquals(resp_data['Number Not Found'], 1) + self.assertEquals(resp_data['Errors'], []) + + def test_handle_multipart_delete_segment_404(self): + req = Request.blank( + '/test_delete_404/A/c/man?multipart-manifest=delete', + environ={'REQUEST_METHOD': 'DELETE', + 'HTTP_ACCEPT': 'application/json'}) + app_iter = self.slo(req.environ, fake_start_response) + app_iter = list(app_iter) # iterate through whole response + resp_data = json.loads(app_iter[0]) + self.assertEquals(self.app.calls, 4) + self.assertEquals(self.app.req_method_paths, + [('GET', '/test_delete_404/A/c/man'), + ('DELETE', '/test_delete_404/A/c/a_1'), + ('DELETE', '/test_delete_404/A/d/b_2'), + ('DELETE', '/test_delete_404/A/c/man')]) + self.assertEquals(resp_data['Response Status'], '200 OK') + self.assertEquals(resp_data['Number Deleted'], 2) + self.assertEquals(resp_data['Number Not Found'], 1) def test_handle_multipart_delete_whole(self): req = Request.blank( @@ -407,9 +456,28 @@ def test_handle_multipart_delete_nested(self): ('DELETE', '/test_delete_nested/A/d/d_3'), ('DELETE', '/test_delete_nested/A/c/man')])) + def test_handle_multipart_delete_nested_404(self): + req = Request.blank( + '/test_delete_nested_404/A/c/man?multipart-manifest=delete', + environ={'REQUEST_METHOD': 'DELETE', + 'HTTP_ACCEPT': 'application/json'}) + app_iter = self.slo(req.environ, fake_start_response) + app_iter = list(app_iter) # iterate through whole response + resp_data = json.loads(app_iter[0]) + self.assertEquals(self.app.calls, 5) + self.assertEquals(self.app.req_method_paths, + [('GET', '/test_delete_nested_404/A/c/man'), + ('DELETE', '/test_delete_nested_404/A/a/a_1'), + ('GET', '/test_delete_nested_404/A/a/sub_nest'), + ('DELETE', '/test_delete_nested_404/A/d/d_3'), + ('DELETE', '/test_delete_nested_404/A/c/man')]) + self.assertEquals(resp_data['Response Status'], '200 OK') + self.assertEquals(resp_data['Response Body'], '') + self.assertEquals(resp_data['Number Deleted'], 3) + self.assertEquals(resp_data['Number Not Found'], 1) + self.assertEquals(resp_data['Errors'], []) + def test_handle_multipart_delete_not_a_manifest(self): - # when trying to delete a SLO and its not an SLO, just go ahead - # and delete it req = Request.blank( '/test_delete_bad_man/A/c/man?multipart-manifest=delete', environ={'REQUEST_METHOD': 'DELETE', @@ -417,11 +485,15 @@ def test_handle_multipart_delete_not_a_manifest(self): app_iter = self.slo(req.environ, fake_start_response) app_iter = list(app_iter) # iterate through whole response resp_data = json.loads(app_iter[0]) - self.assertEquals(self.app.calls, 2) + self.assertEquals(self.app.calls, 1) self.assertEquals(self.app.req_method_paths, - [('GET', '/test_delete_bad_man/A/c/man'), - ('DELETE', '/test_delete_bad_man/A/c/man')]) - self.assertEquals(resp_data['Response Status'], '200 OK') + [('GET', '/test_delete_bad_man/A/c/man')]) + self.assertEquals(resp_data['Response Status'], '400 Bad Request') + self.assertEquals(resp_data['Response Body'], '') + self.assertEquals(resp_data['Number Deleted'], 0) + self.assertEquals(resp_data['Number Not Found'], 0) + self.assertEquals(resp_data['Errors'], + [['/c/man', 'Not an SLO manifest']]) def test_handle_multipart_delete_bad_json(self): req = Request.blank( @@ -434,18 +506,33 @@ def test_handle_multipart_delete_bad_json(self): self.assertEquals(self.app.calls, 1) self.assertEquals(self.app.req_method_paths, [('GET', '/test_delete_bad_json/A/c/man')]) - self.assertEquals(resp_data["Response Status"], "500 Internal Error") - - def test_handle_multipart_delete_whole_bad(self): + self.assertEquals(resp_data['Response Status'], '400 Bad Request') + self.assertEquals(resp_data['Response Body'], '') + self.assertEquals(resp_data['Number Deleted'], 0) + self.assertEquals(resp_data['Number Not Found'], 0) + self.assertEquals(resp_data['Errors'], + [['/c/man', 'Unable to load SLO manifest']]) + + def test_handle_multipart_delete_401(self): req = Request.blank( - '/test_delete_bad/A/c/man?multipart-manifest=delete', - environ={'REQUEST_METHOD': 'DELETE'}) + '/test_delete_401/A/c/man?multipart-manifest=delete', + environ={'REQUEST_METHOD': 'DELETE', + 'HTTP_ACCEPT': 'application/json'}) app_iter = self.slo(req.environ, fake_start_response) - list(app_iter) # iterate through whole response - self.assertEquals(self.app.calls, 2) + app_iter = list(app_iter) # iterate through whole response + resp_data = json.loads(app_iter[0]) + self.assertEquals(self.app.calls, 4) self.assertEquals(self.app.req_method_paths, - [('GET', '/test_delete_bad/A/c/man'), - ('DELETE', '/test_delete_bad/A/c/a_1')]) + [('GET', '/test_delete_401/A/c/man'), + ('DELETE', '/test_delete_401/A/c/a_1'), + ('DELETE', '/test_delete_401/A/d/b_2'), + ('DELETE', '/test_delete_401/A/c/man')]) + self.assertEquals(resp_data['Response Status'], '400 Bad Request') + self.assertEquals(resp_data['Response Body'], '') + self.assertEquals(resp_data['Number Deleted'], 2) + self.assertEquals(resp_data['Number Not Found'], 0) + self.assertEquals(resp_data['Errors'], + [['/d/b_2', '401 Unauthorized']]) if __name__ == '__main__': unittest.main() From ce5e810fed8c453f4cd41c3c32162f47cde48f10 Mon Sep 17 00:00:00 2001 From: Samuel Merritt Date: Tue, 24 Sep 2013 16:20:28 -0700 Subject: [PATCH 31/35] Update SAIO doc to have double proxy-logging in pipeline. Change-Id: I0a034ca1420761cbf4e35dcea1d9cd18a92f90bd --- doc/source/development_saio.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 85b5f1c17d..c6a7d7f6ae 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -306,7 +306,8 @@ Sample configuration files are provided with all defaults in line-by-line commen eventlet_debug = true [pipeline:main] - pipeline = healthcheck cache tempauth proxy-logging proxy-server + # Yes, proxy-logging appears twice. This is not a mistake. + pipeline = healthcheck proxy-logging cache tempauth proxy-logging proxy-server [app:proxy-server] use = egg:swift#proxy From d9d7b2135a7020cdf43172ea4fcf0b1020f49101 Mon Sep 17 00:00:00 2001 From: Samuel Merritt Date: Tue, 24 Sep 2013 16:43:33 -0700 Subject: [PATCH 32/35] Install libffi-dev in SAIO docs. If you don't, then newer versions of xattr won't install, and since our xattr requirement is simply ">= 0.4" in requirements.txt, this affects anyone setting up a new SAIO. This happened with xattr 0.7, which was released on 2013-07-19. Change-Id: Iaf335fa25a2908953d1fd218158ebedf5d01cc27 --- doc/source/development_saio.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index c6a7d7f6ae..092c46d7cd 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -35,14 +35,14 @@ Installing dependencies * On apt based systems, #. `apt-get update` - #. `apt-get install curl gcc memcached rsync sqlite3 xfsprogs git-core python-setuptools` + #. `apt-get install curl gcc memcached rsync sqlite3 xfsprogs git-core libffi-dev python-setuptools` #. `apt-get install python-coverage python-dev python-nose python-simplejson python-xattr python-eventlet python-greenlet python-pastedeploy python-netifaces python-pip python-dnspython python-mock` * On yum based systems, - #. `yum install curl gcc memcached rsync sqlite xfsprogs git-core xinetd python-setuptools` + #. `yum install curl gcc memcached rsync sqlite xfsprogs git-core libffi-devel xinetd python-setuptools` #. `yum install python-coverage python-devel python-nose python-simplejson python-xattr python-eventlet python-greenlet python-pastedeploy python-netifaces python-pip python-dnspython python-mock` From d8e0492ea80adae990f35930465d6e905a3be061 Mon Sep 17 00:00:00 2001 From: Samuel Merritt Date: Tue, 27 Aug 2013 18:00:04 -0700 Subject: [PATCH 33/35] Fix internal swift.source tracking. In 1.8.0 (Grizzly), your proxy logs would indicate which middleware was responsible for an internal request, e.g. TU for tempurl or BD for bulk delete. At some point, those all turned into GET_INFO, which does not give you any idea which specific middleware was responsible, only that it came from a get_account_info/get_container_info call. This commit puts it back to how it was in 1.8.0. Also, the new-since-1.8.0 function get_object_info() got swift_source plumbing added to it, so source tracking for the quota middlewares' get_object_info() calls will happen now too. Note that due to the new-since-1.8.0 in-environment caching of account/container info, you may not see as many lines in the proxy log as you would with 1.8.0. This is because there are actually fewer internal requests being made. Change-Id: I2b2ff7823c612dc7ed7f268da979c4500bbbe911 --- swift/proxy/controllers/base.py | 24 ++++++++----- test/unit/proxy/controllers/test_base.py | 46 ++++++++++++++++++------ 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 9b16bf96df..df7828bd0e 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -238,7 +238,8 @@ def get_object_info(env, app, path=None, swift_source=None): """ (version, account, container, obj) = \ split_path(path or env['PATH_INFO'], 4, 4, True) - info = _get_object_info(app, env, account, container, obj) + info = _get_object_info(app, env, account, container, obj, + swift_source=swift_source) if not info: info = headers_to_object_info({}, 0) return info @@ -253,7 +254,8 @@ def get_container_info(env, app, swift_source=None): """ (version, account, container, unused) = \ split_path(env['PATH_INFO'], 3, 4, True) - info = get_info(app, env, account, container, ret_not_found=True) + info = get_info(app, env, account, container, ret_not_found=True, + swift_source=swift_source) if not info: info = headers_to_container_info({}, 0) return info @@ -268,7 +270,8 @@ def get_account_info(env, app, swift_source=None): """ (version, account, _junk, _junk) = \ split_path(env['PATH_INFO'], 2, 4, True) - info = get_info(app, env, account, ret_not_found=True) + info = get_info(app, env, account, ret_not_found=True, + swift_source=swift_source) if not info: info = headers_to_account_info({}, 0) if info.get('container_count') is None: @@ -421,22 +424,24 @@ def _get_info_cache(app, env, account, container=None): return None -def _prepare_pre_auth_info_request(env, path): +def _prepare_pre_auth_info_request(env, path, swift_source): """ Prepares a pre authed request to obtain info using a HEAD. :param env: the environment used by the current request :param path: The unquoted request path + :param swift_source: value for swift.source in WSGI environment :returns: the pre authed request """ # Set the env for the pre_authed call without a query string newenv = make_pre_authed_env(env, 'HEAD', path, agent='Swift', - query_string='', swift_source='GET_INFO') + query_string='', swift_source=swift_source) # Note that Request.blank expects quoted path return Request.blank(quote(path), environ=newenv) -def get_info(app, env, account, container=None, ret_not_found=False): +def get_info(app, env, account, container=None, ret_not_found=False, + swift_source=None): """ Get the info about accounts or containers @@ -462,7 +467,8 @@ def get_info(app, env, account, container=None, ret_not_found=False): return None path += '/' + container - req = _prepare_pre_auth_info_request(env, path) + req = _prepare_pre_auth_info_request( + env, path, (swift_source or 'GET_INFO')) # Whenever we do a GET/HEAD, the GETorHEAD_base will set the info in # the environment under environ[env_key] and in memcache. We will # pick the one from environ[env_key] and use it to set the caller env @@ -478,7 +484,7 @@ def get_info(app, env, account, container=None, ret_not_found=False): return None -def _get_object_info(app, env, account, container, obj): +def _get_object_info(app, env, account, container, obj, swift_source=None): """ Get the info about object @@ -498,7 +504,7 @@ def _get_object_info(app, env, account, container, obj): return info # Not in cached, let's try the object servers path = '/v1/%s/%s/%s' % (account, container, obj) - req = _prepare_pre_auth_info_request(env, path) + req = _prepare_pre_auth_info_request(env, path, swift_source) # Whenever we do a GET/HEAD, the GETorHEAD_base will set the info in # the environment under environ[env_key]. We will # pick the one from environ[env_key] and use it to set the caller env diff --git a/test/unit/proxy/controllers/test_base.py b/test/unit/proxy/controllers/test_base.py index 9af1a9f1d1..ea72cc1a64 100644 --- a/test/unit/proxy/controllers/test_base.py +++ b/test/unit/proxy/controllers/test_base.py @@ -48,19 +48,23 @@ def __init__(self, headers, env, account, container, obj): class FakeRequest(object): - def __init__(self, env, path): + def __init__(self, env, path, swift_source=None): self.environ = env (version, account, container, obj) = split_path(path, 2, 4, True) self.account = account self.container = container self.obj = obj if obj: + stype = 'object' self.headers = {'content-length': 5555, 'content-type': 'text/plain'} else: stype = container and 'container' or 'account' self.headers = {'x-%s-object-count' % (stype): 1000, 'x-%s-bytes-used' % (stype): 6666} + if swift_source: + meta = 'x-%s-meta-fakerequest-swift-source' % stype + self.headers[meta] = swift_source def get_response(self, app): return FakeResponse(self.headers, self.environ, self.account, @@ -127,7 +131,7 @@ def test_get_info(self): self.assertEquals(info_a['bytes'], 6666) self.assertEquals(info_a['total_object_count'], 1000) # Make sure the env cache is set - self.assertEquals(env, {'swift.account/a': info_a}) + self.assertEquals(env.get('swift.account/a'), info_a) # Do an env cached call to account info_a = get_info(None, env, 'a') @@ -136,7 +140,7 @@ def test_get_info(self): self.assertEquals(info_a['bytes'], 6666) self.assertEquals(info_a['total_object_count'], 1000) # Make sure the env cache is set - self.assertEquals(env, {'swift.account/a': info_a}) + self.assertEquals(env.get('swift.account/a'), info_a) # This time do env cached call to account and non cached to container with patch('swift.proxy.controllers.base.' @@ -147,8 +151,8 @@ def test_get_info(self): self.assertEquals(info_c['bytes'], 6666) self.assertEquals(info_c['object_count'], 1000) # Make sure the env cache is set - self.assertEquals(env['swift.account/a'], info_a) - self.assertEquals(env['swift.container/a/c'], info_c) + self.assertEquals(env.get('swift.account/a'), info_a) + self.assertEquals(env.get('swift.container/a/c'), info_c) # This time do a non cached call to account than non cached to # container @@ -161,8 +165,8 @@ def test_get_info(self): self.assertEquals(info_c['bytes'], 6666) self.assertEquals(info_c['object_count'], 1000) # Make sure the env cache is set - self.assertEquals(env['swift.account/a'], info_a) - self.assertEquals(env['swift.container/a/c'], info_c) + self.assertEquals(env.get('swift.account/a'), info_a) + self.assertEquals(env.get('swift.container/a/c'), info_c) # This time do an env cached call to container while account is not # cached @@ -173,7 +177,7 @@ def test_get_info(self): self.assertEquals(info_c['bytes'], 6666) self.assertEquals(info_c['object_count'], 1000) # Make sure the env cache is set and account still not cached - self.assertEquals(env, {'swift.container/a/c': info_c}) + self.assertEquals(env.get('swift.container/a/c'), info_c) # Do a non cached call to account not found with ret_not_found env = {} @@ -189,7 +193,7 @@ def test_get_info(self): self.assertEquals(info_a['bytes'], 6666) self.assertEquals(info_a['total_object_count'], 1000) # Make sure the env cache is set - self.assertEquals(env, {'swift.account/a': info_a}) + self.assertEquals(env.get('swift.account/a'), info_a) # Do a cached call to account not found with ret_not_found info_a = get_info(None, env, 'a', ret_not_found=True) @@ -198,7 +202,7 @@ def test_get_info(self): self.assertEquals(info_a['bytes'], 6666) self.assertEquals(info_a['total_object_count'], 1000) # Make sure the env cache is set - self.assertEquals(env, {'swift.account/a': info_a}) + self.assertEquals(env.get('swift.account/a'), info_a) # Do a non cached call to account not found without ret_not_found env = {} @@ -219,6 +223,21 @@ def test_get_info(self): self.assertEquals(info_a, None) self.assertEquals(env['swift.account/a']['status'], 404) + def test_get_container_info_swift_source(self): + req = Request.blank("/v1/a/c", environ={'swift.cache': FakeCache({})}) + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + resp = get_container_info(req.environ, 'app', swift_source='MC') + self.assertEquals(resp['meta']['fakerequest-swift-source'], 'MC') + + def test_get_object_info_swift_source(self): + req = Request.blank("/v1/a/c/o", + environ={'swift.cache': FakeCache({})}) + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + resp = get_object_info(req.environ, 'app', swift_source='LU') + self.assertEquals(resp['meta']['fakerequest-swift-source'], 'LU') + def test_get_container_info_no_cache(self): req = Request.blank("/v1/AUTH_account/cont", environ={'swift.cache': FakeCache({})}) @@ -250,6 +269,13 @@ def test_get_container_info_env(self): resp = get_container_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 3867) + def test_get_account_info_swift_source(self): + req = Request.blank("/v1/a", environ={'swift.cache': FakeCache({})}) + with patch('swift.proxy.controllers.base.' + '_prepare_pre_auth_info_request', FakeRequest): + resp = get_account_info(req.environ, 'a', swift_source='MC') + self.assertEquals(resp['meta']['fakerequest-swift-source'], 'MC') + def test_get_account_info_no_cache(self): req = Request.blank("/v1/AUTH_account", environ={'swift.cache': FakeCache({})}) From df39602c41605c4c68a47c6532a466ccc1a6633d Mon Sep 17 00:00:00 2001 From: David Goetz Date: Thu, 12 Sep 2013 07:38:23 -0700 Subject: [PATCH 34/35] bulk delete bug with trailing whitespace Change-Id: Ia48224a1a187a8ed6b0c9a3c72cac06f084a6fc8 --- swift/common/middleware/bulk.py | 8 +++++--- test/unit/common/middleware/test_bulk.py | 8 ++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/swift/common/middleware/bulk.py b/swift/common/middleware/bulk.py index 69086e6b3a..940caade9c 100644 --- a/swift/common/middleware/bulk.py +++ b/swift/common/middleware/bulk.py @@ -241,6 +241,7 @@ def get_objs_to_delete(self, req): while data_remaining: if '\n' in line: obj_to_delete, line = line.split('\n', 1) + obj_to_delete = obj_to_delete.strip() objs_to_delete.append( {'name': unquote(obj_to_delete)}) else: @@ -249,9 +250,10 @@ def get_objs_to_delete(self, req): line += data else: data_remaining = False - if line.strip(): + obj_to_delete = line.strip() + if obj_to_delete: objs_to_delete.append( - {'name': unquote(line)}) + {'name': unquote(obj_to_delete)}) if len(objs_to_delete) > self.max_deletes_per_request: raise HTTPRequestEntityTooLarge( 'Maximum Bulk Deletes: %d per request' % @@ -308,7 +310,7 @@ def handle_delete_iter(self, req, objs_to_delete=None, separator = '\r\n\r\n' last_yield = time() yield ' ' - obj_name = obj_to_delete['name'].strip() + obj_name = obj_to_delete['name'] if not obj_name: continue if len(failed_files) >= self.max_failed_deletes: diff --git a/test/unit/common/middleware/test_bulk.py b/test/unit/common/middleware/test_bulk.py index f5ae07a153..e51c4ee6a2 100644 --- a/test/unit/common/middleware/test_bulk.py +++ b/test/unit/common/middleware/test_bulk.py @@ -553,18 +553,18 @@ def fake_start_response(*args, **kwargs): req.method = 'DELETE' req.headers['Transfer-Encoding'] = 'chunked' req.headers['Accept'] = 'application/json' - req.environ['wsgi.input'] = StringIO('/c/f') + req.environ['wsgi.input'] = StringIO('/c/f%20') list(self.bulk(req.environ, fake_start_response)) # iterate over resp self.assertEquals( - self.app.delete_paths, ['/delete_works/AUTH_Acc/c/f']) + self.app.delete_paths, ['/delete_works/AUTH_Acc/c/f ']) self.assertEquals(self.app.calls, 1) def test_bulk_delete_get_objs(self): - req = Request.blank('/delete_works/AUTH_Acc', body='1\r\n2\r\n') + req = Request.blank('/delete_works/AUTH_Acc', body='1%20\r\n2\r\n') req.method = 'DELETE' with patch.object(self.bulk, 'max_deletes_per_request', 2): results = self.bulk.get_objs_to_delete(req) - self.assertEquals(results, [{'name': '1\r'}, {'name': '2\r'}]) + self.assertEquals(results, [{'name': '1 '}, {'name': '2'}]) with patch.object(bulk, 'MAX_PATH_LENGTH', 2): results = [] From 4c4a8abaa500d0d3940d81a4eb5ac21215ddc07a Mon Sep 17 00:00:00 2001 From: Kun Huang Date: Fri, 27 Sep 2013 15:25:53 +0800 Subject: [PATCH 35/35] improve bulk document This a very small change which just tell users request url of bulk delete request. In original docstrings, it just states the request parameters, request body and request method but not request url. Change-Id: I0bbc302a0e072910bb58e4814614d7f761433b10 --- swift/common/middleware/bulk.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/swift/common/middleware/bulk.py b/swift/common/middleware/bulk.py index 69086e6b3a..27934800ba 100644 --- a/swift/common/middleware/bulk.py +++ b/swift/common/middleware/bulk.py @@ -144,11 +144,11 @@ class Bulk(object): Will delete multiple objects or containers from their account with a single request. Responds to DELETE requests with query parameter - ?bulk-delete set. The Content-Type should be set to text/plain. - The body of the DELETE request will be a newline separated list of url - encoded objects to delete. You can delete 10,000 (configurable) objects - per request. The objects specified in the DELETE request body must be URL - encoded and in the form: + ?bulk-delete set. The request url is your storage url. The Content-Type + should be set to text/plain. The body of the DELETE request will be a + newline separated list of url encoded objects to delete. You can delete + 10,000 (configurable) objects per request. The objects specified in the + DELETE request body must be URL encoded and in the form: /container_name/obj_name