diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index e1db05f50d..64755e0851 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -425,6 +425,10 @@ bind_ip 0.0.0.0 IP Address for server to bind to bind_port 6002 Port for server to bind to workers 1 Number of workers to fork user swift User to run as +db_preallocation on Normally Swift will try to preallocate disk + space for new SQLite databases to decrease + fragmentation (at the cost of disk usage). You + may turn this feature off here. ================== ========== ============================================= [account-server] diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample index c521280aba..08a9fe4066 100644 --- a/etc/account-server.conf-sample +++ b/etc/account-server.conf-sample @@ -11,6 +11,10 @@ # log_name = swift # log_facility = LOG_LOCAL0 # log_level = INFO +# Normally Swift will try to preallocate disk space for new SQLite databases to +# decrease fragmentation (at the cost of disk usage). You may turn this feature +# off here. +# db_preallocation = on [pipeline:main] pipeline = account-server diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index 4dda5f38eb..f9f85950cf 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -14,6 +14,10 @@ # log_name = swift # log_facility = LOG_LOCAL0 # log_level = INFO +# Normally Swift will try to preallocate disk space for new SQLite databases to +# decrease fragmentation (at the cost of disk usage). You may turn this feature +# off here. +# db_preallocation = on [pipeline:main] pipeline = container-server diff --git a/swift/account/auditor.py b/swift/account/auditor.py index f5f9f3e76a..b9f3172a5a 100644 --- a/swift/account/auditor.py +++ b/swift/account/auditor.py @@ -17,9 +17,11 @@ import time from random import random +import swift.common.db from swift.account import server as account_server from swift.common.db import AccountBroker -from swift.common.utils import get_logger, audit_location_generator +from swift.common.utils import get_logger, audit_location_generator, \ + TRUE_VALUES from swift.common.daemon import Daemon from eventlet import Timeout @@ -37,6 +39,8 @@ def __init__(self, conf): self.interval = int(conf.get('interval', 1800)) self.account_passes = 0 self.account_failures = 0 + swift.common.db.DB_PREALLOCATION = \ + conf.get('db_preallocation', 't').lower() in TRUE_VALUES def run_forever(self, *args, **kwargs): """Run the account audit until stopped.""" diff --git a/swift/account/reaper.py b/swift/account/reaper.py index 57c64c3b59..f5096581ad 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -21,12 +21,13 @@ from eventlet import GreenPool, sleep, Timeout +import swift.common.db from swift.account.server import DATADIR from swift.common.db import AccountBroker from swift.common.direct_client import ClientException, \ direct_delete_container, direct_delete_object, direct_get_container from swift.common.ring import Ring -from swift.common.utils import get_logger, whataremyips +from swift.common.utils import get_logger, whataremyips, TRUE_VALUES from swift.common.daemon import Daemon @@ -69,6 +70,8 @@ def __init__(self, conf): self.container_concurrency = self.object_concurrency = \ sqrt(self.concurrency) self.container_pool = GreenPool(size=self.container_concurrency) + swift.common.db.DB_PREALLOCATION = \ + conf.get('db_preallocation', 't').lower() in TRUE_VALUES self.delay_reaping = int(conf.get('delay_reaping') or 0) def get_account_ring(self): diff --git a/swift/account/server.py b/swift/account/server.py index 800b3c0fa7..b0d73bd172 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -29,9 +29,10 @@ HTTPPreconditionFailed, HTTPConflict import simplejson +import swift.common.db from swift.common.db import AccountBroker from swift.common.utils import get_logger, get_param, hash_path, \ - normalize_timestamp, split_path, storage_directory + normalize_timestamp, split_path, storage_directory, TRUE_VALUES from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.db_replicator import ReplicatorRpc @@ -52,6 +53,8 @@ def __init__(self, conf): self.mount_check, logger=self.logger) self.auto_create_account_prefix = \ conf.get('auto_create_account_prefix') or '.' + swift.common.db.DB_PREALLOCATION = \ + conf.get('db_preallocation', 't').lower() in TRUE_VALUES def _get_account_broker(self, drive, part, account): hsh = hash_path(account) diff --git a/swift/common/db.py b/swift/common/db.py index 3a09bfa9d2..32f5d104d9 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -37,6 +37,8 @@ from swift.common.exceptions import LockTimeout +#: Whether calls will be made to preallocate disk space for database files. +DB_PREALLOCATION = True #: Timeout for trying to connect to a DB BROKER_TIMEOUT = 25 #: Pickle protocol to use @@ -508,7 +510,7 @@ def _preallocate(self): within 512k of a boundary, it allocates to the next boundary. Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after. """ - if self.db_file == ':memory:': + if not DB_PREALLOCATION or self.db_file == ':memory:': return MB = (1024 * 1024) diff --git a/swift/container/auditor.py b/swift/container/auditor.py index 0735c28119..89855e2e73 100644 --- a/swift/container/auditor.py +++ b/swift/container/auditor.py @@ -19,9 +19,11 @@ from eventlet import Timeout +import swift.common.db from swift.container import server as container_server from swift.common.db import ContainerBroker -from swift.common.utils import get_logger, audit_location_generator +from swift.common.utils import get_logger, audit_location_generator, \ + TRUE_VALUES from swift.common.daemon import Daemon @@ -38,6 +40,8 @@ def __init__(self, conf): swift_dir = conf.get('swift_dir', '/etc/swift') self.container_passes = 0 self.container_failures = 0 + swift.common.db.DB_PREALLOCATION = \ + conf.get('db_preallocation', 't').lower() in TRUE_VALUES def run_forever(self, *args, **kwargs): """Run the container audit until stopped.""" diff --git a/swift/container/server.py b/swift/container/server.py index f42dd63580..a3cfd5123f 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -29,6 +29,7 @@ HTTPCreated, HTTPInternalServerError, HTTPNoContent, \ HTTPNotFound, HTTPPreconditionFailed, HTTPMethodNotAllowed +import swift.common.db from swift.common.db import ContainerBroker from swift.common.utils import get_logger, get_param, hash_path, \ normalize_timestamp, storage_directory, split_path, validate_sync_to, \ @@ -65,6 +66,8 @@ def __init__(self, conf): conf.get('auto_create_account_prefix') or '.' if conf.get('allow_versions', 'f').lower() in TRUE_VALUES: self.save_headers.append('x-versions-location') + swift.common.db.DB_PREALLOCATION = \ + conf.get('db_preallocation', 't').lower() in TRUE_VALUES def _get_container_broker(self, drive, part, account, container): """ diff --git a/swift/container/sync.py b/swift/container/sync.py index 6c3ef3ebf1..441951d00f 100644 --- a/swift/container/sync.py +++ b/swift/container/sync.py @@ -20,6 +20,7 @@ from eventlet import sleep, Timeout +import swift.common.db from swift.container import server as container_server from swift.common.client import ClientException, delete_object, put_object, \ quote @@ -178,6 +179,8 @@ def __init__(self, conf, container_ring=None, object_ring=None): self.object_ring = object_ring or Ring(swift_dir, ring_name='object') self._myips = whataremyips() self._myport = int(conf.get('bind_port', 6001)) + swift.common.db.DB_PREALLOCATION = \ + conf.get('db_preallocation', 't').lower() in TRUE_VALUES def run_forever(self): """ diff --git a/swift/container/updater.py b/swift/container/updater.py index b3fe158a24..6a9fd1294a 100644 --- a/swift/container/updater.py +++ b/swift/container/updater.py @@ -23,12 +23,13 @@ from eventlet import spawn, patcher, Timeout +import swift.common.db from swift.container.server import DATADIR from swift.common.bufferedhttp import http_connect from swift.common.db import ContainerBroker from swift.common.exceptions import ConnectionTimeout from swift.common.ring import Ring -from swift.common.utils import get_logger, whataremyips +from swift.common.utils import get_logger, whataremyips, TRUE_VALUES from swift.common.daemon import Daemon @@ -55,6 +56,8 @@ def __init__(self, conf): self.account_suppression_time = \ float(conf.get('account_suppression_time', 60)) self.new_account_suppressions = None + swift.common.db.DB_PREALLOCATION = \ + conf.get('db_preallocation', 't').lower() in TRUE_VALUES def get_account_ring(self): """Get the account ring. Load it if it hasn't been yet.""" diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py index fcda936af2..f14097d8b1 100644 --- a/test/unit/common/test_db.py +++ b/test/unit/common/test_db.py @@ -97,6 +97,13 @@ def setUp(self): rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) + def test_DB_PREALLOCATION_setting(self): + u = uuid4().hex + b = DatabaseBroker(u) + self.assertRaises(OSError, b._preallocate) + swift.common.db.DB_PREALLOCATION = False + b._preallocate() + def tearDown(self): rmtree(self.testdir, ignore_errors=1)