Skip to content
Browse files

Added option to disable SQLite db preallocation

Added option to disable SQLite db preallocation. This can be very
useful on pure ssd account/container servers where the extra space is
worth more than the lesser fragmentation.

Change-Id: I8fbb028a9b6143775b25b343e97896497a8b63a9
  • Loading branch information...
1 parent 156f27c commit 0becfab62924e3b23d74612dbc5b04ede075e8cb @gholt gholt committed Mar 28, 2012
View
4 doc/source/deployment_guide.rst
@@ -425,6 +425,10 @@ bind_ip 0.0.0.0 IP Address for server to bind to
bind_port 6002 Port for server to bind to
workers 1 Number of workers to fork
user swift User to run as
+db_preallocation on Normally Swift will try to preallocate disk
+ space for new SQLite databases to decrease
+ fragmentation (at the cost of disk usage). You
+ may turn this feature off here.
================== ========== =============================================
[account-server]
View
4 etc/account-server.conf-sample
@@ -11,6 +11,10 @@
# log_name = swift
# log_facility = LOG_LOCAL0
# log_level = INFO
+# Normally Swift will try to preallocate disk space for new SQLite databases to
+# decrease fragmentation (at the cost of disk usage). You may turn this feature
+# off here.
+# db_preallocation = on
[pipeline:main]
pipeline = account-server
View
4 etc/container-server.conf-sample
@@ -14,6 +14,10 @@
# log_name = swift
# log_facility = LOG_LOCAL0
# log_level = INFO
+# Normally Swift will try to preallocate disk space for new SQLite databases to
+# decrease fragmentation (at the cost of disk usage). You may turn this feature
+# off here.
+# db_preallocation = on
[pipeline:main]
pipeline = container-server
View
6 swift/account/auditor.py
@@ -17,9 +17,11 @@
import time
from random import random
+import swift.common.db
from swift.account import server as account_server
from swift.common.db import AccountBroker
-from swift.common.utils import get_logger, audit_location_generator
+from swift.common.utils import get_logger, audit_location_generator, \
+ TRUE_VALUES
from swift.common.daemon import Daemon
from eventlet import Timeout
@@ -37,6 +39,8 @@ def __init__(self, conf):
self.interval = int(conf.get('interval', 1800))
self.account_passes = 0
self.account_failures = 0
+ swift.common.db.DB_PREALLOCATION = \
+ conf.get('db_preallocation', 't').lower() in TRUE_VALUES
def run_forever(self, *args, **kwargs):
"""Run the account audit until stopped."""
View
5 swift/account/reaper.py
@@ -21,12 +21,13 @@
from eventlet import GreenPool, sleep, Timeout
+import swift.common.db
from swift.account.server import DATADIR
from swift.common.db import AccountBroker
from swift.common.direct_client import ClientException, \
direct_delete_container, direct_delete_object, direct_get_container
from swift.common.ring import Ring
-from swift.common.utils import get_logger, whataremyips
+from swift.common.utils import get_logger, whataremyips, TRUE_VALUES
from swift.common.daemon import Daemon
@@ -69,6 +70,8 @@ def __init__(self, conf):
self.container_concurrency = self.object_concurrency = \
sqrt(self.concurrency)
self.container_pool = GreenPool(size=self.container_concurrency)
+ swift.common.db.DB_PREALLOCATION = \
+ conf.get('db_preallocation', 't').lower() in TRUE_VALUES
self.delay_reaping = int(conf.get('delay_reaping') or 0)
def get_account_ring(self):
View
5 swift/account/server.py
@@ -29,9 +29,10 @@
HTTPPreconditionFailed, HTTPConflict
import simplejson
+import swift.common.db
from swift.common.db import AccountBroker
from swift.common.utils import get_logger, get_param, hash_path, \
- normalize_timestamp, split_path, storage_directory
+ normalize_timestamp, split_path, storage_directory, TRUE_VALUES
from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \
check_mount, check_float, check_utf8
from swift.common.db_replicator import ReplicatorRpc
@@ -52,6 +53,8 @@ def __init__(self, conf):
self.mount_check, logger=self.logger)
self.auto_create_account_prefix = \
conf.get('auto_create_account_prefix') or '.'
+ swift.common.db.DB_PREALLOCATION = \
+ conf.get('db_preallocation', 't').lower() in TRUE_VALUES
def _get_account_broker(self, drive, part, account):
hsh = hash_path(account)
View
4 swift/common/db.py
@@ -37,6 +37,8 @@
from swift.common.exceptions import LockTimeout
+#: Whether calls will be made to preallocate disk space for database files.
+DB_PREALLOCATION = True
#: Timeout for trying to connect to a DB
BROKER_TIMEOUT = 25
#: Pickle protocol to use
@@ -508,7 +510,7 @@ def _preallocate(self):
within 512k of a boundary, it allocates to the next boundary.
Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after.
"""
- if self.db_file == ':memory:':
+ if not DB_PREALLOCATION or self.db_file == ':memory:':
return
MB = (1024 * 1024)
View
6 swift/container/auditor.py
@@ -19,9 +19,11 @@
from eventlet import Timeout
+import swift.common.db
from swift.container import server as container_server
from swift.common.db import ContainerBroker
-from swift.common.utils import get_logger, audit_location_generator
+from swift.common.utils import get_logger, audit_location_generator, \
+ TRUE_VALUES
from swift.common.daemon import Daemon
@@ -38,6 +40,8 @@ def __init__(self, conf):
swift_dir = conf.get('swift_dir', '/etc/swift')
self.container_passes = 0
self.container_failures = 0
+ swift.common.db.DB_PREALLOCATION = \
+ conf.get('db_preallocation', 't').lower() in TRUE_VALUES
def run_forever(self, *args, **kwargs):
"""Run the container audit until stopped."""
View
3 swift/container/server.py
@@ -29,6 +29,7 @@
HTTPCreated, HTTPInternalServerError, HTTPNoContent, \
HTTPNotFound, HTTPPreconditionFailed, HTTPMethodNotAllowed
+import swift.common.db
from swift.common.db import ContainerBroker
from swift.common.utils import get_logger, get_param, hash_path, \
normalize_timestamp, storage_directory, split_path, validate_sync_to, \
@@ -65,6 +66,8 @@ def __init__(self, conf):
conf.get('auto_create_account_prefix') or '.'
if conf.get('allow_versions', 'f').lower() in TRUE_VALUES:
self.save_headers.append('x-versions-location')
+ swift.common.db.DB_PREALLOCATION = \
+ conf.get('db_preallocation', 't').lower() in TRUE_VALUES
def _get_container_broker(self, drive, part, account, container):
"""
View
3 swift/container/sync.py
@@ -20,6 +20,7 @@
from eventlet import sleep, Timeout
+import swift.common.db
from swift.container import server as container_server
from swift.common.client import ClientException, delete_object, put_object, \
quote
@@ -178,6 +179,8 @@ def __init__(self, conf, container_ring=None, object_ring=None):
self.object_ring = object_ring or Ring(swift_dir, ring_name='object')
self._myips = whataremyips()
self._myport = int(conf.get('bind_port', 6001))
+ swift.common.db.DB_PREALLOCATION = \
+ conf.get('db_preallocation', 't').lower() in TRUE_VALUES
def run_forever(self):
"""
View
5 swift/container/updater.py
@@ -23,12 +23,13 @@
from eventlet import spawn, patcher, Timeout
+import swift.common.db
from swift.container.server import DATADIR
from swift.common.bufferedhttp import http_connect
from swift.common.db import ContainerBroker
from swift.common.exceptions import ConnectionTimeout
from swift.common.ring import Ring
-from swift.common.utils import get_logger, whataremyips
+from swift.common.utils import get_logger, whataremyips, TRUE_VALUES
from swift.common.daemon import Daemon
@@ -55,6 +56,8 @@ def __init__(self, conf):
self.account_suppression_time = \
float(conf.get('account_suppression_time', 60))
self.new_account_suppressions = None
+ swift.common.db.DB_PREALLOCATION = \
+ conf.get('db_preallocation', 't').lower() in TRUE_VALUES
def get_account_ring(self):
"""Get the account ring. Load it if it hasn't been yet."""
View
7 test/unit/common/test_db.py
@@ -97,6 +97,13 @@ def setUp(self):
rmtree(self.testdir, ignore_errors=1)
os.mkdir(self.testdir)
+ def test_DB_PREALLOCATION_setting(self):
+ u = uuid4().hex
+ b = DatabaseBroker(u)
+ self.assertRaises(OSError, b._preallocate)
+ swift.common.db.DB_PREALLOCATION = False
+ b._preallocate()
+
def tearDown(self):
rmtree(self.testdir, ignore_errors=1)

0 comments on commit 0becfab

Please sign in to comment.
Something went wrong with that request. Please try again.