Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Added option to disable SQLite db preallocation
Added option to disable SQLite db preallocation. This can be very
useful on pure ssd account/container servers where the extra space is
worth more than the lesser fragmentation.

Change-Id: I8fbb028a9b6143775b25b343e97896497a8b63a9
  • Loading branch information
gholt committed Mar 28, 2012
1 parent 156f27c commit 0becfab
Show file tree
Hide file tree
Showing 12 changed files with 50 additions and 6 deletions.
4 changes: 4 additions & 0 deletions doc/source/deployment_guide.rst
Expand Up @@ -425,6 +425,10 @@ bind_ip 0.0.0.0 IP Address for server to bind to
bind_port 6002 Port for server to bind to
workers 1 Number of workers to fork
user swift User to run as
db_preallocation on Normally Swift will try to preallocate disk
space for new SQLite databases to decrease
fragmentation (at the cost of disk usage). You
may turn this feature off here.
================== ========== =============================================

[account-server]
Expand Down
4 changes: 4 additions & 0 deletions etc/account-server.conf-sample
Expand Up @@ -11,6 +11,10 @@
# log_name = swift
# log_facility = LOG_LOCAL0
# log_level = INFO
# Normally Swift will try to preallocate disk space for new SQLite databases to
# decrease fragmentation (at the cost of disk usage). You may turn this feature
# off here.
# db_preallocation = on

[pipeline:main]
pipeline = account-server
Expand Down
4 changes: 4 additions & 0 deletions etc/container-server.conf-sample
Expand Up @@ -14,6 +14,10 @@
# log_name = swift
# log_facility = LOG_LOCAL0
# log_level = INFO
# Normally Swift will try to preallocate disk space for new SQLite databases to
# decrease fragmentation (at the cost of disk usage). You may turn this feature
# off here.
# db_preallocation = on

[pipeline:main]
pipeline = container-server
Expand Down
6 changes: 5 additions & 1 deletion swift/account/auditor.py
Expand Up @@ -17,9 +17,11 @@
import time
from random import random

import swift.common.db
from swift.account import server as account_server
from swift.common.db import AccountBroker
from swift.common.utils import get_logger, audit_location_generator
from swift.common.utils import get_logger, audit_location_generator, \
TRUE_VALUES
from swift.common.daemon import Daemon

from eventlet import Timeout
Expand All @@ -37,6 +39,8 @@ def __init__(self, conf):
self.interval = int(conf.get('interval', 1800))
self.account_passes = 0
self.account_failures = 0
swift.common.db.DB_PREALLOCATION = \
conf.get('db_preallocation', 't').lower() in TRUE_VALUES

def run_forever(self, *args, **kwargs):
"""Run the account audit until stopped."""
Expand Down
5 changes: 4 additions & 1 deletion swift/account/reaper.py
Expand Up @@ -21,12 +21,13 @@

from eventlet import GreenPool, sleep, Timeout

import swift.common.db
from swift.account.server import DATADIR
from swift.common.db import AccountBroker
from swift.common.direct_client import ClientException, \
direct_delete_container, direct_delete_object, direct_get_container
from swift.common.ring import Ring
from swift.common.utils import get_logger, whataremyips
from swift.common.utils import get_logger, whataremyips, TRUE_VALUES
from swift.common.daemon import Daemon


Expand Down Expand Up @@ -69,6 +70,8 @@ def __init__(self, conf):
self.container_concurrency = self.object_concurrency = \
sqrt(self.concurrency)
self.container_pool = GreenPool(size=self.container_concurrency)
swift.common.db.DB_PREALLOCATION = \
conf.get('db_preallocation', 't').lower() in TRUE_VALUES
self.delay_reaping = int(conf.get('delay_reaping') or 0)

def get_account_ring(self):
Expand Down
5 changes: 4 additions & 1 deletion swift/account/server.py
Expand Up @@ -29,9 +29,10 @@
HTTPPreconditionFailed, HTTPConflict
import simplejson

import swift.common.db
from swift.common.db import AccountBroker
from swift.common.utils import get_logger, get_param, hash_path, \
normalize_timestamp, split_path, storage_directory
normalize_timestamp, split_path, storage_directory, TRUE_VALUES
from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \
check_mount, check_float, check_utf8
from swift.common.db_replicator import ReplicatorRpc
Expand All @@ -52,6 +53,8 @@ def __init__(self, conf):
self.mount_check, logger=self.logger)
self.auto_create_account_prefix = \
conf.get('auto_create_account_prefix') or '.'
swift.common.db.DB_PREALLOCATION = \
conf.get('db_preallocation', 't').lower() in TRUE_VALUES

def _get_account_broker(self, drive, part, account):
hsh = hash_path(account)
Expand Down
4 changes: 3 additions & 1 deletion swift/common/db.py
Expand Up @@ -37,6 +37,8 @@
from swift.common.exceptions import LockTimeout


#: Whether calls will be made to preallocate disk space for database files.
DB_PREALLOCATION = True
#: Timeout for trying to connect to a DB
BROKER_TIMEOUT = 25
#: Pickle protocol to use
Expand Down Expand Up @@ -508,7 +510,7 @@ def _preallocate(self):
within 512k of a boundary, it allocates to the next boundary.
Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after.
"""
if self.db_file == ':memory:':
if not DB_PREALLOCATION or self.db_file == ':memory:':
return
MB = (1024 * 1024)

Expand Down
6 changes: 5 additions & 1 deletion swift/container/auditor.py
Expand Up @@ -19,9 +19,11 @@

from eventlet import Timeout

import swift.common.db
from swift.container import server as container_server
from swift.common.db import ContainerBroker
from swift.common.utils import get_logger, audit_location_generator
from swift.common.utils import get_logger, audit_location_generator, \
TRUE_VALUES
from swift.common.daemon import Daemon


Expand All @@ -38,6 +40,8 @@ def __init__(self, conf):
swift_dir = conf.get('swift_dir', '/etc/swift')
self.container_passes = 0
self.container_failures = 0
swift.common.db.DB_PREALLOCATION = \
conf.get('db_preallocation', 't').lower() in TRUE_VALUES

def run_forever(self, *args, **kwargs):
"""Run the container audit until stopped."""
Expand Down
3 changes: 3 additions & 0 deletions swift/container/server.py
Expand Up @@ -29,6 +29,7 @@
HTTPCreated, HTTPInternalServerError, HTTPNoContent, \
HTTPNotFound, HTTPPreconditionFailed, HTTPMethodNotAllowed

import swift.common.db
from swift.common.db import ContainerBroker
from swift.common.utils import get_logger, get_param, hash_path, \
normalize_timestamp, storage_directory, split_path, validate_sync_to, \
Expand Down Expand Up @@ -65,6 +66,8 @@ def __init__(self, conf):
conf.get('auto_create_account_prefix') or '.'
if conf.get('allow_versions', 'f').lower() in TRUE_VALUES:
self.save_headers.append('x-versions-location')
swift.common.db.DB_PREALLOCATION = \
conf.get('db_preallocation', 't').lower() in TRUE_VALUES

def _get_container_broker(self, drive, part, account, container):
"""
Expand Down
3 changes: 3 additions & 0 deletions swift/container/sync.py
Expand Up @@ -20,6 +20,7 @@

from eventlet import sleep, Timeout

import swift.common.db
from swift.container import server as container_server
from swift.common.client import ClientException, delete_object, put_object, \
quote
Expand Down Expand Up @@ -178,6 +179,8 @@ def __init__(self, conf, container_ring=None, object_ring=None):
self.object_ring = object_ring or Ring(swift_dir, ring_name='object')
self._myips = whataremyips()
self._myport = int(conf.get('bind_port', 6001))
swift.common.db.DB_PREALLOCATION = \
conf.get('db_preallocation', 't').lower() in TRUE_VALUES

def run_forever(self):
"""
Expand Down
5 changes: 4 additions & 1 deletion swift/container/updater.py
Expand Up @@ -23,12 +23,13 @@

from eventlet import spawn, patcher, Timeout

import swift.common.db
from swift.container.server import DATADIR
from swift.common.bufferedhttp import http_connect
from swift.common.db import ContainerBroker
from swift.common.exceptions import ConnectionTimeout
from swift.common.ring import Ring
from swift.common.utils import get_logger, whataremyips
from swift.common.utils import get_logger, whataremyips, TRUE_VALUES
from swift.common.daemon import Daemon


Expand All @@ -55,6 +56,8 @@ def __init__(self, conf):
self.account_suppression_time = \
float(conf.get('account_suppression_time', 60))
self.new_account_suppressions = None
swift.common.db.DB_PREALLOCATION = \
conf.get('db_preallocation', 't').lower() in TRUE_VALUES

def get_account_ring(self):
"""Get the account ring. Load it if it hasn't been yet."""
Expand Down
7 changes: 7 additions & 0 deletions test/unit/common/test_db.py
Expand Up @@ -97,6 +97,13 @@ def setUp(self):
rmtree(self.testdir, ignore_errors=1)
os.mkdir(self.testdir)

def test_DB_PREALLOCATION_setting(self):
u = uuid4().hex
b = DatabaseBroker(u)
self.assertRaises(OSError, b._preallocate)
swift.common.db.DB_PREALLOCATION = False
b._preallocate()

def tearDown(self):
rmtree(self.testdir, ignore_errors=1)

Expand Down

0 comments on commit 0becfab

Please sign in to comment.