Skip to content
Browse files

MB-6445: Remove sqlite backend

Sqlite will be deprecated and not supported anymore as of 2.0
release.

Change-Id: I1a5c8025444ca264897958b4053ff7cfe9c97010
Reviewed-on: http://review.couchbase.org/20207
Reviewed-by: Chiyoung Seo <chiyoung.seo@gmail.com>
Tested-by: Chiyoung Seo <chiyoung.seo@gmail.com>
  • Loading branch information...
1 parent a95a3bb commit d34de537c9841bd2f7bc071da4079e6b55150b82 @mikewied mikewied committed with chiyoung
View
12 .gitignore
@@ -50,9 +50,6 @@
/m4/lt~obsolete.m4
/m4/version.m4
/mc-kvstore-test
-/management/*bdbconvert
-/management/sqlite3
-/mbbackup-????.mbb
/sizes
/src/config.h
/src/config.h.in
@@ -61,17 +58,8 @@
/src/stats-info.[ch]
/src/testlogger_libify.cc
/tests/module_tests/*_test
-/wrapper/*badm-online-restore
-/wrapper/*badm-online-update
/wrapper/*badm-tap-registration
-/wrapper/*bbackup
-/wrapper/*bbackup-incremental
-/wrapper/*bbackup-merge-incremental
-/wrapper/*bdbconvert
-/wrapper/*bdbmaint
-/wrapper/*bdbupgrade
/wrapper/*bepctl
-/wrapper/*brestore
/wrapper/*bstats
/wrapper/*bvbucketctl
/wrapper/wrapper
View
87 Makefile.am
@@ -3,8 +3,6 @@ AM_CPPFLAGS += -I$(top_srcdir)/src -I${top_srcdir}/include
bin_SCRIPTS = management/cbanalyze-core \
wrapper/cbadm-tap-registration \
- wrapper/cbdbmaint \
- wrapper/cbdbupgrade \
wrapper/cbepctl \
wrapper/cbstats \
wrapper/cbvbucketctl
@@ -19,7 +17,6 @@ noinst_LTLIBRARIES = \
libconfiguration.la \
libkvstore.la \
libobjectregistry.la \
- libsqlite-kvstore.la \
libcouch-kvstore.la \
libdirutils.la
@@ -29,7 +26,6 @@ fixup_stats_json:
dist_doc_DATA = docs/stats.json
-bin_PROGRAMS = management/cbdbconvert
BUILT_SOURCES = src/generated_configuration.cc \
src/generated_configuration.hh \
src/stats-info.c src/stats-info.h
@@ -102,27 +98,7 @@ libkvstore_la_SOURCES = src/crc32.c src/crc32.h src/kvstore.cc src/kvstore.hh \
src/mutation_log.cc src/mutation_log.hh \
src/mutation_log_compactor.cc \
src/mutation_log_compactor.hh
-libkvstore_la_CPPFLAGS = -I$(top_srcdir)/src/sqlite-kvstore $(AM_CPPFLAGS)
-
-libsqlite_kvstore_la_SOURCES = src/kvstore.hh \
- src/sqlite-kvstore/factory.cc \
- src/sqlite-kvstore/pathexpand.hh \
- src/sqlite-kvstore/pathexpand.cc \
- src/sqlite-kvstore/sqlite-eval.cc \
- src/sqlite-kvstore/sqlite-eval.hh \
- src/sqlite-kvstore/sqlite-kvstore.cc \
- src/sqlite-kvstore/sqlite-kvstore.hh \
- src/sqlite-kvstore/sqlite-stats.hh \
- src/sqlite-kvstore/sqlite-pst.cc \
- src/sqlite-kvstore/sqlite-pst.hh \
- src/sqlite-kvstore/sqlite-strategies.cc \
- src/sqlite-kvstore/sqlite-strategies.hh \
- src/sqlite-kvstore/sqlite-vfs.c \
- src/sqlite-kvstore/sqlite-vfs.h
-libsqlite_kvstore_la_CPPFLAGS = -I$(top_srcdir)/src/sqlite-kvstore \
- $(AM_CPPFLAGS)
-libsqlite_kvstore_la_LIBADD =
-libsqlite_kvstore_la_DEPENDENCIES =
+libkvstore_la_CPPFLAGS = $(AM_CPPFLAGS)
libblackhole_kvstore_la_CPPFLAGS = -I$(top_srcdir)/src/blackhole-kvstore \
$(AM_CPPFLAGS)
@@ -171,46 +147,16 @@ libconfiguration_la_SOURCES = src/generated_configuration.hh \
src/configuration.hh \
src/configuration.cc
-ep_la_LIBADD = libkvstore.la libsqlite-kvstore.la \
+ep_la_LIBADD = libkvstore.la \
libblackhole-kvstore.la libcouch-kvstore.la \
libobjectregistry.la libconfiguration.la $(LTLIBEVENT)
-ep_la_DEPENDENCIES = libkvstore.la libsqlite-kvstore.la \
+ep_la_DEPENDENCIES = libkvstore.la \
libblackhole-kvstore.la \
libobjectregistry.la libconfiguration.la \
libcouch-kvstore.la
ep_testsuite_la_LIBADD =libobjectregistry.la $(LTLIBEVENT)
ep_testsuite_la_DEPENDENCIES = libobjectregistry.la
-management_cbdbconvert_SOURCES = src/atomic.cc src/mutex.cc \
- management/dbconvert.cc src/testlogger.cc \
- src/item.cc src/stored-value.cc src/ep_time.c \
- src/checkpoint.cc src/vbucketmap.cc
-management_cbdbconvert_LDADD = libkvstore.la libsqlite-kvstore.la \
- libblackhole-kvstore.la \
- libobjectregistry.la \
- libconfiguration.la libkvstore.la \
- libblackhole-kvstore.la \
- libcouch-kvstore.la \
- $(LTLIBEVENT)
-management_cbdbconvert_DEPENDENCIES = libkvstore.la libsqlite-kvstore.la \
- libcouch-kvstore.la libobjectregistry.la \
- libconfiguration.la
-
-if BUILD_EMBEDDED_LIBSQLITE3
-ep_la_LIBADD += libsqlite3.la
-ep_la_DEPENDENCIES += libsqlite3.la
-management_cbdbconvert_LDADD += libsqlite3.la
-management_cbdbconvert_DEPENDENCIES += libsqlite3.la
-noinst_LTLIBRARIES += libsqlite3.la
-bin_PROGRAMS += management/sqlite3
-else
-ep_la_LIBADD += $(LIBSQLITE3)
-management_cbdbconvert_LDADD += $(LIBSQLITE3)
-endif
-
-libsqlite3_la_SOURCES = src/embedded/sqlite3.h src/embedded/sqlite3.c
-libsqlite3_la_CFLAGS = $(AM_CFLAGS) ${NO_WERROR} -DSQLITE_THREADSAFE=2
-
check_PROGRAMS=\
atomic_ptr_test \
atomic_test \
@@ -223,7 +169,6 @@ check_PROGRAMS=\
misc_test \
mutation_log_test \
mutex_test \
- pathexpand_test \
priority_test \
ringbuffer_test \
vbucket_test
@@ -345,11 +290,6 @@ gen_config_LDFLAGS = -lm
gen_code_CPPFLAGS = -I$(top_srcdir)/tools $(AM_CPPFLAGS)
gen_code_SOURCES = tools/gencode.cc tools/cJSON.c tools/cJSON.h
-management_sqlite3_SOURCES = src/embedded/sqlite3-shell.c
-management_sqlite3_CFLAGS = $(AM_CFLAGS) ${NO_WERROR}
-management_sqlite3_DEPENDENCIES = libsqlite3.la
-management_sqlite3_LDADD = libsqlite3.la
-
vbucket_test_CXXFLAGS = $(AM_CPPFLAGS) $(AM_CXXFLAGS) ${NO_WERROR}
vbucket_test_SOURCES = tests/module_tests/vbucket_test.cc \
tests/module_tests/threadtests.hh src/vbucket.hh \
@@ -405,12 +345,6 @@ chunk_creation_test_CXXFLAGS = $(AM_CPPFLAGS) $(AM_CXXFLAGS) ${NO_WERROR}
chunk_creation_test_SOURCES = tests/module_tests/chunk_creation_test.cc \
src/common.hh
-pathexpand_test_CPPFLAGS = -I$(top_srcdir)/src/sqlite-kvstore \
- $(AM_CPPFLAGS) ${NO_WERROR}
-pathexpand_test_SOURCES = tests/module_tests/pathexpand_test.cc \
- src/sqlite-kvstore/pathexpand.hh \
- src/sqlite-kvstore/pathexpand.cc
-
ringbuffer_test_CXXFLAGS = $(AM_CPPFLAGS) $(AM_CXXFLAGS) ${NO_WERROR}
ringbuffer_test_SOURCES = tests/module_tests/ringbuffer_test.cc src/ringbuffer.hh
ringbuffer_test_DEPENDENCIES = src/ringbuffer.hh
@@ -421,7 +355,6 @@ hrtime_test_SOURCES += src/gethrtime.c
dispatcher_test_SOURCES += src/gethrtime.c
vbucket_test_SOURCES += src/gethrtime.c
checkpoint_test_SOURCES += src/gethrtime.c
-management_cbdbconvert_SOURCES += src/gethrtime.c
ep_testsuite_la_SOURCES += src/gethrtime.c
hash_table_test_SOURCES += src/gethrtime.c
mutation_log_test_SOURCES += src/gethrtime.c
@@ -430,12 +363,10 @@ endif
if BUILD_BYTEORDER
ep_la_SOURCES += src/byteorder.c
ep_testsuite_la_SOURCES += src/byteorder.c
-management_cbdbconvert_SOURCES += src/byteorder.c
endif
pythonlibdir=$(libdir)/python
pythonlib_DATA= \
- management/capture.py \
management/clitool.py \
management/mc_bin_client.py \
management/mc_bin_server.py \
@@ -445,16 +376,12 @@ pythonlib_DATA= \
pythonlib_SCRIPTS= \
management/cbadm-tap-registration \
- management/cbdbmaint \
- management/cbdbupgrade \
management/cbepctl \
management/cbstats \
management/cbvbucketctl
PYTHON_TOOLS= \
wrapper/cbadm-tap-registration \
- wrapper/cbdbmaint \
- wrapper/cbdbupgrade \
wrapper/cbepctl \
wrapper/cbstats \
wrapper/cbvbucketctl
@@ -498,8 +425,6 @@ ep_la_LIBADD += ep_la-probes.lo
ep_la_DEPENDENCIES += ep_la-probes.lo
ep_testsuite_la_LIBADD += ep_testsuite_la-probes.lo
ep_testsuite_la_DEPENDENCIES += ep_testsuite_la-probes.lo
-management_cbdbconvert_LDADD += .libs/cddbconvert-probes.o
-management_cbdbconvert_DEPENDENCIES += .libs/cddbconvert-probes.o
atomic_ptr_test_LDADD = .libs/atomic_ptr_test-probes.o
atomic_ptr_test_DEPENDENCIES += .libs/atomic_ptr_test-probes.o
atomic_test_LDADD = .libs/atomic_test-probes.o
@@ -550,12 +475,6 @@ ep_testsuite_la-probes.lo: $(ep_testsuite_la_OBJECTS) dtrace/probes.h
sed -e s,ep_la-item,ep_testsuite_la-probes,g ep_la-item.lo \
> ep_testsuite_la-probes.lo
-.libs/cddbconvert-probes.o: $(management_cbdbconvert_OBJECTS) dtrace/probes.h
- $(DTRACE) $(DTRACEFLAGS) -G \
- -o .libs/cddbconvert-probes.o \
- -s ${srcdir}/dtrace/probes.d \
- $(management_cbdbconvert_OBJECTS)
-
.libs/atomic_ptr_test-probes.o: $(atomic_ptr_test_OBJECTS) dtrace/probes.h
$(DTRACE) $(DTRACEFLAGS) -G \
-o .libs/atomic_ptr_test-probes.o \
View
18 configuration.json
@@ -40,15 +40,13 @@
}
},
"backend": {
- "default": "sqlite",
+ "default": "couchdb",
"dynamic": false,
"type": "std::string",
"validator": {
"enum": [
"blackhole",
- "couchdb",
- "sqlite",
- "mccouch"
+ "couchdb"
]
}
},
@@ -117,14 +115,6 @@
"descr": "True if we want to enable data traffic after warmup is complete",
"type": "bool"
},
- "db_shards": {
- "default": "4",
- "type": "size_t"
- },
- "db_strategy": {
- "default": "multiMTVBDB",
- "type": "std::string"
- },
"dbname": {
"default": "/tmp/test",
"descr": "Path to on-disk storage.",
@@ -327,10 +317,6 @@
}
}
},
- "shardpattern": {
- "default": "%d/%b-%i.sqlite",
- "type": "std::string"
- },
"stored_val_type": {
"default": "",
"type": "std::string"
View
12 configure.ac
@@ -40,18 +40,6 @@ AC_ARG_ENABLE([valgrind],
AS_IF([test "$ac_enable_valgrind" = "yes"],
[ AC_DEFINE(VALGRIND, 1, [Enable extra memset to help valgrind])])
-AC_ARG_ENABLE([system-libsqlite3],
- [AS_HELP_STRING([--enable-system-libsqlite3],
- [Build with system installed libsqlite3. @<:@default=off@:>@])],
- [ac_enable_system_libsqlite3="$enableval"],
- [ac_enable_system_libsqlite3="no"])
-
-AS_IF([test "$ac_enable_system_libsqlite3" = "yes"],
- [ PANDORA_REQUIRE_LIBSQLITE3
- AC_DEFINE(USE_SYSTEM_LIBSQLITE3, 1, "Set if you want to use a system installed sqlite3")
- ])
-AM_CONDITIONAL(BUILD_EMBEDDED_LIBSQLITE3, test "$ac_enable_system_libsqlite3" = "no")
-
AC_ARG_ENABLE([generated-tests],
[AS_HELP_STRING([--enable-generated-tests],
[Run generated test suite. @<:@default=off@:>@])],
View
24 docs/engine-params.org
@@ -28,14 +28,8 @@ memcached like this:
|------------------------+--------+--------------------------------------------|
| config_file | string | Path to additional parameters. |
| dbname | string | Path to on-disk storage. |
-| shardpattern | string | File pattern for shards (see below) |
| ht_locks | int | Number of locks per hash table. |
| ht_size | int | Number of buckets per hash table. |
-| initfile | string | Optional SQL script to run after |
-| | | opening DB |
-| postInitfile | string | Optional SQL script to run after |
-| | | all DB shards and statements have |
-| | | been initialized |
| max_item_size | int | Maximum number of bytes allowed for |
| | | an item. |
| max_size | int | Max cumulative item size in bytes. |
@@ -72,9 +66,6 @@ memcached like this:
| failpartialwarmup | bool | If false, continue running after failing |
| | | to load some records. |
| max_vbuckets | int | Maximum number of vbuckets expected (1024) |
-| db_shards | int | Number of shards for db store |
-| db_strategy | string | DB store strategy ("multiDB", "singleDB" |
-| | | or "singleMTDB") |
| concurrentDB | bool | True (default) if concurrent DB reads are |
| | | permitted where possible. |
| chk_remover_stime | int | Interval for the checkpoint remover that |
@@ -123,18 +114,3 @@ memcached like this:
| } | scanner will be scheduled to run. |
| pager_active_vb_pcnt | int | Percentage of active vbucket items among |
| | | all evicted items by item pager. |
-
-** Shard Patterns
-
-When using a multi-sharded database, =shardpattern= specifies how
-shards are named relative to the main database.
-
-=shardpattern= uses a printf-style string to expand the path with the
-given variables:
-
-- =%d= : The directory name containing the main database
- (This will be =.= for relative paths).
-- =%b= : The base name of the main database.
-- =%i= : The shard number.
-
-The default value of =shardpattern= is =%d/%b-%i.sqlite=
View
11 docs/stats.org
@@ -128,8 +128,6 @@ For introductory information on stats within membase, start with the
| ep_tap_keepalive | Tap keepalive time. |
| ep_dbname | DB path. |
| ep_dbinit | Number of seconds to initialize DB. |
-| ep_dbshards | Number of shards for db store |
-| ep_db_strategy | SQLite db strategy |
| ep_io_num_read | Number of io read operations |
| ep_io_num_write | Number of io write operations |
| ep_io_read_bytes | Number of bytes read (key + values) |
@@ -627,15 +625,6 @@ The following stats are available for all database engine:
| writeSize | Size of data in write operations |
| delete | Time spent in delete() calls |
-The following stats are available for the Sqlite database engine:
-
-| sector_size | Disk sector size (bytes) |
-| lock | Number of database lock operations |
-| truncate | Number of truncate operations |
-| sync | Time spent in sync() calls |
-| readSeek | Seek distance in read operations |
-| writeSeek | Seek distance in write operations |
-
The following stats are available for the CouchStore database engine:
| backend_type | Type of backend database engine |
View
46 m4/pandora_have_libpq.m4
@@ -1,46 +0,0 @@
-dnl Copyright (C) 2009 Sun Microsystems
-dnl This file is free software; Sun Microsystems
-dnl gives unlimited permission to copy and/or distribute it,
-dnl with or without modifications, as long as this notice is preserved.
-
-AC_DEFUN([_PANDORA_SEARCH_LIBPQ],[
- AC_REQUIRE([AC_LIB_PREFIX])
-
- dnl --------------------------------------------------------------------
- dnl Check for libpq
- dnl --------------------------------------------------------------------
-
- AC_ARG_ENABLE([libpq],
- [AS_HELP_STRING([--disable-libpq],
- [Build with libpq support @<:@default=on@:>@])],
- [ac_enable_libpq="$enableval"],
- [ac_enable_libpq="yes"])
-
- AS_IF([test "x$ac_enable_libpq" = "xyes"],[
- AC_CHECK_HEADERS([libpq-fe.h])
- AC_LIB_HAVE_LINKFLAGS(pq,,[
- #ifdef HAVE_LIBPQ_FE_H
- # include <libpq-fe.h>
- #else
- # include <postgresql/libpq-fe.h>
- #endif
- ], [
- PGconn *conn;
- conn = PQconnectdb(NULL);
- ])
- ],[
- ac_cv_libpq="no"
- ])
-
- AM_CONDITIONAL(HAVE_LIBPQ, [test "x${ac_cv_libpq}" = "xyes"])
-])
-
-AC_DEFUN([PANDORA_HAVE_LIBPQ],[
- AC_REQUIRE([_PANDORA_SEARCH_LIBPQ])
-])
-
-AC_DEFUN([PANDORA_REQUIRE_LIBPQ],[
- AC_REQUIRE([PANDORA_HAVE_LIBPQ])
- AS_IF([test "x${ac_cv_libpq}" = "xno"],
- AC_MSG_ERROR([libpq is required for ${PACKAGE}]))
-])
View
42 m4/pandora_have_libsqlite3.m4
@@ -1,42 +0,0 @@
-dnl Copyright (C) 2009 Sun Microsystems
-dnl This file is free software; Sun Microsystems
-dnl gives unlimited permission to copy and/or distribute it,
-dnl with or without modifications, as long as this notice is preserved.
-
-AC_DEFUN([_PANDORA_SEARCH_LIBSQLITE3],[
- AC_REQUIRE([AC_LIB_PREFIX])
-
- dnl --------------------------------------------------------------------
- dnl Check for libsqlite3
- dnl --------------------------------------------------------------------
-
- AC_ARG_ENABLE([libsqlite3],
- [AS_HELP_STRING([--disable-libsqlite3],
- [Build with libsqlite3 support @<:@default=on@:>@])],
- [ac_enable_libsqlite3="$enableval"],
- [ac_enable_libsqlite3="yes"])
-
- AS_IF([test "x$ac_enable_libsqlite3" = "xyes"],[
- AC_LIB_HAVE_LINKFLAGS(sqlite3,,[
- #include <stdio.h>
- #include <sqlite3.h>
- ],[
- sqlite3 *db;
- sqlite3_open(NULL, &db);
- ])
- ],[
- ac_cv_libsqlite3="no"
- ])
-
- AM_CONDITIONAL(HAVE_LIBSQLITE3, [test "x${ac_cv_libsqlite3}" = "xyes"])
-])
-
-AC_DEFUN([PANDORA_HAVE_LIBSQLITE3],[
- AC_REQUIRE([_PANDORA_SEARCH_LIBSQLITE3])
-])
-
-AC_DEFUN([PANDORA_REQUIRE_LIBSQLITE3],[
- AC_REQUIRE([_PANDORA_SEARCH_LIBSQLITE3])
- AS_IF([test "x${ac_cv_libsqlite3}" = "xno"],
- AC_MSG_ERROR([libsqlite3 is required for ${PACKAGE}]))
-])
View
69 management/capture.py
@@ -1,69 +0,0 @@
-#!/usr/bin/env python
-
-import sys
-import time
-
-from sqlite3 import dbapi2 as sqlite
-
-import mc_bin_client
-
-CREATE_TABLE = """
-create table if not exists samples (
- host varchar(64) not null,
- uptime integer not null,
- statname varchar(64) not null,
- value varchar(64) not null
-)
-"""
-
-INS = """insert into samples (host, uptime, statname, value) values (?, ?, ?, ?)"""
-
-dbpath = None
-
-def wants_db(orig):
- def f(*args):
- global dbpath
- db = sqlite.connect(dbpath)
- try:
- return orig(*args + (db,))
- finally:
- db.commit()
- db.close()
- return f
-
-@wants_db
-def setup(db):
- db.execute(CREATE_TABLE)
-
-@wants_db
-def sample(hp, db):
- cur = db.cursor()
-
- try:
- h, p = hp.split(':')
- p = int(p)
- except:
- h = hp
- port = 11211
-
- mc = mc_bin_client.MemcachedClient(h, p)
- stats = mc.stats()
- uptime = int(stats['uptime'])
- print "Sampling", h, "at uptime =", uptime
- for k,v in stats.iteritems():
- cur.execute(INS, (h, uptime, k, v))
-
-if __name__ == '__main__':
- dbpath = sys.argv[1]
- hosts = sys.argv[2:]
- assert hosts
-
- setup()
- while True:
- try:
- for h in hosts:
- sample(h)
- except:
- import traceback
- traceback.print_exc()
- time.sleep(60)
View
88 management/cbdbmaint
@@ -1,88 +0,0 @@
-#!/usr/bin/env python
-import sys
-import os
-import glob
-import shutil
-import getopt
-import subprocess
-import mc_bin_client
-
-DEFAULT_PORT="11210"
-
-def usage(err=None):
- print >> sys.stderr, """
-Usage: %s [--vacuum] [--backupto=<dest_dir>] [--port=%s]
-""" % (os.path.basename(sys.argv[0]), DEFAULT_PORT)
-
- if err:
- print >> sys.stderr, err
- sys.exit(1)
-
-def isADBFile(f):
- return f.endswith('.sqlite') or f.endswith('.mb')
-
-def run_sql(sqlite, fn, sql, more_args=[], logger=sys.stderr):
- args = ['-batch', '-bail']
- cmd = [sqlite] + args + more_args + [fn]
- p = subprocess.Popen(cmd,
- stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- (o,e) = p.communicate(sql)
- if p.returncode != 0:
- logger.write("Error running query: %s\n" % sql)
- logger.write(e)
- sys.exit(1)
- return o
-
-def findCmd(cmdName):
- cmd_dir = os.path.dirname(sys.argv[0])
- possible = []
- for bin_dir in [cmd_dir, os.path.join(cmd_dir, "..", "..", "bin")]:
- possible = possible + [os.path.join(bin_dir, p) for p in [cmdName, cmdName + '.exe']]
- cmdbin = [p for p in possible if os.path.exists(p)][0]
- return cmdbin
-
-def main():
- try:
- opts, args = getopt.getopt(sys.argv[1:], '', ['vacuum', 'backupto=', 'port='])
- except getopt.GetoptError, e:
- usage(e.msg)
-
- if not opts:
- usage()
-
- flushctl = findCmd('cbflushctl')
- sqlitebin = findCmd('sqlite3')
-
- optdict = dict(opts)
- shouldVacuum = '--vacuum' in optdict
-
- dest_dir = optdict.get("--backupto")
- port = int(optdict.get("--port", DEFAULT_PORT))
-
- mc = mc_bin_client.MemcachedClient('127.0.0.1', port=port)
- db_path = mc.stats()['ep_dbname']
- db_files = glob.glob('%s*' % db_path)
-
- print 'Pausing persistence... ',
- os.system('"%s" 127.0.0.1:%d stop' % (flushctl, port))
- print 'paused.'
- try:
- for fn in db_files:
-
- if shouldVacuum and (fn == db_path or isADBFile(fn)):
- print "Vacuuming", fn
- run_sql(sqlitebin, fn, "vacuum")
-
- if dest_dir:
- dest_fn = os.path.join(dest_dir, os.path.basename(fn))
- print 'Copying %s to %s' % (fn, dest_fn)
- shutil.copyfile(fn, dest_fn)
- finally:
- print 'Unpausing persistence.'
- os.system('"%s" 127.0.0.1:%d start' % (flushctl, port))
-
-
-if __name__ == '__main__':
- main()
View
187 management/cbdbupgrade
@@ -1,187 +0,0 @@
-#!/usr/bin/env python
-
-import sys
-import os
-import glob
-import subprocess
-import shutil
-
-TARGET_VERSION=2
-
-# Backported any to older versions of python
-try:
- any
-except NameError:
- def any(a):
- for i in a:
- if i:
- return True
- return False
-
-try:
- all
-except NameError:
- def all(iterable):
- for element in iterable:
- if not element:
- return False
- return True
-
-def run_sql(sqlite, fn, sql, more_args=[], logger=sys.stderr):
- args = ['-batch', '-bail']
- cmd = [sqlite] + args + more_args + [fn]
- p = subprocess.Popen(cmd,
- stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- (o,e) = p.communicate(sql)
- if p.returncode != 0:
- logger.write("Error running query: %s\n" % sql)
- logger.write(e)
- sys.exit(1)
- return o
-
-def version(sqlite, fn):
- return int(run_sql(sqlite, fn, "pragma user_version;").strip())
-
-def count_kv_vb_tables(sqlite, fn):
- return int(run_sql(sqlite, fn,
- 'select count(*) from sqlite_master '\
- 'where name like "kv_%";').strip())
-
-def findCmd(cmdName):
- cmd_dir = os.path.dirname(sys.argv[0])
- possible = []
- for bin_dir in [cmd_dir, os.path.join(cmd_dir, "..", "..", "bin")]:
- possible = possible + [os.path.join(bin_dir, p) for p in [cmdName, cmdName + '.exe']]
- cmdbin = [p for p in possible if os.path.exists(p)][0]
- return cmdbin
-
-def findSqlite():
- return findCmd('sqlite3')
-
-def usage(command, msg):
- e = """Error: %s
-
-Usage: %s /path/to/srcdb [...] /path/to/dest
-
-src is the path to the main database. You can specify more than one.
-dest can be either a new main database filename, or a directory
-(the old DB name will be used).
-
-If more than one src db is specified, the destination *must* be a
-directory.""" % (msg, command)
- sys.exit(e)
-
-class NullLogger(object):
-
- def write(self, x):
- pass
-
-def updateVBStatesTableSchema(sqlite, src):
- try:
- run_sql(sqlite, src, 'alter table vbucket_states add column vb_version;',
- logger=NullLogger())
- except SystemExit:
- pass # Already had the column
- try:
- run_sql(sqlite, src, 'alter table vbucket_states add column checkpoint_id;',
- logger=NullLogger())
- except SystemExit:
- pass # Already had the column
-
-def updateKVTableSchema(sqlite, src):
- for d in glob.glob(src + '*.mb') + glob.glob(src + '*.sqlite'):
- if os.path.exists(d):
- try:
- run_sql(sqlite, d, 'alter table kv add column vb_version;',
- logger=NullLogger())
- except SystemExit:
- pass # Already had the column
-
-def ensureNewColumns(sqlite, src):
- updateVBStatesTableSchema(sqlite, src)
- updateKVTableSchema(sqlite, src)
-
-def findTarget(src, dest):
- if os.path.isdir(dest):
- dest = os.path.join(dest, os.path.basename(src))
- return dest
-
-def findTargetCollisions(srcs, dest):
- rv = []
- for fn in (findTarget(s, dest) for s in srcs):
- if os.path.exists(fn):
- rv.append(fn)
- for shard in glob.glob(fn + '*.mb'):
- if os.path.exists(shard):
- rv.append(shard)
- return rv
-
-def doset(sqlite, src, dest):
- v = version(sqlite, src)
- print 'Source version from "%s" is %d' % (src, v)
-
- args = [findCmd('cbdbconvert')]
- if v < 2:
- args.append("--remove-crlf")
- if os.path.exists(src + '-0.mb'):
- args.append('--src-pattern=%d/%b-%i.mb')
- if count_kv_vb_tables(sqlite, src + "-0.mb") > 0:
- args.append("--src-strategy=multiMTVBDB")
- args.append(src)
- args.append(findTarget(src, dest))
- args.append("--report-every=2000")
- mydir = os.path.dirname(sys.argv[0])
- if os.name == 'nt':
- init_file = os.path.join(mydir, '..', 'etc', 'couchbase', 'init.sql')
- else:
- init_file = os.path.join(mydir, '..', '..', 'etc', 'couchbase', 'init.sql')
- args.append("--init-file=" + init_file)
-
- print " ".join(args)
-
- ensureNewColumns(sqlite, src)
-
- returncode = subprocess.call(args)
- if returncode != 0:
- sys.stderr.write("Error running convert, exit code %d (%s)\n" %
- (returncode, ' '.join(args)))
- sys.exit(1)
-
-def main():
- command = sys.argv[0]
- args = sys.argv[1:]
- try:
- dest = args.pop()
- srcs = args
- if not srcs:
- usage(command, "No destination specified")
- if not all(os.path.isfile(p) for p in srcs):
- usage(command, "Source must point to existing main DB files")
- if len(srcs) > 1 and not os.path.isdir(dest):
- usage(command,
- "Multiple sources, but destination is not a directory.")
- except IndexError:
- usage(command, "Too few arguments")
-
- if os.name == 'nt':
- mydir = os.path.dirname(sys.argv[0])
- memcache_dir = os.path.join(mydir, '..', '..', 'memcached')
- erlang_dir = os.path.join(mydir, 'erlang', 'bin')
- path = [mydir, memcache_dir, erlang_dir, os.environ['PATH']]
- os.environ['PATH'] = ';'.join(path)
-
- sqlite = findSqlite()
-
- collisions = findTargetCollisions(srcs, dest)
- if collisions:
- sys.stderr.write("Would overwrite the following files:\n\t")
- sys.stderr.write('\n\t'.join(collisions) + '\n')
- sys.exit(1)
-
- for src in srcs:
- doset(sqlite, src, dest)
-
-if __name__ == '__main__':
- main()
View
286 management/dbconvert.cc
@@ -1,286 +0,0 @@
-/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-
-#include "config.h"
-
-#include <cassert>
-#include <iostream>
-#include <utility>
-#include <cstdlib>
-
-#ifdef HAVE_SYSEXITS_H
-#include <sysexits.h>
-#endif
-
-#include <memcached/engine.h>
-
-#include <getopt.h>
-#include <stats.hh>
-#include <kvstore.hh>
-#include <item.hh>
-#include <callbacks.hh>
-#include "sqlite-kvstore/sqlite-strategies.hh"
-#include "sqlite-kvstore/sqlite-kvstore.hh"
-
-#ifndef EX_USAGE
-#define EX_USAGE 64
-#endif
-
-/* getopt.h on Solaris defines the name member as "char*" and not
- * "const char*". This cause a compile error when you try to assign
- * it to a constant string. To aviod compile errors let's create
- * a macro to cast it to a char pointer.
- */
-#ifdef __sun
-#define OPTNAME(a) (char*)(a)
-#else
-#define OPTNAME(a) (const char*)(a)
-#endif
-
-using namespace std;
-
-static KVStore *getStore(EPStats &st,
- const char *path,
- const char *strategyName,
- const char *shardPattern,
- const char *initFile = NULL) {
- db_type dbStrategy = multi_db;
-
- if (!SqliteKVStoreFactory::stringToType(strategyName, dbStrategy)) {
- cerr << "Unable to parse strategy type: " << strategyName << endl;
- exit(EX_USAGE);
- }
-
- size_t nVBuckets(1024);
- size_t dbShards(4);
- SqliteStrategy *sqliteInstance = NULL;
-
- switch (dbStrategy) {
- case multi_db:
- sqliteInstance = new MultiDBSingleTableSqliteStrategy(path,
- shardPattern,
- initFile,
- NULL, dbShards);
- break;
- case single_db:
- sqliteInstance = new SingleTableSqliteStrategy(path,
- initFile,
- NULL);
- break;
- case single_mt_db:
- sqliteInstance = new MultiTableSqliteStrategy(path,
- initFile,
- NULL, nVBuckets);
- break;
- case multi_mt_db:
- sqliteInstance = new ShardedMultiTableSqliteStrategy(path,
- shardPattern,
- initFile,
- NULL, nVBuckets,
- dbShards);
- break;
- case multi_mt_vb_db:
- sqliteInstance = new ShardedByVBucketSqliteStrategy(path,
- shardPattern,
- initFile,
- NULL, nVBuckets,
- dbShards);
- break;
- }
-
- return new StrategicSqlite3(st,
- shared_ptr<SqliteStrategy>(sqliteInstance));
-}
-
-class MutationVerifier : public Callback<mutation_result> {
-public:
- void callback(mutation_result &mutation) {
- assert(mutation.first == 1);
- }
-};
-
-class Mover : public Callback<GetValue> {
-public:
-
- Mover(KVStore *d, size_t re, bool kc, size_t ts) : dest(d),
- transferred(0),
- txnSize(ts),
- reportEvery(re),
- killCrlf(kc),
- inTxn(false) {
- assert(dest);
- assert(txnSize != 0);
- assert(reportEvery != 0);
-
- if (!enterTransaction()) {
- cout << "Failed to start a transaction" << endl;
- abort();
- }
-
- startTime = gethrtime();
- }
-
- ~Mover() {
- nextTransaction(false);
- cout << "Elapsed time=" << (gethrtime() - startTime)/1000000000 << " seconds."<< std::endl;
- }
-
- void callback(GetValue &gv) {
- Item *i = gv.getValue();
- adjust(&i);
- dest->set(*i, mv);
- delete i;
-
- if ((++transferred % txnSize) == 0) {
- nextTransaction(true);
- }
- if ((transferred % reportEvery) == 0) {
- cout << "." << flush;
- }
- }
-
- size_t getTransferred() {
- return transferred;
- }
-
- bool enterTransaction() {
- if (!inTxn) {
- inTxn = dest->begin();
- }
- return inTxn;
- }
-
- bool nextTransaction(bool next) {
- if (inTxn) {
- while (!dest->commit()) {
- cout << "Failed to commit a transaction. Sleep a while." << endl;
- sleep(1);
- }
- inTxn = false;
- }
- return next ? enterTransaction() : true;
- }
-
-private:
-
- void adjust(Item **i) {
- Item *input(*i);
- if (killCrlf) {
- const char *data = input->getData();
- assert(data[input->getNBytes() - 2] == '\r');
- assert(data[input->getNBytes() - 1] == '\n');
- *i = new Item(input->getKey(), input->getFlags(),
- input->getExptime(),
- data, input->getNBytes() - 2,
- 0, -1, input->getVBucketId());
- delete input;
- } else {
- input->setId(-1);
- }
- }
-
- MutationVerifier mv;
- KVStore *dest;
- size_t transferred;
- size_t txnSize;
- size_t reportEvery;
- bool killCrlf;
- bool inTxn;
- hrtime_t startTime;
-};
-
-static void usage(const char *cmd) {
- cerr << "Usage: " << cmd << " [args] srcPath destPath" << endl
- << endl
- << "Optional arguments:" << endl
- << " --src-strategy=someStrategy (default=multiDB)" << endl
- << " --src-pattern=shardPattern (default=%d/%b-%i.sqlite)" << endl
- << " --dest-strategy=someStrategy (default=multiMTVBDB)" << endl
- << " --dest-pattern=somePattern (default=%d/%b-%i.mb)" << endl
- << " --remove-crlf" << endl
- << " --txn-size=someNumber (default=10000)" << endl
- << " --report-every=someNumber (default=10000)" << endl
- << " --init-file=filepath (default=NULL)" << endl;
- exit(EX_USAGE);
-}
-
-int main(int argc, char **argv) {
- putenv(strdup("ALLOW_NO_STATS_UPDATE=yeah"));
- const char *cmd(argv[0]);
- const char *srcPath(NULL), *srcStrategy("multiDB");
- const char *destPath(NULL), *destStrategy("multiMTVBDB");
- const char *srcShardPattern("%d/%b-%i.sqlite");
- const char *destShardPattern("%d/%b-%i.mb");
- const char *initFile(NULL);
- size_t txnSize(10000), reportEvery(10000);
- int killCrlf(0);
-
- /* options descriptor */
- static struct option longopts[] = {
- { OPTNAME("src-strategy"), required_argument, NULL, 's' },
- { OPTNAME("src-pattern"), required_argument, NULL, 'p' },
- { OPTNAME("dest-strategy"), required_argument, NULL, 'S' },
- { OPTNAME("dest-pattern"), required_argument, NULL, 'P' },
- { OPTNAME("remove-crlf"), no_argument, &killCrlf, 'x' },
- { OPTNAME("txn-size"), required_argument, NULL, 't' },
- { OPTNAME("report-every"), required_argument, NULL, 'r' },
- { OPTNAME("init-file"), required_argument, NULL, 'i' },
- { NULL, 0, NULL, 0 }
- };
-
- int ch(0);
- while ((ch = getopt_long(argc, argv, "s:S:x", longopts, NULL)) != -1) {
- switch (ch) {
- case 's':
- srcStrategy = optarg;
- break;
- case 'p':
- srcShardPattern = optarg;
- break;
- case 'S':
- destStrategy = optarg;
- break;
- case 'P':
- destShardPattern = optarg;
- break;
- case 'i':
- initFile = optarg;
- break;
- case 't':
- txnSize = static_cast<size_t>(atoi(optarg));
- break;
- case 'r':
- reportEvery = static_cast<size_t>(atoi(optarg));
- break;
- case 0: // Path for automatically handled cases (e.g. remove-crlf)
- break;
- default:
- usage(cmd);
- }
- }
- argc -= optind;
- argv += optind;
-
- if (argc != 2) {
- usage(cmd);
- }
- srcPath = argv[0];
- destPath = argv[1];
-
- cout << "src = " << srcStrategy << "@" << srcPath << endl;
- cout << "dest = " << destStrategy << "@" << destPath << endl;
- EPStats srcStats, destStats;
-
- SqliteStrategy::disableSchemaCheck();
-
- KVStore *src(getStore(srcStats, srcPath,
- srcStrategy, srcShardPattern));
- KVStore *dest(getStore(destStats, destPath,
- destStrategy, destShardPattern, initFile));
-
- shared_ptr<Mover> mover(new Mover(dest, txnSize, static_cast<bool>(killCrlf), reportEvery));
- cout << "Each . represents " << reportEvery << " items moved." << endl;
- src->dump(mover);
- cout << endl << "Moved " << mover->getTransferred() << " items." << endl;
-
- return 0;
-}
View
2,733 src/embedded/sqlite3-shell.c
0 additions, 2,733 deletions not shown because the diff is too large. Please use a local Git client to view these changes.
View
119,636 src/embedded/sqlite3.c
0 additions, 119,636 deletions not shown because the diff is too large. Please use a local Git client to view these changes.
View
5,963 src/embedded/sqlite3.h
0 additions, 5,963 deletions not shown because the diff is too large. Please use a local Git client to view these changes.
View
8 src/ep.cc
@@ -2245,14 +2245,6 @@ void EventuallyPersistentStore::warmupCompleted() {
// "0" sleep_time means that the first snapshot task will be executed right after
// warmup. Subsequent snapshot tasks will be scheduled every 60 sec by default.
dispatcher->schedule(sscb, NULL, Priority::StatSnapPriority, 0);
-
- if (engine.getConfiguration().getBackend().compare("sqlite") == 0 &&
- storageProperties.hasEfficientVBDeletion()) {
- shared_ptr<DispatcherCallback> invalidVBTableRemover(new InvalidVBTableRemover(&engine));
- dispatcher->schedule(invalidVBTableRemover, NULL,
- Priority::VBucketDeletionPriority,
- INVALID_VBTABLE_DEL_FREQ);
- }
}
static void warmupLogCallback(void *arg, uint16_t vb,
View
5 src/kvstore.cc
@@ -8,7 +8,6 @@
#include "ep_engine.h"
#include "stats.hh"
#include "kvstore.hh"
-#include "sqlite-kvstore.hh"
#include "blackhole-kvstore/blackhole.hh"
#include "warmup.hh"
#ifdef HAVE_LIBCOUCHSTORE
@@ -23,9 +22,7 @@ KVStore *KVStoreFactory::create(EventuallyPersistentEngine &theEngine,
KVStore *ret = NULL;
std::string backend = c.getBackend();
- if (backend.compare("sqlite") == 0) {
- ret = SqliteKVStoreFactory::create(theEngine, read_only);
- } else if (backend.compare("couchdb") == 0) {
+ if (backend.compare("couchdb") == 0) {
ret = new CouchKVStore(theEngine, read_only);
} else if (backend.compare("blackhole") == 0) {
ret = new BlackholeKVStore(read_only);
View
5 src/sizes.cc
@@ -24,7 +24,6 @@
#include "vbucket.hh"
#include "histo.hh"
#include "checkpoint.hh"
-#include "sqlite-kvstore/sqlite-stats.hh"
static void display(const char *name, size_t size) {
std::cout << name << "\t" << size << std::endl;
@@ -96,9 +95,5 @@ int main(int, char **) {
display("Commit Histo", stats.diskCommitHisto);
display("Hash table depth histo", dv.depthHisto);
- SQLiteStats sqstats;
- display("Sqlite Seek Histo", sqstats.writeSeekHisto);
- display("Sqlite Size Histo", sqstats.writeSizeHisto);
-
return 0;
}
View
121 src/sqlite-kvstore/factory.cc
@@ -1,121 +0,0 @@
-/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-#include "config.h"
-
-#include <string>
-#include <map>
-
-#include "common.hh"
-#include "ep_engine.h"
-#include "stats.hh"
-#include "kvstore.hh"
-#include "sqlite-kvstore.hh"
-
-static const char *stringToCharPtr(std::string str) {
- if (!str.empty()) {
- return strdup(str.c_str());
- }
- return NULL;
-}
-
-KVStore *SqliteKVStoreFactory::create(EventuallyPersistentEngine &theEngine,
- bool read_only) {
- Configuration &c = theEngine.getConfiguration();
- SqliteStrategy *sqliteInstance = NULL;
- enum db_type type = multi_db;
-
- if (!SqliteKVStoreFactory::stringToType(c.getDbStrategy(), type)) {
- getLogger()->log(EXTENSION_LOG_WARNING, NULL,
- "Unhandled db type: %s", c.getDbStrategy().c_str());
- return NULL;
- }
-
- switch (type) {
- case multi_db:
- sqliteInstance = new MultiDBSingleTableSqliteStrategy(
- stringToCharPtr(c.getDbname()),
- stringToCharPtr(c.getShardpattern()),
- stringToCharPtr(c.getInitfile()),
- stringToCharPtr(c.getPostInitfile()), c.getDbShards());
- break;
- case single_db:
- sqliteInstance = new SingleTableSqliteStrategy(
- stringToCharPtr(c.getDbname()),
- stringToCharPtr(c.getInitfile()),
- stringToCharPtr(c.getPostInitfile()));
- break;
- case single_mt_db:
- sqliteInstance = new MultiTableSqliteStrategy(
- stringToCharPtr(c.getDbname()),
- stringToCharPtr(c.getInitfile()),
- stringToCharPtr(c.getPostInitfile()), c.getMaxVbuckets());
- break;
- case multi_mt_db:
- sqliteInstance = new ShardedMultiTableSqliteStrategy(
- stringToCharPtr(c.getDbname()),
- stringToCharPtr(c.getShardpattern()),
- stringToCharPtr(c.getInitfile()),
- stringToCharPtr(c.getPostInitfile()), c.getMaxVbuckets(),
- c.getDbShards());
- break;
- case multi_mt_vb_db:
- sqliteInstance = new ShardedByVBucketSqliteStrategy(
- stringToCharPtr(c.getDbname()),
- stringToCharPtr(c.getShardpattern()),
- stringToCharPtr(c.getInitfile()),
- stringToCharPtr(c.getPostInitfile()), c.getMaxVbuckets(),
- c.getDbShards());
- break;
- }
-
- KVStore *kvstore = new StrategicSqlite3(theEngine.getEpStats(),
- shared_ptr<SqliteStrategy> (sqliteInstance),
- read_only);
- return kvstore;
-}
-
-static const char* MULTI_DB_NAME("multiDB");
-static const char* SINGLE_DB_NAME("singleDB");
-static const char* SINGLE_MT_DB_NAME("singleMTDB");
-static const char* MULTI_MT_DB_NAME("multiMTDB");
-static const char* MULTI_MT_VB_DB_NAME("multiMTVBDB");
-
-const char* SqliteKVStoreFactory::typeToString(db_type type) {
- char *rv(NULL);
- switch (type) {
- case multi_db:
- return MULTI_DB_NAME;
- break;
- case single_db:
- return SINGLE_DB_NAME;
- break;
- case single_mt_db:
- return SINGLE_MT_DB_NAME;
- break;
- case multi_mt_db:
- return MULTI_MT_DB_NAME;
- break;
- case multi_mt_vb_db:
- return MULTI_MT_VB_DB_NAME;
- break;
- }
- assert(rv);
- return rv;
-}
-
-bool SqliteKVStoreFactory::stringToType(std::string name, enum db_type &typeOut) {
- bool rv(true);
- if (name.compare(MULTI_DB_NAME) == 0) {
- typeOut = multi_db;
- } else if (name.compare(SINGLE_DB_NAME) == 0) {
- typeOut = single_db;
- } else if (name.compare(SINGLE_MT_DB_NAME) == 0) {
- typeOut = single_mt_db;
- } else if (name.compare(MULTI_MT_DB_NAME) == 0) {
- typeOut = multi_mt_db;
- } else if (name.compare(MULTI_MT_VB_DB_NAME) == 0) {
- typeOut = multi_mt_vb_db;
- } else {
- rv = false;
- }
- return rv;
-}
View
69 src/sqlite-kvstore/pathexpand.cc
@@ -1,69 +0,0 @@
-/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-
-#include "config.h"
-
-#include <sstream>
-
-#include "pathexpand.hh"
-
-#ifdef WIN32
-const char* path_separator("\\/");
-#else
-const char* path_separator("/");
-#endif
-
-extern const char* path_separator;
-
-static std::string pe_basename(const char *b) {
- assert(b);
- std::string s(b);
- size_t lastthing(s.find_last_of(path_separator));
- if (lastthing == s.npos) {
- return s;
- }
-
- return s.substr(lastthing + 1);
-}
-
-static std::string pe_dirname(const char *b) {
- assert(b);
- std::string s(b);
- size_t lastthing(s.find_last_of(path_separator));
- if (lastthing == s.npos) {
- return std::string(".");
- }
-
- return s.substr(0, lastthing);
-}
-
-PathExpander::PathExpander(const char *p) : dir(pe_dirname(p)),
- base(pe_basename(p)) {
-}
-
-std::string PathExpander::expand(const char *pattern, int shardId) {
- std::stringstream ss;
-
- while (*pattern) {
- if (*pattern == '%') {
- ++pattern;
- switch (*pattern) {
- case 'd':
- ss << dir;
- break;
- case 'b':
- ss << base;
- break;
- case 'i':
- ss << shardId;
- break;
- default:
- ss << '%' << *pattern;
- }
- } else {
- ss << *pattern;
- }
- ++pattern;
- }
-
- return ss.str();
-}
View
36 src/sqlite-kvstore/pathexpand.hh
@@ -1,36 +0,0 @@
-/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-#ifndef PATHEXPAND_HH
-#define PATHEXPAND_HH 1
-
-#include "config.h"
-
-#include <string>
-#include <cassert>
-
-#include <libgen.h>
-
-/**
- * Expand paths for DB shards.
- *
- * Available variables:
- *
- * %d - the directory containing the base shard
- * %b - the basename of the main db
- * %i - the shard ID
- */
-class PathExpander {
-public:
-
- //! Create a path expander with the given main db path.
- PathExpander(const char *p);
-
- //! Expand the path to the given shard ID.
- std::string expand(const char *pattern, int shardId);
-
-private:
-
- std::string dir;
- std::string base;
-};
-
-#endif // PATHEXPAND_HH
View
78 src/sqlite-kvstore/sqlite-eval.cc
@@ -1,78 +0,0 @@
-/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-/*
- * Copyright 2010 NorthScale, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "config.h"
-#include <iostream>
-#include <fstream>
-#include <stdexcept>
-
-#include "sqlite-eval.hh"
-#include "sqlite-pst.hh"
-#include "ep.hh"
-
-void SqliteEvaluator::eval(const std::string &filename) {
- std::ifstream in(filename.c_str(), std::ios_base::in);
- std::string query;
-
- if (!in.good()) {
- getLogger()->log(EXTENSION_LOG_INFO, NULL,
- "Failed to open eval file\n");
- }
-
- while (in.good() && !in.eof()) {
- char c;
- in.get(c);
- query.append(1, c);
- switch (sqlite3_complete(query.c_str())) {
- case 1:
- // READY!
- execute(query);
- query.clear();
- break;
- case 0:
- // Not ready
- break;
- default:
- // Something's very wrong.
- abort();
- }
- }
-
- execute(query);
-}
-
-void SqliteEvaluator::execute(std::string &query) {
- trim(query);
- if (!query.empty()) {
- getLogger()->log(EXTENSION_LOG_DEBUG, NULL,
- "Executing query: ``%s''\n", query.c_str());
- PreparedStatement st(db, query.c_str());
- st.execute();
- }
-}
-
-void SqliteEvaluator::trim(std::string &str) {
- static const std::string whitespaces(" \r\n\t");
-
- while (!str.empty() && whitespaces.find(str[0]) != std::string::npos) {
- str = str.substr(1);
- }
-
- while (!str.empty() && whitespaces.find(str[str.size()-1]) != std::string::npos) {
- str = str.substr(0, str.size()-1);
- }
-}
View
33 src/sqlite-kvstore/sqlite-eval.hh
@@ -1,33 +0,0 @@
-/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-#ifndef SQLITE_EVAL_HH
-#define SQLITE_EVAL_HH 1
-
-#include <cassert>
-
-#ifdef USE_SYSTEM_LIBSQLITE3
-#include <sqlite3.h>
-#else
-#include "embedded/sqlite3.h"
-#endif
-
-/**
- * Evaluates SQLite files within a given DB.
- */
-class SqliteEvaluator {
-public:
-
- SqliteEvaluator(sqlite3 *d) : db(d) {
- assert(db);
- }
-
- void eval(const std::string &filename);
-
-private:
-
- void execute(std::string &query);
- void trim(std::string &str);
-
- sqlite3 *db;
-};
-
-#endif /* SQLITE_EVAL_HH */
View
548 src/sqlite-kvstore/sqlite-kvstore.cc
@@ -1,548 +0,0 @@
-/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-#include "config.h"
-#include <string.h>
-#include <cstdlib>
-#include <cctype>
-#include <algorithm>
-
-#include "sqlite-kvstore.hh"
-#include "sqlite-pst.hh"
-#include "vbucket.hh"
-#include "ep_engine.h"
-#include "warmup.hh"
-
-#define STATWRITER_NAMESPACE sqlite_engine
-#include "statwriter.hh"
-#undef STATWRITER_NAMESPACE
-
-StrategicSqlite3::StrategicSqlite3(EPStats &st, shared_ptr<SqliteStrategy> s,
- bool read_only) :
- KVStore(read_only), stats(st), strategy(s), intransaction(false) {
- open();
-}
-
-StrategicSqlite3::StrategicSqlite3(const StrategicSqlite3 &from) :
- KVStore(from), stats(from.stats),
- strategy(from.strategy), intransaction(false) {
- open();
-}
-
-int64_t StrategicSqlite3::lastRowId() {
- assert(db);
- return static_cast<int64_t>(sqlite3_last_insert_rowid(db));
-}
-
-void StrategicSqlite3::insert(const Item &itm,
- Callback<mutation_result> &cb) {
- assert(itm.getId() <= 0);
-
- PreparedStatement *ins_stmt = strategy->getStatements(itm.getVBucketId(),
- itm.getKey())->ins();
- ins_stmt->bind(1, itm.getKey());
- ins_stmt->bind(2, const_cast<Item&>(itm).getData(), itm.getNBytes());
- ins_stmt->bind(3, itm.getFlags());
- ins_stmt->bind(4, itm.getExptime());
- ins_stmt->bind64(5, itm.getCas());
- ins_stmt->bind(6, itm.getVBucketId());
-
- ++stats.io_num_write;
- stats.io_write_bytes += itm.getKey().length() + itm.getNBytes();
-
- int rv = ins_stmt->execute();
- if (rv < 0) {
- getLogger()->log(EXTENSION_LOG_WARNING, NULL,
- "Fatal sqlite error in inserting key '%s' !!! "
- "Reopen the database...\n",
- itm.getKey().c_str());
- reopen();
- }
-
- int64_t newId = lastRowId();
-
- std::pair<int, int64_t> p(rv, newId);
- cb.callback(p);
- ins_stmt->reset();
-}
-
-void StrategicSqlite3::update(const Item &itm,
- Callback<mutation_result> &cb) {
- assert(itm.getId() > 0);
-
- PreparedStatement *upd_stmt = strategy->getStatements(itm.getVBucketId(),
- itm.getKey())->upd();
-
- upd_stmt->bind(1, itm.getKey());
- upd_stmt->bind(2, const_cast<Item&>(itm).getData(), itm.getNBytes());
- upd_stmt->bind(3, itm.getFlags());
- upd_stmt->bind(4, itm.getExptime());
- upd_stmt->bind64(5, itm.getCas());
- upd_stmt->bind64(6, itm.getId());
-
- int rv = upd_stmt->execute();
- if (rv < 0) {
- getLogger()->log(EXTENSION_LOG_WARNING, NULL,
- "Fatal sqlite error in updating key '%s' !!! "
- "Reopen the database...\n",
- itm.getKey().c_str());
- reopen();
- }
- ++stats.io_num_write;
- stats.io_write_bytes += itm.getKey().length() + itm.getNBytes();
-
- std::pair<int, int64_t> p(rv, 0);
- cb.callback(p);
- upd_stmt->reset();
-}
-
-vbucket_map_t StrategicSqlite3::listPersistedVbuckets() {
- std::map<uint16_t, vbucket_state> rv;
-
- PreparedStatement *st = strategy->getGetVBucketStateST();
-
- while (st->fetch()) {
- ++stats.io_num_read;
- uint16_t vbid = st->column_int(0);
- vbucket_state vb_state;
- vb_state.state = (vbucket_state_t)st->column_int(1);
- vb_state.checkpointId = st->column_int64(2);
- vb_state.maxDeletedSeqno = 0;
- rv[vbid] = vb_state;
- }
-
- st->reset();
-
- return rv;
-}
-
-void StrategicSqlite3::set(const Item &itm,
- Callback<mutation_result> &cb) {
- assert(!isReadOnly());
- if (itm.getId() <= 0) {
- insert(itm, cb);
- } else {
- update(itm, cb);
- }
-}
-
-void StrategicSqlite3::get(const std::string &key, uint64_t rowid,
- uint16_t vb, Callback<GetValue> &cb) {
- PreparedStatement *sel_stmt = strategy->getStatements(vb, key)->sel();
- sel_stmt->bind64(1, rowid);
-
- ++stats.io_num_read;
-
- if(sel_stmt->fetch()) {
- GetValue rv(new Item(key.data(),
- static_cast<uint16_t>(key.length()),
- sel_stmt->column_int(1),
- sel_stmt->column_int(2),
- sel_stmt->column_blob(0),
- sel_stmt->column_bytes(0),
- sel_stmt->column_int64(3),
- sel_stmt->column_int64(4),
- static_cast<uint16_t>(sel_stmt->column_int(5))));
- stats.io_read_bytes += key.length() + rv.getValue()->getNBytes();
- cb.callback(rv);
- } else {
- GetValue rv;
- cb.callback(rv);
- }
- sel_stmt->reset();
-}
-
-void StrategicSqlite3::reset() {
- assert(!isReadOnly());
- if (db) {
- rollback();
- close();
- open();
- strategy->destroyTables();
- close();
- open();
- execute("vacuum");
- }
-}
-
-void StrategicSqlite3::del(const Item &itm, uint64_t rowid,
- Callback<int> &cb) {
- assert(!isReadOnly());
- int rv = 0;
- if (rowid <= 0) {
- cb.callback(rv);
- return;
- }
-
- std::string key = itm.getKey();
- uint16_t vb = itm.getVBucketId();
- PreparedStatement *del_stmt = strategy->getStatements(vb, key)->del();
- del_stmt->bind64(1, rowid);
- rv = del_stmt->execute();
- if (rv < 0) {
- getLogger()->log(EXTENSION_LOG_WARNING, NULL,
- "Fatal sqlite error in deleting key '%s' !!! "
- "Reopen the database...\n",
- key.c_str());
- reopen();
- }
- cb.callback(rv);
- del_stmt->reset();
-}
-
-bool StrategicSqlite3::delVBucket(uint16_t vbucket) {
- assert(!isReadOnly());
- assert(strategy->hasEfficientVBDeletion());
- bool rv = true;
- std::stringstream tmp_table_name;
- tmp_table_name << "invalid_kv_" << vbucket << "_" << gethrtime();
- rv = begin();
- if (rv) {
- strategy->renameVBTable(vbucket, tmp_table_name.str());
- strategy->createVBTable(vbucket);
- rv = commit();
- }
- if (!rv) {
- getLogger()->log(EXTENSION_LOG_WARNING, NULL,
- "Fatal sqlite error in deleting vbucket %d !!! "
- "Reopen the database...\n",
- vbucket);
- reopen();
- }
- return rv;
-}
-
-bool StrategicSqlite3::snapshotVBuckets(const vbucket_map_t &m) {
- assert(!isReadOnly());
- bool rv = storeMap(strategy->getClearVBucketStateST(),
- strategy->getInsVBucketStateST(), m);
- if (!rv) {
- getLogger()->log(EXTENSION_LOG_WARNING, NULL,
- "Fatal sqlite error in snapshot vbucket states!!! "
- "Reopen the database...\n");
- reopen();
- }
- return rv;
-}
-
-bool StrategicSqlite3::snapshotStats(const std::map<std::string, std::string> &m) {
- assert(!isReadOnly());
- bool rv = storeMap(strategy->getClearStatsST(), strategy->getInsStatST(), m);
- if (!rv) {
- getLogger()->log(EXTENSION_LOG_WARNING, NULL,
- "Fatal sqlite error in snapshot engine stats!!! "
- "Reopen the database...\n");
- reopen();
- }
- return rv;
-}
-
-/**
- * Function object to set a series of k,v pairs into a PreparedStatement.
- */
-template <typename T1, typename T2>
-struct map_setter {
-
- /**
- * Constructor.
- *
- * @param i the prepared statement to operate on
- * @param o the location of the return value - will set to false upon failure
- */
- map_setter(PreparedStatement *i, bool &o) : insSt(i), output(o) {}
-
- PreparedStatement *insSt;
- bool &output;
-
- void operator() (const std::pair<T1, T2> &p) {
- int pos = 1;
- pos += insSt->bind(pos, p.first);
- insSt->bind(pos, p.second);
-
- bool inserted = insSt->execute() == 1;
- insSt->reset();
- output &= inserted;
- }
-};
-
-template <typename T1, typename T2>
-bool StrategicSqlite3::storeMap(PreparedStatement *clearSt,
- PreparedStatement *insSt,
- const std::map<T1, T2> &m) {
- bool rv(false);
- if (!begin()) {
- return false;
- }
- try {
- bool deleted = clearSt->execute() >= 0;
- rv &= deleted;
- clearSt->reset();
-
- map_setter<T1, T2> ms(insSt, rv);
- std::for_each(m.begin(), m.end(), ms);
-
- rv = commit();
- } catch(...) {
- rollback();
- }
- return rv;
-}
-
-static void processDumpRow(EPStats &stats,
- PreparedStatement *st, shared_ptr<Callback<GetValue> > &cb) {
- ++stats.io_num_read;
- GetValue rv(new Item(st->column_blob(0),
- static_cast<uint16_t>(st->column_bytes(0)),
- st->column_int(2),
- st->column_int(3),
- st->column_blob(1),
- st->column_bytes(1),
- st->column_int64(4),
- st->column_int64(6),
- static_cast<uint16_t>(st->column_int(5))),
- ENGINE_SUCCESS,
- -1);
- stats.io_read_bytes += rv.getValue()->getKey().length() + rv.getValue()->getNBytes();
- cb->callback(rv);
-}
-
-void StrategicSqlite3::dump(shared_ptr<Callback<GetValue> > cb) {
- const std::vector<Statements*> statements = strategy->allStatements();
- std::vector<Statements*>::const_iterator it;
- for (it = statements.begin(); it != statements.end(); ++it) {
- PreparedStatement *st = (*it)->all();
- st->reset();
- while (st->fetch()) {
- processDumpRow(stats, st, cb);
- }
-
- st->reset();
- }
-}
-
-void StrategicSqlite3::dump(uint16_t vb, shared_ptr<Callback<GetValue> > cb) {
- assert(strategy->hasEfficientVBLoad());
- std::vector<PreparedStatement*> loaders(strategy->getVBStatements(vb, select_all));
-
- std::vector<PreparedStatement*>::iterator it;
- for (it = loaders.begin(); it != loaders.end(); ++it) {
- PreparedStatement *st = *it;
- while (st->fetch()) {
- processDumpRow(stats, st, cb);
- }
- }
-
- strategy->closeVBStatements(loaders);
-}
-
-
-static char lc(const char i) {
- return std::tolower(i);
-}
-
-StorageProperties StrategicSqlite3::getStorageProperties() {
- // Verify we at least compiled in mutexes.
- assert(sqlite3_threadsafe());
- bool allows_concurrency(false);
- {
- PreparedStatement st(db, "pragma journal_mode");
- static const std::string wal_str("wal");
- if (st.fetch()) {
- std::string s(st.column(0));
- std::transform(s.begin(), s.end(), s.begin(), lc);
- getLogger()->log(EXTENSION_LOG_INFO, NULL,
- "journal-mode: %s\n", s.c_str());
- allows_concurrency = s == wal_str;
- }
- }
- if (allows_concurrency) {
- PreparedStatement st(db, "pragma read_uncommitted");
- if (st.fetch()) {
- allows_concurrency = st.column_int(0) == 1;
- getLogger()->log(EXTENSION_LOG_INFO, NULL,
- "read_uncommitted: %s\n",
- allows_concurrency ? "yes" : "no");
- }
- }
- size_t concurrency(allows_concurrency ? 10 : 1);
- StorageProperties rv(concurrency, concurrency - 1, 1,
- strategy->hasEfficientVBLoad(),
- strategy->hasEfficientVBDeletion(),
- strategy->hasPersistedDeletions(), false);
- return rv;
-}
-
-void StrategicSqlite3::addStats(const std::string &prefix,
- ADD_STAT add_stat, const void *c) {
- if (prefix != "rw") {
- return;
- }
-
- SQLiteStats &st(strategy->sqliteStats);
- add_casted_stat("sector_size", st.sectorSize, add_stat, c);
- add_casted_stat("open", st.numOpen, add_stat, c);
- add_casted_stat("close", st.numClose, add_stat, c);
- add_casted_stat("lock", st.numLocks, add_stat, c);
- add_casted_stat("truncate", st.numTruncates, add_stat, c);
-}
-
-
-void StrategicSqlite3::addTimingStats(const std::string &prefix,
- ADD_STAT add_stat, const void *c) {
- if (prefix != "rw") {
- return;
- }
-
- SQLiteStats &st(strategy->sqliteStats);
- add_casted_stat("delete", st.deleteHisto, add_stat, c);
- add_casted_stat("sync", st.syncTimeHisto, add_stat, c);
- add_casted_stat("readTime", st.readTimeHisto, add_stat, c);
- add_casted_stat("readSeek", st.readSeekHisto, add_stat, c);
- add_casted_stat("readSize", st.readSizeHisto, add_stat, c);
- add_casted_stat("writeTime", st.writeTimeHisto, add_stat, c);
- add_casted_stat("writeSeek", st.writeSeekHisto, add_stat, c);
- add_casted_stat("writeSize", st.writeSizeHisto, add_stat, c);
-}
-
-typedef std::map<std::string, std::list<uint64_t> > ShardRowidMap;
-
-struct WarmupCookie {
- WarmupCookie(StrategicSqlite3 *s) :
- store(s)
- { /* EMPTY */ }
- StrategicSqlite3 *store;
- ShardRowidMap objmap;
-};
-
-static void warmupCallback(void *arg, uint16_t vb,
- const std::string &key, uint64_t rowid)
-{
- WarmupCookie *cookie = static_cast<WarmupCookie*>(arg);
- cookie->objmap[cookie->store->getKvTableName(key, vb)].push_back(rowid);
-}
-
-size_t StrategicSqlite3::warmup(MutationLog &lf,
- const std::map<uint16_t, vbucket_state> &vbmap,
- Callback<GetValue> &cb,
- Callback<size_t> &estimate)
-{
- // First build up the various maps...
-
- MutationLogHarvester harvester(lf);
- std::map<uint16_t, vbucket_state>::const_iterator it;
- for (it = vbmap.begin(); it != vbmap.end(); ++it) {
- harvester.setVBucket(it->first);
- }
-
- hrtime_t start = gethrtime();
- if (!harvester.load()) {
- return -1;
- }
- hrtime_t end = gethrtime();
-
- size_t total = harvester.total();
- estimate.callback(total);
-
- getLogger()->log(EXTENSION_LOG_DEBUG, NULL,
- "Completed log read in %s with %ld entries\n",
- hrtime2text(end - start).c_str(), total);
-
- start = gethrtime();
- WarmupCookie cookie(this);
- harvester.apply(&cookie, &warmupCallback);
-
- // Ok, run through all of the lists and apply each one of them..
- total = 0;
- for (ShardRowidMap::iterator iter = cookie.objmap.begin();
- iter != cookie.objmap.end();
- ++iter) {
-
- total += warmupSingleShard(iter->first, iter->second, cb);
- }
- end = gethrtime();
-
- getLogger()->log(EXTENSION_LOG_WARNING, NULL,
- "Warmed up %ld items in %s", (long)total,
- hrtime2text(end - start).c_str());
-
- return total;
-}
-
-static bool list_compare(uint64_t first, uint64_t second) {
- return first < second;
-}
-
-size_t StrategicSqlite3::warmupSingleShard(const std::string &table,
- std::list<uint64_t> &ids,
- Callback<GetValue> &cb)
-{
- using namespace std;
-
- string prefix("select k, v, flags, exptime, cas, vbucket, rowid from ");
- prefix.append(table);
- prefix.append(" where rowid in (");
-
- ids.sort(list_compare);
-
- list<uint64_t>::iterator iter = ids.begin();
-
- size_t batchSize;
- if (engine) {
- batchSize = engine->getConfiguration().getWarmupBatchSize();
- } else {
- batchSize = 1000;
- }
- size_t ret = 0;
- do {
- stringstream ss;
- size_t num = 0;
-
- // @todo make this configurable
- while (iter != ids.end() && num < batchSize) {
- ss << *iter << ",";
- ++num; ++iter;
- }
-
- if (num != 0) {
- // we need to execute the query
- string query = prefix + ss.str();
- query.resize(query.length() - 1);
- query.append(")");
-
- PreparedStatement st(db, query.c_str());
- while (st.fetch()) {
- ++ret;
- Item *it = new Item(st.column_blob(0),
- static_cast<uint16_t>(st.column_bytes(0)),
- st.column_int(2), st.column_int(3),
- st.column_blob(1), st.column_bytes(1),
- st.column_int64(4), st.column_int64(6),
- static_cast<uint16_t>(st.column_int(5)));
- GetValue rv(it, ENGINE_SUCCESS, -1);
- cb.callback(rv);
- }
- }
- } while (iter != ids.end());
-
- return ret;
-}
-
-bool StrategicSqlite3::getEstimatedItemCount(size_t &nItems) {
- if (getenv("COUCHBASE_FORCE_SQLITE_ESTIMATE_COUNT") != NULL) {
- size_t num = 0;
-
- const std::vector<Statements*> statements = strategy->allStatements();
- std::vector<Statements*>::const_iterator it;
- for (it = statements.begin(); it != statements.end(); ++it) {
- PreparedStatement *st = (*it)->count_all();
- st->reset();
- if (st->fetch()) {
- num += static_cast<size_t>(st->column_int(0));
- }
- st->reset();
- }
-
- nItems = num;
- return true;
- } else {
- return false;
- }
-}
View
260 src/sqlite-kvstore/sqlite-kvstore.hh
@@ -1,260 +0,0 @@
-/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-#ifndef SQLITE_BASE_H
-#define SQLITE_BASE_H 1
-
-#include <map>
-#include <vector>
-
-#ifdef USE_SYSTEM_LIBSQLITE3
-#include <sqlite3.h>
-#else
-#include "embedded/sqlite3.h"
-#endif
-
-#include "kvstore.hh"
-#include "sqlite-pst.hh"
-#include "sqlite-strategies.hh"
-#include "item.hh"
-#include "stats.hh"
-
-#include "sqlite-stats.hh"
-
-class EventuallyPersistentEngine;
-class EPStats;
-
-
-class SqliteKVStoreFactory {
-public:
-
- /**
- * Create a KVStore with the given type.
- *
- * @param theEngine the engine instance
- * @param read_only true if the kvstore instance is for read operations only
- */
- static KVStore *create(EventuallyPersistentEngine &theEngine,
- bool read_only = false);
-
- /**
- * Get the name of a db type.
- */
- static const char* typeToString(enum db_type type);
-
- /**
- * Get the type for a given name.
- *
- * @param name the name to parse
- * @param typeOut a reference to a type to fill
- *
- * @return true if we were able to parse the type
- */
- static bool stringToType(std::string name,
- enum db_type &typeOut);
-
-};
-
-
-/**
- * A persistence store based on sqlite that uses a SqliteStrategy to
- * configure itself.
- */
-class StrategicSqlite3 : public KVStore {
-public:
-
- /**
- * Construct an instance of sqlite with the given database name.
- */
- StrategicSqlite3(EPStats &st, shared_ptr<SqliteStrategy> s,
- bool read_only = false);
-
- /**
- * Copying opens a new underlying DB.
- */
- StrategicSqlite3(const StrategicSqlite3 &from);
-
- /**
- * Cleanup.
- */
- ~StrategicSqlite3() {
- close();
- }
-
- bool getEstimatedItemCount(size_t &nItem);
-
- /**
- * Reset database to a clean state.
- */
- void reset();
-
- /**
- * Begin a transaction (if not already in one).
- */
- bool begin() {
- assert(!isReadOnly());
- if(!intransaction) {
- if (execute("begin immediate") != -1) {
- intransaction = true;
- }
- }
- return intransaction;
- }
-
- /**
- * Commit a transaction (unless not currently in one).
- *
- * Returns false if the commit fails.
- */
- bool commit() {
- assert(!isReadOnly());
- if(intransaction) {
- // If commit returns -1, we're still in a transaction.
- intransaction = (execute("commit") == -1);
- }
- // !intransaction == not in a transaction == committed
- return !intransaction;
- }
-
- /**
- * Rollback a transaction (unless not currently in one).
- */
- void rollback() {
- assert(!isReadOnly());
- if(intransaction) {
- intransaction = false;
- execute("rollback");
- }
- }
-
- /**
- * Query the properties of the underlying storage.
- */
- StorageProperties getStorageProperties();
-
- /**
- * Overrides set().
- */
- void set(const Item &item, Callback<mutation_result> &cb);
-
- /**
- * Overrides get().
- */
- void get(const std::string &key, uint64_t rowid,
- uint16_t vb, Callback<GetValue> &cb);
-
- /**
- * Overrides del().
- */
- void del(const Item &itm, uint64_t rowid,
- Callback<int> &cb);
-
- bool delVBucket(uint16_t vbucket);
-
- vbucket_map_t listPersistedVbuckets(void);
-
- /**
- * Take a snapshot of the stats in the main DB.
- */
- bool snapshotStats(const std::map<std::string, std::string> &m);
- /**
- * Take a snapshot of the vbucket states in the main DB.
- */
- bool snapshotVBuckets(const vbucket_map_t &m);
-
- /**
- * Overrides dump
- */
- void dump(shared_ptr<Callback<GetValue> > cb);
-
- void dump(uint16_t vb, shared_ptr<Callback<GetValue> > cb);
-
- size_t getNumShards() {
- return strategy->getNumOfDbShards();
- }
-
- size_t getShardId(const QueuedItem &i) {
- return strategy->getDbShardId(i);
- }
-
- std::string getKvTableName(const std::string &key, uint16_t vb) {
- return strategy->getKvTableName(key, vb);
- }
-
- void optimizeWrites(std::vector<queued_item> &items) {
- strategy->optimizeWrites(items);
- }
-
- void destroyInvalidVBuckets(bool destroyOnlyOne = false) {
- strategy->destroyInvalidTables(destroyOnlyOne);
- }
-
- void addStats(const std::string &prefix, ADD_STAT add_stat, const void *c);
- void addTimingStats(const std::string &prefix, ADD_STAT add_stat, const void *c);
-
- /**
- * Override the warmup method to bulk load the items
- */
- virtual size_t warmup(MutationLog &lf,
- const std::map<uint16_t, vbucket_state> &vbmap,
- Callback<GetValue> &cb,
- Callback<size_t> &estimate);
-
-private:
- size_t warmupSingleShard(const std::string &table,
- std::list<uint64_t> &ids,
- Callback<GetValue> &cb);
-
- /**
- * Shortcut to execute a simple query.
- *
- * @param query a simple query with no bindings to execute directly
- */
- int execute(const char *query) {
- PreparedStatement st(db, query);
- return st.execute();
- }
-
- template <typename T1, typename T2>
- bool storeMap(PreparedStatement *clearSt,
- PreparedStatement *insSt,
- const std::map<T1, T2> &m);
-
- void insert(const Item &itm, Callback<mutation_result> &cb);
- void update(const Item &itm, Callback<mutation_result> &cb);
- int64_t lastRowId();
-
- EPStats &stats;
-
- /**
- * Direct access to the DB.
- */
- sqlite3 *db;
-
- void open() {
- assert(strategy);
- db = strategy->open();