Permalink
Browse files

Update to leveldb 1.6

Highlights
----------
Mmap at most 1000 files on Posix to improve performance for large databases.
Support for more architectures (thanks to Alexander K.)

Building and porting
--------------------
HP/UX support (issue 126)
AtomicPointer for ia64 (issue 123)
Sparc v9 support (issue 124)
Atomic ops for powerpc
Use -fno-builtin-memcmp only when using g++
Simplify IOS build rules (issue 114)
Use CXXFLAGS instead of CFLAGS when invoking C++ compiler (issue 118)
Fix snappy shared library problem (issue 94)
Fix shared library installation path regression
Endian-ness detection tweak for FreeBSD

Bug fixes
---------
Stop ignoring FLAGS_open_files in db_bench
Make bloom test behavior agnostic to endian-ness

Performance
-----------
Limit number of mmapped files to 1000 to improve perf for large dbs
Do not delay for 1 second on shutdown path (issue 125)

Misc
----
Make InMemoryEnv return a no-op logger
C binding now has a wrapper for free (issue 117)
Add thread-safety annotations
Added an in-process lock table (issue 120)
Make RandomAccessFile and SequentialFile non-copyable
  • Loading branch information...
1 parent dd0d562 commit 946e5b5a4ce7980917b22a408f090a4e86c3fa44 @davidsgrogan davidsgrogan committed Oct 12, 2012
View
@@ -15,7 +15,8 @@ OPT ?= -O2 -DNDEBUG # (A) Production use (optimized mode)
#-----------------------------------------------
# detect what platform we're building on
-$(shell ./build_detect_platform build_config.mk)
+$(shell CC=$(CC) CXX=$(CXX) TARGET_OS=$(TARGET_OS) \
+ ./build_detect_platform build_config.mk ./)
# this file is generated by the previous line to set build flags and sources
include build_config.mk
@@ -70,7 +71,7 @@ SHARED = $(SHARED1)
else
# Update db.h if you change these.
SHARED_MAJOR = 1
-SHARED_MINOR = 5
+SHARED_MINOR = 6
SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
SHARED2 = $(SHARED1).$(SHARED_MAJOR)
SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR)
@@ -82,7 +83,7 @@ $(SHARED2): $(SHARED3)
endif
$(SHARED3):
- $(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) -o $(SHARED3)
+ $(CXX) $(SOURCES) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(INSTALL_PATH)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -o $(SHARED3)
endif # PLATFORM_SHARED_EXT
@@ -179,14 +180,14 @@ IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/version CFBu
.cc.o:
mkdir -p ios-x86/$(dir $@)
- $(SIMULATORROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@
+ $(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@
mkdir -p ios-arm/$(dir $@)
$(DEVICEROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@
lipo ios-x86/$@ ios-arm/$@ -create -output $@
.c.o:
mkdir -p ios-x86/$(dir $@)
- $(SIMULATORROOT)/usr/bin/$(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@
+ $(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@
mkdir -p ios-arm/$(dir $@)
$(DEVICEROOT)/usr/bin/$(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@
lipo ios-x86/$@ ios-arm/$@ -create -output $@
View
1 TODO
@@ -7,6 +7,7 @@ db
within [start_key..end_key]? For Chrome, deletion of obsolete
object stores, etc. can be done in the background anyway, so
probably not that important.
+- There have been requests for MultiGet.
After a range is completely deleted, what gets rid of the
corresponding files if we do no future changes to that range. Make
View
@@ -23,8 +23,9 @@
#
OUTPUT=$1
-if test -z "$OUTPUT"; then
- echo "usage: $0 <output-filename>" >&2
+PREFIX=$2
+if test -z "$OUTPUT" || test -z "$PREFIX"; then
+ echo "usage: $0 <output-filename> <directory_prefix>" >&2
exit 1
fi
@@ -55,58 +56,72 @@ PLATFORM_SHARED_LDFLAGS="-shared -Wl,-soname -Wl,"
PLATFORM_SHARED_CFLAGS="-fPIC"
PLATFORM_SHARED_VERSIONED=true
-# On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp
+MEMCMP_FLAG=
+if [ "$CXX" = "g++" ]; then
+ # Use libc's memcmp instead of GCC's memcmp. This results in ~40%
+ # performance improvement on readrandom under gcc 4.4.3 on Linux/x86.
+ MEMCMP_FLAG="-fno-builtin-memcmp"
+fi
+
case "$TARGET_OS" in
Darwin)
PLATFORM=OS_MACOSX
- COMMON_FLAGS="-fno-builtin-memcmp -DOS_MACOSX"
+ COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX"
PLATFORM_SHARED_EXT=dylib
PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name "
PORT_FILE=port/port_posix.cc
;;
Linux)
PLATFORM=OS_LINUX
- COMMON_FLAGS="-fno-builtin-memcmp -pthread -DOS_LINUX"
+ COMMON_FLAGS="$MEMCMP_FLAG -pthread -DOS_LINUX"
PLATFORM_LDFLAGS="-pthread"
PORT_FILE=port/port_posix.cc
;;
SunOS)
PLATFORM=OS_SOLARIS
- COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS"
+ COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_SOLARIS"
PLATFORM_LDFLAGS="-lpthread -lrt"
PORT_FILE=port/port_posix.cc
;;
FreeBSD)
PLATFORM=OS_FREEBSD
- COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
+ COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_FREEBSD"
PLATFORM_LDFLAGS="-lpthread"
PORT_FILE=port/port_posix.cc
;;
NetBSD)
PLATFORM=OS_NETBSD
- COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD"
+ COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_NETBSD"
PLATFORM_LDFLAGS="-lpthread -lgcc_s"
PORT_FILE=port/port_posix.cc
;;
OpenBSD)
PLATFORM=OS_OPENBSD
- COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_OPENBSD"
+ COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_OPENBSD"
PLATFORM_LDFLAGS="-pthread"
PORT_FILE=port/port_posix.cc
;;
DragonFly)
PLATFORM=OS_DRAGONFLYBSD
- COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_DRAGONFLYBSD"
+ COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_DRAGONFLYBSD"
PLATFORM_LDFLAGS="-lpthread"
PORT_FILE=port/port_posix.cc
;;
OS_ANDROID_CROSSCOMPILE)
PLATFORM=OS_ANDROID
- COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX"
+ COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX"
PLATFORM_LDFLAGS="" # All pthread features are in the Android C library
PORT_FILE=port/port_posix.cc
CROSS_COMPILE=true
;;
+ HP-UX)
+ PLATFORM=OS_HPUX
+ COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_HPUX"
+ PLATFORM_LDFLAGS="-pthread"
+ PORT_FILE=port/port_posix.cc
+ # man ld: +h internal_name
+ PLATFORM_SHARED_LDFLAGS="-shared -Wl,+h -Wl,"
+ ;;
*)
echo "Unknown platform!" >&2
exit 1
@@ -116,11 +131,13 @@ esac
# except for the test and benchmark files. By default, find will output a list
# of all files matching either rule, so we need to append -print to make the
# prune take effect.
-DIRS="util db table"
+DIRS="$PREFIX/db $PREFIX/util $PREFIX/table"
+
set -f # temporarily disable globbing so that our patterns aren't expanded
PRUNE_TEST="-name *test*.cc -prune"
PRUNE_BENCH="-name *_bench.cc -prune"
-PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | tr "\n" " "`
+PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "`
+
set +f # re-enable globbing
# The sources consist of the portable files, plus the platform-specific port
@@ -133,7 +150,7 @@ if [ "$CROSS_COMPILE" = "true" ]; then
true
else
# If -std=c++0x works, use <cstdatomic>. Otherwise use port_posix.h.
- $CXX $CFLAGS -std=c++0x -x c++ - -o /dev/null 2>/dev/null <<EOF
+ $CXX $CXXFLAGS -std=c++0x -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <cstdatomic>
int main() {}
EOF
@@ -146,7 +163,7 @@ EOF
# Test whether Snappy library is installed
# http://code.google.com/p/snappy/
- $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+ $CXX $CXXFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <snappy.h>
int main() {}
EOF
@@ -156,7 +173,7 @@ EOF
fi
# Test whether tcmalloc is available
- $CXX $CFLAGS -x c++ - -o /dev/null -ltcmalloc 2>/dev/null <<EOF
+ $CXX $CXXFLAGS -x c++ - -o /dev/null -ltcmalloc 2>/dev/null <<EOF
int main() {}
EOF
if [ "$?" = 0 ]; then
View
@@ -578,4 +578,8 @@ void leveldb_env_destroy(leveldb_env_t* env) {
delete env;
}
+void leveldb_free(void* ptr) {
+ free(ptr);
+}
+
} // end extern "C"
View
@@ -204,6 +204,12 @@ int main(int argc, char** argv) {
CheckCondition(err != NULL);
Free(&err);
+ StartPhase("leveldb_free");
+ db = leveldb_open(options, dbname, &err);
+ CheckCondition(err != NULL);
+ leveldb_free(err);
+ err = NULL;
+
StartPhase("open");
leveldb_options_set_create_if_missing(options, 1);
db = leveldb_open(options, dbname, &err);
View
@@ -693,6 +693,7 @@ class Benchmark {
options.create_if_missing = !FLAGS_use_existing_db;
options.block_cache = cache_;
options.write_buffer_size = FLAGS_write_buffer_size;
+ options.max_open_files = FLAGS_open_files;
options.filter_policy = filter_policy_;
Status s = DB::Open(options, FLAGS_db, &db_);
if (!s.ok()) {
View
@@ -609,7 +609,11 @@ void DBImpl::BackgroundCall() {
assert(bg_compaction_scheduled_);
if (!shutting_down_.Acquire_Load()) {
Status s = BackgroundCompaction();
- if (!s.ok()) {
+ if (s.ok()) {
+ // Success
+ } else if (shutting_down_.Acquire_Load()) {
+ // Error most likely due to shutdown; do not wait
+ } else {
// Wait a little bit before retrying background compaction in
// case this is an environmental problem and we do not want to
// chew up resources for failed compactions for the duration of
View
@@ -13,6 +13,7 @@
#include "leveldb/db.h"
#include "leveldb/env.h"
#include "port/port.h"
+#include "port/thread_annotations.h"
namespace leveldb {
@@ -71,7 +72,7 @@ class DBImpl : public DB {
// Recover the descriptor from persistent storage. May do a significant
// amount of work to recover recently logged updates. Any changes to
// be made to the descriptor are added to *edit.
- Status Recover(VersionEdit* edit);
+ Status Recover(VersionEdit* edit) EXCLUSIVE_LOCKS_REQUIRED(mutex_);
void MaybeIgnoreError(Status* s) const;
@@ -80,27 +81,34 @@ class DBImpl : public DB {
// Compact the in-memory write buffer to disk. Switches to a new
// log-file/memtable and writes a new descriptor iff successful.
- Status CompactMemTable();
+ Status CompactMemTable()
+ EXCLUSIVE_LOCKS_REQUIRED(mutex_);
Status RecoverLogFile(uint64_t log_number,
VersionEdit* edit,
- SequenceNumber* max_sequence);
+ SequenceNumber* max_sequence)
+ EXCLUSIVE_LOCKS_REQUIRED(mutex_);
- Status WriteLevel0Table(MemTable* mem, VersionEdit* edit, Version* base);
+ Status WriteLevel0Table(MemTable* mem, VersionEdit* edit, Version* base)
+ EXCLUSIVE_LOCKS_REQUIRED(mutex_);
- Status MakeRoomForWrite(bool force /* compact even if there is room? */);
+ Status MakeRoomForWrite(bool force /* compact even if there is room? */)
+ EXCLUSIVE_LOCKS_REQUIRED(mutex_);
WriteBatch* BuildBatchGroup(Writer** last_writer);
- void MaybeScheduleCompaction();
+ void MaybeScheduleCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
static void BGWork(void* db);
void BackgroundCall();
- Status BackgroundCompaction();
- void CleanupCompaction(CompactionState* compact);
- Status DoCompactionWork(CompactionState* compact);
+ Status BackgroundCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+ void CleanupCompaction(CompactionState* compact)
+ EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+ Status DoCompactionWork(CompactionState* compact)
+ EXCLUSIVE_LOCKS_REQUIRED(mutex_);
Status OpenCompactionOutputFile(CompactionState* compact);
Status FinishCompactionOutputFile(CompactionState* compact, Iterator* input);
- Status InstallCompactionResults(CompactionState* compact);
+ Status InstallCompactionResults(CompactionState* compact)
+ EXCLUSIVE_LOCKS_REQUIRED(mutex_);
// Constant after construction
Env* const env_;
View
@@ -1442,6 +1442,12 @@ TEST(DBTest, DBOpen_Options) {
db = NULL;
}
+TEST(DBTest, Locking) {
+ DB* db2 = NULL;
+ Status s = DB::Open(CurrentOptions(), dbname_, &db2);
+ ASSERT_TRUE(!s.ok()) << "Locking did not prevent re-opening db";
+}
+
// Check that number of files does not grow when we are out of space
TEST(DBTest, NoSpace) {
Options options = CurrentOptions();
View
@@ -865,7 +865,7 @@ Status VersionSet::Recover() {
if (edit.has_comparator_ &&
edit.comparator_ != icmp_.user_comparator()->Name()) {
s = Status::InvalidArgument(
- edit.comparator_ + "does not match existing comparator ",
+ edit.comparator_ + " does not match existing comparator ",
icmp_.user_comparator()->Name());
}
}
View
@@ -21,6 +21,7 @@
#include "db/dbformat.h"
#include "db/version_edit.h"
#include "port/port.h"
+#include "port/thread_annotations.h"
namespace leveldb {
@@ -159,7 +160,8 @@ class VersionSet {
// current version. Will release *mu while actually writing to the file.
// REQUIRES: *mu is held on entry.
// REQUIRES: no other thread concurrently calls LogAndApply()
- Status LogAndApply(VersionEdit* edit, port::Mutex* mu);
+ Status LogAndApply(VersionEdit* edit, port::Mutex* mu)
+ EXCLUSIVE_LOCKS_REQUIRED(mu);
// Recover the last saved descriptor from persistent storage.
Status Recover();
@@ -618,7 +618,7 @@ class Benchmark {
ErrorCheck(status);
// Execute read statement
- while ((status = sqlite3_step(read_stmt)) == SQLITE_ROW);
+ while ((status = sqlite3_step(read_stmt)) == SQLITE_ROW) {}
StepErrorCheck(status);
// Reset SQLite statement for another use
View
@@ -408,7 +408,7 @@
the number of disk reads substantially.
<pre>
leveldb::Options options;
- options.filter_policy = NewBloomFilter(10);
+ options.filter_policy = NewBloomFilterPolicy(10);
leveldb::DB* db;
leveldb::DB::Open(options, "/tmp/testdb", &amp;db);
... use the database ...
View
@@ -221,6 +221,11 @@ class WritableFileImpl : public WritableFile {
FileState* file_;
};
+class NoOpLogger : public Logger {
+ public:
+ virtual void Logv(const char* format, va_list ap) { }
+};
+
class InMemoryEnv : public EnvWrapper {
public:
explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) { }
@@ -358,6 +363,11 @@ class InMemoryEnv : public EnvWrapper {
return Status::OK();
}
+ virtual Status NewLogger(const std::string& fname, Logger** result) {
+ *result = new NoOpLogger;
+ return Status::OK();
+ }
+
private:
// Map from filenames to FileState objects, representing a simple file system.
typedef std::map<std::string, FileState*> FileSystem;
View
@@ -28,6 +28,7 @@
be true on entry:
*errptr == NULL
*errptr points to a malloc()ed null-terminated error message
+ (On Windows, *errptr must have been malloc()-ed by this library.)
On success, a leveldb routine leaves *errptr unchanged.
On failure, leveldb frees the old value of *errptr and
set *errptr to a malloc()ed error message.
@@ -268,6 +269,15 @@ extern void leveldb_cache_destroy(leveldb_cache_t* cache);
extern leveldb_env_t* leveldb_create_default_env();
extern void leveldb_env_destroy(leveldb_env_t*);
+/* Utility */
+
+/* Calls free(ptr).
+ REQUIRES: ptr was malloc()-ed and returned by one of the routines
+ in this file. Note that in certain cases (typically on Windows), you
+ may need to call this routine instead of free(ptr) to dispose of
+ malloc()-ed memory returned by this library. */
+extern void leveldb_free(void* ptr);
+
#ifdef __cplusplus
} /* end extern "C" */
#endif
Oops, something went wrong.

0 comments on commit 946e5b5

Please sign in to comment.