Permalink
Browse files

reverting disastrous MOE commit, returning to r21

git-svn-id: http://leveldb.googlecode.com/svn/trunk@23 62dab493-f737-651d-591e-8d6aee1b9529
  • Loading branch information...
1 parent 8540066 commit 80d4a4601a20e75a6c4245244e3cac2cf444e012 dgrogan@chromium.org committed Apr 19, 2011
Showing with 1,628 additions and 253 deletions.
  1. 0 {leveldb → }/AUTHORS
  2. 0 {leveldb → }/LICENSE
  3. +5 −0 {leveldb → }/Makefile
  4. +2 −2 {leveldb → }/README
  5. +2 −2 {leveldb → }/TODO
  6. +9 −0 {leveldb → }/db/builder.cc
  7. +3 −3 {leveldb → }/db/builder.h
  8. +25 −1 {leveldb → }/db/corruption_test.cc
  9. +22 −0 {leveldb → }/db/db_bench.cc
  10. +187 −30 {leveldb → }/db/db_impl.cc
  11. +23 −0 {leveldb → }/db/db_impl.h
  12. +100 −1 {leveldb → }/db/db_iter.cc
  13. 0 {leveldb → }/db/db_iter.h
  14. +216 −35 {leveldb → }/db/db_test.cc
  15. +65 −0 {leveldb → }/db/dbformat.cc
  16. +51 −2 {leveldb → }/db/dbformat.h
  17. +15 −0 {leveldb → }/db/dbformat_test.cc
  18. +19 −0 {leveldb → }/db/filename.cc
  19. +14 −2 {leveldb → }/db/filename.h
  20. +156 −0 db/filename_test.cc
  21. 0 {leveldb → }/db/log_format.h
  22. 0 {leveldb → }/db/log_reader.cc
  23. 0 {leveldb → }/db/log_reader.h
  24. 0 {leveldb → }/db/log_test.cc
  25. +2 −2 {leveldb → }/db/log_writer.cc
  26. 0 {leveldb → }/db/log_writer.h
  27. 0 {leveldb → }/db/memtable.cc
  28. 0 {leveldb → }/db/memtable.h
  29. +28 −12 {leveldb → }/db/repair.cc
  30. 0 {leveldb → }/db/skiplist.h
  31. 0 {leveldb → }/db/skiplist_test.cc
  32. 0 {leveldb → }/db/snapshot.h
  33. 0 {leveldb → }/db/table_cache.cc
  34. 0 {leveldb → }/db/table_cache.h
  35. +38 −5 {leveldb → }/db/version_edit.cc
  36. +18 −0 {leveldb → }/db/version_edit.h
  37. +5 −1 {leveldb → }/db/version_edit_test.cc
  38. +111 −18 {leveldb → }/db/version_set.cc
  39. +26 −2 {leveldb → }/db/version_set.h
  40. +16 −0 {leveldb → }/db/write_batch.cc
  41. +4 −0 {leveldb → }/db/write_batch_internal.h
  42. +23 −0 {leveldb → }/db/write_batch_test.cc
  43. 0 {leveldb → }/doc/doc.css
  44. +12 −1 {leveldb → }/doc/impl.html
  45. +11 −0 {leveldb → }/doc/index.html
  46. 0 {leveldb → }/doc/log_format.txt
  47. 0 {leveldb → }/doc/table_format.txt
  48. 0 {leveldb → }/include/leveldb/cache.h
  49. 0 {leveldb → }/include/leveldb/comparator.h
  50. 0 {leveldb → }/include/leveldb/db.h
  51. 0 {leveldb → }/include/leveldb/env.h
  52. 0 {leveldb → }/include/leveldb/iterator.h
  53. +11 −1 {leveldb → }/include/leveldb/options.h
  54. 0 {leveldb → }/include/leveldb/slice.h
  55. 0 {leveldb → }/include/leveldb/status.h
  56. 0 {leveldb → }/include/leveldb/table.h
  57. 0 {leveldb → }/include/leveldb/table_builder.h
  58. 0 {leveldb → }/include/leveldb/write_batch.h
  59. +12 −0 {leveldb → }/leveldb.gyp
  60. +0 −122 leveldb/db/filename_test.cc
  61. 0 {leveldb → }/port/README
  62. 0 {leveldb → }/port/port.h
  63. 0 {leveldb → }/port/port_android.cc
  64. +8 −0 {leveldb → }/port/port_android.h
  65. 0 {leveldb → }/port/port_chromium.cc
  66. +7 −0 {leveldb → }/port/port_chromium.h
  67. +5 −0 {leveldb → }/port/port_example.h
  68. 0 {leveldb → }/port/port_posix.cc
  69. +5 −0 {leveldb → }/port/port_posix.h
  70. +298 −0 port/sha1_portable.cc
  71. +25 −0 port/sha1_portable.h
  72. +39 −0 port/sha1_test.cc
  73. 0 {leveldb → }/port/win/stdint.h
  74. +1 −3 {leveldb → }/table/block.cc
  75. 0 {leveldb → }/table/block.h
  76. +1 −1 {leveldb → }/table/block_builder.cc
  77. 0 {leveldb → }/table/block_builder.h
  78. +2 −2 {leveldb → }/table/format.cc
  79. 0 {leveldb → }/table/format.h
  80. 0 {leveldb → }/table/iterator.cc
  81. 0 {leveldb → }/table/iterator_wrapper.h
  82. 0 {leveldb → }/table/merger.cc
  83. 0 {leveldb → }/table/merger.h
  84. 0 {leveldb → }/table/table.cc
  85. 0 {leveldb → }/table/table_builder.cc
  86. 0 {leveldb → }/table/table_test.cc
  87. 0 {leveldb → }/table/two_level_iterator.cc
  88. 0 {leveldb → }/table/two_level_iterator.h
  89. +1 −1 {leveldb → }/util/arena.cc
  90. 0 {leveldb → }/util/arena.h
  91. 0 {leveldb → }/util/arena_test.cc
  92. 0 {leveldb → }/util/cache.cc
  93. 0 {leveldb → }/util/cache_test.cc
  94. +1 −1 {leveldb → }/util/coding.cc
  95. 0 {leveldb → }/util/coding.h
  96. 0 {leveldb → }/util/coding_test.cc
  97. +1 −1 {leveldb → }/util/comparator.cc
  98. 0 {leveldb → }/util/crc32c.cc
  99. 0 {leveldb → }/util/crc32c.h
  100. 0 {leveldb → }/util/crc32c_test.cc
  101. 0 {leveldb → }/util/env.cc
  102. 0 {leveldb → }/util/env_chromium.cc
  103. 0 {leveldb → }/util/env_posix.cc
  104. 0 {leveldb → }/util/env_test.cc
  105. 0 {leveldb → }/util/hash.cc
  106. 0 {leveldb → }/util/hash.h
  107. 0 {leveldb → }/util/histogram.cc
  108. 0 {leveldb → }/util/histogram.h
  109. +1 −1 {leveldb → }/util/logging.cc
  110. 0 {leveldb → }/util/logging.h
  111. 0 {leveldb → }/util/mutexlock.h
  112. +1 −0 {leveldb → }/util/options.cc
  113. +1 −1 {leveldb → }/util/random.h
  114. 0 {leveldb → }/util/status.cc
  115. 0 {leveldb → }/util/testharness.cc
  116. 0 {leveldb → }/util/testharness.h
  117. 0 {leveldb → }/util/testutil.cc
  118. 0 {leveldb → }/util/testutil.h
File renamed without changes.
File renamed without changes.
@@ -27,6 +27,7 @@ LIBOBJECTS = \
./db/version_set.o \
./db/write_batch.o \
./port/port_posix.o \
+ ./port/sha1_portable.o \
./table/block.o \
./table/block_builder.o \
./table/format.o \
@@ -62,6 +63,7 @@ TESTS = \
env_test \
filename_test \
log_test \
+ sha1_test \
skiplist_test \
table_test \
version_edit_test \
@@ -113,6 +115,9 @@ log_test: db/log_test.o $(LIBOBJECTS) $(TESTHARNESS)
table_test: table/table_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CC) $(LDFLAGS) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@
+sha1_test: port/sha1_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CC) $(LDFLAGS) port/sha1_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@
+
skiplist_test: db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CC) $(LDFLAGS) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@
@@ -2,10 +2,10 @@ leveldb: A key-value store
Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
The code under this directory implements a system for maintaining a
-persistent key/value store.
+persistent key/value store.
See doc/index.html for more explanation.
-See doc/impl.html for a brief overview of the implementation.
+See doc/db_layout.txt for a brief overview of the implementation.
The public interface is in include/*.h. Callers should not include or
rely on the details of any other header files in this package. Those
View
@@ -8,7 +8,7 @@ db
object stores, etc. can be done in the background anyway, so
probably not that important.
-api changes:
-- Make it wrappable
+api changes?
+- Efficient large value reading and writing
Faster Get implementation
@@ -38,6 +38,15 @@ Status BuildTable(const std::string& dbname,
for (; iter->Valid(); iter->Next()) {
Slice key = iter->key();
meta->largest.DecodeFrom(key);
+ if (ExtractValueType(key) == kTypeLargeValueRef) {
+ if (iter->value().size() != LargeValueRef::ByteSize()) {
+ s = Status::Corruption("invalid indirect reference hash value (L0)");
+ break;
+ }
+ edit->AddLargeValueRef(LargeValueRef::FromRef(iter->value()),
+ meta->number,
+ iter->key());
+ }
builder->Add(key, iter->value());
}
@@ -20,9 +20,9 @@ class VersionEdit;
// Build a Table file from the contents of *iter. The generated file
// will be named according to meta->number. On success, the rest of
// *meta will be filled with metadata about the generated table, and
-// the file information will be added to *edit. If no data is present
-// in *iter, meta->file_size will be set to zero, and no Table file
-// will be produced.
+// large value refs and the added file information will be added to
+// *edit. If no data is present in *iter, meta->file_size will be set
+// to zero, and no Table file will be produced.
extern Status BuildTable(const std::string& dbname,
Env* env,
const Options& options,
@@ -121,10 +121,11 @@ class CorruptionTest {
std::vector<std::string> filenames;
ASSERT_OK(env_.GetChildren(dbname_, &filenames));
uint64_t number;
+ LargeValueRef large_ref;
FileType type;
std::vector<std::string> candidates;
for (int i = 0; i < filenames.size(); i++) {
- if (ParseFileName(filenames[i], &number, &type) &&
+ if (ParseFileName(filenames[i], &number, &large_ref, &type) &&
type == filetype) {
candidates.push_back(dbname_ + "/" + filenames[i]);
}
@@ -275,6 +276,29 @@ TEST(CorruptionTest, SequenceNumberRecovery) {
ASSERT_EQ("v6", v);
}
+TEST(CorruptionTest, LargeValueRecovery) {
+ Options options;
+ options.large_value_threshold = 10000;
+ Reopen(&options);
+
+ Random rnd(301);
+ std::string big;
+ ASSERT_OK(db_->Put(WriteOptions(),
+ "foo", test::RandomString(&rnd, 100000, &big)));
+ std::string v;
+ ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
+ ASSERT_EQ(big, v);
+
+ RepairDB();
+ Reopen();
+ ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
+ ASSERT_EQ(big, v);
+
+ Reopen();
+ ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
+ ASSERT_EQ(big, v);
+}
+
TEST(CorruptionTest, CorruptedDescriptor) {
ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello"));
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
@@ -28,6 +28,7 @@
// readreverse -- read N values in reverse order
// readrandom -- read N values in random order
// crc32c -- repeated crc32c of 4K of data
+// sha1 -- repeated SHA1 computation over 4K of data
// Meta operations:
// compact -- Compact the entire DB
// stats -- Print DB stats
@@ -47,6 +48,7 @@ static const char* FLAGS_benchmarks =
"readreverse,"
"fill100K,"
"crc32c,"
+ "sha1,"
"snappycomp,"
"snappyuncomp,"
;
@@ -364,6 +366,8 @@ class Benchmark {
Compact();
} else if (name == Slice("crc32c")) {
Crc32c(4096, "(4K per op)");
+ } else if (name == Slice("sha1")) {
+ SHA1(4096, "(4K per op)");
} else if (name == Slice("snappycomp")) {
SnappyCompress();
} else if (name == Slice("snappyuncomp")) {
@@ -402,6 +406,24 @@ class Benchmark {
message_ = label;
}
+ void SHA1(int size, const char* label) {
+ // SHA1 about 100MB of data total
+ std::string data(size, 'x');
+ int64_t bytes = 0;
+ char sha1[20];
+ while (bytes < 100 * 1048576) {
+ port::SHA1_Hash(data.data(), size, sha1);
+ FinishedSingleOp();
+ bytes += size;
+ }
+
+ // Print so result is not dead
+ fprintf(stderr, "... sha1=%02x...\r", static_cast<unsigned int>(sha1[0]));
+
+ bytes_ = bytes;
+ message_ = label;
+ }
+
void SnappyCompress() {
Slice input = gen_.Generate(Options().block_size);
int64_t bytes = 0;
Oops, something went wrong.

0 comments on commit 80d4a46

Please sign in to comment.