Permalink
Browse files

journal compression

  • Loading branch information...
1 parent 97e6514 commit 9a30ac81c4fa07cf05a19be1d3c465591c05cc44 @dwight dwight committed Jul 21, 2011
Showing with 155 additions and 74 deletions.
  1. +1 −3 db/dur.cpp
  2. +48 −13 db/dur_journal.cpp
  3. +1 −1 db/dur_journal.h
  4. +20 −1 db/dur_journalformat.h
  5. +1 −1 db/dur_journalimpl.h
  6. +3 −23 db/dur_preplogbuffer.cpp
  7. +69 −27 db/dur_recover.cpp
  8. +3 −0 db/dur_recover.h
  9. +0 −2 db/durop.h
  10. +1 −1 db/mongommf.h
  11. +4 −2 util/alignedbuilder.cpp
  12. +4 −0 util/alignedbuilder.h
View
@@ -504,20 +504,18 @@ namespace mongo {
commitJob.notifyCommitted();
return true;
}
-
PREPLOGBUFFER();
RWLockRecursive::Shared lk3(MongoFile::mmmutex);
- unsigned abLen = commitJob._ab.len();
+ unsigned abLen = commitJob._ab.len() + sizeof(JSectFooter);
commitJob.reset(); // must be reset before allowing anyone to write
DEV assert( !commitJob.hasWritten() );
// release the readlock -- allowing others to now write while we are writing to the journal (etc.)
lk1.reset();
// ****** now other threads can do writes ******
-
WRITETOJOURNAL(commitJob._ab);
assert( abLen == commitJob._ab.len() ); // a check that no one touched the builder while we were doing work. if so, our locking is wrong.
View
@@ -95,6 +95,11 @@ namespace mongo {
assert(false);
}
+ JSectFooter::JSectFooter() {
+ memset(this, 0, sizeof(*this));
+ sentinel = JEntry::OpCode_Footer;
+ }
+
JSectFooter::JSectFooter(const void* begin, int len) { // needs buffer to compute hash
sentinel = JEntry::OpCode_Footer;
reserved = 0;
@@ -629,24 +634,52 @@ namespace mongo {
}
}
- /** write to journal
+ /** write to journal (called by WRITETOJOURNAL)
*/
- void journal(const AlignedBuilder& b) {
+ void journal(AlignedBuilder& b) {
j.journal(b);
}
- void Journal::journal(const AlignedBuilder& _b) {
-#if defined(_NOCOMPRESS)
+ void Journal::journal(AlignedBuilder& _b) {
+ RACECHECK
+#if 0
unsigned w = _b.len();
const AlignedBuilder& b = _b;
-#else
- static AlignedBuilder compressed(32*1024*1024);
- compressed.reset( maxCompressedLength(_b.len()) );
+#endif
+ static AlignedBuilder cb(32*1024*1024);
+ const unsigned headTailSize = sizeof(JSectHeader) + sizeof(JSectFooter);
+ cb.reset( maxCompressedLength(_b.len()) + headTailSize );
+
+ {
+ JSectHeader *h = (JSectHeader *) _b.buf();
+ dassert( h->sectionLen() == (unsigned) 0xffffffff );
+ cb.appendStruct(*h);
+ }
+
size_t compressedLength = 0;
- rawCompress(_b.buf(), _b.len(), compressed.atOfs(0), &compressedLength);
+ rawCompress(_b.buf(), _b.len() - headTailSize, cb.cur(), &compressedLength);
+ assert( compressedLength < 0xffffffff );
+ cb.skip(compressedLength);
- unsigned w = (compressedLength+8191)&(~8191);
- const AlignedBuilder& b = compressed;
-#endif
+ // footer
+ unsigned L = 0xffffffff;
+ {
+ // pad to alignment, and set the total section length in the JSectHeader
+ assert( 0xffffe000 == (~(Alignment-1)) );
+ unsigned lenUnpadded = cb.len() + sizeof(JSectFooter);
+ L = (lenUnpadded + Alignment-1) & (~(Alignment-1));
+ dassert( L >= lenUnpadded );
+
+ ((JSectHeader*)cb.atOfs(0))->setSectionLen(lenUnpadded);
+
+ JSectFooter f(cb.buf(), cb.len()); // computes checksum
+ cb.appendStruct(f);
+
+ // need a footer for uncompressed buffer too so that WRITETODATAFILES is happy.
+ // done this way as we do not need a checksum for that.
+ JSectFooter uncompressedFooter;
+ _b.appendStruct(uncompressedFooter);
+ ((JSectHeader*)_b.atOfs(0))->setSectionLen(_b.len());
+ }
try {
mutex::scoped_lock lk(_curLogFileMutex);
@@ -655,9 +688,11 @@ namespace mongo {
assert( _curLogFile );
stats.curr->_uncompressedBytes += _b.len();
- stats.curr->_journaledBytes += w;
+ unsigned w = cb.len();
_written += w;
- _curLogFile->synchronousAppend((void *) b.buf(), w);
+ assert( w <= L );
+ stats.curr->_journaledBytes += L;
+ _curLogFile->synchronousAppend((void *) cb.buf(), L);
_rotate();
}
catch(std::exception& e) {
View
@@ -41,7 +41,7 @@ namespace mongo {
@param buf - a buffer that will be written to the journal.
will not return until on disk
*/
- void journal(const AlignedBuilder& buf);
+ void journal(AlignedBuilder& buf);
/** flag that something has gone wrong during writing to the journal
(not for recovery mode)
View
@@ -22,6 +22,8 @@ namespace mongo {
namespace dur {
+ const unsigned Alignment = 8192;
+
#pragma pack(1)
/** beginning header for a journal/j._<n> file
there is nothing important int this header at this time. except perhaps version #.
@@ -59,11 +61,25 @@ namespace mongo {
/** "Section" header. A section corresponds to a group commit.
len is length of the entire section including header and footer.
+ header and footer are not compressed, just the stuff in between.
*/
struct JSectHeader {
- unsigned len; // length in bytes of the whole section
+ private:
+ unsigned _sectionLen; // unpadded length in bytes of the whole section
+ public:
unsigned long long seqNumber; // sequence number that can be used on recovery to not do too much work
unsigned long long fileId; // matches JHeader::fileId
+ unsigned sectionLen() const { return _sectionLen; }
+
+ // we store the unpadded length so we can use that when we uncompress. to
+ // get the true total size this must be rounded up to the Alignment.
+ void setSectionLen(unsigned lenUnpadded) { _sectionLen = lenUnpadded; }
+
+ unsigned sectionLenWithPadding() const {
+ unsigned x = (sectionLen() + (Alignment-1)) & (~(Alignment-1));
+ dassert( x % Alignment == 0 );
+ return x;
+ }
};
/** an individual write operation within a group commit section. Either the entire section should
@@ -115,6 +131,7 @@ namespace mongo {
/** group commit section footer. md5 is a key field. */
struct JSectFooter {
+ JSectFooter();
JSectFooter(const void* begin, int len); // needs buffer to compute hash
unsigned sentinel;
unsigned char hash[16];
@@ -127,6 +144,8 @@ namespace mongo {
@return true if buffer looks valid
*/
bool checkHash(const void* begin, int len) const;
+
+ void checkMagic() const { assert( *((unsigned*)magic) == 0x0a0a0a0a ); }
};
/** declares "the next entry(s) are for this database / file path prefix" */
View
@@ -35,7 +35,7 @@ namespace mongo {
/** write to journal
*/
- void journal(const AlignedBuilder& b);
+ void journal(AlignedBuilder& b);
boost::filesystem::path getFilePathFor(int filenumber) const;
View
@@ -130,7 +130,7 @@ namespace mongo {
// JSectHeader
JSectHeader h;
- h.len = (unsigned) 0xffffffff; // total length, will fill in later
+ h.setSectionLen(0xffffffff); // total length, will fill in later
h.seqNumber = getLastDataFileFlushTime();
h.fileId = j.curFileId();
@@ -153,7 +153,7 @@ namespace mongo {
}
AlignedBuilder& bb = commitJob._ab;
- resetLogBuffer(bb);
+ resetLogBuffer(bb); // adds JSectHeader
// ops other than basic writes (DurOp's)
{
@@ -162,27 +162,7 @@ namespace mongo {
}
}
- {
- prepBasicWrites(bb);
- }
-
- // pad to alignment, and set the total section length in the JSectHeader
- assert( 0xffffe000 == (~(Alignment-1)) );
- unsigned lenWillBe = bb.len() + sizeof(JSectFooter);
- unsigned L = (lenWillBe + Alignment-1) & (~(Alignment-1));
- dassert( L >= lenWillBe );
- *((unsigned*)bb.atOfs(0)) = L;
-
- {
- JSectFooter f(bb.buf(), bb.len());
- bb.appendStruct(f);
- }
-
- {
- unsigned padding = L - bb.len();
- bb.skip(padding);
- dassert( bb.len() % Alignment == 0 );
- }
+ prepBasicWrites(bb);
return;
}
Oops, something went wrong.

0 comments on commit 9a30ac8

Please sign in to comment.