Permalink
Browse files

SERVER-2771 Retry index builds on startup

  • Loading branch information...
kchodorow committed Nov 12, 2012
1 parent 3abcc53 commit ab1bceba74845666aaad957cab256cadf2a0c8de
@@ -0,0 +1,88 @@
+// Check index rebuild when MongoDB is killed
+
+var ports = allocatePorts(1);
+mongod = new MongodRunner(ports[0], "/data/db/index_retry", null, null, ["--journal"]);
+var conn = mongod.start();
+
+var test = conn.getDB("test");
+
+var name = 'jstests_slownightly_index_retry';
+t = test.getCollection(name);
+t.drop();
+
+// Insert a large number of documents, enough to ensure that an index build on these documents can
+// be interrupted before complete.
+for (i = 0; i < 1e6; ++i) {
+ t.save( { a:i } );
+ if (i % 10000 == 0) {
+ print("i: " + i);
+ }
+}
+test.getLastError();
+
+function debug(x) {
+ printjson(x);
+}
+
+/**
+ * @return if there's a current running index build
+ */
+function indexBuildInProgress() {
+ inprog = test.currentOp().inprog;
+ debug(inprog);
+ indexBuildOpId = -1;
+ inprog.forEach(
+ function( op ) {
+ // Identify the index build as an insert into the 'test.system.indexes'
+ // namespace. It is assumed that no other clients are concurrently
+ // accessing the 'test' database.
+ if ( op.op == 'insert' && op.ns == 'test.system.indexes' ) {
+ debug(op.opid);
+ indexBuildOpId = op.opid;
+ }
+ }
+ );
+ return indexBuildOpId != -1;
+}
+
+function abortDuringIndexBuild(options) {
+
+ // Create an index asynchronously by using a new connection.
+ new Mongo(test.getMongo().host ).getCollection( t.toString() ).createIndex( { a:1 }, options);
+
+ // Wait for the index build to start.
+ var times = 0;
+ assert.soon(
+ function() {
+ return indexBuildInProgress() && times++ >= 2;
+ }
+ );
+
+ print("killing the mongod");
+ stopMongod(ports[0], /* signal */ 9);
+}
+
+abortDuringIndexBuild({background:true});
+
+print("sleeping");
+sleep(2000);
+
+conn = mongod.start(/* reuseData */ true);
+
+assert.soon(
+ function() {
+ try {
+ printjson(conn.getDB("test").getCollection(name).find({a:42}).hint({a:1}).next());
+ } catch (e) {
+ print(e);
+ return false;
+ }
+ return true;
+ },
+ 'index builds successfully'
+);
+
+print("Index built");
+
+stopMongod(ports[0]);
+print("SUCCESS!");
View
@@ -344,6 +344,7 @@ serverOnlyFiles = [ "db/curop.cpp",
"db/extsort.cpp",
"db/index.cpp",
"db/index_update.cpp",
+ "db/index_rebuilder.cpp",
"db/scanandorder.cpp",
"db/explain.cpp",
"db/geo/2d.cpp",
View
@@ -72,6 +72,7 @@ namespace mongo {
bool usingReplSets() const { return !_replSet.empty(); }
std::string rsIndexPrefetch;// --indexPrefetch
+ bool indexBuildRetry; // --noIndexBuildRetry
// for master/slave replication
std::string source; // --source
@@ -141,7 +142,7 @@ namespace mongo {
SSLManager* sslServerManager; // currently leaks on close
#endif
-
+
static void launchOk();
static void addGlobalOptions( boost::program_options::options_description& general ,
@@ -176,7 +177,7 @@ namespace mongo {
// todo move to cmdline.cpp?
inline CmdLine::CmdLine() :
- port(DefaultDBPort), rest(false), jsonp(false), quiet(false),
+ port(DefaultDBPort), rest(false), jsonp(false), indexBuildRetry(true), quiet(false),
noTableScan(false), prealloc(true), preallocj(true), smallfiles(sizeof(int*) == 4),
configsvr(false), quota(false), quotaFiles(8), cpu(false),
durOptions(0), objcheck(false), oplogSize(0), defaultProfile(0),
View
@@ -34,6 +34,7 @@
#include "mongo/db/dbmessage.h"
#include "mongo/db/dbwebserver.h"
#include "mongo/db/dur.h"
+#include "mongo/db/index_rebuilder.h"
#include "mongo/db/initialize_server_global_state.h"
#include "mongo/db/instance.h"
#include "mongo/db/introspect.h"
@@ -78,7 +79,6 @@ namespace mongo {
extern int diagLogging;
extern unsigned lenForNewNsFiles;
extern int lockFile;
- extern bool checkNsFilesOnLoad;
extern string repairpath;
void setupSignals( bool inFork );
@@ -294,9 +294,6 @@ namespace mongo {
Client::GodScope gs;
LOG(1) << "enter repairDatabases (to check pdfile version #)" << endl;
- //verify(checkNsFilesOnLoad);
- checkNsFilesOnLoad = false; // we are mainly just checking the header - don't scan the whole .ns file for every db here.
-
Lock::GlobalWrite lk;
vector< string > dbNames;
getDatabaseNames( dbNames );
@@ -345,8 +342,6 @@ namespace mongo {
cc().shutdown();
dbexit( EXIT_CLEAN );
}
-
- checkNsFilesOnLoad = true;
}
void clearTmpFiles() {
@@ -403,7 +398,7 @@ namespace mongo {
*/
class DataFileSync : public BackgroundJob , public ServerStatusSection {
public:
- DataFileSync()
+ DataFileSync()
: ServerStatusSection( "backgroundFlushing" ),
_total_time( 0 ),
_flushes( 0 ),
@@ -412,7 +407,7 @@ namespace mongo {
virtual bool includeByDefault() const { return true; }
virtual bool adminOnly() const { return false; }
-
+
string name() const { return "DataFileSync"; }
void run() {
@@ -469,7 +464,7 @@ namespace mongo {
_last_time = ms;
_last = jsTime();
}
-
+
long long _total_time;
long long _flushes;
int _last_time;
@@ -485,16 +480,16 @@ namespace mongo {
virtual void appendAtLeaf( BSONObjBuilder& b ) const {
int m = static_cast<int>(MemoryMappedFile::totalMappedLength() / ( 1024 * 1024 ));
b.appendNumber( "mapped" , m );
-
+
if ( cmdLine.dur ) {
m *= 2;
b.appendNumber( "mappedWithJournal" , m );
}
-
+
}
} memJournalServerStatusMetric;
}
-
+
const char * jsInterruptCallback() {
// should be safe to interrupt in js code, even if we have a write lock
@@ -640,6 +635,8 @@ namespace mongo {
/* this is for security on certain platforms (nonce generation) */
srand((unsigned) (curTimeMicros() ^ startupSrandTimer.micros()));
+ indexRebuilder.go();
+
snapshotThread.go();
d.clientCursorMonitor.go();
PeriodicTask::theRunner->go();
@@ -754,6 +751,8 @@ static void buildOptionsDescriptions(po::options_description *pVisible,
("jsonp","allow JSONP access via http (has security implications)")
("noauth", "run without security")
("nohttpinterface", "disable http interface")
+ ("noIndexBuildRetry", po::value<int>(),
+ "don't retry any index builds that were interrupted by shutdown")
("nojournal", "disable journaling (journaling is on by default for 64 bit)")
("noprealloc", "disable data file preallocation - will often hurt performance")
("noscripting", "disable scripting engine")
@@ -1037,6 +1036,9 @@ static void processCommandLineOptions(const std::vector<std::string>& argv) {
if (params.count("replIndexPrefetch")) {
cmdLine.rsIndexPrefetch = params["replIndexPrefetch"].as<std::string>();
}
+ if (params.count("noIndexBuildRetry")) {
+ cmdLine.indexBuildRetry = false;
+ }
if (params.count("only")) {
cmdLine.only = params["only"].as<string>().c_str();
}
@@ -0,0 +1,112 @@
+/**
+ * Copyright (C) 2012 10gen Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mongo/db/index_rebuilder.h"
+
+#include "mongo/db/instance.h"
+#include "mongo/db/pdfile.h"
+
+namespace mongo {
+
+ IndexRebuilder indexRebuilder;
+
+ IndexRebuilder::IndexRebuilder() {}
+
+ std::string IndexRebuilder::name() const {
+ return "IndexRebuilder";
+ }
+
+ void IndexRebuilder::run() {
+ Client::initThread(name().c_str());
+ Lock::GlobalWrite lk;
+ Client::GodScope gs;
+ std::vector<std::string> dbNames;
+ getDatabaseNames(dbNames);
+
+ for (std::vector<std::string>::const_iterator it = dbNames.begin();
+ it < dbNames.end();
+ it++) {
+ checkDB(*it);
+ }
+
+ cc().shutdown();
+ }
+
+ void IndexRebuilder::checkDB(const std::string& dbName) {
+ const std::string systemNS = dbName + ".system.namespaces";
+ DBDirectClient cli;
+ scoped_ptr<DBClientCursor> cursor(cli.query(systemNS, Query()));
+
+ while (cursor->more()) {
+ BSONObj nsDoc = cursor->next();
+ const char* ns = nsDoc["name"].valuestrsafe();
+
+ Client::Context ctx(ns, dbpath, false, false);
+ NamespaceDetails* nsd = nsdetails(ns);
+
+ if (!nsd || !nsd->indexBuildInProgress) {
+ continue;
+ }
+
+ log() << "Found interrupted index build on " << ns << endl;
+
+ // If the indexBuildRetry flag isn't set, just clear the inProg flag
+ if (!cmdLine.indexBuildRetry) {
+ // If we crash between unsetting the inProg flag and cleaning up the index, the
+ // index space will be lost.
+ getDur().writingInt(nsd->indexBuildInProgress) = 0;
+ nsd->idx(nsd->nIndexes).kill_idx();
+ continue;
+ }
+
+ retryIndexBuild(dbName, nsd);
+ }
+ }
+
+ void IndexRebuilder::retryIndexBuild(const std::string& dbName, NamespaceDetails* nsd) {
+ // details.info is always a valid system.indexes entry because DataFileMgr::insert journals
+ // creating the index doc and then insert_makeIndex durably assigns its DiskLoc to info.
+ // indexBuildInProgress is set after that, so if it is set, info must be set.
+ IndexDetails& details = nsd->idx(nsd->nIndexes);
+
+ // First, clean up the in progress index build. Save the system.indexes entry so that we
+ // can add it again afterwards.
+ BSONObj indexObj = details.info.obj().getOwned();
+
+ // Clean up the in-progress index build
+ getDur().writingInt(nsd->indexBuildInProgress) = 0;
+ details.kill_idx();
+ // The index has now been removed from system.indexes, so the only record of it is in-
+ // memory. If there is a journal commit between now and when insert() rewrites the entry and
+ // the db crashes before the new system.indexes entry is journalled, the index will be lost
+ // forever. Thus, we're assuming no journaling will happen between now and the entry being
+ // re-written.
+
+ // We need to force a foreground index build to prevent replication from replaying an
+ // incompatible op (like a drop) during a yield.
+ // TODO: once commands can interrupt/wait for index builds, this can be removed.
+ indexObj = indexObj.removeField("background");
+
+ try {
+ const std::string ns = dbName + ".system.indexes";
+ theDataFileMgr.insert(ns.c_str(), indexObj.objdata(), indexObj.objsize(), false, true);
+ }
+ catch (const DBException& e) {
+ log() << "Rebuilding index failed: " << e.what() << " (" << e.getCode() << ")"
+ << endl;
+ }
+ }
+}
@@ -0,0 +1,45 @@
+/**
+ * Copyright (C) 2012 10gen Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "mongo/db/namespace_details.h"
+#include "mongo/util/background.h"
+
+namespace mongo {
+
+ class IndexRebuilder : public BackgroundJob {
+ public:
+ IndexRebuilder();
+
+ std::string name() const;
+ void run();
+
+ private:
+ /**
+ * Check each collection in a database to see if it has any in-progress index builds that
+ * need to be retried. If so, calls retryIndexBuild.
+ */
+ void checkDB(const std::string& dbname);
+
+ /**
+ * Actually retry the index build on a given namespace.
+ */
+ void retryIndexBuild(const std::string& dbName, NamespaceDetails* nsd);
+ };
+
+ extern IndexRebuilder indexRebuilder;
+}
Oops, something went wrong.

0 comments on commit ab1bceb

Please sign in to comment.