Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

SERVER-8685: mongorestore oplogLimit fixes

Conflicts:

	src/mongo/tools/restore.cpp
  • Loading branch information...
commit 49cb7cab30855e58ef506c97d1280f2ca7d4dc6e 1 parent 2d57d09
Scott Hernandez authored monkey101 committed

Showing 1 changed file with 75 additions and 10 deletions. Show diff stats Hide diff stats

  1. 85  src/mongo/tools/restore.cpp
85  src/mongo/tools/restore.cpp
@@ -21,6 +21,8 @@
21 21
 #include <boost/filesystem/convenience.hpp>
22 22
 #include <boost/filesystem/operations.hpp>
23 23
 #include <boost/program_options.hpp>
  24
+#include <boost/scoped_ptr.hpp>
  25
+#include <boost/lexical_cast.hpp>
24 26
 #include <fcntl.h>
25 27
 #include <fstream>
26 28
 #include <set>
@@ -52,12 +54,16 @@ class Restore : public BSONTool {
52 54
     string _curdb;
53 55
     string _curcoll;
54 56
     set<string> _users; // For restoring users with --drop
55  
-    auto_ptr<Matcher> _opmatcher; // For oplog replay
  57
+    scoped_ptr<Matcher> _opmatcher; // For oplog replay
  58
+    scoped_ptr<OpTime> _oplogLimitTS; // for oplog replay (limit)
  59
+    int _oplogEntrySkips; // oplog entries skipped
  60
+    int _oplogEntryApplies; // oplog entries applied
56 61
     Restore() : BSONTool( "restore" ) , _drop(false) {
57 62
         add_options()
58 63
         ("drop" , "drop each collection before import" )
59 64
         ("oplogReplay", "replay oplog for point-in-time restore")
60  
-        ("oplogLimit", po::value<string>(), "exclude oplog entries newer than provided timestamp (epoch[:ordinal])")
  65
+        ("oplogLimit", po::value<string>(), "include oplog entries before the provided Timestamp "
  66
+                "(seconds[:ordinal]) during the oplog replay; the ordinal value is optional")
61 67
         ("keepIndexVersion" , "don't upgrade indexes to newest version")
62 68
         ("noOptionsRestore" , "don't restore collection options")
63 69
         ("noIndexRestore" , "don't restore indexes")
@@ -140,9 +146,50 @@ class Restore : public BSONTool {
140 146
 
141 147
                     oplogLimit = oplogLimit.substr(0, i);
142 148
                 }
143  
-                
144  
-                if ( ! oplogLimit.empty() ) {
145  
-                    _opmatcher.reset( new Matcher( fromjson( string("{ \"ts\": { \"$lt\": { \"$timestamp\": { \"t\": ") + oplogLimit + string(", \"i\": ") + oplogInc + string(" } } } }") ) ) );
  149
+
  150
+                try {
  151
+                    _oplogLimitTS.reset(new OpTime(
  152
+                        boost::lexical_cast<unsigned long>(oplogLimit.c_str()),
  153
+                        boost::lexical_cast<unsigned long>(oplogInc.c_str())));
  154
+                } catch( const boost::bad_lexical_cast& error) {
  155
+                    log() << "Could not parse oplogLimit into Timestamp from values ( "
  156
+                          << oplogLimit << " , " << oplogInc << " )"
  157
+                          << endl;
  158
+                    return -1;
  159
+                }
  160
+
  161
+                if (!oplogLimit.empty()) {
  162
+                    // Only for a replica set as master will have no-op entries so we would need to
  163
+                    // skip them all to find the real op
  164
+                    scoped_ptr<DBClientCursor> cursor(
  165
+                            conn().query("local.oplog.rs", Query().sort(BSON("$natural" << -1)),
  166
+                                         1 /*return first*/));
  167
+                    OpTime tsOptime;
  168
+                    // get newest oplog entry and make sure it is older than the limit to apply.
  169
+                    if (cursor->more()) {
  170
+                        tsOptime = cursor->next().getField("ts")._opTime();
  171
+                        if (tsOptime > *_oplogLimitTS.get()) {
  172
+                            log() << "The oplogLimit is not newer than"
  173
+                                  << " the last oplog entry on the server."
  174
+                                  << endl;
  175
+                            return -1;
  176
+                        }
  177
+                    }
  178
+
  179
+                    BSONObjBuilder tsRestrictBldr;
  180
+                    if (tsOptime != NULL)
  181
+                        tsRestrictBldr << "$gt" << tsOptime;
  182
+                    tsRestrictBldr << "$lt" << *_oplogLimitTS.get();
  183
+
  184
+                    BSONObj query = BSON("ts" << tsRestrictBldr.obj());
  185
+
  186
+                    if (tsOptime != NULL) {
  187
+                        log() << "Latest oplog entry on the server is " << tsOptime.getSecs()
  188
+                                << ":" << tsOptime.getInc() << endl;
  189
+                        log() << "Only applying oplog entries matching this criteria: "
  190
+                                << query.jsonString() << endl;
  191
+                    }
  192
+                    _opmatcher.reset(new Matcher(query));
146 193
                 }
147 194
             }
148 195
         }
@@ -156,7 +203,7 @@ class Restore : public BSONTool {
156 203
          * given either a root directory that contains only a single
157 204
          * .bson file, or a single .bson file itself (a collection).
158 205
          */
159  
-        drillDown(root, _db != "", _coll != "", true);
  206
+        drillDown(root, _db != "", _coll != "", !(_oplogLimitTS.get() == NULL), true);
160 207
 
161 208
         // should this happen for oplog replay as well?
162 209
         conn().getLastError(_db == "" ? "admin" : _db);
@@ -165,12 +212,20 @@ class Restore : public BSONTool {
165 212
             log() << "\t Replaying oplog" << endl;
166 213
             _curns = OPLOG_SENTINEL;
167 214
             processFile( root / "oplog.bson" );
  215
+            log() << "Applied " << _oplogEntryApplies << " oplog entries out of "
  216
+                  << _oplogEntryApplies + _oplogEntrySkips << " (" << _oplogEntrySkips
  217
+                  << " skipped)." << endl;
168 218
         }
169 219
 
170 220
         return EXIT_CLEAN;
171 221
     }
172 222
 
173  
-    void drillDown( boost::filesystem::path root, bool use_db, bool use_coll, bool top_level=false ) {
  223
+    void drillDown( boost::filesystem::path root,
  224
+                    bool use_db,
  225
+                    bool use_coll,
  226
+                    bool oplogReplayLimit,
  227
+                    bool top_level=false) {
  228
+        bool json_metadata = false;
174 229
         LOG(2) << "drillDown: " << root.string() << endl;
175 230
 
176 231
         // skip hidden files and directories
@@ -210,12 +265,13 @@ class Restore : public BSONTool {
210 265
                 if ( p.leaf() == "system.indexes.bson" ) {
211 266
                     indexes = p;
212 267
                 } else {
213  
-                    drillDown(p, use_db, use_coll);
  268
+                    drillDown(p, use_db, use_coll, oplogReplayLimit);
214 269
                 }
215 270
             }
216 271
 
217  
-            if (!indexes.empty())
218  
-                drillDown(indexes, use_db, use_coll);
  272
+            if (!indexes.empty() && !json_metadata) {
  273
+                drillDown(indexes, use_db, use_coll, oplogReplayLimit);
  274
+            }
219 275
 
220 276
             return;
221 277
         }
@@ -264,6 +320,13 @@ class Restore : public BSONTool {
264 320
             ns += "." + oldCollName;
265 321
         }
266 322
 
  323
+        if (oplogReplayLimit) {
  324
+            error() << "The oplogLimit option cannot be used if "
  325
+                    << "normal databases/collections exist in the dump directory."
  326
+                    << endl;
  327
+            exit(EXIT_FAILURE);
  328
+        }
  329
+
267 330
         log() << "\tgoing into namespace [" << ns << "]" << endl;
268 331
 
269 332
         if ( _drop ) {
@@ -341,6 +404,7 @@ class Restore : public BSONTool {
341 404
             
342 405
             // exclude operations that don't meet (timestamp) criteria
343 406
             if ( _opmatcher.get() && ! _opmatcher->matches ( obj ) ) {
  407
+                _oplogEntrySkips++;
344 408
                 return;
345 409
             }
346 410
 
@@ -350,6 +414,7 @@ class Restore : public BSONTool {
350 414
             BSONObj cmd = BSON( "applyOps" << BSON_ARRAY( obj ) );
351 415
             BSONObj out;
352 416
             conn().runCommand(db, cmd, out);
  417
+            _oplogEntryApplies++;
353 418
 
354 419
             // wait for ops to propagate to "w" nodes (doesn't warn if w used without replset)
355 420
             if ( _w > 1 ) {

0 notes on commit 49cb7ca

Please sign in to comment.
Something went wrong with that request. Please try again.