Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Generalize recloning docs on initial oplog application SERVER-3367

Conflicts:

	db/repl.cpp
	db/repl/rs_sync.cpp
  • Loading branch information...
commit 154f5545b6c6708146d0215408b4029e72a021dd 1 parent e6e308d
Kristina authored November 14, 2011
34  db/oplog.cpp
@@ -625,6 +625,40 @@ namespace mongo {
625 625
         }
626 626
     }
627 627
 
  628
+    bool shouldRetry(const BSONObj& o, const string& hn) {
  629
+        OplogReader missingObjReader;
  630
+
  631
+        // we don't have the object yet, which is possible on initial sync.  get it.
  632
+        log() << "replication info adding missing object" << endl; // rare enough we can log
  633
+        uassert(15916, str::stream() << "Can no longer connect to initial sync source: " << hn, missingObjReader.connect(hn));
  634
+
  635
+        const char *ns = o.getStringField("ns");
  636
+        // might be more than just _id in the update criteria
  637
+        BSONObj query = BSONObjBuilder().append(o.getObjectField("o2")["_id"]).obj();
  638
+        BSONObj missingObj;
  639
+        try {
  640
+            missingObj = missingObjReader.findOne(ns, query);
  641
+        } catch(DBException& e) {
  642
+            log() << "replication assertion fetching missing object: " << e.what() << endl;
  643
+            throw;
  644
+        }
  645
+
  646
+        if( missingObj.isEmpty() ) {
  647
+            log() << "replication missing object not found on source. presumably deleted later in oplog" << endl;
  648
+            log() << "replication o2: " << o.getObjectField("o2").toString() << endl;
  649
+            log() << "replication o firstfield: " << o.getObjectField("o").firstElementFieldName() << endl;
  650
+
  651
+            return false;
  652
+        }
  653
+        else {
  654
+            Client::Context ctx(ns);
  655
+            DiskLoc d = theDataFileMgr.insert(ns, (void*) missingObj.objdata(), missingObj.objsize());
  656
+            uassert(15917, "Got bad disk location when attempting to insert", !d.isNull());
  657
+
  658
+            return true;
  659
+        }
  660
+    }
  661
+
628 662
     /** @param fromRepl false if from ApplyOpsCmd
629 663
         @return true if was and update should have happened and the document DNE.  see replset initial sync code.
630 664
      */
6  db/oplog.h
@@ -129,6 +129,12 @@ namespace mongo {
129 129
      * take an op and apply locally
130 130
      * used for applying from an oplog
131 131
      * @param fromRepl really from replication or for testing/internal/command/etc...
  132
+     * Returns if the op was an update that could not be applied (true on failure)
132 133
      */
133 134
     bool applyOperation_inlock(const BSONObj& op , bool fromRepl = true );
  135
+
  136
+    /**
  137
+     * If applyOperation_inlock should be called again after an update fails.
  138
+     */
  139
+    bool shouldRetry(const BSONObj& op , const string& hn);
134 140
 }
47  db/repl.cpp
@@ -508,12 +508,12 @@ namespace mongo {
508 508
 
509 509
         return;
510 510
     }
511  
-    
  511
+
512 512
     DatabaseIgnorer ___databaseIgnorer;
513  
-    
  513
+
514 514
     void DatabaseIgnorer::doIgnoreUntilAfter( const string &db, const OpTime &futureOplogTime ) {
515 515
         if ( futureOplogTime > _ignores[ db ] ) {
516  
-            _ignores[ db ] = futureOplogTime;   
  516
+            _ignores[ db ] = futureOplogTime;
517 517
         }
518 518
     }
519 519
 
@@ -533,28 +533,28 @@ namespace mongo {
533 533
     bool ReplSource::handleDuplicateDbName( const BSONObj &op, const char *ns, const char *db ) {
534 534
         if ( dbHolder.isLoaded( ns, dbpath ) ) {
535 535
             // Database is already present.
536  
-            return true;   
  536
+            return true;
537 537
         }
538 538
         BSONElement ts = op.getField( "ts" );
539 539
         if ( ( ts.type() == Date || ts.type() == Timestamp ) && ___databaseIgnorer.ignoreAt( db, ts.date() ) ) {
540 540
             // Database is ignored due to a previous indication that it is
541 541
             // missing from master after optime "ts".
542  
-            return false;   
  542
+            return false;
543 543
         }
544 544
         if ( Database::duplicateUncasedName( db, dbpath ).empty() ) {
545 545
             // No duplicate database names are present.
546 546
             return true;
547 547
         }
548  
-        
  548
+
549 549
         OpTime lastTime;
550 550
         bool dbOk = false;
551 551
         {
552 552
             dbtemprelease release;
553  
-        
  553
+
554 554
             // We always log an operation after executing it (never before), so
555 555
             // a database list will always be valid as of an oplog entry generated
556 556
             // before it was retrieved.
557  
-            
  557
+
558 558
             BSONObj last = oplogReader.findOne( this->ns().c_str(), Query().sort( BSON( "$natural" << -1 ) ) );
559 559
             if ( !last.isEmpty() ) {
560 560
 	            BSONElement ts = last.getField( "ts" );
@@ -568,34 +568,34 @@ namespace mongo {
568 568
             BSONObjIterator i( info.getField( "databases" ).embeddedObject() );
569 569
             while( i.more() ) {
570 570
                 BSONElement e = i.next();
571  
-            
  571
+
572 572
                 const char * name = e.embeddedObject().getField( "name" ).valuestr();
573 573
                 if ( strcasecmp( name, db ) != 0 )
574 574
                     continue;
575  
-                
  575
+
576 576
                 if ( strcmp( name, db ) == 0 ) {
577 577
                     // The db exists on master, still need to check that no conflicts exist there.
578 578
                     dbOk = true;
579 579
                     continue;
580 580
                 }
581  
-                
  581
+
582 582
                 // The master has a db name that conflicts with the requested name.
583 583
                 dbOk = false;
584 584
                 break;
585 585
             }
586 586
         }
587  
-        
  587
+
588 588
         if ( !dbOk ) {
589 589
             ___databaseIgnorer.doIgnoreUntilAfter( db, lastTime );
590 590
             incompleteCloneDbs.erase(db);
591 591
             addDbNextPass.erase(db);
592  
-            return false;   
  592
+            return false;
593 593
         }
594  
-        
  594
+
595 595
         // Check for duplicates again, since we released the lock above.
596 596
         set< string > duplicates;
597 597
         Database::duplicateUncasedName( db, dbpath, &duplicates );
598  
-        
  598
+
599 599
         // The database is present on the master and no conflicting databases
600 600
         // are present on the master.  Drop any local conflicts.
601 601
         for( set< string >::const_iterator i = duplicates.begin(); i != duplicates.end(); ++i ) {
@@ -605,7 +605,7 @@ namespace mongo {
605 605
             Client::Context ctx(*i);
606 606
             dropDatabase(*i);
607 607
         }
608  
-        
  608
+
609 609
         massert( 14034, "Duplicate database names present after attempting to delete duplicates",
610 610
                 Database::duplicateUncasedName( db, dbpath ).empty() );
611 611
         return true;
@@ -613,7 +613,10 @@ namespace mongo {
613 613
 
614 614
     void ReplSource::applyOperation(const BSONObj& op) {
615 615
         try {
616  
-            applyOperation_inlock( op );
  616
+            bool failedUpdate = applyOperation_inlock( op );
  617
+            if (failedUpdate && shouldRetry(op, hostName)) {
  618
+                uassert(15914, "Failure retrying initial sync update", applyOperation_inlock(op));
  619
+            }
617 620
         }
618 621
         catch ( UserException& e ) {
619 622
             log() << "sync: caught user assertion " << e << " while applying op: " << op << endl;;
@@ -705,9 +708,9 @@ namespace mongo {
705 708
         }
706 709
 
707 710
         if ( !handleDuplicateDbName( op, ns, clientName ) ) {
708  
-            return;   
  711
+            return;
709 712
         }
710  
-                
  713
+
711 714
         Client::Context ctx( ns );
712 715
         ctx.getClient()->curop()->reset();
713 716
 
@@ -943,7 +946,7 @@ namespace mongo {
943 946
                         }
944 947
                         // otherwise, break out of loop so we can set to completed or clone more dbs
945 948
                     }
946  
-                    
  949
+
947 950
                     if( oplogReader.awaitCapable() && tailing )
948 951
                         okResultCode = 0; // don't sleep
949 952
                     syncedTo = nextOpTime;
@@ -1077,7 +1080,7 @@ namespace mongo {
1077 1080
 
1078 1081
         BSONObj me;
1079 1082
         {
1080  
-            
  1083
+
1081 1084
             dblock l;
1082 1085
             // local.me is an identifier for a server for getLastError w:2+
1083 1086
             if ( ! Helpers::getSingleton( "local.me" , me ) ||
@@ -1123,7 +1126,7 @@ namespace mongo {
1123 1126
         }
1124 1127
         return true;
1125 1128
     }
1126  
-    
  1129
+
1127 1130
     bool OplogReader::connect(string hostName) {
1128 1131
         if (conn() != 0) {
1129 1132
             return true;
4  db/repl.h
@@ -122,11 +122,11 @@ namespace mongo {
122 122
          * @return true iff an op with the specified ns may be applied.
123 123
          */
124 124
         bool handleDuplicateDbName( const BSONObj &op, const char *ns, const char *db );
125  
-        
  125
+
126 126
     public:
127 127
         OplogReader oplogReader;
128 128
 
129  
-        static void applyOperation(const BSONObj& op);
  129
+        void applyOperation(const BSONObj& op);
130 130
         string hostName;    // ip addr or hostname plus optionally, ":<port>"
131 131
         string _sourceName;  // a logical source name.
132 132
         string sourceName() const { return _sourceName.empty() ? "main" : _sourceName; }
61  db/repl/rs_sync.cpp
@@ -32,7 +32,7 @@ namespace mongo {
32 32
         }
33 33
     }
34 34
 
35  
-    /* apply the log op that is in param o 
  35
+    /* apply the log op that is in param o
36 36
        @return bool failedUpdate
37 37
     */
38 38
     bool ReplSetImpl::syncApply(const BSONObj &o) {
@@ -59,7 +59,7 @@ namespace mongo {
59 59
 
60 60
         const string hn = source->h().toString();
61 61
         OplogReader r;
62  
-        OplogReader missingObjReader;
  62
+
63 63
         try {
64 64
             if( !r.connect(hn) ) {
65 65
                 log() << "replSet initial sync error can't connect to " << hn << " to read " << rsoplog << rsLog;
@@ -135,45 +135,8 @@ namespace mongo {
135 135
 
136 136
                     if( ts >= applyGTE ) { // optimes before we started copying need not be applied.
137 137
                         bool failedUpdate = syncApply(o);
138  
-                        if( failedUpdate ) {
139  
-                            // we don't have the object yet, which is possible on initial sync.  get it.
140  
-                            log() << "replSet info adding missing object" << endl; // rare enough we can log
141  
-                            if( !missingObjReader.connect(hn) ) { // ok to call more than once
142  
-                                log() << "replSet initial sync fails, couldn't connect to " << hn << endl;
143  
-                                return false;
144  
-                            }
145  
-                            const char *ns = o.getStringField("ns");
146  
-                            BSONObj query = BSONObjBuilder().append(o.getObjectField("o2")["_id"]).obj(); // might be more than just _id in the update criteria
147  
-                            BSONObj missingObj;
148  
-                            try {
149  
-                                missingObj = missingObjReader.findOne(
150  
-                                    ns, 
151  
-                                    query );
152  
-                            } catch(...) { 
153  
-                                log() << "replSet assertion fetching missing object" << endl;
154  
-                                throw;
155  
-                            }
156  
-                            if( missingObj.isEmpty() ) { 
157  
-                                log() << "replSet missing object not found on source. presumably deleted later in oplog" << endl;
158  
-                                log() << "replSet o2: " << o.getObjectField("o2").toString() << endl;
159  
-                                log() << "replSet o firstfield: " << o.getObjectField("o").firstElementFieldName() << endl;
160  
-                            }
161  
-                            else {
162  
-                                Client::Context ctx(ns);
163  
-                                try {
164  
-                                    DiskLoc d = theDataFileMgr.insert(ns, (void*) missingObj.objdata(), missingObj.objsize());
165  
-                                    assert( !d.isNull() );
166  
-                                } catch(...) { 
167  
-                                    log() << "replSet assertion during insert of missing object" << endl;
168  
-                                    throw;
169  
-                                }
170  
-                                // now reapply the update from above
171  
-                                bool failed = syncApply(o);
172  
-                                if( failed ) {
173  
-                                    log() << "replSet update still fails after adding missing object " << ns << endl;
174  
-                                    assert(false);
175  
-                                }
176  
-                            }
  138
+                        if( failedUpdate && shouldRetry(o, hn)) {
  139
+                            uassert(15915, "replSet update still fails after adding missing object", syncApply(o));
177 140
                         }
178 141
                     }
179 142
                     _logOpObjRS(o);   /* with repl sets we write the ops to our oplog too */
@@ -200,7 +163,7 @@ namespace mongo {
200 163
                 if( e.getCode() == 11000 || e.getCode() == 11001 ) {
201 164
                     continue;
202 165
                 }
203  
-                
  166
+
204 167
                 // handle cursor not found (just requery)
205 168
                 if( e.getCode() == 13127 ) {
206 169
                     r.resetCursor();
@@ -333,7 +296,7 @@ namespace mongo {
333 296
                 target = 0;
334 297
             }
335 298
         }
336  
-            
  299
+
337 300
         // no server found
338 301
         if (target == 0) {
339 302
             // if there is no one to sync from
@@ -341,7 +304,7 @@ namespace mongo {
341 304
             tryToGoLiveAsASecondary(minvalid);
342 305
             return;
343 306
         }
344  
-        
  307
+
345 308
         r.tailingQueryGTE(rsoplog, lastOpTimeWritten);
346 309
         // if target cut connections between connecting and querying (for
347 310
         // example, because it stepped down) we might not have a cursor
@@ -451,7 +414,7 @@ namespace mongo {
451 414
                                     if( !target->hbinfo().hbstate.readable() ) {
452 415
                                         break;
453 416
                                     }
454  
-                                    
  417
+
455 418
                                     if( myConfig().slaveDelay != sd ) // reconf
456 419
                                         break;
457 420
                                 }
@@ -472,7 +435,7 @@ namespace mongo {
472 435
                         }
473 436
 
474 437
                         syncApply(o);
475  
-                        _logOpObjRS(o);   // with repl sets we write the ops to our oplog too 
  438
+                        _logOpObjRS(o);   // with repl sets we write the ops to our oplog too
476 439
                     }
477 440
                     catch (DBException& e) {
478 441
                         sethbmsg(str::stream() << "syncTail: " << e.toString() << ", syncing: " << o);
@@ -487,7 +450,7 @@ namespace mongo {
487 450
                 // TODO : reuse our connection to the primary.
488 451
                 return;
489 452
             }
490  
-            
  453
+
491 454
             if( !target->hbinfo().hbstate.readable() ) {
492 455
                 return;
493 456
             }
@@ -608,10 +571,10 @@ namespace mongo {
608 571
             OCCASIONALLY warning() << "couldn't update slave " << rid << " no entry" << rsLog;
609 572
             return;
610 573
         }
611  
-        
  574
+
612 575
         GhostSlave& slave = i->second;
613 576
         if (!slave.init) {
614  
-            OCCASIONALLY log() << "couldn't update slave " << rid << " not init" << rsLog;            
  577
+            OCCASIONALLY log() << "couldn't update slave " << rid << " not init" << rsLog;
615 578
             return;
616 579
         }
617 580
 

0 notes on commit 154f554

Please sign in to comment.
Something went wrong with that request. Please try again.