Permalink
Browse files

SERVER-1528 account for expensive skipping operations in nscanned

  • Loading branch information...
1 parent 5240b2c commit 701dc65282b7e154aca20ae1882ae7962c11e0b2 @astaple astaple committed Aug 4, 2010
Showing with 197 additions and 73 deletions.
  1. +3 −0 db/btree.h
  2. +17 −6 db/btreecursor.cpp
  3. +4 −1 db/cursor.cpp
  4. +9 −2 db/cursor.h
  5. +4 −0 db/dbhelpers.cpp
  6. +33 −7 db/geo/2d.cpp
  7. +23 −6 db/query.cpp
  8. +33 −27 db/queryoptimizer.cpp
  9. +11 −2 db/queryoptimizer.h
  10. +5 −1 db/update.cpp
  11. +7 −2 dbtests/queryoptimizertests.cpp
  12. +1 −1 jstests/explain2.js
  13. +1 −1 jstests/in3.js
  14. +1 −1 jstests/in4.js
  15. +29 −16 jstests/index_check6.js
  16. +16 −0 jstests/indexi.js
View
@@ -364,6 +364,8 @@ namespace mongo {
_matcher = matcher;
}
+ virtual long long nscanned() { return _nscanned; }
+
// for debugging only
DiskLoc getBucket() const { return bucket; }
@@ -408,6 +410,7 @@ namespace mongo {
const IndexSpec& _spec;
shared_ptr< CoveredIndexMatcher > _matcher;
bool _independentFieldRanges;
+ long long _nscanned;
};
View
@@ -38,7 +38,8 @@ namespace mongo {
_ordering( Ordering::make( order ) ),
direction( _direction ),
_spec( _id.getSpec() ),
- _independentFieldRanges( false )
+ _independentFieldRanges( false ),
+ _nscanned( 0 )
{
audit();
init();
@@ -57,7 +58,8 @@ namespace mongo {
bounds_( ( assert( _bounds.get() ), _bounds ) ),
_boundsIterator( new FieldRangeVector::Iterator( *bounds_ ) ),
_spec( _id.getSpec() ),
- _independentFieldRanges( true )
+ _independentFieldRanges( true ),
+ _nscanned( 0 )
{
massert( 13384, "BtreeCursor FieldRangeVector constructor doesn't accept special indexes", !_spec.getType() );
audit();
@@ -93,6 +95,9 @@ namespace mongo {
bool found;
bucket = indexDetails.head.btree()->
locate(indexDetails, indexDetails.head, startKey, _ordering, keyOfs, found, direction > 0 ? minDiskLoc : maxDiskLoc, direction);
+ if ( ok() ) {
+ _nscanned = 1;
+ }
skipUnusedKeys( false );
checkEnd();
}
@@ -119,8 +124,10 @@ namespace mongo {
bucket = DiskLoc();
return false;
} else if ( ret == -1 ) {
+ ++_nscanned;
return false;
}
+ ++_nscanned;
advanceTo( currKeyNode().key, ret, _boundsIterator->cmp() );
return true;
}
@@ -137,6 +144,8 @@ namespace mongo {
break;
bucket = b->advance(bucket, keyOfs, direction, "skipUnusedKeys");
u++;
+ //don't include unused keys in nscanned
+ //++_nscanned;
if ( mayJump && ( u % 10 == 0 ) ) {
skipOutOfRangeKeysAndCheckEnd();
}
@@ -175,14 +184,16 @@ namespace mongo {
return false;
bucket = bucket.btree()->advance(bucket, keyOfs, direction, "BtreeCursor::advance");
-
+
if ( !_independentFieldRanges ) {
skipUnusedKeys( false );
checkEnd();
- return ok();
+ if ( ok() ) {
+ ++_nscanned;
+ }
+ } else {
+ skipAndCheck();
}
-
- skipAndCheck();
return ok();
}
View
@@ -24,14 +24,15 @@ namespace mongo {
killCurrentOp.checkForInterrupt();
if ( eof() ) {
if ( tailable_ && !last.isNull() ) {
- curr = s->next( last );
+ curr = s->next( last );
} else {
return false;
}
} else {
last = curr;
curr = s->next( curr );
}
+ incNscanned();
return ok();
}
@@ -89,6 +90,7 @@ namespace mongo {
}
curr = start;
s = this;
+ incNscanned();
}
DiskLoc ForwardCappedCursor::next( const DiskLoc &prev ) const {
@@ -125,6 +127,7 @@ namespace mongo {
}
curr = start;
s = this;
+ incNscanned();
}
DiskLoc ReverseCappedCursor::next( const DiskLoc &prev ) const {
View
@@ -96,6 +96,8 @@ namespace mongo {
virtual bool capped() const { return false; }
+ virtual long long nscanned() = 0;
+
// The implementation may return different matchers depending on the
// position of the cursor. If matcher() is nonzero at the start,
// matcher() should be checked each time advance() is called.
@@ -124,10 +126,12 @@ namespace mongo {
protected:
DiskLoc curr, last;
const AdvanceStrategy *s;
+ void incNscanned() { if ( !curr.isNull() ) { ++_nscanned; } }
private:
bool tailable_;
shared_ptr< CoveredIndexMatcher > _matcher;
+ long long _nscanned;
void init() {
tailable_ = false;
}
@@ -153,10 +157,11 @@ namespace mongo {
bool advance();
- BasicCursor(DiskLoc dl, const AdvanceStrategy *_s = forward()) : curr(dl), s( _s ) {
+ BasicCursor(DiskLoc dl, const AdvanceStrategy *_s = forward()) : curr(dl), s( _s ), _nscanned() {
+ incNscanned();
init();
}
- BasicCursor(const AdvanceStrategy *_s = forward()) : s( _s ) {
+ BasicCursor(const AdvanceStrategy *_s = forward()) : s( _s ), _nscanned() {
init();
}
virtual string toString() {
@@ -180,6 +185,8 @@ namespace mongo {
_matcher = matcher;
}
+ virtual long long nscanned() { return _nscanned; }
+
};
/* used for order { $natural: -1 } */
View
@@ -114,6 +114,10 @@ namespace mongo {
c_->advance();
}
}
+ virtual long long nscanned() {
+ assert( c_.get() );
+ return c_->nscanned();
+ }
virtual bool mayRecordPlan() const { return false; }
virtual QueryOp *_createChild() const { return new FindOne( requireIndex_ ); }
BSONObj one() const { return one_; }
View
@@ -976,7 +976,10 @@ namespace mongo {
public:
GeoSearchCursor( shared_ptr<GeoSearch> s )
: GeoCursorBase( s->_spec ) ,
- _s( s ) , _cur( s->_hopper->_points.begin() ) , _end( s->_hopper->_points.end() ) {
+ _s( s ) , _cur( s->_hopper->_points.begin() ) , _end( s->_hopper->_points.end() ), _nscanned() {
+ if ( _cur != _end ) {
+ ++_nscanned;
+ }
}
virtual ~GeoSearchCursor() {}
@@ -988,7 +991,7 @@ namespace mongo {
virtual Record* _current(){ assert(ok()); return _cur->_loc.rec(); }
virtual BSONObj current(){ assert(ok()); return _cur->_o; }
virtual DiskLoc currLoc(){ assert(ok()); return _cur->_loc; }
- virtual bool advance(){ _cur++; return ok(); }
+ virtual bool advance(){ _cur++; incNscanned(); return ok(); }
virtual BSONObj currKey() const { return _cur->_key; }
virtual string toString() {
@@ -1004,36 +1007,49 @@ namespace mongo {
temp.move( 1 , 1 );
return BSON( _s->_spec->_geo << temp.toString() );
}
-
+
+ virtual long long nscanned() { return _nscanned; }
shared_ptr<GeoSearch> _s;
GeoHopper::Holder::iterator _cur;
GeoHopper::Holder::iterator _end;
+
+ void incNscanned() { if ( ok() ) { ++_nscanned; } }
+ long long _nscanned;
};
class GeoBrowse : public GeoCursorBase , public GeoAccumulator {
public:
GeoBrowse( const Geo2dType * g , string type , BSONObj filter = BSONObj() )
: GeoCursorBase( g ) ,GeoAccumulator( g , filter ) ,
- _type( type ) , _filter( filter ) , _firstCall(true) {
+ _type( type ) , _filter( filter ) , _firstCall(true), _nscanned() {
}
virtual string toString() {
return (string)"GeoBrowse-" + _type;
}
virtual bool ok(){
+ bool first = _firstCall;
if ( _firstCall ){
fillStack();
_firstCall = false;
}
- if ( ! _cur.isEmpty() || _stack.size() )
+ if ( ! _cur.isEmpty() || _stack.size() ) {
+ if ( first ) {
+ ++_nscanned;
+ }
return true;
+ }
while ( moreToDo() ){
fillStack();
- if ( ! _cur.isEmpty() )
+ if ( ! _cur.isEmpty() ) {
+ if ( first ) {
+ ++_nscanned;
+ }
return true;
+ }
}
return false;
@@ -1045,6 +1061,7 @@ namespace mongo {
if ( _stack.size() ){
_cur = _stack.front();
_stack.pop_front();
+ ++_nscanned;
return true;
}
@@ -1053,7 +1070,7 @@ namespace mongo {
while ( _cur.isEmpty() && moreToDo() )
fillStack();
- return ! _cur.isEmpty();
+ return ! _cur.isEmpty() && ++_nscanned;
}
virtual Record* _current(){ assert(ok()); return _cur._loc.rec(); }
@@ -1072,12 +1089,21 @@ namespace mongo {
_stack.push_back( GeoPoint( node , d ) );
}
+ virtual long long nscanned() {
+ if ( _firstCall ) {
+ ok();
+ }
+ return _nscanned;
+ }
+
string _type;
BSONObj _filter;
list<GeoPoint> _stack;
GeoPoint _cur;
bool _firstCall;
+
+ long long _nscanned;
};
View
@@ -75,6 +75,10 @@ namespace mongo {
massert( 13340, "cursor dropped during delete", false );
}
}
+ virtual long long nscanned() {
+ assert( c_.get() );
+ return c_->nscanned();
+ }
virtual void next() {
if ( !c_->ok() ) {
setComplete();
@@ -89,7 +93,7 @@ namespace mongo {
}
c_->advance();
- ++_nscanned;
+ _nscanned = c_->nscanned();
if ( count_ > bestCount_ )
bestCount_ = count_;
@@ -409,6 +413,11 @@ namespace mongo {
}
}
+ virtual long long nscanned() {
+ assert( c_.get() );
+ return c_->nscanned();
+ }
+
virtual bool prepareToYield() {
if ( ! _cc ) {
_cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , c_ , _ns.c_str() ) );
@@ -657,6 +666,14 @@ namespace mongo {
}
}
+ virtual long long nscanned() {
+ if ( _findingStartCursor.get() ) {
+ return 0; // should only be one query plan, so value doesn't really matter.
+ }
+ assert( _c.get() );
+ return _c->nscanned();
+ }
+
virtual void next() {
if ( _findingStartCursor.get() ) {
if ( _findingStartCursor->done() ) {
@@ -684,7 +701,7 @@ namespace mongo {
return;
}
- _nscanned++;
+ _nscanned = _c->nscanned();
if ( !matcher()->matches(_c->currKey(), _c->currLoc() , &_details ) ) {
// not a match, continue onward
if ( _details.loadedObject )
@@ -794,7 +811,7 @@ namespace mongo {
}
void finishExplain( const BSONObj &suffix ) {
- BSONObj obj = _eb.finishWithSuffix( nscanned(), nscannedObjects(), n(), _curop.elapsedMillis(), suffix);
+ BSONObj obj = _eb.finishWithSuffix( totalNscanned(), nscannedObjects(), n(), _curop.elapsedMillis(), suffix);
fillQueryResultFromObj(_buf, 0, obj);
_n = 1;
_oldN = 0;
@@ -810,7 +827,7 @@ namespace mongo {
}
UserQueryOp *ret = new UserQueryOp( _pq, _response, _eb, _curop );
ret->_oldN = n();
- ret->_oldNscanned = nscanned();
+ ret->_oldNscanned = totalNscanned();
ret->_oldNscannedObjects = nscannedObjects();
ret->_ntoskip = _ntoskip;
return ret;
@@ -819,7 +836,7 @@ namespace mongo {
bool scanAndOrderRequired() const { return _inMemSort; }
shared_ptr<Cursor> cursor() { return _c; }
int n() const { return _oldN + _n; }
- long long nscanned() const { return _nscanned + _oldNscanned; }
+ long long totalNscanned() const { return _nscanned + _oldNscanned; }
long long nscannedObjects() const { return _nscannedObjects + _oldNscannedObjects; }
bool saveClientCursor() const { return _saveClientCursor; }
bool wouldSaveClientCursor() const { return _wouldSaveClientCursor; }
@@ -1025,7 +1042,7 @@ namespace mongo {
dqo.finishExplain( explainSuffix );
}
n = dqo.n();
- long long nscanned = dqo.nscanned();
+ long long nscanned = dqo.totalNscanned();
if ( dqo.scanAndOrderRequired() )
ss << " scanAndOrder ";
shared_ptr<Cursor> cursor = dqo.cursor();
Oops, something went wrong.

0 comments on commit 701dc65

Please sign in to comment.