Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

SERVER-4148 simplify count implementation and rewrite using query opt…

…imizer cursor
  • Loading branch information...
commit 46d49e316d37667c9a500d19b91353700ca37c95 1 parent 53ba6ca
@astaple astaple authored
View
2  db/btree.h
@@ -1064,8 +1064,6 @@ namespace mongo {
virtual BSONObj prettyIndexBounds() const;
- void forgetEndKey() { endKey = BSONObj(); }
-
virtual CoveredIndexMatcher *matcher() const { return _matcher.get(); }
virtual shared_ptr< CoveredIndexMatcher > matcherPtr() const { return _matcher; }
View
197 db/ops/query.cpp
@@ -220,140 +220,6 @@ namespace mongo {
return qr;
}
- class CountOp : public QueryOp {
- public:
- CountOp( const string& ns , const BSONObj &spec ) :
- _ns(ns), _capped(false), _count(), _myCount(),
- _skip( spec["skip"].numberLong() ),
- _limit( spec["limit"].numberLong() ),
- _nscanned(),
- _bc(),
- _yieldRecoveryFailed() {
- }
-
- virtual void _init() {
- _c = qp().newCursor();
- _capped = _c->capped();
- if ( qp().exactKeyMatch() && ! matcher( _c )->needRecord() ) {
- _query = qp().simplifiedQuery( qp().indexKey() );
- _bc = dynamic_cast< BtreeCursor* >( _c.get() );
- _bc->forgetEndKey();
- }
- }
-
- virtual long long nscanned() {
- return _c.get() ? _c->nscanned() : _nscanned;
- }
-
- virtual bool prepareToYield() {
- if ( _c && !_cc ) {
- _cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , _c , _ns.c_str() ) );
- }
- if ( _cc ) {
- return _cc->prepareToYield( _yieldData );
- }
- // no active cursor - ok to yield
- return true;
- }
-
- virtual void recoverFromYield() {
- if ( _cc && !ClientCursor::recoverFromYield( _yieldData ) ) {
- _yieldRecoveryFailed = true;
- _c.reset();
- _cc.reset();
-
- if ( _capped ) {
- msgassertedNoTrace( 13337, str::stream() << "capped cursor overrun during count: " << _ns );
- }
- else if ( qp().mustAssertOnYieldFailure() ) {
- msgassertedNoTrace( 15891, str::stream() << "CountOp::recoverFromYield() failed to recover: " << _ns );
- }
- else {
- // we don't fail query since we're fine with returning partial data if collection dropped
- }
- }
- }
-
- virtual void next() {
- if ( ! _c || !_c->ok() ) {
- setComplete();
- return;
- }
-
- _nscanned = _c->nscanned();
- if ( _bc ) {
- if ( _firstMatch.isEmpty() ) {
- _firstMatch = _bc->currKey().getOwned();
- // if not match
- if ( _query.woCompare( _firstMatch, BSONObj(), false ) ) {
- setComplete();
- return;
- }
- _gotOne();
- }
- else {
- if ( ! _firstMatch.equal( _bc->currKey() ) ) {
- setComplete();
- return;
- }
- _gotOne();
- }
- }
- else {
- if ( !matcher( _c )->matchesCurrent( _c.get() ) ) {
- }
- else if( !_c->getsetdup(_c->currLoc()) ) {
- _gotOne();
- }
- }
- _c->advance();
- }
- virtual QueryOp *_createChild() const {
- CountOp *ret = new CountOp( _ns , BSONObj() );
- ret->_count = _count;
- ret->_skip = _skip;
- ret->_limit = _limit;
- return ret;
- }
- long long count() const { return _count; }
- virtual bool mayRecordPlan() const {
- return !_yieldRecoveryFailed && ( ( _myCount > _limit / 2 ) || ( complete() && !stopRequested() ) );
- }
- private:
-
- void _gotOne() {
- if ( _skip ) {
- _skip--;
- return;
- }
-
- if ( _limit > 0 && _count >= _limit ) {
- setStop();
- return;
- }
-
- _count++;
- _myCount++;
- }
-
- string _ns;
- bool _capped;
-
- long long _count;
- long long _myCount;
- long long _skip;
- long long _limit;
- long long _nscanned;
- shared_ptr<Cursor> _c;
- BSONObj _query;
- BtreeCursor * _bc;
- BSONObj _firstMatch;
-
- ClientCursor::CleanupPointer _cc;
- ClientCursor::YieldData _yieldData;
- bool _yieldRecoveryFailed;
- };
-
/* { count: "collectionname"[, query: <query>] }
returns -1 on ns does not exist error.
*/
@@ -370,16 +236,59 @@ namespace mongo {
if ( query.isEmpty() ) {
return applySkipLimit( d->stats.nrecords , cmd );
}
- MultiPlanScanner mps( ns, query, BSONObj(), 0, true, BSONObj(), BSONObj(), false, true );
- CountOp original( ns , cmd );
- shared_ptr< CountOp > res = mps.runOp( original );
- if ( !res->complete() ) {
- log() << "Count with ns: " << ns << " and query: " << query
- << " failed with exception: " << res->exception()
- << endl;
- return 0;
- }
- return res->count();
+
+ string exceptionInfo;
+ long long count = 0;
+ long long skip = cmd["skip"].numberLong();
+ long long limit = cmd["limit"].numberLong();
+ bool simpleEqualityMatch;
+ shared_ptr<Cursor> cursor = NamespaceDetailsTransient::getCursor( ns, query, BSONObj(), false, &simpleEqualityMatch );
+ ClientCursor::CleanupPointer ccPointer;
+ ccPointer.reset( new ClientCursor( QueryOption_NoCursorTimeout, cursor, ns ) );
+ try {
+ while( cursor->ok() ) {
+ if ( !ccPointer->yieldSometimes( simpleEqualityMatch ? ClientCursor::DontNeed : ClientCursor::MaybeCovered ) ||
+ !cursor->ok() ) {
+ break;
+ }
+
+ // With simple equality matching there is no need to use the matcher because the bounds
+ // are enforced by the FieldRangeVectorIterator and only key fields have constraints. There
+ // is no need do key deduping because an exact value is specified in the query for all key
+ // fields and duplicate keys are not allowed per document.
+ // NOTE In the distant past we used a min/max bounded BtreeCursor with a shallow
+ // equality comparison to check for matches in the simple match case. That may be
+ // more performant, but I don't think we've measured the performance.
+ if ( simpleEqualityMatch ||
+ ( ( !cursor->matcher() || cursor->matcher()->matchesCurrent( cursor.get() ) ) &&
+ !cursor->getsetdup( cursor->currLoc() ) ) ) {
+
+ if ( skip > 0 ) {
+ --skip;
+ }
+ else {
+ ++count;
+ if ( limit > 0 && count >= limit ) {
+ break;
+ }
+ }
+ }
+ cursor->advance();
+ }
+ ccPointer.reset();
+ return count;
+
+ } catch ( const DBException &e ) {
+ exceptionInfo = e.toString();
+ } catch ( const std::exception &e ) {
+ exceptionInfo = e.what();
+ } catch ( ... ) {
+ exceptionInfo = "unknown exception";
+ }
+ log() << "Count with ns: " << ns << " and query: " << query
+ << " failed with exception: " << exceptionInfo
+ << endl;
+ return 0;
}
class ExplainBuilder {
View
42 jstests/count6.js
@@ -0,0 +1,42 @@
+// Some correctness checks for fast and normal count modes, including with skip and limit.
+
+t = db.jstests_count6;
+
+function checkCountForObject( obj ) {
+ t.drop();
+ t.ensureIndex( {b:1,a:1} );
+
+ function checkCounts( query, expected ) {
+ assert.eq( expected, t.count( query ) );
+ assert.eq( expected, t.find( query ).skip( 0 ).limit( 0 ).count( true ) );
+ // Check proper counts with various skip and limit specs.
+ for( var skip = 1; skip <= 2; ++skip ) {
+ for( var limit = 1; limit <= 2; ++limit ) {
+ assert.eq( Math.max( expected - skip, 0 ), t.find( query ).skip( skip ).count( true ) );
+ assert.eq( Math.min( expected, limit ), t.find( query ).limit( limit ).count( true ) );
+ assert.eq( Math.min( Math.max( expected - skip, 0 ), limit ), t.find( query ).skip( skip ).limit( limit ).count( true ) );
+ }
+ }
+ }
+
+ for( var i = 0; i < 5; ++i ) {
+ checkCounts( {a:obj.a,b:obj.b}, i );
+ checkCounts( {b:obj.b,a:obj.a}, i );
+ t.insert( obj );
+ }
+
+ t.insert( {a:true,b:true} );
+ t.insert( {a:true,b:1} );
+ t.insert( {a:false,b:1} );
+ t.insert( {a:false,b:true} );
+ t.insert( {a:false,b:false} );
+
+ checkCounts( {a:obj.a,b:obj.b}, i );
+ checkCounts( {b:obj.b,a:obj.a}, i );
+}
+
+// Check fast count mode.
+checkCountForObject( {a:true,b:false} );
+
+// Check normal count mode.
+checkCountForObject( {a:1,b:0} );
View
25 jstests/count7.js
@@ -0,0 +1,25 @@
+// Check normal count matching and deduping.
+
+t = db.jstests_count7;
+t.drop();
+
+t.ensureIndex( {a:1} );
+t.save( {a:'algebra'} );
+t.save( {a:'apple'} );
+t.save( {a:'azores'} );
+t.save( {a:'bumper'} );
+t.save( {a:'supper'} );
+t.save( {a:'termite'} );
+t.save( {a:'zeppelin'} );
+t.save( {a:'ziggurat'} );
+t.save( {a:'zope'} );
+
+assert.eq( 5, t.count( {a:/p/} ) );
+
+t.remove();
+
+t.save( {a:[1,2,3]} );
+t.save( {a:[1,2,3]} );
+t.save( {a:[1]} );
+
+assert.eq( 2, t.count( {a:{$gt:1}} ) );
View
48 jstests/count8.js
@@ -0,0 +1,48 @@
+// Test count yielding, in both fast and normal count modes.
+
+t = db.jstests_count8;
+t.drop();
+
+function checkYield( dropCollection, fastCount, query ) {
+
+ obj = fastCount ? {a:true} : {a:1};
+ query = query || obj;
+
+ passed = false;
+ for( nDocs = 20000; nDocs < 100000000; nDocs *= 2 ) {
+
+ t.drop();
+ t.ensureIndex( {a:1} );
+ for( i = 0; i < nDocs; ++i ) {
+ t.insert( obj );
+ }
+ db.getLastError();
+
+ if ( dropCollection ) {
+ p = startParallelShell( 'sleep( 100 ); db.jstests_count8.drop(); db.getLastError();' );
+ } else {
+ p = startParallelShell( 'sleep( 100 ); db.jstests_count8.update( {$atomic:true}, {$set:{a:-1}}, false, true ); db.getLastError();' );
+ }
+
+ printjson( query );
+ count = t.count( query );
+ // Changing documents or dropping collection changes the return value of count,
+ // indicating that a yield occurred.
+ print( 'count: ' + count + ', nDocs: ' + nDocs );
+ if ( count < nDocs ) {
+ passed = true;
+ p();
+ break;
+ }
+
+ p();
+ }
+
+ assert( passed );
+}
+
+checkYield( true, false );
+checkYield( false, false );
+checkYield( true, true );
+checkYield( false, true );
+checkYield( true, false, {$or:[{a:1},{a:2}]} );
View
28 jstests/count9.js
@@ -0,0 +1,28 @@
+// Test fast mode count with multikey entries.
+
+t = db.jstests_count9;
+t.drop();
+
+t.ensureIndex( {a:1} );
+
+t.save( {a:['a','b','a']} );
+assert.eq( 1, t.count( {a:'a'} ) );
+
+t.save( {a:['a','b','a']} );
+assert.eq( 2, t.count( {a:'a'} ) );
+
+t.drop();
+t.ensureIndex( {a:1,b:1} );
+
+t.save( {a:['a','b','a'],b:'r'} );
+assert.eq( 1, t.count( {a:'a',b:'r'} ) );
+assert.eq( 1, t.count( {a:'a'} ) );
+
+t.save( {a:['a','b','a'],b:'r'} );
+assert.eq( 2, t.count( {a:'a',b:'r'} ) );
+assert.eq( 2, t.count( {a:'a'} ) );
+
+t.drop();
+t.ensureIndex( {'a.b':1,'a.c':1} );
+t.save( {a:[{b:'b',c:'c'},{b:'b',c:'c'}]} );
+assert.eq( 1, t.count( {'a.b':'b','a.c':'c'} ) );
View
11 jstests/counta.js
@@ -0,0 +1,11 @@
+// Check that count returns 0 in some exception cases.
+
+t = db.jstests_counta;
+t.drop();
+
+for( i = 0; i < 10; ++i ) {
+ t.save( {a:i} );
+}
+
+// f() is undefined, causing an assertion which results in count returning 0 rather than the running total already counted.
+assert.eq( 0, t.count( { $where:function() { if ( this.a < 5 ) { return true; } else { f(); } } } ) );
Please sign in to comment.
Something went wrong with that request. Please try again.