Permalink
Browse files

SERVER-4161 Report nscannedObjects properly when covered indexes are …

…used.
  • Loading branch information...
1 parent 3f02a78 commit 6e555988bff53d807635ef59796c45c824b770ba @astaple astaple committed Sep 24, 2012
View
@@ -18,9 +18,10 @@ assert.eq( 2, explain.n );
// nscanned and nscannedObjects are reported for the a:1,b:1 plan.
assert.eq( 2, explain.nscanned );
assert.eq( 2, explain.nscannedObjects );
-// nscannedAllPlans and nscannedObjectsAllPlans report the combined total of all plans.
+// nscannedAllPlans reports the combined total of all plans.
assert.eq( 6, explain.nscannedAllPlans );
-assert.eq( 6, explain.nscannedObjectsAllPlans );
+// nscannedObjectsAllPlans reports the total for the set of interleaved plans.
+assert.eq( 4, explain.nscannedObjectsAllPlans );
// A limit of 2.
explain = t.find( { a:{ $gte:0 }, b:{ $gte:0 } } ).limit( -2 ).explain( true );
@@ -40,20 +41,20 @@ explain = t.find( { $or:[ { a:{ $gte:0 }, b:{ $gte:1 } },
assert.eq( 1, explain.clauses[ 0 ].n );
assert.eq( 1, explain.clauses[ 0 ].nscannedObjects );
assert.eq( 2, explain.clauses[ 0 ].nscanned );
-assert.eq( 3, explain.clauses[ 0 ].nscannedObjectsAllPlans );
+assert.eq( 2, explain.clauses[ 0 ].nscannedObjectsAllPlans );
assert.eq( 4, explain.clauses[ 0 ].nscannedAllPlans );
assert.eq( 1, explain.clauses[ 1 ].n );
assert.eq( 1, explain.clauses[ 1 ].nscannedObjects );
assert.eq( 1, explain.clauses[ 1 ].nscanned );
-assert.eq( 3, explain.clauses[ 1 ].nscannedObjectsAllPlans );
+assert.eq( 2, explain.clauses[ 1 ].nscannedObjectsAllPlans );
assert.eq( 3, explain.clauses[ 1 ].nscannedAllPlans );
assert.eq( 2, explain.n );
// These are computed by summing the values for each clause.
assert.eq( 2, explain.n );
assert.eq( 2, explain.nscannedObjects );
assert.eq( 3, explain.nscanned );
-assert.eq( 6, explain.nscannedObjectsAllPlans );
+assert.eq( 4, explain.nscannedObjectsAllPlans );
assert.eq( 7, explain.nscannedAllPlans );
// A non $or case where nscanned != nscannedObjects.
@@ -70,5 +71,6 @@ assert.eq( 2, explain.nscanned );
assert.eq( 1, explain.nscannedObjects );
// Two results were scanned for each plan.
assert.eq( 6, explain.nscannedAllPlans );
-// The indexed plans each loaded one matching object, the unindexed plan loaded two.
-assert.eq( 4, explain.nscannedObjectsAllPlans );
+// One result was generated by { a:1, b:1 }. One result was matched by { b:1, a:1 } but it was a
+// dup. Two results were loaded for matching by the unindexed plan.
+assert.eq( 3, explain.nscannedObjectsAllPlans );
View
@@ -0,0 +1,195 @@
+// Test cases for explain()'s nscannedObjects. SERVER-4161
+
+t = db.jstests_explainc;
+t.drop();
+
+t.save( { a:1 } );
+t.ensureIndex( { a:1 } );
+
+function assertExplain( expected, explain, checkAllPlans ) {
+ for( field in expected ) {
+ assert.eq( expected[ field ], explain[ field ], field );
+ }
+ if ( checkAllPlans && explain.allPlans && explain.allPlans.length == 1 ) {
+ for( field in { n:1, nscanned:1, nscannedObjects:1 } ) {
+ assert.eq( expected[ field ], explain.allPlans[ 0 ][ field ], field );
+ }
+ }
+ return explain;
+}
+
+function assertHintedExplain( expected, cursor ) {
+ return assertExplain( expected, cursor.hint( { a:1 } ).explain( true ), true );
+}
+
+function assertUnhintedExplain( expected, cursor, checkAllPlans ) {
+ return assertExplain( expected, cursor.explain( true ), checkAllPlans );
+}
+
+// Standard query.
+assertHintedExplain( { n:1, nscanned:1, nscannedObjects:1 },
+ t.find( { a:1 } ) );
+
+// Covered index query.
+assertHintedExplain( { n:1, nscanned:1, nscannedObjects:0 /* no object loaded */ },
+ t.find( { a:1 }, { _id:0, a:1 } ) );
+
+// Covered index query, but matching requires loading document.
+assertHintedExplain( { n:1, nscanned:1, nscannedObjects:1 },
+ t.find( { a:1, b:null }, { _id:0, a:1 } ) );
+
+// $returnKey query.
+assertHintedExplain( { n:1, nscanned:1, nscannedObjects:0 },
+ t.find( { a:1 } )._addSpecial( "$returnKey", true ) );
+
+// $returnKey query but matching requires loading document.
+assertHintedExplain( { n:1, nscanned:1, nscannedObjects:1 },
+ t.find( { a:1, b:null } )._addSpecial( "$returnKey", true ) );
+
+// Skip a result.
+assertHintedExplain( { n:0, nscanned:1, nscannedObjects:0 },
+ t.find( { a:1 } ).skip( 1 ) );
+
+// Cursor sorted covered index query.
+assertHintedExplain( { n:1, nscanned:1, nscannedObjects:0, scanAndOrder:false },
+ t.find( { a:1 }, { _id:0, a:1 } ).sort( { a:1 } ) );
+
+t.dropIndex( { a:1 } );
+t.ensureIndex( { a:1, b:1 } );
+
+// In memory sort covered index query.
+assertUnhintedExplain( { n:1, nscanned:1, nscannedObjects:1, scanAndOrder:true },
+ t.find( { a:{ $gt:0 } }, { _id:0, a:1 } ).sort( { b:1 } )
+ .hint( { a:1, b:1 } ) );
+
+// In memory sort $returnKey query.
+assertUnhintedExplain( { n:1, nscanned:1, nscannedObjects:0, scanAndOrder:true },
+ t.find( { a:{ $gt:0 } } )._addSpecial( "$returnKey", true ).sort( { b:1 } )
+ .hint( { a:1, b:1 } ) );
+
+// In memory sort with skip.
+assertUnhintedExplain( { n:0, nscanned:1, nscannedObjects:1 /* The record is still loaded. */ },
+ t.find( { a:{ $gt:0 } } ).sort( { b:1 } ).skip( 1 ).hint( { a:1, b:1 } ),
+ false );
+
+// With a multikey index.
+t.drop();
+t.ensureIndex( { a:1 } );
+t.save( { a:[ 1, 2 ] } );
+
+// A multikey index with duplicate keys matched.
+assertHintedExplain( { n:1, nscanned:2, nscannedObjects:2, scanAndOrder:false },
+ t.find( { a:{ $gt:0 } }, { _id:0, a:1 } ) );
+
+// A multikey index with duplicate keys matched, and an in memory sort.
+assertHintedExplain( { n:1, nscanned:2, nscannedObjects:2, scanAndOrder:true },
+ t.find( { a:{ $gt:0 } }, { _id:0, a:1 } ).sort( { b:1 } ) );
+
+// Dedup matches from multiple query plans.
+t.drop();
+t.ensureIndex( { a:1, b:1 } );
+t.ensureIndex( { b:1, a:1 } );
+t.save( { a:1, b:1 } );
+
+// Document matched by three query plans.
+assertUnhintedExplain( { n:1, nscanned:1, nscannedObjects:1,
+ nscannedObjectsAllPlans:2 /* Result is not loaded if a dup. */ },
+ t.find( { a:{ $gt:0 }, b:{ $gt:0 } } ) );
+
+// Document matched by three query plans, with sorting.
+assertUnhintedExplain( { n:1, nscanned:1, nscannedObjects:1, nscannedObjectsAllPlans:2 },
+ t.find( { a:{ $gt:0 }, b:{ $gt:0 } } ).sort( { c:1 } ) );
+
+// Document matched by three query plans, with a skip.
+assertUnhintedExplain( { n:0, nscanned:1, nscannedObjects:1, nscannedObjectsAllPlans:1 },
+ t.find( { a:{ $gt:0 }, b:{ $gt:0 } } ).skip( 1 ) );
+
+// Hybrid ordered and unordered plans.
+
+t.drop();
+t.ensureIndex( { a:1, b:1 } );
+t.ensureIndex( { b:1 } );
+for( i = 0; i < 30; ++i ) {
+ t.save( { a:i, b:i } );
+}
+
+// Ordered plan chosen.
+assertUnhintedExplain( { cursor:'BtreeCursor a_1_b_1', n:30, nscanned:30, nscannedObjects:30,
+ scanAndOrder:false },
+ t.find( { b:{ $gte:0 } } ).sort( { a:1 } ) );
+
+// Ordered plan chosen with a covered index.
+assertUnhintedExplain( { cursor:'BtreeCursor a_1_b_1', n:30, nscanned:30, nscannedObjects:0,
+ scanAndOrder:false },
+ t.find( { b:{ $gte:0 } }, { _id:0, b:1 } ).sort( { a:1 } ) );
+
+// Ordered plan chosen, with a skip. Skip is not included in counting nscannedObjects for a single
+// plan.
+assertUnhintedExplain( { cursor:'BtreeCursor a_1_b_1', n:29, nscanned:30, nscannedObjects:30,
+ nscannedObjectsAllPlans:89, scanAndOrder:false },
+ t.find( { b:{ $gte:0 } } ).sort( { a:1 } ).skip( 1 ) );
+
+// Unordered plan chosen.
+assertUnhintedExplain( { cursor:'BtreeCursor b_1', n:1, nscanned:1, nscannedObjects:1,
+ nscannedObjectsAllPlans:2, scanAndOrder:true },
+ t.find( { b:1 } ).sort( { a:1 } ) );
+
+// Unordered plan chosen and projected.
+assertUnhintedExplain( { cursor:'BtreeCursor b_1', n:1, nscanned:1, nscannedObjects:1,
+ nscannedObjectsAllPlans:2, scanAndOrder:true },
+ t.find( { b:1 }, { _id:0, b:1 } ).sort( { a:1 } ) );
+
+// Unordered plan chosen, with a skip.
+assertUnhintedExplain( { cursor:'BtreeCursor b_1', n:0, nscanned:1, nscannedObjects:1,
+ nscannedObjectsAllPlans:2, scanAndOrder:true },
+ t.find( { b:1 }, { _id:0, b:1 } ).sort( { a:1 } ).skip( 1 ) );
+
+// Ordered plan chosen, $returnKey specified.
+assertUnhintedExplain( { cursor:'BtreeCursor a_1_b_1', n:30, nscanned:30, nscannedObjects:0,
+ scanAndOrder:false },
+ t.find( { b:{ $gte:0 } }, { _id:0, b:1 } ).sort( { a:1 } )
+ ._addSpecial( "$returnKey", true ) );
+
+// Ordered plan chosen, $returnKey specified, matching requires loading document.
+assertUnhintedExplain( { cursor:'BtreeCursor a_1_b_1', n:30, nscanned:30, nscannedObjects:30,
+ scanAndOrder:false },
+ t.find( { b:{ $gte:0 }, c:null }, { _id:0, b:1 } ).sort( { a:1 } )
+ ._addSpecial( "$returnKey", true ) );
+
+// Unordered plan chosen, $returnKey specified.
+assertUnhintedExplain( { cursor:'BtreeCursor b_1', n:1, nscanned:1, nscannedObjects:0,
+ nscannedObjectsAllPlans:1, scanAndOrder:true },
+ t.find( { b:1 }, { _id:0, b:1 } ).sort( { a:1 } )
+ ._addSpecial( "$returnKey", true ) );
+
+// Unordered plan chosen, $returnKey specified, matching requires loading document.
+assertUnhintedExplain( { cursor:'BtreeCursor b_1', n:1, nscanned:1, nscannedObjects:1,
+ nscannedObjectsAllPlans:2, scanAndOrder:true },
+ t.find( { b:1, c:null }, { _id:0, b:1 } ).sort( { a:1 } )
+ ._addSpecial( "$returnKey", true ) );
+
+t.ensureIndex( { a:1, b:1, c:1 } );
+
+// Documents matched by four query plans.
+assertUnhintedExplain( { n:30, nscanned:30, nscannedObjects:30,
+ nscannedObjectsAllPlans:90 // Not 120 because deduping occurs before
+ // loading results.
+ },
+ t.find( { a:{ $gte:0 }, b:{ $gte:0 } } ).sort( { b:1 } ) );
+
+for( i = 30; i < 150; ++i ) {
+ t.save( { a:i, b:i } );
+}
+
+// The matches in the second $or clause are loaded to dedup against the first clause.
+explain = assertUnhintedExplain( { n:150, nscannedObjects:150, nscannedObjectsAllPlans:150 },
+ t.find( { $or:[ { a:{ $gte:-1, $lte:200 },
+ b:{ $gte:0, $lte:201 } },
+ { a:{ $gte:0, $lte:201 },
+ b:{ $gte:-1, $lte:200 } } ] },
+ { _id:0, a:1, b:1 } ).hint( { a:1, b:1 } ) );
+// Check nscannedObjects for each clause.
+assert.eq( 0, explain.clauses[ 0 ].nscannedObjects );
+assert.eq( 0, explain.clauses[ 0 ].nscannedObjectsAllPlans );
+assert.eq( 150, explain.clauses[ 1 ].nscannedObjects );
+assert.eq( 150, explain.clauses[ 1 ].nscannedObjectsAllPlans );
@@ -0,0 +1,42 @@
+// Check explain() results reported for a sharded cluster, in particular nscannedObjects.
+// SERVER-4161
+
+s = new ShardingTest( "explain1" , 2 , 2 );
+
+// Tests can be invalidated by the balancer.
+s.stopBalancer()
+
+db = s.getDB( "test" );
+
+s.adminCommand( { enablesharding : "test" } );
+s.adminCommand( { shardcollection : "test.foo" , key : { _id : 1 } } );
+
+t = db.foo;
+for( i = 0; i < 10; ++i ) {
+ t.save( { a:i } );
+}
+
+// Without an index.
+explain = t.find( { a:{ $gte:5 } } ).explain();
+assert.eq( explain.cursor, 'BasicCursor' );
+assert.eq( explain.n, 5 );
+assert.eq( explain.nscanned, 10 );
+assert.eq( explain.nscannedObjects, 10 );
+
+// With an index.
+t.ensureIndex( { a:1 } );
+explain = t.find( { a:{ $gte:5 } } ).explain();
+assert.eq( explain.cursor, 'BtreeCursor a_1' );
+assert.eq( explain.n, 5 );
+assert.eq( explain.nscanned, 5 );
+assert.eq( explain.nscannedObjects, 5 );
+
+// With a covered index.
+t.ensureIndex( { a:1 } );
+explain = t.find( { a:{ $gte:5 } }, { _id:0, a:1 } ).explain();
+assert.eq( explain.cursor, 'BtreeCursor a_1' );
+assert.eq( explain.n, 5 );
+assert.eq( explain.nscanned, 5 );
+assert.eq( explain.nscannedObjects, 5 ); // Covered indexes do not work with sharding.
+
+s.stop();
@@ -313,7 +313,8 @@ namespace mongo {
_buf.skip( sizeof( QueryResult ) );
}
- BSONObj ResponseBuildStrategy::current( bool allowCovered ) const {
+ BSONObj ResponseBuildStrategy::current( bool allowCovered,
+ ResultDetails* resultDetails ) const {
if ( _parsedQuery.returnKey() ) {
BSONObjBuilder bob;
bob.appendKeys( _cursor->indexKeyPattern(), _cursor->currKey() );
@@ -325,6 +326,7 @@ namespace mongo {
return keyFieldsOnly->hydrate( _cursor->currKey() );
}
}
+ resultDetails->loadedRecord = true;
BSONObj ret = _cursor->current();
verify( ret.isValid() );
return ret;
@@ -347,10 +349,11 @@ namespace mongo {
--_skip;
return false;
}
+ BSONObj currentDocument = current( true, resultDetails );
// Explain does not obey soft limits, so matches should not be buffered.
if ( !_parsedQuery.isExplain() ) {
fillQueryResultFromObj( _buf, _parsedQuery.getFields(),
- current( true ), &resultDetails->matchDetails,
+ currentDocument, &resultDetails->matchDetails,
( _parsedQuery.showDiskLoc() ? &loc : 0 ) );
++_bufferedMatches;
}
@@ -383,14 +386,15 @@ namespace mongo {
if ( _cursor->getsetdup( _cursor->currLoc() ) ) {
return false;
}
- _handleMatchNoDedup();
+ _handleMatchNoDedup( resultDetails );
resultDetails->match = true;
return true;
}
- void ReorderBuildStrategy::_handleMatchNoDedup() {
+ void ReorderBuildStrategy::_handleMatchNoDedup( ResultDetails* resultDetails ) {
DiskLoc loc = _cursor->currLoc();
- _scanAndOrder->add( current( false ), _parsedQuery.showDiskLoc() ? &loc : 0 );
+ _scanAndOrder->add( current( false, resultDetails ),
+ _parsedQuery.showDiskLoc() ? &loc : 0 );
}
int ReorderBuildStrategy::rewriteMatches() {
@@ -455,7 +459,7 @@ namespace mongo {
}
resultDetails->match = true;
try {
- _reorderBuild->_handleMatchNoDedup();
+ _reorderBuild->_handleMatchNoDedup( resultDetails );
} catch ( const UserException &e ) {
if ( e.getCode() == ScanAndOrderMemoryLimitExceededAssertionCode ) {
if ( _queryOptimizerCursor->hasPossiblyExcludedPlans() ) {
@@ -523,18 +527,11 @@ namespace mongo {
resultDetails.matchDetails.requestElemMatchKey();
}
- if ( !currentMatches( &resultDetails ) ) {
- resultDetails.loadedRecord = resultDetails.matchDetails.hasLoadedRecord();
- _explain->noteIterate( resultDetails );
- return false;
- }
- if ( !chunkMatches( &resultDetails ) ) {
- resultDetails.loadedRecord = true;
- _explain->noteIterate( resultDetails );
- return false;
- }
- bool match = _builder->handleMatch( &resultDetails );
- resultDetails.loadedRecord = true;
+ bool match =
+ currentMatches( &resultDetails ) &&
+ chunkMatches( &resultDetails ) &&
+ _builder->handleMatch( &resultDetails );
+
_explain->noteIterate( resultDetails );
return match;
}
@@ -632,17 +629,19 @@ namespace mongo {
}
bool QueryResponseBuilder::currentMatches( ResultDetails* resultDetails ) {
- if ( _cursor->currentMatches( &resultDetails->matchDetails ) ) {
- return true;
+ bool matches = _cursor->currentMatches( &resultDetails->matchDetails );
+ if ( resultDetails->matchDetails.hasLoadedRecord() ) {
+ resultDetails->loadedRecord = true;
}
- return false;
+ return matches;
}
bool QueryResponseBuilder::chunkMatches( ResultDetails* resultDetails ) {
if ( !_chunkManager ) {
return true;
}
// TODO: should make this covered at some point
+ resultDetails->loadedRecord = true;
if ( _chunkManager->belongsToMe( _cursor->current() ) ) {
return true;
}
Oops, something went wrong.

0 comments on commit 6e55598

Please sign in to comment.