Permalink
Browse files

SERVER-4150 checkpoint finish initial version of QueryResponseBuilder…

… handling in order and scan and order plans
  • Loading branch information...
1 parent cbc14d8 commit eac5c56ed64fe98e4c77719a8f57d42cd5cb2632 @astaple astaple committed Jan 23, 2012
View
@@ -6,17 +6,24 @@ t.drop();
t.save( {a:1} );
t.save( {a:2} );
-function checkA( a, sort, skip ) {
- assert.eq( a, t.find().sort( sort ).skip( skip )[ 0 ].a );
+function checkA( a, sort, skip, query ) {
+ query = query || {};
+ assert.eq( a, t.find( query ).sort( sort ).skip( skip )[ 0 ].a );
}
function checkSortAndSkip() {
checkA( 1, {a:1}, 0 );
checkA( 2, {a:1}, 1 );
+ checkA( 1, {a:1}, 0, {a:{$gt:0},b:null} );
+ checkA( 2, {a:1}, 1, {a:{$gt:0},b:null} );
+
checkA( 2, {a:-1}, 0 );
checkA( 1, {a:-1}, 1 );
+ checkA( 2, {a:-1}, 0, {a:{$gt:0},b:null} );
+ checkA( 1, {a:-1}, 1, {a:{$gt:0},b:null} );
+
checkA( 1, {$natural:1}, 0 );
checkA( 2, {$natural:1}, 1 );
View
@@ -0,0 +1,70 @@
+// Test sorting with dups and multiple candidate query plans.
+
+t = db.jstests_sortd;
+
+function checkNumSorted( n, query ) {
+ docs = query.toArray();
+ assert.eq( n, docs.length );
+ for( i = 1; i < docs.length; ++i ) {
+ assert.lte( docs[ i-1 ].a, docs[ i ].a );
+ }
+}
+
+
+// Test results added by ordered and unordered plans, unordered plan finishes.
+
+t.drop();
+
+t.save( {a:[1,2,3,4,5]} );
+t.save( {a:10} );
+t.ensureIndex( {a:1} );
+
+assert.eq( 2, t.find( {a:{$gt:0}} ).sort( {a:1} ).itcount() );
+assert.eq( 2, t.find( {a:{$gt:0},b:null} ).sort( {a:1} ).itcount() );
+
+// Test results added by ordered and unordered plans, ordered plan finishes.
+
+t.drop();
+
+t.save( {a:1} );
+t.save( {a:10} );
+for( i = 2; i <= 9; ++i ) {
+ t.save( {a:i} );
+}
+for( i = 0; i < 30; ++i ) {
+ t.save( {a:100} );
+}
+t.ensureIndex( {a:1} );
+
+checkNumSorted( 10, t.find( {a:{$gte:0,$lte:10}} ).sort( {a:1} ) );
+checkNumSorted( 10, t.find( {a:{$gte:0,$lte:10},b:null} ).sort( {a:1} ) );
+
+// Test results added by ordered and unordered plans, ordered plan finishes and continues with getmore.
+
+t.drop();
+
+t.save( {a:1} );
+t.save( {a:200} );
+for( i = 2; i <= 199; ++i ) {
+ t.save( {a:i} );
+}
+for( i = 0; i < 30; ++i ) {
+ t.save( {a:2000} );
+}
+t.ensureIndex( {a:1} );
+
+checkNumSorted( 200, t.find( {a:{$gte:0,$lte:200}} ).sort( {a:1} ) );
+checkNumSorted( 200, t.find( {a:{$gte:0,$lte:200},b:null} ).sort( {a:1} ) );
+
+// Test results added by ordered and unordered plans, with unordered results excluded during
+// getmore.
+
+t.drop();
+
+for( i = 399; i >= 0; --i ) {
+ t.save( {a:i} );
+}
+t.ensureIndex( {a:1} );
+
+checkNumSorted( 400, t.find( {a:{$gte:0,$lte:400},b:null} ).batchSize( 50 ).sort( {a:1} ) );
+
View
@@ -0,0 +1,19 @@
+// Unsorted plan on {a:1}, sorted plan on {b:1}. The unsorted plan exhausts its memory limit before
+// the sorted plan is chosen by the query optimizer.
+
+t = db.jstests_sortf;
+t.drop();
+
+t.ensureIndex( {a:1} );
+t.ensureIndex( {b:1} );
+
+for( i = 0; i < 100; ++i ) {
+ t.save( {a:0,b:0} );
+}
+
+big = new Array( 10 * 1000 * 1000 ).toString();
+for( i = 0; i < 5; ++i ) {
+ t.save( {a:1,b:1,big:big} );
+}
+
+assert.eq( 5, t.find( {a:1} ).sort( {b:1} ).itcount() );
View
@@ -11,13 +11,15 @@ for( i = 0; i < 40; ++i ) {
function memoryException( sortSpec, querySpec ) {
querySpec = querySpec || {};
- assert.throws( function() { t.find( querySpec ).sort( sortSpec ).itcount() } );
+ assert.throws( function() {
+ t.find( querySpec ).sort( sortSpec ).batchSize( 1000 ).itcount()
+ } );
assert( db.getLastError().match( /too much data for sort\(\) with no index/ ) );
}
function noMemoryException( sortSpec, querySpec ) {
querySpec = querySpec || {};
- t.find( querySpec ).sort( sortSpec ).itcount();
+ t.find( querySpec ).sort( sortSpec ).batchSize( 1000 ).itcount();
assert( !db.getLastError() );
}
@@ -37,10 +39,10 @@ noMemoryException( {a:1} );
noMemoryException( {b:1} );
// With an indexed plan on _id:1 and an unindexed plan on b:1, the indexed plan
-// should succeed even if the unindexed one exhausts its memory limit.
+// should succeed even if the unindexed one would exhaust its memory limit.
noMemoryException( {_id:1}, {b:null} );
// With an unindexed plan on b:1 recorded for a query, the query should be
// retried when the unindexed plan exhausts its memory limit.
-assert.eq( 'BtreeCursor b_1', t.find( {b:1} ).sort( {_id:1} ).explain().cursor );
+assert.eq( 'BtreeCursor b_1', t.find( {b:1} ).sort( {_id:1} ).explain().cursor ); // Record b:1 plan
noMemoryException( {_id:1}, {b:null} );
@@ -38,7 +38,7 @@
#include "../repl_block.h"
#include "../../server.h"
#include "../d_concurrency.h"
-#include "../queryoptimizercursor.h"
+#include "../queryoptimizercursorimpl.h"
namespace mongo {
@@ -668,22 +668,36 @@ namespace mongo {
void mayAddMatch() {
DiskLoc loc = _cursor->currLoc();
if ( _scanAndOrder ) {
- try {
- _scanAndOrder->add( _cursor->current(), _parsedQuery.showDiskLoc() ? &loc : 0 );
- } catch ( const UserException &e ) {
- if ( e.getCode() == ScanAndOrderMemoryLimitExceededAssertionCode ) {
- _queryOptimizerCursor->multiPlanScanner()->clearIndexesForPatterns();
+ if ( !_scanAndOrderDups.getsetdup( loc ) ) {
+ try {
+ _scanAndOrder->add( _cursor->current(), _parsedQuery.showDiskLoc() ? &loc : 0 );
+ } catch ( const UserException &e ) {
+ bool rethrow = true;
+ if ( e.getCode() == ScanAndOrderMemoryLimitExceededAssertionCode ) {
+ if ( _queryOptimizerCursor->multiPlanScanner()->haveOrderedPlan() ) {
+ _scanAndOrder.reset();
+ _queryOptimizerCursor->abortUnorderedPlans();
+ rethrow = false;
+ }
+ else if ( _queryOptimizerCursor->multiPlanScanner()->usingCachedPlan() ) {
+ _queryOptimizerCursor->multiPlanScanner()->clearIndexesForPatterns();
+ }
+ }
+ if ( rethrow ) {
+ throw;
+ }
}
- throw;
}
}
if ( !iterateNeedsSort() ) {
- if ( _skip > 0 ) {
- --_skip;
- }
- else {
- ++_n;
- fillQueryResultFromObj( _buf, _parsedQuery.getFields(), loc.obj(), ( _parsedQuery.showDiskLoc() ? &loc : 0 ) );
+ if ( !_cursor->getsetdup( loc ) ) {
+ if ( _skip > 0 ) {
+ --_skip;
+ }
+ else {
+ ++_n;
+ fillQueryResultFromObj( _buf, _parsedQuery.getFields(), loc.obj(), ( _parsedQuery.showDiskLoc() ? &loc : 0 ) );
+ }
}
}
}
@@ -693,13 +707,26 @@ namespace mongo {
bool enoughTotalResults() const {
return ( _parsedQuery.enough( _n ) || _buf.len() >= MaxBytesToReturnToClientAtOnce );
}
+ void finishedFirstBatch() {
+ if ( _queryOptimizerCursor ) {
+ _queryOptimizerCursor->abortUnorderedPlans();
+ }
+ }
long long handoff( Message &result ) {
int ret = _n;
if ( resultsNeedSort() ) {
_buf.reset();
_buf.skip( sizeof( QueryResult ) );
- // Handle exception here
- _scanAndOrder->fill( _buf, _parsedQuery.getFields(), ret );
+// try {
+// log() << "going to fill" << endl;
+ _scanAndOrder->fill( _buf, _parsedQuery.getFields(), ret );
+// } catch ( const UserException &e ) {
+// if ( e.getCode() == ScanAndOrderMemoryLimitExceededAssertionCode ) {
+// log() << "filled exception" << endl;
+// _queryOptimizerCursor->multiPlanScanner()->clearIndexesForPatterns();
+// }
+// throw;
+// }
}
if ( _buf.len() > 0 ) {
result.appendData( _buf.buf(), _buf.len() );
@@ -751,6 +778,7 @@ namespace mongo {
QueryOptimizerCursor *_queryOptimizerCursor;
BufBuilder _buf;
shared_ptr<ScanAndOrder> _scanAndOrder;
+ SmallDupSet _scanAndOrderDups;
long long _skip;
long long _n;
};
@@ -925,19 +953,18 @@ namespace mongo {
}
DiskLoc currLoc = cursor->currLoc();
- if ( cursor->getsetdup( currLoc ) ) {
- continue;
- }
+// log() << "idx: " << cursor->indexKeyPattern() << " obj: " << cursor->current() << endl;
BSONObj js = cursor->current();
assert( js.isValid() );
-
- if ( pq.hasOption( QueryOption_OplogReplay ) ) {
- BSONElement e = js["ts"];
- if ( e.type() == Date || e.type() == Timestamp ) {
- slaveReadTill = e._opTime();
- }
- }
+
+ // This should happen after matching?
+// if ( pq.hasOption( QueryOption_OplogReplay ) ) {
+// BSONElement e = js["ts"];
+// if ( e.type() == Date || e.type() == Timestamp ) {
+// slaveReadTill = e._opTime();
+// }
+// }
queryResponseBuilder.mayAddMatch();
if ( !cursor->supportGetMore() ) {
@@ -948,6 +975,7 @@ namespace mongo {
else if ( queryResponseBuilder.enoughForFirstBatch() ) {
/* if only 1 requested, no cursor saved for efficiency...we assume it is findOne() */
if ( pq.wantMore() && pq.getNumToReturn() != 1 ) {
+ queryResponseBuilder.finishedFirstBatch();
cursor->advance();
cursorid = ccPointer->cursorid();
}
@@ -675,6 +675,15 @@ namespace mongo {
}
return _plans[0];
}
+
+ bool QueryPlanSet::haveOrderedPlan() const {
+ for( PlanSet::const_iterator i = _plans.begin(); i != _plans.end(); ++i ) {
+ if ( !(*i)->scanAndOrderRequired() ) {
+ return true;
+ }
+ }
+ return false;
+ }
QueryPlanSet::Runner::Runner( QueryPlanSet &plans, QueryOp &op ) :
_op( op ),
@@ -1067,6 +1076,10 @@ namespace mongo {
void MultiPlanScanner::clearIndexesForPatterns() const {
QueryUtilIndexed::clearIndexesForPatterns( _currentQps->frsp(), _currentQps->order() );
}
+
+ bool MultiPlanScanner::haveOrderedPlan() const {
+ return _currentQps->haveOrderedPlan();
+ }
MultiCursor::MultiCursor( const char *ns, const BSONObj &pattern, const BSONObj &order, shared_ptr<CursorOp> op, bool mayYield )
: _mps( new MultiPlanScanner( ns, pattern, order, BSONObj(), true, BSONObj(), BSONObj(), !op.get(), mayYield ) ), _nscanned() {
@@ -312,6 +312,8 @@ namespace mongo {
const FieldRangeSetPair &frsp() const { return *_frsp; }
BSONObj order() const { return _order; }
+
+ bool haveOrderedPlan() const;
//for testing
const FieldRangeSetPair *originalFrsp() const { return _originalFrsp.get(); }
@@ -473,6 +475,8 @@ namespace mongo {
void clearIndexesForPatterns() const;
+ bool haveOrderedPlan() const;
+
private:
void assertNotOr() const {
massert( 13266, "not implemented for $or query", !_or );
@@ -82,6 +82,7 @@ namespace mongo {
virtual const QueryPlan *queryPlan() const = 0;
virtual const QueryPlan *completeQueryPlan() const = 0;
virtual const MultiPlanScanner *multiPlanScanner() const = 0;
+ virtual void abortUnorderedPlans() = 0;
};
} // namespace mongo
Oops, something went wrong.

0 comments on commit eac5c56

Please sign in to comment.