Permalink
Browse files

SERVER-12868 added memory usage accounting to hashed AND stage

  • Loading branch information...
1 parent 04f7ab3 commit 5e120d33c761d2dd89ae260751ab1368c512a8d5 @benety benety committed Feb 26, 2014
Showing with 332 additions and 6 deletions.
  1. +68 −5 src/mongo/db/exec/and_hash.cpp
  2. +21 −0 src/mongo/db/exec/and_hash.h
  3. +243 −1 src/mongo/dbtests/query_stage_and.cpp
@@ -34,22 +34,60 @@
#include "mongo/db/exec/working_set.h"
#include "mongo/util/mongoutils/str.h"
+namespace {
+
+ using namespace mongo;
+
+ // Upper limit for buffered data.
+ // Stage execution will fail once size of all buffered data exceeds this threshold.
+ const size_t kDefaultMaxMemUsageBytes = 32 * 1024 * 1024;
+
+ /**
+ * Returns expected memory usage of working set member
+ */
+ size_t getMemberMemUsage(WorkingSetMember* member) {
+ size_t memUsage = 0;
+ for (size_t i = 0; i < member->keyData.size(); ++i) {
+ const IndexKeyDatum& keyDatum = member->keyData[i];
+ memUsage += keyDatum.keyData.objsize();
+ }
+ return memUsage;
+ }
+
+} // namespace
+
namespace mongo {
+ using std::auto_ptr;
+
const size_t AndHashStage::kLookAheadWorks = 10;
AndHashStage::AndHashStage(WorkingSet* ws, const MatchExpression* filter)
: _ws(ws),
_filter(filter),
_hashingChildren(true),
- _currentChild(0) {}
+ _currentChild(0),
+ _memUsage(0),
+ _maxMemUsage(kDefaultMaxMemUsageBytes) {}
+
+ AndHashStage::AndHashStage(WorkingSet* ws, const MatchExpression* filter, size_t maxMemUsage)
+ : _ws(ws),
+ _filter(filter),
+ _hashingChildren(true),
+ _currentChild(0),
+ _memUsage(0),
+ _maxMemUsage(maxMemUsage) {}
AndHashStage::~AndHashStage() {
for (size_t i = 0; i < _children.size(); ++i) { delete _children[i]; }
}
void AndHashStage::addChild(PlanStage* child) { _children.push_back(child); }
+ size_t AndHashStage::getMemUsage() const {
+ return _memUsage;
+ }
+
bool AndHashStage::isEOF() {
// This is empty before calling work() and not-empty after.
if (_lookAheadResults.empty()) { return false; }
@@ -139,6 +177,16 @@ namespace mongo {
// We read the first child into our hash table.
if (_hashingChildren) {
+ // Check memory usage of previously hashed results.
+ if (_memUsage > _maxMemUsage) {
+ mongoutils::str::stream ss;
+ ss << "hashed AND stage buffered data usage of " << _memUsage
+ << " bytes exceeds internal limit of " << kDefaultMaxMemUsageBytes << " bytes";
+ Status status(ErrorCodes::Overflow, ss);
+ *out = WorkingSetCommon::allocateStatusMember( _ws, status);
+ return PlanStage::FAILURE;
+ }
+
if (0 == _currentChild) {
return readFirstChild(out);
}
@@ -241,6 +289,10 @@ namespace mongo {
verify(_dataMap.end() == _dataMap.find(member->loc));
_dataMap[member->loc] = id;
+
+ // Update memory stats.
+ _memUsage += getMemberMemUsage(member);
+
++_commonStats.needTime;
return PlanStage::NEED_TIME;
}
@@ -309,7 +361,12 @@ namespace mongo {
// We have a hit. Copy data into the WSM we already have.
_seenMap.insert(member->loc);
WorkingSetMember* olderMember = _ws->get(_dataMap[member->loc]);
+ size_t memUsageBefore = getMemberMemUsage(olderMember);
+
AndCommon::mergeFrom(olderMember, *member);
+
+ // Update memory stats.
+ _memUsage += getMemberMemUsage(olderMember) - memUsageBefore;
}
_ws->free(id);
++_commonStats.needTime;
@@ -325,6 +382,11 @@ namespace mongo {
if (_seenMap.end() == _seenMap.find(it->first)) {
DataMap::iterator toErase = it;
++it;
+
+ // Update memory stats.
+ WorkingSetMember* member = _ws->get(toErase->second);
+ _memUsage -= getMemberMemUsage(member);
+
_ws->free(toErase->second);
_dataMap.erase(toErase);
}
@@ -435,6 +497,9 @@ namespace mongo {
++_specificStats.flaggedButPassed;
}
+ // Update memory stats.
+ _memUsage -= getMemberMemUsage(member);
+
// The loc is about to be invalidated. Fetch it and clear the loc.
WorkingSetCommon::fetchAndInvalidateLoc(member);
@@ -449,10 +514,8 @@ namespace mongo {
PlanStageStats* AndHashStage::getStats() {
_commonStats.isEOF = isEOF();
- // XXX: populate
- // _specificStats.memLimit
- // _specificStats.memUsage
- // when ben's change is in
+ _specificStats.memLimit = _maxMemUsage;
+ _specificStats.memUsage = _memUsage;
auto_ptr<PlanStageStats> ret(new PlanStageStats(_commonStats, STAGE_AND_HASH));
ret->specific.reset(new AndHashStats(_specificStats));
@@ -53,10 +53,22 @@ namespace mongo {
class AndHashStage : public PlanStage {
public:
AndHashStage(WorkingSet* ws, const MatchExpression* filter);
+
+ /**
+ * For testing only. Allows tests to set memory usage threshold.
+ */
+ AndHashStage(WorkingSet* ws, const MatchExpression* filter, size_t maxMemUsage);
+
virtual ~AndHashStage();
void addChild(PlanStage* child);
+ /**
+ * Returns memory usage.
+ * For testing only.
+ */
+ size_t getMemUsage() const;
+
virtual StageState work(WorkingSetID* out);
virtual bool isEOF();
@@ -106,6 +118,15 @@ namespace mongo {
// Stats
CommonStats _commonStats;
AndHashStats _specificStats;
+
+ // The usage in bytes of all buffered data that we're holding.
+ // Memory usage is calculated from keys held in _dataMap only.
+ // For simplicity, results in _lookAheadResults do not count towards the limit.
+ size_t _memUsage;
+
+ // Upper limit for buffered data memory usage.
+ // Defaults to 32 MB (See kMaxBytes in and_hash.cpp).
+ size_t _maxMemUsage;
};
} // namespace mongo
Oops, something went wrong.

0 comments on commit 5e120d3

Please sign in to comment.