Skip to content

Commit

Permalink
SERVER-28030 Remove writes from Chunk
Browse files Browse the repository at this point in the history
  • Loading branch information
@kaloianm
kaloianm committed Feb 18, 2017
1 parent f2744a7 commit 41729ce
Show file tree
Hide file tree
Showing 5 changed files with 78 additions and 111 deletions.
18 changes: 17 additions & 1 deletion src/mongo/db/s/balancer/balancer.cpp
View file
Expand Up @@ -638,7 +638,23 @@ void Balancer::_splitOrMarkJumbo(OperationContext* txn,
ChunkRange(chunk->getMin(), chunk->getMax()),
splitPoints));
} catch (const DBException& ex) {
chunk->markAsJumbo(txn);
log() << "Marking chunk " << redact(chunk->toString()) << " as jumbo.";

chunk->markAsJumbo();

const std::string chunkName = ChunkType::genID(nss.ns(), chunk->getMin());

auto status = Grid::get(txn)->catalogClient(txn)->updateConfigDocument(
txn,
ChunkType::ConfigNS,
BSON(ChunkType::name(chunkName)),
BSON("$set" << BSON(ChunkType::jumbo(true))),
false,
ShardingCatalogClient::kMajorityWriteConcern);
if (!status.isOK()) {
log() << "Couldn't set jumbo for chunk: " << redact(chunkName)
<< causedBy(redact(status.getStatus()));
}
}
}

Expand Down
76 changes: 46 additions & 30 deletions src/mongo/dbtests/chunktests.cpp
View file
Expand Up @@ -58,13 +58,14 @@ class TestableChunkManager : public ChunkManager {
const string shardId = str::stream() << (i - 1);
_shardIds.insert(shardId);

std::shared_ptr<Chunk> chunk(new Chunk(this,
mySplitPoints[i - 1],
mySplitPoints[i],
shardId,
ChunkVersion(0, 0, OID()),
0));
_chunkMap[mySplitPoints[i]] = chunk;
ChunkType chunk;
chunk.setNS(getns());
chunk.setMin(mySplitPoints[i - 1]);
chunk.setMax(mySplitPoints[i]);
chunk.setShard(shardId);
chunk.setVersion(ChunkVersion(0, 0, OID()));

_chunkMap[mySplitPoints[i]] = std::make_shared<Chunk>(chunk);
}

_chunkRangeMap = _constructRanges(_chunkMap);
Expand All @@ -84,8 +85,9 @@ class Base {
auto opCtx = serviceContext.makeOperationContext();

ShardKeyPattern shardKeyPattern(shardKey());
TestableChunkManager chunkManager("", shardKeyPattern, defaultCollator(), false);
chunkManager.setSingleChunkForShards(splitPointsVector());
TestableChunkManager chunkManager(
"TestDB.TestColl", shardKeyPattern, defaultCollator(), false);
chunkManager.setSingleChunkForShards(splitPoints());

set<ShardId> shardIds;
chunkManager.getShardIdsForQuery(opCtx.get(), query(), queryCollation(), &shardIds);
Expand All @@ -106,8 +108,8 @@ class Base {
return {nullptr};
}

virtual BSONArray splitPoints() const {
return BSONArray();
virtual std::vector<BSONObj> splitPoints() const {
return {};
}

virtual BSONObj query() const {
Expand All @@ -121,30 +123,21 @@ class Base {
virtual BSONArray expectedShardNames() const {
return BSON_ARRAY("0");
}

virtual vector<BSONObj> splitPointsVector() const {
vector<BSONObj> ret;
BSONArray a = splitPoints();
BSONObjIterator i(a);
while (i.more()) {
ret.push_back(i.next().Obj().getOwned());
}
return ret;
}
};

class EmptyQuerySingleShard : public Base {};

class MultiShardBase : public Base {
virtual BSONArray splitPoints() const {
return BSON_ARRAY(BSON("a"
<< "x")
<< BSON("a"
<< "y")
<< BSON("a"
<< "z"));
virtual std::vector<BSONObj> splitPoints() const {
return {BSON("a"
<< "x"),
BSON("a"
<< "y"),
BSON("a"
<< "z")};
}
};

class EmptyQueryMultiShard : public MultiShardBase {
virtual BSONArray expectedShardNames() const {
return BSON_ARRAY("0"
Expand Down Expand Up @@ -172,6 +165,7 @@ class EqualityRangeMultiShard : public MultiShardBase {
return BSON("a"
<< "y");
}

virtual BSONArray expectedShardNames() const {
return BSON_ARRAY("2");
}
Expand All @@ -181,6 +175,7 @@ class SetRangeMultiShard : public MultiShardBase {
virtual BSONObj query() const {
return fromjson("{a:{$in:['u','y']}}");
}

virtual BSONArray expectedShardNames() const {
return BSON_ARRAY("0"
<< "2");
Expand All @@ -191,6 +186,7 @@ class GTRangeMultiShard : public MultiShardBase {
virtual BSONObj query() const {
return BSON("a" << GT << "x");
}

virtual BSONArray expectedShardNames() const {
return BSON_ARRAY("1"
<< "2"
Expand All @@ -202,6 +198,7 @@ class GTERangeMultiShard : public MultiShardBase {
virtual BSONObj query() const {
return BSON("a" << GTE << "x");
}

virtual BSONArray expectedShardNames() const {
return BSON_ARRAY("1"
<< "2"
Expand Down Expand Up @@ -229,6 +226,7 @@ class LTERangeMultiShard : public MultiShardBase {
virtual BSONObj query() const {
return BSON("a" << LTE << "y");
}

virtual BSONArray expectedShardNames() const {
return BSON_ARRAY("0"
<< "1"
Expand All @@ -240,6 +238,7 @@ class OrEqualities : public MultiShardBase {
virtual BSONObj query() const {
return fromjson("{$or:[{a:'u'},{a:'y'}]}");
}

virtual BSONArray expectedShardNames() const {
return BSON_ARRAY("0"
<< "2");
Expand All @@ -250,6 +249,7 @@ class OrEqualityInequality : public MultiShardBase {
virtual BSONObj query() const {
return fromjson("{$or:[{a:'u'},{a:{$gte:'y'}}]}");
}

virtual BSONArray expectedShardNames() const {
return BSON_ARRAY("0"
<< "2"
Expand Down Expand Up @@ -297,6 +297,7 @@ class EqualityThenUnsatisfiable : public Unsatisfiable<Base> {
virtual BSONObj shardKey() const {
return BSON("a" << 1 << "b" << 1);
}

virtual BSONObj query() const {
return BSON("a" << 1 << "b" << GT << 4 << LT << 4);
}
Expand All @@ -306,6 +307,7 @@ class InequalityThenUnsatisfiable : public Unsatisfiable<Base> {
virtual BSONObj shardKey() const {
return BSON("a" << 1 << "b" << 1);
}

virtual BSONObj query() const {
return BSON("a" << GT << 1 << "b" << GT << 4 << LT << 4);
}
Expand All @@ -328,8 +330,8 @@ class CompoundKeyBase : public Base {
return BSON("a" << 1 << "b" << 1);
}

virtual BSONArray splitPoints() const {
return BSON_ARRAY(BSON("a" << 5 << "b" << 10) << BSON("a" << 5 << "b" << 20));
virtual std::vector<BSONObj> splitPoints() const {
return {BSON("a" << 5 << "b" << 10), BSON("a" << 5 << "b" << 20)};
}
};

Expand All @@ -354,10 +356,12 @@ class CollationStringsMultiShard : public MultiShardBase {
return BSON("a"
<< "y");
}

virtual BSONObj queryCollation() const {
return BSON("locale"
<< "mock_reverse_string");
}

virtual BSONArray expectedShardNames() const {
return BSON_ARRAY("0"
<< "1"
Expand All @@ -371,11 +375,13 @@ class DefaultCollationStringsMultiShard : public MultiShardBase {
return BSON("a"
<< "y");
}

virtual std::unique_ptr<CollatorInterface> defaultCollator() const {
auto collator = stdx::make_unique<CollatorInterfaceMock>(
CollatorInterfaceMock::MockType::kReverseString);
return {std::move(collator)};
}

virtual BSONArray expectedShardNames() const {
return BSON_ARRAY("0"
<< "1"
Expand All @@ -389,15 +395,18 @@ class SimpleCollationStringsMultiShard : public MultiShardBase {
return BSON("a"
<< "y");
}

virtual std::unique_ptr<CollatorInterface> defaultCollator() const {
auto collator = stdx::make_unique<CollatorInterfaceMock>(
CollatorInterfaceMock::MockType::kReverseString);
return {std::move(collator)};
}

virtual BSONObj queryCollation() const {
return BSON("locale"
<< "simple");
}

virtual BSONArray expectedShardNames() const {
return BSON_ARRAY("2");
}
Expand All @@ -407,10 +416,12 @@ class CollationNumbersMultiShard : public MultiShardBase {
virtual BSONObj query() const {
return BSON("a" << 5);
}

virtual BSONObj queryCollation() const {
return BSON("locale"
<< "mock_reverse_string");
}

virtual BSONArray expectedShardNames() const {
return BSON_ARRAY("0");
}
Expand All @@ -420,11 +431,13 @@ class DefaultCollationNumbersMultiShard : public MultiShardBase {
virtual BSONObj query() const {
return BSON("a" << 5);
}

virtual std::unique_ptr<CollatorInterface> defaultCollator() const {
auto collator = stdx::make_unique<CollatorInterfaceMock>(
CollatorInterfaceMock::MockType::kReverseString);
return {std::move(collator)};
}

virtual BSONArray expectedShardNames() const {
return BSON_ARRAY("0");
}
Expand All @@ -434,15 +447,18 @@ class SimpleCollationNumbersMultiShard : public MultiShardBase {
virtual BSONObj query() const {
return BSON("a" << 5);
}

virtual std::unique_ptr<CollatorInterface> defaultCollator() const {
auto collator = stdx::make_unique<CollatorInterfaceMock>(
CollatorInterfaceMock::MockType::kReverseString);
return {std::move(collator)};
}

virtual BSONObj queryCollation() const {
return BSON("locale"
<< "simple");
}

virtual BSONArray expectedShardNames() const {
return BSON_ARRAY("0");
}
Expand Down
46 changes: 4 additions & 42 deletions src/mongo/s/chunk.cpp
View file
Expand Up @@ -34,9 +34,6 @@

#include "mongo/platform/random.h"
#include "mongo/s/balancer_configuration.h"
#include "mongo/s/catalog/sharding_catalog_client.h"
#include "mongo/s/catalog/type_chunk.h"
#include "mongo/s/chunk_manager.h"
#include "mongo/s/grid.h"
#include "mongo/util/log.h"

Expand All @@ -57,31 +54,15 @@ int64_t mkDataWritten() {

} // namespace

Chunk::Chunk(ChunkManager* manager, const ChunkType& from)
: _manager(manager),
_min(from.getMin().getOwned()),
_max(from.getMax().getOwned()),
Chunk::Chunk(const ChunkType& from)
: _range(from.getMin(), from.getMax()),
_shardId(from.getShard()),
_lastmod(from.getVersion()),
_jumbo(from.getJumbo()),
_dataWritten(mkDataWritten()) {
invariantOK(from.validate());
}

Chunk::Chunk(ChunkManager* manager,
const BSONObj& min,
const BSONObj& max,
const ShardId& shardId,
ChunkVersion lastmod,
uint64_t initialDataWritten)
: _manager(manager),
_min(min),
_max(max),
_shardId(shardId),
_lastmod(lastmod),
_jumbo(false),
_dataWritten(initialDataWritten) {}

bool Chunk::containsKey(const BSONObj& shardKey) const {
return getMin().woCompare(shardKey) <= 0 && shardKey.woCompare(getMax()) < 0;
}
Expand All @@ -106,30 +87,11 @@ void Chunk::randomizeBytesWritten() {
std::string Chunk::toString() const {
return str::stream() << ChunkType::shard() << ": " << _shardId << ", "
<< ChunkType::DEPRECATED_lastmod() << ": " << _lastmod.toString() << ", "
<< ChunkType::min() << ": " << _min << ", " << ChunkType::max() << ": "
<< _max;
<< _range.toString();
}

void Chunk::markAsJumbo(OperationContext* txn) const {
log() << "Marking chunk " << toString() << " as jumbo.";

// set this first
// even if we can't set it in the db
// at least this mongos won't try and keep moving
void Chunk::markAsJumbo() {
_jumbo = true;

const std::string chunkName = ChunkType::genID(_manager->getns(), _min);

auto status = Grid::get(txn)->catalogClient(txn)->updateConfigDocument(
txn,
ChunkType::ConfigNS,
BSON(ChunkType::name(chunkName)),
BSON("$set" << BSON(ChunkType::jumbo(true))),
false,
ShardingCatalogClient::kMajorityWriteConcern);
if (!status.isOK()) {
warning() << "couldn't set jumbo for chunk: " << chunkName << causedBy(status.getStatus());
}
}

} // namespace mongo

0 comments on commit 41729ce

Please sign in to comment.