Skip to content

Commit

Permalink
SERVER-2001 change shard key validation to allow hashed shard keys
Browse files Browse the repository at this point in the history 
This changes the top-level shard key validation to allow shard keys
such as {a : "hashed"}. It also adds some helper functions for
determining when a unique index is compatible with a given shard
key, and a variety of unit tests and a js test.
  • Loading branch information
Kevin Matulef committed Oct 10, 2012
1 parent a50b595 commit 532a051
Show file tree
Hide file tree
Showing 7 changed files with 164 additions and 6 deletions.
62 changes: 61 additions & 1 deletion jstests/sharding/index1.js
View file
Expand Up @@ -3,7 +3,7 @@
s = new ShardingTest( "shard_index", 2, 0, 1 )

// Regenerate fully because of SERVER-2782
for ( var i = 0; i < 15; i++ ) {
for ( var i = 0; i < 19; i++ ) {

var coll = s.admin._mongo.getDB( "test" ).getCollection( "foo" + i )
coll.drop()
Expand Down Expand Up @@ -275,6 +275,66 @@ for ( var i = 0; i < 15; i++ ) {
assert( !passed , "Should not be able to shard collection with mulikey index");

}
if ( i == 15 ) {

// try sharding with a hashed index
coll.ensureIndex( { num : "hashed"} );

try{
s.adminCommand( { shardcollection : "" + coll, key : { num : "hashed" } } );
}
catch( e ){
print(e);
assert( false, "Should be able to shard collection with hashed index.");
}
}
if ( i == 16 ) {

// create hashed index, but try to declare it unique when sharding
coll.ensureIndex( { num : "hashed"} )

passed = false;
try{
s.adminCommand({ shardcollection : "" + coll, key : { num : "hashed" }, unique : true});
passed = true;
}
catch( e ){
print(e);
}
assert( !passed , "Should not be able to declare hashed shard key unique.");

}
if ( i == 17 ) {

// create hashed index, but unrelated unique index present
coll.ensureIndex( { x : "hashed" } );
coll.ensureIndex( { num : 1 }, { unique : true} );

passed = false;
try {
s.adminCommand( { shardcollection : "" + coll, key : { x : "hashed" } } );
passed = true;
}
catch (e) {
print( e );
}
assert( !passed, "Should not be able to shard on hashed index with another unique index" );

}
if ( i == 18 ) {

// create hashed index, and a regular unique index exists on same field
coll.ensureIndex( { num : "hashed" } );
coll.ensureIndex( { num : 1 }, { unique : true } );

try{
s.adminCommand({ shardcollection : "" + coll, key : { num : "hashed" } } );
}
catch( e ){
print(e);
assert( false, "Should be able to shard coll with hashed and regular unique index");
}
}
}

s.stop();
8 changes: 8 additions & 0 deletions src/mongo/bson/bsonobj.h
View file
Expand Up @@ -331,6 +331,14 @@ namespace mongo {
*/
bool isPrefixOf( const BSONObj& otherObj ) const;

/**
* @param otherObj
* @return returns true if the list of field names in 'this' is a prefix
* of the list of field names in otherObj. Similar to 'isPrefixOf',
* but ignores the field values and only looks at field names.
*/
bool isFieldNamePrefixOf( const BSONObj& otherObj ) const;

/** This is "shallow equality" -- ints and doubles won't match. for a
deep equality test use woCompare (which is slower).
*/
Expand Down
15 changes: 15 additions & 0 deletions src/mongo/db/jsobj.cpp
View file
Expand Up @@ -586,6 +586,21 @@ namespace mongo {
return ! a.more();
}

bool BSONObj::isFieldNamePrefixOf( const BSONObj& otherObj ) const {
BSONObjIterator a( *this );
BSONObjIterator b( otherObj );

while ( a.more() && b.more() ) {
BSONElement x = a.next();
BSONElement y = b.next();
if ( ! mongoutils::str::equals( x.fieldName() , y.fieldName() ) ) {
return false;
}
}

return ! a.more();
}

template <typename BSONElementColl>
void _getFieldsDotted( const BSONObj* obj, const StringData& name, BSONElementColl &ret, bool expandLastArray ) {
BSONElement e = obj->getField( name );
Expand Down
5 changes: 5 additions & 0 deletions src/mongo/dbtests/jsobjtests.cpp
View file
Expand Up @@ -198,6 +198,11 @@ namespace JsobjTests {
verify( ! k.isPrefixOf( BSON( "x" << "hi" ) ) );
verify( k.isPrefixOf( BSON( "x" << 1 << "a" << "hi" ) ) );
}
{
BSONObj k = BSON( "x" << 1 );
verify( k.isFieldNamePrefixOf( BSON( "x" << "hi" ) ) );
verify( ! k.isFieldNamePrefixOf( BSON( "a" << 1 ) ) );
}
}
};

Expand Down
35 changes: 30 additions & 5 deletions src/mongo/s/commands_admin.cpp
View file
Expand Up @@ -490,11 +490,36 @@ namespace mongo {
return false;
}

BSONForEach(e, proposedKey) {
if (!e.isNumber() || e.number() != 1.0) {
errmsg = "shard keys must all be ascending";
// Currently the allowable shard keys are either
// i) a hashed single field, e.g. { a : "hashed" }, or
// ii) a compound list of ascending fields, e.g. { a : 1 , b : 1 }
if ( proposedKey.firstElementType() == mongo::String ) {
// case i)
if ( !str::equals( proposedKey.firstElement().valuestrsafe() , "hashed" ) ) {
errmsg = "unrecognized string: " + proposedKey.firstElement().str();
return false;
}
if ( proposedKey.nFields() > 1 ) {
errmsg = "hashed shard keys currently only support single field keys";
return false;
}
if ( cmdObj["unique"].trueValue() ) {
// it's possible to ensure uniqueness on the hashed field by
// declaring an additional (non-hashed) unique index on the field,
// but the hashed shard key itself should not be declared unique
errmsg = "hashed shard keys cannot be declared unique.";
return false;
}
} else {
// case ii)
BSONForEach(e, proposedKey) {
if (!e.isNumber() || e.number() != 1.0) {
errmsg = str::stream() << "Unsupported shard key pattern. Pattern must"
<< " either be a single hashed field, or a list"
<< " of ascending fields.";
return false;
}
}
}

if ( ns.find( ".system." ) != string::npos ) {
Expand Down Expand Up @@ -553,11 +578,11 @@ namespace mongo {
auto_ptr<DBClientCursor> uniqueQueryResult =
conn->get()->query( indexNS , uniqueQuery );

ShardKeyPattern proposedShardKey( proposedKey );
while ( uniqueQueryResult->more() ) {
BSONObj idx = uniqueQueryResult->next();
BSONObj currentKey = idx["key"].embeddedObject();
bool isCurrentID = str::equals( currentKey.firstElementFieldName() , "_id" );
if( ! isCurrentID && ! proposedKey.isPrefixOf( currentKey) ) {
if( ! proposedShardKey.isUniqueIndexCompatible( currentKey ) ) {
errmsg = str::stream() << "can't shard collection '" << ns << "' "
<< "with unique index on " << currentKey << " "
<< "and proposed shard key " << proposedKey << ". "
Expand Down
22 changes: 22 additions & 0 deletions src/mongo/s/shardkey.cpp
View file
Expand Up @@ -61,6 +61,14 @@ namespace mongo {
return pattern.isPrefixOf( otherPattern );
}

bool ShardKeyPattern::isUniqueIndexCompatible( const BSONObj& uniqueIndexPattern ) const {
if ( ! uniqueIndexPattern.isEmpty() &&
str::equals( uniqueIndexPattern.firstElementFieldName(), "_id" ) ){
return true;
}
return pattern.isFieldNamePrefixOf( uniqueIndexPattern );
}

bool ShardKeyPattern::isSpecial() const {
BSONForEach(e, pattern) {
int fieldVal = e.numberInt();
Expand Down Expand Up @@ -210,6 +218,18 @@ namespace mongo {

}

void uniqueIndexCompatibleTest() {
ShardKeyPattern k1( BSON( "a" << 1 ) );
verify( k1.isUniqueIndexCompatible( BSON( "_id" << 1 ) ) );
verify( k1.isUniqueIndexCompatible( BSON( "a" << 1 << "b" << 1 ) ) );
verify( k1.isUniqueIndexCompatible( BSON( "a" << -1 ) ) );
verify( ! k1.isUniqueIndexCompatible( BSON( "b" << 1 ) ) );

ShardKeyPattern k2( BSON( "a" << "hashed") );
verify( k2.isUniqueIndexCompatible( BSON( "a" << 1 ) ) );
verify( ! k2.isUniqueIndexCompatible( BSON( "b" << 1 ) ) );
}

void moveToFrontBenchmark(int numFields) {
BSONObjBuilder bb;
bb.append("_id", 1);
Expand Down Expand Up @@ -263,6 +283,8 @@ namespace mongo {

moveToFrontTest();

uniqueIndexCompatibleTest();

if (0) { // toggle to run benchmark
moveToFrontBenchmark(0);
moveToFrontBenchmark(10);
Expand Down
23 changes: 23 additions & 0 deletions src/mongo/s/shardkey.h
View file
Expand Up @@ -85,6 +85,29 @@ namespace mongo {
*/
bool isPrefixOf( const BSONObj& otherPattern ) const;

/**
* @return
* true if this shard key is compatible with a unique index on 'uniqueIndexPattern'.
* Primarily this just checks whether 'this' is a prefix of 'uniqueIndexPattern',
* However it does not need to be an exact syntactic prefix due to "hashed"
* indexes or mismatches in ascending/descending order. Also, uniqueness of the
* _id field is guaranteed by the generation process (or by the user) so every
* index that begins with _id is unique index compatible with any shard key.
* Examples:
* shard key {a : 1} is compatible with a unique index on {_id : 1}
* shard key {a : 1} is compatible with a unique index on {a : 1 , b : 1}
* shard key {a : 1} is compatible with a unique index on {a : -1 , b : 1 }
* shard key {a : "hashed"} is compatible with a unique index on {a : 1}
* shard key {a : 1} is not compatible with a unique index on {b : 1}
* shard key {a : "hashed" , b : 1 } is not compatible with unique index on { b : 1 }
* Note:
* this method assumes that 'uniqueIndexPattern' is a valid index pattern,
* and is capable of being a unique index. A pattern like { k : "hashed" }
* is never capable of being a unique index, and thus is an invalid setting
* for the 'uniqueIndexPattern' argument.
*/
bool isUniqueIndexCompatible( const BSONObj& uniqueIndexPattern ) const;

/**
* @return
* true if keyPattern contains any computed values, (e.g. {a : "hashed"})
Expand Down

0 comments on commit 532a051

Please sign in to comment.