Skip to content

Commit

Permalink
SERVER-18024 Move collection metadata loading under the catalog manager
Browse files Browse the repository at this point in the history 
Cleans up the CollectionType parsing code and moves all the collection
metadata retrieval logic under the catalog manager.

Also moves type_collection.* under the catalog manager library.
  • Loading branch information
@kaloianm
kaloianm committed Apr 21, 2015
1 parent 1e0624e commit 4777fb8
Show file tree
Hide file tree
Showing 39 changed files with 1,045 additions and 1,318 deletions.
10 changes: 7 additions & 3 deletions jstests/noPassthroughWithMongod/no_balance_collection.js
View file
Expand Up @@ -82,7 +82,11 @@ assert.writeOK(bulk.execute());
printjson( lastMigration )
printjson( sh._lastMigration( collB ) )

if( lastMigration == null ) assert.eq( null, sh._lastMigration( collB ) )
else assert.eq( lastMigration.time, sh._lastMigration( collB ).time )
if(lastMigration == null) {
assert.eq(null, sh._lastMigration(collB));
}
else {
assert.eq(lastMigration.time, sh._lastMigration(collB).time);
}

st.stop()
st.stop();
28 changes: 14 additions & 14 deletions jstests/sharding/drop_sharded_db.js
View file
@@ -1,15 +1,14 @@
// Tests the dropping of a sharded database SERVER-3471 SERVER-1726

var st = new ShardingTest({ name : jsTestName() })
st.stopBalancer()

var mongos = st.s0
var config = mongos.getDB( "config" )

var dbName = "buy"
var dbA = mongos.getDB( dbName )
var dbB = mongos.getDB( dbName + "_201107" )
var dbC = mongos.getDB( dbName + "_201108" )
var dbA = mongos.getDB( "DropSharded_A" );
var dbB = mongos.getDB( "DropSharded_B" );
var dbC = mongos.getDB( "DropSharded_C" );


print( "1: insert some data and colls into all dbs" )

Expand All @@ -21,10 +20,10 @@ for( var i = 0; i < numDocs; i++ ){
dbC.getCollection( "data" + (i % numColls) ).insert({ _id : i })
}


print( "2: shard the colls ")

for( var i = 0; i < numColls; i++ ){

var key = { _id : 1 }
st.shardColl( dbA.getCollection( "data" + i ), key )
st.shardColl( dbB.getCollection( "data" + i ), key )
Expand All @@ -48,16 +47,17 @@ assert.eq( 0, config.databases.find({ _id : "" + dbA }).toArray().length )
assert.eq( 1, config.databases.find({ _id : "" + dbB }).toArray().length )
assert.eq( 1, config.databases.find({ _id : "" + dbC }).toArray().length )

// 10 dropped collections
assert.eq( numColls, config.collections.find({ _id : RegExp( "^" + dbA + "\\..*" ), dropped : true }).toArray().length )
assert.eq( numColls, config.collections.find({ _id : RegExp( "^" + dbB + "\\..*" ), dropped : false }).toArray().length )
assert.eq( numColls, config.collections.find({ _id : RegExp( "^" + dbC + "\\..*" ), dropped : false }).toArray().length )

for( var i = 0; i < numColls; i++ ){

assert.eq( numDocs / numColls, dbB.getCollection( "data" + (i % numColls) ).find().itcount() )
assert.eq( numDocs / numColls, dbC.getCollection( "data" + (i % numColls) ).find().itcount() )

// 20 active (dropped is missing)
assert.eq( numColls, config.collections.find({ _id : RegExp( "^" + dbB + "\\..*" ) }).toArray().length )
assert.eq( numColls, config.collections.find({ _id : RegExp( "^" + dbC + "\\..*" ) }).toArray().length )

for( var i = 0; i < numColls; i++ ) {
assert.eq( numDocs / numColls, dbB.getCollection( "data" + (i % numColls) ).find().itcount() );
assert.eq( numDocs / numColls, dbC.getCollection( "data" + (i % numColls) ).find().itcount() );
}

// Finish
st.stop()
st.stop();
8 changes: 4 additions & 4 deletions jstests/sharding/mrShardedOutputAuth.js
View file
Expand Up @@ -34,7 +34,7 @@ function doMapReduce(connection, outputDb) {
function assertSuccess(configDb, outputDb) {
adminDb.printShardingStatus();
assert.eq(outputDb.numbers_out.count(), 50, "map/reduce failed");
assert.eq(configDb.collections.findOne().dropped, false, "no sharded collections");
assert.eq(configDb.collections.findOne().dropped, undefined, "no sharded collections");
}

function assertFailure(configDb, outputDb) {
Expand All @@ -50,7 +50,7 @@ var st = new ShardingTest( testName = "mrShardedOutputAuth",
{ extraOptions : {"keyFile" : "jstests/libs/key1"} }
);

// setup the users to the input, output and admin databases
// Setup the users to the input, output and admin databases
var mongos = st.s;
var adminDb = mongos.getDB("admin");
adminDb.createUser({user: "user", pwd: "pass", roles: jsTest.adminUserRoles});
Expand All @@ -67,14 +67,14 @@ inputDb.createUser({user: "user", pwd: "pass", roles: jsTest.basicUserRoles});
var outputDb = authenticatedConn.getDB("output");
outputDb.createUser({user: "user", pwd: "pass", roles: jsTest.basicUserRoles});

// setup the input db
// Setup the input db
inputDb.numbers.drop();
for (var i = 0; i < 50; i++) {
inputDb.numbers.insert({ num : i });
}
assert.eq(inputDb.numbers.count(), 50);

// setup a connection authenticated to both input and output db
// Setup a connection authenticated to both input and output db
var inputOutputAuthConn = new Mongo(mongos.host);
inputOutputAuthConn.getDB('input').auth("user", "pass");
inputOutputAuthConn.getDB('output').auth("user", "pass");
Expand Down
10 changes: 7 additions & 3 deletions jstests/sharding/shard1.js
View file
Expand Up @@ -23,14 +23,18 @@ s.adminCommand( shardCommand );
assert.throws( function(){ s.adminCommand({ shardCollection: 'test', key: { x: 1 }}); });
assert.throws( function(){ s.adminCommand({ shardCollection: '.foo', key: { x: 1 }}); });

cconfig = s.config.collections.findOne( { _id : "test.foo" } );
var cconfig = s.config.collections.findOne( { _id : "test.foo" } );
assert( cconfig , "why no collection entry for test.foo" )

delete cconfig.lastmod
delete cconfig.dropped
delete cconfig.lastmodEpoch
assert.eq( cconfig , { _id : "test.foo" , key : { num : 1 } , unique : false } , "Sharded content" );

s.config.collections.find().forEach( printjson )
assert.eq(cconfig,
{ _id : "test.foo" , key : { num : 1 } , unique : false },
"Sharded content mismatch");

s.config.collections.find().forEach( printjson );

assert.eq( 1 , s.config.chunks.count() , "num chunks A");
si = s.config.chunks.findOne();
Expand Down
4 changes: 2 additions & 2 deletions jstests/sharding/top_chunk_autosplit.js
View file
Expand Up @@ -25,9 +25,9 @@ function runTest(test) {

// Setup
// Enable sharding, set primary shard and shard collection
db.adminCommand({enableSharding: dbName});
assert.commandWorked(db.adminCommand({enableSharding: dbName}));
db.adminCommand({movePrimary: dbName, to: 'shard0000'});
db.adminCommand({shardCollection: coll + "", key: {x: 1}});
assert.commandWorked(db.adminCommand({shardCollection: coll + "", key: {x: 1}}));

// Pre-split, move chunks & create tags
for (var i = 0; i < test.shards.length; i++) {
Expand Down
1 change: 0 additions & 1 deletion src/mongo/dbtests/config_upgrade_tests.cpp
View file
Expand Up @@ -33,7 +33,6 @@
#include "mongo/s/catalog/type_shard.h"
#include "mongo/s/chunk_version.h"
#include "mongo/s/type_mongos.h"
#include "mongo/s/type_collection.h"
#include "mongo/s/type_settings.h"
#include "mongo/s/type_config_version.h"
#include "mongo/unittest/unittest.h"
Expand Down
9 changes: 4 additions & 5 deletions src/mongo/dbtests/merge_chunk_tests.cpp
View file
Expand Up @@ -29,12 +29,12 @@
#include "mongo/db/range_arithmetic.h"
#include "mongo/dbtests/config_server_fixture.h"
#include "mongo/s/catalog/type_chunk.h"
#include "mongo/s/catalog/type_collection.h"
#include "mongo/s/chunk.h" // for genID
#include "mongo/s/chunk_version.h"
#include "mongo/s/collection_metadata.h"
#include "mongo/s/d_state.h"
#include "mongo/s/d_merge.h"
#include "mongo/s/type_collection.h"
#include "mongo/unittest/unittest.h"

namespace mongo {
Expand Down Expand Up @@ -64,17 +64,16 @@ namespace mongo {
ASSERT_GREATER_THAN( ranges.size(), 0u );

CollectionType coll;
coll.setNS( nss.ns() );
coll.setNs( nss.ns() );
coll.setKeyPattern( ranges.begin()->keyPattern );
coll.setEpoch( startVersion.epoch() );
coll.setUpdatedAt( 1ULL );
string errMsg;
ASSERT( coll.isValid( &errMsg ) );
ASSERT_OK(coll.validate());

DBDirectClient client(&_txn);

client.update( CollectionType::ConfigNS,
BSON( CollectionType::ns( coll.getNS() ) ),
BSON( CollectionType::fullNs( coll.getNs() ) ),
coll.toBSON(), true, false );

ChunkVersion nextVersion = startVersion;
Expand Down
22 changes: 10 additions & 12 deletions src/mongo/dbtests/sharding.cpp
View file
Expand Up @@ -39,12 +39,12 @@
#include "mongo/dbtests/config_server_fixture.h"
#include "mongo/dbtests/dbtests.h"
#include "mongo/s/catalog/type_chunk.h"
#include "mongo/s/catalog/type_collection.h"
#include "mongo/s/catalog/type_shard.h"
#include "mongo/s/chunk_diff.h"
#include "mongo/s/chunk_manager.h"
#include "mongo/s/chunk_version.h"
#include "mongo/s/config.h"
#include "mongo/s/type_collection.h"
#include "mongo/util/log.h"

namespace ShardingTests {
Expand Down Expand Up @@ -259,17 +259,15 @@ namespace ShardingTests {
ChunkType::DEPRECATED_lastmod());

// Make manager load existing chunks
BSONObjBuilder collDocBuilder;
collDocBuilder << CollectionType::ns(collName());
collDocBuilder << CollectionType::keyPattern(BSON( "_id" << 1 ));
collDocBuilder << CollectionType::unique(false);
collDocBuilder << CollectionType::dropped(false);
collDocBuilder << CollectionType::DEPRECATED_lastmod(jsTime());
collDocBuilder << CollectionType::DEPRECATED_lastmodEpoch(version.epoch());

BSONObj collDoc(collDocBuilder.done());

ChunkManager manager(collDoc);
CollectionType collType;
collType.setNs(collName());
collType.setEpoch(version.epoch());
collType.setUpdatedAt(jsTime());
collType.setKeyPattern(BSON("_id" << 1));
collType.setUnique(false);
collType.setDropped(false);

ChunkManager manager(collType);
manager.loadExistingRanges(shard().getConnString(), NULL);

ASSERT(manager.getVersion().epoch() == version.epoch());
Expand Down
39 changes: 18 additions & 21 deletions src/mongo/s/SConscript
View file
Expand Up @@ -7,7 +7,6 @@ Import("env")
#

env.Library('base', ['mongo_version_range.cpp',
'type_collection.cpp',
'type_database.cpp',
'type_locks.cpp',
'type_lockpings.cpp',
Expand All @@ -27,10 +26,6 @@ env.CppUnitTest('mongo_version_range_test', 'mongo_version_range_test.cpp',
'$BUILD_DIR/mongo/bson',
'$BUILD_DIR/mongo/db/common'])

env.CppUnitTest('type_collection_test', 'type_collection_test.cpp',
LIBDEPS=['base',
'$BUILD_DIR/mongo/db/common'])

env.CppUnitTest('type_config_version_test', 'type_config_version_test.cpp',
LIBDEPS=['base',
'$BUILD_DIR/mongo/db/common'])
Expand Down Expand Up @@ -79,22 +74,24 @@ env.Library(
]
)

env.CppUnitTest('chunk_diff_test',
'chunk_diff_test.cpp',
LIBDEPS=['metadata',
'$BUILD_DIR/mongo/db/common'])

env.CppUnitTest('collection_metadata_test',
'collection_metadata_test.cpp',
LIBDEPS=['metadata',
'$BUILD_DIR/mongo/mocklib',
'$BUILD_DIR/mongo/db/common'])

env.CppUnitTest('metadata_loader_test',
'metadata_loader_test.cpp',
LIBDEPS=['metadata',
'$BUILD_DIR/mongo/mocklib',
'$BUILD_DIR/mongo/db/common'])
env.CppUnitTest(
target='metadata_test',
source=[
'chunk_diff_test.cpp',
'metadata_loader_test.cpp',
'collection_metadata_test.cpp',
],
LIBDEPS=[
'catalog/legacy/catalog_manager_legacy',
'metadata',
'$BUILD_DIR/mongo/coredb',
'$BUILD_DIR/mongo/coreserver',
'$BUILD_DIR/mongo/coreshard',
'$BUILD_DIR/mongo/mocklib',
'$BUILD_DIR/mongo/mongocommon',
'$BUILD_DIR/mongo/mongoscore',
'$BUILD_DIR/mongo/db/common',
])

#
# Write Operations
Expand Down

0 comments on commit 4777fb8

Please sign in to comment.