Skip to content
Browse files

SERVER-3842 - make all initial chunks before marking collection sharded

  • Loading branch information...
1 parent 3e478e6 commit 5f7feee66ca6c1c915852de83a93502a26f64d8e @erh erh committed Oct 19, 2011
Showing with 78 additions and 57 deletions.
  1. +33 −0 jstests/sharding/shard_existing.js
  2. +42 −41 s/chunk.cpp
  3. +1 −3 s/chunk.h
  4. +2 −13 s/config.cpp
View
33 jstests/sharding/shard_existing.js
@@ -0,0 +1,33 @@
+
+s = new ShardingTest( "shard_existing" , 2 /* numShards */, 1 /* verboseLevel */, 1 /* numMongos */, { chunksize : 1 } )
+
+db = s.getDB( "test" )
+
+stringSize = 10000
+
+// we want a lot of data, so lets make a string to cheat :)
+bigString = "";
+while ( bigString.length < stringSize )
+ bigString += "this is a big string. ";
+
+dataSize = 20 * 1024 * 1024;
+
+numToInsert = dataSize / stringSize
+print( "numToInsert: " + numToInsert )
+
+for ( i=0; i<(dataSize/stringSize); i++ ) {
+ db.data.insert( { _id : i , s : bigString } )
+}
+
+db.getLastError();
+
+assert.lt( dataSize , db.data.stats().size )
+
+s.adminCommand( { enablesharding : "test" } );
+res = s.adminCommand( { shardcollection : "test.data" , key : { _id : 1 } } );
+printjson( res );
+
+assert.eq( 40 , s.config.chunks.find().itcount() , "not right number of chunks" );
+
+
+s.stop();
View
83 s/chunk.cpp
@@ -649,44 +649,65 @@ namespace mongo {
return _key.hasShardKey( obj );
}
- void ChunkManager::createFirstChunk( const Shard& shard ) const {
+ void ChunkManager::createFirstChunks( const Shard& shard ) const {
// TODO distlock?
assert( _chunkMap.size() == 0 );
- Chunk c (this, _key.globalMin(), _key.globalMax(), shard);
+ unsigned long long numObjects = 0;
+ {
+ // get stats to see if there is any data
+ ScopedDbConnection shardConn( shard.getConnString() );
+ numObjects = shardConn->count( getns() );
+ shardConn.done();
+ }
// this is the first chunk; start the versioning from scratch
ShardChunkVersion version;
version.incMajor();
- // build update for the chunk collection
- BSONObjBuilder chunkBuilder;
- c.serialize( chunkBuilder , version );
- BSONObj chunkCmd = chunkBuilder.obj();
+ Chunk c(this, _key.globalMin(), _key.globalMax(), shard);
- log() << "about to create first chunk for: " << _ns << endl;
+ vector<BSONObj> splitPoints;
+ if ( numObjects > 0 )
+ c.pickSplitVector( splitPoints , Chunk::MaxChunkSize );
+
+ log() << "going to create " << splitPoints.size() + 1 << " chunk(s) for: " << _ns << endl;
+
- ScopedDbConnection conn( configServer.modelServer() );
- BSONObj res;
- conn->update( Chunk::chunkMetadataNS, QUERY( "_id" << c.genID() ), chunkCmd, true, false );
+ ScopedDbConnection conn( configServer.modelServer() );
+
+ for ( unsigned i=0; i<=splitPoints.size(); i++ ) {
+ BSONObj min = i == 0 ? _key.globalMin() : splitPoints[i-1];
+ BSONObj max = i < splitPoints.size() ? splitPoints[i] : _key.globalMax();
+
+ Chunk temp( this , min , max , shard );
+
+ BSONObjBuilder chunkBuilder;
+ temp.serialize( chunkBuilder , version );
+ BSONObj chunkObj = chunkBuilder.obj();
+
+ conn->update( Chunk::chunkMetadataNS, QUERY( "_id" << temp.genID() ), chunkObj, true, false );
+
+ version.incMinor();
+ }
string errmsg = conn->getLastError();
if ( errmsg.size() ) {
- stringstream ss;
- ss << "saving first chunk failed. cmd: " << chunkCmd << " result: " << errmsg;
- log( LL_ERROR ) << ss.str() << endl;
- msgasserted( 13592 , ss.str() );
+ string ss = str::stream() << "creating first chunks failed. result: " << errmsg;
+ error() << ss << endl;
+ msgasserted( 15903 , ss );
}
-
+
conn.done();
- // the ensure index will have the (desired) indirect effect of creating the collection on the
- // assigned shard, as it sets up the index over the sharding keys.
- ScopedDbConnection shardConn( c.getShard().getConnString() );
- shardConn->ensureIndex( getns() , getShardKey().key() , _unique , "" , false /* do not cache ensureIndex SERVER-1691 */ );
- shardConn.done();
+ if ( numObjects == 0 ) {
+ // the ensure index will have the (desired) indirect effect of creating the collection on the
+ // assigned shard, as it sets up the index over the sharding keys.
+ ScopedDbConnection shardConn( c.getShard().getConnString() );
+ shardConn->ensureIndex( getns() , getShardKey().key() , _unique , "" , false ); // do not cache ensureIndex SERVER-1691
+ shardConn.done();
+ }
- log() << "successfully created first chunk for " << c.toString() << endl;
}
ChunkPtr ChunkManager::findChunk( const BSONObj & obj ) const {
@@ -868,26 +889,6 @@ namespace mongo {
configServer.logChange( "dropCollection" , _ns , BSONObj() );
}
- void ChunkManager::maybeChunkCollection() const {
- uassert( 13346 , "can't pre-split already splitted collection" , (_chunkMap.size() == 1) );
-
- ChunkPtr soleChunk = _chunkMap.begin()->second;
- vector<BSONObj> splitPoints;
- soleChunk->pickSplitVector( splitPoints , Chunk::MaxChunkSize );
- if ( splitPoints.empty() ) {
- LOG(1) << "not enough data to warrant chunking " << getns() << endl;
- return;
- }
-
- BSONObj res;
- ChunkPtr p;
- bool worked = soleChunk->multiSplit( splitPoints , res );
- if (!worked) {
- log( LL_WARNING ) << "could not split '" << getns() << "': " << res << endl;
- return;
- }
- }
-
ShardChunkVersion ChunkManager::getVersion( const Shard& shard ) const {
ShardVersionMap::const_iterator i = _shardVersions.find( shard );
if ( i == _shardVersions.end() )
View
4 s/chunk.h
@@ -306,15 +306,13 @@ namespace mongo {
int numChunks() const { return _chunkMap.size(); }
bool hasShardKey( const BSONObj& obj ) const;
- void createFirstChunk( const Shard& shard ) const; // only call from DBConfig::shardCollection
+ void createFirstChunks( const Shard& shard ) const; // only call from DBConfig::shardCollection
ChunkPtr findChunk( const BSONObj& obj ) const;
ChunkPtr findChunkOnServer( const Shard& shard ) const;
const ShardKeyPattern& getShardKey() const { return _key; }
bool isUnique() const { return _unique; }
- void maybeChunkCollection() const;
-
void getShardsForQuery( set<Shard>& shards , const BSONObj& query ) const;
void getAllShards( set<Shard>& all ) const;
void getShardsForRange(set<Shard>& shards, const BSONObj& min, const BSONObj& max, bool fullKeyReq = true) const; // [min, max)
View
15 s/config.cpp
@@ -143,25 +143,14 @@ namespace mongo {
log() << "enable sharding on: " << ns << " with shard key: " << fieldsAndOrder << endl;
- // From this point on, 'ns' is going to be treated as a sharded collection. We assume this is the first
- // time it is seen by the sharded system and thus create the first chunk for the collection. All the remaining
- // chunks will be created as a by-product of splitting.
ci.shard( ns , fieldsAndOrder , unique );
ChunkManagerPtr cm = ci.getCM();
uassert( 13449 , "collections already sharded" , (cm->numChunks() == 0) );
- cm->createFirstChunk( getPrimary() );
+ cm->createFirstChunks( getPrimary() );
_save();
}
- try {
- getChunkManager(ns, true)->maybeChunkCollection();
- }
- catch ( UserException& e ) {
- // failure to chunk is not critical enough to abort the command (and undo the _save()'d configDB state)
- log() << "couldn't chunk recently created collection: " << ns << " " << e << endl;
- }
-
- return getChunkManager(ns);
+ return getChunkManager(ns,true,true);
}
bool DBConfig::removeSharding( const string& ns ) {

0 comments on commit 5f7feee

Please sign in to comment.
Something went wrong with that request. Please try again.