From 625b798e93d0a1e36c5f424fe19bd9054cd3abc1 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Mon, 3 May 2021 12:25:26 +0200 Subject: [PATCH 1/2] fix(shell-api): sharding methods account for 5.0 config.chunks changes MONGOSH-740 --- packages/shell-api/src/collection.ts | 20 ++++++++++++++------ packages/shell-api/src/helpers.ts | 16 +++++++++------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/packages/shell-api/src/collection.ts b/packages/shell-api/src/collection.ts index b6b1a3d9cf..48ff51a220 100644 --- a/packages/shell-api/src/collection.ts +++ b/packages/shell-api/src/collection.ts @@ -1530,14 +1530,15 @@ export default class Collection extends ShellApiClass { const result = {} as Document; const config = this._mongo.getDB('config'); + const ns = `${this._database._name}.${this._name}`; - const isSharded = !!(await config.getCollection('collections').countDocuments({ - _id: `${this._database._name}.${this._name}`, + const configCollectionsInfo = await config.getCollection('collections').findOne({ + _id: ns, // dropped is gone on newer server versions, so check for !== true // rather than for === false (SERVER-51880 and related) dropped: { $ne: true } - })); - if (!isSharded) { + }); + if (!configCollectionsInfo) { throw new MongoshInvalidInputError( `Collection ${this._name} is not sharded`, ShellApiErrors.NotConnectedToShardedCluster @@ -1545,6 +1546,7 @@ export default class Collection extends ShellApiClass { } const collStats = await (await this.aggregate({ '$collStats': { storageStats: {} } })).toArray(); + const uuid = configCollectionsInfo?.uuid ?? null; const totals = { numChunks: 0, size: 0, count: 0 }; const conciseShardsStats: { @@ -1559,12 +1561,18 @@ export default class Collection extends ShellApiClass { await Promise.all(collStats.map((extShardStats) => ( (async(): Promise => { // Extract and store only the relevant subset of the stats for this shard + const { shard } = extShardStats; + + // If we have an UUID, use that for lookups. If we have only the ns, + // use that. (On 5.0+ servers, config.chunk has uses the UUID, before + // that it had the ns). + const countChunksQuery = uuid ? { $or: [ { uuid }, { ns } ], shard } : { ns, shard }; const [ host, numChunks ] = await Promise.all([ config.getCollection('shards').findOne({ _id: extShardStats.shard }), - config.getCollection('chunks').countDocuments({ ns: extShardStats.ns, shard: extShardStats.shard }) + config.getCollection('chunks').countDocuments(countChunksQuery) ]); const shardStats = { - shardId: extShardStats.shard, + shardId: shard, host: host !== null ? host.host : null, size: extShardStats.storageStats.size, count: extShardStats.storageStats.count, diff --git a/packages/shell-api/src/helpers.ts b/packages/shell-api/src/helpers.ts index f316fa0ecb..17b6d1e20c 100644 --- a/packages/shell-api/src/helpers.ts +++ b/packages/shell-api/src/helpers.ts @@ -403,8 +403,8 @@ export async function getPrintableShardStatus(db: Database, verbose: boolean): P // Special case the config db, since it doesn't have a record in config.databases. databases.push({ '_id': 'config', 'primary': 'config', 'partitioned': true }); - databases.sort((a: any, b: any): any => { - return a._id > b._id; + databases.sort((a: any, b: any): number => { + return a._id.localeCompare(b._id); }); result.databases = await Promise.all(databases.map(async(db) => { @@ -429,12 +429,14 @@ export async function getPrintableShardStatus(db: Database, verbose: boolean): P collRes.balancing = [ !coll.noBalance, { noBalance: coll.noBalance } ]; } const chunksRes = []; + const chunksCollMatch = coll.uuid ? { $or: [ { uuid: coll.uuid, ns: coll._id } ] } : { ns: coll._id }; const chunks = await - (await chunksColl.aggregate({ $match: { ns: coll._id } }, + (await chunksColl.aggregate([ + { $match: chunksCollMatch }, { $group: { _id: '$shard', cnt: { $sum: 1 } } }, { $project: { _id: 0, shard: '$_id', nChunks: '$cnt' } }, - { $sort: { shard: 1 } }) - ).toArray(); + { $sort: { shard: 1 } } + ])).toArray(); let totalChunks = 0; chunks.forEach((z: any) => { totalChunks += z.nChunks; @@ -443,7 +445,7 @@ export async function getPrintableShardStatus(db: Database, verbose: boolean): P // NOTE: this will return the chunk info as a string, and will print ugly BSON if (totalChunks < 20 || verbose) { - (await chunksColl.find({ 'ns': coll._id }) + (await chunksColl.find(chunksCollMatch) .sort({ min: 1 }).toArray()) .forEach((chunk: any) => { const c = { @@ -461,7 +463,7 @@ export async function getPrintableShardStatus(db: Database, verbose: boolean): P const tagsRes: any[] = []; (await configDB.getCollection('tags') - .find({ ns: coll._id }) + .find(chunksCollMatch) .sort({ min: 1 }) .toArray()) .forEach((tag: any) => { From 8b3faf3e1a851fedc0f5a9aa63a8d1aa5727b12e Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Mon, 3 May 2021 14:18:07 +0200 Subject: [PATCH 2/2] fixup! fix(shell-api): sharding methods account for 5.0 config.chunks changes MONGOSH-740 --- packages/shell-api/src/collection.spec.ts | 4 ++++ packages/shell-api/src/helpers.ts | 3 ++- packages/shell-api/src/shard.spec.ts | 14 +++++++------- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/packages/shell-api/src/collection.spec.ts b/packages/shell-api/src/collection.spec.ts index 28343bcece..377f976dc2 100644 --- a/packages/shell-api/src/collection.spec.ts +++ b/packages/shell-api/src/collection.spec.ts @@ -1622,6 +1622,10 @@ describe('Collection', () => { }); describe('getShardDistribution', () => { it('throws when collection is not sharded', async() => { + const serviceProviderCursor = stubInterface(); + serviceProviderCursor.limit.returns(serviceProviderCursor); + serviceProviderCursor.tryNext.returns(null); + serviceProvider.find.returns(serviceProviderCursor as any); const error = await collection.getShardDistribution().catch(e => e); expect(error).to.be.instanceOf(MongoshInvalidInputError); diff --git a/packages/shell-api/src/helpers.ts b/packages/shell-api/src/helpers.ts index 17b6d1e20c..fd320f769b 100644 --- a/packages/shell-api/src/helpers.ts +++ b/packages/shell-api/src/helpers.ts @@ -429,7 +429,8 @@ export async function getPrintableShardStatus(db: Database, verbose: boolean): P collRes.balancing = [ !coll.noBalance, { noBalance: coll.noBalance } ]; } const chunksRes = []; - const chunksCollMatch = coll.uuid ? { $or: [ { uuid: coll.uuid, ns: coll._id } ] } : { ns: coll._id }; + const chunksCollMatch = + coll.uuid ? { $or: [ { uuid: coll.uuid }, { ns: coll._id } ] } : { ns: coll._id }; const chunks = await (await chunksColl.aggregate([ { $match: chunksCollMatch }, diff --git a/packages/shell-api/src/shard.spec.ts b/packages/shell-api/src/shard.spec.ts index b170df525b..2b0822978d 100644 --- a/packages/shell-api/src/shard.spec.ts +++ b/packages/shell-api/src/shard.spec.ts @@ -1159,9 +1159,9 @@ describe('Shard', () => { expect((await sh.status()).value.databases.length).to.equal(2); }); it('enableSharding for a collection and modify documents in it', async() => { - expect(Object.keys((await sh.status()).value.databases[0].collections).length).to.equal(0); + expect(Object.keys((await sh.status()).value.databases[1].collections).length).to.equal(0); expect((await sh.shardCollection(ns, { key: 1 })).collectionsharded).to.equal(ns); - expect((await sh.status()).value.databases[0].collections[ns].shardKey).to.deep.equal({ key: 1 }); + expect((await sh.status()).value.databases[1].collections[ns].shardKey).to.deep.equal({ key: 1 }); const db = internalState.currentDb.getSiblingDB(dbName); await db.coll.insertMany([{ key: 'A', value: 10 }, { key: 'B', value: 20 }]); @@ -1189,20 +1189,20 @@ describe('Shard', () => { }); it('sets a zone key range', async() => { expect((await sh.updateZoneKeyRange(ns, { key: 0 }, { key: 20 }, 'zone1')).ok).to.equal(1); - expect((await sh.status()).value.databases[0].collections[ns].tags[0]).to.deep.equal({ + expect((await sh.status()).value.databases[1].collections[ns].tags[0]).to.deep.equal({ tag: 'zone1', min: { key: 0 }, max: { key: 20 } }); expect((await sh.addTagRange(ns, { key: 21 }, { key: 40 }, 'zone0')).ok).to.equal(1); - expect((await sh.status()).value.databases[0].collections[ns].tags[1]).to.deep.equal({ + expect((await sh.status()).value.databases[1].collections[ns].tags[1]).to.deep.equal({ tag: 'zone0', min: { key: 21 }, max: { key: 40 } }); }); it('removes a key range', async() => { - expect((await sh.status()).value.databases[0].collections[ns].tags.length).to.equal(2); + expect((await sh.status()).value.databases[1].collections[ns].tags.length).to.equal(2); expect((await sh.removeRangeFromZone(ns, { key: 0 }, { key: 20 })).ok).to.equal(1); - expect((await sh.status()).value.databases[0].collections[ns].tags.length).to.equal(1); + expect((await sh.status()).value.databases[1].collections[ns].tags.length).to.equal(1); expect((await sh.removeTagRange(ns, { key: 21 }, { key: 40 })).ok).to.equal(1); - expect((await sh.status()).value.databases[0].collections[ns].tags.length).to.equal(0); + expect((await sh.status()).value.databases[1].collections[ns].tags.length).to.equal(0); }); it('removes zones', async() => { expect((await sh.removeShardFromZone(`${shardId}-1`, 'zone1')).ok).to.equal(1);