From a41878d9c8839f1dae4e303fe0dccc130f9207f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Kosiec?= Date: Sat, 14 Jul 2018 10:30:21 +0200 Subject: [PATCH] Allow dropping collection before importing it --- cli/README.md | 3 ++- cli/src/options.ts | 7 +++++++ core/README.md | 1 + core/src/common/config.ts | 3 ++- core/src/common/types.ts | 1 + core/src/database/Database.ts | 17 +++++++++++++++-- core/src/index.ts | 10 ++++++++-- core/tests/integration/Database.ts | 13 +++++++++++++ core/tests/integration/seedDatabase.ts | 24 ++++++++++++++++++++++++ core/tests/unit/config.ts | 1 + docker-image/README.md | 5 +++-- docker-image/src/index.ts | 3 ++- 12 files changed, 79 insertions(+), 9 deletions(-) diff --git a/cli/README.md b/cli/README.md index c366ac0..cb825e7 100644 --- a/cli/README.md +++ b/cli/README.md @@ -45,7 +45,8 @@ You can use the following parameters while using `seed` binary: | `--db-name $DB_NAME` | `database` | Name of the database | | `--db-username $DB_USERNAME` | database | Username for connecting with database that requires authentication | | `--db-password $DB_PASSWORD` | database | Password for connecting with database that requires authentication | -| `--drop-database` | `false` | Dropping database before data import | +| `--drop-database` | `false` | Dropping entire database before data import | +| `--drop-collection` | `false` | Dropping every collection that is being imported | | `--replace-id` | `false` | Replacing `id` property with `_id` for every document during data import | | `--reconnect-timeout` | `10` (seconds) | Maximum time of waiting for successful MongoDB connection| | `--help` or `-h` | n/a | Help diff --git a/cli/src/options.ts b/cli/src/options.ts index d066b25..df39f8e 100644 --- a/cli/src/options.ts +++ b/cli/src/options.ts @@ -67,6 +67,11 @@ export const optionsDefinition: CommandLineOptionDefinition[] = [ description: 'Drops database before import', type: Boolean, }, + { + name: 'drop-collection', + description: 'Drops collection before importing it', + type: Boolean, + }, { name: 'replace-id', description: 'Replaces `id` property with `_id` for every object to import', @@ -84,6 +89,7 @@ export interface CommandLineOptions { data?: string; [key: string]: string | number | boolean | undefined; 'drop-database': boolean; + 'drop-collection': boolean; 'replace-id': boolean; 'db-protocol'?: string; 'db-host'?: string; @@ -128,6 +134,7 @@ export const convertOptions = ( databaseConnectionUri: options['db-uri'], inputPath: options.data ? resolve(options.data) : resolve('./'), dropDatabase: options['drop-database'], + dropCollection: options['drop-collection'], replaceIdWithUnderscoreId: options['replace-id'], reconnectTimeoutInSeconds: options['reconnect-timeout'], }); diff --git a/core/README.md b/core/README.md index f912a12..df90362 100644 --- a/core/README.md +++ b/core/README.md @@ -81,6 +81,7 @@ const config = { password: undefined, }, databaseConnectionUri: undefined, // if defined, it will be used for DB connection instead of `database` object + dropCollection: false, // drops every collection that is being imported inputPath: resolve(__dirname, '../../data'), // input directory with import data structure dropDatabase: false, // drops database before import replaceIdWithUnderscoreId: false, // rewrites `id` property to `_id` for every document; useful for ORMs diff --git a/core/src/common/config.ts b/core/src/common/config.ts index 709a2a3..cf3d666 100644 --- a/core/src/common/config.ts +++ b/core/src/common/config.ts @@ -13,7 +13,8 @@ export const defaultConfig: AppConfig = { }, databaseConnectionUri: undefined, inputPath: resolve(__dirname, '../../data'), // input directory with import data structure - dropDatabase: false, // drops database before import + dropDatabase: false, // drops entire database before import + dropCollection: false, // drops collection before importing it replaceIdWithUnderscoreId: false, // rewrites `id` property to `_id` for every document supportedExtensions: ['json', 'js'], // files that should be imported reconnectTimeoutInSeconds: 10, // maximum time of waiting for successful MongoDB connection diff --git a/core/src/common/types.ts b/core/src/common/types.ts index 0c26878..76e5204 100644 --- a/core/src/common/types.ts +++ b/core/src/common/types.ts @@ -16,6 +16,7 @@ export interface AppConfig { databaseConnectionUri?: string; inputPath: string; dropDatabase: boolean; + dropCollection: boolean; replaceIdWithUnderscoreId: boolean; supportedExtensions: string[]; reconnectTimeoutInSeconds: number; diff --git a/core/src/database/Database.ts b/core/src/database/Database.ts index 64d3c86..63b2512 100644 --- a/core/src/database/Database.ts +++ b/core/src/database/Database.ts @@ -8,10 +8,23 @@ export class Database { collectionName: string, ) { const documentsCopy = documentsToInsert.map(document => ({ ...document })); - await this.db.collection(collectionName).insertMany(documentsCopy); + return this.db.collection(collectionName).insertMany(documentsCopy); } async drop() { - await this.db.dropDatabase(); + return this.db.dropDatabase(); + } + + async ifCollectionExist(collectionName:string):Promise { + const collections = await this.db.collections(); + return collections.map(collection => collection.collectionName).includes(collectionName); + } + + async dropCollectionIfExists(collectionName:string) { + if (!await this.ifCollectionExist(collectionName)) { + return; + } + + return this.db.collection(collectionName).drop() } } diff --git a/core/src/index.ts b/core/src/index.ts index 745ecb0..8d4a78a 100644 --- a/core/src/index.ts +++ b/core/src/index.ts @@ -1,4 +1,3 @@ -import { MongoClient } from 'mongodb'; import { log, getConfig, DeepPartial, AppConfig } from './common'; import { DatabaseConnector } from './database'; import { @@ -34,12 +33,19 @@ export const seedDatabase = async (partialConfig: DeepPartial) => { databaseConnectionUri: config.databaseConnectionUri, databaseConfig: config.database, }); + + if (!config.dropDatabase && config.dropCollection) { + log('Dropping collections...'); + for (const collection of collections) { + await database.dropCollectionIfExists(collection.name); + } + } if (config.dropDatabase) { log('Dropping database...'); await database.drop(); } - + await new DataImporter(database).import(collections); } catch (err) { throw wrapError(err); diff --git a/core/tests/integration/Database.ts b/core/tests/integration/Database.ts index 8273467..d39f603 100644 --- a/core/tests/integration/Database.ts +++ b/core/tests/integration/Database.ts @@ -73,4 +73,17 @@ describe('Database', () => { await database.drop(); await expect(listExistingCollections(database.db)).resolves.toEqual([]); }); + + it('should drop collection', async () => { + await createCollection(database.db, 'first'); + await createCollection(database.db, 'second'); + + const collections = await listExistingCollections(database.db); + await expect(collections).toHaveLength(2); + await expect(collections).toContainEqual('first'); + await expect(collections).toContainEqual('second'); + + await database.dropCollectionIfExists('first'); + await expect(listExistingCollections(database.db)).resolves.toEqual(["second"]); + }); }); diff --git a/core/tests/integration/seedDatabase.ts b/core/tests/integration/seedDatabase.ts index 44f0319..ea044b0 100644 --- a/core/tests/integration/seedDatabase.ts +++ b/core/tests/integration/seedDatabase.ts @@ -144,6 +144,30 @@ describe('Mongo Seeding', () => { expect(collections).toContainEqual('CollectionTwo'); }); + it('should drop collections before importing data', async () => { + const expectedCollectionNames = ['CollectionOne', 'CollectionTwo']; + createSampleFiles(expectedCollectionNames, TEMP_DIRECTORY_PATH); + + await createCollection(database.db, 'CollectionOne'); + await createCollection(database.db, 'ShouldNotBeRemoved'); + + const config: DeepPartial = { + inputPath: TEMP_DIRECTORY_PATH, + database: { + name: DATABASE_NAME, + }, + dropCollection: true, + }; + + await expect(seedDatabase(config)).resolves.toBeUndefined(); + + const collections = await listExistingCollections(database.db); + expect(collections).toHaveLength(3); + expect(collections).toContainEqual('CollectionOne'); + expect(collections).toContainEqual('CollectionTwo'); + expect(collections).toContainEqual('ShouldNotBeRemoved'); + }); + it('should throw error when wrong path given', async () => { const config: DeepPartial = { inputPath: '/this/path/surely/doesnt/exist', diff --git a/core/tests/unit/config.ts b/core/tests/unit/config.ts index cff3351..21e5c9d 100644 --- a/core/tests/unit/config.ts +++ b/core/tests/unit/config.ts @@ -30,6 +30,7 @@ describe('Config', () => { }, inputPath: '/', dropDatabase: false, + dropCollection: false, replaceIdWithUnderscoreId: true, supportedExtensions: ['md', 'txt'], reconnectTimeoutInSeconds: 20, diff --git a/docker-image/README.md b/docker-image/README.md index 471cf5d..5c051ad 100644 --- a/docker-image/README.md +++ b/docker-image/README.md @@ -2,7 +2,7 @@ # Mongo Seeding Docker Image -[![Build Status](https://travis-ci.org/pkosiec/mongo-seeding-docker.svg?branch=master)](https://travis-ci.org/pkosiec/mongo-seeding-docker) [![David](https://img.shields.io/david/pkosiec/mongo-seeding.svg?path=docker-image)]() [![David](https://img.shields.io/david/dev/pkosiec/mongo-seeding.svg?path=docker-image)]() +[![Build Status](https://travis-ci.org/pkosiec/mongo-seeding.svg?branch=master)](https://travis-ci.org/pkosiec/mongo-seeding) [![David](https://img.shields.io/david/pkosiec/mongo-seeding.svg?path=docker-image)]() [![David](https://img.shields.io/david/dev/pkosiec/mongo-seeding.svg?path=docker-image)]() The ultimate solution for populating your MongoDB database. Define the data in JSON, JavaScript or TypeScript. Import collections and documents! @@ -55,7 +55,8 @@ Specify environmental variables with `-e {key}={value}` parameter. | DB_NAME | `database` | Name of the database | | DB_USERNAME | *`undefined`* | Username for connecting with database that requires authentication | | DB_PASSWORD | *`undefined`* | Password for connecting with database that requires authentication | -| DROP_DATABASE | `false` | Dropping database before data import | +| DROP_DATABASE | `false` | Dropping entire database before data import | +| DROP_COLLECTION | `false` | Dropping every collection that is being imported | | REPLACE_ID_TO_UNDERSCORE_ID | `false` | Replacing `id` property with `_id` for every document during import; useful for ORMs | | RECONNECT_TIMEOUT_IN_SECONDS | `10` | Maximum time, in which app should keep trying connecting to database | diff --git a/docker-image/src/index.ts b/docker-image/src/index.ts index ed2f765..96fd57f 100644 --- a/docker-image/src/index.ts +++ b/docker-image/src/index.ts @@ -1,6 +1,6 @@ import { seedDatabase } from 'mongo-seeding'; import { resolve } from 'path'; -import { DeepPartial, AppConfig } from 'mongo-seeding/common'; +import { DeepPartial, AppConfig } from 'mongo-seeding/dist/common'; const env = process.env; const envOptions: DeepPartial = { @@ -16,6 +16,7 @@ const envOptions: DeepPartial = { ? String(env.DB_CONNECTION_URI) : undefined, dropDatabase: env.DROP_DATABASE === 'true', + dropCollection: env.DROP_COLLECTION === 'true', replaceIdWithUnderscoreId: env.REPLACE_ID_TO_UNDERSCORE_ID === 'true', supportedExtensions: ['ts', 'js', 'json'], inputPath: resolve(__dirname, '../data'),